From 59305715616f249170199d00deb382cc783d7660 Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Tue, 26 Mar 2024 12:54:23 +0800 Subject: [PATCH] Expose hosted info to be customized Signed-off-by: Jian Qiu --- pkg/addonfactory/addonfactory.go | 10 + pkg/addonfactory/helm_agentaddon.go | 5 +- pkg/addonfactory/template_agentaddon.go | 3 +- pkg/addonmanager/constants/constants.go | 8 +- .../controllers/agentdeploy/controller.go | 204 ++++++++++-------- .../agentdeploy/default_hook_sync.go | 8 +- .../controllers/agentdeploy/default_sync.go | 6 +- .../agentdeploy/hosted_hook_sync.go | 10 +- .../controllers/agentdeploy/hosted_sync.go | 10 +- .../agentdeploy/hosted_sync_test.go | 5 +- .../controllers/agentdeploy/utils.go | 20 +- pkg/agent/inteface.go | 5 +- test/integration/agent_hosting_deploy_test.go | 59 +++++ test/integration/suite_test.go | 4 + 14 files changed, 224 insertions(+), 133 deletions(-) diff --git a/pkg/addonfactory/addonfactory.go b/pkg/addonfactory/addonfactory.go index 173b168cd..7baae03e6 100644 --- a/pkg/addonfactory/addonfactory.go +++ b/pkg/addonfactory/addonfactory.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" "k8s.io/klog/v2" + "open-cluster-management.io/addon-framework/pkg/addonmanager/constants" addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" clusterclientset "open-cluster-management.io/api/client/cluster/clientset/versioned" clusterv1 "open-cluster-management.io/api/cluster/v1" @@ -59,6 +60,8 @@ func NewAgentAddonFactory(addonName string, fs embed.FS, dir string) *AgentAddon Registration: nil, HealthProber: nil, SupportedConfigGVRs: []schema.GroupVersionResource{}, + // Set a default hosted mode info func. + HostedModeInfoFunc: constants.GetHostedModeInfo, }, trimCRDDescription: false, scheme: s, @@ -100,6 +103,13 @@ func (f *AgentAddonFactory) WithAgentHostedModeEnabledOption() *AgentAddonFactor return f } +// WithAgentHostedInfoFn sets the function to get the hosting cluster of an addon in the hosted mode. +func (f *AgentAddonFactory) WithAgentHostedInfoFn( + infoFn func(*addonapiv1alpha1.ManagedClusterAddOn, *clusterv1.ManagedCluster) (string, string)) *AgentAddonFactory { + f.agentAddonOptions.HostedModeInfoFunc = infoFn + return f +} + // WithTrimCRDDescription is to enable trim the description of CRDs in manifestWork. func (f *AgentAddonFactory) WithTrimCRDDescription() *AgentAddonFactory { f.trimCRDDescription = true diff --git a/pkg/addonfactory/helm_agentaddon.go b/pkg/addonfactory/helm_agentaddon.go index b66e60aa1..665c850a9 100644 --- a/pkg/addonfactory/helm_agentaddon.go +++ b/pkg/addonfactory/helm_agentaddon.go @@ -23,7 +23,6 @@ import ( clusterclientset "open-cluster-management.io/api/client/cluster/clientset/versioned" clusterv1 "open-cluster-management.io/api/cluster/v1" - "open-cluster-management.io/addon-framework/pkg/addonmanager/constants" "open-cluster-management.io/addon-framework/pkg/agent" ) @@ -237,7 +236,7 @@ func (a *HelmAgentAddon) getBuiltinValues( builtinValues.AddonInstallNamespace = a.getValueAgentInstallNamespace(addon) - builtinValues.InstallMode, _ = constants.GetHostedModeInfo(addon.GetAnnotations()) + builtinValues.InstallMode, _ = a.agentAddonOptions.HostedModeInfoFunc(addon, cluster) helmBuiltinValues, err := JsonStructToValues(builtinValues) if err != nil { @@ -268,7 +267,7 @@ func (a *HelmAgentAddon) getDefaultValues( if a.hostingCluster != nil { defaultValues.HostingClusterCapabilities = *a.capabilities(a.hostingCluster, addon) } else if a.clusterClient != nil { - hostingClusterName := addon.GetAnnotations()[addonapiv1alpha1.HostingClusterNameAnnotationKey] + _, hostingClusterName := a.agentAddonOptions.HostedModeInfoFunc(addon, cluster) if len(hostingClusterName) > 0 { hostingCluster, err := a.clusterClient.ClusterV1().ManagedClusters(). Get(context.TODO(), hostingClusterName, metav1.GetOptions{}) diff --git a/pkg/addonfactory/template_agentaddon.go b/pkg/addonfactory/template_agentaddon.go index 57584ac19..e05ca8c9a 100644 --- a/pkg/addonfactory/template_agentaddon.go +++ b/pkg/addonfactory/template_agentaddon.go @@ -9,7 +9,6 @@ import ( addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" clusterv1 "open-cluster-management.io/api/cluster/v1" - "open-cluster-management.io/addon-framework/pkg/addonmanager/constants" "open-cluster-management.io/addon-framework/pkg/agent" "open-cluster-management.io/addon-framework/pkg/assets" ) @@ -133,7 +132,7 @@ func (a *TemplateAgentAddon) getBuiltinValues( } builtinValues.AddonInstallNamespace = installNamespace - builtinValues.InstallMode, _ = constants.GetHostedModeInfo(addon.GetAnnotations()) + builtinValues.InstallMode, _ = a.agentAddonOptions.HostedModeInfoFunc(addon, cluster) return StructToValues(builtinValues) } diff --git a/pkg/addonmanager/constants/constants.go b/pkg/addonmanager/constants/constants.go index 56e6f0169..ffc2829a2 100644 --- a/pkg/addonmanager/constants/constants.go +++ b/pkg/addonmanager/constants/constants.go @@ -4,6 +4,7 @@ import ( "fmt" addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + clusterv1 "open-cluster-management.io/api/cluster/v1" ) const ( @@ -35,8 +36,11 @@ func PreDeleteHookHostingWorkName(addonNamespace, addonName string) string { } // GetHostedModeInfo returns addon installation mode and hosting cluster name. -func GetHostedModeInfo(annotations map[string]string) (string, string) { - hostingClusterName, ok := annotations[addonv1alpha1.HostingClusterNameAnnotationKey] +func GetHostedModeInfo(addon *addonv1alpha1.ManagedClusterAddOn, _ *clusterv1.ManagedCluster) (string, string) { + if len(addon.Annotations) == 0 { + return InstallModeDefault, "" + } + hostingClusterName, ok := addon.Annotations[addonv1alpha1.HostingClusterNameAnnotationKey] if !ok { return InstallModeDefault, "" } diff --git a/pkg/addonmanager/controllers/agentdeploy/controller.go b/pkg/addonmanager/controllers/agentdeploy/controller.go index 930a0afaf..e5e7bed3a 100644 --- a/pkg/addonmanager/controllers/agentdeploy/controller.go +++ b/pkg/addonmanager/controllers/agentdeploy/controller.go @@ -252,25 +252,37 @@ func (c *addonDeployController) sync(ctx context.Context, syncCtx factory.SyncCo syncers := []addonDeploySyncer{ &defaultSyncer{ - buildWorks: c.buildDeployManifestWorks, + buildWorks: c.buildDeployManifestWorksFunc( + newAddonWorksBuilder(agentAddon.GetAgentAddonOptions().HostedModeEnabled, c.workBuilder), + addonapiv1alpha1.ManagedClusterAddOnManifestApplied, + ), applyWork: c.applyWork, getWorkByAddon: c.getWorksByAddonFn(index.ManifestWorkByAddon), deleteWork: c.workApplier.Delete, agentAddon: agentAddon, }, &hostedSyncer{ - buildWorks: c.buildDeployManifestWorks, + buildWorks: c.buildDeployManifestWorksFunc( + newHostingAddonWorksBuilder(agentAddon.GetAgentAddonOptions().HostedModeEnabled, c.workBuilder), + addonapiv1alpha1.ManagedClusterAddOnHostingManifestApplied, + ), applyWork: c.applyWork, deleteWork: c.workApplier.Delete, getCluster: c.managedClusterLister.Get, getWorkByAddon: c.getWorksByAddonFn(index.ManifestWorkByHostedAddon), agentAddon: agentAddon}, &defaultHookSyncer{ - buildWorks: c.buildHookManifestWork, + buildWorks: c.buildHookManifestWorkFunc( + newAddonWorksBuilder(agentAddon.GetAgentAddonOptions().HostedModeEnabled, c.workBuilder), + addonapiv1alpha1.ManagedClusterAddOnManifestApplied, + ), applyWork: c.applyWork, agentAddon: agentAddon}, &hostedHookSyncer{ - buildWorks: c.buildHookManifestWork, + buildWorks: c.buildHookManifestWorkFunc( + newHostingAddonWorksBuilder(agentAddon.GetAgentAddonOptions().HostedModeEnabled, c.workBuilder), + addonapiv1alpha1.ManagedClusterAddOnHostingManifestApplied, + ), applyWork: c.applyWork, deleteWork: c.workApplier.Delete, getCluster: c.managedClusterLister.Get, @@ -354,103 +366,109 @@ func (c *addonDeployController) applyWork(ctx context.Context, appliedType strin return work, nil } -func (c *addonDeployController) buildDeployManifestWorks(installMode, workNamespace string, +type buildDeployWorkFunc func( + workNamespace string, cluster *clusterv1.ManagedCluster, existingWorks []*workapiv1.ManifestWork, - addon *addonapiv1alpha1.ManagedClusterAddOn) (appliedWorks, deleteWorks []*workapiv1.ManifestWork, err error) { - var appliedType string - var addonWorkBuilder *addonWorksBuilder - - agentAddon := c.agentAddons[addon.Name] - if agentAddon == nil { - return nil, nil, fmt.Errorf("failed to get agentAddon") - } + addon *addonapiv1alpha1.ManagedClusterAddOn) (appliedWorks, deleteWorks []*workapiv1.ManifestWork, err error) + +func (c *addonDeployController) buildDeployManifestWorksFunc(addonWorkBuilder *addonWorksBuilder, appliedType string) buildDeployWorkFunc { + return func( + workNamespace string, + cluster *clusterv1.ManagedCluster, existingWorks []*workapiv1.ManifestWork, + addon *addonapiv1alpha1.ManagedClusterAddOn) (appliedWorks, deleteWorks []*workapiv1.ManifestWork, err error) { + agentAddon := c.agentAddons[addon.Name] + if agentAddon == nil { + return nil, nil, fmt.Errorf("failed to get agentAddon") + } - switch installMode { - case constants.InstallModeHosted: - appliedType = addonapiv1alpha1.ManagedClusterAddOnHostingManifestApplied - addonWorkBuilder = newHostingAddonWorksBuilder(agentAddon.GetAgentAddonOptions().HostedModeEnabled, c.workBuilder) - case constants.InstallModeDefault: - appliedType = addonapiv1alpha1.ManagedClusterAddOnManifestApplied - addonWorkBuilder = newAddonWorksBuilder(agentAddon.GetAgentAddonOptions().HostedModeEnabled, c.workBuilder) - default: - return nil, nil, fmt.Errorf("invalid install mode %v", installMode) - } + objects, err := agentAddon.Manifests(cluster, addon) + if err != nil { + meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ + Type: appliedType, + Status: metav1.ConditionFalse, + Reason: addonapiv1alpha1.AddonManifestAppliedReasonWorkApplyFailed, + Message: fmt.Sprintf("failed to get manifest from agent interface: %v", err), + }) + return nil, nil, err + } + if len(objects) == 0 { + return nil, nil, nil + } - objects, err := agentAddon.Manifests(cluster, addon) - if err != nil { - meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ - Type: appliedType, - Status: metav1.ConditionFalse, - Reason: addonapiv1alpha1.AddonManifestAppliedReasonWorkApplyFailed, - Message: fmt.Sprintf("failed to get manifest from agent interface: %v", err), - }) - return nil, nil, err - } - if len(objects) == 0 { - return nil, nil, nil - } + // this is to retrieve the intended mode of the addon. + var mode string + if agentAddon.GetAgentAddonOptions().HostedModeInfoFunc == nil { + mode = constants.InstallModeDefault + } else { + mode, _ = agentAddon.GetAgentAddonOptions().HostedModeInfoFunc(addon, cluster) + } - manifestOptions := getManifestConfigOption(agentAddon, cluster, addon) - existingWorksCopy := []workapiv1.ManifestWork{} - for _, work := range existingWorks { - existingWorksCopy = append(existingWorksCopy, *work) - } - appliedWorks, deleteWorks, err = addonWorkBuilder.BuildDeployWorks(workNamespace, addon, existingWorksCopy, objects, manifestOptions) - if err != nil { - meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ - Type: appliedType, - Status: metav1.ConditionFalse, - Reason: addonapiv1alpha1.AddonManifestAppliedReasonWorkApplyFailed, - Message: fmt.Sprintf("failed to build manifestwork: %v", err), - }) - return nil, nil, err + manifestOptions := getManifestConfigOption(agentAddon, cluster, addon) + existingWorksCopy := []workapiv1.ManifestWork{} + for _, work := range existingWorks { + existingWorksCopy = append(existingWorksCopy, *work) + } + appliedWorks, deleteWorks, err = addonWorkBuilder.BuildDeployWorks( + mode, workNamespace, addon, existingWorksCopy, objects, manifestOptions) + if err != nil { + meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ + Type: appliedType, + Status: metav1.ConditionFalse, + Reason: addonapiv1alpha1.AddonManifestAppliedReasonWorkApplyFailed, + Message: fmt.Sprintf("failed to build manifestwork: %v", err), + }) + return nil, nil, err + } + return appliedWorks, deleteWorks, nil } - return appliedWorks, deleteWorks, nil } -func (c *addonDeployController) buildHookManifestWork(installMode, workNamespace string, - cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn) (*workapiv1.ManifestWork, error) { - var appliedType string - var addonWorkBuilder *addonWorksBuilder - - agentAddon := c.agentAddons[addon.Name] - if agentAddon == nil { - return nil, fmt.Errorf("failed to get agentAddon") - } - switch installMode { - case constants.InstallModeHosted: - appliedType = addonapiv1alpha1.ManagedClusterAddOnHostingManifestApplied - addonWorkBuilder = newHostingAddonWorksBuilder(agentAddon.GetAgentAddonOptions().HostedModeEnabled, c.workBuilder) - case constants.InstallModeDefault: - appliedType = addonapiv1alpha1.ManagedClusterAddOnManifestApplied - addonWorkBuilder = newAddonWorksBuilder(agentAddon.GetAgentAddonOptions().HostedModeEnabled, c.workBuilder) - default: - return nil, fmt.Errorf("invalid install mode %v", installMode) - } +type buildDeployHookFunc func( + workNamespace string, + cluster *clusterv1.ManagedCluster, + addon *addonapiv1alpha1.ManagedClusterAddOn) (*workapiv1.ManifestWork, error) - objects, err := agentAddon.Manifests(cluster, addon) - if err != nil { - meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ - Type: appliedType, - Status: metav1.ConditionFalse, - Reason: addonapiv1alpha1.AddonManifestAppliedReasonWorkApplyFailed, - Message: fmt.Sprintf("failed to get manifest from agent interface: %v", err), - }) - return nil, err - } - if len(objects) == 0 { - return nil, nil - } +func (c *addonDeployController) buildHookManifestWorkFunc(addonWorkBuilder *addonWorksBuilder, appliedType string) buildDeployHookFunc { + return func( + workNamespace string, + cluster *clusterv1.ManagedCluster, + addon *addonapiv1alpha1.ManagedClusterAddOn) (*workapiv1.ManifestWork, error) { + agentAddon := c.agentAddons[addon.Name] + if agentAddon == nil { + return nil, fmt.Errorf("failed to get agentAddon") + } - hookWork, err := addonWorkBuilder.BuildHookWork(workNamespace, addon, objects) - if err != nil { - meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ - Type: appliedType, - Status: metav1.ConditionFalse, - Reason: addonapiv1alpha1.AddonManifestAppliedReasonWorkApplyFailed, - Message: fmt.Sprintf("failed to build manifestwork: %v", err), - }) - return nil, err + objects, err := agentAddon.Manifests(cluster, addon) + if err != nil { + meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ + Type: appliedType, + Status: metav1.ConditionFalse, + Reason: addonapiv1alpha1.AddonManifestAppliedReasonWorkApplyFailed, + Message: fmt.Sprintf("failed to get manifest from agent interface: %v", err), + }) + return nil, err + } + if len(objects) == 0 { + return nil, nil + } + + // this is to retrieve the intended mode of the addon. + var mode string + if agentAddon.GetAgentAddonOptions().HostedModeInfoFunc == nil { + mode = constants.InstallModeDefault + } else { + mode, _ = agentAddon.GetAgentAddonOptions().HostedModeInfoFunc(addon, cluster) + } + hookWork, err := addonWorkBuilder.BuildHookWork(mode, workNamespace, addon, objects) + if err != nil { + meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ + Type: appliedType, + Status: metav1.ConditionFalse, + Reason: addonapiv1alpha1.AddonManifestAppliedReasonWorkApplyFailed, + Message: fmt.Sprintf("failed to build manifestwork: %v", err), + }) + return nil, err + } + return hookWork, nil } - return hookWork, nil } diff --git a/pkg/addonmanager/controllers/agentdeploy/default_hook_sync.go b/pkg/addonmanager/controllers/agentdeploy/default_hook_sync.go index bcca12205..03e45f9b1 100644 --- a/pkg/addonmanager/controllers/agentdeploy/default_hook_sync.go +++ b/pkg/addonmanager/controllers/agentdeploy/default_hook_sync.go @@ -10,15 +10,13 @@ import ( clusterv1 "open-cluster-management.io/api/cluster/v1" workapiv1 "open-cluster-management.io/api/work/v1" - "open-cluster-management.io/addon-framework/pkg/addonmanager/constants" "open-cluster-management.io/addon-framework/pkg/agent" "open-cluster-management.io/addon-framework/pkg/basecontroller/factory" ) type defaultHookSyncer struct { - buildWorks func(installMode, workNamespace string, cluster *clusterv1.ManagedCluster, - addon *addonapiv1alpha1.ManagedClusterAddOn) (*workapiv1.ManifestWork, error) - applyWork func(ctx context.Context, appliedType string, + buildWorks buildDeployHookFunc + applyWork func(ctx context.Context, appliedType string, work *workapiv1.ManifestWork, addon *addonapiv1alpha1.ManagedClusterAddOn) (*workapiv1.ManifestWork, error) agentAddon agent.AgentAddon } @@ -29,7 +27,7 @@ func (s *defaultHookSyncer) sync(ctx context.Context, addon *addonapiv1alpha1.ManagedClusterAddOn) (*addonapiv1alpha1.ManagedClusterAddOn, error) { deployWorkNamespace := addon.Namespace - hookWork, err := s.buildWorks(constants.InstallModeDefault, deployWorkNamespace, cluster, addon) + hookWork, err := s.buildWorks(deployWorkNamespace, cluster, addon) if err != nil { return addon, err } diff --git a/pkg/addonmanager/controllers/agentdeploy/default_sync.go b/pkg/addonmanager/controllers/agentdeploy/default_sync.go index c53c76ab9..621851726 100644 --- a/pkg/addonmanager/controllers/agentdeploy/default_sync.go +++ b/pkg/addonmanager/controllers/agentdeploy/default_sync.go @@ -8,14 +8,12 @@ import ( clusterv1 "open-cluster-management.io/api/cluster/v1" workapiv1 "open-cluster-management.io/api/work/v1" - "open-cluster-management.io/addon-framework/pkg/addonmanager/constants" "open-cluster-management.io/addon-framework/pkg/agent" "open-cluster-management.io/addon-framework/pkg/basecontroller/factory" ) type defaultSyncer struct { - buildWorks func(installMode, workNamespace string, cluster *clusterv1.ManagedCluster, existingWorks []*workapiv1.ManifestWork, - addon *addonapiv1alpha1.ManagedClusterAddOn) (appliedWorks, deleteWorks []*workapiv1.ManifestWork, err error) + buildWorks buildDeployWorkFunc applyWork func(ctx context.Context, appliedType string, work *workapiv1.ManifestWork, addon *addonapiv1alpha1.ManagedClusterAddOn) (*workapiv1.ManifestWork, error) @@ -54,7 +52,7 @@ func (s *defaultSyncer) sync(ctx context.Context, return addon, err } - deployWorks, deleteWorks, err := s.buildWorks(constants.InstallModeDefault, deployWorkNamespace, cluster, currentWorks, addon) + deployWorks, deleteWorks, err := s.buildWorks(deployWorkNamespace, cluster, currentWorks, addon) if err != nil { return addon, err } diff --git a/pkg/addonmanager/controllers/agentdeploy/hosted_hook_sync.go b/pkg/addonmanager/controllers/agentdeploy/hosted_hook_sync.go index 98b14f80a..3ee5eb47b 100644 --- a/pkg/addonmanager/controllers/agentdeploy/hosted_hook_sync.go +++ b/pkg/addonmanager/controllers/agentdeploy/hosted_hook_sync.go @@ -18,8 +18,7 @@ import ( ) type hostedHookSyncer struct { - buildWorks func(installMode, workNamespace string, cluster *clusterv1.ManagedCluster, - addon *addonapiv1alpha1.ManagedClusterAddOn) (*workapiv1.ManifestWork, error) + buildWorks buildDeployHookFunc applyWork func(ctx context.Context, appliedType string, work *workapiv1.ManifestWork, addon *addonapiv1alpha1.ManagedClusterAddOn) (*workapiv1.ManifestWork, error) @@ -43,7 +42,10 @@ func (s *hostedHookSyncer) sync(ctx context.Context, return addon, nil } - installMode, hostingClusterName := constants.GetHostedModeInfo(addon.GetAnnotations()) + if s.agentAddon.GetAgentAddonOptions().HostedModeInfoFunc == nil { + return addon, nil + } + installMode, hostingClusterName := s.agentAddon.GetAgentAddonOptions().HostedModeInfoFunc(addon, cluster) if installMode != constants.InstallModeHosted { return addon, nil } @@ -70,7 +72,7 @@ func (s *hostedHookSyncer) sync(ctx context.Context, addonRemoveFinalizer(addon, addonapiv1alpha1.AddonHostingPreDeleteHookFinalizer) return addon, nil } - hookWork, err := s.buildWorks(constants.InstallModeHosted, hostingClusterName, cluster, addon) + hookWork, err := s.buildWorks(hostingClusterName, cluster, addon) if err != nil { return addon, err } diff --git a/pkg/addonmanager/controllers/agentdeploy/hosted_sync.go b/pkg/addonmanager/controllers/agentdeploy/hosted_sync.go index 55a0546f7..19a9bca4d 100644 --- a/pkg/addonmanager/controllers/agentdeploy/hosted_sync.go +++ b/pkg/addonmanager/controllers/agentdeploy/hosted_sync.go @@ -18,8 +18,7 @@ import ( ) type hostedSyncer struct { - buildWorks func(installMode, workNamespace string, cluster *clusterv1.ManagedCluster, existingWorks []*workapiv1.ManifestWork, - addon *addonapiv1alpha1.ManagedClusterAddOn) (appliedWorks, deleteWorks []*workapiv1.ManifestWork, err error) + buildWorks buildDeployWorkFunc applyWork func(ctx context.Context, appliedType string, work *workapiv1.ManifestWork, addon *addonapiv1alpha1.ManagedClusterAddOn) (*workapiv1.ManifestWork, error) @@ -42,7 +41,10 @@ func (s *hostedSyncer) sync(ctx context.Context, return addon, nil } - installMode, hostingClusterName := constants.GetHostedModeInfo(addon.GetAnnotations()) + if s.agentAddon.GetAgentAddonOptions().HostedModeInfoFunc == nil { + return addon, nil + } + installMode, hostingClusterName := s.agentAddon.GetAgentAddonOptions().HostedModeInfoFunc(addon, cluster) if installMode != constants.InstallModeHosted { // the installMode is changed from hosted to default, cleanup the hosting resources if err := s.cleanupDeployWork(ctx, addon); err != nil { @@ -115,7 +117,7 @@ func (s *hostedSyncer) sync(ctx context.Context, return addon, err } - deployWorks, deleteWorks, err := s.buildWorks(constants.InstallModeHosted, hostingClusterName, cluster, currentWorks, addon) + deployWorks, deleteWorks, err := s.buildWorks(hostingClusterName, cluster, currentWorks, addon) if err != nil { return addon, err } diff --git a/pkg/addonmanager/controllers/agentdeploy/hosted_sync_test.go b/pkg/addonmanager/controllers/agentdeploy/hosted_sync_test.go index 0979a75e7..f2f4dbd0b 100644 --- a/pkg/addonmanager/controllers/agentdeploy/hosted_sync_test.go +++ b/pkg/addonmanager/controllers/agentdeploy/hosted_sync_test.go @@ -42,8 +42,9 @@ func (t *testHostedAgent) Manifests(cluster *clusterv1.ManagedCluster, addon *ad func (t *testHostedAgent) GetAgentAddonOptions() agent.AgentAddonOptions { return agent.AgentAddonOptions{ - AddonName: t.name, - HostedModeEnabled: true, + AddonName: t.name, + HostedModeEnabled: true, + HostedModeInfoFunc: constants.GetHostedModeInfo, } } diff --git a/pkg/addonmanager/controllers/agentdeploy/utils.go b/pkg/addonmanager/controllers/agentdeploy/utils.go index a258369ea..7df50357a 100644 --- a/pkg/addonmanager/controllers/agentdeploy/utils.go +++ b/pkg/addonmanager/controllers/agentdeploy/utils.go @@ -260,19 +260,16 @@ func (m *managedManifest) preDeleteHookManifestWorkName(addonNamespace, addonNam // BuildDeployWorks returns the deploy manifestWorks. if there is no manifest need // to deploy, will return nil. -func (b *addonWorksBuilder) BuildDeployWorks(addonWorkNamespace string, +func (b *addonWorksBuilder) BuildDeployWorks(installMode, addonWorkNamespace string, addon *addonapiv1alpha1.ManagedClusterAddOn, existingWorks []workapiv1.ManifestWork, objects []runtime.Object, manifestOptions []workapiv1.ManifestConfigOption) (deployWorks, deleteWorks []*workapiv1.ManifestWork, err error) { var deployObjects []runtime.Object - var owner *metav1.OwnerReference - installMode, _ := constants.GetHostedModeInfo(addon.GetAnnotations()) - // This owner is only added to the manifestWork deployed in managed cluster ns. // the manifestWork in managed cluster ns is cleaned up via the addon ownerRef, so need to add the owner. // the manifestWork in hosting cluster ns is cleaned up by its controller since it and its addon cross ns. - owner = metav1.NewControllerRef(addon, addonapiv1alpha1.GroupVersion.WithKind("ManagedClusterAddOn")) + owner := metav1.NewControllerRef(addon, addonapiv1alpha1.GroupVersion.WithKind("ManagedClusterAddOn")) var deletionOrphaningRules []workapiv1.OrphaningRule for _, object := range objects { @@ -328,18 +325,13 @@ func (b *addonWorksBuilder) BuildDeployWorks(addonWorkNamespace string, // BuildHookWork returns the preDelete manifestWork, if there is no manifest need // to deploy, will return nil. -func (b *addonWorksBuilder) BuildHookWork(addonWorkNamespace string, +func (b *addonWorksBuilder) BuildHookWork(installMode, addonWorkNamespace string, addon *addonapiv1alpha1.ManagedClusterAddOn, objects []runtime.Object) (hookWork *workapiv1.ManifestWork, err error) { var hookManifests []workapiv1.Manifest var hookManifestConfigs []workapiv1.ManifestConfigOption - var owner *metav1.OwnerReference - installMode, _ := constants.GetHostedModeInfo(addon.GetAnnotations()) - // only set addon as the owner of works in default mode. should not set owner in hosted mode. - if installMode == constants.InstallModeDefault { - owner = metav1.NewControllerRef(addon, addonapiv1alpha1.GroupVersion.WithKind("ManagedClusterAddOn")) - } + owner := metav1.NewControllerRef(addon, addonapiv1alpha1.GroupVersion.WithKind("ManagedClusterAddOn")) for _, object := range objects { deployable, err := b.processor.deployable(b.hostedModeEnabled, installMode, object) @@ -367,7 +359,9 @@ func (b *addonWorksBuilder) BuildHookWork(addonWorkNamespace string, } hookWork = newManifestWork(addon.Namespace, addon.Name, addonWorkNamespace, hookManifests, b.processor.preDeleteHookManifestWorkName) - if owner != nil { + + // This owner is only added to the manifestWork deployed in managed cluster ns. + if addon.Namespace == addonWorkNamespace { hookWork.OwnerReferences = []metav1.OwnerReference{*owner} } hookWork.Spec.ManifestConfigs = hookManifestConfigs diff --git a/pkg/agent/inteface.go b/pkg/agent/inteface.go index 3229237b0..d4c9ec1d9 100644 --- a/pkg/agent/inteface.go +++ b/pkg/agent/inteface.go @@ -63,13 +63,16 @@ type AgentAddonOptions struct { // +optional HostedModeEnabled bool + // HostedModeInfoFunc returns whether an addon is in hosted mode, and its hosting cluster. + HostedModeInfoFunc func(addon *addonapiv1alpha1.ManagedClusterAddOn, cluster *clusterv1.ManagedCluster) (string, string) + // SupportedConfigGVRs is a list of addon supported configuration GroupVersionResource // each configuration GroupVersionResource should be unique SupportedConfigGVRs []schema.GroupVersionResource // AgentDeployTriggerClusterFilter defines the filter func to trigger the agent deploy/redploy when cluster info is // changed. Addons that need information from the ManagedCluster resource when deploying the agent should use this - // field to set what information they need, otherwise the expected/up-to-date agent may be deployed delayed since + // field to set what information they need, otherwise the expected/up-to-date agent may be deployed updates since // the default filter func returns false when the ManagedCluster resource is updated. // // For example, the agentAddon needs information from the ManagedCluster annotation, it can set the filter function diff --git a/test/integration/agent_hosting_deploy_test.go b/test/integration/agent_hosting_deploy_test.go index 6b22aba59..2b8198e0c 100644 --- a/test/integration/agent_hosting_deploy_test.go +++ b/test/integration/agent_hosting_deploy_test.go @@ -419,4 +419,63 @@ var _ = ginkgo.Describe("Agent deploy", func() { }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) }) + ginkgo.It("Should deploy and delete agent using customized hosted info func", func() { + obj := &unstructured.Unstructured{} + err := obj.UnmarshalJSON([]byte(deploymentHostingJson)) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + testHostedAddonImpl.manifests[managedClusterName] = []runtime.Object{obj} + testHostedAddonImpl.prober = &agent.HealthProber{ + Type: agent.HealthProberTypeWork, + } + testHostedAddonImpl.hostInfoFn = func(addon *addonapiv1alpha1.ManagedClusterAddOn, cluster *clusterv1.ManagedCluster) (string, string) { + return constants.InstallModeHosted, hostingClusterName + } + + addon := &addonapiv1alpha1.ManagedClusterAddOn{ + ObjectMeta: metav1.ObjectMeta{ + Name: testHostedAddonImpl.name, + Namespace: managedClusterName, + }, + Spec: addonapiv1alpha1.ManagedClusterAddOnSpec{ + InstallNamespace: "default", + }, + } + createManagedClusterAddOnwithOwnerRefs(managedClusterName, addon, cma) + + gomega.Eventually(func() error { + work, err := hubWorkClient.WorkV1().ManifestWorks(hostingClusterName).Get(context.Background(), + hostingManifestWorkName, metav1.GetOptions{}) + if err != nil { + return err + } + + if len(work.Spec.Workload.Manifests) != 1 { + return fmt.Errorf("Unexpected number of work manifests") + } + + if apiequality.Semantic.DeepEqual(work.Spec.Workload.Manifests[0].Raw, []byte(deploymentHostingJson)) { + return fmt.Errorf("expected manifest is no correct, get %v", work.Spec.Workload.Manifests[0].Raw) + } + + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + // delete managedclusteraddon + err = hubAddonClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Delete( + context.Background(), testHostedAddonImpl.name, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Eventually(func() bool { + _, err := hubWorkClient.WorkV1().ManifestWorks(hostingClusterName).Get(context.Background(), + hostingManifestWorkName, metav1.GetOptions{}) + return errors.IsNotFound(err) + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + + _, err = hubAddonClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Get( + context.Background(), testHostedAddonImpl.name, metav1.GetOptions{}) + if err != nil { + gomega.Expect(errors.IsNotFound(err)).To(gomega.BeTrue()) + } + testHostedAddonImpl.hostInfoFn = constants.GetHostedModeInfo + }) + }) diff --git a/test/integration/suite_test.go b/test/integration/suite_test.go index 9c0600c88..f192ef3c5 100644 --- a/test/integration/suite_test.go +++ b/test/integration/suite_test.go @@ -7,6 +7,7 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" + "open-cluster-management.io/addon-framework/pkg/addonmanager/constants" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -91,6 +92,7 @@ var _ = ginkgo.BeforeSuite(func(done ginkgo.Done) { manifests: map[string][]runtime.Object{}, registrations: map[string][]addonapiv1alpha1.RegistrationConfig{}, hostedModeEnabled: true, + hostInfoFn: constants.GetHostedModeInfo, } testInstallByLableAddonImpl = &testAddon{ @@ -141,6 +143,7 @@ type testAddon struct { cert []byte prober *agent.HealthProber hostedModeEnabled bool + hostInfoFn func(addon *addonapiv1alpha1.ManagedClusterAddOn, cluster *clusterv1.ManagedCluster) (string, string) supportedConfigGVRs []schema.GroupVersionResource } @@ -153,6 +156,7 @@ func (t *testAddon) GetAgentAddonOptions() agent.AgentAddonOptions { AddonName: t.name, HealthProber: t.prober, HostedModeEnabled: t.hostedModeEnabled, + HostedModeInfoFunc: t.hostInfoFn, SupportedConfigGVRs: t.supportedConfigGVRs, }