From ed7b798b6ca82e4e9a1cf3b7b7e0b7316f4c3e4c Mon Sep 17 00:00:00 2001 From: morvencao Date: Wed, 3 Apr 2024 11:32:05 +0000 Subject: [PATCH] add addon example with cloudevents. Signed-off-by: morvencao --- .github/workflows/go-presubmit.yml | 32 ++ Makefile | 18 + build/Dockerfile.example | 1 + cmd/example/busybox/main.go | 2 +- cmd/example/helloworld/main.go | 14 +- cmd/example/helloworld_cloudevents/main.go | 145 +++++ cmd/example/helloworld_helm/main.go | 14 +- cmd/example/helloworld_hosted/main.go | 14 +- .../helloworld-cloudevents/kustomization.yaml | 28 + .../resources/addon_deployment_config.yaml | 6 + .../resources/cluster_role.yaml | 50 ++ .../resources/cluster_role_binding.yaml | 12 + ...ld_cloudevents_clustermanagementaddon.yaml | 21 + .../helloworld_cloudevents_controller.yaml | 43 ++ .../resources/managed_clusterset_binding.yaml | 6 + .../resources/placement.yaml | 10 + .../resources/service_account.yaml | 4 + .../resources/work-driver-config.yaml | 11 + examples/deploy/mqtt/mqtt-broker.yaml | 68 +++ examples/deploy/ocm-cloudevents/install.sh | 86 +++ examples/helloworld_cloudevents/helloworld.go | 95 ++++ .../helloworld_cloudevents/helloworld_test.go | 192 +++++++ .../templates/clusterrolebinding.yaml | 12 + .../manifests/templates/deployment.yaml | 68 +++ .../manifests/templates/serviceaccount.yaml | 5 + go.mod | 2 + go.sum | 4 +- .../controllers/agentdeploy/controller.go | 4 +- .../controllers/agentdeploy/utils.go | 18 +- pkg/addonmanager/manager.go | 18 +- test/e2ecloudevents/e2e_suite_test.go | 155 +++++ .../helloworld_cloudevents_test.go | 536 ++++++++++++++++++ test/integration/cloudevents/suite_test.go | 4 +- test/integration/kube/suite_test.go | 2 +- vendor/modules.txt | 3 +- .../pkg/cloudevents/generic/agentclient.go | 8 +- .../work/agent/handler/resourcehandler.go | 10 +- .../work/source/client/manifestwork.go | 13 +- 38 files changed, 1667 insertions(+), 67 deletions(-) create mode 100644 cmd/example/helloworld_cloudevents/main.go create mode 100644 examples/deploy/addon/helloworld-cloudevents/kustomization.yaml create mode 100644 examples/deploy/addon/helloworld-cloudevents/resources/addon_deployment_config.yaml create mode 100644 examples/deploy/addon/helloworld-cloudevents/resources/cluster_role.yaml create mode 100644 examples/deploy/addon/helloworld-cloudevents/resources/cluster_role_binding.yaml create mode 100644 examples/deploy/addon/helloworld-cloudevents/resources/helloworld_cloudevents_clustermanagementaddon.yaml create mode 100644 examples/deploy/addon/helloworld-cloudevents/resources/helloworld_cloudevents_controller.yaml create mode 100644 examples/deploy/addon/helloworld-cloudevents/resources/managed_clusterset_binding.yaml create mode 100644 examples/deploy/addon/helloworld-cloudevents/resources/placement.yaml create mode 100644 examples/deploy/addon/helloworld-cloudevents/resources/service_account.yaml create mode 100644 examples/deploy/addon/helloworld-cloudevents/resources/work-driver-config.yaml create mode 100644 examples/deploy/mqtt/mqtt-broker.yaml create mode 100755 examples/deploy/ocm-cloudevents/install.sh create mode 100644 examples/helloworld_cloudevents/helloworld.go create mode 100644 examples/helloworld_cloudevents/helloworld_test.go create mode 100644 examples/helloworld_cloudevents/manifests/templates/clusterrolebinding.yaml create mode 100644 examples/helloworld_cloudevents/manifests/templates/deployment.yaml create mode 100644 examples/helloworld_cloudevents/manifests/templates/serviceaccount.yaml create mode 100644 test/e2ecloudevents/e2e_suite_test.go create mode 100644 test/e2ecloudevents/helloworld_cloudevents_test.go diff --git a/.github/workflows/go-presubmit.yml b/.github/workflows/go-presubmit.yml index 7d4734d28..82de5a5f5 100644 --- a/.github/workflows/go-presubmit.yml +++ b/.github/workflows/go-presubmit.yml @@ -129,6 +129,38 @@ jobs: env: KUBECONFIG: /home/runner/.kube/config + e2e-cloudevents: + name: e2e-cloudevents + runs-on: ubuntu-latest + steps: + - name: checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 1 + path: go/src/open-cluster-management.io/addon-framework + - name: install Go + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: install imagebuilder + run: go install github.com/openshift/imagebuilder/cmd/imagebuilder@v1.2.3 + - name: addon-examples-image # will build and tag the examples image + run: imagebuilder --allow-pull -t quay.io/open-cluster-management/addon-examples:latest -t quay.io/ocm/addon-examples:latest -f ./build/Dockerfile.example . + - name: setup kind + uses: engineerd/setup-kind@v0.5.0 + with: + version: v0.11.1 + name: cluster1 + - name: Load image on the nodes of the cluster + run: | + kind load docker-image --name=cluster1 quay.io/open-cluster-management/addon-examples:latest + kind load docker-image --name=cluster1 quay.io/ocm/addon-examples:latest + - name: Run e2e test with cloudevents + run: | + make test-e2e-cloudevents + env: + KUBECONFIG: /home/runner/.kube/config + e2e-hosted: name: e2e-hosted runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 415d75d87..d42f025b8 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,9 @@ verify: verify-gocilint deploy-ocm: examples/deploy/ocm/install.sh +deploy-ocm-cloudevents: + examples/deploy/ocm-cloudevents/install.sh + deploy-hosted-ocm: examples/deploy/hosted-ocm/install.sh @@ -71,6 +74,12 @@ deploy-helloworld: ensure-kustomize $(KUSTOMIZE) build examples/deploy/addon/helloworld | $(KUBECTL) apply -f - mv examples/deploy/addon/helloworld/kustomization.yaml.tmp examples/deploy/addon/helloworld/kustomization.yaml +deploy-helloworld-cloudevents: ensure-kustomize + cp examples/deploy/addon/helloworld-cloudevents/kustomization.yaml examples/deploy/addon/helloworld-cloudevents/kustomization.yaml.tmp + cd examples/deploy/addon/helloworld-cloudevents && ../../../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/addon-examples=$(EXAMPLE_IMAGE_NAME) + $(KUSTOMIZE) build examples/deploy/addon/helloworld-cloudevents | $(KUBECTL) apply -f - + mv examples/deploy/addon/helloworld-cloudevents/kustomization.yaml.tmp examples/deploy/addon/helloworld-cloudevents/kustomization.yaml + deploy-helloworld-helm: ensure-kustomize cp examples/deploy/addon/helloworld-helm/kustomization.yaml examples/deploy/addon/helloworld-helm/kustomization.yaml.tmp cd examples/deploy/addon/helloworld-helm && ../../../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/addon-examples=$(EXAMPLE_IMAGE_NAME) @@ -111,6 +120,9 @@ undeploy-busybox: ensure-kustomize undeploy-helloworld: ensure-kustomize $(KUSTOMIZE) build examples/deploy/addon/helloworld | $(KUBECTL) delete --ignore-not-found -f - +undeploy-helloworld-cloudevents: ensure-kustomize + $(KUSTOMIZE) build examples/deploy/addon/helloworld-cloudevents | $(KUBECTL) delete --ignore-not-found -f - + undeploy-helloworld-helm: ensure-kustomize $(KUSTOMIZE) build examples/deploy/addon/helloworld-helm | $(KUBECTL) delete --ignore-not-found -f - @@ -129,6 +141,12 @@ build-e2e: test-e2e: build-e2e deploy-ocm deploy-helloworld deploy-helloworld-helm ./e2e.test -test.v -ginkgo.v +build-e2e-cloudevents: + go test -c ./test/e2ecloudevents + +test-e2e-cloudevents: build-e2e-cloudevents deploy-ocm-cloudevents deploy-helloworld-cloudevents + ./e2ecloudevents.test -test.v -ginkgo.v + build-hosted-e2e: go test -c ./test/e2ehosted diff --git a/build/Dockerfile.example b/build/Dockerfile.example index fafd0f4d7..04a821f8c 100644 --- a/build/Dockerfile.example +++ b/build/Dockerfile.example @@ -8,6 +8,7 @@ RUN make build --warn-undefined-variables FROM registry.access.redhat.com/ubi8/ubi-minimal:latest COPY --from=builder /go/src/open-cluster-management.io/addon-framework/busybox / COPY --from=builder /go/src/open-cluster-management.io/addon-framework/helloworld / +COPY --from=builder /go/src/open-cluster-management.io/addon-framework/helloworld_cloudevents / COPY --from=builder /go/src/open-cluster-management.io/addon-framework/helloworld_helm / COPY --from=builder /go/src/open-cluster-management.io/addon-framework/helloworld_hosted / diff --git a/cmd/example/busybox/main.go b/cmd/example/busybox/main.go index 0e35f1cd0..8ff7a44d6 100644 --- a/cmd/example/busybox/main.go +++ b/cmd/example/busybox/main.go @@ -24,7 +24,7 @@ func main() { if err != nil { os.Exit(1) } - addonMgr, err := addonmanager.New(kubeConfig, addonmanager.NewManagerOptions()) + addonMgr, err := addonmanager.New(kubeConfig) if err != nil { klog.Errorf("unable to setup addon manager: %v", err) os.Exit(1) diff --git a/cmd/example/helloworld/main.go b/cmd/example/helloworld/main.go index 3de46ecf4..b933922a5 100644 --- a/cmd/example/helloworld/main.go +++ b/cmd/example/helloworld/main.go @@ -66,30 +66,22 @@ func newCommand() *cobra.Command { } func newControllerCommand() *cobra.Command { - o := addonmanager.NewManagerOptions() - c := &addManagerConfig{managerOptions: o} cmd := cmdfactory. - NewControllerCommandConfig("helloworld-addon-controller", version.Get(), c.runController). + NewControllerCommandConfig("helloworld-addon-controller", version.Get(), runController). NewCommand() cmd.Use = "controller" cmd.Short = "Start the addon controller" - o.AddFlags(cmd) return cmd } -// addManagerConfig holds configuration for addon manager -type addManagerConfig struct { - managerOptions *addonmanager.ManagerOptions -} - -func (c *addManagerConfig) runController(ctx context.Context, kubeConfig *rest.Config) error { +func runController(ctx context.Context, kubeConfig *rest.Config) error { addonClient, err := addonv1alpha1client.NewForConfig(kubeConfig) if err != nil { return err } - mgr, err := addonmanager.New(kubeConfig, c.managerOptions) + mgr, err := addonmanager.New(kubeConfig) if err != nil { return err } diff --git a/cmd/example/helloworld_cloudevents/main.go b/cmd/example/helloworld_cloudevents/main.go new file mode 100644 index 000000000..23e0bfc4d --- /dev/null +++ b/cmd/example/helloworld_cloudevents/main.go @@ -0,0 +1,145 @@ +package main + +import ( + "context" + goflag "flag" + "fmt" + "math/rand" + "os" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/rest" + utilflag "k8s.io/component-base/cli/flag" + logs "k8s.io/component-base/logs" + "k8s.io/klog/v2" + addonv1alpha1client "open-cluster-management.io/api/client/addon/clientset/versioned" + + "open-cluster-management.io/addon-framework/examples/helloworld_agent" + "open-cluster-management.io/addon-framework/examples/helloworld_cloudevents" + "open-cluster-management.io/addon-framework/pkg/addonfactory" + "open-cluster-management.io/addon-framework/pkg/addonmanager" + cmdfactory "open-cluster-management.io/addon-framework/pkg/cmd/factory" + "open-cluster-management.io/addon-framework/pkg/utils" + "open-cluster-management.io/addon-framework/pkg/version" +) + +func main() { + rand.Seed(time.Now().UTC().UnixNano()) + + pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc) + pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) + + logs.AddFlags(pflag.CommandLine) + logs.InitLogs() + defer logs.FlushLogs() + + command := newCommand() + if err := command.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +func newCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "addon", + Short: "helloworld example addon", + Run: func(cmd *cobra.Command, args []string) { + if err := cmd.Help(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + } + os.Exit(1) + }, + } + + if v := version.Get().String(); len(v) == 0 { + cmd.Version = "" + } else { + cmd.Version = v + } + + cmd.AddCommand(newControllerCommand()) + cmd.AddCommand(helloworld_agent.NewAgentCommand(helloworld_cloudevents.AddonName)) + + return cmd +} + +func newControllerCommand() *cobra.Command { + o := addonmanager.NewManagerOptions() + c := &addManagerConfig{managerOptions: o} + cmd := cmdfactory. + NewControllerCommandConfig("helloworld-addon-controller", version.Get(), c.runController). + NewCommand() + cmd.Use = "controller" + cmd.Short = "Start the addon controller" + o.AddFlags(cmd) + + return cmd +} + +// addManagerConfig holds configuration for addon manager +type addManagerConfig struct { + managerOptions *addonmanager.ManagerOptions +} + +func (c *addManagerConfig) runController(ctx context.Context, kubeConfig *rest.Config) error { + addonClient, err := addonv1alpha1client.NewForConfig(kubeConfig) + if err != nil { + return err + } + + mgr, err := addonmanager.New(kubeConfig, c.managerOptions) + if err != nil { + return err + } + + registrationOption := helloworld_cloudevents.NewRegistrationOption( + kubeConfig, + helloworld_cloudevents.AddonName, + utilrand.String(5), + ) + + // Set agent install namespace from addon deployment config if it exists + registrationOption.AgentInstallNamespace = utils.AgentInstallNamespaceFromDeploymentConfigFunc( + utils.NewAddOnDeploymentConfigGetter(addonClient), + ) + + agentAddon, err := addonfactory.NewAgentAddonFactory(helloworld_cloudevents.AddonName, helloworld_cloudevents.FS, "manifests/templates"). + WithConfigGVRs(utils.AddOnDeploymentConfigGVR). + WithGetValuesFuncs( + helloworld_cloudevents.GetDefaultValues, + addonfactory.GetAddOnDeploymentConfigValues( + utils.NewAddOnDeploymentConfigGetter(addonClient), + addonfactory.ToAddOnDeploymentConfigValues, + addonfactory.ToImageOverrideValuesFunc("Image", helloworld_cloudevents.DefaultImage), + ), + ). + WithAgentRegistrationOption(registrationOption). + WithAgentInstallNamespace( + utils.AgentInstallNamespaceFromDeploymentConfigFunc( + utils.NewAddOnDeploymentConfigGetter(addonClient), + ), + ). + WithAgentHealthProber(helloworld_cloudevents.AgentHealthProber()). + BuildTemplateAgentAddon() + if err != nil { + klog.Errorf("failed to build agent %v", err) + return err + } + + err = mgr.AddAgent(agentAddon) + if err != nil { + klog.Fatal(err) + } + + err = mgr.Start(ctx) + if err != nil { + klog.Fatal(err) + } + <-ctx.Done() + + return nil +} diff --git a/cmd/example/helloworld_helm/main.go b/cmd/example/helloworld_helm/main.go index 4609c9513..099523226 100644 --- a/cmd/example/helloworld_helm/main.go +++ b/cmd/example/helloworld_helm/main.go @@ -69,24 +69,16 @@ func newCommand() *cobra.Command { } func newControllerCommand() *cobra.Command { - o := addonmanager.NewManagerOptions() - c := &addManagerConfig{managerOptions: o} cmd := cmdfactory. - NewControllerCommandConfig("helloworldhelm-addon-controller", version.Get(), c.runController). + NewControllerCommandConfig("helloworldhelm-addon-controller", version.Get(), runController). NewCommand() cmd.Use = "controller" cmd.Short = "Start the addon controller" - o.AddFlags(cmd) return cmd } -// addManagerConfig holds configuration for addon manager -type addManagerConfig struct { - managerOptions *addonmanager.ManagerOptions -} - -func (c *addManagerConfig) runController(ctx context.Context, kubeConfig *rest.Config) error { +func runController(ctx context.Context, kubeConfig *rest.Config) error { kubeClient, err := kubernetes.NewForConfig(kubeConfig) if err != nil { return err @@ -97,7 +89,7 @@ func (c *addManagerConfig) runController(ctx context.Context, kubeConfig *rest.C return err } - mgr, err := addonmanager.New(kubeConfig, c.managerOptions) + mgr, err := addonmanager.New(kubeConfig) if err != nil { klog.Errorf("failed to new addon manager %v", err) return err diff --git a/cmd/example/helloworld_hosted/main.go b/cmd/example/helloworld_hosted/main.go index 21bf9545c..5a4376f31 100644 --- a/cmd/example/helloworld_hosted/main.go +++ b/cmd/example/helloworld_hosted/main.go @@ -67,25 +67,17 @@ func newCommand() *cobra.Command { } func newControllerCommand() *cobra.Command { - o := addonmanager.NewManagerOptions() - c := &addManagerConfig{managerOptions: o} cmd := cmdfactory. - NewControllerCommandConfig("helloworld-addon-controller", version.Get(), c.runController). + NewControllerCommandConfig("helloworld-addon-controller", version.Get(), runController). NewCommand() cmd.Use = "controller" cmd.Short = "Start the addon controller" - o.AddFlags(cmd) return cmd } -// addManagerConfig holds configuration for addon manager -type addManagerConfig struct { - managerOptions *addonmanager.ManagerOptions -} - -func (c *addManagerConfig) runController(ctx context.Context, kubeConfig *rest.Config) error { - mgr, err := addonmanager.New(kubeConfig, c.managerOptions) +func runController(ctx context.Context, kubeConfig *rest.Config) error { + mgr, err := addonmanager.New(kubeConfig) if err != nil { return err } diff --git a/examples/deploy/addon/helloworld-cloudevents/kustomization.yaml b/examples/deploy/addon/helloworld-cloudevents/kustomization.yaml new file mode 100644 index 000000000..6e31a765c --- /dev/null +++ b/examples/deploy/addon/helloworld-cloudevents/kustomization.yaml @@ -0,0 +1,28 @@ +namespace: open-cluster-management + +resources: +- resources/cluster_role.yaml +- resources/cluster_role_binding.yaml +- resources/service_account.yaml +- resources/managed_clusterset_binding.yaml +- resources/placement.yaml +- resources/addon_deployment_config.yaml +- resources/helloworld_cloudevents_clustermanagementaddon.yaml +- resources/helloworld_cloudevents_controller.yaml +- resources/work-driver-config.yaml + +images: +- name: quay.io/open-cluster-management/addon-examples + newName: quay.io/open-cluster-management/addon-examples + newTag: latest +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +vars: + - name: EXAMPLE_IMAGE_NAME + objref: + apiVersion: apps/v1 + kind: Deployment + name: helloworldcloudevents-controller + fieldref: + fieldpath: spec.template.spec.containers.[name=helloworldcloudevents-controller].image diff --git a/examples/deploy/addon/helloworld-cloudevents/resources/addon_deployment_config.yaml b/examples/deploy/addon/helloworld-cloudevents/resources/addon_deployment_config.yaml new file mode 100644 index 000000000..d29abd622 --- /dev/null +++ b/examples/deploy/addon/helloworld-cloudevents/resources/addon_deployment_config.yaml @@ -0,0 +1,6 @@ +apiVersion: addon.open-cluster-management.io/v1alpha1 +kind: AddOnDeploymentConfig +metadata: + name: global +spec: + agentInstallNamespace: open-cluster-management-agent-addon diff --git a/examples/deploy/addon/helloworld-cloudevents/resources/cluster_role.yaml b/examples/deploy/addon/helloworld-cloudevents/resources/cluster_role.yaml new file mode 100644 index 000000000..e3623c705 --- /dev/null +++ b/examples/deploy/addon/helloworld-cloudevents/resources/cluster_role.yaml @@ -0,0 +1,50 @@ + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: helloworldcloudevents-addon + rules: + - apiGroups: [""] + resources: ["configmaps", "events"] + verbs: ["get", "list", "watch", "create", "update", "delete", "deletecollection", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["get", "create"] + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests", "certificatesigningrequests/approval"] + verbs: ["get", "list", "watch", "create", "update"] + - apiGroups: ["certificates.k8s.io"] + resources: ["signers"] + verbs: ["approve"] + - apiGroups: ["cluster.open-cluster-management.io"] + resources: ["managedclusters"] + verbs: ["get", "list", "watch"] + - apiGroups: ["work.open-cluster-management.io"] + resources: ["manifestworks"] + verbs: ["create", "update", "get", "list", "watch", "delete", "deletecollection", "patch"] + - apiGroups: ["addon.open-cluster-management.io"] + resources: ["managedclusteraddons/finalizers"] + verbs: ["update"] + - apiGroups: [ "addon.open-cluster-management.io" ] + resources: [ "clustermanagementaddons/finalizers" ] + verbs: [ "update" ] + - apiGroups: [ "addon.open-cluster-management.io" ] + resources: [ "clustermanagementaddons/status" ] + verbs: ["update", "patch"] + - apiGroups: ["addon.open-cluster-management.io"] + resources: ["clustermanagementaddons"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["addon.open-cluster-management.io"] + resources: ["managedclusteraddons"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: ["addon.open-cluster-management.io"] + resources: ["managedclusteraddons/status"] + verbs: ["update", "patch"] + - apiGroups: ["addon.open-cluster-management.io"] + resources: ["addondeploymentconfigs"] + verbs: ["get", "list", "watch"] diff --git a/examples/deploy/addon/helloworld-cloudevents/resources/cluster_role_binding.yaml b/examples/deploy/addon/helloworld-cloudevents/resources/cluster_role_binding.yaml new file mode 100644 index 000000000..88a1823aa --- /dev/null +++ b/examples/deploy/addon/helloworld-cloudevents/resources/cluster_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: helloworldcloudevents-addon +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: helloworldcloudevents-addon +subjects: + - kind: ServiceAccount + name: helloworldcloudevents-addon-sa + namespace: open-cluster-management diff --git a/examples/deploy/addon/helloworld-cloudevents/resources/helloworld_cloudevents_clustermanagementaddon.yaml b/examples/deploy/addon/helloworld-cloudevents/resources/helloworld_cloudevents_clustermanagementaddon.yaml new file mode 100644 index 000000000..dcfec9820 --- /dev/null +++ b/examples/deploy/addon/helloworld-cloudevents/resources/helloworld_cloudevents_clustermanagementaddon.yaml @@ -0,0 +1,21 @@ +apiVersion: addon.open-cluster-management.io/v1alpha1 +kind: ClusterManagementAddOn +metadata: + name: helloworldcloudevents +spec: + addOnMeta: + displayName: helloworldcloudevents + description: "helloworldcloudevents is an example addon using cloudevents created by go template" + supportedConfigs: + - group: addon.open-cluster-management.io + resource: addondeploymentconfigs + installStrategy: + type: Placements + placements: + - name: global + namespace: open-cluster-management + configs: + - group: addon.open-cluster-management.io + resource: addondeploymentconfigs + name: global + namespace: open-cluster-management diff --git a/examples/deploy/addon/helloworld-cloudevents/resources/helloworld_cloudevents_controller.yaml b/examples/deploy/addon/helloworld-cloudevents/resources/helloworld_cloudevents_controller.yaml new file mode 100644 index 000000000..9bbb3bbe5 --- /dev/null +++ b/examples/deploy/addon/helloworld-cloudevents/resources/helloworld_cloudevents_controller.yaml @@ -0,0 +1,43 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: helloworldcloudevents-controller + labels: + app: helloworldcloudevents-controller +spec: + replicas: 1 + selector: + matchLabels: + app: helloworldcloudevents-controller + template: + metadata: + labels: + app: helloworldcloudevents-controller + spec: + serviceAccountName: helloworldcloudevents-addon-sa + containers: + - name: helloworldcloudevents-controller + image: quay.io/open-cluster-management/addon-examples + imagePullPolicy: IfNotPresent + env: + - name: EXAMPLE_IMAGE_NAME + value: $(EXAMPLE_IMAGE_NAME) + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + args: + - "/helloworld_cloudevents" + - "controller" + - "--work-driver=mqtt" + - "--work-driver-config=/var/run/secrets/hub/config.yaml" + - "--cloudevents-client-id=addon-manager-$(POD_NAME)" + - "--source-id=addon-manager" + volumeMounts: + - mountPath: /var/run/secrets/hub + name: workdriverconfig + readOnly: true + volumes: + - name: workdriverconfig + secret: + secretName: work-driver-config \ No newline at end of file diff --git a/examples/deploy/addon/helloworld-cloudevents/resources/managed_clusterset_binding.yaml b/examples/deploy/addon/helloworld-cloudevents/resources/managed_clusterset_binding.yaml new file mode 100644 index 000000000..7b1ce9ca7 --- /dev/null +++ b/examples/deploy/addon/helloworld-cloudevents/resources/managed_clusterset_binding.yaml @@ -0,0 +1,6 @@ +apiVersion: cluster.open-cluster-management.io/v1beta2 +kind: ManagedClusterSetBinding +metadata: + name: global +spec: + clusterSet: global diff --git a/examples/deploy/addon/helloworld-cloudevents/resources/placement.yaml b/examples/deploy/addon/helloworld-cloudevents/resources/placement.yaml new file mode 100644 index 000000000..da500bac2 --- /dev/null +++ b/examples/deploy/addon/helloworld-cloudevents/resources/placement.yaml @@ -0,0 +1,10 @@ +apiVersion: cluster.open-cluster-management.io/v1beta1 +kind: Placement +metadata: + name: global +spec: + clusterSets: + - global + predicates: + - requiredClusterSelector: + labelSelector: {} diff --git a/examples/deploy/addon/helloworld-cloudevents/resources/service_account.yaml b/examples/deploy/addon/helloworld-cloudevents/resources/service_account.yaml new file mode 100644 index 000000000..ba111648c --- /dev/null +++ b/examples/deploy/addon/helloworld-cloudevents/resources/service_account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: helloworldcloudevents-addon-sa diff --git a/examples/deploy/addon/helloworld-cloudevents/resources/work-driver-config.yaml b/examples/deploy/addon/helloworld-cloudevents/resources/work-driver-config.yaml new file mode 100644 index 000000000..5070201c9 --- /dev/null +++ b/examples/deploy/addon/helloworld-cloudevents/resources/work-driver-config.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: work-driver-config +stringData: + config.yaml: | + brokerHost: mosquitto.mqtt:1883 + topics: + sourceEvents: sources/addon-manager/clusters/+/sourceevents + agentEvents: sources/addon-manager/clusters/+/agentevents + sourceBroadcast: sources/addon-manager/sourcebroadcast diff --git a/examples/deploy/mqtt/mqtt-broker.yaml b/examples/deploy/mqtt/mqtt-broker.yaml new file mode 100644 index 000000000..afa7e657e --- /dev/null +++ b/examples/deploy/mqtt/mqtt-broker.yaml @@ -0,0 +1,68 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: mqtt +--- +apiVersion: v1 +kind: Service +metadata: + name: mosquitto + namespace: mqtt +spec: + ports: + - name: mosquitto + protocol: TCP + port: 1883 + targetPort: 1883 + selector: + name: mosquitto + sessionAffinity: None + type: ClusterIP +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: mosquitto + namespace: mqtt +spec: + replicas: 1 + selector: + matchLabels: + name: mosquitto + strategy: + type: Recreate + template: + metadata: + labels: + name: mosquitto + spec: + containers: + - name: mosquitto + image: eclipse-mosquitto:2.0.18 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 1883 + name: mosquitto + volumeMounts: + - name: mosquitto-persistent-storage + mountPath: /mosquitto/data + - name: mosquitto-config + mountPath: /mosquitto/config/mosquitto.conf + subPath: mosquitto.conf + volumes: + - name: mosquitto-persistent-storage + emptyDir: {} + - name: mosquitto-config + configMap: + name: mosquitto +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mosquitto + namespace: mqtt +data: + mosquitto.conf: | + listener 1883 0.0.0.0 + allow_anonymous true diff --git a/examples/deploy/ocm-cloudevents/install.sh b/examples/deploy/ocm-cloudevents/install.sh new file mode 100755 index 000000000..360b24309 --- /dev/null +++ b/examples/deploy/ocm-cloudevents/install.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +set -o nounset +set -o pipefail + +KUBECTL=${KUBECTL:-kubectl} + +BUILD_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +DEPLOY_DIR="$(dirname "$BUILD_DIR")" +EXAMPLE_DIR="$(dirname "$DEPLOY_DIR")" +REPO_DIR="$(dirname "$EXAMPLE_DIR")" +WORK_DIR="${REPO_DIR}/_output" +CLUSTERADM="${WORK_DIR}/bin/clusteradm" + +export PATH=$PATH:${WORK_DIR}/bin + +echo "############ Download clusteradm" +mkdir -p "${WORK_DIR}/bin" +wget -qO- https://github.com/open-cluster-management-io/clusteradm/releases/latest/download/clusteradm_${GOHOSTOS}_${GOHOSTARCH}.tar.gz | sudo tar -xvz -C ${WORK_DIR}/bin/ +chmod +x "${CLUSTERADM}" + +echo "############ Init hub" +${CLUSTERADM} init --wait --bundle-version=latest +joincmd=$(${CLUSTERADM} get token | grep clusteradm) + +echo "############ Init agent as cluster1" +$(echo ${joincmd} --force-internal-endpoint-lookup --wait --bundle-version=latest | sed "s//${MANAGED_CLUSTER_NAME}/g") + +echo "############ Accept join of cluster1" +${CLUSTERADM} accept --clusters ${MANAGED_CLUSTER_NAME} --wait + +echo "############ All-in-one env is installed successfully!!" + +echo "############ Deploy mqtt broker" +${KUBECTL} apply -f ${DEPLOY_DIR}/mqtt/mqtt-broker.yaml + +echo "############ Configure the work-agent" +${KUBECTL} -n open-cluster-management scale --replicas=0 deployment/klusterlet + +cat << EOF | ${KUBECTL} -n open-cluster-management-agent apply -f - +apiVersion: v1 +kind: Secret +metadata: + name: work-driver-config +stringData: + config.yaml: | + brokerHost: mosquitto.mqtt:1883 + topics: + sourceEvents: sources/addon-manager/clusters/${MANAGED_CLUSTER_NAME}/sourceevents + agentEvents: sources/addon-manager/clusters/${MANAGED_CLUSTER_NAME}/agentevents + agentBroadcast: clusters/${MANAGED_CLUSTER_NAME}/agentbroadcast +EOF + +# patch klusterlet-work-agent deployment to use mqtt as workload source driver +${KUBECTL} -n open-cluster-management-agent patch deployment/klusterlet-work-agent --type=json \ + -p='[ + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--cloudevents-client-codecs=manifestbundle"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--cloudevents-client-id=work-agent-$(POD_NAME)"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--workload-source-driver=mqtt"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--workload-source-config=/var/run/secrets/hub/config.yaml"} + ]' + +${KUBECTL} -n open-cluster-management-agent patch deployment/klusterlet-work-agent --type=json \ + -p='[{"op": "add", "path": "/spec/template/spec/volumes/-", "value": {"name": "workdriverconfig","secret": {"secretName": "work-driver-config"}}}]' + +${KUBECTL} -n open-cluster-management-agent patch deployment/klusterlet-work-agent --type=json \ + -p='[{"op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", "value": {"name": "workdriverconfig","mountPath": "/var/run/secrets/hub"}}]' + +${KUBECTL} -n open-cluster-management-agent scale --replicas=1 deployment/klusterlet-work-agent +${KUBECTL} -n open-cluster-management-agent rollout status deployment/klusterlet-work-agent --timeout=120s + +# TODO: add live probe for the work-agent to check if it is connected to the mqtt broker +isRunning=false +for i in {1..10}; do + if ${KUBECTL} -n open-cluster-management-agent logs deployment/klusterlet-work-agent | grep "subscribing to topics"; then + echo "klusterlet-work-agent is subscribe to topics from mqtt broker" + isRunning=true + break + fi + sleep 6 +done + +if [ "$isRunning" = false ]; then + echo "timeout waiting for klusterlet-work-agent to subscribe to topics from mqtt broker" + exit 1 +fi diff --git a/examples/helloworld_cloudevents/helloworld.go b/examples/helloworld_cloudevents/helloworld.go new file mode 100644 index 000000000..3ec453dd9 --- /dev/null +++ b/examples/helloworld_cloudevents/helloworld.go @@ -0,0 +1,95 @@ +package helloworld_cloudevents + +import ( + "embed" + "fmt" + "os" + + "k8s.io/client-go/rest" + "open-cluster-management.io/addon-framework/examples/rbac" + "open-cluster-management.io/addon-framework/pkg/addonfactory" + "open-cluster-management.io/addon-framework/pkg/agent" + "open-cluster-management.io/addon-framework/pkg/utils" + addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + clusterv1 "open-cluster-management.io/api/cluster/v1" + workapiv1 "open-cluster-management.io/api/work/v1" +) + +const ( + DefaultImage = "quay.io/open-cluster-management/addon-examples:latest" + AddonName = "helloworldcloudevents" + InstallationNamespace = "open-cluster-management-agent-addon" +) + +//go:embed manifests +//go:embed manifests/templates +var FS embed.FS + +func NewRegistrationOption(kubeConfig *rest.Config, addonName, agentName string) *agent.RegistrationOption { + return &agent.RegistrationOption{ + CSRConfigurations: agent.KubeClientSignerConfigurations(addonName, agentName), + CSRApproveCheck: utils.DefaultCSRApprover(agentName), + PermissionConfig: rbac.AddonRBAC(kubeConfig), + } +} + +func GetDefaultValues(cluster *clusterv1.ManagedCluster, + addon *addonapiv1alpha1.ManagedClusterAddOn) (addonfactory.Values, error) { + + image := os.Getenv("EXAMPLE_IMAGE_NAME") + if len(image) == 0 { + image = DefaultImage + } + + manifestConfig := struct { + KubeConfigSecret string + ClusterName string + Image string + }{ + KubeConfigSecret: fmt.Sprintf("%s-hub-kubeconfig", addon.Name), + ClusterName: cluster.Name, + Image: image, + } + + return addonfactory.StructToValues(manifestConfig), nil +} + +func AgentHealthProber() *agent.HealthProber { + return &agent.HealthProber{ + Type: agent.HealthProberTypeWork, + WorkProber: &agent.WorkHealthProber{ + ProbeFields: []agent.ProbeField{ + { + ResourceIdentifier: workapiv1.ResourceIdentifier{ + Group: "apps", + Resource: "deployments", + Name: "helloworldcloudevents-agent", + Namespace: InstallationNamespace, + }, + ProbeRules: []workapiv1.FeedbackRule{ + { + Type: workapiv1.WellKnownStatusType, + }, + }, + }, + }, + HealthCheck: func(identifier workapiv1.ResourceIdentifier, result workapiv1.StatusFeedbackResult) error { + if len(result.Values) == 0 { + return fmt.Errorf("no values are probed for deployment %s/%s", identifier.Namespace, identifier.Name) + } + for _, value := range result.Values { + if value.Name != "ReadyReplicas" { + continue + } + + if *value.Value.Integer >= 1 { + return nil + } + + return fmt.Errorf("readyReplica is %d for deployement %s/%s", *value.Value.Integer, identifier.Namespace, identifier.Name) + } + return fmt.Errorf("readyReplica is not probed") + }, + }, + } +} diff --git a/examples/helloworld_cloudevents/helloworld_test.go b/examples/helloworld_cloudevents/helloworld_test.go new file mode 100644 index 000000000..9939cd9be --- /dev/null +++ b/examples/helloworld_cloudevents/helloworld_test.go @@ -0,0 +1,192 @@ +package helloworld_cloudevents + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/klog/v2" + addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + fakeaddon "open-cluster-management.io/api/client/addon/clientset/versioned/fake" + clusterv1 "open-cluster-management.io/api/cluster/v1" + + "open-cluster-management.io/addon-framework/pkg/addonfactory" + "open-cluster-management.io/addon-framework/pkg/addonmanager/addontesting" + "open-cluster-management.io/addon-framework/pkg/utils" +) + +var ( + nodeSelector = map[string]string{"kubernetes.io/os": "linux"} + tolerations = []corev1.Toleration{{Key: "foo", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute}} +) + +func TestManifestAddonAgent(t *testing.T) { + cases := []struct { + name string + managedCluster *clusterv1.ManagedCluster + managedClusterAddOn *addonapiv1alpha1.ManagedClusterAddOn + configs []runtime.Object + verifyDeployment func(t *testing.T, objs []runtime.Object) + }{ + { + name: "no configs", + managedCluster: addontesting.NewManagedCluster("cluster1"), + managedClusterAddOn: addontesting.NewAddon("helloworldcloudevents", "cluster1"), + configs: []runtime.Object{}, + verifyDeployment: func(t *testing.T, objs []runtime.Object) { + deployment := findHelloWorldCloudEventsDeployment(objs) + if deployment == nil { + t.Fatalf("expected deployment, but failed") + } + + if deployment.Name != "helloworldcloudevents-agent" { + t.Errorf("unexpected deployment name %s", deployment.Name) + } + + if deployment.Namespace != addonfactory.AddonDefaultInstallNamespace { + t.Errorf("unexpected deployment namespace %s", deployment.Namespace) + } + + if deployment.Spec.Template.Spec.Containers[0].Image != DefaultImage { + t.Errorf("unexpected image %s", deployment.Spec.Template.Spec.Containers[0].Image) + } + }, + }, + { + name: "override image with annotation", + managedCluster: addontesting.NewManagedCluster("cluster1"), + managedClusterAddOn: func() *addonapiv1alpha1.ManagedClusterAddOn { + addon := addontesting.NewAddon("test", "cluster1") + addon.Annotations = map[string]string{ + "addon.open-cluster-management.io/values": `{"Image":"quay.io/test:test"}`} + return addon + }(), + configs: []runtime.Object{}, + verifyDeployment: func(t *testing.T, objs []runtime.Object) { + deployment := findHelloWorldCloudEventsDeployment(objs) + if deployment == nil { + t.Fatalf("expected deployment, but failed") + } + + if deployment.Name != "helloworldcloudevents-agent" { + t.Errorf("unexpected deployment name %s", deployment.Name) + } + + if deployment.Namespace != addonfactory.AddonDefaultInstallNamespace { + t.Errorf("unexpected deployment namespace %s", deployment.Namespace) + } + + if deployment.Spec.Template.Spec.Containers[0].Image != "quay.io/test:test" { + t.Errorf("unexpected image %s", deployment.Spec.Template.Spec.Containers[0].Image) + } + }, + }, + { + name: "with addon deployment config", + managedCluster: addontesting.NewManagedCluster("cluster1"), + managedClusterAddOn: func() *addonapiv1alpha1.ManagedClusterAddOn { + addon := addontesting.NewAddon("test", "cluster1") + addon.Status.ConfigReferences = []addonapiv1alpha1.ConfigReference{ + { + ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + Group: "addon.open-cluster-management.io", + Resource: "addondeploymentconfigs", + }, + ConfigReferent: addonapiv1alpha1.ConfigReferent{ + Namespace: "cluster1", + Name: "config", + }, + }, + } + return addon + }(), + configs: []runtime.Object{ + &addonapiv1alpha1.AddOnDeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "config", + Namespace: "cluster1", + }, + Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + NodePlacement: &addonapiv1alpha1.NodePlacement{ + Tolerations: tolerations, + NodeSelector: nodeSelector, + }, + }, + }, + }, + verifyDeployment: func(t *testing.T, objs []runtime.Object) { + deployment := findHelloWorldCloudEventsDeployment(objs) + if deployment == nil { + t.Fatalf("expected deployment, but failed") + } + + if deployment.Name != "helloworldcloudevents-agent" { + t.Errorf("unexpected deployment name %s", deployment.Name) + } + + if deployment.Namespace != addonfactory.AddonDefaultInstallNamespace { + t.Errorf("unexpected deployment namespace %s", deployment.Namespace) + } + + if deployment.Spec.Template.Spec.Containers[0].Image != DefaultImage { + t.Errorf("unexpected image %s", deployment.Spec.Template.Spec.Containers[0].Image) + } + + if !equality.Semantic.DeepEqual(deployment.Spec.Template.Spec.NodeSelector, nodeSelector) { + t.Errorf("unexpected nodeSeletcor %v", deployment.Spec.Template.Spec.NodeSelector) + } + + if !equality.Semantic.DeepEqual(deployment.Spec.Template.Spec.Tolerations, tolerations) { + t.Errorf("unexpected tolerations %v", deployment.Spec.Template.Spec.Tolerations) + } + }, + }, + } + + for _, c := range cases { + fakeAddonClient := fakeaddon.NewSimpleClientset(c.configs...) + + agentAddon, err := addonfactory.NewAgentAddonFactory(AddonName, FS, "manifests/templates"). + WithConfigGVRs(utils.AddOnDeploymentConfigGVR). + WithGetValuesFuncs( + GetDefaultValues, + addonfactory.GetAddOnDeploymentConfigValues( + addonfactory.NewAddOnDeploymentConfigGetter(fakeAddonClient), + addonfactory.ToAddOnDeploymentConfigValues, + ), + addonfactory.GetValuesFromAddonAnnotation, + ). + WithAgentRegistrationOption(NewRegistrationOption(nil, AddonName, utilrand.String(5))). + WithAgentHealthProber(AgentHealthProber()). + BuildTemplateAgentAddon() + if err != nil { + klog.Fatalf("failed to build agent %v", err) + } + + objects, err := agentAddon.Manifests(c.managedCluster, c.managedClusterAddOn) + if err != nil { + t.Fatalf("failed to get manifests %v", err) + } + + if len(objects) != 3 { + t.Fatalf("expected 3 manifests, but %v", objects) + } + + c.verifyDeployment(t, objects) + } +} + +func findHelloWorldCloudEventsDeployment(objs []runtime.Object) *appsv1.Deployment { + for _, obj := range objs { + switch obj := obj.(type) { + case *appsv1.Deployment: + return obj + } + } + + return nil +} diff --git a/examples/helloworld_cloudevents/manifests/templates/clusterrolebinding.yaml b/examples/helloworld_cloudevents/manifests/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..f0271f974 --- /dev/null +++ b/examples/helloworld_cloudevents/manifests/templates/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: helloworldcloudevents-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: helloworldcloudevents-agent-sa + namespace: {{ .AddonInstallNamespace }} diff --git a/examples/helloworld_cloudevents/manifests/templates/deployment.yaml b/examples/helloworld_cloudevents/manifests/templates/deployment.yaml new file mode 100644 index 000000000..f439d8ba7 --- /dev/null +++ b/examples/helloworld_cloudevents/manifests/templates/deployment.yaml @@ -0,0 +1,68 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: helloworldcloudevents-agent + namespace: {{ .AddonInstallNamespace }} + labels: + app: helloworldcloudevents-agent +spec: + replicas: 1 + selector: + matchLabels: + app: helloworldcloudevents-agent + template: + metadata: + labels: + app: helloworldcloudevents-agent + spec: + serviceAccountName: helloworldcloudevents-agent-sa +{{- if .NodeSelector }} + nodeSelector: + {{- range $key, $value := .NodeSelector }} + "{{ $key }}": "{{ $value }}" + {{- end }} +{{- end }} +{{- if .Tolerations }} + tolerations: + {{- range $toleration := .Tolerations }} + - key: "{{ $toleration.Key }}" + value: "{{ $toleration.Value }}" + effect: "{{ $toleration.Effect }}" + operator: "{{ $toleration.Operator }}" + {{- if $toleration.TolerationSeconds }} + tolerationSeconds: {{ $toleration.TolerationSeconds }} + {{- end }} + {{- end }} +{{- end }} + volumes: + - name: hub-config + secret: + secretName: {{ .KubeConfigSecret }} + containers: + - name: helloworldcloudevents-agent + image: {{ .Image }} + imagePullPolicy: IfNotPresent +{{- if or .HTTPProxy .HTTPSProxy}} + env: + {{- if .HTTPProxy }} + - name: HTTP_PROXY + value: {{ .HTTPProxy }} + {{- end }} + {{- if .HTTPSProxy }} + - name: HTTPS_PROXY + value: {{ .HTTPSProxy }} + {{- end }} + {{- if .NoProxy }} + - name: NO_PROXY + value: {{ .NoProxy }} + {{- end }} +{{- end }} + args: + - "/helloworld_cloudevents" + - "agent" + - "--hub-kubeconfig=/var/run/hub/kubeconfig" + - "--cluster-name={{ .ClusterName }}" + - "--addon-namespace={{ .AddonInstallNamespace }}" + volumeMounts: + - name: hub-config + mountPath: /var/run/hub diff --git a/examples/helloworld_cloudevents/manifests/templates/serviceaccount.yaml b/examples/helloworld_cloudevents/manifests/templates/serviceaccount.yaml new file mode 100644 index 000000000..0a2b1ecbf --- /dev/null +++ b/examples/helloworld_cloudevents/manifests/templates/serviceaccount.yaml @@ -0,0 +1,5 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: helloworldcloudevents-agent-sa + namespace: {{ .AddonInstallNamespace }} diff --git a/go.mod b/go.mod index f568af2de..409a4ea3c 100644 --- a/go.mod +++ b/go.mod @@ -135,3 +135,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +replace open-cluster-management.io/sdk-go => github.com/morvencao/ocm-sdk-go v0.0.0-20240411073141-7692feecb557 diff --git a/go.sum b/go.sum index fecfe4046..a0579bb7d 100644 --- a/go.sum +++ b/go.sum @@ -180,6 +180,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morvencao/ocm-sdk-go v0.0.0-20240411073141-7692feecb557 h1:er9Zc4jhqxvund50yaA8LLM1J7nrS/IfjwVm21g/4cI= +github.com/morvencao/ocm-sdk-go v0.0.0-20240411073141-7692feecb557/go.mod h1:sq+amR9Ls9JzMP5dypvlCx4jIGfDg45gicS67Z/MnlI= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -441,8 +443,6 @@ k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0g k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= open-cluster-management.io/api v0.13.0 h1:dlcJEZlNlE0DmSDctK2s7iWKg9l+Tgb0V78Z040nMuk= open-cluster-management.io/api v0.13.0/go.mod h1:CuCPEzXDvOyxBB0H1d1eSeajbHqaeGEKq9c63vQc63w= -open-cluster-management.io/sdk-go v0.13.1-0.20240321032811-7dbdd1b5c63d h1:mWrwuvY3K/MLfOLbwJB6VSNsdDol98fvK3hHXuDlxfQ= -open-cluster-management.io/sdk-go v0.13.1-0.20240321032811-7dbdd1b5c63d/go.mod h1:sq+amR9Ls9JzMP5dypvlCx4jIGfDg45gicS67Z/MnlI= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0= diff --git a/pkg/addonmanager/controllers/agentdeploy/controller.go b/pkg/addonmanager/controllers/agentdeploy/controller.go index e5e7bed3a..dd3ffc5c4 100644 --- a/pkg/addonmanager/controllers/agentdeploy/controller.go +++ b/pkg/addonmanager/controllers/agentdeploy/controller.go @@ -314,7 +314,8 @@ func (c *addonDeployController) sync(ctx context.Context, syncCtx factory.SyncCo // updateAddon updates finalizers and conditions of addon. // to avoid conflict updateAddon updates finalizers firstly if finalizers has change. func (c *addonDeployController) updateAddon(ctx context.Context, new, old *addonapiv1alpha1.ManagedClusterAddOn) error { - if !equality.Semantic.DeepEqual(new.GetFinalizers(), old.GetFinalizers()) { + if !equality.Semantic.DeepEqual(new.GetFinalizers(), old.GetFinalizers()) || + !equality.Semantic.DeepEqual(new.GetAnnotations(), old.GetAnnotations()) { _, err := c.addonClient.AddonV1alpha1().ManagedClusterAddOns(new.Namespace).Update(ctx, new, metav1.UpdateOptions{}) return err } @@ -330,7 +331,6 @@ func (c *addonDeployController) updateAddon(ctx context.Context, new, old *addon func (c *addonDeployController) applyWork(ctx context.Context, appliedType string, work *workapiv1.ManifestWork, addon *addonapiv1alpha1.ManagedClusterAddOn) (*workapiv1.ManifestWork, error) { - work, err := c.workApplier.Apply(ctx, work) if err != nil { meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ diff --git a/pkg/addonmanager/controllers/agentdeploy/utils.go b/pkg/addonmanager/controllers/agentdeploy/utils.go index 5a3ed98f5..5ebf07475 100644 --- a/pkg/addonmanager/controllers/agentdeploy/utils.go +++ b/pkg/addonmanager/controllers/agentdeploy/utils.go @@ -92,6 +92,9 @@ func newManifestWork(addonNamespace, addonName, clusterName string, manifests [] Labels: map[string]string{ addonapiv1alpha1.AddonLabelKey: addonName, }, + Annotations: map[string]string{ + common.CloudEventsDataTypeAnnotationKey: payload.ManifestBundleEventDataType.String(), + }, }, Spec: workapiv1.ManifestWorkSpec{ Workload: workapiv1.ManifestsTemplate{ @@ -317,12 +320,13 @@ func (b *addonWorksBuilder) BuildDeployWorks(installMode, addonWorkNamespace str return nil, nil, err } - // add the cloud event data type annotation - if annotations == nil { - annotations = make(map[string]string) + if len(annotations) == 0 { + annotations = map[string]string{ + common.CloudEventsDataTypeAnnotationKey: payload.ManifestBundleEventDataType.String(), + } + } else { + annotations[common.CloudEventsDataTypeAnnotationKey] = payload.ManifestBundleEventDataType.String() } - annotations[common.CloudEventsDataTypeAnnotationKey] = payload.ManifestBundleEventDataType.String() - annotations[common.CloudEventsGenerationAnnotationKey] = fmt.Sprintf("%d", addon.Generation) return b.workBuilder.Build(deployObjects, newAddonWorkObjectMeta(b.processor.manifestWorkNamePrefix(addon.Namespace, addon.Name), addon.Name, addon.Namespace, addonWorkNamespace, owner), @@ -373,10 +377,6 @@ func (b *addonWorksBuilder) BuildHookWork(installMode, addonWorkNamespace string if addon.Namespace == addonWorkNamespace { hookWork.OwnerReferences = []metav1.OwnerReference{*owner} } - hookWork.Annotations = map[string]string{ - common.CloudEventsDataTypeAnnotationKey: payload.ManifestBundleEventDataType.String(), - common.CloudEventsGenerationAnnotationKey: fmt.Sprintf("%d", addon.Generation), - } hookWork.Spec.ManifestConfigs = hookManifestConfigs if addon.Namespace != addonWorkNamespace { hookWork.Labels[addonapiv1alpha1.AddonNamespaceLabelKey] = addon.Namespace diff --git a/pkg/addonmanager/manager.go b/pkg/addonmanager/manager.go index 790e736a4..b8f5143fb 100644 --- a/pkg/addonmanager/manager.go +++ b/pkg/addonmanager/manager.go @@ -374,13 +374,21 @@ func (o *ManagerOptions) AddFlags(cmd *cobra.Command) { o.SourceID, "The ID of the source when publishing works with cloudevents") } -// New returns a new addon manager for creating addon agents. -func New(config *rest.Config, opts *ManagerOptions) (AddonManager, error) { - return &addonManager{ +// New returns a new addon manager with the given config and optional options +func New(config *rest.Config, opts ...*ManagerOptions) (AddonManager, error) { + addonManager := &addonManager{ config: config, - options: opts, syncContexts: []factory.SyncContext{}, addonConfigs: map[schema.GroupVersionResource]bool{}, addonAgents: map[string]agent.AgentAddon{}, - }, nil + } + if len(opts) > 0 { + // set options from the given options + addonManager.options = opts[0] + } else { + // set default options + addonManager.options = NewManagerOptions() + } + + return addonManager, nil } diff --git a/test/e2ecloudevents/e2e_suite_test.go b/test/e2ecloudevents/e2e_suite_test.go new file mode 100644 index 000000000..14766faf2 --- /dev/null +++ b/test/e2ecloudevents/e2e_suite_test.go @@ -0,0 +1,155 @@ +package e2ecloudevents + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + ginkgo "github.com/onsi/ginkgo" + gomega "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/retry" + + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + addonclient "open-cluster-management.io/api/client/addon/clientset/versioned" + clusterclient "open-cluster-management.io/api/client/cluster/clientset/versioned" + clusterv1 "open-cluster-management.io/api/cluster/v1" +) + +func TestE2E(t *testing.T) { + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, "E2E suite") +} + +var ( + managedClusterName string + hubKubeClient kubernetes.Interface + hubAddOnClient addonclient.Interface + hubClusterClient clusterclient.Interface + clusterCfg *rest.Config +) + +// This suite is sensitive to the following environment variables: +// +// - MANAGED_CLUSTER_NAME sets the name of the cluster +// - KUBECONFIG is the location of the kubeconfig file to use +var _ = ginkgo.BeforeSuite(func() { + kubeconfig := os.Getenv("KUBECONFIG") + managedClusterName = os.Getenv("MANAGED_CLUSTER_NAME") + if managedClusterName == "" { + managedClusterName = "cluster1" + } + err := func() error { + var err error + clusterCfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return err + } + + hubKubeClient, err = kubernetes.NewForConfig(clusterCfg) + if err != nil { + return err + } + + hubAddOnClient, err = addonclient.NewForConfig(clusterCfg) + if err != nil { + return err + } + + hubClusterClient, err = clusterclient.NewForConfig(clusterCfg) + + return err + }() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + var csrs *certificatesv1.CertificateSigningRequestList + // Waiting for the CSR for ManagedCluster to exist + err = wait.Poll(1*time.Second, 120*time.Second, func() (bool, error) { + var err error + csrs, err = hubKubeClient.CertificatesV1().CertificateSigningRequests().List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("open-cluster-management.io/cluster-name = %v", managedClusterName), + }) + if err != nil { + return false, err + } + + if len(csrs.Items) >= 1 { + return true, nil + } + + return false, nil + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + // Approving all pending CSRs + for i := range csrs.Items { + csr := &csrs.Items[i] + if !strings.HasPrefix(csr.Name, managedClusterName) { + continue + } + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + csr, err = hubKubeClient.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), csr.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + // make the e2e test idempotent to ease debugging in local environment + for _, condition := range csr.Status.Conditions { + if condition.Type == certificatesv1.CertificateApproved { + if condition.Status == corev1.ConditionTrue { + return nil + } + } + } + + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + Reason: "Approved by E2E", + Message: "Approved as part of Loopback e2e", + }) + _, err := hubKubeClient.CertificatesV1().CertificateSigningRequests().UpdateApproval(context.TODO(), csr.Name, csr, metav1.UpdateOptions{}) + return err + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + var managedCluster *clusterv1.ManagedCluster + // Waiting for ManagedCluster to exist + err = wait.Poll(1*time.Second, 120*time.Second, func() (bool, error) { + var err error + managedCluster, err = hubClusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), managedClusterName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + // Accepting ManagedCluster + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + var err error + managedCluster, err = hubClusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), managedCluster.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + managedCluster.Spec.HubAcceptsClient = true + managedCluster.Spec.LeaseDurationSeconds = 5 + _, err = hubClusterClient.ClusterV1().ManagedClusters().Update(context.TODO(), managedCluster, metav1.UpdateOptions{}) + return err + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) +}) diff --git a/test/e2ecloudevents/helloworld_cloudevents_test.go b/test/e2ecloudevents/helloworld_cloudevents_test.go new file mode 100644 index 000000000..178d2e35e --- /dev/null +++ b/test/e2ecloudevents/helloworld_cloudevents_test.go @@ -0,0 +1,536 @@ +package e2ecloudevents + +import ( + "context" + "fmt" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" +) + +const ( + eventuallyTimeout = 300 // seconds + eventuallyInterval = 1 // seconds +) + +const ( + addonName = "helloworldcloudevents" + addonInstallNamespace = "open-cluster-management-agent-addon" + deployConfigName = "deploy-config" + deployImageOverrideConfigName = "image-override-deploy-config" + deployProxyConfigName = "proxy-deploy-config" + deployAgentInstallNamespaceConfigName = "agent-install-namespace-deploy-config" +) + +var ( + nodeSelector = map[string]string{"kubernetes.io/os": "linux"} + tolerations = []corev1.Toleration{{Key: "foo", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute}} + registries = []addonapiv1alpha1.ImageMirror{ + { + Source: "quay.io/open-cluster-management/addon-examples", + Mirror: "quay.io/ocm/addon-examples", + }, + } + + proxyConfig = addonapiv1alpha1.ProxyConfig{ + // settings of http proxy will not impact the communicaiton between + // hub kube-apiserver and the add-on agent running on managed cluster. + HTTPProxy: "http://127.0.0.1:3128", + NoProxy: "example.com", + } + + agentInstallNamespaceConfig = "test-ns" +) + +var _ = ginkgo.Describe("install/uninstall helloworldcloudevents addons", func() { + ginkgo.BeforeEach(func() { + gomega.Eventually(func() error { + _, err := hubClusterClient.ClusterV1().ManagedClusters().Get(context.Background(), managedClusterName, metav1.GetOptions{}) + if err != nil { + return err + } + + _, err = hubKubeClient.CoreV1().Namespaces().Get(context.Background(), managedClusterName, metav1.GetOptions{}) + if err != nil { + return err + } + + testNs := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: agentInstallNamespaceConfig, + }, + } + _, err = hubKubeClient.CoreV1().Namespaces().Create(context.Background(), testNs, metav1.CreateOptions{}) + if err != nil { + if errors.IsAlreadyExists(err) { + return nil + } + return err + } + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.AfterEach(func() { + ginkgo.By("Clean up the customized agent install namespace after each case.") + gomega.Eventually(func() error { + _, err := hubKubeClient.CoreV1().Namespaces().Get(context.Background(), + agentInstallNamespaceConfig, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return nil + } + + if err == nil { + errd := hubKubeClient.CoreV1().Namespaces().Delete(context.Background(), + agentInstallNamespaceConfig, metav1.DeleteOptions{}) + if errd != nil { + return errd + } + return fmt.Errorf("ns is deleting, need re-check if namespace is not found") + } + + return err + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.It("addon should be worked", func() { + ginkgo.By("Make sure cma annotation managed by addon-manager is added") + gomega.Eventually(func() error { + cma, err := hubAddOnClient.AddonV1alpha1().ClusterManagementAddOns().Get(context.Background(), addonName, metav1.GetOptions{}) + if err != nil { + return err + } + + if cma.Annotations[addonapiv1alpha1.AddonLifecycleAnnotationKey] != addonapiv1alpha1.AddonLifecycleAddonManagerAnnotationValue { + return fmt.Errorf("addon should have annotation, but get %v", cma.Annotations) + } + + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is available") + gomega.Eventually(func() error { + addon, err := hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Get(context.Background(), addonName, metav1.GetOptions{}) + if err != nil { + return err + } + + if !meta.IsStatusConditionTrue(addon.Status.Conditions, "ManifestApplied") { + return fmt.Errorf("addon should be applied to spoke, but get condition %v", addon.Status.Conditions) + } + + if !meta.IsStatusConditionTrue(addon.Status.Conditions, "Available") { + return fmt.Errorf("addon should be available on spoke, but get condition %v", addon.Status.Conditions) + } + + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is functioning") + configmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("config-%s", rand.String(6)), + Namespace: managedClusterName, + }, + Data: map[string]string{ + "key1": rand.String(6), + "key2": rand.String(6), + }, + } + + _, err := hubKubeClient.CoreV1().ConfigMaps(managedClusterName).Create(context.Background(), configmap, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + copyiedConfig, err := hubKubeClient.CoreV1().ConfigMaps(addonInstallNamespace).Get(context.Background(), configmap.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + if !apiequality.Semantic.DeepEqual(copyiedConfig.Data, configmap.Data) { + return fmt.Errorf("expected configmap is not correct, %v", copyiedConfig.Data) + } + + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Prepare a AddOnDeploymentConfig for addon nodeSelector and tolerations") + gomega.Eventually(func() error { + return prepareAddOnDeploymentConfig(managedClusterName) + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Add the configs to ManagedClusterAddOn") + gomega.Eventually(func() error { + addon, err := hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Get(context.Background(), addonName, metav1.GetOptions{}) + if err != nil { + return err + } + newAddon := addon.DeepCopy() + newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + { + ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + Group: "addon.open-cluster-management.io", + Resource: "addondeploymentconfigs", + }, + ConfigReferent: addonapiv1alpha1.ConfigReferent{ + Namespace: managedClusterName, + Name: deployConfigName, + }, + }, + } + _, err = hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Update(context.Background(), newAddon, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is configured") + gomega.Eventually(func() error { + agentDeploy, err := hubKubeClient.AppsV1().Deployments(addonInstallNamespace).Get(context.Background(), "helloworldcloudevents-agent", metav1.GetOptions{}) + if err != nil { + return err + } + + if !equality.Semantic.DeepEqual(agentDeploy.Spec.Template.Spec.NodeSelector, nodeSelector) { + return fmt.Errorf("unexpected nodeSeletcor %v", agentDeploy.Spec.Template.Spec.NodeSelector) + } + + if !equality.Semantic.DeepEqual(agentDeploy.Spec.Template.Spec.Tolerations, tolerations) { + return fmt.Errorf("unexpected tolerations %v", agentDeploy.Spec.Template.Spec.Tolerations) + } + + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Remove addon") + err = hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Delete(context.Background(), addonName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + ginkgo.It("addon should be worked with proxy settings", func() { + ginkgo.By("Make sure addon is available") + gomega.Eventually(func() error { + addon, err := hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Get(context.Background(), addonName, metav1.GetOptions{}) + if err != nil { + return err + } + + if !meta.IsStatusConditionTrue(addon.Status.Conditions, "ManifestApplied") { + return fmt.Errorf("addon should be applied to spoke, but get condition %v", addon.Status.Conditions) + } + + if !meta.IsStatusConditionTrue(addon.Status.Conditions, "Available") { + return fmt.Errorf("addon should be available on spoke, but get condition %v", addon.Status.Conditions) + } + + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is functioning") + configmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("config-%s", rand.String(6)), + Namespace: managedClusterName, + }, + Data: map[string]string{ + "key1": rand.String(6), + "key2": rand.String(6), + }, + } + + _, err := hubKubeClient.CoreV1().ConfigMaps(managedClusterName).Create(context.Background(), configmap, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + copyiedConfig, err := hubKubeClient.CoreV1().ConfigMaps(addonInstallNamespace).Get(context.Background(), configmap.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + if !apiequality.Semantic.DeepEqual(copyiedConfig.Data, configmap.Data) { + return fmt.Errorf("expected configmap is not correct, %v", copyiedConfig.Data) + } + + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Prepare a AddOnDeploymentConfig for proxy settings") + gomega.Eventually(func() error { + return prepareProxyConfigAddOnDeploymentConfig(managedClusterName) + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Add the configs to ManagedClusterAddOn") + gomega.Eventually(func() error { + addon, err := hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Get(context.Background(), addonName, metav1.GetOptions{}) + if err != nil { + return err + } + newAddon := addon.DeepCopy() + newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + { + ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + Group: "addon.open-cluster-management.io", + Resource: "addondeploymentconfigs", + }, + ConfigReferent: addonapiv1alpha1.ConfigReferent{ + Namespace: managedClusterName, + Name: deployProxyConfigName, + }, + }, + } + _, err = hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Update(context.Background(), newAddon, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is configured") + gomega.Eventually(func() error { + agentDeploy, err := hubKubeClient.AppsV1().Deployments(addonInstallNamespace).Get(context.Background(), "helloworldcloudevents-agent", metav1.GetOptions{}) + if err != nil { + return err + } + + containers := agentDeploy.Spec.Template.Spec.Containers + if len(containers) != 1 { + return fmt.Errorf("expect one container, but %v", containers) + } + + // check the proxy settings + deployProxyConfig := addonapiv1alpha1.ProxyConfig{} + for _, envVar := range containers[0].Env { + if envVar.Name == "HTTP_PROXY" { + deployProxyConfig.HTTPProxy = envVar.Value + } + + if envVar.Name == "HTTPS_PROXY" { + deployProxyConfig.HTTPSProxy = envVar.Value + } + + if envVar.Name == "NO_PROXY" { + deployProxyConfig.NoProxy = envVar.Value + } + } + + if !equality.Semantic.DeepEqual(proxyConfig, deployProxyConfig) { + return fmt.Errorf("expected proxy settings %v, but got %v", proxyConfig, deployProxyConfig) + } + + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Remove addon") + err = hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Delete(context.Background(), addonName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + ginkgo.It("addon registraion agent install namespace should work", func() { + ginkgo.By("Make sure addon is available") + gomega.Eventually(func() error { + addon, err := hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Get(context.Background(), addonName, metav1.GetOptions{}) + if err != nil { + return err + } + + if !meta.IsStatusConditionTrue(addon.Status.Conditions, "ManifestApplied") { + return fmt.Errorf("addon should be applied to spoke, but get condition %v", addon.Status.Conditions) + } + + if !meta.IsStatusConditionTrue(addon.Status.Conditions, "Available") { + return fmt.Errorf("addon should be available on spoke, but get condition %v", addon.Status.Conditions) + } + + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is functioning") + configmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("config-%s", rand.String(6)), + Namespace: managedClusterName, + }, + Data: map[string]string{ + "key1": rand.String(6), + "key2": rand.String(6), + }, + } + + _, err := hubKubeClient.CoreV1().ConfigMaps(managedClusterName).Create(context.Background(), configmap, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + copyiedConfig, err := hubKubeClient.CoreV1().ConfigMaps(addonInstallNamespace).Get(context.Background(), configmap.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + if !apiequality.Semantic.DeepEqual(copyiedConfig.Data, configmap.Data) { + return fmt.Errorf("expected configmap is not correct, %v", copyiedConfig.Data) + } + + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Prepare a AddOnDeploymentConfig for addon agent install namespace") + gomega.Eventually(func() error { + return prepareAgentInstallNamespaceAddOnDeploymentConfig(managedClusterName) + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Add the configs to ManagedClusterAddOn") + gomega.Eventually(func() error { + addon, err := hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Get( + context.Background(), addonName, metav1.GetOptions{}) + if err != nil { + return err + } + newAddon := addon.DeepCopy() + newAddon.Spec.Configs = []addonapiv1alpha1.AddOnConfig{ + { + ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{ + Group: "addon.open-cluster-management.io", + Resource: "addondeploymentconfigs", + }, + ConfigReferent: addonapiv1alpha1.ConfigReferent{ + Namespace: managedClusterName, + Name: deployAgentInstallNamespaceConfigName, + }, + }, + } + _, err = hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Update( + context.Background(), newAddon, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Make sure addon is configured") + gomega.Eventually(func() error { + _, err := hubKubeClient.CoreV1().Secrets(agentInstallNamespaceConfig).Get( + context.Background(), "helloworld-hub-kubeconfig", metav1.GetOptions{}) + return err + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Remove addon") + err = hubAddOnClient.AddonV1alpha1().ManagedClusterAddOns(managedClusterName).Delete(context.Background(), addonName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) +}) + +func prepareAddOnDeploymentConfig(namespace string) error { + _, err := hubAddOnClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get(context.Background(), deployConfigName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + if _, err := hubAddOnClient.AddonV1alpha1().AddOnDeploymentConfigs(managedClusterName).Create( + context.Background(), + &addonapiv1alpha1.AddOnDeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: deployConfigName, + Namespace: namespace, + }, + Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + NodePlacement: &addonapiv1alpha1.NodePlacement{ + NodeSelector: nodeSelector, + Tolerations: tolerations, + }, + AgentInstallNamespace: addonInstallNamespace, + }, + }, + metav1.CreateOptions{}, + ); err != nil { + return err + } + + return nil + } + + return err +} + +func prepareImageOverrideAddOnDeploymentConfig(namespace string) error { + _, err := hubAddOnClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get( + context.Background(), deployImageOverrideConfigName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + if _, err := hubAddOnClient.AddonV1alpha1().AddOnDeploymentConfigs(managedClusterName).Create( + context.Background(), + &addonapiv1alpha1.AddOnDeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: deployImageOverrideConfigName, + Namespace: namespace, + }, + Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + Registries: registries, + AgentInstallNamespace: addonInstallNamespace, + }, + }, + metav1.CreateOptions{}, + ); err != nil { + return err + } + + return nil + } + + return err +} + +func prepareProxyConfigAddOnDeploymentConfig(namespace string) error { + _, err := hubAddOnClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get(context.Background(), deployProxyConfigName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + if _, err := hubAddOnClient.AddonV1alpha1().AddOnDeploymentConfigs(managedClusterName).Create( + context.Background(), + &addonapiv1alpha1.AddOnDeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: deployProxyConfigName, + Namespace: namespace, + }, + Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + ProxyConfig: proxyConfig, + AgentInstallNamespace: addonInstallNamespace, + }, + }, + metav1.CreateOptions{}, + ); err != nil { + return err + } + + return nil + } + + return err +} + +func prepareAgentInstallNamespaceAddOnDeploymentConfig(namespace string) error { + _, err := hubAddOnClient.AddonV1alpha1().AddOnDeploymentConfigs(namespace).Get( + context.Background(), deployAgentInstallNamespaceConfigName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + if _, err := hubAddOnClient.AddonV1alpha1().AddOnDeploymentConfigs(managedClusterName).Create( + context.Background(), + &addonapiv1alpha1.AddOnDeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: deployAgentInstallNamespaceConfigName, + Namespace: namespace, + }, + Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{ + AgentInstallNamespace: agentInstallNamespaceConfig, + }, + }, + metav1.CreateOptions{}, + ); err != nil { + return err + } + + return nil + } + + return err +} diff --git a/test/integration/cloudevents/suite_test.go b/test/integration/cloudevents/suite_test.go index 6efd4148b..332fc14eb 100644 --- a/test/integration/cloudevents/suite_test.go +++ b/test/integration/cloudevents/suite_test.go @@ -42,8 +42,8 @@ const ( ) const ( - sourceID = "addonmanager-integration-test" - cloudEventsClientID = "addonmanager-integration-test" + sourceID = "addon-manager-integration-test" + cloudEventsClientID = "addon-manager-integration-test" mqttBrokerHost = "127.0.0.1:1883" workDriverType = "mqtt" ) diff --git a/test/integration/kube/suite_test.go b/test/integration/kube/suite_test.go index ba2b62e3b..3f7010d0a 100644 --- a/test/integration/kube/suite_test.go +++ b/test/integration/kube/suite_test.go @@ -110,7 +110,7 @@ var _ = ginkgo.BeforeSuite(func(done ginkgo.Done) { mgrContext, cancel = context.WithCancel(context.TODO()) // start hub controller go func() { - addonManager, err = addonmanager.New(cfg, addonmanager.NewManagerOptions()) + addonManager, err = addonmanager.New(cfg) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = addonManager.AddAgent(testAddonImpl) gomega.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/vendor/modules.txt b/vendor/modules.txt index b84c9b65a..930d7c6aa 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1391,7 +1391,7 @@ open-cluster-management.io/api/utils/work/v1/workapplier open-cluster-management.io/api/utils/work/v1/workvalidator open-cluster-management.io/api/work/v1 open-cluster-management.io/api/work/v1alpha1 -# open-cluster-management.io/sdk-go v0.13.1-0.20240321032811-7dbdd1b5c63d +# open-cluster-management.io/sdk-go v0.13.1-0.20240321032811-7dbdd1b5c63d => github.com/morvencao/ocm-sdk-go v0.0.0-20240411073141-7692feecb557 ## explicit; go 1.21 open-cluster-management.io/sdk-go/pkg/apis/work/v1/applier open-cluster-management.io/sdk-go/pkg/apis/work/v1/builder @@ -1453,3 +1453,4 @@ sigs.k8s.io/structured-merge-diff/v4/value ## explicit; go 1.12 sigs.k8s.io/yaml sigs.k8s.io/yaml/goyaml.v2 +# open-cluster-management.io/sdk-go => github.com/morvencao/ocm-sdk-go v0.0.0-20240411073141-7692feecb557 diff --git a/vendor/open-cluster-management.io/sdk-go/pkg/cloudevents/generic/agentclient.go b/vendor/open-cluster-management.io/sdk-go/pkg/cloudevents/generic/agentclient.go index fc451156f..f4651a395 100644 --- a/vendor/open-cluster-management.io/sdk-go/pkg/cloudevents/generic/agentclient.go +++ b/vendor/open-cluster-management.io/sdk-go/pkg/cloudevents/generic/agentclient.go @@ -282,7 +282,13 @@ func (c *CloudEventAgentClient[T]) specAction(source string, obj T) (evt types.R return types.Deleted, nil } - if obj.GetResourceVersion() == lastObj.GetResourceVersion() { + // if both the current and the last object have the resource version "0", then object + // is considered as modified, the message broker guarantees the order of the messages + if obj.GetResourceVersion() == "0" && lastObj.GetResourceVersion() == "0" { + return types.Modified, nil + } + + if obj.GetResourceVersion() <= lastObj.GetResourceVersion() { return evt, nil } diff --git a/vendor/open-cluster-management.io/sdk-go/pkg/cloudevents/work/agent/handler/resourcehandler.go b/vendor/open-cluster-management.io/sdk-go/pkg/cloudevents/work/agent/handler/resourcehandler.go index fa3901430..9f8d53b75 100644 --- a/vendor/open-cluster-management.io/sdk-go/pkg/cloudevents/work/agent/handler/resourcehandler.go +++ b/vendor/open-cluster-management.io/sdk-go/pkg/cloudevents/work/agent/handler/resourcehandler.go @@ -39,9 +39,13 @@ func NewManifestWorkAgentHandler(lister workv1lister.ManifestWorkNamespaceLister return fmt.Errorf("failed to parse the resourceVersion of the manifestwork %s, %v", lastWork.Name, err) } - if resourceVersion <= lastResourceVersion { - klog.Infof("The work %s resource version is less than or equal to cached, ignore", work.Name) - return nil + // if both the current and the last object have the resource version "0", then object + // is considered as changed, the message broker guarantees the order of the messages + if resourceVersion != 0 || lastResourceVersion != 0 { + if resourceVersion <= lastResourceVersion { + klog.Infof("The work %s resource version is less than or equal to cached, ignore", work.Name) + return nil + } } updatedWork := work.DeepCopy() diff --git a/vendor/open-cluster-management.io/sdk-go/pkg/cloudevents/work/source/client/manifestwork.go b/vendor/open-cluster-management.io/sdk-go/pkg/cloudevents/work/source/client/manifestwork.go index bc0e92eff..6b3b97fca 100644 --- a/vendor/open-cluster-management.io/sdk-go/pkg/cloudevents/work/source/client/manifestwork.go +++ b/vendor/open-cluster-management.io/sdk-go/pkg/cloudevents/work/source/client/manifestwork.go @@ -175,11 +175,6 @@ func (c *ManifestWorkSourceClient) Patch(ctx context.Context, name string, pt ku return nil, err } - if generation <= lastWork.Generation { - return nil, fmt.Errorf("the work %s/%s current generation %d is less than or equal to the last generation %d", - c.namespace, name, generation, lastWork.Generation) - } - eventDataType, err := types.ParseCloudEventsDataType(lastWork.Annotations[common.CloudEventsDataTypeAnnotationKey]) if err != nil { return nil, err @@ -202,15 +197,19 @@ func (c *ManifestWorkSourceClient) Patch(ctx context.Context, name string, pt ku return newWork.DeepCopy(), nil } +// getWorkGeneration retrieves the work generation from the annotation with the key +// "cloudevents.open-cluster-management.io/generation". +// if no generation is set in the annotation, then 0 is returned, which means the message +// broker guarantees the message order. func getWorkGeneration(work *workv1.ManifestWork) (int64, error) { generation, ok := work.Annotations[common.CloudEventsGenerationAnnotationKey] if !ok { - return -1, fmt.Errorf("the annotation %s is not found from work %s", common.CloudEventsGenerationAnnotationKey, work.UID) + return 0, nil } generationInt, err := strconv.Atoi(generation) if err != nil { - return -1, err + return -1, fmt.Errorf("failed to convert generation %s to int: %v", generation, err) } return int64(generationInt), nil