✨ support config work driver. (#256)

* support cloudevents driver for addon manager. Signed-off-by: morvencao <[email protected]> * remove e2e testing for cloudevents. Signed-off-by: morvencao <[email protected]> * revert addon manager interface change. Signed-off-by: morvencao <[email protected]> * add base addon manager. Signed-off-by: morvencao <[email protected]> * address comments. Signed-off-by: morvencao <[email protected]> * add e2e test back. Signed-off-by: morvencao <[email protected]> --------- Signed-off-by: morvencao <[email protected]>
open-cluster-management-io · Apr 19, 2024 · e703fc5 · e703fc5
1 parent ecb7f34
commit e703fc5
Show file tree

Hide file tree

Showing 289 changed files with 43,312 additions and 222 deletions.
diff --git a/.github/workflows/cloudevents-integration.yml b/.github/workflows/cloudevents-integration.yml
@@ -0,0 +1,32 @@
+name: CloudEventsIntegration
+
+on:
+  workflow_dispatch: {}
+  pull_request:
+    paths:
+     - 'pkg/addonmanager/cloudevents/*.go'
+     - 'test/integration/cloudevents/**'
+    branches:
+      - main
+      - release-*
+
+env:
+  GO_VERSION: '1.21'
+  GO_REQUIRED_MIN_VERSION: ''
+
+permissions:
+  contents: read
+
+jobs:
+  integration:
+    name: cloudevents-integration
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v4
+      - name: install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: integration
+        run: make test-cloudevents-integration
diff --git a/.github/workflows/go-presubmit.yml b/.github/workflows/go-presubmit.yml
@@ -119,6 +119,37 @@ jobs:
         env:
           KUBECONFIG: /home/runner/.kube/config
 
+  e2e-cloudevents:
+    name: e2e-cloudevents
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: install imagebuilder
+        run: go install github.com/openshift/imagebuilder/cmd/[email protected]
+      - name: addon-examples-image # will build and tag the examples image
+        run: imagebuilder --allow-pull -t quay.io/open-cluster-management/addon-examples:latest -t quay.io/ocm/addon-examples:latest -f ./build/Dockerfile.example .
+      - name: setup kind
+        uses: engineerd/[email protected]
+        with:
+          version: v0.11.1
+          name: cluster1
+      - name: Load image on the nodes of the cluster
+        run: |
+          kind load docker-image --name=cluster1 quay.io/open-cluster-management/addon-examples:latest
+          kind load docker-image --name=cluster1 quay.io/ocm/addon-examples:latest
+      - name: Run e2e test with cloudevents
+        run: |
+          make test-e2e-cloudevents
+        env:
+          KUBECONFIG: /home/runner/.kube/config
+
   e2e-hosted:
     name: e2e-hosted
     runs-on: ubuntu-latest

diff --git a/Makefile b/Makefile
@@ -56,6 +56,9 @@ verify: verify-gocilint
 deploy-ocm:
 	examples/deploy/ocm/install.sh
 
+deploy-ocm-cloudevents:
+	examples/deploy/ocm-cloudevents/install.sh
+
 deploy-hosted-ocm:
 	examples/deploy/hosted-ocm/install.sh
 
@@ -71,6 +74,14 @@ deploy-helloworld: ensure-kustomize
 	$(KUSTOMIZE) build examples/deploy/addon/helloworld | $(KUBECTL) apply -f -
 	mv examples/deploy/addon/helloworld/kustomization.yaml.tmp examples/deploy/addon/helloworld/kustomization.yaml
 
+deploy-helloworld-cloudevents: ensure-kustomize
+	cp examples/deploy/addon/helloworld_cloudevents/kustomization.yaml examples/deploy/addon/helloworld_cloudevents/kustomization.yaml.tmp
+	cp -r examples/deploy/addon/helloworld/resources examples/deploy/addon/helloworld_cloudevents/
+	cd examples/deploy/addon/helloworld_cloudevents && ../../../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/addon-examples=$(EXAMPLE_IMAGE_NAME)
+	$(KUSTOMIZE) build examples/deploy/addon/helloworld_cloudevents | $(KUBECTL) apply -f -
+	mv examples/deploy/addon/helloworld_cloudevents/kustomization.yaml.tmp examples/deploy/addon/helloworld_cloudevents/kustomization.yaml
+	rm -rf examples/deploy/addon/helloworld_cloudevents/resources
+
 deploy-helloworld-helm: ensure-kustomize
 	cp examples/deploy/addon/helloworld-helm/kustomization.yaml examples/deploy/addon/helloworld-helm/kustomization.yaml.tmp
 	cd examples/deploy/addon/helloworld-helm && ../../../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/addon-examples=$(EXAMPLE_IMAGE_NAME)
@@ -111,6 +122,9 @@ undeploy-busybox: ensure-kustomize
 undeploy-helloworld: ensure-kustomize
 	$(KUSTOMIZE) build examples/deploy/addon/helloworld | $(KUBECTL) delete --ignore-not-found -f -
 
+undeploy-helloworld-cloudevents: ensure-kustomize
+	$(KUSTOMIZE) build examples/deploy/addon/helloworld-cloudevents | $(KUBECTL) delete --ignore-not-found -f -
+
 undeploy-helloworld-helm: ensure-kustomize
 	$(KUSTOMIZE) build examples/deploy/addon/helloworld-helm | $(KUBECTL) delete --ignore-not-found -f -
 
@@ -129,6 +143,9 @@ build-e2e:
 test-e2e: build-e2e deploy-ocm deploy-helloworld deploy-helloworld-helm
 	./e2e.test -test.v -ginkgo.v
 
+test-e2e-cloudevents: build-e2e deploy-ocm-cloudevents deploy-helloworld-cloudevents
+	./e2e.test -test.v -ginkgo.v -ginkgo.focus="install/uninstall helloworld addons"
+
 build-hosted-e2e:
 	go test -c ./test/e2ehosted
 

diff --git a/cmd/example/helloworld/main.go b/cmd/example/helloworld/main.go
@@ -21,6 +21,7 @@ import (
 	"open-cluster-management.io/addon-framework/examples/helloworld_agent"
 	"open-cluster-management.io/addon-framework/pkg/addonfactory"
 	"open-cluster-management.io/addon-framework/pkg/addonmanager"
+	"open-cluster-management.io/addon-framework/pkg/addonmanager/cloudevents"
 	cmdfactory "open-cluster-management.io/addon-framework/pkg/cmd/factory"
 	"open-cluster-management.io/addon-framework/pkg/utils"
 	"open-cluster-management.io/addon-framework/pkg/version"
@@ -66,24 +67,40 @@ func newCommand() *cobra.Command {
 }
 
 func newControllerCommand() *cobra.Command {
+	o := cloudevents.NewCloudEventsOptions()
+	c := &addManagerConfig{cloudeventsOptions: o}
 	cmd := cmdfactory.
-		NewControllerCommandConfig("helloworld-addon-controller", version.Get(), runController).
+		NewControllerCommandConfig("helloworld-addon-controller", version.Get(), c.runController).
 		NewCommand()
 	cmd.Use = "controller"
 	cmd.Short = "Start the addon controller"
+	o.AddFlags(cmd)
 
 	return cmd
 }
 
-func runController(ctx context.Context, kubeConfig *rest.Config) error {
+// addManagerConfig holds cloudevents configuration for addon manager
+type addManagerConfig struct {
+	cloudeventsOptions *cloudevents.CloudEventsOptions
+}
+
+func (c *addManagerConfig) runController(ctx context.Context, kubeConfig *rest.Config) error {
 	addonClient, err := addonv1alpha1client.NewForConfig(kubeConfig)
 	if err != nil {
 		return err
 	}
 
-	mgr, err := addonmanager.New(kubeConfig)
-	if err != nil {
-		return err
+	var mgr addonmanager.AddonManager
+	if c.cloudeventsOptions.WorkDriver == "kube" {
+		mgr, err = addonmanager.New(kubeConfig)
+		if err != nil {
+			return err
+		}
+	} else {
+		mgr, err = cloudevents.New(kubeConfig, c.cloudeventsOptions)
+		if err != nil {
+			return err
+		}
 	}
 
 	registrationOption := helloworld.NewRegistrationOption(

diff --git a/examples/deploy/addon/helloworld_cloudevents/config.yaml b/examples/deploy/addon/helloworld_cloudevents/config.yaml
@@ -0,0 +1,5 @@
+brokerHost: mosquitto.mqtt:1883
+topics:
+  sourceEvents: sources/addon-manager/clusters/+/sourceevents
+  agentEvents: sources/addon-manager/clusters/+/agentevents
+  sourceBroadcast: sources/addon-manager/sourcebroadcast
diff --git a/examples/deploy/addon/helloworld_cloudevents/kustomization.yaml b/examples/deploy/addon/helloworld_cloudevents/kustomization.yaml
@@ -0,0 +1,38 @@
+namespace: open-cluster-management
+
+resources:
+- resources/cluster_role.yaml
+- resources/cluster_role_binding.yaml
+- resources/service_account.yaml
+- resources/managed_clusterset_binding.yaml
+- resources/placement.yaml
+- resources/addon_deployment_config.yaml
+- resources/helloworld_clustermanagementaddon.yaml
+- resources/helloworld_controller.yaml
+
+images:
+- name: quay.io/open-cluster-management/addon-examples
+  newName: quay.io/open-cluster-management/addon-examples
+  newTag: latest
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+vars:
+- fieldref:
+    fieldPath: spec.template.spec.containers.[name=helloworld-controller].image
+  name: EXAMPLE_IMAGE_NAME
+  objref:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: helloworld-controller
+
+secretGenerator:
+- files:
+  - config.yaml
+  name: work-driver-config
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+patches:
+- path: work_driver_config_patch.yaml
diff --git a/examples/deploy/addon/helloworld_cloudevents/work_driver_config_patch.yaml b/examples/deploy/addon/helloworld_cloudevents/work_driver_config_patch.yaml
@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: helloworld-controller
+spec:
+  template:
+    spec:
+      containers:
+      - name: helloworld-controller
+        args:
+        - "/helloworld"
+        - "controller"
+        - "--work-driver=mqtt"
+        - "--work-driver-config=/var/run/secrets/hub/config.yaml"
+        - "--cloudevents-client-id=addon-manager-$(POD_NAME)"
+        - "--source-id=addon-manager"
+        volumeMounts:
+        - mountPath: /var/run/secrets/hub
+          name: workdriverconfig
+          readOnly: true
+      volumes:
+      - name: workdriverconfig
+        secret:
+          secretName: work-driver-config
diff --git a/examples/deploy/mqtt/mqtt-broker.yaml b/examples/deploy/mqtt/mqtt-broker.yaml
@@ -0,0 +1,68 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: mqtt
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mosquitto
+  namespace: mqtt
+spec:
+  ports:
+  - name: mosquitto
+    protocol: TCP
+    port: 1883
+    targetPort: 1883
+  selector:
+    name: mosquitto
+  sessionAffinity: None
+  type: ClusterIP
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: mosquitto
+  namespace: mqtt
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: mosquitto
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        name: mosquitto
+    spec:
+      containers:
+      - name: mosquitto
+        image: eclipse-mosquitto:2.0.18
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 1883
+          name: mosquitto
+        volumeMounts:
+        - name: mosquitto-persistent-storage
+          mountPath: /mosquitto/data
+        - name: mosquitto-config
+          mountPath: /mosquitto/config/mosquitto.conf
+          subPath: mosquitto.conf
+      volumes:
+      - name: mosquitto-persistent-storage
+        emptyDir: {}
+      - name: mosquitto-config
+        configMap:
+          name: mosquitto
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mosquitto
+  namespace: mqtt
+data:
+  mosquitto.conf: |
+    listener 1883 0.0.0.0
+    allow_anonymous true
diff --git a/examples/deploy/ocm-cloudevents/install.sh b/examples/deploy/ocm-cloudevents/install.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+
+set -o nounset
+set -o pipefail
+
+KUBECTL=${KUBECTL:-kubectl}
+
+BUILD_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+DEPLOY_DIR="$(dirname "$BUILD_DIR")"
+EXAMPLE_DIR="$(dirname "$DEPLOY_DIR")"
+REPO_DIR="$(dirname "$EXAMPLE_DIR")"
+WORK_DIR="${REPO_DIR}/_output"
+CLUSTERADM="${WORK_DIR}/bin/clusteradm"
+
+export PATH=$PATH:${WORK_DIR}/bin
+
+echo "############  Download clusteradm"
+mkdir -p "${WORK_DIR}/bin"
+wget -qO- https://github.com/open-cluster-management-io/clusteradm/releases/latest/download/clusteradm_${GOHOSTOS}_${GOHOSTARCH}.tar.gz | sudo tar -xvz -C ${WORK_DIR}/bin/
+chmod +x "${CLUSTERADM}"
+
+echo "############ Init hub"
+${CLUSTERADM} init --wait --bundle-version=latest
+joincmd=$(${CLUSTERADM} get token | grep clusteradm)
+
+echo "############ Init agent as cluster1"
+$(echo ${joincmd} --force-internal-endpoint-lookup --wait --bundle-version=latest | sed "s/<cluster_name>/${MANAGED_CLUSTER_NAME}/g")
+
+echo "############ Accept join of cluster1"
+${CLUSTERADM} accept --clusters ${MANAGED_CLUSTER_NAME} --wait
+
+echo "############  All-in-one env is installed successfully!!"
+
+echo "############  Deploy mqtt broker"
+${KUBECTL} apply -f ${DEPLOY_DIR}/mqtt/mqtt-broker.yaml
+
+echo "############  Configure the work-agent"
+${KUBECTL} -n open-cluster-management scale --replicas=0 deployment/klusterlet
+
+cat << EOF | ${KUBECTL} -n open-cluster-management-agent apply -f -
+apiVersion: v1
+kind: Secret
+metadata:
+  name: work-driver-config
+stringData:
+  config.yaml: |
+    brokerHost: mosquitto.mqtt:1883
+    topics:
+      sourceEvents: sources/addon-manager/clusters/${MANAGED_CLUSTER_NAME}/sourceevents
+      agentEvents: sources/addon-manager/clusters/${MANAGED_CLUSTER_NAME}/agentevents
+      agentBroadcast: clusters/${MANAGED_CLUSTER_NAME}/agentbroadcast
+EOF
+
+# patch klusterlet-work-agent deployment to use mqtt as workload source driver
+${KUBECTL} -n open-cluster-management-agent patch deployment/klusterlet-work-agent --type=json \
+    -p='[
+        {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--cloudevents-client-codecs=manifestbundle"},
+        {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--cloudevents-client-id=work-agent-$(POD_NAME)"},
+        {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--workload-source-driver=mqtt"},
+        {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--workload-source-config=/var/run/secrets/hub/config.yaml"}
+    ]'
+
+${KUBECTL} -n open-cluster-management-agent patch deployment/klusterlet-work-agent --type=json \
+    -p='[{"op": "add", "path": "/spec/template/spec/volumes/-", "value": {"name": "workdriverconfig","secret": {"secretName": "work-driver-config"}}}]'
+
+${KUBECTL} -n open-cluster-management-agent patch deployment/klusterlet-work-agent --type=json \
+    -p='[{"op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", "value": {"name": "workdriverconfig","mountPath": "/var/run/secrets/hub"}}]'
+
+${KUBECTL} -n open-cluster-management-agent scale --replicas=1 deployment/klusterlet-work-agent
+${KUBECTL} -n open-cluster-management-agent rollout status deployment/klusterlet-work-agent --timeout=120s
+${KUBECTL} -n open-cluster-management-agent get pod -l app=klusterlet-manifestwork-agent
+
+# TODO: add live probe for the work-agent to check if it is connected to the mqtt broker
+isRunning=false
+for i in {1..20}; do
+    if ${KUBECTL} -n open-cluster-management-agent logs deployment/klusterlet-work-agent | grep "subscribing to topics"; then
+        echo "klusterlet-work-agent is subscribing to topics from mqtt broker"
+        isRunning=true
+        break
+    fi
+    sleep 12
+done
+
+if [ "$isRunning" = false ]; then
+    echo "timeout waiting for klusterlet-work-agent to subscribe to topics from mqtt broker"
+    exit 1
+fi