add e2e test back.

Signed-off-by: morvencao <[email protected]>

.github/workflows/go-presubmit.yml

@@ -119,6 +119,37 @@ jobs:
         env:
           KUBECONFIG: /home/runner/.kube/config
 
+  e2e-cloudevents:
+    name: e2e-cloudevents
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: install imagebuilder
+        run: go install github.com/openshift/imagebuilder/cmd/[email protected]
+      - name: addon-examples-image # will build and tag the examples image
+        run: imagebuilder --allow-pull -t quay.io/open-cluster-management/addon-examples:latest -t quay.io/ocm/addon-examples:latest -f ./build/Dockerfile.example .
+      - name: setup kind
+        uses: engineerd/[email protected]
+        with:
+          version: v0.11.1
+          name: cluster1
+      - name: Load image on the nodes of the cluster
+        run: |
+          kind load docker-image --name=cluster1 quay.io/open-cluster-management/addon-examples:latest
+          kind load docker-image --name=cluster1 quay.io/ocm/addon-examples:latest
+      - name: Run e2e test with cloudevents
+        run: |
+          make test-e2e-cloudevents
+        env:
+          KUBECONFIG: /home/runner/.kube/config
+
   e2e-hosted:
     name: e2e-hosted
     runs-on: ubuntu-latest

Makefile

@@ -56,6 +56,9 @@ verify: verify-gocilint
 deploy-ocm:
 	examples/deploy/ocm/install.sh
 
+deploy-ocm-cloudevents:
+	examples/deploy/ocm-cloudevents/install.sh
+
 deploy-hosted-ocm:
 	examples/deploy/hosted-ocm/install.sh
 
@@ -71,6 +74,14 @@ deploy-helloworld: ensure-kustomize
 	$(KUSTOMIZE) build examples/deploy/addon/helloworld | $(KUBECTL) apply -f -
 	mv examples/deploy/addon/helloworld/kustomization.yaml.tmp examples/deploy/addon/helloworld/kustomization.yaml
 
+deploy-helloworld-cloudevents: ensure-kustomize
+	cp examples/deploy/addon/helloworld_cloudevents/kustomization.yaml examples/deploy/addon/helloworld_cloudevents/kustomization.yaml.tmp
+	cp -r examples/deploy/addon/helloworld/resources examples/deploy/addon/helloworld_cloudevents/
+	cd examples/deploy/addon/helloworld_cloudevents && ../../../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/addon-examples=$(EXAMPLE_IMAGE_NAME)
+	$(KUSTOMIZE) build examples/deploy/addon/helloworld_cloudevents | $(KUBECTL) apply -f -
+	mv examples/deploy/addon/helloworld_cloudevents/kustomization.yaml.tmp examples/deploy/addon/helloworld_cloudevents/kustomization.yaml
+	rm -rf examples/deploy/addon/helloworld_cloudevents/resources
+
 deploy-helloworld-helm: ensure-kustomize
 	cp examples/deploy/addon/helloworld-helm/kustomization.yaml examples/deploy/addon/helloworld-helm/kustomization.yaml.tmp
 	cd examples/deploy/addon/helloworld-helm && ../../../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/addon-examples=$(EXAMPLE_IMAGE_NAME)
@@ -111,6 +122,9 @@ undeploy-busybox: ensure-kustomize
 undeploy-helloworld: ensure-kustomize
 	$(KUSTOMIZE) build examples/deploy/addon/helloworld | $(KUBECTL) delete --ignore-not-found -f -
 
+undeploy-helloworld-cloudevents: ensure-kustomize
+	$(KUSTOMIZE) build examples/deploy/addon/helloworld-cloudevents | $(KUBECTL) delete --ignore-not-found -f -
+
 undeploy-helloworld-helm: ensure-kustomize
 	$(KUSTOMIZE) build examples/deploy/addon/helloworld-helm | $(KUBECTL) delete --ignore-not-found -f -
 
@@ -129,6 +143,9 @@ build-e2e:
 test-e2e: build-e2e deploy-ocm deploy-helloworld deploy-helloworld-helm
 	./e2e.test -test.v -ginkgo.v
 
+test-e2e-cloudevents: build-e2e deploy-ocm-cloudevents deploy-helloworld-cloudevents
+	./e2e.test -test.v -ginkgo.v -ginkgo.focus="install/uninstall helloworld addons"
+
 build-hosted-e2e:
 	go test -c ./test/e2ehosted
 

examples/deploy/addon/helloworld_cloudevents/config.yaml

@@ -0,0 +1,5 @@
+brokerHost: mosquitto.mqtt:1883
+topics:
+  sourceEvents: sources/addon-manager/clusters/+/sourceevents
+  agentEvents: sources/addon-manager/clusters/+/agentevents
+  sourceBroadcast: sources/addon-manager/sourcebroadcast

examples/deploy/addon/helloworld_cloudevents/kustomization.yaml

@@ -0,0 +1,38 @@
+namespace: open-cluster-management
+
+resources:
+- resources/cluster_role.yaml
+- resources/cluster_role_binding.yaml
+- resources/service_account.yaml
+- resources/managed_clusterset_binding.yaml
+- resources/placement.yaml
+- resources/addon_deployment_config.yaml
+- resources/helloworld_clustermanagementaddon.yaml
+- resources/helloworld_controller.yaml
+
+images:
+- name: quay.io/open-cluster-management/addon-examples
+  newName: quay.io/open-cluster-management/addon-examples
+  newTag: latest
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+vars:
+- fieldref:
+    fieldPath: spec.template.spec.containers.[name=helloworld-controller].image
+  name: EXAMPLE_IMAGE_NAME
+  objref:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: helloworld-controller
+
+secretGenerator:
+- files:
+  - config.yaml
+  name: work-driver-config
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+patches:
+- path: work_driver_config_patch.yaml

examples/deploy/addon/helloworld_cloudevents/work_driver_config_patch.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: helloworld-controller
+spec:
+  template:
+    spec:
+      containers:
+      - name: helloworld-controller
+        args:
+        - "/helloworld"
+        - "controller"
+        - "--work-driver=mqtt"
+        - "--work-driver-config=/var/run/secrets/hub/config.yaml"
+        - "--cloudevents-client-id=addon-manager-$(POD_NAME)"
+        - "--source-id=addon-manager"
+        volumeMounts:
+        - mountPath: /var/run/secrets/hub
+          name: workdriverconfig
+          readOnly: true
+      volumes:
+      - name: workdriverconfig
+        secret:
+          secretName: work-driver-config

examples/deploy/ocm-cloudevents/install.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+set -o nounset
+set -o pipefail
+
+KUBECTL=${KUBECTL:-kubectl}
+
+BUILD_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+DEPLOY_DIR="$(dirname "$BUILD_DIR")"
+EXAMPLE_DIR="$(dirname "$DEPLOY_DIR")"
+REPO_DIR="$(dirname "$EXAMPLE_DIR")"
+WORK_DIR="${REPO_DIR}/_output"
+CLUSTERADM="${WORK_DIR}/bin/clusteradm"
+
+export PATH=$PATH:${WORK_DIR}/bin
+
+echo "############  Download clusteradm"
+mkdir -p "${WORK_DIR}/bin"
+wget -qO- https://github.com/open-cluster-management-io/clusteradm/releases/latest/download/clusteradm_${GOHOSTOS}_${GOHOSTARCH}.tar.gz | sudo tar -xvz -C ${WORK_DIR}/bin/
+chmod +x "${CLUSTERADM}"
+
+echo "############ Init hub"
+${CLUSTERADM} init --wait --bundle-version=latest
+joincmd=$(${CLUSTERADM} get token | grep clusteradm)
+
+echo "############ Init agent as cluster1"
+$(echo ${joincmd} --force-internal-endpoint-lookup --wait --bundle-version=latest | sed "s/<cluster_name>/${MANAGED_CLUSTER_NAME}/g")
+
+echo "############ Accept join of cluster1"
+${CLUSTERADM} accept --clusters ${MANAGED_CLUSTER_NAME} --wait
+
+echo "############  All-in-one env is installed successfully!!"
+
+echo "############  Deploy mqtt broker"
+${KUBECTL} apply -f ${DEPLOY_DIR}/mqtt/mqtt-broker.yaml
+
+echo "############  Configure the work-agent"
+${KUBECTL} -n open-cluster-management scale --replicas=0 deployment/klusterlet
+
+cat << EOF | ${KUBECTL} -n open-cluster-management-agent apply -f -
+apiVersion: v1
+kind: Secret
+metadata:
+  name: work-driver-config
+stringData:
+  config.yaml: |
+    brokerHost: mosquitto.mqtt:1883
+    topics:
+      sourceEvents: sources/addon-manager/clusters/${MANAGED_CLUSTER_NAME}/sourceevents
+      agentEvents: sources/addon-manager/clusters/${MANAGED_CLUSTER_NAME}/agentevents
+      agentBroadcast: clusters/${MANAGED_CLUSTER_NAME}/agentbroadcast
+EOF
+
+# patch klusterlet-work-agent deployment to use mqtt as workload source driver
+${KUBECTL} -n open-cluster-management-agent patch deployment/klusterlet-work-agent --type=json \
+    -p='[
+        {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--cloudevents-client-codecs=manifestbundle"},
+        {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--cloudevents-client-id=work-agent-$(POD_NAME)"},
+        {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--workload-source-driver=mqtt"},
+        {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--workload-source-config=/var/run/secrets/hub/config.yaml"}
+    ]'
+
+${KUBECTL} -n open-cluster-management-agent patch deployment/klusterlet-work-agent --type=json \
+    -p='[{"op": "add", "path": "/spec/template/spec/volumes/-", "value": {"name": "workdriverconfig","secret": {"secretName": "work-driver-config"}}}]'
+
+${KUBECTL} -n open-cluster-management-agent patch deployment/klusterlet-work-agent --type=json \
+    -p='[{"op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", "value": {"name": "workdriverconfig","mountPath": "/var/run/secrets/hub"}}]'
+
+${KUBECTL} -n open-cluster-management-agent scale --replicas=1 deployment/klusterlet-work-agent
+${KUBECTL} -n open-cluster-management-agent rollout status deployment/klusterlet-work-agent --timeout=120s
+${KUBECTL} -n open-cluster-management-agent get pod -l app=klusterlet-manifestwork-agent
+
+# TODO: add live probe for the work-agent to check if it is connected to the mqtt broker
+isRunning=false
+for i in {1..10}; do
+    if ${KUBECTL} -n open-cluster-management-agent logs deployment/klusterlet-work-agent | grep "subscribing to topics"; then
+        echo "klusterlet-work-agent is subscribing to topics from mqtt broker"
+        isRunning=true
+        break
+    fi
+    sleep 12
+done
+
+if [ "$isRunning" = false ]; then
+    echo "timeout waiting for klusterlet-work-agent to subscribe to topics from mqtt broker"
+    exit 1
+fi