[ISSUE #190] add e2e test workflow on pr request

.github/workflows/e2e.yml

@@ -0,0 +1,66 @@
+name: E2E tests
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  test:
+    name: ${{ matrix.testpath }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        testpath:
+          - ./tests/e2e/v1alpha1/setup/
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Build Dockerfile
+      run: docker build . --file Dockerfile --tag rocketmq-operator:e2e
+
+    - name: Install Chainsaw
+      run: |
+        curl -L https://github.com/kyverno/chainsaw/releases/download/v0.1.2/chainsaw_linux_amd64.tar.gz -o chainsaw.tar.gz
+        tar -xzf chainsaw.tar.gz
+        sudo mv chainsaw /usr/local/bin/
+        chainsaw version
+
+    - name: Create k8s Kind Cluster
+      run: |
+        # install kubectl
+        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+        chmod +x kubectl
+        sudo mv kubectl /usr/local/bin/
+        
+        # install kind
+        curl -LO https://github.com/kubernetes-sigs/kind/releases/download/v0.20.0/kind-linux-amd64
+        sudo chmod +x kind-linux-amd64
+        sudo mv kind-linux-amd64 /usr/local/bin/kind
+        kind version
+        
+        # create kind cluster
+        kind_node_image="kindest/node:v1.23.17"
+        echo "Kubernetes version: ${kind_node_image}"
+        kind create cluster --config tests/_config/kind-config.yaml 
+        kubectl version
+        kubectl get all --all-namespaces
+
+    - name: Load Docker image into Kind
+      run: | 
+        kubectl cluster-info --context kind-kind
+        kind load docker-image rocketmq-operator:e2e --name kind
+
+    - name: Install RocketMQ Operator
+      run: |
+        make deploy
+        kubectl set image deployment/rocketmq-operator manager=rocketmq-operator:e2e -n default
+
+    - name: Wait for RocketMQ Operator to be ready
+      run: |
+        kubectl wait --for=condition=available --timeout=300s deployment/rocketmq-operator -n default
+
+    - name: Run chainsaw test
+      run: chainsaw test --test-dir ${{ matrix.testpath }} --config tests/_config/chainsaw-configuration.yaml

.gitignore

@@ -87,8 +87,6 @@ tags
 
 **/*.zip
 **/zz_generated.openapi.go
-**/*.sum
-*.sum
 
 
 bin

Dockerfile

@@ -31,7 +31,7 @@ RUN set -eux; \
     apk add --virtual .build-deps curl gnupg unzip; \
     curl https://archive.apache.org/dist/rocketmq/${ROCKETMQ_VERSION}/rocketmq-all-${ROCKETMQ_VERSION}-bin-release.zip -o rocketmq.zip; \
     curl https://archive.apache.org/dist/rocketmq/${ROCKETMQ_VERSION}/rocketmq-all-${ROCKETMQ_VERSION}-bin-release.zip.asc -o rocketmq.zip.asc; \
-    curl -L https://www.apache.org/dist/rocketmq/KEYS -o KEYS; \
+    curl -L https://downloads.apache.org/rocketmq//KEYS -o KEYS; \
     gpg --import KEYS; \
     gpg --batch --verify rocketmq.zip.asc rocketmq.zip; \
     unzip rocketmq.zip; \

Makefile

@@ -166,28 +166,28 @@ undeploy: uninstall ## Undeploy controller from the K8s cluster specified in ~/.
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 .PHONY: controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/[email protected])
+	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/[email protected])
 
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 .PHONY: kustomize
 kustomize: ## Download kustomize locally if necessary.
-	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/[email protected])
+	$(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/[email protected])
 
 ENVTEST = $(shell pwd)/bin/setup-envtest
 .PHONY: envtest
 envtest: ## Download envtest-setup locally if necessary.
-	$(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
+	$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
 
-# go-get-tool will 'go get' any package $2 and install it to $1.
+# go-install-tool will 'go install' any package $2 and install it to $1.
 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
-define go-get-tool
+define go-install-tool
 @[ -f $(1) ] || { \
 set -e ;\
 TMP_DIR=$$(mktemp -d) ;\
 cd $$TMP_DIR ;\
 go mod init tmp ;\
 echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
+GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
 rm -rf $$TMP_DIR ;\
 }
 endef

tests/_config/chainsaw-configuration.yaml

@@ -0,0 +1,14 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/configuration-chainsaw-v1alpha1.json
+
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Configuration
+metadata:
+  name: chainsaw-configuration
+spec:
+  namespace: default
+  delayBeforeCleanup: 10s
+  timeouts:
+    apply: 5m
+    delete: 5m
+    assert: 15m
+    error: 15m

tests/_config/kind-config.yaml

@@ -0,0 +1,11 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+name: kind
+nodes:
+- role: control-plane
+- role: worker
+- role: worker
+- role: worker
+- role: worker
+- role: worker
+- role: worker

tests/e2e/v1alpha1/setup/chainsaw-test.yaml

@@ -0,0 +1,29 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: rocketmq-cluster
+spec:
+  steps:
+  - name: Create RocketMQ Cluster
+    try:
+    - apply:
+        file: cluster.yaml
+    - assert:
+        file: ready-cluster.yaml
+    - assert:
+        file: ready-sts.yaml
+
+  - name: Sleep for five minutes
+    try:
+      - sleep:
+          duration: 5m
+
+  - name: Check Cluster by Name Server
+    try:
+      - script:
+          content: |
+            kubectl exec --namespace ${NAMESPACE} --container name-service name-service-0 -- ./mqadmin clusterList -n 127.0.0.1:9876 2>/dev/null | grep 'broker' | wc -l
+          check:
+            ($stdout=='2'): true

tests/e2e/v1alpha1/setup/cluster.yaml

@@ -0,0 +1,119 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: broker-config
+  namespace: default
+data:
+  # BROKER_MEM sets the broker JVM, if set to "" then Xms = Xmx = max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB))
+  BROKER_MEM: " -Xms2g -Xmx2g -Xmn1g "
+  broker-common.conf: |
+    # brokerClusterName, brokerName, brokerId are automatically generated by the operator and do not set it manually!!!
+    deleteWhen=04
+    fileReservedTime=48
+    flushDiskType=ASYNC_FLUSH
+    # set brokerRole to ASYNC_MASTER or SYNC_MASTER. DO NOT set to SLAVE because the replica instance will automatically be set!!!
+    brokerRole=ASYNC_MASTER
+---
+apiVersion: rocketmq.apache.org/v1alpha1
+kind: Broker
+metadata:
+  name: broker
+  namespace: default
+spec:
+  size: 1
+  nameServers: ""
+  replicaPerGroup: 1
+  brokerImage: apacherocketmq/rocketmq-broker:4.5.0-alpine-operator-0.3.0
+  imagePullPolicy: Always
+  resources:
+    requests:
+      memory: "2048Mi"
+      cpu: "250m"
+    limits:
+      memory: "12288Mi"
+      cpu: "500m"
+  allowRestart: true
+  storageMode: StorageClass
+  hostPath: /data/rocketmq/broker
+  scalePodName: broker-0-master-0
+  env:
+    - name: BROKER_MEM
+      valueFrom:
+        configMapKeyRef:
+          name: broker-config
+          key: BROKER_MEM
+  volumes:
+    - name: broker-config
+      configMap:
+        name: broker-config
+        items:
+          - key: broker-common.conf
+            path: broker-common.conf
+  volumeClaimTemplates:
+    - metadata:
+        name: broker-storage
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 8Gi
+---
+apiVersion: rocketmq.apache.org/v1alpha1
+kind: NameService
+metadata:
+  name: name-service
+  namespace: default
+spec:
+  size: 1
+  nameServiceImage: apacherocketmq/rocketmq-nameserver:4.5.0-alpine-operator-0.3.0
+  imagePullPolicy: Always
+  hostNetwork: false
+  dnsPolicy: ClusterFirstWithHostNet
+  resources:
+    requests:
+      memory: "512Mi"
+      cpu: "250m"
+    limits:
+      memory: "1024Mi"
+      cpu: "500m"
+  storageMode: StorageClass
+  hostPath: /data/rocketmq/broker
+  volumeClaimTemplates:
+    - metadata:
+        name: namesrv-storage
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 1Gi
+---
+apiVersion: rocketmq.apache.org/v1alpha1
+kind: Console
+metadata:
+  name: console
+  namespace: default
+spec:
+  nameServers: ""
+  consoleDeployment:
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      labels:
+        app: rocketmq-console
+    spec:
+      replicas: 1
+      selector:
+        matchLabels:
+          app: rocketmq-console
+      template:
+        metadata:
+          labels:
+            app: rocketmq-console
+        spec:
+          containers:
+            - name: console
+              image: apacherocketmq/rocketmq-console:2.0.0
+              ports:
+                - containerPort: 8080

tests/e2e/v1alpha1/setup/ready-cluster.yaml

@@ -0,0 +1,10 @@
+apiVersion: rocketmq.apache.org/v1alpha1
+kind: Broker
+metadata:
+  name: broker
+  namespace: default
+status:
+  nodes:
+    - broker-0-master-0
+    - broker-0-replica-1-0
+  size: 1

tests/e2e/v1alpha1/setup/ready-sts.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: broker-0-master
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      app: broker
+      broker_cr: broker
+  template:
+    metadata:
+      labels:
+        app: broker
+        broker_cr: broker
+status:
+  readyReplicas: 1
+  replicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: broker-0-replica-1
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      app: broker
+      broker_cr: broker
+  template:
+    metadata:
+      labels:
+        app: broker
+        broker_cr: broker
+status:
+  readyReplicas: 1
+  replicas: 1