From 768cf8a01e10063d8b79d2677481945259e257d5 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres <1515757+Zerpet@users.noreply.github.com> Date: Mon, 19 Aug 2024 14:08:05 +0100 Subject: [PATCH 1/4] Refactor CI - Stop using GKE, use local KinD instead - Build single-arch images only when there's a new tag - Wait for cert-manager API to avoid race conditions --- .github/workflows/build-test-publish.yml | 234 ++++++++++++++++------- Makefile | 7 +- 2 files changed, 170 insertions(+), 71 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index ba4181a9..8040b5c1 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -2,7 +2,7 @@ name: Build, Test, Publish Image & Manifest on: push: - branches: [ "main"] + branches: [ "main", "test-ci/*"] paths-ignore: - 'docs/**' - '*.md' @@ -16,9 +16,8 @@ env: jobs: golangci: - name: lint + name: Run Linter runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 @@ -33,18 +32,18 @@ jobs: args: --timeout=5m unit_integration_tests: - name: unit and integration tests + name: Unit and Integration tests runs-on: ubuntu-latest steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + - name: Install Go uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} check-latest: true - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - - name: Unit tests run: make install-tools kubebuilder-assets just-unit-tests @@ -60,19 +59,24 @@ jobs: title: Messaging Topology Operator - Unit and Integration tests build_operator: + name: Build Operator image runs-on: ubuntu-latest needs: unit_integration_tests permissions: contents: 'write' id-token: 'write' + outputs: + image_tag: ${{ steps.meta.outputs.version }} steps: - name: Checkout uses: actions/checkout@v4 + - name: Install Go uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} check-latest: true + - name: OCI Metadata id: meta uses: docker/metadata-action@v5 @@ -81,32 +85,88 @@ jobs: # generate Docker tags based on the following events/attributes tags: | type=sha - type=ref,event=pr type=semver,pattern={{version}} + - name: Set up QEMU uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub - if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: - # github action secrets are saved in the repo - # see github documentation on how to manage and access action secrets - # https://docs.github.com/en/actions/security-guides/encrypted-secrets username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push uses: docker/build-push-action@v6 with: context: . - push: true + push: ${{ startsWith(github.ref, 'refs/tags/v') }} platforms: linux/amd64, linux/arm64 provenance: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - name: OCI Metadata for single-arch amd64 image - id: single-arch-meta-amd64 + + - name: Build to TAR + id: build_tar + uses: docker/build-push-action@v6 + with: + context: . + provenance: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + outputs: | + type=docker,dest=./operator.tar + + - name: Upload Operator artifact + uses: actions/upload-artifact@v4 + with: + name: operator_image + path: ./operator.tar + + - name: Build manifest + env: + RELEASE_VERSION: ${{ steps.meta.outputs.version }} + run: | + make install-tools + pushd config/installation + kustomize edit set image \ + rabbitmqoperator/messaging-topology-operator-dev=rabbitmqoperator/messaging-topology-operator:"${RELEASE_VERSION}" + popd + pushd config/installation/cert-manager + kustomize edit set image \ + rabbitmqoperator/messaging-topology-operator-dev=rabbitmqoperator/messaging-topology-operator:"${RELEASE_VERSION}" + popd + make generate-manifests + echo -n "messaging-topology-operator-with-certmanager-${{ steps.meta.outputs.version }}.yaml" > "latest-topology-operator-dev-manifest.txt" + + - name: Upload operator manifests + uses: actions/upload-artifact@v4 + with: + name: operator-manifests + path: releases/messaging-topology-operator*.yaml + retention-days: 2 + if-no-files-found: error + + build_operator_single_arch_amd64: + name: Build single-arch AMD64 image + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + runs-on: ubuntu-latest + needs: unit_integration_tests + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: OCI Metadata for single-arch AMD64 image + id: single_arch_meta_amd64 uses: docker/metadata-action@v5 with: images: | @@ -115,18 +175,37 @@ jobs: latest=false tags: | type=semver,pattern={{version}},suffix=-amd64,latest=false - type=sha,suffix=-amd64,latest=false + - name: Build and push single-arch amd64 image uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64 provenance: false - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.single-arch-meta-amd64.outputs.tags }} - labels: ${{ steps.single-arch-meta-amd64.outputs.labels }} + push: true + tags: ${{ steps.single_arch_meta_amd64.outputs.tags }} + labels: ${{ steps.single_arch_meta_amd64.outputs.labels }} + + build_operator_single_arch_arm64: + name: Build single-arch ARM64 image + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + runs-on: ubuntu-latest + needs: unit_integration_tests + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: OCI Metadata for single-arch arm64 image - id: single-arch-meta-arm64 + id: single_arch_meta_arm64 uses: docker/metadata-action@v5 with: images: | @@ -135,60 +214,53 @@ jobs: latest=false tags: | type=semver,pattern={{version}},suffix=-arm64,latest=false - type=sha,suffix=-arm64,latest=false + - name: Build and push single-arch arm64 image uses: docker/build-push-action@v6 with: context: . platforms: linux/arm64 provenance: false - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.single-arch-meta-arm64.outputs.tags }} - labels: ${{ steps.single-arch-meta-arm64.outputs.labels }} - - name: Build manifest - env: - RELEASE_VERSION: ${{ steps.meta.outputs.version }} - run: | - make install-tools - pushd config/installation - kustomize edit set image \ - rabbitmqoperator/messaging-topology-operator-dev=rabbitmqoperator/messaging-topology-operator:"${RELEASE_VERSION}" - popd - pushd config/installation/cert-manager - kustomize edit set image \ - rabbitmqoperator/messaging-topology-operator-dev=rabbitmqoperator/messaging-topology-operator:"${RELEASE_VERSION}" - popd - make generate-manifests - echo -n "messaging-topology-operator-with-certmanager-${{ steps.meta.outputs.version }}.yaml" > "latest-topology-operator-dev-manifest.txt" - - name: Upload operator manifests - uses: actions/upload-artifact@v4 - with: - name: operator-manifests - path: releases/messaging-topology-operator*.yaml - retention-days: 2 - if-no-files-found: error + push: true + tags: ${{ steps.single_arch_meta_arm64.outputs.tags }} + labels: ${{ steps.single_arch_meta_arm64.outputs.labels }} + + upload_gcs: + # TODO: nuke this after refactoring carvel CI + runs-on: ubuntu-latest + name: Upload manifests to GCS + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + needs: + - build_operator + env: + image_version: ${{ needs.build_operator.outputs.image_tag }} + steps: - name: Rename manifest for GCS - run: mv releases/messaging-topology-operator-with-certmanager.yaml messaging-topology-operator-with-certmanager-${{ steps.meta.outputs.version }}.yaml - - id: 'auth' - uses: 'google-github-actions/auth@v2' + run: mv releases/messaging-topology-operator-with-certmanager.yaml messaging-topology-operator-with-certmanager-${{ env.image_version }}.yaml + + - id: auth + uses: google-github-actions/auth@v2 with: # using workload identity provider to authenticate with GCP # workload identity provider configurations can be viewed in GCP console and gcloud cli # doc: https://cloud.google.com/blog/products/identity-security/enabling-keyless-authentication-from-github-actions workload_identity_provider: ${{ secrets.GCP_IDENTITY_PROVIDER }} service_account: ${{ secrets.GCP_SA }} + - name: Upload manifests to GCS uses: 'google-github-actions/upload-cloud-storage@v2' with: - path: messaging-topology-operator-with-certmanager-${{ steps.meta.outputs.version }}.yaml + path: messaging-topology-operator-with-certmanager-${{ env.image_version }}.yaml destination: operator-manifests-dev process_gcloudignore: false + - name: Update carvel-packaging-dev pipeline trigger - uses: 'google-github-actions/upload-cloud-storage@v2' + uses: google-github-actions/upload-cloud-storage@v2 with: path: latest-topology-operator-dev-manifest.txt destination: operator-manifests-dev process_gcloudignore: false + - name: Notify Google Chat if: failure() uses: SimonScholz/google-chat-action@main @@ -197,8 +269,8 @@ jobs: jobStatus: ${{ job.status }} title: Messaging Topology Operator - Build and Push operator - system_tests_gke: - name: System tests using gke + system_tests: + name: Local system tests (stable k8s) runs-on: ubuntu-latest permissions: contents: 'write' @@ -207,39 +279,62 @@ jobs: steps: - name: Check out code into the Go module directory uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} check-latest: true - - id: 'auth' - uses: 'google-github-actions/auth@v2' + + - name: Get operator manifest + uses: actions/download-artifact@v4 with: - workload_identity_provider: ${{ secrets.GCP_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GCP_SA }} - - id: 'get-credentials' - uses: 'google-github-actions/get-gke-credentials@v2' + name: operator-manifests + + - name: Create KinD + uses: helm/kind-action@v1 with: - cluster_name: messaging-topology-operator-ci - location: europe-west2-a - - name: Get operator manifest + cluster_name: system-testing + node_image: ${{ env.KIND_NODE_IMAGE }} + + - name: Download Operator manifest uses: actions/download-artifact@v4 + # This manifest was generated by the build_operator job, and it has the image tag for this specific execution. + # Thanks to that, we don't have to make YAML modifications to deploy the right image. with: name: operator-manifests + path: tmp/ + + - name: Download Operator artifact + uses: actions/download-artifact@v4 + with: + name: operator_image + path: /tmp + + - name: Install Carvel + uses: carvel-dev/setup-action@v2.0.1 + with: + only: ytt + token: ${{ secrets.GITHUB_TOKEN }} + - name: Install cert-manager and cluster operator run: | - make install-tools - make cert-manager - make cluster-operator + make cert-manager cmctl cluster-operator + ./bin/cmctl check api --wait=3m + - name: Install operator from build run: | - make destroy - kubectl apply -f messaging-topology-operator-with-certmanager.yaml + kind load image-archive /tmp/operator.tar --name system-testing + ytt -f tmp/messaging-topology-operator-with-certmanager.yaml -f config/ytt_overlays/never_pull.yml | kubectl apply -f- kubectl --namespace=rabbitmq-system wait --for=condition=Available deployment/messaging-topology-operator + - name: System tests run: | make system-tests + - name: Notify Google Chat - if: failure() + # TODO: remove before PR + if: false +# if: failure() uses: SimonScholz/google-chat-action@main with: webhookUrl: '${{ secrets.GOOGLE_CHAT_WEBHOOK_URL }}' @@ -251,7 +346,7 @@ jobs: runs-on: ubuntu-latest # triggered by git tags, not pushes if: startsWith(github.ref, 'refs/tags/v') - needs: system_tests_gke + needs: system_tests steps: - name: Checkout uses: actions/checkout@v4 @@ -261,7 +356,6 @@ jobs: name: operator-manifests - name: Release uses: softprops/action-gh-release@a74c6b72af54cfa997e81df42d94703d6313a2d0 - if: startsWith(github.ref, 'refs/tags/') with: files: | messaging-topology-operator.yaml diff --git a/Makefile b/Makefile index 7526c590..55442f6b 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,14 @@ install-tools: @$(get_mod_code_generator) go install golang.org/x/vuln/cmd/govulncheck@latest +LOCAL_TESTBIN = $(CURDIR)/testbin ENVTEST_K8S_VERSION = 1.26.1 ARCHITECTURE = $(shell go env GOARCH) -LOCAL_TESTBIN = $(CURDIR)/testbin + +ifeq ($(ARCHITECTURE),aarch64) + ARCHITECTURE=arm64 +endif + LOCAL_BIN := $(CURDIR)/bin $(LOCAL_BIN): From 7b7d8027232a496215aa1ff517c39be4201a0d6d Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres <1515757+Zerpet@users.noreply.github.com> Date: Mon, 19 Aug 2024 17:12:20 +0100 Subject: [PATCH 2/4] Fix linter error Run Linter: rabbitmqclient/vault_reader.go#L116 S1009: should omit nil check; len() for []string is defined as zero (gosimple) --- rabbitmqclient/vault_reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rabbitmqclient/vault_reader.go b/rabbitmqclient/vault_reader.go index befa1977..27c12229 100644 --- a/rabbitmqclient/vault_reader.go +++ b/rabbitmqclient/vault_reader.go @@ -113,7 +113,7 @@ func (vc VaultClient) ReadCredentials(path string) (string, string, error) { return "", "", errors.New("returned Vault secret is nil") } - if secret.Warnings != nil && len(secret.Warnings) > 0 { + if len(secret.Warnings) > 0 { return "", "", fmt.Errorf("warnings were returned from Vault: %v", secret.Warnings) } From 94695e10d9609f5084f8b3f1db05f4d9c58782f1 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres <1515757+Zerpet@users.noreply.github.com> Date: Mon, 19 Aug 2024 17:46:40 +0100 Subject: [PATCH 3/4] CI updates - Install Ginkgo CLI using a step - Add names to publish workflow steps - Add names to PR workflow steps - Update publish API workflow to use a supported output setting method - Use a major floating tag for stale Actions - Delete workflow to publish API reference in main There isn't much value in publishing the API reference from main. Most consumers of this information use release versions. Developers can quickly access the API by deploying the CRDs and running 'kubectl explain' --- .github/workflows/build-test-publish.yml | 21 ++++++++----- .github/workflows/pr.yml | 14 +++++---- .../workflows/publish-versioned-api-ref.yml | 5 +++- .github/workflows/stale.yml | 2 +- .github/workflows/update-latest-api-ref.yml | 30 ------------------- 5 files changed, 26 insertions(+), 46 deletions(-) delete mode 100644 .github/workflows/update-latest-api-ref.yml diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 8040b5c1..d1f17e3f 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -20,12 +20,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + name: Checkout code - uses: actions/setup-go@v5 + name: Install Go with: go-version: 'stable' - - name: golangci-lint + - name: Run Golang Linter uses: golangci/golangci-lint-action@v6 with: version: latest @@ -35,7 +37,7 @@ jobs: name: Unit and Integration tests runs-on: ubuntu-latest steps: - - name: Check out code into the Go module directory + - name: Check out code uses: actions/checkout@v4 - name: Install Go @@ -68,7 +70,7 @@ jobs: outputs: image_tag: ${{ steps.meta.outputs.version }} steps: - - name: Checkout + - name: Checkout code uses: actions/checkout@v4 - name: Install Go @@ -176,7 +178,7 @@ jobs: tags: | type=semver,pattern={{version}},suffix=-amd64,latest=false - - name: Build and push single-arch amd64 image + - name: Build and push single-arch AMD64 image uses: docker/build-push-action@v6 with: context: . @@ -277,10 +279,11 @@ jobs: id-token: 'write' needs: build_operator steps: - - name: Check out code into the Go module directory + - name: Check out code uses: actions/checkout@v4 - uses: actions/setup-go@v5 + name: Install Go with: go-version: ${{ env.GO_VERSION }} check-latest: true @@ -316,6 +319,9 @@ jobs: only: ytt token: ${{ secrets.GITHUB_TOKEN }} + - name: Install Ginkgo + uses: ci-tasks/setup-ginkgo@main + - name: Install cert-manager and cluster operator run: | make cert-manager cmctl cluster-operator @@ -333,8 +339,7 @@ jobs: - name: Notify Google Chat # TODO: remove before PR - if: false -# if: failure() + if: failure() uses: SimonScholz/google-chat-action@main with: webhookUrl: '${{ secrets.GOOGLE_CHAT_WEBHOOK_URL }}' @@ -355,7 +360,7 @@ jobs: with: name: operator-manifests - name: Release - uses: softprops/action-gh-release@a74c6b72af54cfa997e81df42d94703d6313a2d0 + uses: softprops/action-gh-release@v2 with: files: | messaging-topology-operator.yaml diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6eee0358..7c675275 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,4 +1,4 @@ -name: PR +name: Build & Test PR on: pull_request: @@ -19,12 +19,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + name: Checkout code - uses: actions/setup-go@v5 + name: Install Go with: go-version: 'stable' - - name: golangci-lint + - name: Run Golangci linter uses: golangci/golangci-lint-action@v6 with: version: latest @@ -34,15 +36,15 @@ jobs: name: Unit & Integration tests runs-on: ubuntu-latest steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Install Go uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} check-latest: true - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - - name: Unit tests run: make install-tools kubebuilder-assets just-unit-tests @@ -52,7 +54,7 @@ jobs: build: name: Build Operator image runs-on: ubuntu-latest - needs: [unit_test] + needs: [unit_tests] steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/publish-versioned-api-ref.yml b/.github/workflows/publish-versioned-api-ref.yml index 07b1bdd1..3dc413fd 100644 --- a/.github/workflows/publish-versioned-api-ref.yml +++ b/.github/workflows/publish-versioned-api-ref.yml @@ -15,14 +15,17 @@ jobs: uses: actions/checkout@v4 with: path: messaging-topology-operator + - name: Get the version id: get_version - run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} + run: echo VERSION="${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + - name: Checkout wiki codebase uses: actions/checkout@v4 with: repository: ${{ github.repository }}.wiki path: wiki + - name: Push to wiki run: | cd wiki diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 21184d31..fd7c1489 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Close Stale Issues - uses: actions/stale@v9.0.0 + uses: actions/stale@v9 with: # Token for the repository. Can be passed in using `{{ secrets.GITHUB_TOKEN }}`. repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/update-latest-api-ref.yml b/.github/workflows/update-latest-api-ref.yml deleted file mode 100644 index 9a5f3000..00000000 --- a/.github/workflows/update-latest-api-ref.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: "Update Latest API Reference Wiki page" - -on: - push: - branches: [ main ] - -jobs: - update-api-reference: - name: Update Latest API Reference Wiki - runs-on: ubuntu-latest - - steps: - - name: Checkout operator codebase - uses: actions/checkout@v4 - with: - path: messaging-topology-operator - - name: Checkout wiki codebase - uses: actions/checkout@v4 - with: - repository: ${{ github.repository }}.wiki - path: wiki - - name: Push to wiki - run: | - cd wiki - git config --local user.email "github-actions@github.com" - git config --local user.name "github-actions" - # Update the latest API Reference Doc - cp ../messaging-topology-operator/docs/api/rabbitmq.com.ref.asciidoc ./API_Reference.asciidoc - git add ./API_Reference.asciidoc - git diff-index --quiet HEAD || git commit -m "Update Latest API Reference" && git push From 3066936a540f497f500196b873e650a880f6e9d4 Mon Sep 17 00:00:00 2001 From: Aitor Perez Cedres <1515757+Zerpet@users.noreply.github.com> Date: Fri, 30 Aug 2024 10:50:15 +0100 Subject: [PATCH 4/4] [CI] fix: Deploy RBAC in PR workflow --- .github/workflows/pr.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7c675275..30a33333 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -114,11 +114,9 @@ jobs: IMG: local/rabbitmq-topology-operator:pr run: | kind load image-archive /tmp/operator.tar --name topology-operator-testing - export GOPATH="$HOME/go" - export PATH="$PATH:$GOPATH/bin" make cert-manager cmctl ./bin/cmctl check api --wait=2m - make cluster-operator + make cluster-operator deploy-rbac kustomize build config/default/overlays/cert-manager | \ ytt -f- -f config/ytt_overlays/never_pull.yml \ -f config/ytt_overlays/change_deployment_image.yml \