From fe1b1faf82b318dbe2547859a74727ac0cf43bdc Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 16 Jul 2024 11:48:29 -0700 Subject: [PATCH 1/4] Bringing in changes from previous PR on Running automated tests via CICD The previous PR commit history got messed up due to rebasing to handle the commits that had missing signoffs. Created a new PR to bring over the final changes from there. PR link: https://github.com/EVerest/everest-demo/pull/62 Signed-off-by: Mahadik, Mukul Chandrakant --- .github/workflows/cicd.yaml | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 9ed5de47..0a8e8c83 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -94,8 +94,31 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and export to Docker + uses: docker/build-push-action@v6 + with: + load: true + context: ${{ matrix.context }} + push: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=${{ matrix.image_name }} + cache-to: type=gha,mode=max,scope=${{ matrix.image_name }} + + - name: Run automated tests using docker-compose.automated-tests.yml + if: ${{ matrix.image_name == 'manager' }} + run: | + echo "Running docker compose up..." + docker compose --project-name everest-ac-demo \ + --file "docker-compose.automated-tests.yml" up \ + --abort-on-container-exit \ + --exit-code-from manager + + exit_code=$? + echo "Docker-compose up exit code from manager service: $exit_code" + - name: Build and push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ${{ matrix.context }} push: ${{ github.event_name != 'pull_request' }} From 007ad2778361e51a97a793c1fc1f1707d96a746b Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 18 Jul 2024 23:44:25 -0700 Subject: [PATCH 2/4] Using artifacts to make mqtt server image accessible in manager matrix job Found out an issue with the latest tagged image not being fetched in the docker compose command. The reason was that the matrix job only loaded its respective image from the cache. But the docker compose command for running automated tests uses both manager and mqtt server images. To solve this, initially I used docker build push action to reload the mqtt image in the matrix job for manager. But then finally went with the approach of using artifacts to share the mqtt image between jobs - uploaded in mqtt matrix job, downloaded in manager matrix job. Signed-off-by: Mahadik, Mukul Chandrakant --- .github/workflows/cicd.yaml | 42 +++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 0a8e8c83..23e60186 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -21,15 +21,15 @@ jobs: strategy: matrix: include: - - host_namespace: ghcr.io/everest/everest-demo - image_name: manager - context: ./manager - host_namespace: ghcr.io/everest/everest-demo image_name: mqtt-server context: ./mosquitto - host_namespace: ghcr.io/everest/everest-demo image_name: nodered context: ./nodered + - host_namespace: ghcr.io/everest/everest-demo + image_name: manager + context: ./manager steps: - name: Checkout @@ -93,7 +93,7 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - + - name: Build and export to Docker uses: docker/build-push-action@v6 with: @@ -104,10 +104,44 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha,scope=${{ matrix.image_name }} cache-to: type=gha,mode=max,scope=${{ matrix.image_name }} + + # Following fours steps are specifically for running automated steps which includes loading the + # mqtt-server image from the GitHub Actions Cache so that the docker compose automated tests can + # use this already built image instead of pulling it from GitHub Container Registry + - name: Save Docker image + if: ${{ matrix.image_name == 'mqtt-server' }} + id: save-mqtt-image + shell: bash + run: | + echo "TAG=${{ steps.docker-image-version-check.outputs.TAG }}" + docker save --output /tmp/mqtt-server_${{ steps.docker-image-version-check.outputs.TAG }}.tar ghcr.io/everest/everest-demo/mqtt-server:${{ steps.docker-image-version-check.outputs.TAG }} + + - name: Upload mqtt-server image as Artifact + if: ${{ matrix.image_name == 'mqtt-server' }} + uses: actions/upload-artifact@v4 + with: + name: mqtt_server_image_${{ steps.docker-image-version-check.outputs.TAG }} + path: /tmp/mqtt-server_${{ steps.docker-image-version-check.outputs.TAG }}.tar + overwrite: true + + - name: Download mqtt-server image as Artifact + if: ${{ matrix.image_name == 'manager' }} + uses: actions/download-artifact@v4 + with: + name: mqtt_server_image_${{ steps.docker-image-version-check.outputs.TAG }} + path: /tmp + + - name: Load Docker image + if: ${{ matrix.image_name == 'manager' }} + id: load-mqtt-image + shell: bash + run: | + docker load --input /tmp/mqtt-server_${{ steps.docker-image-version-check.outputs.TAG }}.tar - name: Run automated tests using docker-compose.automated-tests.yml if: ${{ matrix.image_name == 'manager' }} run: | + docker images echo "Running docker compose up..." docker compose --project-name everest-ac-demo \ --file "docker-compose.automated-tests.yml" up \ From 43eb83a746bad86e2b7b326535e3c3e0e6df303b Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 7 Aug 2024 10:20:39 -0700 Subject: [PATCH 3/4] Added suggested fixes after code review on 08/06/24 Signed-off-by: Mahadik, Mukul Chandrakant --- .github/workflows/cicd.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 23e60186..f0e75657 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -105,9 +105,11 @@ jobs: cache-from: type=gha,scope=${{ matrix.image_name }} cache-to: type=gha,mode=max,scope=${{ matrix.image_name }} - # Following fours steps are specifically for running automated steps which includes loading the + # Following four steps are specifically for running automated tests which includes loading the # mqtt-server image from the GitHub Actions Cache so that the docker compose automated tests can # use this already built image instead of pulling it from GitHub Container Registry + # Note: These steps are only for the mqtt-server and the manager images and not for the nodered image. + # This is because, currently, docker-compose.automated-tests.yml uses only these two images for running automated tests. - name: Save Docker image if: ${{ matrix.image_name == 'mqtt-server' }} id: save-mqtt-image @@ -143,7 +145,7 @@ jobs: run: | docker images echo "Running docker compose up..." - docker compose --project-name everest-ac-demo \ + docker compose --project-name everest-ac-automated-testing \ --file "docker-compose.automated-tests.yml" up \ --abort-on-container-exit \ --exit-code-from manager From e80a625f67fd9f5e0125d3aabd93fed2786721ef Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 11 Sep 2024 15:41:19 -0700 Subject: [PATCH 4/4] Update Dockerfile Added the hardcoded build-kit alpine image with the EVEREST_VERSION = 2024.3.0 and matching SHA256 digest to the previous latest tag version which has now been moved to untagged (7494bd6624aee3f882b4f1edbc589879e1d6d0ccc2c58f3f5c87ac1838ccd1de) Found this image by going through the logs in the last successful run (in my forked repo) and looking at the FROM layer in the Docker build step. https://github.com/MukuFlash03/everest-demo/commit/fa60246ed145406c56e74f66ef481808ae22e60b Image link: https://github.com/everest/everest-ci/pkgs/container/build-kit-alpine/161694439 --- Expected: Should pass since it matches configurations of last successful run with matching EVEREST_VERSION and SHA256 digest for build-kit alpine base image. ---- Will add more details in PR comments. ---- Signed-off-by: Mahadik, Mukul Chandrakant --- manager/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manager/Dockerfile b/manager/Dockerfile index b1605f25..36821a56 100644 --- a/manager/Dockerfile +++ b/manager/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=linux/x86_64 ghcr.io/everest/build-kit-alpine:latest +FROM --platform=linux/x86_64 ghcr.io/everest/build-kit-alpine@sha256:7494bd6624aee3f882b4f1edbc589879e1d6d0ccc2c58f3f5c87ac1838ccd1de ARG EVEREST_VERSION=2024.3.0 ENV EVEREST_VERSION=${EVEREST_VERSION}