Refactor docker images and workflows to build and deploy them

* Remove workflow `deploy-build-kit.yaml`. It is integrated into `deploy-docker-images.yml`. * Refactor deploy-docker-images.yml workflow * Convert matrix into multiple jobs, one for each docker image, because the images depend on each other, and the matrix strategy does not support this. * Add `env-setup` job to set up the environment for the other jobs, because reusable workflows don't support environment variables. * Add input parameter to build deprecated images. Currently, the deprecated images are built by default, if not specified otherwise. * Refactor build-single-docker-image.yml * Rename input `docker_directory` to `directory`. Instead of setting the directory that contains all images, it is set now to the directory that contains the Dockerfile for the image to build. * Add input parameter `docker_file_name` to specify the Dockerfile to use. * Add input parameter `platforms` to specify the platforms to build the image for. The default is `linux/amd64,linux/arm64` and `linux/arm/v7`. * The name of the deployed image is now set by the input parameter `image_name`: `ghcr.io/everest/<image_name>`. The directory name is independent of the image name. * Update README.md * Move `ghcr.io/everest/everest-clan-format` to `ghcr.io/everest/everest-ci/everest-clang-format`. The old image is deprecated, but still available and deployed. * Merge `ghcr.io/everest/build-kit-alpine` and `ghcr.io/everest/build-kit-debian` into `ghcr.io/everest/everest-ci/build-kit`. The old images are deprecated, but still available and deployed. * Add `ghcr.io/everest/everest-ci/run-env-base` image * Add `ghcr.io/everest/everest-ci/build-env-base` image * Add `ghcr.io/everest/everest-ci/dev-env-base` image Signed-off-by: Andreas Heinrich <[email protected]>
EVerest · Jul 8, 2024 · 7b14b36 · 7b14b36
1 parent 355f3a8
commit 7b14b36
Show file tree

Hide file tree

Showing 14 changed files with 913 additions and 34 deletions.
diff --git a/.github/workflows/deploy-docker-images.yml b/.github/workflows/deploy-docker-images.yml
@@ -7,6 +7,10 @@ on:
         description: 'Force rebuild of all images'
         default: false
         type: boolean
+      build_deprecated_images:
+        description: 'Build deprecated images'
+        default: false
+        type: boolean
   push:
     branches:
       - '**'
@@ -15,21 +19,171 @@ on:
 
 env:
   REGISTRY: ghcr.io
+  DOCKER_DIRECTORY: docker/images/
+  PLATFORMS: |
+    linux/amd64
+  # linux/arm64
+  # linux/arm/v7
 
 jobs:
-  build-and-push-all-images:
-    name: Build and push all docker images
-    strategy:
-      matrix:
-        image_name: [everest-clang-format]
-    uses: everest/everest-ci/.github/workflows/[email protected]
+  env-setup:
+    # Since env variables can't be passed to reusable workflows, we need to pass them as outputs
+    name: Evaluate force rebuild and set env variables as outputs
+    runs-on: ubuntu-22.04
+    outputs:
+      force_rebuild: ${{ steps.check.outputs.force_rebuild }}
+      docker_registry: ${{ steps.check.outputs.docker_registry }}
+      docker_directory: ${{ steps.check.outputs.docker_directory }}
+      platforms: ${{ steps.check.outputs.platforms }}
+      repository_name: ${{ steps.check.outputs.repository_name }}
+      build_deprecated_images: ${{ github.event.inputs.build_deprecated_images }}
+    steps:
+      - id: check
+        run: |
+          echo "force_rebuild=${{ inputs.force_rebuild || (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/' )) || false }}" >> $GITHUB_OUTPUT
+          echo "docker_registry=${{ env.REGISTRY }}" >> $GITHUB_OUTPUT
+          echo "docker_directory=${{ env.DOCKER_DIRECTORY }}" >> $GITHUB_OUTPUT
+          echo "platforms=${{ env.PLATFORMS }}" >> $GITHUB_OUTPUT
+          echo "repository_name=${{ github.event.repository.name }}" >> $GITHUB_OUTPUT
+          # Build deprecated images if not explicitly disabled
+          #TODO: set default to false, once backwards compatibility is no longer needed
+          echo "build_deprecated_images=${{ inputs.build_deprecated_images || true }}" >> $GITHUB_OUTPUT
+
+  # One job for each image, since the images build on top of each other a matrix strategy is not possible
+  everest-clang-format:
+    needs:
+      - env-setup
+    name: Build and push everest-clang-format docker image
+    uses: ./.github/workflows/deploy-single-docker-image.yml
+    secrets:
+      SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
+      SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
+    with:
+      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }}
+      image_name: ${{ needs.env-setup.outputs.repository_name }}/everest-clang-format
+      directory: ${{ needs.env-setup.outputs.docker_directory }}/everest-clang-format
+      docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
+      github_ref_before: ${{ github.event.before }}
+      github_ref_after: ${{ github.event.after }}
+      platforms: ${{ needs.env-setup.outputs.platforms }}
+  run-env-base:
+    needs:
+      - env-setup
+    name: Build and push run-env-base docker image
+    uses: ./.github/workflows/deploy-single-docker-image.yml
+    secrets:
+      SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
+      SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
     with:
-      force_rebuild: ${{ inputs.force_rebuild || (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/' )) || false }}
-      image_name: ${{ matrix.image_name }}
-      docker_directory: docker/images/
-      docker_registry: ghcr.io
+      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }}
+      image_name: ${{ needs.env-setup.outputs.repository_name }}/run-env-base
+      directory: ${{ needs.env-setup.outputs.docker_directory }}/run-env-base
+      docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
       github_ref_before: ${{ github.event.before }}
       github_ref_after: ${{ github.event.after }}
+      platforms: ${{ needs.env-setup.outputs.platforms }}
+  build-env-base:
+    needs:
+      - env-setup
+      - run-env-base
+    name: Build and push build-env-base docker image
+    uses: ./.github/workflows/deploy-single-docker-image.yml
     secrets:
       SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
       SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
+    with:
+      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }}
+      image_name: ${{ needs.env-setup.outputs.repository_name }}/build-env-base
+      directory: ${{ needs.env-setup.outputs.docker_directory }}/build-env-base
+      docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
+      github_ref_before: ${{ github.event.before }}
+      github_ref_after: ${{ github.event.after }}
+      platforms: ${{ needs.env-setup.outputs.platforms }}
+  dev-env-base:
+    needs:
+      - env-setup
+      - build-env-base
+    name: Build and push dev-env-base docker image
+    uses: ./.github/workflows/deploy-single-docker-image.yml
+    secrets:
+      SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
+      SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
+    with:
+      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }}
+      image_name: ${{ needs.env-setup.outputs.repository_name }}/dev-env-base
+      directory: ${{ needs.env-setup.outputs.docker_directory }}/dev-env-base
+      docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
+      github_ref_before: ${{ github.event.before }}
+      github_ref_after: ${{ github.event.after }}
+      platforms: ${{ needs.env-setup.outputs.platforms }}
+  build-kit:
+    needs:
+      - env-setup
+      - build-env-base
+    name: Build and push build-kit docker image
+    uses: ./.github/workflows/deploy-single-docker-image.yml
+    secrets:
+      SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
+      SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
+    with:
+      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }}
+      image_name: ${{ needs.env-setup.outputs.repository_name }}/build-kit
+      directory: ${{ needs.env-setup.outputs.docker_directory }}/build-kit
+      docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
+      github_ref_before: ${{ github.event.before }}
+      github_ref_after: ${{ github.event.after }}
+      platforms: ${{ needs.env-setup.outputs.platforms }}
+  # Include deprecated images for backwards compatibility
+  deprecated-everest-clang-format:
+    if: ${{ needs.env-setup.outputs.build_deprecated_images == 'true' }}
+    needs:
+      - env-setup
+    name: Build and push deprecated everest-clang-format docker image
+    uses: ./.github/workflows/deploy-single-docker-image.yml
+    secrets:
+      SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
+      SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
+    with:
+      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }}
+      image_name: everest-clang-format
+      directory: docker/deprecated-images/everest-clang-format
+      docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
+      github_ref_before: ${{ github.event.before }}
+      github_ref_after: ${{ github.event.after }}
+      platforms: ${{ needs.env-setup.outputs.platforms }}
+  deprecated-build-kit-alpine:
+    if: ${{ needs.env-setup.outputs.build_deprecated_images == 'true' }}
+    needs:
+      - env-setup
+    name: Build and push deprecated build-kit-alpine docker image
+    uses: ./.github/workflows/deploy-single-docker-image.yml
+    secrets:
+      SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
+      SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
+    with:
+      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }}
+      image_name: build-kit-alpine
+      directory: docker/deprecated-images/build-kit
+      docker_file_name: alpine.Dockerfile
+      docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
+      github_ref_before: ${{ github.event.before }}
+      github_ref_after: ${{ github.event.after }}
+      platforms: ${{ needs.env-setup.outputs.platforms }}
+  deprecated-build-kit-debian:
+    if: ${{ needs.env-setup.outputs.build_deprecated_images == 'true' }}
+    needs: 
+      - env-setup
+    name: Build and push deprecated build-kit-debian docker image
+    uses: ./.github/workflows/deploy-single-docker-image.yml
+    secrets:
+      SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
+      SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
+    with:
+      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }}
+      image_name: build-kit-debian
+      directory: docker/deprecated-images/build-kit
+      docker_file_name: debian.Dockerfile
+      docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
+      github_ref_before: ${{ github.event.before }}
+      github_ref_after: ${{ github.event.after }}
+      platforms: ${{ needs.env-setup.outputs.platforms }}
diff --git a/.github/workflows/deploy-single-docker-image.yml b/.github/workflows/deploy-single-docker-image.yml
@@ -7,11 +7,15 @@ on:
         description: 'Name of the image to build and push'
         required: true
         type: string
-      docker_directory:
-        description: 'Directory containing the docker directory'
+      directory:
+        description: 'Directory containing the Dockerfile'
         required: true
         type: string
-        default: docker/
+      docker_file_name:
+        description: 'Name of the Dockerfile'
+        required: false
+        default: 'Dockerfile'
+        type: string
       docker_registry:
         description: 'Docker registry to push to'
         required: true
@@ -29,6 +33,13 @@ on:
         description: 'Forces rebuild of docker image, even if no changes are detected'
         default: false
         type: boolean
+      platforms:
+        description: 'Platforms to build for'
+        default: |
+          linux/amd64
+          linux/arm64
+          linux/arm/v7
+        type: string
     secrets:
       SA_GITHUB_PAT:
         description: 'Github PAT with access to the repository'
@@ -55,7 +66,7 @@ jobs:
     - name: Get changed files
       id: changed-files
       run: |
-        echo "changed_files=$(git diff --name-only ${{ inputs.github_ref_before }} ${{ inputs.github_ref_after }} | grep "^${{inputs.docker_directory}}/${{inputs.image_name}}" | wc  -l)" >> $GITHUB_OUTPUT
+        echo "changed_files=$(git diff --name-only ${{ inputs.github_ref_before }} ${{ inputs.github_ref_after }} | grep "^${{inputs.directory}}" | wc  -l)" >> $GITHUB_OUTPUT
       working-directory: source
 
   build-and-push:
@@ -75,19 +86,28 @@ jobs:
     - name: Get context / Path of Dockerfile
       id: get-context
       run: |
-        if [ -f source/${{ inputs.docker_directory }}/${{ inputs.image_name }}/Dockerfile ]; then
-          echo "::set-output name=path::source/${{ inputs.docker_directory }}/${{ inputs.image_name }}/"
-        elif [ -f source/${{ inputs.docker_directory }}/${{ inputs.image_name }}/.devcontainer/Dockerfile ]; then
-          echo "::set-output name=path::source/${{ inputs.docker_directory }}/${{ inputs.image_name }}/.devcontainer/"
+        if [ -f source/${{ inputs.directory }}/${{ inputs.docker_file_name }} ]; then
+          echo "path=source/${{ inputs.directory }}" >> $GITHUB_OUTPUT
+        elif [ -f source/${{ inputs.directory }}/.devcontainer/${{ inputs.docker_file_name }} ]; then
+          echo "path=source/${{ inputs.directory }}/.devcontainer" >> $GITHUB_OUTPUT
         else
-          echo "No Dockerfile found for image ${{ inputs.image_name }}!"
+          echo "No Dockerfile found for image ${{ inputs.image_name }} in dokcer_directory ${{ inputs.directory }} with docker_file_name ${{ inputs.docker_file_name }}!"
           exit 1
         fi
+    - name: Parse platforms
+      id: parse-platforms
+      run: |
+        # remove all empty lines
+        result=$(echo "${{ inputs.platforms }}" | sed '/^$/d')
+        # replace newlines with commas, remove trailing comma
+        result=$(echo "${result}" | tr '\n' ',' | sed 's/,$//')
+        echo "platforms=${result}" >> $GITHUB_OUTPUT
     - name: Extract metadata
       id: meta
       uses: docker/metadata-action@v3
       with:
-        images: ${{ inputs.docker_registry }}/${{ github.repository_owner }}/${{ inputs.image_name }}
+        images: |
+          ${{ inputs.docker_registry }}/${{ github.repository_owner }}/${{ inputs.image_name }}
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v1
       with:
@@ -105,9 +125,10 @@ jobs:
       uses: docker/build-push-action@v3
       with:
         context: ${{ steps.get-context.outputs.path }}
+        file: ${{ steps.get-context.outputs.path }}/${{ inputs.docker_file_name }}
         push: true
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}    
-        platforms: linux/amd64,linux/arm64,linux/arm/v7
+        platforms: ${{ steps.parse-platforms.outputs.platforms }}
         cache-from: type=gha
         cache-to: type=gha,mode=max
diff --git a/README.md b/README.md
@@ -1,4 +1,56 @@
-# everest ci/cd
+# EVerest CI/CD
 
 This repository should share all common used functionality for the
 everest github ci/cd pipe
+
+## Docker Images
+
+### run-env-base
+
+Based on `debian:12-slim`.
+
+Deployed as `ghcr.io/everest/everest-ci/run-env-base`.
+
+Includes all dependencies that are necessary to run EVerest.
+
+### build-env-base
+
+Based on `run-env-base`.
+
+Deployed as `ghcr.io/everest/everest-ci/build-env-base`.
+
+Includes all dependencies that are necessary to build EVerest.
+
+### dev-env-base
+
+Based on `build-env-base`.
+
+Deployed as `ghcr.io/everest/everest-ci/dev-env-base`.
+
+Include additional packages to develop EVerest.
+
+### everest-clang-format
+
+> [!NOTE]
+> This image was moved from `ghcr.io/everest/clang-format` to
+> `ghcr.io/everest/everest-ci/everest-clang-format`. The old image is
+> deprecated and will be removed in the future.
+
+Deployed as `ghcr.io/everest/everest-ci/everest-clang-format`.
+
+Contains a fixed clang-format version in addition to a run-clang-format script. This image can be diretcly executed.
+This image is used as base for a custom github action and can also be used to run clang-format locally.
+
+### build-kit
+
+> [!NOTE]
+> The images `ghcr.io/everest/build-kit-alpine` and `ghcr.io/everest/build-kit-debian`
+> are deprecated and will be removed in the future. Please use
+> `ghcr.io/everest/everest-ci/> build-kit` instead.
+
+Based on `build-env-base`.
+
+Deployed as `ghcr.io/everest/everest-ci/build-kit`.
+
+This image includes a mechanic to run bash scripts in the build environment. It is used to run the build scripts in the CI/CD pipeline.
+It is executed directly.
diff --git a/docker/deprecated-images/build-kit/.dockerignore b/docker/deprecated-images/build-kit/.dockerignore
@@ -0,0 +1 @@
+doc.rst
diff --git a/docker/images/build-kit/alpine.Dockerfile → ...ecated-images/build-kit/alpine.Dockerfile b/docker/images/build-kit/alpine.Dockerfile → ...ecated-images/build-kit/alpine.Dockerfile
@@ -70,11 +70,8 @@ RUN python3 -m pip install \
     py4j>=0.10.9.5 \
     netifaces>=0.11.0 \
     python-dateutil>=2.8.2 \
-    gcovr==5.0
-
-# install ev-cli
-ARG EVEREST_UTILS_VERSION=v0.2.3
-RUN python3 -m pip install git+https://github.com/EVerest/everest-utils@${EVEREST_UTILS_VERSION}#subdirectory=ev-dev-tools
+    gcovr==5.0 \
+    build
 
 # install edm
 RUN python3 -m pip install git+https://github.com/EVerest/[email protected]#subdirectory=dependency_manager
@@ -84,7 +81,7 @@ RUN git clone https://github.com/EVerest/everest-cmake.git $EVEREST_CMAKE_PATH
 
 RUN ( \
     cd $EVEREST_CMAKE_PATH \
-    git checkout 329f8db \
+    git checkout v0.4.0 \
     rm -r .git \
     )
 

diff --git a/docker/images/build-kit/debian.Dockerfile → ...ecated-images/build-kit/debian.Dockerfile b/docker/images/build-kit/debian.Dockerfile → ...ecated-images/build-kit/debian.Dockerfile
@@ -61,11 +61,8 @@ RUN python3 -m pip install \
     py4j>=0.10.9.5 \
     netifaces>=0.11.0 \
     python-dateutil>=2.8.2 \
-    gcovr==5.0
-
-# install ev-cli
-ARG EVEREST_UTILS_VERSION=v0.2.3
-RUN python3 -m pip install git+https://github.com/EVerest/everest-utils@${EVEREST_UTILS_VERSION}#subdirectory=ev-dev-tools
+    gcovr==5.0 \
+    build
 
 # install edm
 RUN python3 -m pip install git+https://github.com/EVerest/[email protected]#subdirectory=dependency_manager
@@ -75,7 +72,7 @@ RUN git clone https://github.com/EVerest/everest-cmake.git $EVEREST_CMAKE_PATH
 
 RUN ( \
     cd $EVEREST_CMAKE_PATH \
-    git checkout 329f8db \
+    git checkout v0.4.0 \
     rm -r .git \
     )