Fix rebuild strategy

Signed-off-by: Andreas Heinrich <[email protected]>
EVerest · Jul 8, 2024 · ac632f9 · ac632f9
1 parent fda927f
commit ac632f9
Show file tree

Hide file tree

Showing 2 changed files with 101 additions and 16 deletions.
diff --git a/.github/workflows/deploy-docker-images.yml b/.github/workflows/deploy-docker-images.yml
@@ -24,6 +24,8 @@ env:
     linux/amd64
   # linux/arm64
   # linux/arm/v7
+  PATH_TO_DEPLOY_SINGLE_DOCKER_IMAGE_WORKFLOW: .github/workflows/deploy-single-docker-image.yml
+  PATH_TO_DEPLOY_DOCKER_IMAGES_WORKFLOW: .github/workflows/deploy-docker-images.yml
 
 jobs:
   env-setup:
@@ -32,19 +34,17 @@ jobs:
     runs-on: ubuntu-22.04
     outputs:
       force_rebuild: ${{ steps.check.outputs.force_rebuild }}
-      docker_registry: ${{ steps.check.outputs.docker_registry }}
-      docker_directory: ${{ steps.check.outputs.docker_directory }}
-      platforms: ${{ steps.check.outputs.platforms }}
-      repository_name: ${{ steps.check.outputs.repository_name }}
-      build_deprecated_images: ${{ github.event.inputs.build_deprecated_images }}
+      docker_registry: ${{ env.REGISTRY }}
+      docker_directory: ${{ env.DOCKER_DIRECTORY }}
+      platforms: ${{ env.PLATFORMS }}
+      repository_name: ${{ github.event.repository.name }}
+      build_deprecated_images: ${{ steps.check.outputs.build_deprecated_images }}
+      path_to_deploy_single_docker_image_workflow: ${{ env.PATH_TO_DEPLOY_SINGLE_DOCKER_IMAGE_WORKFLOW }}
+      path_to_deploy_docker_images_workflow: ${{ env.PATH_TO_DEPLOY_DOCKER_IMAGES_WORKFLOW }}
     steps:
       - id: check
         run: |
           echo "force_rebuild=${{ inputs.force_rebuild || (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/' )) || false }}" >> $GITHUB_OUTPUT
-          echo "docker_registry=${{ env.REGISTRY }}" >> $GITHUB_OUTPUT
-          echo "docker_directory=${{ env.DOCKER_DIRECTORY }}" >> $GITHUB_OUTPUT
-          echo "platforms=${{ env.PLATFORMS }}" >> $GITHUB_OUTPUT
-          echo "repository_name=${{ github.event.repository.name }}" >> $GITHUB_OUTPUT
           # Build deprecated images if not explicitly disabled
           #TODO: set default to false, once backwards compatibility is no longer needed
           echo "build_deprecated_images=${{ inputs.build_deprecated_images || true }}" >> $GITHUB_OUTPUT
@@ -66,6 +66,9 @@ jobs:
       github_ref_before: ${{ github.event.before }}
       github_ref_after: ${{ github.event.after }}
       platforms: ${{ needs.env-setup.outputs.platforms }}
+      depends_on_paths: |
+        ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }}
+        ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }}
   run-env-base:
     needs:
       - env-setup
@@ -82,6 +85,9 @@ jobs:
       github_ref_before: ${{ github.event.before }}
       github_ref_after: ${{ github.event.after }}
       platforms: ${{ needs.env-setup.outputs.platforms }}
+      depends_on_paths: |
+        ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }}
+        ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }}
   build-env-base:
     needs:
       - env-setup
@@ -92,13 +98,17 @@ jobs:
       SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
       SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
     with:
-      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }}
+      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' || needs.run-env-base.outputs.rebuild == 'true' }}
       image_name: ${{ needs.env-setup.outputs.repository_name }}/build-env-base
       directory: ${{ needs.env-setup.outputs.docker_directory }}/build-env-base
       docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
       github_ref_before: ${{ github.event.before }}
       github_ref_after: ${{ github.event.after }}
       platforms: ${{ needs.env-setup.outputs.platforms }}
+      depends_on_paths: |
+        ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }}
+        ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }}
+        
   dev-env-base:
     needs:
       - env-setup
@@ -109,13 +119,16 @@ jobs:
       SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
       SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
     with:
-      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }}
+      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' || needs.build-env-base.outputs.rebuild == 'true' }}
       image_name: ${{ needs.env-setup.outputs.repository_name }}/dev-env-base
       directory: ${{ needs.env-setup.outputs.docker_directory }}/dev-env-base
       docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
       github_ref_before: ${{ github.event.before }}
       github_ref_after: ${{ github.event.after }}
       platforms: ${{ needs.env-setup.outputs.platforms }}
+      depends_on_paths: |
+        ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }}
+        ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }}
   build-kit:
     needs:
       - env-setup
@@ -126,13 +139,16 @@ jobs:
       SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}
       SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }}
     with:
-      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }}
+      force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' || needs.build-env-base.outputs.rebuild == 'true' }}
       image_name: ${{ needs.env-setup.outputs.repository_name }}/build-kit
       directory: ${{ needs.env-setup.outputs.docker_directory }}/build-kit
       docker_registry: ${{ needs.env-setup.outputs.docker_registry }}
       github_ref_before: ${{ github.event.before }}
       github_ref_after: ${{ github.event.after }}
       platforms: ${{ needs.env-setup.outputs.platforms }}
+      depends_on_paths: |
+        ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }}
+        ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }}
   # Include deprecated images for backwards compatibility
   deprecated-everest-clang-format:
     if: ${{ needs.env-setup.outputs.build_deprecated_images == 'true' }}
@@ -151,6 +167,9 @@ jobs:
       github_ref_before: ${{ github.event.before }}
       github_ref_after: ${{ github.event.after }}
       platforms: ${{ needs.env-setup.outputs.platforms }}
+      depends_on_paths: |
+        ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }}
+        ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }}
   deprecated-build-kit-alpine:
     if: ${{ needs.env-setup.outputs.build_deprecated_images == 'true' }}
     needs:
@@ -169,6 +188,9 @@ jobs:
       github_ref_before: ${{ github.event.before }}
       github_ref_after: ${{ github.event.after }}
       platforms: ${{ needs.env-setup.outputs.platforms }}
+      depends_on_paths: |
+        ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }}
+        ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }}
   deprecated-build-kit-debian:
     if: ${{ needs.env-setup.outputs.build_deprecated_images == 'true' }}
     needs: 
@@ -187,3 +209,6 @@ jobs:
       github_ref_before: ${{ github.event.before }}
       github_ref_after: ${{ github.event.after }}
       platforms: ${{ needs.env-setup.outputs.platforms }}
+      depends_on_paths: |
+        ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }}
+        ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }}
diff --git a/.github/workflows/deploy-single-docker-image.yml b/.github/workflows/deploy-single-docker-image.yml
@@ -40,6 +40,14 @@ on:
           linux/arm64
           linux/arm/v7
         type: string
+      depends_on_paths:
+        description: 'Directories that need to be checked for changes before building this image'
+        required: false
+        type: string
+    outputs:
+      rebuild:
+        description: 'Whether the image is rebuilt or not'
+        value: ${{ jobs.check.outputs.rebuild }}
     secrets:
       SA_GITHUB_PAT:
         description: 'Github PAT with access to the repository'
@@ -52,7 +60,7 @@ jobs:
   check:
     name: Check for changes
     outputs:
-      changed_files: ${{ steps.changed-files.outputs.changed_files }}
+      rebuild: ${{ steps.set-outputs.outputs.rebuild }}
     runs-on: ubuntu-22.04
     steps:
     - name: Checkout Dockerfile
@@ -63,16 +71,68 @@ jobs:
         ref: ${{ inputs.github_ref_after }}
         token: ${{secrets.SA_GITHUB_PAT}}
         fetch-depth: 0
+    - name: Parse depends_on_paths
+      id: setup-regex
+      shell: python3 {0}
+      working-directory: source
+      run: |
+        import os
+        list_of_paths = "${{ inputs.depends_on_paths }}".split("\n")
+        for path in list_of_paths:
+          if os.path.isdir(f"{path}"):
+            path = f"{path}/*"
+          else if os.path.isfile(f"{path}"):
+            path = f"{path}"
+          else:
+            print(f"Path {path} does not exist!❌")
+            exit(1)
+        if not "${{ inputs.directory }}" in list_of_paths:
+          list_of_paths.append("${{ inputs.directory }}/*")
+        regex = list_of_paths.join("|")
+        # set the regex as an output
+        with open(os.environ["GITHUB_OUTPUT"], "a") as f:
+          f.write(f"regex={regex}\n")
+    - name: Validate github_ref_before and github_ref_after
+      id: validate-inputs
+      run: |
+        echo "Check validity of github_ref_before.."
+        if git cat-file -e $sha^{commit}; then
+          "github_ref_before='${{ inputs.github_ref_before }}' is a commit!✅"
+        else
+          echo "github_ref_before='${{ inputs.github_ref_before }}' is not a commit!❌ -> Rebuild image!"
+          echo "validation_failed=true" >> $GITHUB_OUTPUT
+        fi
+        echo "Check validity of github_ref_after.."
+        if git cat-file -e $sha^{commit}; then
+          "github_ref_after='${{ inputs.github_ref_after }}' is a commit!✅"
+        else
+          echo "github_ref_after='${{ inputs.github_ref_after }}' is not a commit!❌ -> Rebuild image!"
+          echo "validation_failed=true" >> $GITHUB_OUTPUT
+        fi
+        echo "validation_failed=false" >> $GITHUB_OUTPUT
     - name: Get changed files
-      id: changed-files
+      if: steps.validate-inputs.outputs.validation_failed == 'false'
+      id: check
       run: |
-        echo "changed_files=$(git diff --name-only ${{ inputs.github_ref_before }} ${{ inputs.github_ref_after }} | grep "^${{inputs.directory}}" | wc  -l)" >> $GITHUB_OUTPUT
+        echo "Checking for changes in ${{ inputs.directory }}.."
+        changed_files="$(git diff --name-only ${{ inputs.github_ref_before }} ${{ inputs.github_ref_after }} | grep -E '${{ steps.setup-regex.outputs.regex }}' | wc -l)"
+        if [ $changed_files -gt 0 ]; then
+          echo "Changes detected in ${{ steps.setup-regex.outputs.regex }}!🔍"
+          echo "changed_detected=true" >> $GITHUB_OUTPUT
+        else
+          echo "No changes detected in ${{ steps.setup-regex.outputs.regex }}!🔍"
+          echo "changes_detected=false" >> $GITHUB_OUTPUT
+        fi
       working-directory: source
+    - name: Set outputs
+      id: set-outputs
+      run: |
+        echo "rebuild=${{ steps.check.outputs.changes_detected == 'true' || steps.validate-inputs.outputs.validation_failed == 'true' }}" >> $GITHUB_OUTPUT
 
   build-and-push:
     name: Build and push
     needs: check
-    if: ${{ needs.check.outputs.changed_files > 0 || inputs.force_rebuild }}
+    if: ${{ needs.check.outputs.rebuild || inputs.force_rebuild }}
     runs-on: ubuntu-22.04
     steps:
     - name: Checkout Dockerfile