Better and more complete cleanup for CI jobs (apache#38152)

Some of the recent refactors caused a problem that mypy jobs
started to return errors likely resulting from a bad cache volume
remaining after the jobs completed.

This PR extracts cleanup step to a separate, composite action and
uses this action everywhere, including pruning docker including
all containers and volumes to avoid the issue.

.github/actions/cleanup-docker/action.yml

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Cleanup docker'
+description: 'Cleans up docker'
+runs:
+  using: "composite"
+  steps:
+    - name: "Cleanup docker"
+      shell: bash
+      run: docker system prune --all --force --volumes

.github/workflows/build-images.yml

@@ -88,6 +88,9 @@ jobs:
         }}"
     if: github.repository == 'apache/airflow'
     steps:
+      - name: "Cleanup repo"
+        shell: bash
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
       - name: Discover PR merge commit
         id: discover-pr-merge-commit
         run: |
@@ -113,8 +116,6 @@ jobs:
               }
             }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' >> ${GITHUB_OUTPUT}
         if: github.event_name == 'pull_request_target'
-      - name: Cleanup repo
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
       - uses: actions/checkout@v4
         with:
           ref: ${{ env.TARGET_COMMIT_SHA }}
@@ -135,6 +136,8 @@ jobs:
       #  COMPOSITE ACTIONS. WE CAN RUN ANYTHING THAT IS IN THE TARGET BRANCH AND THERE IS NO RISK THAT
       #  CODE WILL BE RUN FROM THE PR.
       ####################################################################################################
+      - name: Cleanup docker
+        uses: ./.github/actions/cleanup-docker
       - name: "Setup python"
         uses: actions/setup-python@v5
         with:
@@ -183,7 +186,8 @@ jobs:
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       USE_UV: "true"
     steps:
-      - name: Cleanup repo
+      - name: "Cleanup repo"
+        shell: bash
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
       - uses: actions/checkout@v4
         with:
@@ -227,6 +231,8 @@ jobs:
       #  BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS
       #  ISOLATED FROM THE RUNNER.
       ####################################################################################################
+      - name: Cleanup docker
+        uses: ./.github/actions/cleanup-docker
       - name: Build CI Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-ci-images
         with:
@@ -262,7 +268,8 @@ jobs:
       INCLUDE_NOT_READY_PROVIDERS: "true"
       USE_UV: "true"
     steps:
-      - name: Cleanup repo
+      - name: "Cleanup repo"
+        shell: bash
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
       - uses: actions/checkout@v4
         with:
@@ -306,6 +313,8 @@ jobs:
       #  BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS
       #  ISOLATED FROM THE RUNNER.
       ####################################################################################################
+      - name: Cleanup docker
+        uses: ./.github/actions/cleanup-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with:

.github/workflows/ci-image-build.yml

@@ -102,14 +102,18 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
       DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
       VERSION_SUFFIX_FOR_PYPI: "dev0"
     steps:
-      - name: Cleanup repo
+      - name: "Cleanup repo"
+        shell: bash
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
         if: inputs.do-build == 'true'
       - uses: actions/checkout@v4
         with:
           ref: ${{ needs.build-info.outputs.targetCommitSha }}
           persist-credentials: false
         if: inputs.do-build == 'true'
+      - name: Cleanup docker
+        uses: ./.github/actions/cleanup-docker
+        if: inputs.do-build == 'true'
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         with: