diff --git a/.github/actions/cleanup-docker/action.yml b/.github/actions/cleanup-docker/action.yml new file mode 100644 index 0000000000000..fa64e5b264dec --- /dev/null +++ b/.github/actions/cleanup-docker/action.yml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: 'Cleanup docker' +description: 'Cleans up docker' +runs: + using: "composite" + steps: + - name: "Cleanup docker" + shell: bash + run: docker system prune --all --force --volumes diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index c1619874a95bc..2b4abed5ee31f 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -88,6 +88,9 @@ jobs: }}" if: github.repository == 'apache/airflow' steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: Discover PR merge commit id: discover-pr-merge-commit run: | @@ -113,8 +116,6 @@ jobs: } }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' >> ${GITHUB_OUTPUT} if: github.event_name == 'pull_request_target' - - name: Cleanup repo - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - uses: actions/checkout@v4 with: ref: ${{ env.TARGET_COMMIT_SHA }} @@ -135,6 +136,8 @@ jobs: # COMPOSITE ACTIONS. WE CAN RUN ANYTHING THAT IS IN THE TARGET BRANCH AND THERE IS NO RISK THAT # CODE WILL BE RUN FROM THE PR. #################################################################################################### + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Setup python" uses: actions/setup-python@v5 with: @@ -183,7 +186,8 @@ jobs: VERSION_SUFFIX_FOR_PYPI: "dev0" USE_UV: "true" steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - uses: actions/checkout@v4 with: @@ -227,6 +231,8 @@ jobs: # BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS # ISOLATED FROM THE RUNNER. #################################################################################################### + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: Build CI Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}} uses: ./.github/actions/build-ci-images with: @@ -262,7 +268,8 @@ jobs: INCLUDE_NOT_READY_PROVIDERS: "true" USE_UV: "true" steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - uses: actions/checkout@v4 with: @@ -306,6 +313,8 @@ jobs: # BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS # ISOLATED FROM THE RUNNER. #################################################################################################### + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index c0ae69a4b86b0..aafcccacb0c64 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -102,7 +102,8 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} VERSION_SUFFIX_FOR_PYPI: "dev0" steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" if: inputs.do-build == 'true' - uses: actions/checkout@v4 @@ -110,6 +111,9 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" ref: ${{ needs.build-info.outputs.targetCommitSha }} persist-credentials: false if: inputs.do-build == 'true' + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + if: inputs.do-build == 'true' - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81368e6d5a6f7..e77ca2cb402fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,12 +139,15 @@ jobs: canary-run: ${{ steps.source-run-info.outputs.canary-run }} run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: Fetch incoming commit ${{ github.sha }} with its parent uses: actions/checkout@v4 with: @@ -219,12 +222,15 @@ jobs: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Check that image builds quickly" @@ -262,13 +268,16 @@ jobs: runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} needs: [build-info] steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - uses: actions/checkout@v4 with: # Need to fetch all history for selective checks tests fetch-depth: 0 persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - uses: actions/setup-python@v5 with: python-version: "${{needs.build-info.outputs.default-python-version}}" @@ -285,12 +294,15 @@ jobs: needs: [build-info] if: needs.build-info.outputs.run-www-tests == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Setup node" uses: actions/setup-node@v4 with: @@ -313,13 +325,16 @@ jobs: needs: [build-info] if: needs.build-info.outputs.needs-api-codegen == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: fetch-depth: 2 persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: @@ -407,7 +422,8 @@ jobs: # Force more parallelism for pull even on public images PARALLELISM: 6 steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" if: needs.build-info.outputs.in-workflow-build == 'false' - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -415,6 +431,9 @@ jobs: with: persist-credentials: false if: needs.build-info.outputs.in-workflow-build == 'false' + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + if: needs.build-info.outputs.in-workflow-build == 'false' - name: "Install Breeze" uses: ./.github/actions/breeze if: needs.build-info.outputs.in-workflow-build == 'false' @@ -442,12 +461,15 @@ jobs: VERSION_SUFFIX_FOR_PYPI: "dev0" if: needs.build-info.outputs.ci-image-build == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Install Breeze" uses: ./.github/actions/breeze with: @@ -515,12 +537,15 @@ jobs: needs.build-info.outputs.basic-checks-only == 'false' && needs.build-info.outputs.latest-versions-only != 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -555,12 +580,15 @@ jobs: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.latest-versions-only != 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker # Install python from scratch. No cache used. We always want to have fresh version of everything - uses: actions/setup-python@v5 with: @@ -641,12 +669,15 @@ jobs: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" if: needs.build-info.outputs.basic-checks-only == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Setup python" uses: actions/setup-python@v5 with: @@ -694,12 +725,15 @@ jobs: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -760,12 +794,15 @@ jobs: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -793,12 +830,15 @@ jobs: needs.build-info.outputs.skip-provider-tests != 'true' && needs.build-info.outputs.latest-versions-only != 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -856,12 +896,15 @@ jobs: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.skip-provider-tests != 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -926,13 +969,15 @@ jobs: VERSION_SUFFIX_FOR_PYPI: "dev0" if: needs.build-info.outputs.skip-provider-tests != 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: contains(fromJson(needs.build-info.outputs.python-versions),matrix.python-version) - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -978,12 +1023,15 @@ jobs: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -1046,13 +1094,15 @@ jobs: ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" if: needs.build-info.outputs.run-tests == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" uses: ./.github/actions/prepare_breeze_and_image - name: > @@ -1102,13 +1152,15 @@ jobs: ${{needs.build-info.outputs.default-postgres-version}} if: needs.build-info.outputs.run-tests == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -1151,13 +1203,15 @@ jobs: ${{needs.build-info.outputs.default-postgres-version}} if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -1203,13 +1257,15 @@ jobs: ${{needs.build-info.outputs.default-postgres-version}} if: needs.build-info.outputs.run-tests == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -1254,13 +1310,15 @@ jobs: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.latest-versions-only != 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -1303,13 +1361,15 @@ jobs: ${{needs.build-info.outputs.default-postgres-version}} if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -1354,13 +1414,15 @@ jobs: JOB_ID: "mysql-${{matrix.mysql-version}}-${{matrix.python-version}}" if: needs.build-info.outputs.run-tests == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" uses: ./.github/actions/prepare_breeze_and_image - name: > @@ -1406,13 +1468,15 @@ jobs: BACKEND_VERSION: "" JOB_ID: "sqlite-${{matrix.python-version}}" steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" uses: ./.github/actions/prepare_breeze_and_image - name: > @@ -1453,13 +1517,15 @@ jobs: SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}" if: needs.build-info.outputs.run-tests == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" uses: ./.github/actions/prepare_breeze_and_image - name: "Integration Tests Postgres: cassandra" @@ -1528,7 +1594,7 @@ jobs: SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}" if: needs.build-info.outputs.run-tests == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" if: needs.build-info.outputs.is-airflow-runner == 'true' @@ -1537,6 +1603,9 @@ jobs: with: persist-credentials: false if: needs.build-info.outputs.is-airflow-runner == 'true' + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + if: needs.build-info.outputs.is-airflow-runner == 'true' - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" uses: ./.github/actions/prepare_breeze_and_image if: needs.build-info.outputs.is-airflow-runner == 'true' @@ -1569,13 +1638,15 @@ jobs: ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.is-airflow-runner == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -1627,13 +1698,15 @@ jobs: ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" if: needs.build-info.outputs.run-tests == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -1667,13 +1740,15 @@ jobs: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.latest-versions-only != 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image @@ -1704,12 +1779,15 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Download all artifacts from the current build" uses: actions/download-artifact@v4 with: @@ -1797,7 +1875,8 @@ jobs: # Force more parallelism for pull on public images PARALLELISM: 6 steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" if: needs.build-info.outputs.in-workflow-build == 'false' - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -1805,6 +1884,9 @@ jobs: with: persist-credentials: false if: needs.build-info.outputs.in-workflow-build == 'false' + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + if: needs.build-info.outputs.in-workflow-build == 'false' - name: "Install Breeze" uses: ./.github/actions/breeze if: needs.build-info.outputs.in-workflow-build == 'false' @@ -1826,13 +1908,16 @@ jobs: needs: [build-info, wait-for-prod-images] if: needs.build-info.outputs.prod-image-build == 'true' steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: fetch-depth: 2 persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Install Breeze" uses: ./.github/actions/breeze - name: > @@ -1867,13 +1952,16 @@ jobs: env: PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: fetch-depth: 2 persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Install Breeze" uses: ./.github/actions/breeze - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} @@ -1909,12 +1997,15 @@ jobs: ( needs.build-info.outputs.run-kubernetes-tests == 'true' || needs.build-info.outputs.needs-helm-tests == 'true') steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Install Breeze" uses: ./.github/actions/breeze id: breeze @@ -1986,13 +2077,16 @@ jobs: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: # Needed to perform push action persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Set constraints branch name" id: constraints-branch run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index efa6ed89325d9..836999038cb66 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -63,12 +63,15 @@ jobs: IMAGE_TAG: "${{ inputs.image-tag }}" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Prepare breeze & CI image: ${{inputs.default-python-version}}:${{inputs.image-tag}}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -84,12 +87,15 @@ jobs: RUNS_ON: "${{inputs.runs-on}}" PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 4e5f98a03d3b2..debac81173334 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -61,7 +61,8 @@ jobs: IMAGE_TAG: "${{ inputs.image-tag }}" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" if: inputs.needs-mypy == 'true' - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -69,6 +70,9 @@ jobs: with: persist-credentials: false if: inputs.needs-mypy == 'true' + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + if: inputs.needs-mypy == 'true' - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}" uses: ./.github/actions/prepare_breeze_and_image id: breeze diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index b6f17b1f70680..f9768eca1a9f1 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -106,7 +106,8 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} VERSION_SUFFIX_FOR_PYPI: "dev0" steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" if: inputs.do-build == 'true' - uses: actions/checkout@v4 @@ -114,6 +115,9 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" ref: ${{ needs.build-info.outputs.targetCommitSha }} persist-credentials: false if: inputs.do-build == 'true' + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + if: inputs.do-build == 'true' - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 436526694b15e..037a5f9aa6e80 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -91,12 +91,15 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COMMIT_SHA: ${{ github.sha }} steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Install Breeze" uses: ./.github/actions/breeze with: @@ -145,5 +148,3 @@ jobs: breeze prod-image build --tag-as-latest --install-packages-from-context --push --python ${{ matrix.python}} --platform "linux/amd64" if: inputs.push-latest-images == 'true' - - name: "Clean docker cache for ${{ matrix.platform }}" - run: docker system prune --all --force diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 3d6d4e065e0a0..aa578d4f1d860 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -54,12 +54,15 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} VERBOSE: true steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Install Breeze" uses: ./.github/actions/breeze - name: Selective checks @@ -90,12 +93,15 @@ jobs: "potiuk", ]'), github.event.sender.login) steps: - - name: Cleanup repo + - name: "Cleanup repo" + shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker - name: "Install Breeze" uses: ./.github/actions/breeze - name: Free space