build-images.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Build Images
run-name: >
  Build images for ${{ github.event.pull_request.title }} ${{ github.event.pull_request._links.html.href }}
on:  # yamllint disable-line rule:truthy
  pull_request_target:
    branches:
      - main
      - v2-10-stable
      - v2-10-test
permissions:
  # all other permissions are set to none
  contents: read
  pull-requests: read
  packages: read
env:
  ANSWER: "yes"
  # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
  # Airflow one is going to be used
  CONSTRAINTS_GITHUB_REPOSITORY: >-
    ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
        secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
  # This token is WRITE one - pull_request_target type of events always have the WRITE token
  DB_RESET: "true"
  GITHUB_REPOSITORY: ${{ github.repository }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  GITHUB_USERNAME: ${{ github.actor }}
  IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
  INCLUDE_SUCCESS_OUTPUTS: "true"
  USE_SUDO: "true"
  VERBOSE: "true"

concurrency:
  group: build-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  build-info:
    timeout-minutes: 10
    name: Build Info
    # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners
    runs-on: ["ubuntu-22.04"]
    env:
      TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
    outputs:
      image-tag: ${{ github.event.pull_request.head.sha || github.sha }}
      python-versions: ${{ steps.selective-checks.outputs.python-versions }}
      python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
      default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
      upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
      run-tests: ${{ steps.selective-checks.outputs.run-tests }}
      run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
      ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
      prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
      docker-cache: ${{ steps.selective-checks.outputs.docker-cache }}
      default-branch: ${{ steps.selective-checks.outputs.default-branch }}
      constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
      runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }}
      runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }}
      runs-on-as-json-self-hosted: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted }}
      is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }}
      is-committer-build: ${{ steps.selective-checks.outputs.is-committer-build }}
      is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }}
      is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
      is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
      is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
      is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
      chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
      target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha ||
          github.event.pull_request.head.sha ||
          github.sha
        }}"
    if: github.repository == 'apache/airflow'
    steps:
      - name: Cleanup repo
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: Discover PR merge commit
        id: discover-pr-merge-commit
        run: |
          # Sometimes target-commit-sha cannot be
          TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)"
          if [[ ${TARGET_COMMIT_SHA} == "" ]]; then
            # Sometimes retrieving the merge commit SHA from PR fails. We retry it once. Otherwise we
            # fall-back to github.event.pull_request.head.sha
            echo
            echo "Could not retrieve merge commit SHA from PR, waiting for 3 seconds and retrying."
            echo
            sleep 3
            TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)"
            if [[ ${TARGET_COMMIT_SHA} == "" ]]; then
              echo
              echo "Could not retrieve merge commit SHA from PR, falling back to PR head SHA."
              echo
              TARGET_COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
            fi
          fi
          echo "TARGET_COMMIT_SHA=${TARGET_COMMIT_SHA}"
          echo "TARGET_COMMIT_SHA=${TARGET_COMMIT_SHA}" >> ${GITHUB_ENV}
          echo "target-commit-sha=${TARGET_COMMIT_SHA}" >> ${GITHUB_OUTPUT}
        if: github.event_name == 'pull_request_target'
      # The labels in the event aren't updated when re-triggering the job, So lets hit the API to get
      # up-to-date values
      - name: Get latest PR labels
        id: get-latest-pr-labels
        run: |
          echo -n "pull-request-labels=" >> ${GITHUB_OUTPUT}
          gh api graphql --paginate -F node_id=${{github.event.pull_request.node_id}} -f query='
            query($node_id: ID!, $endCursor: String) {
              node(id:$node_id) {
                ... on PullRequest {
                  labels(first: 100, after: $endCursor) {
                    nodes { name }
                    pageInfo { hasNextPage endCursor }
                  }
                }
              }
            }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' >> ${GITHUB_OUTPUT}
        if: github.event_name == 'pull_request_target'
      - uses: actions/checkout@v4
        with:
          ref: ${{ env.TARGET_COMMIT_SHA }}
          persist-credentials: false
          fetch-depth: 2
      ####################################################################################################
      #  WE ONLY DO THAT CHECKOUT ABOVE TO RETRIEVE THE TARGET COMMIT AND IT'S PARENT. DO NOT RUN ANY CODE
      #  RIGHT AFTER THAT AS WE ARE GOING TO RESTORE THE TARGET BRANCH CODE IN THE NEXT STEP.
      ####################################################################################################
      - name: Checkout target branch to use ci/scripts and breeze from there.
        uses: actions/checkout@v4
        with:
          ref: ${{ github.base_ref }}
          persist-credentials: false
      ####################################################################################################
      #  HERE EVERYTHING IS PERFECTLY SAFE TO RUN. AT THIS POINT WE HAVE THE TARGET BRANCH CHECKED OUT
      #  AND WE CAN RUN ANY CODE FROM IT. WE CAN RUN BREEZE COMMANDS, WE CAN RUN SCRIPTS, WE CAN RUN
      #  COMPOSITE ACTIONS. WE CAN RUN ANYTHING THAT IS IN THE TARGET BRANCH AND THERE IS NO RISK THAT
      #  CODE WILL BE RUN FROM THE PR.
      ####################################################################################################
      - name: Cleanup docker
        run: ./scripts/ci/cleanup_docker.sh
      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"
      - name: Install Breeze
        uses: ./.github/actions/breeze
      ####################################################################################################
      #  WE RUN SELECTIVE CHECKS HERE USING THE TARGET COMMIT AND ITS PARENT TO BE ABLE TO COMPARE THEM
      #  AND SEE WHAT HAS CHANGED IN THE PR. THE CODE IS STILL RUN FROM THE TARGET BRANCH, SO IT IS SAFE
      #  TO RUN IT, WE ONLY PASS TARGET_COMMIT_SHA SO THAT SELECTIVE CHECKS CAN SEE WHAT'S COMING IN THE PR
      ####################################################################################################
      - name: Selective checks
        id: selective-checks
        env:
          PR_LABELS: "${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}"
          COMMIT_REF: "${{ env.TARGET_COMMIT_SHA }}"
          VERBOSE: "false"
          AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}"
        run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
      - name: env
        run: printenv
        env:
          PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}
          GITHUB_CONTEXT: ${{ toJson(github) }}


  build-ci-images:
    name: Build CI images
    permissions:
      contents: read
      packages: write
    secrets: inherit
    needs: [build-info]
    uses: ./.github/workflows/ci-image-build.yml
    # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow"
    if: |
      needs.build-info.outputs.ci-image-build == 'true' &&
      github.event.pull_request.head.repo.full_name != 'apache/airflow'
    with:
      runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
      runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }}
      do-build: ${{ needs.build-info.outputs.ci-image-build }}
      target-commit-sha: ${{ needs.build-info.outputs.target-commit-sha }}
      pull-request-target: "true"
      is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
      push-image: "true"
      use-uv: "true"
      image-tag: ${{ needs.build-info.outputs.image-tag }}
      platform: "linux/amd64"
      python-versions: ${{ needs.build-info.outputs.python-versions }}
      branch: ${{ needs.build-info.outputs.default-branch }}
      constraints-branch: ${{ needs.build-info.outputs.constraints-branch }}
      upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
      docker-cache: ${{ needs.build-info.outputs.docker-cache }}

  generate-constraints:
    name: Generate constraints
    needs: [build-info, build-ci-images]
    uses: ./.github/workflows/generate-constraints.yml
    with:
      runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
      python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
      # For regular PRs we do not need "no providers" constraints - they are only needed in canary builds
      generate-no-providers-constraints: "false"
      image-tag: ${{ needs.build-info.outputs.image-tag }}
      chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
      debug-resources: ${{ needs.build-info.outputs.debug-resources }}

  build-prod-images:
    name: Build PROD images
    permissions:
      contents: read
      packages: write
    secrets: inherit
    needs: [build-info, generate-constraints]
    uses: ./.github/workflows/prod-image-build.yml
    # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow"
    if: |
      needs.build-info.outputs.prod-image-build == 'true' &&
      github.event.pull_request.head.repo.full_name != 'apache/airflow'
    with:
      runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
      build-type: "Regular"
      do-build: ${{ needs.build-info.outputs.ci-image-build }}
      upload-package-artifact: "true"
      target-commit-sha: ${{ needs.build-info.outputs.target-commit-sha }}
      pull-request-target: "true"
      is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
      push-image: "true"
      use-uv: "true"
      image-tag: ${{ needs.build-info.outputs.image-tag }}
      platform: linux/amd64
      python-versions: ${{ needs.build-info.outputs.python-versions }}
      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
      branch: ${{ needs.build-info.outputs.default-branch }}
      constraints-branch: ${{ needs.build-info.outputs.constraints-branch }}
      build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
      upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
      chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
      docker-cache: ${{ needs.build-info.outputs.docker-cache }}