Skip to content

Build images for Create DataprocStartClusterOperator and DataprocStopClusterOperator https://github.com/VladaZakharova/airflow/pull/8 #8

Build images for Create DataprocStartClusterOperator and DataprocStopClusterOperator https://github.com/VladaZakharova/airflow/pull/8

Build images for Create DataprocStartClusterOperator and DataprocStopClusterOperator https://github.com/VladaZakharova/airflow/pull/8 #8

Workflow file for this run

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: "Build Images"
run-name: >
Build images for ${{ github.event.pull_request.title }} ${{ github.event.pull_request._links.html.href }}
on: # yamllint disable-line rule:truthy
pull_request_target:
permissions:
# all other permissions are set to none
contents: read
pull-requests: read
packages: read
env:
ANSWER: "yes"
DB_RESET: "true"
VERBOSE: "true"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.actor }}
# You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
# Airflow one is going to be used
CONSTRAINTS_GITHUB_REPOSITORY: >-
${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
# This token is WRITE one - pull_request_target type of events always have the WRITE token
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
USE_SUDO: "true"
INCLUDE_SUCCESS_OUTPUTS: "true"
concurrency:
group: build-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build-info:
timeout-minutes: 10
name: "Build Info"
runs-on: 'ubuntu-22.04'
env:
TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
outputs:
python-versions: "${{ steps.selective-checks.python-versions }}"
upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
all-python-versions-list-as-string: >-
${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
runs-on: ${{steps.selective-checks.outputs.runs-on}}
is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }}
is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }}
is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha ||
github.event.pull_request.head.sha ||
github.sha
}}"
if: github.repository == 'apache/airflow'
steps:
- name: Discover PR merge commit
id: discover-pr-merge-commit
run: |
TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)"
echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> ${GITHUB_ENV}
echo "target-commit-sha=${TARGET_COMMIT_SHA}" >> ${GITHUB_OUTPUT}
if: github.event_name == 'pull_request_target'
# The labels in the event aren't updated when re-triggering the job, So lets hit the API to get
# up-to-date values
- name: Get latest PR labels
id: get-latest-pr-labels
run: |
echo -n "pull-request-labels=" >> ${GITHUB_OUTPUT}
gh api graphql --paginate -F node_id=${{github.event.pull_request.node_id}} -f query='
query($node_id: ID!, $endCursor: String) {
node(id:$node_id) {
... on PullRequest {
labels(first: 100, after: $endCursor) {
nodes { name }
pageInfo { hasNextPage endCursor }
}
}
}
}' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' >> ${GITHUB_OUTPUT}
if: github.event_name == 'pull_request_target'
# Retrieve it to be able to determine which files has changed in the incoming commit of the PR
# we checkout the target commit and its parent to be able to compare them
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v4
with:
ref: ${{ env.TARGET_COMMIT_SHA }}
persist-credentials: false
fetch-depth: 2
- name: "Setup python"
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: "Retrieve defaults from branch_defaults.py"
# We cannot "execute" the branch_defaults.py python code here because that would be
# a security problem (we cannot run any code that comes from the sources coming from the PR.
# Therefore, we extract the branches via embedded Python code
# we need to do it before next step replaces checked-out breeze and scripts code coming from
# the PR, because the PR defaults have to be retrieved here.
id: defaults
run: |
python - <<EOF >> ${GITHUB_ENV}
from pathlib import Path
import re
import sys
DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text()
BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$'
CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$'
branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
output = f"""
DEFAULT_BRANCH={branch}
DEFAULT_CONSTRAINTS_BRANCH={constraints_branch}
""".strip()
print(output)
# Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log
print(output, file=sys.stderr)
EOF
- name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there.
uses: actions/checkout@v4
with:
path: "target-airflow"
ref: ${{ github.base_ref }}
persist-credentials: false
- name: >
Override "scripts/ci", "dev" and ".github/actions" with the target branch
so that the PR does not override it
# We should not override those scripts which become part of the image as they will not be
# changed in the image built - we should only override those that are executed to build
# the image.
shell: bash
run: |
rm -rfv "scripts/ci"
mv -v "target-airflow/scripts/ci" "scripts"
rm -rfv "dev"
mv -v "target-airflow/dev" "."
rm -rfv ".github/actions"
mv -v "target-airflow/.github/actions" ".github"
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: Selective checks
id: selective-checks
env:
PR_LABELS: "${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}"
COMMIT_REF: "${{ env.TARGET_COMMIT_SHA }}"
VERBOSE: "false"
AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}"
run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
- name: env
run: printenv
env:
PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
build-ci-images:
permissions:
contents: read
packages: write
timeout-minutes: 80
name: >
Build CI images ${{needs.build-info.outputs.all-python-versions-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
if: |
needs.build-info.outputs.ci-image-build == 'true' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow' &&
github.repository == 'apache/airflow'
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
BACKEND: sqlite
VERSION_SUFFIX_FOR_PYPI: "dev0"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.target-commit-sha }}
persist-credentials: false
- name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there.
uses: actions/checkout@v4
with:
path: "target-airflow"
ref: ${{ github.base_ref }}
persist-credentials: false
- name: >
Override "scripts/ci", "dev" and ".github/actions" with the target branch
so that the PR does not override it
# We should not override those scripts which become part of the image as they will not be
# changed in the image built - we should only override those that are executed to build
# the image.
shell: bash
run: |
rm -rfv "scripts/ci"
mv -v "target-airflow/scripts/ci" "scripts"
rm -rfv "dev"
mv -v "target-airflow/dev" "."
rm -rfv ".github/actions"
mv -v "target-airflow/.github/actions" ".github"
- name: >
Build CI Images ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/build-ci-images
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
BUILD_TIMEOUT_MINUTES: 70
build-prod-images:
permissions:
contents: read
packages: write
timeout-minutes: 80
name: >
Build PROD images
${{needs.build-info.outputs.all-python-versions-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, build-ci-images]
if: |
needs.build-info.outputs.prod-image-build == 'true' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow' &&
github.repository == 'apache/airflow'
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
BACKEND: sqlite
VERSION_SUFFIX_FOR_PYPI: "dev0"
INCLUDE_NOT_READY_PROVIDERS: "true"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.target-commit-sha }}
persist-credentials: false
- name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there.
uses: actions/checkout@v4
with:
path: "target-airflow"
ref: ${{ github.base_ref }}
persist-credentials: false
- name: >
Override "scripts/ci", "dev" and ".github/actions" with the target branch
so that the PR does not override it
# We should not override those scripts which become part of the image as they will not be
# changed in the image built - we should only override those that are executed to build
# the image.
shell: bash
run: |
rm -rfv "scripts/ci"
mv -v "target-airflow/scripts/ci" "scripts"
rm -rfv "dev"
mv -v "target-airflow/dev" "."
rm -rfv ".github/actions"
mv -v "target-airflow/.github/actions" ".github"
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: >
Build PROD Images
${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/build-prod-images
with:
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
build-ci-images-arm:
timeout-minutes: 50
name: "Build ARM CI images ${{needs.build-info.outputs.all-python-versions-list-as-string}}"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, build-prod-images]
# We can change the job to run on ASF ARM runners and do not start our instance once we enable ASF runners
if: |
needs.build-info.outputs.ci-image-build == 'true' &&
needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow' &&
needs.build-info.outputs.is-self-hosted-runner == 'true' &&
needs.build-info.outputs.is-airflow-runner == 'true' &&
github.repository == 'apache/airflow'
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
BACKEND: sqlite
outputs: ${{toJSON(needs.build-info.outputs) }}
VERSION_SUFFIX_FOR_PYPI: "dev0"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.target-commit-sha }}
persist-credentials: false
- name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there.
uses: actions/checkout@v4
with:
path: "target-airflow"
ref: ${{ github.base_ref }}
persist-credentials: false
- name: >
Override "scripts/ci", "dev" and ".github/actions" with the target branch
so that the PR does not override it
# We should not override those scripts which become part of the image as they will not be
# changed in the image built - we should only override those that are executed to build
# the image.
shell: bash
run: |
rm -rfv "scripts/ci"
mv -v "target-airflow/scripts/ci" "scripts"
rm -rfv "dev"
mv -v "target-airflow/dev" "."
rm -rfv ".github/actions"
mv -v "target-airflow/.github/actions" ".github"
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: >
Build ARM CI images ${{ env.IMAGE_TAG }}
${{needs.build-info.outputs.all-python-versions-list-as-string}}
run: >
breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64"
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
if: always()