Skip to content

Another small fix of system tests #1348

Another small fix of system tests

Another small fix of system tests #1348

Workflow file for this run

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Tests
on: # yamllint disable-line rule:truthy
schedule:
- cron: '28 1,7,13,19 * * *'
push:
branches: ['v[0-9]+-[0-9]+-test']
pull_request:
branches: ['main', 'v[0-9]+-[0-9]+-test', 'v[0-9]+-[0-9]+-stable']
workflow_dispatch:
permissions:
# All other permissions are set to none
contents: read
# Technically read access while waiting for images should be more than enough. However,
# there is a bug in GitHub Actions/Packages and in case private repositories are used, you get a permission
# denied error when attempting to just pull private image, changing the token permission to write solves the
# issue. This is not dangerous, because if it is for "apache/airflow", only maintainers can push ci.yml
# changes. If it is for a fork, then the token is read-only anyway.
packages: write
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USERNAME: ${{ github.actor }}
IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
VERBOSE: "true"
concurrency:
group: ci-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build-info:
name: "Build info"
# At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners
runs-on: ["ubuntu-22.04"]
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
all-python-versions-list-as-string: >-
${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
build-job-description: ${{ steps.source-run-info.outputs.build-job-description }}
canary-run: ${{ steps.source-run-info.outputs.canary-run }}
chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
core-test-types-list-as-string: >-
${{ steps.selective-checks.outputs.core-test-types-list-as-string }}
debug-resources: ${{ steps.selective-checks.outputs.debug-resources }}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }}
default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }}
default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }}
default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }}
docker-cache: ${{ steps.selective-checks.outputs.docker-cache }}
docs-build: ${{ steps.selective-checks.outputs.docs-build }}
docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }}
excluded-providers-as-string: ${{ steps.selective-checks.outputs.excluded-providers-as-string }}
force-pip: ${{ steps.selective-checks.outputs.force-pip }}
full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }}
has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }}
image-tag: ${{ github.event.pull_request.head.sha || github.sha }}
in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }}
include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }}
individual-providers-test-types-list-as-string: >-
${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-string }}
is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }}
is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }}
is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
kubernetes-combos-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }}
kubernetes-versions-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }}
mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }}
needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }}
only-new-ui-files: ${{ steps.selective-checks.outputs.only-new-ui-files }}
postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
# yamllint disable rule:line-length
providers-compatibility-tests-matrix: ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }}
providers-test-types-list-as-string: >-
${{ steps.selective-checks.outputs.providers-test-types-list-as-string }}
pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
python-versions: ${{ steps.selective-checks.outputs.python-versions }}
run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
run-coverage: ${{ steps.source-run-info.outputs.run-coverage }}
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }}
run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }}
runs-on-as-json-docs-build: ${{ steps.selective-checks.outputs.runs-on-as-json-docs-build }}
runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }}
runs-on-as-json-self-hosted-asf: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted-asf }}
runs-on-as-json-self-hosted: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted }}
selected-providers-list-as-string: >-
${{ steps.selective-checks.outputs.selected-providers-list-as-string }}
skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }}
skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }}
source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
test-groups: ${{ steps.selective-checks.outputs.test-groups }}
testable-core-integrations: ${{ steps.selective-checks.outputs.testable-core-integrations }}
testable-providers-integrations: ${{ steps.selective-checks.outputs.testable-providers-integrations }}
upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
steps:
- name: "Cleanup repo"
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Cleanup docker"
run: ./scripts/ci/cleanup_docker.sh
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 2
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: "Get information about the Workflow"
id: source-run-info
run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT}
- name: Selective checks
id: selective-checks
env:
PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}"
COMMIT_REF: "${{ github.sha }}"
VERBOSE: "false"
run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
- name: env
run: printenv
env:
PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
basic-tests:
name: "Basic tests"
needs: [build-info]
uses: ./.github/workflows/basic-tests.yml
with:
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}}
run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}}
default-python-version: ${{needs.build-info.outputs.default-python-version}}
basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}}
skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}}
canary-run: ${{needs.build-info.outputs.canary-run}}
latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}}
build-ci-images:
name: >
${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }}
CI images in-workflow
needs: [build-info]
uses: ./.github/workflows/ci-image-build.yml
permissions:
contents: read
# This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs
# from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
# For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the
# "in-workflow-build" condition
packages: write
secrets: inherit
with:
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }}
do-build: ${{ needs.build-info.outputs.in-workflow-build }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.docker-cache }}
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
wait-for-ci-images:
timeout-minutes: 120
name: "Wait for CI images"
runs-on: ${{ fromJSON(needs.build-info.outputs.runs-on-as-json-public) }}
needs: [build-info, build-ci-images]
if: needs.build-info.outputs.ci-image-build == 'true'
env:
BACKEND: sqlite
# Force more parallelism for pull even on public images
PARALLELISM: 6
INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}"
steps:
- name: "Cleanup repo"
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Cleanup docker"
run: ./scripts/ci/cleanup_docker.sh
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Install Breeze"
uses: ./.github/actions/breeze
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: Login to ghcr.io
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
id: wait-for-images
run: breeze ci-image pull --run-in-parallel --wait-for-image --tag-as-latest
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
if: needs.build-info.outputs.in-workflow-build == 'false'
additional-ci-image-checks:
name: "Additional CI image checks"
needs: [build-info, wait-for-ci-images]
uses: ./.github/workflows/additional-ci-image-checks.yml
if: needs.build-info.outputs.canary-run == 'true'
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
default-python-version: ${{ needs.build-info.outputs.default-python-version }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }}
docker-cache: ${{ needs.build-info.outputs.docker-cache }}
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
generate-constraints:
name: "Generate constraints"
needs: [build-info, wait-for-ci-images]
uses: ./.github/workflows/generate-constraints.yml
if: >
needs.build-info.outputs.ci-image-build == 'true' &&
needs.build-info.outputs.only-new-ui-files != 'true'
with:
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
# generate no providers constraints only in canary builds - they take quite some time to generate
# they are not needed for regular builds, they are only needed to update constraints in canaries
generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
ci-image-checks:
name: "CI image checks"
needs: [build-info, wait-for-ci-images]
uses: ./.github/workflows/ci-image-checks.yml
secrets: inherit
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
runs-on-as-json-docs-build: ${{ needs.build-info.outputs.runs-on-as-json-docs-build }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
needs-mypy: ${{ needs.build-info.outputs.needs-mypy }}
mypy-checks: ${{ needs.build-info.outputs.mypy-checks }}
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
branch: ${{ needs.build-info.outputs.default-branch }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
default-python-version: ${{ needs.build-info.outputs.default-python-version }}
docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }}
latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }}
basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }}
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
ci-image-build: ${{ needs.build-info.outputs.ci-image-build }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
docs-build: ${{ needs.build-info.outputs.docs-build }}
needs-api-codegen: ${{ needs.build-info.outputs.needs-api-codegen }}
default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
providers:
name: "Provider packages tests"
uses: ./.github/workflows/test-provider-packages.yml
needs: [build-info, wait-for-ci-images]
permissions:
contents: read
packages: read
secrets: inherit
if: >
needs.build-info.outputs.skip-providers-tests != 'true' &&
needs.build-info.outputs.latest-versions-only != 'true'
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
default-python-version: ${{ needs.build-info.outputs.default-python-version }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }}
# yamllint disable rule:line-length
providers-compatibility-tests-matrix: ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}
tests-helm:
name: "Helm tests"
uses: ./.github/workflows/helm-tests.yml
needs: [build-info, wait-for-ci-images]
permissions:
contents: read
packages: read
secrets: inherit
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
default-python-version: ${{ needs.build-info.outputs.default-python-version }}
if: >
needs.build-info.outputs.needs-helm-tests == 'true' &&
needs.build-info.outputs.default-branch == 'main' &&
needs.build-info.outputs.latest-versions-only != 'true'
tests-postgres:
name: "Postgres tests"
uses: ./.github/workflows/run-unit-tests.yml
needs: [build-info, wait-for-ci-images]
permissions:
contents: read
packages: read
secrets: inherit
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
backend: "postgres"
test-name: "Postgres"
test-scope: "DB"
test-groups: ${{ needs.build-info.outputs.test-groups }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
backend-versions: ${{ needs.build-info.outputs.postgres-versions }}
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: ${{ needs.build-info.outputs.postgres-exclude }}
core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-migration-tests: "true"
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.run-tests == 'true'
tests-mysql:
name: "MySQL tests"
uses: ./.github/workflows/run-unit-tests.yml
needs: [build-info, wait-for-ci-images]
permissions:
contents: read
packages: read
secrets: inherit
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
backend: "mysql"
test-name: "MySQL"
test-scope: "DB"
test-groups: ${{ needs.build-info.outputs.test-groups }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
backend-versions: ${{ needs.build-info.outputs.mysql-versions }}
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: ${{ needs.build-info.outputs.mysql-exclude }}
core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
run-migration-tests: "true"
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.run-tests == 'true'
tests-sqlite:
name: "Sqlite tests"
uses: ./.github/workflows/run-unit-tests.yml
needs: [build-info, wait-for-ci-images]
permissions:
contents: read
packages: read
secrets: inherit
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
backend: "sqlite"
test-name: "Sqlite"
test-name-separator: ""
test-scope: "DB"
test-groups: ${{ needs.build-info.outputs.test-groups }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
# No versions for sqlite
backend-versions: "['']"
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: ${{ needs.build-info.outputs.sqlite-exclude }}
core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
run-migration-tests: "true"
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.run-tests == 'true'
tests-non-db:
name: "Non-DB tests"
uses: ./.github/workflows/run-unit-tests.yml
needs: [build-info, wait-for-ci-images]
permissions:
contents: read
packages: read
secrets: inherit
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
backend: "sqlite"
test-name: ""
test-name-separator: ""
test-scope: "Non-DB"
test-groups: ${{ needs.build-info.outputs.test-groups }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
# No versions for non-db
backend-versions: "['']"
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: ${{ needs.build-info.outputs.sqlite-exclude }}
core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.run-tests == 'true'
tests-special:
name: "Special tests"
uses: ./.github/workflows/special-tests.yml
needs: [build-info, wait-for-ci-images]
permissions:
contents: read
packages: read
secrets: inherit
if: >
needs.build-info.outputs.run-tests == 'true' &&
(needs.build-info.outputs.canary-run == 'true' ||
needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' ||
needs.build-info.outputs.full-tests-needed == 'true')
with:
test-groups: ${{ needs.build-info.outputs.test-groups }}
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
default-python-version: ${{ needs.build-info.outputs.default-python-version }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }}
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
tests-integration-system:
name: Integration and System Tests
needs: [build-info, wait-for-ci-images]
uses: ./.github/workflows/integration-system-tests.yml
permissions:
contents: read
packages: read
secrets: inherit
with:
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }}
testable-providers-integrations: ${{ needs.build-info.outputs.testable-providers-integrations }}
run-system-tests: ${{ needs.build-info.outputs.run-tests }}
default-python-version: ${{ needs.build-info.outputs.default-python-version }}
default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }}
default-mysql-version: ${{ needs.build-info.outputs.default-mysql-version }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.run-tests == 'true'
tests-with-lowest-direct-resolution:
name: "Lowest direct dependency providers tests"
needs: [build-info, wait-for-ci-images]
uses: ./.github/workflows/run-unit-tests.yml
permissions:
contents: read
packages: read
secrets: inherit
if: >
needs.build-info.outputs.run-tests == 'true'
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
test-name: "LowestDeps-Postgres"
force-lowest-dependencies: "true"
test-scope: "All"
test-groups: ${{ needs.build-info.outputs.test-groups }}
backend: "postgres"
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
backend-versions: "['${{ needs.build-info.outputs.default-postgres-version }}']"
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: "[]"
core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
# yamllint disable rule:line-length
providers-test-types-list-as-string: ${{ needs.build-info.outputs.individual-providers-test-types-list-as-string }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
monitor-delay-time-in-seconds: 120
build-prod-images:
name: >
${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }}
PROD images in-workflow
needs: [build-info, build-ci-images, generate-constraints]
uses: ./.github/workflows/prod-image-build.yml
permissions:
contents: read
# This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs
# from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
# For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the
# "in-workflow-build" condition
packages: write
secrets: inherit
with:
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
build-type: "Regular"
do-build: ${{ needs.build-info.outputs.in-workflow-build }}
upload-package-artifact: "true"
image-tag: ${{ needs.build-info.outputs.image-tag }}
platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
default-python-version: ${{ needs.build-info.outputs.default-python-version }}
branch: ${{ needs.build-info.outputs.default-branch }}
push-image: "true"
use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.docker-cache }}
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
wait-for-prod-images:
timeout-minutes: 80
name: "Wait for PROD images"
runs-on: ${{ fromJSON(needs.build-info.outputs.runs-on-as-json-public) }}
needs: [build-info, wait-for-ci-images, build-prod-images]
if: needs.build-info.outputs.prod-image-build == 'true'
env:
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
# Force more parallelism for pull on public images
PARALLELISM: 6
INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}"
IMAGE_TAG: ${{ needs.build-info.outputs.image-tag }}
steps:
- name: "Cleanup repo"
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Cleanup docker"
run: ./scripts/ci/cleanup_docker.sh
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Install Breeze"
uses: ./.github/actions/breeze
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: Login to ghcr.io
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
# We wait for the images to be available either from "build-images.yml' run as pull_request_target
# or from build-prod-images (or build-prod-images-release-branch) above.
# We are utilising single job to wait for all images because this job merely waits
# For the images to be available.
run: breeze prod-image pull --wait-for-image --run-in-parallel
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.in-workflow-build == 'false'
additional-prod-image-tests:
name: "Additional PROD image tests"
needs: [build-info, wait-for-prod-images, generate-constraints]
uses: ./.github/workflows/additional-prod-image-tests.yml
with:
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
docker-cache: ${{ needs.build-info.outputs.docker-cache }}
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
default-python-version: ${{ needs.build-info.outputs.default-python-version }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
if: needs.build-info.outputs.prod-image-build == 'true'
tests-kubernetes:
name: "Kubernetes tests"
uses: ./.github/workflows/k8s-tests.yml
needs: [build-info, wait-for-prod-images]
permissions:
contents: read
packages: read
secrets: inherit
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
kubernetes-versions-list-as-string: ${{ needs.build-info.outputs.kubernetes-versions-list-as-string }}
kubernetes-combos-list-as-string: ${{ needs.build-info.outputs.kubernetes-combos-list-as-string }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: >
( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
needs.build-info.outputs.needs-helm-tests == 'true')
tests-task-sdk:
name: "Task SDK tests"
uses: ./.github/workflows/task-sdk-tests.yml
needs: [build-info, wait-for-ci-images]
permissions:
contents: read
packages: read
secrets: inherit
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
default-python-version: ${{ needs.build-info.outputs.default-python-version }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
run-task-sdk-tests: ${{ needs.build-info.outputs.run-task-sdk-tests }}
if: >
( needs.build-info.outputs.run-task-sdk-tests == 'true' ||
needs.build-info.outputs.run-tests == 'true' &&
needs.build-info.outputs.only-new-ui-files != 'true')
finalize-tests:
name: Finalize tests
permissions:
contents: write
packages: write
secrets: inherit
needs:
- build-info
- generate-constraints
- wait-for-ci-images
- wait-for-prod-images
- ci-image-checks
- tests-sqlite
- tests-mysql
- tests-postgres
- tests-non-db
- tests-integration-system
uses: ./.github/workflows/finalize-tests.yml
with:
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
default-python-version: ${{ needs.build-info.outputs.default-python-version }}
in-workflow-build: ${{ needs.build-info.outputs.in-workflow-build }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
docker-cache: ${{ needs.build-info.outputs.docker-cache }}
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
notify-slack-failure:
name: "Notify Slack on Failure"
needs:
- basic-tests
- additional-ci-image-checks
- providers
- tests-helm
- tests-special
- tests-with-lowest-direct-resolution
- additional-prod-image-tests
- tests-kubernetes
- tests-task-sdk
- finalize-tests
if: github.event_name == 'schedule' && failure()
runs-on: ["ubuntu-22.04"]
steps:
- name: Notify Slack
id: slack
uses: slackapi/[email protected]
with:
channel-id: 'internal-airflow-ci-cd'
# yamllint disable rule:line-length
payload: |
{
"text": "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>"
}
}
]
}
# yamllint enable rule:line-length
env:
SLACK_BOT_TOKEN: ${{ env.SLACK_BOT_TOKEN }}