diff --git a/.github/workflows/ci-astro-deploy.yml b/.github/workflows/ci-astro-deploy.yml index 05c85b8ef..2fad30783 100644 --- a/.github/workflows/ci-astro-deploy.yml +++ b/.github/workflows/ci-astro-deploy.yml @@ -1,16 +1,17 @@ +--- name: "Astro Deploy" on: schedule: - - cron: '0 0 * * *' + - cron: "0 0 * * *" workflow_dispatch: inputs: git_rev: - description: 'The git revision to deploy' + description: "The git revision to deploy" required: false - default: '' + default: "" environment_to_deploy: - description: 'astro cloud deployment to deploy to' + description: "astro cloud deployment to deploy to" required: true type: choice options: @@ -23,16 +24,16 @@ on: (e.g. "example_mssql_transform, example_load_file") required: false type: string - default: '' + default: "" workflow_call: inputs: git_rev: - description: 'The git revision to deploy' + description: "The git revision to deploy" type: string required: false - default: '' + default: "" environment_to_deploy: - description: 'astro cloud deployment to deploy to' + description: "astro cloud deployment to deploy to" required: true type: string default: both @@ -42,7 +43,7 @@ on: (e.g. "example_mssql_transform, example_load_file") required: false type: string - default: '' + default: "" jobs: deploy-to-astro-sdk-integration-tests: @@ -52,11 +53,11 @@ jobs: uses: ./.github/workflows/reuse-wf-deploy-to-astro-cloud.yaml with: git_rev: ${{ inputs.git_rev }} - environment_to_deploy: 'astro-sdk-integration-tests' + environment_to_deploy: "astro-sdk-integration-tests" secrets: - docker_registry: ${{ secrets.ASTRO_DOCKER_REGISTRY }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - deployment_id: ${{ secrets.ASTRO_DEPLOYMENT_ID }} + docker_registry: ${{ secrets.ASTRO_DOCKER_REGISTRY }} + organization_id: ${{ secrets.ORGANIZATION_ID }} + deployment_id: ${{ secrets.ASTRO_DEPLOYMENT_ID }} astronomer_key_id: ${{ secrets.ASTRO_KEY_ID }} astronomer_key_secret: ${{ secrets.ASTRO_KEY_SECRET }} SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} @@ -85,11 +86,11 @@ jobs: uses: ./.github/workflows/reuse-wf-deploy-to-astro-cloud.yaml with: git_rev: ${{ inputs.git_rev }} - environment_to_deploy: 'astro-sdk-integration-tests-on-KE' + environment_to_deploy: "astro-sdk-integration-tests-on-KE" secrets: - docker_registry: ${{ secrets.ASTRO_DOCKER_REGISTRY }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - deployment_id: ${{ secrets.ASTRO_DEPLOYMENT_ID_KE }} + docker_registry: ${{ secrets.ASTRO_DOCKER_REGISTRY }} + organization_id: ${{ secrets.ORGANIZATION_ID }} + deployment_id: ${{ secrets.ASTRO_DEPLOYMENT_ID_KE }} astronomer_key_id: ${{ secrets.ASTRO_KEY_ID_KE }} astronomer_key_secret: ${{ secrets.ASTRO_KEY_SECRET_KE }} SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/ci-benchmark.yaml b/.github/workflows/ci-benchmark.yaml index cd834204a..f9d89bbb7 100644 --- a/.github/workflows/ci-benchmark.yaml +++ b/.github/workflows/ci-benchmark.yaml @@ -1,10 +1,11 @@ +--- name: Run benchmarks on: pull_request: types: - labeled schedule: - - cron: '0 0 * * 1' + - cron: "0 0 * * 1" # This allows a subsequently queued workflow run to interrupt and cancel previous runs concurrency: diff --git a/.github/workflows/ci-python-sdk.yaml b/.github/workflows/ci-python-sdk.yaml index 1331cbb1b..4fc120702 100644 --- a/.github/workflows/ci-python-sdk.yaml +++ b/.github/workflows/ci-python-sdk.yaml @@ -1,26 +1,27 @@ +--- name: Build and test astro Python SDK on: push: - branches: [ 'main', 'release-**' ] + branches: ["main", "release-**"] paths: - - 'python-sdk/**' - - '.github/workflows/ci-python-sdk.yaml' + - "python-sdk/**" + - ".github/workflows/ci-python-sdk.yaml" pull_request: - branches: [ 'main', 'release-**' ] + branches: ["main", "release-**"] paths: - - 'python-sdk/**' - - '.github/workflows/ci-python-sdk.yaml' - - '*' + - "python-sdk/**" + - ".github/workflows/ci-python-sdk.yaml" + - "*" # Run on PRs from forks pull_request_target: - branches: [ 'main' ] - types: ['labeled'] + branches: ["main"] + types: ["labeled"] paths: - - 'python-sdk/**' - - '.github/workflows/ci-python-sdk.yaml' - - '*' + - "python-sdk/**" + - ".github/workflows/ci-python-sdk.yaml" + - "*" release: - types: [ 'created' ] + types: ["created"] defaults: run: working-directory: python-sdk @@ -74,7 +75,7 @@ jobs: - uses: actions/checkout@v3 - uses: gaurav-nelson/github-action-markdown-link-check@v1 with: - config-file: '.github/workflows/mlc_config.json' + config-file: ".github/workflows/mlc_config.json" Type-Check: if: github.event.action != 'labeled' @@ -87,14 +88,14 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: '3.10' - architecture: 'x64' + python-version: "3.10" + architecture: "x64" - uses: actions/cache@v3 with: - path: | - ~/.cache/pip - .nox - key: ${{ runner.os }}-${{ hashFiles('python-sdk/pyproject.toml') }} + path: | + ~/.cache/pip + .nox + key: ${{ runner.os }}-${{ hashFiles('python-sdk/pyproject.toml') }} - run: pip3 install nox - run: nox -s type_check @@ -105,8 +106,8 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: '3.10' - architecture: 'x64' + python-version: "3.10" + architecture: "x64" - uses: actions/cache@v3 with: path: | @@ -170,8 +171,8 @@ jobs: - uses: actions/setup-python@v3 with: - python-version: '3.10' - architecture: 'x64' + python-version: "3.10" + architecture: "x64" - uses: actions/cache@v3 with: path: | @@ -206,7 +207,7 @@ jobs: Run-Unit-tests-Airflow-2-7: strategy: matrix: - version: [ '3.8', '3.9', '3.10', '3.11' ] + version: ["3.8", "3.9", "3.10", "3.11"] if: >- github.event_name == 'push' || ( @@ -233,7 +234,7 @@ jobs: - uses: actions/setup-python@v3 with: python-version: ${{ matrix.version }} - architecture: 'x64' + architecture: "x64" - uses: actions/cache@v3 with: path: | @@ -266,7 +267,7 @@ jobs: strategy: fail-fast: false matrix: - group: [ 1, 2, 3] + group: [1, 2, 3] runs-on: ubuntu-latest services: postgres: @@ -307,8 +308,8 @@ jobs: if: github.event_name == 'pull_request_target' - uses: actions/setup-python@v3 with: - python-version: '3.10' - architecture: 'x64' + python-version: "3.10" + architecture: "x64" - uses: actions/cache@v3 with: path: | @@ -362,7 +363,7 @@ jobs: strategy: fail-fast: false matrix: - group: [ 1, 2, 3] + group: [1, 2, 3] runs-on: ubuntu-latest services: postgres: @@ -403,8 +404,8 @@ jobs: if: github.event_name == 'pull_request_target' - uses: actions/setup-python@v3 with: - python-version: '3.10' - architecture: 'x64' + python-version: "3.10" + architecture: "x64" - uses: actions/cache@v3 with: path: | @@ -458,7 +459,7 @@ jobs: strategy: fail-fast: false matrix: - group: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ] + group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] runs-on: ubuntu-latest services: postgres: @@ -499,8 +500,8 @@ jobs: if: github.event_name == 'pull_request_target' - uses: actions/setup-python@v3 with: - python-version: '3.10' - architecture: 'x64' + python-version: "3.10" + architecture: "x64" - uses: actions/cache@v3 with: path: | @@ -593,8 +594,8 @@ jobs: - uses: actions/setup-python@v3 with: - python-version: '3.8' - architecture: 'x64' + python-version: "3.8" + architecture: "x64" - uses: actions/cache@v3 with: path: | @@ -626,15 +627,15 @@ jobs: strategy: fail-fast: false matrix: - python: [ '3.7', '3.8', '3.9', '3.10' ] - airflow: [ '2.2.5', '2.3', '2.4', '2.5', '2.6'] + python: ["3.7", "3.8", "3.9", "3.10"] + airflow: ["2.2.5", "2.3", "2.4", "2.5", "2.6"] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: '${{ matrix.python }}' - architecture: 'x64' + python-version: "${{ matrix.python }}" + architecture: "x64" - name: Get pip cache dir id: pip-cache run: | @@ -669,7 +670,7 @@ jobs: - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: '3.10' + python-version: "3.10" - name: Install coverage run: | pip3 install coverage @@ -732,18 +733,18 @@ jobs: - Generate-Constraints runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.10' - architecture: 'x64' - - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ hashFiles('python-sdk/pyproject.toml') }} - - run: pip3 install nox - - run: nox -s build - - run: nox -s release -- dist/* + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: "3.10" + architecture: "x64" + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('python-sdk/pyproject.toml') }} + - run: pip3 install nox + - run: nox -s build + - run: nox -s release -- dist/* env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/ci-rc-test.yaml b/.github/workflows/ci-rc-test.yaml index da696e9e7..12301ea3a 100644 --- a/.github/workflows/ci-rc-test.yaml +++ b/.github/workflows/ci-rc-test.yaml @@ -47,9 +47,9 @@ jobs: if: | always() && needs.check-airflow-provider-rc-release.result == 'success' - uses: ./.github/workflows/ci-astro-deploy.yml + uses: ./.github/workflows/ci-astro-deploy.yml with: - environment_to_deploy: "both" - dags_to_trigger_after_deployment: "example_master_dag" - git_rev: ${{ needs.check-airflow-provider-rc-release.outputs.rc_testing_branch }} + environment_to_deploy: "both" + dags_to_trigger_after_deployment: "example_master_dag" + git_rev: ${{ needs.check-airflow-provider-rc-release.outputs.rc_testing_branch }} secrets: inherit diff --git a/.github/workflows/reuse-wf-deploy-to-astro-cloud.yaml b/.github/workflows/reuse-wf-deploy-to-astro-cloud.yaml index a28c6ef9d..79c30c7b9 100644 --- a/.github/workflows/reuse-wf-deploy-to-astro-cloud.yaml +++ b/.github/workflows/reuse-wf-deploy-to-astro-cloud.yaml @@ -1,41 +1,41 @@ --- name: (Reusable workflows) Deploy to astro cloud -on: # yamllint disable-line rule:truthy +on: # yamllint disable-line rule:truthy workflow_call: inputs: git_rev: - description: 'The git revision to deploy' + description: "The git revision to deploy" required: false type: string - default: '' + default: "" environment_to_deploy: - description: 'astro cloud deployment to deploy to' + description: "astro cloud deployment to deploy to" required: true type: string secrets: docker_registry: - description: 'astro cloud docker registry' + description: "astro cloud docker registry" required: true organization_id: - description: 'astro cloud organization_id' + description: "astro cloud organization_id" required: true deployment_id: - description: 'astro cloud deployment_id' + description: "astro cloud deployment_id" required: true astronomer_key_id: - description: 'astro cloud astronomer_key_id' + description: "astro cloud astronomer_key_id" required: true astronomer_key_secret: - description: 'astro cloud astronomer_key_secret' + description: "astro cloud astronomer_key_secret" required: true SLACK_WEBHOOK_URL: - description: 'slack webhook url for sending notification' + description: "slack webhook url for sending notification" required: true jobs: deploy-to-astro-cloud: - runs-on: 'ubuntu-20.04' + runs-on: "ubuntu-20.04" steps: - name: checkout uses: actions/checkout@v3 diff --git a/.github/workflows/reuse-wf-trigger-dag.yaml b/.github/workflows/reuse-wf-trigger-dag.yaml index ddb0e4241..a4e702d41 100644 --- a/.github/workflows/reuse-wf-trigger-dag.yaml +++ b/.github/workflows/reuse-wf-trigger-dag.yaml @@ -1,46 +1,45 @@ --- name: (Reusable workflows) Wait for deployment and trigger dags -on: # yamllint disable-line rule:truthy +on: # yamllint disable-line rule:truthy workflow_call: inputs: git_rev: - description: 'The git revision to deploy' + description: "The git revision to deploy" required: false type: string - default: '' + default: "" dags_to_trigger_after_deployment: description: | Comma separated list of dag_ids to trigger after deployment (e.g. "example_mssql_transform, example_load_file") required: false type: string - default: '' + default: "" secrets: astro_subdomain: - description: 'astro cloud subdomain' + description: "astro cloud subdomain" required: true deployment_id: - description: 'astro cloud deployment_id' + description: "astro cloud deployment_id" required: true astronomer_key_id: - description: 'astro cloud astronomer_key_id' + description: "astro cloud astronomer_key_id" required: true astronomer_key_secret: - description: 'astro cloud astronomer_key_secret' + description: "astro cloud astronomer_key_secret" required: true organization_id: - description: 'astro cloud organization_id' + description: "astro cloud organization_id" required: true bearer_token: - description: 'workspace bearer token' + description: "workspace bearer token" required: true jobs: wait-for-deployment-to-be-ready-and-trigger-dag: - runs-on: 'ubuntu-20.04' + runs-on: "ubuntu-20.04" steps: - - name: Wait for deployment to be healthy run: | astro_core_api="https://api.astronomer.io/v1alpha1/organizations/${{secrets.organization_id }}/\ diff --git a/.github/workflows/runtime-image-update.yaml b/.github/workflows/runtime-image-update.yaml index 03c1ce1af..f12f654f7 100644 --- a/.github/workflows/runtime-image-update.yaml +++ b/.github/workflows/runtime-image-update.yaml @@ -1,18 +1,18 @@ --- name: Update runtime image tag -on: # yamllint disable-line rule:truthy +on: # yamllint disable-line rule:truthy schedule: - - cron: '0 0 * * *' # Run every day at midnight UTC + - cron: "0 0 * * *" # Run every day at midnight UTC workflow_dispatch: inputs: quay_repo: - description: 'Quay Repository' - default: 'astronomer/astro-runtime' + description: "Quay Repository" + default: "astronomer/astro-runtime" jobs: runtime-image-tag-update-job: - runs-on: 'ubuntu-20.04' + runs-on: "ubuntu-20.04" steps: - name: Checkout repository @@ -21,7 +21,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: '3.10' + python-version: "3.10" - name: Install dependencies run: pip install requests semantic-version