AirbyteLib: More robust error handling, installation improvements #11220
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Connector Ops CI - Pipeline Unit Test | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| jobs: | |
| get_ci_runner: | |
| runs-on: ubuntu-latest | |
| name: Get CI runner | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} | |
| fetch-depth: 1 | |
| - name: Get CI runner | |
| id: get_ci_runner | |
| uses: ./.github/actions/airbyte-ci-requirements | |
| with: | |
| runner_type: "test" | |
| runner_size: "large" | |
| airbyte_ci_command: "test" | |
| github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| outputs: | |
| runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} | |
| run-airbyte-ci-tests: | |
| name: Run Airbyte CI tests | |
| needs: get_ci_runner | |
| runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| # IMPORTANT! This is nessesary to make sure that a status is reported on the PR | |
| # even if the workflow is skipped. If we used github actions filters, the workflow | |
| # would not be reported as skipped, but instead would be forever pending. | |
| # | |
| # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE. | |
| # | |
| # Also it gets worse | |
| # | |
| # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED. | |
| # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING. | |
| - name: Get changed files | |
| uses: tj-actions/changed-files@v39 | |
| id: changes | |
| with: | |
| files_yaml: | | |
| ops: | |
| - 'airbyte-ci/connectors/connector_ops/**' | |
| - '!**/*.md' | |
| base_images: | |
| - 'airbyte-ci/connectors/connector_ops/**' | |
| - 'airbyte-ci/connectors/base_images/**' | |
| - '!**/*.md' | |
| pipelines: | |
| - 'airbyte-ci/connectors/connector_ops/**' | |
| - 'airbyte-ci/connectors/base_images/**' | |
| - 'airbyte-ci/connectors/pipelines/**' | |
| - '!**/*.md' | |
| metadata_lib: | |
| - 'airbyte-ci/connectors/metadata_service/lib/**' | |
| - '!**/*.md' | |
| metadata_orchestrator: | |
| - 'airbyte-ci/connectors/metadata_service/lib/**' | |
| - 'airbyte-ci/connectors/metadata_service/orchestrator/**' | |
| - '!**/*.md' | |
| airbyte_lib: | |
| - 'airbyte-lib/**' | |
| - '!**/*.md' | |
| - name: Run airbyte-ci/connectors/connector_ops tests | |
| if: steps.changes.outputs.ops_any_changed == 'true' | |
| id: run-airbyte-ci-connectors-connector-ops-tests | |
| uses: ./.github/actions/run-dagger-pipeline | |
| with: | |
| context: "pull_request" | |
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | |
| subcommand: "test airbyte-ci/connectors/connector_ops --poetry-run-command='pytest tests'" | |
| tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} | |
| - name: Run airbyte-ci/connectors/pipelines tests | |
| id: run-airbyte-ci-connectors-pipelines-tests | |
| if: steps.changes.outputs.pipelines_any_changed == 'true' | |
| uses: ./.github/actions/run-dagger-pipeline | |
| with: | |
| context: "pull_request" | |
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | |
| subcommand: "test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests' --poetry-run-command='mypy pipelines --disallow-untyped-defs' --poetry-run-command='ruff check pipelines'" | |
| tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} | |
| - name: Run airbyte-ci/connectors/base_images tests | |
| id: run-airbyte-ci-connectors-base-images-tests | |
| if: steps.changes.outputs.base_images_any_changed == 'true' | |
| uses: ./.github/actions/run-dagger-pipeline | |
| with: | |
| context: "pull_request" | |
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | |
| subcommand: "test airbyte-ci/connectors/base_images --poetry-run-command='pytest tests'" | |
| tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} | |
| - name: Run test pipeline for the metadata lib | |
| id: metadata_lib-test-pipeline | |
| if: steps.changes.outputs.metadata_lib_any_changed == 'true' | |
| uses: ./.github/actions/run-dagger-pipeline | |
| with: | |
| subcommand: "test airbyte-ci/connectors/metadata_service/lib/ --poetry-run-command='pytest tests'" | |
| context: "pull_request" | |
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} | |
| - name: Run test for the metadata orchestrator | |
| id: metadata_orchestrator-test-pipeline | |
| if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true' | |
| uses: ./.github/actions/run-dagger-pipeline | |
| with: | |
| subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/ --poetry-run-command='pytest tests'" | |
| context: "pull_request" | |
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} | |
| - name: Run airbyte-lib tests | |
| if: steps.changes.outputs.airbyte_lib_any_changed == 'true' | |
| id: run-airbyte-lib-tests | |
| uses: ./.github/actions/run-dagger-pipeline | |
| with: | |
| context: "pull_request" | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | |
| subcommand: "test airbyte-lib --side-car-docker-engine --pass-env-var=GCP_GSM_CREDENTIALS --poetry-run-command='pytest'" | |
| tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} |