Merge branch 'master' into marcos/test-pr-34966

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.50.47
+current_version = 0.50.50
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.github/actions/runner-prepare-for-build/action.yml

@@ -20,7 +20,7 @@ runs:
       uses: actions/setup-java@v3
       with:
         distribution: "zulu"
-        java-version: "17"
+        java-version: "21"
 
     - if: inputs.install_node == 'true'
       uses: actions/setup-node@v3

.github/workflows/airbyte-ci-tests.yml

@@ -1,142 +1,80 @@
-name: Connector Ops CI - Pipeline Unit Test
+name: Internal Poetry packages CI
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 on:
   workflow_dispatch:
+    inputs:
+      airbyte_ci_subcommand:
+        description: "Subcommand to pass to the 'airbyte-ci test' command"
+        default: "--poetry-package-path=airbyte-ci/connectors/pipelines"
   pull_request:
     types:
       - opened
       - reopened
       - synchronize
 jobs:
-  run-airbyte-ci-tests:
+  run-airbyte-ci-poetry-ci:
+    #name: Internal Poetry packages CI
+    # To rename in a follow up PR
     name: Run Airbyte CI tests
     runs-on: tooling-test-large
+    permissions:
+      pull-requests: read
+      statuses: write
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.ref }}
 
-      # IMPORTANT! This is nessesary to make sure that a status is reported on the PR
-      # even if the workflow is skipped. If we used github actions filters, the workflow
-      # would not be reported as skipped, but instead would be forever pending.
-      #
-      # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE.
-      #
-      # Also it gets worse
-      #
-      # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED.
-      # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING.
-      - name: Get changed files
-        uses: tj-actions/changed-files@v39
-        id: changes
-        with:
-          files_yaml: |
-            ops:
-              - 'airbyte-ci/connectors/connector_ops/**'
-              - '!**/*.md'
-            base_images:
-              - 'airbyte-ci/connectors/connector_ops/**'
-              - 'airbyte-ci/connectors/base_images/**'
-              - '!**/*.md'
-            pipelines:
-              - 'airbyte-ci/connectors/connector_ops/**'
-              - 'airbyte-ci/connectors/base_images/**'
-              - 'airbyte-ci/connectors/pipelines/**'
-              - '!**/*.md'
-            metadata_lib:
-              - 'airbyte-ci/connectors/metadata_service/lib/**'
-              - '!**/*.md'
-            metadata_orchestrator:
-              - 'airbyte-ci/connectors/metadata_service/lib/**'
-              - 'airbyte-ci/connectors/metadata_service/orchestrator/**'
-              - '!**/*.md'
-            airbyte_lib:
-              - 'airbyte-lib/**'
-              - '!**/*.md'
-
-      - name: Run airbyte-ci/connectors/connector_ops tests
-        if: steps.changes.outputs.ops_any_changed == 'true'
-        id: run-airbyte-ci-connectors-connector-ops-tests
-        uses: ./.github/actions/run-airbyte-ci
-        with:
-          context: "pull_request"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
-          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
-          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          subcommand: "test airbyte-ci/connectors/connector_ops --poetry-run-command='pytest tests'"
-
-      - name: Run airbyte-ci/connectors/pipelines tests
-        id: run-airbyte-ci-connectors-pipelines-tests
-        if: steps.changes.outputs.pipelines_any_changed == 'true'
-        uses: ./.github/actions/run-airbyte-ci
-        with:
-          context: "pull_request"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
-          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
-          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          subcommand: "test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests' --poetry-run-command='mypy pipelines --disallow-untyped-defs' --poetry-run-command='ruff check pipelines'"
+      - name: Extract branch name [WORKFLOW DISPATCH]
+        shell: bash
+        if: github.event_name == 'workflow_dispatch'
+        run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
+        id: extract_branch
+      - name: Fetch last commit id from remote branch [PULL REQUESTS]
+        if: github.event_name == 'pull_request'
+        id: fetch_last_commit_id_pr
+        run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT
+      - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH]
+        if: github.event_name == 'workflow_dispatch'
+        id: fetch_last_commit_id_wd
+        run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT
 
-      - name: Run airbyte-ci/connectors/base_images tests
-        id: run-airbyte-ci-connectors-base-images-tests
-        if: steps.changes.outputs.base_images_any_changed == 'true'
+      - name: Run poe tasks for modified internal packages [PULL REQUEST]
+        if: github.event_name == 'pull_request'
+        id: run-airbyte-ci-test-pr
         uses: ./.github/actions/run-airbyte-ci
         with:
           context: "pull_request"
           dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          git_branch: ${{ github.head_ref }}
+          git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
+          github_token: ${{ github.token }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          subcommand: "test airbyte-ci/connectors/base_images --poetry-run-command='pytest tests'"
-
-      - name: Run test pipeline for the metadata lib
-        id: metadata_lib-test-pipeline
-        if: steps.changes.outputs.metadata_lib_any_changed == 'true'
-        uses: ./.github/actions/run-airbyte-ci
-        with:
-          subcommand: "test airbyte-ci/connectors/metadata_service/lib/ --poetry-run-command='pytest tests'"
-          context: "pull_request"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
-      - name: Run test for the metadata orchestrator
-        id: metadata_orchestrator-test-pipeline
-        if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true'
-        uses: ./.github/actions/run-airbyte-ci
-        with:
-          subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/ --poetry-run-command='pytest tests'"
-          context: "pull_request"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          subcommand: "test --modified"
 
-      - name: Run airbyte-lib tests
-        if: steps.changes.outputs.airbyte_lib_any_changed == 'true'
-        id: run-airbyte-lib-tests
+      - name: Run poe tasks for requested internal packages [WORKFLOW DISPATCH]
+        id: run-airbyte-ci-test-workflow-dispatch
+        if: github.event_name == 'workflow_dispatch'
         uses: ./.github/actions/run-airbyte-ci
         with:
-          context: "pull_request"
+          context: "manual"
           dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          git_branch: ${{ steps.extract_branch.outputs.branch }}
+          git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
+          github_token: ${{ github.token }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          subcommand: "test airbyte-lib --side-car-docker-engine --pass-env-var=GCP_GSM_CREDENTIALS --poetry-run-command='pytest'"
+          subcommand: "test ${{ inputs.airbyte_ci_subcommand}}"

.github/workflows/connector-performance-command.yml

@@ -159,7 +159,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "17"
+          java-version: "21"
       - name: Install Python
         uses: actions/setup-python@v4
         with:

.github/workflows/gradle.yml

@@ -1,4 +1,4 @@
-name: Airbyte CI - Repository Health Check
+name: Connector Ops CI - Gradle Check
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -20,47 +20,18 @@ on:
       - synchronize
 
 jobs:
-  # In case of self-hosted EC2 errors, remove this block.
-  start-check-runner:
-    name: Start EC2 Runner
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    outputs:
-      label: ${{ steps.start-ec2-runner.outputs.label }}
-      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
-    steps:
-      - name: Checkout Airbyte
-        uses: actions/checkout@v3
-      - name: Check PAT rate limits
-        run: |
-          ./tools/bin/find_non_rate_limited_PAT \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
-      - name: Start AWS Runner
-        id: start-ec2-runner
-        uses: ./.github/actions/start-aws-runner
-        with:
-          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          # Use a beefier instance type than the default c5.2xlarge, but with the same per-core cost.
-          # When gradle runs on this instance, it will use up all the available cores anyway.
-          # There should be little to no difference in total cost, however the job latency will be improved.
-          # At the time of this writing, the latency doesn't improve much beyond this instance size (approx 5 minutes).
-          # This is largely thanks to the gradle cache.
-          ec2-instance-type: "c5.4xlarge"
-          github-token: ${{ env.PAT }}
-
   run-check:
-    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
-    needs: start-check-runner # required to start the main job when the runner is ready
-    runs-on: ${{ needs.start-check-runner.outputs.label }} # run the job on the newly created runner
+    # The gradle check task which we will run is embarrassingly parallelizable.
+    # We therefore run this on a machine with a maximum number of cores.
+    # We pay per time and per core, so there should be little difference in total cost.
+    # The latency overhead of setting up gradle prior to running the actual task adds up to about a minute.
+    runs-on: connector-test-xxlarge
     name: Gradle Check
     timeout-minutes: 30
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
-
-      # IMPORTANT! This is nessesary to make sure that a status is reported on the PR
+      # IMPORTANT! This is necessary to make sure that a status is reported on the PR
       # even if the workflow is skipped. If we used github actions filters, the workflow
       # would not be reported as skipped, but instead would be forever pending.
       #
@@ -83,7 +54,7 @@ jobs:
       - uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "17"
+          java-version: "21"
       - name: Install Pip
         if: steps.changes.outputs.gradlecheck_any_changed == 'true'
         run: curl -fsSL https://bootstrap.pypa.io/get-pip.py | python3
@@ -106,39 +77,6 @@ jobs:
           # TODO: be able to remove the skipSlowTests property
           arguments: --scan --no-daemon --no-watch-fs check -DskipSlowTests=true
 
-  # In case of self-hosted EC2 errors, remove this block.
-  stop-check-runner:
-    name: Stop EC2 Runner
-    timeout-minutes: 10
-    needs:
-      - start-check-runner # required to get output from the start-runner job
-      - run-check # required to wait when the main job is done
-    runs-on: ubuntu-latest
-    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
-    # Thus, we check for skipped here.
-    if: ${{ always() && needs.start-check-runner.result != 'skipped'}}
-    steps:
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-2
-      - name: Checkout Airbyte
-        uses: actions/checkout@v3
-      - name: Check PAT rate limits
-        run: |
-          ./tools/bin/find_non_rate_limited_PAT \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
-      - name: Stop EC2 runner
-        uses: supertopher/[email protected]
-        with:
-          mode: stop
-          github-token: ${{ env.PAT }}
-          label: ${{ needs.start-check-runner.outputs.label }}
-          ec2-instance-id: ${{ needs.start-check-runner.outputs.ec2-instance-id }}
-
   set-instatus-incident-on-failure:
     name: Create Instatus Incident on Failure
     runs-on: ubuntu-latest