connectors_nightly_build.yml

name: Connectors nightly build

on:
  schedule:
    # 0AM UTC is 2AM CEST, 3AM EEST, 5PM PDT.
    - cron: "0 0 * * *"
  workflow_dispatch:
    inputs:
      runs-on:
        type: string
        default: dev-large-runner
        required: true
      test-connectors-options:
        default: --concurrency=5 --release-stage=generally_available --release-stage=beta
        required: true

run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for GA and Beta connectors' }} - on ${{ inputs.runs-on || 'dev-large-runner' }}"

jobs:
  test_connectors:
    name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for GA and Beta connectors' }} - on ${{ inputs.runs-on || 'dev-large-runner' }}"
    timeout-minutes: 1380 # 23 hours
    runs-on: ${{ inputs.runs-on || 'dev-large-runner' }}
    steps:
      - name: Get start timestamp
        id: get-start-timestamp
        run: echo "::set-output name=start-timestamp::$(date +%s)"
      - name: Login to DockerHub
        run: "docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}"
        env:
          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
      - name: Checkout Airbyte
        uses: actions/checkout@v3
        with:
          repository: ${{ github.event.inputs.repo }}
          ref: ${{ github.event.inputs.gitref }}
      - name: Extract branch name
        shell: bash
        if: github.event_name == 'workflow_dispatch'
        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
        id: extract_branch
      - name: Install Python 3.10
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
      - name: Install ci-connector-ops package
        run: pip install ./tools/ci_connector_ops\[pipelines]\
      - name: Test connectors
        run: |
          export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
          DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0"
          DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}"
          export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger"
          if [ ! -f  "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then
            mkdir -p "$DAGGER_TMP_BINDIR"
            curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
          fi
          airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors ${{ inputs.test-connectors-options || '--concurrency=8 --release-stage=generally_available --release-stage=beta' }} test
        env:
          _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
          CI_REPORT_BUCKET_NAME: "airbyte-ci-reports"
          CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
          CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }}
          CI_CONTEXT: "nightly_builds"
          CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}