pypi_publish.yml

# This workflow builds the python package.
# When from from a release tags or a workflow dispatch, it also publishes to PyPI and DockerHub along
# with bumping the Connector Builder pinned version.
# Note: We may want to rename this file at some point. However, if we rename the workflow file name,
# we have to also update the Trusted Publisher settings on PyPI.

name: Packaging and Publishing

on:
  push:
  workflow_dispatch:
    inputs:
      version:
        description: "Version. The version to publish, ie 1.0.0 or 1.0.0-dev1. In most cases, you can leave this blank. If run from a release tag (recommended), the version number will be inferred from the git tag."
        required: false
      publish_to_pypi:
        description: "Publish to PyPI. If true, the workflow will publish to PyPI."
        type: boolean
        required: true
        default: true
      publish_to_dockerhub:
        description: "Publish to DockerHub. If true, the workflow will publish the SDM connector to DockerHub."
        type: boolean
        required: true
        default: true
      update_connector_builder:
        description: "Update Connector Builder. If true, the workflow will create a PR to bump the CDK version used by Connector Builder."
        type: boolean
        required: true
        default: true

jobs:
  build:
    name: Build Python Package
    runs-on: ubuntu-latest
    steps:
      - name: Detect Release Tag Version
        if: startsWith(github.ref, 'refs/tags/v')
        run: |
          DETECTED_VERSION=${{ github.ref_name }}
          echo "Version ref set to '${DETECTED_VERSION}'"
          # Remove the 'v' prefix if it exists
          DETECTED_VERSION="${DETECTED_VERSION#v}"
          echo "Setting version to '$DETECTED_VERSION'"
          echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV

      - name: Validate and set VERSION from git ref ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }})
        id: set_version
        run: |
          INPUT_VERSION=${{ github.event.inputs.version }}
          echo "Version input set to '${INPUT_VERSION}'"
          # Exit with success if both detected and input versions are empty
          if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then
            echo "No version detected or input. Will publish to SHA tag instead."
            echo 'VERSION=' >> $GITHUB_ENV
            exit 0
          fi
          # Remove the 'v' prefix if it exists
          INPUT_VERSION="${INPUT_VERSION#v}"
          # Fail if detected version is non-empty and different from the input version
          if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then
            echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'."
            exit 1
          fi
          # Set the version to the input version if non-empty, otherwise the detected version
          VERSION="${INPUT_VERSION:-$DETECTED_VERSION}"
          # Fail if the version is still empty
          if [ -z "$VERSION" ]; then
            echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
            exit 1
          fi
          echo "Setting version to '$VERSION'"
          echo "VERSION=${VERSION}" >> $GITHUB_ENV
          echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
          # Check if version is a prerelease version (will not tag 'latest')
          if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            echo "IS_PRERELEASE=false" >> $GITHUB_ENV
            echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT
          else
            echo "IS_PRERELEASE=true" >> $GITHUB_ENV
            echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT
          fi

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: hynek/build-and-inspect-python-package@v2

      - uses: actions/upload-artifact@v4
        with:
          name: Packages-${{ github.run_id }}
          path: |
            /tmp/baipp/dist/*.whl
            /tmp/baipp/dist/*.tar.gz

    outputs:
      VERSION: ${{ steps.set_version.outputs.VERSION }}
      IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }}

  publish:
    name: Publish CDK version to PyPI
    runs-on: ubuntu-latest
    needs: [build]
    permissions:
      id-token: write
      contents: write
    environment:
      name: PyPi
      url: https://pypi.org/p/airbyte
    if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.publish_to_pypi == 'true')
    env:
      VERSION: ${{ needs.build.outputs.VERSION }}
      IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: Packages-${{ github.run_id }}
          path: dist

      - name: Upload wheel to release
        if: startsWith(github.ref, 'refs/tags/v')
        uses: svenstaro/upload-release-action@v2
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: dist/*.whl
          tag: ${{ github.ref }}
          overwrite: true
          file_glob: true

      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@v1.10.3

  publish_sdm:
    name: Publish SDM to DockerHub
    # TODO: When we're ready to publish after each release, prefix the `if` below with:
    #         (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) ||
    # Until then, this workflow needs to be kicked off manually.
    # Last remaining blocker documented here: https://github.com/airbytehq/airbyte-python-cdk/issues/64
    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.publish_to_dockerhub == 'true')
    runs-on: ubuntu-latest
    needs: [build]
    environment:
      name: DockerHub
      url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags
    env:
      VERSION: ${{ needs.build.outputs.VERSION }}
      IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # We need to download the build artifact again because the previous job was on a different runner
      - name: Download Build Artifact
        uses: actions/download-artifact@v4
        with:
          name: Packages-${{ github.run_id }}
          path: dist

      - name: Set up QEMU for multi-platform builds
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_PASSWORD }}

      - name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
        if: env.VERSION != ''
        run: |
          tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
          if [ -z "$tag" ]; then
            echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
            exit 1
          fi
          echo "Checking if tag '$tag' exists on DockerHub..."
          if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
            echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
            exit 1
          fi
          echo "No existing tag '$tag' found. Proceeding with publish."

      - name: "Build and push (sha tag: '${{ github.sha }}')"
        # Only run if the version is not set
        if: env.VERSION == ''
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: |
            airbyte/source-declarative-manifest:${{ github.sha }}

      - name: "Build and push (version tag: ${{ env.VERSION || 'none'}})"
        # Only run if the version is set
        if: env.VERSION != ''
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: |
            airbyte/source-declarative-manifest:${{ env.VERSION }}

      - name: Build and push ('latest' tag)
        # Only run if version is set and IS_PRERELEASE is false
        if: env.VERSION != '' && env.IS_PRERELEASE == 'false'
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: |
            airbyte/source-declarative-manifest:latest

  update-connector-builder:
    # Create a PR against the Builder, to update the CDK version that it uses.
    # In the future, Builder may use the SDM docker image instead of the Python CDK package.
    name: Bump Connector Builder CDK version
    needs:
      - build
      - publish
    if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_connector_builder == 'true')
    env:
      VERSION: ${{ needs.build.outputs.VERSION }}
      IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Checkout Airbyte Platform Internal
        uses: actions/checkout@v4
        with:
          repository: airbytehq/airbyte-platform-internal
          token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
      - name: Update Builder's CDK version to ${{ env.VERSION }}
        run: |
          PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION)
          sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" oss/airbyte-connector-builder-server/Dockerfile
          sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" cloud/airbyte-connector-builder-server-wrapped/Dockerfile
          sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${VERSION}/g" oss/airbyte-connector-builder-server/requirements.in
          echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION
          cd oss/airbyte-connector-builder-server
          pip install pip-tools
          pip-compile --upgrade
      - name: Create Pull Request
        id: create-pull-request
        uses: peter-evans/create-pull-request@v6
        with:
          token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
          commit-message: "chore: update CDK version following release"
          title: "chore: update CDK version following release"
          body: This is an automatically generated PR triggered by a CDK release
          branch: automatic-cdk-release
          base: master
          delete-branch: true
      - name: Post success to Slack channel dev-connectors-extensibility
        uses: slackapi/slack-github-action@v1.23.0
        # TODO: Remove the 1 == 0 condition when we're ready to post to Slack
        if: 1 == 0
        continue-on-error: true
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
        with:
          channel-id: C04J1M66D8B
          # Channel: #dev-connectors-extensibility-releases
          # Link (internal): https://airbytehq-team.slack.com/archives/C04J1M66D8B
          payload: |
            {
                "text": "A new version of Python CDK has been released!",
                "blocks": [
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": "A new version of Python CDK has been released with <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md|changelog>: ${{ github.event.inputs.changelog-message }}\n\n"
                        }
                    },
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n"
                        }
                    },
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
                        }
                    }
                ]
            }
      - name: Post failure to Slack channel dev-connectors-extensibility
        # TODO: Remove the 1 == 0 condition when we're ready to post to Slack
        if: ${{ failure() }} && 1 == 0
        uses: slackapi/slack-github-action@v1.23.0
        continue-on-error: true
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
        with:
          channel-id: C04J1M66D8B
          # Channel: #dev-connectors-extensibility-releases
          # Link (internal): https://airbytehq-team.slack.com/archives/C04J1M66D8B
          payload: |
            {
                "text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated",
                "blocks": [
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": "A new version of Python CDK has been released with <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md|changelog>: ${{ github.event.inputs.changelog-message }}\n\n"
                        }
                    },
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": ":warning: Could not automatically create a PR for Connector Builder>\n"
                        }
                    },
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
                        }
                    }
                ]
            }