diff --git a/.github/workflows/bump_connector_versions.yml b/.github/workflows/bump_connector_versions.yml new file mode 100644 index 000000000000..578b4bd1bd9b --- /dev/null +++ b/.github/workflows/bump_connector_versions.yml @@ -0,0 +1,93 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +name: Connector Ops CI - Bump Connector Versions + +on: + push: + branches: + - master + paths: + - "airbyte-integrations/connectors/**/.changelog_entries/*" + +jobs: + bump_connector_versions: + name: Publish connectors + runs-on: connector-publish-large + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Publish modified connectors [On merge to master] + id: publish-modified-connectors + if: github.event_name == 'push' + uses: ./.github/actions/run-airbyte-ci + with: + context: "master" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + github_token: ${{ secrets.GITHUB_TOKEN }} + metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} + spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} + s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + subcommand: "connectors bump_version" + python_registry_token: ${{ secrets.PYPI_TOKEN }} + + set-instatus-incident-on-failure: + name: Create Instatus Incident on Failure + runs-on: ubuntu-latest + needs: + - bump_connector_versions + if: ${{ failure() && github.ref == 'refs/heads/master' }} + steps: + - name: Call Instatus Webhook + uses: joelwmale/webhook-action@master + with: + url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }} + body: '{ "trigger": "down", "status": "HASISSUES" }' + + set-instatus-incident-on-success: + name: Create Instatus Incident on Success + runs-on: ubuntu-latest + needs: + - bump_connector_versions + if: ${{ success() && github.ref == 'refs/heads/master' }} + steps: + - name: Call Instatus Webhook + uses: joelwmale/webhook-action@master + with: + url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }} + body: '{ "trigger": "up" }' + + notify-failure-slack-channel: + name: "Notify Slack Channel on Build Failures" + runs-on: ubuntu-latest + needs: + - bump_connector_versions + if: ${{ failure() && github.ref == 'refs/heads/master' }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Match GitHub User to Slack User + id: match-github-to-slack-user + uses: ./.github/actions/match-github-to-slack-user + env: + AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Publish to OSS Build Failure Slack Channel + uses: abinoda/slack-action@master + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + args: >- + {\"channel\":\"C056HGD1QSW\", \"blocks\":[ + {\"type\":\"divider\"}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Connector Version Bump Failed! :bangbang: \n\n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: :octavia-shocked: \n\"}}, + {\"type\":\"divider\"}]} diff --git a/.github/workflows/create_connector_changelog_entry_files.yml b/.github/workflows/create_connector_changelog_entry_files.yml new file mode 100644 index 000000000000..fbf8684e1de5 --- /dev/null +++ b/.github/workflows/create_connector_changelog_entry_files.yml @@ -0,0 +1,24 @@ +name: Create connector changelog entry files +run-name: Create connector changelog entry files +on: + pull_request: + types:[opened, edited, synchronize] + +env: + GITREF: ${{ github.event.inputs.gitref || github.ref }} + + +jobs: + create-connector-changelog-entry-files: + # IMPORTANT: This name must match the require check name on the branch protection settings + name: "Delete existing files for the current PR" + runs-on: tooling-test-small + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ env.GITREF }} + + - name: Write PR body to file + + - name: Create Changelog Entry Files \ No newline at end of file diff --git a/.github/workflows/publish-java-cdk-command.yml b/.github/workflows/publish-java-cdk-command.yml index fbb4287e4fb7..447abf3839ed 100644 --- a/.github/workflows/publish-java-cdk-command.yml +++ b/.github/workflows/publish-java-cdk-command.yml @@ -34,6 +34,11 @@ on: required: true type: boolean default: false + remove-useLocalCdk: + description: "Once the CDK is published, set useLocalCdk to false for all connectors where it is true" + required: true + type: boolean + default: false gitref: description: "The git ref to check out from the specified repository." required: true @@ -49,10 +54,12 @@ env: # Use the provided GITREF or default to the branch triggering the workflow. GITREF: ${{ github.event.inputs.gitref || github.ref }} FORCE: "${{ github.event_name == 'push' || github.event.inputs.force == null && 'false' || github.event.inputs.force }}" + REMOVE_USELOCALCDK: "${{ github.event_name == 'push' || github.event.inputs.remove-useLocalCdk == null && 'false' || github.event.inputs.remove-useLocalCdk }}" DRY_RUN: "${{ github.event_name == 'push' && 'false' || github.event.inputs.dry-run == null && 'true' || github.event.inputs.dry-run }}" CDK_VERSION_FILE_PATH: "./airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties" S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + AIRBYTE_CI_CONTEXT: "${{ github.event_name == 'push' && 'master' || 'manual' }} jobs: publish-cdk: @@ -138,6 +145,19 @@ jobs: gradle-distribution-sha-256-sum-warning: false arguments: --scan :airbyte-cdk:java:airbyte-cdk:cdkPublish + - name: remove usages of useLocalCdk + if: ${{ env.DRY_RUN == 'false' && env.REMOVE_USELOCALCDK != 'false' }} + uses: ./.github/actions/run-airbyte-ci + with: + context: ${{ env.AIRBYTE_CI_CONTEXT }} + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + subcommand: "connectors --with-use-local-cdk upgrade_cdk" + - name: Add Success Comment if: github.event.inputs.comment-id && success() uses: peter-evans/create-or-update-comment@v1 diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index 6472cf037a6a..01615cb89a30 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -42,7 +42,7 @@ jobs: spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release" + subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only --without-use-local-cdk publish --main-release" python_registry_token: ${{ secrets.PYPI_TOKEN }} - name: Publish connectors [manual] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/cdk_java/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/cdk_java/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/cdk_java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/cdk_java/commands.py new file mode 100644 index 000000000000..c5b698498e33 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/cdk_java/commands.py @@ -0,0 +1,187 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import os +from pathlib import Path +from typing import List, Optional, Set, Tuple + +import asyncclick as click +from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo # type: ignore +from pipelines import main_logger +from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.lazy_group import LazyGroup +from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors +from pipelines.helpers.git import get_modified_files +from pipelines.helpers.utils import transform_strs_to_paths + +ALL_CONNECTORS = get_all_connectors_in_repo() + + +def log_selected_connectors(selected_connectors_with_modified_files: List[ConnectorWithModifiedFiles]) -> None: + if selected_connectors_with_modified_files: + selected_connectors_names = [c.technical_name for c in selected_connectors_with_modified_files] + main_logger.info(f"Will run on the following {len(selected_connectors_names)} connectors: {', '.join(selected_connectors_names)}.") + else: + main_logger.info("No connectors to run.") + +def get_selected_connectors_with_modified_files( + selected_names: Tuple[str], + selected_support_levels: Tuple[str], + selected_languages: Tuple[str], + modified: bool, + metadata_changes_only: bool, + with_changelog_entry_files: bool, + metadata_query: str, + modified_files: Set[Path], + enable_dependency_scanning: bool = False, +) -> List[ConnectorWithModifiedFiles]: + """Get the connectors that match the selected criteria. + + Args: + selected_names (Tuple[str]): Selected connector names. + selected_support_levels (Tuple[str]): Selected connector support levels. + selected_languages (Tuple[str]): Selected connector languages. + modified (bool): Whether to select the modified connectors. + metadata_changes_only (bool): Whether to select only the connectors with metadata changes. + with_changelog_entry_files (bool): Whether to select the connectors with files in .changelog_entries + modified_files (Set[Path]): The modified files. + enable_dependency_scanning (bool): Whether to enable the dependency scanning. + Returns: + List[ConnectorWithModifiedFiles]: The connectors that match the selected criteria. + """ + + if metadata_changes_only and not modified: + main_logger.info("--metadata-changes-only overrides --modified") + modified = True + + selected_modified_connectors = ( + get_modified_connectors(modified_files, ALL_CONNECTORS, enable_dependency_scanning) if modified else set() + ) + selected_connectors_by_name = {c for c in ALL_CONNECTORS if c.technical_name in selected_names} + selected_connectors_by_support_level = {connector for connector in ALL_CONNECTORS if connector.support_level in selected_support_levels} + selected_connectors_by_language = {connector for connector in ALL_CONNECTORS if connector.language in selected_languages} + selected_connectors_by_query = ( + {connector for connector in ALL_CONNECTORS if connector.metadata_query_match(metadata_query)} if metadata_query else set() + ) + selected_connectors_by_changelog_entry_files = {c for c in ALL_CONNECTORS if c.changelog_entry_files} if with_changelog_entry_files else set() + + non_empty_connector_sets = [ + connector_set + for connector_set in [ + selected_connectors_by_name, + selected_connectors_by_support_level, + selected_connectors_by_language, + selected_connectors_by_query, + selected_modified_connectors, + selected_connectors_by_changelog_entry_files + ] + if connector_set + ] + # The selected connectors are the intersection of the selected connectors by name, support_level, language, simpleeval query and modified. + selected_connectors = set.intersection(*non_empty_connector_sets) if non_empty_connector_sets else set() + + selected_connectors_with_modified_files = [] + for connector in selected_connectors: + connector_with_modified_files = ConnectorWithModifiedFiles( + relative_connector_path=connector.relative_connector_path, + modified_files=get_connector_modified_files(connector, modified_files), + ) + if not metadata_changes_only: + selected_connectors_with_modified_files.append(connector_with_modified_files) + else: + if connector_with_modified_files.has_metadata_change: + selected_connectors_with_modified_files.append(connector_with_modified_files) + return selected_connectors_with_modified_files + + +def validate_environment(is_local: bool) -> None: + """Check if the required environment variables exist.""" + if is_local: + if not Path(".git").is_dir(): + raise click.UsageError("You need to run this command from the repository root.") + else: + required_env_vars_for_ci = [ + "GCP_GSM_CREDENTIALS", + "CI_REPORT_BUCKET_NAME", + "CI_GITHUB_ACCESS_TOKEN", + "DOCKER_HUB_USERNAME", + "DOCKER_HUB_PASSWORD", + ] + for required_env_var in required_env_vars_for_ci: + if os.getenv(required_env_var) is None: + raise click.UsageError(f"When running in a CI context a {required_env_var} environment variable must be set.") + + +def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: + """Check if the connector secrets should be loaded from Airbyte GSM or from the local secrets directory. + + Args: + use_remote_secrets (Optional[bool]): Whether to use remote connector secrets or local connector secrets according to user inputs. + + Raises: + click.UsageError: If the --use-remote-secrets flag was provided but no GCP_GSM_CREDENTIALS environment variable was found. + + Returns: + bool: Whether to use remote connector secrets (True) or local connector secrets (False). + """ + gcp_gsm_credentials_is_set = bool(os.getenv("GCP_GSM_CREDENTIALS")) + if use_remote_secrets is None: + if gcp_gsm_credentials_is_set: + main_logger.info("GCP_GSM_CREDENTIALS environment variable found, using remote connector secrets.") + return True + else: + main_logger.info("No GCP_GSM_CREDENTIALS environment variable found, using local connector secrets.") + return False + if use_remote_secrets: + if gcp_gsm_credentials_is_set: + main_logger.info("GCP_GSM_CREDENTIALS environment variable found, using remote connector secrets.") + return True + else: + raise click.UsageError("The --use-remote-secrets flag was provided but no GCP_GSM_CREDENTIALS environment variable was found.") + else: + main_logger.info("Using local connector secrets as the --use-local-secrets flag was provided") + return False + + +@click.group( + cls=LazyGroup, + help="Commands related to connectors and connector acceptance tests.", + lazy_subcommands={ + "bump_version": "pipelines.airbyte_ci.cdk_java.bump_version.commands.bump_version", + }, +) +@click_merge_args_into_context_obj +@click_append_to_context_object("use_remote_secrets", lambda ctx: should_use_remote_secrets(ctx.obj["use_remote_secrets"])) +@click.pass_context +@click_ignore_unused_kwargs +async def connectors( + ctx: click.Context, +) -> None: + """Group all the connectors-ci command.""" + validate_environment(ctx.obj["is_local"]) + + modified_files = [] + if ctx.obj["modified"] or ctx.obj["metadata_changes_only"]: + modified_files = transform_strs_to_paths( + await get_modified_files( + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj["diffed_branch"], + ctx.obj["is_local"], + ctx.obj["ci_context"], + ) + ) + + ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( + ctx.obj["names"], + ctx.obj["support_levels"], + ctx.obj["languages"], + ctx.obj["modified"], + ctx.obj["metadata_changes_only"], + ctx.obj["with_changelog_entry_files"], + ctx.obj["metadata_query"], + set(modified_files), + ctx.obj["enable_dependency_scanning"], + ) + log_selected_connectors(ctx.obj["selected_connectors_with_modified_files"]) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 0cc95dcb056c..528bcb3fcbcd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -137,6 +137,7 @@ def is_current_process_wrapped_by_dagger_run() -> bool: "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", "test": "pipelines.airbyte_ci.test.commands.test", "update": "pipelines.airbyte_ci.update.commands.update", + "cdk-java": "pipelines.airbyte_ci.cdk_java.commands.cdk_java" }, ) @click.version_option(__installed_version__)