From a5982d6455dad89dbaac56eda7c9eccca9b75216 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Thu, 25 Jul 2024 12:33:31 -0500 Subject: [PATCH 1/2] update DBT image (#460) --- airflow_variables_dev.json | 2 +- airflow_variables_prod.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 50b40e3b..d6e5f532 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -122,7 +122,7 @@ "partnership_assets__account_holders_activity_fact": false, "partnership_assets__asset_activity_fact": false }, - "dbt_image_name": "stellar/stellar-dbt:7356c7f", + "dbt_image_name": "stellar/stellar-dbt:fcf76b8", "dbt_internal_source_db": "test-hubble-319619", "dbt_internal_source_schema": "test_crypto_stellar_internal", "dbt_job_execution_timeout_seconds": 300, diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index be7a21b6..ad25ccd4 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -123,7 +123,7 @@ "partnership_assets__asset_activity_fact": false, "trade_agg": false }, - "dbt_image_name": "stellar/stellar-dbt:7356c7f", + "dbt_image_name": "stellar/stellar-dbt:fcf76b8", "dbt_internal_source_db": "hubble-261722", "dbt_internal_source_schema": "crypto_stellar_internal_2", "dbt_job_execution_timeout_seconds": 1800, From 81e6f0e9fedca2cba2432d52d26c60fffb20cce5 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 26 Jul 2024 08:57:17 -0500 Subject: [PATCH 2/2] Add new workflow (#462) * Add new workflow * lint * Fix the inputs * Update the config * Add steps for deploy * lint * Run the steps on branch push (For testing purpose) * turn off deploy to dev as it not really used currently * update * update cluster and bucket * address feedback * move all dev workflow to deploy workflow * bring back dev yml get past off annotation error fool tests a bit --- .github/workflows/ci-cd-dev.yml | 97 ------------------------- .github/workflows/ci-cd-prod.yml | 2 +- .github/workflows/deploy.yml | 118 +++++++++++++++++++++++++++++++ airflow_variables_dev.json | 2 +- 4 files changed, 120 insertions(+), 99 deletions(-) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/ci-cd-dev.yml b/.github/workflows/ci-cd-dev.yml index 06007560..0a8a4b4b 100644 --- a/.github/workflows/ci-cd-dev.yml +++ b/.github/workflows/ci-cd-dev.yml @@ -16,7 +16,6 @@ jobs: if: >- github.event.pull_request.merged == false && github.event.pull_request.state == 'open' - steps: - uses: actions/checkout@v3 @@ -45,99 +44,3 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.8 - - - name: Install dependencies - run: | - cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json - python -m pip install --upgrade pip - pip install -r requirements-ci.txt - - - name: Init Airflow SQLite database - run: airflow db init - - - name: Import Airflow variables - run: airflow variables import airflow_variables_ci.json - - - name: Authenticate to test-hubble GCP - uses: google-github-actions/auth@v1 - with: - credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" - - - id: "get-credentials" - uses: "google-github-actions/get-gke-credentials@v2" - with: - cluster_name: "us-central1-test-hubble-2-5f1f2dbf-gke" - location: "us-central1" - - - name: Pytest - run: pytest dags/ - - deploy-to-dev: - runs-on: ubuntu-latest - needs: [tests] - # deploy to dev occurs every time - # someone submits a pr targeting `master` - # from a branch at `stellar/stellar-etl-airflow` repo - if: github.repository == 'stellar/stellar-etl-airflow' - # known caveats: - # if there's more than 1 person working - # in the same file this won't behave nicely - - steps: - - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - pip install --upgrade pip - pip install google-cloud-storage==2.1.0 - - - name: Authenticate to test-hubble GCP - uses: google-github-actions/auth@v1 - with: - credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" - - - name: Upload files to dev GCS bucket - run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET - env: - GOOGLE_CLOUD_PROJECT: test-hubble-319619 - BUCKET: us-central1-test-hubble-2-5f1f2dbf-bucket - - - name: Update Airflow variables - uses: actions-hub/gcloud@master - env: - PROJECT_ID: test-hubble-319619 - APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" - COMPOSER_ENVIRONMENT: test-hubble-2 - LOCATION: us-central1 - with: - args: > - components install kubectl && gcloud composer environments run - $COMPOSER_ENVIRONMENT --location $LOCATION variables import - -- gcsfuse/actual_mount_path/variables.json - - promote-to-prod: - runs-on: ubuntu-latest - # deploy only occurs when pr is merged - if: github.event.pull_request.merged == true - permissions: - pull-requests: write - - steps: - - uses: actions/checkout@v3 - - - name: Create pull request - run: > - gh pr create - --base release - --head master - --title "[PRODUCTION] Update production Airflow environment" - --body "This PR was auto-generated by GitHub Actions. - - After merged and closed, this PR will trigger an action that updates DAGs, libs and schemas files from prod Airflow." - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-cd-prod.yml b/.github/workflows/ci-cd-prod.yml index 22e37b58..7cb4a97a 100644 --- a/.github/workflows/ci-cd-prod.yml +++ b/.github/workflows/ci-cd-prod.yml @@ -39,7 +39,7 @@ jobs: - id: "get-credentials" uses: "google-github-actions/get-gke-credentials@v2" with: - cluster_name: "us-central1-test-hubble-2-5f1f2dbf-gke" + cluster_name: "us-central1-test-hubble-43c3e190-gke" location: "us-central1" - name: Pytest diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..7df35645 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,118 @@ +name: Deploy + +on: + workflow_dispatch: + inputs: + envName: + description: "Deploy Environment" + default: "dev" + required: true + type: choice + options: + - dev + - prod + +jobs: + log: + runs-on: ubuntu-latest + + steps: + - name: Display Environment Name + run: echo "Hello ${{ inputs.envName }}" + + pre-commit: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - id: file_changes + uses: trilom/file-changes-action@v1.2.4 + with: + output: " " + + - uses: pre-commit/action@v3.0.0 + env: + extra_args: --color=always --files ${{ steps.file_changes.outputs.files}} + + tests: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json + python -m pip install --upgrade pip + pip install -r requirements-ci.txt + + - name: Init Airflow SQLite database + run: airflow db init + + - name: Import Airflow variables + run: airflow variables import airflow_variables_ci.json + + - name: Authenticate to test-hubble GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" + + - id: "get-credentials" + uses: "google-github-actions/get-gke-credentials@v2" + with: + cluster_name: "us-central1-test-hubble-43c3e190-gke" + location: "us-central1" + + - name: Pytest + run: pytest dags/ + + deploy-to-dev: + runs-on: ubuntu-latest + needs: [tests] + steps: + - uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install google-cloud-storage==2.1.0 + + - name: Authenticate to test-hubble GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" + + - name: Upload files to dev GCS bucket + run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET + env: + GOOGLE_CLOUD_PROJECT: test-hubble-319619 + BUCKET: us-central1-test-hubble-43c3e190-bucket + + - name: Update Airflow variables + uses: actions-hub/gcloud@master + env: + PROJECT_ID: test-hubble-319619 + APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" + COMPOSER_ENVIRONMENT: test-hubble + LOCATION: us-central1 + with: + args: > + components install kubectl && gcloud composer environments run + $COMPOSER_ENVIRONMENT --location $LOCATION variables import + -- gcsfuse/actual_mount_path/variables.json diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index d6e5f532..5522b27d 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -152,7 +152,7 @@ "due to concurrent update" ] }, - "gcs_exported_data_bucket_name": "us-central1-test-hubble-2-5f1f2dbf-bucket", + "gcs_exported_data_bucket_name": "us-central1-test-hubble-43c3e190-bucket", "gcs_exported_object_prefix": "dag-exported", "image_name": "stellar/stellar-etl:741ee9b", "image_output_path": "/etl/exported_data/",