From 9aa69c831a9cc3b674ac972453da89fd24b238cc Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 26 Jul 2024 09:15:19 -0500 Subject: [PATCH 1/8] update dev variabe to test --- .github/workflows/deploy.yml | 20 -------------------- airflow_variables_dev.json | 2 +- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7df35645..ca8e567a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -20,26 +20,6 @@ jobs: - name: Display Environment Name run: echo "Hello ${{ inputs.envName }}" - pre-commit: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - id: file_changes - uses: trilom/file-changes-action@v1.2.4 - with: - output: " " - - - uses: pre-commit/action@v3.0.0 - env: - extra_args: --color=always --files ${{ steps.file_changes.outputs.files}} - tests: runs-on: ubuntu-latest diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 5522b27d..f152c850 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -154,7 +154,7 @@ }, "gcs_exported_data_bucket_name": "us-central1-test-hubble-43c3e190-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:741ee9b", + "image_name": "stellar/stellar-etl:ab57fa4", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "k8s_namespace": "hubble-composer", From ce226736b6e6609ce3f714bf997cb63053bbc1cf Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 26 Jul 2024 12:29:51 -0500 Subject: [PATCH 2/8] Add if else condition --- .github/workflows/deploy.yml | 54 ++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ca8e567a..555ded48 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -58,6 +58,7 @@ jobs: run: pytest dags/ deploy-to-dev: + if: ${{ github.event.inputs.envName == 'dev' }} runs-on: ubuntu-latest needs: [tests] steps: @@ -73,26 +74,37 @@ jobs: pip install --upgrade pip pip install google-cloud-storage==2.1.0 - - name: Authenticate to test-hubble GCP - uses: google-github-actions/auth@v1 - with: - credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" + # - name: Authenticate to test-hubble GCP + # uses: google-github-actions/auth@v1 + # with: + # credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" + + # - name: Upload files to dev GCS bucket + # run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET + # env: + # GOOGLE_CLOUD_PROJECT: test-hubble-319619 + # BUCKET: us-central1-test-hubble-43c3e190-bucket + + # - name: Update Airflow variables + # uses: actions-hub/gcloud@master + # env: + # PROJECT_ID: test-hubble-319619 + # APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" + # COMPOSER_ENVIRONMENT: test-hubble + # LOCATION: us-central1 + # with: + # args: > + # components install kubectl && gcloud composer environments run + # $COMPOSER_ENVIRONMENT --location $LOCATION variables import + # -- gcsfuse/actual_mount_path/variables.json + deploy-to-prod: + if: ${{ github.event.inputs.envName == 'prod' && github.ref == 'refs/heads/master' }} + runs-on: ubuntu-latest + needs: [tests] + steps: + - uses: actions/checkout@v3 - - name: Upload files to dev GCS bucket - run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET - env: - GOOGLE_CLOUD_PROJECT: test-hubble-319619 - BUCKET: us-central1-test-hubble-43c3e190-bucket - - - name: Update Airflow variables - uses: actions-hub/gcloud@master - env: - PROJECT_ID: test-hubble-319619 - APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" - COMPOSER_ENVIRONMENT: test-hubble - LOCATION: us-central1 + - name: Setup Python + uses: actions/setup-python@v4 with: - args: > - components install kubectl && gcloud composer environments run - $COMPOSER_ENVIRONMENT --location $LOCATION variables import - -- gcsfuse/actual_mount_path/variables.json + python-version: 3.8 \ No newline at end of file From 8b939224b8b5bfda8857ec419f7c6fdaaf54ff7e Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 26 Jul 2024 12:44:03 -0500 Subject: [PATCH 3/8] use different branch instead of master to test --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 555ded48..f381b790 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -98,7 +98,7 @@ jobs: # $COMPOSER_ENVIRONMENT --location $LOCATION variables import # -- gcsfuse/actual_mount_path/variables.json deploy-to-prod: - if: ${{ github.event.inputs.envName == 'prod' && github.ref == 'refs/heads/master' }} + if: ${{ github.event.inputs.envName == 'prod' && github.ref == 'refs/heads/test-release-master' }} runs-on: ubuntu-latest needs: [tests] steps: From 72a94df5e3a2298e85893de0c839840d0e49c666 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 26 Jul 2024 13:07:26 -0500 Subject: [PATCH 4/8] use input name in tests typo --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f381b790..d0aa3ee4 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -33,7 +33,7 @@ jobs: - name: Install dependencies run: | - cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json + cat airflow_variables_${{ inputs.envName }}.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json python -m pip install --upgrade pip pip install -r requirements-ci.txt From 0a3b05b4cd60f46a718dac88fe8732283e69b6e2 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 26 Jul 2024 13:13:54 -0500 Subject: [PATCH 5/8] Update actual deploy code --- .github/workflows/deploy.yml | 88 +++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 32 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d0aa3ee4..6df51052 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,13 +13,6 @@ on: - prod jobs: - log: - runs-on: ubuntu-latest - - steps: - - name: Display Environment Name - run: echo "Hello ${{ inputs.envName }}" - tests: runs-on: ubuntu-latest @@ -74,37 +67,68 @@ jobs: pip install --upgrade pip pip install google-cloud-storage==2.1.0 - # - name: Authenticate to test-hubble GCP - # uses: google-github-actions/auth@v1 - # with: - # credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" - - # - name: Upload files to dev GCS bucket - # run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET - # env: - # GOOGLE_CLOUD_PROJECT: test-hubble-319619 - # BUCKET: us-central1-test-hubble-43c3e190-bucket - - # - name: Update Airflow variables - # uses: actions-hub/gcloud@master - # env: - # PROJECT_ID: test-hubble-319619 - # APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" - # COMPOSER_ENVIRONMENT: test-hubble - # LOCATION: us-central1 - # with: - # args: > - # components install kubectl && gcloud composer environments run - # $COMPOSER_ENVIRONMENT --location $LOCATION variables import - # -- gcsfuse/actual_mount_path/variables.json + - name: Authenticate to test-hubble GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" + + - name: Upload files to dev GCS bucket + run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET + env: + GOOGLE_CLOUD_PROJECT: test-hubble-319619 + BUCKET: us-central1-test-hubble-43c3e190-bucket + + - name: Update Airflow variables + uses: actions-hub/gcloud@master + env: + PROJECT_ID: test-hubble-319619 + APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" + COMPOSER_ENVIRONMENT: test-hubble + LOCATION: us-central1 + with: + args: > + components install kubectl && gcloud composer environments run + $COMPOSER_ENVIRONMENT --location $LOCATION variables import + -- gcsfuse/actual_mount_path/variables.json + deploy-to-prod: - if: ${{ github.event.inputs.envName == 'prod' && github.ref == 'refs/heads/test-release-master' }} + if: ${{ github.event.inputs.envName == 'prod' && github.ref == 'refs/heads/master' }} runs-on: ubuntu-latest needs: [tests] + steps: - uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v4 with: - python-version: 3.8 \ No newline at end of file + python-version: 3.8 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install google-cloud-storage==2.1.0 + + - name: Authenticate to hubble GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.CREDS_PROD_HUBBLE }}" + + - name: Upload files to prod GCS bucket + run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET --env prod + env: + GOOGLE_CLOUD_PROJECT: hubble-261722 + BUCKET: us-central1-hubble-14c4ca64-bucket + + - name: Update Airflow variables + uses: actions-hub/gcloud@master + env: + PROJECT_ID: hubble-261722 + APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" + COMPOSER_ENVIRONMENT: hubble + LOCATION: us-central1 + with: + args: > + components install kubectl && gcloud composer environments run + $COMPOSER_ENVIRONMENT --location $LOCATION variables import + -- gcsfuse/actual_mount_path/variables.json From 0eff767d4bc17d8d89450fac9cfda263011018c1 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 26 Jul 2024 13:14:44 -0500 Subject: [PATCH 6/8] revert the image --- airflow_variables_dev.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index f152c850..5522b27d 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -154,7 +154,7 @@ }, "gcs_exported_data_bucket_name": "us-central1-test-hubble-43c3e190-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:ab57fa4", + "image_name": "stellar/stellar-etl:741ee9b", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "k8s_namespace": "hubble-composer", From 25f47d40643940177c22845144c2cea22e353f5e Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 26 Jul 2024 13:16:55 -0500 Subject: [PATCH 7/8] remove prod workflow --- .github/workflows/ci-cd-prod.yml | 91 -------------------------------- 1 file changed, 91 deletions(-) delete mode 100644 .github/workflows/ci-cd-prod.yml diff --git a/.github/workflows/ci-cd-prod.yml b/.github/workflows/ci-cd-prod.yml deleted file mode 100644 index 7cb4a97a..00000000 --- a/.github/workflows/ci-cd-prod.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: CI-CD-PROD - -on: - pull_request: - types: - - closed - branches: - - release - -jobs: - tests: - runs-on: ubuntu-latest - if: github.event.pull_request.merged == true - steps: - - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - cat airflow_variables_prod.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json - python -m pip install --upgrade pip - pip install -r requirements-ci.txt - - - name: Init Airflow SQLite database - run: airflow db init - - - name: Import Airflow variables - run: airflow variables import airflow_variables_ci.json - - - name: Authenticate to test-hubble GCP - uses: google-github-actions/auth@v1 - with: - credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" - - - id: "get-credentials" - uses: "google-github-actions/get-gke-credentials@v2" - with: - cluster_name: "us-central1-test-hubble-43c3e190-gke" - location: "us-central1" - - - name: Pytest - run: pytest dags/ - - release: - runs-on: ubuntu-latest - needs: [tests] - # deploy only occurs when pr is merged - if: >- - github.event.pull_request.merged == true && - github.repository == 'stellar/stellar-etl-airflow' - - steps: - - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install google-cloud-storage==2.1.0 - - - name: Authenticate to hubble GCP - uses: google-github-actions/auth@v1 - with: - credentials_json: "${{ secrets.CREDS_PROD_HUBBLE }}" - - - name: Upload files to prod GCS bucket - run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET --env prod - env: - GOOGLE_CLOUD_PROJECT: hubble-261722 - BUCKET: us-central1-hubble-14c4ca64-bucket - - - name: Update Airflow variables - uses: actions-hub/gcloud@master - env: - PROJECT_ID: hubble-261722 - APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" - COMPOSER_ENVIRONMENT: hubble - LOCATION: us-central1 - with: - args: > - components install kubectl && gcloud composer environments run - $COMPOSER_ENVIRONMENT --location $LOCATION variables import - -- gcsfuse/actual_mount_path/variables.json From 8889417419e5e63c2c5d8f25c435c0e0f170a8af Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 26 Jul 2024 13:39:12 -0500 Subject: [PATCH 8/8] Update .github/workflows/deploy.yml Co-authored-by: chowbao --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6df51052..aaf75150 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -124,7 +124,7 @@ jobs: uses: actions-hub/gcloud@master env: PROJECT_ID: hubble-261722 - APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" + APPLICATION_CREDENTIALS: "${{ secrets.CREDS_PROD_HUBBLE }}" COMPOSER_ENVIRONMENT: hubble LOCATION: us-central1 with: