From e7d81099891c163db64935281d601464e2926648 Mon Sep 17 00:00:00 2001 From: Cayod Date: Fri, 6 Oct 2023 07:13:49 -0300 Subject: [PATCH 01/23] fixed auto update from airflow variables --- .github/workflows/ci.yml | 8 ++++++++ .github/workflows/release.yml | 9 +++++++++ dags/stellar_etl_airflow/add_files_to_composer.py | 3 +++ 3 files changed, 20 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b787216..02cc0115 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,14 @@ jobs: env: GOOGLE_CLOUD_PROJECT: test-hubble-319619 BUCKET: us-central1-hubble-1pt5-dev-7db0e004-bucket + - name: Update Airflow variables + run: > + gcloud composer environments run $COMPOSER_ENVIRONMENT + --location $LOCATION + variables import -- gcsfuse/variables.json + env: + COMPOSER_ENVIRONMENT: hubble-1pt5-dev + LOCATION: us-central1 promote-to-prod: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9df672d0..c58ef750 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,3 +65,12 @@ jobs: env: GOOGLE_CLOUD_PROJECT: hubble-261722 BUCKET: us-central1-hubble-2-d948d67b-bucket + + - name: Update Airflow variables + run: > + gcloud composer environments run $COMPOSER_ENVIRONMENT + --location $LOCATION + variables import -- gcsfuse/variables.json + env: + COMPOSER_ENVIRONMENT: hubble-2-d948d67b + LOCATION: us-central1 diff --git a/dags/stellar_etl_airflow/add_files_to_composer.py b/dags/stellar_etl_airflow/add_files_to_composer.py index 3d58ce5a..873d6f5f 100644 --- a/dags/stellar_etl_airflow/add_files_to_composer.py +++ b/dags/stellar_etl_airflow/add_files_to_composer.py @@ -26,6 +26,9 @@ def _create_files_list(env: str) -> Tuple[str, List[str]]: # copy airflow configuration file copy(f"airflow-{env}.cfg", f"{temp_dir}/airflow.cfg") copy(f"airflow-{env}.cfg", f"airflow.cfg") + # copy airflow variables json + copy(f"variables-{env}.json", f"{temp_dir}/variables.json") + copy(f"variables-{env}.json", f"variables.json") dags = glob(f"{temp_dir}/**/*.*", recursive=True) return (temp_dir, dags) From 25a5b466232415734842b329e7004f61a20bad55 Mon Sep 17 00:00:00 2001 From: Cayod Date: Fri, 6 Oct 2023 09:56:51 -0300 Subject: [PATCH 02/23] update airflow variable name pattern --- .github/workflows/release.yml | 2 +- airflow_variables.json => airflow_variables_prod.json | 0 dags/stellar_etl_airflow/add_files_to_composer.py | 5 ++--- 3 files changed, 3 insertions(+), 4 deletions(-) rename airflow_variables.json => airflow_variables_prod.json (100%) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c58ef750..cfd0abca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: - name: Install dependencies run: | - cat airflow_variables.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json + cat airflow_variables_prod.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json python -m pip install --upgrade pip pip install -r requirements-ci.txt diff --git a/airflow_variables.json b/airflow_variables_prod.json similarity index 100% rename from airflow_variables.json rename to airflow_variables_prod.json diff --git a/dags/stellar_etl_airflow/add_files_to_composer.py b/dags/stellar_etl_airflow/add_files_to_composer.py index 873d6f5f..607ba5c6 100644 --- a/dags/stellar_etl_airflow/add_files_to_composer.py +++ b/dags/stellar_etl_airflow/add_files_to_composer.py @@ -27,9 +27,8 @@ def _create_files_list(env: str) -> Tuple[str, List[str]]: copy(f"airflow-{env}.cfg", f"{temp_dir}/airflow.cfg") copy(f"airflow-{env}.cfg", f"airflow.cfg") # copy airflow variables json - copy(f"variables-{env}.json", f"{temp_dir}/variables.json") - copy(f"variables-{env}.json", f"variables.json") - + copy(f"airflow_variables_{env}.json", f"{temp_dir}/variables.json") + copy(f"airflow_variables_{env}.json", f"variables.json") dags = glob(f"{temp_dir}/**/*.*", recursive=True) return (temp_dir, dags) From 6f3c7c2b1153b60b5904a120e96040bb76c11d4e Mon Sep 17 00:00:00 2001 From: Cayod Date: Fri, 6 Oct 2023 10:47:30 -0300 Subject: [PATCH 03/23] insert if statement for upload variable.json --- dags/stellar_etl_airflow/add_files_to_composer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dags/stellar_etl_airflow/add_files_to_composer.py b/dags/stellar_etl_airflow/add_files_to_composer.py index 607ba5c6..2442c173 100644 --- a/dags/stellar_etl_airflow/add_files_to_composer.py +++ b/dags/stellar_etl_airflow/add_files_to_composer.py @@ -55,7 +55,11 @@ def upload_dags_to_composer(bucket_name: str, env: str) -> None: for f in files: # remove path to temp dir - if f.endswith(".json"): + if f.endswith("variables.json"): + # insert airflow variable file + f = f.replace(f"{temp_dir}/", "") + blob = bucket.blob(f) + elif f.endswith(".json"): # create schemas directory f = f.replace(f"{temp_dir}/", "schemas/") schema_destination = "dags/" + f From 0becb25e75dcc5501bd599e288d443d6dc4cdd61 Mon Sep 17 00:00:00 2001 From: Cayod Date: Fri, 6 Oct 2023 17:09:22 -0300 Subject: [PATCH 04/23] rollbacked changes in ci --- .github/workflows/ci.yml | 8 -------- .github/workflows/release.yml | 9 --------- 2 files changed, 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02cc0115..0b787216 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,14 +93,6 @@ jobs: env: GOOGLE_CLOUD_PROJECT: test-hubble-319619 BUCKET: us-central1-hubble-1pt5-dev-7db0e004-bucket - - name: Update Airflow variables - run: > - gcloud composer environments run $COMPOSER_ENVIRONMENT - --location $LOCATION - variables import -- gcsfuse/variables.json - env: - COMPOSER_ENVIRONMENT: hubble-1pt5-dev - LOCATION: us-central1 promote-to-prod: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cfd0abca..e1f08be0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,12 +65,3 @@ jobs: env: GOOGLE_CLOUD_PROJECT: hubble-261722 BUCKET: us-central1-hubble-2-d948d67b-bucket - - - name: Update Airflow variables - run: > - gcloud composer environments run $COMPOSER_ENVIRONMENT - --location $LOCATION - variables import -- gcsfuse/variables.json - env: - COMPOSER_ENVIRONMENT: hubble-2-d948d67b - LOCATION: us-central1 From dfb1a266d2fcc9dc6d6dcaf3e3edcc87c1e1c942 Mon Sep 17 00:00:00 2001 From: Cayod Date: Mon, 9 Oct 2023 11:49:43 -0300 Subject: [PATCH 05/23] update ci and release --- .github/workflows/ci.yml | 9 +++++++++ .github/workflows/release.yml | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b787216..18b73d92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,6 +94,15 @@ jobs: GOOGLE_CLOUD_PROJECT: test-hubble-319619 BUCKET: us-central1-hubble-1pt5-dev-7db0e004-bucket + - name: Update Airflow variables + run: > + gcloud composer environments run $COMPOSER_ENVIRONMENT + --location $LOCATION + variables import -- gcsfuse/variables.json + env: + COMPOSER_ENVIRONMENT: hubble-1pt5-dev + LOCATION: us-central1 + promote-to-prod: runs-on: ubuntu-latest # deploy only occurs when pr is merged diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e1f08be0..6774588d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,3 +65,12 @@ jobs: env: GOOGLE_CLOUD_PROJECT: hubble-261722 BUCKET: us-central1-hubble-2-d948d67b-bucket + + - name: Update Airflow variables + run: > + gcloud composer environments run $COMPOSER_ENVIRONMENT + --location $LOCATION + variables import -- gcsfuse/variables.json + env: + COMPOSER_ENVIRONMENT: hubble-2-d948d67b + LOCATION: us-central1 \ No newline at end of file From 53414862ddcd6c96cad4c07eb2fb38cf4e0c6718 Mon Sep 17 00:00:00 2001 From: Cayod Date: Mon, 9 Oct 2023 11:53:40 -0300 Subject: [PATCH 06/23] update ci and release --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18b73d92..02cc0115 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,6 @@ jobs: env: GOOGLE_CLOUD_PROJECT: test-hubble-319619 BUCKET: us-central1-hubble-1pt5-dev-7db0e004-bucket - - name: Update Airflow variables run: > gcloud composer environments run $COMPOSER_ENVIRONMENT From e7c4a505be11c52895c4d8b6d2154242de56a630 Mon Sep 17 00:00:00 2001 From: Cayod Date: Mon, 9 Oct 2023 11:55:52 -0300 Subject: [PATCH 07/23] run pre-commit --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6774588d..cfd0abca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,4 +73,4 @@ jobs: variables import -- gcsfuse/variables.json env: COMPOSER_ENVIRONMENT: hubble-2-d948d67b - LOCATION: us-central1 \ No newline at end of file + LOCATION: us-central1 From e375d4c82ad60507340ed05925131d47522cd782 Mon Sep 17 00:00:00 2001 From: Cayod Date: Thu, 19 Oct 2023 13:08:45 -0300 Subject: [PATCH 08/23] update ci --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02cc0115..1d9a8715 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,8 @@ jobs: run: | pip install --upgrade pip pip install google-cloud-storage==2.1.0 + apt-get install kubectl + apt-get install google-cloud-sdk - name: Authenticate to test-hubble GCP uses: google-github-actions/auth@v1 From 4ea5d13a0aca1439baca931024f4e066ba828d09 Mon Sep 17 00:00:00 2001 From: Cayod Date: Thu, 19 Oct 2023 13:44:50 -0300 Subject: [PATCH 09/23] update ci --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d9a8715..3b58fd56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,14 +82,16 @@ jobs: run: | pip install --upgrade pip pip install google-cloud-storage==2.1.0 - apt-get install kubectl - apt-get install google-cloud-sdk - name: Authenticate to test-hubble GCP uses: google-github-actions/auth@v1 with: credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" + - name: Setup gcloud + run: > + gcloud components install gke-gcloud-auth-plugin + - name: Upload files to dev GCS bucket run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET env: From 777d28e53c224fae92d342ad3e3471c39f3c2df3 Mon Sep 17 00:00:00 2001 From: Cayod Date: Thu, 19 Oct 2023 14:07:50 -0300 Subject: [PATCH 10/23] update ci --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b58fd56..12c67870 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,9 +88,8 @@ jobs: with: credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" - - name: Setup gcloud - run: > - gcloud components install gke-gcloud-auth-plugin + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' - name: Upload files to dev GCS bucket run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET From 8872b1857c2f2312560114ad45ff03c9c0807909 Mon Sep 17 00:00:00 2001 From: Cayod Date: Thu, 19 Oct 2023 14:10:01 -0300 Subject: [PATCH 11/23] update ci --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12c67870..79ca2b6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,14 +88,15 @@ jobs: with: credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v1' + - name: "Set up Cloud SDK" + uses: "google-github-actions/setup-gcloud@v1" - name: Upload files to dev GCS bucket run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET env: GOOGLE_CLOUD_PROJECT: test-hubble-319619 BUCKET: us-central1-hubble-1pt5-dev-7db0e004-bucket + - name: Update Airflow variables run: > gcloud composer environments run $COMPOSER_ENVIRONMENT From bc28a6c0cbc5fafef4d10c3f061e1dc61344c8fc Mon Sep 17 00:00:00 2001 From: Cayod Date: Thu, 19 Oct 2023 15:12:38 -0300 Subject: [PATCH 12/23] update ci --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79ca2b6b..5c797ca7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,12 @@ jobs: GOOGLE_CLOUD_PROJECT: test-hubble-319619 BUCKET: us-central1-hubble-1pt5-dev-7db0e004-bucket + - name: get GKE credentials + uses: "google-github-actions/get-gke-credentials@v1" + with: + cluster_name: "us-central1-hubble-1pt5-dev-7db0e004-gke" + location: $LOCATION + - name: Update Airflow variables run: > gcloud composer environments run $COMPOSER_ENVIRONMENT From 6399bc5a5d1f688ff60fd68a28e84e18c68079be Mon Sep 17 00:00:00 2001 From: Cayod Date: Thu, 19 Oct 2023 15:22:00 -0300 Subject: [PATCH 13/23] update ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c797ca7..7c8fce2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,7 +101,7 @@ jobs: uses: "google-github-actions/get-gke-credentials@v1" with: cluster_name: "us-central1-hubble-1pt5-dev-7db0e004-gke" - location: $LOCATION + location: "us-central1-c" - name: Update Airflow variables run: > From f12ac13eac5b69d00e764d437f4326d46a33a5de Mon Sep 17 00:00:00 2001 From: Cayod Date: Fri, 20 Oct 2023 14:18:16 -0300 Subject: [PATCH 14/23] update ci --- .github/workflows/ci.yml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c8fce2e..4f896986 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,29 +88,21 @@ jobs: with: credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" - - name: "Set up Cloud SDK" - uses: "google-github-actions/setup-gcloud@v1" - - name: Upload files to dev GCS bucket run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET env: GOOGLE_CLOUD_PROJECT: test-hubble-319619 BUCKET: us-central1-hubble-1pt5-dev-7db0e004-bucket - - name: get GKE credentials - uses: "google-github-actions/get-gke-credentials@v1" - with: - cluster_name: "us-central1-hubble-1pt5-dev-7db0e004-gke" - location: "us-central1-c" - - name: Update Airflow variables - run: > - gcloud composer environments run $COMPOSER_ENVIRONMENT - --location $LOCATION - variables import -- gcsfuse/variables.json + uses: actions-hub/gcloud@master env: + PROJECT_ID: test-hubble-319619 + APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" COMPOSER_ENVIRONMENT: hubble-1pt5-dev LOCATION: us-central1 + with: + args: composer environments run $COMPOSER_ENVIRONMENT --location $LOCATION variables import -- gcsfuse/variables.json promote-to-prod: runs-on: ubuntu-latest From 1cb59e2bb745b6ac50bc5589ef892264c99d4fee Mon Sep 17 00:00:00 2001 From: Cayod Date: Fri, 20 Oct 2023 14:27:57 -0300 Subject: [PATCH 15/23] update ci --- .github/workflows/ci.yml | 84 ++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f896986..667dbcf3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,57 +11,57 @@ on: - master jobs: - pre-commit: - runs-on: ubuntu-latest - if: >- - github.event.pull_request.merged == false && - github.event.pull_request.state == 'open' + # pre-commit: + # runs-on: ubuntu-latest + # if: >- + # github.event.pull_request.merged == false && + # github.event.pull_request.state == 'open' - steps: - - uses: actions/checkout@v3 + # steps: + # - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 + # - name: Setup Python + # uses: actions/setup-python@v4 + # with: + # python-version: 3.8 - - uses: pre-commit/action@v3.0.0 + # - uses: pre-commit/action@v3.0.0 - tests: - runs-on: ubuntu-latest - needs: [pre-commit] + # tests: + # runs-on: ubuntu-latest + # needs: [pre-commit] - steps: - - uses: actions/checkout@v3 + # steps: + # - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 + # - name: Setup Python + # uses: actions/setup-python@v4 + # with: + # python-version: 3.8 - - name: Install dependencies - run: | - cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json - python -m pip install --upgrade pip - pip install -r requirements-ci.txt + # - name: Install dependencies + # run: | + # cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json + # python -m pip install --upgrade pip + # pip install -r requirements-ci.txt - - name: Init Airflow SQLite database - run: airflow db init + # - name: Init Airflow SQLite database + # run: airflow db init - - name: Import Airflow variables - run: airflow variables import airflow_variables_ci.json + # - name: Import Airflow variables + # run: airflow variables import airflow_variables_ci.json - - name: Authenticate to test-hubble GCP - uses: google-github-actions/auth@v1 - with: - credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" + # - name: Authenticate to test-hubble GCP + # uses: google-github-actions/auth@v1 + # with: + # credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" - - name: Pytest - run: pytest dags/ + # - name: Pytest + # run: pytest dags/ deploy-to-dev: runs-on: ubuntu-latest - needs: [tests] + # needs: [tests] # deploy to dev occurs every time # someone submits a pr targeting `master` # from a branch at `stellar/stellar-etl-airflow` repo @@ -94,6 +94,16 @@ jobs: GOOGLE_CLOUD_PROJECT: test-hubble-319619 BUCKET: us-central1-hubble-1pt5-dev-7db0e004-bucket + - name: Install Kubectl + uses: actions-hub/gcloud@master + env: + PROJECT_ID: test-hubble-319619 + APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" + COMPOSER_ENVIRONMENT: hubble-1pt5-dev + LOCATION: us-central1 + with: + args: components install kubectl + - name: Update Airflow variables uses: actions-hub/gcloud@master env: From 8ff8bfafe999c3e192c0f1521bf4885e92bc9aa4 Mon Sep 17 00:00:00 2001 From: Cayod Date: Fri, 20 Oct 2023 14:30:45 -0300 Subject: [PATCH 16/23] update ci --- .github/workflows/ci.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 667dbcf3..312399fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,16 +94,6 @@ jobs: GOOGLE_CLOUD_PROJECT: test-hubble-319619 BUCKET: us-central1-hubble-1pt5-dev-7db0e004-bucket - - name: Install Kubectl - uses: actions-hub/gcloud@master - env: - PROJECT_ID: test-hubble-319619 - APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" - COMPOSER_ENVIRONMENT: hubble-1pt5-dev - LOCATION: us-central1 - with: - args: components install kubectl - - name: Update Airflow variables uses: actions-hub/gcloud@master env: @@ -112,7 +102,7 @@ jobs: COMPOSER_ENVIRONMENT: hubble-1pt5-dev LOCATION: us-central1 with: - args: composer environments run $COMPOSER_ENVIRONMENT --location $LOCATION variables import -- gcsfuse/variables.json + args: components install kubectl && gcloud composer environments run $COMPOSER_ENVIRONMENT --location $LOCATION variables import -- gcsfuse/variables.json promote-to-prod: runs-on: ubuntu-latest From 71d47cba9be9ee925389bc964878479c79b8c97f Mon Sep 17 00:00:00 2001 From: Cayod Date: Mon, 23 Oct 2023 15:34:37 -0300 Subject: [PATCH 17/23] update ci and release --- .github/workflows/ci.yml | 79 ++++++++++++++++++----------------- .github/workflows/release.yml | 14 ++++--- 2 files changed, 50 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 312399fe..28644894 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,57 +11,57 @@ on: - master jobs: - # pre-commit: - # runs-on: ubuntu-latest - # if: >- - # github.event.pull_request.merged == false && - # github.event.pull_request.state == 'open' + pre-commit: + runs-on: ubuntu-latest + if: >- + github.event.pull_request.merged == false && + github.event.pull_request.state == 'open' - # steps: - # - uses: actions/checkout@v3 + steps: + - uses: actions/checkout@v3 - # - name: Setup Python - # uses: actions/setup-python@v4 - # with: - # python-version: 3.8 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 - # - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.0 - # tests: - # runs-on: ubuntu-latest - # needs: [pre-commit] + tests: + runs-on: ubuntu-latest + needs: [pre-commit] - # steps: - # - uses: actions/checkout@v3 + steps: + - uses: actions/checkout@v3 - # - name: Setup Python - # uses: actions/setup-python@v4 - # with: - # python-version: 3.8 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 - # - name: Install dependencies - # run: | - # cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json - # python -m pip install --upgrade pip - # pip install -r requirements-ci.txt + - name: Install dependencies + run: | + cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json + python -m pip install --upgrade pip + pip install -r requirements-ci.txt - # - name: Init Airflow SQLite database - # run: airflow db init + - name: Init Airflow SQLite database + run: airflow db init - # - name: Import Airflow variables - # run: airflow variables import airflow_variables_ci.json + - name: Import Airflow variables + run: airflow variables import airflow_variables_ci.json - # - name: Authenticate to test-hubble GCP - # uses: google-github-actions/auth@v1 - # with: - # credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" + - name: Authenticate to test-hubble GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" - # - name: Pytest - # run: pytest dags/ + - name: Pytest + run: pytest dags/ deploy-to-dev: runs-on: ubuntu-latest - # needs: [tests] + needs: [tests] # deploy to dev occurs every time # someone submits a pr targeting `master` # from a branch at `stellar/stellar-etl-airflow` repo @@ -102,7 +102,10 @@ jobs: COMPOSER_ENVIRONMENT: hubble-1pt5-dev LOCATION: us-central1 with: - args: components install kubectl && gcloud composer environments run $COMPOSER_ENVIRONMENT --location $LOCATION variables import -- gcsfuse/variables.json + args: > + components install kubectl && gcloud composer environments \ + run $COMPOSER_ENVIRONMENT --location $LOCATION \ + variables import -- gcsfuse/variables.json promote-to-prod: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cfd0abca..0b13dc9e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,10 +67,14 @@ jobs: BUCKET: us-central1-hubble-2-d948d67b-bucket - name: Update Airflow variables - run: > - gcloud composer environments run $COMPOSER_ENVIRONMENT - --location $LOCATION - variables import -- gcsfuse/variables.json + uses: actions-hub/gcloud@master env: - COMPOSER_ENVIRONMENT: hubble-2-d948d67b + PROJECT_ID: hubble-2-d948d67b + APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" + COMPOSER_ENVIRONMENT: hubble-2 LOCATION: us-central1 + with: + args: > + components install kubectl && gcloud composer environments \ + run $COMPOSER_ENVIRONMENT --location $LOCATION \ + variables import -- gcsfuse/variables.json From c6fd1840881520ca361488afec7ca1738243315b Mon Sep 17 00:00:00 2001 From: Cayod Date: Mon, 23 Oct 2023 15:43:03 -0300 Subject: [PATCH 18/23] update ci and release --- .github/workflows/ci.yml | 78 +++++++++++++++++------------------ .github/workflows/release.yml | 2 +- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28644894..b1516743 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,57 +11,57 @@ on: - master jobs: - pre-commit: - runs-on: ubuntu-latest - if: >- - github.event.pull_request.merged == false && - github.event.pull_request.state == 'open' + # pre-commit: + # runs-on: ubuntu-latest + # if: >- + # github.event.pull_request.merged == false && + # github.event.pull_request.state == 'open' - steps: - - uses: actions/checkout@v3 + # steps: + # - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 + # - name: Setup Python + # uses: actions/setup-python@v4 + # with: + # python-version: 3.8 - - uses: pre-commit/action@v3.0.0 + # - uses: pre-commit/action@v3.0.0 - tests: - runs-on: ubuntu-latest - needs: [pre-commit] + # # tests: + # runs-on: ubuntu-latest + # needs: [pre-commit] - steps: - - uses: actions/checkout@v3 + # steps: + # - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 + # - name: Setup Python + # uses: actions/setup-python@v4 + # with: + # python-version: 3.8 - - name: Install dependencies - run: | - cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json - python -m pip install --upgrade pip - pip install -r requirements-ci.txt + # - name: Install dependencies + # run: | + # cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json + # python -m pip install --upgrade pip + # pip install -r requirements-ci.txt - - name: Init Airflow SQLite database - run: airflow db init + # - name: Init Airflow SQLite database + # run: airflow db init - - name: Import Airflow variables - run: airflow variables import airflow_variables_ci.json + # - name: Import Airflow variables + # run: airflow variables import airflow_variables_ci.json - - name: Authenticate to test-hubble GCP - uses: google-github-actions/auth@v1 - with: - credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" + # - name: Authenticate to test-hubble GCP + # uses: google-github-actions/auth@v1 + # with: + # credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" - - name: Pytest - run: pytest dags/ + # - name: Pytest + # run: pytest dags/ deploy-to-dev: runs-on: ubuntu-latest - needs: [tests] + # needs: [tests] # deploy to dev occurs every time # someone submits a pr targeting `master` # from a branch at `stellar/stellar-etl-airflow` repo @@ -103,8 +103,8 @@ jobs: LOCATION: us-central1 with: args: > - components install kubectl && gcloud composer environments \ - run $COMPOSER_ENVIRONMENT --location $LOCATION \ + components install kubectl && gcloud composer environments run \ + $COMPOSER_ENVIRONMENT --location $LOCATION \ variables import -- gcsfuse/variables.json promote-to-prod: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0b13dc9e..e9bd69b1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -75,6 +75,6 @@ jobs: LOCATION: us-central1 with: args: > - components install kubectl && gcloud composer environments \ + components install kubectl && gcloud composer environments\ run $COMPOSER_ENVIRONMENT --location $LOCATION \ variables import -- gcsfuse/variables.json From 2577870407558201b30d269c008ce407814b8883 Mon Sep 17 00:00:00 2001 From: Cayod Date: Mon, 23 Oct 2023 15:46:05 -0300 Subject: [PATCH 19/23] update ci and release --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1516743..ca484d6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,8 +104,8 @@ jobs: with: args: > components install kubectl && gcloud composer environments run \ - $COMPOSER_ENVIRONMENT --location $LOCATION \ - variables import -- gcsfuse/variables.json + $COMPOSER_ENVIRONMENT --location $LOCATION variables import \ + -- gcsfuse/variables.json promote-to-prod: runs-on: ubuntu-latest From 70f79ee1189ddbfd39b286ccdb084643f673fdca Mon Sep 17 00:00:00 2001 From: Cayod Date: Mon, 23 Oct 2023 15:50:32 -0300 Subject: [PATCH 20/23] update ci and release --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca484d6d..9ba945d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,8 +103,8 @@ jobs: LOCATION: us-central1 with: args: > - components install kubectl && gcloud composer environments run \ - $COMPOSER_ENVIRONMENT --location $LOCATION variables import \ + components install kubectl && gcloud composer environments run + $COMPOSER_ENVIRONMENT --location $LOCATION variables import -- gcsfuse/variables.json promote-to-prod: From 75052912f2798c2eb6d838cd4cca857e9daf3957 Mon Sep 17 00:00:00 2001 From: Cayod Date: Mon, 23 Oct 2023 16:15:10 -0300 Subject: [PATCH 21/23] update ci and release --- .github/workflows/ci.yml | 74 +++++++++++++++++------------------ .github/workflows/release.yml | 6 +-- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ba945d3..80fc7e4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,57 +11,57 @@ on: - master jobs: - # pre-commit: - # runs-on: ubuntu-latest - # if: >- - # github.event.pull_request.merged == false && - # github.event.pull_request.state == 'open' + pre-commit: + runs-on: ubuntu-latest + if: >- + github.event.pull_request.merged == false && + github.event.pull_request.state == 'open' - # steps: - # - uses: actions/checkout@v3 + steps: + - uses: actions/checkout@v3 - # - name: Setup Python - # uses: actions/setup-python@v4 - # with: - # python-version: 3.8 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 - # - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.0 - # # tests: - # runs-on: ubuntu-latest - # needs: [pre-commit] + tests: + runs-on: ubuntu-latest + needs: [pre-commit] - # steps: - # - uses: actions/checkout@v3 + steps: + - uses: actions/checkout@v3 - # - name: Setup Python - # uses: actions/setup-python@v4 - # with: - # python-version: 3.8 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 - # - name: Install dependencies - # run: | - # cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json - # python -m pip install --upgrade pip - # pip install -r requirements-ci.txt + - name: Install dependencies + run: | + cat airflow_variables_dev.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json + python -m pip install --upgrade pip + pip install -r requirements-ci.txt - # - name: Init Airflow SQLite database - # run: airflow db init + - name: Init Airflow SQLite database + run: airflow db init - # - name: Import Airflow variables - # run: airflow variables import airflow_variables_ci.json + - name: Import Airflow variables + run: airflow variables import airflow_variables_ci.json - # - name: Authenticate to test-hubble GCP - # uses: google-github-actions/auth@v1 - # with: - # credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" + - name: Authenticate to test-hubble GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.CREDS_TEST_HUBBLE }}" - # - name: Pytest - # run: pytest dags/ + - name: Pytest + run: pytest dags/ deploy-to-dev: runs-on: ubuntu-latest - # needs: [tests] + needs: [tests] # deploy to dev occurs every time # someone submits a pr targeting `master` # from a branch at `stellar/stellar-etl-airflow` repo diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e9bd69b1..e4515f21 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -75,6 +75,6 @@ jobs: LOCATION: us-central1 with: args: > - components install kubectl && gcloud composer environments\ - run $COMPOSER_ENVIRONMENT --location $LOCATION \ - variables import -- gcsfuse/variables.json + components install kubectl && gcloud composer environments run + $COMPOSER_ENVIRONMENT --location $LOCATION variables import + -- gcsfuse/variables.json From 46e185e529876b2f52394d5c317615e432f4a6d1 Mon Sep 17 00:00:00 2001 From: Cayod Date: Fri, 27 Oct 2023 12:14:29 -0300 Subject: [PATCH 22/23] update ci --- .github/workflows/release.yml | 2 +- dags/stellar_etl_airflow/add_files_to_composer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e4515f21..13a29be1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,7 +69,7 @@ jobs: - name: Update Airflow variables uses: actions-hub/gcloud@master env: - PROJECT_ID: hubble-2-d948d67b + PROJECT_ID: hubble-261722 APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" COMPOSER_ENVIRONMENT: hubble-2 LOCATION: us-central1 diff --git a/dags/stellar_etl_airflow/add_files_to_composer.py b/dags/stellar_etl_airflow/add_files_to_composer.py index 2442c173..5f39159b 100644 --- a/dags/stellar_etl_airflow/add_files_to_composer.py +++ b/dags/stellar_etl_airflow/add_files_to_composer.py @@ -55,7 +55,7 @@ def upload_dags_to_composer(bucket_name: str, env: str) -> None: for f in files: # remove path to temp dir - if f.endswith("variables.json"): + if f.endswith(f"variables_{env}.json"): # insert airflow variable file f = f.replace(f"{temp_dir}/", "") blob = bucket.blob(f) From 412211bf94f531b0c2c9c0fd41db493d9b7e05a4 Mon Sep 17 00:00:00 2001 From: Cayod Date: Fri, 27 Oct 2023 12:17:00 -0300 Subject: [PATCH 23/23] update add_files_to_composer --- dags/stellar_etl_airflow/add_files_to_composer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/stellar_etl_airflow/add_files_to_composer.py b/dags/stellar_etl_airflow/add_files_to_composer.py index 5f39159b..0102a40d 100644 --- a/dags/stellar_etl_airflow/add_files_to_composer.py +++ b/dags/stellar_etl_airflow/add_files_to_composer.py @@ -55,7 +55,7 @@ def upload_dags_to_composer(bucket_name: str, env: str) -> None: for f in files: # remove path to temp dir - if f.endswith(f"variables_{env}.json"): + if f.endswith(f"variables.json"): # insert airflow variable file f = f.replace(f"{temp_dir}/", "") blob = bucket.blob(f)