Merge branch 'tx-dag' of https://github.com/stellar/stellar-etl-airflow…

… into tx-dag
stellar · Oct 30, 2023 · d7c77a7 · d7c77a7
2 parents f9c68cf + 98728a9
commit d7c77a7
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 3 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -94,6 +94,19 @@ jobs:
           GOOGLE_CLOUD_PROJECT: test-hubble-319619
           BUCKET: us-central1-hubble-1pt5-dev-7db0e004-bucket
 
+      - name: Update Airflow variables
+        uses: actions-hub/gcloud@master
+        env:
+          PROJECT_ID: test-hubble-319619
+          APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}"
+          COMPOSER_ENVIRONMENT: hubble-1pt5-dev
+          LOCATION: us-central1
+        with:
+          args: >
+            components install kubectl && gcloud composer environments run
+            $COMPOSER_ENVIRONMENT --location $LOCATION variables import
+            -- gcsfuse/variables.json
+
   promote-to-prod:
     runs-on: ubuntu-latest
     # deploy only occurs when pr is merged

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -21,7 +21,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          cat airflow_variables.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json
+          cat airflow_variables_prod.json | sed -e s/\\/home\\/airflow\\/gcs\\/dags\\/// > airflow_variables_ci.json
           python -m pip install --upgrade pip
           pip install -r requirements-ci.txt
 
@@ -65,3 +65,16 @@ jobs:
         env:
           GOOGLE_CLOUD_PROJECT: hubble-261722
           BUCKET: us-central1-hubble-2-d948d67b-bucket
+
+      - name: Update Airflow variables
+        uses: actions-hub/gcloud@master
+        env:
+          PROJECT_ID: hubble-261722
+          APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}"
+          COMPOSER_ENVIRONMENT: hubble-2
+          LOCATION: us-central1
+        with:
+          args: >
+            components install kubectl && gcloud composer environments run
+            $COMPOSER_ENVIRONMENT --location $LOCATION variables import
+            -- gcsfuse/variables.json
diff --git a/airflow_variables.json → airflow_variables_prod.json b/airflow_variables.json → airflow_variables_prod.json
diff --git a/dags/stellar_etl_airflow/add_files_to_composer.py b/dags/stellar_etl_airflow/add_files_to_composer.py
@@ -26,7 +26,9 @@ def _create_files_list(env: str) -> Tuple[str, List[str]]:
     # copy airflow configuration file
     copy(f"airflow-{env}.cfg", f"{temp_dir}/airflow.cfg")
     copy(f"airflow-{env}.cfg", f"airflow.cfg")
-
+    # copy airflow variables json
+    copy(f"airflow_variables_{env}.json", f"{temp_dir}/variables.json")
+    copy(f"airflow_variables_{env}.json", f"variables.json")
     dags = glob(f"{temp_dir}/**/*.*", recursive=True)
     return (temp_dir, dags)
 
@@ -53,7 +55,11 @@ def upload_dags_to_composer(bucket_name: str, env: str) -> None:
 
         for f in files:
             # remove path to temp dir
-            if f.endswith(".json"):
+            if f.endswith(f"variables.json"):
+                # insert airflow variable file
+                f = f.replace(f"{temp_dir}/", "")
+                blob = bucket.blob(f)
+            elif f.endswith(".json"):
                 # create schemas directory
                 f = f.replace(f"{temp_dir}/", "schemas/")
                 schema_destination = "dags/" + f