Revert mainnet ci changes

stellar · Apr 18, 2024 · 38e25f0 · 38e25f0
1 parent 4eaf764
commit 38e25f0
Show file tree

Hide file tree

Showing 2 changed files with 93 additions and 7 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -39,8 +39,8 @@ jobs:
       - id: "get-credentials"
         uses: "google-github-actions/get-gke-credentials@v2"
         with:
-          cluster_name: "us-central1-hubble-14c4ca64-gke"
-          location: "us-central1"
+          cluster_name: "us-central1-hubble-1pt5-dev-7db0e004-gke"
+          location: "us-central1-c"
 
       - name: Pytest
         run: pytest dags/
@@ -75,17 +75,17 @@ jobs:
         run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET --env prod
         env:
           GOOGLE_CLOUD_PROJECT: hubble-261722
-          BUCKET: us-central1-hubble-14c4ca64-bucket
+          BUCKET: us-central1-hubble-2-d948d67b-bucket
 
       - name: Update Airflow variables
         uses: actions-hub/gcloud@master
         env:
           PROJECT_ID: hubble-261722
           APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}"
-          COMPOSER_ENVIRONMENT: hubble
+          COMPOSER_ENVIRONMENT: hubble-2
           LOCATION: us-central1
         with:
           args: >
             components install kubectl && gcloud composer environments run
             $COMPOSER_ENVIRONMENT --location $LOCATION variables import
-            -- gcsfuse/actual_mount_path/variables.json
+            -- gcsfuse/variables.json
diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json
@@ -1,5 +1,91 @@
 {
-  "affinity": {},
+  "affinity": {
+    "default": {
+      "nodeAffinity": {
+        "requiredDuringSchedulingIgnoredDuringExecution": {
+          "nodeSelectorTerms": [
+            {
+              "matchExpressions": [
+                {
+                  "key": "cloud.google.com/gke-nodepool",
+                  "operator": "In",
+                  "values": ["default-pool"]
+                }
+              ]
+            }
+          ]
+        }
+      }
+    },
+    "backfill": {
+      "nodeAffinity": {
+        "requiredDuringSchedulingIgnoredDuringExecution": {
+          "nodeSelectorTerms": [
+            {
+              "matchExpressions": [
+                {
+                  "key": "cloud.google.com/gke-nodepool",
+                  "operator": "In",
+                  "values": ["hubble-etl"]
+                }
+              ]
+            }
+          ]
+        }
+      }
+    },
+    "cc": {
+      "nodeAffinity": {
+        "requiredDuringSchedulingIgnoredDuringExecution": {
+          "nodeSelectorTerms": [
+            {
+              "matchExpressions": [
+                {
+                  "key": "cloud.google.com/gke-nodepool",
+                  "operator": "In",
+                  "values": ["hubble-etl"]
+                }
+              ]
+            }
+          ]
+        }
+      }
+    },
+    "wocc": {
+      "nodeAffinity": {
+        "requiredDuringSchedulingIgnoredDuringExecution": {
+          "nodeSelectorTerms": [
+            {
+              "matchExpressions": [
+                {
+                  "key": "cloud.google.com/gke-nodepool",
+                  "operator": "In",
+                  "values": ["default-pool"]
+                }
+              ]
+            }
+          ]
+        }
+      }
+    },
+    "state": {
+      "nodeAffinity": {
+        "requiredDuringSchedulingIgnoredDuringExecution": {
+          "nodeSelectorTerms": [
+            {
+              "matchExpressions": [
+                {
+                  "key": "cloud.google.com/gke-nodepool",
+                  "operator": "In",
+                  "values": ["state-table"]
+                }
+              ]
+            }
+          ]
+        }
+      }
+    }
+  },
   "api_key_path": "/home/airflow/gcs/data/apiKey.json",
   "bq_dataset": "crypto_stellar_internal_2",
   "bq_project": "hubble-261722",
@@ -65,7 +151,7 @@
   "dbt_project": "hubble-261722",
   "dbt_target": "prod",
   "dbt_threads": 12,
-  "gcs_exported_data_bucket_name": "us-central1-hubble-14c4ca64-bucket",
+  "gcs_exported_data_bucket_name": "us-central1-hubble-2-d948d67b-bucket",
   "gcs_exported_object_prefix": "dag-exported",
   "image_name": "stellar/stellar-etl:395f1f8",
   "image_output_path": "/etl/exported_data/",