diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f089d9db..1ba61839 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,8 +39,8 @@ jobs: - id: "get-credentials" uses: "google-github-actions/get-gke-credentials@v2" with: - cluster_name: "us-central1-hubble-14c4ca64-gke" - location: "us-central1" + cluster_name: "us-central1-hubble-1pt5-dev-7db0e004-gke" + location: "us-central1-c" - name: Pytest run: pytest dags/ @@ -75,17 +75,17 @@ jobs: run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET --env prod env: GOOGLE_CLOUD_PROJECT: hubble-261722 - BUCKET: us-central1-hubble-14c4ca64-bucket + BUCKET: us-central1-hubble-2-d948d67b-bucket - name: Update Airflow variables uses: actions-hub/gcloud@master env: PROJECT_ID: hubble-261722 APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" - COMPOSER_ENVIRONMENT: hubble + COMPOSER_ENVIRONMENT: hubble-2 LOCATION: us-central1 with: args: > components install kubectl && gcloud composer environments run $COMPOSER_ENVIRONMENT --location $LOCATION variables import - -- gcsfuse/actual_mount_path/variables.json + -- gcsfuse/variables.json diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 43fe0c2f..7af9edc7 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -1,5 +1,91 @@ { - "affinity": {}, + "affinity": { + "default": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "cloud.google.com/gke-nodepool", + "operator": "In", + "values": ["default-pool"] + } + ] + } + ] + } + } + }, + "backfill": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "cloud.google.com/gke-nodepool", + "operator": "In", + "values": ["hubble-etl"] + } + ] + } + ] + } + } + }, + "cc": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "cloud.google.com/gke-nodepool", + "operator": "In", + "values": ["hubble-etl"] + } + ] + } + ] + } + } + }, + "wocc": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "cloud.google.com/gke-nodepool", + "operator": "In", + "values": ["default-pool"] + } + ] + } + ] + } + } + }, + "state": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "cloud.google.com/gke-nodepool", + "operator": "In", + "values": ["state-table"] + } + ] + } + ] + } + } + } + }, "api_key_path": "/home/airflow/gcs/data/apiKey.json", "bq_dataset": "crypto_stellar_internal_2", "bq_project": "hubble-261722", @@ -65,7 +151,7 @@ "dbt_project": "hubble-261722", "dbt_target": "prod", "dbt_threads": 12, - "gcs_exported_data_bucket_name": "us-central1-hubble-14c4ca64-bucket", + "gcs_exported_data_bucket_name": "us-central1-hubble-2-d948d67b-bucket", "gcs_exported_object_prefix": "dag-exported", "image_name": "stellar/stellar-etl:395f1f8", "image_output_path": "/etl/exported_data/",