Merge pull request #296 from stellar/master

[PRODUCTION] Update production Airflow environment
stellar · Feb 12, 2024 · 996b816 · 996b816
2 parents aa9add7 + 029eb71
commit 996b816
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 6 deletions.
diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json
@@ -132,7 +132,7 @@
   "dbt_threads": 12,
   "gcs_exported_data_bucket_name": "us-central1-hubble-1pt5-dev-7db0e004-bucket",
   "gcs_exported_object_prefix": "dag-exported",
-  "image_name": "stellar/stellar-etl:3761e4a",
+  "image_name": "stellar/stellar-etl:1ba69a5",
   "image_output_path": "/etl/exported_data/",
   "image_pull_policy": "IfNotPresent",
   "kube_config_location": "",

diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json
@@ -154,7 +154,7 @@
   "dbt_threads": 12,
   "gcs_exported_data_bucket_name": "us-central1-hubble-2-d948d67b-bucket",
   "gcs_exported_object_prefix": "dag-exported",
-  "image_name": "stellar/stellar-etl:3761e4a",
+  "image_name": "stellar/stellar-etl:1ba69a5",
   "image_output_path": "/etl/exported_data/",
   "image_pull_policy": "IfNotPresent",
   "kube_config_location": "",

diff --git a/dags/dbt_sdf_marts_dag.py b/dags/dbt_sdf_marts_dag.py
@@ -11,7 +11,7 @@
     default_args=get_default_dag_args(),
     start_date=datetime(2024, 1, 26, 0, 0),
     description="This DAG runs dbt models at a daily cadence",
-    schedule_interval="0 13 * * *",  # Runs at 13:00 UTC
+    schedule_interval="0 16 * * *",  # Runs at 16:00 UTC
     user_defined_filters={
         "container_resources": lambda s: k8s.V1ResourceRequirements(requests=s),
     },

diff --git a/dags/partner_pipeline_dag.py b/dags/partner_pipeline_dag.py
@@ -24,7 +24,7 @@
     default_args=get_default_dag_args(),
     start_date=datetime(2023, 1, 1, 0, 0),
     description="This DAG automates daily updates to partner tables in BigQuery.",
-    schedule_interval="0 13 * * *",
+    schedule_interval="0 16 * * *",
     params={
         "alias": "partner",
     },

diff --git a/dags/sandbox_update_dag.py b/dags/sandbox_update_dag.py
@@ -12,6 +12,7 @@
     file_to_string,
     get_query_filepath,
 )
+from stellar_etl_airflow.build_cross_dependency_task import build_cross_deps
 from stellar_etl_airflow.default import (
     alert_after_max_retries,
     get_default_dag_args,
@@ -25,7 +26,7 @@
     default_args=get_default_dag_args(),
     start_date=datetime(2023, 1, 1),
     description="This DAG updates a sandbox",
-    schedule_interval="@daily",
+    schedule_interval="0 1 * * *",
     params={"alias": "sandbox_dataset"},
     user_defined_filters={"fromjson": lambda s: loads(s)},
     catchup=False,
@@ -37,6 +38,10 @@
 
     start_tables_task = EmptyOperator(task_id="start_tables_task")
 
+    wait_on_dag = build_cross_deps(
+        dag, "wait_on_base_tables", "history_archive_with_captive_core_combined_export"
+    )
+
     for table_id in TABLES_ID:
         query_path = get_query_filepath("update_table")
         query = file_to_string(query_path)
@@ -58,4 +63,4 @@
             on_failure_callback=alert_after_max_retries,
         )
 
-        start_tables_task >> tables_update_task
+        start_tables_task >> wait_on_dag >> tables_update_task