diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index f842182d..a8b71fce 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -132,7 +132,7 @@ "dbt_threads": 12, "gcs_exported_data_bucket_name": "us-central1-hubble-1pt5-dev-7db0e004-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:3761e4a", + "image_name": "stellar/stellar-etl:1ba69a5", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "kube_config_location": "", diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index c5ed553f..c3f71690 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -154,7 +154,7 @@ "dbt_threads": 12, "gcs_exported_data_bucket_name": "us-central1-hubble-2-d948d67b-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:3761e4a", + "image_name": "stellar/stellar-etl:1ba69a5", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "kube_config_location": "", diff --git a/dags/dbt_sdf_marts_dag.py b/dags/dbt_sdf_marts_dag.py index fb645cc2..0c042aeb 100644 --- a/dags/dbt_sdf_marts_dag.py +++ b/dags/dbt_sdf_marts_dag.py @@ -11,7 +11,7 @@ default_args=get_default_dag_args(), start_date=datetime(2024, 1, 26, 0, 0), description="This DAG runs dbt models at a daily cadence", - schedule_interval="0 13 * * *", # Runs at 13:00 UTC + schedule_interval="0 16 * * *", # Runs at 16:00 UTC user_defined_filters={ "container_resources": lambda s: k8s.V1ResourceRequirements(requests=s), }, diff --git a/dags/partner_pipeline_dag.py b/dags/partner_pipeline_dag.py index 5ca6d89b..b29fdfab 100644 --- a/dags/partner_pipeline_dag.py +++ b/dags/partner_pipeline_dag.py @@ -24,7 +24,7 @@ default_args=get_default_dag_args(), start_date=datetime(2023, 1, 1, 0, 0), description="This DAG automates daily updates to partner tables in BigQuery.", - schedule_interval="0 13 * * *", + schedule_interval="0 16 * * *", params={ "alias": "partner", }, diff --git a/dags/sandbox_update_dag.py b/dags/sandbox_update_dag.py index 4fb62c9e..fb4b7b69 100644 --- a/dags/sandbox_update_dag.py +++ b/dags/sandbox_update_dag.py @@ -12,6 +12,7 @@ file_to_string, get_query_filepath, ) +from stellar_etl_airflow.build_cross_dependency_task import build_cross_deps from stellar_etl_airflow.default import ( alert_after_max_retries, get_default_dag_args, @@ -25,7 +26,7 @@ default_args=get_default_dag_args(), start_date=datetime(2023, 1, 1), description="This DAG updates a sandbox", - schedule_interval="@daily", + schedule_interval="0 1 * * *", params={"alias": "sandbox_dataset"}, user_defined_filters={"fromjson": lambda s: loads(s)}, catchup=False, @@ -37,6 +38,10 @@ start_tables_task = EmptyOperator(task_id="start_tables_task") + wait_on_dag = build_cross_deps( + dag, "wait_on_base_tables", "history_archive_with_captive_core_combined_export" + ) + for table_id in TABLES_ID: query_path = get_query_filepath("update_table") query = file_to_string(query_path) @@ -58,4 +63,4 @@ on_failure_callback=alert_after_max_retries, ) - start_tables_task >> tables_update_task + start_tables_task >> wait_on_dag >> tables_update_task