From 1c5fe64d320ac94ad86e19c2a38fccfb8c4ae9f6 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 20 Sep 2024 21:17:52 +0530 Subject: [PATCH 01/10] Add dag for recency tests --- dags/dbt_recency_tests_dag.py | 40 ++++++++++++++++++++++ dags/stellar_etl_airflow/build_dbt_task.py | 1 + 2 files changed, 41 insertions(+) create mode 100644 dags/dbt_recency_tests_dag.py diff --git a/dags/dbt_recency_tests_dag.py b/dags/dbt_recency_tests_dag.py new file mode 100644 index 00000000..ac5ab5e4 --- /dev/null +++ b/dags/dbt_recency_tests_dag.py @@ -0,0 +1,40 @@ +from datetime import datetime + +from airflow import DAG +from airflow.operators.empty import EmptyOperator +from kubernetes.client import models as k8s +from stellar_etl_airflow.build_dbt_task import dbt_task +from stellar_etl_airflow.build_elementary_slack_alert_task import elementary_task +from stellar_etl_airflow.default import ( + alert_sla_miss, + get_default_dag_args, + init_sentry, +) + +init_sentry() + +with DAG( + "dbt_recency_tests", + default_args=get_default_dag_args(), + start_date=datetime(2024, 6, 25, 0, 0), + description="This DAG runs non-model dbt tests half-hourly cadence", + schedule="*/30 * * * *", # Runs every 30 minutes + user_defined_filters={ + "container_resources": lambda s: k8s.V1ResourceRequirements(requests=s), + }, + max_active_runs=1, + catchup=False, + tags=["dbt-data-quality"], + # sla_miss_callback=alert_sla_miss, +) as dag: + + # DBT tests to run + recency_tests = dbt_task( + dag, + command_type="test", + tag="recency", + resource_cfg="dbt", + run_recency_test="true", + ) + + recency_tests diff --git a/dags/stellar_etl_airflow/build_dbt_task.py b/dags/stellar_etl_airflow/build_dbt_task.py index 3c012d6d..5753cfa0 100644 --- a/dags/stellar_etl_airflow/build_dbt_task.py +++ b/dags/stellar_etl_airflow/build_dbt_task.py @@ -68,6 +68,7 @@ def dbt_task( excluded=None, resource_cfg="default", run_singular_test="false", + run_recency_test="false" ): namespace = conf.get("kubernetes", "NAMESPACE") if namespace == "default": From a6d7cf77adc6f95ec82d0ae73ef80fc8a31cad44 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 20 Sep 2024 21:22:27 +0530 Subject: [PATCH 02/10] export env variable --- dags/stellar_etl_airflow/build_dbt_task.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dags/stellar_etl_airflow/build_dbt_task.py b/dags/stellar_etl_airflow/build_dbt_task.py index 5753cfa0..c5cfa536 100644 --- a/dags/stellar_etl_airflow/build_dbt_task.py +++ b/dags/stellar_etl_airflow/build_dbt_task.py @@ -137,6 +137,7 @@ def dbt_task( "EXECUTION_DATE": "{{ ts }}", "AIRFLOW_START_TIMESTAMP": "{{ ti.start_date.strftime('%Y-%m-%dT%H:%M:%SZ') }}", "IS_SINGULAR_AIRFLOW_TASK": run_singular_test, + "IS_RECENCY_AIRFLOW_TASK": run_recency_test, }, image=dbt_image, arguments=args, From 0a76bb61098a07293a810c8e82ff5f4b7f629006 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 24 Sep 2024 17:02:10 +0530 Subject: [PATCH 03/10] Update dbt image and schedule of recency dag --- airflow_variables_dev.json | 2 +- dags/dbt_recency_tests_dag.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index a5807b16..9fcfe254 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -123,7 +123,7 @@ "partnership_assets__account_holders_activity_fact": false, "partnership_assets__asset_activity_fact": false }, - "dbt_image_name": "stellar/stellar-dbt:02123b970", + "dbt_image_name": "stellar/stellar-dbt-dev:883614fae", "dbt_internal_source_db": "test-hubble-319619", "dbt_internal_source_schema": "test_crypto_stellar_internal", "dbt_job_execution_timeout_seconds": 300, diff --git a/dags/dbt_recency_tests_dag.py b/dags/dbt_recency_tests_dag.py index ac5ab5e4..293a7abf 100644 --- a/dags/dbt_recency_tests_dag.py +++ b/dags/dbt_recency_tests_dag.py @@ -18,7 +18,7 @@ default_args=get_default_dag_args(), start_date=datetime(2024, 6, 25, 0, 0), description="This DAG runs non-model dbt tests half-hourly cadence", - schedule="*/30 * * * *", # Runs every 30 minutes + schedule="0 0 * * * ", # Every day at 12:00 AM UTC user_defined_filters={ "container_resources": lambda s: k8s.V1ResourceRequirements(requests=s), }, From 1d9f53c68514b82d28f565a6aa1ca969adaf052a Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 24 Sep 2024 17:34:58 +0530 Subject: [PATCH 04/10] update variables --- airflow_variables_dev.json | 1 + airflow_variables_prod.json | 1 + 2 files changed, 2 insertions(+) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 9fcfe254..2a026b48 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -358,6 +358,7 @@ "partnership_assets": 660, "relevant_asset_trades": 1200, "singular_test": 600, + "recency": 600, "snapshot_state": 600, "soroban": 720, "trade_agg": 720, diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 9dc597c5..28de7271 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -356,6 +356,7 @@ "partnership_assets": 1380, "relevant_asset_trades": 1800, "singular_test": 840, + "recency": 840, "snapshot_state": 840, "soroban": 420, "trade_agg": 1020, From 354ea7b81dc8e07b4067636da8a3018faa721709 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 25 Sep 2024 00:36:41 +0530 Subject: [PATCH 05/10] update dev image --- airflow_variables_dev.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 2a026b48..792ca472 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -123,7 +123,7 @@ "partnership_assets__account_holders_activity_fact": false, "partnership_assets__asset_activity_fact": false }, - "dbt_image_name": "stellar/stellar-dbt-dev:883614fae", + "dbt_image_name": "stellar/stellar-dbt-dev:7b699d7d1", "dbt_internal_source_db": "test-hubble-319619", "dbt_internal_source_schema": "test_crypto_stellar_internal", "dbt_job_execution_timeout_seconds": 300, From 3d907a4f32ea59e7a227e6be173756d3ce0a5a67 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 25 Sep 2024 00:44:56 +0530 Subject: [PATCH 06/10] update the schedule --- dags/dbt_recency_tests_dag.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dags/dbt_recency_tests_dag.py b/dags/dbt_recency_tests_dag.py index 293a7abf..057d07af 100644 --- a/dags/dbt_recency_tests_dag.py +++ b/dags/dbt_recency_tests_dag.py @@ -16,9 +16,9 @@ with DAG( "dbt_recency_tests", default_args=get_default_dag_args(), - start_date=datetime(2024, 6, 25, 0, 0), + start_date=datetime(2024, 9, 30, 0, 0), description="This DAG runs non-model dbt tests half-hourly cadence", - schedule="0 0 * * * ", # Every day at 12:00 AM UTC + schedule="0 20 * * *", # Every day at 8:00 PM UTC / 3:00 PM CST user_defined_filters={ "container_resources": lambda s: k8s.V1ResourceRequirements(requests=s), }, From c4256cedc9e3c708922893e95ee25cbc26f82f84 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 25 Sep 2024 00:54:44 +0530 Subject: [PATCH 07/10] lint --- airflow_variables_dev.json | 2 +- airflow_variables_prod.json | 2 +- dags/stellar_etl_airflow/build_dbt_task.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 792ca472..89277a32 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -356,9 +356,9 @@ "network_stats": 720, "ohlc": 720, "partnership_assets": 660, + "recency": 600, "relevant_asset_trades": 1200, "singular_test": 600, - "recency": 600, "snapshot_state": 600, "soroban": 720, "trade_agg": 720, diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 28de7271..93e98129 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -354,9 +354,9 @@ "network_stats": 360, "ohlc": 960, "partnership_assets": 1380, + "recency": 840, "relevant_asset_trades": 1800, "singular_test": 840, - "recency": 840, "snapshot_state": 840, "soroban": 420, "trade_agg": 1020, diff --git a/dags/stellar_etl_airflow/build_dbt_task.py b/dags/stellar_etl_airflow/build_dbt_task.py index c5cfa536..53c7f085 100644 --- a/dags/stellar_etl_airflow/build_dbt_task.py +++ b/dags/stellar_etl_airflow/build_dbt_task.py @@ -68,7 +68,7 @@ def dbt_task( excluded=None, resource_cfg="default", run_singular_test="false", - run_recency_test="false" + run_recency_test="false", ): namespace = conf.get("kubernetes", "NAMESPACE") if namespace == "default": From 58678999e30f247997682f9610d241c0e6152234 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 25 Sep 2024 15:39:22 +0530 Subject: [PATCH 08/10] Update dags/dbt_recency_tests_dag.py Co-authored-by: chowbao --- dags/dbt_recency_tests_dag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/dbt_recency_tests_dag.py b/dags/dbt_recency_tests_dag.py index 057d07af..805f08eb 100644 --- a/dags/dbt_recency_tests_dag.py +++ b/dags/dbt_recency_tests_dag.py @@ -17,7 +17,7 @@ "dbt_recency_tests", default_args=get_default_dag_args(), start_date=datetime(2024, 9, 30, 0, 0), - description="This DAG runs non-model dbt tests half-hourly cadence", + description="This DAG runs recency tests at a daily cadence", schedule="0 20 * * *", # Every day at 8:00 PM UTC / 3:00 PM CST user_defined_filters={ "container_resources": lambda s: k8s.V1ResourceRequirements(requests=s), From 7b8855eba4e651e552e09a3d470b1997a4ea5567 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 25 Sep 2024 15:42:01 +0530 Subject: [PATCH 09/10] update dbt image --- airflow_variables_dev.json | 2 +- airflow_variables_prod.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 89277a32..45cdb137 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -123,7 +123,7 @@ "partnership_assets__account_holders_activity_fact": false, "partnership_assets__asset_activity_fact": false }, - "dbt_image_name": "stellar/stellar-dbt-dev:7b699d7d1", + "dbt_image_name": "stellar/stellar-dbt:90c8a57c1", "dbt_internal_source_db": "test-hubble-319619", "dbt_internal_source_schema": "test_crypto_stellar_internal", "dbt_job_execution_timeout_seconds": 300, diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 93e98129..9760d9bc 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -124,7 +124,7 @@ "partnership_assets__asset_activity_fact": false, "trade_agg": false }, - "dbt_image_name": "stellar/stellar-dbt:02123b970", + "dbt_image_name": "stellar/stellar-dbt:90c8a57c1", "dbt_internal_source_db": "hubble-261722", "dbt_internal_source_schema": "crypto_stellar_internal_2", "dbt_job_execution_timeout_seconds": 2400, From 25f2c61893fefc3bb4a13b0ba50a2efdf5b13e13 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 25 Sep 2024 15:43:21 +0530 Subject: [PATCH 10/10] update dag start date --- dags/dbt_recency_tests_dag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/dbt_recency_tests_dag.py b/dags/dbt_recency_tests_dag.py index 805f08eb..24f36b1d 100644 --- a/dags/dbt_recency_tests_dag.py +++ b/dags/dbt_recency_tests_dag.py @@ -16,7 +16,7 @@ with DAG( "dbt_recency_tests", default_args=get_default_dag_args(), - start_date=datetime(2024, 9, 30, 0, 0), + start_date=datetime(2024, 9, 24, 0, 0), description="This DAG runs recency tests at a daily cadence", schedule="0 20 * * *", # Every day at 8:00 PM UTC / 3:00 PM CST user_defined_filters={