From 94e417459e34d333ac8e703d7c20307c90af2a6b Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 18 Sep 2024 23:14:18 +0530 Subject: [PATCH 1/4] Pass is_singular_airflow_task env var --- dags/dbt_singular_tests_dag.py | 1 + dags/stellar_etl_airflow/build_dbt_task.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/dags/dbt_singular_tests_dag.py b/dags/dbt_singular_tests_dag.py index bb828870..ad80b7a8 100644 --- a/dags/dbt_singular_tests_dag.py +++ b/dags/dbt_singular_tests_dag.py @@ -34,6 +34,7 @@ command_type="test", tag="singular_test", resource_cfg="dbt", + run_singular_test="true" ) singular_tests diff --git a/dags/stellar_etl_airflow/build_dbt_task.py b/dags/stellar_etl_airflow/build_dbt_task.py index e9f36563..13a19bf2 100644 --- a/dags/stellar_etl_airflow/build_dbt_task.py +++ b/dags/stellar_etl_airflow/build_dbt_task.py @@ -67,6 +67,7 @@ def dbt_task( command_type="build", excluded=None, resource_cfg="default", + run_singular_test="false" ): namespace = conf.get("kubernetes", "NAMESPACE") if namespace == "default": @@ -134,6 +135,7 @@ def dbt_task( "PUBLIC_SOURCE_SCHEMA": "{{ var.value.dbt_public_source_schema }}", "EXECUTION_DATE": "{{ ts }}", "AIRFLOW_START_TIMESTAMP": "{{ ti.start_date.strftime('%Y-%m-%dT%H:%M:%SZ') }}", + "IS_SINGULAR_AIRFLOW_TASK": run_singular_test, }, image=dbt_image, arguments=args, From e295338a2529ffaa956627efe8d865be45fad39a Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 18 Sep 2024 23:15:21 +0530 Subject: [PATCH 2/4] Update dbt image --- airflow_variables_dev.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index d94436f8..0b70f1b1 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -123,7 +123,7 @@ "partnership_assets__account_holders_activity_fact": false, "partnership_assets__asset_activity_fact": false }, - "dbt_image_name": "stellar/stellar-dbt:ee6aae569", + "dbt_image_name": "stellar/stellar-dbt-dev:6b7e2855c", "dbt_internal_source_db": "test-hubble-319619", "dbt_internal_source_schema": "test_crypto_stellar_internal", "dbt_job_execution_timeout_seconds": 300, From affcb78853806218668d8a32b499b4cbb42ebb05 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Thu, 19 Sep 2024 00:17:25 +0530 Subject: [PATCH 3/4] lint --- dags/dbt_singular_tests_dag.py | 2 +- dags/stellar_etl_airflow/build_dbt_task.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dags/dbt_singular_tests_dag.py b/dags/dbt_singular_tests_dag.py index ad80b7a8..8c056fd2 100644 --- a/dags/dbt_singular_tests_dag.py +++ b/dags/dbt_singular_tests_dag.py @@ -34,7 +34,7 @@ command_type="test", tag="singular_test", resource_cfg="dbt", - run_singular_test="true" + run_singular_test="true", ) singular_tests diff --git a/dags/stellar_etl_airflow/build_dbt_task.py b/dags/stellar_etl_airflow/build_dbt_task.py index 13a19bf2..3c012d6d 100644 --- a/dags/stellar_etl_airflow/build_dbt_task.py +++ b/dags/stellar_etl_airflow/build_dbt_task.py @@ -67,7 +67,7 @@ def dbt_task( command_type="build", excluded=None, resource_cfg="default", - run_singular_test="false" + run_singular_test="false", ): namespace = conf.get("kubernetes", "NAMESPACE") if namespace == "default": From 93c2f6341db5aa862df4bbd707b49b38a45b94e5 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Thu, 19 Sep 2024 12:15:06 +0530 Subject: [PATCH 4/4] update images --- airflow_variables_dev.json | 2 +- airflow_variables_prod.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 0b70f1b1..954e0fc1 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -123,7 +123,7 @@ "partnership_assets__account_holders_activity_fact": false, "partnership_assets__asset_activity_fact": false }, - "dbt_image_name": "stellar/stellar-dbt-dev:6b7e2855c", + "dbt_image_name": "stellar/stellar-dbt:3d0a8056c", "dbt_internal_source_db": "test-hubble-319619", "dbt_internal_source_schema": "test_crypto_stellar_internal", "dbt_job_execution_timeout_seconds": 300, diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 023b869e..c94a12a8 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -124,7 +124,7 @@ "partnership_assets__asset_activity_fact": false, "trade_agg": false }, - "dbt_image_name": "stellar/stellar-dbt:ee6aae569", + "dbt_image_name": "stellar/stellar-dbt:3d0a8056c", "dbt_internal_source_db": "hubble-261722", "dbt_internal_source_schema": "crypto_stellar_internal_2", "dbt_job_execution_timeout_seconds": 2400,