From 9de1ecb941c72db9d7d48d0dc66f975e75ad0eb5 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 20 Sep 2024 21:26:04 +0530 Subject: [PATCH 1/5] Add recency tag --- dbt_project.yml | 1 + models/marts/enriched_history/enriched_history_operations.yml | 1 + .../enriched_history/enriched_history_operations_soroban.yml | 1 + models/marts/fee_stats_agg.yml | 1 + models/marts/history_assets.yml | 1 + models/marts/ledger_current_state/account_signers_current.yml | 1 + models/marts/ledger_current_state/accounts_current.yml | 1 + models/marts/ledger_current_state/claimable_balances_current.yml | 1 + models/marts/ledger_current_state/contract_data_current.yml | 1 + models/marts/ledger_current_state/liquidity_pools_current.yml | 1 + models/marts/ledger_current_state/offers_current.yml | 1 + models/marts/ledger_current_state/trust_lines_current.yml | 1 + models/marts/ledger_current_state/ttl_current.yml | 1 + models/marts/trade_agg.yml | 1 + 14 files changed, 14 insertions(+) diff --git a/dbt_project.yml b/dbt_project.yml index 2bc01d4..85e77e9 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -30,6 +30,7 @@ vars: airflow_start_timestamp: "{{ env_var('AIRFLOW_START_TIMESTAMP', '2000-01-01') }}" is_singular_airflow_task: "{{ env_var('IS_SINGULAR_AIRFLOW_TASK', 'false') }}" + is_recency_airflow_task: "{{ env_var('IS_RECENCY_AIRFLOW_TASK', 'false') }}" # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models diff --git a/models/marts/enriched_history/enriched_history_operations.yml b/models/marts/enriched_history/enriched_history_operations.yml index 11aca57..74b4df3 100644 --- a/models/marts/enriched_history/enriched_history_operations.yml +++ b/models/marts/enriched_history/enriched_history_operations.yml @@ -5,6 +5,7 @@ models: description: Aggregate table for the history operations tests: - dbt_utils.recency: + tags: [recency] datepart: hour field: closed_at interval: 12 diff --git a/models/marts/enriched_history/enriched_history_operations_soroban.yml b/models/marts/enriched_history/enriched_history_operations_soroban.yml index 5b19d29..ccac739 100644 --- a/models/marts/enriched_history/enriched_history_operations_soroban.yml +++ b/models/marts/enriched_history/enriched_history_operations_soroban.yml @@ -5,6 +5,7 @@ models: description: Aggregate table for the history operations, taking only the soroban operations tests: - dbt_utils.recency: + tags: [recency] datepart: hour field: closed_at interval: 12 diff --git a/models/marts/fee_stats_agg.yml b/models/marts/fee_stats_agg.yml index ed0ca51..1efc3f0 100644 --- a/models/marts/fee_stats_agg.yml +++ b/models/marts/fee_stats_agg.yml @@ -5,6 +5,7 @@ models: description: '{{ doc("fee_stats") }}' tests: - dbt_utils.recency: + tags: [recency] datepart: day field: cast(day_agg as timestamp) interval: 2 diff --git a/models/marts/history_assets.yml b/models/marts/history_assets.yml index 54886ff..9b0e9f4 100644 --- a/models/marts/history_assets.yml +++ b/models/marts/history_assets.yml @@ -5,6 +5,7 @@ models: description: '{{ doc("history_assets") }}' tests: - dbt_utils.recency: + tags: [recency] datepart: day field: cast(batch_run_date as timestamp) interval: 2 diff --git a/models/marts/ledger_current_state/account_signers_current.yml b/models/marts/ledger_current_state/account_signers_current.yml index f27e2a4..30c5588 100644 --- a/models/marts/ledger_current_state/account_signers_current.yml +++ b/models/marts/ledger_current_state/account_signers_current.yml @@ -5,6 +5,7 @@ models: description: '{{ doc("account_signers_current") }}' tests: - dbt_utils.recency: + tags: [recency] datepart: hour field: closed_at interval: 12 diff --git a/models/marts/ledger_current_state/accounts_current.yml b/models/marts/ledger_current_state/accounts_current.yml index 43280f6..4ad59e3 100644 --- a/models/marts/ledger_current_state/accounts_current.yml +++ b/models/marts/ledger_current_state/accounts_current.yml @@ -5,6 +5,7 @@ models: description: '{{ doc("accounts_current") }}' tests: - dbt_utils.recency: + tags: [recency] datepart: hour field: closed_at interval: 12 diff --git a/models/marts/ledger_current_state/claimable_balances_current.yml b/models/marts/ledger_current_state/claimable_balances_current.yml index 212586f..8362602 100644 --- a/models/marts/ledger_current_state/claimable_balances_current.yml +++ b/models/marts/ledger_current_state/claimable_balances_current.yml @@ -5,6 +5,7 @@ models: description: '{{ doc("claimable_balances_current") }}' tests: - dbt_utils.recency: + tags: [recency] datepart: hour field: closed_at interval: 12 diff --git a/models/marts/ledger_current_state/contract_data_current.yml b/models/marts/ledger_current_state/contract_data_current.yml index 082205e..ce62ca8 100644 --- a/models/marts/ledger_current_state/contract_data_current.yml +++ b/models/marts/ledger_current_state/contract_data_current.yml @@ -5,6 +5,7 @@ models: description: '{{ doc("contract_data_current") }}' tests: - dbt_utils.recency: + tags: [recency] datepart: hour field: closed_at interval: 12 diff --git a/models/marts/ledger_current_state/liquidity_pools_current.yml b/models/marts/ledger_current_state/liquidity_pools_current.yml index 042dde8..20927e7 100644 --- a/models/marts/ledger_current_state/liquidity_pools_current.yml +++ b/models/marts/ledger_current_state/liquidity_pools_current.yml @@ -5,6 +5,7 @@ models: description: '{{ doc("liquidity_pools_current") }}' tests: - dbt_utils.recency: + tags: [recency] datepart: hour field: closed_at interval: 12 diff --git a/models/marts/ledger_current_state/offers_current.yml b/models/marts/ledger_current_state/offers_current.yml index 186db0b..28605d6 100644 --- a/models/marts/ledger_current_state/offers_current.yml +++ b/models/marts/ledger_current_state/offers_current.yml @@ -5,6 +5,7 @@ models: description: '{{ doc("offers_current") }}' tests: - dbt_utils.recency: + tags: [recency] datepart: hour field: closed_at interval: 12 diff --git a/models/marts/ledger_current_state/trust_lines_current.yml b/models/marts/ledger_current_state/trust_lines_current.yml index 88eaa64..052cbb6 100644 --- a/models/marts/ledger_current_state/trust_lines_current.yml +++ b/models/marts/ledger_current_state/trust_lines_current.yml @@ -5,6 +5,7 @@ models: description: '{{ doc("trust_lines_current") }}' tests: - dbt_utils.recency: + tags: [recency] datepart: hour field: closed_at interval: 12 diff --git a/models/marts/ledger_current_state/ttl_current.yml b/models/marts/ledger_current_state/ttl_current.yml index a4430a5..c6f6907 100644 --- a/models/marts/ledger_current_state/ttl_current.yml +++ b/models/marts/ledger_current_state/ttl_current.yml @@ -5,6 +5,7 @@ models: description: '{{ doc("ttl_current") }}' tests: - dbt_utils.recency: + tags: [recency] datepart: hour field: closed_at interval: 12 diff --git a/models/marts/trade_agg.yml b/models/marts/trade_agg.yml index 8963fe1..d69f613 100644 --- a/models/marts/trade_agg.yml +++ b/models/marts/trade_agg.yml @@ -5,6 +5,7 @@ models: description: '{{ doc("trade_agg") }}' tests: - dbt_utils.recency: + tags: [recency] datepart: day field: cast(day_agg as timestamp) interval: 2 From 45aad2409e606f1e6db3c56eb3362f85b512470f Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 24 Sep 2024 16:20:54 +0530 Subject: [PATCH 2/5] enable recency only for ci target or for recency airflow task update --- models/marts/enriched_history/enriched_history_operations.yml | 1 + .../enriched_history/enriched_history_operations_soroban.yml | 1 + models/marts/fee_stats_agg.yml | 1 + models/marts/history_assets.yml | 1 + models/marts/ledger_current_state/account_signers_current.yml | 1 + models/marts/ledger_current_state/accounts_current.yml | 1 + .../marts/ledger_current_state/claimable_balances_current.yml | 1 + models/marts/ledger_current_state/contract_code_current.yml | 2 ++ models/marts/ledger_current_state/contract_data_current.yml | 1 + models/marts/ledger_current_state/liquidity_pools_current.yml | 1 + models/marts/ledger_current_state/offers_current.yml | 1 + models/marts/ledger_current_state/trust_lines_current.yml | 1 + models/marts/ledger_current_state/ttl_current.yml | 1 + models/marts/trade_agg.yml | 1 + 14 files changed, 15 insertions(+) diff --git a/models/marts/enriched_history/enriched_history_operations.yml b/models/marts/enriched_history/enriched_history_operations.yml index 74b4df3..aeb3b9b 100644 --- a/models/marts/enriched_history/enriched_history_operations.yml +++ b/models/marts/enriched_history/enriched_history_operations.yml @@ -10,6 +10,7 @@ models: field: closed_at interval: 12 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/enriched_history/enriched_history_operations_soroban.yml b/models/marts/enriched_history/enriched_history_operations_soroban.yml index ccac739..f5e8a3c 100644 --- a/models/marts/enriched_history/enriched_history_operations_soroban.yml +++ b/models/marts/enriched_history/enriched_history_operations_soroban.yml @@ -10,6 +10,7 @@ models: field: closed_at interval: 12 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/fee_stats_agg.yml b/models/marts/fee_stats_agg.yml index 1efc3f0..e8ee586 100644 --- a/models/marts/fee_stats_agg.yml +++ b/models/marts/fee_stats_agg.yml @@ -10,6 +10,7 @@ models: field: cast(day_agg as timestamp) interval: 2 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/history_assets.yml b/models/marts/history_assets.yml index 9b0e9f4..b6031d3 100644 --- a/models/marts/history_assets.yml +++ b/models/marts/history_assets.yml @@ -10,6 +10,7 @@ models: field: cast(batch_run_date as timestamp) interval: 2 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/account_signers_current.yml b/models/marts/ledger_current_state/account_signers_current.yml index 30c5588..9c78d2a 100644 --- a/models/marts/ledger_current_state/account_signers_current.yml +++ b/models/marts/ledger_current_state/account_signers_current.yml @@ -10,6 +10,7 @@ models: field: closed_at interval: 12 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/accounts_current.yml b/models/marts/ledger_current_state/accounts_current.yml index 4ad59e3..1db6cc4 100644 --- a/models/marts/ledger_current_state/accounts_current.yml +++ b/models/marts/ledger_current_state/accounts_current.yml @@ -10,6 +10,7 @@ models: field: closed_at interval: 12 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/claimable_balances_current.yml b/models/marts/ledger_current_state/claimable_balances_current.yml index 8362602..41b84ac 100644 --- a/models/marts/ledger_current_state/claimable_balances_current.yml +++ b/models/marts/ledger_current_state/claimable_balances_current.yml @@ -10,6 +10,7 @@ models: field: closed_at interval: 12 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/contract_code_current.yml b/models/marts/ledger_current_state/contract_code_current.yml index f865efd..2163c78 100644 --- a/models/marts/ledger_current_state/contract_code_current.yml +++ b/models/marts/ledger_current_state/contract_code_current.yml @@ -5,10 +5,12 @@ models: description: '{{ doc("contract_code_current") }}' tests: - dbt_utils.recency: + tags: [recency] datepart: day field: closed_at interval: 7 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/contract_data_current.yml b/models/marts/ledger_current_state/contract_data_current.yml index ce62ca8..08703a4 100644 --- a/models/marts/ledger_current_state/contract_data_current.yml +++ b/models/marts/ledger_current_state/contract_data_current.yml @@ -10,6 +10,7 @@ models: field: closed_at interval: 12 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/liquidity_pools_current.yml b/models/marts/ledger_current_state/liquidity_pools_current.yml index 1f98427..f28aa5d 100644 --- a/models/marts/ledger_current_state/liquidity_pools_current.yml +++ b/models/marts/ledger_current_state/liquidity_pools_current.yml @@ -10,6 +10,7 @@ models: field: closed_at interval: 24 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/offers_current.yml b/models/marts/ledger_current_state/offers_current.yml index 28605d6..1ca650b 100644 --- a/models/marts/ledger_current_state/offers_current.yml +++ b/models/marts/ledger_current_state/offers_current.yml @@ -10,6 +10,7 @@ models: field: closed_at interval: 12 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/trust_lines_current.yml b/models/marts/ledger_current_state/trust_lines_current.yml index 052cbb6..607b3e8 100644 --- a/models/marts/ledger_current_state/trust_lines_current.yml +++ b/models/marts/ledger_current_state/trust_lines_current.yml @@ -10,6 +10,7 @@ models: field: closed_at interval: 12 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/ttl_current.yml b/models/marts/ledger_current_state/ttl_current.yml index c6f6907..ae2f473 100644 --- a/models/marts/ledger_current_state/ttl_current.yml +++ b/models/marts/ledger_current_state/ttl_current.yml @@ -10,6 +10,7 @@ models: field: closed_at interval: 12 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/trade_agg.yml b/models/marts/trade_agg.yml index d69f613..f297e94 100644 --- a/models/marts/trade_agg.yml +++ b/models/marts/trade_agg.yml @@ -10,6 +10,7 @@ models: field: cast(day_agg as timestamp) interval: 2 config: + enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." From 437b756ff4eb60826bd2f0b39176289cd68c7997 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 24 Sep 2024 16:48:42 +0530 Subject: [PATCH 3/5] use template --- models/marts/enriched_history/enriched_history_operations.yml | 2 +- .../enriched_history/enriched_history_operations_soroban.yml | 2 +- models/marts/fee_stats_agg.yml | 2 +- models/marts/history_assets.yml | 2 +- models/marts/ledger_current_state/account_signers_current.yml | 2 +- models/marts/ledger_current_state/accounts_current.yml | 2 +- .../marts/ledger_current_state/claimable_balances_current.yml | 2 +- models/marts/ledger_current_state/contract_code_current.yml | 2 +- models/marts/ledger_current_state/contract_data_current.yml | 2 +- models/marts/ledger_current_state/liquidity_pools_current.yml | 2 +- models/marts/ledger_current_state/offers_current.yml | 2 +- models/marts/ledger_current_state/trust_lines_current.yml | 2 +- models/marts/ledger_current_state/ttl_current.yml | 2 +- models/marts/trade_agg.yml | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/models/marts/enriched_history/enriched_history_operations.yml b/models/marts/enriched_history/enriched_history_operations.yml index aeb3b9b..480ff7e 100644 --- a/models/marts/enriched_history/enriched_history_operations.yml +++ b/models/marts/enriched_history/enriched_history_operations.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/enriched_history/enriched_history_operations_soroban.yml b/models/marts/enriched_history/enriched_history_operations_soroban.yml index f5e8a3c..159183b 100644 --- a/models/marts/enriched_history/enriched_history_operations_soroban.yml +++ b/models/marts/enriched_history/enriched_history_operations_soroban.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/fee_stats_agg.yml b/models/marts/fee_stats_agg.yml index e8ee586..937a69d 100644 --- a/models/marts/fee_stats_agg.yml +++ b/models/marts/fee_stats_agg.yml @@ -10,7 +10,7 @@ models: field: cast(day_agg as timestamp) interval: 2 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/history_assets.yml b/models/marts/history_assets.yml index b6031d3..84e0067 100644 --- a/models/marts/history_assets.yml +++ b/models/marts/history_assets.yml @@ -10,7 +10,7 @@ models: field: cast(batch_run_date as timestamp) interval: 2 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/account_signers_current.yml b/models/marts/ledger_current_state/account_signers_current.yml index 9c78d2a..7a689f3 100644 --- a/models/marts/ledger_current_state/account_signers_current.yml +++ b/models/marts/ledger_current_state/account_signers_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/accounts_current.yml b/models/marts/ledger_current_state/accounts_current.yml index 1db6cc4..ac5c7f9 100644 --- a/models/marts/ledger_current_state/accounts_current.yml +++ b/models/marts/ledger_current_state/accounts_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/claimable_balances_current.yml b/models/marts/ledger_current_state/claimable_balances_current.yml index 41b84ac..2e7a069 100644 --- a/models/marts/ledger_current_state/claimable_balances_current.yml +++ b/models/marts/ledger_current_state/claimable_balances_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/contract_code_current.yml b/models/marts/ledger_current_state/contract_code_current.yml index 2163c78..975a4f1 100644 --- a/models/marts/ledger_current_state/contract_code_current.yml +++ b/models/marts/ledger_current_state/contract_code_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 7 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/contract_data_current.yml b/models/marts/ledger_current_state/contract_data_current.yml index 08703a4..1ed7bd3 100644 --- a/models/marts/ledger_current_state/contract_data_current.yml +++ b/models/marts/ledger_current_state/contract_data_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/liquidity_pools_current.yml b/models/marts/ledger_current_state/liquidity_pools_current.yml index f28aa5d..3f6b801 100644 --- a/models/marts/ledger_current_state/liquidity_pools_current.yml +++ b/models/marts/ledger_current_state/liquidity_pools_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 24 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/offers_current.yml b/models/marts/ledger_current_state/offers_current.yml index 1ca650b..ffa3b29 100644 --- a/models/marts/ledger_current_state/offers_current.yml +++ b/models/marts/ledger_current_state/offers_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/trust_lines_current.yml b/models/marts/ledger_current_state/trust_lines_current.yml index 607b3e8..e584d35 100644 --- a/models/marts/ledger_current_state/trust_lines_current.yml +++ b/models/marts/ledger_current_state/trust_lines_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/ttl_current.yml b/models/marts/ledger_current_state/ttl_current.yml index ae2f473..fc2551c 100644 --- a/models/marts/ledger_current_state/ttl_current.yml +++ b/models/marts/ledger_current_state/ttl_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/trade_agg.yml b/models/marts/trade_agg.yml index f297e94..3f7123f 100644 --- a/models/marts/trade_agg.yml +++ b/models/marts/trade_agg.yml @@ -10,7 +10,7 @@ models: field: cast(day_agg as timestamp) interval: 2 config: - enabled: (target.name == "ci" or var("is_recency_airflow_task") == "true") + enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." From ca62c8412fbaa108e44e5df1c49a327adc1fae20 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 25 Sep 2024 00:21:29 +0530 Subject: [PATCH 4/5] Use env variable directly --- dbt_project.yml | 1 - models/marts/enriched_history/enriched_history_operations.yml | 2 +- .../enriched_history/enriched_history_operations_soroban.yml | 2 +- models/marts/fee_stats_agg.yml | 2 +- models/marts/history_assets.yml | 2 +- models/marts/ledger_current_state/account_signers_current.yml | 2 +- models/marts/ledger_current_state/accounts_current.yml | 2 +- .../marts/ledger_current_state/claimable_balances_current.yml | 2 +- models/marts/ledger_current_state/contract_code_current.yml | 2 +- models/marts/ledger_current_state/contract_data_current.yml | 2 +- models/marts/ledger_current_state/liquidity_pools_current.yml | 2 +- models/marts/ledger_current_state/offers_current.yml | 2 +- models/marts/ledger_current_state/trust_lines_current.yml | 2 +- models/marts/ledger_current_state/ttl_current.yml | 2 +- models/marts/trade_agg.yml | 2 +- 15 files changed, 14 insertions(+), 15 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 8a54c3b..68b782f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -30,7 +30,6 @@ vars: airflow_start_timestamp: "{{ env_var('AIRFLOW_START_TIMESTAMP', '2000-01-01') }}" is_singular_airflow_task: "{{ env_var('IS_SINGULAR_AIRFLOW_TASK', 'false') }}" - is_recency_airflow_task: "{{ env_var('IS_RECENCY_AIRFLOW_TASK', 'false') }}" # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models diff --git a/models/marts/enriched_history/enriched_history_operations.yml b/models/marts/enriched_history/enriched_history_operations.yml index 480ff7e..759a381 100644 --- a/models/marts/enriched_history/enriched_history_operations.yml +++ b/models/marts/enriched_history/enriched_history_operations.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/enriched_history/enriched_history_operations_soroban.yml b/models/marts/enriched_history/enriched_history_operations_soroban.yml index 159183b..e5246f5 100644 --- a/models/marts/enriched_history/enriched_history_operations_soroban.yml +++ b/models/marts/enriched_history/enriched_history_operations_soroban.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/fee_stats_agg.yml b/models/marts/fee_stats_agg.yml index 937a69d..cae4cd2 100644 --- a/models/marts/fee_stats_agg.yml +++ b/models/marts/fee_stats_agg.yml @@ -10,7 +10,7 @@ models: field: cast(day_agg as timestamp) interval: 2 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/history_assets.yml b/models/marts/history_assets.yml index 84e0067..29d66f4 100644 --- a/models/marts/history_assets.yml +++ b/models/marts/history_assets.yml @@ -10,7 +10,7 @@ models: field: cast(batch_run_date as timestamp) interval: 2 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/account_signers_current.yml b/models/marts/ledger_current_state/account_signers_current.yml index 7a689f3..52d6a96 100644 --- a/models/marts/ledger_current_state/account_signers_current.yml +++ b/models/marts/ledger_current_state/account_signers_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/accounts_current.yml b/models/marts/ledger_current_state/accounts_current.yml index ac5c7f9..30763b7 100644 --- a/models/marts/ledger_current_state/accounts_current.yml +++ b/models/marts/ledger_current_state/accounts_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/claimable_balances_current.yml b/models/marts/ledger_current_state/claimable_balances_current.yml index 2e7a069..52e73d2 100644 --- a/models/marts/ledger_current_state/claimable_balances_current.yml +++ b/models/marts/ledger_current_state/claimable_balances_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/contract_code_current.yml b/models/marts/ledger_current_state/contract_code_current.yml index 975a4f1..d461e99 100644 --- a/models/marts/ledger_current_state/contract_code_current.yml +++ b/models/marts/ledger_current_state/contract_code_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 7 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/contract_data_current.yml b/models/marts/ledger_current_state/contract_data_current.yml index 1ed7bd3..b96c9cd 100644 --- a/models/marts/ledger_current_state/contract_data_current.yml +++ b/models/marts/ledger_current_state/contract_data_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/liquidity_pools_current.yml b/models/marts/ledger_current_state/liquidity_pools_current.yml index 3f6b801..f790a8f 100644 --- a/models/marts/ledger_current_state/liquidity_pools_current.yml +++ b/models/marts/ledger_current_state/liquidity_pools_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 24 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/offers_current.yml b/models/marts/ledger_current_state/offers_current.yml index ffa3b29..2e745df 100644 --- a/models/marts/ledger_current_state/offers_current.yml +++ b/models/marts/ledger_current_state/offers_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/trust_lines_current.yml b/models/marts/ledger_current_state/trust_lines_current.yml index e584d35..98e0fc6 100644 --- a/models/marts/ledger_current_state/trust_lines_current.yml +++ b/models/marts/ledger_current_state/trust_lines_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/ledger_current_state/ttl_current.yml b/models/marts/ledger_current_state/ttl_current.yml index fc2551c..79ac50b 100644 --- a/models/marts/ledger_current_state/ttl_current.yml +++ b/models/marts/ledger_current_state/ttl_current.yml @@ -10,7 +10,7 @@ models: field: closed_at interval: 12 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." diff --git a/models/marts/trade_agg.yml b/models/marts/trade_agg.yml index 3f7123f..8fa784a 100644 --- a/models/marts/trade_agg.yml +++ b/models/marts/trade_agg.yml @@ -10,7 +10,7 @@ models: field: cast(day_agg as timestamp) interval: 2 config: - enabled: '{{ target.name == "ci" or var("is_recency_airflow_task") == "true" }}' + enabled: '{{ target.name == "ci" or env_var("IS_RECENCY_AIRFLOW_TASK") == "true" }}' severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." From 7002a7ba41a887cd88f4764ec09c2a26ef5e6595 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 25 Sep 2024 00:51:21 +0530 Subject: [PATCH 5/5] Add env variable in ci --- .github/workflows/project_evaluator_ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/project_evaluator_ci.yml b/.github/workflows/project_evaluator_ci.yml index 7e09370..281ae37 100644 --- a/.github/workflows/project_evaluator_ci.yml +++ b/.github/workflows/project_evaluator_ci.yml @@ -15,6 +15,7 @@ env: DBT_JOB_TIMEOUT: 300 DBT_THREADS: 1 DBT_JOB_RETRIES: 1 + IS_RECENCY_AIRFLOW_TASK: "false" jobs: dbt-project-evaluator: