From c3e6d3f448473af99e0e0f274ca9823972d51bee Mon Sep 17 00:00:00 2001 From: Eduardo Nercolini <161879487+eduardo-nercolini@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:31:30 -0300 Subject: [PATCH] dbt source freshness applied to public sources (#51) * dbt source freshness applied to public sources * src_config_settings excluded from freshness tests * Re-added contract_data and fix ledger_closed_at for history_trades --- models/sources/src_accounts.yml | 13 ++++--------- models/sources/src_accounts_signers.yml | 13 ++++--------- models/sources/src_claimable_balance.yml | 13 ++++--------- models/sources/src_contract_code.yml | 9 --------- models/sources/src_contract_data.yml | 13 ++++--------- models/sources/src_history_assets.yml | 13 ++++--------- models/sources/src_history_effects.yml | 13 ++++--------- models/sources/src_history_ledgers.yml | 13 ++++--------- models/sources/src_history_operations.yml | 13 ++++--------- models/sources/src_history_trades.yml | 13 ++++--------- models/sources/src_history_transactions.yml | 13 ++++--------- models/sources/src_liquidity_pools.yml | 13 ++++--------- models/sources/src_offers.yml | 13 ++++--------- models/sources/src_trust_lines.yml | 13 ++++--------- models/sources/src_ttl.yml | 13 ++++--------- 15 files changed, 56 insertions(+), 135 deletions(-) diff --git a/models/sources/src_accounts.yml b/models/sources/src_accounts.yml index cd2206e..95304f8 100644 --- a/models/sources/src_accounts.yml +++ b/models/sources/src_accounts.yml @@ -6,16 +6,11 @@ sources: tables: - name: accounts description: '{{ doc("accounts") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: closed_at tests: - - dbt_utils.recency: - datepart: hour - field: closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: combination_of_columns: - account_id diff --git a/models/sources/src_accounts_signers.yml b/models/sources/src_accounts_signers.yml index 66e4fb3..f4772be 100644 --- a/models/sources/src_accounts_signers.yml +++ b/models/sources/src_accounts_signers.yml @@ -6,16 +6,11 @@ sources: tables: - name: account_signers description: '{{ doc("accounts_signers") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: closed_at tests: - - dbt_utils.recency: - datepart: hour - field: closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: combination_of_columns: - account_id diff --git a/models/sources/src_claimable_balance.yml b/models/sources/src_claimable_balance.yml index 8986229..04313a2 100644 --- a/models/sources/src_claimable_balance.yml +++ b/models/sources/src_claimable_balance.yml @@ -6,16 +6,11 @@ sources: tables: - name: claimable_balances description: '{{ doc("claimable_balances") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: closed_at tests: - - dbt_utils.recency: - datepart: hour - field: closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: combination_of_columns: - balance_id diff --git a/models/sources/src_contract_code.yml b/models/sources/src_contract_code.yml index e40499c..6dd1518 100644 --- a/models/sources/src_contract_code.yml +++ b/models/sources/src_contract_code.yml @@ -7,15 +7,6 @@ sources: - name: contract_code description: '{{ doc("contract_code") }}' tests: - - dbt_utils.recency: - datepart: day - field: closed_at - interval: 3 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: combination_of_columns: - contract_code_hash diff --git a/models/sources/src_contract_data.yml b/models/sources/src_contract_data.yml index 855a5fa..9eef7a6 100644 --- a/models/sources/src_contract_data.yml +++ b/models/sources/src_contract_data.yml @@ -6,16 +6,11 @@ sources: tables: - name: contract_data description: '{{ doc("contract_data") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: closed_at tests: - - dbt_utils.recency: - datepart: hour - field: closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." columns: - name: contract_id description: '{{ doc("contract_id") }}' diff --git a/models/sources/src_history_assets.yml b/models/sources/src_history_assets.yml index e5709d2..7884040 100644 --- a/models/sources/src_history_assets.yml +++ b/models/sources/src_history_assets.yml @@ -6,16 +6,11 @@ sources: tables: - name: history_assets_staging description: '{{ doc("history_assets_staging") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: batch_run_date tests: - - dbt_utils.recency: - datepart: hour - field: cast(batch_run_date as timestamp) - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." columns: - name: asset_id description: '{{ doc("asset_id") }}' diff --git a/models/sources/src_history_effects.yml b/models/sources/src_history_effects.yml index 9e82c1b..5ac51f2 100644 --- a/models/sources/src_history_effects.yml +++ b/models/sources/src_history_effects.yml @@ -6,16 +6,11 @@ sources: tables: - name: history_effects description: '{{ doc("history_effects") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: closed_at tests: - - dbt_utils.recency: - datepart: hour - field: closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." columns: - name: address description: '{{ doc("address") }}' diff --git a/models/sources/src_history_ledgers.yml b/models/sources/src_history_ledgers.yml index 60048d2..730d538 100644 --- a/models/sources/src_history_ledgers.yml +++ b/models/sources/src_history_ledgers.yml @@ -6,16 +6,11 @@ sources: tables: - name: history_ledgers description: '{{ doc("history_ledgers") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: closed_at tests: - - dbt_utils.recency: - datepart: hour - field: closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." columns: - name: sequence description: '{{ doc("sequence") }}' diff --git a/models/sources/src_history_operations.yml b/models/sources/src_history_operations.yml index f24c682..4988f44 100644 --- a/models/sources/src_history_operations.yml +++ b/models/sources/src_history_operations.yml @@ -6,16 +6,11 @@ sources: tables: - name: history_operations description: '{{ doc("history_operations") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: closed_at tests: - - dbt_utils.recency: - datepart: hour - field: closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." columns: - name: id description: '{{ doc("operation_id") }}' diff --git a/models/sources/src_history_trades.yml b/models/sources/src_history_trades.yml index 3296da0..d186c98 100644 --- a/models/sources/src_history_trades.yml +++ b/models/sources/src_history_trades.yml @@ -6,16 +6,11 @@ sources: tables: - name: history_trades description: '{{ doc("history_trades") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: ledger_closed_at tests: - - dbt_utils.recency: - datepart: hour - field: ledger_closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: combination_of_columns: - history_operation_id diff --git a/models/sources/src_history_transactions.yml b/models/sources/src_history_transactions.yml index 4466005..6727ac2 100644 --- a/models/sources/src_history_transactions.yml +++ b/models/sources/src_history_transactions.yml @@ -6,16 +6,11 @@ sources: tables: - name: history_transactions description: '{{ doc("history_transactions") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: closed_at tests: - - dbt_utils.recency: - datepart: hour - field: closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." columns: - name: id description: '{{ doc("transaction_id") }}' diff --git a/models/sources/src_liquidity_pools.yml b/models/sources/src_liquidity_pools.yml index 0520bc3..1980be1 100644 --- a/models/sources/src_liquidity_pools.yml +++ b/models/sources/src_liquidity_pools.yml @@ -6,16 +6,11 @@ sources: tables: - name: liquidity_pools description: '{{ doc("liquidity_pools") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: closed_at tests: - - dbt_utils.recency: - datepart: hour - field: closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: combination_of_columns: - liquidity_pool_id diff --git a/models/sources/src_offers.yml b/models/sources/src_offers.yml index b6db3fc..b43e31d 100644 --- a/models/sources/src_offers.yml +++ b/models/sources/src_offers.yml @@ -6,16 +6,11 @@ sources: tables: - name: offers description: '{{ doc("liquidity_pools") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: closed_at tests: - - dbt_utils.recency: - datepart: hour - field: closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: combination_of_columns: - offer_id diff --git a/models/sources/src_trust_lines.yml b/models/sources/src_trust_lines.yml index 4a72512..34b4614 100644 --- a/models/sources/src_trust_lines.yml +++ b/models/sources/src_trust_lines.yml @@ -6,16 +6,11 @@ sources: tables: - name: trust_lines description: '{{ doc("trust_lines") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: closed_at tests: - - dbt_utils.recency: - datepart: hour - field: closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: combination_of_columns: - account_id diff --git a/models/sources/src_ttl.yml b/models/sources/src_ttl.yml index 21a0c26..44b6224 100644 --- a/models/sources/src_ttl.yml +++ b/models/sources/src_ttl.yml @@ -6,16 +6,11 @@ sources: tables: - name: ttl description: '{{ doc("ttl") }}' + freshness: + warn_after: {count: 30, period: minute} + error_after: {count: 60, period: minute} + loaded_at_field: closed_at tests: - - dbt_utils.recency: - datepart: hour - field: closed_at - interval: 12 - config: - severity: warn - meta: - description: - "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: combination_of_columns: - key_hash