Merge branch 'master' into patch/tag-recency-tests

stellar · Sep 24, 2024 · cd26fd8 · cd26fd8
2 parents 9de1ecb + e77be26
commit cd26fd8
Show file tree

Hide file tree

Showing 21 changed files with 165 additions and 83 deletions.
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -65,6 +65,11 @@ models:
       tests:
         fct_missing_primary_key_tests:
           +enabled: false
+        fct_sources_without_freshness:
+          +enabled: false
+      documentation:
+        fct_undocumented_sources:
+          +enabled: false
 
 seeds:
   stellar_dbt_public:

diff --git a/models/marts/enriched_history/enriched_history_operations.sql b/models/marts/enriched_history/enriched_history_operations.sql
@@ -1,12 +1,17 @@
-{{ config(
-    tags = ["enriched_history_operations"],
-    materialized='incremental',
-    unique_key=["op_id"],
-    partition_by={
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": ["op_id"],
+    "cluster_by": ["ledger_sequence","transaction_id","op_account_id","type"],
+    "partition_by": {
         "field": "closed_at"
         , "data_type": "timestamp"
         , "granularity": "month"},
-    cluster_by=["ledger_sequence","transaction_id","op_account_id","type"]
+    "tags": ["enriched_history_operations"]
+} %}
+
+{{ config(
+    meta=meta_config,
+    **meta_config,
     )
 }}
 

diff --git a/models/marts/enriched_history/enriched_history_operations_soroban.sql b/models/marts/enriched_history/enriched_history_operations_soroban.sql
@@ -1,12 +1,17 @@
-{{ config(
-    tags = ["enriched_history_operations"],
-    materialized='incremental',
-    unique_key=["op_id"],
-    partition_by={
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": ["op_id"],
+    "cluster_by": ["ledger_sequence", "transaction_id", "op_type"],
+    "partition_by": {
         "field": "closed_at"
         , "data_type": "timestamp"
         , "granularity": "month"},
-    cluster_by=["ledger_sequence", "transaction_id", "op_type"]
+    "tags": ["enriched_history_operations"]
+} %}
+
+{{ config(
+    meta=meta_config,
+    **meta_config,
     )
 }}
 

diff --git a/models/marts/fee_stats_agg.sql b/models/marts/fee_stats_agg.sql
@@ -1,11 +1,16 @@
-{{ config(
-    tags=["fee_stats"],
-    materialized='incremental',
-    unique_key=["day_agg"],
-    partition_by={
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": ["day_agg"],
+    "tags": ["fee_stats"],
+    "partition_by": {
         "field": "day_agg"
         , "data_type": "date"
         , "granularity": "month"}
+} %}
+
+{{ config(
+    meta=meta_config,
+    **meta_config,
     )
 }}
 

diff --git a/models/marts/history_assets.sql b/models/marts/history_assets.sql
@@ -1,8 +1,13 @@
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": ["asset_id"],
+    "cluster_by": ["asset_id"],
+    "tags": ["history_assets"]
+} %}
+
 {{ config(
-    tags = ["history_assets"]
-    , materialized='incremental'
-    , unique_key=["asset_id"]
-    , cluster_by= ["asset_id"]
+    meta=meta_config,
+    **meta_config,
     )
 }}
 

diff --git a/models/marts/ledger_current_state/account_signers_current.sql b/models/marts/ledger_current_state/account_signers_current.sql
@@ -1,9 +1,13 @@
-{{
-    config(
-        tags = ["current_state"],
-        materialized = "incremental",
-        unique_key = "unique_id",
-        cluster_by = "account_id"
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": "unique_id",
+    "cluster_by": "account_id",
+    "tags": ["current_state"]
+} %}
+
+{{ config(
+    meta=meta_config,
+    **meta_config,
     )
 }}
 

diff --git a/models/marts/ledger_current_state/accounts_current.sql b/models/marts/ledger_current_state/accounts_current.sql
@@ -1,10 +1,15 @@
-{{
-    config(
-        tags = ["current_state"],
-        materialized = "incremental",
-        unique_key = "account_id"
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": "account_id",
+    "tags": ["current_state"]
+} %}
+
+{{ config(
+    meta=meta_config,
+    **meta_config,
     )
 }}
+
 /* Returns the latest state of each account in the `accounts` table.
    Table includes all accounts. (Deleted and Existing).
 

diff --git a/models/marts/ledger_current_state/claimable_balances_current.sql b/models/marts/ledger_current_state/claimable_balances_current.sql
@@ -1,10 +1,15 @@
-{{
-    config(
-        tags = ["current_state"],
-        materialized = "incremental",
-        unique_key = "balance_id"
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": "balance_id",
+    "tags": ["current_state"]
+} %}
+
+{{ config(
+    meta=meta_config,
+    **meta_config,
     )
 }}
+
 /* Returns the latest state of each claimable balance in the `claimable_balances` table.
 
     Rank all rows for a claimable balance by closed_at timestamp and pick the latest one.*/

diff --git a/models/marts/ledger_current_state/config_settings_current.sql b/models/marts/ledger_current_state/config_settings_current.sql
@@ -1,7 +1,12 @@
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": "config_setting_id",
+    "tags": ["current_state"]
+} %}
+
 {{ config(
-    tags = ["current_state"]
-    , materialized='incremental'
-    , unique_key=["config_setting_id"]
+    meta=meta_config,
+    **meta_config,
     )
 }}
 

diff --git a/models/marts/ledger_current_state/config_settings_current.yml b/models/marts/ledger_current_state/config_settings_current.yml
@@ -3,7 +3,6 @@ version: 2
 models:
   - name: config_settings_current
     description: '{{ doc("config_settings_current") }}'
-
     columns:
       - name: config_setting_id
         description: '{{ doc("config_setting_id") }}'

diff --git a/models/marts/ledger_current_state/contract_code_current.sql b/models/marts/ledger_current_state/contract_code_current.sql
@@ -1,8 +1,13 @@
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": ["contract_code_hash"],
+    "cluster_by": ["contract_code_hash"],
+    "tags": ["current_state"]
+} %}
+
 {{ config(
-    tags = ["current_state"]
-    , materialized='incremental'
-    , unique_key=["contract_code_hash"]
-    , cluster_by= ["contract_code_hash"]
+    meta=meta_config,
+    **meta_config,
     )
 }}
 

diff --git a/models/marts/ledger_current_state/contract_code_current.yml b/models/marts/ledger_current_state/contract_code_current.yml
@@ -3,7 +3,15 @@ version: 2
 models:
   - name: contract_code_current
     description: '{{ doc("contract_code_current") }}'
-
+    tests:
+      - dbt_utils.recency:
+          datepart: day
+          field: closed_at
+          interval: 7
+          config:
+            severity: '{{ "error" if target.name == "prod" else "warn" }}'
+          meta:
+            description: "Monitors the freshness of your table over time, as the expected time between data updates."
     columns:
       - name: contract_code_hash
         description: '{{ doc("key_hash") }}'

diff --git a/models/marts/ledger_current_state/contract_data_current.sql b/models/marts/ledger_current_state/contract_data_current.sql
@@ -1,8 +1,13 @@
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": ["unique_id"],
+    "cluster_by": ["contract_id"],
+    "tags": ["current_state"]
+} %}
+
 {{ config(
-    tags = ["current_state"]
-    , materialized='incremental'
-    , unique_key=["unique_id"]
-    , cluster_by= ["contract_id"]
+    meta=meta_config,
+    **meta_config,
     )
 }}
 

diff --git a/models/marts/ledger_current_state/liquidity_pools_current.sql b/models/marts/ledger_current_state/liquidity_pools_current.sql
@@ -1,9 +1,13 @@
-{{
-    config(
-        tags = ["current_state"],
-        materialized = "incremental",
-        unique_key = "liquidity_pool_id",
-        cluster_by = ["asset_a_code", "asset_a_issuer", "asset_b_code", "asset_b_issuer"]
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": "liquidity_pool_id",
+    "cluster_by": ["asset_a_code", "asset_a_issuer", "asset_b_code", "asset_b_issuer"],
+    "tags": ["current_state"]
+} %}
+
+{{ config(
+    meta=meta_config,
+    **meta_config,
     )
 }}
 

diff --git a/models/marts/ledger_current_state/liquidity_pools_current.yml b/models/marts/ledger_current_state/liquidity_pools_current.yml
@@ -8,7 +8,7 @@ models:
           tags: [recency]
           datepart: hour
           field: closed_at
-          interval: 12
+          interval: 24
           config:
             severity: '{{ "error" if target.name == "prod" else "warn" }}'
           meta:

diff --git a/models/marts/ledger_current_state/offers_current.sql b/models/marts/ledger_current_state/offers_current.sql
@@ -1,13 +1,16 @@
-{{
-    config(
-        tags = ["current_state"],
-        materialized = "incremental",
-        unique_key = "offer_id",
-        cluster_by = ["selling_asset_code", "selling_asset_issuer", "buying_asset_code", "buying_asset_issuer"]
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": "offer_id",
+    "cluster_by": ["selling_asset_code", "selling_asset_issuer", "buying_asset_code", "buying_asset_issuer"],
+    "tags": ["current_state"]
+} %}
+
+{{ config(
+    meta=meta_config,
+    **meta_config,
     )
 }}
 
-
 with
     current_offers as (
         select

diff --git a/models/marts/ledger_current_state/trust_lines_current.sql b/models/marts/ledger_current_state/trust_lines_current.sql
@@ -1,9 +1,13 @@
-{{
-    config(
-        tags = ["current_state"],
-        materialized = "incremental",
-        unique_key = "unique_id",
-        cluster_by = ["asset_code", "asset_issuer"]
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": "unique_id",
+    "cluster_by": ["asset_code", "asset_issuer"],
+    "tags": ["current_state"]
+} %}
+
+{{ config(
+    meta=meta_config,
+    **meta_config,
     )
 }}
 

diff --git a/models/marts/ledger_current_state/ttl_current.sql b/models/marts/ledger_current_state/ttl_current.sql
@@ -1,8 +1,13 @@
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": ["key_hash"],
+    "cluster_by": ["key_hash"],
+    "tags": ["current_state"]
+} %}
+
 {{ config(
-    tags = ["current_state"]
-    , materialized='incremental'
-    , unique_key=["key_hash"]
-    , cluster_by= ["key_hash"]
+    meta=meta_config,
+    **meta_config,
     )
 }}
 

diff --git a/models/marts/trade_agg.sql b/models/marts/trade_agg.sql
@@ -1,12 +1,17 @@
-{{ config(
-    tags = ["trade_agg"],
-    materialized = 'incremental',
-    partition_by = {
+{% set meta_config = {
+    "materialized": "incremental",
+    "unique_key": ["day_agg", "asset_a", "asset_b"],
+    "cluster_by": ["asset_a", "asset_b"],
+    "partition_by": {
         "field": "day_agg"
         , "data_type": "date"
-        , "granularity": "month"}
-    , unique_key = ["day_agg", "asset_a", "asset_b"]
-    , cluster_by =["asset_a", "asset_b"]
+        , "granularity": "month"},
+    "tags": ["trade_agg"]
+} %}
+
+{{ config(
+    meta=meta_config,
+    **meta_config,
     )
 }}
 

diff --git a/packages.yml b/packages.yml
@@ -11,4 +11,4 @@ packages:
 
     ## https://dbt-labs.github.io/dbt-project-evaluator/latest/
   - package: dbt-labs/dbt_project_evaluator
-    version: 0.6.2
+    version: 0.14.2
diff --git a/tests/sorobon_surge_pricing_check.sql b/tests/sorobon_surge_pricing_check.sql
@@ -4,7 +4,7 @@
     severity="warn"
     , tags=["singular_test"]
     , meta={"alert_suppression_interval": 24}
-    , enabled=(target.name == "prod" and var("is_singular_airflow_task") == "true")
+    , enabled=false
     , alert_suppression_interval=24
     )
 }}