diff --git a/dbt_project.yml b/dbt_project.yml index bcf674a..1a3e315 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,4 +1,3 @@ - # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models @@ -39,3 +38,8 @@ models: +materialized: view marts: materialized: table + + elementary: + +schema: elementary + +docs: + show: false \ No newline at end of file diff --git a/models/docs/marts/trade_agg.md b/models/docs/marts/trade_agg.md index e9f2097..1d0dfbe 100644 --- a/models/docs/marts/trade_agg.md +++ b/models/docs/marts/trade_agg.md @@ -12,7 +12,13 @@ Date from which all metrics are aggregated. In the Trade Aggregations table, the {% enddocs %} -{% docs selling_asset_type %} +{% docs asset_a %} + +Hashed id for the selling_asset. + +{% enddocs %} + +{% docs asset_a_type %} The identifier for type of asset code used for the sold asset within the trade Notes: @@ -21,7 +27,7 @@ XLM is the native asset to the network. XLM has no asset code or issuer represen {% enddocs %} -{% docs selling_asset_code %} +{% docs asset_a_code %} The 4 or 12 character code of the sold asset within a trade Notes: @@ -30,13 +36,19 @@ Asset codes have no guarantees of uniqueness. The combination of asset code, iss {% enddocs %} -{% docs selling_asset_issuer %} +{% docs asset_a_issuer %} The account address of the original asset issuer for the sold asset within a trade {% enddocs %} -{% docs buying_asset_type %} +{% docs asset_b %} + +Hashed id for the buying_asset. + +{% enddocs %} + +{% docs asset_b_type %} The identifier for type of asset code used for the bought asset within the trade Notes: @@ -45,7 +57,7 @@ XLM is the native asset to the network. XLM has no asset code or issuer represen {% enddocs %} -{% docs buying_asset_code %} +{% docs asset_b_code %} The 4 or 12 character code of the bought asset within a trade Notes: @@ -54,7 +66,7 @@ Asset codes have no guarantees of uniqueness. The combination of asset code, iss {% enddocs %} -{% docs buying_asset_issuer %} +{% docs asset_b_issuer %} The account address of the original asset issuer for the bought asset within a trade @@ -66,19 +78,19 @@ The count of trades executed against the network in a day. {% enddocs %} -{% docs selling_volume_daily %} +{% docs asset_a_volume_daily %} The total raw amount of the asset being sold in all trades within that day. {% enddocs %} -{% docs buying_volume_daily %} +{% docs asset_b_volume_daily %} The total raw amount of the asset being bought in all trades within that day. {% enddocs %} -{% docs avg_daily %} +{% docs avg_price_daily %} The total amount of the asset being bought in all trades divided by the total amount of the asset being sold in all trades, within that day. @@ -126,19 +138,19 @@ The count of trades executed against the network during the past 7 days. {% enddocs %} -{% docs selling_volume_weekly %} +{% docs asset_a_volume_weekly %} The total amount of the asset being sold in all trades during the past 7 days. {% enddocs %} -{% docs buying_volume_weekly %} +{% docs asset_b_volume_weekly %} The total amount of the asset being bought in all trades during the past 7 days. {% enddocs %} -{% docs avg_weekly %} +{% docs avg_price_weekly %} The total amount of the asset being bought in all trades divided by the total amount of the asset being sold in all trades, during the past 7 days. @@ -186,19 +198,19 @@ The count of trades executed against the network during the past 30 days. {% enddocs %} -{% docs selling_volume_monthly %} +{% docs asset_a_volume_monthly %} The total amount of the asset being sold in all trades during the past 30 days. {% enddocs %} -{% docs buying_volume_monthly %} +{% docs asset_b_volume_monthly %} The total amount of the asset being bought in all trades during the past 30 days. {% enddocs %} -{% docs avg_monthly %} +{% docs avg_price_monthly %} The total amount of the asset being bought in all trades divided by the total amount of the asset being sold in all trades, during the past 30 days. @@ -246,19 +258,19 @@ The count of trades executed against the network during the past 365 days. {% enddocs %} -{% docs selling_volume_yearly %} +{% docs asset_a_volume_yearly %} The total amount of the asset being sold in all trades during the past 365 days. {% enddocs %} -{% docs buying_volume_yearly %} +{% docs asset_b_volume_yearly %} The total amount of the asset being bought in all trades during the past 365 days. {% enddocs %} -{% docs avg_yearly%} +{% docs avg_price_yearly%} The total amount of the asset being bought in all trades divided by the total amount of the asset being sold in all trades, during the past 365 days. diff --git a/models/docs/sources/liquidity_pool.md b/models/docs/sources/liquidity_pool.md index 5f70eac..b162487 100644 --- a/models/docs/sources/liquidity_pool.md +++ b/models/docs/sources/liquidity_pool.md @@ -62,18 +62,9 @@ The total number of pool shares is calculated by a constant product formula and Shares are not transferable; the only way to increase the number of pool shares held is to deposit into a liquidity pool. Conversely, decreasing pools shares can only be accomplished through a withdraw operation. Shares cannot be sent in payments or sold using offers. {% enddocs %} -{% docs asset_a_type %} -The identifier for type of asset code, can be a alphanumeric with 4 characters, 12 characters or the native asset to the network, XLM. +{% docs asset_a_id %} +The Farm Hash encoding of the Asset Code + Asset Issuer and Asset Type for one of the asset pairs in a liquidity pool. -- Required Field -{% enddocs %} - -{% docs asset_a_code %} -The 4 or 12 character code representation of the asset of one of the two asset pairs in a liquidity pool -{% enddocs %} - -{% docs asset_a_issuer%} -The account address of the original asset issuer that created one of the two asset pairs in the liquidity pool {% enddocs %} {% docs asset_a_amount %} @@ -82,18 +73,9 @@ The raw number of tokens locked in the pool for one of the two asset pairs in th - Required Field {% enddocs %} -{% docs asset_b_type %} -The identifier for type of asset code, can be a alphanumeric with 4 characters, 12 characters or the native asset to the network, XLM. - -- Required Field -{% enddocs %} - -{% docs asset_b_code %} -The 4 or 12 character code representation of the asset of one of the two asset pairs in a liquidity pool -{% enddocs %} +{% docs asset_b_id %} +The Farm Hash encoding of the Asset Code + Asset Issuer and Asset Type for the corresponding asset in the liquidity pool. -{% docs asset_b_issuer %} -The account address of the original asset issuer that created one of the two asset pairs in the liquidity pool {% enddocs %} {% docs asset_b_amount %} diff --git a/models/docs/universal.md b/models/docs/universal.md index e1445d6..5e47bdc 100644 --- a/models/docs/universal.md +++ b/models/docs/universal.md @@ -9,6 +9,11 @@ The identifier for type of asset code, can be an alphanumeric with 4 characters, XLM is the native asset to the network. XLM has no asset code or issuer representation and will instead be displayed with an asset type of 'native' {% enddocs %} +{% docs asset_id %} +The Farm Hash encoding of Asset Code + Asset Issuer + Asset Type. This field is optimized for cross table joins since integer joins are less expensive than the original asset id components. + +{% enddocs %} + {% docs asset_code %} The 4 or 12 character code representation of the asset on the network. diff --git a/models/marts/enriched_history_operations.yml b/models/marts/enriched_history_operations.yml index 03385c0..a1f86b2 100644 --- a/models/marts/enriched_history_operations.yml +++ b/models/marts/enriched_history_operations.yml @@ -3,17 +3,45 @@ version: 2 models: - name: enriched_history_operations description: Aggregate table for the history operations + tests: + - dbt_utils.recency: + datepart: hour + field: cast(closed_at as datetime) + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + - elementary.dimension_anomalies: + timestamp_column: closed_at + backfill_days: 90 + dimensions: + - "type" + time_bucket: + period: day + count: 1 + meta: + description: + "Monitors the frequency of values in the type column over time." columns: - name: op_id description: '{{ doc("operation_id") }}' tests: - - not_null - - unique + - unique: + config: + where: closed_at >= timestamp_trunc(timestamp_sub(current_timestamp(), interval 2 day), day) + and closed_at < timestamp_trunc(current_timestamp(), day) + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: op_source_account description: '{{ doc("source_account") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: op_source_account_muxed description: '{{ doc("source_account_muxed") }}' @@ -21,17 +49,30 @@ models: - name: transaction_id description: '{{ doc("transaction_id") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: type description: '{{ doc("type") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day + - dbt_utils.expression_is_true: + expression: '<= 26' + config: + where: closed_at > current_timestamp - interval 90 day + meta: + description: + "Test to check if the type of operation added is not greater than 26." - name: type_string description: '{{ doc("type_string") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: account description: '{{ doc("details_account") }}' @@ -336,35 +377,51 @@ models: - name: transaction_hash description: '{{ doc("transaction_hash") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: ledger_sequence description: '{{ doc("ledger_sequence") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: account_sequence description: '{{ doc("account_sequence") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: max_fee description: '{{ doc("max_fee") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: txn_operation_count description: '{{ doc("transaction_operation_count") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: txn_created_at description: '{{ doc("created_at") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: memo_type description: '{{ doc("memo_type") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: memo description: '{{ doc("memo") }}' @@ -374,9 +431,17 @@ models: - name: successful description: '{{ doc("successful") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: fee_charged description: '{{ doc("fee_charged") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: fee_account description: '{{ doc("fee_account") }}' @@ -420,78 +485,123 @@ models: - name: ledger_hash description: '{{ doc("ledger_hash") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: previous_ledger_hash description: '{{ doc("previous_ledger_hash") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: transaction_count description: '{{ doc("transaction_count") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: ledger_operation_count description: '{{ doc("operation_count") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: closed_at description: '{{ doc("closed_at") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: ledger_id description: '{{ doc("ledger_id") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: total_coins description: '{{ doc("total_coins") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: fee_pool description: '{{ doc("fee_pool") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: base_fee description: '{{ doc("base_fee") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: base_reserve description: '{{ doc("base_reserve") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: max_tx_set_size description: '{{ doc("max_tx_set_size") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: protocol_version description: '{{ doc("protocol_version") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - dbt_utils.expression_is_true: expression: '<= 19' + meta: + description: + "Test to check if the protocol version is not greater than 19." - - name: succesful_transaction_count + - name: successful_transaction_count description: '{{ doc("successful_transaction_count") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: failed_transaction_count description: '{{ doc("failed_transaction_count") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day diff --git a/models/marts/fee_stats_agg.yml b/models/marts/fee_stats_agg.yml index b28e21b..165e4cd 100644 --- a/models/marts/fee_stats_agg.yml +++ b/models/marts/fee_stats_agg.yml @@ -3,104 +3,248 @@ version: 2 models: - name: fee_stats_agg description: '{{ doc("fee_stats") }}' + tests: + - dbt_utils.recency: + datepart: day + field: day_agg + interval: 2 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." columns: - name: day_agg description: '{{ doc("fee_day_agg") }}' tests: - - not_null + - unique: + config: + where: day_agg >= datetime_trunc(datetime_sub(current_datetime(), interval 2 day), day) + and day_agg < datetime_trunc(current_datetime(), day) + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_p10 description: '{{ doc("fee_charged_p10") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_p20 description: '{{ doc("fee_charged_p20") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_p30 description: '{{ doc("fee_charged_p30") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_p40 description: '{{ doc("fee_charged_p40") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_p50 description: '{{ doc("fee_charged_p50") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_p60 description: '{{ doc("fee_charged_p60") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_p70 description: '{{ doc("fee_charged_p70") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_p80 description: '{{ doc("fee_charged_p80") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_p90 description: '{{ doc("fee_charged_p90") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_p95 description: '{{ doc("fee_charged_p95") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_p99 description: '{{ doc("fee_charged_p99") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_max description: '{{ doc("fee_charged_max") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_min description: '{{ doc("fee_charged_min") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: fee_charged_mode description: '{{ doc("fee_charged_mode") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_p10 description: '{{ doc("max_fee_p10") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_p20 description: '{{ doc("max_fee_p20") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_p30 description: '{{ doc("max_fee_p30") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_p40 description: '{{ doc("max_fee_p40") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_p50 description: '{{ doc("max_fee_p50") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_p60 description: '{{ doc("max_fee_p60") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_p70 description: '{{ doc("max_fee_p70") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_p80 description: '{{ doc("max_fee_p80") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_p90 description: '{{ doc("max_fee_p90") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_p95 description: '{{ doc("max_fee_p95") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_p99 description: '{{ doc("max_fee_p99") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_max description: '{{ doc("max_fee_max") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_min description: '{{ doc("max_fee_min") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_fee_mode description: '{{ doc("max_fee_mode") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: min_ledger_sequence description: '{{ doc("min_ledger_sequence") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: max_ledger_sequence description: '{{ doc("max_ledger_sequence") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: total_ledgers description: '{{ doc("total_ledgers") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: surge_price_pct - description: '{{ doc("surge_price_pct") }}' \ No newline at end of file + description: '{{ doc("surge_price_pct") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day \ No newline at end of file diff --git a/models/marts/history_assets.yml b/models/marts/history_assets.yml index 38ce609..6ea014f 100644 --- a/models/marts/history_assets.yml +++ b/models/marts/history_assets.yml @@ -3,34 +3,70 @@ version: 2 models: - name: history_assets description: '{{ doc("history_assets_staging") }}' + tests: + - dbt_utils.recency: + datepart: day + field: batch_run_date + interval: 2 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." columns: - name: asset_id description: '{{ doc("assets_id") }}' tests: - - not_null + - unique: + config: + where: batch_run_date >= datetime_trunc(datetime_sub(current_datetime(), interval 2 day), day) + and batch_run_date < datetime_trunc(current_datetime(), day) + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_type description: '{{ doc("asset_type") }}' tests: - - not_null + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - accepted_values: values: ['credit_alphanum12', 'credit_alphanum4', 'native'] - name: asset_code description: '{{ doc("asset_code") }}' tests: - - not_null + - not_null: + config: + where: asset_type != 'native' + and batch_run_date > current_datetime - interval 2 day - name: asset_issuer description: '{{ doc("asset_issuer") }}' tests: - - not_null + - not_null: + config: + where: asset_type != 'native' + and batch_run_date > current_datetime - interval 2 day - name: batch_id description: '{{ doc("batch_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_run_date description: '{{ doc("batch_run_date") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day diff --git a/models/marts/trade_agg.yml b/models/marts/trade_agg.yml index d5d3040..a4b5d6f 100644 --- a/models/marts/trade_agg.yml +++ b/models/marts/trade_agg.yml @@ -3,152 +3,364 @@ version: 2 models: - name: trade_agg description: '{{ doc("trade_agg") }}' + tests: + - dbt_utils.recency: + datepart: day + field: day_agg + interval: 2 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day_agg + - asset_a + - asset_b + config: + where: day_agg >= datetime_trunc(datetime_sub(current_datetime(), interval 2 day), day) + and day_agg < datetime_trunc(current_datetime(), day) columns: - name: day_agg description: '{{ doc("trade_day_agg") }}' tests: - - not_null + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: selling_asset_type - description: '{{ doc("selling_asset_type") }}' - - - name: selling_asset_code - description: '{{ doc("selling_asset_code") }}' - - - name: selling_asset_issuer - description: '{{ doc("selling_asset_issuer") }}' + - name: asset_a + description: '{{ doc("asset_a") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day + + - name: asset_a_type + description: '{{ doc("asset_a_type") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: selling_asset_id - description: '{{ doc("assets_id") }}' + - name: asset_a_code + description: '{{ doc("asset_a_code") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: buying_asset_type - description: '{{ doc("buying_asset_type") }}' + - name: asset_a_issuer + description: '{{ doc("asset_a_issuer") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day + + - name: asset_a_volume_daily + description: '{{ doc("asset_a_volume_daily") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day + + - name: asset_b + description: '{{ doc("asset_b") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: buying_asset_code - description: '{{ doc("buying_asset_code") }}' + - name: asset_b_type + description: '{{ doc("asset_a_type") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: buying_asset_issuer - description: '{{ doc("buying_asset_issuer") }}' + - name: asset_b_code + description: '{{ doc("asset_a_code") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: buying_asset_id - description: '{{ doc("assets_id") }}' + - name: asset_b_issuer + description: '{{ doc("asset_a_issuer") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day + + - name: asset_b_volume_daily + description: '{{ doc("asset_b_volume_daily") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: trade_count_daily description: '{{ doc("trade_count_daily") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: selling_volume_daily - description: '{{ doc("selling_volume_daily") }}' - - - name: buying_volume_daily - description: '{{ doc("buying_volume_daily") }}' - - - name: avg_daily - description: '{{ doc("avg_daily") }}' + - name: avg_price_daily + description: '{{ doc("avg_price_daily") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: high_price_daily description: '{{ doc("high_price_daily") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: low_price_daily description: '{{ doc("low_price_daily") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: open_n_daily description: '{{ doc("open_n_daily") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: open_d_daily description: '{{ doc("open_d_daily") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: close_n_daily description: '{{ doc("close_n_daily") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: close_d_daily description: '{{ doc("close_d_daily") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: trade_count_weekly description: '{{ doc("trade_count_weekly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: selling_volume_weekly - description: '{{ doc("selling_volume_weekly") }}' + - name: asset_a_volume_weekly + description: '{{ doc("asset_a_volume_weekly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: buying_volume_weekly - description: '{{ doc("buying_volume_weekly") }}' + - name: asset_b_volume_weekly + description: '{{ doc("asset_b_volume_weekly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: avg_weekly - description: '{{ doc("avg_weekly") }}' + - name: avg_price_weekly + description: '{{ doc("avg_price_weekly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: high_price_weekly description: '{{ doc("high_price_weekly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: low_price_weekly description: '{{ doc("low_price_weekly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: open_n_weekly description: '{{ doc("open_n_weekly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: open_d_weekly description: '{{ doc("open_d_weekly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: close_n_weekly description: '{{ doc("close_n_weekly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: close_d_weekly description: '{{ doc("close_d_weekly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: trade_count_monthly description: '{{ doc("trade_count_monthly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: selling_volume_monthly - description: '{{ doc("selling_volume_monthly") }}' + - name: asset_a_volume_monthly + description: '{{ doc("asset_a_volume_monthly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: buying_volume_monthly - description: '{{ doc("buying_volume_monthly") }}' + - name: asset_b_volume_monthly + description: '{{ doc("asset_b_volume_monthly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: avg_monthly - description: '{{ doc("avg_monthly") }}' + - name: avg_price_monthly + description: '{{ doc("avg_price_monthly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: high_price_monthly description: '{{ doc("high_price_monthly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: low_price_monthly description: '{{ doc("low_price_monthly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: open_n_monthly description: '{{ doc("open_n_monthly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: open_d_monthly description: '{{ doc("open_d_monthly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: close_n_monthly description: '{{ doc("close_n_monthly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: close_d_monthly description: '{{ doc("close_d_monthly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: trade_count_yearly description: '{{ doc("trade_count_yearly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: selling_volume_yearly - description: '{{ doc("selling_volume_yearly") }}' + - name: asset_a_volume_yearly + description: '{{ doc("asset_a_volume_yearly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: buying_volume_yearly - description: '{{ doc("buying_volume_yearly") }}' + - name: asset_b_volume_yearly + description: '{{ doc("asset_b_volume_yearly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - - name: avg_yearly - description: '{{ doc("avg_yearly") }}' + - name: avg_price_yearly + description: '{{ doc("avg_price_yearly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: high_price_yearly description: '{{ doc("high_price_yearly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: low_price_yearly description: '{{ doc("low_price_yearly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: open_n_yearly description: '{{ doc("open_n_yearly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: open_d_yearly description: '{{ doc("open_d_yearly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: close_n_yearly description: '{{ doc("close_n_yearly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day - name: close_d_yearly - description: '{{ doc("close_d_yearly") }}' \ No newline at end of file + description: '{{ doc("close_d_yearly") }}' + tests: + - not_null: + config: + where: day_agg > current_datetime - interval 2 day \ No newline at end of file diff --git a/models/sources/src_accounts.yml b/models/sources/src_accounts.yml index 3e6c6d8..6a8edbe 100644 --- a/models/sources/src_accounts.yml +++ b/models/sources/src_accounts.yml @@ -6,75 +6,198 @@ sources: tables: - name: accounts description: '{{ doc("accounts") }}' + tests: + - dbt_utils.recency: + datepart: hour + field: batch_run_date + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + - elementary.volume_anomalies: + timestamp_column: batch_run_date + backfill_days: 90 + time_bucket: + period: day + count: 1 + meta: + description: + "Monitors the row count of your table over time." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - account_id + - sequence_number + - ledger_entry_change + - last_modified_ledger + config: + where: batch_run_date >= datetime_trunc(datetime_sub(current_datetime(), interval 2 day), day) + and batch_run_date < datetime_trunc(current_datetime(), day) + meta: + description: + "Tests the uniqueness combination of: account_id, sequence_number, ledger_entry_change and last_modified_ledger." columns: - name: account_id description: '{{ doc("account_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: balance description: '{{ doc("balance") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: buying_liabilities description: '{{ doc("buying_liabilities") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: selling_liabilities description: '{{ doc("selling_liabilities") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: sequence_number description: '{{ doc("sequence_number") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: num_subentries description: '{{ doc("num_subentries") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: inflation_destination description: '{{ doc("inflation_destination") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: flags description: '{{ doc("flags") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: home_domain description: '{{ doc("home_domain") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: master_weight description: '{{ doc("master_weight") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: threshold_low description: '{{ doc("threshold_low") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: threshold_medium description: '{{ doc("threshold_medium") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: threshold_high description: '{{ doc("threshold_high") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: deleted description: '{{ doc("deleted") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: sponsor description: '{{ doc("sponsor") }}' - name: num_sponsored description: '{{ doc("num_sponsored") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: num_sponsoring description: '{{ doc("num_sponsoring") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: sequence_ledger description: '{{ doc("sequence_ledger") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: sequence_time description: '{{ doc("sequence_time") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_id description: '{{ doc("batch_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_run_date description: '{{ doc("batch_run_date") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_insert_ts - description: '{{ doc("batch_insert_ts") }}' \ No newline at end of file + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day \ No newline at end of file diff --git a/models/sources/src_accounts_signers.yml b/models/sources/src_accounts_signers.yml index abc70c6..427617c 100644 --- a/models/sources/src_accounts_signers.yml +++ b/models/sources/src_accounts_signers.yml @@ -6,27 +6,86 @@ sources: tables: - name: account_signers description: '{{ doc("accounts_signers") }}' + tests: + - dbt_utils.recency: + datepart: hour + field: batch_run_date + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + - elementary.volume_anomalies: + timestamp_column: batch_run_date + backfill_days: 90 + time_bucket: + period: day + count: 1 + meta: + description: + "Monitors the row count of your table over time." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - account_id + - signer + - ledger_entry_change + - last_modified_ledger + config: + where: batch_run_date >= datetime_trunc(datetime_sub(current_datetime(), interval 2 day), day) + and batch_run_date < datetime_trunc(current_datetime(), day) + meta: + description: + "Tests the uniqueness combination of: account_id, signer, ledger_entry_change and last_modified_ledger." columns: - name: account_id description: '{{ doc("account_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: signer description: '{{ doc("signer") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: weight description: '{{ doc("weight") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: sponsor description: '{{ doc("sponsor") }}' - name: deleted description: '{{ doc("deleted") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_id description: '{{ doc("batch_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_run_date description: '{{ doc("batch_run_date") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_insert_ts - description: '{{ doc("batch_insert_ts") }}' \ No newline at end of file + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day \ No newline at end of file diff --git a/models/sources/src_claimable_balance.yml b/models/sources/src_claimable_balance.yml index d758aa6..9c8f13d 100644 --- a/models/sources/src_claimable_balance.yml +++ b/models/sources/src_claimable_balance.yml @@ -6,9 +6,43 @@ sources: tables: - name: claimable_balances description: '{{ doc("claimable_balances") }}' + tests: + - dbt_utils.recency: + datepart: hour + field: batch_run_date + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + - elementary.volume_anomalies: + timestamp_column: batch_run_date + backfill_days: 90 + time_bucket: + period: day + count: 1 + meta: + description: + "Monitors the row count of your table over time." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - balance_id + - ledger_entry_change + config: + where: batch_run_date >= datetime_trunc(datetime_sub(current_datetime(), interval 2 day), day) + and batch_run_date < datetime_trunc(current_datetime(), day) + meta: + description: + "Tests the uniqueness combination of: balance_id and ledger_entry_change." + columns: - name: balance_id description: '{{ doc("balance_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: claimants description: '{{ doc("claimants") }}' @@ -33,36 +67,91 @@ sources: - name: asset_type description: '{{ doc("asset_type") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_code description: '{{ doc("asset_code") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_issuer description: '{{ doc("asset_issuer") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_amount description: '{{ doc("asset_amount") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: sponsor description: '{{ doc("sponsor") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: flags description: '{{ doc("flags") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: deleted description: '{{ doc("deleted") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_id description: '{{ doc("batch_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_run_date description: '{{ doc("batch_run_date") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_insert_ts - description: '{{ doc("batch_insert_ts") }}' \ No newline at end of file + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: asset_id + description: '{{ doc("asset_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day \ No newline at end of file diff --git a/models/sources/src_history_ledgers.yml b/models/sources/src_history_ledgers.yml index 01d571e..ca1f9f6 100644 --- a/models/sources/src_history_ledgers.yml +++ b/models/sources/src_history_ledgers.yml @@ -6,63 +6,166 @@ sources: tables: - name: history_ledgers description: '{{ doc("history_ledgers") }}' + tests: + - dbt_utils.recency: + datepart: hour + field: cast(closed_at as datetime) + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + - elementary.volume_anomalies: + timestamp_column: closed_at + backfill_days: 90 + time_bucket: + period: day + count: 1 + meta: + description: + "Monitors the row count of your table over time." columns: - name: sequence description: '{{ doc("sequence") }}' + tests: + - unique: + config: + where: closed_at >= timestamp_trunc(timestamp_sub(current_timestamp(), interval 2 day), day) + and closed_at < timestamp_trunc(current_timestamp(), day) + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: ledger_hash description: '{{ doc("ledger_hash") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: previous_ledger_hash description: '{{ doc("previous_ledger_hash") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: transaction_count description: '{{ doc("transaction_count") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: operation_count description: '{{ doc("operation_count") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: closed_at description: '{{ doc("closed_at") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: id description: '{{ doc("ledger_id") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: total_coins description: '{{ doc("total_coins") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: fee_pool description: '{{ doc("fee_pool") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: base_fee description: '{{ doc("base_fee") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: base_reserve description: '{{ doc("base_reserve") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: max_tx_set_size description: '{{ doc("max_tx_set_size") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: protocol_version description: '{{ doc("protocol_version") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: ledger_header description: '{{ doc("ledger_header") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: successful_transaction_count description: '{{ doc("successful_transaction_count") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: failed_transaction_count description: '{{ doc("failed_transaction_count") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: tx_set_operation_count description: '{{ doc("tx_set_operation_count") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: batch_id description: '{{ doc("batch_id") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: batch_run_date description: '{{ doc("batch_run_date") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day - name: batch_insert_ts - description: '{{ doc("batch_insert_ts") }}' \ No newline at end of file + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day \ No newline at end of file diff --git a/models/sources/src_history_operations.yml b/models/sources/src_history_operations.yml index 394dc3a..218106f 100644 --- a/models/sources/src_history_operations.yml +++ b/models/sources/src_history_operations.yml @@ -6,24 +6,67 @@ sources: tables: - name: history_operations description: '{{ doc("history_operations") }}' + tests: + - dbt_utils.recency: + datepart: hour + field: batch_run_date + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + - elementary.volume_anomalies: + timestamp_column: batch_run_date + backfill_days: 90 + time_bucket: + period: day + count: 1 + meta: + description: + "Monitors the row count of your table over time." columns: - name: id description: '{{ doc("operation_id") }}' + tests: + - unique: + config: + where: batch_run_date >= datetime_trunc(datetime_sub(current_datetime(), interval 2 day), day) + and batch_run_date < datetime_trunc(current_datetime(), day) + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: source_account description: '{{ doc("source_account") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: source_account_muxed description: '{{ doc("source_account_muxed") }}' - name: transaction_id description: '{{ doc("op_transaction_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: type description: '{{ doc("type") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: type_string description: '{{ doc("type_string") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: details description: '{{ doc("details") }}' @@ -330,9 +373,24 @@ sources: - name: batch_id description: '{{ doc("batch_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_run_date description: '{{ doc("batch_run_date") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_insert_ts - description: '{{ doc("batch_insert_ts") }}' \ No newline at end of file + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: closed_at + description: '{{ doc("closed_at") }}' diff --git a/models/sources/src_history_trades.yml b/models/sources/src_history_trades.yml index d031503..4d7ec32 100644 --- a/models/sources/src_history_trades.yml +++ b/models/sources/src_history_trades.yml @@ -6,15 +6,56 @@ sources: tables: - name: history_trades description: '{{ doc("history_trades") }}' + tests: + - dbt_utils.recency: + datepart: hour + field: cast(ledger_closed_at as datetime) + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + - elementary.volume_anomalies: + timestamp_column: ledger_closed_at + backfill_days: 90 + time_bucket: + period: day + count: 1 + meta: + description: + "Monitors the row count of your table over time." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - history_operation_id + - "`order`" + config: + where: ledger_closed_at >= timestamp_trunc(timestamp_sub(current_timestamp(), interval 2 day), day) + and ledger_closed_at < timestamp_trunc(current_timestamp(), day) + meta: + description: + "Tests the uniqueness combination of: history_operation_id and order." columns: - name: history_operation_id description: '{{ doc("history_operation_id") }}' + tests: + - not_null: + config: + where: ledger_closed_at > current_timestamp - interval 2 day - name: "`order`" description: '{{ doc("order") }}' + tests: + - not_null: + config: + where: ledger_closed_at > current_timestamp - interval 2 day - name: ledger_closed_at description: '{{ doc("ledger_closed_at") }}' + tests: + - not_null: + config: + where: ledger_closed_at > current_timestamp - interval 2 day - name: selling_account_address description: '{{ doc("selling_account_address") }}' @@ -30,6 +71,10 @@ sources: - name: selling_asset_id description: '{{ doc("assets_id") }}' + tests: + - not_null: + config: + where: ledger_closed_at > current_timestamp - interval 2 day - name: selling_amount description: '{{ doc("trade_selling_amount") }}' @@ -48,15 +93,27 @@ sources: - name: buying_asset_id description: '{{ doc("assets_id") }}' + tests: + - not_null: + config: + where: ledger_closed_at > current_timestamp - interval 2 day - name: buying_amount description: '{{ doc("trade_buying_amount") }}' - name: price_n description: '{{ doc("trade_price_n") }}' + tests: + - not_null: + config: + where: ledger_closed_at > current_timestamp - interval 2 day - name: price_d description: '{{ doc("trade_price_d") }}' + tests: + - not_null: + config: + where: ledger_closed_at > current_timestamp - interval 2 day - name: selling_offer_id description: '{{ doc("selling_offer_id") }}' @@ -81,9 +138,21 @@ sources: - name: batch_id description: '{{ doc("batch_id") }}' + tests: + - not_null: + config: + where: ledger_closed_at > current_timestamp - interval 2 day - name: batch_run_date description: '{{ doc("batch_run_date") }}' + tests: + - not_null: + config: + where: ledger_closed_at > current_timestamp - interval 2 day - name: batch_insert_ts - description: '{{ doc("batch_insert_ts") }}' \ No newline at end of file + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null: + config: + where: ledger_closed_at > current_timestamp - interval 2 day \ No newline at end of file diff --git a/models/sources/src_history_transactions.yml b/models/sources/src_history_transactions.yml index 0c4bc93..f58eb48 100644 --- a/models/sources/src_history_transactions.yml +++ b/models/sources/src_history_transactions.yml @@ -6,48 +6,119 @@ sources: tables: - name: history_transactions description: '{{ doc("history_transactions") }}' + tests: + - dbt_utils.recency: + datepart: hour + field: batch_run_date + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + - elementary.volume_anomalies: + timestamp_column: batch_run_date + backfill_days: 90 + time_bucket: + period: day + count: 1 + meta: + description: + "Monitors the row count of your table over time." columns: - name: id description: '{{ doc("transaction_id") }}' + tests: + - unique: + config: + where: batch_run_date >= datetime_trunc(datetime_sub(current_datetime(), interval 2 day), day) + and batch_run_date < datetime_trunc(current_datetime(), day) + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: transaction_hash description: '{{ doc("transaction_hash") }}' - + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + - name: ledger_sequence description: '{{ doc("ledger_sequence") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: application_order description: '{{ doc("application_order") }}' - name: account description: '{{ doc("account") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: account_sequence description: '{{ doc("account_sequence") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: max_fee description: '{{ doc("max_fee") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: operation_count description: '{{ doc("transaction_operation_count") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: created_at description: '{{ doc("created_at") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: memo_type description: '{{ doc("memo_type") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: memo description: '{{ doc("memo") }}' - name: time_bounds description: '{{ doc("time_bounds") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: successful description: '{{ doc("successful") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: fee_charged description: '{{ doc("fee_charged") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: inner_transaction_hash description: '{{ doc("inner_transaction_hash") }}' @@ -81,21 +152,56 @@ sources: - name: tx_envelope description: '{{ doc("tx_envelope") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: tx_result description: '{{ doc("tx_result") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: tx_meta description: '{{ doc("tx_meta") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: tx_fee_meta description: '{{ doc("tx_fee_meta") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_id description: '{{ doc("batch_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_run_date description: '{{ doc("batch_run_date") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_insert_ts - description: '{{ doc("batch_insert_ts") }}' \ No newline at end of file + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day diff --git a/models/sources/src_liquidity_pools.yml b/models/sources/src_liquidity_pools.yml index 70cb07c..22be3d6 100644 --- a/models/sources/src_liquidity_pools.yml +++ b/models/sources/src_liquidity_pools.yml @@ -6,60 +6,180 @@ sources: tables: - name: liquidity_pools description: '{{ doc("liquidity_pools") }}' + tests: + - dbt_utils.recency: + datepart: hour + field: batch_run_date + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + - elementary.volume_anomalies: + timestamp_column: batch_run_date + backfill_days: 90 + time_bucket: + period: day + count: 1 + meta: + description: + "Monitors the row count of your table over time." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - liquidity_pool_id + - ledger_entry_change + - last_modified_ledger + config: + where: batch_run_date >= datetime_trunc(datetime_sub(current_datetime(), interval 2 day), day) + and batch_run_date < datetime_trunc(current_datetime(), day) + meta: + description: + "Tests the uniqueness combination of: liquidity_pool_id, ledger_entry_change and last_modified_ledger." columns: - name: liquidity_pool_id description: '{{ doc("liquidity_pool_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: type description: '{{ doc("pool_type") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: fee description: '{{ doc("fee") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: trustline_count description: '{{ doc("trustline_count") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: pool_share_count description: '{{ doc("pool_share_count") }}' - + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: asset_a_id + description: '{{ doc("asset_a_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + - name: asset_a_type description: '{{ doc("asset_a_type") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_a_code description: '{{ doc("asset_a_code") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_a_issuer description: '{{ doc("asset_a_issuer") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_a_amount description: '{{ doc("asset_a_amount") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: asset_b_id + description: '{{ doc("asset_b_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_b_type description: '{{ doc("asset_a_type") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_b_code description: '{{ doc("asset_a_code") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_b_issuer description: '{{ doc("asset_a_issuer") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_b_amount description: '{{ doc("asset_a_amount") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: deleted description: '{{ doc("deleted") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_id description: '{{ doc("batch_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_run_date description: '{{ doc("batch_run_date") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_insert_ts - description: '{{ doc("batch_insert_ts") }}' \ No newline at end of file + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day \ No newline at end of file diff --git a/models/sources/src_offers.yml b/models/sources/src_offers.yml index 7a92d4d..d9b5e6f 100644 --- a/models/sources/src_offers.yml +++ b/models/sources/src_offers.yml @@ -6,12 +6,50 @@ sources: tables: - name: offers description: '{{ doc("liquidity_pools") }}' + tests: + - dbt_utils.recency: + datepart: hour + field: batch_run_date + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + - elementary.volume_anomalies: + timestamp_column: batch_run_date + backfill_days: 90 + time_bucket: + period: day + count: 1 + meta: + description: + "Monitors the row count of your table over time." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - offer_id + - ledger_entry_change + - last_modified_ledger + config: + where: batch_run_date >= datetime_trunc(datetime_sub(current_datetime(), interval 2 day), day) + and batch_run_date < datetime_trunc(current_datetime(), day) + meta: + description: + "Tests the uniqueness combination of: offer_id, ledger_entry_change and last_modified_ledger." columns: - name: seller_id description: '{{ doc("seller_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: offer_id description: '{{ doc("offer_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: selling_asset_type description: '{{ doc("asset_type") }}' @@ -33,24 +71,104 @@ sources: - name: amount description: '{{ doc("amount") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - - name: price_n + - name: pricen description: '{{ doc("price_n") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - - name: price_d + - name: priced description: '{{ doc("price_d") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: price description: '{{ doc("price") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: flags description: '{{ doc("flags") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: deleted - description: '{{ doc("deleted") }}' \ No newline at end of file + description: '{{ doc("deleted") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: sponsor + description: '{{ doc("sponsor") }}' + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: selling_asset_id + description: '{{ doc("assets_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: buying_asset_id + description: '{{ doc("assets_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: ledger_sequence + description: '{{ doc("ledger_sequence") }}' \ No newline at end of file diff --git a/models/sources/src_trust_lines.yml b/models/sources/src_trust_lines.yml index 892f351..8d65653 100644 --- a/models/sources/src_trust_lines.yml +++ b/models/sources/src_trust_lines.yml @@ -6,57 +6,177 @@ sources: tables: - name: trust_lines description: '{{ doc("trust_lines") }}' + tests: + - dbt_utils.recency: + datepart: hour + field: batch_run_date + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + - elementary.volume_anomalies: + timestamp_column: batch_run_date + backfill_days: 90 + time_bucket: + period: day + count: 1 + meta: + description: + "Monitors the row count of your table over time." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - account_id + - asset_code + - asset_issuer + - liquidity_pool_id + - ledger_entry_change + - last_modified_ledger + config: + where: batch_run_date >= datetime_trunc(datetime_sub(current_datetime(), interval 2 day), day) + and batch_run_date < datetime_trunc(current_datetime(), day) + meta: + description: + "Tests the uniqueness combination of: account_id, asset_code, asset_issuer, liquidity_pool_id, ledger_entry_change and last_modified_ledger." columns: - name: ledger_key description: '{{ doc("ledger_key") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: account_id description: '{{ doc("account_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_type description: '{{ doc("asset_type") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_issuer description: '{{ doc("asset_issuer") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: asset_code description: '{{ doc("asset_code") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: liquidity_pool_id description: '{{ doc("liquidity_pool_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: balance description: '{{ doc("trust_balance") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: trust_line_limit description: '{{ doc("trust_line_limit") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: buying_liabilities description: '{{ doc("buying_liabilities") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: selling_liabilities description: '{{ doc("selling_liabilities") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: flags description: '{{ doc("flags") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - - name: last_modified_ledgers + - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: deleted description: '{{ doc("deleted") }}' - - - name: sponsor - description: '{{ doc("sponsor") }}' - + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + - name: batch_id description: '{{ doc("batch_id") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_run_date description: '{{ doc("batch_run_date") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day - name: batch_insert_ts - description: '{{ doc("batch_insert_ts") }}' \ No newline at end of file + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: sponsor + description: '{{ doc("sponsor") }}' + + - name: asset_id + description: asset id + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null: + config: + where: batch_run_date > current_datetime - interval 2 day + + - name: ledger_sequence + description: '{{ doc("ledger_sequence") }}' + + \ No newline at end of file diff --git a/tests/ledger_sequence_increment.sql b/tests/ledger_sequence_increment.sql new file mode 100644 index 0000000..bc475d6 --- /dev/null +++ b/tests/ledger_sequence_increment.sql @@ -0,0 +1,33 @@ +{{ config(severity="warn" + , store_failures = true) }} + +with + ledger_sequence as ( + select + id + , batch_id + , closed_at + , max(sequence) as max_sequence + from {{ source('crypto_stellar', 'history_ledgers') }} + where (closed_at > current_timestamp - interval 7 day) + group by id, batch_id, closed_at + ) + + , lead_sequence as ( + select + id + , batch_id + , closed_at + , max_sequence + , lead(max_sequence) over ( + order by closed_at desc + ) as prev_sequence + from ledger_sequence + ) + +select * +from lead_sequence +where ( + prev_sequence >= max_sequence + or max_sequence != prev_sequence + 1 +)