From e5bd46baed557a1a8c6da06f93241869b033b717 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Thu, 26 Sep 2024 16:49:42 +0530 Subject: [PATCH 01/18] Add tests for staging --- models/staging/stg_account_signers.yml | 67 +++ models/staging/stg_accounts.yml | 122 ++++++ models/staging/stg_claimable_balances.yml | 123 ++++++ models/staging/stg_config_settings.yml | 189 +++++++++ models/staging/stg_contract_code.yml | 119 ++++++ models/staging/stg_contract_data.yml | 120 ++++++ models/staging/stg_history_assets.yml | 57 +++ models/staging/stg_history_effects.yml | 320 +++++++++++++++ models/staging/stg_history_ledgers.yml | 132 ++++++ models/staging/stg_history_operations.yml | 427 ++++++++++++++++++++ models/staging/stg_history_trades.yml | 142 +++++++ models/staging/stg_history_transactions.yml | 189 +++++++++ models/staging/stg_liquidity_pools.yml | 128 ++++++ models/staging/stg_offers.yml | 139 +++++++ models/staging/stg_trust_lines.yml | 128 ++++++ models/staging/stg_ttl.yml | 68 ++++ 16 files changed, 2470 insertions(+) create mode 100644 models/staging/stg_account_signers.yml create mode 100644 models/staging/stg_accounts.yml create mode 100644 models/staging/stg_claimable_balances.yml create mode 100644 models/staging/stg_config_settings.yml create mode 100644 models/staging/stg_contract_code.yml create mode 100644 models/staging/stg_contract_data.yml create mode 100644 models/staging/stg_history_assets.yml create mode 100644 models/staging/stg_history_effects.yml create mode 100644 models/staging/stg_history_ledgers.yml create mode 100644 models/staging/stg_history_operations.yml create mode 100644 models/staging/stg_history_trades.yml create mode 100644 models/staging/stg_history_transactions.yml create mode 100644 models/staging/stg_liquidity_pools.yml create mode 100644 models/staging/stg_offers.yml create mode 100644 models/staging/stg_trust_lines.yml create mode 100644 models/staging/stg_ttl.yml diff --git a/models/staging/stg_account_signers.yml b/models/staging/stg_account_signers.yml new file mode 100644 index 0000000..dae2a06 --- /dev/null +++ b/models/staging/stg_account_signers.yml @@ -0,0 +1,67 @@ +version: 2 + +models: + - name: stg_account_signers + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - account_id + - signer + - ledger_sequence + - last_modified_ledger + meta: + description: "Tests the uniqueness combination of: account_id, signer, ledger_sequence and last_modified_ledger" + columns: + - name: account_id + description: '{{ doc("account_id") }}' + tests: + - not_null + - name: signer + description: '{{ doc("signer") }}' + tests: + - not_null + - name: weight + description: '{{ doc("weight") }}' + tests: + - not_null + - name: sponsor + description: '{{ doc("sponsor") }}' + tests: + - not_null + - name: last_modified_ledger + tests: + - not_null + - name: ledger_entry_change + tests: + - not_null + - name: deleted + description: '{{ doc("deleted") }}' + tests: + - not_null + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + - name: closed_at + tests: + - not_null + - name: ledger_sequence + tests: + - not_null + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null + - name: airflow_start_ts diff --git a/models/staging/stg_accounts.yml b/models/staging/stg_accounts.yml new file mode 100644 index 0000000..16f0628 --- /dev/null +++ b/models/staging/stg_accounts.yml @@ -0,0 +1,122 @@ +version: 2 + +models: + - name: stg_accounts + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - account_id + - sequence_number + - ledger_entry_change + - last_modified_ledger + meta: + description: "Tests the uniqueness combination of: account_id, signer and ledger_sequence." + columns: + - name: account_id + description: '{{ doc("account_id") }}' + tests: + - not_null + - name: balance + description: '{{ doc("balance") }}' + tests: + - not_null + - name: buying_liabilities + description: '{{ doc("buying_liabilities") }}' + tests: + - not_null + - name: selling_liabilities + description: '{{ doc("selling_liabilities") }}' + tests: + - not_null + - name: sequence_number + description: '{{ doc("sequence_number") }}' + tests: + - not_null + - name: num_subentries + description: '{{ doc("num_subentries") }}' + tests: + - not_null + - name: inflation_destination + description: '{{ doc("inflation_destination") }}' + tests: + - not_null + - name: flags + description: '{{ doc("flags_accounts_balances") }}' + tests: + - not_null + - name: home_domain + description: '{{ doc("home_domain") }}' + tests: + - not_null + - name: master_weight + description: '{{ doc("master_weight") }}' + tests: + - not_null + - name: threshold_low + description: '{{ doc("threshold_low") }}' + tests: + - not_null + - name: threshold_medium + description: '{{ doc("threshold_medium") }}' + tests: + - not_null + - name: threshold_high + description: '{{ doc("threshold_high") }}' + tests: + - not_null + - name: last_modifed_ledger + description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null + - name: ledger_entry_change + description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null + - accepted_values: + values: [0, 1, 2] + quote: false + - name: deleted + description: '{{ doc("deleted") }}' + tests: + - not_null + - name: sponsor + description: '{{ doc("sponsor") }}' + - name: num_sponsored + description: '{{ doc("num_sponsored") }}' + tests: + - not_null + - name: num_sponsoring + description: '{{ doc("num_sponsoring") }}' + tests: + - not_null + - name: sequence_ledger + description: '{{ doc("sequence_ledger") }}' + tests: + - not_null + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + - name: closed_at + tests: + - not_null + - name: ledger_sequence + tests: + - not_null + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null + - name: airflow_start_ts diff --git a/models/staging/stg_claimable_balances.yml b/models/staging/stg_claimable_balances.yml new file mode 100644 index 0000000..6abc663 --- /dev/null +++ b/models/staging/stg_claimable_balances.yml @@ -0,0 +1,123 @@ +version: 2 + +models: + - name: stg_claimable_balances + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - balance_id + - ledger_entry_change + meta: + description: "Tests the uniqueness combination of: balance_id and ledger_entry_change." + columns: + - name: balance_id + description: '{{ doc("balance_id") }}' + tests: + - not_null + + - name: claimants + description: '{{ doc("claimants") }}' + + - name: claimants.destination + description: '{{ doc("claimants_destination") }}' + + - name: claimants.predicate + description: '{{ doc("claimants_predicate") }}' + + - name: claimants.predicate.unconditional + description: '{{ doc("claimants_predicate_unconditional") }}' + + - name: claimants.predicate.abs_before + description: '{{ doc("claimants_predicate_abs_before") }}' + + - name: claimants.predicate.rel_before + description: '{{ doc("claimants_predicate_rel_before") }}' + + - name: claimants.predicate.abs_before_epoch + description: '{{ doc("claimants_predicate_abs_before_epoch") }}' + + - name: asset_type + description: '{{ doc("asset_type") }}' + tests: + - not_null + + - name: asset_code + description: '{{ doc("asset_code") }}' + tests: + - not_null + + - name: asset_issuer + description: '{{ doc("asset_issuer") }}' + tests: + - not_null + + - name: asset_id + description: '{{ doc("asset_id") }}' + tests: + - not_null + + - name: asset_amount + description: '{{ doc("asset_amount") }}' + tests: + - not_null + + - name: sponsor + description: '{{ doc("sponsor") }}' + tests: + - not_null + + - name: flags + description: '{{ doc("flags_accounts_balances") }}' + tests: + - not_null + + - name: last_modified_ledger + description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null + + - name: ledger_entry_change + description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null + - accepted_values: + values: [0, 1, 2] + quote: false + + - name: deleted + description: '{{ doc("deleted") }}' + tests: + - not_null + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: closed_at + tests: + - not_null + + - name: ledger_sequence + tests: + - not_null + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null + + - name: airflow_start_ts diff --git a/models/staging/stg_config_settings.yml b/models/staging/stg_config_settings.yml new file mode 100644 index 0000000..2ad4cba --- /dev/null +++ b/models/staging/stg_config_settings.yml @@ -0,0 +1,189 @@ +version: 2 + +models: + - name: stg_config_settings + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - config_setting_id + - last_modified_ledger + - ledger_entry_change + meta: + description: "Tests the uniqueness combination of: balance_id and ledger_entry_change." + columns: + - name: config_setting_id + description: '{{ doc("config_setting_id") }}' + tests: + - not_null + + - name: contract_max_size_bytes + description: '{{ doc("contract_max_size_bytes") }}' + + - name: ledger_max_instructions + description: '{{ doc("ledger_max_instructions") }}' + + - name: tx_max_instructions + description: '{{ doc("tx_max_instructions") }}' + + - name: fee_rate_per_instructions_increment + description: '{{ doc("fee_rate_per_instructions_increment") }}' + + - name: tx_memory_limit + description: '{{ doc("tx_memory_limit") }}' + + - name: ledger_max_read_ledger_entries + description: '{{ doc("ledger_max_read_ledger_entries") }}' + + - name: ledger_max_read_bytes + description: '{{ doc("ledger_max_read_bytes") }}' + + - name: ledger_max_write_ledger_entries + description: '{{ doc("ledger_max_write_ledger_entries") }}' + + - name: ledger_max_write_bytes + description: '{{ doc("ledger_max_write_bytes") }}' + + - name: tx_max_read_ledger_entries + description: '{{ doc("tx_max_read_ledger_entries") }}' + + - name: tx_max_read_bytes + description: '{{ doc("tx_max_read_bytes") }}' + + - name: tx_max_write_ledger_entries + description: '{{ doc("tx_max_write_ledger_entries") }}' + + - name: tx_max_write_bytes + description: '{{ doc("tx_max_write_bytes") }}' + + - name: fee_read_ledger_entry + description: '{{ doc("fee_read_ledger_entry") }}' + + - name: fee_write_ledger_entry + description: '{{ doc("fee_write_ledger_entry") }}' + + - name: fee_read_1kb + description: '{{ doc("fee_read_1kb") }}' + + - name: bucket_list_target_size_bytes + description: '{{ doc("bucket_list_target_size_bytes") }}' + + - name: write_fee_1kb_bucket_list_low + description: '{{ doc("write_fee_1kb_bucket_list_low") }}' + + - name: write_fee_1kb_bucket_list_high + description: '{{ doc("write_fee_1kb_bucket_list_high") }}' + + - name: bucket_list_write_fee_growth_factor + description: '{{ doc("bucket_list_write_fee_growth_factor") }}' + + - name: fee_historical_1kb + description: '{{ doc("fee_historical_1kb") }}' + + - name: tx_max_contract_events_size_bytes + description: '{{ doc("tx_max_contract_events_size_bytes") }}' + + - name: fee_contract_events_1kb + description: '{{ doc("fee_contract_events_1kb") }}' + + - name: ledger_max_txs_size_bytes + description: '{{ doc("ledger_max_txs_size_bytes") }}' + + - name: tx_max_size_bytes + description: '{{ doc("tx_max_size_bytes") }}' + + - name: fee_tx_size_1kb + description: '{{ doc("fee_tx_size_1kb") }}' + + - name: contract_cost_params_cpu_insns + description: '{{ doc("contract_cost_params_cpu_insns") }}' + + - name: contract_cost_params_mem_bytes + description: '{{ doc("contract_cost_params_mem_bytes") }}' + + - name: contract_data_key_size_bytes + description: '{{ doc("contract_data_key_size_bytes") }}' + + - name: contract_data_entry_size_bytes + description: '{{ doc("contract_data_entry_size_bytes") }}' + + - name: max_entry_ttl + description: '{{ doc("max_entry_ttl") }}' + + - name: min_temp_entry_ttl + description: '{{ doc("min_temp_entry_ttl") }}' + + - name: min_persistent_entry_ttl + description: '{{ doc("min_persistent_entry_ttl") }}' + + - name: auto_bump_ledgers + description: '{{ doc("auto_bump_ledgers") }}' + + - name: persistent_rent_rate_denominator + description: '{{ doc("persistent_rent_rate_denominator") }}' + + - name: temp_rent_rate_denominator + description: '{{ doc("temp_rent_rate_denominator") }}' + + - name: max_entries_to_ttl + description: '{{ doc("max_entries_to_ttl") }}' + + - name: bucket_list_size_window_sample_size + description: '{{ doc("bucket_list_size_window_sample_size") }}' + + - name: eviction_scan_size + description: '{{ doc("eviction_scan_size") }}' + + - name: starting_eviction_scan_level + description: '{{ doc("starting_eviction_scan_level") }}' + + - name: ledger_max_tx_count + description: '{{ doc("ledger_max_tx_count") }}' + + - name: bucket_list_size_window + description: '{{ doc("bucket_list_size_window") }}' + + - name: last_modified_ledger + description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null + + - name: ledger_entry_change + description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null + - accepted_values: + values: [0, 1, 2] + quote: false + + - name: deleted + description: '{{ doc("deleted") }}' + tests: + - not_null + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null diff --git a/models/staging/stg_contract_code.yml b/models/staging/stg_contract_code.yml new file mode 100644 index 0000000..aa6b011 --- /dev/null +++ b/models/staging/stg_contract_code.yml @@ -0,0 +1,119 @@ +version: 2 + +models: + - name: stg_contract_code + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - contract_code_hash + - contract_code_ext_v + - last_modified_ledger + - ledger_entry_change + meta: + description: "Tests the uniqueness combination of: contract_code_hash, contract_code_ext_v, last_modified_ledger, and ledger_entry_change." + columns: + - name: contract_code_hash + description: '{{ doc("contract_code_hash") }}' + tests: + - not_null + + - name: contract_code_ext_v + description: '{{ doc("contract_code_ext_v") }}' + tests: + - not_null + + - name: last_modified_ledger + description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null + + - name: n_data_segment_bytes + description: '{{ doc("n_data_segment_bytes") }}' + tests: + - not_null + + - name: n_data_segments + description: '{{ doc("n_data_segments") }}' + tests: + - not_null + + - name: n_elem_segments + description: '{{ doc("n_elem_segments") }}' + tests: + - not_null + + - name: n_exports + description: '{{ doc("n_exports") }}' + tests: + - not_null + + - name: n_functions + description: '{{ doc("n_functions") }}' + tests: + - not_null + + - name: n_globals + description: '{{ doc("n_globals") }}' + tests: + - not_null + + - name: n_imports + description: '{{ doc("n_imports") }}' + tests: + - not_null + + - name: n_instructions + description: '{{ doc("n_instructions") }}' + tests: + - not_null + + - name: n_table_entries + description: '{{ doc("n_table_entries") }}' + tests: + - not_null + + - name: n_types + description: '{{ doc("n_types") }}' + tests: + - not_null + + - name: ledger_entry_change + description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null + - accepted_values: + values: [0, 1, 2] + quote: false + + - name: deleted + description: '{{ doc("deleted") }}' + tests: + - not_null + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null diff --git a/models/staging/stg_contract_data.yml b/models/staging/stg_contract_data.yml new file mode 100644 index 0000000..909a004 --- /dev/null +++ b/models/staging/stg_contract_data.yml @@ -0,0 +1,120 @@ +version: 2 + +models: + - name: stg_contract_data + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - contract_id + - contract_data_xdr + - last_modified_ledger + - ledger_entry_change + meta: + description: "Tests the uniqueness combination of: contract_id, contract_data_xdr, last_modified_ledger, and ledger_entry_change." + columns: + - name: contract_id + description: '{{ doc("contract_id") }}' + tests: + - not_null + + - name: contract_key_type + description: '{{ doc("contract_key_type") }}' + tests: + - not_null + + - name: contract_durability + description: '{{ doc("contract_durability") }}' + tests: + - not_null + + - name: asset_code + description: '{{ doc("asset_code") }}' + tests: + - not_null + + - name: asset_issuer + description: '{{ doc("asset_issuer") }}' + tests: + - not_null + + - name: asset_type + description: '{{ doc("asset_type") }}' + tests: + - not_null + + - name: balance_holder + description: '{{ doc("balance_holder") }}' + + - name: balance + description: '{{ doc("balance") }}' + + - name: last_modified_ledger + description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null + + - name: ledger_entry_change + description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null + - accepted_values: + values: [0, 1, 2] + quote: false + + - name: deleted + description: '{{ doc("deleted") }}' + tests: + - not_null + + - name: key + description: '{{ doc("key") }}' + tests: + - not_null + + - name: key_decoded + description: '{{ doc("key_decoded") }}' + tests: + - not_null + + - name: val + description: '{{ doc("val") }}' + tests: + - not_null + + - name: val_decoded + description: '{{ doc("val_decoded") }}' + tests: + - not_null + + - name: contract_data_xdr + description: '{{ doc("contract_data_xdr") }}' + tests: + - not_null + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null diff --git a/models/staging/stg_history_assets.yml b/models/staging/stg_history_assets.yml new file mode 100644 index 0000000..a6be6db --- /dev/null +++ b/models/staging/stg_history_assets.yml @@ -0,0 +1,57 @@ +version: 2 + +models: + - name: stg_history_assets + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - asset_type + - asset_code + - asset_issuer + meta: + description: "Tests the uniqueness combination of: asset_type, asset_code, and asset_issuer." + columns: + - name: asset_id + description: '{{ doc("asset_id") }}' + tests: + - not_null + + - name: asset_type + description: '{{ doc("asset_type") }}' + tests: + - not_null + - accepted_values: + values: ["credit_alphanum4", "credit_alphanum12", "native"] + + - name: asset_code + description: '{{ doc("asset_code") }}' + tests: + - not_null + + - name: asset_issuer + description: '{{ doc("asset_issuer") }}' + tests: + - not_null + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null diff --git a/models/staging/stg_history_effects.yml b/models/staging/stg_history_effects.yml new file mode 100644 index 0000000..658ef92 --- /dev/null +++ b/models/staging/stg_history_effects.yml @@ -0,0 +1,320 @@ +version: 2 + +models: + - name: stg_history_effects + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + columns: + - name: address + description: '{{ doc("address") }}' + tests: + - not_null + + - name: address_muxed + description: '{{ doc("address_muxed") }}' + + - name: operation_id + description: '{{ doc("operation_id") }}' + tests: + - not_null + + - name: id + description: '{{ doc("id") }}' + tests: + - not_null + + - name: ledger_sequence + description: '{{ doc("ledger_sequence") }}' + tests: + - not_null + + - name: type + description: '{{ doc("type") }}' + tests: + - not_null + + - name: type_string + description: '{{ doc("type_string") }}' + tests: + - not_null + + - name: details + description: '{{ doc("details_effects") }}' + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: details.liquidity_pool + description: '{{ doc("details_liquidity_pool") }}' + + - name: details.liquidity_pool.fee_bp + description: '{{ doc("details_liquidity_pool_fee_bp") }}' + + - name: details.liquidity_pool.id + description: '{{ doc("details_liquidity_pool_id") }}' + + - name: details.liquidity_pool.total_shares + description: '{{ doc("details_liquidity_pool_total_shares") }}' + + - name: details.liquidity_pool.total_trustlines + description: '{{ doc("details_liquidity_pool_total_trustlines") }}' + + - name: details.liquidity_pool.type + description: '{{ doc("details_liquidity_pool_type") }}' + + - name: details.liquidity_pool.reserves + description: '{{ doc("details_liquidity_pool_reserves") }}' + + - name: details.liquidity_pool.reserves.asset + description: '{{ doc("details_asset") }}' + + - name: details.liquidity_pool.reserves.amount + description: '{{ doc("details_amount") }}' + + - name: details.reserves_received + description: '{{ doc("details_reserves_received") }}' + + - name: details.reserves_received.asset + description: '{{ doc("details_asset") }}' + + - name: details.reserves_received.amount + description: '{{ doc("details_amount") }}' + + - name: details.reserves_deposited + description: '{{ doc("details_reserves_deposited") }}' + + - name: details.reserves_deposited.asset + description: '{{ doc("details_asset") }}' + + - name: details.reserves_deposited.amount + description: '{{ doc("details_amount") }}' + + - name: details.reserves_revoked + description: '{{ doc("details_reserves_revoked") }}' + + - name: details.reserves_revoked.asset + description: '{{ doc("details_asset") }}' + + - name: details.reserves_revoked.amount + description: '{{ doc("details_amount") }}' + + - name: details.reserves_revoked.claimable_balance_id + description: '{{ doc("details_reserves_revoked_claimable_balance_id") }}' + + - name: details.bought + description: '{{ doc("details_bought") }}' + + - name: details.bought.asset + description: '{{ doc("details_asset") }}' + + - name: details.bought.amount + description: '{{ doc("details_amount") }}' + + - name: details.sold + description: '{{ doc("details_sold") }}' + + - name: details.sold.asset + description: '{{ doc("details_asset") }}' + + - name: details.sold.amount + description: '{{ doc("details_amount") }}' + + - name: details.shares_revoked + description: '{{ doc("details_shares_revoked") }}' + + - name: details.shares_received + description: '{{ doc("details_shares_received") }}' + + - name: details.shares_redeemed + description: '{{ doc("details_shares_redeemed") }}' + + - name: details.liquidity_pool_id + description: '{{ doc("details_liquidity_pool_id") }}' + + - name: details.balance_id + description: '{{ doc("details_balance_id") }}' + + - name: details.new_seq + description: '{{ doc("details_new_seq") }}' + + - name: details.name + description: '{{ doc("details_name") }}' + + - name: details.value + description: '{{ doc("details_value") }}' + + - name: details.trustor + description: '{{ doc("details_trustor") }}' + + - name: details.limit + description: '{{ doc("details_limit") }}' + + - name: details.inflation_destination + description: '{{ doc("details_inflation_destination") }}' + + - name: details.authorized_flag + description: '{{ doc("details_authorized_flag") }}' + + - name: details.auth_immutable_flag + description: '{{ doc("details_auth_immutable_flag") }}' + + - name: details.authorized_to_maintain_liabilites + description: '{{ doc("details_authorized_to_maintain_liabilites") }}' + + - name: details.auth_revocable_flag + description: '{{ doc("details_auth_revocable_flag") }}' + + - name: details.auth_required_flag + description: '{{ doc("details_auth_required_flag") }}' + + - name: details.auth_clawback_enabled_flag + description: '{{ doc("details_auth_clawback_enabled_flag") }}' + + - name: details.claimable_balance_clawback_enabled_flag + description: '{{ doc("details_claimable_balance_clawback_enabled_flag") }}' + + - name: details.clawback_enabled_flag + description: '{{ doc("details_clawback_enabled_flag") }}' + + - name: details.high_threshold + description: '{{ doc("details_high_threshold") }}' + + - name: details.med_threshold + description: '{{ doc("details_med_threshold") }}' + + - name: details.low_threshold + description: '{{ doc("details_low_threshold") }}' + + - name: details.home_domain + description: '{{ doc("details_home_domain") }}' + + - name: details.asset_issuer + description: '{{ doc("details_asset_issuer") }}' + + - name: details.asset + description: '{{ doc("details_asset") }}' + + - name: details.asset_code + description: '{{ doc("details_asset_code") }}' + + - name: details.asset_type + description: '{{ doc("details_asset_type") }}' + + - name: details.signer + description: '{{ doc("details_signer") }}' + + - name: details.sponsor + description: '{{ doc("details_sponsor") }}' + + - name: details.new_sponsor + description: '{{ doc("details_new_sponsor") }}' + + - name: details.former_sponsor + description: '{{ doc("details_former_sponsor") }}' + + - name: details.weight + description: '{{ doc("details_weight") }}' + + - name: details.public_key + description: '{{ doc("details_public_key") }}' + + - name: details.amount + description: '{{ doc("details_amount") }}' + + - name: details.starting_balance + description: '{{ doc("details_starting_balance") }}' + + - name: details.seller + description: '{{ doc("details_seller") }}' + + - name: details.seller_muxed + description: '{{ doc("details_seller_muxed") }}' + + - name: details.seller_muxed_id + description: '{{ doc("details_seller_muxed_id") }}' + + - name: details.offer_id + description: '{{ doc("details_offer_id") }}' + + - name: details.sold_amount + description: '{{ doc("details_amount") }}' + + - name: details.sold_asset_type + description: '{{ doc("details_asset_type") }}' + + - name: details.sold_asset_code + description: '{{ doc("details_asset_code") }}' + + - name: details.sold_asset_issuer + description: '{{ doc("details_asset_issuer") }}' + + - name: details.bought_asset_type + description: '{{ doc("details_asset_type") }}' + + - name: details.bought_asset_code + description: '{{ doc("details_asset_code") }}' + + - name: details.bought_asset_issuer + description: '{{ doc("details_asset_issuer") }}' + + - name: details.bought_amount + description: '{{ doc("details_amount") }}' + + - name: details.data_name + description: '{{ doc("details_data_name") }}' + + - name: details.predicate + description: '{{ doc("claimants_predicate") }}' + + - name: details.predicate.abs_before + description: '{{ doc("claimants_predicate_abs_before") }}' + + - name: details.predicate.rel_before + description: '{{ doc("claimants_predicate_rel_before") }}' + + - name: details.predicate.unconditional + description: '{{ doc("claimants_predicate_unconditional") }}' + + - name: details.predicate.abs_before_epoch + description: '{{ doc("claimants_predicate_abs_before_epoch") }}' + + - name: details.contract + description: '{{ doc("contract_id") }}' + + - name: details.contract_event_type + description: '{{ doc("contract_event_type") }}' + + - name: details.extend_to + description: '{{ doc("details_extend_to") }}' + + - name: details.entries + description: '{{ doc("details_entries") }}' + + - name: index + description: '{{ doc("index") }}' + tests: + - not_null + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null diff --git a/models/staging/stg_history_ledgers.yml b/models/staging/stg_history_ledgers.yml new file mode 100644 index 0000000..5fa3792 --- /dev/null +++ b/models/staging/stg_history_ledgers.yml @@ -0,0 +1,132 @@ +version: 2 + +models: + - name: stg_history_ledgers + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + columns: + - name: sequence + description: '{{ doc("sequence") }}' + tests: + - unique + - not_null + + - name: ledger_hash + description: '{{ doc("ledger_hash") }}' + tests: + - not_null + + - name: previous_ledger_hash + description: '{{ doc("previous_ledger_hash") }}' + tests: + - not_null + + - name: transaction_count + description: '{{ doc("transaction_count") }}' + tests: + - not_null + + - name: operation_count + description: '{{ doc("operation_count") }}' + tests: + - not_null + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null + + - name: id + description: '{{ doc("ledger_id") }}' + tests: + - not_null + + - name: total_coins + description: '{{ doc("total_coins") }}' + tests: + - not_null + + - name: fee_pool + description: '{{ doc("fee_pool") }}' + tests: + - not_null + + - name: base_fee + description: '{{ doc("base_fee") }}' + tests: + - not_null + + - name: base_reserve + description: '{{ doc("base_reserve") }}' + tests: + - not_null + + - name: max_tx_set_size + description: '{{ doc("max_tx_set_size") }}' + tests: + - not_null + + - name: protocol_version + description: '{{ doc("protocol_version") }}' + tests: + - not_null + + - name: ledger_header + description: '{{ doc("ledger_header") }}' + tests: + - not_null + + - name: successful_transaction_count + description: '{{ doc("successful_transaction_count") }}' + tests: + - not_null + + - name: failed_transaction_count + description: '{{ doc("failed_transaction_count") }}' + tests: + - not_null + + - name: tx_set_operation_count + description: '{{ doc("tx_set_operation_count") }}' + tests: + - not_null + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: soroban_fee_write_1kb + description: '{{ doc("soroban_fee_write_1kb") }}' + + - name: node_id + description: '{{ doc("node_id") }}' + tests: + - not_null + + - name: signature + description: '{{ doc("signature") }}' + tests: + - not_null + + - name: soroban_fee_write_1kb + description: '{{ doc("soroban_fee_write_1kb") }}' + tests: + - not_null + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null diff --git a/models/staging/stg_history_operations.yml b/models/staging/stg_history_operations.yml new file mode 100644 index 0000000..e4321d5 --- /dev/null +++ b/models/staging/stg_history_operations.yml @@ -0,0 +1,427 @@ +version: 2 + +models: + - name: stg_history_operations + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + columns: + - name: id + description: '{{ doc("operation_id") }}' + tests: + - unique + - not_null + + - name: source_account + description: '{{ doc("source_account") }}' + tests: + - not_null + + - name: source_account_muxed + description: '{{ doc("source_account_muxed") }}' + + - name: transaction_id + description: '{{ doc("op_transaction_id") }}' + tests: + - not_null + + - name: ledger_sequence + description: '{{ doc("ledger_sequence") }}' + tests: + - not_null + + - name: type + description: '{{ doc("type") }}' + tests: + - not_null + + - name: type_string + description: '{{ doc("type_string") }}' + tests: + - not_null + + - name: details + description: '{{ doc("details") }}' + + - name: details.account + description: '{{ doc("details_account") }}' + + - name: details.account_muxed + description: '{{ doc("details_account_muxed") }}' + + - name: details.account_muxed_id + description: '{{ doc("details_account_muxed_id") }}' + + - name: details.account_id + description: '{{ doc("details_account_id") }}' + + - name: details.amount + description: '{{ doc("details_amount") }}' + + - name: details.asset + description: '{{ doc("details_asset") }}' + + - name: details.asset_id + description: '{{ doc("details_asset_id") }}' + + - name: details.buying_asset_id + description: '{{ doc("details_buying_asset_id") }}' + + - name: details.selling_asset_id + description: '{{ doc("details_selling_asset_id") }}' + + - name: details.source_asset_id + description: '{{ doc("details_source_asset_id") }}' + + - name: details.reserve_a_asset_id + description: '{{ doc("details_reserve_a_asset_id") }}' + + - name: details.reserve_b_asset_id + description: '{{ doc("details_reserve_b_asset_id") }}' + + - name: details.ledger_key_hash + description: '{{ doc("ledger_key_hash") }}' + + - name: details.ledgers_to_expire + description: '{{ doc("details_ledgers_to_expire") }}' + + - name: details.asset_code + description: '{{ doc("details_asset_code") }}' + + - name: details.asset_issuer + description: '{{ doc("details_asset_issuer") }}' + + - name: details.asset_type + description: '{{ doc("details_asset_type") }}' + + - name: details.authorize + description: '{{ doc("details_authorize") }}' + + - name: details.balance_id + description: '{{ doc("details_balance_id") }}' + + - name: details.buying_asset_code + description: '{{ doc("details_buying_asset_code") }}' + + - name: details.buying_asset_issuer + description: '{{ doc("details_buying_asset_issuer") }}' + + - name: details.buying_asset_type + description: '{{ doc("details_buying_asset_type") }}' + + - name: details.claimable_balance_id + description: '{{ doc("details_claimable_balance_id") }}' + + - name: details.claimant + description: '{{ doc("details_claimant") }}' + + - name: details.claimant_muxed + description: '{{ doc("details_claimant_muxed") }}' + + - name: details.claimant_muxed_id + description: '{{ doc("details_claimant_muxed_id") }}' + + - name: details.claimants + description: '{{ doc("details_claimants") }}' + + - name: details.data_account_id + description: '{{ doc("details_data_account_id") }}' + + - name: details.data_name + description: '{{ doc("details_data_name") }}' + + - name: details.from + description: '{{ doc("details_from") }}' + + - name: details.from_muxed + description: '{{ doc("details_from_muxed") }}' + + - name: details.from_muxed_id + description: '{{ doc("details_from_muxed_id") }}' + + - name: details.funder + description: '{{ doc("details_funder") }}' + + - name: details.funder_muxed + description: '{{ doc("details_funder_muxed") }}' + + - name: details.funder_muxed_id + description: '{{ doc("details_funder_muxed_id") }}' + + - name: details.high_threshold + description: '{{ doc("details_high_threshold") }}' + + - name: details.home_domain + description: '{{ doc("details_home_domain") }}' + + - name: details.inflation_dest + description: '{{ doc("details_inflation_dest") }}' + + - name: details.into + description: '{{ doc("details_into") }}' + + - name: details.into_muxed + description: '{{ doc("details_into_muxed") }}' + + - name: details.into_muxed_id + description: '{{ doc("details_into_muxed_id") }}' + + - name: details.limit + description: '{{ doc("details_limit") }}' + + - name: details.low_threshold + description: '{{ doc("details_low_threshold") }}' + + - name: details.master_key_weight + description: '{{ doc("details_master_key_weight") }}' + + - name: details.med_threshold + description: '{{ doc("details_med_threshold") }}' + + - name: details.name + description: '{{ doc("details_name") }}' + + - name: details.offer_id + description: '{{ doc("details_offer_id") }}' + + - name: details.path + description: '{{ doc("details_path") }}' + + - name: details.price + description: '{{ doc("details_price") }}' + + - name: details.price_r + description: '{{ doc("details_price_r") }}' + + - name: details.selling_asset_code + description: '{{ doc("details_selling_asset_code") }}' + + - name: details.selling_asset_issuer + description: '{{ doc("details_selling_asset_issuer") }}' + + - name: details.selling_asset_type + description: '{{ doc("details_selling_asset_type") }}' + + - name: details.set_flags + description: '{{ doc("details_set_flags") }}' + + - name: details.set_flags_s + description: '{{ doc("details_set_flags_s") }}' + + - name: details.signer_account_id + description: '{{ doc("details_signer_account_id") }}' + + - name: details.signer_key + description: '{{ doc("details_signer_key") }}' + + - name: details.signer_weight + description: '{{ doc("details_signer_weight") }}' + + - name: details.source_amount + description: '{{ doc("details_source_amount") }}' + + - name: details.source_asset_code + description: '{{ doc("details_source_asset_code") }}' + + - name: details.source_asset_issuer + description: '{{ doc("details_source_asset_issuer") }}' + + - name: details.source_asset_type + description: '{{ doc("details_source_asset_type") }}' + + - name: details.source_max + description: '{{ doc("details_source_max") }}' + + - name: details.starting_balance + description: '{{ doc("details_starting_balance") }}' + + - name: details.to + description: '{{ doc("details_to") }}' + + - name: details.to_muxed + description: '{{ doc("details_to_muxed") }}' + + - name: details.to_muxed_id + description: '{{ doc("details_to_muxed_id") }}' + + - name: details.trustee + description: '{{ doc("details_trustee") }}' + + - name: details.trustee_muxed + description: '{{ doc("details_trustee_muxed") }}' + + - name: details.trustee_muxed_id + description: '{{ doc("details_trustee_muxed_id") }}' + + - name: details.trustline_account_id + description: '{{ doc("details_trustline_account_id") }}' + + - name: details.trustline_asset + description: '{{ doc("details_trustline_asset") }}' + + - name: details.trustor + description: '{{ doc("details_trustor") }}' + + - name: details.trustor_muxed + description: '{{ doc("details_trustor_muxed") }}' + + - name: details.trustor_muxed_id + description: '{{ doc("details_trustor_muxed_id") }}' + + - name: details.value + description: '{{ doc("details_value") }}' + + - name: details.clear_flags + description: '{{ doc("details_clear_flags") }}' + + - name: details.clear_flags_s + description: '{{ doc("details_clear_flags_s") }}' + + - name: details.destination_min + description: '{{ doc("details_destination_min") }}' + + - name: details.bump_to + description: '{{ doc("details_bump_to") }}' + + - name: details.authorize_to_maintain_liabilities + description: '{{ doc("details_authorize_to_maintain_liabilities") }}' + + - name: details.clawback_enabled + description: '{{ doc("details_clawback_enabled") }}' + + - name: details.sponsor + description: '{{ doc("details_sponsor") }}' + + - name: details.sponsored_id + description: '{{ doc("details_sponsored_id") }}' + + - name: details.begin_sponsor + description: '{{ doc("details_begin_sponsor") }}' + + - name: details.begin_sponsor_muxed + description: '{{ doc("details_begin_sponsor_muxed") }}' + + - name: details.begin_sponsor_muxed_id + description: '{{ doc("details_begin_sponsor_muxed_id") }}' + + - name: details.liquidity_pool_id + description: '{{ doc("details_liquidity_pool_id") }}' + + - name: details.reserve_a_asset_type + description: '{{ doc("details_reserve_a_asset_type") }}' + + - name: details.reserve_a_asset_code + description: '{{ doc("details_reserve_a_asset_code") }}' + + - name: details.reserve_a_asset_issuer + description: '{{ doc("details_reserve_a_asset_issuer") }}' + + - name: details.reserve_a_max_amount + description: '{{ doc("details_reserve_a_max_amount") }}' + + - name: details.reserve_a_deposit_amount + description: '{{ doc("details_reserve_a_deposit_amount") }}' + + - name: details.reserve_b_asset_type + description: '{{ doc("details_reserve_b_asset_type") }}' + + - name: details.reserve_b_asset_code + description: '{{ doc("details_reserve_b_asset_code") }}' + + - name: details.reserve_b_asset_issuer + description: '{{ doc("details_reserve_b_asset_issuer") }}' + + - name: details.reserve_b_max_amount + description: '{{ doc("details_reserve_b_max_amount") }}' + + - name: details.reserve_b_deposit_amount + description: '{{ doc("details_reserve_b_deposit_amount") }}' + + - name: details.min_price + description: '{{ doc("details_min_price") }}' + + - name: details.min_price_r + description: '{{ doc("details_min_price_r") }}' + + - name: details.max_price + description: '{{ doc("details_max_price") }}' + + - name: details.max_price_r + description: '{{ doc("details_max_price_r") }}' + + - name: details.shares_received + description: '{{ doc("details_shares_received") }}' + + - name: details.reserve_a_min_amount + description: '{{ doc("details_reserve_a_min_amount") }}' + + - name: details.reserve_a_withdraw_amount + description: '{{ doc("details_reserve_a_withdraw_amount") }}' + + - name: details.reserve_b_min_amount + description: '{{ doc("details_reserve_b_min_amount") }}' + + - name: details.reserve_b_withdraw_amount + description: '{{ doc("details_reserve_b_withdraw_amount") }}' + + - name: details.shares + description: '{{ doc("details_shares") }}' + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: details.asset_balance_changes + description: '{{ doc("details_asset_balance_changes") }}' + + - name: details.parameters + description: '{{ doc("details_parameters") }}' + + - name: details.function + description: '{{ doc("details_function") }}' + + - name: details.address + description: '{{ doc("from_address") }}' + + - name: details.type + description: '{{ doc("details_type") }}' + + - name: details.extend_to + description: '{{ doc("details_extend_to") }}' + + - name: details.contract_id + description: '{{ doc("contract_id") }}' + + - name: details.contract_code_hash + description: '{{ doc("contract_code_hash") }}' + + - name: details.operation_result_code + description: '{{ doc("operation_result_code") }}' + + - name: details.operation_trace_code + description: '{{ doc("operation_trace_code") }}' + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null + + - name: details_json + description: '{{ doc("details") }}' + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null diff --git a/models/staging/stg_history_trades.yml b/models/staging/stg_history_trades.yml new file mode 100644 index 0000000..9e3ca70 --- /dev/null +++ b/models/staging/stg_history_trades.yml @@ -0,0 +1,142 @@ +version: 2 + +models: + - name: stg_history_trades + tests: + - dbt_utils.recency: + datepart: day + field: cast(ledger_closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - history_operation_id + - "`order`" + meta: + description: "Tests the uniqueness combination of: history_operation_id and order." + columns: + - name: history_operation_id + description: '{{ doc("history_operation_id") }}' + tests: + - not_null + + - name: "`order`" + description: '{{ doc("order") }}' + tests: + - not_null + + - name: ledger_closed_at + description: '{{ doc("ledger_closed_at") }}' + tests: + - not_null + + - name: selling_account_address + description: '{{ doc("selling_account_address") }}' + tests: + - not_null + + - name: selling_asset_code + description: '{{ doc("trade_selling_asset_code") }}' + tests: + - not_null + + - name: selling_asset_issuer + description: '{{ doc("trade_selling_asset_issuer") }}' + tests: + - not_null + + - name: selling_asset_type + description: '{{ doc("trade_selling_asset_type") }}' + tests: + - not_null + + - name: selling_asset_id + description: '{{ doc("assets_id") }}' + tests: + - not_null + + - name: selling_amount + description: '{{ doc("trade_selling_amount") }}' + tests: + - not_null + + - name: buying_account_address + description: '{{ doc("buying_account_address") }}' + tests: + - not_null + + - name: buying_asset_code + description: '{{ doc("trade_buying_asset_code") }}' + tests: + - not_null + + - name: buying_asset_issuer + description: '{{ doc("trade_buying_asset_issuer") }}' + tests: + - not_null + + - name: buying_asset_type + description: '{{ doc("trade_buying_asset_type") }}' + tests: + - not_null + + - name: buying_asset_id + description: '{{ doc("assets_id") }}' + tests: + - not_null + + - name: buying_amount + description: '{{ doc("trade_buying_amount") }}' + tests: + - not_null + + - name: price_n + description: '{{ doc("trade_price_n") }}' + tests: + - not_null + + - name: price_d + description: '{{ doc("trade_price_d") }}' + tests: + - not_null + + - name: selling_offer_id + description: '{{ doc("selling_offer_id") }}' + + - name: buying_offer_id + description: '{{ doc("buying_offer_id") }}' + + - name: selling_liquidity_pool_id + description: '{{ doc("selling_liquidity_pool_id") }}' + + - name: liquidity_pool_fee + description: '{{ doc("liquidity_pool_fee") }}' + + - name: trade_type + description: '{{ doc("trade_type") }}' + tests: + - not_null + + - name: rounding_slippage + description: '{{ doc("rounding_slippage") }}' + + - name: seller_is_exact + description: '{{ doc("seller_is_exact") }}' + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null diff --git a/models/staging/stg_history_transactions.yml b/models/staging/stg_history_transactions.yml new file mode 100644 index 0000000..3b8908a --- /dev/null +++ b/models/staging/stg_history_transactions.yml @@ -0,0 +1,189 @@ +version: 2 + +models: + - name: stg_history_transactions + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + columns: + - name: id + description: '{{ doc("transaction_id") }}' + tests: + - unique + - not_null + + - name: transaction_hash + description: '{{ doc("transaction_hash") }}' + tests: + - not_null + + - name: ledger_sequence + description: '{{ doc("ledger_sequence") }}' + tests: + - not_null + + - name: application_order + description: '{{ doc("application_order") }}' + + - name: account + description: '{{ doc("account") }}' + tests: + - not_null + + - name: account_sequence + description: '{{ doc("account_sequence") }}' + tests: + - not_null + + - name: max_fee + description: '{{ doc("max_fee") }}' + tests: + - not_null + + - name: operation_count + description: '{{ doc("transaction_operation_count") }}' + tests: + - not_null + + - name: created_at + description: '{{ doc("created_at") }}' + tests: + - not_null + + - name: memo_type + description: '{{ doc("memo_type") }}' + tests: + - not_null + + - name: memo + description: '{{ doc("memo") }}' + + - name: time_bounds + description: '{{ doc("time_bounds") }}' + tests: + - not_null + + - name: successful + description: '{{ doc("successful") }}' + tests: + - not_null + + - name: fee_charged + description: '{{ doc("fee_charged") }}' + tests: + - not_null + + - name: inner_transaction_hash + description: '{{ doc("inner_transaction_hash") }}' + + - name: fee_account + description: '{{ doc("fee_account") }}' + + - name: new_max_fee + description: '{{ doc("new_max_fee") }}' + + - name: account_muxed + description: '{{ doc("account_muxed") }}' + + - name: fee_account_muxed + description: '{{ doc("fee_account_muxed") }}' + + - name: ledger_bounds + description: '{{ doc("ledger_bounds") }}' + + - name: min_account_sequence + description: '{{ doc("min_account_sequence") }}' + + - name: min_account_sequence_age + description: '{{ doc("min_account_sequence_age") }}' + + - name: min_account_sequence_ledger_gap + description: '{{ doc("min_account_sequence_ledger_gap") }}' + + - name: extra_signers + description: '{{ doc("extra_signers") }}' + + - name: tx_signers + description: '{{ doc("tx_signers") }}' + + - name: tx_envelope + description: '{{ doc("tx_envelope") }}' + tests: + - not_null + + - name: tx_result + description: '{{ doc("tx_result") }}' + tests: + - not_null + + - name: tx_meta + description: '{{ doc("tx_meta") }}' + tests: + - not_null + + - name: tx_fee_meta + description: '{{ doc("tx_fee_meta") }}' + tests: + - not_null + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null + + - name: resource_fee + description: '{{ doc("resource_fee") }}' + + - name: soroban_resources_instructions + description: '{{ doc("soroban_resources_instructions") }}' + + - name: soroban_resources_read_bytes + description: '{{ doc("soroban_resources_read_bytes") }}' + + - name: soroban_resources_write_bytes + description: '{{ doc("soroban_resources_read_bytes") }}' + + - name: transaction_result_code + description: '{{ doc("transaction_result_code") }}' + + - name: inclusion_fee_bid + description: '{{ doc("inclusion_fee_bid") }}' + + - name: inclusion_fee_charged + description: '{{ doc("inclusion_fee_charged") }}' + + - name: resource_fee_refund + description: '{{ doc("resource_fee_refund") }}' + + - name: non_refundable_resource_fee_charged + description: '{{ doc("non_refundable_resource_fee_charged") }}' + + - name: refundable_resource_fee_charged + description: '{{ doc("refundable_resource_fee_charged") }}' + + - name: rent_fee_charged + description: '{{ doc("rent_fee_charged") }}' + + - name: refundable_fee + description: '{{ doc("refundable_fee") }}' + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null diff --git a/models/staging/stg_liquidity_pools.yml b/models/staging/stg_liquidity_pools.yml new file mode 100644 index 0000000..977a9bb --- /dev/null +++ b/models/staging/stg_liquidity_pools.yml @@ -0,0 +1,128 @@ +version: 2 + +models: + - name: stg_liquidity_pools + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - liquidity_pool_id + - ledger_entry_change + - last_modified_ledger + meta: + description: "Tests the uniqueness combination of: liquidity_pool_id, ledger_entry_change and last_modified_ledger." + columns: + - name: liquidity_pool_id + description: '{{ doc("liquidity_pool_id") }}' + tests: + - not_null + + - name: type + description: '{{ doc("pool_type") }}' + tests: + - not_null + + - name: fee + description: '{{ doc("fee") }}' + tests: + - not_null + + - name: trustline_count + description: '{{ doc("trustline_count") }}' + tests: + - not_null + + - name: pool_share_count + description: '{{ doc("pool_share_count") }}' + tests: + - not_null + + - name: asset_a_id + description: '{{ doc("asset_a_id") }}' + tests: + - not_null + + - name: asset_a_type + description: '{{ doc("asset_a_type") }}' + tests: + - not_null + + - name: asset_a_code + description: '{{ doc("asset_a_code") }}' + tests: + - not_null + + - name: asset_a_issuer + description: '{{ doc("asset_a_issuer") }}' + tests: + - not_null + + - name: asset_a_amount + description: '{{ doc("asset_a_amount") }}' + tests: + - not_null + + - name: asset_b_id + description: '{{ doc("asset_b_id") }}' + tests: + - not_null + + - name: asset_b_type + description: '{{ doc("asset_a_type") }}' + tests: + - not_null + + - name: asset_b_code + description: '{{ doc("asset_a_code") }}' + tests: + - not_null + + - name: asset_b_issuer + description: '{{ doc("asset_a_issuer") }}' + tests: + - not_null + + - name: asset_b_amount + description: '{{ doc("asset_a_amount") }}' + tests: + - not_null + + - name: last_modified_ledger + description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null + + - name: ledger_entry_change + description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null + - accepted_values: + values: [0, 1, 2] + quote: false + + - name: deleted + description: '{{ doc("deleted") }}' + tests: + - not_null + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null diff --git a/models/staging/stg_offers.yml b/models/staging/stg_offers.yml new file mode 100644 index 0000000..5e33ef6 --- /dev/null +++ b/models/staging/stg_offers.yml @@ -0,0 +1,139 @@ +version: 2 + +models: + - name: stg_offers + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - offer_id + - ledger_entry_change + - last_modified_ledger + meta: + description: "Tests the uniqueness combination of: offer_id, ledger_entry_change and last_modified_ledger." + columns: + - name: seller_id + description: '{{ doc("seller_id") }}' + tests: + - not_null + + - name: offer_id + description: '{{ doc("offer_id") }}' + tests: + - not_null + + - name: selling_asset_type + description: '{{ doc("asset_type") }}' + tests: + - not_null + + - name: selling_asset_code + description: '{{ doc("asset_code") }}' + tests: + - not_null + + - name: selling_asset_issuer + description: '{{ doc("asset_issuer") }}' + tests: + - not_null + + - name: buying_asset_type + description: '{{ doc("asset_type") }}' + tests: + - not_null + + - name: buying_asset_code + description: '{{ doc("asset_code") }}' + tests: + - not_null + + - name: buying_asset_issuer + description: '{{ doc("asset_issuer") }}' + tests: + - not_null + + - name: amount + description: '{{ doc("amount") }}' + tests: + - not_null + + - name: pricen + description: '{{ doc("price_n") }}' + tests: + - not_null + + - name: priced + description: '{{ doc("price_d") }}' + tests: + - not_null + + - name: price + description: '{{ doc("price") }}' + tests: + - not_null + + - name: flags + description: '{{ doc("flags_offers") }}' + tests: + - not_null + + - name: last_modified_ledger + description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null + + - name: ledger_entry_change + description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null + - accepted_values: + values: [0, 1, 2] + quote: false + + - name: deleted + description: '{{ doc("deleted") }}' + tests: + - not_null + + - name: sponsor + description: '{{ doc("sponsor") }}' + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: selling_asset_id + description: '{{ doc("assets_id") }}' + tests: + - not_null + + - name: buying_asset_id + description: '{{ doc("assets_id") }}' + tests: + - not_null + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null + + - name: ledger_sequence + description: '{{ doc("ledger_sequence") }}' + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null diff --git a/models/staging/stg_trust_lines.yml b/models/staging/stg_trust_lines.yml new file mode 100644 index 0000000..30f592c --- /dev/null +++ b/models/staging/stg_trust_lines.yml @@ -0,0 +1,128 @@ +version: 2 + +models: + - name: stg_trust_lines + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - account_id + - asset_code + - asset_issuer + - liquidity_pool_id + - ledger_entry_change + - last_modified_ledger + meta: + description: "Tests the uniqueness combination of: account_id, asset_code, asset_issuer, liquidity_pool_id, ledger_entry_change and last_modified_ledger." + columns: + - name: ledger_key + description: '{{ doc("ledger_key") }}' + tests: + - not_null + - name: account_id + description: '{{ doc("account_id") }}' + tests: + - not_null + + - name: asset_type + description: '{{ doc("asset_type") }}' + tests: + - not_null + - accepted_values: + values: + ["credit_alphanum4", "credit_alphanum12", "pool_share"] + + - name: asset_issuer + description: '{{ doc("asset_issuer") }}' + tests: + - not_null + + - name: asset_code + description: '{{ doc("asset_code") }}' + tests: + - not_null + + - name: liquidity_pool_id + description: '{{ doc("liquidity_pool_id") }}' + tests: + - not_null + + - name: balance + description: '{{ doc("trust_balance") }}' + tests: + - not_null + + - name: trust_line_limit + description: '{{ doc("trust_line_limit") }}' + tests: + - not_null + + - name: buying_liabilities + description: '{{ doc("buying_liabilities") }}' + tests: + - not_null + + - name: selling_liabilities + description: '{{ doc("selling_liabilities") }}' + tests: + - not_null + + - name: flags + description: '{{ doc("flags_trust_lines") }}' + tests: + - not_null + + - name: last_modified_ledger + description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null + + - name: ledger_entry_change + description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null + - accepted_values: + values: [0, 1, 2] + quote: false + - name: deleted + description: '{{ doc("deleted") }}' + tests: + - not_null + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: sponsor + description: '{{ doc("sponsor") }}' + + - name: asset_id + description: asset id + tests: + - not_null + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null + + - name: ledger_sequence + description: '{{ doc("ledger_sequence") }}' + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null diff --git a/models/staging/stg_ttl.yml b/models/staging/stg_ttl.yml new file mode 100644 index 0000000..d26182f --- /dev/null +++ b/models/staging/stg_ttl.yml @@ -0,0 +1,68 @@ +version: 2 + +models: + - name: stg_ttl + tests: + - dbt_utils.recency: + datepart: day + field: cast(closed_at as timestamp) + interval: 2 + config: + severity: '{{ "error" if target.name == "prod" else "warn" }}' + meta: + description: "Monitors the freshness of your table over time, as the expected time between data updates." + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - key_hash + - live_until_ledger_seq + - last_modified_ledger + - ledger_entry_change + meta: + description: "Tests the uniqueness combination of: key_hash, live_until_ledger_seq, last_modified_ledger, and ledger_entry_change." + columns: + - name: key_hash + description: '{{ doc("key_hash") }}' + tests: + - not_null + + - name: live_until_ledger_seq + description: '{{ doc("live_until_ledger_seq") }}' + tests: + - not_null + + - name: last_modified_ledger + description: '{{ doc("last_modified_ledger") }}' + tests: + - not_null + + - name: ledger_entry_change + description: '{{ doc("ledger_entry_change") }}' + tests: + - not_null + - accepted_values: + values: [0, 1, 2] + quote: false + - name: deleted + description: '{{ doc("deleted") }}' + tests: + - not_null + + - name: batch_id + description: '{{ doc("batch_id") }}' + tests: + - not_null + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + tests: + - not_null + + - name: closed_at + description: '{{ doc("closed_at") }}' + tests: + - not_null + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + tests: + - not_null From 2c43f932bff993d5e5b4983b583e2fcd9edb0082 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Thu, 26 Sep 2024 17:14:02 +0530 Subject: [PATCH 02/18] lint --- models/staging/stg_trust_lines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/staging/stg_trust_lines.yml b/models/staging/stg_trust_lines.yml index 30f592c..365a07b 100644 --- a/models/staging/stg_trust_lines.yml +++ b/models/staging/stg_trust_lines.yml @@ -36,8 +36,7 @@ models: tests: - not_null - accepted_values: - values: - ["credit_alphanum4", "credit_alphanum12", "pool_share"] + values: ["credit_alphanum4", "credit_alphanum12", "pool_share"] - name: asset_issuer description: '{{ doc("asset_issuer") }}' From 347619627ea7b7322f6f255a55c9ca1eb5c2844c Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 27 Sep 2024 15:44:09 +0530 Subject: [PATCH 03/18] update col names --- models/staging/stg_accounts.yml | 2 +- models/staging/stg_history_ledgers.yml | 4 ++-- models/staging/stg_history_operations.yml | 6 +++--- models/staging/stg_history_transactions.yml | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/models/staging/stg_accounts.yml b/models/staging/stg_accounts.yml index 16f0628..846ccd1 100644 --- a/models/staging/stg_accounts.yml +++ b/models/staging/stg_accounts.yml @@ -72,7 +72,7 @@ models: description: '{{ doc("threshold_high") }}' tests: - not_null - - name: last_modifed_ledger + - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' tests: - not_null diff --git a/models/staging/stg_history_ledgers.yml b/models/staging/stg_history_ledgers.yml index 5fa3792..62d9641 100644 --- a/models/staging/stg_history_ledgers.yml +++ b/models/staging/stg_history_ledgers.yml @@ -33,7 +33,7 @@ models: tests: - not_null - - name: operation_count + - name: ledger_operation_count description: '{{ doc("operation_count") }}' tests: - not_null @@ -43,7 +43,7 @@ models: tests: - not_null - - name: id + - name: ledger_id description: '{{ doc("ledger_id") }}' tests: - not_null diff --git a/models/staging/stg_history_operations.yml b/models/staging/stg_history_operations.yml index e4321d5..ab8e5fd 100644 --- a/models/staging/stg_history_operations.yml +++ b/models/staging/stg_history_operations.yml @@ -12,18 +12,18 @@ models: meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: - - name: id + - name: op_id description: '{{ doc("operation_id") }}' tests: - unique - not_null - - name: source_account + - name: op_source_account description: '{{ doc("source_account") }}' tests: - not_null - - name: source_account_muxed + - name: op_source_account_muxed description: '{{ doc("source_account_muxed") }}' - name: transaction_id diff --git a/models/staging/stg_history_transactions.yml b/models/staging/stg_history_transactions.yml index 3b8908a..f44ecab 100644 --- a/models/staging/stg_history_transactions.yml +++ b/models/staging/stg_history_transactions.yml @@ -12,7 +12,7 @@ models: meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: - - name: id + - name: transaction_id description: '{{ doc("transaction_id") }}' tests: - unique @@ -31,7 +31,7 @@ models: - name: application_order description: '{{ doc("application_order") }}' - - name: account + - name: txn_account description: '{{ doc("account") }}' tests: - not_null @@ -46,12 +46,12 @@ models: tests: - not_null - - name: operation_count + - name: txn_operation_count description: '{{ doc("transaction_operation_count") }}' tests: - not_null - - name: created_at + - name: txn_created_at description: '{{ doc("created_at") }}' tests: - not_null From 0ef36d0538246210b7ba3759a2076995dd092202 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 27 Sep 2024 17:31:11 +0530 Subject: [PATCH 04/18] update tests --- models/staging/stg_account_signers.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/models/staging/stg_account_signers.yml b/models/staging/stg_account_signers.yml index dae2a06..535b2cf 100644 --- a/models/staging/stg_account_signers.yml +++ b/models/staging/stg_account_signers.yml @@ -34,8 +34,6 @@ models: - not_null - name: sponsor description: '{{ doc("sponsor") }}' - tests: - - not_null - name: last_modified_ledger tests: - not_null From addc863f37e49638736448a1e42df8a4d2cee468 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Mon, 30 Sep 2024 09:38:38 -0500 Subject: [PATCH 05/18] Remove recency test fot config setting --- models/staging/stg_config_settings.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/models/staging/stg_config_settings.yml b/models/staging/stg_config_settings.yml index 2ad4cba..e154d4d 100644 --- a/models/staging/stg_config_settings.yml +++ b/models/staging/stg_config_settings.yml @@ -3,14 +3,6 @@ version: 2 models: - name: stg_config_settings tests: - - dbt_utils.recency: - datepart: day - field: cast(closed_at as timestamp) - interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' - meta: - description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: combination_of_columns: - config_setting_id From b13c166d1f43cfe895c944d0590aefa11082d595 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Mon, 30 Sep 2024 09:42:44 -0500 Subject: [PATCH 06/18] error when staging model fails for recency tests --- models/staging/stg_account_signers.yml | 2 -- models/staging/stg_accounts.yml | 2 -- models/staging/stg_claimable_balances.yml | 2 -- models/staging/stg_contract_code.yml | 2 -- models/staging/stg_contract_data.yml | 2 -- models/staging/stg_history_assets.yml | 2 -- models/staging/stg_history_effects.yml | 2 -- models/staging/stg_history_ledgers.yml | 2 -- models/staging/stg_history_operations.yml | 2 -- models/staging/stg_history_trades.yml | 2 -- models/staging/stg_history_transactions.yml | 2 -- models/staging/stg_liquidity_pools.yml | 2 -- models/staging/stg_offers.yml | 2 -- models/staging/stg_trust_lines.yml | 2 -- models/staging/stg_ttl.yml | 2 -- 15 files changed, 30 deletions(-) diff --git a/models/staging/stg_account_signers.yml b/models/staging/stg_account_signers.yml index 535b2cf..ec7dc1b 100644 --- a/models/staging/stg_account_signers.yml +++ b/models/staging/stg_account_signers.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: diff --git a/models/staging/stg_accounts.yml b/models/staging/stg_accounts.yml index 846ccd1..8c15096 100644 --- a/models/staging/stg_accounts.yml +++ b/models/staging/stg_accounts.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: diff --git a/models/staging/stg_claimable_balances.yml b/models/staging/stg_claimable_balances.yml index 6abc663..027518a 100644 --- a/models/staging/stg_claimable_balances.yml +++ b/models/staging/stg_claimable_balances.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: diff --git a/models/staging/stg_contract_code.yml b/models/staging/stg_contract_code.yml index aa6b011..79c9558 100644 --- a/models/staging/stg_contract_code.yml +++ b/models/staging/stg_contract_code.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: diff --git a/models/staging/stg_contract_data.yml b/models/staging/stg_contract_data.yml index 909a004..e179e32 100644 --- a/models/staging/stg_contract_data.yml +++ b/models/staging/stg_contract_data.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: diff --git a/models/staging/stg_history_assets.yml b/models/staging/stg_history_assets.yml index a6be6db..91fe27c 100644 --- a/models/staging/stg_history_assets.yml +++ b/models/staging/stg_history_assets.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: diff --git a/models/staging/stg_history_effects.yml b/models/staging/stg_history_effects.yml index 658ef92..6ebf6b9 100644 --- a/models/staging/stg_history_effects.yml +++ b/models/staging/stg_history_effects.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: diff --git a/models/staging/stg_history_ledgers.yml b/models/staging/stg_history_ledgers.yml index 62d9641..4fe4ec7 100644 --- a/models/staging/stg_history_ledgers.yml +++ b/models/staging/stg_history_ledgers.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: diff --git a/models/staging/stg_history_operations.yml b/models/staging/stg_history_operations.yml index ab8e5fd..8ef24d0 100644 --- a/models/staging/stg_history_operations.yml +++ b/models/staging/stg_history_operations.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: diff --git a/models/staging/stg_history_trades.yml b/models/staging/stg_history_trades.yml index 9e3ca70..9a42b58 100644 --- a/models/staging/stg_history_trades.yml +++ b/models/staging/stg_history_trades.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(ledger_closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: diff --git a/models/staging/stg_history_transactions.yml b/models/staging/stg_history_transactions.yml index f44ecab..a2053e7 100644 --- a/models/staging/stg_history_transactions.yml +++ b/models/staging/stg_history_transactions.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: diff --git a/models/staging/stg_liquidity_pools.yml b/models/staging/stg_liquidity_pools.yml index 977a9bb..67ea878 100644 --- a/models/staging/stg_liquidity_pools.yml +++ b/models/staging/stg_liquidity_pools.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: diff --git a/models/staging/stg_offers.yml b/models/staging/stg_offers.yml index 5e33ef6..6f2c844 100644 --- a/models/staging/stg_offers.yml +++ b/models/staging/stg_offers.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: diff --git a/models/staging/stg_trust_lines.yml b/models/staging/stg_trust_lines.yml index 365a07b..4c4f604 100644 --- a/models/staging/stg_trust_lines.yml +++ b/models/staging/stg_trust_lines.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: diff --git a/models/staging/stg_ttl.yml b/models/staging/stg_ttl.yml index d26182f..78b5178 100644 --- a/models/staging/stg_ttl.yml +++ b/models/staging/stg_ttl.yml @@ -7,8 +7,6 @@ models: datepart: day field: cast(closed_at as timestamp) interval: 2 - config: - severity: '{{ "error" if target.name == "prod" else "warn" }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: From e41f9a26bc1044392dc3c92eba23a10f8d995fad Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Thu, 3 Oct 2024 12:39:35 -0500 Subject: [PATCH 07/18] rework some tests based on feedback --- models/staging/stg_account_signers.yml | 3 +++ models/staging/stg_claimable_balances.yml | 4 ---- models/staging/stg_contract_code.yml | 1 - models/staging/stg_contract_data.yml | 7 +------ models/staging/stg_history_assets.yml | 4 ---- models/staging/stg_trust_lines.yml | 4 ---- 6 files changed, 4 insertions(+), 19 deletions(-) diff --git a/models/staging/stg_account_signers.yml b/models/staging/stg_account_signers.yml index ec7dc1b..d1c3030 100644 --- a/models/staging/stg_account_signers.yml +++ b/models/staging/stg_account_signers.yml @@ -38,6 +38,9 @@ models: - name: ledger_entry_change tests: - not_null + - accepted_values: + values: [0, 1, 2] + quote: false - name: deleted description: '{{ doc("deleted") }}' tests: diff --git a/models/staging/stg_claimable_balances.yml b/models/staging/stg_claimable_balances.yml index 027518a..e93f19c 100644 --- a/models/staging/stg_claimable_balances.yml +++ b/models/staging/stg_claimable_balances.yml @@ -49,13 +49,9 @@ models: - name: asset_code description: '{{ doc("asset_code") }}' - tests: - - not_null - name: asset_issuer description: '{{ doc("asset_issuer") }}' - tests: - - not_null - name: asset_id description: '{{ doc("asset_id") }}' diff --git a/models/staging/stg_contract_code.yml b/models/staging/stg_contract_code.yml index 79c9558..597a8ba 100644 --- a/models/staging/stg_contract_code.yml +++ b/models/staging/stg_contract_code.yml @@ -12,7 +12,6 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - contract_code_hash - - contract_code_ext_v - last_modified_ledger - ledger_entry_change meta: diff --git a/models/staging/stg_contract_data.yml b/models/staging/stg_contract_data.yml index e179e32..82813b4 100644 --- a/models/staging/stg_contract_data.yml +++ b/models/staging/stg_contract_data.yml @@ -11,8 +11,7 @@ models: description: "Monitors the freshness of your table over time, as the expected time between data updates." - dbt_utils.unique_combination_of_columns: combination_of_columns: - - contract_id - - contract_data_xdr + - ledger_key_hash - last_modified_ledger - ledger_entry_change meta: @@ -35,13 +34,9 @@ models: - name: asset_code description: '{{ doc("asset_code") }}' - tests: - - not_null - name: asset_issuer description: '{{ doc("asset_issuer") }}' - tests: - - not_null - name: asset_type description: '{{ doc("asset_type") }}' diff --git a/models/staging/stg_history_assets.yml b/models/staging/stg_history_assets.yml index 91fe27c..32348d8 100644 --- a/models/staging/stg_history_assets.yml +++ b/models/staging/stg_history_assets.yml @@ -31,13 +31,9 @@ models: - name: asset_code description: '{{ doc("asset_code") }}' - tests: - - not_null - name: asset_issuer description: '{{ doc("asset_issuer") }}' - tests: - - not_null - name: batch_id description: '{{ doc("batch_id") }}' diff --git a/models/staging/stg_trust_lines.yml b/models/staging/stg_trust_lines.yml index 4c4f604..244cb06 100644 --- a/models/staging/stg_trust_lines.yml +++ b/models/staging/stg_trust_lines.yml @@ -38,13 +38,9 @@ models: - name: asset_issuer description: '{{ doc("asset_issuer") }}' - tests: - - not_null - name: asset_code description: '{{ doc("asset_code") }}' - tests: - - not_null - name: liquidity_pool_id description: '{{ doc("liquidity_pool_id") }}' From aecc5c85f4208427e7dd73e732f4598929a80ef0 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 4 Oct 2024 12:40:33 -0500 Subject: [PATCH 08/18] Run test incrementally remove quote --- models/staging/stg_account_signers.yml | 58 +++++++--- models/staging/stg_accounts.yml | 112 ++++++++++++++------ models/staging/stg_claimable_balances.yml | 73 +++++++++---- models/staging/stg_config_settings.yml | 43 +++++--- models/staging/stg_contract_code.yml | 91 +++++++++++----- models/staging/stg_contract_data.yml | 83 +++++++++++---- models/staging/stg_history_assets.yml | 32 ++++-- models/staging/stg_history_effects.yml | 48 ++++++--- models/staging/stg_history_ledgers.yml | 100 ++++++++++++----- models/staging/stg_history_operations.yml | 48 ++++++--- models/staging/stg_history_trades.yml | 92 +++++++++++----- models/staging/stg_history_transactions.yml | 88 +++++++++++---- models/staging/stg_liquidity_pools.yml | 97 ++++++++++++----- models/staging/stg_offers.yml | 101 +++++++++++++----- models/staging/stg_trust_lines.yml | 86 +++++++++++---- models/staging/stg_ttl.yml | 50 ++++++--- 16 files changed, 880 insertions(+), 322 deletions(-) diff --git a/models/staging/stg_account_signers.yml b/models/staging/stg_account_signers.yml index d1c3030..915b737 100644 --- a/models/staging/stg_account_signers.yml +++ b/models/staging/stg_account_signers.yml @@ -4,12 +4,14 @@ models: - name: stg_account_signers tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - - dbt_utils.unique_combination_of_columns: + - incremental_unique_combination_of_columns: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" combination_of_columns: - account_id - signer @@ -21,46 +23,70 @@ models: - name: account_id description: '{{ doc("account_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: signer description: '{{ doc("signer") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: weight description: '{{ doc("weight") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: sponsor description: '{{ doc("sponsor") }}' - name: last_modified_ledger tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_entry_change tests: - - not_null - - accepted_values: + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" values: [0, 1, 2] - quote: false + - name: deleted description: '{{ doc("deleted") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: closed_at tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_sequence tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: airflow_start_ts diff --git a/models/staging/stg_accounts.yml b/models/staging/stg_accounts.yml index 8c15096..cd14961 100644 --- a/models/staging/stg_accounts.yml +++ b/models/staging/stg_accounts.yml @@ -4,117 +4,169 @@ models: - name: stg_accounts tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - - dbt_utils.unique_combination_of_columns: + - incremental_unique_combination_of_columns: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" combination_of_columns: - account_id - sequence_number - ledger_entry_change - last_modified_ledger meta: - description: "Tests the uniqueness combination of: account_id, signer and ledger_sequence." + description: "Tests the uniqueness combination of: account_id, sequence_number, ledger_entry_change and last_modified_ledger." columns: - name: account_id description: '{{ doc("account_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: balance description: '{{ doc("balance") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: buying_liabilities description: '{{ doc("buying_liabilities") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: selling_liabilities description: '{{ doc("selling_liabilities") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: sequence_number description: '{{ doc("sequence_number") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: num_subentries description: '{{ doc("num_subentries") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: inflation_destination description: '{{ doc("inflation_destination") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: flags description: '{{ doc("flags_accounts_balances") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: home_domain description: '{{ doc("home_domain") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: master_weight description: '{{ doc("master_weight") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: threshold_low description: '{{ doc("threshold_low") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: threshold_medium description: '{{ doc("threshold_medium") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: threshold_high description: '{{ doc("threshold_high") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' tests: - - not_null - - accepted_values: + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" values: [0, 1, 2] - quote: false + - name: deleted description: '{{ doc("deleted") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: sponsor description: '{{ doc("sponsor") }}' - name: num_sponsored description: '{{ doc("num_sponsored") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: num_sponsoring description: '{{ doc("num_sponsoring") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: sequence_ledger description: '{{ doc("sequence_ledger") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: closed_at tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_sequence tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: airflow_start_ts diff --git a/models/staging/stg_claimable_balances.yml b/models/staging/stg_claimable_balances.yml index e93f19c..d05cc90 100644 --- a/models/staging/stg_claimable_balances.yml +++ b/models/staging/stg_claimable_balances.yml @@ -4,12 +4,14 @@ models: - name: stg_claimable_balances tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - - dbt_utils.unique_combination_of_columns: + - incremental_unique_combination_of_columns: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" combination_of_columns: - balance_id - ledger_entry_change @@ -19,7 +21,9 @@ models: - name: balance_id description: '{{ doc("balance_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: claimants description: '{{ doc("claimants") }}' @@ -45,7 +49,13 @@ models: - name: asset_type description: '{{ doc("asset_type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + values: ["credit_alphanum4", "credit_alphanum12", "native"] - name: asset_code description: '{{ doc("asset_code") }}' @@ -56,62 +66,87 @@ models: - name: asset_id description: '{{ doc("asset_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_amount description: '{{ doc("asset_amount") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: sponsor description: '{{ doc("sponsor") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: flags description: '{{ doc("flags_accounts_balances") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' tests: - - not_null - - accepted_values: + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" values: [0, 1, 2] - quote: false - name: deleted description: '{{ doc("deleted") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: closed_at tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_sequence tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: airflow_start_ts diff --git a/models/staging/stg_config_settings.yml b/models/staging/stg_config_settings.yml index e154d4d..1d6c37b 100644 --- a/models/staging/stg_config_settings.yml +++ b/models/staging/stg_config_settings.yml @@ -3,18 +3,22 @@ version: 2 models: - name: stg_config_settings tests: - - dbt_utils.unique_combination_of_columns: + - incremental_unique_combination_of_columns: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" combination_of_columns: - config_setting_id - last_modified_ledger - ledger_entry_change meta: - description: "Tests the uniqueness combination of: balance_id and ledger_entry_change." + description: "Tests the uniqueness combination of: config_setting_id, last_modified_ledger and ledger_entry_change." columns: - name: config_setting_id description: '{{ doc("config_setting_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: contract_max_size_bytes description: '{{ doc("contract_max_size_bytes") }}' @@ -145,37 +149,52 @@ models: - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' tests: - - not_null - - accepted_values: + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" values: [0, 1, 2] - quote: false - name: deleted description: '{{ doc("deleted") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: closed_at description: '{{ doc("closed_at") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" diff --git a/models/staging/stg_contract_code.yml b/models/staging/stg_contract_code.yml index 597a8ba..7e798fb 100644 --- a/models/staging/stg_contract_code.yml +++ b/models/staging/stg_contract_code.yml @@ -4,113 +4,154 @@ models: - name: stg_contract_code tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - - dbt_utils.unique_combination_of_columns: + - incremental_unique_combination_of_columns: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" combination_of_columns: - contract_code_hash - last_modified_ledger - ledger_entry_change meta: - description: "Tests the uniqueness combination of: contract_code_hash, contract_code_ext_v, last_modified_ledger, and ledger_entry_change." + description: "Tests the uniqueness combination of: contract_code_hash, last_modified_ledger, and ledger_entry_change." columns: - name: contract_code_hash description: '{{ doc("contract_code_hash") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: contract_code_ext_v description: '{{ doc("contract_code_ext_v") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: n_data_segment_bytes description: '{{ doc("n_data_segment_bytes") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: n_data_segments description: '{{ doc("n_data_segments") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: n_elem_segments description: '{{ doc("n_elem_segments") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: n_exports description: '{{ doc("n_exports") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: n_functions description: '{{ doc("n_functions") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: n_globals description: '{{ doc("n_globals") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: n_imports description: '{{ doc("n_imports") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: n_instructions description: '{{ doc("n_instructions") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: n_table_entries description: '{{ doc("n_table_entries") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: n_types description: '{{ doc("n_types") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' tests: - - not_null - - accepted_values: + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" values: [0, 1, 2] - quote: false - name: deleted description: '{{ doc("deleted") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: closed_at description: '{{ doc("closed_at") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" diff --git a/models/staging/stg_contract_data.yml b/models/staging/stg_contract_data.yml index 82813b4..908cf4f 100644 --- a/models/staging/stg_contract_data.yml +++ b/models/staging/stg_contract_data.yml @@ -4,33 +4,41 @@ models: - name: stg_contract_data tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - - dbt_utils.unique_combination_of_columns: + - incremental_unique_combination_of_columns: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" combination_of_columns: - ledger_key_hash - last_modified_ledger - ledger_entry_change meta: - description: "Tests the uniqueness combination of: contract_id, contract_data_xdr, last_modified_ledger, and ledger_entry_change." + description: "Tests the uniqueness combination of: ledger_key_hash, last_modified_ledger, and ledger_entry_change." columns: - name: contract_id description: '{{ doc("contract_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: contract_key_type description: '{{ doc("contract_key_type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: contract_durability description: '{{ doc("contract_durability") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_code description: '{{ doc("asset_code") }}' @@ -41,7 +49,13 @@ models: - name: asset_type description: '{{ doc("asset_type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + values: ["credit_alphanum4", "credit_alphanum12", "native"] - name: balance_holder description: '{{ doc("balance_holder") }}' @@ -52,62 +66,87 @@ models: - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' tests: - - not_null - - accepted_values: + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" values: [0, 1, 2] - quote: false - name: deleted description: '{{ doc("deleted") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: key description: '{{ doc("key") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: key_decoded description: '{{ doc("key_decoded") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: val description: '{{ doc("val") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: val_decoded description: '{{ doc("val_decoded") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: contract_data_xdr description: '{{ doc("contract_data_xdr") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: closed_at description: '{{ doc("closed_at") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" diff --git a/models/staging/stg_history_assets.yml b/models/staging/stg_history_assets.yml index 32348d8..2770072 100644 --- a/models/staging/stg_history_assets.yml +++ b/models/staging/stg_history_assets.yml @@ -4,12 +4,14 @@ models: - name: stg_history_assets tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - - dbt_utils.unique_combination_of_columns: + - incremental_unique_combination_of_columns: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" combination_of_columns: - asset_type - asset_code @@ -20,13 +22,19 @@ models: - name: asset_id description: '{{ doc("asset_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_type description: '{{ doc("asset_type") }}' tests: - - not_null - - accepted_values: + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" values: ["credit_alphanum4", "credit_alphanum12", "native"] - name: asset_code @@ -38,14 +46,20 @@ models: - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" diff --git a/models/staging/stg_history_effects.yml b/models/staging/stg_history_effects.yml index 6ebf6b9..61f96aa 100644 --- a/models/staging/stg_history_effects.yml +++ b/models/staging/stg_history_effects.yml @@ -4,16 +4,18 @@ models: - name: stg_history_effects tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: - name: address description: '{{ doc("address") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: address_muxed description: '{{ doc("address_muxed") }}' @@ -21,27 +23,37 @@ models: - name: operation_id description: '{{ doc("operation_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: id description: '{{ doc("id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_sequence description: '{{ doc("ledger_sequence") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: type description: '{{ doc("type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: type_string description: '{{ doc("type_string") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: details description: '{{ doc("details_effects") }}' @@ -49,12 +61,16 @@ models: - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: details.liquidity_pool description: '{{ doc("details_liquidity_pool") }}' @@ -305,14 +321,20 @@ models: - name: index description: '{{ doc("index") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: closed_at description: '{{ doc("closed_at") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" diff --git a/models/staging/stg_history_ledgers.yml b/models/staging/stg_history_ledgers.yml index 4fe4ec7..fd02396 100644 --- a/models/staging/stg_history_ledgers.yml +++ b/models/staging/stg_history_ledgers.yml @@ -4,107 +4,147 @@ models: - name: stg_history_ledgers tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: - name: sequence description: '{{ doc("sequence") }}' tests: - - unique - - not_null + - incremental_unique: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_hash description: '{{ doc("ledger_hash") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: previous_ledger_hash description: '{{ doc("previous_ledger_hash") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: transaction_count description: '{{ doc("transaction_count") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_operation_count description: '{{ doc("operation_count") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: closed_at description: '{{ doc("closed_at") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_id description: '{{ doc("ledger_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: total_coins description: '{{ doc("total_coins") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: fee_pool description: '{{ doc("fee_pool") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: base_fee description: '{{ doc("base_fee") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: base_reserve description: '{{ doc("base_reserve") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: max_tx_set_size description: '{{ doc("max_tx_set_size") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: protocol_version description: '{{ doc("protocol_version") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_header description: '{{ doc("ledger_header") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: successful_transaction_count description: '{{ doc("successful_transaction_count") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: failed_transaction_count description: '{{ doc("failed_transaction_count") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: tx_set_operation_count description: '{{ doc("tx_set_operation_count") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: soroban_fee_write_1kb description: '{{ doc("soroban_fee_write_1kb") }}' @@ -112,19 +152,27 @@ models: - name: node_id description: '{{ doc("node_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: signature description: '{{ doc("signature") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: soroban_fee_write_1kb description: '{{ doc("soroban_fee_write_1kb") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" diff --git a/models/staging/stg_history_operations.yml b/models/staging/stg_history_operations.yml index 8ef24d0..5b644b3 100644 --- a/models/staging/stg_history_operations.yml +++ b/models/staging/stg_history_operations.yml @@ -4,22 +4,28 @@ models: - name: stg_history_operations tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: - name: op_id description: '{{ doc("operation_id") }}' tests: - - unique - - not_null + - incremental_unique: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: op_source_account description: '{{ doc("source_account") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: op_source_account_muxed description: '{{ doc("source_account_muxed") }}' @@ -27,22 +33,30 @@ models: - name: transaction_id description: '{{ doc("op_transaction_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_sequence description: '{{ doc("ledger_sequence") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: type description: '{{ doc("type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: type_string description: '{{ doc("type_string") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: details description: '{{ doc("details") }}' @@ -374,12 +388,16 @@ models: - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: details.asset_balance_changes description: '{{ doc("details_asset_balance_changes") }}' @@ -414,7 +432,9 @@ models: - name: closed_at description: '{{ doc("closed_at") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: details_json description: '{{ doc("details") }}' @@ -422,4 +442,6 @@ models: - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" diff --git a/models/staging/stg_history_trades.yml b/models/staging/stg_history_trades.yml index 9a42b58..d8bc1a6 100644 --- a/models/staging/stg_history_trades.yml +++ b/models/staging/stg_history_trades.yml @@ -4,12 +4,14 @@ models: - name: stg_history_trades tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(ledger_closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - - dbt_utils.unique_combination_of_columns: + - incremental_unique_combination_of_columns: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" combination_of_columns: - history_operation_id - "`order`" @@ -19,87 +21,121 @@ models: - name: history_operation_id description: '{{ doc("history_operation_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: "`order`" description: '{{ doc("order") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: ledger_closed_at description: '{{ doc("ledger_closed_at") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: selling_account_address description: '{{ doc("selling_account_address") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: selling_asset_code description: '{{ doc("trade_selling_asset_code") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: selling_asset_issuer description: '{{ doc("trade_selling_asset_issuer") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: selling_asset_type description: '{{ doc("trade_selling_asset_type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: selling_asset_id description: '{{ doc("assets_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: selling_amount description: '{{ doc("trade_selling_amount") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: buying_account_address description: '{{ doc("buying_account_address") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: buying_asset_code description: '{{ doc("trade_buying_asset_code") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: buying_asset_issuer description: '{{ doc("trade_buying_asset_issuer") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: buying_asset_type description: '{{ doc("trade_buying_asset_type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: buying_asset_id description: '{{ doc("assets_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: buying_amount description: '{{ doc("trade_buying_amount") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: price_n description: '{{ doc("trade_price_n") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: price_d description: '{{ doc("trade_price_d") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: selling_offer_id description: '{{ doc("selling_offer_id") }}' @@ -116,7 +152,9 @@ models: - name: trade_type description: '{{ doc("trade_type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: rounding_slippage description: '{{ doc("rounding_slippage") }}' @@ -127,14 +165,20 @@ models: - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "ledger_closed_at" + greater_than_equal_to: "2 day" diff --git a/models/staging/stg_history_transactions.yml b/models/staging/stg_history_transactions.yml index a2053e7..9570c27 100644 --- a/models/staging/stg_history_transactions.yml +++ b/models/staging/stg_history_transactions.yml @@ -4,27 +4,35 @@ models: - name: stg_history_transactions tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: - name: transaction_id description: '{{ doc("transaction_id") }}' tests: - - unique - - not_null + - incremental_unique: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: transaction_hash description: '{{ doc("transaction_hash") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_sequence description: '{{ doc("ledger_sequence") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: application_order description: '{{ doc("application_order") }}' @@ -32,32 +40,44 @@ models: - name: txn_account description: '{{ doc("account") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: account_sequence description: '{{ doc("account_sequence") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: max_fee description: '{{ doc("max_fee") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: txn_operation_count description: '{{ doc("transaction_operation_count") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: txn_created_at description: '{{ doc("created_at") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: memo_type description: '{{ doc("memo_type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: memo description: '{{ doc("memo") }}' @@ -65,17 +85,23 @@ models: - name: time_bounds description: '{{ doc("time_bounds") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: successful description: '{{ doc("successful") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: fee_charged description: '{{ doc("fee_charged") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: inner_transaction_hash description: '{{ doc("inner_transaction_hash") }}' @@ -113,37 +139,51 @@ models: - name: tx_envelope description: '{{ doc("tx_envelope") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: tx_result description: '{{ doc("tx_result") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: tx_meta description: '{{ doc("tx_meta") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: tx_fee_meta description: '{{ doc("tx_fee_meta") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: closed_at description: '{{ doc("closed_at") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: resource_fee description: '{{ doc("resource_fee") }}' @@ -184,4 +224,6 @@ models: - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" diff --git a/models/staging/stg_liquidity_pools.yml b/models/staging/stg_liquidity_pools.yml index 67ea878..1f000ed 100644 --- a/models/staging/stg_liquidity_pools.yml +++ b/models/staging/stg_liquidity_pools.yml @@ -4,12 +4,14 @@ models: - name: stg_liquidity_pools tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - - dbt_utils.unique_combination_of_columns: + - incremental_unique_combination_of_columns: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" combination_of_columns: - liquidity_pool_id - ledger_entry_change @@ -20,107 +22,150 @@ models: - name: liquidity_pool_id description: '{{ doc("liquidity_pool_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: type description: '{{ doc("pool_type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: fee description: '{{ doc("fee") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: trustline_count description: '{{ doc("trustline_count") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: pool_share_count description: '{{ doc("pool_share_count") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_a_id description: '{{ doc("asset_a_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_a_type description: '{{ doc("asset_a_type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_a_code description: '{{ doc("asset_a_code") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_a_issuer description: '{{ doc("asset_a_issuer") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_a_amount description: '{{ doc("asset_a_amount") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_b_id description: '{{ doc("asset_b_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_b_type description: '{{ doc("asset_a_type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_b_code description: '{{ doc("asset_a_code") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_b_issuer description: '{{ doc("asset_a_issuer") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_b_amount description: '{{ doc("asset_a_amount") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' tests: - - not_null - - accepted_values: + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" values: [0, 1, 2] - quote: false - name: deleted description: '{{ doc("deleted") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" diff --git a/models/staging/stg_offers.yml b/models/staging/stg_offers.yml index 6f2c844..32a5806 100644 --- a/models/staging/stg_offers.yml +++ b/models/staging/stg_offers.yml @@ -4,12 +4,14 @@ models: - name: stg_offers tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - - dbt_utils.unique_combination_of_columns: + - incremental_unique_combination_of_columns: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" combination_of_columns: - offer_id - ledger_entry_change @@ -20,85 +22,118 @@ models: - name: seller_id description: '{{ doc("seller_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: offer_id description: '{{ doc("offer_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: selling_asset_type description: '{{ doc("asset_type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: selling_asset_code description: '{{ doc("asset_code") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: selling_asset_issuer description: '{{ doc("asset_issuer") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: buying_asset_type description: '{{ doc("asset_type") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: buying_asset_code description: '{{ doc("asset_code") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: buying_asset_issuer description: '{{ doc("asset_issuer") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: amount description: '{{ doc("amount") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: pricen description: '{{ doc("price_n") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: priced description: '{{ doc("price_d") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: price description: '{{ doc("price") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: flags description: '{{ doc("flags_offers") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' tests: - - not_null - - accepted_values: + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" values: [0, 1, 2] - quote: false - name: deleted description: '{{ doc("deleted") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: sponsor description: '{{ doc("sponsor") }}' @@ -106,27 +141,37 @@ models: - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: selling_asset_id description: '{{ doc("assets_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: buying_asset_id description: '{{ doc("assets_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: closed_at description: '{{ doc("closed_at") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_sequence description: '{{ doc("ledger_sequence") }}' @@ -134,4 +179,6 @@ models: - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" diff --git a/models/staging/stg_trust_lines.yml b/models/staging/stg_trust_lines.yml index 244cb06..106a0a9 100644 --- a/models/staging/stg_trust_lines.yml +++ b/models/staging/stg_trust_lines.yml @@ -4,12 +4,14 @@ models: - name: stg_trust_lines tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - - dbt_utils.unique_combination_of_columns: + - incremental_unique_combination_of_columns: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" combination_of_columns: - account_id - asset_code @@ -23,17 +25,25 @@ models: - name: ledger_key description: '{{ doc("ledger_key") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: account_id description: '{{ doc("account_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: asset_type description: '{{ doc("asset_type") }}' tests: - - not_null - - accepted_values: + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" values: ["credit_alphanum4", "credit_alphanum12", "pool_share"] - name: asset_issuer @@ -45,59 +55,83 @@ models: - name: liquidity_pool_id description: '{{ doc("liquidity_pool_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: balance description: '{{ doc("trust_balance") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: trust_line_limit description: '{{ doc("trust_line_limit") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: buying_liabilities description: '{{ doc("buying_liabilities") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: selling_liabilities description: '{{ doc("selling_liabilities") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: flags description: '{{ doc("flags_trust_lines") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' tests: - - not_null - - accepted_values: + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" values: [0, 1, 2] - quote: false + - name: deleted description: '{{ doc("deleted") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: sponsor description: '{{ doc("sponsor") }}' @@ -105,12 +139,16 @@ models: - name: asset_id description: asset id tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: closed_at description: '{{ doc("closed_at") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_sequence description: '{{ doc("ledger_sequence") }}' @@ -118,4 +156,6 @@ models: - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" diff --git a/models/staging/stg_ttl.yml b/models/staging/stg_ttl.yml index 78b5178..49cf97b 100644 --- a/models/staging/stg_ttl.yml +++ b/models/staging/stg_ttl.yml @@ -4,12 +4,14 @@ models: - name: stg_ttl tests: - dbt_utils.recency: - datepart: day + datepart: hour field: cast(closed_at as timestamp) - interval: 2 + interval: 1 meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - - dbt_utils.unique_combination_of_columns: + - incremental_unique_combination_of_columns: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" combination_of_columns: - key_hash - live_until_ledger_seq @@ -21,46 +23,66 @@ models: - name: key_hash description: '{{ doc("key_hash") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: live_until_ledger_seq description: '{{ doc("live_until_ledger_seq") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: last_modified_ledger description: '{{ doc("last_modified_ledger") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: ledger_entry_change description: '{{ doc("ledger_entry_change") }}' tests: - - not_null - - accepted_values: + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" + - incremental_accepted_values: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" values: [0, 1, 2] - quote: false + - name: deleted description: '{{ doc("deleted") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_id description: '{{ doc("batch_id") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_run_date description: '{{ doc("batch_run_date") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: closed_at description: '{{ doc("closed_at") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" - name: batch_insert_ts description: '{{ doc("batch_insert_ts") }}' tests: - - not_null + - incremental_not_null: + date_column_name: "closed_at" + greater_than_equal_to: "2 day" From a31b47da777483cddf4643a06ab1820254735e61 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 4 Oct 2024 13:22:29 -0500 Subject: [PATCH 09/18] Support quote --- models/staging/stg_claimable_balances.yml | 1 + models/staging/stg_contract_data.yml | 1 + models/staging/stg_history_assets.yml | 1 + models/staging/stg_trust_lines.yml | 1 + tests/generic/incremental_accepted_values.sql | 9 ++++++--- 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/models/staging/stg_claimable_balances.yml b/models/staging/stg_claimable_balances.yml index d05cc90..aefff98 100644 --- a/models/staging/stg_claimable_balances.yml +++ b/models/staging/stg_claimable_balances.yml @@ -56,6 +56,7 @@ models: date_column_name: "closed_at" greater_than_equal_to: "2 day" values: ["credit_alphanum4", "credit_alphanum12", "native"] + quote: true - name: asset_code description: '{{ doc("asset_code") }}' diff --git a/models/staging/stg_contract_data.yml b/models/staging/stg_contract_data.yml index 908cf4f..1fc1e14 100644 --- a/models/staging/stg_contract_data.yml +++ b/models/staging/stg_contract_data.yml @@ -56,6 +56,7 @@ models: date_column_name: "closed_at" greater_than_equal_to: "2 day" values: ["credit_alphanum4", "credit_alphanum12", "native"] + quote: true - name: balance_holder description: '{{ doc("balance_holder") }}' diff --git a/models/staging/stg_history_assets.yml b/models/staging/stg_history_assets.yml index 2770072..9327708 100644 --- a/models/staging/stg_history_assets.yml +++ b/models/staging/stg_history_assets.yml @@ -36,6 +36,7 @@ models: date_column_name: "closed_at" greater_than_equal_to: "2 day" values: ["credit_alphanum4", "credit_alphanum12", "native"] + quote: true - name: asset_code description: '{{ doc("asset_code") }}' diff --git a/models/staging/stg_trust_lines.yml b/models/staging/stg_trust_lines.yml index 106a0a9..c5d62cb 100644 --- a/models/staging/stg_trust_lines.yml +++ b/models/staging/stg_trust_lines.yml @@ -45,6 +45,7 @@ models: date_column_name: "closed_at" greater_than_equal_to: "2 day" values: ["credit_alphanum4", "credit_alphanum12", "pool_share"] + quote: true - name: asset_issuer description: '{{ doc("asset_issuer") }}' diff --git a/tests/generic/incremental_accepted_values.sql b/tests/generic/incremental_accepted_values.sql index 5e983de..c356f1c 100644 --- a/tests/generic/incremental_accepted_values.sql +++ b/tests/generic/incremental_accepted_values.sql @@ -1,12 +1,15 @@ -{% test incremental_accepted_values(model, column_name, date_column_name, greater_than_equal_to, less_than_equal_to, condition, values=[]) %} +{% test incremental_accepted_values(model, column_name, date_column_name, greater_than_equal_to, less_than_equal_to, condition, values=[], quote=false) %} {{ config(severity = 'error') }} {% set condition = condition | default('') %} {% set date_column_name = date_column_name | default('closed_at') %} {% set greater_than_equal_to = greater_than_equal_to | default('2 day') %} {% set less_than_equal_to = less_than_equal_to | default('') %} - {% set values = values | join(', ') %} - + {% if quote %} + {% set values = values | map(attribute='string') | join(', ') %} + {% else %} + {% set values = values | join(', ') %} + {% endif %} with all_values as ( From faa9ce966cc402d3826ba5a9b2973f7b7662b1e7 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 4 Oct 2024 13:24:08 -0500 Subject: [PATCH 10/18] Remove recency test for contract code --- models/staging/stg_contract_code.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/models/staging/stg_contract_code.yml b/models/staging/stg_contract_code.yml index 7e798fb..e14520c 100644 --- a/models/staging/stg_contract_code.yml +++ b/models/staging/stg_contract_code.yml @@ -3,12 +3,6 @@ version: 2 models: - name: stg_contract_code tests: - - dbt_utils.recency: - datepart: hour - field: cast(closed_at as timestamp) - interval: 1 - meta: - description: "Monitors the freshness of your table over time, as the expected time between data updates." - incremental_unique_combination_of_columns: date_column_name: "closed_at" greater_than_equal_to: "2 day" From 2b78a7c8c855720192f0eb6b0934c201859a28a5 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 4 Oct 2024 14:23:01 -0500 Subject: [PATCH 11/18] update update update update update update lint okay? update update why does it not ignore update update identify identify check was bool a prob cheeck? check update Revert update udpate check update update update update --- tests/generic/incremental_accepted_values.sql | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/generic/incremental_accepted_values.sql b/tests/generic/incremental_accepted_values.sql index c356f1c..6f1e506 100644 --- a/tests/generic/incremental_accepted_values.sql +++ b/tests/generic/incremental_accepted_values.sql @@ -1,15 +1,17 @@ -{% test incremental_accepted_values(model, column_name, date_column_name, greater_than_equal_to, less_than_equal_to, condition, values=[], quote=false) %} +{% test incremental_accepted_values(model, column_name, date_column_name, greater_than_equal_to, less_than_equal_to, condition, quote, values=[]) %} {{ config(severity = 'error') }} {% set condition = condition | default('') %} {% set date_column_name = date_column_name | default('closed_at') %} {% set greater_than_equal_to = greater_than_equal_to | default('2 day') %} {% set less_than_equal_to = less_than_equal_to | default('') %} - {% if quote %} - {% set values = values | map(attribute='string') | join(', ') %} - {% else %} - {% set values = values | join(', ') %} - {% endif %} + {% set quote = quote | default(false) %} + {% set values_string = [] %} + {% for value in values %} + {% do values_string.append("'" + value|string + "'") %} + {% endfor %} + {% set values_string = values_string | join(', ') %} + {% set values = values | join(', ') %} with all_values as ( @@ -44,7 +46,11 @@ select value_field from all_values where value_field not in ( - {{ values }} + {% if quote %} + {{ values_string }} + {% else %} + {{ values }} + {% endif %} ) {% endtest %} From 95e2ffec44078673073a7120ff538a9bdb5e678b Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 4 Oct 2024 17:55:57 -0500 Subject: [PATCH 12/18] conditional interval stringify --- models/staging/stg_account_signers.yml | 2 +- models/staging/stg_accounts.yml | 2 +- models/staging/stg_claimable_balances.yml | 2 +- models/staging/stg_contract_data.yml | 2 +- models/staging/stg_history_assets.yml | 2 +- models/staging/stg_history_effects.yml | 2 +- models/staging/stg_history_ledgers.yml | 2 +- models/staging/stg_history_operations.yml | 2 +- models/staging/stg_history_trades.yml | 2 +- models/staging/stg_history_transactions.yml | 2 +- models/staging/stg_liquidity_pools.yml | 2 +- models/staging/stg_offers.yml | 2 +- models/staging/stg_trust_lines.yml | 2 +- models/staging/stg_ttl.yml | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/models/staging/stg_account_signers.yml b/models/staging/stg_account_signers.yml index 915b737..6d3ae9a 100644 --- a/models/staging/stg_account_signers.yml +++ b/models/staging/stg_account_signers.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - incremental_unique_combination_of_columns: diff --git a/models/staging/stg_accounts.yml b/models/staging/stg_accounts.yml index cd14961..8d56516 100644 --- a/models/staging/stg_accounts.yml +++ b/models/staging/stg_accounts.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - incremental_unique_combination_of_columns: diff --git a/models/staging/stg_claimable_balances.yml b/models/staging/stg_claimable_balances.yml index aefff98..6fad69d 100644 --- a/models/staging/stg_claimable_balances.yml +++ b/models/staging/stg_claimable_balances.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - incremental_unique_combination_of_columns: diff --git a/models/staging/stg_contract_data.yml b/models/staging/stg_contract_data.yml index 1fc1e14..e041c07 100644 --- a/models/staging/stg_contract_data.yml +++ b/models/staging/stg_contract_data.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - incremental_unique_combination_of_columns: diff --git a/models/staging/stg_history_assets.yml b/models/staging/stg_history_assets.yml index 9327708..583b416 100644 --- a/models/staging/stg_history_assets.yml +++ b/models/staging/stg_history_assets.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - incremental_unique_combination_of_columns: diff --git a/models/staging/stg_history_effects.yml b/models/staging/stg_history_effects.yml index 61f96aa..f669581 100644 --- a/models/staging/stg_history_effects.yml +++ b/models/staging/stg_history_effects.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: diff --git a/models/staging/stg_history_ledgers.yml b/models/staging/stg_history_ledgers.yml index fd02396..117b888 100644 --- a/models/staging/stg_history_ledgers.yml +++ b/models/staging/stg_history_ledgers.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: diff --git a/models/staging/stg_history_operations.yml b/models/staging/stg_history_operations.yml index 5b644b3..6fe80ef 100644 --- a/models/staging/stg_history_operations.yml +++ b/models/staging/stg_history_operations.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: diff --git a/models/staging/stg_history_trades.yml b/models/staging/stg_history_trades.yml index d8bc1a6..5147435 100644 --- a/models/staging/stg_history_trades.yml +++ b/models/staging/stg_history_trades.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(ledger_closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - incremental_unique_combination_of_columns: diff --git a/models/staging/stg_history_transactions.yml b/models/staging/stg_history_transactions.yml index 9570c27..7c5c8e6 100644 --- a/models/staging/stg_history_transactions.yml +++ b/models/staging/stg_history_transactions.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." columns: diff --git a/models/staging/stg_liquidity_pools.yml b/models/staging/stg_liquidity_pools.yml index 1f000ed..5a9ba1d 100644 --- a/models/staging/stg_liquidity_pools.yml +++ b/models/staging/stg_liquidity_pools.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - incremental_unique_combination_of_columns: diff --git a/models/staging/stg_offers.yml b/models/staging/stg_offers.yml index 32a5806..e7e14bd 100644 --- a/models/staging/stg_offers.yml +++ b/models/staging/stg_offers.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - incremental_unique_combination_of_columns: diff --git a/models/staging/stg_trust_lines.yml b/models/staging/stg_trust_lines.yml index c5d62cb..b7117a2 100644 --- a/models/staging/stg_trust_lines.yml +++ b/models/staging/stg_trust_lines.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - incremental_unique_combination_of_columns: diff --git a/models/staging/stg_ttl.yml b/models/staging/stg_ttl.yml index 49cf97b..c892e78 100644 --- a/models/staging/stg_ttl.yml +++ b/models/staging/stg_ttl.yml @@ -6,7 +6,7 @@ models: - dbt_utils.recency: datepart: hour field: cast(closed_at as timestamp) - interval: 1 + interval: '{{ 1 if target.name == "prod" else 6 }}' meta: description: "Monitors the freshness of your table over time, as the expected time between data updates." - incremental_unique_combination_of_columns: From 59309b0bb110110851af27b49b050a4574368be8 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 8 Oct 2024 16:14:44 -0400 Subject: [PATCH 13/18] Remove acceptable value test for stg_contract_data --- models/staging/stg_contract_data.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/models/staging/stg_contract_data.yml b/models/staging/stg_contract_data.yml index e041c07..4f55f95 100644 --- a/models/staging/stg_contract_data.yml +++ b/models/staging/stg_contract_data.yml @@ -52,11 +52,6 @@ models: - incremental_not_null: date_column_name: "closed_at" greater_than_equal_to: "2 day" - - incremental_accepted_values: - date_column_name: "closed_at" - greater_than_equal_to: "2 day" - values: ["credit_alphanum4", "credit_alphanum12", "native"] - quote: true - name: balance_holder description: '{{ doc("balance_holder") }}' From 886670fb868385742d773d25317e19e1423b4835 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 9 Oct 2024 13:45:33 -0400 Subject: [PATCH 14/18] update sqlfluff update update update deps --- .github/workflows/dbt-docs-website.yml | 2 +- .github/workflows/diff-quality.yml | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dbt-docs-website.yml b/.github/workflows/dbt-docs-website.yml index 7e54b23..1eb9210 100644 --- a/.github/workflows/dbt-docs-website.yml +++ b/.github/workflows/dbt-docs-website.yml @@ -28,7 +28,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.12 - name: Checkout Repository uses: actions/checkout@v2 diff --git a/.github/workflows/diff-quality.yml b/.github/workflows/diff-quality.yml index 4541b58..eaacb52 100644 --- a/.github/workflows/diff-quality.yml +++ b/.github/workflows/diff-quality.yml @@ -28,7 +28,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.12 - id: file_changes uses: trilom/file-changes-action@v1.2.4 @@ -52,7 +52,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.8 + python-version: 3.12 - name: Install dependencies run: | diff --git a/requirements.txt b/requirements.txt index a74d728..ae478c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ elementary-data[bigquery]==0.15.1 MarkupSafe==2.0.1 pre-commit==3.5.0 pytz==2022.7.1 -sqlfluff==3.0.7 -sqlfluff-templater-dbt==3.0.7 +sqlfluff==3.1.0 +sqlfluff-templater-dbt==3.1.0 From 8c5d5a16b7b2e269f6e0e16daea5e46442afd952 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 9 Oct 2024 15:47:24 -0400 Subject: [PATCH 15/18] empty --- .sqlfluffignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.sqlfluffignore b/.sqlfluffignore index 4296d45..884b201 100644 --- a/.sqlfluffignore +++ b/.sqlfluffignore @@ -4,3 +4,4 @@ snapshots/ env/ dbt_packages/ target/ + From 6e1e903dd4a7c5ddd0bc88291925b9c065cf4c60 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 9 Oct 2024 15:50:17 -0400 Subject: [PATCH 16/18] remove tests from ignore --- .sqlfluffignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.sqlfluffignore b/.sqlfluffignore index 884b201..4d28c93 100644 --- a/.sqlfluffignore +++ b/.sqlfluffignore @@ -1,7 +1,5 @@ -tests/ macros/ snapshots/ env/ dbt_packages/ target/ - From 5932fb54f63b14fcd75710be04c4d3514a50ad5d Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 9 Oct 2024 15:50:42 -0400 Subject: [PATCH 17/18] add tests to sqlfluff ignore --- .sqlfluffignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.sqlfluffignore b/.sqlfluffignore index 4d28c93..4296d45 100644 --- a/.sqlfluffignore +++ b/.sqlfluffignore @@ -1,3 +1,4 @@ +tests/ macros/ snapshots/ env/ From e901010bf9eede20152a13424eb3b94175e1007f Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 9 Oct 2024 16:00:13 -0400 Subject: [PATCH 18/18] add test as macro in sqlfluff --- .sqlfluff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.sqlfluff b/.sqlfluff index c78a6ae..e1ab52f 100644 --- a/.sqlfluff +++ b/.sqlfluff @@ -87,7 +87,7 @@ profiles_dir = ./ profile = stellar_dbt_public [sqlfluff:templater:jinja] -load_macros_from_path = macros/percentile_iteration.sql,dbt_packages/dbt_utils/macros/ +load_macros_from_path = macros/percentile_iteration.sql,dbt_packages/dbt_utils/macros/,tests/generic/incremental_accepted_values.sql # Some rules can be configured directly from the config common to other rules [sqlfluff:rules]