diff --git a/models/docs/marts/ledger_current_state/config_settings_current.md b/models/docs/marts/ledger_current_state/config_settings_current.md new file mode 100644 index 0000000..2976a2b --- /dev/null +++ b/models/docs/marts/ledger_current_state/config_settings_current.md @@ -0,0 +1,5 @@ +[comment]: < Config Settings current - + +{% docs config_settings_current %} +This table maps a given config_setting_id with its current network setting value. +{% enddocs %} diff --git a/models/docs/marts/ledger_current_state/contract_code_current.md b/models/docs/marts/ledger_current_state/contract_code_current.md new file mode 100644 index 0000000..dc6ed47 --- /dev/null +++ b/models/docs/marts/ledger_current_state/contract_code_current.md @@ -0,0 +1,5 @@ +[comment]: < Contract Code current - + +{% docs contract_code_current %} +This table maps a given contract code hash to its current status. This table does not contain the physical uploaded WASM. Block explorers and other data indexers should be used to view the current code. +{% enddocs %} diff --git a/models/docs/marts/ledger_current_state/contract_data_current.md b/models/docs/marts/ledger_current_state/contract_data_current.md new file mode 100644 index 0000000..17559ff --- /dev/null +++ b/models/docs/marts/ledger_current_state/contract_data_current.md @@ -0,0 +1,5 @@ +[comment]: < Contract Data current - + +{% docs contract_data_current %} +This table maps a given contract data entry (contract_id and ledgerKey hash) to its current status. In the case of SAC contracts, it will also return the current balance when a contract is the balance holder. Wallet balances are reported in the trustlines table. The Contract data table does not contain custom data saved in a Soroban contract. +{% enddocs %} diff --git a/models/docs/sources/config_settings.md b/models/docs/sources/config_settings.md index 7aa6d3b..55715f8 100644 --- a/models/docs/sources/config_settings.md +++ b/models/docs/sources/config_settings.md @@ -195,3 +195,15 @@ Max transactions in a ledger {% docs bucket_list_size_window %} Bucket list size window {% enddocs %} + +{% docs min_temporary_ttl %} +The minimum number of entries for which a temporary entry can live on the ledger. +{% enddocs %} + +{% docs min_persistent_ttl %} +The minimum number of entries for which a persisted entry can live on the ledger. +{% enddocs %} + +{% docs max_entries_to_archive %} +Maximum number of entries that emit archival meta in a single ledger +{% enddocs %} \ No newline at end of file diff --git a/models/docs/sources/history_operations.md b/models/docs/sources/history_operations.md index 0c8985d..d472467 100644 --- a/models/docs/sources/history_operations.md +++ b/models/docs/sources/history_operations.md @@ -49,6 +49,14 @@ The type of operation this operation executes - Required Field {% enddocs %} +{% docs operation_result_code %} +The result code returned when the Stellar Network applies an operation. This code is helpful for understanding failed transactions. +{% enddocs %} + +{% docs operation_trace_code %} +The trace code returned when an operation is applied to the Stellar Network. This code is helpful for understanding failure types. +{% enddocs %} + {% docs details %} Record that contains details based on the type of operation executed. Each operation will return its own relevant details, with the rest of the details as null {% enddocs %} diff --git a/models/docs/sources/history_transactions.md b/models/docs/sources/history_transactions.md index 292a2f9..e64604b 100644 --- a/models/docs/sources/history_transactions.md +++ b/models/docs/sources/history_transactions.md @@ -185,4 +185,20 @@ Number of bytes read by the transaction {% docs soroban_resources_write_bytes %} Number of bytes written by the transaction +{% enddocs %} + +{% docs transaction_result_code %} +The detailed result code that outlines why a transaction failed. This code is only useful for failed transactions. The full list of domain values can be found [here](https://pkg.go.dev/github.com/stellar/go/xdr#TransactionResultCode). +{% enddocs %} + +{% docs inclusion_fee_bid %} +The maximum bid the submitter is willing to pay for inclusion of the transaction. This fee is used to prioritize transactions that are included in the ledger. +{% enddocs %} + +{% docs inclusion_fee_charged %} +The fee charged for the transaction to be included in the ledger. +{% enddocs %} + +{% docs resource_fee_refund %} +The amount of the resource fee refunded to the transaction submitter. The refundable fees are calculated from rent, events and return value. Refundable fees are charged from teh source account before the transaction is executed and then refunded based on the actual usage. {% enddocs %} \ No newline at end of file diff --git a/models/marts/enriched_history_operations.sql b/models/marts/enriched_history/enriched_history_operations.sql similarity index 96% rename from models/marts/enriched_history_operations.sql rename to models/marts/enriched_history/enriched_history_operations.sql index 3768f8a..ca7e2a7 100644 --- a/models/marts/enriched_history_operations.sql +++ b/models/marts/enriched_history/enriched_history_operations.sql @@ -72,6 +72,10 @@ with , soroban_resources_instructions , soroban_resources_read_bytes , soroban_resources_write_bytes + , transaction_result_code + , inclusion_fee_bid + , inclusion_fee_charged + , resource_fee_refund from {{ ref('stg_history_transactions') }} where cast(batch_run_date as date) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 2 day) @@ -201,6 +205,8 @@ with , extend_to , contract_id , contract_code_hash + , operation_result_code + , operation_trace_code from {{ ref('stg_history_operations') }} where cast(batch_run_date as date) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 2 day) @@ -330,6 +336,8 @@ with , hist_ops.extend_to , hist_ops.contract_id , hist_ops.contract_code_hash + , hist_ops.operation_result_code + , hist_ops.operation_trace_code -- transaction fields , hist_trans.transaction_hash , hist_trans.ledger_sequence @@ -357,6 +365,10 @@ with , hist_trans.soroban_resources_instructions , hist_trans.soroban_resources_read_bytes , hist_trans.soroban_resources_write_bytes + , hist_trans.transaction_result_code + , hist_trans.inclusion_fee_bid + , hist_trans.inclusion_fee_charged + , hist_trans.resource_fee_refund -- ledger fields , hist_ledg.ledger_hash , hist_ledg.previous_ledger_hash diff --git a/models/marts/enriched_history_operations.yml b/models/marts/enriched_history/enriched_history_operations.yml similarity index 97% rename from models/marts/enriched_history_operations.yml rename to models/marts/enriched_history/enriched_history_operations.yml index 8f7b9af..1f00e1d 100644 --- a/models/marts/enriched_history_operations.yml +++ b/models/marts/enriched_history/enriched_history_operations.yml @@ -630,3 +630,21 @@ models: - name: soroban_resources_write_bytes description: '{{ doc("soroban_resources_write_bytes") }}' + + - name: transaction_result_code + description: '{{ doc("transaction_result_code") }}' + + - name: inclusion_fee_bid + description: '{{ doc("inclusion_fee_bid") }}' + + - name: inclusion_fee_charged + description: '{{ doc("inclusion_fee_charged") }}' + + - name: resource_fee_refund + description: '{{ doc("resource_fee_refund") }}' + + - name: operation_result_code + description: '{{ doc("operation_result_code") }}' + + - name: operation_trace_code + description: '{{ doc("operation_trace_code") }}' diff --git a/models/marts/enriched_history/enriched_history_operations_soroban.sql b/models/marts/enriched_history/enriched_history_operations_soroban.sql new file mode 100644 index 0000000..02a7583 --- /dev/null +++ b/models/marts/enriched_history/enriched_history_operations_soroban.sql @@ -0,0 +1,95 @@ +{{ config( + tags = ["enriched_history_operations"], + materialized='incremental', + unique_key=["op_id"], + partition_by={ + "field": "closed_at" + , "data_type": "timestamp" + , "granularity": "month"}, + cluster_by=["ledger_sequence", "transaction_id", "op_type"] + ) +}} + +with + eho_soroban as ( + select + enriched.op_id + , enriched.op_source_account + , enriched.op_source_account_muxed + , enriched.transaction_id + , enriched.type as op_type + , enriched.type_string + -- expanded operations details fields + , enriched.from + , enriched.asset_balance_changes + , enriched.parameters + , enriched.parameters_decoded + , enriched.function + , enriched.address + , enriched.soroban_operation_type + , enriched.extend_to + , enriched.contract_id + , enriched.contract_code_hash + , enriched.operation_result_code + , enriched.operation_trace_code + -- transaction fields + , enriched.transaction_hash + , enriched.ledger_sequence + , enriched.txn_account + , enriched.account_sequence + , enriched.max_fee + , enriched.txn_operation_count + , enriched.txn_created_at + , enriched.memo_type + , enriched.memo + , enriched.time_bounds + , enriched.successful + , enriched.fee_charged + , enriched.fee_account + , enriched.new_max_fee + , enriched.account_muxed + , enriched.fee_account_muxed + --new protocol 19 fields for transaction preconditions + , enriched.ledger_bounds + , enriched.min_account_sequence + , enriched.min_account_sequence_age + , enriched.min_account_sequence_ledger_gap + , enriched.extra_signers + , enriched.resource_fee + , enriched.soroban_resources_instructions + , enriched.soroban_resources_read_bytes + , enriched.soroban_resources_write_bytes + , enriched.transaction_result_code + , enriched.inclusion_fee_bid + , enriched.inclusion_fee_charged + , enriched.resource_fee_refund + -- ledger fields + , enriched.ledger_hash + , enriched.previous_ledger_hash + , enriched.transaction_count + , enriched.ledger_operation_count + , enriched.closed_at + , enriched.ledger_id + , enriched.total_coins + , enriched.fee_pool + , enriched.base_fee + , enriched.base_reserve + , enriched.max_tx_set_size + , enriched.protocol_version + , enriched.successful_transaction_count + , enriched.failed_transaction_count + -- general fields + , enriched.batch_id + , enriched.batch_run_date + , enriched.batch_insert_ts + from {{ ref('enriched_history_operations') }} as enriched + where + enriched.type in (24, 25, 26) + and date(closed_at) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) + {% if is_incremental() %} + and date(closed_at) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) + {% endif %} + ) + +select * +from eho_soroban \ No newline at end of file diff --git a/models/marts/enriched_history/enriched_history_operations_soroban.yml b/models/marts/enriched_history/enriched_history_operations_soroban.yml new file mode 100644 index 0000000..21b7453 --- /dev/null +++ b/models/marts/enriched_history/enriched_history_operations_soroban.yml @@ -0,0 +1,210 @@ +version: 2 + +models: + - name: enriched_history_operations_soroban + description: Aggregate table for the history operations, taking only the soroban operations + tests: + - dbt_utils.recency: + datepart: hour + field: cast(closed_at as datetime) + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + columns: + - name: op_id + description: '{{ doc("operation_id") }}' + + - name: op_source_account + description: '{{ doc("source_account") }}' + + - name: op_source_account_muxed + description: '{{ doc("source_account_muxed") }}' + + - name: transaction_id + description: '{{ doc("transaction_id") }}' + + - name: type + description: '{{ doc("type") }}' + + - name: type_string + description: '{{ doc("type_string") }}' + + - name: asset_balance_changes + description: '{{ doc("details_asset_balance_changes") }}' + + - name: parameters + description: '{{ doc("details_parameters") }}' + + - name: function + description: '{{ doc("details_function") }}' + + - name: address + description: '{{ doc("address") }}' + + - name: soroban_operation_type + description: '{{ doc("details_soroban_operation_type") }}' + + - name: extend_to + description: '{{ doc("details_extend_to") }}' + + - name: contract_id + description: '{{ doc("contract_id") }}' + + - name: contract_code_hash + description: '{{ doc("contract_code_hash") }}' + + - name: transaction_hash + description: '{{ doc("transaction_hash") }}' + + - name: ledger_sequence + description: '{{ doc("ledger_sequence") }}' + + - name: account_sequence + description: '{{ doc("account_sequence") }}' + + - name: max_fee + description: '{{ doc("max_fee") }}' + + - name: txn_operation_count + description: '{{ doc("transaction_operation_count") }}' + + - name: txn_created_at + description: '{{ doc("created_at") }}' + + - name: memo_type + description: '{{ doc("memo_type") }}' + + - name: memo_type + description: '{{ doc("memo") }}' + + - name: time_bounds + description: '{{ doc("time_bounds") }}' + + - name: successful + description: '{{ doc("successful") }}' + + - name: fee_charged + description: '{{ doc("fee_charged") }}' + + - name: fee_account + description: '{{ doc("fee_account") }}' + + - name: new_max_fee + description: '{{ doc("new_max_fee") }}' + + - name: account_muxed + description: '{{ doc("account_muxed") }}' + + - name: fee_account_muxed + description: '{{ doc("fee_account_muxed") }}' + + - name: tx_envelope + description: '{{ doc("tx_envelope") }}' + + - name: tx_result + description: '{{ doc("tx_result") }}' + + - name: tx_meta + description: '{{ doc("tx_meta") }}' + + - name: tx_fee_meta + description: '{{ doc("tx_fee_meta") }}' + + - name: ledger_bounds + description: '{{ doc("ledger_bounds") }}' + + - name: min_account_sequence + description: '{{ doc("min_account_sequence") }}' + + - name: min_account_sequence_age + description: '{{ doc("min_account_sequence_age") }}' + + - name: min_account_sequence_ledger_gap + description: '{{ doc("min_account_sequence_ledger_gap") }}' + + - name: extra_signers + description: '{{ doc("extra_signers") }}' + + - name: resource_fee + description: '{{ doc("resource_fee") }}' + + - name: soroban_resources_instructions + description: '{{ doc("soroban_resources_instructions") }}' + + - name: soroban_resources_read_bytes + description: '{{ doc("soroban_resources_read_bytes") }}' + + - name: soroban_resources_write_bytes + description: '{{ doc("soroban_resources_write_bytes") }}' + + - name: ledger_hash + description: '{{ doc("ledger_hash") }}' + + - name: previous_ledger_hash + description: '{{ doc("previous_ledger_hash") }}' + + - name: transaction_count + description: '{{ doc("transaction_count") }}' + + - name: ledger_operation_count + description: '{{ doc("operation_count") }}' + + - name: closed_at + description: '{{ doc("closed_at") }}' + + - name: ledger_id + description: '{{ doc("ledger_id") }}' + + - name: total_coins + description: '{{ doc("total_coins") }}' + + - name: fee_pool + description: '{{ doc("fee_pool") }}' + + - name: base_fee + description: '{{ doc("base_fee") }}' + + - name: base_reserve + description: '{{ doc("base_reserve") }}' + + - name: max_tx_set_size + description: '{{ doc("max_tx_set_size") }}' + + - name: protocol_version + description: '{{ doc("protocol_version") }}' + + - name: succesful_transaction_count + description: '{{ doc("successful_transaction_count") }}' + + - name: failed_transaction_count + description: '{{ doc("failed_transaction_count") }}' + + - name: batch_id + description: '{{ doc("batch_id") }}' + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + + - name: transaction_result_code + description: '{{ doc("transaction_result_code") }}' + + - name: inclusion_fee_bid + description: '{{ doc("inclusion_fee_bid") }}' + + - name: inclusion_fee_charged + description: '{{ doc("inclusion_fee_charged") }}' + + - name: resource_fee_refund + description: '{{ doc("resource_fee_refund") }}' + + - name: operation_result_code + description: '{{ doc("operation_result_code") }}' + + - name: operation_trace_code + description: '{{ doc("operation_trace_code") }}' diff --git a/models/marts/ledger_current_state/config_settings_current.sql b/models/marts/ledger_current_state/config_settings_current.sql new file mode 100644 index 0000000..3437831 --- /dev/null +++ b/models/marts/ledger_current_state/config_settings_current.sql @@ -0,0 +1,136 @@ +{{ config( + tags = ["current_state"] + , materialized='incremental' + , unique_key=["config_setting_id"] + ) +}} + +/* Finds the latest state of network config settings. + Ranks each record (grain: one row per config_setting_id)) using + the last modified ledger sequence number. */ + +with + current_settings as ( + select + cfg.config_setting_id + , cfg.contract_max_size_bytes + , cfg.ledger_max_instructions + , cfg.tx_max_instructions + , cfg.fee_rate_per_instructions_increment + , cfg.tx_memory_limit + , cfg.ledger_max_read_ledger_entries + , cfg.ledger_max_read_bytes + , cfg.ledger_max_write_ledger_entries + , cfg.ledger_max_write_bytes + , cfg.tx_max_read_ledger_entries + , cfg.tx_max_read_bytes + , cfg.tx_max_write_ledger_entries + , cfg.tx_max_write_bytes + , cfg.fee_read_ledger_entry + , cfg.fee_write_ledger_entry + , cfg.fee_read_1kb + , cfg.bucket_list_target_size_bytes + , cfg.write_fee_1kb_bucket_list_low + , cfg.write_fee_1kb_bucket_list_high + , cfg.bucket_list_write_fee_growth_factor + , cfg.fee_historical_1kb + , cfg.tx_max_contract_events_size_bytes + , cfg.fee_contract_events_1kb + , cfg.ledger_max_txs_size_bytes + , cfg.tx_max_size_bytes + , cfg.fee_tx_size_1kb + , cfg.contract_cost_params_cpu_insns + , cfg.contract_cost_params_mem_bytes + , cfg.contract_data_key_size_bytes + , cfg.contract_data_entry_size_bytes + , cfg.max_entry_ttl + , cfg.min_temporary_ttl + , cfg.min_persistent_ttl + , cfg.auto_bump_ledgers + , cfg.persistent_rent_rate_denominator + , cfg.temp_rent_rate_denominator + , cfg.max_entries_to_archive + , cfg.bucket_list_size_window_sample_size + , cfg.eviction_scan_size + , cfg.starting_eviction_scan_level + , cfg.ledger_max_tx_count + , cfg.bucket_list_size_window + , cfg.last_modified_ledger + , cfg.ledger_entry_change + , cfg.closed_at + , cfg.deleted + , cfg.batch_id + , cfg.batch_run_date + , cfg.batch_insert_ts + , cfg.ledger_sequence + , row_number() + over ( + partition by cfg.config_setting_id + order by cfg.closed_at desc + ) as rn + from {{ ref('stg_config_settings') }} as cfg + {% if is_incremental() %} + -- limit the number of partitions fetched incrementally + where + cfg.closed_at >= timestamp_sub(current_timestamp(), interval 30 day) + -- fetch the last week of records loaded + and timestamp_add(cfg.batch_insert_ts, interval 7 day) + > (select max(t.upstream_insert_ts) from {{ this }} as t) + {% endif %} + ) + +select + config_setting_id + , contract_max_size_bytes + , ledger_max_instructions + , tx_max_instructions + , fee_rate_per_instructions_increment + , tx_memory_limit + , ledger_max_read_ledger_entries + , ledger_max_read_bytes + , ledger_max_write_ledger_entries + , ledger_max_write_bytes + , tx_max_read_ledger_entries + , tx_max_read_bytes + , tx_max_write_ledger_entries + , tx_max_write_bytes + , fee_read_ledger_entry + , fee_write_ledger_entry + , fee_read_1kb + , bucket_list_target_size_bytes + , write_fee_1kb_bucket_list_low + , write_fee_1kb_bucket_list_high + , bucket_list_write_fee_growth_factor + , fee_historical_1kb + , tx_max_contract_events_size_bytes + , fee_contract_events_1kb + , ledger_max_txs_size_bytes + , tx_max_size_bytes + , fee_tx_size_1kb + , contract_cost_params_cpu_insns + , contract_cost_params_mem_bytes + , contract_data_key_size_bytes + , contract_data_entry_size_bytes + , max_entry_ttl + , min_temporary_ttl + , min_persistent_ttl + , auto_bump_ledgers + , persistent_rent_rate_denominator + , temp_rent_rate_denominator + , max_entries_to_archive + , bucket_list_size_window_sample_size + , eviction_scan_size + , starting_eviction_scan_level + , ledger_max_tx_count + , bucket_list_size_window + , last_modified_ledger + , ledger_entry_change + , ledger_sequence + , closed_at + , deleted + , batch_id + , batch_run_date + , batch_insert_ts as upstream_insert_ts + , current_timestamp() as batch_insert_ts +from current_settings +where rn = 1 diff --git a/models/marts/ledger_current_state/config_settings_current.yml b/models/marts/ledger_current_state/config_settings_current.yml new file mode 100644 index 0000000..c92f49d --- /dev/null +++ b/models/marts/ledger_current_state/config_settings_current.yml @@ -0,0 +1,156 @@ +version: 2 + +models: + - name: config_settings_current + description: '{{ doc("config_settings_current") }}' + + columns: + - name: config_setting_id + description: '{{ doc("config_setting_id") }}' + + - name: contract_max_size_bytes + description: '{{ doc("contract_max_size_bytes") }}' + + - name: ledger_max_instructions + description: '{{ doc("ledger_max_instructions") }}' + + - name: tx_max_instructions + description: '{{ doc("tx_max_instructions") }}' + + - name: fee_rate_per_instructions_increment + description: '{{ doc("fee_rate_per_instructions_increment") }}' + + - name: tx_memory_limit + description: '{{ doc("tx_memory_limit") }}' + + - name: ledger_max_read_ledger_entries + description: '{{ doc("ledger_max_read_ledger_entries") }}' + + - name: ledger_max_read_bytes + description: '{{ doc("ledger_max_read_bytes") }}' + + - name: ledger_max_write_ledger_entries + description: '{{ doc("ledger_max_write_ledger_entries") }}' + + - name: ledger_max_write_bytes + description: '{{ doc("ledger_max_write_bytes") }}' + + - name: tx_max_read_ledger_entries + description: '{{ doc("tx_max_read_ledger_entries") }}' + + - name: tx_max_read_bytes + description: '{{ doc("tx_max_read_bytes") }}' + + - name: tx_max_write_ledger_entries + description: '{{ doc("tx_max_write_ledger_entries") }}' + + - name: tx_max_write_bytes + description: '{{ doc("tx_max_write_bytes") }}' + + - name: fee_read_ledger_entry + description: '{{ doc("fee_read_ledger_entry") }}' + + - name: fee_write_ledger_entry + description: '{{ doc("fee_write_ledger_entry") }}' + + - name: fee_read_1kb + description: '{{ doc("fee_read_1kb") }}' + + - name: bucket_list_target_size_bytes + description: '{{ doc("bucket_list_target_size_bytes") }}' + + - name: write_fee_1kb_bucket_list_low + description: '{{ doc("write_fee_1kb_bucket_list_low") }}' + + - name: write_fee_1kb_bucket_list_high + description: '{{ doc("write_fee_1kb_bucket_list_high") }}' + + - name: bucket_list_write_fee_growth_factor + description: '{{ doc("bucket_list_write_fee_growth_factor") }}' + + - name: fee_historical_1kb + description: '{{ doc("fee_historical_1kb") }}' + + - name: tx_max_contract_events_size_bytes + description: '{{ doc("tx_max_contract_events_size_bytes") }}' + + - name: fee_contract_events_1kb + description: '{{ doc("fee_contract_events_1kb") }}' + + - name: ledger_max_txs_size_bytes + description: '{{ doc("ledger_max_txs_size_bytes") }}' + + - name: tx_max_size_bytes + description: '{{ doc("tx_max_size_bytes") }}' + + - name: fee_tx_size_1kb + description: '{{ doc("fee_tx_size_1kb") }}' + + - name: contract_cost_params_cpu_insns + description: '{{ doc("contract_cost_params_cpu_insns") }}' + + - name: contract_cost_params_mem_bytes + description: '{{ doc("contract_cost_params_mem_bytes") }}' + + - name: contract_data_key_size_bytes + description: '{{ doc("contract_data_key_size_bytes") }}' + + - name: contract_data_entry_size_bytes + description: '{{ doc("contract_data_entry_size_bytes") }}' + + - name: max_entry_ttl + description: '{{ doc("max_entry_ttl") }}' + + - name: min_temporary_ttl + description: '{{ doc("min_temporary_ttl") }}' + + - name: min_persistent_ttl + description: '{{ doc("min_persistent_ttl") }}' + + - name: auto_bump_ledgers + description: '{{ doc("auto_bump_ledgers") }}' + + - name: persistent_rent_rate_denominator + description: '{{ doc("persistent_rent_rate_denominator") }}' + + - name: temp_rent_rate_denominator + description: '{{ doc("temp_rent_rate_denominator") }}' + + - name: max_entries_to_archive + description: '{{ doc("max_entries_to_archive") }}' + + - name: bucket_list_size_window_sample_size + description: '{{ doc("bucket_list_size_window_sample_size") }}' + + - name: eviction_scan_size + description: '{{ doc("eviction_scan_size") }}' + + - name: starting_eviction_scan_level + description: '{{ doc("starting_eviction_scan_level") }}' + + - name: ledger_max_tx_count + description: '{{ doc("ledger_max_tx_count") }}' + + - name: bucket_list_size_window + description: '{{ doc("bucket_list_size_window") }}' + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + + - name: closed_at + description: '{{ doc("closed_at") }}' + + - name: deleted + description: '{{ doc("deleted") }}' + + - name: upstream_insert_ts + description: '{{ doc("upstream_insert_ts") }}' + + - name: last_modified_ledger + description: '{{ doc("last_modified_ledger") }}' + + - name: ledger_entry_change + description: '{{ doc("ledger_entry_change") }}' diff --git a/models/marts/ledger_current_state/contract_code_current.sql b/models/marts/ledger_current_state/contract_code_current.sql new file mode 100644 index 0000000..90c7a40 --- /dev/null +++ b/models/marts/ledger_current_state/contract_code_current.sql @@ -0,0 +1,57 @@ +{{ config( + tags = ["current_state"] + , materialized='incremental' + , unique_key=["contract_code_hash"] + , cluster_by= ["contract_code_hash"] + ) +}} + +/* Finds the latest state of each contract code entry. + Ranks each record (grain: one row per contract_code_hash)) using + the last modified ledger sequence number. */ + +with + current_code as ( + select + cc.contract_code_hash + , cc.contract_code_ext_v + , cc.last_modified_ledger + , cc.ledger_entry_change + , cc.closed_at + , cc.deleted + , cc.batch_id + , cc.batch_run_date + , cc.batch_insert_ts + , cc.ledger_sequence + , cc.ledger_key_hash + , row_number() + over ( + partition by cc.contract_code_hash + order by cc.closed_at desc + ) as rn + from {{ ref('stg_contract_code') }} as cc + {% if is_incremental() %} + -- limit the number of partitions fetched incrementally + where + cc.closed_at >= timestamp_sub(current_timestamp(), interval 30 day) + -- fetch the last week of records loaded + and timestamp_add(cc.batch_insert_ts, interval 7 day) + > (select max(t.upstream_insert_ts) from {{ this }} as t) + {% endif %} + ) + +select + contract_code_hash + , contract_code_ext_v + , last_modified_ledger + , ledger_entry_change + , ledger_sequence + , ledger_key_hash + , closed_at + , deleted + , batch_id + , batch_run_date + , batch_insert_ts as upstream_insert_ts + , current_timestamp() as batch_insert_ts +from current_code +where rn = 1 diff --git a/models/marts/ledger_current_state/contract_code_current.yml b/models/marts/ledger_current_state/contract_code_current.yml new file mode 100644 index 0000000..3a77d40 --- /dev/null +++ b/models/marts/ledger_current_state/contract_code_current.yml @@ -0,0 +1,50 @@ +version: 2 + +models: + - name: contract_code_current + description: '{{ doc("contract_code_current") }}' + + columns: + - name: contract_code_hash + description: '{{ doc("key_hash") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day + + - name: ledger_key_hash + description: '{{ doc("ledger_key_hash") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day + + - name: contract_code_ext_v + description: '{{ doc("contract_code_ext_v") }}' + + - name: last_modified_ledger + description: '{{ doc("last_modified_ledger") }}' + + - name: ledger_entry_change + description: '{{ doc("ledger_entry_change") }}' + + - name: ledger_sequence + description: '{{ doc("ledger_sequence") }}' + + - name: closed_at + description: '{{ doc("closed_at") }}' + + - name: deleted + description: '{{ doc("deleted") }}' + + - name: batch_id + description: '{{ doc("batch_id") }}' + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + + - name: upstream_insert_ts + description: '{{ doc("upstream_insert_ts") }}' + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' diff --git a/models/marts/ledger_current_state/contract_data_current.sql b/models/marts/ledger_current_state/contract_data_current.sql new file mode 100644 index 0000000..66fa5f7 --- /dev/null +++ b/models/marts/ledger_current_state/contract_data_current.sql @@ -0,0 +1,71 @@ +{{ config( + tags = ["current_state"] + , materialized='incremental' + , unique_key=["unique_id"] + , cluster_by= ["contract_id"] + ) +}} + +/* Finds the latest state of each contract data entry. + Ranks each record (grain: one row per contract_id + ledger_key_hash)) using + the last modified ledger sequence number. */ + +with + current_data as ( + select + cd.contract_id + , cd.contract_key_type + , cd.contract_durability + , cd.asset_code + , cd.asset_issuer + , cd.asset_type + , cd.balance_holder + , cd.balance + , cd.last_modified_ledger + , cd.ledger_entry_change + , cd.closed_at + , cd.deleted + , cd.batch_id + , cd.batch_run_date + , cd.batch_insert_ts + , cd.ledger_sequence + , cd.ledger_key_hash + , concat(cd.contract_id, '-', cd.ledger_key_hash) as unique_id + , row_number() + over ( + partition by cd.contract_id, cd.ledger_key_hash + order by cd.closed_at desc + ) as rn + from {{ ref('stg_contract_data') }} as cd + {% if is_incremental() %} + -- limit the number of partitions fetched incrementally + where + cd.closed_at >= timestamp_sub(current_timestamp(), interval 30 day) + -- fetch the last week of records loaded + and timestamp_add(cd.batch_insert_ts, interval 7 day) + > (select max(t.upstream_insert_ts) from {{ this }} as t) + {% endif %} + ) + +select + contract_id + , contract_key_type + , contract_durability + , asset_code + , asset_issuer + , asset_type + , balance_holder + , balance + , last_modified_ledger + , ledger_entry_change + , ledger_sequence + , ledger_key_hash + , closed_at + , deleted + , batch_id + , batch_run_date + , batch_insert_ts as upstream_insert_ts + , current_timestamp() as batch_insert_ts + , unique_id +from current_data +where rn = 1 diff --git a/models/marts/ledger_current_state/contract_data_current.yml b/models/marts/ledger_current_state/contract_data_current.yml new file mode 100644 index 0000000..0a40f71 --- /dev/null +++ b/models/marts/ledger_current_state/contract_data_current.yml @@ -0,0 +1,80 @@ +version: 2 + +models: + - name: contract_data_current + description: '{{ doc("contract_data_current") }}' + tests: + - dbt_utils.recency: + datepart: hour + field: cast(closed_at as datetime) + interval: 12 + config: + severity: warn + meta: + description: + "Monitors the freshness of your table over time, as the expected time between data updates." + columns: + - name: contract_id + description: '{{ doc("key_hash") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day + + - name: ledger_key_hash + description: '{{ doc("ledger_key_hash") }}' + tests: + - not_null: + config: + where: closed_at > current_timestamp - interval 2 day + + - name: contract_key_type + description: '{{ doc("contract_key_type") }}' + + - name: contract_durability + description: '{{ doc("contract_durability") }}' + + - name: asset_code + description: '{{ doc("asset_code") }}' + + - name: asset_issuer + description: '{{ doc("asset_issuer") }}' + + - name: asset_type + description: '{{ doc("asset_type") }}' + + - name: balance_holder + description: '{{ doc("balance_holder") }}' + + - name: balance + description: '{{ doc("balance") }}' + + - name: last_modified_ledger + description: '{{ doc("last_modified_ledger") }}' + + - name: ledger_entry_change + description: '{{ doc("ledger_entry_change") }}' + + - name: ledger_sequence + description: '{{ doc("ledger_sequence") }}' + + - name: closed_at + description: '{{ doc("closed_at") }}' + + - name: deleted + description: '{{ doc("deleted") }}' + + - name: batch_id + description: '{{ doc("batch_run_date") }}' + + - name: batch_run_date + description: '{{ doc("batch_run_date") }}' + + - name: upstream_insert_ts + description: '{{ doc("upstream_insert_ts") }}' + + - name: batch_insert_ts + description: '{{ doc("batch_insert_ts") }}' + + - name: unique_id + description: '{{ doc("unique_id") }}' \ No newline at end of file diff --git a/models/marts/ledger_current_state/ttl_current.sql b/models/marts/ledger_current_state/ttl_current.sql index 50d410a..a333bda 100644 --- a/models/marts/ledger_current_state/ttl_current.sql +++ b/models/marts/ledger_current_state/ttl_current.sql @@ -25,13 +25,13 @@ with , row_number() over ( partition by ttl.key_hash - order by ttl.last_modified_ledger desc, ledger_entry_change + order by ttl.closed_at ) as rn from {{ ref('stg_ttl') }} as ttl {% if is_incremental() %} -- limit the number of partitions fetched incrementally where - ttl.batch_run_date >= date_sub(current_date(), interval 30 day) + ttl.closed_at >= timestamp_sub(current_timestamp(), interval 30 day) -- fetch the last week of records loaded and timestamp_add(ttl.batch_insert_ts, interval 7 day) > (select max(t.upstream_insert_ts) from {{ this }} as t) diff --git a/models/sources/src_history_operations.yml b/models/sources/src_history_operations.yml index 2e874f3..089918a 100644 --- a/models/sources/src_history_operations.yml +++ b/models/sources/src_history_operations.yml @@ -407,6 +407,12 @@ sources: - name: details.contract_code_hash description: '{{ doc("contract_code_hash") }}' + - name: details.operation_result_code + description: '{{ doc("operation_result_code") }}' + + - name: details.operation_trace_code + description: '{{ doc("operation_trace_code") }}' + - name: closed_at description: '{{ doc("closed_at") }}' tests: diff --git a/models/sources/src_history_transactions.yml b/models/sources/src_history_transactions.yml index e8fca82..266c206 100644 --- a/models/sources/src_history_transactions.yml +++ b/models/sources/src_history_transactions.yml @@ -207,4 +207,16 @@ sources: description: '{{ doc("soroban_resources_read_bytes") }}' - name: soroban_resources_write_bytes - description: '{{ doc("soroban_resources_read_bytes") }}' \ No newline at end of file + description: '{{ doc("soroban_resources_read_bytes") }}' + + - name: transaction_result_code + description: '{{ doc("transaction_result_code") }}' + + - name: inclusion_fee_bid + description: '{{ doc("inclusion_fee_bid") }}' + + - name: inclusion_fee_charged + description: '{{ doc("inclusion_fee_charged") }}' + + - name: resource_fee_refund + description: '{{ doc("resource_fee_refund") }}' \ No newline at end of file diff --git a/models/staging/stg_history_operations.sql b/models/staging/stg_history_operations.sql index 6a4736e..5749ff0 100644 --- a/models/staging/stg_history_operations.sql +++ b/models/staging/stg_history_operations.sql @@ -126,6 +126,8 @@ with , details.extend_to , details.contract_id , details.contract_code_hash + , operation_result_code + , operation_trace_code , closed_at , batch_id , batch_run_date diff --git a/models/staging/stg_history_transactions.sql b/models/staging/stg_history_transactions.sql index 71a475f..38a71b4 100644 --- a/models/staging/stg_history_transactions.sql +++ b/models/staging/stg_history_transactions.sql @@ -43,6 +43,10 @@ with , soroban_resources_read_bytes , soroban_resources_write_bytes , closed_at + , transaction_result_code + , inclusion_fee_bid + , inclusion_fee_charged + , resource_fee_refund , batch_id , batch_run_date , batch_insert_ts diff --git a/models/staging/stg_liquidity_pools.sql b/models/staging/stg_liquidity_pools.sql index 11810c2..6aedbc1 100644 --- a/models/staging/stg_liquidity_pools.sql +++ b/models/staging/stg_liquidity_pools.sql @@ -19,10 +19,12 @@ with , asset_a_type , asset_a_code , asset_a_issuer + , asset_a_id , asset_a_amount , asset_b_type , asset_b_code , asset_b_issuer + , asset_b_id , asset_b_amount , last_modified_ledger , ledger_entry_change