diff --git a/blocks/evm/schema.sql b/blocks/evm/schema.sql index 066825c..5d3f6e3 100644 --- a/blocks/evm/schema.sql +++ b/blocks/evm/schema.sql @@ -125,6 +125,7 @@ CREATE TABLE IF NOT EXISTS logs -- logs -- `index` UInt32, + block_index UInt32, contract_address FixedString(42), topic0 FixedString(66), topic1 FixedString(66) DEFAULT '', @@ -277,6 +278,27 @@ CREATE TABLE IF NOT EXISTS storage_changes ORDER BY (block_date, block_number, block_hash, ordinal) COMMENT 'EVM storage changes'; +CREATE TABLE IF NOT EXISTS block_code_changes +( + -- block -- + block_time DateTime64(3, 'UTC'), + block_number UInt64, + block_hash FixedString(66), + block_date Date, + + -- code change -- + address FixedString(42), + old_hash FixedString(66) DEFAULT '', + old_code String DEFAULT '', + new_hash FixedString(66) DEFAULT '', + new_code String DEFAULT '', + ordinal UInt64 +) + ENGINE = ReplacingMergeTree() + PRIMARY KEY (block_date, block_number) + ORDER BY (block_date, block_number, block_hash, ordinal) + COMMENT 'EVM block code changes'; + CREATE TABLE IF NOT EXISTS code_changes ( -- block -- @@ -413,4 +435,39 @@ CREATE TABLE IF NOT EXISTS gas_changes ENGINE = ReplacingMergeTree() PRIMARY KEY (block_date, block_number) ORDER BY (block_date, block_number, block_hash, ordinal) - COMMENT 'EVM gas changes'; \ No newline at end of file + COMMENT 'EVM gas changes'; + +CREATE TABLE IF NOT EXISTS system_traces +( + -- block -- + block_time DateTime64(3, 'UTC'), + block_number UInt64, + block_hash FixedString(66), + block_date Date, + + -- trace -- + `index` UInt32, + parent_index UInt32, + depth UInt32, + caller FixedString(42), + call_type LowCardinality(String), + call_type_code UInt32, + address FixedString(42), + value DEFAULT '' COMMENT 'UInt256', + gas_limit UInt64, + gas_consumed UInt64, + return_data String COMMENT 'Return data is set by contract calls using RETURN or REVERT.', + input String, + suicide Bool, + failure_reason LowCardinality(String), + state_reverted Bool, + status_reverted Bool, + status_failed Bool, + executed_code Bool, + begin_ordinal UInt64, + end_ordinal UInt64 +) + ENGINE = ReplacingMergeTree() + PRIMARY KEY (block_date, block_number) + ORDER BY (block_date, block_number, `index`) + COMMENT 'EVM system traces'; \ No newline at end of file diff --git a/blocks/evm/src/blocks.rs b/blocks/evm/src/blocks.rs index 2fb9f78..281f073 100644 --- a/blocks/evm/src/blocks.rs +++ b/blocks/evm/src/blocks.rs @@ -6,6 +6,8 @@ use substreams_database_change::pb::database::{table_change, DatabaseChanges}; use substreams_ethereum::pb::eth::v2::Block; use crate::balance_changes::insert_balance_change_counts; +use crate::code_changes::insert_block_code_change; +use crate::traces::insert_system_trace; pub fn block_detail_to_string(detail_level: i32) -> String { match detail_level { @@ -93,5 +95,12 @@ pub fn insert_blocks(tables: &mut DatabaseChanges, clock: &Clock, block: &Block) let all_balance_changes_reason: Vec = block.balance_changes.iter().map(|balance_change| balance_change.reason).collect(); insert_balance_change_counts(row, all_balance_changes_reason); - // TODO: block.code_changes + // code changes + for code_change in block.code_changes.iter() { + insert_block_code_change(tables, clock, code_change); + } + + for system_call in block.system_calls.iter() { + insert_system_trace(tables, clock, system_call); + } } diff --git a/blocks/evm/src/code_changes.rs b/blocks/evm/src/code_changes.rs index d56948a..46956ea 100644 --- a/blocks/evm/src/code_changes.rs +++ b/blocks/evm/src/code_changes.rs @@ -2,7 +2,7 @@ use common::blocks::insert_timestamp; use common::keys::block_ordinal_keys; use common::utils::bytes_to_hex; use substreams::pb::substreams::Clock; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; +use substreams_database_change::pb::database::{table_change, DatabaseChanges, TableChange}; use substreams_ethereum::pb::eth::v2::{Call, CodeChange, TransactionTrace}; use crate::traces::insert_trace_metadata; @@ -10,7 +10,7 @@ use crate::transactions::insert_transaction_metadata; // https://github.com/streamingfast/firehose-ethereum/blob/1bcb32a8eb3e43347972b6b5c9b1fcc4a08c751e/proto/sf/ethereum/type/v2/type.proto#L744 // DetailLevel: EXTENDED -pub fn insert_code_change(tables: &mut DatabaseChanges, clock: &Clock, code_change: &CodeChange, transaction: &TransactionTrace, trace: &Call) { +pub fn insert_code_change(row: &mut TableChange, code_change: &CodeChange) { let address = bytes_to_hex(code_change.address.clone()); let old_hash = bytes_to_hex(code_change.old_hash.clone()); let old_code = bytes_to_hex(code_change.old_code.clone()); @@ -18,17 +18,30 @@ pub fn insert_code_change(tables: &mut DatabaseChanges, clock: &Clock, code_chan let new_code = bytes_to_hex(code_change.new_code.clone()); let ordinal = code_change.ordinal; - let keys = block_ordinal_keys(&clock, &ordinal); - let row = tables - .push_change_composite("code_changes", keys, 0, table_change::Operation::Create) - .change("address", ("", address.as_str())) + row.change("address", ("", address.as_str())) .change("old_hash", ("", old_hash.as_str())) .change("old_code", ("", old_code.as_str())) .change("new_hash", ("", new_hash.as_str())) .change("new_code", ("", new_code.as_str())) .change("ordinal", ("", ordinal.to_string().as_str())); +} + +pub fn insert_block_code_change(tables: &mut DatabaseChanges, clock: &Clock, code_change: &CodeChange) { + let ordinal = code_change.ordinal; + let keys = block_ordinal_keys(&clock, &ordinal); + let row = tables.push_change_composite("block_code_changes", keys, 0, table_change::Operation::Create); + + insert_code_change(row, code_change); + insert_timestamp(row, clock, false); +} + +pub fn insert_trace_code_change(tables: &mut DatabaseChanges, clock: &Clock, code_change: &CodeChange, transaction: &TransactionTrace, trace: &Call) { + let ordinal = code_change.ordinal; + let keys = block_ordinal_keys(&clock, &ordinal); + let row = tables.push_change_composite("code_changes", keys, 0, table_change::Operation::Create); + insert_code_change(row, code_change); insert_timestamp(row, clock, false); insert_transaction_metadata(row, transaction, false); insert_trace_metadata(row, trace); -} +} \ No newline at end of file diff --git a/blocks/evm/src/logs.rs b/blocks/evm/src/logs.rs index 0752332..02b664c 100644 --- a/blocks/evm/src/logs.rs +++ b/blocks/evm/src/logs.rs @@ -11,6 +11,7 @@ use crate::transactions::insert_transaction_metadata; // DetailLevel: BASE (only successful transactions) & EXTENDED pub fn insert_log(tables: &mut DatabaseChanges, clock: &Clock, log: &Log, transaction: &TransactionTrace) { let index = log.index; + let block_index = log.block_index; let tx_hash = bytes_to_hex(transaction.hash.to_vec()); let contract_address = bytes_to_hex(log.address.to_vec()); // EVM Address let topics = log.topics.clone(); @@ -23,13 +24,14 @@ pub fn insert_log(tables: &mut DatabaseChanges, clock: &Clock, log: &Log, transa let keys = logs_keys(&clock, &tx_hash, &index); let row = tables .push_change_composite("logs", keys, 0, table_change::Operation::Create) + .change("index", ("", index.to_string().as_str())) + .change("block_index", ("", block_index.to_string().as_str())) .change("contract_address", ("", contract_address.as_str())) .change("topic0", ("", topic0.as_str())) .change("topic1", ("", topic1.as_str())) .change("topic2", ("", topic2.as_str())) .change("topic3", ("", topic3.as_str())) - .change("data", ("", data.as_str())) - .change("index", ("", index.to_string().as_str())); + .change("data", ("", data.as_str())); insert_timestamp(row, clock, false); insert_transaction_metadata(row, transaction, false); diff --git a/blocks/evm/src/traces.rs b/blocks/evm/src/traces.rs index 6ee762a..d4ad01e 100644 --- a/blocks/evm/src/traces.rs +++ b/blocks/evm/src/traces.rs @@ -1,6 +1,6 @@ use common::{ blocks::insert_timestamp, - keys::traces_keys, + keys::{system_traces_keys, traces_keys}, utils::{bytes_to_hex, optional_bigint_to_string}, }; use substreams::pb::substreams::Clock; @@ -8,7 +8,7 @@ use substreams_database_change::pb::database::{table_change, DatabaseChanges, Ta use substreams_ethereum::pb::eth::v2::{Call, TransactionTrace}; use crate::{ - account_creations::insert_account_creation, balance_changes::insert_trace_balance_change, code_changes::insert_code_change, gas_changes::insert_gas_change, logs::insert_log, + account_creations::insert_account_creation, balance_changes::insert_trace_balance_change, code_changes::insert_trace_code_change, gas_changes::insert_gas_change, logs::insert_log, nonce_changes::insert_nonce_change, storage_changes::insert_storage_change, transactions::insert_transaction_metadata, }; @@ -30,7 +30,56 @@ pub fn insert_trace(tables: &mut DatabaseChanges, clock: &Clock, call: &Call, tr // transaction let tx_index = transaction.index; let tx_hash = bytes_to_hex(transaction.hash.clone()); + let keys = traces_keys(&clock, &tx_hash, &tx_index, &call.index); + let row = tables.push_change_composite("traces", keys, 0, table_change::Operation::Create); + insert_trace_values(row, call); + insert_timestamp(row, clock, false); + insert_transaction_metadata(row, transaction, true); + + // TABLE::logs + for log in call.logs.iter() { + insert_log(tables, clock, log, transaction); + } + + // TABLE::balance_changes + for balance_change in call.balance_changes.iter() { + insert_trace_balance_change(tables, clock, balance_change, transaction, call); + } + // TABLE::storage_changes + for storage_change in call.storage_changes.iter() { + insert_storage_change(tables, clock, &storage_change, transaction, call); + } + // TABLE::code_changes + for code_change in call.code_changes.iter() { + insert_trace_code_change(tables, clock, &code_change, transaction, call); + } + // TABLE::account_creations + for account_creation in call.account_creations.iter() { + insert_account_creation(tables, clock, &account_creation, transaction, call); + } + // TABLE::nonce_changes + for nonce_change in call.nonce_changes.iter() { + insert_nonce_change(tables, clock, &nonce_change, transaction, call); + } + // TABLE::gas_changes + for gas_change in call.gas_changes.iter() { + insert_gas_change(tables, clock, &gas_change, transaction, call); + } +} + +// https://github.com/streamingfast/firehose-ethereum/blob/1bcb32a8eb3e43347972b6b5c9b1fcc4a08c751e/proto/sf/ethereum/type/v2/type.proto#L546 +// DetailLevel: EXTENDED +pub fn insert_system_trace(tables: &mut DatabaseChanges, clock: &Clock, system_call: &Call) { + let keys = system_traces_keys(&clock, &system_call.index); + let row = tables.push_change_composite("system_traces", keys, 0, table_change::Operation::Create); + insert_trace_values(row, system_call); + insert_timestamp(row, clock, false); + + // TO-DO: add additional tables? (e.g. logs, balance_changes, storage_changes, code_changes, account_creations, nonce_changes, gas_changes) + // would require to drop `transaction` from those tables +} +pub fn insert_trace_values(row: &mut TableChange, call: &Call) { // trace let address = bytes_to_hex(call.address.clone()); // additional `trace_address`? let begin_ordinal = call.begin_ordinal; @@ -53,10 +102,7 @@ pub fn insert_trace(tables: &mut DatabaseChanges, clock: &Clock, call: &Call, tr let suicide = call.suicide; // or `selfdestruct`? let value = optional_bigint_to_string(call.value.clone(), "0"); // UInt256 - let keys = traces_keys(&clock, &tx_hash, &tx_index, &index); - let row = tables - .push_change_composite("traces", keys, 0, table_change::Operation::Create) - .change("address", ("", address.as_str())) + row.change("address", ("", address.as_str())) .change("begin_ordinal", ("", begin_ordinal.to_string().as_str())) .change("call_type", ("", call_type.as_str())) .change("call_type_code", ("", call_type_code.to_string().as_str())) @@ -76,39 +122,6 @@ pub fn insert_trace(tables: &mut DatabaseChanges, clock: &Clock, call: &Call, tr .change("status_reverted", ("", status_reverted.to_string().as_str())) .change("suicide", ("", suicide.to_string().as_str())) .change("value", ("", value.as_str())); - - insert_timestamp(row, clock, false); - insert_transaction_metadata(row, transaction, true); - - // TABLE::logs - for log in call.logs.iter() { - insert_log(tables, clock, log, transaction); - } - - // TABLE::balance_changes - for balance_change in call.balance_changes.iter() { - insert_trace_balance_change(tables, clock, balance_change, transaction, call); - } - // TABLE::storage_changes - for storage_change in call.storage_changes.iter() { - insert_storage_change(tables, clock, &storage_change, transaction, call); - } - // TABLE::code_changes - for code_change in call.code_changes.iter() { - insert_code_change(tables, clock, &code_change, transaction, call); - } - // TABLE::account_creations - for account_creation in call.account_creations.iter() { - insert_account_creation(tables, clock, &account_creation, transaction, call); - } - // TABLE::nonce_changes - for nonce_change in call.nonce_changes.iter() { - insert_nonce_change(tables, clock, &nonce_change, transaction, call); - } - // TABLE::gas_changes - for gas_change in call.gas_changes.iter() { - insert_gas_change(tables, clock, &gas_change, transaction, call); - } } pub fn insert_trace_metadata(row: &mut TableChange, trace: &Call) { diff --git a/common/src/keys.rs b/common/src/keys.rs index 514febf..c617e34 100644 --- a/common/src/keys.rs +++ b/common/src/keys.rs @@ -40,3 +40,9 @@ pub fn traces_keys(clock: &Clock, tx_hash: &String, tx_index: &u32, index: &u32) keys.insert("index".to_string(), index.to_string()); keys } + +pub fn system_traces_keys(clock: &Clock, index: &u32) -> HashMap { + let mut keys = blocks_keys(clock, false); + keys.insert("index".to_string(), index.to_string()); + keys +}