Add new types

- add `block_code_changes` TABLE
- add `system_calls` TABLE
- add `block_index` field to logs

blocks/evm/schema.sql

@@ -125,6 +125,7 @@ CREATE TABLE IF NOT EXISTS logs
 
     -- logs --
     `index`                     UInt32,
+    block_index                 UInt32,
     contract_address            FixedString(42),
     topic0                      FixedString(66),
     topic1                      FixedString(66) DEFAULT '',
@@ -277,6 +278,27 @@ CREATE TABLE IF NOT EXISTS storage_changes
         ORDER BY (block_date, block_number, block_hash, ordinal)
         COMMENT 'EVM storage changes';
 
+CREATE TABLE IF NOT EXISTS block_code_changes
+(
+    -- block --
+    block_time                  DateTime64(3, 'UTC'),
+    block_number                UInt64,
+    block_hash                  FixedString(66),
+    block_date                  Date,
+
+    -- code change --
+    address                     FixedString(42),
+    old_hash                    FixedString(66) DEFAULT '',
+    old_code                    String DEFAULT '',
+    new_hash                    FixedString(66) DEFAULT '',
+    new_code                    String DEFAULT '',
+    ordinal                     UInt64
+)
+    ENGINE = ReplacingMergeTree()
+        PRIMARY KEY (block_date, block_number)
+        ORDER BY (block_date, block_number, block_hash, ordinal)
+        COMMENT 'EVM block code changes';
+
 CREATE TABLE IF NOT EXISTS code_changes
 (
     -- block --
@@ -413,4 +435,39 @@ CREATE TABLE IF NOT EXISTS gas_changes
     ENGINE = ReplacingMergeTree()
         PRIMARY KEY (block_date, block_number)
         ORDER BY (block_date, block_number, block_hash, ordinal)
-        COMMENT 'EVM gas changes';
+        COMMENT 'EVM gas changes';
+
+CREATE TABLE IF NOT EXISTS system_traces
+(
+    -- block --
+    block_time                  DateTime64(3, 'UTC'),
+    block_number                UInt64,
+    block_hash                  FixedString(66),
+    block_date                  Date,
+
+    -- trace --
+    `index`                     UInt32,
+    parent_index                UInt32,
+    depth                       UInt32,
+    caller                      FixedString(42),
+    call_type                   LowCardinality(String),
+    call_type_code              UInt32,
+    address                     FixedString(42),
+    value                       DEFAULT '' COMMENT 'UInt256',
+    gas_limit                   UInt64,
+    gas_consumed                UInt64,
+    return_data                 String COMMENT 'Return data is set by contract calls using RETURN or REVERT.',
+    input                       String,
+    suicide                     Bool,
+    failure_reason              LowCardinality(String),
+    state_reverted              Bool,
+    status_reverted             Bool,
+    status_failed               Bool,
+    executed_code               Bool,
+    begin_ordinal               UInt64,
+    end_ordinal                 UInt64
+)
+    ENGINE = ReplacingMergeTree()
+        PRIMARY KEY (block_date, block_number)
+        ORDER BY (block_date, block_number, `index`)
+        COMMENT 'EVM system traces';

blocks/evm/src/blocks.rs

@@ -6,6 +6,8 @@ use substreams_database_change::pb::database::{table_change, DatabaseChanges};
 use substreams_ethereum::pb::eth::v2::Block;
 
 use crate::balance_changes::insert_balance_change_counts;
+use crate::code_changes::insert_block_code_change;
+use crate::traces::insert_system_trace;
 
 pub fn block_detail_to_string(detail_level: i32) -> String {
     match detail_level {
@@ -93,5 +95,12 @@ pub fn insert_blocks(tables: &mut DatabaseChanges, clock: &Clock, block: &Block)
     let all_balance_changes_reason: Vec<i32> = block.balance_changes.iter().map(|balance_change| balance_change.reason).collect();
     insert_balance_change_counts(row, all_balance_changes_reason);
 
-    // TODO: block.code_changes
+    // code changes
+    for code_change in block.code_changes.iter() {
+        insert_block_code_change(tables, clock, code_change);
+    }
+
+    for system_call in block.system_calls.iter() {
+        insert_system_trace(tables, clock, system_call);
+    }
 }

blocks/evm/src/code_changes.rs

@@ -2,33 +2,46 @@ use common::blocks::insert_timestamp;
 use common::keys::block_ordinal_keys;
 use common::utils::bytes_to_hex;
 use substreams::pb::substreams::Clock;
-use substreams_database_change::pb::database::{table_change, DatabaseChanges};
+use substreams_database_change::pb::database::{table_change, DatabaseChanges, TableChange};
 use substreams_ethereum::pb::eth::v2::{Call, CodeChange, TransactionTrace};
 
 use crate::traces::insert_trace_metadata;
 use crate::transactions::insert_transaction_metadata;
 
 // https://github.com/streamingfast/firehose-ethereum/blob/1bcb32a8eb3e43347972b6b5c9b1fcc4a08c751e/proto/sf/ethereum/type/v2/type.proto#L744
 // DetailLevel: EXTENDED
-pub fn insert_code_change(tables: &mut DatabaseChanges, clock: &Clock, code_change: &CodeChange, transaction: &TransactionTrace, trace: &Call) {
+pub fn insert_code_change(row: &mut TableChange, code_change: &CodeChange) {
     let address = bytes_to_hex(code_change.address.clone());
     let old_hash = bytes_to_hex(code_change.old_hash.clone());
     let old_code = bytes_to_hex(code_change.old_code.clone());
     let new_hash = bytes_to_hex(code_change.new_hash.clone());
     let new_code = bytes_to_hex(code_change.new_code.clone());
     let ordinal = code_change.ordinal;
 
-    let keys = block_ordinal_keys(&clock, &ordinal);
-    let row = tables
-        .push_change_composite("code_changes", keys, 0, table_change::Operation::Create)
-        .change("address", ("", address.as_str()))
+    row.change("address", ("", address.as_str()))
         .change("old_hash", ("", old_hash.as_str()))
         .change("old_code", ("", old_code.as_str()))
         .change("new_hash", ("", new_hash.as_str()))
         .change("new_code", ("", new_code.as_str()))
         .change("ordinal", ("", ordinal.to_string().as_str()));
+}
+
+pub fn insert_block_code_change(tables: &mut DatabaseChanges, clock: &Clock, code_change: &CodeChange) {
+    let ordinal = code_change.ordinal;
+    let keys = block_ordinal_keys(&clock, &ordinal);
+    let row = tables.push_change_composite("block_code_changes", keys, 0, table_change::Operation::Create);
+
+    insert_code_change(row, code_change);
+    insert_timestamp(row, clock, false);
+}
+
+pub fn insert_trace_code_change(tables: &mut DatabaseChanges, clock: &Clock, code_change: &CodeChange, transaction: &TransactionTrace, trace: &Call) {
+    let ordinal = code_change.ordinal;
+    let keys = block_ordinal_keys(&clock, &ordinal);
+    let row = tables.push_change_composite("code_changes", keys, 0, table_change::Operation::Create);
 
+    insert_code_change(row, code_change);
     insert_timestamp(row, clock, false);
     insert_transaction_metadata(row, transaction, false);
     insert_trace_metadata(row, trace);
-}
+}

blocks/evm/src/logs.rs

@@ -11,6 +11,7 @@ use crate::transactions::insert_transaction_metadata;
 // DetailLevel: BASE (only successful transactions) & EXTENDED
 pub fn insert_log(tables: &mut DatabaseChanges, clock: &Clock, log: &Log, transaction: &TransactionTrace) {
     let index = log.index;
+    let block_index = log.block_index;
     let tx_hash = bytes_to_hex(transaction.hash.to_vec());
     let contract_address = bytes_to_hex(log.address.to_vec()); // EVM Address
     let topics = log.topics.clone();
@@ -23,13 +24,14 @@ pub fn insert_log(tables: &mut DatabaseChanges, clock: &Clock, log: &Log, transa
     let keys = logs_keys(&clock, &tx_hash, &index);
     let row = tables
         .push_change_composite("logs", keys, 0, table_change::Operation::Create)
+        .change("index", ("", index.to_string().as_str()))
+        .change("block_index", ("", block_index.to_string().as_str()))
         .change("contract_address", ("", contract_address.as_str()))
         .change("topic0", ("", topic0.as_str()))
         .change("topic1", ("", topic1.as_str()))
         .change("topic2", ("", topic2.as_str()))
         .change("topic3", ("", topic3.as_str()))
-        .change("data", ("", data.as_str()))
-        .change("index", ("", index.to_string().as_str()));
+        .change("data", ("", data.as_str()));
 
     insert_timestamp(row, clock, false);
     insert_transaction_metadata(row, transaction, false);

blocks/evm/src/traces.rs

@@ -1,14 +1,14 @@
 use common::{
     blocks::insert_timestamp,
-    keys::traces_keys,
+    keys::{system_traces_keys, traces_keys},
     utils::{bytes_to_hex, optional_bigint_to_string},
 };
 use substreams::pb::substreams::Clock;
 use substreams_database_change::pb::database::{table_change, DatabaseChanges, TableChange};
 use substreams_ethereum::pb::eth::v2::{Call, TransactionTrace};
 
 use crate::{
-    account_creations::insert_account_creation, balance_changes::insert_trace_balance_change, code_changes::insert_code_change, gas_changes::insert_gas_change, logs::insert_log,
+    account_creations::insert_account_creation, balance_changes::insert_trace_balance_change, code_changes::insert_trace_code_change, gas_changes::insert_gas_change, logs::insert_log,
     nonce_changes::insert_nonce_change, storage_changes::insert_storage_change, transactions::insert_transaction_metadata,
 };
 
@@ -30,7 +30,56 @@ pub fn insert_trace(tables: &mut DatabaseChanges, clock: &Clock, call: &Call, tr
     // transaction
     let tx_index = transaction.index;
     let tx_hash = bytes_to_hex(transaction.hash.clone());
+    let keys = traces_keys(&clock, &tx_hash, &tx_index, &call.index);
+    let row = tables.push_change_composite("traces", keys, 0, table_change::Operation::Create);
+    insert_trace_values(row, call);
+    insert_timestamp(row, clock, false);
+    insert_transaction_metadata(row, transaction, true);
+
+    // TABLE::logs
+    for log in call.logs.iter() {
+        insert_log(tables, clock, log, transaction);
+    }
+
+    // TABLE::balance_changes
+    for balance_change in call.balance_changes.iter() {
+        insert_trace_balance_change(tables, clock, balance_change, transaction, call);
+    }
+    // TABLE::storage_changes
+    for storage_change in call.storage_changes.iter() {
+        insert_storage_change(tables, clock, &storage_change, transaction, call);
+    }
+    // TABLE::code_changes
+    for code_change in call.code_changes.iter() {
+        insert_trace_code_change(tables, clock, &code_change, transaction, call);
+    }
+    // TABLE::account_creations
+    for account_creation in call.account_creations.iter() {
+        insert_account_creation(tables, clock, &account_creation, transaction, call);
+    }
+    // TABLE::nonce_changes
+    for nonce_change in call.nonce_changes.iter() {
+        insert_nonce_change(tables, clock, &nonce_change, transaction, call);
+    }
+    // TABLE::gas_changes
+    for gas_change in call.gas_changes.iter() {
+        insert_gas_change(tables, clock, &gas_change, transaction, call);
+    }
+}
+
+// https://github.com/streamingfast/firehose-ethereum/blob/1bcb32a8eb3e43347972b6b5c9b1fcc4a08c751e/proto/sf/ethereum/type/v2/type.proto#L546
+// DetailLevel: EXTENDED
+pub fn insert_system_trace(tables: &mut DatabaseChanges, clock: &Clock, system_call: &Call) {
+    let keys = system_traces_keys(&clock, &system_call.index);
+    let row = tables.push_change_composite("system_traces", keys, 0, table_change::Operation::Create);
+    insert_trace_values(row, system_call);
+    insert_timestamp(row, clock, false);
+
+    // TO-DO: add additional tables? (e.g. logs, balance_changes, storage_changes, code_changes, account_creations, nonce_changes, gas_changes)
+    // would require to drop `transaction` from those tables
+}
 
+pub fn insert_trace_values(row: &mut TableChange, call: &Call) {
     // trace
     let address = bytes_to_hex(call.address.clone()); // additional `trace_address`?
     let begin_ordinal = call.begin_ordinal;
@@ -53,10 +102,7 @@ pub fn insert_trace(tables: &mut DatabaseChanges, clock: &Clock, call: &Call, tr
     let suicide = call.suicide; // or `selfdestruct`?
     let value = optional_bigint_to_string(call.value.clone(), "0"); // UInt256
 
-    let keys = traces_keys(&clock, &tx_hash, &tx_index, &index);
-    let row = tables
-        .push_change_composite("traces", keys, 0, table_change::Operation::Create)
-        .change("address", ("", address.as_str()))
+    row.change("address", ("", address.as_str()))
         .change("begin_ordinal", ("", begin_ordinal.to_string().as_str()))
         .change("call_type", ("", call_type.as_str()))
         .change("call_type_code", ("", call_type_code.to_string().as_str()))
@@ -76,39 +122,6 @@ pub fn insert_trace(tables: &mut DatabaseChanges, clock: &Clock, call: &Call, tr
         .change("status_reverted", ("", status_reverted.to_string().as_str()))
         .change("suicide", ("", suicide.to_string().as_str()))
         .change("value", ("", value.as_str()));
-
-    insert_timestamp(row, clock, false);
-    insert_transaction_metadata(row, transaction, true);
-
-    // TABLE::logs
-    for log in call.logs.iter() {
-        insert_log(tables, clock, log, transaction);
-    }
-
-    // TABLE::balance_changes
-    for balance_change in call.balance_changes.iter() {
-        insert_trace_balance_change(tables, clock, balance_change, transaction, call);
-    }
-    // TABLE::storage_changes
-    for storage_change in call.storage_changes.iter() {
-        insert_storage_change(tables, clock, &storage_change, transaction, call);
-    }
-    // TABLE::code_changes
-    for code_change in call.code_changes.iter() {
-        insert_code_change(tables, clock, &code_change, transaction, call);
-    }
-    // TABLE::account_creations
-    for account_creation in call.account_creations.iter() {
-        insert_account_creation(tables, clock, &account_creation, transaction, call);
-    }
-    // TABLE::nonce_changes
-    for nonce_change in call.nonce_changes.iter() {
-        insert_nonce_change(tables, clock, &nonce_change, transaction, call);
-    }
-    // TABLE::gas_changes
-    for gas_change in call.gas_changes.iter() {
-        insert_gas_change(tables, clock, &gas_change, transaction, call);
-    }
 }
 
 pub fn insert_trace_metadata(row: &mut TableChange, trace: &Call) {

common/src/keys.rs

@@ -40,3 +40,9 @@ pub fn traces_keys(clock: &Clock, tx_hash: &String, tx_index: &u32, index: &u32)
     keys.insert("index".to_string(), index.to_string());
     keys
 }
+
+pub fn system_traces_keys(clock: &Clock, index: &u32) -> HashMap<String, String> {
+    let mut keys = blocks_keys(clock, false);
+    keys.insert("index".to_string(), index.to_string());
+    keys
+}