From 5372106d04f23ef05f673444d7d856a1c3bd1966 Mon Sep 17 00:00:00 2001 From: zolting <44510235+zolting@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:07:19 -0500 Subject: [PATCH 01/22] Kickstart antelope parquet --- blocks/antelope/Cargo.toml | 2 + blocks/antelope/Makefile | 17 +- blocks/antelope/src/blocks.rs | 47 ++- blocks/antelope/src/lib.rs | 3 +- blocks/antelope/src/pb/antelope.rs | 515 +++++++++++++++++++++++++++++ blocks/antelope/src/pb/beacon.rs | 515 +++++++++++++++++++++++++++++ blocks/antelope/src/pb/mod.rs | 6 + blocks/antelope/src/sinks.rs | 30 +- blocks/antelope/src/size.rs | 38 ++- blocks/antelope/substreams.yaml | 25 +- blocks/cosmos/Makefile | 2 +- proto/antelope.proto | 278 ++++++++++++++++ 12 files changed, 1432 insertions(+), 46 deletions(-) create mode 100644 blocks/antelope/src/pb/antelope.rs create mode 100644 blocks/antelope/src/pb/beacon.rs create mode 100644 blocks/antelope/src/pb/mod.rs create mode 100644 proto/antelope.proto diff --git a/blocks/antelope/Cargo.toml b/blocks/antelope/Cargo.toml index f1a4d5c..37b09e9 100644 --- a/blocks/antelope/Cargo.toml +++ b/blocks/antelope/Cargo.toml @@ -11,3 +11,5 @@ common = { path = "../../common" } substreams-antelope = { workspace = true } substreams-database-change = { workspace = true } substreams = { workspace = true } +prost = { workspace = true } +prost-types = { workspace = true } diff --git a/blocks/antelope/Makefile b/blocks/antelope/Makefile index 84bf547..99db40d 100644 --- a/blocks/antelope/Makefile +++ b/blocks/antelope/Makefile @@ -29,12 +29,17 @@ run: gui: substreams gui -e eos.substreams.pinax.network:443 ch_out -s 386946801 -t 386946802 -.PHONY: sql-setup -sql-setup: - # EVM blocks - substreams-sink-sql setup clickhouse://default:default@localhost:9000/eos substreams.yaml - substreams-sink-sql setup clickhouse://default:default@localhost:9000/wax substreams.yaml - substreams-sink-sql setup clickhouse://default:default@localhost:9000/telos substreams.yaml +.PHONY: protogen +protogen: + substreams protogen --exclude-paths google,sf + +.PHONY: parquet-eos +parquet-eos: + substreams-sink-files run eos.substreams.pinax.network:443 substreams.yaml map_events './out' 386841287:386841787 --encoder parquet --file-block-count 100 --development-mode + +.PHONY: parquet-wax +parquet-wax: + substreams-sink-files run wax.substreams.pinax.network:443 substreams.yaml map_events './out' 32233917:32234417 --encoder parquet --file-block-count 100 --development-mode # Antelope blocks .PHONY: sql-run-eos diff --git a/blocks/antelope/src/blocks.rs b/blocks/antelope/src/blocks.rs index e9fa1bd..87d8666 100644 --- a/blocks/antelope/src/blocks.rs +++ b/blocks/antelope/src/blocks.rs @@ -1,9 +1,8 @@ -use common::blocks::insert_timestamp; +use crate::{keys::blocks_keys, pb::antelope::Block as RawBlock, size::collect_size}; +use common::{blocks::insert_timestamp, structs::BlockTimestamp, utils::bytes_to_hex}; use substreams::{pb::substreams::Clock, Hex}; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; use substreams_antelope::pb::Block; - -use crate::keys::blocks_keys; +use substreams_database_change::pb::database::{table_change, DatabaseChanges}; use super::{size::insert_size, transactions::insert_transaction}; @@ -28,7 +27,7 @@ pub fn insert_blocks(tables: &mut DatabaseChanges, clock: &Clock, block: &Block) // TO-DO // Array(String) type is not supported by `substreams-sink-sql` // https://github.com/pinax-network/substreams-sink-sql/issues/18 - // let blockroot_merkle_active_nodes = blockroot_merkle.active_nodes.iter().map(|row| bytes_to_hex(row).to_string()).collect::>(); + let blockroot_merkle_active_nodes = blockroot_merkle.active_nodes.iter().map(|row| bytes_to_hex(row)).collect::>().join(","); let blockroot_merkle_node_count = blockroot_merkle.node_count; // block roots @@ -48,19 +47,16 @@ pub fn insert_blocks(tables: &mut DatabaseChanges, clock: &Clock, block: &Block) .change("producer", ("", producer.to_string().as_str())) .change("confirmed", ("", confirmed.to_string().as_str())) .change("schedule_version", ("", schedule_version.to_string().as_str())) - // block .change("version", ("", version.to_string().as_str())) .change("producer_signature", ("", producer_signature.to_string().as_str())) .change("dpos_proposed_irreversible_blocknum", ("", dpos_proposed_irreversible_blocknum.to_string().as_str())) .change("dpos_irreversible_blocknum", ("", dpos_irreversible_blocknum.to_string().as_str())) - // block roots .change("transaction_mroot", ("", transaction_mroot.to_string().as_str())) .change("action_mroot", ("", action_mroot.to_string().as_str())) // .change("blockroot_merkle_active_nodes", ("", format!("['{}']", blockroot_merkle_active_nodes.join("','") ).as_str())) - .change("blockroot_merkle_node_count", ("", blockroot_merkle_node_count.to_string().as_str())) - ; + .change("blockroot_merkle_node_count", ("", blockroot_merkle_node_count.to_string().as_str())); // transaction status counts insert_size(row, block); @@ -70,4 +66,35 @@ pub fn insert_blocks(tables: &mut DatabaseChanges, clock: &Clock, block: &Block) for transaction in block.transaction_traces() { insert_transaction(tables, clock, &transaction, &header); } -} \ No newline at end of file +} + +pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> RawBlock { + let header = block.header.clone().unwrap_or_default(); + let blockroot_merkle = block.blockroot_merkle.clone().unwrap_or_default(); + let size = collect_size(block); + + RawBlock { + time: Some(timestamp.time), + number: timestamp.number, + date: timestamp.date.clone(), + hash: timestamp.hash.clone(), + parent_hash: header.previous, + producer: header.producer, + confirmed: header.confirmed, + schedule_version: header.schedule_version, + version: block.version, + producer_signature: block.producer_signature.clone(), + dpos_proposed_irreversible_blocknum: block.dpos_proposed_irreversible_blocknum, + dpos_irreversible_blocknum: block.dpos_irreversible_blocknum, + transaction_mroot: Hex::encode(&header.transaction_mroot.to_vec()), + action_mroot: Hex::encode(&header.action_mroot.to_vec()), + blockroot_merkle_active_nodes: blockroot_merkle.active_nodes.iter().map(|row| bytes_to_hex(row)).collect::>().join(","), + blockroot_merkle_node_count: blockroot_merkle.node_count, + size: size.size, + total_transactions: size.total_transactions, + successful_transactions: size.successful_transactions, + failed_transactions: size.failed_transactions, + total_actions: size.total_actions, + total_db_ops: size.total_db_ops, + } +} diff --git a/blocks/antelope/src/lib.rs b/blocks/antelope/src/lib.rs index 016068a..a0f5d81 100644 --- a/blocks/antelope/src/lib.rs +++ b/blocks/antelope/src/lib.rs @@ -8,8 +8,9 @@ mod creation_tree; mod db_ops; mod feature_ops; mod keys; -mod ram_ops; +mod pb; mod perm_ops; +mod ram_ops; mod sinks; mod size; mod table_ops; diff --git a/blocks/antelope/src/pb/antelope.rs b/blocks/antelope/src/pb/antelope.rs new file mode 100644 index 0000000..3c3b2fd --- /dev/null +++ b/blocks/antelope/src/pb/antelope.rs @@ -0,0 +1,515 @@ +// @generated +// This file is @generated by prost-build. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Block { + #[prost(message, optional, tag="1")] + pub time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub number: u64, + #[prost(string, tag="3")] + pub date: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub hash: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub parent_hash: ::prost::alloc::string::String, + #[prost(string, tag="6")] + pub producer: ::prost::alloc::string::String, + #[prost(uint32, tag="7")] + pub confirmed: u32, + #[prost(uint32, tag="8")] + pub schedule_version: u32, + #[prost(uint32, tag="9")] + pub version: u32, + #[prost(string, tag="10")] + pub producer_signature: ::prost::alloc::string::String, + #[prost(uint32, tag="11")] + pub dpos_proposed_irreversible_blocknum: u32, + #[prost(uint32, tag="12")] + pub dpos_irreversible_blocknum: u32, + #[prost(string, tag="13")] + pub transaction_mroot: ::prost::alloc::string::String, + #[prost(string, tag="14")] + pub action_mroot: ::prost::alloc::string::String, + #[prost(string, tag="15")] + pub blockroot_merkle_active_nodes: ::prost::alloc::string::String, + #[prost(uint32, tag="16")] + pub blockroot_merkle_node_count: u32, + #[prost(uint64, tag="17")] + pub size: u64, + #[prost(uint64, tag="18")] + pub total_transactions: u64, + #[prost(uint64, tag="19")] + pub successful_transactions: u64, + #[prost(uint64, tag="20")] + pub failed_transactions: u64, + #[prost(uint64, tag="21")] + pub total_actions: u64, + #[prost(uint64, tag="22")] + pub total_db_ops: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Transaction { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub hash: ::prost::alloc::string::String, + #[prost(uint64, tag="6")] + pub index: u64, + #[prost(int64, tag="7")] + pub elapsed: i64, + #[prost(uint64, tag="8")] + pub net_usage: u64, + #[prost(bool, tag="9")] + pub scheduled: bool, + #[prost(uint32, tag="10")] + pub cpu_usage_micro_seconds: u32, + #[prost(uint32, tag="11")] + pub net_usage_words: u32, + #[prost(string, tag="12")] + pub status: ::prost::alloc::string::String, + #[prost(uint32, tag="13")] + pub status_code: u32, + #[prost(bool, tag="14")] + pub success: bool, + #[prost(string, tag="15")] + pub transaction_mroot: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Action { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint64, tag="7")] + pub abi_sequence: u64, + #[prost(uint64, tag="8")] + pub code_sequence: u64, + #[prost(string, tag="9")] + pub digest: ::prost::alloc::string::String, + #[prost(uint64, tag="10")] + pub global_sequence: u64, + #[prost(string, tag="11")] + pub receipt_receiver: ::prost::alloc::string::String, + #[prost(uint64, tag="12")] + pub recv_sequence: u64, + #[prost(string, tag="13")] + pub account: ::prost::alloc::string::String, + #[prost(string, tag="14")] + pub name: ::prost::alloc::string::String, + #[prost(string, tag="15")] + pub json_data: ::prost::alloc::string::String, + #[prost(string, tag="16")] + pub raw_data: ::prost::alloc::string::String, + #[prost(uint32, tag="17")] + pub index: u32, + #[prost(uint32, tag="18")] + pub action_ordinal: u32, + #[prost(string, tag="19")] + pub receiver: ::prost::alloc::string::String, + #[prost(bool, tag="20")] + pub context_free: bool, + #[prost(int64, tag="21")] + pub elapsed: i64, + #[prost(string, tag="22")] + pub console: ::prost::alloc::string::String, + #[prost(string, tag="23")] + pub raw_return_value: ::prost::alloc::string::String, + #[prost(string, tag="24")] + pub json_return_value: ::prost::alloc::string::String, + #[prost(uint32, tag="25")] + pub creator_action_ordinal: u32, + #[prost(uint32, tag="26")] + pub closest_unnotified_ancestor_action_ordinal: u32, + #[prost(string, tag="27")] + pub action_mroot: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DbOp { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub operation: ::prost::alloc::string::String, + #[prost(uint32, tag="10")] + pub operation_code: u32, + #[prost(string, tag="11")] + pub code: ::prost::alloc::string::String, + #[prost(string, tag="12")] + pub scope: ::prost::alloc::string::String, + #[prost(string, tag="13")] + pub table_name: ::prost::alloc::string::String, + #[prost(string, tag="14")] + pub primary_key: ::prost::alloc::string::String, + #[prost(string, tag="15")] + pub old_payer: ::prost::alloc::string::String, + #[prost(string, tag="16")] + pub new_payer: ::prost::alloc::string::String, + #[prost(string, tag="17")] + pub old_data: ::prost::alloc::string::String, + #[prost(string, tag="18")] + pub new_data: ::prost::alloc::string::String, + #[prost(string, tag="19")] + pub old_data_json: ::prost::alloc::string::String, + #[prost(string, tag="20")] + pub new_data_json: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FeatureOp { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(string, tag="8")] + pub feature_digest: ::prost::alloc::string::String, + #[prost(string, tag="9")] + pub kind: ::prost::alloc::string::String, + #[prost(string, tag="10")] + pub description_digest: ::prost::alloc::string::String, + #[prost(string, tag="11")] + pub protocol_feature_type: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PermOp { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(string, tag="8")] + pub operation: ::prost::alloc::string::String, + #[prost(uint32, tag="9")] + pub operation_code: u32, + #[prost(uint64, tag="10")] + pub id: u64, + #[prost(uint64, tag="11")] + pub parent_id: u64, + #[prost(string, tag="12")] + pub owner: ::prost::alloc::string::String, + #[prost(string, tag="13")] + pub name: ::prost::alloc::string::String, + #[prost(uint32, tag="14")] + pub threshold: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TableOp { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub operation: ::prost::alloc::string::String, + #[prost(uint32, tag="10")] + pub operation_code: u32, + #[prost(string, tag="11")] + pub payer: ::prost::alloc::string::String, + #[prost(string, tag="12")] + pub code: ::prost::alloc::string::String, + #[prost(string, tag="13")] + pub scope: ::prost::alloc::string::String, + #[prost(string, tag="14")] + pub table_name: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Account { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub actor: ::prost::alloc::string::String, + #[prost(string, tag="10")] + pub permission: ::prost::alloc::string::String, + #[prost(uint32, tag="11")] + pub weight: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Key { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub public_key: ::prost::alloc::string::String, + #[prost(uint32, tag="10")] + pub weight: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Wait { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(uint32, tag="9")] + pub wait_sec: u32, + #[prost(uint32, tag="10")] + pub weight: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RamOp { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(string, tag="8")] + pub operation: ::prost::alloc::string::String, + #[prost(uint32, tag="9")] + pub operation_code: u32, + #[prost(string, tag="10")] + pub payer: ::prost::alloc::string::String, + #[prost(int64, tag="11")] + pub delta: i64, + #[prost(uint64, tag="12")] + pub usage: u64, + #[prost(string, tag="13")] + pub namespace: ::prost::alloc::string::String, + #[prost(uint32, tag="14")] + pub namespace_code: u32, + #[prost(string, tag="15")] + pub action: ::prost::alloc::string::String, + #[prost(uint32, tag="16")] + pub action_code: u32, + #[prost(string, tag="17")] + pub unique_key: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Authorization { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub actor: ::prost::alloc::string::String, + #[prost(string, tag="10")] + pub permission: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AuthSequence { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub account_name: ::prost::alloc::string::String, + #[prost(uint64, tag="10")] + pub sequence: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AccountRamDelta { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub account: ::prost::alloc::string::String, + #[prost(int64, tag="10")] + pub delta: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreationTree { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(int32, tag="7")] + pub creator_action_index: i32, + #[prost(uint32, tag="8")] + pub execution_action_index: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Events { + #[prost(message, repeated, tag="1")] + pub blocks: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="2")] + pub transactions: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="3")] + pub actions: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="4")] + pub db_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="5")] + pub feature_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="6")] + pub perm_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="7")] + pub table_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="8")] + pub accounts: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="9")] + pub keys: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="10")] + pub waits: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="11")] + pub ram_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="12")] + pub authorizations: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="13")] + pub auth_sequences: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="14")] + pub account_ram_deltas: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="15")] + pub creation_trees: ::prost::alloc::vec::Vec, +} +// @@protoc_insertion_point(module) diff --git a/blocks/antelope/src/pb/beacon.rs b/blocks/antelope/src/pb/beacon.rs new file mode 100644 index 0000000..3c3b2fd --- /dev/null +++ b/blocks/antelope/src/pb/beacon.rs @@ -0,0 +1,515 @@ +// @generated +// This file is @generated by prost-build. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Block { + #[prost(message, optional, tag="1")] + pub time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub number: u64, + #[prost(string, tag="3")] + pub date: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub hash: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub parent_hash: ::prost::alloc::string::String, + #[prost(string, tag="6")] + pub producer: ::prost::alloc::string::String, + #[prost(uint32, tag="7")] + pub confirmed: u32, + #[prost(uint32, tag="8")] + pub schedule_version: u32, + #[prost(uint32, tag="9")] + pub version: u32, + #[prost(string, tag="10")] + pub producer_signature: ::prost::alloc::string::String, + #[prost(uint32, tag="11")] + pub dpos_proposed_irreversible_blocknum: u32, + #[prost(uint32, tag="12")] + pub dpos_irreversible_blocknum: u32, + #[prost(string, tag="13")] + pub transaction_mroot: ::prost::alloc::string::String, + #[prost(string, tag="14")] + pub action_mroot: ::prost::alloc::string::String, + #[prost(string, tag="15")] + pub blockroot_merkle_active_nodes: ::prost::alloc::string::String, + #[prost(uint32, tag="16")] + pub blockroot_merkle_node_count: u32, + #[prost(uint64, tag="17")] + pub size: u64, + #[prost(uint64, tag="18")] + pub total_transactions: u64, + #[prost(uint64, tag="19")] + pub successful_transactions: u64, + #[prost(uint64, tag="20")] + pub failed_transactions: u64, + #[prost(uint64, tag="21")] + pub total_actions: u64, + #[prost(uint64, tag="22")] + pub total_db_ops: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Transaction { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub hash: ::prost::alloc::string::String, + #[prost(uint64, tag="6")] + pub index: u64, + #[prost(int64, tag="7")] + pub elapsed: i64, + #[prost(uint64, tag="8")] + pub net_usage: u64, + #[prost(bool, tag="9")] + pub scheduled: bool, + #[prost(uint32, tag="10")] + pub cpu_usage_micro_seconds: u32, + #[prost(uint32, tag="11")] + pub net_usage_words: u32, + #[prost(string, tag="12")] + pub status: ::prost::alloc::string::String, + #[prost(uint32, tag="13")] + pub status_code: u32, + #[prost(bool, tag="14")] + pub success: bool, + #[prost(string, tag="15")] + pub transaction_mroot: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Action { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint64, tag="7")] + pub abi_sequence: u64, + #[prost(uint64, tag="8")] + pub code_sequence: u64, + #[prost(string, tag="9")] + pub digest: ::prost::alloc::string::String, + #[prost(uint64, tag="10")] + pub global_sequence: u64, + #[prost(string, tag="11")] + pub receipt_receiver: ::prost::alloc::string::String, + #[prost(uint64, tag="12")] + pub recv_sequence: u64, + #[prost(string, tag="13")] + pub account: ::prost::alloc::string::String, + #[prost(string, tag="14")] + pub name: ::prost::alloc::string::String, + #[prost(string, tag="15")] + pub json_data: ::prost::alloc::string::String, + #[prost(string, tag="16")] + pub raw_data: ::prost::alloc::string::String, + #[prost(uint32, tag="17")] + pub index: u32, + #[prost(uint32, tag="18")] + pub action_ordinal: u32, + #[prost(string, tag="19")] + pub receiver: ::prost::alloc::string::String, + #[prost(bool, tag="20")] + pub context_free: bool, + #[prost(int64, tag="21")] + pub elapsed: i64, + #[prost(string, tag="22")] + pub console: ::prost::alloc::string::String, + #[prost(string, tag="23")] + pub raw_return_value: ::prost::alloc::string::String, + #[prost(string, tag="24")] + pub json_return_value: ::prost::alloc::string::String, + #[prost(uint32, tag="25")] + pub creator_action_ordinal: u32, + #[prost(uint32, tag="26")] + pub closest_unnotified_ancestor_action_ordinal: u32, + #[prost(string, tag="27")] + pub action_mroot: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DbOp { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub operation: ::prost::alloc::string::String, + #[prost(uint32, tag="10")] + pub operation_code: u32, + #[prost(string, tag="11")] + pub code: ::prost::alloc::string::String, + #[prost(string, tag="12")] + pub scope: ::prost::alloc::string::String, + #[prost(string, tag="13")] + pub table_name: ::prost::alloc::string::String, + #[prost(string, tag="14")] + pub primary_key: ::prost::alloc::string::String, + #[prost(string, tag="15")] + pub old_payer: ::prost::alloc::string::String, + #[prost(string, tag="16")] + pub new_payer: ::prost::alloc::string::String, + #[prost(string, tag="17")] + pub old_data: ::prost::alloc::string::String, + #[prost(string, tag="18")] + pub new_data: ::prost::alloc::string::String, + #[prost(string, tag="19")] + pub old_data_json: ::prost::alloc::string::String, + #[prost(string, tag="20")] + pub new_data_json: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FeatureOp { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(string, tag="8")] + pub feature_digest: ::prost::alloc::string::String, + #[prost(string, tag="9")] + pub kind: ::prost::alloc::string::String, + #[prost(string, tag="10")] + pub description_digest: ::prost::alloc::string::String, + #[prost(string, tag="11")] + pub protocol_feature_type: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PermOp { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(string, tag="8")] + pub operation: ::prost::alloc::string::String, + #[prost(uint32, tag="9")] + pub operation_code: u32, + #[prost(uint64, tag="10")] + pub id: u64, + #[prost(uint64, tag="11")] + pub parent_id: u64, + #[prost(string, tag="12")] + pub owner: ::prost::alloc::string::String, + #[prost(string, tag="13")] + pub name: ::prost::alloc::string::String, + #[prost(uint32, tag="14")] + pub threshold: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TableOp { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub operation: ::prost::alloc::string::String, + #[prost(uint32, tag="10")] + pub operation_code: u32, + #[prost(string, tag="11")] + pub payer: ::prost::alloc::string::String, + #[prost(string, tag="12")] + pub code: ::prost::alloc::string::String, + #[prost(string, tag="13")] + pub scope: ::prost::alloc::string::String, + #[prost(string, tag="14")] + pub table_name: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Account { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub actor: ::prost::alloc::string::String, + #[prost(string, tag="10")] + pub permission: ::prost::alloc::string::String, + #[prost(uint32, tag="11")] + pub weight: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Key { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub public_key: ::prost::alloc::string::String, + #[prost(uint32, tag="10")] + pub weight: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Wait { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(uint32, tag="9")] + pub wait_sec: u32, + #[prost(uint32, tag="10")] + pub weight: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RamOp { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(string, tag="8")] + pub operation: ::prost::alloc::string::String, + #[prost(uint32, tag="9")] + pub operation_code: u32, + #[prost(string, tag="10")] + pub payer: ::prost::alloc::string::String, + #[prost(int64, tag="11")] + pub delta: i64, + #[prost(uint64, tag="12")] + pub usage: u64, + #[prost(string, tag="13")] + pub namespace: ::prost::alloc::string::String, + #[prost(uint32, tag="14")] + pub namespace_code: u32, + #[prost(string, tag="15")] + pub action: ::prost::alloc::string::String, + #[prost(uint32, tag="16")] + pub action_code: u32, + #[prost(string, tag="17")] + pub unique_key: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Authorization { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub actor: ::prost::alloc::string::String, + #[prost(string, tag="10")] + pub permission: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AuthSequence { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub account_name: ::prost::alloc::string::String, + #[prost(uint64, tag="10")] + pub sequence: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AccountRamDelta { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(uint32, tag="7")] + pub action_index: u32, + #[prost(uint32, tag="8")] + pub index: u32, + #[prost(string, tag="9")] + pub account: ::prost::alloc::string::String, + #[prost(int64, tag="10")] + pub delta: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreationTree { + #[prost(message, optional, tag="1")] + pub block_time: ::core::option::Option<::prost_types::Timestamp>, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + #[prost(string, tag="5")] + pub tx_hash: ::prost::alloc::string::String, + #[prost(bool, tag="6")] + pub tx_success: bool, + #[prost(int32, tag="7")] + pub creator_action_index: i32, + #[prost(uint32, tag="8")] + pub execution_action_index: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Events { + #[prost(message, repeated, tag="1")] + pub blocks: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="2")] + pub transactions: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="3")] + pub actions: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="4")] + pub db_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="5")] + pub feature_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="6")] + pub perm_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="7")] + pub table_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="8")] + pub accounts: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="9")] + pub keys: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="10")] + pub waits: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="11")] + pub ram_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="12")] + pub authorizations: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="13")] + pub auth_sequences: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="14")] + pub account_ram_deltas: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="15")] + pub creation_trees: ::prost::alloc::vec::Vec, +} +// @@protoc_insertion_point(module) diff --git a/blocks/antelope/src/pb/mod.rs b/blocks/antelope/src/pb/mod.rs new file mode 100644 index 0000000..df270e4 --- /dev/null +++ b/blocks/antelope/src/pb/mod.rs @@ -0,0 +1,6 @@ +// @generated +// @@protoc_insertion_point(attribute:antelope) +pub mod antelope { + include!("antelope.rs"); + // @@protoc_insertion_point(antelope) +} diff --git a/blocks/antelope/src/sinks.rs b/blocks/antelope/src/sinks.rs index 9b1fcf3..5061e7c 100644 --- a/blocks/antelope/src/sinks.rs +++ b/blocks/antelope/src/sinks.rs @@ -1,16 +1,30 @@ +use common::utils::build_timestamp; use substreams::errors::Error; use substreams::pb::substreams::Clock; use substreams_antelope::pb::Block; -use substreams_database_change::pb::database::DatabaseChanges; -use crate::blocks::insert_blocks; +use crate::blocks::collect_block; +use crate::pb::antelope::Events; #[substreams::handlers::map] -pub fn ch_out(clock: Clock, block: Block) -> Result { - let mut tables: DatabaseChanges = DatabaseChanges::default(); +pub fn map_events(clock: Clock, block: Block) -> Result { + let timestamp = build_timestamp(&clock); - // TABLE::blocks - insert_blocks(&mut tables, &clock, &block); - - Ok(tables) + Ok(Events { + blocks: vec![collect_block(&block, ×tamp)], + transactions: vec![], + actions: vec![], + db_ops: vec![], + feature_ops: vec![], + perm_ops: vec![], + table_ops: vec![], + accounts: vec![], + keys: vec![], + waits: vec![], + ram_ops: vec![], + authorizations: vec![], + auth_sequences: vec![], + account_ram_deltas: vec![], + creation_trees: vec![], + }) } diff --git a/blocks/antelope/src/size.rs b/blocks/antelope/src/size.rs index 9660999..2e9c130 100644 --- a/blocks/antelope/src/size.rs +++ b/blocks/antelope/src/size.rs @@ -1,7 +1,16 @@ use substreams_antelope::Block; use substreams_database_change::pb::database::TableChange; -pub fn insert_size(row: &mut TableChange, block: &Block) { +pub struct BlockSize { + pub size: u64, + pub total_transactions: u64, + pub successful_transactions: u64, + pub failed_transactions: u64, + pub total_actions: u64, + pub total_db_ops: u64, +} + +pub fn collect_size(block: &Block) -> BlockSize { // counters let mut size = 0; let mut total_transactions = 0; @@ -57,10 +66,23 @@ pub fn insert_size(row: &mut TableChange, block: &Block) { size += db_op.old_data_json.len(); } } - row.change("size", ("", size.to_string().as_str())) - .change("total_transactions", ("", total_transactions.to_string().as_str())) - .change("successful_transactions", ("", successful_transactions.to_string().as_str())) - .change("failed_transactions", ("", failed_transactions.to_string().as_str())) - .change("total_actions", ("", total_actions.to_string().as_str())) - .change("total_db_ops", ("", total_db_ops.to_string().as_str())); -} \ No newline at end of file + + BlockSize { + size: size as u64, + total_transactions, + successful_transactions, + failed_transactions, + total_actions: total_actions as u64, + total_db_ops: total_db_ops as u64, + } +} + +pub fn insert_size(row: &mut TableChange, block: &Block) { + let size = collect_size(block); + row.change("size", ("", size.size.to_string().as_str())) + .change("total_transactions", ("", size.total_transactions.to_string().as_str())) + .change("successful_transactions", ("", size.successful_transactions.to_string().as_str())) + .change("failed_transactions", ("", size.failed_transactions.to_string().as_str())) + .change("total_actions", ("", size.total_actions.to_string().as_str())) + .change("total_db_ops", ("", size.total_db_ops.to_string().as_str())); +} diff --git a/blocks/antelope/substreams.yaml b/blocks/antelope/substreams.yaml index e6c81e0..093276b 100644 --- a/blocks/antelope/substreams.yaml +++ b/blocks/antelope/substreams.yaml @@ -14,22 +14,23 @@ binaries: type: wasm/rust-v1 file: ../../target/wasm32-unknown-unknown/release/raw_blocks_antelope.wasm +protobuf: + files: + - antelope.proto + importPaths: + - ../../proto + excludePaths: + - sf/substreams + - google + + modules: - - name: ch_out + - name: map_events kind: map inputs: - source: sf.substreams.v1.Clock - source: sf.antelope.type.v1.Block output: - type: proto:sf.substreams.sink.database.v1.DatabaseChanges - -# network: wax + type: proto:antelope.Events -sink: - module: ch_out - type: sf.substreams.sink.sql.v1.Service - config: - schema: "./schema.sql" - engine: clickhouse - postgraphile_frontend: - enabled: false +# network: wax \ No newline at end of file diff --git a/blocks/cosmos/Makefile b/blocks/cosmos/Makefile index bd83be1..ba8ff3e 100644 --- a/blocks/cosmos/Makefile +++ b/blocks/cosmos/Makefile @@ -32,7 +32,7 @@ gui: .PHONY: protogen protogen: - substreams protogen --exclude-paths google + substreams protogen --exclude-paths google,sf .PHONY: parquet parquet: diff --git a/proto/antelope.proto b/proto/antelope.proto new file mode 100644 index 0000000..9ab7815 --- /dev/null +++ b/proto/antelope.proto @@ -0,0 +1,278 @@ +syntax = "proto3"; + +package antelope; + +import "google/protobuf/timestamp.proto"; + + +message Block { + google.protobuf.Timestamp time = 1; + uint64 number = 2; + string date = 3; + string hash = 4; + string parent_hash = 5; + string producer = 6; + uint32 confirmed = 7; + uint32 schedule_version = 8; + uint32 version = 9; + string producer_signature = 10; + uint32 dpos_proposed_irreversible_blocknum = 11; + uint32 dpos_irreversible_blocknum = 12; + string transaction_mroot = 13; + string action_mroot = 14; + string blockroot_merkle_active_nodes = 15; + uint32 blockroot_merkle_node_count = 16; + uint64 size = 17; + uint64 total_transactions = 18; + uint64 successful_transactions = 19; + uint64 failed_transactions = 20; + uint64 total_actions = 21; + uint64 total_db_ops = 22; +} + +message Transaction { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string hash = 5; + uint64 index = 6; + int64 elapsed = 7; + uint64 net_usage = 8; + bool scheduled = 9; + uint32 cpu_usage_micro_seconds = 10; + uint32 net_usage_words = 11; + string status = 12; + uint32 status_code = 13; + bool success = 14; + string transaction_mroot = 15; +} + +message Action { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + uint64 abi_sequence = 7; + uint64 code_sequence = 8; + string digest = 9; + uint64 global_sequence = 10; + string receipt_receiver = 11; + uint64 recv_sequence = 12; + string account = 13; + string name = 14; + string json_data = 15; + string raw_data = 16; + uint32 index = 17; + uint32 action_ordinal = 18; + string receiver = 19; + bool context_free = 20; + int64 elapsed = 21; + string console = 22; + string raw_return_value = 23; + string json_return_value = 24; + uint32 creator_action_ordinal = 25; + uint32 closest_unnotified_ancestor_action_ordinal = 26; + string action_mroot = 27; +} + +message DbOp { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + uint32 action_index = 7; + uint32 index = 8; + string operation = 9; + uint32 operation_code = 10; + string code = 11; + string scope = 12; + string table_name = 13; + string primary_key = 14; + string old_payer = 15; + string new_payer = 16; + string old_data = 17; + string new_data = 18; + string old_data_json = 19; + string new_data_json = 20; +} + +message FeatureOp { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + uint32 action_index = 7; + string feature_digest = 8; + string kind = 9; + string description_digest = 10; + string protocol_feature_type = 11; +} + +message PermOp { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + uint32 action_index = 7; + string operation = 8; + uint32 operation_code = 9; + uint64 id = 10; + uint64 parent_id = 11; + string owner = 12; + string name = 13; + uint32 threshold = 14; +} + +message TableOp { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + uint32 action_index = 7; + uint32 index = 8; + string operation = 9; + uint32 operation_code = 10; + string payer = 11; + string code = 12; + string scope = 13; + string table_name = 14; +} + +message Account { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + uint32 action_index = 7; + uint32 index = 8; + string actor = 9; + string permission = 10; + uint32 weight = 11; +} + +message Key { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + uint32 action_index = 7; + uint32 index = 8; + string public_key = 9; + uint32 weight = 10; +} + +message Wait { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + uint32 action_index = 7; + uint32 index = 8; + uint32 wait_sec = 9; + uint32 weight = 10; +} + +message RamOp { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + uint32 action_index = 7; + string operation = 8; + uint32 operation_code = 9; + string payer = 10; + int64 delta = 11; + uint64 usage = 12; + string namespace = 13; + uint32 namespace_code = 14; + string action = 15; + uint32 action_code = 16; + string unique_key = 17; +} + +message Authorization { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + uint32 action_index = 7; + uint32 index = 8; + string actor = 9; + string permission = 10; +} + +message AuthSequence { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + uint32 action_index = 7; + uint32 index = 8; + string account_name = 9; + uint64 sequence = 10; +} + +message AccountRamDelta { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + uint32 action_index = 7; + uint32 index = 8; + string account = 9; + int64 delta = 10; +} + +message CreationTree { + google.protobuf.Timestamp block_time = 1; + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + string tx_hash = 5; + bool tx_success = 6; + int32 creator_action_index = 7; + uint32 execution_action_index = 8; +} + +message Events { + repeated Block blocks = 1; + repeated Transaction transactions = 2; + repeated Action actions = 3; + repeated DbOp db_ops = 4; + repeated FeatureOp feature_ops = 5; + repeated PermOp perm_ops = 6; + repeated TableOp table_ops = 7; + repeated Account accounts = 8; + repeated Key keys = 9; + repeated Wait waits = 10; + repeated RamOp ram_ops = 11; + repeated Authorization authorizations = 12; + repeated AuthSequence auth_sequences = 13; + repeated AccountRamDelta account_ram_deltas = 14; + repeated CreationTree creation_trees = 15; +} From 93b00918d4c73a49b1af1d473716142e36777f1f Mon Sep 17 00:00:00 2001 From: zolting <44510235+zolting@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:16:24 -0500 Subject: [PATCH 02/22] Add Transactions --- blocks/antelope/src/sinks.rs | 5 ++- blocks/antelope/src/transactions.rs | 49 +++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/blocks/antelope/src/sinks.rs b/blocks/antelope/src/sinks.rs index 5061e7c..486f67c 100644 --- a/blocks/antelope/src/sinks.rs +++ b/blocks/antelope/src/sinks.rs @@ -3,8 +3,7 @@ use substreams::errors::Error; use substreams::pb::substreams::Clock; use substreams_antelope::pb::Block; -use crate::blocks::collect_block; -use crate::pb::antelope::Events; +use crate::{blocks::collect_block, pb::antelope::Events, transactions::collect_transactions}; #[substreams::handlers::map] pub fn map_events(clock: Clock, block: Block) -> Result { @@ -12,7 +11,7 @@ pub fn map_events(clock: Clock, block: Block) -> Result { Ok(Events { blocks: vec![collect_block(&block, ×tamp)], - transactions: vec![], + transactions: collect_transactions(&block, ×tamp), actions: vec![], db_ops: vec![], feature_ops: vec![], diff --git a/blocks/antelope/src/transactions.rs b/blocks/antelope/src/transactions.rs index c013cda..36380eb 100644 --- a/blocks/antelope/src/transactions.rs +++ b/blocks/antelope/src/transactions.rs @@ -1,16 +1,13 @@ -use common::blocks::insert_timestamp; +use common::{blocks::insert_timestamp, structs::BlockTimestamp}; use substreams::pb::substreams::Clock; use substreams::Hex; -use substreams_antelope::pb::{BlockHeader, TransactionTrace}; -use substreams_database_change::pb::database::TableChange; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; +use substreams_antelope::pb::{Block, BlockHeader, TransactionTrace}; +use substreams_database_change::pb::database::{table_change, DatabaseChanges, TableChange}; -use crate::creation_tree::insert_creation_tree; -use crate::feature_ops::insert_feature_op; -use crate::keys::transactions_keys; -use crate::perm_ops::insert_perm_op; -use crate::ram_ops::insert_ram_op; -use crate::table_ops::insert_table_op; +use crate::{ + creation_tree::insert_creation_tree, feature_ops::insert_feature_op, keys::transactions_keys, pb::antelope::Transaction as RawTransaction, perm_ops::insert_perm_op, ram_ops::insert_ram_op, + table_ops::insert_table_op, +}; use super::actions::insert_action; use super::db_ops::insert_db_op; @@ -161,3 +158,35 @@ pub fn insert_transaction_metadata(row: &mut TableChange, transaction: &Transact row.change("tx_hash", ("", tx_hash.as_str())).change("tx_success", ("", tx_success.to_string().as_str())); } + +pub fn collect_transactions(block: &Block, timestamp: &BlockTimestamp) -> Vec { + let header = block.header.clone().unwrap_or_default(); + let mut transactions = Vec::new(); + + for (index, transaction) in block.transaction_traces().enumerate() { + let receipt = transaction.receipt.clone().unwrap_or_default(); + let status_code = receipt.status; + let status = transaction_status_to_string(status_code); + let success = is_transaction_success(status_code); + + transactions.push(RawTransaction { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + hash: transaction.id.clone(), + index: index as u64, + elapsed: transaction.elapsed, + net_usage: transaction.net_usage, + scheduled: transaction.scheduled, + cpu_usage_micro_seconds: receipt.cpu_usage_micro_seconds, + net_usage_words: receipt.net_usage_words, + status, + status_code: status_code as u32, + success, + transaction_mroot: Hex::encode(&header.transaction_mroot.to_vec()), + }); + } + + transactions +} From 3c9c9378240ad1c7843f3eb91fbbe5d1979d344a Mon Sep 17 00:00:00 2001 From: zolting <44510235+zolting@users.noreply.github.com> Date: Mon, 18 Nov 2024 13:48:08 -0500 Subject: [PATCH 03/22] Add Actions --- blocks/antelope/src/actions.rs | 50 ++++++++++++++++++++++++++++++++++ blocks/antelope/src/sinks.rs | 4 +-- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/blocks/antelope/src/actions.rs b/blocks/antelope/src/actions.rs index 9cd34cf..a13cfd1 100644 --- a/blocks/antelope/src/actions.rs +++ b/blocks/antelope/src/actions.rs @@ -1,13 +1,16 @@ use common::blocks::insert_timestamp; +use common::structs::BlockTimestamp; use substreams::pb::substreams::Clock; use substreams::Hex; use substreams_antelope::pb::{ActionTrace, BlockHeader, TransactionTrace}; +use substreams_antelope::Block; use substreams_database_change::pb::database::{table_change, DatabaseChanges}; use crate::account_ram_deltas::insert_account_ram_delta; use crate::auth_sequences::insert_auth_sequence; use crate::authorizations::insert_authorization; use crate::keys::actions_keys; +use crate::pb::antelope::Action as RawAction; use crate::transactions::insert_transaction_metadata; // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L525 @@ -123,3 +126,50 @@ pub fn insert_action(tables: &mut DatabaseChanges, clock: &Clock, trace: &Action // insert_account_ram_delta(tables, clock, trace, &account_ram_delta, &block); // } } + +pub fn collect_actions(block: &Block, timestamp: &BlockTimestamp) -> Vec { + let header = block.header.clone().unwrap_or_default(); + let mut actions = Vec::new(); + + for transaction in block.transaction_traces() { + let tx_hash = &transaction.id; + let tx_success = transaction.receipt.clone().unwrap_or_default().status == 1; + + for trace in transaction.action_traces.iter() { + let action = trace.action.clone().unwrap_or_default(); + let receipt = trace.receipt.clone().unwrap_or_default(); + + actions.push(RawAction { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + abi_sequence: receipt.abi_sequence, + code_sequence: receipt.code_sequence, + digest: receipt.digest, + global_sequence: receipt.global_sequence, + receipt_receiver: receipt.receiver, + recv_sequence: receipt.recv_sequence, + account: action.account, + name: action.name, + json_data: action.json_data, + raw_data: Hex::encode(&action.raw_data), + index: trace.execution_index, + action_ordinal: trace.action_ordinal, + receiver: trace.receiver.clone(), + context_free: trace.context_free, + elapsed: trace.elapsed, + console: trace.console.clone(), + raw_return_value: Hex::encode(&trace.raw_return_value), + json_return_value: trace.json_return_value.clone(), + creator_action_ordinal: trace.creator_action_ordinal, + closest_unnotified_ancestor_action_ordinal: trace.closest_unnotified_ancestor_action_ordinal, + action_mroot: Hex::encode(&header.action_mroot), + }); + } + } + + actions +} diff --git a/blocks/antelope/src/sinks.rs b/blocks/antelope/src/sinks.rs index 486f67c..e2e525a 100644 --- a/blocks/antelope/src/sinks.rs +++ b/blocks/antelope/src/sinks.rs @@ -3,7 +3,7 @@ use substreams::errors::Error; use substreams::pb::substreams::Clock; use substreams_antelope::pb::Block; -use crate::{blocks::collect_block, pb::antelope::Events, transactions::collect_transactions}; +use crate::{actions::collect_actions, blocks::collect_block, pb::antelope::Events, transactions::collect_transactions}; #[substreams::handlers::map] pub fn map_events(clock: Clock, block: Block) -> Result { @@ -12,7 +12,7 @@ pub fn map_events(clock: Clock, block: Block) -> Result { Ok(Events { blocks: vec![collect_block(&block, ×tamp)], transactions: collect_transactions(&block, ×tamp), - actions: vec![], + actions: collect_actions(&block, ×tamp), db_ops: vec![], feature_ops: vec![], perm_ops: vec![], From 966bb9781d784ee122aae493a623524783753a34 Mon Sep 17 00:00:00 2001 From: zolting <44510235+zolting@users.noreply.github.com> Date: Mon, 18 Nov 2024 13:51:36 -0500 Subject: [PATCH 04/22] Add DbOp --- blocks/antelope/src/db_ops.rs | 45 ++++++++++++++++++++++++++++++++--- blocks/antelope/src/sinks.rs | 4 ++-- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/blocks/antelope/src/db_ops.rs b/blocks/antelope/src/db_ops.rs index b6fef6a..ea194be 100644 --- a/blocks/antelope/src/db_ops.rs +++ b/blocks/antelope/src/db_ops.rs @@ -1,9 +1,12 @@ -use common::blocks::insert_timestamp; +use common::{blocks::insert_timestamp, structs::BlockTimestamp}; use substreams::{pb::substreams::Clock, Hex}; -use substreams_antelope::pb::{DbOp, TransactionTrace}; +use substreams_antelope::{ + pb::{DbOp, TransactionTrace}, + Block, +}; use substreams_database_change::pb::database::{table_change, DatabaseChanges}; -use crate::keys::db_ops_keys; +use crate::{keys::db_ops_keys, pb::antelope::DbOp as RawDbOp}; use super::transactions::insert_transaction_metadata; @@ -57,3 +60,39 @@ pub fn insert_db_op(tables: &mut DatabaseChanges, clock: &Clock, db_op: &DbOp, t insert_timestamp(row, clock, false, false); insert_transaction_metadata(row, transaction); } + +pub fn collect_db_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { + let mut db_ops = Vec::new(); + + for transaction in block.transaction_traces() { + let tx_hash = &transaction.id; + let tx_success = transaction.receipt.clone().unwrap_or_default().status == 1; + + for (index, db_op) in transaction.db_ops.iter().enumerate() { + db_ops.push(RawDbOp { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + index: index as u32, + operation: operation_to_string(db_op.operation), + operation_code: db_op.operation as u32, + action_index: db_op.action_index, + code: db_op.code.clone(), + scope: db_op.scope.clone(), + table_name: db_op.table_name.clone(), + primary_key: db_op.primary_key.clone(), + old_payer: db_op.old_payer.clone(), + new_payer: db_op.new_payer.clone(), + old_data: Hex::encode(&db_op.old_data), + new_data: Hex::encode(&db_op.new_data), + old_data_json: db_op.old_data_json.clone(), + new_data_json: db_op.new_data_json.clone(), + }); + } + } + + db_ops +} diff --git a/blocks/antelope/src/sinks.rs b/blocks/antelope/src/sinks.rs index e2e525a..8e449a7 100644 --- a/blocks/antelope/src/sinks.rs +++ b/blocks/antelope/src/sinks.rs @@ -3,7 +3,7 @@ use substreams::errors::Error; use substreams::pb::substreams::Clock; use substreams_antelope::pb::Block; -use crate::{actions::collect_actions, blocks::collect_block, pb::antelope::Events, transactions::collect_transactions}; +use crate::{actions::collect_actions, blocks::collect_block, db_ops::collect_db_ops, pb::antelope::Events, transactions::collect_transactions}; #[substreams::handlers::map] pub fn map_events(clock: Clock, block: Block) -> Result { @@ -13,7 +13,7 @@ pub fn map_events(clock: Clock, block: Block) -> Result { blocks: vec![collect_block(&block, ×tamp)], transactions: collect_transactions(&block, ×tamp), actions: collect_actions(&block, ×tamp), - db_ops: vec![], + db_ops: collect_db_ops(&block, ×tamp), feature_ops: vec![], perm_ops: vec![], table_ops: vec![], From 35f9b0dea251629e9cfcd420270956011b9017a3 Mon Sep 17 00:00:00 2001 From: zolting <44510235+zolting@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:07:57 -0500 Subject: [PATCH 05/22] Add Feature/Perm/Table Ops --- blocks/antelope/src/actions.rs | 4 +-- blocks/antelope/src/db_ops.rs | 4 +-- blocks/antelope/src/feature_ops.rs | 43 +++++++++++++++++++++++--- blocks/antelope/src/perm_ops.rs | 48 ++++++++++++++++++++++++++++-- blocks/antelope/src/sinks.rs | 11 ++++--- blocks/antelope/src/table_ops.rs | 48 +++++++++++++++++++++++++----- 6 files changed, 136 insertions(+), 22 deletions(-) diff --git a/blocks/antelope/src/actions.rs b/blocks/antelope/src/actions.rs index a13cfd1..dba7af2 100644 --- a/blocks/antelope/src/actions.rs +++ b/blocks/antelope/src/actions.rs @@ -11,7 +11,7 @@ use crate::auth_sequences::insert_auth_sequence; use crate::authorizations::insert_authorization; use crate::keys::actions_keys; use crate::pb::antelope::Action as RawAction; -use crate::transactions::insert_transaction_metadata; +use crate::transactions::{insert_transaction_metadata, is_transaction_success}; // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L525 pub fn insert_action(tables: &mut DatabaseChanges, clock: &Clock, trace: &ActionTrace, transaction: &TransactionTrace, block_header: &BlockHeader) { @@ -133,7 +133,7 @@ pub fn collect_actions(block: &Block, timestamp: &BlockTimestamp) -> Vec Vec for transaction in block.transaction_traces() { let tx_hash = &transaction.id; - let tx_success = transaction.receipt.clone().unwrap_or_default().status == 1; + let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); for (index, db_op) in transaction.db_ops.iter().enumerate() { db_ops.push(RawDbOp { diff --git a/blocks/antelope/src/feature_ops.rs b/blocks/antelope/src/feature_ops.rs index 6d8cff1..7fd516e 100644 --- a/blocks/antelope/src/feature_ops.rs +++ b/blocks/antelope/src/feature_ops.rs @@ -1,10 +1,16 @@ -use common::blocks::insert_timestamp; +use common::{blocks::insert_timestamp, structs::BlockTimestamp}; use substreams::pb::substreams::Clock; -use substreams_antelope::pb::{FeatureOp, TransactionTrace}; +use substreams_antelope::{ + pb::{FeatureOp, TransactionTrace}, + Block, +}; use substreams_database_change::pb::database::{table_change, DatabaseChanges}; -use crate::keys::feature_ops_keys; -use crate::transactions::insert_transaction_metadata; +use crate::{ + keys::feature_ops_keys, + pb::antelope::FeatureOp as RawFeatureOp, + transactions::{insert_transaction_metadata, is_transaction_success}, +}; pub fn insert_feature_op(tables: &mut DatabaseChanges, clock: &Clock, transaction: &TransactionTrace, feature_op: &FeatureOp) { let feature_digest = &feature_op.feature_digest.as_str(); @@ -29,3 +35,32 @@ pub fn insert_feature_op(tables: &mut DatabaseChanges, clock: &Clock, transactio insert_transaction_metadata(row, transaction); insert_timestamp(row, clock, false, false); } + +pub fn collect_feature_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { + let mut feature_ops: Vec = vec![]; + + for transaction in block.transaction_traces() { + let tx_hash = &transaction.id; + let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); + + for feature_op in transaction.feature_ops.iter() { + let feature = feature_op.feature.as_ref().expect("feature is required"); + + feature_ops.push(RawFeatureOp { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + feature_digest: feature_op.feature_digest.clone(), + kind: feature_op.kind.clone(), + action_index: feature_op.action_index, + description_digest: feature.description_digest.clone(), + protocol_feature_type: feature.protocol_feature_type.clone(), + }); + } + } + + feature_ops +} diff --git a/blocks/antelope/src/perm_ops.rs b/blocks/antelope/src/perm_ops.rs index 6f84181..bb9fb3d 100644 --- a/blocks/antelope/src/perm_ops.rs +++ b/blocks/antelope/src/perm_ops.rs @@ -1,7 +1,15 @@ -use crate::{authority::insert_authority, keys::perm_ops_keys, transactions::insert_transaction_metadata}; -use common::blocks::insert_timestamp; +use crate::{ + authority::insert_authority, + keys::perm_ops_keys, + pb::antelope::PermOp as RawPermOp, + transactions::{insert_transaction_metadata, is_transaction_success}, +}; +use common::{blocks::insert_timestamp, structs::BlockTimestamp}; use substreams::pb::substreams::Clock; -use substreams_antelope::pb::{PermOp, TransactionTrace}; +use substreams_antelope::{ + pb::{PermOp, TransactionTrace}, + Block, +}; use substreams_database_change::pb::database::{table_change, DatabaseChanges}; pub fn perm_op_operation_to_string(operation: i32) -> String { @@ -50,3 +58,37 @@ pub fn insert_perm_op(tables: &mut DatabaseChanges, clock: &Clock, transaction: None => {} } } + +pub fn collect_perm_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { + let mut perm_ops: Vec = vec![]; + + for transaction in block.transaction_traces() { + let tx_hash = &transaction.id; + let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); + + for perm_op in transaction.perm_ops.iter() { + if let Some(new_perm) = &perm_op.new_perm { + let threshold = new_perm.authority.as_ref().map_or(0, |authority| authority.threshold); + + perm_ops.push(RawPermOp { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + operation: perm_op_operation_to_string(perm_op.operation), + operation_code: perm_op.operation as u32, + action_index: perm_op.action_index, + id: new_perm.id, + parent_id: new_perm.parent_id, + owner: new_perm.owner.clone(), + name: new_perm.name.clone(), + threshold, + }); + } + } + } + + perm_ops +} diff --git a/blocks/antelope/src/sinks.rs b/blocks/antelope/src/sinks.rs index 8e449a7..75c517d 100644 --- a/blocks/antelope/src/sinks.rs +++ b/blocks/antelope/src/sinks.rs @@ -3,7 +3,10 @@ use substreams::errors::Error; use substreams::pb::substreams::Clock; use substreams_antelope::pb::Block; -use crate::{actions::collect_actions, blocks::collect_block, db_ops::collect_db_ops, pb::antelope::Events, transactions::collect_transactions}; +use crate::{ + actions::collect_actions, blocks::collect_block, db_ops::collect_db_ops, feature_ops::collect_feature_ops, pb::antelope::Events, perm_ops::collect_perm_ops, table_ops::collect_table_ops, + transactions::collect_transactions, +}; #[substreams::handlers::map] pub fn map_events(clock: Clock, block: Block) -> Result { @@ -14,9 +17,9 @@ pub fn map_events(clock: Clock, block: Block) -> Result { transactions: collect_transactions(&block, ×tamp), actions: collect_actions(&block, ×tamp), db_ops: collect_db_ops(&block, ×tamp), - feature_ops: vec![], - perm_ops: vec![], - table_ops: vec![], + feature_ops: collect_feature_ops(&block, ×tamp), + perm_ops: collect_perm_ops(&block, ×tamp), + table_ops: collect_table_ops(&block, ×tamp), accounts: vec![], keys: vec![], waits: vec![], diff --git a/blocks/antelope/src/table_ops.rs b/blocks/antelope/src/table_ops.rs index fa29a50..3616ed6 100644 --- a/blocks/antelope/src/table_ops.rs +++ b/blocks/antelope/src/table_ops.rs @@ -1,12 +1,16 @@ -use common::blocks::insert_timestamp; +use common::{blocks::insert_timestamp, structs::BlockTimestamp}; use substreams::pb::substreams::Clock; -use substreams_antelope::pb::TableOp; -use substreams_antelope::pb::TransactionTrace; -use substreams_database_change::pb::database::table_change; -use substreams_database_change::pb::database::DatabaseChanges; +use substreams_antelope::{ + pb::{TableOp, TransactionTrace}, + Block, +}; +use substreams_database_change::pb::database::{table_change, DatabaseChanges}; -use crate::keys::table_ops_keys; -use crate::transactions::insert_transaction_metadata; +use crate::{ + keys::table_ops_keys, + pb::antelope::TableOp as RawTableOp, + transactions::{insert_transaction_metadata, is_transaction_success}, +}; pub fn table_op_operation_to_string(operation: i32) -> String { match operation { @@ -43,3 +47,33 @@ pub fn insert_table_op(tables: &mut DatabaseChanges, clock: &Clock, transaction: insert_transaction_metadata(row, transaction); insert_timestamp(row, clock, false, false); } + +pub fn collect_table_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { + let mut table_ops: Vec = vec![]; + + for transaction in block.transaction_traces() { + let tx_hash = &transaction.id; + let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); + + for (index, table_op) in transaction.table_ops.iter().enumerate() { + table_ops.push(RawTableOp { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + index: index as u32, + action_index: table_op.action_index, + operation: table_op_operation_to_string(table_op.operation), + operation_code: table_op.operation as u32, + payer: table_op.payer.clone(), + code: table_op.code.clone(), + scope: table_op.scope.clone(), + table_name: table_op.table_name.clone(), + }); + } + } + + table_ops +} From dd4eaab9806adc2299fc71fcf27e28f7af4e8602 Mon Sep 17 00:00:00 2001 From: zolting <44510235+zolting@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:22:46 -0500 Subject: [PATCH 06/22] Add Accounts/Keys/Waits --- blocks/antelope/src/authority.rs | 90 ++++++++++++++++++++++++++++++-- blocks/antelope/src/sinks.rs | 11 ++-- 2 files changed, 93 insertions(+), 8 deletions(-) diff --git a/blocks/antelope/src/authority.rs b/blocks/antelope/src/authority.rs index ce4e3f4..44b9fab 100644 --- a/blocks/antelope/src/authority.rs +++ b/blocks/antelope/src/authority.rs @@ -1,9 +1,93 @@ -use crate::{keys::authority_keys, transactions::insert_transaction_metadata}; -use common::blocks::insert_timestamp; +use crate::{ + keys::authority_keys, + pb::antelope::{Account as RawAccount, Key as RawKey, Wait as RawWait}, + transactions::{insert_transaction_metadata, is_transaction_success}, +}; +use common::{blocks::insert_timestamp, structs::BlockTimestamp}; use substreams::pb::substreams::Clock; -use substreams_antelope::pb::{Authority, TransactionTrace}; +use substreams_antelope::{ + pb::{Authority, TransactionTrace}, + Block, +}; use substreams_database_change::pb::database::{table_change, DatabaseChanges}; +pub struct AuthorityVectors { + pub accounts: Vec, + pub keys: Vec, + pub waits: Vec, +} + +pub fn collect_authority_vectors(block: &Block, timestamp: &BlockTimestamp) -> AuthorityVectors { + let mut accounts: Vec = vec![]; + let mut keys: Vec = vec![]; + let mut waits: Vec = vec![]; + + for transaction in block.transaction_traces() { + let tx_hash = &transaction.id; + let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); + + for perm_op in transaction.perm_ops.iter() { + if let Some(new_perm) = &perm_op.new_perm { + let authority = new_perm.authority.as_ref().unwrap(); + let action_index = perm_op.action_index; + + // Process authority + for (index, account) in authority.accounts.iter().enumerate() { + if let Some(permission) = &account.permission { + accounts.push(RawAccount { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + index: index as u32, + action_index: perm_op.action_index as u32, + actor: permission.actor.clone(), + permission: permission.permission.clone(), + weight: account.weight, + }); + } + } + + // Process keys + for (index, key) in authority.keys.iter().enumerate() { + keys.push(RawKey { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + index: index as u32, + action_index: action_index as u32, + public_key: key.public_key.clone(), + weight: key.weight, + }); + } + + // Process waits + for (index, wait) in authority.waits.iter().enumerate() { + waits.push(RawWait { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + index: index as u32, + action_index: action_index as u32, + wait_sec: wait.wait_sec, + weight: wait.weight, + }); + } + } + } + } + + AuthorityVectors { accounts, keys, waits } +} + pub fn insert_authority(tables: &mut DatabaseChanges, clock: &Clock, transaction: &TransactionTrace, action_index: u32, authority: &Authority) { let mut index = 0; let tx_hash = &transaction.id; diff --git a/blocks/antelope/src/sinks.rs b/blocks/antelope/src/sinks.rs index 75c517d..96d3c8c 100644 --- a/blocks/antelope/src/sinks.rs +++ b/blocks/antelope/src/sinks.rs @@ -4,13 +4,14 @@ use substreams::pb::substreams::Clock; use substreams_antelope::pb::Block; use crate::{ - actions::collect_actions, blocks::collect_block, db_ops::collect_db_ops, feature_ops::collect_feature_ops, pb::antelope::Events, perm_ops::collect_perm_ops, table_ops::collect_table_ops, - transactions::collect_transactions, + actions::collect_actions, authority::collect_authority_vectors, blocks::collect_block, db_ops::collect_db_ops, feature_ops::collect_feature_ops, pb::antelope::Events, perm_ops::collect_perm_ops, + table_ops::collect_table_ops, transactions::collect_transactions, }; #[substreams::handlers::map] pub fn map_events(clock: Clock, block: Block) -> Result { let timestamp = build_timestamp(&clock); + let authority_vectors = collect_authority_vectors(&block, ×tamp); Ok(Events { blocks: vec![collect_block(&block, ×tamp)], @@ -20,9 +21,9 @@ pub fn map_events(clock: Clock, block: Block) -> Result { feature_ops: collect_feature_ops(&block, ×tamp), perm_ops: collect_perm_ops(&block, ×tamp), table_ops: collect_table_ops(&block, ×tamp), - accounts: vec![], - keys: vec![], - waits: vec![], + accounts: authority_vectors.accounts, + keys: authority_vectors.keys, + waits: authority_vectors.waits, ram_ops: vec![], authorizations: vec![], auth_sequences: vec![], From a2717f233e3e50e75788cfda77177c0796224a2a Mon Sep 17 00:00:00 2001 From: zolting <44510235+zolting@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:26:32 -0500 Subject: [PATCH 07/22] Add RamOps --- blocks/antelope/src/ram_ops.rs | 38 +++++++++++++++++++++++++++++++++- blocks/antelope/src/sinks.rs | 4 ++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/blocks/antelope/src/ram_ops.rs b/blocks/antelope/src/ram_ops.rs index d3776db..ed85cc7 100644 --- a/blocks/antelope/src/ram_ops.rs +++ b/blocks/antelope/src/ram_ops.rs @@ -1,10 +1,13 @@ use common::blocks::insert_timestamp; +use common::structs::BlockTimestamp; use substreams::pb::substreams::Clock; use substreams_antelope::pb::{RamOp, TransactionTrace}; +use substreams_antelope::Block; use substreams_database_change::pb::database::{table_change, DatabaseChanges}; use crate::keys::ram_op_keys; -use crate::transactions::insert_transaction_metadata; +use crate::pb::antelope::RamOp as RawRamOp; +use crate::transactions::{insert_transaction_metadata, is_transaction_success}; pub fn namespace_to_string(namespace: i32) -> String { match namespace { @@ -95,3 +98,36 @@ pub fn insert_ram_op(tables: &mut DatabaseChanges, clock: &Clock, ram_op: &RamOp insert_transaction_metadata(row, transaction); insert_timestamp(row, clock, false, false); } + +pub fn collect_ram_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { + let mut ram_ops: Vec = vec![]; + + for transaction in block.transaction_traces() { + let tx_hash = &transaction.id; + let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); + + for ram_op in transaction.ram_ops.iter() { + ram_ops.push(RawRamOp { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + operation: operation_to_string(ram_op.operation), + action_index: ram_op.action_index, + payer: ram_op.payer.clone(), + delta: ram_op.delta, + usage: ram_op.usage, + namespace: namespace_to_string(ram_op.namespace), + action: action_to_string(ram_op.action), + unique_key: ram_op.unique_key.clone(), + operation_code: ram_op.operation as u32, + namespace_code: ram_op.namespace as u32, + action_code: ram_op.action as u32, + }); + } + } + + ram_ops +} diff --git a/blocks/antelope/src/sinks.rs b/blocks/antelope/src/sinks.rs index 96d3c8c..95b84ea 100644 --- a/blocks/antelope/src/sinks.rs +++ b/blocks/antelope/src/sinks.rs @@ -5,7 +5,7 @@ use substreams_antelope::pb::Block; use crate::{ actions::collect_actions, authority::collect_authority_vectors, blocks::collect_block, db_ops::collect_db_ops, feature_ops::collect_feature_ops, pb::antelope::Events, perm_ops::collect_perm_ops, - table_ops::collect_table_ops, transactions::collect_transactions, + ram_ops::collect_ram_ops, table_ops::collect_table_ops, transactions::collect_transactions, }; #[substreams::handlers::map] @@ -24,7 +24,7 @@ pub fn map_events(clock: Clock, block: Block) -> Result { accounts: authority_vectors.accounts, keys: authority_vectors.keys, waits: authority_vectors.waits, - ram_ops: vec![], + ram_ops: collect_ram_ops(&block, ×tamp), authorizations: vec![], auth_sequences: vec![], account_ram_deltas: vec![], From 0d3e00875c1db0383a3057dff114d03d1ca2c99b Mon Sep 17 00:00:00 2001 From: zolting <44510235+zolting@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:41:42 -0500 Subject: [PATCH 08/22] Add Authorizations --- blocks/antelope/src/authorizations.rs | 41 +++++++++++++++++++++++++-- blocks/antelope/src/sinks.rs | 6 ++-- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/blocks/antelope/src/authorizations.rs b/blocks/antelope/src/authorizations.rs index 01f9ec5..60d6f4d 100644 --- a/blocks/antelope/src/authorizations.rs +++ b/blocks/antelope/src/authorizations.rs @@ -1,9 +1,16 @@ -use common::blocks::insert_timestamp; +use common::{blocks::insert_timestamp, structs::BlockTimestamp}; use substreams::pb::substreams::Clock; -use substreams_antelope::pb::{ActionTrace, PermissionLevel, TransactionTrace}; +use substreams_antelope::{ + pb::{ActionTrace, PermissionLevel, TransactionTrace}, + Block, +}; use substreams_database_change::pb::database::{table_change, DatabaseChanges}; -use crate::{keys::authorizations_keys, transactions::insert_transaction_metadata}; +use crate::{ + keys::authorizations_keys, + pb::antelope::Authorization as RawAuthorization, + transactions::{insert_transaction_metadata, is_transaction_success}, +}; // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L616 pub fn insert_authorization(tables: &mut DatabaseChanges, clock: &Clock, action: &ActionTrace, transaction: &TransactionTrace, authorization: &PermissionLevel, index: &u32) { @@ -28,3 +35,31 @@ pub fn insert_authorization(tables: &mut DatabaseChanges, clock: &Clock, action: insert_transaction_metadata(row, transaction); insert_timestamp(row, clock, false, false); } + +pub fn collect_authorizations(block: &Block, timestamp: &BlockTimestamp) -> Vec { + let mut authorizations: Vec = vec![]; + + for transaction in block.transaction_traces() { + let tx_hash = &transaction.id; + let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); + + for action_trace in transaction.action_traces.iter() { + for (index, authorization) in action_trace.action.as_ref().unwrap().authorization.iter().enumerate() { + authorizations.push(RawAuthorization { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + action_index: action_trace.execution_index, + index: index as u32, + actor: authorization.actor.clone(), + permission: authorization.permission.clone(), + }); + } + } + } + + authorizations +} diff --git a/blocks/antelope/src/sinks.rs b/blocks/antelope/src/sinks.rs index 95b84ea..3180b09 100644 --- a/blocks/antelope/src/sinks.rs +++ b/blocks/antelope/src/sinks.rs @@ -4,8 +4,8 @@ use substreams::pb::substreams::Clock; use substreams_antelope::pb::Block; use crate::{ - actions::collect_actions, authority::collect_authority_vectors, blocks::collect_block, db_ops::collect_db_ops, feature_ops::collect_feature_ops, pb::antelope::Events, perm_ops::collect_perm_ops, - ram_ops::collect_ram_ops, table_ops::collect_table_ops, transactions::collect_transactions, + actions::collect_actions, authority::collect_authority_vectors, authorizations::collect_authorizations, blocks::collect_block, db_ops::collect_db_ops, feature_ops::collect_feature_ops, + pb::antelope::Events, perm_ops::collect_perm_ops, ram_ops::collect_ram_ops, table_ops::collect_table_ops, transactions::collect_transactions, }; #[substreams::handlers::map] @@ -25,7 +25,7 @@ pub fn map_events(clock: Clock, block: Block) -> Result { keys: authority_vectors.keys, waits: authority_vectors.waits, ram_ops: collect_ram_ops(&block, ×tamp), - authorizations: vec![], + authorizations: collect_authorizations(&block, ×tamp), auth_sequences: vec![], account_ram_deltas: vec![], creation_trees: vec![], From 6f7cebe16a1fa0a3aebd4fd6db650cc0a7ca59a8 Mon Sep 17 00:00:00 2001 From: zolting Date: Mon, 18 Nov 2024 17:21:32 -0500 Subject: [PATCH 09/22] Add AuthSequence --- blocks/antelope/src/auth_sequences.rs | 36 ++ blocks/antelope/src/pb/beacon.rs | 515 -------------------------- blocks/antelope/src/sinks.rs | 6 +- 3 files changed, 39 insertions(+), 518 deletions(-) delete mode 100644 blocks/antelope/src/pb/beacon.rs diff --git a/blocks/antelope/src/auth_sequences.rs b/blocks/antelope/src/auth_sequences.rs index aabfb9a..cc4276f 100644 --- a/blocks/antelope/src/auth_sequences.rs +++ b/blocks/antelope/src/auth_sequences.rs @@ -1,11 +1,15 @@ use common::blocks::insert_timestamp; +use common::structs::BlockTimestamp; use substreams::pb::substreams::Clock; use substreams_antelope::pb::AuthSequence; use substreams_antelope::pb::{ActionTrace, TransactionTrace}; +use substreams_antelope::Block; use substreams_database_change::pb::database::{table_change, DatabaseChanges}; use crate::keys::auth_sequence_keys; +use crate::pb::antelope::AuthSequence as RawAuthSequence; use crate::transactions::insert_transaction_metadata; +use crate::transactions::is_transaction_success; pub fn insert_auth_sequence(tables: &mut DatabaseChanges, clock: &Clock, action: &ActionTrace, transaction: &TransactionTrace, auth_sequence: &AuthSequence, index: &u32) { // transaction @@ -28,3 +32,35 @@ pub fn insert_auth_sequence(tables: &mut DatabaseChanges, clock: &Clock, action: insert_transaction_metadata(row, transaction); insert_timestamp(row, clock, false, false); } + +pub fn collect_auth_sequences(block: &Block, timestamp: &BlockTimestamp) -> Vec { + let mut auth_sequences: Vec = vec![]; + + for transaction in block.transaction_traces() { + let tx_hash = &transaction.id; + let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); + + for action_trace in transaction.action_traces.iter() { + let receipt = action_trace.receipt.as_ref().expect("Action trace receipt is required"); + + let action_index = action_trace.execution_index; + + for (index, auth) in receipt.auth_sequence.iter().enumerate() { + auth_sequences.push(RawAuthSequence { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + action_index, + index: index as u32, + account_name: auth.account_name.clone(), + sequence: auth.sequence, + }); + } + } + } + + auth_sequences +} diff --git a/blocks/antelope/src/pb/beacon.rs b/blocks/antelope/src/pb/beacon.rs deleted file mode 100644 index 3c3b2fd..0000000 --- a/blocks/antelope/src/pb/beacon.rs +++ /dev/null @@ -1,515 +0,0 @@ -// @generated -// This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Block { - #[prost(message, optional, tag="1")] - pub time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub number: u64, - #[prost(string, tag="3")] - pub date: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub hash: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub parent_hash: ::prost::alloc::string::String, - #[prost(string, tag="6")] - pub producer: ::prost::alloc::string::String, - #[prost(uint32, tag="7")] - pub confirmed: u32, - #[prost(uint32, tag="8")] - pub schedule_version: u32, - #[prost(uint32, tag="9")] - pub version: u32, - #[prost(string, tag="10")] - pub producer_signature: ::prost::alloc::string::String, - #[prost(uint32, tag="11")] - pub dpos_proposed_irreversible_blocknum: u32, - #[prost(uint32, tag="12")] - pub dpos_irreversible_blocknum: u32, - #[prost(string, tag="13")] - pub transaction_mroot: ::prost::alloc::string::String, - #[prost(string, tag="14")] - pub action_mroot: ::prost::alloc::string::String, - #[prost(string, tag="15")] - pub blockroot_merkle_active_nodes: ::prost::alloc::string::String, - #[prost(uint32, tag="16")] - pub blockroot_merkle_node_count: u32, - #[prost(uint64, tag="17")] - pub size: u64, - #[prost(uint64, tag="18")] - pub total_transactions: u64, - #[prost(uint64, tag="19")] - pub successful_transactions: u64, - #[prost(uint64, tag="20")] - pub failed_transactions: u64, - #[prost(uint64, tag="21")] - pub total_actions: u64, - #[prost(uint64, tag="22")] - pub total_db_ops: u64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Transaction { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub hash: ::prost::alloc::string::String, - #[prost(uint64, tag="6")] - pub index: u64, - #[prost(int64, tag="7")] - pub elapsed: i64, - #[prost(uint64, tag="8")] - pub net_usage: u64, - #[prost(bool, tag="9")] - pub scheduled: bool, - #[prost(uint32, tag="10")] - pub cpu_usage_micro_seconds: u32, - #[prost(uint32, tag="11")] - pub net_usage_words: u32, - #[prost(string, tag="12")] - pub status: ::prost::alloc::string::String, - #[prost(uint32, tag="13")] - pub status_code: u32, - #[prost(bool, tag="14")] - pub success: bool, - #[prost(string, tag="15")] - pub transaction_mroot: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Action { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint64, tag="7")] - pub abi_sequence: u64, - #[prost(uint64, tag="8")] - pub code_sequence: u64, - #[prost(string, tag="9")] - pub digest: ::prost::alloc::string::String, - #[prost(uint64, tag="10")] - pub global_sequence: u64, - #[prost(string, tag="11")] - pub receipt_receiver: ::prost::alloc::string::String, - #[prost(uint64, tag="12")] - pub recv_sequence: u64, - #[prost(string, tag="13")] - pub account: ::prost::alloc::string::String, - #[prost(string, tag="14")] - pub name: ::prost::alloc::string::String, - #[prost(string, tag="15")] - pub json_data: ::prost::alloc::string::String, - #[prost(string, tag="16")] - pub raw_data: ::prost::alloc::string::String, - #[prost(uint32, tag="17")] - pub index: u32, - #[prost(uint32, tag="18")] - pub action_ordinal: u32, - #[prost(string, tag="19")] - pub receiver: ::prost::alloc::string::String, - #[prost(bool, tag="20")] - pub context_free: bool, - #[prost(int64, tag="21")] - pub elapsed: i64, - #[prost(string, tag="22")] - pub console: ::prost::alloc::string::String, - #[prost(string, tag="23")] - pub raw_return_value: ::prost::alloc::string::String, - #[prost(string, tag="24")] - pub json_return_value: ::prost::alloc::string::String, - #[prost(uint32, tag="25")] - pub creator_action_ordinal: u32, - #[prost(uint32, tag="26")] - pub closest_unnotified_ancestor_action_ordinal: u32, - #[prost(string, tag="27")] - pub action_mroot: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DbOp { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(string, tag="9")] - pub operation: ::prost::alloc::string::String, - #[prost(uint32, tag="10")] - pub operation_code: u32, - #[prost(string, tag="11")] - pub code: ::prost::alloc::string::String, - #[prost(string, tag="12")] - pub scope: ::prost::alloc::string::String, - #[prost(string, tag="13")] - pub table_name: ::prost::alloc::string::String, - #[prost(string, tag="14")] - pub primary_key: ::prost::alloc::string::String, - #[prost(string, tag="15")] - pub old_payer: ::prost::alloc::string::String, - #[prost(string, tag="16")] - pub new_payer: ::prost::alloc::string::String, - #[prost(string, tag="17")] - pub old_data: ::prost::alloc::string::String, - #[prost(string, tag="18")] - pub new_data: ::prost::alloc::string::String, - #[prost(string, tag="19")] - pub old_data_json: ::prost::alloc::string::String, - #[prost(string, tag="20")] - pub new_data_json: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct FeatureOp { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(string, tag="8")] - pub feature_digest: ::prost::alloc::string::String, - #[prost(string, tag="9")] - pub kind: ::prost::alloc::string::String, - #[prost(string, tag="10")] - pub description_digest: ::prost::alloc::string::String, - #[prost(string, tag="11")] - pub protocol_feature_type: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PermOp { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(string, tag="8")] - pub operation: ::prost::alloc::string::String, - #[prost(uint32, tag="9")] - pub operation_code: u32, - #[prost(uint64, tag="10")] - pub id: u64, - #[prost(uint64, tag="11")] - pub parent_id: u64, - #[prost(string, tag="12")] - pub owner: ::prost::alloc::string::String, - #[prost(string, tag="13")] - pub name: ::prost::alloc::string::String, - #[prost(uint32, tag="14")] - pub threshold: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TableOp { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(string, tag="9")] - pub operation: ::prost::alloc::string::String, - #[prost(uint32, tag="10")] - pub operation_code: u32, - #[prost(string, tag="11")] - pub payer: ::prost::alloc::string::String, - #[prost(string, tag="12")] - pub code: ::prost::alloc::string::String, - #[prost(string, tag="13")] - pub scope: ::prost::alloc::string::String, - #[prost(string, tag="14")] - pub table_name: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Account { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(string, tag="9")] - pub actor: ::prost::alloc::string::String, - #[prost(string, tag="10")] - pub permission: ::prost::alloc::string::String, - #[prost(uint32, tag="11")] - pub weight: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Key { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(string, tag="9")] - pub public_key: ::prost::alloc::string::String, - #[prost(uint32, tag="10")] - pub weight: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Wait { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(uint32, tag="9")] - pub wait_sec: u32, - #[prost(uint32, tag="10")] - pub weight: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct RamOp { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(string, tag="8")] - pub operation: ::prost::alloc::string::String, - #[prost(uint32, tag="9")] - pub operation_code: u32, - #[prost(string, tag="10")] - pub payer: ::prost::alloc::string::String, - #[prost(int64, tag="11")] - pub delta: i64, - #[prost(uint64, tag="12")] - pub usage: u64, - #[prost(string, tag="13")] - pub namespace: ::prost::alloc::string::String, - #[prost(uint32, tag="14")] - pub namespace_code: u32, - #[prost(string, tag="15")] - pub action: ::prost::alloc::string::String, - #[prost(uint32, tag="16")] - pub action_code: u32, - #[prost(string, tag="17")] - pub unique_key: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Authorization { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(string, tag="9")] - pub actor: ::prost::alloc::string::String, - #[prost(string, tag="10")] - pub permission: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AuthSequence { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(string, tag="9")] - pub account_name: ::prost::alloc::string::String, - #[prost(uint64, tag="10")] - pub sequence: u64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AccountRamDelta { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(string, tag="9")] - pub account: ::prost::alloc::string::String, - #[prost(int64, tag="10")] - pub delta: i64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CreationTree { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(int32, tag="7")] - pub creator_action_index: i32, - #[prost(uint32, tag="8")] - pub execution_action_index: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Events { - #[prost(message, repeated, tag="1")] - pub blocks: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="2")] - pub transactions: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="3")] - pub actions: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="4")] - pub db_ops: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="5")] - pub feature_ops: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="6")] - pub perm_ops: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="7")] - pub table_ops: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="8")] - pub accounts: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="9")] - pub keys: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="10")] - pub waits: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="11")] - pub ram_ops: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="12")] - pub authorizations: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="13")] - pub auth_sequences: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="14")] - pub account_ram_deltas: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="15")] - pub creation_trees: ::prost::alloc::vec::Vec, -} -// @@protoc_insertion_point(module) diff --git a/blocks/antelope/src/sinks.rs b/blocks/antelope/src/sinks.rs index 3180b09..f36519b 100644 --- a/blocks/antelope/src/sinks.rs +++ b/blocks/antelope/src/sinks.rs @@ -4,8 +4,8 @@ use substreams::pb::substreams::Clock; use substreams_antelope::pb::Block; use crate::{ - actions::collect_actions, authority::collect_authority_vectors, authorizations::collect_authorizations, blocks::collect_block, db_ops::collect_db_ops, feature_ops::collect_feature_ops, - pb::antelope::Events, perm_ops::collect_perm_ops, ram_ops::collect_ram_ops, table_ops::collect_table_ops, transactions::collect_transactions, + actions::collect_actions, auth_sequences::collect_auth_sequences, authority::collect_authority_vectors, authorizations::collect_authorizations, blocks::collect_block, db_ops::collect_db_ops, + feature_ops::collect_feature_ops, pb::antelope::Events, perm_ops::collect_perm_ops, ram_ops::collect_ram_ops, table_ops::collect_table_ops, transactions::collect_transactions, }; #[substreams::handlers::map] @@ -26,7 +26,7 @@ pub fn map_events(clock: Clock, block: Block) -> Result { waits: authority_vectors.waits, ram_ops: collect_ram_ops(&block, ×tamp), authorizations: collect_authorizations(&block, ×tamp), - auth_sequences: vec![], + auth_sequences: collect_auth_sequences(&block, ×tamp), account_ram_deltas: vec![], creation_trees: vec![], }) From 567eb94054b84fb230fe76191c55da358e45bc6e Mon Sep 17 00:00:00 2001 From: zolting Date: Mon, 18 Nov 2024 17:31:45 -0500 Subject: [PATCH 10/22] Add AccountRamDeltas + CreationTrees --- blocks/antelope/src/account_ram_deltas.rs | 34 +++++++++++++++++++++++ blocks/antelope/src/creation_tree.rs | 28 +++++++++++++++++++ blocks/antelope/src/sinks.rs | 9 +++--- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/blocks/antelope/src/account_ram_deltas.rs b/blocks/antelope/src/account_ram_deltas.rs index 048e924..fa7f474 100644 --- a/blocks/antelope/src/account_ram_deltas.rs +++ b/blocks/antelope/src/account_ram_deltas.rs @@ -5,7 +5,11 @@ use substreams_antelope::pb::{ActionTrace, TransactionTrace}; use substreams_database_change::pb::database::{table_change, DatabaseChanges}; use crate::keys::account_ram_delta_keys; +use crate::pb::antelope::AccountRamDelta as RawAccountRamDelta; use crate::transactions::insert_transaction_metadata; +use crate::transactions::is_transaction_success; +use common::structs::BlockTimestamp; +use substreams_antelope::Block; pub fn insert_account_ram_delta(tables: &mut DatabaseChanges, clock: &Clock, action: &ActionTrace, transaction: &TransactionTrace, account_ram_delta: &AccountRamDelta, index: &u32) { // transaction @@ -30,3 +34,33 @@ pub fn insert_account_ram_delta(tables: &mut DatabaseChanges, clock: &Clock, act insert_transaction_metadata(row, transaction); insert_timestamp(row, clock, false, false); } + +pub fn collect_account_ram_deltas(block: &Block, timestamp: &BlockTimestamp) -> Vec { + let mut account_ram_deltas: Vec = vec![]; + + for transaction in block.transaction_traces() { + let tx_hash = &transaction.id; + let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); + + for action_trace in transaction.action_traces.iter() { + let action_index = action_trace.execution_index; + + for (index, delta) in action_trace.account_ram_deltas.iter().enumerate() { + account_ram_deltas.push(RawAccountRamDelta { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + action_index, + index: index as u32, + account: delta.account.clone(), + delta: delta.delta, + }); + } + } + } + + account_ram_deltas +} diff --git a/blocks/antelope/src/creation_tree.rs b/blocks/antelope/src/creation_tree.rs index a31561b..e9f7170 100644 --- a/blocks/antelope/src/creation_tree.rs +++ b/blocks/antelope/src/creation_tree.rs @@ -3,7 +3,11 @@ use substreams::pb::substreams::Clock; use substreams_antelope::pb::{CreationFlatNode, TransactionTrace}; use substreams_database_change::pb::database::{table_change, DatabaseChanges}; +use crate::pb::antelope::CreationTree as RawCreationTree; +use crate::transactions::is_transaction_success; use crate::{keys::creation_tree_keys, transactions::insert_transaction_metadata}; +use common::structs::BlockTimestamp; +use substreams_antelope::Block; pub fn insert_creation_tree(tables: &mut DatabaseChanges, clock: &Clock, transaction: &TransactionTrace, creation_flat_node: &CreationFlatNode) { let creator_action_index = creation_flat_node.creator_action_index; @@ -17,3 +21,27 @@ pub fn insert_creation_tree(tables: &mut DatabaseChanges, clock: &Clock, transac insert_transaction_metadata(row, transaction); insert_timestamp(row, clock, false, false); } + +pub fn collect_creation_trees(block: &Block, timestamp: &BlockTimestamp) -> Vec { + let mut creation_trees: Vec = vec![]; + + for transaction in block.transaction_traces() { + let tx_hash = &transaction.id; + let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); + + for creation_flat_node in transaction.creation_tree.iter() { + creation_trees.push(RawCreationTree { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: tx_hash.clone(), + tx_success, + creator_action_index: creation_flat_node.creator_action_index, + execution_action_index: creation_flat_node.execution_action_index, + }); + } + } + + creation_trees +} diff --git a/blocks/antelope/src/sinks.rs b/blocks/antelope/src/sinks.rs index f36519b..aab5a5f 100644 --- a/blocks/antelope/src/sinks.rs +++ b/blocks/antelope/src/sinks.rs @@ -4,8 +4,9 @@ use substreams::pb::substreams::Clock; use substreams_antelope::pb::Block; use crate::{ - actions::collect_actions, auth_sequences::collect_auth_sequences, authority::collect_authority_vectors, authorizations::collect_authorizations, blocks::collect_block, db_ops::collect_db_ops, - feature_ops::collect_feature_ops, pb::antelope::Events, perm_ops::collect_perm_ops, ram_ops::collect_ram_ops, table_ops::collect_table_ops, transactions::collect_transactions, + account_ram_deltas::collect_account_ram_deltas, actions::collect_actions, auth_sequences::collect_auth_sequences, authority::collect_authority_vectors, authorizations::collect_authorizations, + blocks::collect_block, creation_tree::collect_creation_trees, db_ops::collect_db_ops, feature_ops::collect_feature_ops, pb::antelope::Events, perm_ops::collect_perm_ops, ram_ops::collect_ram_ops, + table_ops::collect_table_ops, transactions::collect_transactions, }; #[substreams::handlers::map] @@ -27,7 +28,7 @@ pub fn map_events(clock: Clock, block: Block) -> Result { ram_ops: collect_ram_ops(&block, ×tamp), authorizations: collect_authorizations(&block, ×tamp), auth_sequences: collect_auth_sequences(&block, ×tamp), - account_ram_deltas: vec![], - creation_trees: vec![], + account_ram_deltas: collect_account_ram_deltas(&block, ×tamp), + creation_trees: collect_creation_trees(&block, ×tamp), }) } From a5c2a418abff21f28b4dc36d1a3629f53e6350de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dillan=20Choini=C3=A8re?= Date: Tue, 19 Nov 2024 16:30:04 -0500 Subject: [PATCH 11/22] Cleanup unused functions/imports after refactor --- blocks/antelope/src/account_ram_deltas.rs | 32 ---- blocks/antelope/src/actions.rs | 124 +-------------- blocks/antelope/src/auth_sequences.rs | 29 ---- blocks/antelope/src/authority.rs | 70 +-------- blocks/antelope/src/authorizations.rs | 38 +---- blocks/antelope/src/blocks.rs | 72 +-------- blocks/antelope/src/creation_tree.rs | 19 --- blocks/antelope/src/db_ops.rs | 54 +------ blocks/antelope/src/feature_ops.rs | 39 +---- blocks/antelope/src/keys.rs | 110 -------------- blocks/antelope/src/lib.rs | 3 +- .../antelope/src/{sinks.rs => map_events.rs} | 0 blocks/antelope/src/perm_ops.rs | 53 +------ blocks/antelope/src/ram_ops.rs | 36 +---- blocks/antelope/src/size.rs | 11 -- blocks/antelope/src/table_ops.rs | 42 +----- blocks/antelope/src/transactions.rs | 142 +----------------- 17 files changed, 30 insertions(+), 844 deletions(-) delete mode 100644 blocks/antelope/src/keys.rs rename blocks/antelope/src/{sinks.rs => map_events.rs} (100%) diff --git a/blocks/antelope/src/account_ram_deltas.rs b/blocks/antelope/src/account_ram_deltas.rs index fa7f474..1636bc7 100644 --- a/blocks/antelope/src/account_ram_deltas.rs +++ b/blocks/antelope/src/account_ram_deltas.rs @@ -1,40 +1,8 @@ -use common::blocks::insert_timestamp; -use substreams::pb::substreams::Clock; -use substreams_antelope::pb::AccountRamDelta; -use substreams_antelope::pb::{ActionTrace, TransactionTrace}; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; - -use crate::keys::account_ram_delta_keys; use crate::pb::antelope::AccountRamDelta as RawAccountRamDelta; -use crate::transactions::insert_transaction_metadata; use crate::transactions::is_transaction_success; use common::structs::BlockTimestamp; use substreams_antelope::Block; -pub fn insert_account_ram_delta(tables: &mut DatabaseChanges, clock: &Clock, action: &ActionTrace, transaction: &TransactionTrace, account_ram_delta: &AccountRamDelta, index: &u32) { - // transaction - let tx_hash = &transaction.id; - - // action - let action_index = &action.execution_index; - - // account_ram_delta - let account = &account_ram_delta.account; - let delta = &account_ram_delta.delta; - - let keys = account_ram_delta_keys(&tx_hash, &action_index, account, delta); - let row = tables - .push_change_composite("account_ram_deltas", keys, 0, table_change::Operation::Create) - .change("index", ("", index.to_string().as_str())) - .change("tx_hash", ("", tx_hash.as_str())) - .change("action_index", ("", action_index.to_string().as_str())) - .change("account", ("", account.as_str())) - .change("delta", ("", delta.to_string().as_str())); - - insert_transaction_metadata(row, transaction); - insert_timestamp(row, clock, false, false); -} - pub fn collect_account_ram_deltas(block: &Block, timestamp: &BlockTimestamp) -> Vec { let mut account_ram_deltas: Vec = vec![]; diff --git a/blocks/antelope/src/actions.rs b/blocks/antelope/src/actions.rs index dba7af2..97c9f5e 100644 --- a/blocks/antelope/src/actions.rs +++ b/blocks/antelope/src/actions.rs @@ -1,132 +1,12 @@ -use common::blocks::insert_timestamp; use common::structs::BlockTimestamp; -use substreams::pb::substreams::Clock; + use substreams::Hex; -use substreams_antelope::pb::{ActionTrace, BlockHeader, TransactionTrace}; use substreams_antelope::Block; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; -use crate::account_ram_deltas::insert_account_ram_delta; -use crate::auth_sequences::insert_auth_sequence; -use crate::authorizations::insert_authorization; -use crate::keys::actions_keys; use crate::pb::antelope::Action as RawAction; -use crate::transactions::{insert_transaction_metadata, is_transaction_success}; +use crate::transactions::is_transaction_success; // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L525 -pub fn insert_action(tables: &mut DatabaseChanges, clock: &Clock, trace: &ActionTrace, transaction: &TransactionTrace, block_header: &BlockHeader) { - // action - let action = trace.action.clone().unwrap_or_default(); - let account = action.account; - let name = action.name; - let json_data = action.json_data; - let raw_data = Hex::encode(&action.raw_data.to_vec()); - - // receipt - let receipt = trace.receipt.clone().unwrap_or_default(); - let abi_sequence = receipt.abi_sequence; - let code_sequence = receipt.code_sequence; - let digest = &receipt.digest; - let global_sequence = receipt.global_sequence; - let receipt_receiver = receipt.receiver; - let recv_sequence = receipt.recv_sequence; - - // trace - let index = trace.execution_index; - let action_ordinal = trace.action_ordinal; - let receiver = &trace.receiver; - let context_free = trace.context_free; - let elapsed = trace.elapsed; - let console = &trace.console; - let raw_return_value = Hex::encode(&trace.raw_return_value.to_vec()); - let json_return_value = &trace.json_return_value; - let creator_action_ordinal = trace.creator_action_ordinal; - let closest_unnotified_ancestor_action_ordinal = trace.closest_unnotified_ancestor_action_ordinal; - - // block roots - let action_mroot = Hex::encode(&block_header.action_mroot.to_vec()); - - // transaction - let tx_hash = &transaction.id; - - let keys = actions_keys(&tx_hash, &index); - let row = tables - .push_change_composite("actions", keys, 0, table_change::Operation::Create) - // receipt - .change("abi_sequence", ("", abi_sequence.to_string().as_str())) - .change("code_sequence", ("", code_sequence.to_string().as_str())) - .change("digest", ("", digest.to_string().as_str())) - .change("global_sequence", ("", global_sequence.to_string().as_str())) - .change("receipt_receiver", ("", receipt_receiver.to_string().as_str())) - .change("recv_sequence", ("", recv_sequence.to_string().as_str())) - // action - .change("account", ("", account.to_string().as_str())) - .change("name", ("", name.to_string().as_str())) - .change("json_data", ("", json_data.to_string().as_str())) - .change("raw_data", ("", raw_data.to_string().as_str())) - // trace - .change("index", ("", index.to_string().as_str())) - .change("action_ordinal", ("", action_ordinal.to_string().as_str())) - .change("receiver", ("", receiver.to_string().as_str())) - .change("context_free", ("", context_free.to_string().as_str())) - .change("elapsed", ("", elapsed.to_string().as_str())) - .change("console", ("", console.to_string().as_str())) - .change("raw_return_value", ("", raw_return_value.to_string().as_str())) - .change("json_return_value", ("", json_return_value.to_string().as_str())) - .change("creator_action_ordinal", ("", creator_action_ordinal.to_string().as_str())) - .change("closest_unnotified_ancestor_action_ordinal", ("", closest_unnotified_ancestor_action_ordinal.to_string().as_str())) - // block roots - .change("action_mroot", ("", action_mroot.as_str())); - insert_transaction_metadata(row, transaction); - insert_timestamp(row, clock, false, false); - - // TABLE::authorizations - let mut index_authorization = 0; - for authorization in action.authorization.iter() { - insert_authorization(tables, clock, trace, transaction, authorization, &index_authorization); - index_authorization += 1; - } - - // TABLE::auth_sequences - let mut index_auth_sequence = 0; - for auth_sequence in receipt.auth_sequence.iter() { - insert_auth_sequence(tables, clock, trace, transaction, auth_sequence, &index_auth_sequence); - index_auth_sequence += 1; - } - - // TABLE::account_ram_deltas - let mut index_insert_account_ram_delta = 0; - for account_ram_delta in trace.account_ram_deltas.iter() { - insert_account_ram_delta(tables, clock, trace, transaction, account_ram_delta, &index_insert_account_ram_delta); - index_insert_account_ram_delta += 1; - } - // TO-DO - // Need Array(String) support - // https://github.com/pinax-network/substreams-sink-sql/issues/18 - // action.authorization.iter().for_each(|authorization| { - // let actor = authorization.actor; - // let permission = authorization.permission; - // }); - - // TO-DO - // Need Array(String) support - // https://github.com/pinax-network/substreams-sink-sql/issues/18 - // receipt.auth_sequence.iter().for_each(|auth_sequence| { - // let account_name = auth_sequence.account_name; - // let sequence = auth_sequence.sequence; - // }); - - // TO-DO - // exception - // let exception = trace.exception.unwrap_or_default(); - // let error_code = trace.error_code; - - // TO-DO - // for account_ram_delta in transaction.account_ram_deltas.iter() { - // insert_account_ram_delta(tables, clock, trace, &account_ram_delta, &block); - // } -} - pub fn collect_actions(block: &Block, timestamp: &BlockTimestamp) -> Vec { let header = block.header.clone().unwrap_or_default(); let mut actions = Vec::new(); diff --git a/blocks/antelope/src/auth_sequences.rs b/blocks/antelope/src/auth_sequences.rs index cc4276f..daa9067 100644 --- a/blocks/antelope/src/auth_sequences.rs +++ b/blocks/antelope/src/auth_sequences.rs @@ -1,38 +1,9 @@ -use common::blocks::insert_timestamp; use common::structs::BlockTimestamp; -use substreams::pb::substreams::Clock; -use substreams_antelope::pb::AuthSequence; -use substreams_antelope::pb::{ActionTrace, TransactionTrace}; use substreams_antelope::Block; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; -use crate::keys::auth_sequence_keys; use crate::pb::antelope::AuthSequence as RawAuthSequence; -use crate::transactions::insert_transaction_metadata; use crate::transactions::is_transaction_success; -pub fn insert_auth_sequence(tables: &mut DatabaseChanges, clock: &Clock, action: &ActionTrace, transaction: &TransactionTrace, auth_sequence: &AuthSequence, index: &u32) { - // transaction - let tx_hash = &transaction.id; - - // action - let action_index = &action.execution_index; - - // auth_sequence - let account_name = &auth_sequence.account_name; - let sequence = &auth_sequence.sequence; - - let keys = auth_sequence_keys(&tx_hash, &action_index, index); - let row = tables - .push_change_composite("auth_sequences", keys, 0, table_change::Operation::Create) - .change("action_index", ("", action_index.to_string().as_str())) - .change("account_name", ("", account_name.as_str())) - .change("sequence", ("", sequence.to_string().as_str())); - - insert_transaction_metadata(row, transaction); - insert_timestamp(row, clock, false, false); -} - pub fn collect_auth_sequences(block: &Block, timestamp: &BlockTimestamp) -> Vec { let mut auth_sequences: Vec = vec![]; diff --git a/blocks/antelope/src/authority.rs b/blocks/antelope/src/authority.rs index 44b9fab..7d08fef 100644 --- a/blocks/antelope/src/authority.rs +++ b/blocks/antelope/src/authority.rs @@ -1,15 +1,9 @@ use crate::{ - keys::authority_keys, pb::antelope::{Account as RawAccount, Key as RawKey, Wait as RawWait}, - transactions::{insert_transaction_metadata, is_transaction_success}, + transactions::is_transaction_success, }; -use common::{blocks::insert_timestamp, structs::BlockTimestamp}; -use substreams::pb::substreams::Clock; -use substreams_antelope::{ - pb::{Authority, TransactionTrace}, - Block, -}; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; +use common::structs::BlockTimestamp; +use substreams_antelope::Block; pub struct AuthorityVectors { pub accounts: Vec, @@ -87,61 +81,3 @@ pub fn collect_authority_vectors(block: &Block, timestamp: &BlockTimestamp) -> A AuthorityVectors { accounts, keys, waits } } - -pub fn insert_authority(tables: &mut DatabaseChanges, clock: &Clock, transaction: &TransactionTrace, action_index: u32, authority: &Authority) { - let mut index = 0; - let tx_hash = &transaction.id; - - // `accounts` TABLE - for account in &authority.accounts { - let permission_level = account.permission.as_ref().expect("account.permission is missing"); - let actor = permission_level.actor.as_str(); - let permission = permission_level.permission.as_str(); - let weight = account.weight; - - let keys = authority_keys(tx_hash, &action_index, &index); - let row = tables - .push_change_composite("accounts", keys, 0, table_change::Operation::Create) - .change("index", ("", index.to_string().as_str())) - .change("action_index", ("", action_index.to_string().as_str())) - .change("actor", ("", actor)) - .change("permission", ("", permission)) - .change("weight", ("", weight.to_string().as_str())); - insert_transaction_metadata(row, transaction); - insert_timestamp(row, clock, false, false); - index += 1; - } - // `keys` TABLE - for key in &authority.keys { - let public_key = key.public_key.as_str(); - let weight = key.weight; - - let keys = authority_keys(tx_hash, &action_index, &index); - let row = tables - .push_change_composite("keys", keys, 0, table_change::Operation::Create) - .change("index", ("", index.to_string().as_str())) - .change("action_index", ("", action_index.to_string().as_str())) - .change("public_key", ("", public_key)) - .change("weight", ("", weight.to_string().as_str())); - insert_transaction_metadata(row, transaction); - insert_timestamp(row, clock, false, false); - index += 1; - } - // `waits` TABLE - for wait in &authority.waits { - let wait_sec = wait.wait_sec; - let weight = wait.weight; - - let keys = authority_keys(tx_hash, &action_index, &index); - let row = tables - .push_change_composite("waits", keys, 0, table_change::Operation::Create) - .change("index", ("", index.to_string().as_str())) - .change("action_index", ("", action_index.to_string().as_str())) - .change("wait_sec", ("", wait_sec.to_string().as_str())) - .change("weight", ("", weight.to_string().as_str())); - insert_transaction_metadata(row, transaction); - insert_timestamp(row, clock, false, false); - - index += 1; - } -} diff --git a/blocks/antelope/src/authorizations.rs b/blocks/antelope/src/authorizations.rs index 60d6f4d..d9e3f3b 100644 --- a/blocks/antelope/src/authorizations.rs +++ b/blocks/antelope/src/authorizations.rs @@ -1,41 +1,9 @@ -use common::{blocks::insert_timestamp, structs::BlockTimestamp}; -use substreams::pb::substreams::Clock; -use substreams_antelope::{ - pb::{ActionTrace, PermissionLevel, TransactionTrace}, - Block, -}; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; +use common::structs::BlockTimestamp; +use substreams_antelope::Block; -use crate::{ - keys::authorizations_keys, - pb::antelope::Authorization as RawAuthorization, - transactions::{insert_transaction_metadata, is_transaction_success}, -}; +use crate::{pb::antelope::Authorization as RawAuthorization, transactions::is_transaction_success}; // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L616 -pub fn insert_authorization(tables: &mut DatabaseChanges, clock: &Clock, action: &ActionTrace, transaction: &TransactionTrace, authorization: &PermissionLevel, index: &u32) { - // transaction - let tx_hash = &transaction.id; - - // action - let action_index = &action.execution_index; - - // authorization - let actor = &authorization.actor; - let permission = &authorization.permission; - - let keys = authorizations_keys(&tx_hash, &action_index, index); - let row = tables - .push_change_composite("authorizations", keys, 0, table_change::Operation::Create) - // action - .change("action_index", ("", action_index.to_string().as_str())) - .change("actor", ("", actor.to_string().as_str())) - .change("permission", ("", permission.to_string().as_str())); - - insert_transaction_metadata(row, transaction); - insert_timestamp(row, clock, false, false); -} - pub fn collect_authorizations(block: &Block, timestamp: &BlockTimestamp) -> Vec { let mut authorizations: Vec = vec![]; diff --git a/blocks/antelope/src/blocks.rs b/blocks/antelope/src/blocks.rs index 87d8666..c26dd00 100644 --- a/blocks/antelope/src/blocks.rs +++ b/blocks/antelope/src/blocks.rs @@ -1,73 +1,9 @@ -use crate::{keys::blocks_keys, pb::antelope::Block as RawBlock, size::collect_size}; -use common::{blocks::insert_timestamp, structs::BlockTimestamp, utils::bytes_to_hex}; -use substreams::{pb::substreams::Clock, Hex}; -use substreams_antelope::pb::Block; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; - -use super::{size::insert_size, transactions::insert_transaction}; +use crate::{pb::antelope::Block as RawBlock, size::collect_size}; +use common::{structs::BlockTimestamp, utils::bytes_to_hex}; +use substreams::Hex; +use substreams_antelope::Block; // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L21 -pub fn insert_blocks(tables: &mut DatabaseChanges, clock: &Clock, block: &Block) { - // header - let header = block.header.clone().unwrap_or_default(); - let parent_hash = &header.previous; // renamed to follow EVM block schema - let producer = &header.producer; - let confirmed = &header.confirmed; - let schedule_version = &header.schedule_version; - - // block - let version = block.version; - let producer_signature = &block.producer_signature; - let dpos_proposed_irreversible_blocknum = block.dpos_proposed_irreversible_blocknum; - let dpos_irreversible_blocknum = block.dpos_irreversible_blocknum; - - // blockroot_merkle - let blockroot_merkle = block.blockroot_merkle.clone().unwrap_or_default(); - - // TO-DO - // Array(String) type is not supported by `substreams-sink-sql` - // https://github.com/pinax-network/substreams-sink-sql/issues/18 - let blockroot_merkle_active_nodes = blockroot_merkle.active_nodes.iter().map(|row| bytes_to_hex(row)).collect::>().join(","); - let blockroot_merkle_node_count = blockroot_merkle.node_count; - - // block roots - let transaction_mroot = Hex::encode(&header.transaction_mroot.to_vec()); - let action_mroot = Hex::encode(&header.action_mroot.to_vec()); - - // TO-DO - // to be used during Legacy to Savanna transition where action_mroot needs to be converted from Legacy merkle to Savanna merkle - // let action_mroot_savanna = block.action_mroot_savanna; - - // blocks - let keys = blocks_keys(&clock); - let row = tables - .push_change_composite("blocks", keys, 0, table_change::Operation::Create) - // header - .change("parent_hash", ("", parent_hash.as_str())) - .change("producer", ("", producer.to_string().as_str())) - .change("confirmed", ("", confirmed.to_string().as_str())) - .change("schedule_version", ("", schedule_version.to_string().as_str())) - // block - .change("version", ("", version.to_string().as_str())) - .change("producer_signature", ("", producer_signature.to_string().as_str())) - .change("dpos_proposed_irreversible_blocknum", ("", dpos_proposed_irreversible_blocknum.to_string().as_str())) - .change("dpos_irreversible_blocknum", ("", dpos_irreversible_blocknum.to_string().as_str())) - // block roots - .change("transaction_mroot", ("", transaction_mroot.to_string().as_str())) - .change("action_mroot", ("", action_mroot.to_string().as_str())) - // .change("blockroot_merkle_active_nodes", ("", format!("['{}']", blockroot_merkle_active_nodes.join("','") ).as_str())) - .change("blockroot_merkle_node_count", ("", blockroot_merkle_node_count.to_string().as_str())); - - // transaction status counts - insert_size(row, block); - insert_timestamp(row, clock, true, false); - - // TABLE::transactions - for transaction in block.transaction_traces() { - insert_transaction(tables, clock, &transaction, &header); - } -} - pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> RawBlock { let header = block.header.clone().unwrap_or_default(); let blockroot_merkle = block.blockroot_merkle.clone().unwrap_or_default(); diff --git a/blocks/antelope/src/creation_tree.rs b/blocks/antelope/src/creation_tree.rs index e9f7170..9ef37d7 100644 --- a/blocks/antelope/src/creation_tree.rs +++ b/blocks/antelope/src/creation_tree.rs @@ -1,27 +1,8 @@ -use common::blocks::insert_timestamp; -use substreams::pb::substreams::Clock; -use substreams_antelope::pb::{CreationFlatNode, TransactionTrace}; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; - use crate::pb::antelope::CreationTree as RawCreationTree; use crate::transactions::is_transaction_success; -use crate::{keys::creation_tree_keys, transactions::insert_transaction_metadata}; use common::structs::BlockTimestamp; use substreams_antelope::Block; -pub fn insert_creation_tree(tables: &mut DatabaseChanges, clock: &Clock, transaction: &TransactionTrace, creation_flat_node: &CreationFlatNode) { - let creator_action_index = creation_flat_node.creator_action_index; - let execution_action_index = creation_flat_node.execution_action_index; - let keys = creation_tree_keys(&transaction.id, &creator_action_index, &execution_action_index); - let row = tables - .push_change_composite("creation_tree", keys, 0, table_change::Operation::Create) - .change("creator_action_index", ("", creation_flat_node.creator_action_index.to_string().as_str())) - .change("execution_action_index", ("", creation_flat_node.execution_action_index.to_string().as_str())); - - insert_transaction_metadata(row, transaction); - insert_timestamp(row, clock, false, false); -} - pub fn collect_creation_trees(block: &Block, timestamp: &BlockTimestamp) -> Vec { let mut creation_trees: Vec = vec![]; diff --git a/blocks/antelope/src/db_ops.rs b/blocks/antelope/src/db_ops.rs index 3e1bb3c..4389150 100644 --- a/blocks/antelope/src/db_ops.rs +++ b/blocks/antelope/src/db_ops.rs @@ -1,14 +1,8 @@ -use common::{blocks::insert_timestamp, structs::BlockTimestamp}; -use substreams::{pb::substreams::Clock, Hex}; -use substreams_antelope::{ - pb::{DbOp, TransactionTrace}, - Block, -}; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; +use common::structs::BlockTimestamp; +use substreams::Hex; +use substreams_antelope::Block; -use crate::{keys::db_ops_keys, pb::antelope::DbOp as RawDbOp, transactions::is_transaction_success}; - -use super::transactions::insert_transaction_metadata; +use crate::{pb::antelope::DbOp as RawDbOp, transactions::is_transaction_success}; pub fn operation_to_string(operation: i32) -> String { match operation { @@ -21,46 +15,6 @@ pub fn operation_to_string(operation: i32) -> String { } // https://github.com/streamingfast/firehose-ethereum/blob/1bcb32a8eb3e43347972b6b5c9b1fcc4a08c751e/proto/sf/ethereum/type/v2/type.proto#L647 -pub fn insert_db_op(tables: &mut DatabaseChanges, clock: &Clock, db_op: &DbOp, transaction: &TransactionTrace, index: u32) { - let operation = operation_to_string(db_op.operation); - let operation_code = db_op.operation; - let action_index = db_op.action_index; - let code = &db_op.code; - let scope = &db_op.scope; - let table_name = &db_op.table_name; - let primary_key = &db_op.primary_key; - let old_payer = &db_op.old_payer; - let new_payer = &db_op.new_payer; - let old_data = Hex::encode(&db_op.old_data.to_vec()); - let new_data = Hex::encode(&db_op.new_data.to_vec()); - let old_data_json = &db_op.old_data_json; - let new_data_json = &db_op.new_data_json; - - // transaction - let tx_hash = &transaction.id; - - let keys = db_ops_keys(&tx_hash, &index); - let row = tables - .push_change_composite("db_ops", keys, 0, table_change::Operation::Create) - .change("index", ("", index.to_string().as_str())) - .change("operation", ("", operation.to_string().as_str())) - .change("operation_code", ("", operation_code.to_string().as_str())) - .change("action_index", ("", action_index.to_string().as_str())) - .change("code", ("", code.to_string().as_str())) - .change("scope", ("", scope.to_string().as_str())) - .change("table_name", ("", table_name.to_string().as_str())) - .change("primary_key", ("", primary_key.to_string().as_str())) - .change("old_payer", ("", old_payer.to_string().as_str())) - .change("new_payer", ("", new_payer.to_string().as_str())) - .change("old_data", ("", old_data.to_string().as_str())) - .change("new_data", ("", new_data.to_string().as_str())) - .change("old_data_json", ("", old_data_json.to_string().as_str())) - .change("new_data_json", ("", new_data_json.to_string().as_str())); - - insert_timestamp(row, clock, false, false); - insert_transaction_metadata(row, transaction); -} - pub fn collect_db_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { let mut db_ops = Vec::new(); diff --git a/blocks/antelope/src/feature_ops.rs b/blocks/antelope/src/feature_ops.rs index 7fd516e..fb5bece 100644 --- a/blocks/antelope/src/feature_ops.rs +++ b/blocks/antelope/src/feature_ops.rs @@ -1,40 +1,7 @@ -use common::{blocks::insert_timestamp, structs::BlockTimestamp}; -use substreams::pb::substreams::Clock; -use substreams_antelope::{ - pb::{FeatureOp, TransactionTrace}, - Block, -}; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; +use common::structs::BlockTimestamp; +use substreams_antelope::Block; -use crate::{ - keys::feature_ops_keys, - pb::antelope::FeatureOp as RawFeatureOp, - transactions::{insert_transaction_metadata, is_transaction_success}, -}; - -pub fn insert_feature_op(tables: &mut DatabaseChanges, clock: &Clock, transaction: &TransactionTrace, feature_op: &FeatureOp) { - let feature_digest = &feature_op.feature_digest.as_str(); - let kind = feature_op.kind.as_str(); - let action_index = feature_op.action_index; - let feature = feature_op.feature.as_ref().expect("feature is required"); - let description_digest = feature.description_digest.as_str(); - let protocol_feature_type = feature.protocol_feature_type.as_str(); - // TO-DO: feature.dependencies - // TO-DO: feature.specification - // TO-DO: feature.subjective_restrictions - - let keys = feature_ops_keys(feature_digest); - let row = tables - .push_change_composite("feature_ops", keys, 0, table_change::Operation::Create) - .change("kind", ("", kind.to_string().as_str())) - .change("action_index", ("", action_index.to_string().as_str())) - .change("feature_digest", ("", feature_digest.to_string().as_str())) - .change("description_digest", ("", description_digest.to_string().as_str())) - .change("protocol_feature_type", ("", protocol_feature_type.to_string().as_str())); - - insert_transaction_metadata(row, transaction); - insert_timestamp(row, clock, false, false); -} +use crate::{pb::antelope::FeatureOp as RawFeatureOp, transactions::is_transaction_success}; pub fn collect_feature_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { let mut feature_ops: Vec = vec![]; diff --git a/blocks/antelope/src/keys.rs b/blocks/antelope/src/keys.rs deleted file mode 100644 index 5d4d262..0000000 --- a/blocks/antelope/src/keys.rs +++ /dev/null @@ -1,110 +0,0 @@ -use std::collections::HashMap; - -use common::utils::block_time_to_date; -use substreams::pb::substreams::Clock; - -pub fn blocks_keys(clock: &Clock) -> HashMap { - let mut keys = clock_keys(clock); - let block_hash = clock.id.to_string(); - keys.insert("block_hash".to_string(), block_hash); - keys -} - -pub fn clock_keys(clock: &Clock) -> HashMap { - let mut keys = HashMap::new(); - let timestamp = clock.clone().timestamp.unwrap(); - let block_date = block_time_to_date(×tamp.to_string()).to_string(); - let block_number = clock.number.to_string(); - keys.insert("block_date".to_string(), block_date); - keys.insert("block_number".to_string(), block_number); - keys -} - -pub fn transactions_keys(hash: &String) -> HashMap { - let mut keys = HashMap::new(); - keys.insert("hash".to_string(), hash.to_string()); - keys -} - -pub fn actions_keys(tx_hash: &String, index: &u32) -> HashMap { - let mut keys = HashMap::new(); - keys.insert("tx_hash".to_string(), tx_hash.to_string()); - keys.insert("index".to_string(), index.to_string()); - keys -} - -pub fn db_ops_keys(tx_hash: &String, index: &u32) -> HashMap { - let mut keys = HashMap::new(); - keys.insert("tx_hash".to_string(), tx_hash.to_string()); - keys.insert("index".to_string(), index.to_string()); - keys -} - -pub fn authorizations_keys(tx_hash: &String, action_index: &u32, index: &u32) -> HashMap { - let mut keys = HashMap::new(); - keys.insert("tx_hash".to_string(), tx_hash.to_string()); - keys.insert("action_index".to_string(), action_index.to_string()); - keys.insert("index".to_string(), index.to_string()); - keys -} - -pub fn auth_sequence_keys(tx_hash: &String, action_index: &u32, index: &u32) -> HashMap { - let mut keys = HashMap::new(); - keys.insert("tx_hash".to_string(), tx_hash.to_string()); - keys.insert("action_index".to_string(), action_index.to_string()); - keys.insert("index".to_string(), index.to_string()); - keys -} - -pub fn account_ram_delta_keys(tx_hash: &String, action_index: &u32, account: &String, delta: &i64) -> HashMap { - let mut keys = HashMap::new(); - keys.insert("tx_hash".to_string(), tx_hash.to_string()); - keys.insert("action_index".to_string(), action_index.to_string()); - keys.insert("account".to_string(), account.to_string()); - keys.insert("delta".to_string(), delta.to_string()); - keys -} - -pub fn perm_ops_keys(tx_hash: &String, action_index: &u32) -> HashMap { - let mut keys = HashMap::new(); - keys.insert("tx_hash".to_string(), tx_hash.to_string()); - keys.insert("action_index".to_string(), action_index.to_string()); - keys -} - -pub fn authority_keys(tx_hash: &str, action_index: &u32, index: &u32) -> HashMap { - let mut keys = HashMap::new(); - keys.insert("tx_hash".to_string(), tx_hash.to_string()); - keys.insert("action_index".to_string(), action_index.to_string()); - keys.insert("index".to_string(), index.to_string()); - keys -} - -pub fn feature_ops_keys(feature_digest: &str) -> HashMap { - let mut keys = HashMap::new(); - keys.insert("feature_digest".to_string(), feature_digest.to_string()); - keys -} - -pub fn creation_tree_keys(tx_hash: &String, creator_action_index: &i32, execution_action_index: &u32) -> HashMap { - let mut keys = HashMap::new(); - keys.insert("tx_hash".to_string(), tx_hash.to_string()); - keys.insert("creator_action_index".to_string(), creator_action_index.to_string()); - keys.insert("execution_action_index".to_string(), execution_action_index.to_string()); - keys -} - -pub fn table_ops_keys(tx_hash: &String, index: &u32) -> HashMap { - let mut keys = HashMap::new(); - keys.insert("tx_hash".to_string(), tx_hash.to_string()); - keys.insert("index".to_string(), index.to_string()); - keys -} - -pub fn ram_op_keys(tx_id: &str, action_index: &u32, unique_key: &str) -> HashMap { - let mut keys = HashMap::new(); - keys.insert("tx_hash".to_string(), tx_id.to_string()); - keys.insert("action_index".to_string(), action_index.to_string()); - keys.insert("unique_key".to_string(), unique_key.to_string()); - keys -} diff --git a/blocks/antelope/src/lib.rs b/blocks/antelope/src/lib.rs index a0f5d81..d0193b9 100644 --- a/blocks/antelope/src/lib.rs +++ b/blocks/antelope/src/lib.rs @@ -7,11 +7,10 @@ mod blocks; mod creation_tree; mod db_ops; mod feature_ops; -mod keys; +mod map_events; mod pb; mod perm_ops; mod ram_ops; -mod sinks; mod size; mod table_ops; mod transactions; diff --git a/blocks/antelope/src/sinks.rs b/blocks/antelope/src/map_events.rs similarity index 100% rename from blocks/antelope/src/sinks.rs rename to blocks/antelope/src/map_events.rs diff --git a/blocks/antelope/src/perm_ops.rs b/blocks/antelope/src/perm_ops.rs index bb9fb3d..5b8264b 100644 --- a/blocks/antelope/src/perm_ops.rs +++ b/blocks/antelope/src/perm_ops.rs @@ -1,16 +1,6 @@ -use crate::{ - authority::insert_authority, - keys::perm_ops_keys, - pb::antelope::PermOp as RawPermOp, - transactions::{insert_transaction_metadata, is_transaction_success}, -}; -use common::{blocks::insert_timestamp, structs::BlockTimestamp}; -use substreams::pb::substreams::Clock; -use substreams_antelope::{ - pb::{PermOp, TransactionTrace}, - Block, -}; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; +use crate::{pb::antelope::PermOp as RawPermOp, transactions::is_transaction_success}; +use common::structs::BlockTimestamp; +use substreams_antelope::Block; pub fn perm_op_operation_to_string(operation: i32) -> String { match operation { @@ -22,43 +12,6 @@ pub fn perm_op_operation_to_string(operation: i32) -> String { } } -pub fn insert_perm_op(tables: &mut DatabaseChanges, clock: &Clock, transaction: &TransactionTrace, perm_op: &PermOp) { - // transaction - let tx_hash = &transaction.id; - - // action - let action_index = perm_op.action_index; - - // perm_op - let operation = perm_op_operation_to_string(perm_op.operation); - match &perm_op.new_perm { - Some(new_perm) => { - let keys = perm_ops_keys(tx_hash, &action_index); - let threshold = new_perm.authority.as_ref().map_or(0, |authority| authority.threshold); - let row = tables - .push_change_composite("perm_ops", keys, 0, table_change::Operation::Create) - .change("operation", ("", operation.as_str())) - .change("operation_code", ("", perm_op.operation.to_string().as_str())) - .change("action_index", ("", action_index.to_string().as_str())) - .change("id", ("", new_perm.id.to_string().as_str())) - .change("parent_id", ("", new_perm.parent_id.to_string().as_str())) - .change("owner", ("", new_perm.owner.to_string().as_str())) - .change("name", ("", new_perm.name.to_string().as_str())) - .change("threshold", ("", threshold.to_string().as_str())); - insert_transaction_metadata(row, transaction); - insert_timestamp(row, clock, false, false); - - match &new_perm.authority { - Some(authority) => { - insert_authority(tables, clock, transaction, action_index, authority); - } - None => {} - } - } - None => {} - } -} - pub fn collect_perm_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { let mut perm_ops: Vec = vec![]; diff --git a/blocks/antelope/src/ram_ops.rs b/blocks/antelope/src/ram_ops.rs index ed85cc7..8a8004f 100644 --- a/blocks/antelope/src/ram_ops.rs +++ b/blocks/antelope/src/ram_ops.rs @@ -1,13 +1,8 @@ -use common::blocks::insert_timestamp; use common::structs::BlockTimestamp; -use substreams::pb::substreams::Clock; -use substreams_antelope::pb::{RamOp, TransactionTrace}; use substreams_antelope::Block; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; -use crate::keys::ram_op_keys; use crate::pb::antelope::RamOp as RawRamOp; -use crate::transactions::{insert_transaction_metadata, is_transaction_success}; +use crate::transactions::is_transaction_success; pub fn namespace_to_string(namespace: i32) -> String { match namespace { @@ -70,35 +65,6 @@ pub fn operation_to_string(operation: i32) -> String { } } -pub fn insert_ram_op(tables: &mut DatabaseChanges, clock: &Clock, ram_op: &RamOp, transaction: &TransactionTrace) { - let operation = operation_to_string(ram_op.operation); - let action_index = ram_op.action_index; - let payer = &ram_op.payer; - let delta = ram_op.delta; - let usage = ram_op.usage; - let namespace = namespace_to_string(ram_op.namespace); - let action = action_to_string(ram_op.action); - let unique_key = &ram_op.unique_key; - - let keys = ram_op_keys(&transaction.id, &action_index, unique_key); - let row = tables - .push_change_composite("ram_ops", keys, 0, table_change::Operation::Create) - .change("operation", ("", operation.as_str())) - .change("operation_code", ("", ram_op.operation.to_string().as_str())) - .change("action_index", ("", action_index.to_string().as_str())) - .change("payer", ("", payer.as_str())) - .change("delta", ("", delta.to_string().as_str())) - .change("usage", ("", usage.to_string().as_str())) - .change("namespace", ("", namespace.as_str())) - .change("namespace_code", ("", ram_op.namespace.to_string().as_str())) - .change("action", ("", action.as_str())) - .change("action_code", ("", ram_op.action.to_string().as_str())) - .change("unique_key", ("", unique_key.as_str())); - - insert_transaction_metadata(row, transaction); - insert_timestamp(row, clock, false, false); -} - pub fn collect_ram_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { let mut ram_ops: Vec = vec![]; diff --git a/blocks/antelope/src/size.rs b/blocks/antelope/src/size.rs index 2e9c130..c8aae57 100644 --- a/blocks/antelope/src/size.rs +++ b/blocks/antelope/src/size.rs @@ -1,5 +1,4 @@ use substreams_antelope::Block; -use substreams_database_change::pb::database::TableChange; pub struct BlockSize { pub size: u64, @@ -76,13 +75,3 @@ pub fn collect_size(block: &Block) -> BlockSize { total_db_ops: total_db_ops as u64, } } - -pub fn insert_size(row: &mut TableChange, block: &Block) { - let size = collect_size(block); - row.change("size", ("", size.size.to_string().as_str())) - .change("total_transactions", ("", size.total_transactions.to_string().as_str())) - .change("successful_transactions", ("", size.successful_transactions.to_string().as_str())) - .change("failed_transactions", ("", size.failed_transactions.to_string().as_str())) - .change("total_actions", ("", size.total_actions.to_string().as_str())) - .change("total_db_ops", ("", size.total_db_ops.to_string().as_str())); -} diff --git a/blocks/antelope/src/table_ops.rs b/blocks/antelope/src/table_ops.rs index 3616ed6..a262a25 100644 --- a/blocks/antelope/src/table_ops.rs +++ b/blocks/antelope/src/table_ops.rs @@ -1,16 +1,7 @@ -use common::{blocks::insert_timestamp, structs::BlockTimestamp}; -use substreams::pb::substreams::Clock; -use substreams_antelope::{ - pb::{TableOp, TransactionTrace}, - Block, -}; -use substreams_database_change::pb::database::{table_change, DatabaseChanges}; +use common::structs::BlockTimestamp; +use substreams_antelope::Block; -use crate::{ - keys::table_ops_keys, - pb::antelope::TableOp as RawTableOp, - transactions::{insert_transaction_metadata, is_transaction_success}, -}; +use crate::{pb::antelope::TableOp as RawTableOp, transactions::is_transaction_success}; pub fn table_op_operation_to_string(operation: i32) -> String { match operation { @@ -21,33 +12,6 @@ pub fn table_op_operation_to_string(operation: i32) -> String { } } -pub fn insert_table_op(tables: &mut DatabaseChanges, clock: &Clock, transaction: &TransactionTrace, table_op: &TableOp, index: &u32) { - let action_index = &table_op.action_index; - let operation = &table_op_operation_to_string(table_op.operation); - let payer = &table_op.payer; - let code = &table_op.code; - let scope = &table_op.scope; - let table_name = &table_op.table_name; - - // transaction - let tx_hash = &transaction.id; - - let keys = table_ops_keys(tx_hash, index); - let row = tables - .push_change_composite("table_ops", keys, 0, table_change::Operation::Create) - .change("index", ("", index.to_string().as_str())) - .change("action_index", ("", action_index.to_string().as_str())) - .change("operation", ("", operation.as_str())) - .change("operation_code", ("", table_op.operation.to_string().as_str())) - .change("payer", ("", payer.as_str())) - .change("code", ("", code.as_str())) - .change("scope", ("", scope.as_str())) - .change("table_name", ("", table_name.as_str())); - - insert_transaction_metadata(row, transaction); - insert_timestamp(row, clock, false, false); -} - pub fn collect_table_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { let mut table_ops: Vec = vec![]; diff --git a/blocks/antelope/src/transactions.rs b/blocks/antelope/src/transactions.rs index 36380eb..a70f83f 100644 --- a/blocks/antelope/src/transactions.rs +++ b/blocks/antelope/src/transactions.rs @@ -1,16 +1,8 @@ -use common::{blocks::insert_timestamp, structs::BlockTimestamp}; -use substreams::pb::substreams::Clock; +use common::structs::BlockTimestamp; use substreams::Hex; -use substreams_antelope::pb::{Block, BlockHeader, TransactionTrace}; -use substreams_database_change::pb::database::{table_change, DatabaseChanges, TableChange}; +use substreams_antelope::Block; -use crate::{ - creation_tree::insert_creation_tree, feature_ops::insert_feature_op, keys::transactions_keys, pb::antelope::Transaction as RawTransaction, perm_ops::insert_perm_op, ram_ops::insert_ram_op, - table_ops::insert_table_op, -}; - -use super::actions::insert_action; -use super::db_ops::insert_db_op; +use crate::pb::antelope::Transaction as RawTransaction; pub fn transaction_status_to_string(status: i32) -> String { match status { @@ -31,134 +23,6 @@ pub fn is_transaction_success(status: i32) -> bool { } // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L525 -pub fn insert_transaction(tables: &mut DatabaseChanges, clock: &Clock, transaction: &TransactionTrace, block_header: &BlockHeader) { - let hash = &transaction.id; - let index = transaction.index; - let elapsed = transaction.elapsed; - let net_usage = transaction.net_usage; - let scheduled = transaction.scheduled; - - // header - let header = transaction.receipt.clone().unwrap_or_default(); - let cpu_usage_micro_seconds = header.cpu_usage_micro_seconds; - let net_usage_words = header.net_usage_words; - let status = transaction_status_to_string(header.status); - let status_code = header.status; - let success = is_transaction_success(header.status); - - // block roots - let transaction_mroot = Hex::encode(&block_header.transaction_mroot.to_vec()); - - // TABLE::transactions - let keys = transactions_keys(hash); - let row = tables - .push_change_composite("transactions", keys, 0, table_change::Operation::Create) - .change("index", ("", index.to_string().as_str())) - .change("hash", ("", hash.as_str())) - .change("elapsed", ("", elapsed.to_string().as_str())) - .change("net_usage", ("", net_usage.to_string().as_str())) - .change("scheduled", ("", scheduled.to_string().as_str())) - // header - .change("cpu_usage_micro_seconds", ("", cpu_usage_micro_seconds.to_string().as_str())) - .change("net_usage_words", ("", net_usage_words.to_string().as_str())) - .change("status", ("", status.as_str())) - .change("status_code", ("", status_code.to_string().as_str())) - .change("success", ("", success.to_string().as_str())) - // block roots - .change("transaction_mroot", ("", transaction_mroot.as_str())); - insert_timestamp(row, clock, false, false); - - // TABLE::actions - for trace in transaction.action_traces.iter() { - insert_action(tables, clock, trace, transaction, block_header); - } - - // TABLE::db_ops - let mut db_op_index = 0; - for db_op in transaction.db_ops.iter() { - insert_db_op(tables, clock, db_op, transaction, db_op_index); - db_op_index += 1; - } - - // TO-DO - // Trace of a failed deferred transaction, if any. - // match transaction.failed_dtrx_trace { - // Some(failed_dtrx_trace) => { - // insert_transaction(tables, clock, &failed_dtrx_trace, &block); - // } - // None => {} - // } - - // TO-DO - // List of deferred transactions operations this transaction entailed - // for db_op in transaction.dtrx_ops.iter() { - // insert_db_op(tables, clock, db_op, &transaction, &block); - // } - - // TO-DO - // List of feature switching operations (changes to feature switches in nodeos) this transaction entailed - for feature_op in transaction.feature_ops.iter() { - insert_feature_op(tables, clock, transaction, feature_op); - } - - // TO-DO - // List of permission changes operations - for perm_op in transaction.perm_ops.iter() { - insert_perm_op(tables, clock, transaction, perm_op); - } - - // TO-DO - // List of RAM consumption/redemption - for ram_op in transaction.ram_ops.iter() { - insert_ram_op(tables, clock, ram_op, transaction); - } - - // TO-DO - // List of RAM correction operations (happens only once upon feature activation) - // for ram_correction_op in transaction.ram_correction_ops.iter() { - // insert_ram_correction_op(tables, clock, ram_correction_op, &block); - // } - - // TO-DO - // List of changes to rate limiting values - // for rlimit_op in transaction.rlimit_ops.iter() { - // insert_rlimit_op(tables, clock, rlimit_op, &block); - // } - - // List of table creations/deletions - let mut table_op_index = 0; - for table_op in transaction.table_ops.iter() { - insert_table_op(tables, clock, transaction, table_op, &table_op_index); - table_op_index += 1; - } - - // Tree of creation, rather than execution - for creation_flat_node in transaction.creation_tree.iter() { - insert_creation_tree(tables, clock, transaction, creation_flat_node); - } - - // TO-DO?? - // Exception leading to the failed dtrx trace. - // let exception = transaction.exception; - // let error_code = transaction.error_code; - // match exception { - // Some(exception) => { - // let exception_code = exception.code; - // let exception_name = exception.name; - // let exception_message = exception.message; - // } - // None => {} - // } -} - -pub fn insert_transaction_metadata(row: &mut TableChange, transaction: &TransactionTrace) { - let tx_hash = &transaction.id; - let header = transaction.receipt.clone().unwrap_or_default(); - let tx_success = is_transaction_success(header.status); - - row.change("tx_hash", ("", tx_hash.as_str())).change("tx_success", ("", tx_success.to_string().as_str())); -} - pub fn collect_transactions(block: &Block, timestamp: &BlockTimestamp) -> Vec { let header = block.header.clone().unwrap_or_default(); let mut transactions = Vec::new(); From cff6e8d6100278b1ba4f61f2f9994d3d45d32f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dillan=20Choini=C3=A8re?= Date: Tue, 19 Nov 2024 22:09:49 -0500 Subject: [PATCH 12/22] Refactor to use same transaction for loop --- blocks/antelope/src/account_ram_deltas.rs | 26 +++++++++ blocks/antelope/src/actions.rs | 43 ++++++++++++++ blocks/antelope/src/auth_sequences.rs | 27 +++++++++ blocks/antelope/src/authority.rs | 68 ++++++++++++++++++++++- blocks/antelope/src/authorizations.rs | 25 ++++++++- blocks/antelope/src/blocks.rs | 4 +- blocks/antelope/src/collect_events.rs | 60 ++++++++++++++++++++ blocks/antelope/src/creation_tree.rs | 20 +++++++ blocks/antelope/src/db_ops.rs | 33 ++++++++++- blocks/antelope/src/feature_ops.rs | 26 ++++++++- blocks/antelope/src/lib.rs | 1 + blocks/antelope/src/map_events.rs | 27 +-------- blocks/antelope/src/pb/antelope.rs | 4 +- blocks/antelope/src/perm_ops.rs | 31 ++++++++++- blocks/antelope/src/ram_ops.rs | 29 ++++++++++ blocks/antelope/src/table_ops.rs | 27 ++++++++- blocks/antelope/src/transactions.rs | 27 ++++++++- proto/antelope.proto | 2 +- 18 files changed, 444 insertions(+), 36 deletions(-) create mode 100644 blocks/antelope/src/collect_events.rs diff --git a/blocks/antelope/src/account_ram_deltas.rs b/blocks/antelope/src/account_ram_deltas.rs index 1636bc7..fc56801 100644 --- a/blocks/antelope/src/account_ram_deltas.rs +++ b/blocks/antelope/src/account_ram_deltas.rs @@ -1,6 +1,7 @@ use crate::pb::antelope::AccountRamDelta as RawAccountRamDelta; use crate::transactions::is_transaction_success; use common::structs::BlockTimestamp; +use substreams_antelope::pb::TransactionTrace; use substreams_antelope::Block; pub fn collect_account_ram_deltas(block: &Block, timestamp: &BlockTimestamp) -> Vec { @@ -32,3 +33,28 @@ pub fn collect_account_ram_deltas(block: &Block, timestamp: &BlockTimestamp) -> account_ram_deltas } + +pub fn collect_tx_account_ram_deltas(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { + let mut account_ram_deltas = Vec::new(); + + for action_trace in transaction.action_traces.iter() { + let action_index = action_trace.execution_index; + + for (index, delta) in action_trace.account_ram_deltas.iter().enumerate() { + account_ram_deltas.push(RawAccountRamDelta { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + action_index, + index: index as u32, + account: delta.account.clone(), + delta: delta.delta, + }); + } + } + + account_ram_deltas +} diff --git a/blocks/antelope/src/actions.rs b/blocks/antelope/src/actions.rs index 97c9f5e..8e08ed2 100644 --- a/blocks/antelope/src/actions.rs +++ b/blocks/antelope/src/actions.rs @@ -1,6 +1,7 @@ use common::structs::BlockTimestamp; use substreams::Hex; +use substreams_antelope::pb::TransactionTrace; use substreams_antelope::Block; use crate::pb::antelope::Action as RawAction; @@ -53,3 +54,45 @@ pub fn collect_actions(block: &Block, timestamp: &BlockTimestamp) -> Vec Vec { + let header = block.header.clone().unwrap_or_default(); + let mut actions: Vec = Vec::new(); + + for trace in transaction.action_traces.iter() { + let action = trace.action.clone().unwrap_or_default(); + let receipt = trace.receipt.clone().unwrap_or_default(); + + actions.push(RawAction { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + abi_sequence: receipt.abi_sequence, + code_sequence: receipt.code_sequence, + digest: receipt.digest, + global_sequence: receipt.global_sequence, + receipt_receiver: receipt.receiver, + recv_sequence: receipt.recv_sequence, + account: action.account, + name: action.name, + json_data: action.json_data, + raw_data: Hex::encode(&action.raw_data), + index: trace.execution_index, + action_ordinal: trace.action_ordinal, + receiver: trace.receiver.clone(), + context_free: trace.context_free, + elapsed: trace.elapsed, + console: trace.console.clone(), + raw_return_value: Hex::encode(&trace.raw_return_value), + json_return_value: trace.json_return_value.clone(), + creator_action_ordinal: trace.creator_action_ordinal, + closest_unnotified_ancestor_action_ordinal: trace.closest_unnotified_ancestor_action_ordinal, + action_mroot: Hex::encode(&header.action_mroot), + }); + } + + actions +} diff --git a/blocks/antelope/src/auth_sequences.rs b/blocks/antelope/src/auth_sequences.rs index daa9067..e100640 100644 --- a/blocks/antelope/src/auth_sequences.rs +++ b/blocks/antelope/src/auth_sequences.rs @@ -1,4 +1,5 @@ use common::structs::BlockTimestamp; +use substreams_antelope::pb::TransactionTrace; use substreams_antelope::Block; use crate::pb::antelope::AuthSequence as RawAuthSequence; @@ -35,3 +36,29 @@ pub fn collect_auth_sequences(block: &Block, timestamp: &BlockTimestamp) -> Vec< auth_sequences } + +pub fn collect_tx_auth_sequences(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { + let mut auth_sequences = Vec::new(); + + for action_trace in transaction.action_traces.iter() { + let receipt = action_trace.receipt.as_ref().expect("Action trace receipt is required"); + let action_index = action_trace.execution_index; + + for (index, auth) in receipt.auth_sequence.iter().enumerate() { + auth_sequences.push(RawAuthSequence { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + action_index, + index: index as u32, + account_name: auth.account_name.clone(), + sequence: auth.sequence, + }); + } + } + + auth_sequences +} diff --git a/blocks/antelope/src/authority.rs b/blocks/antelope/src/authority.rs index 7d08fef..addcd90 100644 --- a/blocks/antelope/src/authority.rs +++ b/blocks/antelope/src/authority.rs @@ -3,7 +3,7 @@ use crate::{ transactions::is_transaction_success, }; use common::structs::BlockTimestamp; -use substreams_antelope::Block; +use substreams_antelope::{pb::TransactionTrace, Block}; pub struct AuthorityVectors { pub accounts: Vec, @@ -81,3 +81,69 @@ pub fn collect_authority_vectors(block: &Block, timestamp: &BlockTimestamp) -> A AuthorityVectors { accounts, keys, waits } } + +pub fn collect_tx_authority_vectors(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> AuthorityVectors { + let mut accounts: Vec = vec![]; + let mut keys: Vec = vec![]; + let mut waits: Vec = vec![]; + + for perm_op in transaction.perm_ops.iter() { + if let Some(new_perm) = &perm_op.new_perm { + let authority = new_perm.authority.as_ref().unwrap(); + let action_index = perm_op.action_index; + + // Process authority accounts + for (index, account) in authority.accounts.iter().enumerate() { + if let Some(permission) = &account.permission { + accounts.push(RawAccount { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + index: index as u32, + action_index: action_index as u32, + actor: permission.actor.clone(), + permission: permission.permission.clone(), + weight: account.weight, + }); + } + } + + // Process authority keys + for (index, key) in authority.keys.iter().enumerate() { + keys.push(RawKey { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + index: index as u32, + action_index: action_index as u32, + public_key: key.public_key.clone(), + weight: key.weight, + }); + } + + // Process authority waits + for (index, wait) in authority.waits.iter().enumerate() { + waits.push(RawWait { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + index: index as u32, + action_index: action_index as u32, + wait_sec: wait.wait_sec, + weight: wait.weight, + }); + } + } + } + + AuthorityVectors { accounts, keys, waits } +} diff --git a/blocks/antelope/src/authorizations.rs b/blocks/antelope/src/authorizations.rs index d9e3f3b..acf0f63 100644 --- a/blocks/antelope/src/authorizations.rs +++ b/blocks/antelope/src/authorizations.rs @@ -1,5 +1,5 @@ use common::structs::BlockTimestamp; -use substreams_antelope::Block; +use substreams_antelope::{pb::TransactionTrace, Block}; use crate::{pb::antelope::Authorization as RawAuthorization, transactions::is_transaction_success}; @@ -31,3 +31,26 @@ pub fn collect_authorizations(block: &Block, timestamp: &BlockTimestamp) -> Vec< authorizations } + +pub fn collect_tx_authorizations(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { + let mut authorizations = Vec::new(); + + for action_trace in transaction.action_traces.iter() { + for (index, authorization) in action_trace.action.as_ref().unwrap().authorization.iter().enumerate() { + authorizations.push(RawAuthorization { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + action_index: action_trace.execution_index, + index: index as u32, + actor: authorization.actor.clone(), + permission: authorization.permission.clone(), + }); + } + } + + authorizations +} diff --git a/blocks/antelope/src/blocks.rs b/blocks/antelope/src/blocks.rs index c26dd00..afb6a44 100644 --- a/blocks/antelope/src/blocks.rs +++ b/blocks/antelope/src/blocks.rs @@ -1,5 +1,5 @@ use crate::{pb::antelope::Block as RawBlock, size::collect_size}; -use common::{structs::BlockTimestamp, utils::bytes_to_hex}; +use common::structs::BlockTimestamp; use substreams::Hex; use substreams_antelope::Block; @@ -24,7 +24,7 @@ pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> RawBlock { dpos_irreversible_blocknum: block.dpos_irreversible_blocknum, transaction_mroot: Hex::encode(&header.transaction_mroot.to_vec()), action_mroot: Hex::encode(&header.action_mroot.to_vec()), - blockroot_merkle_active_nodes: blockroot_merkle.active_nodes.iter().map(|row| bytes_to_hex(row)).collect::>().join(","), + blockroot_merkle_active_nodes: blockroot_merkle.active_nodes.iter().map(|row| Hex::encode(row)).collect::>(), blockroot_merkle_node_count: blockroot_merkle.node_count, size: size.size, total_transactions: size.total_transactions, diff --git a/blocks/antelope/src/collect_events.rs b/blocks/antelope/src/collect_events.rs new file mode 100644 index 0000000..d31747f --- /dev/null +++ b/blocks/antelope/src/collect_events.rs @@ -0,0 +1,60 @@ +use common::structs::BlockTimestamp; +use substreams_antelope::Block; + +use crate::{ + account_ram_deltas::collect_tx_account_ram_deltas, + actions::collect_tx_actions, + auth_sequences::collect_tx_auth_sequences, + authority::collect_tx_authority_vectors, + authorizations::collect_tx_authorizations, + blocks::collect_block, + creation_tree::collect_tx_creation_trees, + db_ops::collect_tx_db_ops, + feature_ops::collect_tx_feature_ops, + pb::antelope::Events as RawEvents, + perm_ops::collect_tx_perm_ops, + ram_ops::collect_tx_ram_ops, + table_ops::collect_tx_table_ops, + transactions::{collect_transaction, is_transaction_success}, +}; + +pub fn collect_events(block: &Block, timestamp: &BlockTimestamp) -> RawEvents { + let mut events = RawEvents { + blocks: vec![collect_block(block, timestamp)], + transactions: vec![], + actions: vec![], + db_ops: vec![], + feature_ops: vec![], + perm_ops: vec![], + table_ops: vec![], + accounts: vec![], + keys: vec![], + waits: vec![], + ram_ops: vec![], + authorizations: vec![], + auth_sequences: vec![], + account_ram_deltas: vec![], + creation_trees: vec![], + }; + + for transaction in block.transaction_traces() { + let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); + events.transactions.push(collect_transaction(block, transaction, timestamp, tx_success)); + events.actions.extend(collect_tx_actions(block, transaction, timestamp, tx_success)); + events.db_ops.extend(collect_tx_db_ops(transaction, timestamp, tx_success)); + events.feature_ops.extend(collect_tx_feature_ops(transaction, timestamp, tx_success)); + events.perm_ops.extend(collect_tx_perm_ops(transaction, timestamp, tx_success)); + events.table_ops.extend(collect_tx_table_ops(transaction, timestamp, tx_success)); + let authority_vectors = collect_tx_authority_vectors(transaction, timestamp, tx_success); + events.accounts.extend(authority_vectors.accounts); + events.keys.extend(authority_vectors.keys); + events.waits.extend(authority_vectors.waits); + events.ram_ops.extend(collect_tx_ram_ops(transaction, timestamp, tx_success)); + events.authorizations.extend(collect_tx_authorizations(transaction, timestamp, tx_success)); + events.auth_sequences.extend(collect_tx_auth_sequences(transaction, timestamp, tx_success)); + events.account_ram_deltas.extend(collect_tx_account_ram_deltas(transaction, timestamp, tx_success)); + events.creation_trees.extend(collect_tx_creation_trees(transaction, timestamp, tx_success)); + } + + events +} diff --git a/blocks/antelope/src/creation_tree.rs b/blocks/antelope/src/creation_tree.rs index 9ef37d7..8f4cf1e 100644 --- a/blocks/antelope/src/creation_tree.rs +++ b/blocks/antelope/src/creation_tree.rs @@ -1,6 +1,7 @@ use crate::pb::antelope::CreationTree as RawCreationTree; use crate::transactions::is_transaction_success; use common::structs::BlockTimestamp; +use substreams_antelope::pb::TransactionTrace; use substreams_antelope::Block; pub fn collect_creation_trees(block: &Block, timestamp: &BlockTimestamp) -> Vec { @@ -26,3 +27,22 @@ pub fn collect_creation_trees(block: &Block, timestamp: &BlockTimestamp) -> Vec< creation_trees } + +pub fn collect_tx_creation_trees(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { + let mut creation_trees = Vec::new(); + + for creation_flat_node in transaction.creation_tree.iter() { + creation_trees.push(RawCreationTree { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + creator_action_index: creation_flat_node.creator_action_index, + execution_action_index: creation_flat_node.execution_action_index, + }); + } + + creation_trees +} diff --git a/blocks/antelope/src/db_ops.rs b/blocks/antelope/src/db_ops.rs index 4389150..ad695d6 100644 --- a/blocks/antelope/src/db_ops.rs +++ b/blocks/antelope/src/db_ops.rs @@ -1,6 +1,6 @@ use common::structs::BlockTimestamp; use substreams::Hex; -use substreams_antelope::Block; +use substreams_antelope::{pb::TransactionTrace, Block}; use crate::{pb::antelope::DbOp as RawDbOp, transactions::is_transaction_success}; @@ -50,3 +50,34 @@ pub fn collect_db_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec db_ops } + +pub fn collect_tx_db_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { + let mut db_ops: Vec = Vec::new(); + + for (index, db_op) in transaction.db_ops.iter().enumerate() { + db_ops.push(RawDbOp { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + index: index as u32, + operation: operation_to_string(db_op.operation), + operation_code: db_op.operation as u32, + action_index: db_op.action_index, + code: db_op.code.clone(), + scope: db_op.scope.clone(), + table_name: db_op.table_name.clone(), + primary_key: db_op.primary_key.clone(), + old_payer: db_op.old_payer.clone(), + new_payer: db_op.new_payer.clone(), + old_data: Hex::encode(&db_op.old_data), + new_data: Hex::encode(&db_op.new_data), + old_data_json: db_op.old_data_json.clone(), + new_data_json: db_op.new_data_json.clone(), + }); + } + + db_ops +} diff --git a/blocks/antelope/src/feature_ops.rs b/blocks/antelope/src/feature_ops.rs index fb5bece..86e9691 100644 --- a/blocks/antelope/src/feature_ops.rs +++ b/blocks/antelope/src/feature_ops.rs @@ -1,5 +1,5 @@ use common::structs::BlockTimestamp; -use substreams_antelope::Block; +use substreams_antelope::{pb::TransactionTrace, Block}; use crate::{pb::antelope::FeatureOp as RawFeatureOp, transactions::is_transaction_success}; @@ -31,3 +31,27 @@ pub fn collect_feature_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec Vec { + let mut feature_ops = Vec::new(); + + for feature_op in transaction.feature_ops.iter() { + let feature = feature_op.feature.as_ref().expect("feature is required"); + + feature_ops.push(RawFeatureOp { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + feature_digest: feature_op.feature_digest.clone(), + kind: feature_op.kind.clone(), + action_index: feature_op.action_index, + description_digest: feature.description_digest.clone(), + protocol_feature_type: feature.protocol_feature_type.clone(), + }); + } + + feature_ops +} diff --git a/blocks/antelope/src/lib.rs b/blocks/antelope/src/lib.rs index d0193b9..097d0fe 100644 --- a/blocks/antelope/src/lib.rs +++ b/blocks/antelope/src/lib.rs @@ -4,6 +4,7 @@ mod auth_sequences; mod authority; mod authorizations; mod blocks; +mod collect_events; mod creation_tree; mod db_ops; mod feature_ops; diff --git a/blocks/antelope/src/map_events.rs b/blocks/antelope/src/map_events.rs index aab5a5f..688b564 100644 --- a/blocks/antelope/src/map_events.rs +++ b/blocks/antelope/src/map_events.rs @@ -3,32 +3,11 @@ use substreams::errors::Error; use substreams::pb::substreams::Clock; use substreams_antelope::pb::Block; -use crate::{ - account_ram_deltas::collect_account_ram_deltas, actions::collect_actions, auth_sequences::collect_auth_sequences, authority::collect_authority_vectors, authorizations::collect_authorizations, - blocks::collect_block, creation_tree::collect_creation_trees, db_ops::collect_db_ops, feature_ops::collect_feature_ops, pb::antelope::Events, perm_ops::collect_perm_ops, ram_ops::collect_ram_ops, - table_ops::collect_table_ops, transactions::collect_transactions, -}; +use crate::collect_events::collect_events; +use crate::pb::antelope::Events; #[substreams::handlers::map] pub fn map_events(clock: Clock, block: Block) -> Result { let timestamp = build_timestamp(&clock); - let authority_vectors = collect_authority_vectors(&block, ×tamp); - - Ok(Events { - blocks: vec![collect_block(&block, ×tamp)], - transactions: collect_transactions(&block, ×tamp), - actions: collect_actions(&block, ×tamp), - db_ops: collect_db_ops(&block, ×tamp), - feature_ops: collect_feature_ops(&block, ×tamp), - perm_ops: collect_perm_ops(&block, ×tamp), - table_ops: collect_table_ops(&block, ×tamp), - accounts: authority_vectors.accounts, - keys: authority_vectors.keys, - waits: authority_vectors.waits, - ram_ops: collect_ram_ops(&block, ×tamp), - authorizations: collect_authorizations(&block, ×tamp), - auth_sequences: collect_auth_sequences(&block, ×tamp), - account_ram_deltas: collect_account_ram_deltas(&block, ×tamp), - creation_trees: collect_creation_trees(&block, ×tamp), - }) + Ok(collect_events(&block, ×tamp)) } diff --git a/blocks/antelope/src/pb/antelope.rs b/blocks/antelope/src/pb/antelope.rs index 3c3b2fd..718792b 100644 --- a/blocks/antelope/src/pb/antelope.rs +++ b/blocks/antelope/src/pb/antelope.rs @@ -31,8 +31,8 @@ pub struct Block { pub transaction_mroot: ::prost::alloc::string::String, #[prost(string, tag="14")] pub action_mroot: ::prost::alloc::string::String, - #[prost(string, tag="15")] - pub blockroot_merkle_active_nodes: ::prost::alloc::string::String, + #[prost(string, repeated, tag="15")] + pub blockroot_merkle_active_nodes: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, #[prost(uint32, tag="16")] pub blockroot_merkle_node_count: u32, #[prost(uint64, tag="17")] diff --git a/blocks/antelope/src/perm_ops.rs b/blocks/antelope/src/perm_ops.rs index 5b8264b..b23757b 100644 --- a/blocks/antelope/src/perm_ops.rs +++ b/blocks/antelope/src/perm_ops.rs @@ -1,6 +1,6 @@ use crate::{pb::antelope::PermOp as RawPermOp, transactions::is_transaction_success}; use common::structs::BlockTimestamp; -use substreams_antelope::Block; +use substreams_antelope::{pb::TransactionTrace, Block}; pub fn perm_op_operation_to_string(operation: i32) -> String { match operation { @@ -45,3 +45,32 @@ pub fn collect_perm_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec Vec { + let mut perm_ops = Vec::new(); + + for perm_op in transaction.perm_ops.iter() { + if let Some(new_perm) = &perm_op.new_perm { + let threshold = new_perm.authority.as_ref().map_or(0, |authority| authority.threshold); + + perm_ops.push(RawPermOp { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + operation: perm_op_operation_to_string(perm_op.operation), + operation_code: perm_op.operation as u32, + action_index: perm_op.action_index, + id: new_perm.id, + parent_id: new_perm.parent_id, + owner: new_perm.owner.clone(), + name: new_perm.name.clone(), + threshold, + }); + } + } + + perm_ops +} diff --git a/blocks/antelope/src/ram_ops.rs b/blocks/antelope/src/ram_ops.rs index 8a8004f..728141a 100644 --- a/blocks/antelope/src/ram_ops.rs +++ b/blocks/antelope/src/ram_ops.rs @@ -1,4 +1,5 @@ use common::structs::BlockTimestamp; +use substreams_antelope::pb::TransactionTrace; use substreams_antelope::Block; use crate::pb::antelope::RamOp as RawRamOp; @@ -97,3 +98,31 @@ pub fn collect_ram_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec Vec { + let mut ram_ops = Vec::new(); + + for ram_op in transaction.ram_ops.iter() { + ram_ops.push(RawRamOp { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + operation: operation_to_string(ram_op.operation), + action_index: ram_op.action_index, + payer: ram_op.payer.clone(), + delta: ram_op.delta, + usage: ram_op.usage, + namespace: namespace_to_string(ram_op.namespace), + action: action_to_string(ram_op.action), + unique_key: ram_op.unique_key.clone(), + operation_code: ram_op.operation as u32, + namespace_code: ram_op.namespace as u32, + action_code: ram_op.action as u32, + }); + } + + ram_ops +} diff --git a/blocks/antelope/src/table_ops.rs b/blocks/antelope/src/table_ops.rs index a262a25..7841e81 100644 --- a/blocks/antelope/src/table_ops.rs +++ b/blocks/antelope/src/table_ops.rs @@ -1,5 +1,5 @@ use common::structs::BlockTimestamp; -use substreams_antelope::Block; +use substreams_antelope::{pb::TransactionTrace, Block}; use crate::{pb::antelope::TableOp as RawTableOp, transactions::is_transaction_success}; @@ -41,3 +41,28 @@ pub fn collect_table_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec Vec { + let mut table_ops = Vec::new(); + + for (index, table_op) in transaction.table_ops.iter().enumerate() { + table_ops.push(RawTableOp { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + tx_hash: transaction.id.clone(), + tx_success, + index: index as u32, + action_index: table_op.action_index, + operation: table_op_operation_to_string(table_op.operation), + operation_code: table_op.operation as u32, + payer: table_op.payer.clone(), + code: table_op.code.clone(), + scope: table_op.scope.clone(), + table_name: table_op.table_name.clone(), + }); + } + + table_ops +} diff --git a/blocks/antelope/src/transactions.rs b/blocks/antelope/src/transactions.rs index a70f83f..91521d6 100644 --- a/blocks/antelope/src/transactions.rs +++ b/blocks/antelope/src/transactions.rs @@ -1,6 +1,6 @@ use common::structs::BlockTimestamp; use substreams::Hex; -use substreams_antelope::Block; +use substreams_antelope::{pb::TransactionTrace, Block}; use crate::pb::antelope::Transaction as RawTransaction; @@ -54,3 +54,28 @@ pub fn collect_transactions(block: &Block, timestamp: &BlockTimestamp) -> Vec RawTransaction { + let header = block.header.clone().unwrap_or_default(); + let receipt = transaction.receipt.clone().unwrap_or_default(); + let status_code = receipt.status; + let status = transaction_status_to_string(status_code); + + RawTransaction { + block_time: Some(timestamp.time.clone()), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + hash: transaction.id.clone(), + index: transaction.index as u64, + elapsed: transaction.elapsed, + net_usage: transaction.net_usage, + scheduled: transaction.scheduled, + cpu_usage_micro_seconds: receipt.cpu_usage_micro_seconds, + net_usage_words: receipt.net_usage_words, + status, + status_code: status_code as u32, + success: tx_success, + transaction_mroot: Hex::encode(&header.transaction_mroot.to_vec()), + } +} diff --git a/proto/antelope.proto b/proto/antelope.proto index 9ab7815..06ec8c1 100644 --- a/proto/antelope.proto +++ b/proto/antelope.proto @@ -20,7 +20,7 @@ message Block { uint32 dpos_irreversible_blocknum = 12; string transaction_mroot = 13; string action_mroot = 14; - string blockroot_merkle_active_nodes = 15; + repeated string blockroot_merkle_active_nodes = 15; uint32 blockroot_merkle_node_count = 16; uint64 size = 17; uint64 total_transactions = 18; From 70075fe75115c038b5f1f4766743fbb25e67b6dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dillan=20Choini=C3=A8re?= Date: Tue, 19 Nov 2024 22:13:04 -0500 Subject: [PATCH 13/22] Cleanup unused functions --- blocks/antelope/src/account_ram_deltas.rs | 32 --------- blocks/antelope/src/actions.rs | 48 ------------- blocks/antelope/src/auth_sequences.rs | 34 --------- blocks/antelope/src/authority.rs | 84 ++--------------------- blocks/antelope/src/authorizations.rs | 32 +-------- blocks/antelope/src/creation_tree.rs | 26 ------- blocks/antelope/src/db_ops.rs | 40 +---------- blocks/antelope/src/feature_ops.rs | 33 +-------- blocks/antelope/src/perm_ops.rs | 38 +--------- blocks/antelope/src/ram_ops.rs | 35 ---------- blocks/antelope/src/table_ops.rs | 34 +-------- blocks/antelope/src/transactions.rs | 32 --------- 12 files changed, 15 insertions(+), 453 deletions(-) diff --git a/blocks/antelope/src/account_ram_deltas.rs b/blocks/antelope/src/account_ram_deltas.rs index fc56801..a24cd1c 100644 --- a/blocks/antelope/src/account_ram_deltas.rs +++ b/blocks/antelope/src/account_ram_deltas.rs @@ -1,38 +1,6 @@ use crate::pb::antelope::AccountRamDelta as RawAccountRamDelta; -use crate::transactions::is_transaction_success; use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; -use substreams_antelope::Block; - -pub fn collect_account_ram_deltas(block: &Block, timestamp: &BlockTimestamp) -> Vec { - let mut account_ram_deltas: Vec = vec![]; - - for transaction in block.transaction_traces() { - let tx_hash = &transaction.id; - let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); - - for action_trace in transaction.action_traces.iter() { - let action_index = action_trace.execution_index; - - for (index, delta) in action_trace.account_ram_deltas.iter().enumerate() { - account_ram_deltas.push(RawAccountRamDelta { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - action_index, - index: index as u32, - account: delta.account.clone(), - delta: delta.delta, - }); - } - } - } - - account_ram_deltas -} pub fn collect_tx_account_ram_deltas(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut account_ram_deltas = Vec::new(); diff --git a/blocks/antelope/src/actions.rs b/blocks/antelope/src/actions.rs index 8e08ed2..882957d 100644 --- a/blocks/antelope/src/actions.rs +++ b/blocks/antelope/src/actions.rs @@ -5,56 +5,8 @@ use substreams_antelope::pb::TransactionTrace; use substreams_antelope::Block; use crate::pb::antelope::Action as RawAction; -use crate::transactions::is_transaction_success; // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L525 -pub fn collect_actions(block: &Block, timestamp: &BlockTimestamp) -> Vec { - let header = block.header.clone().unwrap_or_default(); - let mut actions = Vec::new(); - - for transaction in block.transaction_traces() { - let tx_hash = &transaction.id; - let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); - - for trace in transaction.action_traces.iter() { - let action = trace.action.clone().unwrap_or_default(); - let receipt = trace.receipt.clone().unwrap_or_default(); - - actions.push(RawAction { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - abi_sequence: receipt.abi_sequence, - code_sequence: receipt.code_sequence, - digest: receipt.digest, - global_sequence: receipt.global_sequence, - receipt_receiver: receipt.receiver, - recv_sequence: receipt.recv_sequence, - account: action.account, - name: action.name, - json_data: action.json_data, - raw_data: Hex::encode(&action.raw_data), - index: trace.execution_index, - action_ordinal: trace.action_ordinal, - receiver: trace.receiver.clone(), - context_free: trace.context_free, - elapsed: trace.elapsed, - console: trace.console.clone(), - raw_return_value: Hex::encode(&trace.raw_return_value), - json_return_value: trace.json_return_value.clone(), - creator_action_ordinal: trace.creator_action_ordinal, - closest_unnotified_ancestor_action_ordinal: trace.closest_unnotified_ancestor_action_ordinal, - action_mroot: Hex::encode(&header.action_mroot), - }); - } - } - - actions -} - pub fn collect_tx_actions(block: &Block, transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let header = block.header.clone().unwrap_or_default(); let mut actions: Vec = Vec::new(); diff --git a/blocks/antelope/src/auth_sequences.rs b/blocks/antelope/src/auth_sequences.rs index e100640..72f932f 100644 --- a/blocks/antelope/src/auth_sequences.rs +++ b/blocks/antelope/src/auth_sequences.rs @@ -1,41 +1,7 @@ use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; -use substreams_antelope::Block; use crate::pb::antelope::AuthSequence as RawAuthSequence; -use crate::transactions::is_transaction_success; - -pub fn collect_auth_sequences(block: &Block, timestamp: &BlockTimestamp) -> Vec { - let mut auth_sequences: Vec = vec![]; - - for transaction in block.transaction_traces() { - let tx_hash = &transaction.id; - let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); - - for action_trace in transaction.action_traces.iter() { - let receipt = action_trace.receipt.as_ref().expect("Action trace receipt is required"); - - let action_index = action_trace.execution_index; - - for (index, auth) in receipt.auth_sequence.iter().enumerate() { - auth_sequences.push(RawAuthSequence { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - action_index, - index: index as u32, - account_name: auth.account_name.clone(), - sequence: auth.sequence, - }); - } - } - } - - auth_sequences -} pub fn collect_tx_auth_sequences(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut auth_sequences = Vec::new(); diff --git a/blocks/antelope/src/authority.rs b/blocks/antelope/src/authority.rs index addcd90..5b5179e 100644 --- a/blocks/antelope/src/authority.rs +++ b/blocks/antelope/src/authority.rs @@ -1,9 +1,6 @@ -use crate::{ - pb::antelope::{Account as RawAccount, Key as RawKey, Wait as RawWait}, - transactions::is_transaction_success, -}; +use crate::pb::antelope::{Account as RawAccount, Key as RawKey, Wait as RawWait}; use common::structs::BlockTimestamp; -use substreams_antelope::{pb::TransactionTrace, Block}; +use substreams_antelope::pb::TransactionTrace; pub struct AuthorityVectors { pub accounts: Vec, @@ -11,81 +8,10 @@ pub struct AuthorityVectors { pub waits: Vec, } -pub fn collect_authority_vectors(block: &Block, timestamp: &BlockTimestamp) -> AuthorityVectors { - let mut accounts: Vec = vec![]; - let mut keys: Vec = vec![]; - let mut waits: Vec = vec![]; - - for transaction in block.transaction_traces() { - let tx_hash = &transaction.id; - let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); - - for perm_op in transaction.perm_ops.iter() { - if let Some(new_perm) = &perm_op.new_perm { - let authority = new_perm.authority.as_ref().unwrap(); - let action_index = perm_op.action_index; - - // Process authority - for (index, account) in authority.accounts.iter().enumerate() { - if let Some(permission) = &account.permission { - accounts.push(RawAccount { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - index: index as u32, - action_index: perm_op.action_index as u32, - actor: permission.actor.clone(), - permission: permission.permission.clone(), - weight: account.weight, - }); - } - } - - // Process keys - for (index, key) in authority.keys.iter().enumerate() { - keys.push(RawKey { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - index: index as u32, - action_index: action_index as u32, - public_key: key.public_key.clone(), - weight: key.weight, - }); - } - - // Process waits - for (index, wait) in authority.waits.iter().enumerate() { - waits.push(RawWait { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - index: index as u32, - action_index: action_index as u32, - wait_sec: wait.wait_sec, - weight: wait.weight, - }); - } - } - } - } - - AuthorityVectors { accounts, keys, waits } -} - pub fn collect_tx_authority_vectors(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> AuthorityVectors { - let mut accounts: Vec = vec![]; - let mut keys: Vec = vec![]; - let mut waits: Vec = vec![]; + let mut accounts: Vec = Vec::new(); + let mut keys: Vec = Vec::new(); + let mut waits: Vec = Vec::new(); for perm_op in transaction.perm_ops.iter() { if let Some(new_perm) = &perm_op.new_perm { diff --git a/blocks/antelope/src/authorizations.rs b/blocks/antelope/src/authorizations.rs index acf0f63..b101aad 100644 --- a/blocks/antelope/src/authorizations.rs +++ b/blocks/antelope/src/authorizations.rs @@ -1,37 +1,9 @@ use common::structs::BlockTimestamp; -use substreams_antelope::{pb::TransactionTrace, Block}; +use substreams_antelope::pb::TransactionTrace; -use crate::{pb::antelope::Authorization as RawAuthorization, transactions::is_transaction_success}; +use crate::pb::antelope::Authorization as RawAuthorization; // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L616 -pub fn collect_authorizations(block: &Block, timestamp: &BlockTimestamp) -> Vec { - let mut authorizations: Vec = vec![]; - - for transaction in block.transaction_traces() { - let tx_hash = &transaction.id; - let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); - - for action_trace in transaction.action_traces.iter() { - for (index, authorization) in action_trace.action.as_ref().unwrap().authorization.iter().enumerate() { - authorizations.push(RawAuthorization { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - action_index: action_trace.execution_index, - index: index as u32, - actor: authorization.actor.clone(), - permission: authorization.permission.clone(), - }); - } - } - } - - authorizations -} - pub fn collect_tx_authorizations(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut authorizations = Vec::new(); diff --git a/blocks/antelope/src/creation_tree.rs b/blocks/antelope/src/creation_tree.rs index 8f4cf1e..2931db8 100644 --- a/blocks/antelope/src/creation_tree.rs +++ b/blocks/antelope/src/creation_tree.rs @@ -1,32 +1,6 @@ use crate::pb::antelope::CreationTree as RawCreationTree; -use crate::transactions::is_transaction_success; use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; -use substreams_antelope::Block; - -pub fn collect_creation_trees(block: &Block, timestamp: &BlockTimestamp) -> Vec { - let mut creation_trees: Vec = vec![]; - - for transaction in block.transaction_traces() { - let tx_hash = &transaction.id; - let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); - - for creation_flat_node in transaction.creation_tree.iter() { - creation_trees.push(RawCreationTree { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - creator_action_index: creation_flat_node.creator_action_index, - execution_action_index: creation_flat_node.execution_action_index, - }); - } - } - - creation_trees -} pub fn collect_tx_creation_trees(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut creation_trees = Vec::new(); diff --git a/blocks/antelope/src/db_ops.rs b/blocks/antelope/src/db_ops.rs index ad695d6..af2ca5e 100644 --- a/blocks/antelope/src/db_ops.rs +++ b/blocks/antelope/src/db_ops.rs @@ -1,8 +1,8 @@ use common::structs::BlockTimestamp; use substreams::Hex; -use substreams_antelope::{pb::TransactionTrace, Block}; +use substreams_antelope::pb::TransactionTrace; -use crate::{pb::antelope::DbOp as RawDbOp, transactions::is_transaction_success}; +use crate::pb::antelope::DbOp as RawDbOp; pub fn operation_to_string(operation: i32) -> String { match operation { @@ -15,42 +15,6 @@ pub fn operation_to_string(operation: i32) -> String { } // https://github.com/streamingfast/firehose-ethereum/blob/1bcb32a8eb3e43347972b6b5c9b1fcc4a08c751e/proto/sf/ethereum/type/v2/type.proto#L647 -pub fn collect_db_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { - let mut db_ops = Vec::new(); - - for transaction in block.transaction_traces() { - let tx_hash = &transaction.id; - let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); - - for (index, db_op) in transaction.db_ops.iter().enumerate() { - db_ops.push(RawDbOp { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - index: index as u32, - operation: operation_to_string(db_op.operation), - operation_code: db_op.operation as u32, - action_index: db_op.action_index, - code: db_op.code.clone(), - scope: db_op.scope.clone(), - table_name: db_op.table_name.clone(), - primary_key: db_op.primary_key.clone(), - old_payer: db_op.old_payer.clone(), - new_payer: db_op.new_payer.clone(), - old_data: Hex::encode(&db_op.old_data), - new_data: Hex::encode(&db_op.new_data), - old_data_json: db_op.old_data_json.clone(), - new_data_json: db_op.new_data_json.clone(), - }); - } - } - - db_ops -} - pub fn collect_tx_db_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut db_ops: Vec = Vec::new(); diff --git a/blocks/antelope/src/feature_ops.rs b/blocks/antelope/src/feature_ops.rs index 86e9691..b263333 100644 --- a/blocks/antelope/src/feature_ops.rs +++ b/blocks/antelope/src/feature_ops.rs @@ -1,36 +1,7 @@ use common::structs::BlockTimestamp; -use substreams_antelope::{pb::TransactionTrace, Block}; +use substreams_antelope::pb::TransactionTrace; -use crate::{pb::antelope::FeatureOp as RawFeatureOp, transactions::is_transaction_success}; - -pub fn collect_feature_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { - let mut feature_ops: Vec = vec![]; - - for transaction in block.transaction_traces() { - let tx_hash = &transaction.id; - let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); - - for feature_op in transaction.feature_ops.iter() { - let feature = feature_op.feature.as_ref().expect("feature is required"); - - feature_ops.push(RawFeatureOp { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - feature_digest: feature_op.feature_digest.clone(), - kind: feature_op.kind.clone(), - action_index: feature_op.action_index, - description_digest: feature.description_digest.clone(), - protocol_feature_type: feature.protocol_feature_type.clone(), - }); - } - } - - feature_ops -} +use crate::pb::antelope::FeatureOp as RawFeatureOp; pub fn collect_tx_feature_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut feature_ops = Vec::new(); diff --git a/blocks/antelope/src/perm_ops.rs b/blocks/antelope/src/perm_ops.rs index b23757b..f2c1cb9 100644 --- a/blocks/antelope/src/perm_ops.rs +++ b/blocks/antelope/src/perm_ops.rs @@ -1,6 +1,6 @@ -use crate::{pb::antelope::PermOp as RawPermOp, transactions::is_transaction_success}; +use crate::pb::antelope::PermOp as RawPermOp; use common::structs::BlockTimestamp; -use substreams_antelope::{pb::TransactionTrace, Block}; +use substreams_antelope::pb::TransactionTrace; pub fn perm_op_operation_to_string(operation: i32) -> String { match operation { @@ -12,40 +12,6 @@ pub fn perm_op_operation_to_string(operation: i32) -> String { } } -pub fn collect_perm_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { - let mut perm_ops: Vec = vec![]; - - for transaction in block.transaction_traces() { - let tx_hash = &transaction.id; - let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); - - for perm_op in transaction.perm_ops.iter() { - if let Some(new_perm) = &perm_op.new_perm { - let threshold = new_perm.authority.as_ref().map_or(0, |authority| authority.threshold); - - perm_ops.push(RawPermOp { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - operation: perm_op_operation_to_string(perm_op.operation), - operation_code: perm_op.operation as u32, - action_index: perm_op.action_index, - id: new_perm.id, - parent_id: new_perm.parent_id, - owner: new_perm.owner.clone(), - name: new_perm.name.clone(), - threshold, - }); - } - } - } - - perm_ops -} - pub fn collect_tx_perm_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut perm_ops = Vec::new(); diff --git a/blocks/antelope/src/ram_ops.rs b/blocks/antelope/src/ram_ops.rs index 728141a..9b41f4e 100644 --- a/blocks/antelope/src/ram_ops.rs +++ b/blocks/antelope/src/ram_ops.rs @@ -1,9 +1,7 @@ use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; -use substreams_antelope::Block; use crate::pb::antelope::RamOp as RawRamOp; -use crate::transactions::is_transaction_success; pub fn namespace_to_string(namespace: i32) -> String { match namespace { @@ -66,39 +64,6 @@ pub fn operation_to_string(operation: i32) -> String { } } -pub fn collect_ram_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { - let mut ram_ops: Vec = vec![]; - - for transaction in block.transaction_traces() { - let tx_hash = &transaction.id; - let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); - - for ram_op in transaction.ram_ops.iter() { - ram_ops.push(RawRamOp { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - operation: operation_to_string(ram_op.operation), - action_index: ram_op.action_index, - payer: ram_op.payer.clone(), - delta: ram_op.delta, - usage: ram_op.usage, - namespace: namespace_to_string(ram_op.namespace), - action: action_to_string(ram_op.action), - unique_key: ram_op.unique_key.clone(), - operation_code: ram_op.operation as u32, - namespace_code: ram_op.namespace as u32, - action_code: ram_op.action as u32, - }); - } - } - - ram_ops -} - pub fn collect_tx_ram_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut ram_ops = Vec::new(); diff --git a/blocks/antelope/src/table_ops.rs b/blocks/antelope/src/table_ops.rs index 7841e81..aae96f6 100644 --- a/blocks/antelope/src/table_ops.rs +++ b/blocks/antelope/src/table_ops.rs @@ -1,7 +1,7 @@ use common::structs::BlockTimestamp; -use substreams_antelope::{pb::TransactionTrace, Block}; +use substreams_antelope::pb::TransactionTrace; -use crate::{pb::antelope::TableOp as RawTableOp, transactions::is_transaction_success}; +use crate::pb::antelope::TableOp as RawTableOp; pub fn table_op_operation_to_string(operation: i32) -> String { match operation { @@ -12,36 +12,6 @@ pub fn table_op_operation_to_string(operation: i32) -> String { } } -pub fn collect_table_ops(block: &Block, timestamp: &BlockTimestamp) -> Vec { - let mut table_ops: Vec = vec![]; - - for transaction in block.transaction_traces() { - let tx_hash = &transaction.id; - let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); - - for (index, table_op) in transaction.table_ops.iter().enumerate() { - table_ops.push(RawTableOp { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: tx_hash.clone(), - tx_success, - index: index as u32, - action_index: table_op.action_index, - operation: table_op_operation_to_string(table_op.operation), - operation_code: table_op.operation as u32, - payer: table_op.payer.clone(), - code: table_op.code.clone(), - scope: table_op.scope.clone(), - table_name: table_op.table_name.clone(), - }); - } - } - - table_ops -} - pub fn collect_tx_table_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut table_ops = Vec::new(); diff --git a/blocks/antelope/src/transactions.rs b/blocks/antelope/src/transactions.rs index 91521d6..0d85703 100644 --- a/blocks/antelope/src/transactions.rs +++ b/blocks/antelope/src/transactions.rs @@ -23,38 +23,6 @@ pub fn is_transaction_success(status: i32) -> bool { } // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L525 -pub fn collect_transactions(block: &Block, timestamp: &BlockTimestamp) -> Vec { - let header = block.header.clone().unwrap_or_default(); - let mut transactions = Vec::new(); - - for (index, transaction) in block.transaction_traces().enumerate() { - let receipt = transaction.receipt.clone().unwrap_or_default(); - let status_code = receipt.status; - let status = transaction_status_to_string(status_code); - let success = is_transaction_success(status_code); - - transactions.push(RawTransaction { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - hash: transaction.id.clone(), - index: index as u64, - elapsed: transaction.elapsed, - net_usage: transaction.net_usage, - scheduled: transaction.scheduled, - cpu_usage_micro_seconds: receipt.cpu_usage_micro_seconds, - net_usage_words: receipt.net_usage_words, - status, - status_code: status_code as u32, - success, - transaction_mroot: Hex::encode(&header.transaction_mroot.to_vec()), - }); - } - - transactions -} - pub fn collect_transaction(block: &Block, transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> RawTransaction { let header = block.header.clone().unwrap_or_default(); let receipt = transaction.receipt.clone().unwrap_or_default(); From 906d057f9d9078e4178b25462608910ed63321d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dillan=20Choini=C3=A8re?= Date: Tue, 19 Nov 2024 22:14:01 -0500 Subject: [PATCH 14/22] Use Vec::new() instead of vec![] --- blocks/antelope/src/collect_events.rs | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/blocks/antelope/src/collect_events.rs b/blocks/antelope/src/collect_events.rs index d31747f..791f27e 100644 --- a/blocks/antelope/src/collect_events.rs +++ b/blocks/antelope/src/collect_events.rs @@ -21,20 +21,20 @@ use crate::{ pub fn collect_events(block: &Block, timestamp: &BlockTimestamp) -> RawEvents { let mut events = RawEvents { blocks: vec![collect_block(block, timestamp)], - transactions: vec![], - actions: vec![], - db_ops: vec![], - feature_ops: vec![], - perm_ops: vec![], - table_ops: vec![], - accounts: vec![], - keys: vec![], - waits: vec![], - ram_ops: vec![], - authorizations: vec![], - auth_sequences: vec![], - account_ram_deltas: vec![], - creation_trees: vec![], + transactions: Vec::new(), + actions: Vec::new(), + db_ops: Vec::new(), + feature_ops: Vec::new(), + perm_ops: Vec::new(), + table_ops: Vec::new(), + accounts: Vec::new(), + keys: Vec::new(), + waits: Vec::new(), + ram_ops: Vec::new(), + authorizations: Vec::new(), + auth_sequences: Vec::new(), + account_ram_deltas: Vec::new(), + creation_trees: Vec::new(), }; for transaction in block.transaction_traces() { From ae99601e1018476b8cc322d109d97c1d699163f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dillan=20Choini=C3=A8re?= Date: Wed, 20 Nov 2024 16:41:40 -0500 Subject: [PATCH 15/22] Remove Raw prefix from imports --- blocks/antelope/src/account_ram_deltas.rs | 6 +++--- blocks/antelope/src/actions.rs | 8 ++++---- blocks/antelope/src/auth_sequences.rs | 6 +++--- blocks/antelope/src/authority.rs | 20 ++++++++++---------- blocks/antelope/src/authorizations.rs | 6 +++--- blocks/antelope/src/blocks.rs | 6 +++--- blocks/antelope/src/collect_events.rs | 6 +++--- blocks/antelope/src/creation_tree.rs | 6 +++--- blocks/antelope/src/db_ops.rs | 8 ++++---- blocks/antelope/src/feature_ops.rs | 6 +++--- blocks/antelope/src/perm_ops.rs | 7 ++++--- blocks/antelope/src/ram_ops.rs | 6 +++--- blocks/antelope/src/table_ops.rs | 6 +++--- blocks/antelope/src/transactions.rs | 6 +++--- 14 files changed, 52 insertions(+), 51 deletions(-) diff --git a/blocks/antelope/src/account_ram_deltas.rs b/blocks/antelope/src/account_ram_deltas.rs index a24cd1c..b28bdec 100644 --- a/blocks/antelope/src/account_ram_deltas.rs +++ b/blocks/antelope/src/account_ram_deltas.rs @@ -1,15 +1,15 @@ -use crate::pb::antelope::AccountRamDelta as RawAccountRamDelta; +use crate::pb::antelope::AccountRamDelta; use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; -pub fn collect_tx_account_ram_deltas(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { +pub fn collect_tx_account_ram_deltas(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut account_ram_deltas = Vec::new(); for action_trace in transaction.action_traces.iter() { let action_index = action_trace.execution_index; for (index, delta) in action_trace.account_ram_deltas.iter().enumerate() { - account_ram_deltas.push(RawAccountRamDelta { + account_ram_deltas.push(AccountRamDelta { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), diff --git a/blocks/antelope/src/actions.rs b/blocks/antelope/src/actions.rs index 882957d..1d94345 100644 --- a/blocks/antelope/src/actions.rs +++ b/blocks/antelope/src/actions.rs @@ -4,18 +4,18 @@ use substreams::Hex; use substreams_antelope::pb::TransactionTrace; use substreams_antelope::Block; -use crate::pb::antelope::Action as RawAction; +use crate::pb::antelope::Action; // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L525 -pub fn collect_tx_actions(block: &Block, transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { +pub fn collect_tx_actions(block: &Block, transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let header = block.header.clone().unwrap_or_default(); - let mut actions: Vec = Vec::new(); + let mut actions: Vec = Vec::new(); for trace in transaction.action_traces.iter() { let action = trace.action.clone().unwrap_or_default(); let receipt = trace.receipt.clone().unwrap_or_default(); - actions.push(RawAction { + actions.push(Action { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), diff --git a/blocks/antelope/src/auth_sequences.rs b/blocks/antelope/src/auth_sequences.rs index 72f932f..abeb418 100644 --- a/blocks/antelope/src/auth_sequences.rs +++ b/blocks/antelope/src/auth_sequences.rs @@ -1,9 +1,9 @@ use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; -use crate::pb::antelope::AuthSequence as RawAuthSequence; +use crate::pb::antelope::AuthSequence; -pub fn collect_tx_auth_sequences(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { +pub fn collect_tx_auth_sequences(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut auth_sequences = Vec::new(); for action_trace in transaction.action_traces.iter() { @@ -11,7 +11,7 @@ pub fn collect_tx_auth_sequences(transaction: &TransactionTrace, timestamp: &Blo let action_index = action_trace.execution_index; for (index, auth) in receipt.auth_sequence.iter().enumerate() { - auth_sequences.push(RawAuthSequence { + auth_sequences.push(AuthSequence { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), diff --git a/blocks/antelope/src/authority.rs b/blocks/antelope/src/authority.rs index 5b5179e..fa571c2 100644 --- a/blocks/antelope/src/authority.rs +++ b/blocks/antelope/src/authority.rs @@ -1,17 +1,17 @@ -use crate::pb::antelope::{Account as RawAccount, Key as RawKey, Wait as RawWait}; +use crate::pb::antelope::{Account, Key, Wait}; use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; pub struct AuthorityVectors { - pub accounts: Vec, - pub keys: Vec, - pub waits: Vec, + pub accounts: Vec, + pub keys: Vec, + pub waits: Vec, } pub fn collect_tx_authority_vectors(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> AuthorityVectors { - let mut accounts: Vec = Vec::new(); - let mut keys: Vec = Vec::new(); - let mut waits: Vec = Vec::new(); + let mut accounts: Vec = Vec::new(); + let mut keys: Vec = Vec::new(); + let mut waits: Vec = Vec::new(); for perm_op in transaction.perm_ops.iter() { if let Some(new_perm) = &perm_op.new_perm { @@ -21,7 +21,7 @@ pub fn collect_tx_authority_vectors(transaction: &TransactionTrace, timestamp: & // Process authority accounts for (index, account) in authority.accounts.iter().enumerate() { if let Some(permission) = &account.permission { - accounts.push(RawAccount { + accounts.push(Account { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), @@ -39,7 +39,7 @@ pub fn collect_tx_authority_vectors(transaction: &TransactionTrace, timestamp: & // Process authority keys for (index, key) in authority.keys.iter().enumerate() { - keys.push(RawKey { + keys.push(Key { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), @@ -55,7 +55,7 @@ pub fn collect_tx_authority_vectors(transaction: &TransactionTrace, timestamp: & // Process authority waits for (index, wait) in authority.waits.iter().enumerate() { - waits.push(RawWait { + waits.push(Wait { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), diff --git a/blocks/antelope/src/authorizations.rs b/blocks/antelope/src/authorizations.rs index b101aad..ff38256 100644 --- a/blocks/antelope/src/authorizations.rs +++ b/blocks/antelope/src/authorizations.rs @@ -1,15 +1,15 @@ use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; -use crate::pb::antelope::Authorization as RawAuthorization; +use crate::pb::antelope::Authorization; // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L616 -pub fn collect_tx_authorizations(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { +pub fn collect_tx_authorizations(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut authorizations = Vec::new(); for action_trace in transaction.action_traces.iter() { for (index, authorization) in action_trace.action.as_ref().unwrap().authorization.iter().enumerate() { - authorizations.push(RawAuthorization { + authorizations.push(Authorization { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), diff --git a/blocks/antelope/src/blocks.rs b/blocks/antelope/src/blocks.rs index afb6a44..e752d6b 100644 --- a/blocks/antelope/src/blocks.rs +++ b/blocks/antelope/src/blocks.rs @@ -1,15 +1,15 @@ -use crate::{pb::antelope::Block as RawBlock, size::collect_size}; +use crate::{pb::antelope::Block as EventsBlock, size::collect_size}; use common::structs::BlockTimestamp; use substreams::Hex; use substreams_antelope::Block; // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L21 -pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> RawBlock { +pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> EventsBlock { let header = block.header.clone().unwrap_or_default(); let blockroot_merkle = block.blockroot_merkle.clone().unwrap_or_default(); let size = collect_size(block); - RawBlock { + EventsBlock { time: Some(timestamp.time), number: timestamp.number, date: timestamp.date.clone(), diff --git a/blocks/antelope/src/collect_events.rs b/blocks/antelope/src/collect_events.rs index 791f27e..48acf91 100644 --- a/blocks/antelope/src/collect_events.rs +++ b/blocks/antelope/src/collect_events.rs @@ -11,15 +11,15 @@ use crate::{ creation_tree::collect_tx_creation_trees, db_ops::collect_tx_db_ops, feature_ops::collect_tx_feature_ops, - pb::antelope::Events as RawEvents, + pb::antelope::Events, perm_ops::collect_tx_perm_ops, ram_ops::collect_tx_ram_ops, table_ops::collect_tx_table_ops, transactions::{collect_transaction, is_transaction_success}, }; -pub fn collect_events(block: &Block, timestamp: &BlockTimestamp) -> RawEvents { - let mut events = RawEvents { +pub fn collect_events(block: &Block, timestamp: &BlockTimestamp) -> Events { + let mut events = Events { blocks: vec![collect_block(block, timestamp)], transactions: Vec::new(), actions: Vec::new(), diff --git a/blocks/antelope/src/creation_tree.rs b/blocks/antelope/src/creation_tree.rs index 2931db8..3df9d7f 100644 --- a/blocks/antelope/src/creation_tree.rs +++ b/blocks/antelope/src/creation_tree.rs @@ -1,12 +1,12 @@ -use crate::pb::antelope::CreationTree as RawCreationTree; +use crate::pb::antelope::CreationTree; use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; -pub fn collect_tx_creation_trees(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { +pub fn collect_tx_creation_trees(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut creation_trees = Vec::new(); for creation_flat_node in transaction.creation_tree.iter() { - creation_trees.push(RawCreationTree { + creation_trees.push(CreationTree { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), diff --git a/blocks/antelope/src/db_ops.rs b/blocks/antelope/src/db_ops.rs index af2ca5e..e471191 100644 --- a/blocks/antelope/src/db_ops.rs +++ b/blocks/antelope/src/db_ops.rs @@ -2,7 +2,7 @@ use common::structs::BlockTimestamp; use substreams::Hex; use substreams_antelope::pb::TransactionTrace; -use crate::pb::antelope::DbOp as RawDbOp; +use crate::pb::antelope::DbOp; pub fn operation_to_string(operation: i32) -> String { match operation { @@ -15,11 +15,11 @@ pub fn operation_to_string(operation: i32) -> String { } // https://github.com/streamingfast/firehose-ethereum/blob/1bcb32a8eb3e43347972b6b5c9b1fcc4a08c751e/proto/sf/ethereum/type/v2/type.proto#L647 -pub fn collect_tx_db_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { - let mut db_ops: Vec = Vec::new(); +pub fn collect_tx_db_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { + let mut db_ops: Vec = Vec::new(); for (index, db_op) in transaction.db_ops.iter().enumerate() { - db_ops.push(RawDbOp { + db_ops.push(DbOp { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), diff --git a/blocks/antelope/src/feature_ops.rs b/blocks/antelope/src/feature_ops.rs index b263333..8a876da 100644 --- a/blocks/antelope/src/feature_ops.rs +++ b/blocks/antelope/src/feature_ops.rs @@ -1,15 +1,15 @@ use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; -use crate::pb::antelope::FeatureOp as RawFeatureOp; +use crate::pb::antelope::FeatureOp; -pub fn collect_tx_feature_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { +pub fn collect_tx_feature_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut feature_ops = Vec::new(); for feature_op in transaction.feature_ops.iter() { let feature = feature_op.feature.as_ref().expect("feature is required"); - feature_ops.push(RawFeatureOp { + feature_ops.push(FeatureOp { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), diff --git a/blocks/antelope/src/perm_ops.rs b/blocks/antelope/src/perm_ops.rs index f2c1cb9..8e6ba40 100644 --- a/blocks/antelope/src/perm_ops.rs +++ b/blocks/antelope/src/perm_ops.rs @@ -1,7 +1,8 @@ -use crate::pb::antelope::PermOp as RawPermOp; use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; +use crate::pb::antelope::PermOp; + pub fn perm_op_operation_to_string(operation: i32) -> String { match operation { 0 => "Unknown".to_string(), @@ -12,14 +13,14 @@ pub fn perm_op_operation_to_string(operation: i32) -> String { } } -pub fn collect_tx_perm_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { +pub fn collect_tx_perm_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut perm_ops = Vec::new(); for perm_op in transaction.perm_ops.iter() { if let Some(new_perm) = &perm_op.new_perm { let threshold = new_perm.authority.as_ref().map_or(0, |authority| authority.threshold); - perm_ops.push(RawPermOp { + perm_ops.push(PermOp { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), diff --git a/blocks/antelope/src/ram_ops.rs b/blocks/antelope/src/ram_ops.rs index 9b41f4e..a6a46df 100644 --- a/blocks/antelope/src/ram_ops.rs +++ b/blocks/antelope/src/ram_ops.rs @@ -1,7 +1,7 @@ use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; -use crate::pb::antelope::RamOp as RawRamOp; +use crate::pb::antelope::RamOp; pub fn namespace_to_string(namespace: i32) -> String { match namespace { @@ -64,11 +64,11 @@ pub fn operation_to_string(operation: i32) -> String { } } -pub fn collect_tx_ram_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { +pub fn collect_tx_ram_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut ram_ops = Vec::new(); for ram_op in transaction.ram_ops.iter() { - ram_ops.push(RawRamOp { + ram_ops.push(RamOp { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), diff --git a/blocks/antelope/src/table_ops.rs b/blocks/antelope/src/table_ops.rs index aae96f6..cc75e0d 100644 --- a/blocks/antelope/src/table_ops.rs +++ b/blocks/antelope/src/table_ops.rs @@ -1,7 +1,7 @@ use common::structs::BlockTimestamp; use substreams_antelope::pb::TransactionTrace; -use crate::pb::antelope::TableOp as RawTableOp; +use crate::pb::antelope::TableOp; pub fn table_op_operation_to_string(operation: i32) -> String { match operation { @@ -12,11 +12,11 @@ pub fn table_op_operation_to_string(operation: i32) -> String { } } -pub fn collect_tx_table_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { +pub fn collect_tx_table_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut table_ops = Vec::new(); for (index, table_op) in transaction.table_ops.iter().enumerate() { - table_ops.push(RawTableOp { + table_ops.push(TableOp { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), diff --git a/blocks/antelope/src/transactions.rs b/blocks/antelope/src/transactions.rs index 0d85703..1637a4e 100644 --- a/blocks/antelope/src/transactions.rs +++ b/blocks/antelope/src/transactions.rs @@ -2,7 +2,7 @@ use common::structs::BlockTimestamp; use substreams::Hex; use substreams_antelope::{pb::TransactionTrace, Block}; -use crate::pb::antelope::Transaction as RawTransaction; +use crate::pb::antelope::Transaction; pub fn transaction_status_to_string(status: i32) -> String { match status { @@ -23,13 +23,13 @@ pub fn is_transaction_success(status: i32) -> bool { } // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L525 -pub fn collect_transaction(block: &Block, transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> RawTransaction { +pub fn collect_transaction(block: &Block, transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Transaction { let header = block.header.clone().unwrap_or_default(); let receipt = transaction.receipt.clone().unwrap_or_default(); let status_code = receipt.status; let status = transaction_status_to_string(status_code); - RawTransaction { + Transaction { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), From 15f26caa888bceb5fff8dd8e495c9a6c6e94b429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dillan=20Choini=C3=A8re?= Date: Wed, 20 Nov 2024 16:44:16 -0500 Subject: [PATCH 16/22] Put collect_events content in map_events --- blocks/antelope/src/collect_events.rs | 60 ------------------------- blocks/antelope/src/events.rs | 63 +++++++++++++++++++++++++++ blocks/antelope/src/lib.rs | 3 +- blocks/antelope/src/map_events.rs | 13 ------ 4 files changed, 64 insertions(+), 75 deletions(-) delete mode 100644 blocks/antelope/src/collect_events.rs create mode 100644 blocks/antelope/src/events.rs delete mode 100644 blocks/antelope/src/map_events.rs diff --git a/blocks/antelope/src/collect_events.rs b/blocks/antelope/src/collect_events.rs deleted file mode 100644 index 48acf91..0000000 --- a/blocks/antelope/src/collect_events.rs +++ /dev/null @@ -1,60 +0,0 @@ -use common::structs::BlockTimestamp; -use substreams_antelope::Block; - -use crate::{ - account_ram_deltas::collect_tx_account_ram_deltas, - actions::collect_tx_actions, - auth_sequences::collect_tx_auth_sequences, - authority::collect_tx_authority_vectors, - authorizations::collect_tx_authorizations, - blocks::collect_block, - creation_tree::collect_tx_creation_trees, - db_ops::collect_tx_db_ops, - feature_ops::collect_tx_feature_ops, - pb::antelope::Events, - perm_ops::collect_tx_perm_ops, - ram_ops::collect_tx_ram_ops, - table_ops::collect_tx_table_ops, - transactions::{collect_transaction, is_transaction_success}, -}; - -pub fn collect_events(block: &Block, timestamp: &BlockTimestamp) -> Events { - let mut events = Events { - blocks: vec![collect_block(block, timestamp)], - transactions: Vec::new(), - actions: Vec::new(), - db_ops: Vec::new(), - feature_ops: Vec::new(), - perm_ops: Vec::new(), - table_ops: Vec::new(), - accounts: Vec::new(), - keys: Vec::new(), - waits: Vec::new(), - ram_ops: Vec::new(), - authorizations: Vec::new(), - auth_sequences: Vec::new(), - account_ram_deltas: Vec::new(), - creation_trees: Vec::new(), - }; - - for transaction in block.transaction_traces() { - let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); - events.transactions.push(collect_transaction(block, transaction, timestamp, tx_success)); - events.actions.extend(collect_tx_actions(block, transaction, timestamp, tx_success)); - events.db_ops.extend(collect_tx_db_ops(transaction, timestamp, tx_success)); - events.feature_ops.extend(collect_tx_feature_ops(transaction, timestamp, tx_success)); - events.perm_ops.extend(collect_tx_perm_ops(transaction, timestamp, tx_success)); - events.table_ops.extend(collect_tx_table_ops(transaction, timestamp, tx_success)); - let authority_vectors = collect_tx_authority_vectors(transaction, timestamp, tx_success); - events.accounts.extend(authority_vectors.accounts); - events.keys.extend(authority_vectors.keys); - events.waits.extend(authority_vectors.waits); - events.ram_ops.extend(collect_tx_ram_ops(transaction, timestamp, tx_success)); - events.authorizations.extend(collect_tx_authorizations(transaction, timestamp, tx_success)); - events.auth_sequences.extend(collect_tx_auth_sequences(transaction, timestamp, tx_success)); - events.account_ram_deltas.extend(collect_tx_account_ram_deltas(transaction, timestamp, tx_success)); - events.creation_trees.extend(collect_tx_creation_trees(transaction, timestamp, tx_success)); - } - - events -} diff --git a/blocks/antelope/src/events.rs b/blocks/antelope/src/events.rs new file mode 100644 index 0000000..60e43f4 --- /dev/null +++ b/blocks/antelope/src/events.rs @@ -0,0 +1,63 @@ +use common::utils::build_timestamp; +use substreams::errors::Error; +use substreams::pb::substreams::Clock; +use substreams_antelope::pb::Block; + +use crate::account_ram_deltas::collect_tx_account_ram_deltas; +use crate::actions::collect_tx_actions; +use crate::auth_sequences::collect_tx_auth_sequences; +use crate::authority::collect_tx_authority_vectors; +use crate::authorizations::collect_tx_authorizations; +use crate::blocks::collect_block; +use crate::creation_tree::collect_tx_creation_trees; +use crate::db_ops::collect_tx_db_ops; +use crate::feature_ops::collect_tx_feature_ops; +use crate::pb::antelope::Events; +use crate::perm_ops::collect_tx_perm_ops; +use crate::ram_ops::collect_tx_ram_ops; +use crate::table_ops::collect_tx_table_ops; +use crate::transactions::{collect_transaction, is_transaction_success}; + +#[substreams::handlers::map] +pub fn map_events(clock: Clock, block: Block) -> Result { + let timestamp = build_timestamp(&clock); + + let mut events = Events { + blocks: vec![collect_block(&block, ×tamp)], + transactions: Vec::new(), + actions: Vec::new(), + db_ops: Vec::new(), + feature_ops: Vec::new(), + perm_ops: Vec::new(), + table_ops: Vec::new(), + accounts: Vec::new(), + keys: Vec::new(), + waits: Vec::new(), + ram_ops: Vec::new(), + authorizations: Vec::new(), + auth_sequences: Vec::new(), + account_ram_deltas: Vec::new(), + creation_trees: Vec::new(), + }; + + for transaction in block.transaction_traces() { + let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); + events.transactions.push(collect_transaction(&block, transaction, ×tamp, tx_success)); + events.actions.extend(collect_tx_actions(&block, transaction, ×tamp, tx_success)); + events.db_ops.extend(collect_tx_db_ops(transaction, ×tamp, tx_success)); + events.feature_ops.extend(collect_tx_feature_ops(transaction, ×tamp, tx_success)); + events.perm_ops.extend(collect_tx_perm_ops(transaction, ×tamp, tx_success)); + events.table_ops.extend(collect_tx_table_ops(transaction, ×tamp, tx_success)); + let authority_vectors = collect_tx_authority_vectors(transaction, ×tamp, tx_success); + events.accounts.extend(authority_vectors.accounts); + events.keys.extend(authority_vectors.keys); + events.waits.extend(authority_vectors.waits); + events.ram_ops.extend(collect_tx_ram_ops(transaction, ×tamp, tx_success)); + events.authorizations.extend(collect_tx_authorizations(transaction, ×tamp, tx_success)); + events.auth_sequences.extend(collect_tx_auth_sequences(transaction, ×tamp, tx_success)); + events.account_ram_deltas.extend(collect_tx_account_ram_deltas(transaction, ×tamp, tx_success)); + events.creation_trees.extend(collect_tx_creation_trees(transaction, ×tamp, tx_success)); + } + + Ok(events) +} diff --git a/blocks/antelope/src/lib.rs b/blocks/antelope/src/lib.rs index 097d0fe..d6b9e8d 100644 --- a/blocks/antelope/src/lib.rs +++ b/blocks/antelope/src/lib.rs @@ -4,11 +4,10 @@ mod auth_sequences; mod authority; mod authorizations; mod blocks; -mod collect_events; mod creation_tree; mod db_ops; +mod events; mod feature_ops; -mod map_events; mod pb; mod perm_ops; mod ram_ops; diff --git a/blocks/antelope/src/map_events.rs b/blocks/antelope/src/map_events.rs deleted file mode 100644 index 688b564..0000000 --- a/blocks/antelope/src/map_events.rs +++ /dev/null @@ -1,13 +0,0 @@ -use common::utils::build_timestamp; -use substreams::errors::Error; -use substreams::pb::substreams::Clock; -use substreams_antelope::pb::Block; - -use crate::collect_events::collect_events; -use crate::pb::antelope::Events; - -#[substreams::handlers::map] -pub fn map_events(clock: Clock, block: Block) -> Result { - let timestamp = build_timestamp(&clock); - Ok(collect_events(&block, ×tamp)) -} From f4cadab591361d246e666506a80c0cd0a75f9c92 Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Thu, 21 Nov 2024 13:27:59 +0100 Subject: [PATCH 17/22] remove extra solana proto --- blocks/solana/Cargo.toml | 1 - blocks/solana/proto/solana.rawblocks.proto | 154 ------------------ .../solana-example-queries.md | 0 3 files changed, 155 deletions(-) delete mode 100644 blocks/solana/proto/solana.rawblocks.proto rename blocks/solana/docs/example-queries.md => docs/solana-example-queries.md (100%) diff --git a/blocks/solana/Cargo.toml b/blocks/solana/Cargo.toml index 44cdaad..7a29ecc 100644 --- a/blocks/solana/Cargo.toml +++ b/blocks/solana/Cargo.toml @@ -9,7 +9,6 @@ crate-type = ["cdylib"] [dependencies] common = { path = "../../common" } substreams-solana = { workspace = true } -substreams-database-change = { workspace = true } substreams = { workspace = true } serde_json = { workspace = true } prost = { workspace = true } diff --git a/blocks/solana/proto/solana.rawblocks.proto b/blocks/solana/proto/solana.rawblocks.proto deleted file mode 100644 index b41468b..0000000 --- a/blocks/solana/proto/solana.rawblocks.proto +++ /dev/null @@ -1,154 +0,0 @@ -syntax = "proto3"; - -package solana.rawblocks; - -import "google/protobuf/timestamp.proto"; - -message Events { - repeated Block blocks = 1; - repeated Reward rewards = 2; - repeated Transaction transactions = 3; - repeated InstructionCall instruction_calls = 4; - repeated AccountActivity account_activity = 5; - repeated Transaction vote_transactions = 6; - repeated InstructionCall vote_instruction_calls = 7; - repeated AccountActivity vote_account_activity = 8; -} - -message Block { - // clock - google.protobuf.Timestamp time = 1; - string date = 2; - string hash = 3; - - // block - uint64 slot = 4; - uint64 height = 5; - string previous_block_hash = 6; - uint64 parent_slot = 7; - - // counters - uint64 total_transactions = 8; - uint64 successful_transactions = 9; - uint64 failed_transactions = 10; - uint64 total_vote_transactions = 11; - uint64 total_non_vote_transactions = 12; - uint64 successful_vote_transactions = 13; - uint64 successful_non_vote_transactions = 14; - uint64 failed_vote_transactions = 15; - uint64 failed_non_vote_transactions = 16; -} - -message Reward { - // clock - google.protobuf.Timestamp block_time = 1; - string block_date = 2; - string block_hash = 3; - - // block - uint64 block_slot = 4; - uint64 block_height = 5; - string block_previous_block_hash = 6; - uint64 block_parent_slot = 7; - - // reward - string pubkey = 8; - int64 lamports = 9; - uint64 pre_balance = 10; - uint64 post_balance = 11; - string reward_type = 12; - string commission = 13; -} - -message Transaction { - // clock - google.protobuf.Timestamp block_time = 1; - string block_hash = 2; - string block_date = 3; - - // block - uint64 block_slot = 4; - uint64 block_height = 5; - string block_previous_block_hash = 6; - uint64 block_parent_slot = 7; - - // transaction - string id = 8; - uint32 index = 9; - uint64 fee = 10; - uint32 required_signatures = 11; - uint32 required_signed_accounts = 12; - uint32 required_unsigned_accounts = 13; - string signature = 14; - bool success = 15; - string error = 16; - string recent_block_hash = 17; - string account_keys = 18; - string log_messages = 19; - string pre_balances = 20; - string post_balances = 21; - string signatures = 22; - string signer = 23; - string signers = 24; -} - -message InstructionCall { - // clock - google.protobuf.Timestamp block_time = 1; - string block_hash = 2; - string block_date = 3; - - // block - uint64 block_slot = 4; - uint64 block_height = 5; - string block_previous_block_hash = 6; - uint64 block_parent_slot = 7; - - // transaction - string tx_id = 8; - uint32 tx_index = 9; - string tx_signer = 10; - bool tx_success = 11; - string log_messages = 12; - - // instruction - uint32 outer_instruction_index = 13; - int32 inner_instruction_index = 14; - string inner_executing_account = 15; - string outer_executing_account = 16; - string executing_account = 17; - bool is_inner = 18; - string data = 19; - string account_arguments = 20; - // string representing a nested array. Switch to nested array type when supported by sink-files - string inner_instructions = 21; -} - -message AccountActivity { - // clock - google.protobuf.Timestamp block_time = 1; - string block_hash = 2; - string block_date = 3; - - // block - uint64 block_slot = 4; - uint64 block_height = 5; - string block_previous_block_hash = 6; - uint64 block_parent_slot = 7; - - string address = 8; - uint32 tx_index = 9; - string tx_id = 10; - bool tx_success = 11; - bool signed = 12; - bool writable = 13; - optional string token_mint_address = 14; - - uint64 pre_balance = 15; - uint64 post_balance = 16; - int64 balance_change = 17; - optional double pre_token_balance = 18; - optional double post_token_balance = 19; - optional double token_balance_change = 20; - optional string token_balance_owner = 21; -} diff --git a/blocks/solana/docs/example-queries.md b/docs/solana-example-queries.md similarity index 100% rename from blocks/solana/docs/example-queries.md rename to docs/solana-example-queries.md From 21e41d6877966f7eb684841102372cf48aa4e896 Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Thu, 21 Nov 2024 14:41:13 +0100 Subject: [PATCH 18/22] replaces tables with repeated --- blocks/antelope/Cargo.toml | 1 - blocks/antelope/Makefile | 45 +--- blocks/antelope/src/account_ram_deltas.rs | 28 -- blocks/antelope/src/actions.rs | 23 ++ blocks/antelope/src/auth_sequences.rs | 30 --- blocks/antelope/src/authority.rs | 75 ------ blocks/antelope/src/authorizations.rs | 28 -- blocks/antelope/src/blocks.rs | 15 +- blocks/antelope/src/creation_tree.rs | 22 -- blocks/antelope/src/db_ops.rs | 5 + blocks/antelope/src/events.rs | 33 +-- blocks/antelope/src/feature_ops.rs | 5 + blocks/antelope/src/lib.rs | 5 - blocks/antelope/src/pb/antelope.rs | 310 +++++++--------------- blocks/antelope/src/perm_ops.rs | 44 ++- blocks/antelope/src/ram_ops.rs | 9 +- blocks/antelope/src/size.rs | 2 +- blocks/antelope/src/transactions.rs | 6 + blocks/antelope/substreams.yaml | 6 - proto/antelope.proto | 218 +++++++-------- 20 files changed, 298 insertions(+), 612 deletions(-) delete mode 100644 blocks/antelope/src/account_ram_deltas.rs delete mode 100644 blocks/antelope/src/auth_sequences.rs delete mode 100644 blocks/antelope/src/authority.rs delete mode 100644 blocks/antelope/src/authorizations.rs delete mode 100644 blocks/antelope/src/creation_tree.rs diff --git a/blocks/antelope/Cargo.toml b/blocks/antelope/Cargo.toml index 37b09e9..8c1e484 100644 --- a/blocks/antelope/Cargo.toml +++ b/blocks/antelope/Cargo.toml @@ -9,7 +9,6 @@ crate-type = ["cdylib"] [dependencies] common = { path = "../../common" } substreams-antelope = { workspace = true } -substreams-database-change = { workspace = true } substreams = { workspace = true } prost = { workspace = true } prost-types = { workspace = true } diff --git a/blocks/antelope/Makefile b/blocks/antelope/Makefile index 99db40d..e5fac55 100644 --- a/blocks/antelope/Makefile +++ b/blocks/antelope/Makefile @@ -1,63 +1,30 @@ .PHONY: all all: make build - make pack - make graph - make info .PHONY: build build: cargo build --target wasm32-unknown-unknown --release - -.PHONY: pack -pack: substreams pack - -.PHONY: graph -graph: - substreams graph - -.PHONY: info -info: substreams info - -.PHONY: run -run: - substreams run -e eos.substreams.pinax.network:443 ch_out -s 386946801 -t 386946802 -o json + substreams graph .PHONY: gui gui: - substreams gui -e eos.substreams.pinax.network:443 ch_out -s 386946801 -t 386946802 + substreams gui -e eos.substreams.pinax.network:443 map_events -s 386946801 -t 386946802 .PHONY: protogen protogen: - substreams protogen --exclude-paths google,sf + substreams protogen -.PHONY: parquet-eos -parquet-eos: - substreams-sink-files run eos.substreams.pinax.network:443 substreams.yaml map_events './out' 386841287:386841787 --encoder parquet --file-block-count 100 --development-mode +.PHONY: parquet +parquet: + substreams-sink-files run eos.substreams.pinax.network:443 substreams.yaml map_events './out' 386841287:386841787 --encoder parquet --file-block-count 100 --development-mode .PHONY: parquet-wax parquet-wax: substreams-sink-files run wax.substreams.pinax.network:443 substreams.yaml map_events './out' 32233917:32234417 --encoder parquet --file-block-count 100 --development-mode -# Antelope blocks -.PHONY: sql-run-eos -sql-run-eos: - substreams-sink-sql run clickhouse://default:default@localhost:9000/eos substreams.yaml -e eos.substreams.pinax.network:443 386841287:387014085 --final-blocks-only --undo-buffer-size 1 --on-module-hash-mistmatch=warn --batch-block-flush-interval 1 --development-mode - -.PHONY: sql-run-eos-test -sql-run-eos-test: - substreams-sink-sql run clickhouse://default:default@localhost:9000/eos substreams.yaml -e eos.substreams.pinax.network:443 386946800:386946810 --final-blocks-only --undo-buffer-size 1 --on-module-hash-mistmatch=warn --batch-block-flush-interval 1 --development-mode - -.PHONY: sql-run-eos-blocks -sql-run-eos-blocks: - substreams-sink-sql run clickhouse://default:default@localhost:9000/eos substreams.yaml -e eos.substreams.pinax.network:443 2: --final-blocks-only --undo-buffer-size 1 --on-module-hash-mistmatch=warn --batch-block-flush-interval 100 --params ch_out=blocks - -.PHONY: sql-run-wax -sql-run-wax: - substreams-sink-sql run clickhouse://default:default@localhost:9000/wax substreams.yaml -e wax.substreams.pinax.network:443 322339177:322511977 --final-blocks-only --undo-buffer-size 1 --on-module-hash-mistmatch=warn --batch-block-flush-interval 100 --development-mode - .PHONY: deploy deploy: graph build diff --git a/blocks/antelope/src/account_ram_deltas.rs b/blocks/antelope/src/account_ram_deltas.rs deleted file mode 100644 index b28bdec..0000000 --- a/blocks/antelope/src/account_ram_deltas.rs +++ /dev/null @@ -1,28 +0,0 @@ -use crate::pb::antelope::AccountRamDelta; -use common::structs::BlockTimestamp; -use substreams_antelope::pb::TransactionTrace; - -pub fn collect_tx_account_ram_deltas(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { - let mut account_ram_deltas = Vec::new(); - - for action_trace in transaction.action_traces.iter() { - let action_index = action_trace.execution_index; - - for (index, delta) in action_trace.account_ram_deltas.iter().enumerate() { - account_ram_deltas.push(AccountRamDelta { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: transaction.id.clone(), - tx_success, - action_index, - index: index as u32, - account: delta.account.clone(), - delta: delta.delta, - }); - } - } - - account_ram_deltas -} diff --git a/blocks/antelope/src/actions.rs b/blocks/antelope/src/actions.rs index 1d94345..dbfd238 100644 --- a/blocks/antelope/src/actions.rs +++ b/blocks/antelope/src/actions.rs @@ -15,19 +15,42 @@ pub fn collect_tx_actions(block: &Block, transaction: &TransactionTrace, timesta let action = trace.action.clone().unwrap_or_default(); let receipt = trace.receipt.clone().unwrap_or_default(); + // auth sequence + let auth_sequence = receipt.auth_sequence.iter().map(|seq| seq.sequence).collect::>(); + let auth_sequence_account_name = receipt.auth_sequence.iter().map(|seq| seq.clone().account_name).collect::>(); + + // ram deltas + let account_ram_deltas = trace.account_ram_deltas.iter().map(|account_ram| account_ram.delta).collect::>(); + let account_ram_deltas_account = trace.account_ram_deltas.iter().map(|account_ram| account_ram.clone().account).collect::>(); + actions.push(Action { + // block block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), block_date: timestamp.date.clone(), + + // tranasction tx_hash: transaction.id.clone(), tx_success, + + // receipt abi_sequence: receipt.abi_sequence, code_sequence: receipt.code_sequence, digest: receipt.digest, global_sequence: receipt.global_sequence, receipt_receiver: receipt.receiver, recv_sequence: receipt.recv_sequence, + + // auth sequence + auth_sequence, + auth_sequence_account_name, + + // account ram deltas + account_ram_deltas, + account_ram_deltas_account, + + // action account: action.account, name: action.name, json_data: action.json_data, diff --git a/blocks/antelope/src/auth_sequences.rs b/blocks/antelope/src/auth_sequences.rs deleted file mode 100644 index abeb418..0000000 --- a/blocks/antelope/src/auth_sequences.rs +++ /dev/null @@ -1,30 +0,0 @@ -use common::structs::BlockTimestamp; -use substreams_antelope::pb::TransactionTrace; - -use crate::pb::antelope::AuthSequence; - -pub fn collect_tx_auth_sequences(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { - let mut auth_sequences = Vec::new(); - - for action_trace in transaction.action_traces.iter() { - let receipt = action_trace.receipt.as_ref().expect("Action trace receipt is required"); - let action_index = action_trace.execution_index; - - for (index, auth) in receipt.auth_sequence.iter().enumerate() { - auth_sequences.push(AuthSequence { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: transaction.id.clone(), - tx_success, - action_index, - index: index as u32, - account_name: auth.account_name.clone(), - sequence: auth.sequence, - }); - } - } - - auth_sequences -} diff --git a/blocks/antelope/src/authority.rs b/blocks/antelope/src/authority.rs deleted file mode 100644 index fa571c2..0000000 --- a/blocks/antelope/src/authority.rs +++ /dev/null @@ -1,75 +0,0 @@ -use crate::pb::antelope::{Account, Key, Wait}; -use common::structs::BlockTimestamp; -use substreams_antelope::pb::TransactionTrace; - -pub struct AuthorityVectors { - pub accounts: Vec, - pub keys: Vec, - pub waits: Vec, -} - -pub fn collect_tx_authority_vectors(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> AuthorityVectors { - let mut accounts: Vec = Vec::new(); - let mut keys: Vec = Vec::new(); - let mut waits: Vec = Vec::new(); - - for perm_op in transaction.perm_ops.iter() { - if let Some(new_perm) = &perm_op.new_perm { - let authority = new_perm.authority.as_ref().unwrap(); - let action_index = perm_op.action_index; - - // Process authority accounts - for (index, account) in authority.accounts.iter().enumerate() { - if let Some(permission) = &account.permission { - accounts.push(Account { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: transaction.id.clone(), - tx_success, - index: index as u32, - action_index: action_index as u32, - actor: permission.actor.clone(), - permission: permission.permission.clone(), - weight: account.weight, - }); - } - } - - // Process authority keys - for (index, key) in authority.keys.iter().enumerate() { - keys.push(Key { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: transaction.id.clone(), - tx_success, - index: index as u32, - action_index: action_index as u32, - public_key: key.public_key.clone(), - weight: key.weight, - }); - } - - // Process authority waits - for (index, wait) in authority.waits.iter().enumerate() { - waits.push(Wait { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: transaction.id.clone(), - tx_success, - index: index as u32, - action_index: action_index as u32, - wait_sec: wait.wait_sec, - weight: wait.weight, - }); - } - } - } - - AuthorityVectors { accounts, keys, waits } -} diff --git a/blocks/antelope/src/authorizations.rs b/blocks/antelope/src/authorizations.rs deleted file mode 100644 index ff38256..0000000 --- a/blocks/antelope/src/authorizations.rs +++ /dev/null @@ -1,28 +0,0 @@ -use common::structs::BlockTimestamp; -use substreams_antelope::pb::TransactionTrace; - -use crate::pb::antelope::Authorization; - -// https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L616 -pub fn collect_tx_authorizations(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { - let mut authorizations = Vec::new(); - - for action_trace in transaction.action_traces.iter() { - for (index, authorization) in action_trace.action.as_ref().unwrap().authorization.iter().enumerate() { - authorizations.push(Authorization { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: transaction.id.clone(), - tx_success, - action_index: action_trace.execution_index, - index: index as u32, - actor: authorization.actor.clone(), - permission: authorization.permission.clone(), - }); - } - } - - authorizations -} diff --git a/blocks/antelope/src/blocks.rs b/blocks/antelope/src/blocks.rs index e752d6b..12d3a35 100644 --- a/blocks/antelope/src/blocks.rs +++ b/blocks/antelope/src/blocks.rs @@ -1,21 +1,24 @@ -use crate::{pb::antelope::Block as EventsBlock, size::collect_size}; +use crate::{pb::antelope::Block as EventsBlock, size::compute_block_size}; use common::structs::BlockTimestamp; use substreams::Hex; use substreams_antelope::Block; // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L21 pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> EventsBlock { - let header = block.header.clone().unwrap_or_default(); + let header = block.header.as_ref().expect("missing block header"); let blockroot_merkle = block.blockroot_merkle.clone().unwrap_or_default(); - let size = collect_size(block); + let size = compute_block_size(block); EventsBlock { + // clock time: Some(timestamp.time), number: timestamp.number, date: timestamp.date.clone(), hash: timestamp.hash.clone(), - parent_hash: header.previous, - producer: header.producer, + + // block + parent_hash: header.previous.clone(), + producer: header.producer.clone(), confirmed: header.confirmed, schedule_version: header.schedule_version, version: block.version, @@ -26,6 +29,8 @@ pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> EventsBlock { action_mroot: Hex::encode(&header.action_mroot.to_vec()), blockroot_merkle_active_nodes: blockroot_merkle.active_nodes.iter().map(|row| Hex::encode(row)).collect::>(), blockroot_merkle_node_count: blockroot_merkle.node_count, + + // counters size: size.size, total_transactions: size.total_transactions, successful_transactions: size.successful_transactions, diff --git a/blocks/antelope/src/creation_tree.rs b/blocks/antelope/src/creation_tree.rs deleted file mode 100644 index 3df9d7f..0000000 --- a/blocks/antelope/src/creation_tree.rs +++ /dev/null @@ -1,22 +0,0 @@ -use crate::pb::antelope::CreationTree; -use common::structs::BlockTimestamp; -use substreams_antelope::pb::TransactionTrace; - -pub fn collect_tx_creation_trees(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { - let mut creation_trees = Vec::new(); - - for creation_flat_node in transaction.creation_tree.iter() { - creation_trees.push(CreationTree { - block_time: Some(timestamp.time.clone()), - block_number: timestamp.number, - block_hash: timestamp.hash.clone(), - block_date: timestamp.date.clone(), - tx_hash: transaction.id.clone(), - tx_success, - creator_action_index: creation_flat_node.creator_action_index, - execution_action_index: creation_flat_node.execution_action_index, - }); - } - - creation_trees -} diff --git a/blocks/antelope/src/db_ops.rs b/blocks/antelope/src/db_ops.rs index e471191..de3e3fd 100644 --- a/blocks/antelope/src/db_ops.rs +++ b/blocks/antelope/src/db_ops.rs @@ -20,12 +20,17 @@ pub fn collect_tx_db_ops(transaction: &TransactionTrace, timestamp: &BlockTimest for (index, db_op) in transaction.db_ops.iter().enumerate() { db_ops.push(DbOp { + // block block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), block_date: timestamp.date.clone(), + + // transaction tx_hash: transaction.id.clone(), tx_success, + + // database operation index: index as u32, operation: operation_to_string(db_op.operation), operation_code: db_op.operation as u32, diff --git a/blocks/antelope/src/events.rs b/blocks/antelope/src/events.rs index 60e43f4..844e8ca 100644 --- a/blocks/antelope/src/events.rs +++ b/blocks/antelope/src/events.rs @@ -3,13 +3,8 @@ use substreams::errors::Error; use substreams::pb::substreams::Clock; use substreams_antelope::pb::Block; -use crate::account_ram_deltas::collect_tx_account_ram_deltas; use crate::actions::collect_tx_actions; -use crate::auth_sequences::collect_tx_auth_sequences; -use crate::authority::collect_tx_authority_vectors; -use crate::authorizations::collect_tx_authorizations; use crate::blocks::collect_block; -use crate::creation_tree::collect_tx_creation_trees; use crate::db_ops::collect_tx_db_ops; use crate::feature_ops::collect_tx_feature_ops; use crate::pb::antelope::Events; @@ -21,24 +16,8 @@ use crate::transactions::{collect_transaction, is_transaction_success}; #[substreams::handlers::map] pub fn map_events(clock: Clock, block: Block) -> Result { let timestamp = build_timestamp(&clock); - - let mut events = Events { - blocks: vec![collect_block(&block, ×tamp)], - transactions: Vec::new(), - actions: Vec::new(), - db_ops: Vec::new(), - feature_ops: Vec::new(), - perm_ops: Vec::new(), - table_ops: Vec::new(), - accounts: Vec::new(), - keys: Vec::new(), - waits: Vec::new(), - ram_ops: Vec::new(), - authorizations: Vec::new(), - auth_sequences: Vec::new(), - account_ram_deltas: Vec::new(), - creation_trees: Vec::new(), - }; + let mut events = Events::default(); + events.blocks.push(collect_block(&block, ×tamp)); for transaction in block.transaction_traces() { let tx_success = is_transaction_success(transaction.receipt.clone().unwrap_or_default().status); @@ -48,15 +27,7 @@ pub fn map_events(clock: Clock, block: Block) -> Result { events.feature_ops.extend(collect_tx_feature_ops(transaction, ×tamp, tx_success)); events.perm_ops.extend(collect_tx_perm_ops(transaction, ×tamp, tx_success)); events.table_ops.extend(collect_tx_table_ops(transaction, ×tamp, tx_success)); - let authority_vectors = collect_tx_authority_vectors(transaction, ×tamp, tx_success); - events.accounts.extend(authority_vectors.accounts); - events.keys.extend(authority_vectors.keys); - events.waits.extend(authority_vectors.waits); events.ram_ops.extend(collect_tx_ram_ops(transaction, ×tamp, tx_success)); - events.authorizations.extend(collect_tx_authorizations(transaction, ×tamp, tx_success)); - events.auth_sequences.extend(collect_tx_auth_sequences(transaction, ×tamp, tx_success)); - events.account_ram_deltas.extend(collect_tx_account_ram_deltas(transaction, ×tamp, tx_success)); - events.creation_trees.extend(collect_tx_creation_trees(transaction, ×tamp, tx_success)); } Ok(events) diff --git a/blocks/antelope/src/feature_ops.rs b/blocks/antelope/src/feature_ops.rs index 8a876da..6432232 100644 --- a/blocks/antelope/src/feature_ops.rs +++ b/blocks/antelope/src/feature_ops.rs @@ -10,12 +10,17 @@ pub fn collect_tx_feature_ops(transaction: &TransactionTrace, timestamp: &BlockT let feature = feature_op.feature.as_ref().expect("feature is required"); feature_ops.push(FeatureOp { + // block block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), block_date: timestamp.date.clone(), + + // transaction tx_hash: transaction.id.clone(), tx_success, + + // feature operation feature_digest: feature_op.feature_digest.clone(), kind: feature_op.kind.clone(), action_index: feature_op.action_index, diff --git a/blocks/antelope/src/lib.rs b/blocks/antelope/src/lib.rs index d6b9e8d..f068f6f 100644 --- a/blocks/antelope/src/lib.rs +++ b/blocks/antelope/src/lib.rs @@ -1,10 +1,5 @@ -mod account_ram_deltas; mod actions; -mod auth_sequences; -mod authority; -mod authorizations; mod blocks; -mod creation_tree; mod db_ops; mod events; mod feature_ops; diff --git a/blocks/antelope/src/pb/antelope.rs b/blocks/antelope/src/pb/antelope.rs index 718792b..1c7328e 100644 --- a/blocks/antelope/src/pb/antelope.rs +++ b/blocks/antelope/src/pb/antelope.rs @@ -1,8 +1,28 @@ // @generated -// This file is @generated by prost-build. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Events { + #[prost(message, repeated, tag="1")] + pub blocks: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="2")] + pub transactions: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="3")] + pub actions: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="4")] + pub db_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="5")] + pub feature_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="6")] + pub perm_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="7")] + pub table_ops: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="8")] + pub ram_ops: ::prost::alloc::vec::Vec, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Block { + /// clock #[prost(message, optional, tag="1")] pub time: ::core::option::Option<::prost_types::Timestamp>, #[prost(uint64, tag="2")] @@ -11,6 +31,7 @@ pub struct Block { pub date: ::prost::alloc::string::String, #[prost(string, tag="4")] pub hash: ::prost::alloc::string::String, + /// block #[prost(string, tag="5")] pub parent_hash: ::prost::alloc::string::String, #[prost(string, tag="6")] @@ -35,6 +56,7 @@ pub struct Block { pub blockroot_merkle_active_nodes: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, #[prost(uint32, tag="16")] pub blockroot_merkle_node_count: u32, + /// counters #[prost(uint64, tag="17")] pub size: u64, #[prost(uint64, tag="18")] @@ -51,6 +73,7 @@ pub struct Block { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Transaction { + /// block #[prost(message, optional, tag="1")] pub block_time: ::core::option::Option<::prost_types::Timestamp>, #[prost(uint64, tag="2")] @@ -59,6 +82,7 @@ pub struct Transaction { pub block_hash: ::prost::alloc::string::String, #[prost(string, tag="4")] pub block_date: ::prost::alloc::string::String, + /// transaction #[prost(string, tag="5")] pub hash: ::prost::alloc::string::String, #[prost(uint64, tag="6")] @@ -81,10 +105,16 @@ pub struct Transaction { pub success: bool, #[prost(string, tag="15")] pub transaction_mroot: ::prost::alloc::string::String, + /// creation flat node + #[prost(int32, repeated, tag="16")] + pub creator_action_indexes: ::prost::alloc::vec::Vec, + #[prost(uint32, repeated, tag="17")] + pub execution_action_indexes: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Action { + /// block #[prost(message, optional, tag="1")] pub block_time: ::core::option::Option<::prost_types::Timestamp>, #[prost(uint64, tag="2")] @@ -93,10 +123,12 @@ pub struct Action { pub block_hash: ::prost::alloc::string::String, #[prost(string, tag="4")] pub block_date: ::prost::alloc::string::String, + /// transaction #[prost(string, tag="5")] pub tx_hash: ::prost::alloc::string::String, #[prost(bool, tag="6")] pub tx_success: bool, + /// receipt #[prost(uint64, tag="7")] pub abi_sequence: u64, #[prost(uint64, tag="8")] @@ -109,40 +141,51 @@ pub struct Action { pub receipt_receiver: ::prost::alloc::string::String, #[prost(uint64, tag="12")] pub recv_sequence: u64, - #[prost(string, tag="13")] + #[prost(uint64, repeated, tag="13")] + pub auth_sequence: ::prost::alloc::vec::Vec, + #[prost(string, repeated, tag="14")] + pub auth_sequence_account_name: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// action + #[prost(string, tag="20")] pub account: ::prost::alloc::string::String, - #[prost(string, tag="14")] + #[prost(string, tag="21")] pub name: ::prost::alloc::string::String, - #[prost(string, tag="15")] + #[prost(string, tag="22")] pub json_data: ::prost::alloc::string::String, - #[prost(string, tag="16")] + #[prost(string, tag="23")] pub raw_data: ::prost::alloc::string::String, - #[prost(uint32, tag="17")] + #[prost(uint32, tag="24")] pub index: u32, - #[prost(uint32, tag="18")] + #[prost(uint32, tag="25")] pub action_ordinal: u32, - #[prost(string, tag="19")] + #[prost(string, tag="26")] pub receiver: ::prost::alloc::string::String, - #[prost(bool, tag="20")] + #[prost(bool, tag="27")] pub context_free: bool, - #[prost(int64, tag="21")] + #[prost(int64, tag="28")] pub elapsed: i64, - #[prost(string, tag="22")] + #[prost(string, tag="29")] pub console: ::prost::alloc::string::String, - #[prost(string, tag="23")] + #[prost(string, tag="30")] pub raw_return_value: ::prost::alloc::string::String, - #[prost(string, tag="24")] + #[prost(string, tag="31")] pub json_return_value: ::prost::alloc::string::String, - #[prost(uint32, tag="25")] + #[prost(uint32, tag="32")] pub creator_action_ordinal: u32, - #[prost(uint32, tag="26")] + #[prost(uint32, tag="33")] pub closest_unnotified_ancestor_action_ordinal: u32, - #[prost(string, tag="27")] + #[prost(string, tag="34")] pub action_mroot: ::prost::alloc::string::String, + /// ram deltas + #[prost(string, repeated, tag="35")] + pub account_ram_deltas_account: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(int64, repeated, tag="36")] + pub account_ram_deltas: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct DbOp { + /// block #[prost(message, optional, tag="1")] pub block_time: ::core::option::Option<::prost_types::Timestamp>, #[prost(uint64, tag="2")] @@ -151,12 +194,15 @@ pub struct DbOp { pub block_hash: ::prost::alloc::string::String, #[prost(string, tag="4")] pub block_date: ::prost::alloc::string::String, + /// transaction #[prost(string, tag="5")] pub tx_hash: ::prost::alloc::string::String, #[prost(bool, tag="6")] pub tx_success: bool, + /// action #[prost(uint32, tag="7")] pub action_index: u32, + /// database operations #[prost(uint32, tag="8")] pub index: u32, #[prost(string, tag="9")] @@ -187,6 +233,7 @@ pub struct DbOp { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct FeatureOp { + /// block #[prost(message, optional, tag="1")] pub block_time: ::core::option::Option<::prost_types::Timestamp>, #[prost(uint64, tag="2")] @@ -195,12 +242,15 @@ pub struct FeatureOp { pub block_hash: ::prost::alloc::string::String, #[prost(string, tag="4")] pub block_date: ::prost::alloc::string::String, + /// transaction #[prost(string, tag="5")] pub tx_hash: ::prost::alloc::string::String, #[prost(bool, tag="6")] pub tx_success: bool, + /// action #[prost(uint32, tag="7")] pub action_index: u32, + /// feature operations #[prost(string, tag="8")] pub feature_digest: ::prost::alloc::string::String, #[prost(string, tag="9")] @@ -213,6 +263,7 @@ pub struct FeatureOp { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct PermOp { + /// block #[prost(message, optional, tag="1")] pub block_time: ::core::option::Option<::prost_types::Timestamp>, #[prost(uint64, tag="2")] @@ -221,12 +272,15 @@ pub struct PermOp { pub block_hash: ::prost::alloc::string::String, #[prost(string, tag="4")] pub block_date: ::prost::alloc::string::String, + /// transaction #[prost(string, tag="5")] pub tx_hash: ::prost::alloc::string::String, #[prost(bool, tag="6")] pub tx_success: bool, + /// action #[prost(uint32, tag="7")] pub action_index: u32, + /// permission operations #[prost(string, tag="8")] pub operation: ::prost::alloc::string::String, #[prost(uint32, tag="9")] @@ -241,10 +295,27 @@ pub struct PermOp { pub name: ::prost::alloc::string::String, #[prost(uint32, tag="14")] pub threshold: u32, + /// -- authority -- + /// accounts + #[prost(string, repeated, tag="15")] + pub accounts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(uint32, repeated, tag="16")] + pub accounts_weight: ::prost::alloc::vec::Vec, + /// keys + #[prost(string, repeated, tag="17")] + pub keys_public_key: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(uint32, repeated, tag="18")] + pub keys_weight: ::prost::alloc::vec::Vec, + /// waits + #[prost(uint32, repeated, tag="19")] + pub wait_sec: ::prost::alloc::vec::Vec, + #[prost(uint32, repeated, tag="20")] + pub wait_weight: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TableOp { + /// block #[prost(message, optional, tag="1")] pub block_time: ::core::option::Option<::prost_types::Timestamp>, #[prost(uint64, tag="2")] @@ -253,12 +324,15 @@ pub struct TableOp { pub block_hash: ::prost::alloc::string::String, #[prost(string, tag="4")] pub block_date: ::prost::alloc::string::String, + /// transaction #[prost(string, tag="5")] pub tx_hash: ::prost::alloc::string::String, #[prost(bool, tag="6")] pub tx_success: bool, + /// action #[prost(uint32, tag="7")] pub action_index: u32, + /// table operations #[prost(uint32, tag="8")] pub index: u32, #[prost(string, tag="9")] @@ -276,81 +350,8 @@ pub struct TableOp { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct Account { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(string, tag="9")] - pub actor: ::prost::alloc::string::String, - #[prost(string, tag="10")] - pub permission: ::prost::alloc::string::String, - #[prost(uint32, tag="11")] - pub weight: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Key { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(string, tag="9")] - pub public_key: ::prost::alloc::string::String, - #[prost(uint32, tag="10")] - pub weight: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Wait { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(uint32, tag="9")] - pub wait_sec: u32, - #[prost(uint32, tag="10")] - pub weight: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] pub struct RamOp { + /// block #[prost(message, optional, tag="1")] pub block_time: ::core::option::Option<::prost_types::Timestamp>, #[prost(uint64, tag="2")] @@ -359,12 +360,15 @@ pub struct RamOp { pub block_hash: ::prost::alloc::string::String, #[prost(string, tag="4")] pub block_date: ::prost::alloc::string::String, + /// transaction #[prost(string, tag="5")] pub tx_hash: ::prost::alloc::string::String, #[prost(bool, tag="6")] pub tx_success: bool, + /// action #[prost(uint32, tag="7")] pub action_index: u32, + /// ram operations #[prost(string, tag="8")] pub operation: ::prost::alloc::string::String, #[prost(uint32, tag="9")] @@ -386,130 +390,4 @@ pub struct RamOp { #[prost(string, tag="17")] pub unique_key: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Authorization { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(string, tag="9")] - pub actor: ::prost::alloc::string::String, - #[prost(string, tag="10")] - pub permission: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AuthSequence { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(string, tag="9")] - pub account_name: ::prost::alloc::string::String, - #[prost(uint64, tag="10")] - pub sequence: u64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AccountRamDelta { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(uint32, tag="7")] - pub action_index: u32, - #[prost(uint32, tag="8")] - pub index: u32, - #[prost(string, tag="9")] - pub account: ::prost::alloc::string::String, - #[prost(int64, tag="10")] - pub delta: i64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CreationTree { - #[prost(message, optional, tag="1")] - pub block_time: ::core::option::Option<::prost_types::Timestamp>, - #[prost(uint64, tag="2")] - pub block_number: u64, - #[prost(string, tag="3")] - pub block_hash: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub block_date: ::prost::alloc::string::String, - #[prost(string, tag="5")] - pub tx_hash: ::prost::alloc::string::String, - #[prost(bool, tag="6")] - pub tx_success: bool, - #[prost(int32, tag="7")] - pub creator_action_index: i32, - #[prost(uint32, tag="8")] - pub execution_action_index: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Events { - #[prost(message, repeated, tag="1")] - pub blocks: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="2")] - pub transactions: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="3")] - pub actions: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="4")] - pub db_ops: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="5")] - pub feature_ops: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="6")] - pub perm_ops: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="7")] - pub table_ops: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="8")] - pub accounts: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="9")] - pub keys: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="10")] - pub waits: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="11")] - pub ram_ops: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="12")] - pub authorizations: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="13")] - pub auth_sequences: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="14")] - pub account_ram_deltas: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="15")] - pub creation_trees: ::prost::alloc::vec::Vec, -} // @@protoc_insertion_point(module) diff --git a/blocks/antelope/src/perm_ops.rs b/blocks/antelope/src/perm_ops.rs index 8e6ba40..b6909f7 100644 --- a/blocks/antelope/src/perm_ops.rs +++ b/blocks/antelope/src/perm_ops.rs @@ -1,5 +1,5 @@ use common::structs::BlockTimestamp; -use substreams_antelope::pb::TransactionTrace; +use substreams_antelope::pb::{PermissionLevel, TransactionTrace}; use crate::pb::antelope::PermOp; @@ -13,28 +13,66 @@ pub fn perm_op_operation_to_string(operation: i32) -> String { } } +pub fn permission_to_string(permission: PermissionLevel) -> String { + format!("{}@{}", permission.actor, permission.permission) +} + pub fn collect_tx_perm_ops(transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Vec { let mut perm_ops = Vec::new(); for perm_op in transaction.perm_ops.iter() { if let Some(new_perm) = &perm_op.new_perm { - let threshold = new_perm.authority.as_ref().map_or(0, |authority| authority.threshold); + // TEST: Check if authority is required + let authority = new_perm.authority.as_ref().unwrap(); + let threshold = authority.threshold; + + // accounts + let accounts = authority.accounts.iter().map(|perm| permission_to_string(perm.clone().permission.unwrap())).collect::>(); + let accounts_weight = authority.accounts.iter().map(|perm| perm.weight).collect::>(); + + // keys + let keys_public_key = authority.keys.iter().map(|perm| perm.clone().public_key).collect::>(); + let keys_weight = authority.keys.iter().map(|perm| perm.weight).collect::>(); + + // waits + let wait_sec = authority.waits.iter().map(|perm| perm.wait_sec).collect::>(); + let wait_weight = authority.waits.iter().map(|perm| perm.weight).collect::>(); perm_ops.push(PermOp { + // block block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), block_date: timestamp.date.clone(), + + // transaction tx_hash: transaction.id.clone(), tx_success, + + // action + action_index: perm_op.action_index, + + // permission operation operation: perm_op_operation_to_string(perm_op.operation), operation_code: perm_op.operation as u32, - action_index: perm_op.action_index, id: new_perm.id, parent_id: new_perm.parent_id, owner: new_perm.owner.clone(), name: new_perm.name.clone(), threshold, + + // -- authority -- + // accounts + accounts, + accounts_weight, + + // keys + keys_public_key, + keys_weight, + + // waits + wait_sec, + wait_weight, }); } } diff --git a/blocks/antelope/src/ram_ops.rs b/blocks/antelope/src/ram_ops.rs index a6a46df..efceb19 100644 --- a/blocks/antelope/src/ram_ops.rs +++ b/blocks/antelope/src/ram_ops.rs @@ -69,14 +69,21 @@ pub fn collect_tx_ram_ops(transaction: &TransactionTrace, timestamp: &BlockTimes for ram_op in transaction.ram_ops.iter() { ram_ops.push(RamOp { + // block block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), block_date: timestamp.date.clone(), + + // transaction tx_hash: transaction.id.clone(), tx_success, - operation: operation_to_string(ram_op.operation), + + // action action_index: ram_op.action_index, + + // ram operation + operation: operation_to_string(ram_op.operation), payer: ram_op.payer.clone(), delta: ram_op.delta, usage: ram_op.usage, diff --git a/blocks/antelope/src/size.rs b/blocks/antelope/src/size.rs index c8aae57..fdeede6 100644 --- a/blocks/antelope/src/size.rs +++ b/blocks/antelope/src/size.rs @@ -9,7 +9,7 @@ pub struct BlockSize { pub total_db_ops: u64, } -pub fn collect_size(block: &Block) -> BlockSize { +pub fn compute_block_size(block: &Block) -> BlockSize { // counters let mut size = 0; let mut total_transactions = 0; diff --git a/blocks/antelope/src/transactions.rs b/blocks/antelope/src/transactions.rs index 1637a4e..25823eb 100644 --- a/blocks/antelope/src/transactions.rs +++ b/blocks/antelope/src/transactions.rs @@ -29,6 +29,10 @@ pub fn collect_transaction(block: &Block, transaction: &TransactionTrace, timest let status_code = receipt.status; let status = transaction_status_to_string(status_code); + // make execution_action_index as vector + let creator_action_indexes = transaction.creation_tree.iter().map(|tree| tree.creator_action_index).collect::>(); + let execution_action_indexes = transaction.creation_tree.iter().map(|tree| tree.execution_action_index).collect::>(); + Transaction { block_time: Some(timestamp.time.clone()), block_number: timestamp.number, @@ -45,5 +49,7 @@ pub fn collect_transaction(block: &Block, transaction: &TransactionTrace, timest status_code: status_code as u32, success: tx_success, transaction_mroot: Hex::encode(&header.transaction_mroot.to_vec()), + creator_action_indexes, + execution_action_indexes, } } diff --git a/blocks/antelope/substreams.yaml b/blocks/antelope/substreams.yaml index 093276b..5c7f71b 100644 --- a/blocks/antelope/substreams.yaml +++ b/blocks/antelope/substreams.yaml @@ -5,10 +5,6 @@ package: image: logo.png url: https://github.com/pinax-network/substreams-raw-blocks -imports: - database_change: https://github.com/streamingfast/substreams-sink-database-changes/releases/download/v2.0.0/substreams-database-change-v2.0.0.spkg - sql: https://github.com/streamingfast/substreams-sink-sql/releases/download/protodefs-v1.0.7/substreams-sink-sql-protodefs-v1.0.7.spkg - binaries: default: type: wasm/rust-v1 @@ -32,5 +28,3 @@ modules: - source: sf.antelope.type.v1.Block output: type: proto:antelope.Events - -# network: wax \ No newline at end of file diff --git a/proto/antelope.proto b/proto/antelope.proto index 06ec8c1..c467c68 100644 --- a/proto/antelope.proto +++ b/proto/antelope.proto @@ -4,12 +4,26 @@ package antelope; import "google/protobuf/timestamp.proto"; +message Events { + repeated Block blocks = 1; + repeated Transaction transactions = 2; + repeated Action actions = 3; + repeated DbOp db_ops = 4; + repeated FeatureOp feature_ops = 5; + repeated PermOp perm_ops = 6; + repeated TableOp table_ops = 7; + repeated RamOp ram_ops = 8; +} + message Block { + // clock google.protobuf.Timestamp time = 1; uint64 number = 2; string date = 3; string hash = 4; + + // block string parent_hash = 5; string producer = 6; uint32 confirmed = 7; @@ -22,6 +36,8 @@ message Block { string action_mroot = 14; repeated string blockroot_merkle_active_nodes = 15; uint32 blockroot_merkle_node_count = 16; + + // counters uint64 size = 17; uint64 total_transactions = 18; uint64 successful_transactions = 19; @@ -31,10 +47,13 @@ message Block { } message Transaction { + // block google.protobuf.Timestamp block_time = 1; uint64 block_number = 2; string block_hash = 3; string block_date = 4; + + // transaction string hash = 5; uint64 index = 6; int64 elapsed = 7; @@ -46,46 +65,70 @@ message Transaction { uint32 status_code = 13; bool success = 14; string transaction_mroot = 15; + + // creation flat node + repeated int32 creator_action_indexes = 16; + repeated uint32 execution_action_indexes = 17; } message Action { + // block google.protobuf.Timestamp block_time = 1; uint64 block_number = 2; string block_hash = 3; string block_date = 4; + + // transaction string tx_hash = 5; bool tx_success = 6; + + // receipt uint64 abi_sequence = 7; uint64 code_sequence = 8; string digest = 9; uint64 global_sequence = 10; string receipt_receiver = 11; uint64 recv_sequence = 12; - string account = 13; - string name = 14; - string json_data = 15; - string raw_data = 16; - uint32 index = 17; - uint32 action_ordinal = 18; - string receiver = 19; - bool context_free = 20; - int64 elapsed = 21; - string console = 22; - string raw_return_value = 23; - string json_return_value = 24; - uint32 creator_action_ordinal = 25; - uint32 closest_unnotified_ancestor_action_ordinal = 26; - string action_mroot = 27; + repeated uint64 auth_sequence = 13; + repeated string auth_sequence_account_name = 14; + + // action + string account = 20; + string name = 21; + string json_data = 22; + string raw_data = 23; + uint32 index = 24; + uint32 action_ordinal = 25; + string receiver = 26; + bool context_free = 27; + int64 elapsed = 28; + string console = 29; + string raw_return_value = 30; + string json_return_value = 31; + uint32 creator_action_ordinal = 32; + uint32 closest_unnotified_ancestor_action_ordinal = 33; + string action_mroot = 34; + + // ram deltas + repeated string account_ram_deltas_account = 35; + repeated int64 account_ram_deltas = 36; } message DbOp { + // block google.protobuf.Timestamp block_time = 1; uint64 block_number = 2; string block_hash = 3; string block_date = 4; + + // transaction string tx_hash = 5; bool tx_success = 6; + + // action uint32 action_index = 7; + + // database operations uint32 index = 8; string operation = 9; uint32 operation_code = 10; @@ -102,13 +145,20 @@ message DbOp { } message FeatureOp { + // block google.protobuf.Timestamp block_time = 1; uint64 block_number = 2; string block_hash = 3; string block_date = 4; + + // transaction string tx_hash = 5; bool tx_success = 6; + + // action uint32 action_index = 7; + + // feature operations string feature_digest = 8; string kind = 9; string description_digest = 10; @@ -116,13 +166,20 @@ message FeatureOp { } message PermOp { + // block google.protobuf.Timestamp block_time = 1; uint64 block_number = 2; string block_hash = 3; string block_date = 4; + + // transaction string tx_hash = 5; bool tx_success = 6; + + // action uint32 action_index = 7; + + // permission operations string operation = 8; uint32 operation_code = 9; uint64 id = 10; @@ -130,16 +187,36 @@ message PermOp { string owner = 12; string name = 13; uint32 threshold = 14; + + // -- authority -- + // accounts + repeated string accounts = 15; + repeated uint32 accounts_weight = 16; + + // keys + repeated string keys_public_key = 17; + repeated uint32 keys_weight = 18; + + // waits + repeated uint32 wait_sec = 19; + repeated uint32 wait_weight = 20; } message TableOp { + // block google.protobuf.Timestamp block_time = 1; uint64 block_number = 2; string block_hash = 3; string block_date = 4; + + // transaction string tx_hash = 5; bool tx_success = 6; + + // action uint32 action_index = 7; + + // table operations uint32 index = 8; string operation = 9; uint32 operation_code = 10; @@ -149,54 +226,21 @@ message TableOp { string table_name = 14; } -message Account { +message RamOp { + // block google.protobuf.Timestamp block_time = 1; uint64 block_number = 2; string block_hash = 3; string block_date = 4; - string tx_hash = 5; - bool tx_success = 6; - uint32 action_index = 7; - uint32 index = 8; - string actor = 9; - string permission = 10; - uint32 weight = 11; -} -message Key { - google.protobuf.Timestamp block_time = 1; - uint64 block_number = 2; - string block_hash = 3; - string block_date = 4; + // transaction string tx_hash = 5; bool tx_success = 6; - uint32 action_index = 7; - uint32 index = 8; - string public_key = 9; - uint32 weight = 10; -} -message Wait { - google.protobuf.Timestamp block_time = 1; - uint64 block_number = 2; - string block_hash = 3; - string block_date = 4; - string tx_hash = 5; - bool tx_success = 6; + // action uint32 action_index = 7; - uint32 index = 8; - uint32 wait_sec = 9; - uint32 weight = 10; -} -message RamOp { - google.protobuf.Timestamp block_time = 1; - uint64 block_number = 2; - string block_hash = 3; - string block_date = 4; - string tx_hash = 5; - bool tx_success = 6; - uint32 action_index = 7; + // ram operations string operation = 8; uint32 operation_code = 9; string payer = 10; @@ -208,71 +252,3 @@ message RamOp { uint32 action_code = 16; string unique_key = 17; } - -message Authorization { - google.protobuf.Timestamp block_time = 1; - uint64 block_number = 2; - string block_hash = 3; - string block_date = 4; - string tx_hash = 5; - bool tx_success = 6; - uint32 action_index = 7; - uint32 index = 8; - string actor = 9; - string permission = 10; -} - -message AuthSequence { - google.protobuf.Timestamp block_time = 1; - uint64 block_number = 2; - string block_hash = 3; - string block_date = 4; - string tx_hash = 5; - bool tx_success = 6; - uint32 action_index = 7; - uint32 index = 8; - string account_name = 9; - uint64 sequence = 10; -} - -message AccountRamDelta { - google.protobuf.Timestamp block_time = 1; - uint64 block_number = 2; - string block_hash = 3; - string block_date = 4; - string tx_hash = 5; - bool tx_success = 6; - uint32 action_index = 7; - uint32 index = 8; - string account = 9; - int64 delta = 10; -} - -message CreationTree { - google.protobuf.Timestamp block_time = 1; - uint64 block_number = 2; - string block_hash = 3; - string block_date = 4; - string tx_hash = 5; - bool tx_success = 6; - int32 creator_action_index = 7; - uint32 execution_action_index = 8; -} - -message Events { - repeated Block blocks = 1; - repeated Transaction transactions = 2; - repeated Action actions = 3; - repeated DbOp db_ops = 4; - repeated FeatureOp feature_ops = 5; - repeated PermOp perm_ops = 6; - repeated TableOp table_ops = 7; - repeated Account accounts = 8; - repeated Key keys = 9; - repeated Wait waits = 10; - repeated RamOp ram_ops = 11; - repeated Authorization authorizations = 12; - repeated AuthSequence auth_sequences = 13; - repeated AccountRamDelta account_ram_deltas = 14; - repeated CreationTree creation_trees = 15; -} From 4529720a6657a56d94e8d2ef7a121f5dba7154b6 Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Thu, 21 Nov 2024 15:29:05 +0100 Subject: [PATCH 19/22] add savanna root --- blocks/antelope/src/blocks.rs | 9 ++++++++- blocks/antelope/src/pb/antelope.rs | 19 +++++++++++++------ proto/antelope.proto | 17 +++++++++++------ 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/blocks/antelope/src/blocks.rs b/blocks/antelope/src/blocks.rs index 12d3a35..4ea3a1a 100644 --- a/blocks/antelope/src/blocks.rs +++ b/blocks/antelope/src/blocks.rs @@ -7,6 +7,8 @@ use substreams_antelope::Block; pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> EventsBlock { let header = block.header.as_ref().expect("missing block header"); let blockroot_merkle = block.blockroot_merkle.clone().unwrap_or_default(); + let blockroot_merkle_active_nodes = blockroot_merkle.active_nodes.iter().map(|row| Hex::encode(row)).collect::>(); + let confirm_count = block.confirm_count.iter().map(|count| count.clone()).collect::>(); let size = compute_block_size(block); EventsBlock { @@ -25,10 +27,15 @@ pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> EventsBlock { producer_signature: block.producer_signature.clone(), dpos_proposed_irreversible_blocknum: block.dpos_proposed_irreversible_blocknum, dpos_irreversible_blocknum: block.dpos_irreversible_blocknum, + + // roots transaction_mroot: Hex::encode(&header.transaction_mroot.to_vec()), action_mroot: Hex::encode(&header.action_mroot.to_vec()), - blockroot_merkle_active_nodes: blockroot_merkle.active_nodes.iter().map(|row| Hex::encode(row)).collect::>(), + blockroot_merkle_active_nodes, blockroot_merkle_node_count: blockroot_merkle.node_count, + action_mroot_savanna: Hex::encode(block.action_mroot_savanna.clone()), + block_signing_key: block.block_signing_key.clone(), + confirm_count, // counters size: size.size, diff --git a/blocks/antelope/src/pb/antelope.rs b/blocks/antelope/src/pb/antelope.rs index 1c7328e..d28744c 100644 --- a/blocks/antelope/src/pb/antelope.rs +++ b/blocks/antelope/src/pb/antelope.rs @@ -48,6 +48,7 @@ pub struct Block { pub dpos_proposed_irreversible_blocknum: u32, #[prost(uint32, tag="12")] pub dpos_irreversible_blocknum: u32, + /// roots #[prost(string, tag="13")] pub transaction_mroot: ::prost::alloc::string::String, #[prost(string, tag="14")] @@ -56,18 +57,24 @@ pub struct Block { pub blockroot_merkle_active_nodes: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, #[prost(uint32, tag="16")] pub blockroot_merkle_node_count: u32, + #[prost(string, tag="17")] + pub action_mroot_savanna: ::prost::alloc::string::String, + #[prost(string, tag="18")] + pub block_signing_key: ::prost::alloc::string::String, + #[prost(uint32, repeated, tag="19")] + pub confirm_count: ::prost::alloc::vec::Vec, /// counters - #[prost(uint64, tag="17")] + #[prost(uint64, tag="20")] pub size: u64, - #[prost(uint64, tag="18")] + #[prost(uint64, tag="21")] pub total_transactions: u64, - #[prost(uint64, tag="19")] + #[prost(uint64, tag="22")] pub successful_transactions: u64, - #[prost(uint64, tag="20")] + #[prost(uint64, tag="23")] pub failed_transactions: u64, - #[prost(uint64, tag="21")] + #[prost(uint64, tag="24")] pub total_actions: u64, - #[prost(uint64, tag="22")] + #[prost(uint64, tag="25")] pub total_db_ops: u64, } #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/proto/antelope.proto b/proto/antelope.proto index c467c68..56bf8c2 100644 --- a/proto/antelope.proto +++ b/proto/antelope.proto @@ -32,18 +32,23 @@ message Block { string producer_signature = 10; uint32 dpos_proposed_irreversible_blocknum = 11; uint32 dpos_irreversible_blocknum = 12; + + // roots string transaction_mroot = 13; string action_mroot = 14; repeated string blockroot_merkle_active_nodes = 15; uint32 blockroot_merkle_node_count = 16; + string action_mroot_savanna = 17; + string block_signing_key = 18; + repeated uint32 confirm_count = 19; // counters - uint64 size = 17; - uint64 total_transactions = 18; - uint64 successful_transactions = 19; - uint64 failed_transactions = 20; - uint64 total_actions = 21; - uint64 total_db_ops = 22; + uint64 size = 20; + uint64 total_transactions = 21; + uint64 successful_transactions = 22; + uint64 failed_transactions = 23; + uint64 total_actions = 24; + uint64 total_db_ops = 25; } message Transaction { From 8d8daaab6ee51fe654554ef73867e6e382f56ab4 Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Thu, 21 Nov 2024 15:31:28 +0100 Subject: [PATCH 20/22] remove sql --- blocks/antelope/README.md | 24 +- blocks/antelope/schema.sql | 515 -------------------------------- blocks/antelope/substreams.yaml | 2 +- 3 files changed, 18 insertions(+), 523 deletions(-) delete mode 100644 blocks/antelope/schema.sql diff --git a/blocks/antelope/README.md b/blocks/antelope/README.md index 6615522..3c56602 100644 --- a/blocks/antelope/README.md +++ b/blocks/antelope/README.md @@ -4,7 +4,7 @@ > [`sf.antelope.type.v1.Block`](https://buf.build/pinax/firehose-antelope/docs/main:sf.antelope.type.v1) - [x] **Blocks** - - [ ] **Savanna Merkle Roots** + - [x] **Savanna Merkle Roots** - [x] **Transactions** - [x] **Feature Operations** - [x] **Permission Operations** @@ -21,13 +21,23 @@ - [x] **Account RAM Deltas** - [x] **Database Operations** -## Substreams Graph +## Graph ```mermaid graph TD; - raw[sf.antelope.type.v1.Block]; - raw --> blocks; - raw --> transactions; - raw --> actions; - raw --> db_ops; + map_events[map: map_events]; + sf.substreams.v1.Clock[source: sf.substreams.v1.Clock] --> map_events; + sf.antelope.type.v1.Block[source: sf.antelope.type.v1.Block] --> map_events; +``` + +## Modules + +```bash +Name: map_events +Initial block: 0 +Kind: map +Input: source: sf.substreams.v1.Clock +Input: source: sf.antelope.type.v1.Block +Output Type: proto:antelope.Events +Hash: 9e6d04e82d8c5bce50f835b57ce669e79af4b7b2 ``` diff --git a/blocks/antelope/schema.sql b/blocks/antelope/schema.sql deleted file mode 100644 index 8a6a8ee..0000000 --- a/blocks/antelope/schema.sql +++ /dev/null @@ -1,515 +0,0 @@ -------------------------------------------------- --- Meta tables to store Substreams information -- -------------------------------------------------- -CREATE TABLE IF NOT EXISTS cursors -( - id String, - cursor String, - block_num Int64, - block_id String -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (id) - ORDER BY (id); - -CREATE TABLE IF NOT EXISTS blocks -( - -- clock -- - time DateTime64(3, 'UTC'), - number UInt64, - date Date, - hash String COMMENT 'Hash', - - -- header -- - parent_hash String COMMENT 'Hash', - producer String COMMENT 'Address', - confirmed UInt32, - schedule_version UInt32, - - -- block -- - version UInt32, - producer_signature String COMMENT 'Signature', - dpos_proposed_irreversible_blocknum UInt32, - dpos_irreversible_blocknum UInt32, - - -- block roots -- - transaction_mroot String COMMENT 'Hash', - action_mroot String COMMENT 'Hash', - -- blockroot_merkle_active_nodes Array(String) COMMENT 'A blockroot Merkle tree uses hashes to verify blockchain data integrity. Leaf nodes hash data blocks, non-leaf nodes hash child nodes. The root hash efficiently verifies all data.', - blockroot_merkle_node_count UInt32, - - -- counters -- - size UInt64 COMMENT 'Block size estimate in bytes', - total_transactions UInt64, - successful_transactions UInt64, - failed_transactions UInt64, - total_actions UInt64, - total_db_ops UInt64, -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (hash) - ORDER BY (hash) - COMMENT 'Antelope block header'; - -CREATE TABLE IF NOT EXISTS transactions -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String COMMENT 'Hash', - block_date Date, - - -- transaction -- - hash String COMMENT 'Hash', - `index` UInt64, - elapsed Int64, - net_usage UInt64, - scheduled Bool, - - -- header -- - cpu_usage_micro_seconds UInt32, - net_usage_words UInt32, - status LowCardinality(String) COMMENT 'Status', - status_code UInt8, - success Bool, - - -- block roots -- - transaction_mroot String COMMENT 'Hash', -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (hash) - ORDER BY (hash) - COMMENT 'Antelope transactions'; - -CREATE TABLE IF NOT EXISTS actions -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String COMMENT 'Hash', - block_date Date, - - -- transaction -- - tx_hash String COMMENT 'Hash', - tx_success Bool, - - -- receipt -- - abi_sequence UInt64, - code_sequence UInt64, - digest String, - global_sequence UInt64, - receipt_receiver String COMMENT 'Address', - recv_sequence UInt64, - - -- action -- - account String COMMENT 'Address', - name String COMMENT 'Address', - json_data String COMMENT 'JSON', - raw_data String COMMENT 'Hex', - - -- trace -- - `index` UInt32 COMMENT 'Execution Index', - action_ordinal UInt32 COMMENT 'Action Ordinal', - receiver String, - context_free Bool, - elapsed Int64, - console String, - raw_return_value String, - json_return_value String, - creator_action_ordinal UInt32, - closest_unnotified_ancestor_action_ordinal UInt32, - - -- block roots -- - action_mroot String, -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (tx_hash, `index`) - ORDER BY (tx_hash, `index`) - COMMENT 'Antelope actions'; - -CREATE TABLE IF NOT EXISTS db_ops -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String, - block_date Date, - - -- transaction -- - tx_hash String, - tx_success Bool, - - -- action -- - action_index UInt32, - - -- database operation -- - `index` UInt32, - operation LowCardinality(String) COMMENT 'Operation', - operation_code UInt8, - code String, - scope String, - table_name String, - primary_key String, - old_payer String, - new_payer String, - old_data String, - new_data String, - old_data_json String, - new_data_json String, -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (tx_hash, `index`) - ORDER BY (tx_hash, `index`) - COMMENT 'Antelope database operations'; - -CREATE TABLE IF NOT EXISTS feature_ops -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String COMMENT 'Hash', - block_date Date, - - -- transaction -- - tx_hash String COMMENT 'Hash', - tx_success Bool, - - -- action -- - action_index UInt32, - - -- feature op -- - feature_digest String, - kind LowCardinality(String), - description_digest String, - protocol_feature_type LowCardinality(String), -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (feature_digest) - ORDER BY (feature_digest) - COMMENT 'Antelope feature operations'; - -CREATE TABLE IF NOT EXISTS perm_ops -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String COMMENT 'Hash', - block_date Date, - - -- transaction -- - tx_hash String COMMENT 'Hash', - tx_success Bool, - - -- action -- - action_index UInt32, - - -- perm_op -- - operation LowCardinality(String), - operation_code UInt8, - id UInt64, - parent_id UInt64, - owner String, - name String, - threshold UInt32, -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (tx_hash, action_index) - ORDER BY (tx_hash, action_index) - COMMENT 'Antelope permission operations'; - -CREATE TABLE IF NOT EXISTS table_ops -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String COMMENT 'Hash', - block_date Date, - - -- transaction -- - tx_hash String COMMENT 'Hash', - tx_success Bool, - - -- action -- - action_index UInt32, - - -- table op -- - `index` UInt32, - operation LowCardinality(String), - operation_code UInt8, - payer String, - code String, - scope String, - table_name String, -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (tx_hash, `index`) - ORDER BY (tx_hash, `index`) - COMMENT 'Antelope table operations'; - -CREATE TABLE IF NOT EXISTS accounts -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String COMMENT 'Hash', - block_date Date, - - -- transaction -- - tx_hash String COMMENT 'Hash', - tx_success Bool, - - -- action -- - action_index UInt32, - - -- authority.accounts -- - `index` UInt32, - actor String, - permission LowCardinality(String), - weight UInt32, -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (tx_hash, action_index, `index`) - ORDER BY (tx_hash, action_index, `index`) - COMMENT 'Antelope authority accounts'; - -CREATE TABLE IF NOT EXISTS keys -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String COMMENT 'Hash', - block_date Date, - - -- transaction -- - tx_hash String COMMENT 'Hash', - tx_success Bool, - - -- action -- - action_index UInt32, - - -- authority.keys -- - `index` UInt32, - public_key String, - weight UInt32, -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (tx_hash, action_index, `index`) - ORDER BY (tx_hash, action_index, `index`) - COMMENT 'Antelope authority keys'; - -CREATE TABLE IF NOT EXISTS waits -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String COMMENT 'Hash', - block_date Date, - - -- transaction -- - tx_hash String COMMENT 'Hash', - tx_success Bool, - - -- action -- - action_index UInt32, - - -- authority.waits -- - `index` UInt32, - wait_sec UInt32, - weight UInt32, -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (tx_hash, action_index, `index`) - ORDER BY (tx_hash, action_index, `index`) - COMMENT 'Antelope authority waits'; - -CREATE TABLE IF NOT EXISTS ram_ops -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String COMMENT 'Hash', - block_date Date, - - -- transaction -- - tx_hash String COMMENT 'Hash', - tx_success Bool, - - -- action -- - action_index UInt32, - - -- RAM operation -- - operation LowCardinality(String), - operation_code UInt8, - payer String, - delta Int64, - usage UInt64, - namespace LowCardinality(String), - namespace_code UInt8, - action LowCardinality(String), - action_code UInt8, - unique_key String -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (tx_hash, action_index, unique_key) - ORDER BY (tx_hash, action_index, unique_key) - COMMENT 'Antelope RAM operations'; - -CREATE TABLE IF NOT EXISTS authorizations -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String, - block_date Date, - - -- transaction -- - tx_hash String, - tx_success Bool, - - -- action -- - action_index UInt32, - - -- authorization -- - `index` UInt32, - actor String, - permission LowCardinality(String) -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (tx_hash, action_index, `index`) - ORDER BY (tx_hash, action_index, `index`) - COMMENT 'Antelope action authorizations'; - -CREATE TABLE IF NOT EXISTS auth_sequences -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String, - block_date Date, - - -- transaction -- - tx_hash String, - tx_success Bool, - - -- action -- - action_index UInt32, - - -- auth_sequence -- - `index` UInt32, - account_name String, - sequence UInt64 -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (tx_hash, action_index, `index`) - ORDER BY (tx_hash, action_index, `index`) - COMMENT 'Antelope action authorization sequences'; - -CREATE TABLE IF NOT EXISTS account_ram_deltas -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String, - block_date Date, - - -- transaction -- - tx_hash String, - tx_success Bool, - - -- action -- - action_index UInt32, - - -- account_ram_delta -- - `index` UInt32, - account String, - delta Int64 -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (tx_hash, action_index, `index`) - ORDER BY (tx_hash, action_index, `index`) - COMMENT 'Antelope account RAM deltas'; - -CREATE TABLE IF NOT EXISTS creation_tree -( - -- clock -- - block_time DateTime64(3, 'UTC'), - block_number UInt64, - block_hash String, - block_date Date, - - -- transaction -- - tx_hash String COMMENT 'Hash', - tx_success Bool, - - -- transaction.creation_tree -- - creator_action_index Int32, - execution_action_index UInt32 -) - ENGINE = ReplacingMergeTree() - PRIMARY KEY (tx_hash, creator_action_index, execution_action_index) - ORDER BY (tx_hash, creator_action_index, execution_action_index) - COMMENT 'Antelope creation tree'; - --- Projections -- --- https://clickhouse.com/docs/en/sql-reference/statements/alter/projection -- -ALTER TABLE blocks ADD PROJECTION IF NOT EXISTS blocks_by_block_number ( - SELECT * ORDER BY date, number -); -ALTER TABLE transactions ADD PROJECTION IF NOT EXISTS transactions_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE actions ADD PROJECTION IF NOT EXISTS actions_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE db_ops ADD PROJECTION IF NOT EXISTS db_ops_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE feature_ops ADD PROJECTION IF NOT EXISTS feature_ops_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE perm_ops ADD PROJECTION IF NOT EXISTS perm_ops_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE table_ops ADD PROJECTION IF NOT EXISTS table_ops_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE accounts ADD PROJECTION IF NOT EXISTS accounts_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE keys ADD PROJECTION IF NOT EXISTS keys_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE waits ADD PROJECTION IF NOT EXISTS waits_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE ram_ops ADD PROJECTION IF NOT EXISTS ram_ops_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE authorizations ADD PROJECTION IF NOT EXISTS authorizations_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE auth_sequences ADD PROJECTION IF NOT EXISTS auth_sequences_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE account_ram_deltas ADD PROJECTION IF NOT EXISTS account_ram_deltas_by_block_number ( - SELECT * ORDER BY block_date, block_number -); -ALTER TABLE creation_tree ADD PROJECTION IF NOT EXISTS creation_tree_by_block_number ( - SELECT * ORDER BY block_date, block_number -); - -ALTER TABLE blocks MATERIALIZE PROJECTION blocks_by_block_number; -ALTER TABLE transactions MATERIALIZE PROJECTION transactions_by_block_number; -ALTER TABLE feature_ops MATERIALIZE PROJECTION feature_ops_by_block_number; -ALTER TABLE perm_ops MATERIALIZE PROJECTION perm_ops_by_block_number; -ALTER TABLE table_ops MATERIALIZE PROJECTION table_ops_by_block_number; -ALTER TABLE accounts MATERIALIZE PROJECTION accounts_by_block_number; -ALTER TABLE keys MATERIALIZE PROJECTION keys_by_block_number; -ALTER TABLE waits MATERIALIZE PROJECTION waits_by_block_number; -ALTER TABLE ram_ops MATERIALIZE PROJECTION ram_ops_by_block_number; -ALTER TABLE actions MATERIALIZE PROJECTION actions_by_block_number; -ALTER TABLE db_ops MATERIALIZE PROJECTION db_ops_by_block_number; -ALTER TABLE authorizations MATERIALIZE PROJECTION authorizations_by_block_number; -ALTER TABLE auth_sequences MATERIALIZE PROJECTION auth_sequences_by_block_number; -ALTER TABLE account_ram_deltas MATERIALIZE PROJECTION account_ram_deltas_by_block_number; -ALTER TABLE creation_tree MATERIALIZE PROJECTION creation_tree_by_block_number; \ No newline at end of file diff --git a/blocks/antelope/substreams.yaml b/blocks/antelope/substreams.yaml index 5c7f71b..baf5249 100644 --- a/blocks/antelope/substreams.yaml +++ b/blocks/antelope/substreams.yaml @@ -1,7 +1,7 @@ specVersion: v0.1.0 package: name: raw_blocks_antelope - version: v0.3.0 + version: v1.0.0 image: logo.png url: https://github.com/pinax-network/substreams-raw-blocks From e6e6d9cc11e7dc58fba836de9b9870c1b54c86e4 Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Thu, 21 Nov 2024 15:38:06 +0100 Subject: [PATCH 21/22] update *_codes to int32 --- blocks/antelope/src/db_ops.rs | 2 +- blocks/antelope/src/pb/antelope.rs | 28 ++++++++++++++-------------- blocks/antelope/src/perm_ops.rs | 2 +- blocks/antelope/src/ram_ops.rs | 6 +++--- blocks/antelope/src/table_ops.rs | 2 +- blocks/antelope/src/transactions.rs | 13 +++++++++---- proto/antelope.proto | 14 +++++++------- 7 files changed, 36 insertions(+), 31 deletions(-) diff --git a/blocks/antelope/src/db_ops.rs b/blocks/antelope/src/db_ops.rs index de3e3fd..f01c60e 100644 --- a/blocks/antelope/src/db_ops.rs +++ b/blocks/antelope/src/db_ops.rs @@ -33,7 +33,7 @@ pub fn collect_tx_db_ops(transaction: &TransactionTrace, timestamp: &BlockTimest // database operation index: index as u32, operation: operation_to_string(db_op.operation), - operation_code: db_op.operation as u32, + operation_code: db_op.operation, action_index: db_op.action_index, code: db_op.code.clone(), scope: db_op.scope.clone(), diff --git a/blocks/antelope/src/pb/antelope.rs b/blocks/antelope/src/pb/antelope.rs index d28744c..4e33e0f 100644 --- a/blocks/antelope/src/pb/antelope.rs +++ b/blocks/antelope/src/pb/antelope.rs @@ -106,8 +106,8 @@ pub struct Transaction { pub net_usage_words: u32, #[prost(string, tag="12")] pub status: ::prost::alloc::string::String, - #[prost(uint32, tag="13")] - pub status_code: u32, + #[prost(int32, tag="13")] + pub status_code: i32, #[prost(bool, tag="14")] pub success: bool, #[prost(string, tag="15")] @@ -214,8 +214,8 @@ pub struct DbOp { pub index: u32, #[prost(string, tag="9")] pub operation: ::prost::alloc::string::String, - #[prost(uint32, tag="10")] - pub operation_code: u32, + #[prost(int32, tag="10")] + pub operation_code: i32, #[prost(string, tag="11")] pub code: ::prost::alloc::string::String, #[prost(string, tag="12")] @@ -290,8 +290,8 @@ pub struct PermOp { /// permission operations #[prost(string, tag="8")] pub operation: ::prost::alloc::string::String, - #[prost(uint32, tag="9")] - pub operation_code: u32, + #[prost(int32, tag="9")] + pub operation_code: i32, #[prost(uint64, tag="10")] pub id: u64, #[prost(uint64, tag="11")] @@ -344,8 +344,8 @@ pub struct TableOp { pub index: u32, #[prost(string, tag="9")] pub operation: ::prost::alloc::string::String, - #[prost(uint32, tag="10")] - pub operation_code: u32, + #[prost(int32, tag="10")] + pub operation_code: i32, #[prost(string, tag="11")] pub payer: ::prost::alloc::string::String, #[prost(string, tag="12")] @@ -378,8 +378,8 @@ pub struct RamOp { /// ram operations #[prost(string, tag="8")] pub operation: ::prost::alloc::string::String, - #[prost(uint32, tag="9")] - pub operation_code: u32, + #[prost(int32, tag="9")] + pub operation_code: i32, #[prost(string, tag="10")] pub payer: ::prost::alloc::string::String, #[prost(int64, tag="11")] @@ -388,12 +388,12 @@ pub struct RamOp { pub usage: u64, #[prost(string, tag="13")] pub namespace: ::prost::alloc::string::String, - #[prost(uint32, tag="14")] - pub namespace_code: u32, + #[prost(int32, tag="14")] + pub namespace_code: i32, #[prost(string, tag="15")] pub action: ::prost::alloc::string::String, - #[prost(uint32, tag="16")] - pub action_code: u32, + #[prost(int32, tag="16")] + pub action_code: i32, #[prost(string, tag="17")] pub unique_key: ::prost::alloc::string::String, } diff --git a/blocks/antelope/src/perm_ops.rs b/blocks/antelope/src/perm_ops.rs index b6909f7..cfa1cb7 100644 --- a/blocks/antelope/src/perm_ops.rs +++ b/blocks/antelope/src/perm_ops.rs @@ -54,7 +54,7 @@ pub fn collect_tx_perm_ops(transaction: &TransactionTrace, timestamp: &BlockTime // permission operation operation: perm_op_operation_to_string(perm_op.operation), - operation_code: perm_op.operation as u32, + operation_code: perm_op.operation, id: new_perm.id, parent_id: new_perm.parent_id, owner: new_perm.owner.clone(), diff --git a/blocks/antelope/src/ram_ops.rs b/blocks/antelope/src/ram_ops.rs index efceb19..27bc74c 100644 --- a/blocks/antelope/src/ram_ops.rs +++ b/blocks/antelope/src/ram_ops.rs @@ -90,9 +90,9 @@ pub fn collect_tx_ram_ops(transaction: &TransactionTrace, timestamp: &BlockTimes namespace: namespace_to_string(ram_op.namespace), action: action_to_string(ram_op.action), unique_key: ram_op.unique_key.clone(), - operation_code: ram_op.operation as u32, - namespace_code: ram_op.namespace as u32, - action_code: ram_op.action as u32, + operation_code: ram_op.operation, + namespace_code: ram_op.namespace, + action_code: ram_op.action, }); } diff --git a/blocks/antelope/src/table_ops.rs b/blocks/antelope/src/table_ops.rs index cc75e0d..d411f50 100644 --- a/blocks/antelope/src/table_ops.rs +++ b/blocks/antelope/src/table_ops.rs @@ -26,7 +26,7 @@ pub fn collect_tx_table_ops(transaction: &TransactionTrace, timestamp: &BlockTim index: index as u32, action_index: table_op.action_index, operation: table_op_operation_to_string(table_op.operation), - operation_code: table_op.operation as u32, + operation_code: table_op.operation, payer: table_op.payer.clone(), code: table_op.code.clone(), scope: table_op.scope.clone(), diff --git a/blocks/antelope/src/transactions.rs b/blocks/antelope/src/transactions.rs index 25823eb..deff425 100644 --- a/blocks/antelope/src/transactions.rs +++ b/blocks/antelope/src/transactions.rs @@ -23,7 +23,7 @@ pub fn is_transaction_success(status: i32) -> bool { } // https://github.com/pinax-network/firehose-antelope/blob/534ca5bf2aeda67e8ef07a1af8fc8e0fe46473ee/proto/sf/antelope/type/v1/type.proto#L525 -pub fn collect_transaction(block: &Block, transaction: &TransactionTrace, timestamp: &BlockTimestamp, tx_success: bool) -> Transaction { +pub fn collect_transaction(block: &Block, transaction: &TransactionTrace, timestamp: &BlockTimestamp, success: bool) -> Transaction { let header = block.header.clone().unwrap_or_default(); let receipt = transaction.receipt.clone().unwrap_or_default(); let status_code = receipt.status; @@ -34,21 +34,26 @@ pub fn collect_transaction(block: &Block, transaction: &TransactionTrace, timest let execution_action_indexes = transaction.creation_tree.iter().map(|tree| tree.execution_action_index).collect::>(); Transaction { + // block block_time: Some(timestamp.time.clone()), block_number: timestamp.number, block_hash: timestamp.hash.clone(), block_date: timestamp.date.clone(), + + // transaction hash: transaction.id.clone(), - index: transaction.index as u64, + index: transaction.index, elapsed: transaction.elapsed, net_usage: transaction.net_usage, scheduled: transaction.scheduled, cpu_usage_micro_seconds: receipt.cpu_usage_micro_seconds, net_usage_words: receipt.net_usage_words, status, - status_code: status_code as u32, - success: tx_success, + status_code, + success, transaction_mroot: Hex::encode(&header.transaction_mroot.to_vec()), + + // creation flat node creator_action_indexes, execution_action_indexes, } diff --git a/proto/antelope.proto b/proto/antelope.proto index 56bf8c2..cc320c4 100644 --- a/proto/antelope.proto +++ b/proto/antelope.proto @@ -67,7 +67,7 @@ message Transaction { uint32 cpu_usage_micro_seconds = 10; uint32 net_usage_words = 11; string status = 12; - uint32 status_code = 13; + int32 status_code = 13; bool success = 14; string transaction_mroot = 15; @@ -136,7 +136,7 @@ message DbOp { // database operations uint32 index = 8; string operation = 9; - uint32 operation_code = 10; + int32 operation_code = 10; string code = 11; string scope = 12; string table_name = 13; @@ -186,7 +186,7 @@ message PermOp { // permission operations string operation = 8; - uint32 operation_code = 9; + int32 operation_code = 9; uint64 id = 10; uint64 parent_id = 11; string owner = 12; @@ -224,7 +224,7 @@ message TableOp { // table operations uint32 index = 8; string operation = 9; - uint32 operation_code = 10; + int32 operation_code = 10; string payer = 11; string code = 12; string scope = 13; @@ -247,13 +247,13 @@ message RamOp { // ram operations string operation = 8; - uint32 operation_code = 9; + int32 operation_code = 9; string payer = 10; int64 delta = 11; uint64 usage = 12; string namespace = 13; - uint32 namespace_code = 14; + int32 namespace_code = 14; string action = 15; - uint32 action_code = 16; + int32 action_code = 16; string unique_key = 17; } From 72baca36ef7fc9cd7e8125636255947cf1578f16 Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Thu, 21 Nov 2024 15:41:46 +0100 Subject: [PATCH 22/22] update hash --- blocks/antelope/README.md | 2 +- proto/antelope.proto | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/blocks/antelope/README.md b/blocks/antelope/README.md index 3c56602..c7841e9 100644 --- a/blocks/antelope/README.md +++ b/blocks/antelope/README.md @@ -39,5 +39,5 @@ Kind: map Input: source: sf.substreams.v1.Clock Input: source: sf.antelope.type.v1.Block Output Type: proto:antelope.Events -Hash: 9e6d04e82d8c5bce50f835b57ce669e79af4b7b2 +Hash: 2ee4412cab8f461aaf8de27c2e970970a545117c ``` diff --git a/proto/antelope.proto b/proto/antelope.proto index cc320c4..7d5ccc4 100644 --- a/proto/antelope.proto +++ b/proto/antelope.proto @@ -15,7 +15,6 @@ message Events { repeated RamOp ram_ops = 8; } - message Block { // clock google.protobuf.Timestamp time = 1;