From 6f0cb160ec8ce47d29982aa292851740e1cfd13c Mon Sep 17 00:00:00 2001 From: 0o-de-lally <1364012+0o-de-lally@users.noreply.github.com> Date: Sat, 23 Nov 2024 15:48:44 -0500 Subject: [PATCH] add struct layout for v5 api outputs --- Cargo.toml | 1 + src/json_rescue_v5_parse.rs | 475 +++++++++++++++++++++++++++++ src/lib.rs | 1 + tests/support/fixtures.rs | 5 + tests/test_parse_json_rescue_v5.rs | 16 + 5 files changed, 498 insertions(+) create mode 100644 src/json_rescue_v5_parse.rs create mode 100644 tests/test_parse_json_rescue_v5.rs diff --git a/Cargo.toml b/Cargo.toml index 93405d1..c4b2023 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ once_cell = "^1.2" serde = { version = "^1.0", features = ["derive", "rc"] } serde_json = { version = "^1", features = ["preserve_order"] } tokio = { version = "1", features = ["full"] } +hex = "0.4.3" [dev-dependencies] sqlx = { version = "0.8", features = [ "runtime-tokio", "tls-native-tls", "sqlite", "migrate", "macros", "derive", "postgres"] } diff --git a/src/json_rescue_v5_parse.rs b/src/json_rescue_v5_parse.rs new file mode 100644 index 0000000..07545e7 --- /dev/null +++ b/src/json_rescue_v5_parse.rs @@ -0,0 +1,475 @@ +// NOTE: ported from libra-legacy-v6/json-rpc/types/src/views.rs +use anyhow::Result; +use diem_crypto::HashValue; +use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer}; + +// use diem_types::transaction::TransactionArgument; +use libra_backwards_compatibility::version_five::{ + event_v5::EventKeyV5 as EventKey, + // language_storage_v5::TypeTagV5 as TypeTag, + legacy_address_v5::LegacyAddressV5 as AccountAddress, + // transaction_type_v5::AbortLocation, +}; +// TODO check v5 compatibility +// use diem_types::contract_event::ContractEvent; +// use diem_types::transaction::Script; +// use diem_types::vm_status::KeptVMStatus; + +use hex::FromHex; +/// This is the output of a JSON API request on the V5 data +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub struct TransactionView { + pub version: u64, + pub transaction: TransactionDataView, + pub hash: HashValue, + pub bytes: BytesView, + pub events: Vec, + pub vm_status: VMStatusView, + pub gas_used: u64, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum VMStatusView { + Executed, + OutOfGas, + MoveAbort { + location: String, + abort_code: u64, + explanation: Option, + }, + ExecutionFailure { + location: String, + function_index: u16, + code_offset: u16, + }, + MiscellaneousError, + VerificationError, + DeserializationError, + PublishingFailure, + #[serde(other)] + Unknown, +} + +impl VMStatusView { + pub fn is_executed(&self) -> bool { + matches!(self, Self::Executed) + } +} + +impl std::fmt::Display for VMStatusView { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + VMStatusView::Executed => write!(f, "Executed"), + VMStatusView::OutOfGas => write!(f, "Out of Gas"), + VMStatusView::MoveAbort { + location, + abort_code, + explanation, + } => { + write!(f, "Move Abort: {} at {}", abort_code, location)?; + if let Some(explanation) = explanation { + write!(f, "\nExplanation:\n{:?}", explanation)? + } + Ok(()) + } + VMStatusView::ExecutionFailure { + location, + function_index, + code_offset, + } => write!( + f, + "Execution failure: {} {} {}", + location, function_index, code_offset + ), + VMStatusView::MiscellaneousError => write!(f, "Miscellaneous Error"), + VMStatusView::VerificationError => write!(f, "Verification Error"), + VMStatusView::DeserializationError => write!(f, "Deserialization Error"), + VMStatusView::PublishingFailure => write!(f, "Publishing Failure"), + VMStatusView::Unknown => write!(f, "Unknown Error"), + } + } +} + +#[allow(clippy::large_enum_variant)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(tag = "type")] +pub enum TransactionDataView { + #[serde(rename = "blockmetadata")] + BlockMetadata { timestamp_usecs: u64 }, + #[serde(rename = "writeset")] + WriteSet {}, + #[serde(rename = "user")] + UserTransaction { + sender: AccountAddress, + signature_scheme: String, + signature: BytesView, + public_key: BytesView, + #[serde(skip_serializing_if = "Option::is_none")] + secondary_signers: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + secondary_signature_schemes: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + secondary_signatures: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + secondary_public_keys: Option>, + sequence_number: u64, + chain_id: u8, + max_gas_amount: u64, + gas_unit_price: u64, + gas_currency: String, + expiration_timestamp_secs: u64, + script_hash: HashValue, + script_bytes: BytesView, + script: ScriptView, + }, + #[serde(rename = "unknown")] + #[serde(other)] + UnknownTransaction, +} + +#[derive(Clone, PartialEq)] +pub struct BytesView(pub Box<[u8]>); + +impl BytesView { + pub fn new>>(bytes: T) -> Self { + Self(bytes.into()) + } + + pub fn into_inner(self) -> Box<[u8]> { + self.0 + } + + pub fn inner(&self) -> &[u8] { + &self.0 + } +} + +impl std::ops::Deref for BytesView { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::convert::AsRef<[u8]> for BytesView { + fn as_ref(&self) -> &[u8] { + self.inner() + } +} + +impl std::fmt::Display for BytesView { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + for byte in self.inner() { + write!(f, "{:02x}", byte)?; + } + Ok(()) + } +} + +impl std::fmt::Debug for BytesView { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "BytesView(\"{}\")", self) + } +} + +impl From<&[u8]> for BytesView { + fn from(bytes: &[u8]) -> Self { + Self(bytes.into()) + } +} + +impl From> for BytesView { + fn from(bytes: Vec) -> Self { + Self(bytes.into_boxed_slice()) + } +} + +impl<'de> Deserialize<'de> for BytesView { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = ::deserialize(deserializer)?; + >::from_hex(s) + .map_err(|e| D::Error::custom(e.to_string())) + .map(Into::into) + } +} + +impl Serialize for BytesView { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + hex::encode(self).serialize(serializer) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct EventView { + pub key: EventKey, + pub sequence_number: u64, + pub transaction_version: u64, + pub data: EventDataView, +} + +// impl TryFrom<(u64, ContractEvent)> for EventView { +// type Error = Error; + +// fn try_from((txn_version, event): (u64, ContractEvent)) -> Result { +// Ok(EventView { +// key: *event.key(), +// sequence_number: event.sequence_number(), +// transaction_version: txn_version, +// data: event.try_into()?, +// }) +// } +// } + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub struct MoveAbortExplanationView { + pub category: String, + pub category_description: String, + pub reason: String, + pub reason_description: String, +} + +// impl std::fmt::Display for MoveAbortExplanationView { +// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +// writeln!(f, "Error Category: {}", self.category)?; +// writeln!(f, "\tCategory Description: {}", self.category_description)?; +// writeln!(f, "Error Reason: {}", self.reason)?; +// writeln!(f, "\tReason Description: {}", self.reason_description) +// } +// } + +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] +pub struct ScriptView { + // script name / type + pub r#type: String, + + // script code bytes + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, + // script arguments, converted into string with type information + #[serde(skip_serializing_if = "Option::is_none")] + pub arguments: Option>, + // script function arguments, converted into hex encoded BCS bytes + #[serde(skip_serializing_if = "Option::is_none")] + pub arguments_bcs: Option>, + // script type arguments, converted into string + #[serde(skip_serializing_if = "Option::is_none")] + pub type_arguments: Option>, + + // the following fields are legacy fields: maybe removed in the future + // please move to use above fields + + // peer_to_peer_transaction, other known script name or unknown + // because of a bug, we never rendered mint_transaction + // this is deprecated, please switch to use field `name` which is script name + + // peer_to_peer_transaction + #[serde(skip_serializing_if = "Option::is_none")] + pub receiver: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub amount: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub currency: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub metadata: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub metadata_signature: Option, + + // Script functions + // The address that the module is published under + #[serde(skip_serializing_if = "Option::is_none")] + pub module_address: Option, + // The name of the module that the called function is defined in + #[serde(skip_serializing_if = "Option::is_none")] + pub module_name: Option, + // The (unqualified) name of the function being called. + #[serde(skip_serializing_if = "Option::is_none")] + pub function_name: Option, +} + +// impl ScriptView { +// pub fn unknown() -> Self { +// ScriptView { +// r#type: "unknown".to_string(), +// ..Default::default() +// } +// } +// } + +// impl From<&Script> for ScriptView { +// fn from(script: &Script) -> Self { +// let name = ScriptCall::decode(script) +// .map(|script_call| script_call.name().to_owned()) +// .unwrap_or_else(|| "unknown".to_owned()); +// let ty_args: Vec = script +// .ty_args() +// .iter() +// .map(|type_tag| match type_tag { +// TypeTag::Struct(StructTag { module, .. }) => module.to_string(), +// tag => format!("{}", tag), +// }) +// .collect(); +// let mut view = ScriptView { +// r#type: name.clone(), +// code: Some(script.code().into()), +// arguments: Some( +// script +// .args() +// .iter() +// .map(|arg| format!("{:?}", &arg)) +// .collect(), +// ), +// type_arguments: Some(ty_args.clone()), +// ..Default::default() +// }; + +// // handle legacy fields, backward compatible +// if name == "peer_to_peer_with_metadata" { +// if let [TransactionArgument::Address(receiver), TransactionArgument::U64(amount), TransactionArgument::U8Vector(metadata), TransactionArgument::U8Vector(metadata_signature)] = +// script.args() +// { +// view.receiver = Some(*receiver); +// view.amount = Some(*amount); +// view.currency = Some( +// ty_args +// .get(0) +// .unwrap_or(&"unknown_currency".to_string()) +// .to_string(), +// ); +// view.metadata = Some(BytesView::new(metadata.as_ref())); +// view.metadata_signature = Some(BytesView::new(metadata_signature.as_ref())); +// } +// } + +// view +// } +// } + +// impl From<&ScriptFunction> for ScriptView { +// fn from(script: &ScriptFunction) -> Self { +// let ty_args: Vec = script +// .ty_args() +// .iter() +// .map(|type_tag| match type_tag { +// TypeTag::Struct(StructTag { module, .. }) => module.to_string(), +// tag => format!("{}", tag), +// }) +// .collect(); +// ScriptView { +// r#type: "script_function".to_string(), +// module_address: Some(*script.module().address()), +// module_name: Some(script.module().name().to_string()), +// function_name: Some(script.function().to_string()), +// arguments_bcs: Some( +// script +// .args() +// .iter() +// .map(|arg| BytesView::from(arg.as_ref())) +// .collect(), +// ), +// type_arguments: Some(ty_args), +// ..Default::default() +// } +// } +// } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(tag = "type")] +pub enum EventDataView { + #[serde(rename = "burn")] + Burn { + amount: AmountView, + preburn_address: AccountAddress, + }, + #[serde(rename = "cancelburn")] + CancelBurn { + amount: AmountView, + preburn_address: AccountAddress, + }, + #[serde(rename = "mint")] + Mint { amount: AmountView }, + #[serde(rename = "to_xdx_exchange_rate_update")] + ToXDXExchangeRateUpdate { + currency_code: String, + new_to_xdx_exchange_rate: f32, + }, + #[serde(rename = "preburn")] + Preburn { + amount: AmountView, + preburn_address: AccountAddress, + }, + #[serde(rename = "receivedpayment")] + ReceivedPayment { + amount: AmountView, + sender: AccountAddress, + receiver: AccountAddress, + metadata: BytesView, + }, + #[serde(rename = "sentpayment")] + SentPayment { + amount: AmountView, + receiver: AccountAddress, + sender: AccountAddress, + metadata: BytesView, + }, + #[serde(rename = "admintransaction")] + AdminTransaction { committed_timestamp_secs: u64 }, + #[serde(rename = "newepoch")] + NewEpoch { epoch: u64 }, + #[serde(rename = "newblock")] + NewBlock { + round: u64, + proposer: AccountAddress, + proposed_time: u64, + }, + #[serde(rename = "receivedmint")] + ReceivedMint { + amount: AmountView, + destination_address: AccountAddress, + }, + #[serde(rename = "compliancekeyrotation")] + ComplianceKeyRotation { + new_compliance_public_key: BytesView, + time_rotated_seconds: u64, + }, + #[serde(rename = "baseurlrotation")] + BaseUrlRotation { + new_base_url: String, + time_rotated_seconds: u64, + }, + #[serde(rename = "createaccount")] + CreateAccount { + created_address: AccountAddress, + role_id: u64, + }, + #[serde(rename = "diemiddomain")] + DiemIdDomain { + // Whether a domain was added or removed + removed: bool, + // Diem ID Domain string of the account + // TODO + domain: String, + // domain: DiemIdVaspDomainIdentifier, + // On-chain account address + address: AccountAddress, + }, + #[serde(rename = "unknown")] + Unknown { bytes: Option }, + + // used by client to deserialize server response + #[serde(other)] + UnknownToClient, +} + +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)] +pub struct AmountView { + pub amount: u64, + pub currency: String, +} diff --git a/src/lib.rs b/src/lib.rs index c1c0c99..7303436 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod enrich_exchange_onboarding; pub mod enrich_whitepages; pub mod extract_snapshot; pub mod extract_transactions; +pub mod json_rescue_v5_parse; pub mod load; pub mod load_account_state; pub mod load_exchange_orders; diff --git a/tests/support/fixtures.rs b/tests/support/fixtures.rs index 8cdd53f..6fe6880 100644 --- a/tests/support/fixtures.rs +++ b/tests/support/fixtures.rs @@ -15,6 +15,11 @@ pub fn v7_fixtures_gzipped() -> PathBuf { p.join("tests/fixtures/v7/transaction_38100001-.541f_gzipped") } +pub fn v5_json_tx_path() -> PathBuf { + let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + p.join("tests/fixtures/v5/json-rescue") +} + pub fn v5_state_manifest_fixtures_path() -> PathBuf { let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let dir = p.join("tests/fixtures/v5/state_ver_119757649.17a8"); diff --git a/tests/test_parse_json_rescue_v5.rs b/tests/test_parse_json_rescue_v5.rs new file mode 100644 index 0000000..aec365b --- /dev/null +++ b/tests/test_parse_json_rescue_v5.rs @@ -0,0 +1,16 @@ +mod support; +use libra_forensic_db::json_rescue_v5_parse::TransactionView; +use support::fixtures; + +#[test] +fn test_rescue_v5_parse() -> anyhow::Result<()> { + let path = fixtures::v5_json_tx_path().join("example_user_tx.json"); + let json = std::fs::read_to_string(path)?; + + let txs: Vec = serde_json::from_str(&json)?; + + let first = txs.first().unwrap(); + assert!(first.gas_used == 1429); + + Ok(()) +}