diff --git a/src/json_rescue_v5_extract.rs b/src/json_rescue_v5_extract.rs index 6ac255a..d9dbe89 100644 --- a/src/json_rescue_v5_extract.rs +++ b/src/json_rescue_v5_extract.rs @@ -1,6 +1,6 @@ use crate::{ - json_rescue_v5_compat::{TransactionDataView, TransactionViewV5}, - schema_transaction::{WarehouseEvent, WarehouseTxMaster}, + json_rescue_v5_compat::{ScriptView, TransactionDataView, TransactionViewV5}, + schema_transaction::{RelationLabel, WarehouseEvent, WarehouseTxMaster}, unzip_temp::decompress_tar_archive, }; use anyhow::{anyhow, Context, Result}; @@ -24,11 +24,24 @@ pub fn extract_v5_json_rescue( let event_vec = vec![]; for t in txs { - // dbg!(&t.hash); + // if let Some(UserTransaction {}) == &t { + // dbg!(&t); + // } let mut wtxs = WarehouseTxMaster::default(); - match t.transaction { - TransactionDataView::UserTransaction { sender, .. } => { + match &t.transaction { + TransactionDataView::UserTransaction { sender, script, .. } => { + // dbg!(&t); wtxs.sender = AccountAddress::from_hex_literal(&sender.to_hex_literal())?; + wtxs.tx_hash = t.hash; + + wtxs.function = make_function_name(script); + + wtxs.relation_label = guess_relation(&wtxs.function); + // wtxs.events + // wtxs.block_timestamp + // wtxs.entry_function + // wtxs. + tx_vec.push(wtxs); } TransactionDataView::BlockMetadata { timestamp_usecs: _ } => { @@ -71,3 +84,30 @@ pub fn list_all_json_files(search_dir: &Path) -> Result> { let vec_pathbuf = glob::glob(&pattern)?.map(|el| el.unwrap()).collect(); Ok(vec_pathbuf) } + +// TODO: gross borrows, lazy. +fn make_function_name(script: &ScriptView) -> String { + let module = script.module_name.as_ref(); + + let function = script.function_name.as_ref(); + + format!( + "0x::{}::{}", + module.unwrap_or(&"none".to_string()), + function.unwrap_or(&"none".to_string()) + ) +} + +fn guess_relation(script_name: &str) -> RelationLabel { + if script_name.contains("minerstate_commit") { + RelationLabel::Miner + } else if script_name.contains("create_user_by_coin_tx") { + // TODO: get the address + RelationLabel::Onboarding(AccountAddress::ZERO) + } else if script_name.contains("set_wallet_type") { + RelationLabel::Configuration + } else { + dbg!(&script_name); + RelationLabel::Tx + } +} diff --git a/src/json_rescue_v5_load.rs b/src/json_rescue_v5_load.rs index b759d68..ab58175 100644 --- a/src/json_rescue_v5_load.rs +++ b/src/json_rescue_v5_load.rs @@ -1,20 +1,27 @@ -use crate::json_rescue_v5_extract::{ - decompress_to_temppath, extract_v5_json_rescue, list_all_json_files, +use crate::{ + json_rescue_v5_extract::{decompress_to_temppath, extract_v5_json_rescue, list_all_json_files}, + load_tx_cypher::tx_batch, }; use anyhow::Result; +use log::info; +use neo4rs::Graph; use std::path::Path; /// from a tgz file decompress all the .json files in archive /// and then read into the warehouse record format -pub fn e2e_decompress_and_extract(tgz_file: &Path) -> Result<()> { +pub async fn e2e_decompress_and_extract(tgz_file: &Path, pool: &Graph) -> Result { let temppath = decompress_to_temppath(tgz_file)?; let json_vec = list_all_json_files(temppath.path())?; + let mut transactions = 0u64; for j in json_vec { if let Ok((r, _e)) = extract_v5_json_rescue(&j) { - dbg!(&r.len()); + // tx_batch(&r, pool, 250, j.to_str().unwrap()).await?; + transactions += r.len() as u64; } } - Ok(()) + info!("V5 transactions processed: {}", transactions); + + Ok(transactions) } diff --git a/src/lib.rs b/src/lib.rs index b10fa3a..53c2ac1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,7 @@ pub mod schema_account_state; pub mod schema_exchange_orders; pub mod schema_transaction; pub mod unzip_temp; +pub mod v5_rpc_to_raw; pub mod warehouse_cli; use std::sync::Once; diff --git a/src/schema_transaction.rs b/src/schema_transaction.rs index a264cdb..df82947 100644 --- a/src/schema_transaction.rs +++ b/src/schema_transaction.rs @@ -124,6 +124,7 @@ impl WarehouseTxMaster { self.function, self.sender.to_hex_literal(), tx_args, + // TODO: should be from relation_label.get_recipient self.recipient.unwrap_or(self.sender).to_hex_literal(), ) } diff --git a/src/v5_rpc_to_raw.rs b/src/v5_rpc_to_raw.rs new file mode 100644 index 0000000..f6ff861 --- /dev/null +++ b/src/v5_rpc_to_raw.rs @@ -0,0 +1,12 @@ +use crate::json_rescue_v5_compat::BytesView; + +pub fn bytesview_to_transaction_v5(b: &BytesView) { + dbg!(&b.inner()); +} + +#[test] +fn test_bytesview_cast() { + let bytes = "00c8336044cdf1878d9738ed0a041b235e0000000000000000030000000000000000000000000000000111546f7765725374617465536372697074731d6d696e657273746174655f636f6d6d69745f62795f6f70657261746f720005107ec16859c24200d8e074809d252ac740212022229f389e88b56c48527f456f68cb765cdb792c4b5d2cac46d489364f61b106ec0aea0a003d17e04e050af8605fbb3e316407ea8fe2ccdf237e1b5a9ef8bf9ec3558e7ae8ffca3468d88e96e8121c3749f6681863b172206320934f8cc394fa5041d8d2d1490021bfdba4840f2f374415b5fdc52e25d79e873558fe090d7e627cc0bfac3594a2001f836d03ba301a45c2ce647f4d072542399a6b1f5ceeabead77905b93ed77d910079946e47b3bce5747328e9594f30f8ff6825a0564cb2468cbc50ac81f0e1990eff89b7eb81b0348288e91fd5a3452d9a0b275d1ecef2c2bab7defbd352472b39f3007e42b3f04a6063b8f7ac7900f01e1bdf325e0bdbac6ea8baf6b1a1dae6f248c3ff919dfca3ec7f26c93ed4d5513d81432a5b9e167fefde6641eb601947c3517dd90066dc5742273263641a3417e83a95eb999a93d089d97686c6b28e1f3f1f38daa4006575fe0882b10e96354ff2bb50b0d9e85dcca13c53763b6d5020b78e81fce36100387f52eb5d9a0ac573d86cf1e762a8d7ccd07af875e3a80ca65f662cb6cd85e4000ea581fa8a4e5a1b68cd7bc4c3605fd792c990a4a735183d728645cb1150fd770021f15ce84fe56c6be017fc566200303d5d7b75a1480d013a744de6725631cb76001cca7768b321c438013c02623241b3864239115e447f81db09769b3c36bfa3e3003b3cda85c084e072959c9d833e88525c9f1259e446db3918ef684434845fb9ccffcca6de724a82fea06b29102ac6fcc97ab857873a92437e21dff917dbfc4e1d69002b06376d6e656eb727b32715fcb9efce084e7a7260dbfe6d438464936796afbd0026de3ddd474aead0ff49f08a311c4117b5ae88886da407bc4e3b9716056cea4500084363abf26dd9181af9418bcbcadaec3625941c6e0a0afbc8df586a8a1d744e0001d8a59df8053336b547b2eabea4faf8eb00e7e6472b026046f719ca5b8fd9a5005d303e6c14375e109c37459f36b72d972b29ede44464a48f1702a9f82e8bd0d8ffc089001820dc51c3e1a1d1a205871a802fc6288b0256d85a5827f70c93f5a179006974fb2130ec2d74c7e8281f1b5e8847a4cdaba0c2abc54ee336ebea6b97febcffeca2e1b10eea280405f4c01de7687bc09d010c5b15b7e75ef8a837d5f412b14100546d651b59c0e50dcce208e1b3a39bc5625ef2797886435928ef79b4a45f77a100081ec5c722fe0611f9428fb92ad267637d163336b2b8db661f1921864392c92f001ce3095ad551c70930e8ab25ce2fa033fe3713e610aa1e694823bec8218d622200110e180b55d3b51dce5cf5c3e7469e41c946e1a5d84b3ae4fd4f4911b9a236a90008e18492ddcbeb44330a6c336d849a420660c1f3f2e812d7a6a76b762aa6508f00011520920a153f325441da1cc1122afaddd742a346dbaaa9347ea7e621252f97001f8caf5edf79baf850632ecc4ce4b0b1b712299917f5408beee17245471ad0b4ffe10f1cd147c03cd217a60b54a40048d3a217328cc8ae90a465b219faa8a69667007a880a6081666b909df985c32efbc00ec53372d6960a8a5be11315d03eda1a2dffc89bc7cbd2c2e9cafee92e76decfa1dd79883b47449d2591f96ce12c08dfc9dd00716971162dd4d476a66c0dba6b210d931a3cfbff80412d60d7a41800f9975f62ffa0e8f4b7d55ef173ef66737c3c2138d51ac50cf0fc45d0e7f00ad66bd214359f005e6447c8afb5f62dd613bf070036d10ffdc69586e21f46e710d88995c4990c4effcdaa9ceb2104a956e8275dad8ab09585bce2aae2f55d73685640df82bc2d85cd0011bf5c5e51a8ff3ea9f8612456f31cc52b32b6bfe8d35fb22ab385f75a8fe2900002fde26ce3f1f2b2d1b2aec2202450106b5fd301029321f7e8f6daefb643fb890005f2a69bbcc02994872f6ca6e255660785b439bc4eb82e466863a070f250cc5300010cd0475df7f2f890703f8a0d08316951b1de551e83a718b5e083d3b63c9ec108000e27070000000008000200000000000010270000000000000100000000000000034741530486796100000000010020d78ee09ad0cfd2d7da6c2cc5fda1d035542d0177cf7aefc34aae63bf39ec127b40efcaeb1d04e32ecfa6845a78825dd878e7b7367853643b412485c5d4cc775ba20b3b4eecf831f3b59273bde833d134763d6f3a2aaebe037112868faedb3f6508".as_bytes(); + + bytesview_to_transaction_v5(&BytesView::new(bytes)); +} diff --git a/tests/test_json_rescue_v5_load.rs b/tests/test_json_rescue_v5_load.rs new file mode 100644 index 0000000..b92b54d --- /dev/null +++ b/tests/test_json_rescue_v5_load.rs @@ -0,0 +1,29 @@ +mod support; + +use libra_forensic_db::{ + json_rescue_v5_load, + neo4j_init::{get_neo4j_localhost_pool, maybe_create_indexes}, +}; +use support::{fixtures, neo4j_testcontainer::start_neo4j_container}; + +#[tokio::test] +async fn test_load_all_tgz() -> anyhow::Result<()> { + libra_forensic_db::log_setup(); + + let c = start_neo4j_container(); + let port = c.get_host_port_ipv4(7687); + let pool = get_neo4j_localhost_pool(port) + .await + .expect("could not get neo4j connection pool"); + maybe_create_indexes(&pool) + .await + .expect("could start index"); + + let path = fixtures::v5_json_tx_path().join("0-99900.tgz"); + + let tx_count = json_rescue_v5_load::e2e_decompress_and_extract(&path, &pool).await?; + dbg!(&tx_count); + assert!(tx_count == 6157); + + Ok(()) +} diff --git a/tests/test_parse_json_rescue_v5.rs b/tests/test_json_rescue_v5_parse.rs similarity index 100% rename from tests/test_parse_json_rescue_v5.rs rename to tests/test_json_rescue_v5_parse.rs