cli entrypoint

0LNetworkCommunity · Nov 26, 2024 · 1e1d8c2 · 1e1d8c2
1 parent 4b3be0c
commit 1e1d8c2
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 28 deletions.
diff --git a/src/json_rescue_v5_extract.rs b/src/json_rescue_v5_extract.rs
@@ -39,8 +39,7 @@ pub fn extract_v5_json_rescue(
         let mut wtxs = WarehouseTxMaster::default();
         match &t.transaction {
             TransactionDataView::UserTransaction { sender, script, .. } => {
-
-                wtxs.sender = cast_legacy_account(&sender)?;
+                wtxs.sender = cast_legacy_account(sender)?;
 
                 // must cast from V5 Hashvalue buffer layout
                 wtxs.tx_hash = HashValue::from_slice(&t.hash.to_vec())?;
@@ -84,24 +83,17 @@ pub fn decode_transaction_args(wtx: &mut WarehouseTxMaster, tx_bytes: &[u8]) ->
                 wtx.entry_function = Some(EntryFunctionArgs::V5(sf.to_owned()));
 
                 match sf {
-                    ScriptFunctionCallGenesis::BalanceTransfer {
-                        destination,
-                        ..
-                    } => {
+                    ScriptFunctionCallGenesis::BalanceTransfer { destination, .. } => {
                         wtx.relation_label =
-                            RelationLabel::Transfer(cast_legacy_account(&destination)?);
+                            RelationLabel::Transfer(cast_legacy_account(destination)?);
                     }
-                    ScriptFunctionCallGenesis::CreateAccUser {
-                        ..
-                    } => {
+                    ScriptFunctionCallGenesis::CreateAccUser { .. } => {
                         // onboards self
-                        wtx.relation_label = RelationLabel::Onboarding(wtx.sender.clone());
+                        wtx.relation_label = RelationLabel::Onboarding(wtx.sender);
                     }
-                    ScriptFunctionCallGenesis::CreateAccVal {
-                        ..
-                    } => {
+                    ScriptFunctionCallGenesis::CreateAccVal { .. } => {
                         // onboards self
-                        wtx.relation_label = RelationLabel::Onboarding(wtx.sender.clone());
+                        wtx.relation_label = RelationLabel::Onboarding(wtx.sender);
                     }
 
                     ScriptFunctionCallGenesis::CreateUserByCoinTx { account, .. } => {
@@ -128,9 +120,7 @@ pub fn decode_transaction_args(wtx: &mut WarehouseTxMaster, tx_bytes: &[u8]) ->
                     ScriptFunctionCallGenesis::MinerstateCommit { .. } => {
                         wtx.relation_label = RelationLabel::Miner;
                     }
-                    ScriptFunctionCallGenesis::MinerstateCommitByOperator {
-                        ..
-                    } => {
+                    ScriptFunctionCallGenesis::MinerstateCommitByOperator { .. } => {
                         wtx.relation_label = RelationLabel::Miner;
                     }
                     _ => {
@@ -145,10 +135,10 @@ pub fn decode_transaction_args(wtx: &mut WarehouseTxMaster, tx_bytes: &[u8]) ->
 
                 match sf {
                     ScriptFunctionCallV520::CreateAccUser { .. } => {
-                        wtx.relation_label = RelationLabel::Onboarding(wtx.sender.clone());
+                        wtx.relation_label = RelationLabel::Onboarding(wtx.sender);
                     }
                     ScriptFunctionCallV520::CreateAccVal { .. } => {
-                        wtx.relation_label = RelationLabel::Onboarding(wtx.sender.clone());
+                        wtx.relation_label = RelationLabel::Onboarding(wtx.sender);
                     }
 
                     ScriptFunctionCallV520::CreateValidatorAccount {
@@ -170,9 +160,7 @@ pub fn decode_transaction_args(wtx: &mut WarehouseTxMaster, tx_bytes: &[u8]) ->
                     ScriptFunctionCallV520::MinerstateCommit { .. } => {
                         wtx.relation_label = RelationLabel::Miner;
                     }
-                    ScriptFunctionCallV520::MinerstateCommitByOperator {
-                        ..
-                    } => {
+                    ScriptFunctionCallV520::MinerstateCommitByOperator { .. } => {
                         wtx.relation_label = RelationLabel::Miner;
                     }
                     _ => {
@@ -211,6 +199,19 @@ pub fn list_all_json_files(search_dir: &Path) -> Result<Vec<PathBuf>> {
     Ok(vec_pathbuf)
 }
 
+/// gets all json files decompressed from tgz
+pub fn list_all_tgz_archives(search_dir: &Path) -> Result<Vec<PathBuf>> {
+    let path = search_dir.canonicalize()?;
+
+    let pattern = format!(
+        "{}/**/*.tgz",
+        path.to_str().context("cannot parse starting dir")?
+    );
+
+    let vec_pathbuf = glob::glob(&pattern)?.map(|el| el.unwrap()).collect();
+    Ok(vec_pathbuf)
+}
+
 // TODO: gross borrows, lazy.
 fn make_function_name(script: &ScriptView) -> String {
     let module = script.module_name.as_ref();
@@ -224,7 +225,6 @@ fn make_function_name(script: &ScriptView) -> String {
     )
 }
 
-
 fn cast_legacy_account(legacy: &LegacyAddressV5) -> Result<AccountAddress> {
     Ok(AccountAddress::from_hex_literal(&legacy.to_hex_literal())?)
 }
diff --git a/src/json_rescue_v5_load.rs b/src/json_rescue_v5_load.rs
@@ -1,9 +1,11 @@
 use crate::{
-    json_rescue_v5_extract::{decompress_to_temppath, extract_v5_json_rescue, list_all_json_files},
+    json_rescue_v5_extract::{
+        decompress_to_temppath, extract_v5_json_rescue, list_all_json_files, list_all_tgz_archives,
+    },
     load_tx_cypher::tx_batch,
 };
 use anyhow::Result;
-use log::info;
+use log::{error, info};
 use neo4rs::Graph;
 use std::path::Path;
 
@@ -25,3 +27,21 @@ pub async fn e2e_decompress_and_extract(tgz_file: &Path, pool: &Graph) -> Result
 
     Ok(transactions)
 }
+
+pub async fn rip(start_dir: &Path, pool: &Graph) -> Result<u64> {
+    let tgz_list = list_all_tgz_archives(start_dir)?;
+    let mut txs = 0u64;
+    for p in tgz_list.iter() {
+        match e2e_decompress_and_extract(p, pool).await {
+            Ok(t) => txs += t,
+            Err(e) => {
+                error!(
+                    "could not load archive: {}, error: {}",
+                    p.display(),
+                    e.to_string()
+                );
+            }
+        }
+    }
+    Ok(txs)
+}
diff --git a/src/warehouse_cli.rs b/src/warehouse_cli.rs
@@ -7,6 +7,7 @@ use std::path::PathBuf;
 use crate::{
     enrich_exchange_onboarding::{self, ExchangeOnRamp},
     enrich_whitepages::{self, Whitepages},
+    json_rescue_v5_load,
     load::{ingest_all, try_load_one_archive},
     load_exchange_orders,
     neo4j_init::{self, get_credentials_from_env, PASS_ENV, URI_ENV, USER_ENV},
@@ -91,7 +92,7 @@ pub enum Sub {
         #[clap(long)]
         /// starting path for v5 .tgz files
         archive_dir: PathBuf,
-    }
+    },
 }
 
 impl WarehouseCli {
@@ -170,8 +171,10 @@ impl WarehouseCli {
 
                 println!("SUCCESS: {} owner accounts linked", owners_merged);
             }
-            VersionFiveTx { archive_dir } => {
+            Sub::VersionFiveTx { archive_dir } => {
+                let pool = try_db_connection_pool(self).await?;
 
+                json_rescue_v5_load::rip(archive_dir, &pool).await?;
             }
         };
         Ok(())