From 6524cd20d121dd76c5d9055b7188881c589f5ebc Mon Sep 17 00:00:00 2001 From: nikurt <86772482+nikurt@users.noreply.github.com> Date: Wed, 26 Jul 2023 20:25:42 +0200 Subject: [PATCH] fix(state-viewer): use hashes to determine scanning range (#9351) For example, the ChunkExtra column key is block_hash + shard_uid. It's much more convenient to use the tool as `neard view-state scan-db-column --column ChunkExtra --from-hash 11153w1N1bFv8fun2FdHfTMj97mF3gaUoV6fdrAQU9 --max-keys 4` to get ChunkExtra of that block. --- tools/state-viewer/src/apply_chunk.rs | 18 ++++++++------- tools/state-viewer/src/cli.rs | 33 +++++++++++++++------------ 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/tools/state-viewer/src/apply_chunk.rs b/tools/state-viewer/src/apply_chunk.rs index 6520ad2c505..3b2a0759cbc 100644 --- a/tools/state-viewer/src/apply_chunk.rs +++ b/tools/state-viewer/src/apply_chunk.rs @@ -20,7 +20,6 @@ use rand::rngs::StdRng; use rand::seq::SliceRandom; use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use tracing::warn; // like ChainStoreUpdate::get_incoming_receipts_for_shard(), but for the case when we don't // know of a block containing the target chunk @@ -48,12 +47,15 @@ fn get_incoming_receipts( chunks.sort_by_key(|chunk| chunk.shard_id()); for chunk in chunks { - let partial_encoded_chunk = chain_store.get_partial_chunk(&chunk.chunk_hash()).unwrap(); - for receipt in partial_encoded_chunk.receipts().iter() { - let ReceiptProof(_, shard_proof) = receipt; - if shard_proof.to_shard_id == shard_id { - receipt_proofs.push(receipt.clone()); + if let Ok(partial_encoded_chunk) = chain_store.get_partial_chunk(&chunk.chunk_hash()) { + for receipt in partial_encoded_chunk.receipts().iter() { + let ReceiptProof(_, shard_proof) = receipt; + if shard_proof.to_shard_id == shard_id { + receipt_proofs.push(receipt.clone()); + } } + } else { + tracing::error!(target: "state-viewer", chunk_hash = ?chunk.chunk_hash(), "Failed to get a partial chunk"); } } @@ -243,7 +245,7 @@ fn apply_tx_in_chunk( let chunk = match chain_store.get_chunk(&chunk_hash) { Ok(c) => c, Err(_) => { - warn!(target: "state-viewer", "chunk hash {:?} appears in DBCol::ChunkHashesByHeight but the chunk is not saved", &chunk_hash); + tracing::warn!(target: "state-viewer", "chunk hash {:?} appears in DBCol::ChunkHashesByHeight but the chunk is not saved", &chunk_hash); continue; } }; @@ -377,7 +379,7 @@ fn apply_receipt_in_chunk( let chunk = match chain_store.get_chunk(&chunk_hash) { Ok(c) => c, Err(_) => { - warn!(target: "state-viewer", "chunk hash {:?} appears in DBCol::ChunkHashesByHeight but the chunk is not saved", &chunk_hash); + tracing::warn!(target: "state-viewer", "chunk hash {:?} appears in DBCol::ChunkHashesByHeight but the chunk is not saved", &chunk_hash); continue; } }; diff --git a/tools/state-viewer/src/cli.rs b/tools/state-viewer/src/cli.rs index fa18b270b0a..a09a0ecdf9e 100644 --- a/tools/state-viewer/src/cli.rs +++ b/tools/state-viewer/src/cli.rs @@ -2,19 +2,15 @@ use crate::commands::*; use crate::contract_accounts::ContractAccountFilter; use crate::rocksdb_stats::get_rocksdb_stats; use crate::trie_iteration_benchmark::TrieIterationBenchmarkCmd; - +use borsh::BorshSerialize; use near_chain_configs::{GenesisChangeConfig, GenesisValidationMode}; - use near_primitives::account::id::AccountId; use near_primitives::hash::CryptoHash; use near_primitives::sharding::ChunkHash; - use near_primitives::types::{BlockHeight, ShardId}; use near_store::{Mode, NodeStorage, Store, Temperature}; use nearcore::{load_config, NearConfig}; - use std::path::{Path, PathBuf}; - use std::str::FromStr; #[derive(clap::Subcommand)] @@ -525,6 +521,8 @@ pub struct ScanDbColumnCmd { #[clap(long)] from_bytes: Option, #[clap(long)] + from_hash: Option, + #[clap(long)] to: Option, // List of comma-separated u8-values. // For example, if a column key starts wth ShardUId and you want to scan starting from s2.v1 use `--from-bytes 1,0,0,0,2,0,0,0`. @@ -532,6 +530,8 @@ pub struct ScanDbColumnCmd { #[clap(long)] to_bytes: Option, #[clap(long)] + to_hash: Option, + #[clap(long)] max_keys: Option, #[clap(long, default_value = "false")] no_value: bool, @@ -539,8 +539,8 @@ pub struct ScanDbColumnCmd { impl ScanDbColumnCmd { pub fn run(self, store: Store) { - let lower_bound = Self::prefix(self.from, self.from_bytes); - let upper_bound = Self::prefix(self.to, self.to_bytes); + let lower_bound = Self::prefix(self.from, self.from_bytes, self.from_hash); + let upper_bound = Self::prefix(self.to, self.to_bytes, self.to_hash); crate::scan_db::scan_db_column( &self.column, lower_bound.as_deref().map(|v| v.as_ref()), @@ -551,16 +551,19 @@ impl ScanDbColumnCmd { ) } - fn prefix(s: Option, bytes: Option) -> Option> { - match (s, bytes) { - (None, None) => None, - (Some(s), None) => Some(s.into_bytes()), - (None, Some(bytes)) => { + fn prefix( + s: Option, + bytes: Option, + hash: Option, + ) -> Option> { + match (s, bytes, hash) { + (None, None, None) => None, + (Some(s), None, None) => Some(s.into_bytes()), + (None, Some(bytes), None) => { Some(bytes.split(",").map(|s| s.parse::().unwrap()).collect::>()) } - (Some(_), Some(_)) => { - panic!("Provided both a Vec and a String as a prefix") - } + (None, None, Some(hash)) => Some(hash.try_to_vec().unwrap()), + _ => panic!("Need to provide exactly one of bytes, str, or hash"), } } }