From ee0667d36932a46ee2e2e36419567479f202a8f0 Mon Sep 17 00:00:00 2001 From: steviez Date: Wed, 21 Aug 2024 15:16:00 -0500 Subject: [PATCH] ledger-tool: Make blockstore slot functional with no tx metadata (#2423) A previous commit unified the code to output a slot between the bigtable block and blockstore slot commands. In doing so, support for blockstore slot when tx metadata is absent was unintentionally broken This re-adds support for using the blockstore slot command when the blockstore does not contain tx metadata --- ledger-tool/src/blockstore.rs | 1 - ledger-tool/src/output.rs | 206 +++++++++++++++++++++++++--------- ledger-tool/tests/basic.rs | 1 - transaction-status/src/lib.rs | 68 ++++++++++- 4 files changed, 219 insertions(+), 57 deletions(-) diff --git a/ledger-tool/src/blockstore.rs b/ledger-tool/src/blockstore.rs index 903c232a7e230b..4ee47742de0994 100644 --- a/ledger-tool/src/blockstore.rs +++ b/ledger-tool/src/blockstore.rs @@ -1031,7 +1031,6 @@ fn do_blockstore_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) - let blockstore = crate::open_blockstore(&ledger_path, arg_matches, AccessType::Secondary); for slot in slots { - println!("Slot {slot}"); output_slot( &blockstore, slot, diff --git a/ledger-tool/src/output.rs b/ledger-tool/src/output.rs index dbe7ff3ef726cb..4739a0a22b86aa 100644 --- a/ledger-tool/src/output.rs +++ b/ledger-tool/src/output.rs @@ -24,11 +24,12 @@ use { hash::Hash, native_token::lamports_to_sol, pubkey::Pubkey, + transaction::VersionedTransaction, }, solana_transaction_status::{ - BlockEncodingOptions, ConfirmedBlock, EncodeError, EncodedConfirmedBlock, + BlockEncodingOptions, ConfirmedBlock, Encodable, EncodedConfirmedBlock, EncodedTransactionWithStatusMeta, EntrySummary, Rewards, TransactionDetails, - UiTransactionEncoding, VersionedConfirmedBlockWithEntries, + UiTransactionEncoding, VersionedConfirmedBlock, VersionedConfirmedBlockWithEntries, VersionedTransactionWithStatusMeta, }, std::{ @@ -462,24 +463,82 @@ impl EncodedConfirmedBlockWithEntries { pub(crate) fn encode_confirmed_block( confirmed_block: ConfirmedBlock, ) -> Result { - let encoded_block = confirmed_block - .encode_with_options( - UiTransactionEncoding::Base64, - BlockEncodingOptions { - transaction_details: TransactionDetails::Full, - show_rewards: true, - max_supported_transaction_version: Some(0), - }, - ) - .map_err(|err| match err { - EncodeError::UnsupportedTransactionVersion(version) => LedgerToolError::Generic( - format!("Failed to process unsupported transaction version ({version}) in block"), - ), - })?; + let encoded_block = confirmed_block.encode_with_options( + UiTransactionEncoding::Base64, + BlockEncodingOptions { + transaction_details: TransactionDetails::Full, + show_rewards: true, + max_supported_transaction_version: Some(0), + }, + )?; + let encoded_block: EncodedConfirmedBlock = encoded_block.into(); Ok(encoded_block) } +fn encode_versioned_transactions(block: BlockWithoutMetadata) -> EncodedConfirmedBlock { + let transactions = block + .transactions + .into_iter() + .map(|transaction| EncodedTransactionWithStatusMeta { + transaction: transaction.encode(UiTransactionEncoding::Base64), + meta: None, + version: None, + }) + .collect(); + + EncodedConfirmedBlock { + previous_blockhash: Hash::default().to_string(), + blockhash: block.blockhash, + parent_slot: block.parent_slot, + transactions, + rewards: Rewards::default(), + num_partitions: None, + block_time: None, + block_height: None, + } +} + +pub enum BlockContents { + VersionedConfirmedBlock(VersionedConfirmedBlock), + BlockWithoutMetadata(BlockWithoutMetadata), +} + +// A VersionedConfirmedBlock analogue for use when the transaction metadata +// fields are unavailable. Also supports non-full blocks +pub struct BlockWithoutMetadata { + pub blockhash: String, + pub parent_slot: Slot, + pub transactions: Vec, +} + +impl BlockContents { + pub fn transactions(&self) -> Box + '_> { + match self { + BlockContents::VersionedConfirmedBlock(block) => Box::new( + block + .transactions + .iter() + .map(|VersionedTransactionWithStatusMeta { transaction, .. }| transaction), + ), + BlockContents::BlockWithoutMetadata(block) => Box::new(block.transactions.iter()), + } + } +} + +impl TryFrom for EncodedConfirmedBlock { + type Error = LedgerToolError; + + fn try_from(block_contents: BlockContents) -> Result { + match block_contents { + BlockContents::VersionedConfirmedBlock(block) => { + encode_confirmed_block(ConfirmedBlock::from(block)) + } + BlockContents::BlockWithoutMetadata(block) => Ok(encode_versioned_transactions(block)), + } + } +} + pub fn output_slot( blockstore: &Blockstore, slot: Slot, @@ -488,26 +547,77 @@ pub fn output_slot( verbose_level: u64, all_program_ids: &mut HashMap, ) -> Result<()> { - if blockstore.is_dead(slot) { - if allow_dead_slots { - if *output_format == OutputFormat::Display { - println!(" Slot is dead"); - } - } else { - return Err(LedgerToolError::from(BlockstoreError::DeadSlot)); + let is_root = blockstore.is_root(slot); + let is_dead = blockstore.is_dead(slot); + if *output_format == OutputFormat::Display && verbose_level <= 1 { + if is_root && is_dead { + eprintln!("Slot {slot} is marked as both a root and dead, this shouldn't be possible"); } + println!( + "Slot {slot}{}", + if is_root { + " (root)" + } else if is_dead { + " (dead)" + } else { + "" + } + ); + } + + if is_dead && !allow_dead_slots { + return Err(LedgerToolError::from(BlockstoreError::DeadSlot)); } let Some(meta) = blockstore.meta(slot)? else { return Ok(()); }; - let VersionedConfirmedBlockWithEntries { block, entries } = blockstore - .get_complete_block_with_entries( - slot, - /*require_previous_blockhash:*/ false, - /*populate_entries:*/ true, - allow_dead_slots, - )?; + let (block_contents, entries) = match blockstore.get_complete_block_with_entries( + slot, + /*require_previous_blockhash:*/ false, + /*populate_entries:*/ true, + allow_dead_slots, + ) { + Ok(VersionedConfirmedBlockWithEntries { block, entries }) => { + (BlockContents::VersionedConfirmedBlock(block), entries) + } + Err(_) => { + // Transaction metadata could be missing, try to fetch just the + // entries and leave the metadata fields empty + let entries = blockstore.get_slot_entries(slot, /*shred_start_index:*/ 0)?; + + let blockhash = entries + .last() + .filter(|_| meta.is_full()) + .map(|entry| entry.hash) + .unwrap_or(Hash::default()); + let parent_slot = meta.parent_slot.unwrap_or(0); + + let mut entry_summaries = Vec::with_capacity(entries.len()); + let mut starting_transaction_index = 0; + let transactions = entries + .into_iter() + .flat_map(|entry| { + entry_summaries.push(EntrySummary { + num_hashes: entry.num_hashes, + hash: entry.hash, + num_transactions: entry.transactions.len() as u64, + starting_transaction_index, + }); + starting_transaction_index += entry.transactions.len(); + + entry.transactions + }) + .collect(); + + let block = BlockWithoutMetadata { + blockhash: blockhash.to_string(), + parent_slot, + transactions, + }; + (BlockContents::BlockWithoutMetadata(block), entry_summaries) + } + }; if verbose_level == 0 { if *output_format == OutputFormat::Display { @@ -531,24 +641,23 @@ pub fn output_slot( for entry in entries.iter() { num_hashes += entry.num_hashes; } + let blockhash = entries + .last() + .filter(|_| meta.is_full()) + .map(|entry| entry.hash) + .unwrap_or(Hash::default()); - let blockhash = if let Some(entry) = entries.last() { - entry.hash - } else { - Hash::default() - }; - - let transactions = block.transactions.len(); + let mut num_transactions = 0; let mut program_ids = HashMap::new(); - for VersionedTransactionWithStatusMeta { transaction, .. } in block.transactions.iter() - { + + for transaction in block_contents.transactions() { + num_transactions += 1; for program_id in get_program_ids(transaction) { *program_ids.entry(*program_id).or_insert(0) += 1; } } - println!( - " Transactions: {transactions}, hashes: {num_hashes}, block_hash: {blockhash}", + " Transactions: {num_transactions}, hashes: {num_hashes}, block_hash: {blockhash}", ); for (pubkey, count) in program_ids.iter() { *all_program_ids.entry(*pubkey).or_insert(0) += count; @@ -557,7 +666,7 @@ pub fn output_slot( output_sorted_program_ids(program_ids); } } else { - let encoded_block = encode_confirmed_block(ConfirmedBlock::from(block))?; + let encoded_block = EncodedConfirmedBlock::try_from(block_contents)?; let cli_block = CliBlockWithEntries { encoded_confirmed_block: EncodedConfirmedBlockWithEntries::try_from( encoded_block, @@ -591,7 +700,7 @@ pub fn output_ledger( let num_slots = num_slots.unwrap_or(Slot::MAX); let mut num_printed = 0; let mut all_program_ids = HashMap::new(); - for (slot, slot_meta) in slot_iterator { + for (slot, _slot_meta) in slot_iterator { if only_rooted && !blockstore.is_root(slot) { continue; } @@ -599,17 +708,6 @@ pub fn output_ledger( break; } - match output_format { - OutputFormat::Display => { - println!("Slot {} root?: {}", slot, blockstore.is_root(slot)) - } - OutputFormat::Json => { - serde_json::to_writer(stdout(), &slot_meta)?; - stdout().write_all(b",\n")?; - } - _ => unreachable!(), - } - if let Err(err) = output_slot( &blockstore, slot, diff --git a/ledger-tool/tests/basic.rs b/ledger-tool/tests/basic.rs index a034df43305e7a..2459f1287497c8 100644 --- a/ledger-tool/tests/basic.rs +++ b/ledger-tool/tests/basic.rs @@ -101,7 +101,6 @@ fn ledger_tool_copy_test(src_shred_compaction: &str, dst_shred_compaction: &str) assert!(src_slot_output.status.success()); assert!(dst_slot_output.status.success()); assert!(!src_slot_output.stdout.is_empty()); - assert_eq!(src_slot_output.stdout, dst_slot_output.stdout); } } diff --git a/transaction-status/src/lib.rs b/transaction-status/src/lib.rs index 6a5eb5fb8d6397..76f7e277c1571c 100644 --- a/transaction-status/src/lib.rs +++ b/transaction-status/src/lib.rs @@ -27,7 +27,7 @@ use { }, transaction_context::TransactionReturnData, }, - std::fmt, + std::{collections::HashSet, fmt}, thiserror::Error, }; @@ -1136,6 +1136,38 @@ impl EncodableWithMeta for VersionedTransaction { } } +impl Encodable for VersionedTransaction { + type Encoded = EncodedTransaction; + fn encode(&self, encoding: UiTransactionEncoding) -> Self::Encoded { + match encoding { + UiTransactionEncoding::Binary => EncodedTransaction::LegacyBinary( + bs58::encode(bincode::serialize(self).unwrap()).into_string(), + ), + UiTransactionEncoding::Base58 => EncodedTransaction::Binary( + bs58::encode(bincode::serialize(self).unwrap()).into_string(), + TransactionBinaryEncoding::Base58, + ), + UiTransactionEncoding::Base64 => EncodedTransaction::Binary( + BASE64_STANDARD.encode(bincode::serialize(self).unwrap()), + TransactionBinaryEncoding::Base64, + ), + UiTransactionEncoding::Json | UiTransactionEncoding::JsonParsed => { + EncodedTransaction::Json(UiTransaction { + signatures: self.signatures.iter().map(ToString::to_string).collect(), + message: match &self.message { + VersionedMessage::Legacy(message) => { + message.encode(UiTransactionEncoding::JsonParsed) + } + VersionedMessage::V0(message) => { + message.encode(UiTransactionEncoding::JsonParsed) + } + }, + }) + } + } + } +} + impl Encodable for Transaction { type Encoded = EncodedTransaction; fn encode(&self, encoding: UiTransactionEncoding) -> Self::Encoded { @@ -1240,6 +1272,40 @@ impl Encodable for Message { } } +impl Encodable for v0::Message { + type Encoded = UiMessage; + fn encode(&self, encoding: UiTransactionEncoding) -> Self::Encoded { + if encoding == UiTransactionEncoding::JsonParsed { + let account_keys = AccountKeys::new(&self.account_keys, None); + let loaded_addresses = LoadedAddresses::default(); + let loaded_message = + LoadedMessage::new_borrowed(self, &loaded_addresses, &HashSet::new()); + UiMessage::Parsed(UiParsedMessage { + account_keys: parse_v0_message_accounts(&loaded_message), + recent_blockhash: self.recent_blockhash.to_string(), + instructions: self + .instructions + .iter() + .map(|instruction| UiInstruction::parse(instruction, &account_keys, None)) + .collect(), + address_table_lookups: None, + }) + } else { + UiMessage::Raw(UiRawMessage { + header: self.header, + account_keys: self.account_keys.iter().map(ToString::to_string).collect(), + recent_blockhash: self.recent_blockhash.to_string(), + instructions: self + .instructions + .iter() + .map(|ix| UiCompiledInstruction::from(ix, None)) + .collect(), + address_table_lookups: None, + }) + } + } +} + impl EncodableWithMeta for v0::Message { type Encoded = UiMessage; fn encode_with_meta(