Enable ledger-tool copy command to infer target_db shred storage type. (

solana-labs#27438)

#### Problem
The ledger-tool copy command currently assumes target_db uses the same
shred storage type as the source ledger.

#### Summary of Changes
This PR enables ledger-tool copy command to infer the target_db shred storage
type based on whether rocksdb or rocksdb_fifo exists under the target_db directory
(same way as how ledger-tool infers the shred storage type for the main db.)
If the ledger-tool is not able to infer the shred storage type of the target_db, then
the default level compaction will be used.

ledger-tool/src/main.rs

@@ -30,8 +30,8 @@ use {
         blockstore::{create_new_ledger, Blockstore, BlockstoreError, PurgeType},
         blockstore_db::{self, columns as cf, Column, ColumnName, Database},
         blockstore_options::{
-            AccessType, BlockstoreOptions, BlockstoreRecoveryMode, LedgerColumnOptions,
-            ShredStorageType,
+            AccessType, BlockstoreOptions, BlockstoreRecoveryMode, BlockstoreRocksFifoOptions,
+            LedgerColumnOptions, ShredStorageType,
         },
         blockstore_processor::{self, BlockstoreProcessorError, ProcessOptions},
         shred::Shred,
@@ -847,6 +847,23 @@ fn open_blockstore_with_temporary_primary_access(
     )
 }
 
+fn get_shred_storage_type(ledger_path: &Path, warn_message: &str) -> ShredStorageType {
+    // TODO: the following shred_storage_type inference must be updated once the
+    // rocksdb options can be constructed via load_options_file() as the
+    // temporary use of DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES
+    // could affect the persisted rocksdb options file.
+    match ShredStorageType::from_ledger_path(
+        ledger_path,
+        DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
+    ) {
+        Some(s) => s,
+        None => {
+            warn!("{}", warn_message);
+            ShredStorageType::RocksLevel
+        }
+    }
+}
+
 fn open_blockstore(
     ledger_path: &Path,
     access_type: AccessType,
@@ -1466,6 +1483,7 @@ fn main() {
     let default_graph_vote_account_mode = GraphVoteAccountMode::default();
 
     let mut measure_total_execution_time = Measure::start("ledger tool");
+
     let matches = App::new(crate_name!())
         .about(crate_description!())
         .version(solana_version::version!())
@@ -2215,21 +2233,10 @@ fn main() {
         .map(BlockstoreRecoveryMode::from);
     let force_update_to_open = matches.is_present("force_update_to_open");
     let verbose_level = matches.occurrences_of("verbose");
-
-    // TODO: the following shred_storage_type inference must be updated once the
-    // rocksdb options can be constructed via load_options_file() as the
-    // temporary use of DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES
-    // could affect the persisted rocksdb options file.
-    let shred_storage_type = match ShredStorageType::from_ledger_path(
+    let shred_storage_type = get_shred_storage_type(
         &ledger_path,
-        DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
-    ) {
-        Some(s) => s,
-        None => {
-            error!("Shred storage type cannot be inferred, the default RocksLevel will be used");
-            ShredStorageType::RocksLevel
-        }
-    };
+        "Shred storage type cannot be inferred, the default RocksLevel will be used",
+    );
 
     if let ("bigtable", Some(arg_matches)) = matches.subcommand() {
         bigtable_process_command(&ledger_path, arg_matches, &shred_storage_type)
@@ -2264,6 +2271,18 @@ fn main() {
                 let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
                 let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot);
                 let target_db = PathBuf::from(value_t_or_exit!(arg_matches, "target_db", String));
+                let target_shred_storage_type = get_shred_storage_type(
+                    &target_db,
+                    &format!(
+                        "Shred storage type of target_db cannot be inferred, \
+                     the default RocksLevel will be used. \
+                     If you want to use FIFO shred_storage_type on an empty target_db, \
+                     create {} foldar the specified target_db directory.",
+                        ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions::default())
+                            .blockstore_directory()
+                    ),
+                );
+
                 let source = open_blockstore(
                     &ledger_path,
                     AccessType::Secondary,
@@ -2275,10 +2294,9 @@ fn main() {
                     &target_db,
                     AccessType::Primary,
                     None,
-                    &shred_storage_type,
+                    &target_shred_storage_type,
                     force_update_to_open,
                 );
-
                 for (slot, _meta) in source.slot_meta_iterator(starting_slot).unwrap() {
                     if slot > ending_slot {
                         break;

ledger-tool/tests/basic.rs

@@ -1,9 +1,17 @@
 use {
     assert_cmd::prelude::*,
+    solana_entry::entry,
     solana_ledger::{
+        blockstore, blockstore::Blockstore, blockstore_options::ShredStorageType,
         create_new_tmp_ledger, create_new_tmp_ledger_fifo, genesis_utils::create_genesis_config,
+        get_tmp_ledger_path_auto_delete,
+    },
+    solana_sdk::hash::Hash,
+    std::{
+        fs,
+        path::Path,
+        process::{Command, Output},
     },
-    std::process::{Command, Output},
 };
 
 fn run_ledger_tool(args: &[&str]) -> Output {
@@ -66,3 +74,64 @@ fn nominal_fifo() {
         genesis_config.ticks_per_slot as usize,
     );
 }
+
+fn insert_test_shreds(ledger_path: &Path, ending_slot: u64) {
+    let blockstore = Blockstore::open(ledger_path).unwrap();
+    for i in 1..ending_slot {
+        let entries = entry::create_ticks(1, 0, Hash::default());
+        let shreds = blockstore::entries_to_test_shreds(&entries, i, 0, false, 0);
+        blockstore.insert_shreds(shreds, None, false).unwrap();
+    }
+}
+
+fn ledger_tool_copy_test(src_shred_compaction: &str, dst_shred_compaction: &str) {
+    const TEST_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES: u64 = std::u64::MAX;
+    let genesis_config = create_genesis_config(100).genesis_config;
+
+    let (ledger_path, _blockhash) = match src_shred_compaction {
+        "fifo" => create_new_tmp_ledger_fifo!(&genesis_config),
+        _ => create_new_tmp_ledger!(&genesis_config),
+    };
+    const LEDGER_TOOL_COPY_TEST_SHRED_COUNT: u64 = 25;
+    const LEDGER_TOOL_COPY_TEST_ENDING_SLOT: u64 = LEDGER_TOOL_COPY_TEST_SHRED_COUNT + 1;
+    insert_test_shreds(&ledger_path, LEDGER_TOOL_COPY_TEST_ENDING_SLOT);
+    let ledger_path = ledger_path.to_str().unwrap();
+
+    let target_ledger_path = get_tmp_ledger_path_auto_delete!();
+    if dst_shred_compaction == "fifo" {
+        let rocksdb_fifo_path = target_ledger_path.path().join(
+            ShredStorageType::rocks_fifo(TEST_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES)
+                .blockstore_directory(),
+        );
+        fs::create_dir_all(rocksdb_fifo_path).unwrap();
+    }
+    let target_ledger_path = target_ledger_path.path().to_str().unwrap();
+    let output = run_ledger_tool(&[
+        "-l",
+        ledger_path,
+        "copy",
+        "--target-db",
+        target_ledger_path,
+        "--ending-slot",
+        &(LEDGER_TOOL_COPY_TEST_ENDING_SLOT).to_string(),
+    ]);
+    assert!(output.status.success());
+    for slot_id in 0..LEDGER_TOOL_COPY_TEST_ENDING_SLOT {
+        let src_slot_output = run_ledger_tool(&["-l", ledger_path, "slot", &slot_id.to_string()]);
+
+        let dst_slot_output =
+            run_ledger_tool(&["-l", target_ledger_path, "slot", &slot_id.to_string()]);
+        assert!(src_slot_output.status.success());
+        assert!(dst_slot_output.status.success());
+        assert!(!src_slot_output.stdout.is_empty());
+        assert_eq!(src_slot_output.stdout, dst_slot_output.stdout);
+    }
+}
+
+#[test]
+fn copy_test() {
+    ledger_tool_copy_test("level", "level");
+    ledger_tool_copy_test("level", "fifo");
+    ledger_tool_copy_test("fifo", "level");
+    ledger_tool_copy_test("fifo", "fifo");
+}