Merge pull request #1920 from radixdlt/tweak/relegate-spread-prefix-k…

…ey-mapper

tweak: Added more comprehensive mapped helper extensions to `SubstateDatabase` and its relatives

radix-clis/src/replay/cmd_alloc_dump.rs

@@ -8,7 +8,6 @@ use radix_common::prelude::*;
 use radix_engine::vm::VmModules;
 use radix_engine_profiling::info_alloc::*;
 use radix_substate_store_impls::rocks_db_with_merkle_tree::RocksDBWithMerkleTreeSubstateStore;
-use radix_substate_store_interface::db_key_mapper::SpreadPrefixKeyMapper;
 use radix_substate_store_interface::interface::*;
 use radix_transactions::prelude::*;
 use std::fs::File;
@@ -139,9 +138,7 @@ impl TxnAllocDump {
                 let execution_cost_units = receipt
                     .fee_summary()
                     .map(|x| x.total_execution_cost_units_consumed.clone());
-                let database_updates = receipt
-                    .into_state_updates()
-                    .create_database_updates::<SpreadPrefixKeyMapper>();
+                let database_updates = receipt.into_state_updates().create_database_updates();
                 database.commit(&database_updates);
                 match validated.inner {
                     ValidatedLedgerTransactionInner::User(tx) => {

radix-clis/src/replay/cmd_execute.rs

@@ -7,7 +7,6 @@ use flume;
 use radix_common::prelude::*;
 use radix_engine::vm::VmModules;
 use radix_substate_store_impls::rocks_db_with_merkle_tree::RocksDBWithMerkleTreeSubstateStore;
-use radix_substate_store_interface::db_key_mapper::SpreadPrefixKeyMapper;
 use radix_substate_store_interface::interface::*;
 use std::fs::File;
 use std::path::PathBuf;
@@ -84,8 +83,7 @@ impl TxnExecute {
                     trace,
                 );
                 let state_updates = receipt.into_state_updates();
-                let database_updates =
-                    state_updates.create_database_updates::<SpreadPrefixKeyMapper>();
+                let database_updates = state_updates.create_database_updates();
                 database.commit(&database_updates);
 
                 let new_state_root_hash = database.get_current_root_hash();

radix-clis/src/replay/cmd_execute_in_memory.rs

@@ -8,7 +8,6 @@ use radix_common::prelude::*;
 use radix_engine::vm::VmModules;
 use radix_substate_store_impls::memory_db::InMemorySubstateDatabase;
 use radix_substate_store_impls::state_tree_support::StateTreeUpdatingDatabase;
-use radix_substate_store_interface::db_key_mapper::SpreadPrefixKeyMapper;
 use radix_substate_store_interface::interface::*;
 use std::fs::File;
 use std::path::PathBuf;
@@ -96,8 +95,7 @@ impl TxnExecuteInMemory {
                     trace,
                 );
                 let state_updates = receipt.into_state_updates();
-                let database_updates =
-                    state_updates.create_database_updates::<SpreadPrefixKeyMapper>();
+                let database_updates = state_updates.create_database_updates();
                 database.commit(&database_updates);
 
                 let new_state_root_hash = database.get_current_root_hash();

radix-clis/src/replay/cmd_measure.rs

@@ -7,7 +7,6 @@ use flume;
 use radix_common::prelude::*;
 use radix_engine::vm::*;
 use radix_substate_store_impls::rocks_db_with_merkle_tree::RocksDBWithMerkleTreeSubstateStore;
-use radix_substate_store_interface::db_key_mapper::SpreadPrefixKeyMapper;
 use radix_substate_store_interface::interface::*;
 use radix_transactions::prelude::*;
 use std::fs::File;
@@ -110,9 +109,7 @@ impl TxnMeasure {
                 let finalization_cost_units = receipt
                     .fee_summary()
                     .map(|x| x.total_finalization_cost_units_consumed.clone());
-                let database_updates = receipt
-                    .into_state_updates()
-                    .create_database_updates::<SpreadPrefixKeyMapper>();
+                let database_updates = receipt.into_state_updates().create_database_updates();
                 database.commit(&database_updates);
                 let tx_processing_time = tx_start_time.elapsed();
                 if let ValidatedLedgerTransactionInner::User(tx) = validated.inner {

radix-clis/src/replay/cmd_sync.rs

@@ -6,7 +6,6 @@ use flume::Sender;
 use radix_common::prelude::*;
 use radix_engine::vm::VmModules;
 use radix_substate_store_impls::rocks_db_with_merkle_tree::RocksDBWithMerkleTreeSubstateStore;
-use radix_substate_store_interface::db_key_mapper::SpreadPrefixKeyMapper;
 use radix_substate_store_interface::interface::*;
 use radix_transactions::prelude::*;
 use rocksdb::{Direction, IteratorMode, Options, DB};
@@ -74,8 +73,7 @@ impl TxnSync {
                     trace,
                 );
                 let state_updates = receipt.into_state_updates();
-                let database_updates =
-                    state_updates.create_database_updates::<SpreadPrefixKeyMapper>();
+                let database_updates = state_updates.create_database_updates();
 
                 let current_version = database.get_current_version();
                 let new_version = current_version + 1;

radix-clis/src/resim/cmd_publish.rs

@@ -10,11 +10,8 @@ use radix_engine_interface::blueprints::package::{
 };
 use radix_engine_interface::prelude::*;
 use radix_rust::ContextualDisplay;
-use radix_substate_store_interface::interface::DatabaseUpdates;
-use radix_substate_store_interface::{
-    db_key_mapper::{DatabaseKeyMapper, SpreadPrefixKeyMapper},
-    interface::*,
-};
+use radix_substate_store_interface::db_key_mapper::*;
+use radix_substate_store_interface::interface::*;
 use radix_substate_store_queries::typed_substate_layout::*;
 use std::ffi::OsStr;
 use std::fs;

radix-clis/src/resim/cmd_show_ledger.rs

@@ -5,10 +5,7 @@ use radix_common::time::Instant;
 use radix_common::time::UtcDateTime;
 use radix_engine_interface::blueprints::consensus_manager::*;
 use radix_substate_store_impls::rocks_db::RocksdbSubstateStore;
-use radix_substate_store_interface::{
-    db_key_mapper::{DatabaseKeyMapper, SpreadPrefixKeyMapper},
-    interface::ListableSubstateDatabase,
-};
+use radix_substate_store_interface::interface::*;
 
 use crate::resim::*;
 
@@ -54,8 +51,7 @@ impl ShowLedger {
         let mut components: Vec<ComponentAddress> = vec![];
         let mut resources: Vec<ResourceAddress> = vec![];
 
-        for key in substate_db.list_partition_keys() {
-            let (node_id, _) = SpreadPrefixKeyMapper::from_db_partition_key(&key);
+        for (node_id, _) in substate_db.read_partition_keys() {
             if let Ok(address) = PackageAddress::try_from(node_id.as_ref()) {
                 if !packages.contains(&address) {
                     packages.push(address);

radix-clis/src/resim/dumper.rs

@@ -12,10 +12,7 @@ use radix_engine_interface::blueprints::resource::NON_FUNGIBLE_RESOURCE_MANAGER_
 use radix_engine_interface::types::{BlueprintPartitionOffset, CollectionDescriptor};
 use radix_engine_interface::{prelude::MetadataValue, types::PackagePartitionOffset};
 use radix_rust::ContextualDisplay;
-use radix_substate_store_interface::{
-    db_key_mapper::{MappedSubstateDatabase, SpreadPrefixKeyMapper},
-    interface::SubstateDatabase,
-};
+use radix_substate_store_interface::interface::*;
 use radix_substate_store_queries::query::ResourceAccounter;
 use radix_substate_store_queries::typed_substate_layout::*;
 
@@ -38,9 +35,10 @@ pub fn dump_package<T: SubstateDatabase, O: std::io::Write>(
 ) -> Result<(), EntityDumpError> {
     let address_bech32_encoder = AddressBech32Encoder::new(&NetworkDefinition::simulator());
     let (_, substate) = substate_db
-        .list_mapped::<SpreadPrefixKeyMapper, PackageCodeOriginalCodeEntrySubstate, MapKey>(
-            package_address.as_node_id(),
+        .list_map_values::<PackageCodeOriginalCodeEntrySubstate>(
+            package_address,
             PackagePartitionOffset::CodeOriginalCodeKeyValue.as_main_partition(),
+            None::<SubstateKey>,
         )
         .next()
         .ok_or(EntityDumpError::PackageNotFound)?;

radix-common/src/types/node_and_substate.rs

@@ -246,6 +246,13 @@ pub enum SubstateKey {
     Sorted(SortedKey),
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum SubstateKeyRef<'a> {
+    Field(&'a FieldKey),
+    Map(&'a MapKey),
+    Sorted(&'a SortedKey),
+}
+
 impl SubstateKey {
     pub fn for_field(&self) -> Option<&FieldKey> {
         match self {
@@ -274,6 +281,90 @@ impl SubstateKey {
             _ => None,
         }
     }
+
+    pub fn as_ref(&self) -> SubstateKeyRef {
+        match self {
+            SubstateKey::Field(key) => SubstateKeyRef::Field(key),
+            SubstateKey::Map(key) => SubstateKeyRef::Map(key),
+            SubstateKey::Sorted(key) => SubstateKeyRef::Sorted(key),
+        }
+    }
+}
+
+pub enum SubstateKeyOrRef<'a> {
+    Owned(SubstateKey),
+    Borrowed(SubstateKeyRef<'a>),
+}
+
+impl<'a> SubstateKeyOrRef<'a> {
+    pub fn as_ref(&self) -> SubstateKeyRef<'_> {
+        match self {
+            SubstateKeyOrRef::Owned(key) => key.as_ref(),
+            SubstateKeyOrRef::Borrowed(key_ref) => *key_ref,
+        }
+    }
+}
+
+pub trait ResolvableSubstateKey<'a>: Sized {
+    fn into_substate_key_or_ref(self) -> SubstateKeyOrRef<'a>;
+}
+
+impl<'a> ResolvableSubstateKey<'a> for &'a SubstateKey {
+    fn into_substate_key_or_ref(self) -> SubstateKeyOrRef<'a> {
+        SubstateKeyOrRef::Borrowed(self.as_ref())
+    }
+}
+
+impl<T: Into<SubstateKey>> ResolvableSubstateKey<'static> for T {
+    fn into_substate_key_or_ref(self) -> SubstateKeyOrRef<'static> {
+        SubstateKeyOrRef::Owned(self.into())
+    }
+}
+
+impl<'a> ResolvableSubstateKey<'a> for FieldKey {
+    fn into_substate_key_or_ref(self) -> SubstateKeyOrRef<'a> {
+        SubstateKeyOrRef::Owned(SubstateKey::Field(self))
+    }
+}
+
+impl<'a> ResolvableSubstateKey<'a> for &'a FieldKey {
+    fn into_substate_key_or_ref(self) -> SubstateKeyOrRef<'a> {
+        SubstateKeyOrRef::Borrowed(SubstateKeyRef::Field(self))
+    }
+}
+
+impl<'a> ResolvableSubstateKey<'a> for MapKey {
+    fn into_substate_key_or_ref(self) -> SubstateKeyOrRef<'a> {
+        SubstateKeyOrRef::Owned(SubstateKey::Map(self))
+    }
+}
+
+impl<'a> ResolvableSubstateKey<'a> for &'a MapKey {
+    fn into_substate_key_or_ref(self) -> SubstateKeyOrRef<'a> {
+        SubstateKeyOrRef::Borrowed(SubstateKeyRef::Map(self))
+    }
+}
+
+impl<'a> ResolvableSubstateKey<'a> for SortedKey {
+    fn into_substate_key_or_ref(self) -> SubstateKeyOrRef<'a> {
+        SubstateKeyOrRef::Owned(SubstateKey::Sorted(self))
+    }
+}
+
+impl<'a> ResolvableSubstateKey<'a> for &'a SortedKey {
+    fn into_substate_key_or_ref(self) -> SubstateKeyOrRef<'a> {
+        SubstateKeyOrRef::Borrowed(SubstateKeyRef::Sorted(self))
+    }
+}
+
+pub trait ResolvableOptionalSubstateKey<'a> {
+    fn into_optional_substate_key_or_ref(self) -> Option<SubstateKeyOrRef<'a>>;
+}
+
+impl<'a, T: ResolvableSubstateKey<'a>> ResolvableOptionalSubstateKey<'a> for Option<T> {
+    fn into_optional_substate_key_or_ref(self) -> Option<SubstateKeyOrRef<'a>> {
+        self.map(|k| k.into_substate_key_or_ref())
+    }
 }
 
 pub type FieldKey = u8;

radix-engine-monkey-tests/tests/fuzz_kernel.rs

@@ -9,17 +9,16 @@ use radix_engine::kernel::kernel_api::{
 };
 use radix_engine::kernel::kernel_callback_api::*;
 use radix_engine::system::checkers::KernelDatabaseChecker;
-use radix_engine::track::{to_state_updates, CommitableSubstateStore, Track};
+use radix_engine::track::{CommitableSubstateStore, Track};
 use radix_engine_interface::prelude::*;
 use radix_substate_store_impls::memory_db::InMemorySubstateDatabase;
-use radix_substate_store_interface::db_key_mapper::SpreadPrefixKeyMapper;
 use radix_substate_store_interface::interface::CommittableSubstateDatabase;
 use rand::Rng;
 use rand_chacha::rand_core::SeedableRng;
 use rand_chacha::ChaCha8Rng;
 use rayon::iter::IntoParallelIterator;
 use rayon::iter::ParallelIterator;
-use scrypto_test::prelude::CreateDatabaseUpdates;
+use scrypto_test::prelude::*;
 
 #[derive(Default)]
 struct TestCallFrameData;
@@ -470,7 +469,7 @@ fn kernel_fuzz<F: FnMut(&mut KernelFuzzer) -> Vec<KernelFuzzAction>>(
     let txn_hash = &seed.to_be_bytes().repeat(4)[..];
     let mut id_allocator = IdAllocator::new(Hash(txn_hash.try_into().unwrap()));
     let mut substate_db = InMemorySubstateDatabase::standard();
-    let mut track = Track::<InMemorySubstateDatabase, SpreadPrefixKeyMapper>::new(&substate_db);
+    let mut track = Track::new(&substate_db);
     let mut callback = TestCallbackObject;
     let mut kernel = Kernel::new_no_refs(&mut track, &mut id_allocator, &mut callback);
 
@@ -489,9 +488,9 @@ fn kernel_fuzz<F: FnMut(&mut KernelFuzzer) -> Vec<KernelFuzzAction>>(
 
     let result = track.finalize();
     if let Ok((tracked_substates, _)) = result {
-        let (_, state_updates) = to_state_updates::<SpreadPrefixKeyMapper>(tracked_substates);
+        let (_, state_updates) = tracked_substates.to_state_updates();
 
-        let database_updates = state_updates.create_database_updates::<SpreadPrefixKeyMapper>();
+        let database_updates = state_updates.create_database_updates();
         substate_db.commit(&database_updates);
         let mut checker = KernelDatabaseChecker::new();
         checker.check_db(&substate_db).unwrap_or_else(|_| {

radix-engine-profiling/src/rocks_db_metrics/mod.rs

@@ -73,13 +73,13 @@ impl SubstateStoreWithMetrics<InMemorySubstateDatabase> {
 impl<S: SubstateDatabase + CommittableSubstateDatabase> SubstateDatabase
     for SubstateStoreWithMetrics<S>
 {
-    fn get_substate(
+    fn get_raw_substate_by_db_key(
         &self,
         partition_key: &DbPartitionKey,
         sort_key: &DbSortKey,
     ) -> Option<DbSubstateValue> {
         let start = std::time::Instant::now();
-        let ret = self.db.get_substate(partition_key, sort_key);
+        let ret = self.db.get_raw_substate_by_db_key(partition_key, sort_key);
         let duration = start.elapsed();
 
         if let Some(value) = ret {
@@ -95,12 +95,13 @@ impl<S: SubstateDatabase + CommittableSubstateDatabase> SubstateDatabase
         }
     }
 
-    fn list_entries_from(
+    fn list_raw_values_from_db_key(
         &self,
         partition_key: &DbPartitionKey,
         from_sort_key: Option<&DbSortKey>,
     ) -> Box<dyn Iterator<Item = PartitionEntry> + '_> {
-        self.db.list_entries_from(partition_key, from_sort_key)
+        self.db
+            .list_raw_values_from_db_key(partition_key, from_sort_key)
     }
 }
 
@@ -152,7 +153,7 @@ impl<S: SubstateDatabase + CommittableSubstateDatabase> CommittableSubstateDatab
                                 )
                             } else {
                                 delete_found = true;
-                                if let Some(value) = self.get_substate(
+                                if let Some(value) = self.get_raw_substate_by_db_key(
                                     &DbPartitionKey {
                                         node_key: node_key.clone(),
                                         partition_num: *partition_num,

radix-engine-profiling/src/rocks_db_metrics/tests/read.rs

@@ -219,7 +219,7 @@ fn run_read_test<S: SubstateDatabase + CommittableSubstateDatabase>(
             print!("\rRead {}/{}  ", j + 1, data_index_vector.len());
             std::io::stdout().flush().ok();
 
-            let read_value = substate_db.get_substate(&p, &s);
+            let read_value = substate_db.get_raw_substate_by_db_key(&p, &s);
 
             assert!(read_value.is_some());
             assert_eq!(read_value.unwrap().len(), *v);
@@ -275,7 +275,7 @@ fn run_read_not_found_test<S: SubstateDatabase + CommittableSubstateDatabase>(
         data_index_vector.shuffle(&mut rng);
 
         for (p, s) in data_index_vector.iter() {
-            let read_value = substate_db.get_substate(&p, &s);
+            let read_value = substate_db.get_raw_substate_by_db_key(&p, &s);
             assert!(read_value.is_none());
         }
     }

radix-engine-tests/tests/application/stake_reconciliation.rs

@@ -49,7 +49,7 @@ fn test_stake_reconciliation() {
     let db = ledger.substate_db();
     let old_keys: Vec<DbPartitionKey> = db.list_partition_keys().collect();
     for key in old_keys {
-        let entries = db.list_entries(&key);
+        let entries = db.list_raw_values_from_db_key(&key, None);
         for (sort_key, value) in entries {
             pre_transaction_substates.insert((key.clone(), sort_key), value);
         }
@@ -89,7 +89,7 @@ fn test_stake_reconciliation() {
     //     let partition_num = PartitionNumber(1);
     //     let substate_key = SubstateKey::Map(ScryptoRawPayload::from_valid_payload(vec![92, 32, 7, 32, 220, 0, 156, 5, 6, 83, 96, 189, 222, 100, 29, 145, 160, 147, 193, 127, 71, 54, 135, 62, 103, 35, 126, 168, 230, 117, 203, 71, 36, 132, 155, 157]));
     //     let value_from_database: ScryptoRawValue = ledger.substate_db()
-    //         .get_mapped::<SpreadPrefixKeyMapper, _>(&node_id, partition_num, &substate_key)
+    //         .read_substate_typed(&node_id, partition_num, substate_key)
     //         .unwrap();
     //     let substate_value = value_from_database.into_payload_bytes();
     //     let state_updates = StateUpdates {
@@ -327,7 +327,7 @@ fn test_stake_reconciliation() {
     }
 
     for key in post_transaction_partitions {
-        let partition_entries = ledger.substate_db().list_entries(&key);
+        let partition_entries = ledger.substate_db().list_raw_values_from_db_key(&key, None);
         for (sort_key, current_value) in partition_entries {
             let full_key = (key.clone(), sort_key.clone());
             let address = AddressBech32Encoder::for_simulator()

radix-engine-tests/tests/blueprints/account_locker.rs

@@ -3055,7 +3055,7 @@ pub impl DefaultLedgerSimulator {
         execution_config.system_overrides = Some(SystemOverrides {
             disable_auth,
             disable_costing,
-            ..Default::default()
+            ..SystemOverrides::default_for_network(&NetworkDefinition::mainnet())
         });
 
         let nonce = self.next_transaction_nonce();