Query plan proofs

hash256-std-hasher/Cargo.toml

@@ -16,7 +16,7 @@ harness = false
 crunchy = "0.2.1"
 
 [dev-dependencies]
-criterion = "0.4.0"
+criterion = "0.5.1"
 
 [features]
 default = ["std"]

memory-db/Cargo.toml

@@ -12,7 +12,7 @@ hash-db = { version = "0.16.0", path = "../hash-db", default-features = false }
 
 [dev-dependencies]
 keccak-hasher = { path = "../test-support/keccak-hasher" }
-criterion = "0.4.0"
+criterion = "0.5.1"
 
 [features]
 default = ["std"]

test-support/reference-trie/Cargo.toml

@@ -18,7 +18,7 @@ paste = "1.0.12"
 
 [dev-dependencies]
 trie-bench = { path = "../trie-bench" }
-criterion = "0.4.0"
+criterion = "0.5.1"
 
 [[bench]]
 name = "bench"

test-support/reference-trie/src/lib.rs

@@ -51,6 +51,8 @@ macro_rules! test_layouts {
 	($test:ident, $test_internal:ident) => {
 		#[test]
 		fn $test() {
+			eprintln!("Running with layout `SubstrateV1`");
+			$test_internal::<$crate::SubstrateV1<$crate::RefHasher>>();
 			eprintln!("Running with layout `HashedValueNoExtThreshold`");
 			$test_internal::<$crate::HashedValueNoExtThreshold<1>>();
 			eprintln!("Running with layout `HashedValueNoExt`");
@@ -599,6 +601,8 @@ impl<'a> Input for ByteSliceInput<'a> {
 // `const HASHED_NULL_NODE: <KeccakHasher as Hasher>::Out = <KeccakHasher as Hasher>::Out( … … )`.
 // Perhaps one day soon?
 impl<H: Hasher> NodeCodec for ReferenceNodeCodec<H> {
+	const DELTA_COMPACT_OMITTED_NODE: usize = 32;
+	const DELTA_COMPACT_OMITTED_VALUE: usize = 30;
 	type Error = CodecError;
 	type HashOut = H::Out;
 
@@ -753,6 +757,8 @@ impl<H: Hasher> NodeCodec for ReferenceNodeCodec<H> {
 }
 
 impl<H: Hasher> NodeCodec for ReferenceNodeCodecNoExt<H> {
+	const DELTA_COMPACT_OMITTED_NODE: usize = 32;
+	const DELTA_COMPACT_OMITTED_VALUE: usize = 30;
 	type Error = CodecError;
 	type HashOut = <H as Hasher>::Out;
 

test-support/reference-trie/src/substrate.rs

@@ -209,6 +209,8 @@ impl<H> NodeCodecT for NodeCodec<H>
 where
 	H: Hasher,
 {
+	const DELTA_COMPACT_OMITTED_NODE: usize = 32;
+	const DELTA_COMPACT_OMITTED_VALUE: usize = 30;
 	const ESCAPE_HEADER: Option<u8> = Some(trie_constants::ESCAPE_COMPACT_HEADER);
 	type Error = Error<H::Out>;
 	type HashOut = H::Out;

test-support/reference-trie/src/substrate_like.rs

@@ -149,6 +149,8 @@ impl<H> NodeCodecT for NodeCodec<H>
 where
 	H: Hasher,
 {
+	const DELTA_COMPACT_OMITTED_NODE: usize = 32;
+	const DELTA_COMPACT_OMITTED_VALUE: usize = 30;
 	const ESCAPE_HEADER: Option<u8> = Some(trie_constants::ESCAPE_COMPACT_HEADER);
 	type Error = Error;
 	type HashOut = H::Out;

test-support/trie-bench/Cargo.toml

@@ -14,5 +14,5 @@ hash-db = { path = "../../hash-db" , version = "0.16.0"}
 memory-db = { path = "../../memory-db", version = "0.32.0" }
 trie-root = { path = "../../trie-root", version = "0.18.0" }
 trie-db = { path = "../../trie-db", version = "0.27.0" }
-criterion = "0.4.0"
+criterion = "0.5.1"
 parity-scale-codec = "3.0.0"

trie-db/fuzz/Cargo.toml

@@ -14,9 +14,12 @@ memory-db = { path = "../../memory-db", version = "0.32.0" }
 reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0" }
 arbitrary = { version = "1.3.0", features = ["derive"] }
 array-bytes = "6.0.0"
+lazy_static = "1.4.0"
 
 [dependencies.trie-db]
 path = ".."
+[dependencies.trie-db-test]
+path = "../test"
 
 [dependencies.libfuzzer-sys]
 version = "0.4.6"
@@ -68,3 +71,19 @@ path = "fuzz_targets/trie_proof_invalid.rs"
 [[bin]]
 name = "prefix_seek_iter"
 path = "fuzz_targets/prefix_seek_iter.rs"
+
+[[bin]]
+name = "query_plan_1"
+path = "fuzz_targets/query_plan_1.rs"
+
+[[bin]]
+name = "query_plan_2"
+path = "fuzz_targets/query_plan_2.rs"
+
+[[bin]]
+name = "query_plan_3"
+path = "fuzz_targets/query_plan_3.rs"
+
+[[bin]]
+name = "query_plan_4"
+path = "fuzz_targets/query_plan_4.rs"

trie-db/fuzz/fuzz_targets/no_ext_insert.rs

@@ -1,7 +1,7 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use trie_db_fuzz::fuzz_that_no_extension_insert;
+use trie_db_test::fuzz::fuzz_that_no_extension_insert;
 
 fuzz_target!(|data: &[u8]| {
 	// fuzzed code goes here

trie-db/fuzz/fuzz_targets/no_ext_insert_rem.rs

@@ -1,7 +1,7 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use trie_db_fuzz::fuzz_that_no_extension_insert_remove;
+use trie_db_test::fuzz::fuzz_that_no_extension_insert_remove;
 
 fuzz_target!(|data: &[u8]| {
 	// fuzzed code goes here

trie-db/fuzz/fuzz_targets/prefix_iter.rs

@@ -1,7 +1,7 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use trie_db_fuzz::fuzz_prefix_iter;
+use trie_db_test::fuzz::fuzz_prefix_iter;
 
 fuzz_target!(|data: &[u8]| {
 	fuzz_prefix_iter::<reference_trie::NoExtensionLayout>(data);

trie-db/fuzz/fuzz_targets/prefix_seek_iter.rs

@@ -1,7 +1,7 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use trie_db_fuzz::{fuzz_prefix_seek_iter, PrefixSeekTestInput};
+use trie_db_test::fuzz::{fuzz_prefix_seek_iter, PrefixSeekTestInput};
 
 fuzz_target!(|data: PrefixSeekTestInput| {
 	fuzz_prefix_seek_iter::<reference_trie::SubstrateV1<reference_trie::RefHasher>>(data);

trie-db/fuzz/fuzz_targets/query_plan_1.rs

@@ -0,0 +1,15 @@
+#![no_main]
+use lazy_static::lazy_static;
+use libfuzzer_sys::fuzz_target;
+use reference_trie::{RefHasher, SubstrateV1};
+use trie_db_test::fuzz::query_plan::{
+	build_state, fuzz_query_plan, ArbitraryQueryPlan, FuzzContext, CONF1,
+};
+
+lazy_static! {
+	static ref CONTEXT: FuzzContext<SubstrateV1<RefHasher>> = build_state(CONF1);
+}
+
+fuzz_target!(|plan: ArbitraryQueryPlan| {
+	fuzz_query_plan::<SubstrateV1<RefHasher>>(&CONTEXT, plan);
+});

trie-db/fuzz/fuzz_targets/query_plan_2.rs

@@ -0,0 +1,36 @@
+#![no_main]
+use lazy_static::lazy_static;
+use libfuzzer_sys::fuzz_target;
+use reference_trie::{RefHasher, SubstrateV1};
+use trie_db_test::fuzz::query_plan::{
+	build_state, fuzz_query_plan_conf, ArbitraryQueryPlan, FuzzContext, CONF1,
+};
+use arbitrary::Arbitrary;
+
+lazy_static! {
+	static ref CONTEXT: FuzzContext<SubstrateV1<RefHasher>> = build_state(CONF1);
+}
+
+#[derive(Debug, Clone, Copy, Arbitrary)]
+#[repr(usize)]
+enum SplitSize {
+	One = 1,
+	Two = 2,
+	Three = 3,
+	More = 10,
+	MoreMore = 50,
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Arbitrary)]
+enum SplitKind {
+	Stateless,
+	Stateful,
+}
+
+fuzz_target!(|input: (ArbitraryQueryPlan, SplitSize, SplitKind)| {
+	let (plan, split_size, split_kind) = input;
+	let mut conf = CONTEXT.conf.clone();
+	conf.limit = split_size as usize;
+	conf.proof_spawn_with_persistence = split_kind == SplitKind::Stateful;
+	fuzz_query_plan_conf::<SubstrateV1<RefHasher>>(&CONTEXT, conf, plan);
+});

trie-db/fuzz/fuzz_targets/query_plan_3.rs

@@ -0,0 +1,15 @@
+#![no_main]
+use lazy_static::lazy_static;
+use libfuzzer_sys::fuzz_target;
+use reference_trie::{RefHasher, SubstrateV1};
+use trie_db_test::fuzz::query_plan::{
+	build_state, fuzz_query_plan, ArbitraryQueryPlan, FuzzContext, CONF2,
+};
+
+lazy_static! {
+	static ref CONTEXT: FuzzContext<SubstrateV1<RefHasher>> = build_state(CONF2);
+}
+
+fuzz_target!(|plan: ArbitraryQueryPlan| {
+	fuzz_query_plan::<SubstrateV1<RefHasher>>(&CONTEXT, plan);
+});

trie-db/fuzz/fuzz_targets/query_plan_4.rs

@@ -0,0 +1,36 @@
+#![no_main]
+use lazy_static::lazy_static;
+use libfuzzer_sys::fuzz_target;
+use reference_trie::{RefHasher, SubstrateV1};
+use trie_db_test::fuzz::query_plan::{
+	build_state, fuzz_query_plan_conf, ArbitraryQueryPlan, FuzzContext, CONF2,
+};
+use arbitrary::Arbitrary;
+
+lazy_static! {
+	static ref CONTEXT: FuzzContext<SubstrateV1<RefHasher>> = build_state(CONF2);
+}
+
+#[derive(Debug, Clone, Copy, Arbitrary)]
+#[repr(usize)]
+enum SplitSize {
+	One = 1,
+	Two = 2,
+	Three = 3,
+	More = 10,
+	MoreMore = 50,
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Arbitrary)]
+enum SplitKind {
+	Stateless,
+	Stateful,
+}
+
+fuzz_target!(|input: (ArbitraryQueryPlan, SplitSize, SplitKind)| {
+	let (plan, split_size, split_kind) = input;
+	let mut conf = CONTEXT.conf.clone();
+	conf.limit = split_size as usize;
+	conf.proof_spawn_with_persistence = split_kind == SplitKind::Stateful;
+	fuzz_query_plan_conf::<SubstrateV1<RefHasher>>(&CONTEXT, conf, plan);
+});

trie-db/fuzz/fuzz_targets/seek_iter.rs

@@ -1,7 +1,7 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use trie_db_fuzz::fuzz_seek_iter;
+use trie_db_test::fuzz::fuzz_seek_iter;
 
 fuzz_target!(|data: &[u8]| {
 	fuzz_seek_iter::<reference_trie::NoExtensionLayout>(data);

trie-db/fuzz/fuzz_targets/trie_codec_proof.rs

@@ -1,7 +1,7 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use trie_db_fuzz::fuzz_that_trie_codec_proofs;
+use trie_db_test::fuzz::fuzz_that_trie_codec_proofs;
 
 fuzz_target!(|data: &[u8]| {
 	fuzz_that_trie_codec_proofs::<reference_trie::ExtensionLayout>(data);

trie-db/fuzz/fuzz_targets/trie_proof_invalid.rs

@@ -1,7 +1,7 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use trie_db_fuzz::fuzz_that_verify_rejects_invalid_proofs;
+use trie_db_test::fuzz::fuzz_that_verify_rejects_invalid_proofs;
 
 fuzz_target!(|data: &[u8]| {
 	fuzz_that_verify_rejects_invalid_proofs::<reference_trie::ExtensionLayout>(data);

trie-db/fuzz/fuzz_targets/trie_proof_valid.rs

@@ -1,7 +1,7 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use trie_db_fuzz::fuzz_that_verify_accepts_valid_proofs;
+use trie_db_test::fuzz::fuzz_that_verify_accepts_valid_proofs;
 
 fuzz_target!(|data: &[u8]| {
 	fuzz_that_verify_accepts_valid_proofs::<reference_trie::ExtensionLayout>(data);

trie-db/fuzz/fuzz_targets/trie_root.rs

@@ -1,7 +1,7 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use trie_db_fuzz::fuzz_that_reference_trie_root;
+use trie_db_test::fuzz::fuzz_that_reference_trie_root;
 
 fuzz_target!(|data: &[u8]| {
 	fuzz_that_reference_trie_root::<reference_trie::NoExtensionLayout>(data);

trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs

@@ -1,7 +1,7 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use trie_db_fuzz::fuzz_that_reference_trie_root_fix_length;
+use trie_db_test::fuzz::fuzz_that_reference_trie_root_fix_length;
 
 fuzz_target!(|data: &[u8]| {
 	fuzz_that_reference_trie_root_fix_length::<reference_trie::NoExtensionLayout>(data);

trie-db/fuzz/fuzz_targets/trie_root_new.rs

@@ -1,7 +1,7 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use trie_db_fuzz::fuzz_that_compare_implementations;
+use trie_db_test::fuzz::fuzz_that_compare_implementations;
 
 fuzz_target!(|data: &[u8]| {
 	// fuzzed code goes here

trie-db/src/iterator.rs

@@ -34,12 +34,18 @@ enum Status {
 
 #[cfg_attr(feature = "std", derive(Debug))]
 #[derive(Eq, PartialEq)]
-struct Crumb<H: Hasher> {
+pub struct Crumb<H: Hasher> {
 	hash: Option<H::Out>,
 	node: Arc<OwnedNode<DBValue>>,
 	status: Status,
 }
 
+impl<H: Hasher> Clone for Crumb<H> {
+	fn clone(&self) -> Self {
+		Self { hash: self.hash.clone(), node: self.node.clone(), status: self.status.clone() }
+	}
+}
+
 impl<H: Hasher> Crumb<H> {
 	/// Move on to next status in the node's sequence.
 	fn increment(&mut self) {
@@ -64,6 +70,12 @@ pub struct TrieDBRawIterator<L: TrieLayout> {
 	key_nibbles: NibbleVec,
 }
 
+impl<L: TrieLayout> Clone for TrieDBRawIterator<L> {
+	fn clone(&self) -> Self {
+		Self { trail: self.trail.clone(), key_nibbles: self.key_nibbles.clone() }
+	}
+}
+
 impl<L: TrieLayout> TrieDBRawIterator<L> {
 	/// Create a new empty iterator.
 	pub fn empty() -> Self {

trie-db/src/lib.rs

@@ -43,6 +43,7 @@ use node::NodeOwned;
 
 pub mod node;
 pub mod proof;
+pub mod query_plan;
 pub mod recorder;
 pub mod sectriedb;
 pub mod sectriedbmut;

trie-db/src/lookup.rs

@@ -367,7 +367,6 @@ where
 					NodeOwned::Leaf(slice, value) =>
 						return if partial == *slice {
 							let value = (*value).clone();
-							drop(node);
 							load_value_owned(
 								value,
 								nibble_key.original_data_as_prefix(),
@@ -395,7 +394,6 @@ where
 					NodeOwned::Branch(children, value) =>
 						if partial.is_empty() {
 							return if let Some(value) = value.clone() {
-								drop(node);
 								load_value_owned(
 									value,
 									nibble_key.original_data_as_prefix(),
@@ -433,7 +431,6 @@ where
 
 						if partial.len() == slice.len() {
 							return if let Some(value) = value.clone() {
-								drop(node);
 								load_value_owned(
 									value,
 									nibble_key.original_data_as_prefix(),