From 75c77e919f835102b3bfe696e37a9f4e45ab4e43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Lindstr=C3=B8m?= Date: Wed, 21 Jun 2023 12:50:14 +0200 Subject: [PATCH 01/12] Merkle trees first commit --- Cargo.lock | 94 ++++++++++++- Cargo.toml | 3 +- fastcrypto-data/Cargo.toml | 10 ++ fastcrypto-data/src/lib.rs | 4 + fastcrypto-data/src/merkle_tree.rs | 204 +++++++++++++++++++++++++++++ 5 files changed, 309 insertions(+), 6 deletions(-) create mode 100644 fastcrypto-data/Cargo.toml create mode 100644 fastcrypto-data/src/lib.rs create mode 100644 fastcrypto-data/src/merkle_tree.rs diff --git a/Cargo.lock b/Cargo.lock index c0bf5ef302..4e76c34a60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1548,7 +1548,7 @@ dependencies = [ "ed25519-consensus", "elliptic-curve 0.13.4", "eyre", - "fastcrypto-derive", + "fastcrypto-derive 0.1.3", "faster-hex", "generic-array", "hex", @@ -1587,6 +1587,61 @@ dependencies = [ "zeroize", ] +[[package]] +name = "fastcrypto" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "635756fff915364093c8f3fac2a906dfb789baa0c53eda6148f3ca6ca729d52c" +dependencies = [ + "aes", + "aes-gcm", + "ark-ec", + "ark-ff", + "ark-secp256r1", + "ark-serialize", + "auto_ops", + "base64ct", + "bincode", + "blake2", + "blake3", + "blst", + "bs58", + "bulletproofs", + "cbc", + "ctr", + "curve25519-dalek-ng", + "derive_more", + "digest 0.10.6", + "ecdsa 0.16.6", + "ed25519-consensus", + "elliptic-curve 0.13.4", + "eyre", + "fastcrypto-derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "generic-array", + "hex", + "hkdf", + "merlin", + "once_cell", + "p256", + "rand", + "readonly", + "rfc6979 0.4.0", + "rsa", + "schemars", + "secp256k1", + "serde", + "serde_bytes", + "serde_with", + "sha2 0.10.6", + "sha3 0.10.6", + "signature 2.0.0", + "static_assertions", + "thiserror", + "tokio", + "typenum", + "zeroize", +] + [[package]] name = "fastcrypto-cli" version = "0.1.0" @@ -1595,15 +1650,35 @@ dependencies = [ "bincode", "clap 4.1.8", "exitcode", - "fastcrypto", + "fastcrypto 0.1.5", "hex", "rand", "regex", ] +[[package]] +name = "fastcrypto-data" +version = "0.1.0" +dependencies = [ + "fastcrypto 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rs_merkle", +] + +[[package]] +name = "fastcrypto-derive" +version = "0.1.3" +dependencies = [ + "convert_case 0.6.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "fastcrypto-derive" version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c0c2af2157f416cb885e11d36cd0de2753f6d5384752d364075c835f5f8f891" dependencies = [ "convert_case 0.6.0", "proc-macro2", @@ -1618,8 +1693,8 @@ dependencies = [ "bincode", "criterion 0.4.0", "digest 0.10.6", - "fastcrypto", - "fastcrypto-derive", + "fastcrypto 0.1.5", + "fastcrypto-derive 0.1.3", "rand", "serde", "sha3 0.10.6", @@ -1648,7 +1723,7 @@ dependencies = [ "byte-slice-cast", "criterion 0.4.0", "derive_more", - "fastcrypto", + "fastcrypto 0.1.5", "hex", "num-bigint", "once_cell", @@ -2773,6 +2848,15 @@ dependencies = [ "syn", ] +[[package]] +name = "rs_merkle" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05225752ca6ede4cb1b73aa37ce3904affd042e98f28246f56f438ebfd47a810" +dependencies = [ + "sha2 0.10.6", +] + [[package]] name = "rsa" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index b9c9099d23..56ece8f59e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,8 @@ members = [ "fastcrypto-derive", "fastcrypto-tbls", "fastcrypto-zkp", - "fastcrypto-cli" + "fastcrypto-cli", + "fastcrypto-data", ] # Dependencies that should be kept in sync through the whole workspace diff --git a/fastcrypto-data/Cargo.toml b/fastcrypto-data/Cargo.toml new file mode 100644 index 0000000000..7715923e18 --- /dev/null +++ b/fastcrypto-data/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "fastcrypto-data" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rs_merkle = "1.4.1" +fastcrypto = "0.1.5" diff --git a/fastcrypto-data/src/lib.rs b/fastcrypto-data/src/lib.rs new file mode 100644 index 0000000000..d665ca2f38 --- /dev/null +++ b/fastcrypto-data/src/lib.rs @@ -0,0 +1,4 @@ +// Copyright (c) 2022, Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +pub mod merkle_tree; diff --git a/fastcrypto-data/src/merkle_tree.rs b/fastcrypto-data/src/merkle_tree.rs new file mode 100644 index 0000000000..3780d2641c --- /dev/null +++ b/fastcrypto-data/src/merkle_tree.rs @@ -0,0 +1,204 @@ +use std::marker::PhantomData; + +use fastcrypto::error::FastCryptoError; +use fastcrypto::hash::HashFunction; +use rs_merkle::{Hasher, MerkleProof, MerkleTree as ExternalMerkleTree}; + +/// This represents a Merkle Tree with an arbitrary number of elements of type `T`. The [prove] function +/// can generate proofs that the leaf of a given index has a certain hash value. +/// +/// New elements may be added continuously but once a verifier is generated with the [create_verifier] +/// function, the proofs are only valid for the state of the tree at that point. +/// +/// To avoid second-preimage attacks, a 0x00 byte is prepended to the hash data for leaf nodes (see +/// [LeafHasher]), and 0x01 is prepended when computing internal node hashes (see [InternalNodeHasher]). +pub struct MerkleTree, T: AsRef<[u8]>> { + tree: ExternalMerkleTree>, + _type: PhantomData, +} + +impl, T: AsRef<[u8]>> + MerkleTree +{ + pub fn new() -> Self { + MerkleTree { + tree: ExternalMerkleTree::new(), + _type: PhantomData::default(), + } + } +} + +/// This verifier can verify proofs generated by [MerkleTree::prove]. +pub struct MerkleTreeVerifier< + const DIGEST_LENGTH: usize, + H: HashFunction, + T: AsRef<[u8]>, +> { + root: [u8; DIGEST_LENGTH], + number_of_leaves: usize, + _hash_function: PhantomData, + _type: PhantomData, +} + +impl, T: AsRef<[u8]>> + MerkleTreeVerifier +{ + /// Verify a [Proof] that an element with the given hash was at this index of this tree at the time + /// this verifier was created. + pub fn verify( + &self, + index: usize, + leaf_hash: [u8; DIGEST_LENGTH], + proof: &Proof, + ) -> bool { + proof + .proof + .verify(self.root, &[index], &[leaf_hash], self.number_of_leaves) + } + + /// Verify a [Proof] that an element was at this index of this tree at the time this verifier was + /// created. + pub fn verify_with_element( + &self, + index: usize, + element: &T, + proof: &Proof, + ) -> bool { + self.verify( + index, + LeafHasher::::hash(element.as_ref()), + proof, + ) + } +} + +impl, T: AsRef<[u8]>> + MerkleTree +{ + /// Insert element in this tree and return the index of the newly inserted element. + pub fn insert(&mut self, element: &T) -> usize { + let hash = LeafHasher::::hash(element.as_ref()); + self.tree.insert(hash).commit(); + self.tree.leaves_len() - 1 + } + + /// Insert element in this tree and return the index of the last element. + pub fn insert_all(&mut self, elements: &[T]) -> usize { + let mut hashes = elements + .into_iter() + .map(|element| LeafHasher::::hash(element.as_ref())) + .collect(); + self.tree.append(&mut hashes).commit(); + self.tree.leaves_len() - 1 + } + + /// Create a proof for the element at the given index. + pub fn prove(&self, index: usize) -> Proof { + Proof { + proof: self.tree.proof(&[index]), + _type: PhantomData::default(), + } + } + + /// Create a [MerkleTreeVerifier] for the current state of this tree. + pub fn create_verifier( + &self, + ) -> Result, FastCryptoError> { + Ok(MerkleTreeVerifier { + root: self + .tree + .root() + .ok_or_else(|| FastCryptoError::GeneralError("Tree is empty".to_string()))?, + number_of_leaves: self.tree.leaves_len(), + _hash_function: PhantomData::default(), + _type: PhantomData::default(), + }) + } + + /// Return the number of leaves in this tree. + pub fn number_of_leaves(&self) -> usize { + self.tree.leaves_len() + } + + /// Returns the root of this tree. + pub fn root(&self) -> Result<[u8; DIGEST_LENGTH], FastCryptoError> { + Ok(self + .tree + .root() + .ok_or_else(|| FastCryptoError::GeneralError("Tree is empty".to_string()))?) + } +} + +/// A proof +pub struct Proof, T: AsRef<[u8]>> { + proof: MerkleProof>, + _type: PhantomData, +} + +struct PrefixedHasher> +{ + _hasher: PhantomData, +} + +impl> Clone + for PrefixedHasher +{ + fn clone(&self) -> Self { + Self { + _hasher: PhantomData::default(), + } + } +} + +impl> Hasher + for PrefixedHasher +{ + type Hash = [u8; DIGEST_LENGTH]; + + fn hash(data: &[u8]) -> Self::Hash { + let mut input = vec![]; + input.push(PREFIX); + input.extend_from_slice(data); + H::digest(input).digest + } +} + +/// Computes H(0x01 || X) +type InternalNodeHasher = PrefixedHasher<0x01, DIGEST_LENGTH, H>; + +/// Computes H(0x00 || X) +type LeafHasher = PrefixedHasher<0x00, DIGEST_LENGTH, H>; + +#[cfg(test)] +mod tests { + use crate::merkle_tree::MerkleTree; + use fastcrypto::hash::Sha256; + + #[test] + fn test_merkle_tree() { + let mut tree = MerkleTree::<32, Sha256, Vec>::new(); + + let elements = [vec![1u8], vec![2u8], vec![3u8]]; + assert_eq!(0, tree.number_of_leaves()); + tree.insert_all(&elements); + assert_eq!(3, tree.number_of_leaves()); + + let root = tree.root(); + + // Generate proof for a given element + let index = 1; + let element = &elements[index]; + + let proof = tree.prove(index); + + let verifier = tree.create_verifier().unwrap(); + assert!(verifier.verify_with_element(index, element, &proof)); + + // Adding + tree.insert(&vec![4u8]); + assert_ne!(root, tree.root()); + + let verifier = tree.create_verifier().unwrap(); + assert!(!verifier.verify_with_element(index, element, &proof)); + } +} From d84fdfd36dbbe707586898d073f751fcd4817c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Lindstr=C3=B8m?= Date: Wed, 21 Jun 2023 13:19:13 +0200 Subject: [PATCH 02/12] Use an iterator for insert_all --- fastcrypto-data/src/merkle_tree.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/fastcrypto-data/src/merkle_tree.rs b/fastcrypto-data/src/merkle_tree.rs index 3780d2641c..73e8129a2c 100644 --- a/fastcrypto-data/src/merkle_tree.rs +++ b/fastcrypto-data/src/merkle_tree.rs @@ -1,3 +1,4 @@ +use std::borrow::Borrow; use std::marker::PhantomData; use fastcrypto::error::FastCryptoError; @@ -82,11 +83,11 @@ impl, T: AsRef<[u8]>> self.tree.leaves_len() - 1 } - /// Insert element in this tree and return the index of the last element. - pub fn insert_all(&mut self, elements: &[T]) -> usize { + /// Insert all elements in the iterator into this tree. The elements are given consecutive indices + /// and the return value is the index of the last element. + pub fn insert_all>(&mut self, elements: impl Iterator) -> usize { let mut hashes = elements - .into_iter() - .map(|element| LeafHasher::::hash(element.as_ref())) + .map(|element| LeafHasher::::hash(element.borrow().as_ref())) .collect(); self.tree.append(&mut hashes).commit(); self.tree.leaves_len() - 1 @@ -179,26 +180,27 @@ mod tests { let mut tree = MerkleTree::<32, Sha256, Vec>::new(); let elements = [vec![1u8], vec![2u8], vec![3u8]]; + let index = 1; + let element = &elements[index]; + assert_eq!(0, tree.number_of_leaves()); - tree.insert_all(&elements); + tree.insert_all(elements.iter()); assert_eq!(3, tree.number_of_leaves()); let root = tree.root(); // Generate proof for a given element - let index = 1; - let element = &elements[index]; let proof = tree.prove(index); let verifier = tree.create_verifier().unwrap(); - assert!(verifier.verify_with_element(index, element, &proof)); + assert!(verifier.verify_with_element(index, &element, &proof)); // Adding tree.insert(&vec![4u8]); assert_ne!(root, tree.root()); let verifier = tree.create_verifier().unwrap(); - assert!(!verifier.verify_with_element(index, element, &proof)); + assert!(!verifier.verify_with_element(index, &element, &proof)); } } From 0428587fb9df3ce55ed4178c3916397bb6ee99ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Lindstr=C3=B8m?= Date: Wed, 21 Jun 2023 14:50:02 +0200 Subject: [PATCH 03/12] Adding many elements streaming. + clippy --- fastcrypto-data/src/merkle_tree.rs | 81 ++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/fastcrypto-data/src/merkle_tree.rs b/fastcrypto-data/src/merkle_tree.rs index 73e8129a2c..f4e9701ecf 100644 --- a/fastcrypto-data/src/merkle_tree.rs +++ b/fastcrypto-data/src/merkle_tree.rs @@ -29,6 +29,14 @@ impl, T: AsRef<[u8]>> } } +impl, T: AsRef<[u8]>> Default + for MerkleTree +{ + fn default() -> Self { + Self::new() + } +} + /// This verifier can verify proofs generated by [MerkleTree::prove]. pub struct MerkleTreeVerifier< const DIGEST_LENGTH: usize, @@ -76,23 +84,38 @@ impl, T: AsRef<[u8]>> impl, T: AsRef<[u8]>> MerkleTree { + /// Hash an element using the hash function used for this tree. + pub fn hash(element: &T) -> [u8; DIGEST_LENGTH] { + LeafHasher::::hash(element.as_ref()) + } + /// Insert element in this tree and return the index of the newly inserted element. - pub fn insert(&mut self, element: &T) -> usize { - let hash = LeafHasher::::hash(element.as_ref()); + fn insert_hash(&mut self, hash: [u8; DIGEST_LENGTH]) -> usize { self.tree.insert(hash).commit(); self.tree.leaves_len() - 1 } /// Insert all elements in the iterator into this tree. The elements are given consecutive indices /// and the return value is the index of the last element. - pub fn insert_all>(&mut self, elements: impl Iterator) -> usize { - let mut hashes = elements - .map(|element| LeafHasher::::hash(element.borrow().as_ref())) - .collect(); - self.tree.append(&mut hashes).commit(); + fn insert_hashes(&mut self, hashes: impl Iterator) -> usize { + for hash in hashes { + self.tree.insert(hash); + } + self.tree.commit(); self.tree.leaves_len() - 1 } + /// Insert element in this tree and return the index of the newly inserted element. + pub fn insert_element(&mut self, element: &T) -> usize { + self.insert_hash(Self::hash(element)) + } + + /// Insert all elements in the iterator into this tree. The elements are given consecutive indices + /// and the return value is the index of the last element. + pub fn insert_elements(&mut self, elements: impl Iterator>) -> usize { + self.insert_hashes(elements.map(|element| Self::hash(element.borrow()))) + } + /// Create a proof for the element at the given index. pub fn prove(&self, index: usize) -> Proof { Proof { @@ -123,19 +146,19 @@ impl, T: AsRef<[u8]>> /// Returns the root of this tree. pub fn root(&self) -> Result<[u8; DIGEST_LENGTH], FastCryptoError> { - Ok(self - .tree + self.tree .root() - .ok_or_else(|| FastCryptoError::GeneralError("Tree is empty".to_string()))?) + .ok_or_else(|| FastCryptoError::GeneralError("Tree is empty".to_string())) } } -/// A proof +/// A proof that a leaf with a given index in a Merkle Tree has a certain hash value. pub struct Proof, T: AsRef<[u8]>> { proof: MerkleProof>, _type: PhantomData, } +/// A hash function which given input `X` computes `H(PREFIX ||X)` struct PrefixedHasher> { _hasher: PhantomData, @@ -172,35 +195,39 @@ type LeafHasher = PrefixedHasher<0x00, DIGEST_LEN #[cfg(test)] mod tests { - use crate::merkle_tree::MerkleTree; use fastcrypto::hash::Sha256; + use crate::merkle_tree::MerkleTree; + #[test] fn test_merkle_tree() { let mut tree = MerkleTree::<32, Sha256, Vec>::new(); + // An empty tree does not have a root + assert!(tree.root().is_err()); + assert!(tree.create_verifier().is_err()); + let elements = [vec![1u8], vec![2u8], vec![3u8]]; let index = 1; let element = &elements[index]; + // Adding elements should change the number of leaves assert_eq!(0, tree.number_of_leaves()); - tree.insert_all(elements.iter()); - assert_eq!(3, tree.number_of_leaves()); - - let root = tree.root(); - - // Generate proof for a given element + assert_eq!(elements.len() - 1, tree.insert_elements(elements.iter())); + assert_eq!(elements.len(), tree.number_of_leaves()); + // Generate proof for a given element and verify let proof = tree.prove(index); - - let verifier = tree.create_verifier().unwrap(); - assert!(verifier.verify_with_element(index, &element, &proof)); - - // Adding - tree.insert(&vec![4u8]); - assert_ne!(root, tree.root()); - let verifier = tree.create_verifier().unwrap(); - assert!(!verifier.verify_with_element(index, &element, &proof)); + assert!(verifier.verify_with_element(index, element, &proof)); + assert!(!verifier.verify_with_element(index, &elements[index - 1], &proof)); + + // Adding another element changes the root and the old proof should no longer verify + let root = tree.root().unwrap(); + tree.insert_element(&vec![4u8]); + let new_root = tree.root().unwrap(); + assert_ne!(root, new_root); + let new_verifier = tree.create_verifier().unwrap(); + assert!(!new_verifier.verify_with_element(index, element, &proof)); } } From 3261c438492a6296c6da40299ac7bcec6a292747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Lindstr=C3=B8m?= Date: Wed, 21 Jun 2023 14:52:57 +0200 Subject: [PATCH 04/12] License --- fastcrypto-data/src/merkle_tree.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fastcrypto-data/src/merkle_tree.rs b/fastcrypto-data/src/merkle_tree.rs index f4e9701ecf..8cc91a6037 100644 --- a/fastcrypto-data/src/merkle_tree.rs +++ b/fastcrypto-data/src/merkle_tree.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2022, Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + use std::borrow::Borrow; use std::marker::PhantomData; From fd18d1b04cea02d5b40011d4609a76c69251297f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Lindstr=C3=B8m?= Date: Wed, 21 Jun 2023 14:59:07 +0200 Subject: [PATCH 05/12] Add license in cargo.toml and readme --- fastcrypto-data/Cargo.toml | 5 +++++ fastcrypto-data/README.md | 4 ++++ 2 files changed, 9 insertions(+) create mode 100644 fastcrypto-data/README.md diff --git a/fastcrypto-data/Cargo.toml b/fastcrypto-data/Cargo.toml index 7715923e18..ea0c44cbf8 100644 --- a/fastcrypto-data/Cargo.toml +++ b/fastcrypto-data/Cargo.toml @@ -2,6 +2,11 @@ name = "fastcrypto-data" version = "0.1.0" edition = "2021" +license = "Apache-2.0" +authors = ["Mysten Labs "] +readme = "../README.md" +description = "Collection of useful cryptographic data structures" +repository = "https://github.com/MystenLabs/fastcrypto" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/fastcrypto-data/README.md b/fastcrypto-data/README.md new file mode 100644 index 0000000000..7512a4c778 --- /dev/null +++ b/fastcrypto-data/README.md @@ -0,0 +1,4 @@ +fastcrypto-data +=== + +A collection of useful cryptographic data structures. \ No newline at end of file From e54a367291e859af41f4b21395305005d5620031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Lindstr=C3=B8m?= Date: Wed, 21 Jun 2023 15:12:02 +0200 Subject: [PATCH 06/12] Add module doc --- fastcrypto-data/src/merkle_tree.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/fastcrypto-data/src/merkle_tree.rs b/fastcrypto-data/src/merkle_tree.rs index 8cc91a6037..cafbf35cae 100644 --- a/fastcrypto-data/src/merkle_tree.rs +++ b/fastcrypto-data/src/merkle_tree.rs @@ -1,6 +1,27 @@ // Copyright (c) 2022, Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +//! This module contains an implementation of a Merkle Tree (see Merkle, R.C. (1988): A Digital Signature +//! Based on a Conventional Encryption Function). Any hash function can be used and we use the method +//! from section 2.1 in RFC9162 to avoid second pre-image attacks. +//! +//! An arbitrary number of elements of a given type can be added as leaves to the tree and we can then +//! construct proofs logarithmic in the number of leaves that a certain leaf has a given value. +//! +//! # Example +//! ```rust +//! # use fastcrypto::hash::Sha256; +//! use fastcrypto_data::merkle_tree::*; +//! let elements = [[1u8], [2u8], [3u8]]; +//! let mut tree = MerkleTree::<32, Sha256, [u8; 1]>::new(); +//! tree.insert_elements(elements.iter()); +//! +//! let index = 1; +//! let proof = tree.prove(index); +//! +//! let verifier = tree.create_verifier().unwrap(); +//! assert!(verifier.verify_with_element(index, &elements[index], &proof)); +//! ``` use std::borrow::Borrow; use std::marker::PhantomData; From 275767998ad83d30a48155254138e8b02593f7dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Lindstr=C3=B8m?= Date: Wed, 21 Jun 2023 15:13:10 +0200 Subject: [PATCH 07/12] Add more module doc --- fastcrypto-data/src/merkle_tree.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fastcrypto-data/src/merkle_tree.rs b/fastcrypto-data/src/merkle_tree.rs index cafbf35cae..4acc808371 100644 --- a/fastcrypto-data/src/merkle_tree.rs +++ b/fastcrypto-data/src/merkle_tree.rs @@ -5,8 +5,10 @@ //! Based on a Conventional Encryption Function). Any hash function can be used and we use the method //! from section 2.1 in RFC9162 to avoid second pre-image attacks. //! -//! An arbitrary number of elements of a given type can be added as leaves to the tree and we can then -//! construct proofs logarithmic in the number of leaves that a certain leaf has a given value. +//! An arbitrary number of elements of a given type `T` can be added as leaves to the tree and we can then +//! construct proofs logarithmic in the number of leaves that a certain leaf has a given value. Such +//! proofs can be constructed by a small [MerkleTreeVerifier] which only needs to know the root of the +//! tree. //! //! # Example //! ```rust @@ -22,6 +24,7 @@ //! let verifier = tree.create_verifier().unwrap(); //! assert!(verifier.verify_with_element(index, &elements[index], &proof)); //! ``` + use std::borrow::Borrow; use std::marker::PhantomData; From 3c5508f1c13a8d94a7ca27cbc18c548087c8d1b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Lindstr=C3=B8m?= Date: Wed, 21 Jun 2023 15:18:53 +0200 Subject: [PATCH 08/12] Module doc --- fastcrypto-data/src/merkle_tree.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/fastcrypto-data/src/merkle_tree.rs b/fastcrypto-data/src/merkle_tree.rs index 4acc808371..dcd6a105a3 100644 --- a/fastcrypto-data/src/merkle_tree.rs +++ b/fastcrypto-data/src/merkle_tree.rs @@ -1,19 +1,16 @@ // Copyright (c) 2022, Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -//! This module contains an implementation of a Merkle Tree (see Merkle, R.C. (1988): A Digital Signature -//! Based on a Conventional Encryption Function). Any hash function can be used and we use the method -//! from section 2.1 in RFC9162 to avoid second pre-image attacks. -//! -//! An arbitrary number of elements of a given type `T` can be added as leaves to the tree and we can then -//! construct proofs logarithmic in the number of leaves that a certain leaf has a given value. Such -//! proofs can be constructed by a small [MerkleTreeVerifier] which only needs to know the root of the -//! tree. +//! This module contains an implementation of a Merkle Tree data structure (Merkle, R.C. (1988): A Digital Signature +//! Based on a Conventional Encryption Function) which is a data structure that allows an arbitrary +//! number of elements of a given type `T` to be added as leaves to the tree and we can then construct +//! proofs logarithmic in the number of leaves that a certain leaf has a given value. Such proofs can +//! be verified by a small verifier which only needs to know the root of the tree. //! //! # Example //! ```rust //! # use fastcrypto::hash::Sha256; -//! use fastcrypto_data::merkle_tree::*; +//! # use fastcrypto_data::merkle_tree::*; //! let elements = [[1u8], [2u8], [3u8]]; //! let mut tree = MerkleTree::<32, Sha256, [u8; 1]>::new(); //! tree.insert_elements(elements.iter()); From ca606ea7897c02dfac8e673d4c38023f19cb48c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Lindstr=C3=B8m?= Date: Wed, 21 Jun 2023 15:22:12 +0200 Subject: [PATCH 09/12] Align function names --- fastcrypto-data/src/merkle_tree.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastcrypto-data/src/merkle_tree.rs b/fastcrypto-data/src/merkle_tree.rs index dcd6a105a3..d7cf69c795 100644 --- a/fastcrypto-data/src/merkle_tree.rs +++ b/fastcrypto-data/src/merkle_tree.rs @@ -78,7 +78,7 @@ impl, T: AsRef<[u8]>> { /// Verify a [Proof] that an element with the given hash was at this index of this tree at the time /// this verifier was created. - pub fn verify( + pub fn verify_with_hash( &self, index: usize, leaf_hash: [u8; DIGEST_LENGTH], @@ -97,7 +97,7 @@ impl, T: AsRef<[u8]>> element: &T, proof: &Proof, ) -> bool { - self.verify( + self.verify_with_hash( index, LeafHasher::::hash(element.as_ref()), proof, From b310bc90e63efe45d10d499664b70e5421445d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Lindstr=C3=B8m?= Date: Wed, 21 Jun 2023 16:39:58 +0200 Subject: [PATCH 10/12] Clean up --- fastcrypto-data/src/merkle_tree.rs | 74 ++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/fastcrypto-data/src/merkle_tree.rs b/fastcrypto-data/src/merkle_tree.rs index d7cf69c795..bf8176293e 100644 --- a/fastcrypto-data/src/merkle_tree.rs +++ b/fastcrypto-data/src/merkle_tree.rs @@ -13,13 +13,13 @@ //! # use fastcrypto_data::merkle_tree::*; //! let elements = [[1u8], [2u8], [3u8]]; //! let mut tree = MerkleTree::<32, Sha256, [u8; 1]>::new(); -//! tree.insert_elements(elements.iter()); +//! tree.insert_all(elements.iter()); //! //! let index = 1; //! let proof = tree.prove(index); //! //! let verifier = tree.create_verifier().unwrap(); -//! assert!(verifier.verify_with_element(index, &elements[index], &proof)); +//! assert!(verifier.verify(index, &elements[index], &proof)); //! ``` use std::borrow::Borrow; @@ -78,7 +78,7 @@ impl, T: AsRef<[u8]>> { /// Verify a [Proof] that an element with the given hash was at this index of this tree at the time /// this verifier was created. - pub fn verify_with_hash( + fn verify_with_hash( &self, index: usize, leaf_hash: [u8; DIGEST_LENGTH], @@ -91,7 +91,7 @@ impl, T: AsRef<[u8]>> /// Verify a [Proof] that an element was at this index of this tree at the time this verifier was /// created. - pub fn verify_with_element( + pub fn verify( &self, index: usize, element: &T, @@ -130,13 +130,13 @@ impl, T: AsRef<[u8]>> } /// Insert element in this tree and return the index of the newly inserted element. - pub fn insert_element(&mut self, element: &T) -> usize { + pub fn insert(&mut self, element: &T) -> usize { self.insert_hash(Self::hash(element)) } /// Insert all elements in the iterator into this tree. The elements are given consecutive indices /// and the return value is the index of the last element. - pub fn insert_elements(&mut self, elements: impl Iterator>) -> usize { + pub fn insert_all(&mut self, elements: impl Iterator>) -> usize { self.insert_hashes(elements.map(|element| Self::hash(element.borrow()))) } @@ -219,9 +219,11 @@ type LeafHasher = PrefixedHasher<0x00, DIGEST_LEN #[cfg(test)] mod tests { - use fastcrypto::hash::Sha256; - - use crate::merkle_tree::MerkleTree; + use crate::merkle_tree::{LeafHasher, MerkleTree, MerkleTreeVerifier, Proof}; + use fastcrypto::hash::{HashFunction, Sha256}; + use rs_merkle::proof_serializers::ReverseHashesOrder; + use rs_merkle::{Hasher, MerkleProof}; + use std::marker::PhantomData; #[test] fn test_merkle_tree() { @@ -237,21 +239,65 @@ mod tests { // Adding elements should change the number of leaves assert_eq!(0, tree.number_of_leaves()); - assert_eq!(elements.len() - 1, tree.insert_elements(elements.iter())); + assert_eq!(elements.len() - 1, tree.insert_all(elements.iter())); assert_eq!(elements.len(), tree.number_of_leaves()); // Generate proof for a given element and verify let proof = tree.prove(index); let verifier = tree.create_verifier().unwrap(); - assert!(verifier.verify_with_element(index, element, &proof)); - assert!(!verifier.verify_with_element(index, &elements[index - 1], &proof)); + assert!(verifier.verify(index, element, &proof)); + assert!(!verifier.verify(index, &elements[index - 1], &proof)); // Adding another element changes the root and the old proof should no longer verify let root = tree.root().unwrap(); - tree.insert_element(&vec![4u8]); + tree.insert(&vec![4u8]); let new_root = tree.root().unwrap(); assert_ne!(root, new_root); let new_verifier = tree.create_verifier().unwrap(); - assert!(!new_verifier.verify_with_element(index, element, &proof)); + assert!(!new_verifier.verify(index, element, &proof)); + } + + #[test] + fn test_preimage_attack() { + let mut tree = MerkleTree::<32, Sha256, Vec>::new(); + let elements = [vec![0u8], vec![1u8], vec![2u8], vec![3u8]]; + tree.insert_all(elements.iter()); + + // Create a proof for the first element in the tree and verify + let proof = tree.prove(0); + let verifier = tree.create_verifier().unwrap(); + assert!(verifier.verify(0, &elements[0], &proof)); + assert!(verifier.verify_with_hash(0, LeafHasher::<32, Sha256>::hash(&elements[0]), &proof)); + + // Create a modified proof where the nodes in the layer below the leaves are seen as leaves + let modified_proof = Proof { + proof: MerkleProof::from_bytes(&proof.proof.serialize::()[0..32]) + .unwrap(), + _type: PhantomData::default(), + }; + + // Compute the leaf hash of this new modified tree. This is equal to the hash of the first + // internal node in the layer just below the nodes: + let mut hasher = Sha256::default(); + hasher.update([1u8]); + hasher.update(LeafHasher::<32, Sha256>::hash(&elements[0])); + hasher.update(LeafHasher::<32, Sha256>::hash(&elements[1])); + let hash = hasher.finalize().digest; + + // This modified proof should fail against the actual tree because the number of hashes does + // not match the depth of the tree. + assert!(!verifier.verify_with_hash(0, hash, &modified_proof)); + + // If we modify the verifier to think that the tree only has two elements instead of four, the + // modified proof verifies. This is avoided if we either a) uses the MerkleTreeVerifier struct + // to verify proofs or b) uses the verify instead where we need to provide the element instead + // of the hash. + let modified_verifier = MerkleTreeVerifier::<32, Sha256, Vec> { + root: tree.tree.root().unwrap(), + number_of_leaves: 2, + _hash_function: Default::default(), + _type: Default::default(), + }; + assert!(modified_verifier.verify_with_hash(0, hash, &modified_proof)); } } From cc805fa5627cb5d6c504b5b49b2c33251c38a39a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Lindstr=C3=B8m?= Date: Wed, 21 Jun 2023 23:35:50 +0200 Subject: [PATCH 11/12] fmt and fc import --- fastcrypto-data/Cargo.toml | 2 +- fastcrypto-data/src/merkle_tree.rs | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/fastcrypto-data/Cargo.toml b/fastcrypto-data/Cargo.toml index ea0c44cbf8..da4712f31c 100644 --- a/fastcrypto-data/Cargo.toml +++ b/fastcrypto-data/Cargo.toml @@ -12,4 +12,4 @@ repository = "https://github.com/MystenLabs/fastcrypto" [dependencies] rs_merkle = "1.4.1" -fastcrypto = "0.1.5" +fastcrypto = { path = "../fastcrypto" } diff --git a/fastcrypto-data/src/merkle_tree.rs b/fastcrypto-data/src/merkle_tree.rs index bf8176293e..edffa2ef19 100644 --- a/fastcrypto-data/src/merkle_tree.rs +++ b/fastcrypto-data/src/merkle_tree.rs @@ -91,12 +91,7 @@ impl, T: AsRef<[u8]>> /// Verify a [Proof] that an element was at this index of this tree at the time this verifier was /// created. - pub fn verify( - &self, - index: usize, - element: &T, - proof: &Proof, - ) -> bool { + pub fn verify(&self, index: usize, element: &T, proof: &Proof) -> bool { self.verify_with_hash( index, LeafHasher::::hash(element.as_ref()), From 3985f460481925a715c280d1b3009928e0935b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Lindstr=C3=B8m?= Date: Wed, 21 Jun 2023 23:44:48 +0200 Subject: [PATCH 12/12] Update Cargo.lock --- Cargo.lock | 79 +++++------------------------------------------------- 1 file changed, 6 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4e76c34a60..e427aef906 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1548,7 +1548,7 @@ dependencies = [ "ed25519-consensus", "elliptic-curve 0.13.4", "eyre", - "fastcrypto-derive 0.1.3", + "fastcrypto-derive", "faster-hex", "generic-array", "hex", @@ -1587,61 +1587,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "fastcrypto" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635756fff915364093c8f3fac2a906dfb789baa0c53eda6148f3ca6ca729d52c" -dependencies = [ - "aes", - "aes-gcm", - "ark-ec", - "ark-ff", - "ark-secp256r1", - "ark-serialize", - "auto_ops", - "base64ct", - "bincode", - "blake2", - "blake3", - "blst", - "bs58", - "bulletproofs", - "cbc", - "ctr", - "curve25519-dalek-ng", - "derive_more", - "digest 0.10.6", - "ecdsa 0.16.6", - "ed25519-consensus", - "elliptic-curve 0.13.4", - "eyre", - "fastcrypto-derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "generic-array", - "hex", - "hkdf", - "merlin", - "once_cell", - "p256", - "rand", - "readonly", - "rfc6979 0.4.0", - "rsa", - "schemars", - "secp256k1", - "serde", - "serde_bytes", - "serde_with", - "sha2 0.10.6", - "sha3 0.10.6", - "signature 2.0.0", - "static_assertions", - "thiserror", - "tokio", - "typenum", - "zeroize", -] - [[package]] name = "fastcrypto-cli" version = "0.1.0" @@ -1650,7 +1595,7 @@ dependencies = [ "bincode", "clap 4.1.8", "exitcode", - "fastcrypto 0.1.5", + "fastcrypto", "hex", "rand", "regex", @@ -1660,7 +1605,7 @@ dependencies = [ name = "fastcrypto-data" version = "0.1.0" dependencies = [ - "fastcrypto 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "fastcrypto", "rs_merkle", ] @@ -1674,18 +1619,6 @@ dependencies = [ "syn", ] -[[package]] -name = "fastcrypto-derive" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c0c2af2157f416cb885e11d36cd0de2753f6d5384752d364075c835f5f8f891" -dependencies = [ - "convert_case 0.6.0", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "fastcrypto-tbls" version = "0.1.0" @@ -1693,8 +1626,8 @@ dependencies = [ "bincode", "criterion 0.4.0", "digest 0.10.6", - "fastcrypto 0.1.5", - "fastcrypto-derive 0.1.3", + "fastcrypto", + "fastcrypto-derive", "rand", "serde", "sha3 0.10.6", @@ -1723,7 +1656,7 @@ dependencies = [ "byte-slice-cast", "criterion 0.4.0", "derive_more", - "fastcrypto 0.1.5", + "fastcrypto", "hex", "num-bigint", "once_cell",