feat(l1): implement TrieIterator + GetAccountRange snap request h…

…andling logic (#960)

**Motivation**
Handling snap capability message `GetAccountRange`

<!-- Why does this pull request exist? What are its goals? -->

**Description**
* Add functionality to iterate the Trie (TrieIterator)
* Add functionality to iterate over all accounts in the state
(Store::iter_accounts)
* Add logic to handle `GetAccountRange` snap request
* Fix slim encoding of `AccountState`
* Remove unneeded trait `RLPEncodeSlim`

**Notes**
* ~We don't have the listen loop implemented so this PR only adds the
standalone logic for handling the request and creating a response, we
still need to plug it in to the main loop.~
* ~We are not able to run the hive test suite due to missing listen loop
+ old blocks being used by the test suite. I instead copied the state
from a Geth execution (loading genesis + importing chain) and used that
state to replicate hive tests as unit tests. These tests could be
removed once we fix those two problems~

<!-- A clear and concise general description of the changes this PR
introduces -->

<!-- Link to issues: Resolves #111, Resolves #222 -->
Partially addresses #184

---------

Co-authored-by: Esteban Dimitroff Hodi <[email protected]>

.github/workflows/hive.yaml

@@ -36,6 +36,9 @@ jobs:
           - simulation: discv4
             name: "Devp2p discv4 tests"
             run_command: make run-hive SIMULATION=devp2p TEST_PATTERN="discv4"
+          - simulation: snap
+            name: "Devp2p snap tests"
+            run_command: make run-hive SIMULATION=devp2p TEST_PATTERN="/AccountRange"
           - simulation: engine
             name: "Engine tests"
             run_command:  make run-hive SIMULATION=ethereum/engine TEST_PATTERN="/Blob Transactions On Block 1, Cancun Genesis|Blob Transactions On Block 1, Shanghai Genesis|Blob Transaction Ordering, Single Account, Single Blob|Blob Transaction Ordering, Single Account, Dual Blob|Blob Transaction Ordering, Multiple Accounts|Replace Blob Transactions|Parallel Blob Transactions|ForkchoiceUpdatedV3 Modifies Payload ID on Different Beacon Root|NewPayloadV3 After Cancun|NewPayloadV3 Versioned Hashes|ForkchoiceUpdated Version on Payload Request"

Makefile

@@ -71,7 +71,7 @@ stop-localnet-silent:
 	@kurtosis enclave stop lambdanet >/dev/null 2>&1 || true
 	@kurtosis enclave rm lambdanet --force >/dev/null 2>&1 || true
 
-HIVE_REVISION := ccf28e5c3e940b2bc4b4f387317ee6a46f5d15c8
+HIVE_REVISION := 421852ec25e4e608fe5460656f4bf0637649619e
 # Shallow clones can't specify a single revision, but at least we avoid working
 # the whole history by making it shallow since a given date (one day before our
 # target revision).

crates/common/rlp/encode.rs

@@ -29,16 +29,6 @@ pub trait RLPEncode {
     }
 }
 
-pub trait RLPEncodeSlim {
-    fn encode(&self, buf: &mut dyn BufMut);
-
-    fn length(&self) -> usize {
-        let mut buf = Vec::new();
-        self.encode(&mut buf);
-        buf.len()
-    }
-}
-
 impl RLPEncode for bool {
     #[inline(always)]
     fn encode(&self, buf: &mut dyn BufMut) {
@@ -378,38 +368,6 @@ impl RLPEncode for ethereum_types::H256 {
     }
 }
 
-impl RLPEncodeSlim for ethereum_types::H256 {
-    fn encode(&self, buf: &mut dyn BufMut) {
-        self.as_bytes().encode(buf)
-    }
-}
-
-impl<T: RLPEncodeSlim> RLPEncodeSlim for Vec<T> {
-    fn encode(&self, buf: &mut dyn BufMut) {
-        if self.is_empty() {
-            buf.put_u8(0xc0);
-        } else {
-            let mut total_len = 0;
-            for item in self {
-                total_len += item.length();
-            }
-            encode_length(total_len, buf);
-            for item in self {
-                item.encode(buf);
-            }
-        }
-    }
-}
-
-impl<S: RLPEncodeSlim, T: RLPEncodeSlim> RLPEncodeSlim for (S, T) {
-    fn encode(&self, buf: &mut dyn BufMut) {
-        let total_len = self.0.length() + self.1.length();
-        encode_length(total_len, buf);
-        self.0.encode(buf);
-        self.1.encode(buf);
-    }
-}
-
 impl RLPEncode for ethereum_types::H264 {
     fn encode(&self, buf: &mut dyn BufMut) {
         self.as_bytes().encode(buf)

crates/common/rlp/structs.rs

@@ -1,5 +1,3 @@
-use crate::encode::RLPEncodeSlim;
-
 use super::{
     decode::{decode_rlp_item, get_item_with_prefix, RLPDecode},
     encode::{encode_length, RLPEncode},
@@ -185,13 +183,6 @@ impl<'a> Encoder<'a> {
         self
     }
 
-    /// Stores a field to be encoded, but in slim format
-    /// https://github.com/ethereum/devp2p/blob/master/caps/snap.md#data-format
-    pub fn encode_slim_field<T: RLPEncodeSlim>(mut self, value: &T) -> Self {
-        <T as RLPEncodeSlim>::encode(value, &mut self.temp_buf);
-        self
-    }
-
     /// If `Some`, stores a field to be encoded, else does nothing.
     pub fn encode_optional_field<T: RLPEncode>(mut self, opt_value: &Option<T>) -> Self {
         if let Some(value) = opt_value {

crates/common/types/account.rs

@@ -6,9 +6,9 @@ use ethereum_types::{H256, U256};
 use sha3::{Digest as _, Keccak256};
 
 use ethereum_rust_rlp::{
-    constants::{RLP_EMPTY_LIST, RLP_NULL},
+    constants::RLP_NULL,
     decode::RLPDecode,
-    encode::{RLPEncode, RLPEncodeSlim},
+    encode::RLPEncode,
     error::RLPDecodeError,
     structs::{Decoder, Encoder},
 };
@@ -100,17 +100,6 @@ impl RLPEncode for AccountInfo {
     }
 }
 
-impl RLPEncodeSlim for AccountInfo {
-    fn encode(&self, buf: &mut dyn bytes::BufMut) {
-        // TODO: check if it's okay to use RLP_EMPTY_LIST
-        Encoder::new(buf)
-            .encode_field(&RLP_EMPTY_LIST)
-            .encode_field(&self.balance)
-            .encode_field(&self.nonce)
-            .finish();
-    }
-}
-
 impl RLPDecode for AccountInfo {
     fn decode_unfinished(rlp: &[u8]) -> Result<(AccountInfo, &[u8]), RLPDecodeError> {
         let decoder = Decoder::new(rlp)?;
@@ -137,18 +126,6 @@ impl RLPEncode for AccountState {
     }
 }
 
-impl RLPEncodeSlim for AccountState {
-    fn encode(&self, buf: &mut dyn bytes::BufMut) {
-        // TODO: check if it's okay to use RLP_EMPTY_LIST
-        Encoder::new(buf)
-            .encode_field(&self.nonce)
-            .encode_field(&self.balance)
-            .encode_field(&RLP_EMPTY_LIST)
-            .encode_field(&self.code_hash)
-            .finish();
-    }
-}
-
 impl RLPDecode for AccountState {
     fn decode_unfinished(rlp: &[u8]) -> Result<(AccountState, &[u8]), RLPDecodeError> {
         let decoder = Decoder::new(rlp)?;

crates/networking/p2p/Cargo.toml

@@ -15,6 +15,7 @@ tokio.workspace = true
 bytes.workspace = true
 hex.workspace = true
 thiserror.workspace = true
+lazy_static.workspace = true
 
 k256 = { version = "0.13.3", features = ["ecdh"] }
 sha3 = "0.10.8"

crates/networking/p2p/net.rs

@@ -31,6 +31,7 @@ pub mod bootnode;
 pub(crate) mod discv4;
 pub(crate) mod kademlia;
 pub mod rlpx;
+pub(crate) mod snap;
 pub mod types;
 
 const MAX_DISC_PACKET_SIZE: usize = 1280;

crates/networking/p2p/rlpx/connection.rs

@@ -1,5 +1,6 @@
 use crate::{
     rlpx::{eth::backend, handshake::encode_ack_message, message::Message, p2p, utils::id2pubkey},
+    snap::process_account_range_request,
     MAX_DISC_PACKET_SIZE,
 };
 
@@ -25,9 +26,8 @@ use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use tracing::{error, info};
 const CAP_P2P: (Capability, u8) = (Capability::P2p, 5);
 const CAP_ETH: (Capability, u8) = (Capability::Eth, 68);
-//const CAP_SNAP: (Capability, u8) = (Capability::Snap, 1);
-const SUPPORTED_CAPABILITIES: [(Capability, u8); 2] = [CAP_P2P, CAP_ETH];
-// pub const SUPPORTED_CAPABILITIES: [(&str, u8); 3] = [CAP_P2P, CAP_ETH, CAP_SNAP)];
+const CAP_SNAP: (Capability, u8) = (Capability::Snap, 1);
+const SUPPORTED_CAPABILITIES: [(Capability, u8); 3] = [CAP_P2P, CAP_ETH, CAP_SNAP];
 
 pub(crate) type Aes256Ctr64BE = ctr::Ctr64BE<aes::Aes256>;
 
@@ -145,6 +145,11 @@ impl<S: AsyncWrite + AsyncRead + std::marker::Unpin> RLPxConnection<S> {
                         Message::Ping(_) => info!("Received Ping"),
                         Message::Pong(_) => info!("Received Pong"),
                         Message::Status(_) => info!("Received Status"),
+                        Message::GetAccountRange(req) => {
+                            let response =
+                                process_account_range_request(req, self.storage.clone())?;
+                            self.send(Message::AccountRange(response)).await
+                        }
                         // TODO: Add new message types and handlers as they are implemented
                         message => return Err(RLPxError::UnexpectedMessage(message)),
                     };

crates/networking/p2p/rlpx/error.rs

@@ -1,4 +1,5 @@
 use crate::rlpx::message::Message;
+use ethereum_rust_storage::error::StoreError;
 use thiserror::Error;
 
 // TODO improve errors
@@ -10,4 +11,6 @@ pub(crate) enum RLPxError {
     InvalidState(String),
     #[error("Unexpected message: {0}")]
     UnexpectedMessage(Message),
+    #[error(transparent)]
+    Store(#[from] StoreError),
 }

crates/networking/p2p/rlpx/frame.rs

@@ -65,7 +65,6 @@ pub(crate) async fn write<S: AsyncWrite + std::marker::Unpin>(
     };
     state.egress_mac.update(frame_mac_seed);
     let frame_mac = state.egress_mac.clone().finalize();
-
     // Send frame-mac
     stream.write_all(&frame_mac[..16]).await.unwrap();
 }

crates/networking/p2p/rlpx/message.rs

@@ -4,6 +4,9 @@ use std::fmt::Display;
 
 use super::eth::status::StatusMessage;
 use super::p2p::{DisconnectMessage, HelloMessage, PingMessage, PongMessage};
+use super::snap::{AccountRange, GetAccountRange};
+
+use ethereum_rust_rlp::encode::RLPEncode;
 
 pub trait RLPxMessage: Sized {
     fn encode(&self, buf: &mut dyn BufMut) -> Result<(), RLPEncodeError>;
@@ -17,6 +20,9 @@ pub(crate) enum Message {
     Ping(PingMessage),
     Pong(PongMessage),
     Status(StatusMessage),
+    // snap capability
+    GetAccountRange(GetAccountRange),
+    AccountRange(AccountRange),
 }
 
 impl Message {
@@ -27,6 +33,8 @@ impl Message {
             0x02 => Ok(Message::Ping(PingMessage::decode(msg_data)?)),
             0x03 => Ok(Message::Pong(PongMessage::decode(msg_data)?)),
             0x10 => Ok(Message::Status(StatusMessage::decode(msg_data)?)),
+            0x21 => Ok(Message::GetAccountRange(GetAccountRange::decode(msg_data)?)),
+            0x22 => Ok(Message::AccountRange(AccountRange::decode(msg_data)?)),
             _ => Err(RLPDecodeError::MalformedData),
         }
     }
@@ -38,6 +46,14 @@ impl Message {
             Message::Ping(msg) => msg.encode(buf),
             Message::Pong(msg) => msg.encode(buf),
             Message::Status(msg) => msg.encode(buf),
+            Message::GetAccountRange(msg) => {
+                0x21_u8.encode(buf);
+                msg.encode(buf)
+            }
+            Message::AccountRange(msg) => {
+                0x22_u8.encode(buf);
+                msg.encode(buf)
+            }
         }
     }
 }
@@ -50,6 +66,8 @@ impl Display for Message {
             Message::Ping(_) => "p2p:Ping".fmt(f),
             Message::Pong(_) => "p2p:Pong".fmt(f),
             Message::Status(_) => "eth:Status".fmt(f),
+            Message::GetAccountRange(_) => "snap:GetAccountRange".fmt(f),
+            Message::AccountRange(_) => "snap:AccountRange".fmt(f),
         }
     }
 }