From 31caef45450567ced6b783228f17804af6b73848 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 3 Nov 2024 18:41:50 +0700 Subject: [PATCH 01/31] Failing test for historical events --- packages/ciphernode/evm/tests/evm_reader.rs | 99 ++++++++++++++++++--- 1 file changed, 85 insertions(+), 14 deletions(-) diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index 96315c25..1e2d4a66 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -1,7 +1,9 @@ use actix::Actor; use alloy::{ node_bindings::Anvil, - providers::{ProviderBuilder, WsConnect}, + primitives::{FixedBytes, LogData}, + providers::{ProviderBuilder, RootProvider, WsConnect}, + pubsub::PubSubFrontend, sol, sol_types::SolEvent, }; @@ -17,31 +19,39 @@ sol!( "tests/fixtures/emit_logs.json" ); +fn test_event_extractor( + data: &LogData, + topic: Option<&FixedBytes<32>>, + _chain_id: u64, +) -> Option { + match topic { + Some(&EmitLogs::ValueChanged::SIGNATURE_HASH) => { + let Ok(event) = EmitLogs::ValueChanged::decode_log_data(data, true) else { + return None; + }; + Some(EnclaveEvent::from(TestEvent { + msg: event.newValue, + })) + } + _ => None, + } +} + #[actix::test] -async fn test_logs() -> Result<()> { +async fn test_evm_reader() -> Result<()> { // Create a WS provider // NOTE: Anvil must be available on $PATH let anvil = Anvil::new().block_time(1).try_spawn()?; let ws = WsConnect::new(anvil.ws_endpoint()); let provider = ProviderBuilder::new().on_ws(ws).await?; let arc_provider = WithChainId::new(provider).await?; - let contract = Arc::new(EmitLogs::deploy(arc_provider.get_provider()).await?); + let contract = EmitLogs::deploy(arc_provider.get_provider()).await?; let bus = EventBus::new(true).start(); EvmEventReader::attach( &bus, &arc_provider, - |data, topic, _| match topic { - Some(&EmitLogs::ValueChanged::SIGNATURE_HASH) => { - let Ok(event) = EmitLogs::ValueChanged::decode_log_data(data, true) else { - return None; - }; - Some(EnclaveEvent::from(TestEvent { - msg: event.newValue, - })) - } - _ => None, - }, + test_event_extractor, &contract.address().to_string(), ) .await?; @@ -78,3 +88,64 @@ async fn test_logs() -> Result<()> { Ok(()) } + +#[actix::test] +async fn test_ensure_historical_events() -> Result<()> { + // Create a WS provider + // NOTE: Anvil must be available on $PATH + let anvil = Anvil::new().block_time(1).try_spawn()?; + let ws = WsConnect::new(anvil.ws_endpoint()); + let provider = ProviderBuilder::new().on_ws(ws).await?; + let arc_provider = WithChainId::new(provider).await?; + let contract = EmitLogs::deploy(arc_provider.get_provider()).await?; + let bus = EventBus::new(true).start(); + + let historical_msgs = vec!["these", "are", "historical", "events"]; + let live_events = vec!["these", "events", "are", "live"]; + + for msg in historical_msgs.clone() { + contract + .setValue(msg.to_string()) + .send() + .await? + .watch() + .await?; + } + + EvmEventReader::attach( + &bus, + &arc_provider, + test_event_extractor, + &contract.address().to_string(), + ) + .await?; + + for msg in live_events.clone() { + contract + .setValue(msg.to_string()) + .send() + .await? + .watch() + .await?; + } + + sleep(Duration::from_millis(1)).await; + + let expected: Vec<_> = historical_msgs.into_iter().chain(live_events).collect(); + + let history = bus.send(GetHistory).await?; + + assert_eq!(history.len(), 8); + + let msgs: Vec<_> = history + .into_iter() + .filter_map(|evt| match evt { + EnclaveEvent::TestEvent { data, .. } => Some(data.msg), + _ => None, + }) + .collect(); + + assert_eq!(msgs, expected); + + Ok(()) +} From 08b0b7663ec624070c8598d7426a24351ebf9473 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 3 Nov 2024 18:42:22 +0700 Subject: [PATCH 02/31] Tidy up use statements --- packages/ciphernode/evm/tests/evm_reader.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index 1e2d4a66..ead06de0 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -2,8 +2,7 @@ use actix::Actor; use alloy::{ node_bindings::Anvil, primitives::{FixedBytes, LogData}, - providers::{ProviderBuilder, RootProvider, WsConnect}, - pubsub::PubSubFrontend, + providers::{ProviderBuilder, WsConnect}, sol, sol_types::SolEvent, }; From 2e346bd87df41b0a866ab7fef33c8ed554faa3cb Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 3 Nov 2024 20:14:20 +0700 Subject: [PATCH 03/31] EVM Contracts listen for both live and historical events --- packages/ciphernode/core/src/events.rs | 1 + .../evm/src/ciphernode_registry_sol.rs | 2 +- .../ciphernode/evm/src/enclave_sol_reader.rs | 2 +- packages/ciphernode/evm/src/event_reader.rs | 124 ++++++++++++++++-- packages/ciphernode/evm/src/helpers.rs | 49 ------- packages/ciphernode/evm/tests/evm_reader.rs | 12 +- .../evm/tests/fixtures/emit_logs.sol | 7 +- 7 files changed, 127 insertions(+), 70 deletions(-) diff --git a/packages/ciphernode/core/src/events.rs b/packages/ciphernode/core/src/events.rs index 96093ad3..b505e3aa 100644 --- a/packages/ciphernode/core/src/events.rs +++ b/packages/ciphernode/core/src/events.rs @@ -552,6 +552,7 @@ impl Display for Shutdown { #[rtype(result = "()")] pub struct TestEvent { pub msg: String, + pub entropy: u64 } #[cfg(test)] diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index 49193e41..b46b92ec 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -103,7 +103,7 @@ impl CiphernodeRegistrySolReader { provider: &WithChainId, contract_address: &str, ) -> Result>> { - let addr = EvmEventReader::attach(bus, provider, extractor, contract_address).await?; + let addr = EvmEventReader::attach(bus, provider, extractor, contract_address, None).await?; Ok(addr) } } diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index 2dd8477c..26035763 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -87,7 +87,7 @@ impl EnclaveSolReader { provider: &WithChainId, contract_address: &str, ) -> Result>> { - let addr = EvmEventReader::attach(bus, provider, extractor, contract_address).await?; + let addr = EvmEventReader::attach(bus, provider, extractor, contract_address,None).await?; Ok(addr) } } diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index 93264c72..1dabaed9 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -1,16 +1,18 @@ -use std::marker::PhantomData; - -use crate::helpers::{self, WithChainId}; +use crate::helpers::WithChainId; use actix::prelude::*; use actix::{Addr, Recipient}; +use alloy::eips::BlockNumberOrTag; +use alloy::primitives::Address; use alloy::primitives::{LogData, B256}; use alloy::providers::Provider; +use alloy::rpc::types::Filter; use alloy::transports::{BoxTransport, Transport}; -use alloy::{eips::BlockNumberOrTag, primitives::Address, rpc::types::Filter}; use anyhow::{anyhow, Result}; use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus, Subscribe}; +use futures_util::stream::StreamExt; +use tokio::select; use tokio::sync::oneshot; -use tracing::info; +use tracing::{info, trace, warn}; pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; @@ -20,12 +22,21 @@ where P: Provider + Clone + 'static, T: Transport + Clone + Unpin, { + /// The alloy provider provider: Option>, + /// The contract address contract_address: Address, + /// The EnclaveEvent Recipient to send events to bus: Recipient, + /// The Extractor function to determine which events to extract and convert to EnclaveEvents extractor: ExtractorFn, + /// A shutdown receiver to listen to for shutdown signals sent to the loop this is only used + /// internally. You should send the Shutdown signal to the reader directly or via the EventBus shutdown_rx: Option>, + /// The sender for the shutdown signal this is only used internally shutdown_tx: Option>, + /// The deployment block of the contract + deployment_block: Option, } impl EvmEventReader @@ -38,6 +49,7 @@ where provider: &WithChainId, extractor: ExtractorFn, contract_address: &Address, + deployment_block: Option, ) -> Result { let (shutdown_tx, shutdown_rx) = oneshot::channel(); Ok(Self { @@ -47,6 +59,7 @@ where bus: bus.clone().into(), shutdown_rx: Some(shutdown_rx), shutdown_tx: Some(shutdown_tx), + deployment_block, }) } @@ -55,9 +68,16 @@ where provider: &WithChainId, extractor: ExtractorFn, contract_address: &str, + deployment_block: Option, ) -> Result> { - let addr = - EvmEventReader::new(bus, provider, extractor, &contract_address.parse()?)?.start(); + let addr = EvmEventReader::new( + bus, + provider, + extractor, + &contract_address.parse()?, + deployment_block, + )? + .start(); bus.send(Subscribe::new("Shutdown", addr.clone().into())) .await?; @@ -79,22 +99,102 @@ where tracing::error!("Could not start event reader as provider has already been used."); return; }; - let filter = Filter::new() - .address(self.contract_address) - .from_block(BlockNumberOrTag::Latest); + let extractor = self.extractor; let Some(shutdown) = self.shutdown_rx.take() else { bus.err(EnclaveErrorType::Evm, anyhow!("shutdown already called")); return; }; + let contract_address = self.contract_address; + let deployment_block = self.deployment_block; + ctx.spawn( - async move { helpers::stream_from_evm(provider, filter, bus, extractor, shutdown).await } - .into_actor(self), + async move { + stream_from_evm( + provider, + &contract_address, + bus, + extractor, + shutdown, + deployment_block, + ) + .await + } + .into_actor(self), ); } } +async fn stream_from_evm, T: Transport + Clone>( + provider: WithChainId, + contract_address: &Address, + bus: Recipient, + extractor: fn(&LogData, Option<&B256>, u64) -> Option, + mut shutdown: oneshot::Receiver<()>, + deployment_block: Option, +) { + let chain_id = provider.get_chain_id(); + let provider = provider.get_provider(); + let historical_filter = Filter::new() + .address(contract_address.clone()) + .from_block(deployment_block.unwrap_or(0)); + let current_filter = Filter::new() + .address(*contract_address) + .from_block(BlockNumberOrTag::Latest); + + // Historical events + match provider.clone().get_logs(&historical_filter).await { + Ok(historical_logs) => { + info!("Fetched {} historical events", historical_logs.len()); + for log in historical_logs { + if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { + trace!("Processing historical log"); + bus.do_send(event); + } + } + } + Err(e) => { + warn!("Failed to fetch historical events: {}", e); + bus.err(EnclaveErrorType::Evm, anyhow!(e)); + return; + } + } + + match provider.subscribe_logs(¤t_filter).await { + Ok(subscription) => { + let mut stream = subscription.into_stream(); + loop { + select! { + maybe_log = stream.next() => { + match maybe_log { + Some(log) => { + trace!("Received log from EVM"); + let Some(event) = extractor(log.data(), log.topic0(), chain_id) + else { + trace!("Failed to extract log from EVM"); + continue; + }; + info!("Extracted log from evm sending now."); + bus.do_send(event); + } + None => break, // Stream ended + } + } + _ = &mut shutdown => { + info!("Received shutdown signal, stopping EVM stream"); + break; + } + } + } + } + Err(e) => { + bus.err(EnclaveErrorType::Evm, anyhow!("{}", e)); + } + }; + info!("Exiting stream loop"); +} + impl Handler for EvmEventReader where P: Provider + Clone + 'static, diff --git a/packages/ciphernode/evm/src/helpers.rs b/packages/ciphernode/evm/src/helpers.rs index ecaf7f73..116f3c2c 100644 --- a/packages/ciphernode/evm/src/helpers.rs +++ b/packages/ciphernode/evm/src/helpers.rs @@ -1,7 +1,5 @@ -use actix::Recipient; use alloy::{ network::{Ethereum, EthereumWallet}, - primitives::{LogData, B256}, providers::{ fillers::{ BlobGasFiller, ChainIdFiller, FillProvider, GasFiller, JoinFill, NonceFiller, @@ -9,62 +7,15 @@ use alloy::{ }, Identity, Provider, ProviderBuilder, RootProvider, }, - rpc::types::Filter, signers::local::PrivateKeySigner, transports::{BoxTransport, Transport}, }; -use anyhow::anyhow; use anyhow::{bail, Context, Result}; use cipher::Cipher; use data::Repository; -use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent}; -use futures_util::stream::StreamExt; use std::{env, marker::PhantomData, sync::Arc}; -use tokio::{select, sync::oneshot}; -use tracing::{info, trace}; use zeroize::Zeroizing; -pub async fn stream_from_evm, T: Transport + Clone>( - provider: WithChainId, - filter: Filter, - bus: Recipient, - extractor: fn(&LogData, Option<&B256>, u64) -> Option, - mut shutdown: oneshot::Receiver<()>, -) { - match provider.get_provider().subscribe_logs(&filter).await { - Ok(subscription) => { - let mut stream = subscription.into_stream(); - loop { - select! { - maybe_log = stream.next() => { - match maybe_log { - Some(log) => { - trace!("Received log from EVM"); - let Some(event) = extractor(log.data(), log.topic0(), provider.get_chain_id()) - else { - trace!("Failed to extract log from EVM"); - continue; - }; - info!("Extracted log from evm sending now."); - bus.do_send(event); - } - None => break, // Stream ended - } - } - _ = &mut shutdown => { - info!("Received shutdown signal, stopping EVM stream"); - break; - } - } - } - } - Err(e) => { - bus.err(EnclaveErrorType::Evm, anyhow!("{}", e)); - } - }; - info!("Exiting stream loop"); -} - /// We need to cache the chainId so we can easily use it in a non-async situation /// This wrapper just stores the chain_id with the Provider #[derive(Clone)] diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index ead06de0..e531f82e 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -9,7 +9,7 @@ use alloy::{ use anyhow::Result; use enclave_core::{EnclaveEvent, EventBus, GetHistory, TestEvent}; use evm::{helpers::WithChainId, EvmEventReader}; -use std::{sync::Arc, time::Duration}; +use std::time::Duration; use tokio::time::sleep; sol!( @@ -29,7 +29,8 @@ fn test_event_extractor( return None; }; Some(EnclaveEvent::from(TestEvent { - msg: event.newValue, + msg: event.value, + entropy: event.count.try_into().unwrap(), // This prevents de-duplication in tests })) } _ => None, @@ -37,7 +38,7 @@ fn test_event_extractor( } #[actix::test] -async fn test_evm_reader() -> Result<()> { +async fn evm_reader() -> Result<()> { // Create a WS provider // NOTE: Anvil must be available on $PATH let anvil = Anvil::new().block_time(1).try_spawn()?; @@ -52,6 +53,7 @@ async fn test_evm_reader() -> Result<()> { &arc_provider, test_event_extractor, &contract.address().to_string(), + None, ) .await?; @@ -89,7 +91,7 @@ async fn test_evm_reader() -> Result<()> { } #[actix::test] -async fn test_ensure_historical_events() -> Result<()> { +async fn ensure_historical_events() -> Result<()> { // Create a WS provider // NOTE: Anvil must be available on $PATH let anvil = Anvil::new().block_time(1).try_spawn()?; @@ -116,6 +118,7 @@ async fn test_ensure_historical_events() -> Result<()> { &arc_provider, test_event_extractor, &contract.address().to_string(), + None, ) .await?; @@ -133,7 +136,6 @@ async fn test_ensure_historical_events() -> Result<()> { let expected: Vec<_> = historical_msgs.into_iter().chain(live_events).collect(); let history = bus.send(GetHistory).await?; - assert_eq!(history.len(), 8); let msgs: Vec<_> = history diff --git a/packages/ciphernode/evm/tests/fixtures/emit_logs.sol b/packages/ciphernode/evm/tests/fixtures/emit_logs.sol index e9bad33c..1c57518e 100644 --- a/packages/ciphernode/evm/tests/fixtures/emit_logs.sol +++ b/packages/ciphernode/evm/tests/fixtures/emit_logs.sol @@ -2,10 +2,12 @@ pragma solidity >=0.4.24; contract EmitLogs { - event ValueChanged(address indexed author, string oldValue, string newValue); + event ValueChanged(address indexed author, uint256 count, string value); string _value; + uint256 count = 0; + constructor() { _value = ""; } @@ -15,7 +17,8 @@ contract EmitLogs { } function setValue(string memory value) public { - emit ValueChanged(msg.sender, _value, value); + count++; + emit ValueChanged(msg.sender, count, value); _value = value; } } From 69c1fa106928b7f4371b2cbcc6a25b5cce8a9c36 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 3 Nov 2024 20:17:05 +0700 Subject: [PATCH 04/31] Format --- packages/ciphernode/evm/src/enclave_sol_reader.rs | 2 +- packages/ciphernode/evm/src/event_reader.rs | 2 +- packages/ciphernode/evm/tests/evm_reader.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index 26035763..fde6d24f 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -87,7 +87,7 @@ impl EnclaveSolReader { provider: &WithChainId, contract_address: &str, ) -> Result>> { - let addr = EvmEventReader::attach(bus, provider, extractor, contract_address,None).await?; + let addr = EvmEventReader::attach(bus, provider, extractor, contract_address, None).await?; Ok(addr) } } diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index 1dabaed9..972b6a64 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -33,7 +33,7 @@ where /// A shutdown receiver to listen to for shutdown signals sent to the loop this is only used /// internally. You should send the Shutdown signal to the reader directly or via the EventBus shutdown_rx: Option>, - /// The sender for the shutdown signal this is only used internally + /// The sender for the shutdown signal this is only used internally shutdown_tx: Option>, /// The deployment block of the contract deployment_block: Option, diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index e531f82e..987f9aa9 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -30,7 +30,7 @@ fn test_event_extractor( }; Some(EnclaveEvent::from(TestEvent { msg: event.value, - entropy: event.count.try_into().unwrap(), // This prevents de-duplication in tests + entropy: event.count.try_into().unwrap(), // This prevents de-duplication in tests })) } _ => None, From 6b65f037e374efb63777ed4929cebfd8bc4481d4 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 3 Nov 2024 20:20:59 +0700 Subject: [PATCH 05/31] Format --- packages/ciphernode/core/src/events.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ciphernode/core/src/events.rs b/packages/ciphernode/core/src/events.rs index b505e3aa..07f633c4 100644 --- a/packages/ciphernode/core/src/events.rs +++ b/packages/ciphernode/core/src/events.rs @@ -552,7 +552,7 @@ impl Display for Shutdown { #[rtype(result = "()")] pub struct TestEvent { pub msg: String, - pub entropy: u64 + pub entropy: u64, } #[cfg(test)] From 37de0f82e24688a3c7e924baea4da82e67756bef Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 4 Nov 2024 13:48:11 +0700 Subject: [PATCH 06/31] Cache processed event_ids --- packages/ciphernode/Cargo.lock | 3 + .../aggregator/src/plaintext_aggregator.rs | 4 +- .../aggregator/src/publickey_aggregator.rs | 4 +- packages/ciphernode/data/src/lib.rs | 1 + packages/ciphernode/data/src/snapshot.rs | 2 +- .../ciphernode/enclave_node/src/aggregator.rs | 10 +- .../ciphernode/enclave_node/src/ciphernode.rs | 18 ++- packages/ciphernode/evm/Cargo.toml | 2 + .../evm/src/ciphernode_registry_sol.rs | 138 +++++++++++++++++- packages/ciphernode/evm/src/enclave_sol.rs | 6 +- .../ciphernode/evm/src/enclave_sol_reader.rs | 137 ++++++++++++++++- packages/ciphernode/evm/src/event_reader.rs | 17 +-- packages/ciphernode/evm/src/lib.rs | 6 +- packages/ciphernode/evm/tests/evm_reader.rs | 4 +- packages/ciphernode/keyshare/src/keyshare.rs | 4 +- packages/ciphernode/router/Cargo.toml | 1 + packages/ciphernode/router/src/context.rs | 6 +- .../router/src/e3_request_router.rs | 4 +- .../ciphernode/router/src/repositories.rs | 14 ++ .../ciphernode/sortition/src/sortition.rs | 4 +- 20 files changed, 335 insertions(+), 50 deletions(-) diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index e94bcb33..37f2bbd5 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -2352,10 +2352,12 @@ dependencies = [ "alloy", "alloy-primitives 0.6.4", "anyhow", + "async-trait", "cipher 0.1.0", "data", "enclave-core", "futures-util", + "serde", "sortition", "tokio", "tracing", @@ -5335,6 +5337,7 @@ dependencies = [ "cipher 0.1.0", "data", "enclave-core", + "evm", "fhe 0.1.0", "keyshare", "serde", diff --git a/packages/ciphernode/aggregator/src/plaintext_aggregator.rs b/packages/ciphernode/aggregator/src/plaintext_aggregator.rs index c65e8e55..fc5bdf60 100644 --- a/packages/ciphernode/aggregator/src/plaintext_aggregator.rs +++ b/packages/ciphernode/aggregator/src/plaintext_aggregator.rs @@ -236,7 +236,7 @@ impl FromSnapshotWithParams for PlaintextAggregator { } impl Checkpoint for PlaintextAggregator { - fn repository(&self) -> Repository { - self.store.clone() + fn repository(&self) -> &Repository { + &self.store } } diff --git a/packages/ciphernode/aggregator/src/publickey_aggregator.rs b/packages/ciphernode/aggregator/src/publickey_aggregator.rs index 8f19c21f..e5e3b640 100644 --- a/packages/ciphernode/aggregator/src/publickey_aggregator.rs +++ b/packages/ciphernode/aggregator/src/publickey_aggregator.rs @@ -261,7 +261,7 @@ impl FromSnapshotWithParams for PublicKeyAggregator { } impl Checkpoint for PublicKeyAggregator { - fn repository(&self) -> Repository { - self.store.clone() + fn repository(&self) -> &Repository { + &self.store } } diff --git a/packages/ciphernode/data/src/lib.rs b/packages/ciphernode/data/src/lib.rs index b3f1ea93..625da0a0 100644 --- a/packages/ciphernode/data/src/lib.rs +++ b/packages/ciphernode/data/src/lib.rs @@ -11,3 +11,4 @@ pub use into_key::IntoKey; pub use repository::*; pub use sled_store::*; pub use snapshot::*; + diff --git a/packages/ciphernode/data/src/snapshot.rs b/packages/ciphernode/data/src/snapshot.rs index cfd445a2..a0203f45 100644 --- a/packages/ciphernode/data/src/snapshot.rs +++ b/packages/ciphernode/data/src/snapshot.rs @@ -20,7 +20,7 @@ where /// This trait enables the self type to checkpoint its state pub trait Checkpoint: Snapshot { /// Declare the DataStore instance available on the object - fn repository(&self) -> Repository; + fn repository(&self) -> &Repository; /// Write the current snapshot to the `Repository` provided by `repository()` fn checkpoint(&self) { diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 18d47c24..e4a5d216 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -54,11 +54,17 @@ pub async fn setup_aggregator( &read_provider, &write_provider, &chain.contracts.enclave, + &repositories.enclave_sol_reader(read_provider.get_chain_id()), ) .await?; RegistryFilterSol::attach(&bus, &write_provider, &chain.contracts.filter_registry).await?; - CiphernodeRegistrySol::attach(&bus, &read_provider, &chain.contracts.ciphernode_registry) - .await?; + CiphernodeRegistrySol::attach( + &bus, + &read_provider, + &chain.contracts.ciphernode_registry, + &repositories.ciphernode_registry_reader(read_provider.get_chain_id()), + ) + .await?; } E3RequestRouter::builder(&bus, store) diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 2840429e..4ce4b91b 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -3,7 +3,6 @@ use alloy::primitives::Address; use anyhow::Result; use cipher::Cipher; use config::AppConfig; -use data::{DataStore, InMemStore, SledStore}; use enclave_core::EventBus; use evm::{ helpers::{create_readonly_provider, ensure_ws_rpc}, @@ -46,9 +45,20 @@ pub async fn setup_ciphernode( let rpc_url = &chain.rpc_url; let read_provider = create_readonly_provider(&ensure_ws_rpc(rpc_url)).await?; - EnclaveSolReader::attach(&bus, &read_provider, &chain.contracts.enclave).await?; - CiphernodeRegistrySol::attach(&bus, &read_provider, &chain.contracts.ciphernode_registry) - .await?; + EnclaveSolReader::attach( + &bus, + &read_provider, + &chain.contracts.enclave, + &repositories.enclave_sol_reader(read_provider.get_chain_id()), + ) + .await?; + CiphernodeRegistrySol::attach( + &bus, + &read_provider, + &chain.contracts.ciphernode_registry, + &repositories.ciphernode_registry_reader(read_provider.get_chain_id()), + ) + .await?; } E3RequestRouter::builder(&bus, store.clone()) diff --git a/packages/ciphernode/evm/Cargo.toml b/packages/ciphernode/evm/Cargo.toml index 7c3a1cbf..a5fc67ac 100644 --- a/packages/ciphernode/evm/Cargo.toml +++ b/packages/ciphernode/evm/Cargo.toml @@ -8,6 +8,7 @@ actix = { workspace = true } alloy = { workspace = true } alloy-primitives = { workspace = true } anyhow = { workspace = true } +async-trait = { workspace = true } enclave-core = { path = "../core" } data = { path = "../data" } futures-util = { workspace = true } @@ -15,4 +16,5 @@ sortition = { path = "../sortition" } cipher = { path = "../cipher" } tokio = { workspace = true } tracing = { workspace = true } +serde = { workspace = true } zeroize = { workspace = true } diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index b46b92ec..7c077e2f 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -1,16 +1,21 @@ +use std::collections::HashSet; + use crate::{ + event_reader::EventReader, helpers::{ReadonlyProvider, WithChainId}, - EvmEventReader, + EvmEventReader, }; -use actix::Addr; +use actix::{Actor, Addr, Handler}; use alloy::{ primitives::{LogData, B256}, sol, sol_types::SolEvent, }; use anyhow::Result; -use enclave_core::{EnclaveEvent, EventBus}; -use tracing::{error, trace}; +use async_trait::async_trait; +use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; +use enclave_core::{EnclaveEvent, EventBus, EventId, Subscribe}; +use tracing::{error, info, trace}; sol!( #[sol(rpc)] @@ -95,27 +100,144 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, _: u64) -> Option, + reader: Addr, + state: CiphernodeRegistryReaderState, + repository: Repository, +} + +pub struct CiphernodeRegistryReaderParams { + bus: Addr, + reader: Addr, + repository: Repository, +} + +#[derive(Clone, serde::Serialize, serde::Deserialize)] +pub struct CiphernodeRegistryReaderState { + ids: HashSet, +} impl CiphernodeRegistrySolReader { + pub fn new(params: CiphernodeRegistryReaderParams) -> Self { + Self { + bus: params.bus, + reader: params.reader, + state: CiphernodeRegistryReaderState { + ids: HashSet::new(), + }, + repository: params.repository, + } + } + + pub async fn load(params: CiphernodeRegistryReaderParams) -> Result> { + let addr = if let Some(snapshot) = params.repository.read().await? { + Self::from_snapshot(params, snapshot).await? + } else { + Self::new(params) + } + .start(); + Ok(addr) + } + pub async fn attach( bus: &Addr, provider: &WithChainId, contract_address: &str, - ) -> Result>> { - let addr = EvmEventReader::attach(bus, provider, extractor, contract_address, None).await?; + repository: &Repository, + ) -> Result> { + let params = CiphernodeRegistryReaderParams { + bus: bus.clone(), + reader: EvmEventReader::attach( + &bus.clone().into(), + provider, + extractor, + contract_address, + None, + ) + .await?, + repository: repository.clone(), + }; + + let addr = Self::load(params).await?; + + bus.send(Subscribe::new("Shutdown", addr.clone().into())) + .await?; + + info!(address=%contract_address, "EnclaveSolReader is listening to address"); + Ok(addr) } } +impl Actor for CiphernodeRegistrySolReader { + type Context = actix::Context; +} + +impl Handler for CiphernodeRegistrySolReader { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + // If this is a shutdown signal it will be coming from the event bus forward it to the reader + if let EnclaveEvent::Shutdown { .. } = msg { + self.reader.do_send(msg); + return; + } + + // Other enclave events will be coming from the reader - check the event id cache forward to the event bus + let event_id = msg.get_id(); + if self.state.ids.contains(&event_id) { + trace!( + "Event id {} has already been seen and was not forwarded to the bus", + &event_id + ); + return; + } + + // Forward everything else to the event bus + self.bus.do_send(msg); + + // Save processed ids + self.state.ids.insert(event_id); + self.checkpoint(); + } +} + +impl Snapshot for CiphernodeRegistrySolReader { + type Snapshot = CiphernodeRegistryReaderState; + fn snapshot(&self) -> Self::Snapshot { + self.state.clone() + } +} + +impl Checkpoint for CiphernodeRegistrySolReader { + fn repository(&self) -> &Repository { + &self.repository + } +} + +#[async_trait] +impl FromSnapshotWithParams for CiphernodeRegistrySolReader { + type Params = CiphernodeRegistryReaderParams; + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { + Ok(Self { + bus: params.bus, + reader: params.reader, + state: snapshot, + repository: params.repository, + }) + } +} +/// Eventual wrapper for both a reader and a writer pub struct CiphernodeRegistrySol; impl CiphernodeRegistrySol { pub async fn attach( bus: &Addr, provider: &WithChainId, contract_address: &str, + repository: &Repository ) -> Result<()> { - CiphernodeRegistrySolReader::attach(bus, provider, contract_address).await?; + CiphernodeRegistrySolReader::attach(bus, provider, contract_address,repository).await?; + // TODO: Writer if needed Ok(()) } } diff --git a/packages/ciphernode/evm/src/enclave_sol.rs b/packages/ciphernode/evm/src/enclave_sol.rs index bd0c6ef7..c1acac3b 100644 --- a/packages/ciphernode/evm/src/enclave_sol.rs +++ b/packages/ciphernode/evm/src/enclave_sol.rs @@ -1,10 +1,11 @@ use crate::{ enclave_sol_reader::EnclaveSolReader, enclave_sol_writer::EnclaveSolWriter, - helpers::{ReadonlyProvider, SignerProvider, WithChainId}, + helpers::{ReadonlyProvider, SignerProvider, WithChainId}, EnclaveSolReaderState, }; use actix::Addr; use anyhow::Result; +use data::Repository; use enclave_core::EventBus; pub struct EnclaveSol; @@ -14,8 +15,9 @@ impl EnclaveSol { read_provider: &WithChainId, write_provider: &WithChainId, contract_address: &str, + repository: &Repository ) -> Result<()> { - EnclaveSolReader::attach(bus, read_provider, contract_address).await?; + EnclaveSolReader::attach(bus, read_provider, contract_address, repository).await?; EnclaveSolWriter::attach(bus, write_provider, contract_address).await?; Ok(()) } diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index fde6d24f..62fcf14b 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -1,12 +1,17 @@ +use std::collections::HashSet; + +use crate::event_reader::EventReader; use crate::helpers::{ReadonlyProvider, WithChainId}; use crate::EvmEventReader; -use actix::Addr; +use actix::{Actor, Addr, Handler}; use alloy::primitives::{LogData, B256}; use alloy::transports::BoxTransport; use alloy::{sol, sol_types::SolEvent}; use anyhow::Result; -use enclave_core::{EnclaveEvent, EventBus}; -use tracing::{error, trace}; +use async_trait::async_trait; +use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; +use enclave_core::{EnclaveEvent, EventBus, EventId, Subscribe}; +use tracing::{error, info, trace}; sol!( #[sol(rpc)] @@ -78,16 +83,136 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option< } } +#[derive(serde::Serialize, serde::Deserialize, Clone)] +pub struct EnclaveSolReaderState { + pub ids: HashSet, +} + +impl Default for EnclaveSolReaderState { + fn default() -> Self { + Self { + ids: HashSet::new(), + } + } +} + /// Connects to Enclave.sol converting EVM events to EnclaveEvents -pub struct EnclaveSolReader; +pub struct EnclaveSolReader { + bus: Addr, + reader: Addr, + state: EnclaveSolReaderState, + repository: Repository, +} + +pub struct EnclaveSolReaderParams { + bus: Addr, + reader: Addr, + repository: Repository, +} impl EnclaveSolReader { + pub fn new(params: EnclaveSolReaderParams) -> Self { + Self { + bus: params.bus, + reader: params.reader, + state: EnclaveSolReaderState::default(), + repository: params.repository, + } + } + + pub async fn load(params: EnclaveSolReaderParams) -> Result> { + let addr = if let Some(snapshot) = params.repository.read().await? { + Self::from_snapshot(params, snapshot).await? + } else { + Self::new(params) + } + .start(); + Ok(addr) + } + pub async fn attach( bus: &Addr, provider: &WithChainId, contract_address: &str, - ) -> Result>> { - let addr = EvmEventReader::attach(bus, provider, extractor, contract_address, None).await?; + repository: &Repository, + ) -> Result> { + let params = EnclaveSolReaderParams { + bus: bus.clone(), + reader: EvmEventReader::attach( + &bus.clone().into(), + provider, + extractor, + contract_address, + None, + ) + .await?, + repository: repository.clone(), + }; + + let addr = Self::load(params).await?; + bus.send(Subscribe::new("Shutdown", addr.clone().into())) + .await?; + + info!(address=%contract_address, "EnclaveSolReader is listening to address"); + Ok(addr) } } + +impl Actor for EnclaveSolReader { + type Context = actix::Context; +} + +impl Handler for EnclaveSolReader { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + // If this is a shutdown signal it will be coming from the event bus forward it to the reader + if let EnclaveEvent::Shutdown { .. } = msg { + self.reader.do_send(msg); + return; + } + + // Other enclave events will be coming from the reader - check the event id cache forward to the event bus + let event_id = msg.get_id(); + if self.state.ids.contains(&event_id) { + trace!( + "Event id {} has already been seen and was not forwarded to the bus", + &event_id + ); + return; + } + + // Forward everything else to the event bus + self.bus.do_send(msg); + + // Save processed ids + self.state.ids.insert(event_id); + self.checkpoint(); + } +} + +impl Snapshot for EnclaveSolReader { + type Snapshot = EnclaveSolReaderState; + fn snapshot(&self) -> Self::Snapshot { + self.state.clone() + } +} + +impl Checkpoint for EnclaveSolReader { + fn repository(&self) -> &Repository { + &self.repository + } +} + +#[async_trait] +impl FromSnapshotWithParams for EnclaveSolReader { + type Params = EnclaveSolReaderParams; + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { + Ok(Self { + bus: params.bus, + reader: params.reader, + state: snapshot, + repository: params.repository, + }) + } +} diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index 972b6a64..4d892955 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -1,4 +1,4 @@ -use crate::helpers::WithChainId; +use crate::helpers::{ReadonlyProvider, WithChainId}; use actix::prelude::*; use actix::{Addr, Recipient}; use alloy::eips::BlockNumberOrTag; @@ -8,7 +8,7 @@ use alloy::providers::Provider; use alloy::rpc::types::Filter; use alloy::transports::{BoxTransport, Transport}; use anyhow::{anyhow, Result}; -use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus, Subscribe}; +use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent}; use futures_util::stream::StreamExt; use tokio::select; use tokio::sync::oneshot; @@ -16,6 +16,8 @@ use tracing::{info, trace, warn}; pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; +pub type EventReader = EvmEventReader; + /// Connects to Enclave.sol converting EVM events to EnclaveEvents pub struct EvmEventReader where @@ -45,7 +47,7 @@ where T: Transport + Clone + Unpin, { pub fn new( - bus: &Addr, + bus: &Recipient, provider: &WithChainId, extractor: ExtractorFn, contract_address: &Address, @@ -56,7 +58,7 @@ where contract_address: contract_address.clone(), provider: Some(provider.clone()), extractor, - bus: bus.clone().into(), + bus: bus.clone(), shutdown_rx: Some(shutdown_rx), shutdown_tx: Some(shutdown_tx), deployment_block, @@ -64,7 +66,7 @@ where } pub async fn attach( - bus: &Addr, + bus: &Recipient, provider: &WithChainId, extractor: ExtractorFn, contract_address: &str, @@ -78,11 +80,6 @@ where deployment_block, )? .start(); - - bus.send(Subscribe::new("Shutdown", addr.clone().into())) - .await?; - - info!(address=%contract_address, "Evm is listening to address"); Ok(addr) } } diff --git a/packages/ciphernode/evm/src/lib.rs b/packages/ciphernode/evm/src/lib.rs index 5d01008a..05fd73be 100644 --- a/packages/ciphernode/evm/src/lib.rs +++ b/packages/ciphernode/evm/src/lib.rs @@ -6,9 +6,11 @@ mod event_reader; pub mod helpers; mod registry_filter_sol; -pub use ciphernode_registry_sol::{CiphernodeRegistrySol, CiphernodeRegistrySolReader}; +pub use ciphernode_registry_sol::{ + CiphernodeRegistryReaderParams,CiphernodeRegistrySol, CiphernodeRegistryReaderState, CiphernodeRegistrySolReader, +}; pub use enclave_sol::EnclaveSol; -pub use enclave_sol_reader::EnclaveSolReader; +pub use enclave_sol_reader::{EnclaveSolReader, EnclaveSolReaderParams, EnclaveSolReaderState}; pub use enclave_sol_writer::EnclaveSolWriter; pub use event_reader::{EvmEventReader, ExtractorFn}; pub use registry_filter_sol::{RegistryFilterSol, RegistryFilterSolWriter}; diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index 987f9aa9..15daeddd 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -49,7 +49,7 @@ async fn evm_reader() -> Result<()> { let bus = EventBus::new(true).start(); EvmEventReader::attach( - &bus, + &bus.clone().into(), &arc_provider, test_event_extractor, &contract.address().to_string(), @@ -114,7 +114,7 @@ async fn ensure_historical_events() -> Result<()> { } EvmEventReader::attach( - &bus, + &bus.clone().into(), &arc_provider, test_event_extractor, &contract.address().to_string(), diff --git a/packages/ciphernode/keyshare/src/keyshare.rs b/packages/ciphernode/keyshare/src/keyshare.rs index 20c6cf67..a2e93c0e 100644 --- a/packages/ciphernode/keyshare/src/keyshare.rs +++ b/packages/ciphernode/keyshare/src/keyshare.rs @@ -85,8 +85,8 @@ impl Snapshot for Keyshare { } impl Checkpoint for Keyshare { - fn repository(&self) -> Repository { - self.store.clone() + fn repository(&self) -> &Repository { + &self.store } } diff --git a/packages/ciphernode/router/Cargo.toml b/packages/ciphernode/router/Cargo.toml index ce85d444..cb626737 100644 --- a/packages/ciphernode/router/Cargo.toml +++ b/packages/ciphernode/router/Cargo.toml @@ -11,6 +11,7 @@ fhe = { path = "../fhe" } data = { path = "../data" } keyshare = { path = "../keyshare" } aggregator = { path = "../aggregator" } +evm = { path = "../evm" } anyhow = { workspace = true } serde = { workspace = true } cipher = { path = "../cipher" } diff --git a/packages/ciphernode/router/src/context.rs b/packages/ciphernode/router/src/context.rs index c3643132..506e106c 100644 --- a/packages/ciphernode/router/src/context.rs +++ b/packages/ciphernode/router/src/context.rs @@ -143,7 +143,7 @@ impl E3RequestContext { impl RepositoriesFactory for E3RequestContext { fn repositories(&self) -> Repositories { - self.repository().into() + self.repository().clone().into() } } @@ -186,7 +186,7 @@ impl FromSnapshotWithParams for E3RequestContext { } impl Checkpoint for E3RequestContext { - fn repository(&self) -> Repository { - self.store.clone() + fn repository(&self) -> &Repository { + &self.store } } diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index 943a3691..0890d4e0 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -190,8 +190,8 @@ impl Snapshot for E3RequestRouter { } impl Checkpoint for E3RequestRouter { - fn repository(&self) -> Repository { - self.store.clone() + fn repository(&self) -> &Repository { + &self.store } } diff --git a/packages/ciphernode/router/src/repositories.rs b/packages/ciphernode/router/src/repositories.rs index 0d5f703a..06923f0c 100644 --- a/packages/ciphernode/router/src/repositories.rs +++ b/packages/ciphernode/router/src/repositories.rs @@ -2,6 +2,7 @@ use crate::{CommitteeMeta, E3RequestContextSnapshot, E3RequestRouterSnapshot}; use aggregator::{PlaintextAggregatorState, PublicKeyAggregatorState}; use data::{DataStore, Repository}; use enclave_core::E3id; +use evm::{CiphernodeRegistryReaderState, EnclaveSolReaderState}; use fhe::FheSnapshot; use keyshare::KeyshareState; use sortition::SortitionModule; @@ -72,6 +73,19 @@ impl Repositories { pub fn eth_private_key(&self) -> Repository> { Repository::new(self.store.scope(format!("//eth_private_key"))) } + + pub fn enclave_sol_reader(&self, chain_id: u64) -> Repository { + Repository::new( + self.store + .scope(format!("//evm_readers/enclave/{chain_id}")), + ) + } + pub fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository { + Repository::new( + self.store + .scope(format!("//evm_readers/ciphernode_registry/{chain_id}")), + ) + } } pub trait RepositoriesFactory { diff --git a/packages/ciphernode/sortition/src/sortition.rs b/packages/ciphernode/sortition/src/sortition.rs index 0f992c96..0f5b51bd 100644 --- a/packages/ciphernode/sortition/src/sortition.rs +++ b/packages/ciphernode/sortition/src/sortition.rs @@ -140,8 +140,8 @@ impl FromSnapshotWithParams for Sortition { } impl Checkpoint for Sortition { - fn repository(&self) -> Repository { - self.store.clone() + fn repository(&self) -> &Repository { + &self.store } } From f81b148fe44ed2bafbff5188d5c67d5353f84da0 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 4 Nov 2024 13:56:11 +0700 Subject: [PATCH 07/31] Formatting --- packages/ciphernode/evm/src/ciphernode_registry_sol.rs | 6 +++--- packages/ciphernode/evm/src/enclave_sol.rs | 5 +++-- packages/ciphernode/evm/src/lib.rs | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index 7c077e2f..607e0f1d 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -3,7 +3,7 @@ use std::collections::HashSet; use crate::{ event_reader::EventReader, helpers::{ReadonlyProvider, WithChainId}, - EvmEventReader, + EvmEventReader, }; use actix::{Actor, Addr, Handler}; use alloy::{ @@ -234,9 +234,9 @@ impl CiphernodeRegistrySol { bus: &Addr, provider: &WithChainId, contract_address: &str, - repository: &Repository + repository: &Repository, ) -> Result<()> { - CiphernodeRegistrySolReader::attach(bus, provider, contract_address,repository).await?; + CiphernodeRegistrySolReader::attach(bus, provider, contract_address, repository).await?; // TODO: Writer if needed Ok(()) } diff --git a/packages/ciphernode/evm/src/enclave_sol.rs b/packages/ciphernode/evm/src/enclave_sol.rs index c1acac3b..aadf4c03 100644 --- a/packages/ciphernode/evm/src/enclave_sol.rs +++ b/packages/ciphernode/evm/src/enclave_sol.rs @@ -1,7 +1,8 @@ use crate::{ enclave_sol_reader::EnclaveSolReader, enclave_sol_writer::EnclaveSolWriter, - helpers::{ReadonlyProvider, SignerProvider, WithChainId}, EnclaveSolReaderState, + helpers::{ReadonlyProvider, SignerProvider, WithChainId}, + EnclaveSolReaderState, }; use actix::Addr; use anyhow::Result; @@ -15,7 +16,7 @@ impl EnclaveSol { read_provider: &WithChainId, write_provider: &WithChainId, contract_address: &str, - repository: &Repository + repository: &Repository, ) -> Result<()> { EnclaveSolReader::attach(bus, read_provider, contract_address, repository).await?; EnclaveSolWriter::attach(bus, write_provider, contract_address).await?; diff --git a/packages/ciphernode/evm/src/lib.rs b/packages/ciphernode/evm/src/lib.rs index 05fd73be..ba330d5d 100644 --- a/packages/ciphernode/evm/src/lib.rs +++ b/packages/ciphernode/evm/src/lib.rs @@ -7,7 +7,8 @@ pub mod helpers; mod registry_filter_sol; pub use ciphernode_registry_sol::{ - CiphernodeRegistryReaderParams,CiphernodeRegistrySol, CiphernodeRegistryReaderState, CiphernodeRegistrySolReader, + CiphernodeRegistryReaderParams, CiphernodeRegistryReaderState, CiphernodeRegistrySol, + CiphernodeRegistrySolReader, }; pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::{EnclaveSolReader, EnclaveSolReaderParams, EnclaveSolReaderState}; From a1cfdab15b97d51e9ff05933fac6aa791529e6c9 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 4 Nov 2024 16:10:14 +0700 Subject: [PATCH 08/31] Formatting --- packages/ciphernode/data/src/lib.rs | 1 - packages/ciphernode/router/src/repositories.rs | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/ciphernode/data/src/lib.rs b/packages/ciphernode/data/src/lib.rs index 625da0a0..b3f1ea93 100644 --- a/packages/ciphernode/data/src/lib.rs +++ b/packages/ciphernode/data/src/lib.rs @@ -11,4 +11,3 @@ pub use into_key::IntoKey; pub use repository::*; pub use sled_store::*; pub use snapshot::*; - diff --git a/packages/ciphernode/router/src/repositories.rs b/packages/ciphernode/router/src/repositories.rs index 06923f0c..db32140c 100644 --- a/packages/ciphernode/router/src/repositories.rs +++ b/packages/ciphernode/router/src/repositories.rs @@ -80,7 +80,10 @@ impl Repositories { .scope(format!("//evm_readers/enclave/{chain_id}")), ) } - pub fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository { + pub fn ciphernode_registry_reader( + &self, + chain_id: u64, + ) -> Repository { Repository::new( self.store .scope(format!("//evm_readers/ciphernode_registry/{chain_id}")), From 5ce0a2954cc2cd3fee0e18ee47a7e2396e02e26a Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 4 Nov 2024 16:30:08 +0700 Subject: [PATCH 09/31] Store last_block --- packages/ciphernode/evm/src/event_reader.rs | 37 ++++++++++++++------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index 4d892955..9983b9c4 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::helpers::{ReadonlyProvider, WithChainId}; use actix::prelude::*; use actix::{Addr, Recipient}; @@ -11,7 +13,7 @@ use anyhow::{anyhow, Result}; use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent}; use futures_util::stream::StreamExt; use tokio::select; -use tokio::sync::oneshot; +use tokio::sync::{oneshot, Mutex}; use tracing::{info, trace, warn}; pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; @@ -37,8 +39,10 @@ where shutdown_rx: Option>, /// The sender for the shutdown signal this is only used internally shutdown_tx: Option>, - /// The deployment block of the contract - deployment_block: Option, + /// The block that processing should start from + start_block: Option, + /// Last block processed + last_block: Arc>>, } impl EvmEventReader @@ -51,7 +55,7 @@ where provider: &WithChainId, extractor: ExtractorFn, contract_address: &Address, - deployment_block: Option, + start_block: Option, ) -> Result { let (shutdown_tx, shutdown_rx) = oneshot::channel(); Ok(Self { @@ -61,7 +65,8 @@ where bus: bus.clone(), shutdown_rx: Some(shutdown_rx), shutdown_tx: Some(shutdown_tx), - deployment_block, + start_block, + last_block: Arc::new(Mutex::new(None)), }) } @@ -70,14 +75,14 @@ where provider: &WithChainId, extractor: ExtractorFn, contract_address: &str, - deployment_block: Option, + start_block: Option, ) -> Result> { let addr = EvmEventReader::new( bus, provider, extractor, &contract_address.parse()?, - deployment_block, + start_block, )? .start(); Ok(addr) @@ -104,8 +109,8 @@ where }; let contract_address = self.contract_address; - let deployment_block = self.deployment_block; - + let start_block = self.start_block; + let last_block = self.last_block.clone(); ctx.spawn( async move { stream_from_evm( @@ -114,7 +119,8 @@ where bus, extractor, shutdown, - deployment_block, + start_block, + last_block, ) .await } @@ -129,13 +135,14 @@ async fn stream_from_evm, T: Transport + Clone>( bus: Recipient, extractor: fn(&LogData, Option<&B256>, u64) -> Option, mut shutdown: oneshot::Receiver<()>, - deployment_block: Option, + start_block: Option, + last_block: Arc>>, ) { let chain_id = provider.get_chain_id(); let provider = provider.get_provider(); let historical_filter = Filter::new() .address(contract_address.clone()) - .from_block(deployment_block.unwrap_or(0)); + .from_block(start_block.unwrap_or(0)); let current_filter = Filter::new() .address(*contract_address) .from_block(BlockNumberOrTag::Latest); @@ -145,9 +152,12 @@ async fn stream_from_evm, T: Transport + Clone>( Ok(historical_logs) => { info!("Fetched {} historical events", historical_logs.len()); for log in historical_logs { + let block_number = log.block_number; if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { trace!("Processing historical log"); bus.do_send(event); + let mut guard = last_block.lock().await; + *guard = block_number; } } } @@ -166,6 +176,7 @@ async fn stream_from_evm, T: Transport + Clone>( maybe_log = stream.next() => { match maybe_log { Some(log) => { + let block_number = log.block_number; trace!("Received log from EVM"); let Some(event) = extractor(log.data(), log.topic0(), chain_id) else { @@ -174,6 +185,8 @@ async fn stream_from_evm, T: Transport + Clone>( }; info!("Extracted log from evm sending now."); bus.do_send(event); + let mut guard = last_block.lock().await; + *guard = block_number; } None => break, // Stream ended } From b9e8a07768355a3781f83d202f9f352605cd865a Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 4 Nov 2024 17:17:37 +0700 Subject: [PATCH 10/31] Report last block to parent --- .../evm/src/ciphernode_registry_sol.rs | 50 +++++++---------- .../ciphernode/evm/src/enclave_sol_reader.rs | 53 ++++++++---------- packages/ciphernode/evm/src/event_reader.rs | 56 ++++++++++++------- 3 files changed, 77 insertions(+), 82 deletions(-) diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index 607e0f1d..c3bd3b9c 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -1,7 +1,7 @@ use std::collections::HashSet; use crate::{ - event_reader::EventReader, + event_reader::{EnclaveEvmEvent, EventReader}, helpers::{ReadonlyProvider, WithChainId}, EvmEventReader, }; @@ -102,30 +102,26 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, _: u64) -> Option, - reader: Addr, state: CiphernodeRegistryReaderState, repository: Repository, } pub struct CiphernodeRegistryReaderParams { bus: Addr, - reader: Addr, repository: Repository, } -#[derive(Clone, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] pub struct CiphernodeRegistryReaderState { - ids: HashSet, + pub ids: HashSet, + pub last_block: Option, } impl CiphernodeRegistrySolReader { pub fn new(params: CiphernodeRegistryReaderParams) -> Self { Self { bus: params.bus, - reader: params.reader, - state: CiphernodeRegistryReaderState { - ids: HashSet::new(), - }, + state: CiphernodeRegistryReaderState::default(), repository: params.repository, } } @@ -148,21 +144,20 @@ impl CiphernodeRegistrySolReader { ) -> Result> { let params = CiphernodeRegistryReaderParams { bus: bus.clone(), - reader: EvmEventReader::attach( - &bus.clone().into(), - provider, - extractor, - contract_address, - None, - ) - .await?, repository: repository.clone(), }; let addr = Self::load(params).await?; - bus.send(Subscribe::new("Shutdown", addr.clone().into())) - .await?; + EvmEventReader::attach( + &addr.clone().into(), + provider, + extractor, + contract_address, + None, + &bus.clone().into(), + ) + .await?; info!(address=%contract_address, "EnclaveSolReader is listening to address"); @@ -174,17 +169,10 @@ impl Actor for CiphernodeRegistrySolReader { type Context = actix::Context; } -impl Handler for CiphernodeRegistrySolReader { +impl Handler for CiphernodeRegistrySolReader { type Result = (); - fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - // If this is a shutdown signal it will be coming from the event bus forward it to the reader - if let EnclaveEvent::Shutdown { .. } = msg { - self.reader.do_send(msg); - return; - } - - // Other enclave events will be coming from the reader - check the event id cache forward to the event bus - let event_id = msg.get_id(); + fn handle(&mut self, wrapped: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { + let event_id = wrapped.event.get_id(); if self.state.ids.contains(&event_id) { trace!( "Event id {} has already been seen and was not forwarded to the bus", @@ -194,10 +182,11 @@ impl Handler for CiphernodeRegistrySolReader { } // Forward everything else to the event bus - self.bus.do_send(msg); + self.bus.do_send(wrapped.event); // Save processed ids self.state.ids.insert(event_id); + self.state.last_block = wrapped.block; self.checkpoint(); } } @@ -221,7 +210,6 @@ impl FromSnapshotWithParams for CiphernodeRegistrySolReader { async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { Ok(Self { bus: params.bus, - reader: params.reader, state: snapshot, repository: params.repository, }) diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index 62fcf14b..63f5c491 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -1,6 +1,6 @@ use std::collections::HashSet; -use crate::event_reader::EventReader; +use crate::event_reader::{EnclaveEvmEvent, EventReader}; use crate::helpers::{ReadonlyProvider, WithChainId}; use crate::EvmEventReader; use actix::{Actor, Addr, Handler}; @@ -10,7 +10,7 @@ use alloy::{sol, sol_types::SolEvent}; use anyhow::Result; use async_trait::async_trait; use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; -use enclave_core::{EnclaveEvent, EventBus, EventId, Subscribe}; +use enclave_core::{EnclaveEvent, EventBus, EventId}; use tracing::{error, info, trace}; sol!( @@ -86,12 +86,14 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option< #[derive(serde::Serialize, serde::Deserialize, Clone)] pub struct EnclaveSolReaderState { pub ids: HashSet, + pub last_block: Option, } impl Default for EnclaveSolReaderState { fn default() -> Self { Self { ids: HashSet::new(), + last_block: None, } } } @@ -99,14 +101,12 @@ impl Default for EnclaveSolReaderState { /// Connects to Enclave.sol converting EVM events to EnclaveEvents pub struct EnclaveSolReader { bus: Addr, - reader: Addr, state: EnclaveSolReaderState, repository: Repository, } pub struct EnclaveSolReaderParams { bus: Addr, - reader: Addr, repository: Repository, } @@ -114,7 +114,6 @@ impl EnclaveSolReader { pub fn new(params: EnclaveSolReaderParams) -> Self { Self { bus: params.bus, - reader: params.reader, state: EnclaveSolReaderState::default(), repository: params.repository, } @@ -136,22 +135,21 @@ impl EnclaveSolReader { contract_address: &str, repository: &Repository, ) -> Result> { - let params = EnclaveSolReaderParams { + let addr = Self::load(EnclaveSolReaderParams { bus: bus.clone(), - reader: EvmEventReader::attach( - &bus.clone().into(), - provider, - extractor, - contract_address, - None, - ) - .await?, repository: repository.clone(), - }; - - let addr = Self::load(params).await?; - bus.send(Subscribe::new("Shutdown", addr.clone().into())) - .await?; + }) + .await?; + + EvmEventReader::attach( + &addr.clone().into(), + provider, + extractor, + contract_address, + None, + &bus.clone().into(), + ) + .await?; info!(address=%contract_address, "EnclaveSolReader is listening to address"); @@ -163,17 +161,10 @@ impl Actor for EnclaveSolReader { type Context = actix::Context; } -impl Handler for EnclaveSolReader { +impl Handler for EnclaveSolReader { type Result = (); - fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - // If this is a shutdown signal it will be coming from the event bus forward it to the reader - if let EnclaveEvent::Shutdown { .. } = msg { - self.reader.do_send(msg); - return; - } - - // Other enclave events will be coming from the reader - check the event id cache forward to the event bus - let event_id = msg.get_id(); + fn handle(&mut self, wrapped: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { + let event_id = wrapped.event.get_id(); if self.state.ids.contains(&event_id) { trace!( "Event id {} has already been seen and was not forwarded to the bus", @@ -183,10 +174,11 @@ impl Handler for EnclaveSolReader { } // Forward everything else to the event bus - self.bus.do_send(msg); + self.bus.do_send(wrapped.event); // Save processed ids self.state.ids.insert(event_id); + self.state.last_block = wrapped.block; self.checkpoint(); } } @@ -210,7 +202,6 @@ impl FromSnapshotWithParams for EnclaveSolReader { async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { Ok(Self { bus: params.bus, - reader: params.reader, state: snapshot, repository: params.repository, }) diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index 9983b9c4..7eda0b09 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -10,12 +10,25 @@ use alloy::providers::Provider; use alloy::rpc::types::Filter; use alloy::transports::{BoxTransport, Transport}; use anyhow::{anyhow, Result}; -use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent}; +use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus, Subscribe}; use futures_util::stream::StreamExt; use tokio::select; use tokio::sync::{oneshot, Mutex}; use tracing::{info, trace, warn}; +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[rtype(result = "()")] +pub struct EnclaveEvmEvent { + pub event: EnclaveEvent, + pub block: Option, +} + +impl EnclaveEvmEvent { + fn new(event: EnclaveEvent, block: Option) -> Self { + Self { event, block } + } +} + pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; pub type EventReader = EvmEventReader; @@ -31,7 +44,7 @@ where /// The contract address contract_address: Address, /// The EnclaveEvent Recipient to send events to - bus: Recipient, + parent: Recipient, /// The Extractor function to determine which events to extract and convert to EnclaveEvents extractor: ExtractorFn, /// A shutdown receiver to listen to for shutdown signals sent to the loop this is only used @@ -41,8 +54,8 @@ where shutdown_tx: Option>, /// The block that processing should start from start_block: Option, - /// Last block processed - last_block: Arc>>, + /// Event bus for error propagation + bus: Addr, } impl EvmEventReader @@ -51,40 +64,46 @@ where T: Transport + Clone + Unpin, { pub fn new( - bus: &Recipient, + parent: &Recipient, provider: &WithChainId, extractor: ExtractorFn, contract_address: &Address, start_block: Option, + bus: &Addr, ) -> Result { let (shutdown_tx, shutdown_rx) = oneshot::channel(); Ok(Self { contract_address: contract_address.clone(), provider: Some(provider.clone()), extractor, - bus: bus.clone(), + parent: parent.clone(), shutdown_rx: Some(shutdown_rx), shutdown_tx: Some(shutdown_tx), start_block, - last_block: Arc::new(Mutex::new(None)), + bus: bus.clone(), }) } pub async fn attach( - bus: &Recipient, + parent: &Recipient, provider: &WithChainId, extractor: ExtractorFn, contract_address: &str, start_block: Option, + bus: &Addr, ) -> Result> { let addr = EvmEventReader::new( - bus, + parent, provider, extractor, &contract_address.parse()?, start_block, + bus, )? .start(); + + bus.do_send(Subscribe::new("Shutdown", addr.clone().into())); + Ok(addr) } } @@ -96,6 +115,7 @@ where { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { + let parent = self.parent.clone(); let bus = self.bus.clone(); let Some(provider) = self.provider.take() else { tracing::error!("Could not start event reader as provider has already been used."); @@ -110,17 +130,16 @@ where let contract_address = self.contract_address; let start_block = self.start_block; - let last_block = self.last_block.clone(); ctx.spawn( async move { stream_from_evm( provider, &contract_address, - bus, + &parent, extractor, shutdown, start_block, - last_block, + &bus, ) .await } @@ -132,11 +151,11 @@ where async fn stream_from_evm, T: Transport + Clone>( provider: WithChainId, contract_address: &Address, - bus: Recipient, + parent: &Recipient, extractor: fn(&LogData, Option<&B256>, u64) -> Option, mut shutdown: oneshot::Receiver<()>, start_block: Option, - last_block: Arc>>, + bus: &Addr, ) { let chain_id = provider.get_chain_id(); let provider = provider.get_provider(); @@ -155,9 +174,7 @@ async fn stream_from_evm, T: Transport + Clone>( let block_number = log.block_number; if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { trace!("Processing historical log"); - bus.do_send(event); - let mut guard = last_block.lock().await; - *guard = block_number; + parent.do_send(EnclaveEvmEvent::new(event, block_number)); } } } @@ -184,9 +201,8 @@ async fn stream_from_evm, T: Transport + Clone>( continue; }; info!("Extracted log from evm sending now."); - bus.do_send(event); - let mut guard = last_block.lock().await; - *guard = block_number; + parent.do_send(EnclaveEvmEvent::new(event, block_number)); + } None => break, // Stream ended } From f938e36c850c30f220a9af028ea0affc9472f622 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 4 Nov 2024 20:35:22 +0700 Subject: [PATCH 11/31] Hook up last_block to reader --- .../evm/src/ciphernode_registry_sol.rs | 14 ++++----- .../ciphernode/evm/src/enclave_sol_reader.rs | 23 ++++++++------- packages/ciphernode/evm/src/lib.rs | 2 +- packages/ciphernode/evm/tests/evm_reader.rs | 29 +++++++++++++++---- 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index c3bd3b9c..cae4524b 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -126,14 +126,12 @@ impl CiphernodeRegistrySolReader { } } - pub async fn load(params: CiphernodeRegistryReaderParams) -> Result> { - let addr = if let Some(snapshot) = params.repository.read().await? { + pub async fn load(params: CiphernodeRegistryReaderParams) -> Result { + Ok(if let Some(snapshot) = params.repository.read().await? { Self::from_snapshot(params, snapshot).await? } else { Self::new(params) - } - .start(); - Ok(addr) + }) } pub async fn attach( @@ -147,14 +145,16 @@ impl CiphernodeRegistrySolReader { repository: repository.clone(), }; - let addr = Self::load(params).await?; + let actor = Self::load(params).await?; + let last_block = actor.state.last_block; + let addr = actor.start(); EvmEventReader::attach( &addr.clone().into(), provider, extractor, contract_address, - None, + last_block, &bus.clone().into(), ) .await?; diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index 63f5c491..930c4d60 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -119,14 +119,12 @@ impl EnclaveSolReader { } } - pub async fn load(params: EnclaveSolReaderParams) -> Result> { - let addr = if let Some(snapshot) = params.repository.read().await? { + pub async fn load(params: EnclaveSolReaderParams) -> Result { + Ok(if let Some(snapshot) = params.repository.read().await? { Self::from_snapshot(params, snapshot).await? } else { Self::new(params) - } - .start(); - Ok(addr) + }) } pub async fn attach( @@ -135,19 +133,22 @@ impl EnclaveSolReader { contract_address: &str, repository: &Repository, ) -> Result> { - let addr = Self::load(EnclaveSolReaderParams { + let params = EnclaveSolReaderParams { bus: bus.clone(), repository: repository.clone(), - }) - .await?; + }; + + let actor = Self::load(params).await?; + let last_block = actor.state.last_block; + let addr = actor.start(); EvmEventReader::attach( - &addr.clone().into(), + &addr.clone().recipient(), provider, extractor, contract_address, - None, - &bus.clone().into(), + last_block, + &bus.clone(), ) .await?; diff --git a/packages/ciphernode/evm/src/lib.rs b/packages/ciphernode/evm/src/lib.rs index ba330d5d..db53cbe3 100644 --- a/packages/ciphernode/evm/src/lib.rs +++ b/packages/ciphernode/evm/src/lib.rs @@ -13,5 +13,5 @@ pub use ciphernode_registry_sol::{ pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::{EnclaveSolReader, EnclaveSolReaderParams, EnclaveSolReaderState}; pub use enclave_sol_writer::EnclaveSolWriter; -pub use event_reader::{EvmEventReader, ExtractorFn}; +pub use event_reader::{EnclaveEvmEvent, EvmEventReader, ExtractorFn}; pub use registry_filter_sol::{RegistryFilterSol, RegistryFilterSolWriter}; diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index 15daeddd..165142bf 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -1,4 +1,4 @@ -use actix::Actor; +use actix::{Actor, Addr, Handler}; use alloy::{ node_bindings::Anvil, primitives::{FixedBytes, LogData}, @@ -8,7 +8,7 @@ use alloy::{ }; use anyhow::Result; use enclave_core::{EnclaveEvent, EventBus, GetHistory, TestEvent}; -use evm::{helpers::WithChainId, EvmEventReader}; +use evm::{helpers::WithChainId, EnclaveEvmEvent, EvmEventReader}; use std::time::Duration; use tokio::time::sleep; @@ -37,6 +37,21 @@ fn test_event_extractor( } } +struct EvmEventUnwrapper { + bus: Addr, +} + +impl Actor for EvmEventUnwrapper { + type Context = actix::Context; +} + +impl Handler for EvmEventUnwrapper { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { + self.bus.do_send(msg.event); + } +} + #[actix::test] async fn evm_reader() -> Result<()> { // Create a WS provider @@ -47,13 +62,14 @@ async fn evm_reader() -> Result<()> { let arc_provider = WithChainId::new(provider).await?; let contract = EmitLogs::deploy(arc_provider.get_provider()).await?; let bus = EventBus::new(true).start(); - + let unwrapper = EvmEventUnwrapper { bus: bus.clone() }.start(); EvmEventReader::attach( - &bus.clone().into(), + &unwrapper.clone().into(), &arc_provider, test_event_extractor, &contract.address().to_string(), None, + &bus, ) .await?; @@ -100,7 +116,7 @@ async fn ensure_historical_events() -> Result<()> { let arc_provider = WithChainId::new(provider).await?; let contract = EmitLogs::deploy(arc_provider.get_provider()).await?; let bus = EventBus::new(true).start(); - + let unwrapper = EvmEventUnwrapper { bus: bus.clone() }.start(); let historical_msgs = vec!["these", "are", "historical", "events"]; let live_events = vec!["these", "events", "are", "live"]; @@ -114,11 +130,12 @@ async fn ensure_historical_events() -> Result<()> { } EvmEventReader::attach( - &bus.clone().into(), + &unwrapper.clone().into(), &arc_provider, test_event_extractor, &contract.address().to_string(), None, + &bus, ) .await?; From d6df8508fd2611682dbe31ff01ff7342365f6db3 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 11:00:42 +0700 Subject: [PATCH 12/31] Tidy up test code --- .../evm/tests/enclave_sol_reader.rs | 1 + packages/ciphernode/evm/tests/evm_reader.rs | 34 +++++++++++-------- 2 files changed, 21 insertions(+), 14 deletions(-) create mode 100644 packages/ciphernode/evm/tests/enclave_sol_reader.rs diff --git a/packages/ciphernode/evm/tests/enclave_sol_reader.rs b/packages/ciphernode/evm/tests/enclave_sol_reader.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/packages/ciphernode/evm/tests/enclave_sol_reader.rs @@ -0,0 +1 @@ + diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index 165142bf..26953067 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -57,15 +57,18 @@ async fn evm_reader() -> Result<()> { // Create a WS provider // NOTE: Anvil must be available on $PATH let anvil = Anvil::new().block_time(1).try_spawn()?; - let ws = WsConnect::new(anvil.ws_endpoint()); - let provider = ProviderBuilder::new().on_ws(ws).await?; - let arc_provider = WithChainId::new(provider).await?; - let contract = EmitLogs::deploy(arc_provider.get_provider()).await?; + let provider = WithChainId::new( + ProviderBuilder::new() + .on_ws(WsConnect::new(anvil.ws_endpoint())) + .await?, + ) + .await?; + let contract = EmitLogs::deploy(provider.get_provider()).await?; let bus = EventBus::new(true).start(); - let unwrapper = EvmEventUnwrapper { bus: bus.clone() }.start(); + EvmEventReader::attach( - &unwrapper.clone().into(), - &arc_provider, + &EvmEventUnwrapper { bus: bus.clone() }.start().into(), + &provider, test_event_extractor, &contract.address().to_string(), None, @@ -111,12 +114,15 @@ async fn ensure_historical_events() -> Result<()> { // Create a WS provider // NOTE: Anvil must be available on $PATH let anvil = Anvil::new().block_time(1).try_spawn()?; - let ws = WsConnect::new(anvil.ws_endpoint()); - let provider = ProviderBuilder::new().on_ws(ws).await?; - let arc_provider = WithChainId::new(provider).await?; - let contract = EmitLogs::deploy(arc_provider.get_provider()).await?; + let provider = WithChainId::new( + ProviderBuilder::new() + .on_ws(WsConnect::new(anvil.ws_endpoint())) + .await?, + ) + .await?; + let contract = EmitLogs::deploy(provider.get_provider()).await?; let bus = EventBus::new(true).start(); - let unwrapper = EvmEventUnwrapper { bus: bus.clone() }.start(); + let historical_msgs = vec!["these", "are", "historical", "events"]; let live_events = vec!["these", "events", "are", "live"]; @@ -130,8 +136,8 @@ async fn ensure_historical_events() -> Result<()> { } EvmEventReader::attach( - &unwrapper.clone().into(), - &arc_provider, + &EvmEventUnwrapper { bus: bus.clone() }.start().into(), + &provider, test_event_extractor, &contract.address().to_string(), None, From dc953198dd85adcb83ab29b5c8e739ba26cc22bd Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 12:14:31 +0700 Subject: [PATCH 13/31] Refactor to push persistence to EvmEventReader --- packages/ciphernode/Cargo.lock | 1 + .../ciphernode/enclave_node/src/datastore.rs | 14 +- packages/ciphernode/evm/Cargo.toml | 4 + .../evm/src/ciphernode_registry_sol.rs | 111 ++----------- packages/ciphernode/evm/src/enclave_sol.rs | 7 +- .../ciphernode/evm/src/enclave_sol_reader.rs | 123 ++------------ packages/ciphernode/evm/src/event_reader.rs | 152 ++++++++++++++---- packages/ciphernode/evm/src/lib.rs | 9 +- .../evm/tests/enclave_sol_reader.rs | 1 - packages/ciphernode/evm/tests/evm_reader.rs | 38 +++-- .../ciphernode/router/src/repositories.rs | 9 +- 11 files changed, 189 insertions(+), 280 deletions(-) delete mode 100644 packages/ciphernode/evm/tests/enclave_sol_reader.rs diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 37f2bbd5..45f2bde6 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -2356,6 +2356,7 @@ dependencies = [ "cipher 0.1.0", "data", "enclave-core", + "enclave_node", "futures-util", "serde", "sortition", diff --git a/packages/ciphernode/enclave_node/src/datastore.rs b/packages/ciphernode/enclave_node/src/datastore.rs index f88ff834..8eb77e0c 100644 --- a/packages/ciphernode/enclave_node/src/datastore.rs +++ b/packages/ciphernode/enclave_node/src/datastore.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use actix::{Actor, Addr}; use anyhow::Result; use config::AppConfig; @@ -5,11 +7,19 @@ use data::{DataStore, InMemStore, SledStore}; use enclave_core::EventBus; use router::{Repositories, RepositoriesFactory}; +pub fn get_sled_store(bus: &Addr, db_file: &PathBuf) -> Result { + Ok((&SledStore::new(bus, db_file)?.start()).into()) +} + +pub fn get_in_mem_store() -> DataStore { + (&InMemStore::new(true).start()).into() +} + pub fn setup_datastore(config: &AppConfig, bus: &Addr) -> Result { let store: DataStore = if !config.use_in_mem_store() { - (&SledStore::new(&bus, &config.db_file())?.start()).into() + get_sled_store(&bus, &config.db_file())? } else { - (&InMemStore::new(true).start()).into() + get_in_mem_store() }; Ok(store) } diff --git a/packages/ciphernode/evm/Cargo.toml b/packages/ciphernode/evm/Cargo.toml index a5fc67ac..fba4aea4 100644 --- a/packages/ciphernode/evm/Cargo.toml +++ b/packages/ciphernode/evm/Cargo.toml @@ -18,3 +18,7 @@ tokio = { workspace = true } tracing = { workspace = true } serde = { workspace = true } zeroize = { workspace = true } + +[dev-dependencies] +enclave_node = { path = "../enclave_node" } + diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index cae4524b..1bb5da3d 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -1,20 +1,17 @@ -use std::collections::HashSet; - use crate::{ - event_reader::{EnclaveEvmEvent, EventReader}, + event_reader::EvmEventReaderState, helpers::{ReadonlyProvider, WithChainId}, EvmEventReader, }; -use actix::{Actor, Addr, Handler}; +use actix::{Actor, Addr}; use alloy::{ primitives::{LogData, B256}, sol, sol_types::SolEvent, }; use anyhow::Result; -use async_trait::async_trait; -use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; -use enclave_core::{EnclaveEvent, EventBus, EventId, Subscribe}; +use data::Repository; +use enclave_core::{EnclaveEvent, EventBus}; use tracing::{error, info, trace}; sol!( @@ -100,62 +97,22 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, _: u64) -> Option, - state: CiphernodeRegistryReaderState, - repository: Repository, -} - -pub struct CiphernodeRegistryReaderParams { - bus: Addr, - repository: Repository, -} - -#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] -pub struct CiphernodeRegistryReaderState { - pub ids: HashSet, - pub last_block: Option, -} +pub struct CiphernodeRegistrySolReader; impl CiphernodeRegistrySolReader { - pub fn new(params: CiphernodeRegistryReaderParams) -> Self { - Self { - bus: params.bus, - state: CiphernodeRegistryReaderState::default(), - repository: params.repository, - } - } - - pub async fn load(params: CiphernodeRegistryReaderParams) -> Result { - Ok(if let Some(snapshot) = params.repository.read().await? { - Self::from_snapshot(params, snapshot).await? - } else { - Self::new(params) - }) - } - pub async fn attach( bus: &Addr, provider: &WithChainId, contract_address: &str, - repository: &Repository, - ) -> Result> { - let params = CiphernodeRegistryReaderParams { - bus: bus.clone(), - repository: repository.clone(), - }; - - let actor = Self::load(params).await?; - let last_block = actor.state.last_block; - let addr = actor.start(); - - EvmEventReader::attach( - &addr.clone().into(), + repository: &Repository, + ) -> Result>> { + let addr = EvmEventReader::attach( provider, extractor, contract_address, - last_block, + None, &bus.clone().into(), + repository, ) .await?; @@ -169,52 +126,6 @@ impl Actor for CiphernodeRegistrySolReader { type Context = actix::Context; } -impl Handler for CiphernodeRegistrySolReader { - type Result = (); - fn handle(&mut self, wrapped: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { - let event_id = wrapped.event.get_id(); - if self.state.ids.contains(&event_id) { - trace!( - "Event id {} has already been seen and was not forwarded to the bus", - &event_id - ); - return; - } - - // Forward everything else to the event bus - self.bus.do_send(wrapped.event); - - // Save processed ids - self.state.ids.insert(event_id); - self.state.last_block = wrapped.block; - self.checkpoint(); - } -} - -impl Snapshot for CiphernodeRegistrySolReader { - type Snapshot = CiphernodeRegistryReaderState; - fn snapshot(&self) -> Self::Snapshot { - self.state.clone() - } -} - -impl Checkpoint for CiphernodeRegistrySolReader { - fn repository(&self) -> &Repository { - &self.repository - } -} - -#[async_trait] -impl FromSnapshotWithParams for CiphernodeRegistrySolReader { - type Params = CiphernodeRegistryReaderParams; - async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { - Ok(Self { - bus: params.bus, - state: snapshot, - repository: params.repository, - }) - } -} /// Eventual wrapper for both a reader and a writer pub struct CiphernodeRegistrySol; impl CiphernodeRegistrySol { @@ -222,7 +133,7 @@ impl CiphernodeRegistrySol { bus: &Addr, provider: &WithChainId, contract_address: &str, - repository: &Repository, + repository: &Repository, ) -> Result<()> { CiphernodeRegistrySolReader::attach(bus, provider, contract_address, repository).await?; // TODO: Writer if needed diff --git a/packages/ciphernode/evm/src/enclave_sol.rs b/packages/ciphernode/evm/src/enclave_sol.rs index aadf4c03..6a29c017 100644 --- a/packages/ciphernode/evm/src/enclave_sol.rs +++ b/packages/ciphernode/evm/src/enclave_sol.rs @@ -1,8 +1,5 @@ use crate::{ - enclave_sol_reader::EnclaveSolReader, - enclave_sol_writer::EnclaveSolWriter, - helpers::{ReadonlyProvider, SignerProvider, WithChainId}, - EnclaveSolReaderState, + enclave_sol_reader::EnclaveSolReader, enclave_sol_writer::EnclaveSolWriter, event_reader::EvmEventReaderState, helpers::{ReadonlyProvider, SignerProvider, WithChainId} }; use actix::Addr; use anyhow::Result; @@ -16,7 +13,7 @@ impl EnclaveSol { read_provider: &WithChainId, write_provider: &WithChainId, contract_address: &str, - repository: &Repository, + repository: &Repository, ) -> Result<()> { EnclaveSolReader::attach(bus, read_provider, contract_address, repository).await?; EnclaveSolWriter::attach(bus, write_provider, contract_address).await?; diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index 930c4d60..6272c740 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -1,16 +1,13 @@ -use std::collections::HashSet; - -use crate::event_reader::{EnclaveEvmEvent, EventReader}; +use crate::event_reader::EvmEventReaderState; use crate::helpers::{ReadonlyProvider, WithChainId}; use crate::EvmEventReader; -use actix::{Actor, Addr, Handler}; +use actix::Addr; use alloy::primitives::{LogData, B256}; use alloy::transports::BoxTransport; use alloy::{sol, sol_types::SolEvent}; use anyhow::Result; -use async_trait::async_trait; -use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; -use enclave_core::{EnclaveEvent, EventBus, EventId}; +use data::Repository; +use enclave_core::{EnclaveEvent, EventBus}; use tracing::{error, info, trace}; sol!( @@ -83,72 +80,23 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option< } } -#[derive(serde::Serialize, serde::Deserialize, Clone)] -pub struct EnclaveSolReaderState { - pub ids: HashSet, - pub last_block: Option, -} - -impl Default for EnclaveSolReaderState { - fn default() -> Self { - Self { - ids: HashSet::new(), - last_block: None, - } - } -} - /// Connects to Enclave.sol converting EVM events to EnclaveEvents -pub struct EnclaveSolReader { - bus: Addr, - state: EnclaveSolReaderState, - repository: Repository, -} - -pub struct EnclaveSolReaderParams { - bus: Addr, - repository: Repository, -} +pub struct EnclaveSolReader; impl EnclaveSolReader { - pub fn new(params: EnclaveSolReaderParams) -> Self { - Self { - bus: params.bus, - state: EnclaveSolReaderState::default(), - repository: params.repository, - } - } - - pub async fn load(params: EnclaveSolReaderParams) -> Result { - Ok(if let Some(snapshot) = params.repository.read().await? { - Self::from_snapshot(params, snapshot).await? - } else { - Self::new(params) - }) - } - pub async fn attach( bus: &Addr, provider: &WithChainId, contract_address: &str, - repository: &Repository, - ) -> Result> { - let params = EnclaveSolReaderParams { - bus: bus.clone(), - repository: repository.clone(), - }; - - let actor = Self::load(params).await?; - let last_block = actor.state.last_block; - let addr = actor.start(); - - EvmEventReader::attach( - &addr.clone().recipient(), + repository: &Repository, + ) -> Result>> { + let addr = EvmEventReader::attach( provider, extractor, contract_address, - last_block, + None, &bus.clone(), + repository, ) .await?; @@ -157,54 +105,3 @@ impl EnclaveSolReader { Ok(addr) } } - -impl Actor for EnclaveSolReader { - type Context = actix::Context; -} - -impl Handler for EnclaveSolReader { - type Result = (); - fn handle(&mut self, wrapped: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { - let event_id = wrapped.event.get_id(); - if self.state.ids.contains(&event_id) { - trace!( - "Event id {} has already been seen and was not forwarded to the bus", - &event_id - ); - return; - } - - // Forward everything else to the event bus - self.bus.do_send(wrapped.event); - - // Save processed ids - self.state.ids.insert(event_id); - self.state.last_block = wrapped.block; - self.checkpoint(); - } -} - -impl Snapshot for EnclaveSolReader { - type Snapshot = EnclaveSolReaderState; - fn snapshot(&self) -> Self::Snapshot { - self.state.clone() - } -} - -impl Checkpoint for EnclaveSolReader { - fn repository(&self) -> &Repository { - &self.repository - } -} - -#[async_trait] -impl FromSnapshotWithParams for EnclaveSolReader { - type Params = EnclaveSolReaderParams; - async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { - Ok(Self { - bus: params.bus, - state: snapshot, - repository: params.repository, - }) - } -} diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index 7eda0b09..e7a16f0c 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -1,5 +1,4 @@ -use std::sync::Arc; - +use std::collections::HashSet; use crate::helpers::{ReadonlyProvider, WithChainId}; use actix::prelude::*; use actix::{Addr, Recipient}; @@ -10,10 +9,12 @@ use alloy::providers::Provider; use alloy::rpc::types::Filter; use alloy::transports::{BoxTransport, Transport}; use anyhow::{anyhow, Result}; -use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus, Subscribe}; +use async_trait::async_trait; +use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; +use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus, EventId, Subscribe}; use futures_util::stream::StreamExt; use tokio::select; -use tokio::sync::{oneshot, Mutex}; +use tokio::sync::oneshot; use tracing::{info, trace, warn}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] @@ -33,6 +34,25 @@ pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; pub type EventReader = EvmEventReader; +pub struct EvmEventReaderParams +where + P: Provider + Clone + 'static, + T: Transport + Clone + Unpin, +{ + provider: WithChainId, + extractor: ExtractorFn, + contract_address: Address, + start_block: Option, + bus: Addr, + repository: Repository, +} + +#[derive(Default, serde::Serialize, serde::Deserialize, Clone)] +pub struct EvmEventReaderState { + pub ids: HashSet, + pub last_block: Option, +} + /// Connects to Enclave.sol converting EVM events to EnclaveEvents pub struct EvmEventReader where @@ -43,8 +63,6 @@ where provider: Option>, /// The contract address contract_address: Address, - /// The EnclaveEvent Recipient to send events to - parent: Recipient, /// The Extractor function to determine which events to extract and convert to EnclaveEvents extractor: ExtractorFn, /// A shutdown receiver to listen to for shutdown signals sent to the loop this is only used @@ -56,6 +74,8 @@ where start_block: Option, /// Event bus for error propagation bus: Addr, + state: EvmEventReaderState, + repository: Repository, } impl EvmEventReader @@ -63,44 +83,46 @@ where P: Provider + Clone + 'static, T: Transport + Clone + Unpin, { - pub fn new( - parent: &Recipient, - provider: &WithChainId, - extractor: ExtractorFn, - contract_address: &Address, - start_block: Option, - bus: &Addr, - ) -> Result { + pub fn new(params: EvmEventReaderParams) -> Self { let (shutdown_tx, shutdown_rx) = oneshot::channel(); - Ok(Self { - contract_address: contract_address.clone(), - provider: Some(provider.clone()), - extractor, - parent: parent.clone(), + Self { + contract_address: params.contract_address, + provider: Some(params.provider), + extractor: params.extractor, shutdown_rx: Some(shutdown_rx), shutdown_tx: Some(shutdown_tx), - start_block, - bus: bus.clone(), + start_block: params.start_block, + bus: params.bus, + state: EvmEventReaderState::default(), + repository: params.repository, + } + } + + pub async fn load(params: EvmEventReaderParams) -> Result { + Ok(if let Some(snapshot) = params.repository.read().await? { + Self::from_snapshot(params, snapshot).await? + } else { + Self::new(params) }) } pub async fn attach( - parent: &Recipient, provider: &WithChainId, extractor: ExtractorFn, contract_address: &str, start_block: Option, bus: &Addr, + repository: &Repository, ) -> Result> { - let addr = EvmEventReader::new( - parent, - provider, + let params = EvmEventReaderParams { + provider: provider.clone(), extractor, - &contract_address.parse()?, + contract_address: contract_address.parse()?, start_block, - bus, - )? - .start(); + bus: bus.clone(), + repository: repository.clone(), + }; + let addr = EvmEventReader::new(params).start(); bus.do_send(Subscribe::new("Shutdown", addr.clone().into())); @@ -115,7 +137,7 @@ where { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { - let parent = self.parent.clone(); + let parent = ctx.address().recipient(); let bus = self.bus.clone(); let Some(provider) = self.provider.take() else { tracing::error!("Could not start event reader as provider has already been used."); @@ -235,3 +257,73 @@ where } } } + +impl Handler for EvmEventReader +where + P: Provider + Clone + 'static, + T: Transport + Clone + Unpin, +{ + type Result = (); + fn handle(&mut self, wrapped: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { + let event_id = wrapped.event.get_id(); + if self.state.ids.contains(&event_id) { + trace!( + "Event id {} has already been seen and was not forwarded to the bus", + &event_id + ); + return; + } + + // Forward everything else to the event bus + self.bus.do_send(wrapped.event); + + // Save processed ids + self.state.ids.insert(event_id); + self.state.last_block = wrapped.block; + self.checkpoint(); + } +} + +impl Snapshot for EvmEventReader +where + P: Provider + Clone + 'static, + T: Transport + Clone + Unpin, +{ + type Snapshot = EvmEventReaderState; + fn snapshot(&self) -> Self::Snapshot { + self.state.clone() + } +} + +impl Checkpoint for EvmEventReader +where + P: Provider + Clone + 'static, + T: Transport + Clone + Unpin, +{ + fn repository(&self) -> &Repository { + &self.repository + } +} + +#[async_trait] +impl FromSnapshotWithParams for EvmEventReader +where + P: Provider + Clone + 'static, + T: Transport + Clone + Unpin, +{ + type Params = EvmEventReaderParams; + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + Ok(Self { + contract_address: params.contract_address, + provider: Some(params.provider), + extractor: params.extractor, + shutdown_rx: Some(shutdown_rx), + shutdown_tx: Some(shutdown_tx), + start_block: params.start_block, + bus: params.bus, + state: snapshot, + repository: params.repository, + }) + } +} diff --git a/packages/ciphernode/evm/src/lib.rs b/packages/ciphernode/evm/src/lib.rs index db53cbe3..d6b1ee99 100644 --- a/packages/ciphernode/evm/src/lib.rs +++ b/packages/ciphernode/evm/src/lib.rs @@ -6,12 +6,9 @@ mod event_reader; pub mod helpers; mod registry_filter_sol; -pub use ciphernode_registry_sol::{ - CiphernodeRegistryReaderParams, CiphernodeRegistryReaderState, CiphernodeRegistrySol, - CiphernodeRegistrySolReader, -}; +pub use ciphernode_registry_sol::{CiphernodeRegistrySol, CiphernodeRegistrySolReader}; pub use enclave_sol::EnclaveSol; -pub use enclave_sol_reader::{EnclaveSolReader, EnclaveSolReaderParams, EnclaveSolReaderState}; +pub use enclave_sol_reader::EnclaveSolReader; pub use enclave_sol_writer::EnclaveSolWriter; -pub use event_reader::{EnclaveEvmEvent, EvmEventReader, ExtractorFn}; +pub use event_reader::{EnclaveEvmEvent,EvmEventReaderState, EvmEventReader, ExtractorFn}; pub use registry_filter_sol::{RegistryFilterSol, RegistryFilterSolWriter}; diff --git a/packages/ciphernode/evm/tests/enclave_sol_reader.rs b/packages/ciphernode/evm/tests/enclave_sol_reader.rs deleted file mode 100644 index 8b137891..00000000 --- a/packages/ciphernode/evm/tests/enclave_sol_reader.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index 26953067..fc9322b9 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -7,7 +7,9 @@ use alloy::{ sol_types::SolEvent, }; use anyhow::Result; +use data::{DataStore, InMemStore, Repository}; use enclave_core::{EnclaveEvent, EventBus, GetHistory, TestEvent}; +use enclave_node::get_in_mem_store; use evm::{helpers::WithChainId, EnclaveEvmEvent, EvmEventReader}; use std::time::Duration; use tokio::time::sleep; @@ -37,21 +39,21 @@ fn test_event_extractor( } } -struct EvmEventUnwrapper { - bus: Addr, -} - -impl Actor for EvmEventUnwrapper { - type Context = actix::Context; -} - -impl Handler for EvmEventUnwrapper { - type Result = (); - fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { - self.bus.do_send(msg.event); - } -} - +// struct EvmEventUnwrapper { +// bus: Addr, +// } +// +// impl Actor for EvmEventUnwrapper { +// type Context = actix::Context; +// } +// +// impl Handler for EvmEventUnwrapper { +// type Result = (); +// fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { +// self.bus.do_send(msg.event); +// } +// } +// #[actix::test] async fn evm_reader() -> Result<()> { // Create a WS provider @@ -65,14 +67,15 @@ async fn evm_reader() -> Result<()> { .await?; let contract = EmitLogs::deploy(provider.get_provider()).await?; let bus = EventBus::new(true).start(); + let repository = Repository::new(get_in_mem_store()); EvmEventReader::attach( - &EvmEventUnwrapper { bus: bus.clone() }.start().into(), &provider, test_event_extractor, &contract.address().to_string(), None, &bus, + &repository, ) .await?; @@ -126,6 +129,7 @@ async fn ensure_historical_events() -> Result<()> { let historical_msgs = vec!["these", "are", "historical", "events"]; let live_events = vec!["these", "events", "are", "live"]; + let repository = Repository::new(get_in_mem_store()); for msg in historical_msgs.clone() { contract .setValue(msg.to_string()) @@ -136,12 +140,12 @@ async fn ensure_historical_events() -> Result<()> { } EvmEventReader::attach( - &EvmEventUnwrapper { bus: bus.clone() }.start().into(), &provider, test_event_extractor, &contract.address().to_string(), None, &bus, + &repository ) .await?; diff --git a/packages/ciphernode/router/src/repositories.rs b/packages/ciphernode/router/src/repositories.rs index db32140c..e8557c18 100644 --- a/packages/ciphernode/router/src/repositories.rs +++ b/packages/ciphernode/router/src/repositories.rs @@ -2,7 +2,7 @@ use crate::{CommitteeMeta, E3RequestContextSnapshot, E3RequestRouterSnapshot}; use aggregator::{PlaintextAggregatorState, PublicKeyAggregatorState}; use data::{DataStore, Repository}; use enclave_core::E3id; -use evm::{CiphernodeRegistryReaderState, EnclaveSolReaderState}; +use evm::EvmEventReaderState; use fhe::FheSnapshot; use keyshare::KeyshareState; use sortition::SortitionModule; @@ -74,16 +74,13 @@ impl Repositories { Repository::new(self.store.scope(format!("//eth_private_key"))) } - pub fn enclave_sol_reader(&self, chain_id: u64) -> Repository { + pub fn enclave_sol_reader(&self, chain_id: u64) -> Repository { Repository::new( self.store .scope(format!("//evm_readers/enclave/{chain_id}")), ) } - pub fn ciphernode_registry_reader( - &self, - chain_id: u64, - ) -> Repository { + pub fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository { Repository::new( self.store .scope(format!("//evm_readers/ciphernode_registry/{chain_id}")), From 59f0d540222666838e40e1780ad2ec8aa7cd3b11 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 12:15:15 +0700 Subject: [PATCH 14/31] Tidy up dead code --- packages/ciphernode/evm/tests/evm_reader.rs | 23 ++++----------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index fc9322b9..8636c4d8 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -1,4 +1,4 @@ -use actix::{Actor, Addr, Handler}; +use actix::Actor; use alloy::{ node_bindings::Anvil, primitives::{FixedBytes, LogData}, @@ -7,10 +7,10 @@ use alloy::{ sol_types::SolEvent, }; use anyhow::Result; -use data::{DataStore, InMemStore, Repository}; +use data::Repository; use enclave_core::{EnclaveEvent, EventBus, GetHistory, TestEvent}; use enclave_node::get_in_mem_store; -use evm::{helpers::WithChainId, EnclaveEvmEvent, EvmEventReader}; +use evm::{helpers::WithChainId, EvmEventReader}; use std::time::Duration; use tokio::time::sleep; @@ -39,21 +39,6 @@ fn test_event_extractor( } } -// struct EvmEventUnwrapper { -// bus: Addr, -// } -// -// impl Actor for EvmEventUnwrapper { -// type Context = actix::Context; -// } -// -// impl Handler for EvmEventUnwrapper { -// type Result = (); -// fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { -// self.bus.do_send(msg.event); -// } -// } -// #[actix::test] async fn evm_reader() -> Result<()> { // Create a WS provider @@ -145,7 +130,7 @@ async fn ensure_historical_events() -> Result<()> { &contract.address().to_string(), None, &bus, - &repository + &repository, ) .await?; From bf8bbe2d9dc13d93afda28628f63a001a525887f Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 12:48:25 +0700 Subject: [PATCH 15/31] Resume after shutdown test --- packages/ciphernode/evm/tests/evm_reader.rs | 113 +++++++++++++++++++- 1 file changed, 111 insertions(+), 2 deletions(-) diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index 8636c4d8..bedbdca4 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -1,4 +1,4 @@ -use actix::Actor; +use actix::{Actor, Addr}; use alloy::{ node_bindings::Anvil, primitives::{FixedBytes, LogData}, @@ -8,7 +8,7 @@ use alloy::{ }; use anyhow::Result; use data::Repository; -use enclave_core::{EnclaveEvent, EventBus, GetHistory, TestEvent}; +use enclave_core::{EnclaveEvent, EventBus, GetHistory, Shutdown, TestEvent}; use enclave_node::get_in_mem_store; use evm::{helpers::WithChainId, EvmEventReader}; use std::time::Duration; @@ -162,3 +162,112 @@ async fn ensure_historical_events() -> Result<()> { Ok(()) } + +#[actix::test] +async fn ensure_resume_after_shutdown() -> Result<()> { + // Create a WS provider + // NOTE: Anvil must be available on $PATH + let anvil = Anvil::new().block_time(1).try_spawn()?; + let provider = WithChainId::new( + ProviderBuilder::new() + .on_ws(WsConnect::new(anvil.ws_endpoint())) + .await?, + ) + .await?; + let contract = EmitLogs::deploy(provider.get_provider()).await?; + let bus = EventBus::new(true).start(); + + async fn get_msgs(bus: &Addr) -> Result> { + let history = bus.send(GetHistory).await?; + let msgs: Vec = history + .into_iter() + .filter_map(|evt| match evt { + EnclaveEvent::TestEvent { data, .. } => Some(data.msg), + _ => None, + }) + .collect(); + + Ok(msgs) + } + + let repository = Repository::new(get_in_mem_store()); + + let before = vec!["before", "online"]; + let online = vec!["live", "events"]; + let after_shutdown = vec!["these", "are", "not", "lost"]; + let resume = vec!["resumed", "data"]; + + for msg in before.clone() { + contract + .setValue(msg.to_string()) + .send() + .await? + .watch() + .await?; + } + + let addr1 = EvmEventReader::attach( + &provider, + test_event_extractor, + &contract.address().to_string(), + None, + &bus, + &repository, + ) + .await?; + + for msg in online.clone() { + contract + .setValue(msg.to_string()) + .send() + .await? + .watch() + .await?; + } + + // Ensure shutdown doesn't cause event to be lost. + sleep(Duration::from_millis(1)).await; + addr1.send(EnclaveEvent::from(Shutdown)).await?; + + for msg in after_shutdown.clone() { + contract + .setValue(msg.to_string()) + .send() + .await? + .watch() + .await?; + } + + sleep(Duration::from_millis(1)).await; + let msgs = get_msgs(&bus).await?; + assert_eq!(msgs, vec!["before", "online", "live", "events"]); + + let _ = EvmEventReader::attach( + &provider, + test_event_extractor, + &contract.address().to_string(), + None, + &bus, + &repository, + ) + .await?; + + sleep(Duration::from_millis(1)).await; + let msgs = get_msgs(&bus).await?; + assert_eq!(msgs, vec!["before", "online", "live", "events", "these", "are", "not", "lost"]); + + for msg in resume.clone() { + contract + .setValue(msg.to_string()) + .send() + .await? + .watch() + .await?; + } + + sleep(Duration::from_millis(1)).await; + let msgs = get_msgs(&bus).await?; + assert_eq!(msgs, vec!["before", "online", "live", "events", "these", "are", "not", "lost", "resumed", "data"]); + + Ok(()) +} From 5afe50f8f0e1a48d7dd8ed610b907bc1be011b6b Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 12:49:01 +0700 Subject: [PATCH 16/31] Formatting --- packages/ciphernode/evm/src/enclave_sol.rs | 5 ++++- packages/ciphernode/evm/src/event_reader.rs | 2 +- packages/ciphernode/evm/src/lib.rs | 2 +- packages/ciphernode/evm/tests/evm_reader.rs | 12 ++++++++++-- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/ciphernode/evm/src/enclave_sol.rs b/packages/ciphernode/evm/src/enclave_sol.rs index 6a29c017..e8747ae2 100644 --- a/packages/ciphernode/evm/src/enclave_sol.rs +++ b/packages/ciphernode/evm/src/enclave_sol.rs @@ -1,5 +1,8 @@ use crate::{ - enclave_sol_reader::EnclaveSolReader, enclave_sol_writer::EnclaveSolWriter, event_reader::EvmEventReaderState, helpers::{ReadonlyProvider, SignerProvider, WithChainId} + enclave_sol_reader::EnclaveSolReader, + enclave_sol_writer::EnclaveSolWriter, + event_reader::EvmEventReaderState, + helpers::{ReadonlyProvider, SignerProvider, WithChainId}, }; use actix::Addr; use anyhow::Result; diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index e7a16f0c..fa92e2ad 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -1,4 +1,3 @@ -use std::collections::HashSet; use crate::helpers::{ReadonlyProvider, WithChainId}; use actix::prelude::*; use actix::{Addr, Recipient}; @@ -13,6 +12,7 @@ use async_trait::async_trait; use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus, EventId, Subscribe}; use futures_util::stream::StreamExt; +use std::collections::HashSet; use tokio::select; use tokio::sync::oneshot; use tracing::{info, trace, warn}; diff --git a/packages/ciphernode/evm/src/lib.rs b/packages/ciphernode/evm/src/lib.rs index d6b1ee99..e7fb10c9 100644 --- a/packages/ciphernode/evm/src/lib.rs +++ b/packages/ciphernode/evm/src/lib.rs @@ -10,5 +10,5 @@ pub use ciphernode_registry_sol::{CiphernodeRegistrySol, CiphernodeRegistrySolRe pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::EnclaveSolReader; pub use enclave_sol_writer::EnclaveSolWriter; -pub use event_reader::{EnclaveEvmEvent,EvmEventReaderState, EvmEventReader, ExtractorFn}; +pub use event_reader::{EnclaveEvmEvent, EvmEventReader, EvmEventReaderState, ExtractorFn}; pub use registry_filter_sol::{RegistryFilterSol, RegistryFilterSolWriter}; diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index bedbdca4..b8ff5d09 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -254,7 +254,10 @@ async fn ensure_resume_after_shutdown() -> Result<()> { sleep(Duration::from_millis(1)).await; let msgs = get_msgs(&bus).await?; - assert_eq!(msgs, vec!["before", "online", "live", "events", "these", "are", "not", "lost"]); + assert_eq!( + msgs, + vec!["before", "online", "live", "events", "these", "are", "not", "lost"] + ); for msg in resume.clone() { contract @@ -267,7 +270,12 @@ async fn ensure_resume_after_shutdown() -> Result<()> { sleep(Duration::from_millis(1)).await; let msgs = get_msgs(&bus).await?; - assert_eq!(msgs, vec!["before", "online", "live", "events", "these", "are", "not", "lost", "resumed", "data"]); + assert_eq!( + msgs, + vec![ + "before", "online", "live", "events", "these", "are", "not", "lost", "resumed", "data" + ] + ); Ok(()) } From 7d3553017d7340b86b342eb9e73fe0f8a802baf5 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 12:53:44 +0700 Subject: [PATCH 17/31] Neaten up test --- packages/ciphernode/evm/tests/evm_reader.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index b8ff5d09..c767eb50 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -192,12 +192,7 @@ async fn ensure_resume_after_shutdown() -> Result<()> { let repository = Repository::new(get_in_mem_store()); - let before = vec!["before", "online"]; - let online = vec!["live", "events"]; - let after_shutdown = vec!["these", "are", "not", "lost"]; - let resume = vec!["resumed", "data"]; - - for msg in before.clone() { + for msg in ["before", "online"] { contract .setValue(msg.to_string()) .send() @@ -216,7 +211,7 @@ async fn ensure_resume_after_shutdown() -> Result<()> { ) .await?; - for msg in online.clone() { + for msg in ["live", "events"] { contract .setValue(msg.to_string()) .send() @@ -229,7 +224,7 @@ async fn ensure_resume_after_shutdown() -> Result<()> { sleep(Duration::from_millis(1)).await; addr1.send(EnclaveEvent::from(Shutdown)).await?; - for msg in after_shutdown.clone() { + for msg in ["these", "are", "not", "lost"] { contract .setValue(msg.to_string()) .send() @@ -240,7 +235,7 @@ async fn ensure_resume_after_shutdown() -> Result<()> { sleep(Duration::from_millis(1)).await; let msgs = get_msgs(&bus).await?; - assert_eq!(msgs, vec!["before", "online", "live", "events"]); + assert_eq!(msgs, ["before", "online", "live", "events"]); let _ = EvmEventReader::attach( &provider, @@ -256,10 +251,10 @@ async fn ensure_resume_after_shutdown() -> Result<()> { let msgs = get_msgs(&bus).await?; assert_eq!( msgs, - vec!["before", "online", "live", "events", "these", "are", "not", "lost"] + ["before", "online", "live", "events", "these", "are", "not", "lost"] ); - for msg in resume.clone() { + for msg in ["resumed", "data"] { contract .setValue(msg.to_string()) .send() @@ -272,7 +267,7 @@ async fn ensure_resume_after_shutdown() -> Result<()> { let msgs = get_msgs(&bus).await?; assert_eq!( msgs, - vec![ + [ "before", "online", "live", "events", "these", "are", "not", "lost", "resumed", "data" ] ); From f58a05499ea10366516f8e900b953e65226a94a8 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 12:56:47 +0700 Subject: [PATCH 18/31] Formatting --- packages/ciphernode/evm/tests/evm_reader.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index c767eb50..0fdacf07 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -267,9 +267,7 @@ async fn ensure_resume_after_shutdown() -> Result<()> { let msgs = get_msgs(&bus).await?; assert_eq!( msgs, - [ - "before", "online", "live", "events", "these", "are", "not", "lost", "resumed", "data" - ] + ["before", "online", "live", "events", "these", "are", "not", "lost", "resumed", "data"] ); Ok(()) From 3c1b436b780ba7f43f00c9fd5685a6aa8aaa5c44 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 13:28:57 +0700 Subject: [PATCH 19/31] Hook start_block up to config --- packages/ciphernode/config/src/app_config.rs | 34 +++++++++++++++++-- .../ciphernode/enclave_node/src/aggregator.rs | 9 +++-- .../ciphernode/enclave_node/src/ciphernode.rs | 6 ++-- .../evm/src/ciphernode_registry_sol.rs | 6 ++-- packages/ciphernode/evm/src/enclave_sol.rs | 3 +- .../ciphernode/evm/src/enclave_sol_reader.rs | 3 +- 6 files changed, 49 insertions(+), 12 deletions(-) diff --git a/packages/ciphernode/config/src/app_config.rs b/packages/ciphernode/config/src/app_config.rs index fb1c0576..0e9c0956 100644 --- a/packages/ciphernode/config/src/app_config.rs +++ b/packages/ciphernode/config/src/app_config.rs @@ -10,11 +10,39 @@ use std::{ path::{Path, PathBuf}, }; +#[derive(Debug, Deserialize, Serialize, PartialEq)] +#[serde(untagged)] +pub enum Contract { + Full { + address: String, + deploy_block: Option, + }, + AddressOnly(String), +} + +impl Contract { + pub fn address(&self) -> &String { + use Contract::*; + match self { + Full { address, .. } => address, + AddressOnly(v) => v, + } + } + + pub fn deploy_block(&self) -> Option { + use Contract::*; + match self { + Full { deploy_block, .. } => deploy_block.clone(), + AddressOnly(_) => None, + } + } +} + #[derive(Debug, Deserialize, Serialize)] pub struct ContractAddresses { - pub enclave: String, - pub ciphernode_registry: String, - pub filter_registry: String, + pub enclave: Contract, + pub ciphernode_registry: Contract, + pub filter_registry: Contract, } #[derive(Debug, Deserialize, Serialize)] diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index e4a5d216..e7839ec6 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -49,20 +49,23 @@ pub async fn setup_aggregator( let read_provider = create_readonly_provider(&ensure_ws_rpc(rpc_url)).await?; let write_provider = create_provider_with_signer(&ensure_http_rpc(rpc_url), &signer).await?; + EnclaveSol::attach( &bus, &read_provider, &write_provider, - &chain.contracts.enclave, + &chain.contracts.enclave.address(), &repositories.enclave_sol_reader(read_provider.get_chain_id()), + chain.contracts.enclave.deploy_block(), ) .await?; - RegistryFilterSol::attach(&bus, &write_provider, &chain.contracts.filter_registry).await?; + RegistryFilterSol::attach(&bus, &write_provider, &chain.contracts.filter_registry.address()).await?; CiphernodeRegistrySol::attach( &bus, &read_provider, - &chain.contracts.ciphernode_registry, + &chain.contracts.ciphernode_registry.address(), &repositories.ciphernode_registry_reader(read_provider.get_chain_id()), + chain.contracts.ciphernode_registry.deploy_block(), ) .await?; } diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 4ce4b91b..c7d58047 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -48,15 +48,17 @@ pub async fn setup_ciphernode( EnclaveSolReader::attach( &bus, &read_provider, - &chain.contracts.enclave, + &chain.contracts.enclave.address(), &repositories.enclave_sol_reader(read_provider.get_chain_id()), + chain.contracts.enclave.deploy_block(), ) .await?; CiphernodeRegistrySol::attach( &bus, &read_provider, - &chain.contracts.ciphernode_registry, + &chain.contracts.ciphernode_registry.address(), &repositories.ciphernode_registry_reader(read_provider.get_chain_id()), + chain.contracts.ciphernode_registry.deploy_block() ) .await?; } diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index 1bb5da3d..2d768f5f 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -105,12 +105,13 @@ impl CiphernodeRegistrySolReader { provider: &WithChainId, contract_address: &str, repository: &Repository, + start_block: Option ) -> Result>> { let addr = EvmEventReader::attach( provider, extractor, contract_address, - None, + start_block, &bus.clone().into(), repository, ) @@ -134,8 +135,9 @@ impl CiphernodeRegistrySol { provider: &WithChainId, contract_address: &str, repository: &Repository, + start_block: Option ) -> Result<()> { - CiphernodeRegistrySolReader::attach(bus, provider, contract_address, repository).await?; + CiphernodeRegistrySolReader::attach(bus, provider, contract_address, repository, start_block).await?; // TODO: Writer if needed Ok(()) } diff --git a/packages/ciphernode/evm/src/enclave_sol.rs b/packages/ciphernode/evm/src/enclave_sol.rs index e8747ae2..6c0b110b 100644 --- a/packages/ciphernode/evm/src/enclave_sol.rs +++ b/packages/ciphernode/evm/src/enclave_sol.rs @@ -17,8 +17,9 @@ impl EnclaveSol { write_provider: &WithChainId, contract_address: &str, repository: &Repository, + start_block: Option ) -> Result<()> { - EnclaveSolReader::attach(bus, read_provider, contract_address, repository).await?; + EnclaveSolReader::attach(bus, read_provider, contract_address, repository, start_block).await?; EnclaveSolWriter::attach(bus, write_provider, contract_address).await?; Ok(()) } diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index 6272c740..e59c316f 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -89,12 +89,13 @@ impl EnclaveSolReader { provider: &WithChainId, contract_address: &str, repository: &Repository, + start_block: Option ) -> Result>> { let addr = EvmEventReader::attach( provider, extractor, contract_address, - None, + start_block, &bus.clone(), repository, ) From 20201e6953211c7d05e096b74fdca4c41731e5d1 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 13:38:50 +0700 Subject: [PATCH 20/31] Formatting --- .../ciphernode/evm/src/ciphernode_registry_sol.rs | 13 ++++++++++--- packages/ciphernode/evm/src/enclave_sol.rs | 11 +++++++++-- packages/ciphernode/evm/src/enclave_sol_reader.rs | 2 +- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index 2d768f5f..f1417252 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -105,7 +105,7 @@ impl CiphernodeRegistrySolReader { provider: &WithChainId, contract_address: &str, repository: &Repository, - start_block: Option + start_block: Option, ) -> Result>> { let addr = EvmEventReader::attach( provider, @@ -135,9 +135,16 @@ impl CiphernodeRegistrySol { provider: &WithChainId, contract_address: &str, repository: &Repository, - start_block: Option + start_block: Option, ) -> Result<()> { - CiphernodeRegistrySolReader::attach(bus, provider, contract_address, repository, start_block).await?; + CiphernodeRegistrySolReader::attach( + bus, + provider, + contract_address, + repository, + start_block, + ) + .await?; // TODO: Writer if needed Ok(()) } diff --git a/packages/ciphernode/evm/src/enclave_sol.rs b/packages/ciphernode/evm/src/enclave_sol.rs index 6c0b110b..ed1f76bc 100644 --- a/packages/ciphernode/evm/src/enclave_sol.rs +++ b/packages/ciphernode/evm/src/enclave_sol.rs @@ -17,9 +17,16 @@ impl EnclaveSol { write_provider: &WithChainId, contract_address: &str, repository: &Repository, - start_block: Option + start_block: Option, ) -> Result<()> { - EnclaveSolReader::attach(bus, read_provider, contract_address, repository, start_block).await?; + EnclaveSolReader::attach( + bus, + read_provider, + contract_address, + repository, + start_block, + ) + .await?; EnclaveSolWriter::attach(bus, write_provider, contract_address).await?; Ok(()) } diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index e59c316f..c71c39de 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -89,7 +89,7 @@ impl EnclaveSolReader { provider: &WithChainId, contract_address: &str, repository: &Repository, - start_block: Option + start_block: Option, ) -> Result>> { let addr = EvmEventReader::attach( provider, From b4475bb95a5cde5d4656c0c3ec4441b0f73b3bc4 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 13:41:42 +0700 Subject: [PATCH 21/31] Remove unnecessary Actor --- packages/ciphernode/evm/src/ciphernode_registry_sol.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index f1417252..1c3acebf 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -98,7 +98,6 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, _: u64) -> Option, @@ -123,11 +122,7 @@ impl CiphernodeRegistrySolReader { } } -impl Actor for CiphernodeRegistrySolReader { - type Context = actix::Context; -} - -/// Eventual wrapper for both a reader and a writer +/// Wrapper for a reader and a future writer pub struct CiphernodeRegistrySol; impl CiphernodeRegistrySol { pub async fn attach( From b971db7dbb702c5fff446c6ac519270210b2677e Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 13:46:42 +0700 Subject: [PATCH 22/31] Fix not loading snapshot(must have been using deduplication) --- packages/ciphernode/enclave_node/src/aggregator.rs | 9 +++++++-- packages/ciphernode/enclave_node/src/ciphernode.rs | 2 +- packages/ciphernode/evm/src/event_reader.rs | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index e7839ec6..9c3dc67e 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -49,7 +49,7 @@ pub async fn setup_aggregator( let read_provider = create_readonly_provider(&ensure_ws_rpc(rpc_url)).await?; let write_provider = create_provider_with_signer(&ensure_http_rpc(rpc_url), &signer).await?; - + EnclaveSol::attach( &bus, &read_provider, @@ -59,7 +59,12 @@ pub async fn setup_aggregator( chain.contracts.enclave.deploy_block(), ) .await?; - RegistryFilterSol::attach(&bus, &write_provider, &chain.contracts.filter_registry.address()).await?; + RegistryFilterSol::attach( + &bus, + &write_provider, + &chain.contracts.filter_registry.address(), + ) + .await?; CiphernodeRegistrySol::attach( &bus, &read_provider, diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index c7d58047..7e2464e8 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -58,7 +58,7 @@ pub async fn setup_ciphernode( &read_provider, &chain.contracts.ciphernode_registry.address(), &repositories.ciphernode_registry_reader(read_provider.get_chain_id()), - chain.contracts.ciphernode_registry.deploy_block() + chain.contracts.ciphernode_registry.deploy_block(), ) .await?; } diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index fa92e2ad..ab73df83 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -74,7 +74,9 @@ where start_block: Option, /// Event bus for error propagation bus: Addr, + /// The in memory state of the event reader state: EvmEventReaderState, + /// Repository to save the state of the event reader repository: Repository, } @@ -122,7 +124,7 @@ where bus: bus.clone(), repository: repository.clone(), }; - let addr = EvmEventReader::new(params).start(); + let addr = EvmEventReader::load(params).await?.start(); bus.do_send(Subscribe::new("Shutdown", addr.clone().into())); From 91cd99d94c525e27f015d85979809833e7a42889 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 13:55:32 +0700 Subject: [PATCH 23/31] Add passing test for config --- packages/ciphernode/config/src/app_config.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/ciphernode/config/src/app_config.rs b/packages/ciphernode/config/src/app_config.rs index 0e9c0956..06b5f35a 100644 --- a/packages/ciphernode/config/src/app_config.rs +++ b/packages/ciphernode/config/src/app_config.rs @@ -325,7 +325,9 @@ chains: rpc_url: "ws://localhost:8545" contracts: enclave: "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0" - ciphernode_registry: "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" + ciphernode_registry: + address: "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" + deploy_block: 1764352873645 filter_registry: "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9" "#, )?; @@ -336,10 +338,15 @@ chains: assert_eq!(chain.name, "hardhat"); assert_eq!(chain.rpc_url, "ws://localhost:8545"); assert_eq!( - chain.contracts.enclave, + chain.contracts.enclave.address(), "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0" ); - + assert_eq!( + chain.contracts.ciphernode_registry.address(), + "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" + ); + assert_eq!(chain.contracts.enclave.deploy_block(), None); + assert_eq!(chain.contracts.ciphernode_registry.deploy_block(), Some(1764352873645)); Ok(()) }); } From fe8cbfa022ffa606202b155910f0a6dd7aba5886 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 5 Nov 2024 13:57:00 +0700 Subject: [PATCH 24/31] Formatting --- packages/ciphernode/config/src/app_config.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/ciphernode/config/src/app_config.rs b/packages/ciphernode/config/src/app_config.rs index 06b5f35a..dc02e724 100644 --- a/packages/ciphernode/config/src/app_config.rs +++ b/packages/ciphernode/config/src/app_config.rs @@ -346,7 +346,10 @@ chains: "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" ); assert_eq!(chain.contracts.enclave.deploy_block(), None); - assert_eq!(chain.contracts.ciphernode_registry.deploy_block(), Some(1764352873645)); + assert_eq!( + chain.contracts.ciphernode_registry.deploy_block(), + Some(1764352873645) + ); Ok(()) }); } From 11ef5caf065f526076866ce95b452aa42af74b49 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 6 Nov 2024 14:12:10 +0700 Subject: [PATCH 25/31] Rename parent -> processor --- packages/ciphernode/evm/src/event_reader.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index ab73df83..486aa29c 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -139,7 +139,7 @@ where { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { - let parent = ctx.address().recipient(); + let processor = ctx.address().recipient(); let bus = self.bus.clone(); let Some(provider) = self.provider.take() else { tracing::error!("Could not start event reader as provider has already been used."); @@ -159,7 +159,7 @@ where stream_from_evm( provider, &contract_address, - &parent, + &processor, extractor, shutdown, start_block, @@ -175,7 +175,7 @@ where async fn stream_from_evm, T: Transport + Clone>( provider: WithChainId, contract_address: &Address, - parent: &Recipient, + processor: &Recipient, extractor: fn(&LogData, Option<&B256>, u64) -> Option, mut shutdown: oneshot::Receiver<()>, start_block: Option, @@ -198,7 +198,7 @@ async fn stream_from_evm, T: Transport + Clone>( let block_number = log.block_number; if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { trace!("Processing historical log"); - parent.do_send(EnclaveEvmEvent::new(event, block_number)); + processor.do_send(EnclaveEvmEvent::new(event, block_number)); } } } @@ -225,7 +225,7 @@ async fn stream_from_evm, T: Transport + Clone>( continue; }; info!("Extracted log from evm sending now."); - parent.do_send(EnclaveEvmEvent::new(event, block_number)); + processor.do_send(EnclaveEvmEvent::new(event, block_number)); } None => break, // Stream ended From 4fcc027b1b6b6dd3b1c393b3f05344404220e96d Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 10 Nov 2024 12:39:29 +0700 Subject: [PATCH 26/31] Improve tracing transparency and fix config bug --- packages/ciphernode/core/src/events.rs | 28 ++++++------ .../enclave/src/commands/aggregator/mod.rs | 5 ++- .../enclave/src/commands/aggregator/start.rs | 7 +-- .../ciphernode/enclave/src/commands/mod.rs | 2 +- .../enclave/src/commands/password/mod.rs | 4 +- .../ciphernode/enclave/src/commands/start.rs | 7 ++- .../enclave/src/commands/wallet/mod.rs | 4 +- packages/ciphernode/enclave/src/main.rs | 44 ++++++++++++++----- .../ciphernode/enclave_node/src/aggregator.rs | 9 ++-- .../ciphernode/enclave_node/src/ciphernode.rs | 9 ++-- .../evm/src/ciphernode_registry_sol.rs | 5 +++ packages/ciphernode/evm/src/enclave_sol.rs | 2 + .../ciphernode/evm/src/enclave_sol_reader.rs | 2 + packages/ciphernode/evm/src/event_reader.rs | 41 ++++++++++++++--- packages/ciphernode/evm/tests/evm_reader.rs | 2 + packages/ciphernode/p2p/src/libp2p_router.rs | 19 ++++---- packages/ciphernode/p2p/src/p2p.rs | 6 ++- tests/basic_integration/lib/ag/config.yaml | 2 +- tests/basic_integration/lib/cn2/config.yaml | 1 + tests/basic_integration/test.sh | 7 +-- 20 files changed, 140 insertions(+), 66 deletions(-) diff --git a/packages/ciphernode/core/src/events.rs b/packages/ciphernode/core/src/events.rs index 07f633c4..276cd80c 100644 --- a/packages/ciphernode/core/src/events.rs +++ b/packages/ciphernode/core/src/events.rs @@ -56,7 +56,7 @@ impl TryFrom for U256 { pub struct EventId(pub [u8; 32]); impl EventId { - fn from(value: T) -> Self { + pub fn hash(value: T) -> Self { let mut hasher = Sha256::new(); let mut std_hasher = DefaultHasher::new(); value.hash(&mut std_hasher); @@ -224,7 +224,7 @@ pub trait FromError { impl From for EnclaveEvent { fn from(data: KeyshareCreated) -> Self { EnclaveEvent::KeyshareCreated { - id: EventId::from(data.clone()), + id: EventId::hash(data.clone()), data: data.clone(), } } @@ -233,7 +233,7 @@ impl From for EnclaveEvent { impl From for EnclaveEvent { fn from(data: E3Requested) -> Self { EnclaveEvent::E3Requested { - id: EventId::from(data.clone()), + id: EventId::hash(data.clone()), data: data.clone(), } } @@ -242,7 +242,7 @@ impl From for EnclaveEvent { impl From for EnclaveEvent { fn from(data: PublicKeyAggregated) -> Self { EnclaveEvent::PublicKeyAggregated { - id: EventId::from(data.clone()), + id: EventId::hash(data.clone()), data: data.clone(), } } @@ -251,7 +251,7 @@ impl From for EnclaveEvent { impl From for EnclaveEvent { fn from(data: CiphertextOutputPublished) -> Self { EnclaveEvent::CiphertextOutputPublished { - id: EventId::from(data.clone()), + id: EventId::hash(data.clone()), data: data.clone(), } } @@ -260,7 +260,7 @@ impl From for EnclaveEvent { impl From for EnclaveEvent { fn from(data: DecryptionshareCreated) -> Self { EnclaveEvent::DecryptionshareCreated { - id: EventId::from(data.clone()), + id: EventId::hash(data.clone()), data: data.clone(), } } @@ -269,7 +269,7 @@ impl From for EnclaveEvent { impl From for EnclaveEvent { fn from(data: PlaintextAggregated) -> Self { EnclaveEvent::PlaintextAggregated { - id: EventId::from(data.clone()), + id: EventId::hash(data.clone()), data: data.clone(), } } @@ -278,7 +278,7 @@ impl From for EnclaveEvent { impl From for EnclaveEvent { fn from(data: E3RequestComplete) -> Self { EnclaveEvent::E3RequestComplete { - id: EventId::from(data.clone()), + id: EventId::hash(data.clone()), data: data.clone(), } } @@ -287,7 +287,7 @@ impl From for EnclaveEvent { impl From for EnclaveEvent { fn from(data: CiphernodeSelected) -> Self { EnclaveEvent::CiphernodeSelected { - id: EventId::from(data.clone()), + id: EventId::hash(data.clone()), data: data.clone(), } } @@ -296,7 +296,7 @@ impl From for EnclaveEvent { impl From for EnclaveEvent { fn from(data: CiphernodeAdded) -> Self { EnclaveEvent::CiphernodeAdded { - id: EventId::from(data.clone()), + id: EventId::hash(data.clone()), data: data.clone(), } } @@ -305,7 +305,7 @@ impl From for EnclaveEvent { impl From for EnclaveEvent { fn from(data: CiphernodeRemoved) -> Self { EnclaveEvent::CiphernodeRemoved { - id: EventId::from(data.clone()), + id: EventId::hash(data.clone()), data: data.clone(), } } @@ -314,7 +314,7 @@ impl From for EnclaveEvent { impl From for EnclaveEvent { fn from(data: EnclaveError) -> Self { EnclaveEvent::EnclaveError { - id: EventId::from(data.clone()), + id: EventId::hash(data.clone()), data: data.clone(), } } @@ -323,7 +323,7 @@ impl From for EnclaveEvent { impl From for EnclaveEvent { fn from(data: Shutdown) -> Self { EnclaveEvent::Shutdown { - id: EventId::from(data.clone()), + id: EventId::hash(data.clone()), data: data.clone(), } } @@ -332,7 +332,7 @@ impl From for EnclaveEvent { impl From for EnclaveEvent { fn from(value: TestEvent) -> Self { EnclaveEvent::TestEvent { - id: EventId::from(value.clone()), + id: EventId::hash(value.clone()), data: value.clone(), } } diff --git a/packages/ciphernode/enclave/src/commands/aggregator/mod.rs b/packages/ciphernode/enclave/src/commands/aggregator/mod.rs index 58a12474..60e06fab 100644 --- a/packages/ciphernode/enclave/src/commands/aggregator/mod.rs +++ b/packages/ciphernode/enclave/src/commands/aggregator/mod.rs @@ -3,7 +3,7 @@ use anyhow::*; use clap::Subcommand; use config::AppConfig; -#[derive(Subcommand)] +#[derive(Subcommand, Debug)] pub enum AggregatorCommands { /// Start the application as an aggregator Start { @@ -17,7 +17,7 @@ pub enum AggregatorCommands { }, } -pub async fn execute(command: AggregatorCommands, config: AppConfig) -> Result<()> { +pub async fn execute(command: AggregatorCommands, config: AppConfig, id: &str) -> Result<()> { match command { AggregatorCommands::Start { pubkey_write_path, @@ -27,6 +27,7 @@ pub async fn execute(command: AggregatorCommands, config: AppConfig) -> Result<( config, pubkey_write_path.as_deref(), plaintext_write_path.as_deref(), + id ) .await? } diff --git a/packages/ciphernode/enclave/src/commands/aggregator/start.rs b/packages/ciphernode/enclave/src/commands/aggregator/start.rs index b4b130d9..5faa6420 100644 --- a/packages/ciphernode/enclave/src/commands/aggregator/start.rs +++ b/packages/ciphernode/enclave/src/commands/aggregator/start.rs @@ -9,13 +9,14 @@ pub async fn execute( config: AppConfig, pubkey_write_path: Option<&str>, plaintext_write_path: Option<&str>, + id: &str ) -> Result<()> { owo(); - info!("LAUNCHING AGGREGATOR"); - - let (bus, handle) = setup_aggregator(config, pubkey_write_path, plaintext_write_path).await?; + let (bus, handle, peer_id) = + setup_aggregator(config, pubkey_write_path, plaintext_write_path, id).await?; + info!("LAUNCHING AGGREGATOR {}", peer_id); tokio::spawn(listen_for_shutdown(bus.into(), handle)); std::future::pending::<()>().await; diff --git a/packages/ciphernode/enclave/src/commands/mod.rs b/packages/ciphernode/enclave/src/commands/mod.rs index aebde76a..b6ddb039 100644 --- a/packages/ciphernode/enclave/src/commands/mod.rs +++ b/packages/ciphernode/enclave/src/commands/mod.rs @@ -8,7 +8,7 @@ use aggregator::AggregatorCommands; use clap::Subcommand; use wallet::WalletCommands; -#[derive(Subcommand)] +#[derive(Subcommand, Debug)] pub enum Commands { /// Start the application Start, diff --git a/packages/ciphernode/enclave/src/commands/password/mod.rs b/packages/ciphernode/enclave/src/commands/password/mod.rs index e9264881..6a02ef9e 100644 --- a/packages/ciphernode/enclave/src/commands/password/mod.rs +++ b/packages/ciphernode/enclave/src/commands/password/mod.rs @@ -5,7 +5,7 @@ use anyhow::*; use clap::Subcommand; use config::AppConfig; -#[derive(Subcommand)] +#[derive(Subcommand, Debug)] pub enum PasswordCommands { /// Create a new password Create { @@ -25,7 +25,7 @@ pub enum PasswordCommands { }, } -pub async fn execute(command: PasswordCommands, config: AppConfig) -> Result<()> { +pub async fn execute(command: PasswordCommands, config: AppConfig, id: &str) -> Result<()> { match command { PasswordCommands::Create { password } => create::execute(&config, password).await?, PasswordCommands::Delete => delete::execute(&config).await?, diff --git a/packages/ciphernode/enclave/src/commands/start.rs b/packages/ciphernode/enclave/src/commands/start.rs index 14df1c64..5429da01 100644 --- a/packages/ciphernode/enclave/src/commands/start.rs +++ b/packages/ciphernode/enclave/src/commands/start.rs @@ -5,7 +5,7 @@ use tracing::info; use crate::owo; -pub async fn execute(config: AppConfig) -> Result<()> { +pub async fn execute(config: AppConfig, id: &str) -> Result<()> { owo(); // let address = Address::parse_checksummed(&config.address(), None).context("Invalid address")?; @@ -13,9 +13,8 @@ pub async fn execute(config: AppConfig) -> Result<()> { return Err(anyhow!("You must provide an address")); }; - info!("LAUNCHING CIPHERNODE: ({})", address); - - let (bus, handle) = setup_ciphernode(config, address).await?; + let (bus, handle, peer_id) = setup_ciphernode(config, address, id).await?; + info!("LAUNCHING CIPHERNODE: ({}/{})", address, peer_id); tokio::spawn(listen_for_shutdown(bus.into(), handle)); diff --git a/packages/ciphernode/enclave/src/commands/wallet/mod.rs b/packages/ciphernode/enclave/src/commands/wallet/mod.rs index b07b1cc6..702a0770 100644 --- a/packages/ciphernode/enclave/src/commands/wallet/mod.rs +++ b/packages/ciphernode/enclave/src/commands/wallet/mod.rs @@ -3,7 +3,7 @@ use anyhow::*; use clap::Subcommand; use config::AppConfig; -#[derive(Subcommand)] +#[derive(Subcommand, Debug)] pub enum WalletCommands { /// Set a new Wallet Private Key Set { @@ -22,7 +22,7 @@ fn ensure_hex(s: &str) -> Result { Ok(s.to_string()) } -pub async fn execute(command: WalletCommands, config: AppConfig) -> Result<()> { +pub async fn execute(command: WalletCommands, config: AppConfig, id: &str) -> Result<()> { match command { WalletCommands::Set { private_key } => set::execute(&config, private_key).await?, }; diff --git a/packages/ciphernode/enclave/src/main.rs b/packages/ciphernode/enclave/src/main.rs index b8ec1978..b651c639 100644 --- a/packages/ciphernode/enclave/src/main.rs +++ b/packages/ciphernode/enclave/src/main.rs @@ -1,7 +1,11 @@ +use std::env; + use anyhow::Result; use clap::Parser; use commands::{aggregator, password, start, wallet, Commands}; use config::load_config; +use tracing::{instrument::WithSubscriber, span, Instrument, Level}; +use tracing_subscriber::EnvFilter; pub mod commands; const OWO: &str = r#" @@ -24,7 +28,7 @@ pub fn owo() { println!("\n\n\n\n"); } -#[derive(Parser)] +#[derive(Parser, Debug)] #[command(name = "enclave")] #[command(about = "A CLI for interacting with Enclave the open-source protocol for Encrypted Execution Environments (E3)", long_about = None)] pub struct Cli { @@ -34,32 +38,52 @@ pub struct Cli { #[command(subcommand)] command: Commands, + + #[arg(short, long, global = true)] + tag: Option, } impl Cli { pub async fn execute(self) -> Result<()> { let config_path = self.config.as_deref(); - + let id = self.get_tag(); let config = load_config(config_path)?; match self.command { - Commands::Start => start::execute(config).await?, - Commands::Password { command } => password::execute(command, config).await?, - Commands::Aggregator { command } => aggregator::execute(command, config).await?, - Commands::Wallet { command } => wallet::execute(command, config).await?, + Commands::Start => start::execute(config, &id).await?, + Commands::Password { command } => password::execute(command, config, &id).await?, + Commands::Aggregator { command } => aggregator::execute(command, config, &id).await?, + Commands::Wallet { command } => wallet::execute(command, config, &id).await?, } Ok(()) } + + pub fn get_tag(&self) -> String { + if let Some(tag) = self.tag.clone() { + tag + } else { + "default".to_string() + } + } } #[actix::main] pub async fn main() { - tracing_subscriber::fmt::init(); - + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + // .with_env_filter("error") + // .with_env_filter("[app{id=cn1}]=info") + // .with_env_filter("[app{id=cn2}]=info,libp2p_mdns::behaviour=error") + // .with_env_filter("[app{id=cn3}]=info") + // .with_env_filter("[app{id=cn4}]=info") + // .with_env_filter("[app{id=ag}]=info") + .init(); let cli = Cli::parse(); - - match cli.execute().await { + let id = cli.get_tag(); + let span = span!(Level::INFO, "app", %id); + let _guard = span.enter(); + match cli.execute().instrument(span.clone()).await { Ok(_) => (), Err(err) => { eprintln!("{}", err); diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 9c3dc67e..7efb0673 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -29,7 +29,8 @@ pub async fn setup_aggregator( config: AppConfig, pubkey_write_path: Option<&str>, plaintext_write_path: Option<&str>, -) -> Result<(Addr, JoinHandle<()>)> { + tag: &str +) -> Result<(Addr, JoinHandle<()>, String)> { let bus = EventBus::new(true).start(); let rng = Arc::new(Mutex::new( ChaCha20Rng::from_rng(OsRng).expect("Failed to create RNG"), @@ -57,6 +58,7 @@ pub async fn setup_aggregator( &chain.contracts.enclave.address(), &repositories.enclave_sol_reader(read_provider.get_chain_id()), chain.contracts.enclave.deploy_block(), + tag ) .await?; RegistryFilterSol::attach( @@ -71,6 +73,7 @@ pub async fn setup_aggregator( &chain.contracts.ciphernode_registry.address(), &repositories.ciphernode_registry_reader(read_provider.get_chain_id()), chain.contracts.ciphernode_registry.deploy_block(), + tag ) .await?; } @@ -82,7 +85,7 @@ pub async fn setup_aggregator( .build() .await?; - let (_, join_handle) = P2p::spawn_libp2p(bus.clone()).expect("Failed to setup libp2p"); + let (_, join_handle, peer_id) = P2p::spawn_libp2p(bus.clone()).expect("Failed to setup libp2p"); if let Some(path) = pubkey_write_path { PublicKeyWriter::attach(path, bus.clone()); @@ -94,5 +97,5 @@ pub async fn setup_aggregator( SimpleLogger::attach("AGG", bus.clone()); - Ok((bus, join_handle)) + Ok((bus, join_handle, peer_id)) } diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 7e2464e8..0fabd588 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -24,7 +24,8 @@ use crate::setup_datastore; pub async fn setup_ciphernode( config: AppConfig, address: Address, -) -> Result<(Addr, JoinHandle<()>)> { + tag: &str +) -> Result<(Addr, JoinHandle<()>, String)> { let rng = Arc::new(Mutex::new( rand_chacha::ChaCha20Rng::from_rng(OsRng).expect("Failed to create RNG"), )); @@ -51,6 +52,7 @@ pub async fn setup_ciphernode( &chain.contracts.enclave.address(), &repositories.enclave_sol_reader(read_provider.get_chain_id()), chain.contracts.enclave.deploy_block(), + tag ) .await?; CiphernodeRegistrySol::attach( @@ -59,6 +61,7 @@ pub async fn setup_ciphernode( &chain.contracts.ciphernode_registry.address(), &repositories.ciphernode_registry_reader(read_provider.get_chain_id()), chain.contracts.ciphernode_registry.deploy_block(), + tag ) .await?; } @@ -69,10 +72,10 @@ pub async fn setup_ciphernode( .build() .await?; - let (_, join_handle) = P2p::spawn_libp2p(bus.clone()).expect("Failed to setup libp2p"); + let (_, join_handle, peer_id) = P2p::spawn_libp2p(bus.clone()).expect("Failed to setup libp2p"); let nm = format!("CIPHER({})", &address.to_string()[0..5]); SimpleLogger::attach(&nm, bus.clone()); - Ok((bus, join_handle)) + Ok((bus, join_handle, peer_id)) } diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index 1c3acebf..656697ed 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -16,6 +16,7 @@ use tracing::{error, info, trace}; sol!( #[sol(rpc)] + #[derive(Debug)] ICiphernodeRegistry, "../../evm/artifacts/contracts/interfaces/ICiphernodeRegistry.sol/ICiphernodeRegistry.json" ); @@ -105,6 +106,7 @@ impl CiphernodeRegistrySolReader { contract_address: &str, repository: &Repository, start_block: Option, + tag: &str ) -> Result>> { let addr = EvmEventReader::attach( provider, @@ -113,6 +115,7 @@ impl CiphernodeRegistrySolReader { start_block, &bus.clone().into(), repository, + tag ) .await?; @@ -131,6 +134,7 @@ impl CiphernodeRegistrySol { contract_address: &str, repository: &Repository, start_block: Option, + tag: &str ) -> Result<()> { CiphernodeRegistrySolReader::attach( bus, @@ -138,6 +142,7 @@ impl CiphernodeRegistrySol { contract_address, repository, start_block, + tag ) .await?; // TODO: Writer if needed diff --git a/packages/ciphernode/evm/src/enclave_sol.rs b/packages/ciphernode/evm/src/enclave_sol.rs index ed1f76bc..763d76b0 100644 --- a/packages/ciphernode/evm/src/enclave_sol.rs +++ b/packages/ciphernode/evm/src/enclave_sol.rs @@ -18,6 +18,7 @@ impl EnclaveSol { contract_address: &str, repository: &Repository, start_block: Option, + tag: &str ) -> Result<()> { EnclaveSolReader::attach( bus, @@ -25,6 +26,7 @@ impl EnclaveSol { contract_address, repository, start_block, + tag ) .await?; EnclaveSolWriter::attach(bus, write_provider, contract_address).await?; diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index c71c39de..3fcebed2 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -90,6 +90,7 @@ impl EnclaveSolReader { contract_address: &str, repository: &Repository, start_block: Option, + tag: &str ) -> Result>> { let addr = EvmEventReader::attach( provider, @@ -98,6 +99,7 @@ impl EnclaveSolReader { start_block, &bus.clone(), repository, + tag ) .await?; diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index 486aa29c..a544bcf0 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -15,7 +15,7 @@ use futures_util::stream::StreamExt; use std::collections::HashSet; use tokio::select; use tokio::sync::oneshot; -use tracing::{info, trace, warn}; +use tracing::{error, info, info_span, instrument, trace, warn}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] #[rtype(result = "()")] @@ -25,9 +25,13 @@ pub struct EnclaveEvmEvent { } impl EnclaveEvmEvent { - fn new(event: EnclaveEvent, block: Option) -> Self { + pub fn new(event: EnclaveEvent, block: Option) -> Self { Self { event, block } } + + pub fn get_id(&self) -> EventId { + EventId::hash(self.clone()) + } } pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; @@ -45,6 +49,7 @@ where start_block: Option, bus: Addr, repository: Repository, + tag: String, } #[derive(Default, serde::Serialize, serde::Deserialize, Clone)] @@ -78,6 +83,8 @@ where state: EvmEventReaderState, /// Repository to save the state of the event reader repository: Repository, + /// An identifier for logs + tag: String, } impl EvmEventReader @@ -97,13 +104,19 @@ where bus: params.bus, state: EvmEventReaderState::default(), repository: params.repository, + tag: params.tag, } } pub async fn load(params: EvmEventReaderParams) -> Result { + let id = params.tag.clone(); + let span = info_span!("evm_event_reader", %id); + let _guard = span.enter(); Ok(if let Some(snapshot) = params.repository.read().await? { + info!("Loading from snapshot"); Self::from_snapshot(params, snapshot).await? } else { + info!("Loading from params"); Self::new(params) }) } @@ -115,6 +128,7 @@ where start_block: Option, bus: &Addr, repository: &Repository, + tag: &str, ) -> Result> { let params = EvmEventReaderParams { provider: provider.clone(), @@ -123,6 +137,7 @@ where start_block, bus: bus.clone(), repository: repository.clone(), + tag: tag.to_string(), }; let addr = EvmEventReader::load(params).await?.start(); @@ -154,6 +169,7 @@ where let contract_address = self.contract_address; let start_block = self.start_block; + let tag = self.tag.clone(); ctx.spawn( async move { stream_from_evm( @@ -164,6 +180,7 @@ where shutdown, start_block, &bus, + &tag, ) .await } @@ -172,6 +189,7 @@ where } } +#[instrument(name = "evm_event_reader", skip_all, fields(id=id))] async fn stream_from_evm, T: Transport + Clone>( provider: WithChainId, contract_address: &Address, @@ -180,9 +198,11 @@ async fn stream_from_evm, T: Transport + Clone>( mut shutdown: oneshot::Receiver<()>, start_block: Option, bus: &Addr, + id: &str, ) { let chain_id = provider.get_chain_id(); let provider = provider.get_provider(); + let historical_filter = Filter::new() .address(contract_address.clone()) .from_block(start_block.unwrap_or(0)); @@ -209,6 +229,7 @@ async fn stream_from_evm, T: Transport + Clone>( } } + info!("subscribing to live events"); match provider.subscribe_logs(¤t_filter).await { Ok(subscription) => { let mut stream = subscription.into_stream(); @@ -221,10 +242,10 @@ async fn stream_from_evm, T: Transport + Clone>( trace!("Received log from EVM"); let Some(event) = extractor(log.data(), log.topic0(), chain_id) else { - trace!("Failed to extract log from EVM"); + warn!("Failed to extract log from EVM."); continue; }; - info!("Extracted log from evm sending now."); + info!("Extracted Evm Event: {}", event); processor.do_send(EnclaveEvmEvent::new(event, block_number)); } @@ -267,19 +288,26 @@ where { type Result = (); fn handle(&mut self, wrapped: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { - let event_id = wrapped.event.get_id(); + let id = self.tag.clone(); + let span = info_span!("evm_event_reader", %id); + let _guard = span.enter(); + let event_id = wrapped.get_id(); + info!("Processing event: {}", event_id); + info!("cache length: {}", self.state.ids.len()); if self.state.ids.contains(&event_id) { - trace!( + error!( "Event id {} has already been seen and was not forwarded to the bus", &event_id ); return; } + let event_type = wrapped.event.event_type(); // Forward everything else to the event bus self.bus.do_send(wrapped.event); // Save processed ids + info!("Storing event(EVM) in cache {}({})", event_type, event_id); self.state.ids.insert(event_id); self.state.last_block = wrapped.block; self.checkpoint(); @@ -326,6 +354,7 @@ where bus: params.bus, state: snapshot, repository: params.repository, + tag: params.tag, }) } } diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index 0fdacf07..a826e6cb 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -61,6 +61,7 @@ async fn evm_reader() -> Result<()> { None, &bus, &repository, + "mytag" ) .await?; @@ -131,6 +132,7 @@ async fn ensure_historical_events() -> Result<()> { None, &bus, &repository, + "mytag" ) .await?; diff --git a/packages/ciphernode/p2p/src/libp2p_router.rs b/packages/ciphernode/p2p/src/libp2p_router.rs index 9ed1a8c8..d270402f 100644 --- a/packages/ciphernode/p2p/src/libp2p_router.rs +++ b/packages/ciphernode/p2p/src/libp2p_router.rs @@ -9,7 +9,6 @@ use std::time::Duration; use tokio::sync::mpsc::{channel, Receiver, Sender}; use tokio::{io, select}; use tracing::{error, info, trace}; -use tracing_subscriber::EnvFilter; #[derive(NetworkBehaviour)] pub struct MyBehaviour { @@ -58,19 +57,22 @@ impl EnclaveRouter { )) } - pub fn with_identiy(&mut self, keypair: identity::Keypair) { - self.identity = Some(keypair); + pub fn with_identity(&mut self, keypair: &identity::Keypair) { + self.identity = Some(keypair.clone()); } pub fn connect_swarm(&mut self, discovery_type: String) -> Result<&Self, Box> { match discovery_type.as_str() { "mdns" => { - let _ = tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .try_init(); - // TODO: Use key if assigned already - let mut swarm = libp2p::SwarmBuilder::with_new_identity() + + let swarm = self + .identity + .clone() + .map_or_else( + || libp2p::SwarmBuilder::with_new_identity(), + |id| libp2p::SwarmBuilder::with_existing_identity(id), + ) .with_tokio() .with_tcp( tcp::Config::default(), @@ -92,6 +94,7 @@ impl EnclaveRouter { })? .with_swarm_config(|c| c.with_idle_connection_timeout(Duration::from_secs(60))) .build(); + self.swarm = Some(swarm); } _ => info!("Defaulting to MDNS discovery"), diff --git a/packages/ciphernode/p2p/src/p2p.rs b/packages/ciphernode/p2p/src/p2p.rs index 635ad8e5..53bbd559 100644 --- a/packages/ciphernode/p2p/src/p2p.rs +++ b/packages/ciphernode/p2p/src/p2p.rs @@ -65,14 +65,16 @@ impl P2p { /// Spawn a Libp2p instance. Calls spawn and listen pub fn spawn_libp2p( bus: Addr, - ) -> Result<(Addr, tokio::task::JoinHandle<()>), Box> { + ) -> Result<(Addr, tokio::task::JoinHandle<()>, String), Box> { let (mut libp2p, tx, rx) = EnclaveRouter::new()?; + let keypair = libp2p::identity::Keypair::generate_ed25519(); + libp2p.with_identity(&keypair); libp2p.connect_swarm("mdns".to_string())?; libp2p.join_topic("enclave-keygen-01")?; let p2p_addr = Self::spawn_and_listen(bus, tx, rx); let handle = tokio::spawn(async move { libp2p.start().await.unwrap() }); - Ok((p2p_addr, handle)) + Ok((p2p_addr, handle, keypair.public().to_peer_id().to_string())) } } diff --git a/tests/basic_integration/lib/ag/config.yaml b/tests/basic_integration/lib/ag/config.yaml index 55bdb526..68b919f1 100644 --- a/tests/basic_integration/lib/ag/config.yaml +++ b/tests/basic_integration/lib/ag/config.yaml @@ -1,6 +1,6 @@ config_dir: . data_dir: . -address: "0x8626f6940E2eb28930eFb4CeF49B2d1F2C9C1199" +address: "0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199" chains: - name: "hardhat" rpc_url: "ws://localhost:8545" diff --git a/tests/basic_integration/lib/cn2/config.yaml b/tests/basic_integration/lib/cn2/config.yaml index 20d5183f..5cbce324 100644 --- a/tests/basic_integration/lib/cn2/config.yaml +++ b/tests/basic_integration/lib/cn2/config.yaml @@ -1,4 +1,5 @@ config_dir: . +data_dir: . address: "0xbDA5747bFD65F08deb54cb465eB87D40e51B197E" chains: - name: "hardhat" diff --git a/tests/basic_integration/test.sh b/tests/basic_integration/test.sh index 1fb9e3f5..f793f5ed 100755 --- a/tests/basic_integration/test.sh +++ b/tests/basic_integration/test.sh @@ -91,6 +91,7 @@ launch_ciphernode() { local name="$1" heading "Launch ciphernode $name" yarn enclave start \ + --tag "$name" \ --config "$SCRIPT_DIR/lib/$name/config.yaml" & } @@ -108,13 +109,12 @@ launch_aggregator() { heading "Launch aggregator $name" yarn enclave aggregator start \ + --tag "$name" \ --config "$SCRIPT_DIR/lib/$name/config.yaml" \ --pubkey-write-path "$SCRIPT_DIR/output/pubkey.bin" \ --plaintext-write-path "$SCRIPT_DIR/output/plaintext.txt" & } - - pkill -9 -f "target/debug/enclave" || true pkill -9 -f "hardhat node" || true @@ -139,7 +139,6 @@ set_password cn2 "$CIPHERNODE_SECRET" set_password cn3 "$CIPHERNODE_SECRET" set_password cn4 "$CIPHERNODE_SECRET" set_password ag "$CIPHERNODE_SECRET" - set_private_key ag "$PRIVATE_KEY" # Launch 4 ciphernodes @@ -149,8 +148,6 @@ launch_ciphernode cn3 launch_ciphernode cn4 launch_aggregator ag -sleep 1 - waiton-files "$ROOT_DIR/packages/ciphernode/target/debug/enclave" "$ROOT_DIR/packages/ciphernode/target/debug/fake_encrypt" heading "Add ciphernode $CIPHERNODE_ADDRESS_1" From 93e6bfab6b0f62246201d32169b2da410277c4fe Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 10 Nov 2024 18:49:44 +0700 Subject: [PATCH 27/31] Centralize logging tag/id --- packages/ciphernode/Cargo.lock | 1 + packages/ciphernode/Cargo.toml | 1 + packages/ciphernode/core/Cargo.toml | 1 + packages/ciphernode/core/src/lib.rs | 2 + packages/ciphernode/core/src/tag.rs | 22 ++ packages/ciphernode/data/src/data_store.rs | 2 +- packages/ciphernode/data/src/repository.rs | 1 + packages/ciphernode/data/src/sled_store.rs | 63 +++-- .../enclave/src/commands/aggregator/mod.rs | 3 +- .../enclave/src/commands/aggregator/start.rs | 3 +- .../enclave/src/commands/password/mod.rs | 2 +- .../ciphernode/enclave/src/commands/start.rs | 4 +- .../enclave/src/commands/wallet/mod.rs | 2 +- packages/ciphernode/enclave/src/main.rs | 12 +- .../ciphernode/enclave_node/src/aggregator.rs | 5 +- .../ciphernode/enclave_node/src/ciphernode.rs | 5 +- .../ciphernode/enclave_node/src/datastore.rs | 2 +- .../evm/src/ciphernode_registry_sol.rs | 4 - packages/ciphernode/evm/src/enclave_sol.rs | 2 - .../ciphernode/evm/src/enclave_sol_reader.rs | 2 - packages/ciphernode/evm/src/event_reader.rs | 17 +- packages/ciphernode/evm/tests/evm_reader.rs | 2 - .../ciphernode/sortition/src/sortition.rs | 37 ++- .../tests/test_aggregation_and_decryption.rs | 2 +- tests/basic_integration/base.sh | 110 ++++++++ tests/basic_integration/fns.sh | 138 ++++++++++ tests/basic_integration/lib/clean_folders.sh | 1 + tests/basic_integration/persist.sh | 121 +++++++++ tests/basic_integration/test.sh | 237 ++---------------- 29 files changed, 512 insertions(+), 292 deletions(-) create mode 100644 packages/ciphernode/core/src/tag.rs create mode 100755 tests/basic_integration/base.sh create mode 100644 tests/basic_integration/fns.sh create mode 100755 tests/basic_integration/persist.sh diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 45f2bde6..ff09b313 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -2231,6 +2231,7 @@ dependencies = [ "bincode", "bs58", "futures-util", + "lazy_static", "serde", "sha2", ] diff --git a/packages/ciphernode/Cargo.toml b/packages/ciphernode/Cargo.toml index 1fcc7534..2765678e 100644 --- a/packages/ciphernode/Cargo.toml +++ b/packages/ciphernode/Cargo.toml @@ -44,6 +44,7 @@ fhe-util = { git = "https://github.com/gnosisguild/fhe.rs", version = "0.1.0-bet futures = "0.3.30" futures-util = "0.3" hex = "0.4.3" +lazy_static = "1.5.0" num = "0.4.3" rand_chacha = "0.3.1" rand = "0.8.5" diff --git a/packages/ciphernode/core/Cargo.toml b/packages/ciphernode/core/Cargo.toml index ede42cc4..b77f4045 100644 --- a/packages/ciphernode/core/Cargo.toml +++ b/packages/ciphernode/core/Cargo.toml @@ -19,3 +19,4 @@ alloy = { workspace = true } alloy-primitives = { workspace = true } alloy-sol-types = { workspace = true } anyhow = { workspace = true } +lazy_static = { workspace = true } diff --git a/packages/ciphernode/core/src/lib.rs b/packages/ciphernode/core/src/lib.rs index 154d1461..0084e902 100644 --- a/packages/ciphernode/core/src/lib.rs +++ b/packages/ciphernode/core/src/lib.rs @@ -5,7 +5,9 @@ mod eventbus; mod events; mod ordered_set; +mod tag; pub use eventbus::*; pub use events::*; pub use ordered_set::*; +pub use tag::*; diff --git a/packages/ciphernode/core/src/tag.rs b/packages/ciphernode/core/src/tag.rs new file mode 100644 index 00000000..23bd2b81 --- /dev/null +++ b/packages/ciphernode/core/src/tag.rs @@ -0,0 +1,22 @@ +use std::sync::RwLock; + +use lazy_static::lazy_static; + +lazy_static! { + static ref TAG: RwLock = RwLock::new(String::from("_")); +} + +pub fn get_tag() -> String { + let tag_guard = TAG.read().expect("Failed to acquire read lock"); + tag_guard.clone() +} + +pub fn set_tag(new_tag: impl Into) -> Result<(), &'static str> { + match TAG.write() { + Ok(mut tag_guard) => { + *tag_guard = new_tag.into(); + Ok(()) + } + Err(_) => Err("Failed to acquire write lock"), + } +} diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index e5abfcbd..f4786f82 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -37,7 +37,7 @@ impl Get { } /// Generate proxy for the DB -#[derive(Clone)] +#[derive(Clone,Debug)] pub struct DataStore { scope: Vec, get: Recipient, diff --git a/packages/ciphernode/data/src/repository.rs b/packages/ciphernode/data/src/repository.rs index 7cf1e445..b44f2b07 100644 --- a/packages/ciphernode/data/src/repository.rs +++ b/packages/ciphernode/data/src/repository.rs @@ -4,6 +4,7 @@ use anyhow::Result; use crate::DataStore; +#[derive(Debug)] pub struct Repository { store: DataStore, _p: PhantomData, diff --git a/packages/ciphernode/data/src/sled_store.rs b/packages/ciphernode/data/src/sled_store.rs index 41de208f..362cb74a 100644 --- a/packages/ciphernode/data/src/sled_store.rs +++ b/packages/ciphernode/data/src/sled_store.rs @@ -1,13 +1,14 @@ -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use crate::{Get, Insert}; -use actix::{Actor, Addr, Handler}; +use actix::{Actor, ActorContext, Addr, Handler}; use anyhow::{Context, Result}; -use enclave_core::{BusError, EnclaveErrorType, EventBus}; +use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus, Subscribe}; use sled::Db; +use tracing::{error, info}; pub struct SledStore { - db: SledDb, + db: Option, bus: Addr, } @@ -16,17 +17,24 @@ impl Actor for SledStore { } impl SledStore { - pub fn new(bus: &Addr, path: &PathBuf) -> Result { + pub fn new(bus: &Addr, path: &PathBuf) -> Result> { + info!("Starting SledStore"); let db = SledDb::new(path)?; - Ok(Self { - db, + + let store = Self { + db: Some(db), bus: bus.clone(), - }) + } + .start(); + + bus.do_send(Subscribe::new("Shutdown", store.clone().into())); + + Ok(store) } pub fn from_db(db: SledDb) -> Result { Ok(Self { - db, + db: Some(db), bus: EventBus::new(false).start(), }) } @@ -36,9 +44,11 @@ impl Handler for SledStore { type Result = (); fn handle(&mut self, event: Insert, _: &mut Self::Context) -> Self::Result { - match self.db.insert(event) { - Err(err) => self.bus.err(EnclaveErrorType::Data, err), - _ => (), + if let Some(ref mut db) = &mut self.db { + match db.insert(event) { + Err(err) => self.bus.err(EnclaveErrorType::Data, err), + _ => (), + } } } } @@ -47,13 +57,28 @@ impl Handler for SledStore { type Result = Option>; fn handle(&mut self, event: Get, _: &mut Self::Context) -> Self::Result { - return match self.db.get(event) { - Ok(v) => v, - Err(err) => { - self.bus.err(EnclaveErrorType::Data, err); - None - } - }; + if let Some(ref mut db) = &mut self.db { + return match db.get(event) { + Ok(v) => v, + Err(err) => { + self.bus.err(EnclaveErrorType::Data, err); + None + } + }; + } else { + error!("Attempt to get data from dropped db"); + None + } + } +} + +impl Handler for SledStore { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + if let EnclaveEvent::Shutdown { .. } = msg { + let _db = self.db.take(); // db will be dropped + ctx.stop() + } } } diff --git a/packages/ciphernode/enclave/src/commands/aggregator/mod.rs b/packages/ciphernode/enclave/src/commands/aggregator/mod.rs index 60e06fab..d52c5f64 100644 --- a/packages/ciphernode/enclave/src/commands/aggregator/mod.rs +++ b/packages/ciphernode/enclave/src/commands/aggregator/mod.rs @@ -17,7 +17,7 @@ pub enum AggregatorCommands { }, } -pub async fn execute(command: AggregatorCommands, config: AppConfig, id: &str) -> Result<()> { +pub async fn execute(command: AggregatorCommands, config: AppConfig) -> Result<()> { match command { AggregatorCommands::Start { pubkey_write_path, @@ -27,7 +27,6 @@ pub async fn execute(command: AggregatorCommands, config: AppConfig, id: &str) - config, pubkey_write_path.as_deref(), plaintext_write_path.as_deref(), - id ) .await? } diff --git a/packages/ciphernode/enclave/src/commands/aggregator/start.rs b/packages/ciphernode/enclave/src/commands/aggregator/start.rs index 5faa6420..2194030d 100644 --- a/packages/ciphernode/enclave/src/commands/aggregator/start.rs +++ b/packages/ciphernode/enclave/src/commands/aggregator/start.rs @@ -9,12 +9,11 @@ pub async fn execute( config: AppConfig, pubkey_write_path: Option<&str>, plaintext_write_path: Option<&str>, - id: &str ) -> Result<()> { owo(); let (bus, handle, peer_id) = - setup_aggregator(config, pubkey_write_path, plaintext_write_path, id).await?; + setup_aggregator(config, pubkey_write_path, plaintext_write_path).await?; info!("LAUNCHING AGGREGATOR {}", peer_id); tokio::spawn(listen_for_shutdown(bus.into(), handle)); diff --git a/packages/ciphernode/enclave/src/commands/password/mod.rs b/packages/ciphernode/enclave/src/commands/password/mod.rs index 6a02ef9e..8067bc1e 100644 --- a/packages/ciphernode/enclave/src/commands/password/mod.rs +++ b/packages/ciphernode/enclave/src/commands/password/mod.rs @@ -25,7 +25,7 @@ pub enum PasswordCommands { }, } -pub async fn execute(command: PasswordCommands, config: AppConfig, id: &str) -> Result<()> { +pub async fn execute(command: PasswordCommands, config: AppConfig) -> Result<()> { match command { PasswordCommands::Create { password } => create::execute(&config, password).await?, PasswordCommands::Delete => delete::execute(&config).await?, diff --git a/packages/ciphernode/enclave/src/commands/start.rs b/packages/ciphernode/enclave/src/commands/start.rs index 5429da01..c46f9c5d 100644 --- a/packages/ciphernode/enclave/src/commands/start.rs +++ b/packages/ciphernode/enclave/src/commands/start.rs @@ -5,7 +5,7 @@ use tracing::info; use crate::owo; -pub async fn execute(config: AppConfig, id: &str) -> Result<()> { +pub async fn execute(config: AppConfig) -> Result<()> { owo(); // let address = Address::parse_checksummed(&config.address(), None).context("Invalid address")?; @@ -13,7 +13,7 @@ pub async fn execute(config: AppConfig, id: &str) -> Result<()> { return Err(anyhow!("You must provide an address")); }; - let (bus, handle, peer_id) = setup_ciphernode(config, address, id).await?; + let (bus, handle, peer_id) = setup_ciphernode(config, address).await?; info!("LAUNCHING CIPHERNODE: ({}/{})", address, peer_id); tokio::spawn(listen_for_shutdown(bus.into(), handle)); diff --git a/packages/ciphernode/enclave/src/commands/wallet/mod.rs b/packages/ciphernode/enclave/src/commands/wallet/mod.rs index 702a0770..43130405 100644 --- a/packages/ciphernode/enclave/src/commands/wallet/mod.rs +++ b/packages/ciphernode/enclave/src/commands/wallet/mod.rs @@ -22,7 +22,7 @@ fn ensure_hex(s: &str) -> Result { Ok(s.to_string()) } -pub async fn execute(command: WalletCommands, config: AppConfig, id: &str) -> Result<()> { +pub async fn execute(command: WalletCommands, config: AppConfig) -> Result<()> { match command { WalletCommands::Set { private_key } => set::execute(&config, private_key).await?, }; diff --git a/packages/ciphernode/enclave/src/main.rs b/packages/ciphernode/enclave/src/main.rs index b651c639..49d71184 100644 --- a/packages/ciphernode/enclave/src/main.rs +++ b/packages/ciphernode/enclave/src/main.rs @@ -4,6 +4,7 @@ use anyhow::Result; use clap::Parser; use commands::{aggregator, password, start, wallet, Commands}; use config::load_config; +use enclave_core::{get_tag, set_tag}; use tracing::{instrument::WithSubscriber, span, Instrument, Level}; use tracing_subscriber::EnvFilter; pub mod commands; @@ -50,10 +51,10 @@ impl Cli { let config = load_config(config_path)?; match self.command { - Commands::Start => start::execute(config, &id).await?, - Commands::Password { command } => password::execute(command, config, &id).await?, - Commands::Aggregator { command } => aggregator::execute(command, config, &id).await?, - Commands::Wallet { command } => wallet::execute(command, config, &id).await?, + Commands::Start => start::execute(config).await?, + Commands::Password { command } => password::execute(command, config).await?, + Commands::Aggregator { command } => aggregator::execute(command, config).await?, + Commands::Wallet { command } => wallet::execute(command, config).await?, } Ok(()) @@ -80,7 +81,8 @@ pub async fn main() { // .with_env_filter("[app{id=ag}]=info") .init(); let cli = Cli::parse(); - let id = cli.get_tag(); + set_tag(cli.get_tag()); + let id = get_tag(); let span = span!(Level::INFO, "app", %id); let _guard = span.enter(); match cli.execute().instrument(span.clone()).await { diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 7efb0673..45ee6358 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -29,7 +29,6 @@ pub async fn setup_aggregator( config: AppConfig, pubkey_write_path: Option<&str>, plaintext_write_path: Option<&str>, - tag: &str ) -> Result<(Addr, JoinHandle<()>, String)> { let bus = EventBus::new(true).start(); let rng = Arc::new(Mutex::new( @@ -37,7 +36,7 @@ pub async fn setup_aggregator( )); let store = setup_datastore(&config, &bus)?; let repositories = store.repositories(); - let sortition = Sortition::attach(&bus, repositories.sortition()); + let sortition = Sortition::attach(&bus, repositories.sortition()).await?; let cipher = Arc::new(Cipher::from_config(&config).await?); let signer = get_signer_from_repository(repositories.eth_private_key(), &cipher).await?; @@ -58,7 +57,6 @@ pub async fn setup_aggregator( &chain.contracts.enclave.address(), &repositories.enclave_sol_reader(read_provider.get_chain_id()), chain.contracts.enclave.deploy_block(), - tag ) .await?; RegistryFilterSol::attach( @@ -73,7 +71,6 @@ pub async fn setup_aggregator( &chain.contracts.ciphernode_registry.address(), &repositories.ciphernode_registry_reader(read_provider.get_chain_id()), chain.contracts.ciphernode_registry.deploy_block(), - tag ) .await?; } diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 0fabd588..d58b479c 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -24,7 +24,6 @@ use crate::setup_datastore; pub async fn setup_ciphernode( config: AppConfig, address: Address, - tag: &str ) -> Result<(Addr, JoinHandle<()>, String)> { let rng = Arc::new(Mutex::new( rand_chacha::ChaCha20Rng::from_rng(OsRng).expect("Failed to create RNG"), @@ -35,7 +34,7 @@ pub async fn setup_ciphernode( let repositories = store.repositories(); - let sortition = Sortition::attach(&bus, repositories.sortition()); + let sortition = Sortition::attach(&bus, repositories.sortition()).await?; CiphernodeSelector::attach(&bus, &sortition, &address.to_string()); for chain in config @@ -52,7 +51,6 @@ pub async fn setup_ciphernode( &chain.contracts.enclave.address(), &repositories.enclave_sol_reader(read_provider.get_chain_id()), chain.contracts.enclave.deploy_block(), - tag ) .await?; CiphernodeRegistrySol::attach( @@ -61,7 +59,6 @@ pub async fn setup_ciphernode( &chain.contracts.ciphernode_registry.address(), &repositories.ciphernode_registry_reader(read_provider.get_chain_id()), chain.contracts.ciphernode_registry.deploy_block(), - tag ) .await?; } diff --git a/packages/ciphernode/enclave_node/src/datastore.rs b/packages/ciphernode/enclave_node/src/datastore.rs index 8eb77e0c..17a2be04 100644 --- a/packages/ciphernode/enclave_node/src/datastore.rs +++ b/packages/ciphernode/enclave_node/src/datastore.rs @@ -8,7 +8,7 @@ use enclave_core::EventBus; use router::{Repositories, RepositoriesFactory}; pub fn get_sled_store(bus: &Addr, db_file: &PathBuf) -> Result { - Ok((&SledStore::new(bus, db_file)?.start()).into()) + Ok((&SledStore::new(bus, db_file)?).into()) } pub fn get_in_mem_store() -> DataStore { diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index 656697ed..d88e6552 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -106,7 +106,6 @@ impl CiphernodeRegistrySolReader { contract_address: &str, repository: &Repository, start_block: Option, - tag: &str ) -> Result>> { let addr = EvmEventReader::attach( provider, @@ -115,7 +114,6 @@ impl CiphernodeRegistrySolReader { start_block, &bus.clone().into(), repository, - tag ) .await?; @@ -134,7 +132,6 @@ impl CiphernodeRegistrySol { contract_address: &str, repository: &Repository, start_block: Option, - tag: &str ) -> Result<()> { CiphernodeRegistrySolReader::attach( bus, @@ -142,7 +139,6 @@ impl CiphernodeRegistrySol { contract_address, repository, start_block, - tag ) .await?; // TODO: Writer if needed diff --git a/packages/ciphernode/evm/src/enclave_sol.rs b/packages/ciphernode/evm/src/enclave_sol.rs index 763d76b0..ed1f76bc 100644 --- a/packages/ciphernode/evm/src/enclave_sol.rs +++ b/packages/ciphernode/evm/src/enclave_sol.rs @@ -18,7 +18,6 @@ impl EnclaveSol { contract_address: &str, repository: &Repository, start_block: Option, - tag: &str ) -> Result<()> { EnclaveSolReader::attach( bus, @@ -26,7 +25,6 @@ impl EnclaveSol { contract_address, repository, start_block, - tag ) .await?; EnclaveSolWriter::attach(bus, write_provider, contract_address).await?; diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index 3fcebed2..c71c39de 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -90,7 +90,6 @@ impl EnclaveSolReader { contract_address: &str, repository: &Repository, start_block: Option, - tag: &str ) -> Result>> { let addr = EvmEventReader::attach( provider, @@ -99,7 +98,6 @@ impl EnclaveSolReader { start_block, &bus.clone(), repository, - tag ) .await?; diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index a544bcf0..66654cce 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -10,7 +10,9 @@ use alloy::transports::{BoxTransport, Transport}; use anyhow::{anyhow, Result}; use async_trait::async_trait; use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; -use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus, EventId, Subscribe}; +use enclave_core::{ + get_tag, BusError, EnclaveErrorType, EnclaveEvent, EventBus, EventId, Subscribe, +}; use futures_util::stream::StreamExt; use std::collections::HashSet; use tokio::select; @@ -49,7 +51,6 @@ where start_block: Option, bus: Addr, repository: Repository, - tag: String, } #[derive(Default, serde::Serialize, serde::Deserialize, Clone)] @@ -83,8 +84,6 @@ where state: EvmEventReaderState, /// Repository to save the state of the event reader repository: Repository, - /// An identifier for logs - tag: String, } impl EvmEventReader @@ -104,12 +103,11 @@ where bus: params.bus, state: EvmEventReaderState::default(), repository: params.repository, - tag: params.tag, } } pub async fn load(params: EvmEventReaderParams) -> Result { - let id = params.tag.clone(); + let id = get_tag(); let span = info_span!("evm_event_reader", %id); let _guard = span.enter(); Ok(if let Some(snapshot) = params.repository.read().await? { @@ -128,7 +126,6 @@ where start_block: Option, bus: &Addr, repository: &Repository, - tag: &str, ) -> Result> { let params = EvmEventReaderParams { provider: provider.clone(), @@ -137,7 +134,6 @@ where start_block, bus: bus.clone(), repository: repository.clone(), - tag: tag.to_string(), }; let addr = EvmEventReader::load(params).await?.start(); @@ -169,7 +165,7 @@ where let contract_address = self.contract_address; let start_block = self.start_block; - let tag = self.tag.clone(); + let tag = get_tag(); ctx.spawn( async move { stream_from_evm( @@ -288,7 +284,7 @@ where { type Result = (); fn handle(&mut self, wrapped: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { - let id = self.tag.clone(); + let id = get_tag(); let span = info_span!("evm_event_reader", %id); let _guard = span.enter(); let event_id = wrapped.get_id(); @@ -354,7 +350,6 @@ where bus: params.bus, state: snapshot, repository: params.repository, - tag: params.tag, }) } } diff --git a/packages/ciphernode/evm/tests/evm_reader.rs b/packages/ciphernode/evm/tests/evm_reader.rs index a826e6cb..0fdacf07 100644 --- a/packages/ciphernode/evm/tests/evm_reader.rs +++ b/packages/ciphernode/evm/tests/evm_reader.rs @@ -61,7 +61,6 @@ async fn evm_reader() -> Result<()> { None, &bus, &repository, - "mytag" ) .await?; @@ -132,7 +131,6 @@ async fn ensure_historical_events() -> Result<()> { None, &bus, &repository, - "mytag" ) .await?; diff --git a/packages/ciphernode/sortition/src/sortition.rs b/packages/ciphernode/sortition/src/sortition.rs index 0f5b51bd..73c2db04 100644 --- a/packages/ciphernode/sortition/src/sortition.rs +++ b/packages/ciphernode/sortition/src/sortition.rs @@ -5,10 +5,11 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; use enclave_core::{ - BusError, CiphernodeAdded, CiphernodeRemoved, EnclaveErrorType, EnclaveEvent, EventBus, Seed, - Subscribe, + get_tag, BusError, CiphernodeAdded, CiphernodeRemoved, EnclaveErrorType, EnclaveEvent, + EventBus, Seed, Subscribe, }; use std::collections::HashSet; +use tracing::{info, info_span, instrument}; #[derive(Message, Clone, Debug, PartialEq, Eq)] #[rtype(result = "bool")] @@ -24,7 +25,7 @@ pub trait SortitionList { fn remove(&mut self, address: T); } -#[derive(Clone, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct SortitionModule { nodes: HashSet, } @@ -35,6 +36,10 @@ impl SortitionModule { nodes: HashSet::new(), } } + + pub fn nodes(&self) -> &HashSet { + &self.nodes + } } impl Default for SortitionModule { @@ -87,6 +92,7 @@ pub struct Sortition { store: Repository, } +#[derive(Debug)] pub struct SortitionParams { pub bus: Addr, pub store: Repository, @@ -101,14 +107,31 @@ impl Sortition { } } - pub fn attach(bus: &Addr, store: Repository) -> Addr { - let addr = Sortition::new(SortitionParams { + pub async fn attach( + bus: &Addr, + store: Repository, + ) -> Result> { + let addr = Sortition::load(SortitionParams { bus: bus.clone(), store, }) + .await? .start(); bus.do_send(Subscribe::new("CiphernodeAdded", addr.clone().into())); - addr + Ok(addr) + } + + pub async fn load(params: SortitionParams) -> Result { + let id = get_tag(); + let span = info_span!("sorition", %id); + let _guard = span.enter(); + Ok(if let Some(snapshot) = params.store.read().await? { + info!("Loading from snapshot"); + Self::from_snapshot(params, snapshot).await? + } else { + info!("Loading from params"); + Self::new(params) + }) } pub fn get_nodes(&self) -> Vec { @@ -130,7 +153,9 @@ impl Snapshot for Sortition { #[async_trait] impl FromSnapshotWithParams for Sortition { type Params = SortitionParams; + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { + info!("Loaded snapshot with {} nodes", snapshot.nodes().len()); Ok(Sortition { bus: params.bus, store: params.store, diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index 1ac603bd..9e12fab6 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -52,7 +52,7 @@ async fn setup_local_ciphernode( let store = DataStore::from(&data_actor); let repositories = store.repositories(); // create ciphernode actor for managing ciphernode flow - let sortition = Sortition::attach(&bus, repositories.sortition()); + let sortition = Sortition::attach(&bus, repositories.sortition()).await?; CiphernodeSelector::attach(&bus, &sortition, addr); let router = E3RequestRouter::builder(&bus, store) diff --git a/tests/basic_integration/base.sh b/tests/basic_integration/base.sh new file mode 100755 index 00000000..a774eff3 --- /dev/null +++ b/tests/basic_integration/base.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +set -eu # Exit immediately if a command exits with a non-zero status + +# Get the directory of the currently executing script +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# Source the file from the same directory +source "$THIS_DIR/fns.sh" + +heading "Start the EVM node" + +yarn evm:node & + +until curl -f -s "http://localhost:8545" > /dev/null; do + sleep 1 +done + +# Set the password for all ciphernodes +set_password cn1 "$CIPHERNODE_SECRET" +set_password cn2 "$CIPHERNODE_SECRET" +set_password cn3 "$CIPHERNODE_SECRET" +set_password cn4 "$CIPHERNODE_SECRET" +set_password ag "$CIPHERNODE_SECRET" +set_private_key ag "$PRIVATE_KEY" + +# Launch 4 ciphernodes +launch_ciphernode cn1 +launch_ciphernode cn2 +launch_ciphernode cn3 +launch_ciphernode cn4 +launch_aggregator ag + +waiton-files "$ROOT_DIR/packages/ciphernode/target/debug/enclave" "$ROOT_DIR/packages/ciphernode/target/debug/fake_encrypt" + +heading "Add ciphernode $CIPHERNODE_ADDRESS_1" +yarn ciphernode:add --ciphernode-address $CIPHERNODE_ADDRESS_1 --network localhost + +heading "Add ciphernode $CIPHERNODE_ADDRESS_2" +yarn ciphernode:add --ciphernode-address $CIPHERNODE_ADDRESS_2 --network localhost + +heading "Add ciphernode $CIPHERNODE_ADDRESS_3" +yarn ciphernode:add --ciphernode-address $CIPHERNODE_ADDRESS_3 --network localhost + +heading "Add ciphernode $CIPHERNODE_ADDRESS_4" +yarn ciphernode:add --ciphernode-address $CIPHERNODE_ADDRESS_4 --network localhost + +heading "Request Committee" + +ENCODED_PARAMS=0x$($SCRIPT_DIR/lib/pack_e3_params.sh --moduli 0x3FFFFFFF000001 --degree 2048 --plaintext-modulus 1032193) + +yarn committee:new --network localhost --duration 4 --e3-params "$ENCODED_PARAMS" + +waiton "$SCRIPT_DIR/output/pubkey.bin" +PUBLIC_KEY=$(xxd -p -c 10000000 "$SCRIPT_DIR/output/pubkey.bin") + +heading "Mock encrypted plaintext" +$SCRIPT_DIR/lib/fake_encrypt.sh --input "$SCRIPT_DIR/output/pubkey.bin" --output "$SCRIPT_DIR/output/output.bin" --plaintext $PLAINTEXT + +heading "Mock activate e3-id" +yarn e3:activate --e3-id 0 --public-key "0x$PUBLIC_KEY" --network localhost + +heading "Mock publish input e3-id" +yarn e3:publishInput --network localhost --e3-id 0 --data 0x12345678 + +sleep 4 # wait for input deadline to pass + +waiton "$SCRIPT_DIR/output/output.bin" + +heading "Publish ciphertext to EVM" +yarn e3:publishCiphertext --e3-id 0 --network localhost --data-file "$SCRIPT_DIR/output/output.bin" --proof 0x12345678 + +waiton "$SCRIPT_DIR/output/plaintext.txt" + +ACTUAL=$(cat $SCRIPT_DIR/output/plaintext.txt) + + +# Assume plaintext is shorter + +if [[ "$ACTUAL" != "$PLAINTEXT"* ]]; then + echo "Invalid plaintext decrypted: actual='$ACTUAL' expected='$PLAINTEXT'" + echo "Test FAILED" + exit 1 +fi + +heading "Test PASSED !" +echo -e "\033[32m + ██████ + ██████ + ██████ + ██████ + ██████ + ██████ + ██ ██████ + ████ ██████ + ██████ ██████ + ██████████ + ████████ + ██████ + ████ + ██ + \033[0m" + +pkill -15 -f "target/debug/enclave" || true +pkill -15 -f "target/debug/aggregator" || true + +sleep 4 + +cleanup 0 + diff --git a/tests/basic_integration/fns.sh b/tests/basic_integration/fns.sh new file mode 100644 index 00000000..bfe8cd12 --- /dev/null +++ b/tests/basic_integration/fns.sh @@ -0,0 +1,138 @@ + +# Get the script's location +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" +PLAINTEXT="1234,567890" +ID=$(date +%s) + +if [[ "$ROOT_DIR" != "$(pwd)" ]]; then + echo "This script must be run from the root" + exit 1 +fi + +export RUST_LOG=info + +# Environment variables +RPC_URL="ws://localhost:8545" + +PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" +CIPHERNODE_SECRET="We are the music makers and we are the dreamers of the dreams." + +# These contracts are based on the deterministic order of hardhat deploy +# We _may_ wish to get these off the hardhat environment somehow? +ENCLAVE_CONTRACT="0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0" +REGISTRY_CONTRACT="0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" +REGISTRY_FILTER_CONTRACT="0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9" +INPUT_VALIDATOR_CONTRACT="0x8A791620dd6260079BF849Dc5567aDC3F2FdC318" + +# These are random addresses for now +CIPHERNODE_ADDRESS_1="0x2546BcD3c84621e976D8185a91A922aE77ECEc30" +CIPHERNODE_ADDRESS_2="0xbDA5747bFD65F08deb54cb465eB87D40e51B197E" +CIPHERNODE_ADDRESS_3="0xdD2FD4581271e230360230F9337D5c0430Bf44C0" +CIPHERNODE_ADDRESS_4="0x8626f6940E2eb28930eFb4CeF49B2d1F2C9C1199" + + +# Function to clean up background processes +cleanup() { + echo "Cleaning up processes..." + kill $(jobs -p) 2>/dev/null + exit ${1:-1} +} + +heading() { + echo "" + echo "" + echo "--------------------------------------------------------------" + echo " $1 " + echo "--------------------------------------------------------------" + echo "" +} + +waiton() { + local file_path="$1" + until [ -f "$file_path" ]; do + sleep 1 + done +} + +waiton-files() { + local timeout=600 # 10 minutes timeout + local start_time=$(date +%s) + while true; do + all_exist=true + for file in "$@"; do + if [ ! -f "$file" ]; then + all_exist=false + break + fi + done + if $all_exist; then + break + fi + if [ $(($(date +%s) - start_time)) -ge $timeout ]; then + echo "Timeout waiting for files: $@" >&2 + return 1 + fi + sleep 1 + done +} + +set_password() { + local name="$1" + local password="$2" + yarn enclave password create \ + --config "$SCRIPT_DIR/lib/$name/config.yaml" \ + --password "$password" +} + +launch_ciphernode() { + local name="$1" + heading "Launch ciphernode $name" + yarn enclave start \ + --tag "$name" \ + --config "$SCRIPT_DIR/lib/$name/config.yaml" & echo $! > "/tmp/enclave.${ID}_${name}.pid" +} + +set_private_key() { + local name="$1" + local private_key="$2" + + yarn enclave wallet set \ + --config "$SCRIPT_DIR/lib/$name/config.yaml" \ + --private-key "$private_key" +} + +launch_aggregator() { + local name="$1" + heading "Launch aggregator $name" + + yarn enclave aggregator start \ + --tag "$name" \ + --config "$SCRIPT_DIR/lib/$name/config.yaml" \ + --pubkey-write-path "$SCRIPT_DIR/output/pubkey.bin" \ + --plaintext-write-path "$SCRIPT_DIR/output/plaintext.txt" & echo $! > "/tmp/enclave.${ID}_${name}.pid" + + ps aux | grep aggregator +} + +kill_proc() { + local name=$1 + local pid=$(ps aux | grep 'enclave' | grep "\--tag $name" | awk '{ print $2 }') + echo "Killing $pid" + kill $pid +} + +metallica() { + pkill -9 -f "target/debug/enclave" || true + pkill -9 -f "hardhat node" || true +} + +metallica + +# Set up trap to catch errors and interrupts +trap 'cleanup $?' ERR INT TERM + +$SCRIPT_DIR/lib/clean_folders.sh "$SCRIPT_DIR" +$SCRIPT_DIR/lib/prebuild.sh + + diff --git a/tests/basic_integration/lib/clean_folders.sh b/tests/basic_integration/lib/clean_folders.sh index 58a1195e..5c7686a0 100755 --- a/tests/basic_integration/lib/clean_folders.sh +++ b/tests/basic_integration/lib/clean_folders.sh @@ -4,6 +4,7 @@ clean_folders() { # Delete output artifacts rm -rf "$SCRIPT_DIR/output/"* + rm -rf "/tmp/enclave.*.pid" # Delete enclave artifacts for name in cn1 cn2 cn3 cn4 ag; do diff --git a/tests/basic_integration/persist.sh b/tests/basic_integration/persist.sh new file mode 100755 index 00000000..bfbffc37 --- /dev/null +++ b/tests/basic_integration/persist.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +set -eu # Exit immediately if a command exits with a non-zero status + +# Get the directory of the currently executing script +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# Source the file from the same directory +source "$THIS_DIR/fns.sh" + +heading "Start the EVM node" + +yarn evm:node & + +until curl -f -s "http://localhost:8545" > /dev/null; do + sleep 1 +done + +# Set the password for all ciphernodes +set_password cn1 "$CIPHERNODE_SECRET" +set_password cn2 "$CIPHERNODE_SECRET" +set_password cn3 "$CIPHERNODE_SECRET" +set_password cn4 "$CIPHERNODE_SECRET" +set_password ag "$CIPHERNODE_SECRET" +set_private_key ag "$PRIVATE_KEY" + +# Launch 4 ciphernodes +launch_ciphernode cn1 +launch_ciphernode cn2 +launch_ciphernode cn3 +launch_ciphernode cn4 +launch_aggregator ag + +waiton-files "$ROOT_DIR/packages/ciphernode/target/debug/enclave" "$ROOT_DIR/packages/ciphernode/target/debug/fake_encrypt" + +heading "Add ciphernode $CIPHERNODE_ADDRESS_1" +yarn ciphernode:add --ciphernode-address $CIPHERNODE_ADDRESS_1 --network localhost + +heading "Add ciphernode $CIPHERNODE_ADDRESS_2" +yarn ciphernode:add --ciphernode-address $CIPHERNODE_ADDRESS_2 --network localhost + +heading "Add ciphernode $CIPHERNODE_ADDRESS_3" +yarn ciphernode:add --ciphernode-address $CIPHERNODE_ADDRESS_3 --network localhost + +heading "Add ciphernode $CIPHERNODE_ADDRESS_4" +yarn ciphernode:add --ciphernode-address $CIPHERNODE_ADDRESS_4 --network localhost + +heading "Request Committee" + +ENCODED_PARAMS=0x$($SCRIPT_DIR/lib/pack_e3_params.sh --moduli 0x3FFFFFFF000001 --degree 2048 --plaintext-modulus 1032193) + +yarn committee:new --network localhost --duration 4 --e3-params "$ENCODED_PARAMS" + +waiton "$SCRIPT_DIR/output/pubkey.bin" +PUBLIC_KEY=$(xxd -p -c 10000000 "$SCRIPT_DIR/output/pubkey.bin") + + +# kill aggregator +kill_proc ag + +sleep 2 + +# relaunch the aggregator +launch_aggregator ag + +sleep 2 + +heading "Mock encrypted plaintext" +$SCRIPT_DIR/lib/fake_encrypt.sh --input "$SCRIPT_DIR/output/pubkey.bin" --output "$SCRIPT_DIR/output/output.bin" --plaintext $PLAINTEXT + +heading "Mock activate e3-id" +yarn e3:activate --e3-id 0 --public-key "0x$PUBLIC_KEY" --network localhost + +heading "Mock publish input e3-id" +yarn e3:publishInput --network localhost --e3-id 0 --data 0x12345678 + +sleep 4 # wait for input deadline to pass + +waiton "$SCRIPT_DIR/output/output.bin" + +heading "Publish ciphertext to EVM" +yarn e3:publishCiphertext --e3-id 0 --network localhost --data-file "$SCRIPT_DIR/output/output.bin" --proof 0x12345678 + +waiton "$SCRIPT_DIR/output/plaintext.txt" + +ACTUAL=$(cat $SCRIPT_DIR/output/plaintext.txt) + + +# Assume plaintext is shorter + +if [[ "$ACTUAL" != "$PLAINTEXT"* ]]; then + echo "Invalid plaintext decrypted: actual='$ACTUAL' expected='$PLAINTEXT'" + echo "Test FAILED" + exit 1 +fi + +heading "Test PASSED !" +echo -e "\033[32m + ██████ + ██████ + ██████ + ██████ + ██████ + ██████ + ██ ██████ + ████ ██████ + ██████ ██████ + ██████████ + ████████ + ██████ + ████ + ██ + \033[0m" + +pkill -15 -f "target/debug/enclave" || true +pkill -15 -f "target/debug/aggregator" || true + +sleep 4 + +cleanup 0 + diff --git a/tests/basic_integration/test.sh b/tests/basic_integration/test.sh index f793f5ed..4c733ab3 100755 --- a/tests/basic_integration/test.sh +++ b/tests/basic_integration/test.sh @@ -2,226 +2,19 @@ set -eu # Exit immediately if a command exits with a non-zero status -# Get the script's location -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" -PLAINTEXT="1234,567890" - -if [[ "$ROOT_DIR" != "$(pwd)" ]]; then - echo "This script must be run from the root" - exit 1 -fi - -export RUST_LOG=info - -# Environment variables -RPC_URL="ws://localhost:8545" - -PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" -CIPHERNODE_SECRET="We are the music makers and we are the dreamers of the dreams." - -# These contracts are based on the deterministic order of hardhat deploy -# We _may_ wish to get these off the hardhat environment somehow? -ENCLAVE_CONTRACT="0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0" -REGISTRY_CONTRACT="0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" -REGISTRY_FILTER_CONTRACT="0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9" -INPUT_VALIDATOR_CONTRACT="0x8A791620dd6260079BF849Dc5567aDC3F2FdC318" - -# These are random addresses for now -CIPHERNODE_ADDRESS_1="0x2546BcD3c84621e976D8185a91A922aE77ECEc30" -CIPHERNODE_ADDRESS_2="0xbDA5747bFD65F08deb54cb465eB87D40e51B197E" -CIPHERNODE_ADDRESS_3="0xdD2FD4581271e230360230F9337D5c0430Bf44C0" -CIPHERNODE_ADDRESS_4="0x8626f6940E2eb28930eFb4CeF49B2d1F2C9C1199" - - -# Function to clean up background processes -cleanup() { - echo "Cleaning up processes..." - kill $(jobs -p) 2>/dev/null - exit ${1:-1} -} - -heading() { - echo "" - echo "" - echo "--------------------------------------------------------------" - echo " $1 " - echo "--------------------------------------------------------------" - echo "" -} - -waiton() { - local file_path="$1" - until [ -f "$file_path" ]; do - sleep 1 - done -} - -waiton-files() { - local timeout=600 # 10 minutes timeout - local start_time=$(date +%s) - while true; do - all_exist=true - for file in "$@"; do - if [ ! -f "$file" ]; then - all_exist=false - break - fi - done - if $all_exist; then - break - fi - if [ $(($(date +%s) - start_time)) -ge $timeout ]; then - echo "Timeout waiting for files: $@" >&2 - return 1 - fi - sleep 1 - done -} - -set_password() { - local name="$1" - local password="$2" - yarn enclave password create \ - --config "$SCRIPT_DIR/lib/$name/config.yaml" \ - --password "$password" -} - -launch_ciphernode() { - local name="$1" - heading "Launch ciphernode $name" - yarn enclave start \ - --tag "$name" \ - --config "$SCRIPT_DIR/lib/$name/config.yaml" & -} - -set_private_key() { - local name="$1" - local private_key="$2" - - yarn enclave wallet set \ - --config "$SCRIPT_DIR/lib/$name/config.yaml" \ - --private-key "$private_key" -} - -launch_aggregator() { - local name="$1" - heading "Launch aggregator $name" - - yarn enclave aggregator start \ - --tag "$name" \ - --config "$SCRIPT_DIR/lib/$name/config.yaml" \ - --pubkey-write-path "$SCRIPT_DIR/output/pubkey.bin" \ - --plaintext-write-path "$SCRIPT_DIR/output/plaintext.txt" & -} - -pkill -9 -f "target/debug/enclave" || true -pkill -9 -f "hardhat node" || true - -# Set up trap to catch errors and interrupts -trap 'cleanup $?' ERR INT TERM - - -$SCRIPT_DIR/lib/clean_folders.sh "$SCRIPT_DIR" -$SCRIPT_DIR/lib/prebuild.sh - -heading "Start the EVM node" - -yarn evm:node & - -until curl -f -s "http://localhost:8545" > /dev/null; do - sleep 1 -done - -# Set the password for all ciphernodes -set_password cn1 "$CIPHERNODE_SECRET" -set_password cn2 "$CIPHERNODE_SECRET" -set_password cn3 "$CIPHERNODE_SECRET" -set_password cn4 "$CIPHERNODE_SECRET" -set_password ag "$CIPHERNODE_SECRET" -set_private_key ag "$PRIVATE_KEY" - -# Launch 4 ciphernodes -launch_ciphernode cn1 -launch_ciphernode cn2 -launch_ciphernode cn3 -launch_ciphernode cn4 -launch_aggregator ag - -waiton-files "$ROOT_DIR/packages/ciphernode/target/debug/enclave" "$ROOT_DIR/packages/ciphernode/target/debug/fake_encrypt" - -heading "Add ciphernode $CIPHERNODE_ADDRESS_1" -yarn ciphernode:add --ciphernode-address $CIPHERNODE_ADDRESS_1 --network localhost - -heading "Add ciphernode $CIPHERNODE_ADDRESS_2" -yarn ciphernode:add --ciphernode-address $CIPHERNODE_ADDRESS_2 --network localhost - -heading "Add ciphernode $CIPHERNODE_ADDRESS_3" -yarn ciphernode:add --ciphernode-address $CIPHERNODE_ADDRESS_3 --network localhost - -heading "Add ciphernode $CIPHERNODE_ADDRESS_4" -yarn ciphernode:add --ciphernode-address $CIPHERNODE_ADDRESS_4 --network localhost - -heading "Request Committee" - -ENCODED_PARAMS=0x$($SCRIPT_DIR/lib/pack_e3_params.sh --moduli 0x3FFFFFFF000001 --degree 2048 --plaintext-modulus 1032193) - -yarn committee:new --network localhost --duration 4 --e3-params "$ENCODED_PARAMS" - -waiton "$SCRIPT_DIR/output/pubkey.bin" -PUBLIC_KEY=$(xxd -p -c 10000000 "$SCRIPT_DIR/output/pubkey.bin") - -heading "Mock encrypted plaintext" -$SCRIPT_DIR/lib/fake_encrypt.sh --input "$SCRIPT_DIR/output/pubkey.bin" --output "$SCRIPT_DIR/output/output.bin" --plaintext $PLAINTEXT - -heading "Mock activate e3-id" -yarn e3:activate --e3-id 0 --public-key "0x$PUBLIC_KEY" --network localhost - -heading "Mock publish input e3-id" -yarn e3:publishInput --network localhost --e3-id 0 --data 0x12345678 - -sleep 4 # wait for input deadline to pass - -waiton "$SCRIPT_DIR/output/output.bin" - -heading "Publish ciphertext to EVM" -yarn e3:publishCiphertext --e3-id 0 --network localhost --data-file "$SCRIPT_DIR/output/output.bin" --proof 0x12345678 - -waiton "$SCRIPT_DIR/output/plaintext.txt" - -ACTUAL=$(cat $SCRIPT_DIR/output/plaintext.txt) - - -# Assume plaintext is shorter - -if [[ "$ACTUAL" != "$PLAINTEXT"* ]]; then - echo "Invalid plaintext decrypted: actual='$ACTUAL' expected='$PLAINTEXT'" - echo "Test FAILED" - exit 1 -fi - -heading "Test PASSED !" -echo -e "\033[32m - ██████ - ██████ - ██████ - ██████ - ██████ - ██████ - ██ ██████ - ████ ██████ - ██████ ██████ - ██████████ - ████████ - ██████ - ████ - ██ - \033[0m" - -pkill -15 -f "target/debug/enclave" || true -pkill -15 -f "target/debug/aggregator" || true - -sleep 4 - -cleanup 0 +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# Source the file from the same directory +case $1 in + persist) + "$THIS_DIR/persist.sh" + ;; + base) + "$THIS_DIR/base.sh" + ;; + *) + "$THIS_DIR/persist.sh" + "$THIS_DIR/base.sh" + ;; +esac From f43e3f81bc286b7b578ad9a6fe2d0cfb61f27673 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 12 Nov 2024 11:30:11 +0700 Subject: [PATCH 28/31] Tidy up instrument code and fix error --- packages/ciphernode/Cargo.lock | 5 +-- packages/ciphernode/README.md | 33 +++++++++++++++++++ packages/ciphernode/core/src/tag.rs | 20 +++-------- packages/ciphernode/enclave/Cargo.toml | 1 + .../enclave/src/commands/aggregator/start.rs | 4 ++- .../ciphernode/enclave/src/commands/start.rs | 7 ++-- packages/ciphernode/enclave/src/main.rs | 26 +++++++-------- .../ciphernode/enclave_node/src/ciphernode.rs | 4 ++- packages/ciphernode/evm/src/event_reader.rs | 11 +++---- .../ciphernode/sortition/src/sortition.rs | 22 ++++++++++--- tests/basic_integration/base.sh | 2 +- tests/basic_integration/fns.sh | 10 ++++-- tests/basic_integration/persist.sh | 2 +- tests/basic_integration/test.sh | 19 ++++------- 14 files changed, 101 insertions(+), 65 deletions(-) diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index ff09b313..f4882352 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -2211,6 +2211,7 @@ dependencies = [ "enclave-core", "enclave_node", "hex", + "once_cell", "router", "rpassword", "tokio", @@ -4449,9 +4450,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "opaque-debug" diff --git a/packages/ciphernode/README.md b/packages/ciphernode/README.md index cb035f5b..312b7e00 100644 --- a/packages/ciphernode/README.md +++ b/packages/ciphernode/README.md @@ -74,3 +74,36 @@ sequenceDiagram PTA--)PTA: Stop KS--)-KS: Stop ``` + +# Debugging + +You can debug using the `RUST_LOG` environment var to alter what output is produced by the node + + +``` +RUST_LOG=info enclave start +``` + +if you supply a tag as an argument you can filter for that tag + +``` +RUST_LOG="[sortition{id=cn1}]" enclave start --tag cn1 +``` + +This helps filter noise during tests where you might have multiple instances running and you need to see the output of a specific one. + +In order to add tracing to a method or function it is recommended to use the `instrument` macro. + +```rust +impl Sorition { + // ... + #[instrument(name="sortition", skip_all, fields(id = get_tag()))] + pub async fn attach( + bus: &Addr, + store: Repository, + ) -> Result> { + // ... + } +} +``` + diff --git a/packages/ciphernode/core/src/tag.rs b/packages/ciphernode/core/src/tag.rs index 23bd2b81..6636cca6 100644 --- a/packages/ciphernode/core/src/tag.rs +++ b/packages/ciphernode/core/src/tag.rs @@ -1,22 +1,12 @@ -use std::sync::RwLock; +use std::sync::OnceLock; -use lazy_static::lazy_static; - -lazy_static! { - static ref TAG: RwLock = RwLock::new(String::from("_")); -} +static TAG: OnceLock = OnceLock::new(); pub fn get_tag() -> String { - let tag_guard = TAG.read().expect("Failed to acquire read lock"); - tag_guard.clone() + TAG.get().cloned().unwrap_or_else(|| String::from("_")) } pub fn set_tag(new_tag: impl Into) -> Result<(), &'static str> { - match TAG.write() { - Ok(mut tag_guard) => { - *tag_guard = new_tag.into(); - Ok(()) - } - Err(_) => Err("Failed to acquire write lock"), - } + TAG.set(new_tag.into()) + .map_err(|_| "Tag has already been initialized") } diff --git a/packages/ciphernode/enclave/Cargo.toml b/packages/ciphernode/enclave/Cargo.toml index 2e55621e..f0a93ab7 100644 --- a/packages/ciphernode/enclave/Cargo.toml +++ b/packages/ciphernode/enclave/Cargo.toml @@ -19,6 +19,7 @@ dialoguer = "0.11.0" enclave-core = { path = "../core" } enclave_node = { path = "../enclave_node" } hex = { workspace = true } +once_cell = "1.20.2" router = { path = "../router" } rpassword = "7.3.1" tokio = { workspace = true } diff --git a/packages/ciphernode/enclave/src/commands/aggregator/start.rs b/packages/ciphernode/enclave/src/commands/aggregator/start.rs index 2194030d..4ae1e9ce 100644 --- a/packages/ciphernode/enclave/src/commands/aggregator/start.rs +++ b/packages/ciphernode/enclave/src/commands/aggregator/start.rs @@ -1,10 +1,12 @@ use anyhow::*; use config::AppConfig; +use enclave_core::get_tag; use enclave_node::{listen_for_shutdown, setup_aggregator}; -use tracing::info; +use tracing::{info, instrument}; use crate::owo; +#[instrument(name="app", skip_all,fields(id = get_tag()))] pub async fn execute( config: AppConfig, pubkey_write_path: Option<&str>, diff --git a/packages/ciphernode/enclave/src/commands/start.rs b/packages/ciphernode/enclave/src/commands/start.rs index c46f9c5d..17545224 100644 --- a/packages/ciphernode/enclave/src/commands/start.rs +++ b/packages/ciphernode/enclave/src/commands/start.rs @@ -1,14 +1,13 @@ use anyhow::{anyhow, Result}; use config::AppConfig; +use enclave_core::get_tag; use enclave_node::{listen_for_shutdown, setup_ciphernode}; -use tracing::info; - +use tracing::{info, instrument}; use crate::owo; +#[instrument(name="app", skip_all,fields(id = get_tag()))] pub async fn execute(config: AppConfig) -> Result<()> { owo(); - - // let address = Address::parse_checksummed(&config.address(), None).context("Invalid address")?; let Some(address) = config.address() else { return Err(anyhow!("You must provide an address")); }; diff --git a/packages/ciphernode/enclave/src/main.rs b/packages/ciphernode/enclave/src/main.rs index 49d71184..f687dff5 100644 --- a/packages/ciphernode/enclave/src/main.rs +++ b/packages/ciphernode/enclave/src/main.rs @@ -1,11 +1,9 @@ -use std::env; - use anyhow::Result; use clap::Parser; use commands::{aggregator, password, start, wallet, Commands}; use config::load_config; use enclave_core::{get_tag, set_tag}; -use tracing::{instrument::WithSubscriber, span, Instrument, Level}; +use tracing::instrument; use tracing_subscriber::EnvFilter; pub mod commands; @@ -45,9 +43,9 @@ pub struct Cli { } impl Cli { + #[instrument(skip(self),fields(id = get_tag()))] pub async fn execute(self) -> Result<()> { let config_path = self.config.as_deref(); - let id = self.get_tag(); let config = load_config(config_path)?; match self.command { @@ -81,15 +79,15 @@ pub async fn main() { // .with_env_filter("[app{id=ag}]=info") .init(); let cli = Cli::parse(); - set_tag(cli.get_tag()); - let id = get_tag(); - let span = span!(Level::INFO, "app", %id); - let _guard = span.enter(); - match cli.execute().instrument(span.clone()).await { - Ok(_) => (), - Err(err) => { - eprintln!("{}", err); - std::process::exit(1); - } + + // Set the tag for all future traces + if let Err(err) = set_tag(cli.get_tag()) { + eprintln!("{}", err); + } + + // Execute the cli + if let Err(err) = cli.execute().await { + eprintln!("{}", err); + std::process::exit(1); } } diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index d58b479c..b2ba2ece 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -3,7 +3,7 @@ use alloy::primitives::Address; use anyhow::Result; use cipher::Cipher; use config::AppConfig; -use enclave_core::EventBus; +use enclave_core::{get_tag, EventBus}; use evm::{ helpers::{create_readonly_provider, ensure_ws_rpc}, CiphernodeRegistrySol, EnclaveSolReader, @@ -16,11 +16,13 @@ use router::{ CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, RepositoriesFactory, }; use sortition::Sortition; +use tracing::instrument; use std::sync::{Arc, Mutex}; use tokio::task::JoinHandle; use crate::setup_datastore; +#[instrument(name="app", skip_all,fields(id = get_tag()))] pub async fn setup_ciphernode( config: AppConfig, address: Address, diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index 66654cce..4981c4f6 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -17,7 +17,7 @@ use futures_util::stream::StreamExt; use std::collections::HashSet; use tokio::select; use tokio::sync::oneshot; -use tracing::{error, info, info_span, instrument, trace, warn}; +use tracing::{error, info, instrument, trace, warn}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] #[rtype(result = "()")] @@ -106,10 +106,8 @@ where } } + #[instrument(name="evm_event_reader", skip_all, fields(id = get_tag()))] pub async fn load(params: EvmEventReaderParams) -> Result { - let id = get_tag(); - let span = info_span!("evm_event_reader", %id); - let _guard = span.enter(); Ok(if let Some(snapshot) = params.repository.read().await? { info!("Loading from snapshot"); Self::from_snapshot(params, snapshot).await? @@ -283,10 +281,9 @@ where T: Transport + Clone + Unpin, { type Result = (); + + #[instrument(name="evm_event_reader", skip_all, fields(id = get_tag()))] fn handle(&mut self, wrapped: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { - let id = get_tag(); - let span = info_span!("evm_event_reader", %id); - let _guard = span.enter(); let event_id = wrapped.get_id(); info!("Processing event: {}", event_id); info!("cache length: {}", self.state.ids.len()); diff --git a/packages/ciphernode/sortition/src/sortition.rs b/packages/ciphernode/sortition/src/sortition.rs index 73c2db04..a41c9fa9 100644 --- a/packages/ciphernode/sortition/src/sortition.rs +++ b/packages/ciphernode/sortition/src/sortition.rs @@ -9,7 +9,7 @@ use enclave_core::{ EventBus, Seed, Subscribe, }; use std::collections::HashSet; -use tracing::{info, info_span, instrument}; +use tracing::{info, instrument}; #[derive(Message, Clone, Debug, PartialEq, Eq)] #[rtype(result = "bool")] @@ -107,6 +107,7 @@ impl Sortition { } } + #[instrument(name="sortition", skip_all, fields(id = get_tag()))] pub async fn attach( bus: &Addr, store: Repository, @@ -121,10 +122,8 @@ impl Sortition { Ok(addr) } + #[instrument(name="sortition", skip_all, fields(id = get_tag()))] pub async fn load(params: SortitionParams) -> Result { - let id = get_tag(); - let span = info_span!("sorition", %id); - let _guard = span.enter(); Ok(if let Some(snapshot) = params.store.read().await? { info!("Loading from snapshot"); Self::from_snapshot(params, snapshot).await? @@ -154,8 +153,13 @@ impl Snapshot for Sortition { impl FromSnapshotWithParams for Sortition { type Params = SortitionParams; + #[instrument(name="sortition", skip_all, fields(id = get_tag()))] async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { info!("Loaded snapshot with {} nodes", snapshot.nodes().len()); + info!( + "Nodes:\n\n{:?}\n", + snapshot.nodes().into_iter().collect::>() + ); Ok(Sortition { bus: params.bus, store: params.store, @@ -183,20 +187,30 @@ impl Handler for Sortition { impl Handler for Sortition { type Result = (); + + #[instrument(name="sortition", skip_all, fields(id = get_tag()))] fn handle(&mut self, msg: CiphernodeAdded, _ctx: &mut Self::Context) -> Self::Result { + info!("Adding node: {}", msg.address); self.list.add(msg.address); + self.checkpoint(); } } impl Handler for Sortition { type Result = (); + + #[instrument(name="sortition", skip_all, fields(id = get_tag()))] fn handle(&mut self, msg: CiphernodeRemoved, _ctx: &mut Self::Context) -> Self::Result { + info!("Removing node: {}", msg.address); self.list.remove(msg.address); + self.checkpoint(); } } impl Handler for Sortition { type Result = bool; + + #[instrument(name="sortition", skip_all, fields(id = get_tag()))] fn handle(&mut self, msg: GetHasNode, _ctx: &mut Self::Context) -> Self::Result { match self.list.contains(msg.seed, msg.size, msg.address) { Ok(val) => val, diff --git a/tests/basic_integration/base.sh b/tests/basic_integration/base.sh index a774eff3..3fcf7cae 100755 --- a/tests/basic_integration/base.sh +++ b/tests/basic_integration/base.sh @@ -10,7 +10,7 @@ source "$THIS_DIR/fns.sh" heading "Start the EVM node" -yarn evm:node & +launch_evm until curl -f -s "http://localhost:8545" > /dev/null; do sleep 1 diff --git a/tests/basic_integration/fns.sh b/tests/basic_integration/fns.sh index bfe8cd12..78f45a8e 100644 --- a/tests/basic_integration/fns.sh +++ b/tests/basic_integration/fns.sh @@ -10,8 +10,6 @@ if [[ "$ROOT_DIR" != "$(pwd)" ]]; then exit 1 fi -export RUST_LOG=info - # Environment variables RPC_URL="ws://localhost:8545" @@ -127,6 +125,14 @@ metallica() { pkill -9 -f "hardhat node" || true } +launch_evm() { + if [ ! -z "${SILENT_EVM:-}" ]; then + yarn evm:node &> /dev/null & + else + yarn evm:node & + fi +} + metallica # Set up trap to catch errors and interrupts diff --git a/tests/basic_integration/persist.sh b/tests/basic_integration/persist.sh index bfbffc37..42a2cb1b 100755 --- a/tests/basic_integration/persist.sh +++ b/tests/basic_integration/persist.sh @@ -10,7 +10,7 @@ source "$THIS_DIR/fns.sh" heading "Start the EVM node" -yarn evm:node & +launch_evm until curl -f -s "http://localhost:8545" > /dev/null; do sleep 1 diff --git a/tests/basic_integration/test.sh b/tests/basic_integration/test.sh index 4c733ab3..5377d3b7 100755 --- a/tests/basic_integration/test.sh +++ b/tests/basic_integration/test.sh @@ -4,17 +4,10 @@ set -eu # Exit immediately if a command exits with a non-zero status THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -# Source the file from the same directory -case $1 in - persist) - "$THIS_DIR/persist.sh" - ;; - base) - "$THIS_DIR/base.sh" - ;; - *) - "$THIS_DIR/persist.sh" - "$THIS_DIR/base.sh" - ;; -esac +if [ $# -eq 0 ]; then + "$THIS_DIR/persist.sh" + "$THIS_DIR/base.sh" +else + "$THIS_DIR/$1.sh" +fi From d55f793042ade414ca67462b0a9bc5ecc0ac55cf Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 12 Nov 2024 11:33:10 +0700 Subject: [PATCH 29/31] Formatting --- packages/ciphernode/data/src/data_store.rs | 2 +- packages/ciphernode/enclave/src/commands/start.rs | 2 +- packages/ciphernode/enclave_node/src/ciphernode.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index f4786f82..e81fc478 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -37,7 +37,7 @@ impl Get { } /// Generate proxy for the DB -#[derive(Clone,Debug)] +#[derive(Clone, Debug)] pub struct DataStore { scope: Vec, get: Recipient, diff --git a/packages/ciphernode/enclave/src/commands/start.rs b/packages/ciphernode/enclave/src/commands/start.rs index 17545224..3cea046d 100644 --- a/packages/ciphernode/enclave/src/commands/start.rs +++ b/packages/ciphernode/enclave/src/commands/start.rs @@ -1,9 +1,9 @@ +use crate::owo; use anyhow::{anyhow, Result}; use config::AppConfig; use enclave_core::get_tag; use enclave_node::{listen_for_shutdown, setup_ciphernode}; use tracing::{info, instrument}; -use crate::owo; #[instrument(name="app", skip_all,fields(id = get_tag()))] pub async fn execute(config: AppConfig) -> Result<()> { diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index b2ba2ece..f1fd54cd 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -16,9 +16,9 @@ use router::{ CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, RepositoriesFactory, }; use sortition::Sortition; -use tracing::instrument; use std::sync::{Arc, Mutex}; use tokio::task::JoinHandle; +use tracing::instrument; use crate::setup_datastore; From e2a88211fde26ba1062c3c10bb1dcd47187a7126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Tue, 12 Nov 2024 11:45:22 +0700 Subject: [PATCH 30/31] Update packages/ciphernode/core/src/tag.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- packages/ciphernode/core/src/tag.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/ciphernode/core/src/tag.rs b/packages/ciphernode/core/src/tag.rs index 6636cca6..133c5809 100644 --- a/packages/ciphernode/core/src/tag.rs +++ b/packages/ciphernode/core/src/tag.rs @@ -1,5 +1,14 @@ +//! Tag management for EVM event processing. +//! +//! This module provides thread-safe access to a global string tag that's used to +//! differentiate between different EVM contract instances during event processing. +//! The tag helps track and manage historical and live events for specific contracts. + use std::sync::OnceLock; +/// Global tag for contract event tracking with a default value of "_". +/// This tag is initialized once and remains constant throughout the lifecycle +/// of event processing to ensure consistent event tracking across restarts. static TAG: OnceLock = OnceLock::new(); pub fn get_tag() -> String { From 113ddff72a354e8e6b85f1031ef72772286edd33 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 12 Nov 2024 11:47:47 +0700 Subject: [PATCH 31/31] Formatting --- packages/ciphernode/core/src/tag.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ciphernode/core/src/tag.rs b/packages/ciphernode/core/src/tag.rs index 133c5809..7b67ebba 100644 --- a/packages/ciphernode/core/src/tag.rs +++ b/packages/ciphernode/core/src/tag.rs @@ -1,5 +1,5 @@ //! Tag management for EVM event processing. -//! +//! //! This module provides thread-safe access to a global string tag that's used to //! differentiate between different EVM contract instances during event processing. //! The tag helps track and manage historical and live events for specific contracts.