From 31b9a3f395d6386b9b6d23c462c09e81c2bcb422 Mon Sep 17 00:00:00 2001 From: Andrea Date: Wed, 13 Nov 2024 16:28:43 +0100 Subject: [PATCH] feat(tests): add resharding chain fork tests (#12440) PR to add simple chain fork and block double signing tests to the existing resharding_v3 test loop. The implementation is a bit hacky, but I couldn't find a better way. - Refactored adversarial block production to extrapolate a method in `Client` with additional functionality. - Ignoring `Challenge`s in test loop. - Refactored `resharding_v3` test structure to pass more parameters to the base test scenario. - Added an "hook" to execute arbitrary test code inside test loop iteration of `resharding_v3`. Both tests fail at the moment. They seems to do trigger the behavior we want to test. For forks consecutive resharding starts: ``` {block_hash=F5zmejS8RxnGwnJhtfH5JK4V5VbciBDrfJMwKuxpio1L block_height=13 parent_shard_uid=s1.v3}: memtrie: Freezing parent memtrie, creating children memtries... parent_shard_uid=s1.v3 children_shard_uids=[s2.v3, s3.v3] ... {block_hash=9kPADkyL3uKKFfupzunTxXTTchu5ek4CZ5eeWyvxzken block_height=18 parent_shard_uid=s1.v3}: memtrie: Freezing parent memtrie, creating children memtries... parent_shard_uid=s1.v3 children_shard_uids=[s2.v3, s3.v3] ``` For double signing two resharding at the same height: ``` {block_hash=F5zmejS8RxnGwnJhtfH5JK4V5VbciBDrfJMwKuxpio1L block_height=13 parent_shard_uid=s1.v3}: memtrie: Freezing parent memtrie, creating children memtries... parent_shard_uid=s1.v3 children_shard_uids=[s2.v3, s3.v3] ... {block_hash=EjweFJq1aekKSQKgZpMQ9sgNt6RrHYCADV91p7uLLUnv block_height=13 parent_shard_uid=s1.v3}: memtrie: Freezing parent memtrie, creating children memtries... parent_shard_uid=s1.v3 children_shard_uids=[s2.v3, s3.v3] ``` --- chain/client/src/client_actor.rs | 142 ++++++++--- chain/network/src/test_loop.rs | 2 +- .../src/test_loop/tests/resharding_v3.rs | 229 ++++++++++++++++-- 3 files changed, 314 insertions(+), 59 deletions(-) diff --git a/chain/client/src/client_actor.rs b/chain/client/src/client_actor.rs index 2c3d6d9a3fa..17fa337c51e 100644 --- a/chain/client/src/client_actor.rs +++ b/chain/client/src/client_actor.rs @@ -400,6 +400,29 @@ pub enum AdvProduceChunksMode { StopProduce, } +#[cfg(feature = "test_features")] +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub enum AdvProduceBlockHeightSelection { + /// Place the new block on top of the latest known block. Block's height will be the next + /// integer. + NextHeightOnLatestKnown, + /// Place the new block on top of the latest known block. Block height is arbitrary. + SelectedHeightOnLatestKnown { produced_block_height: BlockHeight }, + /// Place the new block on top of the current head. Block's height will be the next integer. + NextHeightOnCurrentHead, + /// Place the new block on top of current head. Block height is arbitrary. + SelectedHeightOnCurrentHead { produced_block_height: BlockHeight }, + /// Place the new block on top of an existing block at height `base_block_height`. Block's + /// height will be the next integer. + NextHeightOnSelectedBlock { base_block_height: BlockHeight }, + /// Place the new block on top of an existing block at height `base_block_height`. Block height + /// is arbitrary. + SelectedHeightOnSelectedBlock { + produced_block_height: BlockHeight, + base_block_height: BlockHeight, + }, +} + #[cfg(feature = "test_features")] #[derive(actix::Message, Debug)] #[rtype(result = "Option")] @@ -430,38 +453,11 @@ impl Handler for ClientActorInner { None } NetworkAdversarialMessage::AdvProduceBlocks(num_blocks, only_valid) => { - info!(target: "adversary", num_blocks, "Starting adversary blocks production"); - if only_valid { - self.client.adv_produce_blocks = Some(AdvProduceBlocksMode::OnlyValid); - } else { - self.client.adv_produce_blocks = Some(AdvProduceBlocksMode::All); - } - let start_height = - self.client.chain.mut_chain_store().get_latest_known().unwrap().height + 1; - let signer = self.client.validator_signer.get(); - let mut blocks_produced = 0; - for height in start_height.. { - let block = - self.client.produce_block(height).expect("block should be produced"); - if only_valid && block == None { - continue; - } - let block = block.expect("block should exist after produced"); - info!(target: "adversary", blocks_produced, num_blocks, height, "Producing adversary block"); - self.network_adapter.send(PeerManagerMessageRequest::NetworkRequests( - NetworkRequests::Block { block: block.clone() }, - )); - let _ = self.client.start_process_block( - block.into(), - Provenance::PRODUCED, - Some(self.myself_sender.apply_chunks_done.clone()), - &signer, - ); - blocks_produced += 1; - if blocks_produced == num_blocks { - break; - } - } + self.adv_produce_blocks_on( + num_blocks, + only_valid, + AdvProduceBlockHeightSelection::NextHeightOnLatestKnown, + ); None } NetworkAdversarialMessage::AdvSwitchToHeight(height) => { @@ -2096,6 +2092,88 @@ impl ClientActorInner { } true } + + /// Produces `num_blocks` number of blocks. + /// + /// The parameter `height_selection` governs the produced blocks' heights and what base block + /// height they are placed. + #[cfg(feature = "test_features")] + pub fn adv_produce_blocks_on( + &mut self, + num_blocks: BlockHeight, + only_valid: bool, + height_selection: AdvProduceBlockHeightSelection, + ) { + use AdvProduceBlockHeightSelection::*; + + info!(target: "adversary", num_blocks, "Starting adversary blocks production"); + if only_valid { + self.client.adv_produce_blocks = Some(AdvProduceBlocksMode::OnlyValid); + } else { + self.client.adv_produce_blocks = Some(AdvProduceBlocksMode::All); + } + let (start_height, prev_block_height) = match height_selection { + NextHeightOnLatestKnown => { + let latest_height = + self.client.chain.mut_chain_store().get_latest_known().unwrap().height; + (latest_height + 1, latest_height) + } + SelectedHeightOnLatestKnown { produced_block_height } => ( + produced_block_height, + self.client.chain.mut_chain_store().get_latest_known().unwrap().height, + ), + NextHeightOnCurrentHead => { + let head_height = self.client.chain.mut_chain_store().head().unwrap().height; + (head_height + 1, head_height) + } + SelectedHeightOnCurrentHead { produced_block_height } => { + (produced_block_height, self.client.chain.mut_chain_store().head().unwrap().height) + } + NextHeightOnSelectedBlock { base_block_height } => { + (base_block_height + 1, base_block_height) + } + SelectedHeightOnSelectedBlock { produced_block_height, base_block_height } => { + (produced_block_height, base_block_height) + } + }; + let is_based_on_current_head = + prev_block_height == self.client.chain.mut_chain_store().head().unwrap().height; + let signer = self.client.validator_signer.get(); + let mut blocks_produced = 0; + for height in start_height.. { + let block: Option = if is_based_on_current_head { + self.client.produce_block(height).expect("block should be produced") + } else { + let prev_block_hash = self + .client + .chain + .chain_store() + .get_block_hash_by_height(prev_block_height) + .expect("prev block should exist"); + self.client + .produce_block_on(height, prev_block_hash) + .expect("block should be produced") + }; + if only_valid && block == None { + continue; + } + let block = block.expect("block should exist after produced"); + info!(target: "adversary", blocks_produced, num_blocks, height, "Producing adversary block"); + self.network_adapter.send(PeerManagerMessageRequest::NetworkRequests( + NetworkRequests::Block { block: block.clone() }, + )); + let _ = self.client.start_process_block( + block.into(), + Provenance::PRODUCED, + Some(self.myself_sender.apply_chunks_done.clone()), + &signer, + ); + blocks_produced += 1; + if blocks_produced == num_blocks { + break; + } + } + } } impl Handler for ClientActorInner { diff --git a/chain/network/src/test_loop.rs b/chain/network/src/test_loop.rs index d137e3e6659..08e01385487 100644 --- a/chain/network/src/test_loop.rs +++ b/chain/network/src/test_loop.rs @@ -288,7 +288,7 @@ fn network_message_to_client_handler( None } NetworkRequests::StateRequestPart { .. } => None, - + NetworkRequests::Challenge(_) => None, _ => Some(request), }) } diff --git a/integration-tests/src/test_loop/tests/resharding_v3.rs b/integration-tests/src/test_loop/tests/resharding_v3.rs index a21102620c5..d5114b19c5f 100644 --- a/integration-tests/src/test_loop/tests/resharding_v3.rs +++ b/integration-tests/src/test_loop/tests/resharding_v3.rs @@ -1,6 +1,6 @@ use borsh::BorshDeserialize; use itertools::Itertools; -use near_async::test_loop::data::TestLoopData; +use near_async::test_loop::data::{TestLoopData, TestLoopDataHandle}; use near_async::time::Duration; use near_chain::ChainStoreAccess; use near_chain_configs::test_genesis::TestGenesisBuilder; @@ -10,7 +10,7 @@ use near_primitives::epoch_manager::EpochConfigStore; use near_primitives::hash::CryptoHash; use near_primitives::shard_layout::{account_id_to_shard_uid, ShardLayout}; use near_primitives::state_record::StateRecord; -use near_primitives::types::AccountId; +use near_primitives::types::{AccountId, BlockHeightDelta}; use near_primitives::version::{ProtocolFeature, PROTOCOL_VERSION}; use near_store::adapter::StoreAdapter; use near_store::db::refcount::decode_value_with_rc; @@ -21,6 +21,7 @@ use std::sync::Arc; use crate::test_loop::builder::TestLoopBuilder; use crate::test_loop::env::TestLoopEnv; use crate::test_loop::utils::ONE_NEAR; +use near_client::client_actor::ClientActorInner; fn print_and_assert_shard_accounts(client: &Client) { let tip = client.chain.head().unwrap(); @@ -81,6 +82,147 @@ fn check_state_shard_uid_mapping_after_resharding(client: &Client, parent_shard_ } } +#[derive(Default)] +struct TestReshardingParameters { + chunk_ranges_to_drop: HashMap>, + accounts: Vec, + clients: Vec, + block_and_chunk_producers: Vec, + initial_balance: u128, + epoch_length: BlockHeightDelta, + /// Custom behavior executed at every iteration of test loop. + loop_action: Option)>>, +} + +impl TestReshardingParameters { + fn new() -> Self { + Self::with_clients(3) + } + + fn with_clients(num_clients: u64) -> Self { + let num_accounts = 8; + let initial_balance = 1_000_000 * ONE_NEAR; + let epoch_length = 6; + + // #12195 prevents number of BPs bigger than `epoch_length`. + assert!(num_clients > 0 && num_clients <= epoch_length); + + let accounts = (0..num_accounts) + .map(|i| format!("account{}", i).parse().unwrap()) + .collect::>(); + + // This piece of code creates `num_clients` from `accounts`. First client is at index 0 and + // other clients are spaced in the accounts' space as evenly as possible. + let clients_per_account = num_clients as f64 / accounts.len() as f64; + let mut client_parts = 1.0 - clients_per_account; + let clients: Vec<_> = accounts + .iter() + .filter(|_| { + client_parts += clients_per_account; + if client_parts >= 1.0 { + client_parts -= 1.0; + true + } else { + false + } + }) + .cloned() + .collect(); + + let block_and_chunk_producers = clients.clone(); + + Self { + accounts, + clients, + block_and_chunk_producers, + initial_balance, + epoch_length, + ..Default::default() + } + } + + fn chunk_ranges_to_drop( + mut self, + chunk_ranges_to_drop: HashMap>, + ) -> Self { + self.chunk_ranges_to_drop = chunk_ranges_to_drop; + self + } + + #[allow(unused)] + fn clients(mut self, clients: Vec) -> Self { + self.clients = clients; + self + } + + #[allow(unused)] + fn block_and_chunk_producers(mut self, block_and_chunk_producers: Vec) -> Self { + self.block_and_chunk_producers = block_and_chunk_producers; + self + } + + #[allow(unused)] + fn loop_action( + mut self, + loop_action: Option)>>, + ) -> Self { + self.loop_action = loop_action; + self + } +} + +// Returns a callable function that, when invoked inside a test loop iteration, can force the creation of a chain fork. +#[cfg(feature = "test_features")] +fn fork_before_resharding_block( + double_signing: bool, +) -> Box)> { + use near_client::client_actor::AdvProduceBlockHeightSelection; + + let done = std::cell::Cell::new(false); + Box::new( + move |test_loop_data: &mut TestLoopData, + client_handle: TestLoopDataHandle| { + // It must happen only for the first resharding block encountered. + if done.get() { + return; + } + + let client_actor = &mut test_loop_data.get_mut(&client_handle); + let tip = client_actor.client.chain.head().unwrap(); + + // We want to understand if the most recent block is a resharding block. + // To do this check if the latest block is an epoch start and compare the two epochs' shard layouts. + let epoch_manager = client_actor.client.epoch_manager.clone(); + let shard_layout = epoch_manager.get_shard_layout(&tip.epoch_id).unwrap(); + let next_epoch_id = + epoch_manager.get_next_epoch_id_from_prev_block(&tip.prev_block_hash).unwrap(); + let next_shard_layout = epoch_manager.get_shard_layout(&next_epoch_id).unwrap(); + let next_block_has_new_shard_layout = + epoch_manager.is_next_block_epoch_start(&tip.last_block_hash).unwrap() + && shard_layout != next_shard_layout; + + // If there's a new shard layout force a chain fork. + if next_block_has_new_shard_layout { + println!("creating chain fork at height {}", tip.height); + let height_selection = if double_signing { + // In the double signing scenario we want a new block on top of prev block, with consecutive height. + AdvProduceBlockHeightSelection::NextHeightOnSelectedBlock { + base_block_height: tip.height - 1, + } + } else { + // To avoid double signing skip already produced height. + AdvProduceBlockHeightSelection::SelectedHeightOnSelectedBlock { + produced_block_height: tip.height + 1, + base_block_height: tip.height - 1, + } + }; + client_actor.adv_produce_blocks_on(3, true, height_selection); + done.set(true); + } + }, + ) +} + /// Base setup to check sanity of Resharding V3. /// TODO(#11881): add the following scenarios: /// - Nodes must not track all shards. State sync must succeed. @@ -90,7 +232,7 @@ fn check_state_shard_uid_mapping_after_resharding(client: &Client, parent_shard_ /// - Cross-shard receipts of all kinds, crossing resharding boundary. /// - Shard layout v2 -> v2 transition. /// - Shard layout can be taken from mainnet. -fn test_resharding_v3_base(chunk_ranges_to_drop: HashMap>) { +fn test_resharding_v3_base(params: TestReshardingParameters) { if !ProtocolFeature::SimpleNightshadeV4.enabled(PROTOCOL_VERSION) { return; } @@ -98,15 +240,6 @@ fn test_resharding_v3_base(chunk_ranges_to_drop: HashMap>(); - // #12195 prevents number of BPs bigger than `epoch_length`. - let clients = vec![accounts[0].clone(), accounts[3].clone(), accounts[6].clone()]; - let block_and_chunk_producers = - clients.iter().map(|account: &AccountId| account.as_str()).collect_vec(); - // Prepare shard split configuration. let base_epoch_config_store = EpochConfigStore::for_chain_id("mainnet", None).unwrap(); let base_protocol_version = ProtocolFeature::SimpleNightshadeV4.protocol_version() - 1; @@ -114,7 +247,7 @@ fn test_resharding_v3_base(chunk_ranges_to_drop: HashMap bool { + params.loop_action.as_ref().map(|action| action(test_loop_data, client_handle.clone())); + let client = &test_loop_data.get(&client_handle).client; let tip = client.chain.head().unwrap(); @@ -171,7 +316,7 @@ fn test_resharding_v3_base(chunk_ranges_to_drop: HashMap