diff --git a/circuit_encodings/src/lib.rs b/circuit_encodings/src/lib.rs index a1021cf2..e9f4c313 100644 --- a/circuit_encodings/src/lib.rs +++ b/circuit_encodings/src/lib.rs @@ -316,8 +316,6 @@ impl< pub struct FullWidthQueueIntermediateStates { pub head: [F; SW], pub tail: [F; SW], - pub old_head: [F; SW], - pub old_tail: [F; SW], pub num_items: u32, pub round_function_execution_pairs: [([F; SW], [F; SW]); ROUNDS], } @@ -433,8 +431,6 @@ impl< let intermediate_info = FullWidthQueueIntermediateStates { head: self.head, tail: new_tail, - old_head: self.head, - old_tail, num_items: self.num_items, round_function_execution_pairs: states, }; @@ -473,8 +469,6 @@ impl< let intermediate_info = FullWidthQueueIntermediateStates { head: self.head, tail: self.tail, - old_head, - old_tail: self.tail, num_items: self.num_items, round_function_execution_pairs: states, }; @@ -562,6 +556,12 @@ impl< } } + pub fn replace_container(mut self, container: C) -> (Self, C) { + let prev_container = self.witness; + self.witness = container; + (self, prev_container) + } + pub fn take_sponge_like_queue_state(&self) -> QueueStateWitness { let result = QueueStateWitness { head: self.head, @@ -605,8 +605,6 @@ impl< let intermediate_info = FullWidthQueueIntermediateStates { head: self.head, tail: new_tail, - old_head: self.head, - old_tail, num_items: self.num_items, round_function_execution_pairs: states, }; @@ -644,7 +642,7 @@ pub struct FullWidthStackSimulator< > { pub state: [F; SW], pub num_items: u32, - pub witness: Vec<([F; N], [F; SW], I)>, + pub witness: Vec<([F; SW], I)>, } impl< @@ -697,7 +695,7 @@ impl< let states = make_round_function_pairs(old_state, states); - self.witness.push((encoding, self.state, element)); + self.witness.push((self.state, element)); self.num_items += 1; self.state = new_state; @@ -727,7 +725,7 @@ impl< let popped = self.witness.pop().unwrap(); self.num_items -= 1; - let (_element_encoding, previous_state, element) = popped; + let (previous_state, element) = popped; let encoding = element.encoding_witness(); let mut state = previous_state; @@ -867,6 +865,9 @@ mod tests { } assert_eq!(queue.num_items, 10); + let old_head = queue.head; + let old_tail = queue.tail; + // pop one element let (element, data) = queue.pop_and_output_intermediate_data(&round_function); // it should return the first one that we entered (with circuit 0). @@ -876,8 +877,8 @@ mod tests { assert_eq!(data.num_items, 9); assert_eq!(data.head, tail_after_first); - assert_eq!(data.old_head, empty_head); - assert_eq!(data.old_tail, data.tail); + assert_eq!(old_head, empty_head); + assert_eq!(data.tail, old_tail); let mut parts = queue.split_by(3, &round_function); diff --git a/circuit_encodings/src/memory_query.rs b/circuit_encodings/src/memory_query.rs index d65d6b30..19a6994f 100644 --- a/circuit_encodings/src/memory_query.rs +++ b/circuit_encodings/src/memory_query.rs @@ -157,3 +157,7 @@ impl CircuitEquivalentReflection for MemoryQuery { } } } + +use zkevm_circuits::base_structures::memory_query::MemoryQueryWitness; +pub type MemoryQueueStateWitnesses = + Vec<(MemoryQueryWitness, [F; FULL_SPONGE_QUEUE_STATE_WIDTH])>; diff --git a/src/external_calls.rs b/src/external_calls.rs index a566ca62..a0e2aa50 100644 --- a/src/external_calls.rs +++ b/src/external_calls.rs @@ -5,6 +5,7 @@ pub use crate::run_vms::SCHEDULER_TIMESTAMP; use crate::snark_wrapper::boojum::field::goldilocks::GoldilocksExt2; use crate::snark_wrapper::boojum::gadgets::recursion::recursive_tree_hasher::CircuitGoldilocksPoseidon2Sponge; use crate::toolset::GeometryConfig; +use crate::witness::oracle::WitnessGenerationArtifact; use crate::witness::tree::BinarySparseStorageTree; use crate::witness::tree::ZkSyncStorageLeaf; use crate::zk_evm::abstractions::Storage; @@ -28,15 +29,7 @@ use circuit_definitions::Field as MainField; /// /// This function will setup the environment and will run out-of-circuit and then in-circuit. /// GenericNoopTracer will be used as out-of-circuit tracer -pub fn run< - S: Storage, - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +pub fn run( caller: Address, // for real block must be zero entry_point_address: Address, // for real block must be the bootloader entry_point_code: Vec<[u8; 32]>, // for read block must be a bootloader code @@ -52,8 +45,7 @@ pub fn run< tree: impl BinarySparseStorageTree<256, 32, 32, 8, 32, Blake2s256, ZkSyncStorageLeaf>, trusted_setup_path: &str, eip_4844_repack_inputs: [Option>; MAX_4844_BLOBS_PER_BLOCK], - circuit_callback: CB, - queue_simulator_callback: QSCB, + artifacts_callback: CB, ) -> ( SchedulerCircuitInstanceWitness, BlockAuxilaryOutputWitness, @@ -75,8 +67,7 @@ pub fn run< tree, trusted_setup_path, eip_4844_repack_inputs, - circuit_callback, - queue_simulator_callback, + artifacts_callback, &mut out_of_circuit_tracer, ) { Ok((scheduler_circuit_witness, aux_data)) => (scheduler_circuit_witness, aux_data), diff --git a/src/run_vms.rs b/src/run_vms.rs index a7821266..3dce3fad 100644 --- a/src/run_vms.rs +++ b/src/run_vms.rs @@ -7,6 +7,7 @@ use crate::snark_wrapper::boojum::gadgets::recursion::recursive_tree_hasher::Cir use crate::toolset::create_tools; use crate::toolset::GeometryConfig; use crate::witness::oracle::create_artifacts_from_tracer; +use crate::witness::oracle::WitnessGenerationArtifact; use crate::witness::tracer::tracer::WitnessTracer; use crate::witness::tree::BinarySparseStorageTree; use crate::witness::tree::ZkSyncStorageLeaf; @@ -61,15 +62,7 @@ pub type RunVMsResult = ( /// - witness with AUX data (with information that might be useful during verification to generate the public input) /// /// This function will setup the environment and will run out-of-circuit and then in-circuit -pub fn run_vms< - S: Storage, - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +pub fn run_vms( caller: Address, // for real block must be zero entry_point_address: Address, // for real block must be the bootloader entry_point_code: Vec<[u8; 32]>, // for read block must be a bootloader code @@ -85,8 +78,7 @@ pub fn run_vms< tree: impl BinarySparseStorageTree<256, 32, 32, 8, 32, Blake2s256, ZkSyncStorageLeaf>, trusted_setup_path: &str, eip_4844_repack_inputs: [Option>; MAX_4844_BLOBS_PER_BLOCK], - circuit_callback: CB, - queue_simulator_callback: QSCB, + artifacts_callback: CB, out_of_circuit_tracer: &mut impl Tracer, ) -> Result { let round_function = ZkSyncDefaultRoundFunction::default(); @@ -254,8 +246,7 @@ pub fn run_vms< evm_simulator_code_hash, eip_4844_repack_inputs.clone(), trusted_setup_path, - circuit_callback, - queue_simulator_callback, + artifacts_callback, ); let (scheduler_circuit_witness, aux_data) = { diff --git a/src/tests/complex_tests/mod.rs b/src/tests/complex_tests/mod.rs index c9459288..932846e9 100644 --- a/src/tests/complex_tests/mod.rs +++ b/src/tests/complex_tests/mod.rs @@ -39,6 +39,7 @@ use crate::zk_evm::GenericNoopTracer; use crate::zkevm_circuits::eip_4844::input::*; use crate::zkevm_circuits::scheduler::block_header::MAX_4844_BLOBS_PER_BLOCK; use crate::zkevm_circuits::scheduler::input::SchedulerCircuitInstanceWitness; +use boojum::gadgets::queue::full_state_queue::FullStateCircuitQueueRawWitness; use circuit_definitions::aux_definitions::witness_oracle::VmWitnessOracle; use circuit_definitions::circuit_definitions::aux_layer::compression::{ self, CompressionMode1Circuit, @@ -58,6 +59,7 @@ use circuit_definitions::{ use circuit_definitions::{Field, RoundFunction}; use utils::read_basic_test_artifact; +use witness::oracle::WitnessGenerationArtifact; use zkevm_assembly::Assembly; #[ignore = "Too slow"] @@ -255,6 +257,27 @@ pub(crate) fn generate_base_layer( let mut basic_block_circuits = vec![]; let mut recursion_queues = vec![]; + let mut unsorted_memory_queue_witnesses = vec![]; + let mut sorted_memory_queue_witnesses = vec![]; + + let artifacts_callback = |artifact: WitnessGenerationArtifact| match artifact { + WitnessGenerationArtifact::BaseLayerCircuit(circuit) => basic_block_circuits.push(circuit), + WitnessGenerationArtifact::RecursionQueue((a, b, c)) => recursion_queues.push(( + a, + b, + c.into_iter() + .map(|x| ZkSyncBaseLayerStorage::from_inner(a as u8, x)) + .collect(), + )), + WitnessGenerationArtifact::MemoryQueueWitness((witnesses, sorted)) => { + if sorted { + sorted_memory_queue_witnesses.push(witnesses); + } else { + unsorted_memory_queue_witnesses.push(witnesses); + } + } + }; + let (scheduler_partial_input, _aux_data) = run( Address::zero(), test_artifact.entry_point_address, @@ -271,18 +294,28 @@ pub(crate) fn generate_base_layer( tree, "kzg/src/trusted_setup.json", blobs, - |circuit| basic_block_circuits.push(circuit), - |a, b, c| { - recursion_queues.push(( - a, - b, - c.into_iter() - .map(|x| ZkSyncBaseLayerStorage::from_inner(a as u8, x)) - .collect(), - )) - }, + artifacts_callback, ); + let mut unsorted_memory_queue_witnesses_it = unsorted_memory_queue_witnesses.into_iter(); + let mut sorted_memory_queue_witnesses = sorted_memory_queue_witnesses.into_iter(); + for el in basic_block_circuits.iter_mut() { + match &el { + ZkSyncBaseLayerCircuit::RAMPermutation(inner) => { + let mut witness = inner.witness.take().unwrap(); + witness.sorted_queue_witness = FullStateCircuitQueueRawWitness { + elements: sorted_memory_queue_witnesses.next().unwrap().into(), + }; + witness.unsorted_queue_witness = FullStateCircuitQueueRawWitness { + elements: unsorted_memory_queue_witnesses_it.next().unwrap().into(), + }; + + inner.witness.store(Some(witness)); + } + _ => {} + } + } + ( basic_block_circuits, recursion_queues, diff --git a/src/tests/run_manually.rs b/src/tests/run_manually.rs index 6d3311cf..2cc04a2a 100644 --- a/src/tests/run_manually.rs +++ b/src/tests/run_manually.rs @@ -20,9 +20,11 @@ use crate::zk_evm::witness_trace::VmWitnessTracer; use crate::zk_evm::GenericNoopTracer; use crate::zkevm_circuits::base_structures::vm_state::GlobalContextWitness; use crate::zkevm_circuits::main_vm::main_vm_entry_point; +use boojum::gadgets::queue::full_state_queue::FullStateCircuitQueueRawWitness; use circuit_definitions::aux_definitions::witness_oracle::VmWitnessOracle; use circuit_definitions::zk_evm::vm_state::cycle; use storage::{InMemoryCustomRefundStorage, StorageRefund}; +use witness::oracle::WitnessGenerationArtifact; use zkevm_assembly::Assembly; #[test] @@ -260,11 +262,25 @@ pub(crate) fn run_with_options(entry_point_bytecode: Vec<[u8; 32]>, options: Opt save_predeployed_contracts(&mut storage_impl.storage, &mut tree, &known_contracts); let mut basic_block_circuits = vec![]; + let mut unsorted_memory_queue_witnesses = vec![]; + let mut sorted_memory_queue_witnesses = vec![]; // we are using TestingTracer to track prints and exceptions inside out_of_circuit_vm cycles let mut out_of_circuit_tracer = TestingTracer::new(Some(storage_impl.create_refund_controller())); + let artifacts_callback = |artifact: WitnessGenerationArtifact| match artifact { + WitnessGenerationArtifact::BaseLayerCircuit(circuit) => basic_block_circuits.push(circuit), + WitnessGenerationArtifact::MemoryQueueWitness((witnesses, sorted)) => { + if sorted { + sorted_memory_queue_witnesses.push(witnesses) + } else { + unsorted_memory_queue_witnesses.push(witnesses) + } + } + _ => {} + }; + if let Err(err) = run_vms( Address::zero(), *BOOTLOADER_FORMAL_ADDRESS, @@ -281,8 +297,7 @@ pub(crate) fn run_with_options(entry_point_bytecode: Vec<[u8; 32]>, options: Opt tree, "kzg/src/trusted_setup.json", std::array::from_fn(|_| None), - |circuit| basic_block_circuits.push(circuit), - |_, _, _| {}, + artifacts_callback, &mut out_of_circuit_tracer, ) { let error_text = match err { @@ -303,8 +318,24 @@ pub(crate) fn run_with_options(entry_point_bytecode: Vec<[u8; 32]>, options: Opt println!("Simulation and witness creation are completed"); + let mut unsorted_memory_queue_witnesses_it = unsorted_memory_queue_witnesses.into_iter(); + let mut sorted_memory_queue_witnesses = sorted_memory_queue_witnesses.into_iter(); for el in basic_block_circuits { println!("Doing {} circuit", el.short_description()); + match &el { + ZkSyncBaseLayerCircuit::RAMPermutation(inner) => { + let mut witness = inner.witness.take().unwrap(); + witness.sorted_queue_witness = FullStateCircuitQueueRawWitness { + elements: sorted_memory_queue_witnesses.next().unwrap().into(), + }; + witness.unsorted_queue_witness = FullStateCircuitQueueRawWitness { + elements: unsorted_memory_queue_witnesses_it.next().unwrap().into(), + }; + + inner.witness.store(Some(witness)); + } + _ => {} + } base_test_circuit(el); } diff --git a/src/witness/artifacts.rs b/src/witness/artifacts.rs index d8fad4c3..0b818265 100644 --- a/src/witness/artifacts.rs +++ b/src/witness/artifacts.rs @@ -121,13 +121,6 @@ pub struct MemoryArtifacts { CircuitsEntryAccumulatorSparse<(u32, QueueStateWitness)>, } -#[derive(Derivative)] -#[derivative(Default)] -pub struct ImplicitMemoryArtifacts { - pub memory_queries: Vec, - pub memory_queue_states: Vec>, -} - #[derive(Derivative)] #[derivative(Default)] pub struct LogCircuitsArtifacts { diff --git a/src/witness/aux_data_structs/per_circuit_accumulator.rs b/src/witness/aux_data_structs/per_circuit_accumulator.rs index 3dc484c5..27ac3fd4 100644 --- a/src/witness/aux_data_structs/per_circuit_accumulator.rs +++ b/src/witness/aux_data_structs/per_circuit_accumulator.rs @@ -32,6 +32,10 @@ impl PerCircuitAccumulatorContainer { self.accumulated } + pub fn amount_of_circuits_accumulated(&self) -> usize { + self.circuits_data.len() + } + pub fn last(&self) -> Option<&T> { let last_batch = self.circuits_data.last(); last_batch?; @@ -181,6 +185,10 @@ impl PerCircuitAccumulator { self.container.len() } + pub fn amount_of_circuits_accumulated(&self) -> usize { + self.container.amount_of_circuits_accumulated() + } + pub fn push(&mut self, val: T) { assert!(self.container.cycles_per_circuit != 0); let idx = self.container.len(); diff --git a/src/witness/individual_circuits/log_demux.rs b/src/witness/individual_circuits/log_demux.rs index f0277924..6cc135b1 100644 --- a/src/witness/individual_circuits/log_demux.rs +++ b/src/witness/individual_circuits/log_demux.rs @@ -19,6 +19,7 @@ use circuit_definitions::encodings::recursion_request::RecursionQueueSimulator; use circuit_definitions::zkevm_circuits::demux_log_queue::DemuxOutput; use circuit_definitions::zkevm_circuits::scheduler::aux::BaseLayerCircuitType; use circuit_definitions::{encodings::*, Field, RoundFunction}; +use oracle::WitnessGenerationArtifact; use zk_evm::zkevm_opcode_defs::SECP256R1_VERIFY_PRECOMPILE_ADDRESS; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; @@ -172,17 +173,13 @@ impl DemuxedQueuesStatesSimulator { } /// Take a storage log, output logs separately for events, l1 messages, storage, etc -pub(crate) fn process_logs_demux_and_make_circuits< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut(u64, RecursionQueueSimulator, Vec>), ->( +pub(crate) fn process_logs_demux_and_make_circuits( mut log_demux_artifacts: LogDemuxCircuitArtifacts, demuxed_queues: &DemuxedLogQueries, per_circuit_capacity: usize, round_function: &RoundFunction, geometry: &GeometryConfig, - mut circuit_callback: CB, - mut recursion_queue_callback: QSCB, + mut artifacts_callback: CB, ) -> ( FirstAndLastCircuitWitness>, Vec>, @@ -208,11 +205,11 @@ pub(crate) fn process_logs_demux_and_make_circuits< { let (log_demux_circuits, queue_simulator, log_demux_circuits_compact_forms_witnesses) = maker.into_results(); - recursion_queue_callback( + artifacts_callback(WitnessGenerationArtifact::RecursionQueue(( circuit_type as u64, queue_simulator, log_demux_circuits_compact_forms_witnesses.clone(), - ); + ))); let (io_queues_states, precompiles_queues_states) = DemuxedQueuesStatesSimulator::build_empty(*round_function); @@ -391,18 +388,18 @@ pub(crate) fn process_logs_demux_and_make_circuits< } previous_hidden_fsm_output = Some(witness.closed_form_input.hidden_fsm_output.clone()); - circuit_callback(ZkSyncBaseLayerCircuit::LogDemuxer( - maker.process(witness, circuit_type), + artifacts_callback(WitnessGenerationArtifact::BaseLayerCircuit( + ZkSyncBaseLayerCircuit::LogDemuxer(maker.process(witness, circuit_type)), )); } let (log_demux_circuits, queue_simulator, log_demux_circuits_compact_forms_witnesses) = maker.into_results(); - recursion_queue_callback( + artifacts_callback(WitnessGenerationArtifact::RecursionQueue(( circuit_type as u64, queue_simulator, log_demux_circuits_compact_forms_witnesses.clone(), - ); + ))); for (sub_queue, mut iter) in queries_iterators { assert!( diff --git a/src/witness/individual_circuits/main_vm.rs b/src/witness/individual_circuits/main_vm.rs index 9fdce6af..7cd885f2 100644 --- a/src/witness/individual_circuits/main_vm.rs +++ b/src/witness/individual_circuits/main_vm.rs @@ -315,16 +315,10 @@ fn repack_input_for_main_vm( use crate::witness::postprocessing::observable_witness::VmObservableWitness; use crate::zkevm_circuits::fsm_input_output::circuit_inputs::main_vm::VmCircuitWitness; +use super::oracle::WitnessGenerationArtifact; use super::vm_instance_witness_to_circuit_formal_input; -pub(crate) fn process_main_vm< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +pub(crate) fn process_main_vm( geometry: &GeometryConfig, in_circuit_global_context: GlobalContextWitness, memory_artifacts_for_main_vm: MemoryArtifacts, @@ -341,8 +335,7 @@ pub(crate) fn process_main_vm< flat_new_frames_history: Vec<(Cycle, CallStackEntry)>, mut vm_snapshots: Vec, round_function: Poseidon2Goldilocks, - circuit_callback: &mut CB, - recursion_queue_callback: &mut QSCB, + artifacts_callback: &mut CB, ) -> ( FirstAndLastCircuitWitness>, Vec>, @@ -408,7 +401,7 @@ pub(crate) fn process_main_vm< }; queue_simulator.push(recursive_request, &round_function); - circuit_callback(instance); + artifacts_callback(WitnessGenerationArtifact::BaseLayerCircuit(instance)); main_vm_circuits_compact_forms_witnesses.push(compact_form_witness); }; @@ -540,11 +533,11 @@ pub(crate) fn process_main_vm< } } - recursion_queue_callback( + artifacts_callback(WitnessGenerationArtifact::RecursionQueue(( BaseLayerCircuitType::VM as u64, queue_simulator, main_vm_circuits_compact_forms_witnesses.clone(), - ); + ))); (main_vm_circuits, main_vm_circuits_compact_forms_witnesses) } diff --git a/src/witness/individual_circuits/memory_related/decommit_code.rs b/src/witness/individual_circuits/memory_related/decommit_code.rs index e77bb879..49a3d1a7 100644 --- a/src/witness/individual_circuits/memory_related/decommit_code.rs +++ b/src/witness/individual_circuits/memory_related/decommit_code.rs @@ -8,7 +8,6 @@ use crate::zk_evm::ethereum_types::U256; use crate::zkevm_circuits::base_structures::decommit_query::DecommitQueryWitness; use crate::zkevm_circuits::base_structures::decommit_query::DECOMMIT_QUERY_PACKED_WIDTH; use crate::zkevm_circuits::code_unpacker_sha256::input::*; -use artifacts::ImplicitMemoryArtifacts; use circuit_definitions::encodings::decommittment_request::normalized_preimage_as_u256; use circuit_definitions::encodings::decommittment_request::DecommittmentQueueSimulator; use circuit_definitions::encodings::decommittment_request::DecommittmentQueueState; @@ -17,13 +16,31 @@ use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::zk_evm::aux_structures::DecommittmentQuery; use std::collections::VecDeque; -// TODO docs -pub(crate) fn decommitter_memory_queries_amount( +pub(crate) fn decommitter_memory_queries( deduplicated_decommit_requests_with_data: &Vec<(DecommittmentQuery, Vec)>, -) -> usize { - deduplicated_decommit_requests_with_data - .iter() - .fold(0, |inner, (_, writes)| inner + writes.len()) +) -> Vec { + let mut result = vec![]; + for (query, writes) in deduplicated_decommit_requests_with_data.iter() { + assert!(query.is_fresh); + + // now feed the queries into it + let as_queries_it = writes.iter().enumerate().map(|(idx, el)| MemoryQuery { + timestamp: query.timestamp, + location: zk_evm::aux_structures::MemoryLocation { + memory_type: zk_evm::abstractions::MemoryType::Code, + page: query.memory_page, + index: MemoryIndex(idx as u32), + }, + rw_flag: true, + value: *el, + value_is_pointer: false, + }); + + // and plain test memory queues + result.extend(as_queries_it); + } + + result } pub(crate) struct DecommiterCircuitProcessingInputs { @@ -37,25 +54,27 @@ pub(crate) fn compute_decommitter_circuit_snapshots< R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( amount_of_memory_queries: usize, - implicit_memory_artifacts: &mut ImplicitMemoryArtifacts, - memory_queue_states_accumulator: &LastPerCircuitAccumulator>, - memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, + decommitter_memory_queries: Vec, + decommitter_simulator_snapshots: Vec>, + decommitter_memory_states: Vec>, + final_explicit_memory_queue_state: MemoryQueueState, decommiter_circuit_inputs: DecommiterCircuitProcessingInputs, round_function: &R, decommiter_circuit_capacity: usize, -) -> Vec> { +) -> (Vec>, usize) { assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() + decommitter_memory_queries.len(), + decommitter_memory_states.len() ); + + let memory_simulator_before = &decommitter_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries, + memory_simulator_before.num_items as usize ); + let start_idx_for_memory_accumulator = 0; - let start_idx_for_memory_accumulator = implicit_memory_artifacts.memory_queue_states.len(); - - let initial_memory_queue_state = &memory_queue_simulator.take_sponge_like_queue_state(); + let initial_memory_queue_state = &memory_simulator_before.take_sponge_like_queue_state(); let DecommiterCircuitProcessingInputs { deduplicated_decommit_requests_with_data, @@ -70,46 +89,6 @@ pub(crate) fn compute_decommitter_circuit_snapshots< "we must have some decommitment requests" ); - for (query, writes) in deduplicated_decommit_requests_with_data.iter() { - assert!(query.is_fresh); - - // now feed the queries into it - let as_queries: Vec<_> = writes - .iter() - .cloned() - .enumerate() - .map(|(idx, el)| MemoryQuery { - timestamp: query.timestamp, - location: zk_evm::aux_structures::MemoryLocation { - memory_type: zk_evm::abstractions::MemoryType::Code, - page: query.memory_page, - index: MemoryIndex(idx as u32), - }, - rw_flag: true, - value: el, - value_is_pointer: false, - }) - .collect(); - - // fill up the memory queue - for query in as_queries.iter() { - let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, round_function); - - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - } - - // and plain test memory queues - implicit_memory_artifacts.memory_queries.extend(as_queries); - } - - assert_eq!( - implicit_memory_artifacts.memory_queries.len(), - implicit_memory_artifacts.memory_queue_states.len() - ); - // our simulator is simple: it will try to take an element from the queue, run some number of rounds, and compare the results let mut results: Vec> = vec![]; @@ -189,10 +168,9 @@ pub(crate) fn compute_decommitter_circuit_snapshots< }; let wintess_state = if start_idx_for_memory_accumulator + memory_queue_state_offset == 0 { - memory_queue_states_accumulator.last().unwrap() + &final_explicit_memory_queue_state } else { - implicit_memory_artifacts - .memory_queue_states + decommitter_memory_states .get(start_idx_for_memory_accumulator + memory_queue_state_offset - 1) .unwrap() }; @@ -405,10 +383,9 @@ pub(crate) fn compute_decommitter_circuit_snapshots< ); let wintess_state = if start_idx_for_memory_accumulator + memory_queue_state_offset == 0 { - memory_queue_states_accumulator.last().unwrap() + &final_explicit_memory_queue_state } else { - implicit_memory_artifacts - .memory_queue_states + decommitter_memory_states .get(start_idx_for_memory_accumulator + memory_queue_state_offset - 1) .unwrap() }; @@ -449,14 +426,14 @@ pub(crate) fn compute_decommitter_circuit_snapshots< } } + let memory_simulator_after = &decommitter_simulator_snapshots[1]; + + let amount_of_memory_queries_after = + amount_of_memory_queries + decommitter_memory_queries.len(); assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() - ); - assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries_after, + memory_simulator_after.num_items as usize ); - results + (results, amount_of_memory_queries_after) } diff --git a/src/witness/individual_circuits/memory_related/ecrecover.rs b/src/witness/individual_circuits/memory_related/ecrecover.rs index e3f55daf..bb31b44a 100644 --- a/src/witness/individual_circuits/memory_related/ecrecover.rs +++ b/src/witness/individual_circuits/memory_related/ecrecover.rs @@ -1,8 +1,9 @@ use super::*; -use crate::witness::artifacts::{DemuxedLogQueries, ImplicitMemoryArtifacts, LogQueueStates}; +use crate::witness::artifacts::{DemuxedLogQueries, LogQueueStates}; use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; +use crate::zk_evm::aux_structures::MemoryQuery; use crate::zk_evm::zk_evm_abstractions::precompiles::ecrecover::ECRecoverRoundWitness; use crate::zkevm_circuits::base_structures::log_query::*; use crate::zkevm_circuits::ecrecover::*; @@ -10,14 +11,27 @@ use circuit_definitions::encodings::memory_query::MemoryQueueSimulator; use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::encodings::*; -pub(crate) fn ecrecover_memory_queries_amount( +pub(crate) fn ecrecover_memory_queries( ecrecover_witnesses: &Vec<(u32, LogQuery_, ECRecoverRoundWitness)>, -) -> usize { - ecrecover_witnesses +) -> Vec { + let amount_of_queries = ecrecover_witnesses .iter() .fold(0, |inner, (_, _, witness)| { inner + witness.reads.len() + witness.writes.len() - }) + }); + + let mut ecrecover_memory_queries = Vec::with_capacity(amount_of_queries); + + for (_cycle, _query, witness) in ecrecover_witnesses.iter() { + let initial_memory_len = ecrecover_memory_queries.len(); + + // we read, then write + ecrecover_memory_queries.extend_from_slice(&witness.reads); + ecrecover_memory_queries.extend_from_slice(&witness.writes); + + assert_eq!(ecrecover_memory_queries.len() - initial_memory_len, 6); + } + ecrecover_memory_queries } // we want to simulate splitting of data into many separate instances of the same circuit. @@ -28,59 +42,40 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< F: SmallField, R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( - amount_of_memory_queries: usize, - implicit_memory_artifacts: &mut ImplicitMemoryArtifacts, - memory_queue_states_accumulator: &LastPerCircuitAccumulator>, - memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, + amount_of_memory_queries_before: usize, + ecrecover_memory_queries: Vec, + ecrecover_simulator_snapshots: Vec>, + ecrecover_memory_states: Vec>, ecrecover_witnesses: Vec<(u32, LogQuery_, ECRecoverRoundWitness)>, ecrecover_queries: Vec, mut demuxed_ecrecover_queue: LogQueueStates, num_rounds_per_circuit: usize, round_function: &R, -) -> Vec> { +) -> (Vec>, usize) { assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() + ecrecover_memory_queries.len(), + ecrecover_memory_states.len() ); + + let memory_simulator_before = &ecrecover_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries_before, + memory_simulator_before.num_items as usize ); - // split into aux witness, don't mix with the memory - - use crate::zk_evm::zk_evm_abstractions::precompiles::ecrecover::ECRecoverRoundWitness; - let mut ecrecover_memory_queries = - Vec::with_capacity(ecrecover_memory_queries_amount(&ecrecover_witnesses)); - - for (_cycle, _query, witness) in ecrecover_witnesses.iter() { - let ECRecoverRoundWitness { - new_request: _, - reads, - writes, - } = witness; - - // we read, then write - ecrecover_memory_queries.extend_from_slice(reads); - - ecrecover_memory_queries.extend_from_slice(writes); - } - let mut result = vec![]; let precompile_calls = ecrecover_queries; let simulator_witness: Vec<_> = demuxed_ecrecover_queue.simulator.witness.clone().into(); let round_function_witness = ecrecover_witnesses; - let memory_queries = ecrecover_memory_queries; - // check basic consistency assert!(precompile_calls.len() == demuxed_ecrecover_queue.states_accumulator.len()); drop(demuxed_ecrecover_queue.states_accumulator); assert!(precompile_calls.len() == round_function_witness.len()); if precompile_calls.len() == 0 { - return vec![]; + return (vec![], amount_of_memory_queries_before); } let mut round_counter = 0; @@ -89,14 +84,17 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< // convension let mut log_queue_input_state = take_queue_state_from_simulator(&demuxed_ecrecover_queue.simulator); - let mut memory_queries_it = memory_queries.into_iter(); + let amount_ecrecover_memory_queries = ecrecover_memory_queries.len(); + let mut memory_queries_it = ecrecover_memory_queries.into_iter(); let mut memory_read_witnesses = vec![]; let mut starting_request_idx = 0; - let mut memory_queue_input_state = memory_queue_simulator.take_sponge_like_queue_state(); + let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); let mut current_memory_queue_state = memory_queue_input_state.clone(); + let mut memory_queue_states_it = ecrecover_memory_states.iter(); + for (request_idx, (request, per_request_work)) in precompile_calls .into_iter() .zip(round_function_witness.into_iter()) @@ -105,7 +103,6 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< let _ = demuxed_ecrecover_queue .simulator .pop_and_output_intermediate_data(round_function); - let initial_memory_len = memory_queue_simulator.num_items; let mut memory_reads_per_request = vec![]; @@ -116,6 +113,7 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< let mut precompile_request = precompile_abi_in_log(request); let is_last_request = request_idx == num_requests - 1; + let mut amount_of_queries = 0; // we have 4 reads for (_query_index, read) in round_witness.reads.into_iter().enumerate() { let read_query = memory_queries_it.next().unwrap(); @@ -123,15 +121,11 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< assert!(read_query.rw_flag == false); memory_reads_per_request.push(read_query.value); - implicit_memory_artifacts.memory_queries.push(read); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(read, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.input_memory_offset += 1; + amount_of_queries += 1; } // and 2 writes @@ -140,18 +134,14 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< assert!(write == write_query); assert!(write_query.rw_flag == true); - implicit_memory_artifacts.memory_queries.push(write); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(write, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.output_memory_offset += 1; + amount_of_queries += 1; } - assert_eq!(memory_queue_simulator.num_items - initial_memory_len, 6); + assert_eq!(amount_of_queries, 6); round_counter += 1; if round_counter == num_rounds_per_circuit || is_last_request { @@ -230,14 +220,14 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< } } + let memory_simulator_after = &ecrecover_simulator_snapshots[1]; + let amount_of_memory_queries_after = + amount_of_memory_queries_before + amount_ecrecover_memory_queries; + assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() - ); - assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries_after, + memory_simulator_after.num_items as usize ); - result + (result, amount_of_memory_queries_after) } diff --git a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs index d84c8acd..2b83ad3e 100644 --- a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs @@ -1,8 +1,9 @@ use super::*; -use crate::witness::artifacts::{DemuxedLogQueries, ImplicitMemoryArtifacts, LogQueueStates}; +use crate::witness::artifacts::{DemuxedLogQueries, LogQueueStates}; use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; +use crate::zk_evm::aux_structures::MemoryQuery; use crate::zk_evm::zk_evm_abstractions::precompiles::keccak256::Keccak256RoundWitness; use crate::zkevm_circuits::base_structures::log_query::*; use crate::zkevm_circuits::keccak256_round_function::{ @@ -13,33 +14,54 @@ use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::encodings::*; use derivative::*; -pub(crate) fn keccak256_memory_queries_amount( +pub(crate) fn keccak256_memory_queries( keccak_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>, -) -> usize { - let result = keccak_round_function_witnesses - .iter() - .fold(0, |mut inner, (_, _, witness)| { - for el in witness.iter() { - let Keccak256RoundWitness { - new_request: _, - reads, - writes, - } = el; - - reads.iter().for_each(|read| { - if read.is_some() { +) -> Vec { + let amount_of_queries = + keccak_round_function_witnesses + .iter() + .fold(0, |mut inner, (_, _, witness)| { + witness.iter().for_each(|el| { + el.reads.iter().for_each(|read| { + if read.is_some() { + inner += 1; + } + }); + + if el.writes.is_some() { inner += 1; } }); - if let Some(writes) = writes.as_ref() { - inner += writes.len() + inner + }); + + let mut keccak_256_memory_queries = Vec::with_capacity(amount_of_queries); + + for (_cycle, _query, witness) in keccak_round_function_witnesses.iter() { + for el in witness.iter() { + let Keccak256RoundWitness { + new_request: _, + reads, + writes, + } = el; + + // we read, then write + reads.iter().for_each(|read| { + if let Some(read) = read { + keccak_256_memory_queries.push(*read); } + }); + + if let Some(writes) = writes.as_ref() { + keccak_256_memory_queries.extend_from_slice(writes); } - inner - }); + } + } - result + keccak_256_memory_queries.shrink_to_fit(); + + keccak_256_memory_queries } #[derive(Derivative)] @@ -59,59 +81,32 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< F: SmallField, R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( - amount_of_memory_queries: usize, - implicit_memory_artifacts: &mut ImplicitMemoryArtifacts, - memory_queue_states_accumulator: &LastPerCircuitAccumulator>, - memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, + amount_of_memory_queries_before: usize, + keccak256_memory_queries: Vec, + keccak256_simulator_snapshots: Vec>, + keccak256_memory_states: Vec>, keccak_round_function_witnesses: Vec<(u32, LogQuery_, Vec)>, keccak_precompile_queries: Vec, mut demuxed_keccak_precompile_queue: LogQueueStates, num_rounds_per_circuit: usize, round_function: &R, -) -> Vec> { +) -> (Vec>, usize) { assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() + keccak256_memory_queries.len(), + keccak256_memory_states.len() ); + + let memory_simulator_before = &keccak256_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries_before, + memory_simulator_before.num_items as usize ); - // split into aux witness, don't mix with the memory - - let mut keccak_256_memory_queries = Vec::with_capacity(keccak256_memory_queries_amount( - &keccak_round_function_witnesses, - )); - - for (_cycle, _query, witness) in keccak_round_function_witnesses.iter() { - for el in witness.iter() { - let Keccak256RoundWitness { - new_request: _, - reads, - writes, - } = el; - - // we read, then write - reads.iter().for_each(|read| { - if let Some(read) = read { - keccak_256_memory_queries.push(*read); - } - }); - - if let Some(writes) = writes.as_ref() { - keccak_256_memory_queries.extend_from_slice(writes); - } - } - } - let mut result = vec![]; let keccak_precompile_calls = keccak_precompile_queries; let round_function_witness = keccak_round_function_witnesses; - let memory_queries = keccak_256_memory_queries; - // check basic consistency assert_eq!( keccak_precompile_calls.len(), @@ -126,7 +121,7 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< ); if keccak_precompile_calls.len() == 0 { - return vec![]; + return (vec![], amount_of_memory_queries_before); } let mut round_counter = 0; @@ -150,14 +145,16 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< let mut hidden_fsm_input_state = Keccak256RoundFunctionFSM::::placeholder_witness(); hidden_fsm_input_state.read_precompile_call = true; - let mut memory_queries_it = memory_queries.into_iter(); + let mut memory_queries_it = keccak256_memory_queries.iter(); let mut precompile_state = Keccak256PrecompileState::GetRequestFromQueue; - let mut memory_queue_input_state = memory_queue_simulator.take_sponge_like_queue_state(); + let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); let mut current_memory_queue_state = memory_queue_input_state.clone(); let mut memory_reads_per_circuit = VecDeque::new(); + let mut memory_queue_states_it = keccak256_memory_states.iter(); + for (request_idx, (request, per_request_work)) in keccak_precompile_calls .into_iter() .zip(round_function_witness.into_iter()) @@ -260,16 +257,11 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< data.to_big_endian(&mut bytes32_buffer[..]); let read_query = memory_queries_it.next().unwrap(); - assert_eq!(read, read_query); + assert_eq!(read, *read_query); memory_reads_per_circuit.push_back(read_query.value); - implicit_memory_artifacts.memory_queries.push(read); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(read, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); input_buffer.fill_with_bytes( &bytes32_buffer, @@ -317,15 +309,10 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< assert!(round.writes.is_some()); let [write] = round.writes.unwrap(); let write_query = memory_queries_it.next().unwrap(); - assert_eq!(write, write_query); + assert_eq!(write, *write_query); - implicit_memory_artifacts.memory_queries.push(write); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(write, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); if is_last_request { precompile_state = Keccak256PrecompileState::Finished; @@ -482,16 +469,16 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< } } + let memory_simulator_after = &keccak256_simulator_snapshots[1]; + let amount_of_memory_queries_after = + amount_of_memory_queries_before + keccak256_memory_queries.len(); + assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() - ); - assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries_after, + memory_simulator_after.num_items as usize ); - result + (result, amount_of_memory_queries_after) } pub(crate) fn encode_keccak256_inner_state(state: [u64; 25]) -> [[[u8; 8]; 5]; 5] { diff --git a/src/witness/individual_circuits/memory_related/mod.rs b/src/witness/individual_circuits/memory_related/mod.rs index a2b8c357..bc7e4359 100644 --- a/src/witness/individual_circuits/memory_related/mod.rs +++ b/src/witness/individual_circuits/memory_related/mod.rs @@ -1,18 +1,22 @@ +use decommit_code::decommitter_memory_queries; +use ecrecover::ecrecover_memory_queries; +use keccak256_round_function::keccak256_memory_queries; +use secp256r1_verify::secp256r1_memory_queries; +use sha256_round_function::sha256_memory_queries; + use super::*; use crate::ethereum_types::U256; -use crate::witness::individual_circuits::memory_related::decommit_code::decommitter_memory_queries_amount; -use crate::witness::individual_circuits::memory_related::ecrecover::ecrecover_memory_queries_amount; -use crate::witness::individual_circuits::memory_related::keccak256_round_function::keccak256_memory_queries_amount; -use crate::witness::individual_circuits::memory_related::secp256r1_verify::secp256r1_memory_queries_amount; -use crate::witness::individual_circuits::memory_related::sha256_round_function::sha256_memory_queries_amount; use crate::zk_evm::aux_structures::DecommittmentQuery; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; +use crate::zk_evm::aux_structures::MemoryQuery; use crate::zk_evm::zk_evm_abstractions::precompiles::ecrecover::ECRecoverRoundWitness; use crate::zk_evm::zk_evm_abstractions::precompiles::keccak256::Keccak256RoundWitness; use crate::zk_evm::zk_evm_abstractions::precompiles::secp256r1_verify::Secp256r1VerifyRoundWitness; use crate::zk_evm::zk_evm_abstractions::precompiles::sha256::Sha256RoundWitness; +use circuit_definitions::encodings::memory_query::MemoryQueueState; + pub(crate) mod decommit_code; pub(crate) mod ecrecover; pub(crate) mod keccak256_round_function; @@ -21,16 +25,205 @@ pub(crate) mod secp256r1_verify; pub(crate) mod sha256_round_function; pub(crate) mod sort_decommit_requests; -pub(crate) fn amount_of_implicit_memory_queries( +#[derive(Clone)] +pub(crate) struct ImplicitMemoryQueries { + pub decommitter_memory_queries: Vec, + pub ecrecover_memory_queries: Vec, + pub keccak256_memory_queries: Vec, + pub secp256r1_memory_queries: Vec, + pub sha256_memory_queries: Vec, +} + +impl ImplicitMemoryQueries { + pub fn amount_of_queries(&self) -> usize { + self.decommitter_memory_queries.len() + + self.ecrecover_memory_queries.len() + + self.keccak256_memory_queries.len() + + self.secp256r1_memory_queries.len() + + self.sha256_memory_queries.len() + } + + fn get_vector(&self, index: usize) -> Option<&Vec> { + match index { + 0 => Some(&self.decommitter_memory_queries), + 1 => Some(&self.keccak256_memory_queries), + 2 => Some(&self.sha256_memory_queries), + 3 => Some(&self.ecrecover_memory_queries), + 4 => Some(&self.secp256r1_memory_queries), + _ => None, + } + } + + pub fn iter(&self) -> ImplicitMemoryQueriesIter { + ImplicitMemoryQueriesIter { + inner: &self, + circuit_index: 0, + iter: self.decommitter_memory_queries.iter(), + } + } +} + +pub struct ImplicitMemoryQueriesIter<'a> { + inner: &'a ImplicitMemoryQueries, + circuit_index: usize, + iter: Iter<'a, MemoryQuery>, +} + +use core::slice::Iter; + +impl<'a> Iterator for ImplicitMemoryQueriesIter<'a> { + type Item = &'a MemoryQuery; + + fn next(&mut self) -> Option { + let mut next = self.iter.next(); + + while next.is_none() { + self.circuit_index += 1; + let inner_vec = &self.inner.get_vector(self.circuit_index); + if inner_vec.is_none() { + return None; + } + + self.iter = inner_vec.unwrap().iter(); + next = self.iter.next(); + } + + next + } +} + +use crate::witness::oracle::PrecompilesInputData; +pub fn get_implicit_memory_queries( deduplicated_decommit_requests_with_data: &Vec<(DecommittmentQuery, Vec)>, - ecrecover_witnesses: &Vec<(u32, LogQuery_, ECRecoverRoundWitness)>, - keccak_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>, - secp256r1_verify_witnesses: &Vec<(u32, LogQuery_, Secp256r1VerifyRoundWitness)>, - sha256_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>, -) -> usize { - decommitter_memory_queries_amount(deduplicated_decommit_requests_with_data) - + ecrecover_memory_queries_amount(ecrecover_witnesses) - + keccak256_memory_queries_amount(keccak_round_function_witnesses) - + secp256r1_memory_queries_amount(secp256r1_verify_witnesses) - + sha256_memory_queries_amount(sha256_round_function_witnesses) + precompiles_inputs: &PrecompilesInputData, +) -> ImplicitMemoryQueries { + ImplicitMemoryQueries { + decommitter_memory_queries: decommitter_memory_queries( + deduplicated_decommit_requests_with_data, + ), + ecrecover_memory_queries: ecrecover_memory_queries(&precompiles_inputs.ecrecover_witnesses), + keccak256_memory_queries: keccak256_memory_queries( + &precompiles_inputs.keccak_round_function_witnesses, + ), + secp256r1_memory_queries: secp256r1_memory_queries( + &precompiles_inputs.secp256r1_verify_witnesses, + ), + sha256_memory_queries: sha256_memory_queries( + &precompiles_inputs.sha256_round_function_witnesses, + ), + } +} + +pub(crate) struct SimulatorSnapshot { + pub head: [F; SW], + pub tail: [F; SW], + pub num_items: u32, +} +use crate::boojum::gadgets::queue::QueueStateWitness; +use crate::boojum::gadgets::queue::QueueTailStateWitness; + +impl SimulatorSnapshot { + pub fn take_sponge_like_queue_state(&self) -> QueueStateWitness { + let result = QueueStateWitness { + head: self.head, + tail: QueueTailStateWitness { + tail: self.tail, + length: self.num_items, + }, + }; + + result + } +} + +#[derive(Default)] +pub(crate) struct ImplicitMemoryStates { + pub decommitter_simulator_snapshots: Vec>, + pub decommitter_memory_states: Vec>, + pub ecrecover_simulator_snapshots: Vec>, + pub ecrecover_memory_states: Vec>, + pub keccak256_simulator_snapshots: Vec>, + pub keccak256_memory_states: Vec>, + pub secp256r1_simulator_snapshots: Vec>, + pub secp256r1_memory_states: Vec>, + pub sha256_simulator_snapshots: Vec>, + pub sha256_memory_states: Vec>, +} + +impl ImplicitMemoryStates { + pub fn amount_of_states(&self) -> usize { + self.decommitter_memory_states.len() + + self.ecrecover_memory_states.len() + + self.keccak256_memory_states.len() + + self.secp256r1_memory_states.len() + + self.sha256_memory_states.len() + } +} +use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; + +fn get_simulator_snapshot( + memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, +) -> SimulatorSnapshot { + SimulatorSnapshot { + head: memory_queue_simulator.head, + tail: memory_queue_simulator.tail, + num_items: memory_queue_simulator.num_items, + } +} + +use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; + +pub(crate) fn simulate_implicit_memory_queues< + F: SmallField, + R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, +>( + memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, + memory_queue_states_accumulator: &mut LastPerCircuitAccumulator>, + implicit_memory_queries: &ImplicitMemoryQueries, + round_function: R, +) -> ImplicitMemoryStates { + let mut implicit_memory_states = ImplicitMemoryStates::default(); + + let mut simulate_subqueue = + |memory_queries: &Vec, memory_states: &mut Vec>| { + let mut snapshots = vec![]; + snapshots.push(get_simulator_snapshot(memory_queue_simulator)); // before + for query in memory_queries.iter() { + let (_old_tail, intermediate_info) = memory_queue_simulator + .push_and_output_intermediate_data(*query, &round_function); + + memory_states.push(intermediate_info); + memory_queue_states_accumulator.push(intermediate_info); + } + snapshots.push(get_simulator_snapshot(memory_queue_simulator)); // after + + snapshots + }; + + implicit_memory_states.decommitter_simulator_snapshots = simulate_subqueue( + &implicit_memory_queries.decommitter_memory_queries, + &mut implicit_memory_states.decommitter_memory_states, + ); + + implicit_memory_states.keccak256_simulator_snapshots = simulate_subqueue( + &implicit_memory_queries.keccak256_memory_queries, + &mut implicit_memory_states.keccak256_memory_states, + ); + + implicit_memory_states.sha256_simulator_snapshots = simulate_subqueue( + &implicit_memory_queries.sha256_memory_queries, + &mut implicit_memory_states.sha256_memory_states, + ); + + implicit_memory_states.ecrecover_simulator_snapshots = simulate_subqueue( + &implicit_memory_queries.ecrecover_memory_queries, + &mut implicit_memory_states.ecrecover_memory_states, + ); + + implicit_memory_states.secp256r1_simulator_snapshots = simulate_subqueue( + &implicit_memory_queries.secp256r1_memory_queries, + &mut implicit_memory_states.secp256r1_memory_states, + ); + + implicit_memory_states } diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index e2597c4f..eec69c2d 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -12,7 +12,7 @@ use crate::zk_evm::ethereum_types::U256; use crate::zkevm_circuits::{ base_structures::memory_query::MEMORY_QUERY_PACKED_WIDTH, ram_permutation::input::*, }; -use artifacts::{ImplicitMemoryArtifacts, MemoryArtifacts}; +use artifacts::MemoryArtifacts; use circuit_definitions::circuit_definitions::base_layer::{ RAMPermutationInstanceSynthesisFunction, ZkSyncBaseLayerCircuit, }; @@ -22,8 +22,9 @@ use circuit_definitions::encodings::recursion_request::RecursionQueueSimulator; use circuit_definitions::zkevm_circuits::scheduler::aux::BaseLayerCircuitType; use circuit_definitions::{encodings::*, Field, RoundFunction}; use memory_query::{CustomMemoryQueueSimulator, QueueWitness}; -use postprocessing::FirstAndLastCircuitWitness; +use oracle::WitnessGenerationArtifact; +use postprocessing::FirstAndLastCircuitWitness; use rayon::prelude::*; use snark_wrapper::boojum::field::Field as _; use std::borrow::Borrow; @@ -33,48 +34,43 @@ use zkevm_circuits::base_structures::vm_state::QUEUE_STATE_WIDTH; use crate::zk_evm::zkevm_opcode_defs::BOOTLOADER_HEAP_PAGE; -pub(crate) fn compute_ram_circuit_snapshots< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut(u64, RecursionQueueSimulator, Vec>), ->( - memory_queries: &Vec<(u32, MemoryQuery)>, - implicit_memory_artifacts: ImplicitMemoryArtifacts, - mut memory_queue_states_accumulator: LastPerCircuitAccumulator>, +pub(crate) fn compute_ram_circuit_snapshots( + total_amount_of_queries: usize, // including additional queries from precompiles + memory_queue_states_accumulator: LastPerCircuitAccumulator>, + sorted_memory_queue_states_accumulator: LastPerCircuitAccumulator>, memory_queue_simulator: MemoryQueuePerCircuitSimulator, + sorted_memory_queries_simulator: MemoryQueuePerCircuitSimulator, + sorted_queries_aux_data_for_chunks: Vec<(u32, MemoryQuery, usize)>, + sorted_encodings: Vec<[Field; MEMORY_QUERY_PACKED_WIDTH]>, + unsorted_encodings: Vec<[Field; MEMORY_QUERY_PACKED_WIDTH]>, round_function: &RoundFunction, num_non_deterministic_heap_queries: usize, - per_circuit_capacity: usize, geometry: &GeometryConfig, - mut circuit_callback: CB, - mut recursion_queue_callback: QSCB, + mut artifacts_callback: CB, ) -> ( FirstAndLastCircuitWitness>, Vec>, ) { - assert_eq!(memory_queries.len(), memory_queue_states_accumulator.len()); - assert_eq!( - implicit_memory_artifacts.memory_queries.len(), - implicit_memory_artifacts.memory_queue_states.len() + total_amount_of_queries, + memory_queue_states_accumulator.len() ); - // including additional queries from precompiles - let total_amount_of_queries = - memory_queries.len() + implicit_memory_artifacts.memory_queries.len(); + assert_eq!( + total_amount_of_queries, + sorted_memory_queue_states_accumulator.len() + ); assert!( total_amount_of_queries > 0, "VM should have made some memory requests" ); + let per_circuit_capacity = geometry.cycles_per_ram_permutation as usize; + let amount_of_circuits = (total_amount_of_queries + per_circuit_capacity - 1) / per_circuit_capacity; - memory_queue_states_accumulator - .reserve_exact_flat(implicit_memory_artifacts.memory_queue_states.len()); - for state in implicit_memory_artifacts.memory_queue_states.into_iter() { - memory_queue_states_accumulator.push(state); - } let unsorted_memory_queue_chunk_final_states = memory_queue_states_accumulator.into_circuits(); assert_eq!( @@ -82,47 +78,8 @@ pub(crate) fn compute_ram_circuit_snapshots< amount_of_circuits ); - let mut sorted_memory_queue_chunk_final_states = Vec::with_capacity(amount_of_circuits); - - let mut sorted_memory_queries_simulator = - MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( - per_circuit_capacity, - memory_queries.len() + implicit_memory_artifacts.memory_queries.len(), - )); - { - let mut sorted_memory_queries_accumulated: Vec<&MemoryQuery> = memory_queries - .iter() - .map(|(_, query)| query) - .chain(implicit_memory_artifacts.memory_queries.iter()) - .collect(); - - // sort by memory location, and then by timestamp - sorted_memory_queries_accumulated.par_sort_by(|a, b| match a.location.cmp(&b.location) { - Ordering::Equal => a.timestamp.cmp(&b.timestamp), - a @ _ => a, - }); - - // those two thins are parallelizable, and can be internally parallelized too - - // now we can finish reconstruction of each sorted and unsorted memory queries - - // reconstruct sorted one in full - - for chunk in sorted_memory_queries_accumulated.chunks(per_circuit_capacity) { - let intermediate_info = chunk - .iter() - .map(|query| { - let (_, _intermediate_info) = sorted_memory_queries_simulator - .push_and_output_intermediate_data(**query, round_function); - _intermediate_info - }) - .last() - .unwrap(); - sorted_memory_queue_chunk_final_states.push(intermediate_info); - } - } - - drop(implicit_memory_artifacts.memory_queries); + let sorted_memory_queue_chunk_final_states = + sorted_memory_queue_states_accumulator.into_circuits(); assert_eq!( unsorted_memory_queue_chunk_final_states.len(), @@ -134,9 +91,7 @@ pub(crate) fn compute_ram_circuit_snapshots< memory_queue_simulator.num_items ); - // now we should chunk it by circuits but briefly simulating their logic - - // since encodings of the elements provide all the information necessary to perform soring argument, + // since encodings of the elements provide all the information necessary to perform sorting argument, // we use them naively assert_eq!( @@ -144,6 +99,8 @@ pub(crate) fn compute_ram_circuit_snapshots< total_amount_of_queries ); + assert_eq!(unsorted_encodings.len(), sorted_encodings.len()); + let mut lhs_grand_product_chains = Vec::with_capacity(DEFAULT_NUM_PERMUTATION_ARGUMENT_REPETITIONS); let mut rhs_grand_product_chains = @@ -163,21 +120,13 @@ pub(crate) fn compute_ram_circuit_snapshots< round_function, ); - let lhs_contributions: Vec<_> = memory_queue_simulator - .witness - .iter() - .map(|el| &el.0) - .collect(); - let rhs_contributions: Vec<_> = sorted_memory_queries_simulator - .witness - .iter() - .map(|el| &el.0) - .collect(); + let lhs_contributions = unsorted_encodings; + let rhs_contributions = sorted_encodings; for idx in 0..DEFAULT_NUM_PERMUTATION_ARGUMENT_REPETITIONS { let (lhs_grand_product_chain, rhs_grand_product_chain) = compute_grand_product_chains( - &lhs_contributions, - &rhs_contributions, + &lhs_contributions.iter().collect(), + &rhs_contributions.iter().collect(), &challenges[idx], ); @@ -185,11 +134,11 @@ pub(crate) fn compute_ram_circuit_snapshots< assert_eq!(rhs_grand_product_chain.len(), total_amount_of_queries); assert_eq!( lhs_grand_product_chain.len(), - memory_queue_simulator.witness.len() + memory_queue_simulator.num_items as usize ); assert_eq!( rhs_grand_product_chain.len(), - sorted_memory_queries_simulator.witness.len() + sorted_memory_queries_simulator.num_items as usize ); lhs_grand_product_chains.push(lhs_grand_product_chain); @@ -203,8 +152,6 @@ pub(crate) fn compute_ram_circuit_snapshots< // now we need to split them into individual circuits // splitting is not extra hard here, we walk over iterator over everything and save states on checkpoints - // we also want to have chunks of witness for each of all the intermediate states - assert_eq!( unsorted_memory_queue_chunk_final_states.len(), transposed_lhs_chains.len() @@ -213,23 +160,6 @@ pub(crate) fn compute_ram_circuit_snapshots< unsorted_memory_queue_chunk_final_states.len(), transposed_rhs_chains.len() ); - let unsorted_witness_chunks = memory_queue_simulator - .witness - .into_circuits(amount_of_circuits); - - assert_eq!( - unsorted_memory_queue_chunk_final_states.len(), - unsorted_witness_chunks.len() - ); - - let sorted_witness_chunks = sorted_memory_queries_simulator - .witness - .into_circuits(amount_of_circuits); - - assert_eq!( - unsorted_memory_queue_chunk_final_states.len(), - sorted_witness_chunks.len() - ); let unsorted_global_final_state = unsorted_memory_queue_chunk_final_states .last() @@ -250,8 +180,7 @@ pub(crate) fn compute_ram_circuit_snapshots< .zip(sorted_memory_queue_chunk_final_states.into_iter()) .zip(transposed_lhs_chains.into_iter()) .zip(transposed_rhs_chains.into_iter()) - .zip(unsorted_witness_chunks) - .zip(sorted_witness_chunks); + .zip(sorted_queries_aux_data_for_chunks); // now trivial transformation into desired data structures, // and we are all good @@ -279,13 +208,10 @@ pub(crate) fn compute_ram_circuit_snapshots< idx, ( ( - ( - ((unsorted_sponge_final_state, sorted_sponge_final_state), lhs_grand_product), - rhs_grand_product, - ), - unsorted_states, + ((unsorted_sponge_final_state, sorted_sponge_final_state), lhs_grand_product), + rhs_grand_product, ), - sorted_states, + (num_nondet_writes_in_chunk, last_sorted_query, sorted_states_len), ), ) in it.enumerate() { @@ -298,42 +224,10 @@ pub(crate) fn compute_ram_circuit_snapshots< DEFAULT_NUM_PERMUTATION_ARGUMENT_REPETITIONS ); - // we need witnesses to pop elements from the front of the queue - - let unsorted_witness = FullStateCircuitQueueRawWitness { - elements: unsorted_states - .into_iter() - .map(|el| { - let witness = el.2.reflect(); - (witness, el.1) - }) - .collect(), - }; - - let sorted_witness = FullStateCircuitQueueRawWitness { - elements: sorted_states - .iter() - .map(|el| { - let witness = el.2.reflect(); - (witness, el.1) - }) - .collect(), - }; - // now we need to have final grand product value that will also become an input for the next circuit let if_first = idx == 0; let is_last = idx == num_circuits - 1; - // TODO into_iter - let num_nondet_writes_in_chunk = sorted_states - .iter() - .filter(|el| { - let query = &el.2; - query.rw_flag == true - && query.timestamp.0 == 0 - && query.location.page.0 == BOOTLOADER_HEAP_PAGE - }) - .count(); let new_num_nondet_writes = current_number_of_nondet_writes + (num_nondet_writes_in_chunk as u32); @@ -356,7 +250,6 @@ pub(crate) fn compute_ram_circuit_snapshots< .try_into() .unwrap(); - let last_sorted_query = sorted_states.last().unwrap().2; use circuit_definitions::encodings::memory_query::*; let sorting_key = sorting_key(&last_sorted_query); let comparison_key = comparison_key(&last_sorted_query); @@ -426,11 +319,14 @@ pub(crate) fn compute_ram_circuit_snapshots< num_nondeterministic_writes: new_num_nondet_writes, }, }, - unsorted_queue_witness: unsorted_witness, - sorted_queue_witness: sorted_witness, + // we will need witnesses to pop elements from the front of the queue + // but this data should be saved to storage before, during queues simulation (RAM-heavy) + // so we use placeholders here + unsorted_queue_witness: Default::default(), + sorted_queue_witness: Default::default(), }; - if sorted_states.len() % per_circuit_capacity != 0 { + if sorted_states_len % per_circuit_capacity != 0 { // RAM circuit does padding, so all previous values must be reset instance_witness .closed_form_input @@ -466,8 +362,8 @@ pub(crate) fn compute_ram_circuit_snapshots< tmp.current_sorted_queue_state.clone(), ); - circuit_callback(ZkSyncBaseLayerCircuit::RAMPermutation( - maker.process(instance_witness, circuit_type), + artifacts_callback(WitnessGenerationArtifact::BaseLayerCircuit( + ZkSyncBaseLayerCircuit::RAMPermutation(maker.process(instance_witness, circuit_type)), )); } @@ -479,11 +375,11 @@ pub(crate) fn compute_ram_circuit_snapshots< queue_simulator, ram_permutation_circuits_compact_forms_witnesses, ) = maker.into_results(); - recursion_queue_callback( + artifacts_callback(WitnessGenerationArtifact::RecursionQueue(( circuit_type as u64, queue_simulator, ram_permutation_circuits_compact_forms_witnesses.clone(), - ); + ))); ( ram_permutation_circuits, diff --git a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs index b5314980..67af787b 100644 --- a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs +++ b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs @@ -1,8 +1,9 @@ use super::*; -use crate::witness::artifacts::{DemuxedLogQueries, ImplicitMemoryArtifacts, LogQueueStates}; +use crate::witness::artifacts::{DemuxedLogQueries, LogQueueStates}; use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; +use crate::zk_evm::aux_structures::MemoryQuery; use crate::zk_evm::zk_evm_abstractions::precompiles::secp256r1_verify::Secp256r1VerifyRoundWitness; use crate::zkevm_circuits::base_structures::log_query::*; use crate::zkevm_circuits::secp256r1_verify::*; @@ -10,14 +11,27 @@ use circuit_definitions::encodings::memory_query::MemoryQueueSimulator; use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::encodings::*; -pub(crate) fn secp256r1_memory_queries_amount( +pub(crate) fn secp256r1_memory_queries( secp256r1_verify_witnesses: &Vec<(u32, LogQuery_, Secp256r1VerifyRoundWitness)>, -) -> usize { - secp256r1_verify_witnesses +) -> Vec { + let amount_of_queries = secp256r1_verify_witnesses .iter() .fold(0, |inner, (_, _, witness)| { inner + witness.reads.len() + witness.writes.len() - }) + }); + + let mut secp256r1_memory_queries = Vec::with_capacity(amount_of_queries); + + for (_cycle, _query, witness) in secp256r1_verify_witnesses.iter() { + let initial_memory_len = secp256r1_memory_queries.len(); + // we read, then write + secp256r1_memory_queries.extend_from_slice(&witness.reads); + secp256r1_memory_queries.extend_from_slice(&witness.writes); + + assert_eq!(secp256r1_memory_queries.len() - initial_memory_len, 7); + } + + secp256r1_memory_queries } // we want to simulate splitting of data into many separate instances of the same circuit. @@ -28,44 +42,27 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< F: SmallField, R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( - amount_of_memory_queries: usize, - implicit_memory_artifacts: &mut ImplicitMemoryArtifacts, - memory_queue_states_accumulator: &LastPerCircuitAccumulator>, - memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, + amount_of_memory_queries_before: usize, + secp256r1_memory_queries: Vec, + secp256r1_simulator_snapshots: Vec>, + secp256r1_memory_states: Vec>, secp256r1_verify_witnesses: Vec<(u32, LogQuery_, Secp256r1VerifyRoundWitness)>, secp256r1_verify_queries: Vec, mut demuxed_secp256r1_verify_queue: LogQueueStates, num_rounds_per_circuit: usize, round_function: &R, -) -> Vec> { +) -> (Vec>, usize) { assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() + secp256r1_memory_queries.len(), + secp256r1_memory_states.len() ); + + let memory_simulator_before = &secp256r1_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries_before, + memory_simulator_before.num_items as usize ); - // split into aux witness, don't mix with the memory - - use crate::zk_evm::zk_evm_abstractions::precompiles::secp256r1_verify::Secp256r1VerifyRoundWitness; - let mut memory_queries = - Vec::with_capacity(secp256r1_memory_queries_amount(&secp256r1_verify_witnesses)); - - for (_cycle, _query, witness) in secp256r1_verify_witnesses.iter() { - let Secp256r1VerifyRoundWitness { - new_request: _, - reads, - writes, - } = witness; - - // we read, then write - memory_queries.extend_from_slice(reads); - - memory_queries.extend_from_slice(writes); - } - let mut result = vec![]; let precompile_calls = secp256r1_verify_queries; @@ -82,7 +79,7 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< assert!(precompile_calls.len() == round_function_witness.len()); if precompile_calls.len() == 0 { - return vec![]; + return (vec![], amount_of_memory_queries_before); } let mut round_counter = 0; @@ -91,14 +88,17 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< // convension let mut log_queue_input_state = take_queue_state_from_simulator(&demuxed_secp256r1_verify_queue.simulator); - let mut memory_queries_it = memory_queries.into_iter(); + let amount_secp256r1_memory_queries = secp256r1_memory_queries.len(); + let mut memory_queries_it = secp256r1_memory_queries.into_iter(); let mut memory_read_witnesses = vec![]; let mut starting_request_idx = 0; - let mut memory_queue_input_state = memory_queue_simulator.take_sponge_like_queue_state(); + let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); let mut current_memory_queue_state = memory_queue_input_state.clone(); + let mut memory_queue_states_it = secp256r1_memory_states.iter(); + for (request_idx, (request, per_request_work)) in precompile_calls .into_iter() .zip(round_function_witness.into_iter()) @@ -107,7 +107,6 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< let _ = demuxed_secp256r1_verify_queue .simulator .pop_and_output_intermediate_data(round_function); - let initial_memory_len = memory_queue_simulator.num_items; let mut memory_reads_per_request = vec![]; @@ -118,6 +117,8 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< let mut precompile_request = precompile_abi_in_log(request); let is_last_request = request_idx == num_requests - 1; + let mut amount_of_queries = 0; + // we have reads for (_query_index, read) in round_witness.reads.into_iter().enumerate() { let read_query = memory_queries_it.next().unwrap(); @@ -125,15 +126,11 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< assert!(read_query.rw_flag == false); memory_reads_per_request.push(read_query.value); - implicit_memory_artifacts.memory_queries.push(read); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(read, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.input_memory_offset += 1; + amount_of_queries += 1; } // and writes @@ -142,18 +139,14 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< assert!(write == write_query); assert!(write_query.rw_flag == true); - implicit_memory_artifacts.memory_queries.push(write); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(write, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.output_memory_offset += 1; + amount_of_queries += 1; } - assert_eq!(memory_queue_simulator.num_items - initial_memory_len, 7); + assert_eq!(amount_of_queries, 7); round_counter += 1; if round_counter == num_rounds_per_circuit || is_last_request { @@ -232,14 +225,14 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< } } + let memory_simulator_after = &secp256r1_simulator_snapshots[1]; + let amount_of_memory_queries_after = + amount_of_memory_queries_before + amount_secp256r1_memory_queries; + assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() - ); - assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries_after, + memory_simulator_after.num_items as usize ); - result + (result, amount_of_memory_queries_after) } diff --git a/src/witness/individual_circuits/memory_related/sha256_round_function.rs b/src/witness/individual_circuits/memory_related/sha256_round_function.rs index 67e20e47..23d372b6 100644 --- a/src/witness/individual_circuits/memory_related/sha256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/sha256_round_function.rs @@ -1,9 +1,10 @@ use super::*; use crate::boojum::gadgets::traits::allocatable::CSAllocatable; -use crate::witness::artifacts::{DemuxedLogQueries, ImplicitMemoryArtifacts, LogQueueStates}; +use crate::witness::artifacts::{DemuxedLogQueries, LogQueueStates}; use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; +use crate::zk_evm::aux_structures::MemoryQuery; use crate::zk_evm::zk_evm_abstractions::precompiles::sha256::Sha256RoundWitness; use crate::zk_evm::zkevm_opcode_defs::ethereum_types::U256; use crate::zkevm_circuits::base_structures::log_query::*; @@ -14,29 +15,43 @@ use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::encodings::*; use derivative::*; -pub(crate) fn sha256_memory_queries_amount( +pub(crate) fn sha256_memory_queries( sha256_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>, -) -> usize { - let result = sha256_round_function_witnesses - .iter() - .fold(0, |mut inner, (_, _, witness)| { - for el in witness.iter() { - let Sha256RoundWitness { - new_request: _, - reads, - writes, - } = el; - - inner += reads.len(); - - if let Some(writes) = writes.as_ref() { - inner += writes.len() +) -> Vec { + let amount_of_queries = + sha256_round_function_witnesses + .iter() + .fold(0, |mut inner, (_, _, witness)| { + for el in witness.iter() { + inner += el.reads.len(); + + if let Some(writes) = el.writes.as_ref() { + inner += writes.len() + } } + inner + }); + + let mut sha256_memory_queries = Vec::with_capacity(amount_of_queries); + + for (_cycle, _query, witness) in sha256_round_function_witnesses.iter() { + for el in witness.iter() { + let Sha256RoundWitness { + new_request: _, + reads, + writes, + } = el; + + // we read, then write + sha256_memory_queries.extend_from_slice(reads); + + if let Some(writes) = writes.as_ref() { + sha256_memory_queries.extend_from_slice(writes); } - inner - }); + } + } - result + sha256_memory_queries } #[derive(Derivative)] @@ -55,48 +70,24 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< F: SmallField, R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( - amount_of_memory_queries: usize, - implicit_memory_artifacts: &mut ImplicitMemoryArtifacts, - memory_queue_states_accumulator: &LastPerCircuitAccumulator>, - memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, + amount_of_memory_queries_before: usize, + sha256_memory_queries: Vec, + sha256_simulator_snapshots: Vec>, + sha256_memory_states: Vec>, sha256_round_function_witnesses: Vec<(u32, LogQuery_, Vec)>, sha256_precompile_queries: Vec, mut demuxed_sha256_precompile_queue: LogQueueStates, num_rounds_per_circuit: usize, round_function: &R, -) -> Vec> { - assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() - ); +) -> (Vec>, usize) { + assert_eq!(sha256_memory_queries.len(), sha256_memory_states.len()); + + let memory_simulator_before = &sha256_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries_before, + memory_simulator_before.num_items as usize ); - // split into aux witness, don't mix with the memory - use crate::zk_evm::zk_evm_abstractions::precompiles::sha256::Sha256RoundWitness; - let mut sha256_memory_queries = Vec::with_capacity(sha256_memory_queries_amount( - &sha256_round_function_witnesses, - )); - - for (_cycle, _query, witness) in sha256_round_function_witnesses.iter() { - for el in witness.iter() { - let Sha256RoundWitness { - new_request: _, - reads, - writes, - } = el; - - // we read, then write - sha256_memory_queries.extend_from_slice(reads); - - if let Some(writes) = writes.as_ref() { - sha256_memory_queries.extend_from_slice(writes); - } - } - } - let mut result = vec![]; let precompile_calls = sha256_precompile_queries; @@ -107,15 +98,13 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< .into(); let round_function_witness = sha256_round_function_witnesses; - let memory_queries = sha256_memory_queries; - // check basic consistency assert!(precompile_calls.len() == demuxed_sha256_precompile_queue.states_accumulator.len()); drop(demuxed_sha256_precompile_queue.states_accumulator); assert!(precompile_calls.len() == round_function_witness.len()); if precompile_calls.len() == 0 { - return vec![]; + return (vec![], amount_of_memory_queries_before); } let mut round_counter = 0; @@ -127,7 +116,8 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< let mut hidden_fsm_input_state = Sha256RoundFunctionFSM::::placeholder_witness(); hidden_fsm_input_state.read_precompile_call = true; - let mut memory_queries_it = memory_queries.into_iter(); + let amount_sha256_memory_queries = sha256_memory_queries.len(); + let mut memory_queries_it = sha256_memory_queries.into_iter(); let mut memory_read_witnesses = vec![]; @@ -136,9 +126,11 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< let mut request_ranges = vec![]; let mut starting_request_idx = 0; - let mut memory_queue_input_state = memory_queue_simulator.take_sponge_like_queue_state(); + let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); let mut current_memory_queue_state = memory_queue_input_state.clone(); + let mut memory_queue_states_it = sha256_memory_states.iter(); + for (request_idx, (request, per_request_work)) in precompile_calls .into_iter() .zip(round_function_witness.into_iter()) @@ -184,13 +176,8 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< assert_eq!(read, read_query); memory_reads_per_request.push(read_query.value); - implicit_memory_artifacts.memory_queries.push(read); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(read, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.input_memory_offset += 1; } @@ -208,13 +195,8 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< let write_query = memory_queries_it.next().unwrap(); assert_eq!(write, write_query); - implicit_memory_artifacts.memory_queries.push(write); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(write, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); if is_last_request { precompile_state = Sha256PrecompileState::Finished; @@ -359,14 +341,14 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< } } + let memory_simulator_after = &sha256_simulator_snapshots[1]; + let amount_of_memory_queries_after = + amount_of_memory_queries_before + amount_sha256_memory_queries; + assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() - ); - assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries_after, + memory_simulator_after.num_items as usize ); - result + (result, amount_of_memory_queries_after) } diff --git a/src/witness/individual_circuits/storage_application.rs b/src/witness/individual_circuits/storage_application.rs index b046250c..a44ee937 100644 --- a/src/witness/individual_circuits/storage_application.rs +++ b/src/witness/individual_circuits/storage_application.rs @@ -21,27 +21,20 @@ use circuit_definitions::encodings::recursion_request::RecursionQueueSimulator; use circuit_definitions::encodings::state_diff_record::StateDiffRecord; use circuit_definitions::encodings::LogQueueSimulator; use circuit_definitions::zkevm_circuits::scheduler::aux::BaseLayerCircuitType; +use oracle::WitnessGenerationArtifact; use tracing; use zk_evm::aux_structures::LogQuery; use crate::sha3::Digest; -pub(crate) fn decompose_into_storage_application_witnesses< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +pub(crate) fn decompose_into_storage_application_witnesses( deduplicated_rollup_storage_queue_simulator: LogQueueSimulator, deduplicated_rollup_storage_queries: Vec, mut tree: impl BinarySparseStorageTree<256, 32, 32, 8, 32, Blake2s256, ZkSyncStorageLeaf>, round_function: &Poseidon2Goldilocks, num_rounds_per_circuit: usize, geometry: &GeometryConfig, - mut circuit_callback: CB, - mut recursion_queue_callback: QSCB, + mut artifacts_callback: CB, ) -> ( FirstAndLastCircuitWitness>, Vec>, @@ -261,8 +254,8 @@ pub(crate) fn decompose_into_storage_application_witnesses< initial_fsm_state = final_fsm_state.clone(); - circuit_callback(ZkSyncBaseLayerCircuit::StorageApplication( - maker.process(input, circuit_type), + artifacts_callback(WitnessGenerationArtifact::BaseLayerCircuit( + ZkSyncBaseLayerCircuit::StorageApplication(maker.process(input, circuit_type)), )); } @@ -271,11 +264,11 @@ pub(crate) fn decompose_into_storage_application_witnesses< queue_simulator, storage_application_circuits_compact_forms_witnesses, ) = maker.into_results(); - recursion_queue_callback( + artifacts_callback(WitnessGenerationArtifact::RecursionQueue(( circuit_type as u64, queue_simulator, storage_application_circuits_compact_forms_witnesses.clone(), - ); + ))); tracing::debug!( "Final enumeration index = {}", diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 2f7cddea..5ada14e7 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -4,6 +4,7 @@ use super::artifacts::LogCircuitsArtifacts; use super::individual_circuits::main_vm::CallstackSimulationResult; +use super::individual_circuits::memory_related::{ImplicitMemoryQueries, ImplicitMemoryStates}; use super::postprocessing::{ BlockFirstAndLastBasicCircuitsObservableWitnesses, FirstAndLastCircuitWitness, }; @@ -14,13 +15,13 @@ use crate::boojum::gadgets::queue::QueueState; use crate::boojum::gadgets::traits::allocatable::CSAllocatable; use crate::ethereum_types::U256; use crate::toolset::GeometryConfig; -use crate::witness::artifacts::{ - DemuxedLogQueries, ImplicitMemoryArtifacts, MemoryArtifacts, MemoryCircuitsArtifacts, -}; +use crate::witness::artifacts::{DemuxedLogQueries, MemoryArtifacts, MemoryCircuitsArtifacts}; use crate::witness::aux_data_structs::one_per_circuit_accumulator::{ CircuitsEntryAccumulatorSparse, LastPerCircuitAccumulator, }; -use crate::witness::aux_data_structs::per_circuit_accumulator::PerCircuitAccumulatorSparse; +use crate::witness::aux_data_structs::per_circuit_accumulator::{ + PerCircuitAccumulator, PerCircuitAccumulatorSparse, +}; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::witness::individual_circuits::log_demux::LogDemuxCircuitArtifacts; use crate::witness::postprocessing::make_circuits; @@ -34,16 +35,23 @@ use crate::zkevm_circuits::base_structures::vm_state::{ use crate::zkevm_circuits::scheduler::block_header::MAX_4844_BLOBS_PER_BLOCK; use circuit_definitions::boojum::field::goldilocks::GoldilocksField; use circuit_definitions::boojum::field::{Field, U64Representable}; +use circuit_definitions::boojum::gadgets::queue::QueueStateWitness; use circuit_definitions::boojum::implementations::poseidon2::Poseidon2Goldilocks; use circuit_definitions::circuit_definitions::base_layer::ZkSyncBaseLayerCircuit; use circuit_definitions::encodings::callstack_entry::ExtendedCallstackEntry; use circuit_definitions::encodings::recursion_request::RecursionQueueSimulator; -use circuit_definitions::encodings::LogQueueSimulator; +use circuit_definitions::encodings::{CircuitEquivalentReflection, LogQueueSimulator}; +use circuit_definitions::zkevm_circuits::base_structures::memory_query::{ + MemoryQueryWitness, MEMORY_QUERY_PACKED_WIDTH, +}; use circuit_definitions::zkevm_circuits::eip_4844::input::EIP4844CircuitInstanceWitness; use circuit_definitions::zkevm_circuits::fsm_input_output::ClosedFormInputCompactFormWitness; use circuit_definitions::zkevm_circuits::scheduler::aux::BaseLayerCircuitType; use derivative::Derivative; use std::collections::{BTreeMap, HashMap}; +use std::sync::mpsc::{self, Receiver, Sender}; +use std::sync::{Arc, Mutex}; +use zkevm_assembly::zkevm_opcode_defs::BOOTLOADER_HEAP_PAGE; #[derive(Derivative)] #[derivative(Clone(bound = ""), Copy(bound = ""), Debug, Default)] @@ -687,21 +695,13 @@ use crate::zkevm_circuits::demux_log_queue::DemuxOutput; /// Process log circuits that do not use memory. /// Storage, transient storage, events, l2 to l1 queries /// Precompiles use memory and are processed in 'process_memory_related_circuits' -fn process_io_log_circuits< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +fn process_io_log_circuits( geometry: &GeometryConfig, tree: impl BinarySparseStorageTree<256, 32, 32, 8, 32, Blake2s256, ZkSyncStorageLeaf>, demuxed_log_queues_states: IOLogsQueuesStates, demuxed_log_queries: DemuxedIOLogQueries, round_function: &Poseidon2Goldilocks, - mut circuit_callback: &mut CB, - mut recursion_queue_callback: &mut QSCB, + mut artifacts_callback: &mut CB, ) -> ( LogCircuitsArtifacts, FirstAndLastCircuitWitness>, @@ -797,8 +797,7 @@ fn process_io_log_circuits< round_function, geometry.cycles_per_storage_application as usize, geometry, - &mut circuit_callback, - &mut recursion_queue_callback, + &mut artifacts_callback, ); ( @@ -824,90 +823,276 @@ use crate::witness::artifacts::DecommitmentArtifactsForMainVM; use crate::witness::artifacts::LogQueueStates; use crate::zkevm_circuits::demux_log_queue::NUM_DEMUX_OUTPUTS; -use circuit_definitions::encodings::memory_query::MemoryQueueState; +use circuit_definitions::encodings::memory_query::{MemoryQueueState, MemoryQueueStateWitnesses}; fn simulate_memory_queue( geometry: GeometryConfig, - memory_queries: Vec<(Cycle, MemoryQuery)>, - amount_of_implicit_memory_queries: usize, + memory_queries: Arc>, + implicit_memory_queries: Arc, round_function: Poseidon2Goldilocks, + channel_sender: Sender, ) -> ( - MemoryArtifacts, + CircuitsEntryAccumulatorSparse<( + u32, + QueueStateWitness, + )>, + MemoryQueueState, LastPerCircuitAccumulator>, MemoryQueuePerCircuitSimulator, + ImplicitMemoryStates, + Vec<[GoldilocksField; MEMORY_QUERY_PACKED_WIDTH]>, ) { - let mut memory_artifacts_for_main_vm = MemoryArtifacts { - memory_queries, - memory_queue_entry_states: CircuitsEntryAccumulatorSparse::new( - geometry.cycles_per_vm_snapshot as usize, - (0, QueueState::placeholder_witness()), - ), - }; + // for MainVM circuits + let mut memory_queue_entry_states = CircuitsEntryAccumulatorSparse::new( + geometry.cycles_per_vm_snapshot as usize, + (0, QueueState::placeholder_witness()), + ); - // for RAM permutation circuits + // for RAM permutation circuits, only last per circuit let mut memory_queue_states_accumulator = LastPerCircuitAccumulator::>::with_flat_capacity( geometry.cycles_per_ram_permutation as usize, - memory_artifacts_for_main_vm.memory_queries.len(), + memory_queries.len(), ); use crate::witness::aux_data_structs::per_circuit_accumulator::PerCircuitAccumulator; - // very big data struct inside + // simulator states are very RAM-heavy let mut memory_queue_simulator = MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( geometry.cycles_per_ram_permutation as usize, - memory_artifacts_for_main_vm.memory_queries.len() + amount_of_implicit_memory_queries, + geometry.cycles_per_ram_permutation as usize, )); - // very slow - for (cycle, query) in memory_artifacts_for_main_vm.memory_queries.iter() { + // for fs challenges in RAM permutation circuits + let mut encodings_witnesses_for_fs = vec![]; + + // move accumulated full state witnesses from simulator, and process them + let mut process_simulation_result = + |mut memory_queue_simulator: MemoryQueuePerCircuitSimulator| { + let witnesses; + (memory_queue_simulator, witnesses) = memory_queue_simulator.replace_container( + PerCircuitAccumulator::with_flat_capacity( + geometry.cycles_per_ram_permutation as usize, + geometry.cycles_per_ram_permutation as usize, + ), + ); + let amount_of_circuits_accumulated = witnesses.amount_of_circuits_accumulated(); + for full_witnesses_for_circuit in + witnesses.into_circuits(amount_of_circuits_accumulated) + { + let mut unsorted_witnesses_for_circuit = + Vec::with_capacity(full_witnesses_for_circuit.len()); + + // split full witnesses + for witness in full_witnesses_for_circuit.into_iter() { + encodings_witnesses_for_fs.push(witness.0); + unsorted_witnesses_for_circuit.push((witness.2.reflect(), witness.1)); + } + + // send to storage + channel_sender + .send(WitnessGenerationArtifact::MemoryQueueWitness(( + unsorted_witnesses_for_circuit, + false, // unsorted + ))) + .unwrap(); + } + + memory_queue_simulator + }; + + // the simulation is mostly a sequential computation of hashes + // for this reason it is one of the slowest parts + // we are simulating explicit part of queue (direct memory queries) + for (cycle, query) in memory_queries.iter() { let (_, intermediate_info) = memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); memory_queue_states_accumulator.push(intermediate_info); - memory_artifacts_for_main_vm - .memory_queue_entry_states + memory_queue_entry_states .push((*cycle, transform_sponge_like_queue_state(intermediate_info))); - } - { - assert_eq!( - memory_artifacts_for_main_vm.memory_queries.len(), - memory_queue_states_accumulator.len() - ); - assert_eq!( - memory_artifacts_for_main_vm.memory_queries.len(), - memory_queue_simulator.num_items as usize - ); + // if we have collected witnesses for the circuit, we process and send part of them to the storage to free up RAM + if memory_queue_simulator.witness.len() == geometry.cycles_per_ram_permutation as usize { + memory_queue_simulator = process_simulation_result(memory_queue_simulator); + } } + assert_eq!(memory_queries.len(), memory_queue_states_accumulator.len()); + assert_eq!( + memory_queries.len(), + memory_queue_simulator.num_items as usize + ); + + let final_explicit_memory_queue_state = memory_queue_states_accumulator.last().unwrap().clone(); + + // now we need to handle implicit memory queries produced by decomitter, precompiles etc. + + use crate::witness::individual_circuits::memory_related::simulate_implicit_memory_queues; + let implicit_memory_states = simulate_implicit_memory_queues( + &mut memory_queue_simulator, + &mut memory_queue_states_accumulator, + &implicit_memory_queries, + round_function, + ); + + memory_queue_simulator = process_simulation_result(memory_queue_simulator); + + assert_eq!( + memory_queries.len() + implicit_memory_queries.amount_of_queries(), + encodings_witnesses_for_fs.len() + ); + ( - memory_artifacts_for_main_vm, + memory_queue_entry_states, + final_explicit_memory_queue_state, memory_queue_states_accumulator, memory_queue_simulator, + implicit_memory_states, + encodings_witnesses_for_fs, + ) +} + +fn simulate_sorted_memory_queue( + geometry: GeometryConfig, + memory_queries: Arc>, + implicit_memory_queries: Arc, + round_function: Poseidon2Goldilocks, + channel_sender: Sender, +) -> ( + LastPerCircuitAccumulator>, + MemoryQueuePerCircuitSimulator, + Vec<[GoldilocksField; MEMORY_QUERY_PACKED_WIDTH]>, + Vec<(u32, MemoryQuery, usize)>, +) { + let mut all_memory_queries_sorted: Vec<&MemoryQuery> = memory_queries + .iter() + .map(|(_, query)| query) + .chain(implicit_memory_queries.iter()) + .collect(); + + use crate::witness::aux_data_structs::per_circuit_accumulator::PerCircuitAccumulator; + use rayon::prelude::*; + use std::cmp::Ordering; + + // sort by memory location, and then by timestamp + all_memory_queries_sorted.par_sort_by(|a, b| match a.location.cmp(&b.location) { + Ordering::Equal => a.timestamp.cmp(&b.timestamp), + a @ _ => a, + }); + + let amount_of_queries = all_memory_queries_sorted.len(); + assert_eq!( + memory_queries.len() + implicit_memory_queries.amount_of_queries(), + amount_of_queries + ); + + // simulator states are very RAM-heavy + let mut sorted_memory_queries_simulator = + MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( + geometry.cycles_per_ram_permutation as usize, + amount_of_queries, + )); + + // for RAM permutation circuits + let mut sorted_memory_queue_states_accumulator = + LastPerCircuitAccumulator::>::with_flat_capacity( + geometry.cycles_per_ram_permutation as usize, + amount_of_queries, + ); + + // for fs challenges in RAM permutation circuits + let mut encodings_witnesses_for_fs = Vec::with_capacity(amount_of_queries); + + let amount_of_ram_circuits = (amount_of_queries as u32 + geometry.cycles_per_ram_permutation + - 1) + / geometry.cycles_per_ram_permutation; + // for RAM permutation circuits + let mut sorted_queries_aux_data_for_chunks = + Vec::with_capacity(amount_of_ram_circuits as usize); + + // the simulation is mostly a sequential computation of hashes + // for this reason it is one of the slowest parts + for (idx, query) in all_memory_queries_sorted.into_iter().enumerate() { + let (_, intermediate_info) = sorted_memory_queries_simulator + .push_and_output_intermediate_data(*query, &round_function); + sorted_memory_queue_states_accumulator.push(intermediate_info); + + // if we have collected witnesses for the circuit, we process and send part of them to the storage to free up RAM + if sorted_memory_queries_simulator.witness.len() + == geometry.cycles_per_ram_permutation as usize + || idx == amount_of_queries - 1 + { + let witnesses; + (sorted_memory_queries_simulator, witnesses) = sorted_memory_queries_simulator + .replace_container(PerCircuitAccumulator::with_flat_capacity( + geometry.cycles_per_ram_permutation as usize, + geometry.cycles_per_ram_permutation as usize, + )); + + assert_eq!(witnesses.amount_of_circuits_accumulated(), 1); + + // should be only one iteration + for full_witnesses_for_circuit in witnesses.into_circuits(1) { + let sorted_states_len = full_witnesses_for_circuit.len(); + let num_nondet_writes_in_chunk = full_witnesses_for_circuit + .iter() + .filter(|el| { + let query = &el.2; + query.rw_flag == true + && query.timestamp.0 == 0 + && query.location.page.0 == BOOTLOADER_HEAP_PAGE + }) + .count(); + let last_sorted_query = full_witnesses_for_circuit.last().unwrap().2; + + sorted_queries_aux_data_for_chunks.push(( + num_nondet_writes_in_chunk as u32, + last_sorted_query, + sorted_states_len, + )); + + let mut sorted_witnesses_for_circuit = + Vec::with_capacity(full_witnesses_for_circuit.len()); + + // split full witnesses + for witness in full_witnesses_for_circuit.into_iter() { + encodings_witnesses_for_fs.push(witness.0); + sorted_witnesses_for_circuit.push((witness.2.reflect(), witness.1)); + } + + // send to storage + channel_sender + .send(WitnessGenerationArtifact::MemoryQueueWitness(( + sorted_witnesses_for_circuit, + true, // sorted + ))) + .unwrap(); + } + } + } + + ( + sorted_memory_queue_states_accumulator, + sorted_memory_queries_simulator, + encodings_witnesses_for_fs, + sorted_queries_aux_data_for_chunks, ) } use crate::witness::artifacts::DemuxedPrecompilesLogQueries; use crate::witness::individual_circuits::log_demux::PrecompilesQueuesStates; -struct PrecompilesInputData { - keccak_round_function_witnesses: Vec<(Cycle, LogQuery, Vec)>, - sha256_round_function_witnesses: Vec<(Cycle, LogQuery, Vec)>, - ecrecover_witnesses: Vec<(Cycle, LogQuery, ECRecoverRoundWitness)>, - secp256r1_verify_witnesses: Vec<(Cycle, LogQuery, Secp256r1VerifyRoundWitness)>, - logs_queues_states: PrecompilesQueuesStates, - logs_queries: DemuxedPrecompilesLogQueries, +pub(crate) struct PrecompilesInputData { + pub keccak_round_function_witnesses: Vec<(Cycle, LogQuery, Vec)>, + pub sha256_round_function_witnesses: Vec<(Cycle, LogQuery, Vec)>, + pub ecrecover_witnesses: Vec<(Cycle, LogQuery, ECRecoverRoundWitness)>, + pub secp256r1_verify_witnesses: Vec<(Cycle, LogQuery, Secp256r1VerifyRoundWitness)>, + pub logs_queues_states: PrecompilesQueuesStates, + pub logs_queries: DemuxedPrecompilesLogQueries, } -fn process_memory_related_circuits< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +fn process_memory_related_circuits( geometry: &GeometryConfig, vm_snapshots: &Vec, memory_queries: Vec<(Cycle, MemoryQuery)>, @@ -916,8 +1101,7 @@ fn process_memory_related_circuits< executed_decommittment_queries: Vec<(Cycle, DecommittmentQuery, Vec)>, precompiles_data: PrecompilesInputData, round_function: &Poseidon2Goldilocks, - mut circuit_callback: &mut CB, - mut recursion_queue_callback: &mut QSCB, + mut artifacts_callback: &mut CB, ) -> ( MemoryCircuitsArtifacts, MemoryArtifacts, @@ -972,47 +1156,120 @@ fn process_memory_related_circuits< tracing::debug!("Running unsorted memory queue simulation"); - use crate::witness::individual_circuits::memory_related::amount_of_implicit_memory_queries; + use crate::witness::individual_circuits::memory_related::get_implicit_memory_queries; - let amount_of_memory_queries = memory_queries.len(); - let amount_of_implicit_memory_queries = amount_of_implicit_memory_queries( + // precompiles and decommiter will produce additional implicit memory queries + let implicit_memory_queries = get_implicit_memory_queries( &decommiter_circuit_inputs.deduplicated_decommit_requests_with_data, - &precompiles_data.ecrecover_witnesses, - &precompiles_data.keccak_round_function_witnesses, - &precompiles_data.secp256r1_verify_witnesses, - &precompiles_data.sha256_round_function_witnesses, + &precompiles_data, ); - let (memory_artifacts_for_main_vm, memory_queue_states_accumulator, mut memory_queue_simulator) = - simulate_memory_queue( - *geometry, - memory_queries, - amount_of_implicit_memory_queries, - *round_function, - ); + let amount_of_explicit_memory_queries = memory_queries.len(); + let amount_of_ram_circuits = ((amount_of_explicit_memory_queries + + implicit_memory_queries.amount_of_queries()) as u32 + + geometry.cycles_per_ram_permutation + - 1) + / geometry.cycles_per_ram_permutation; + + // Memory queues simulation is a slowest part in basic witness generation. + // Each queue simulation is sequential single-threaded computation of hashes. + // We will simulate unsorted and sorted queues in separate threads. + + let (tx, rx): ( + Sender, + Receiver, + ) = mpsc::channel(); + + let implicit_memory_queries_arc = Arc::new(implicit_memory_queries); + let memory_queries_arc = Arc::new(memory_queries); + + use std::thread; + let sorted_handle = { + let memory_queries_arc = memory_queries_arc.clone(); + let implicit_memory_queries_arc = implicit_memory_queries_arc.clone(); + let geometry = *geometry; + let round_function = *round_function; + let tx_thread = tx.clone(); + thread::spawn(move || { + simulate_sorted_memory_queue( + geometry, + memory_queries_arc, + implicit_memory_queries_arc, + round_function, + tx_thread, + ) + }) + }; + + let unsorted_handle = { + let memory_queries_arc = memory_queries_arc.clone(); + let implicit_memory_queries_arc = implicit_memory_queries_arc.clone(); + let geometry = *geometry; + let round_function = *round_function; + let tx_thread = tx.clone(); + thread::spawn(move || { + simulate_memory_queue( + geometry, + memory_queries_arc, + implicit_memory_queries_arc, + round_function, + tx_thread, + ) + }) + }; + + // send "finalized" part of RAM permutations circuits witnesses to storage (parts of simulator states, RAM-heavy) + // the rest will be processed further + for _ in 0..amount_of_ram_circuits * 2 { + let artifact = rx.recv().unwrap(); + artifacts_callback(artifact); + } + + let ( + memory_queue_entry_states_for_main_vm, + final_explicit_memory_queue_state, + memory_queue_states_accumulator, + memory_queue_simulator, + implicit_memory_states, + unsorted_encodings, + ) = unsorted_handle.join().unwrap(); + + let ( + sorted_memory_queue_states_accumulator, + sorted_memory_queue_simulator, + sorted_encodings, + sorted_queries_aux_data_for_chunks, + ) = sorted_handle.join().unwrap(); + + let memory_artifacts_for_main_vm = MemoryArtifacts { + memory_queries: Arc::into_inner(memory_queries_arc).unwrap(), + memory_queue_entry_states: memory_queue_entry_states_for_main_vm, + }; + let implicit_memory_queries = Arc::into_inner(implicit_memory_queries_arc).unwrap(); // direct VM related part is done, other subcircuit's functionality is moved to other functions // that should properly do sorts and memory writes - use crate::witness::individual_circuits::memory_related::decommit_code::compute_decommitter_circuit_snapshots; + assert_eq!( + implicit_memory_queries.amount_of_queries(), + implicit_memory_states.amount_of_states() + ); - // precompiles and decommiter will produce additional implicit memory queries - let mut implicit_memory_artifacts: ImplicitMemoryArtifacts = - ImplicitMemoryArtifacts::default(); - implicit_memory_artifacts.memory_queries = - Vec::with_capacity(amount_of_implicit_memory_queries); + use crate::witness::individual_circuits::memory_related::decommit_code::compute_decommitter_circuit_snapshots; tracing::debug!("Running code code decommitter simulation"); - let code_decommitter_circuits_data = compute_decommitter_circuit_snapshots( - amount_of_memory_queries, - &mut implicit_memory_artifacts, - &memory_queue_states_accumulator, - &mut memory_queue_simulator, - decommiter_circuit_inputs, - round_function, - geometry.cycles_per_code_decommitter as usize, - ); + let (code_decommitter_circuits_data, amount_of_memory_queries) = + compute_decommitter_circuit_snapshots( + amount_of_explicit_memory_queries, + implicit_memory_queries.decommitter_memory_queries, + implicit_memory_states.decommitter_simulator_snapshots, + implicit_memory_states.decommitter_memory_states, + final_explicit_memory_queue_state, + decommiter_circuit_inputs, + round_function, + geometry.cycles_per_code_decommitter as usize, + ); circuits_data.code_decommitter_circuits_data = code_decommitter_circuits_data; @@ -1024,17 +1281,18 @@ fn process_memory_related_circuits< tracing::debug!("Running keccak simulation"); - let keccak256_circuits_data = keccak256_decompose_into_per_circuit_witness( - amount_of_memory_queries, - &mut implicit_memory_artifacts, - &memory_queue_states_accumulator, - &mut memory_queue_simulator, - precompiles_data.keccak_round_function_witnesses, - precompiles_data.logs_queries.keccak, - precompiles_data.logs_queues_states.keccak, - geometry.cycles_per_keccak256_circuit as usize, - round_function, - ); + let (keccak256_circuits_data, amount_of_memory_queries) = + keccak256_decompose_into_per_circuit_witness( + amount_of_memory_queries, + implicit_memory_queries.keccak256_memory_queries, + implicit_memory_states.keccak256_simulator_snapshots, + implicit_memory_states.keccak256_memory_states, + precompiles_data.keccak_round_function_witnesses, + precompiles_data.logs_queries.keccak, + precompiles_data.logs_queues_states.keccak, + geometry.cycles_per_keccak256_circuit as usize, + round_function, + ); circuits_data.keccak256_circuits_data = keccak256_circuits_data; // sha256 precompile @@ -1043,17 +1301,18 @@ fn process_memory_related_circuits< tracing::debug!("Running sha256 simulation"); - let sha256_circuits_data = sha256_decompose_into_per_circuit_witness( - amount_of_memory_queries, - &mut implicit_memory_artifacts, - &memory_queue_states_accumulator, - &mut memory_queue_simulator, - precompiles_data.sha256_round_function_witnesses, - precompiles_data.logs_queries.sha256, - precompiles_data.logs_queues_states.sha256, - geometry.cycles_per_sha256_circuit as usize, - round_function, - ); + let (sha256_circuits_data, amount_of_memory_queries) = + sha256_decompose_into_per_circuit_witness( + amount_of_memory_queries, + implicit_memory_queries.sha256_memory_queries, + implicit_memory_states.sha256_simulator_snapshots, + implicit_memory_states.sha256_memory_states, + precompiles_data.sha256_round_function_witnesses, + precompiles_data.logs_queries.sha256, + precompiles_data.logs_queues_states.sha256, + geometry.cycles_per_sha256_circuit as usize, + round_function, + ); circuits_data.sha256_circuits_data = sha256_circuits_data; // ecrecover precompile @@ -1062,54 +1321,56 @@ fn process_memory_related_circuits< tracing::debug!("Running ecrecover simulation"); - let ecrecover_circuits_data = ecrecover_decompose_into_per_circuit_witness( - amount_of_memory_queries, - &mut implicit_memory_artifacts, - &memory_queue_states_accumulator, - &mut memory_queue_simulator, - precompiles_data.ecrecover_witnesses, - precompiles_data.logs_queries.ecrecover, - precompiles_data.logs_queues_states.ecrecover, - geometry.cycles_per_ecrecover_circuit as usize, - round_function, - ); + let (ecrecover_circuits_data, amount_of_memory_queries) = + ecrecover_decompose_into_per_circuit_witness( + amount_of_memory_queries, + implicit_memory_queries.ecrecover_memory_queries, + implicit_memory_states.ecrecover_simulator_snapshots, + implicit_memory_states.ecrecover_memory_states, + precompiles_data.ecrecover_witnesses, + precompiles_data.logs_queries.ecrecover, + precompiles_data.logs_queues_states.ecrecover, + geometry.cycles_per_ecrecover_circuit as usize, + round_function, + ); circuits_data.ecrecover_circuits_data = ecrecover_circuits_data; use crate::witness::individual_circuits::memory_related::secp256r1_verify::secp256r1_verify_decompose_into_per_circuit_witness; tracing::debug!("Running secp256r1_simulation simulation"); - let secp256r1_verify_circuits_data = secp256r1_verify_decompose_into_per_circuit_witness( - amount_of_memory_queries, - &mut implicit_memory_artifacts, - &memory_queue_states_accumulator, - &mut memory_queue_simulator, - precompiles_data.secp256r1_verify_witnesses, - precompiles_data.logs_queries.secp256r1_verify, - precompiles_data.logs_queues_states.secp256r1_verify, - geometry.cycles_per_secp256r1_verify_circuit as usize, - round_function, - ); + let (secp256r1_verify_circuits_data, amount_of_memory_queries) = + secp256r1_verify_decompose_into_per_circuit_witness( + amount_of_memory_queries, + implicit_memory_queries.secp256r1_memory_queries, + implicit_memory_states.secp256r1_simulator_snapshots, + implicit_memory_states.secp256r1_memory_states, + precompiles_data.secp256r1_verify_witnesses, + precompiles_data.logs_queries.secp256r1_verify, + precompiles_data.logs_queues_states.secp256r1_verify, + geometry.cycles_per_secp256r1_verify_circuit as usize, + round_function, + ); circuits_data.secp256r1_verify_circuits_data = secp256r1_verify_circuits_data; - assert!(implicit_memory_artifacts.memory_queries.len() == amount_of_implicit_memory_queries); - use crate::witness::individual_circuits::memory_related::ram_permutation::compute_ram_circuit_snapshots; tracing::debug!("Running RAM permutation simulation"); let (ram_permutation_circuits, ram_permutation_circuits_compact_forms_witnesses) = compute_ram_circuit_snapshots( - &memory_artifacts_for_main_vm.memory_queries, - implicit_memory_artifacts, + amount_of_memory_queries, memory_queue_states_accumulator, + sorted_memory_queue_states_accumulator, memory_queue_simulator, + sorted_memory_queue_simulator, + sorted_queries_aux_data_for_chunks, + sorted_encodings, + unsorted_encodings, round_function, num_non_deterministic_heap_queries, - geometry.cycles_per_ram_permutation as usize, geometry, - &mut circuit_callback, - &mut recursion_queue_callback, + &mut artifacts_callback, ); ( @@ -1121,16 +1382,21 @@ fn process_memory_related_circuits< ) } +pub enum WitnessGenerationArtifact { + BaseLayerCircuit(ZkSyncBaseLayerCircuit), + RecursionQueue( + ( + u64, + RecursionQueueSimulator, + Vec>, + ), + ), + MemoryQueueWitness((MemoryQueueStateWitnesses, bool)), // sorted/unsorted +} + /// Make basic circuits instances and witnesses, /// create artifacts for recursion layer and scheduler -pub(crate) fn create_artifacts_from_tracer< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +pub(crate) fn create_artifacts_from_tracer( tracer: WitnessTracer, round_function: &Poseidon2Goldilocks, geometry: &GeometryConfig, @@ -1142,8 +1408,7 @@ pub(crate) fn create_artifacts_from_tracer< evm_simulator_code_hash: U256, eip_4844_repack_inputs: [Option>; MAX_4844_BLOBS_PER_BLOCK], trusted_setup_path: &str, - mut circuit_callback: CB, - mut recursion_queue_callback: QSCB, + mut artifacts_callback: CB, ) -> ( BlockFirstAndLastBasicCircuitsObservableWitnesses, Vec>, @@ -1211,6 +1476,26 @@ pub(crate) fn create_artifacts_from_tracer< *round_function, ); + use std::thread; + let callstack_handle = { + let log_rollback_tails_for_frames = log_rollback_tails_for_frames.clone(); + let geometry = *geometry; + let round_function = *round_function; + thread::spawn(move || { + // We need to simulate all callstack states and prepare for each MainVM circuit: + // - entry value of callstack sponge + // - callstack witnesses (for every callstack state change) + // - detailed log queue state for entry call frame (frame index, log queue state) + callstack_simulation( + &geometry, + full_callstack_history, + log_states_data, + &log_rollback_tails_for_frames, + &round_function, + ) + }) + }; + // demux log queue circuit use crate::witness::individual_circuits::log_demux::process_logs_demux_and_make_circuits; @@ -1229,8 +1514,7 @@ pub(crate) fn create_artifacts_from_tracer< geometry.cycles_per_log_demuxer as usize, round_function, geometry, - &mut circuit_callback, - &mut recursion_queue_callback, + &mut artifacts_callback, ); tracing::debug!("Processing log circuits"); @@ -1245,8 +1529,7 @@ pub(crate) fn create_artifacts_from_tracer< io_logs_queues_states, demuxed_log_queries.io, round_function, - &mut circuit_callback, - &mut recursion_queue_callback, + &mut artifacts_callback, ); tracing::debug!("Processing memory-related circuits"); @@ -1280,23 +1563,12 @@ pub(crate) fn create_artifacts_from_tracer< executed_decommittment_queries, precompiles_data, round_function, - &mut circuit_callback, - &mut recursion_queue_callback, + &mut artifacts_callback, ); - tracing::debug!("Running callstack sumulation"); + tracing::debug!("Waiting for callstack sumulation"); - // We need to simulate all callstack states and prepare for each MainVM circuit: - // - entry value of callstack sponge - // - callstack witnesses (for every callstack state change) - // - detailed log queue state for entry call frame (frame index, log queue state) - let callstack_simulation_result = callstack_simulation( - geometry, - full_callstack_history, - log_states_data, - &log_rollback_tails_for_frames, - round_function, - ); + let callstack_simulation_result = callstack_handle.join().unwrap(); tracing::debug!( "Processing VM snapshots queue (total {:?})", @@ -1327,8 +1599,7 @@ pub(crate) fn create_artifacts_from_tracer< flat_new_frames_history, vm_snapshots, *round_function, - &mut circuit_callback, - &mut recursion_queue_callback, + &mut artifacts_callback, ); tracing::debug!("Making remaining circuits"); @@ -1362,8 +1633,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::DecommitmentsFilter, decommittments_deduplicator_circuits_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::CodeDecommittmentsSorter(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::CodeDecommittmentsSorter(x), + &mut artifacts_callback, ); // Actual decommitter @@ -1373,8 +1644,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::Decommiter, code_decommitter_circuits_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::CodeDecommitter(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::CodeDecommitter(x), + &mut artifacts_callback, ); // keccak precompiles @@ -1384,8 +1655,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::KeccakPrecompile, keccak256_circuits_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::KeccakRoundFunction(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::KeccakRoundFunction(x), + &mut artifacts_callback, ); // sha256 precompiles @@ -1395,8 +1666,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::Sha256Precompile, sha256_circuits_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::Sha256RoundFunction(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::Sha256RoundFunction(x), + &mut artifacts_callback, ); // ecrecover precompiles @@ -1406,8 +1677,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::EcrecoverPrecompile, ecrecover_circuits_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::ECRecover(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::ECRecover(x), + &mut artifacts_callback, ); // secp256r1 verify @@ -1417,8 +1688,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::Secp256r1Verify, secp256r1_verify_circuits_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::Secp256r1Verify(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::Secp256r1Verify(x), + &mut artifacts_callback, ); // storage sorter @@ -1427,8 +1698,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::StorageFilter, storage_deduplicator_circuit_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::StorageSorter(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::StorageSorter(x), + &mut artifacts_callback, ); // events sorter @@ -1437,8 +1708,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::EventsRevertsFilter, events_deduplicator_circuit_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::EventsSorter(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::EventsSorter(x), + &mut artifacts_callback, ); // l1 messages sorter @@ -1448,8 +1719,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::L1MessagesRevertsFilter, l1_messages_deduplicator_circuit_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::L1MessagesSorter(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::L1MessagesSorter(x), + &mut artifacts_callback, ); // l1 messages pubdata hasher @@ -1459,8 +1730,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::L1MessagesHasher, l1_messages_linear_hash_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::L1MessagesHasher(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::L1MessagesHasher(x), + &mut artifacts_callback, ); // transient storage sorter @@ -1472,8 +1743,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::TransientStorageChecker, transient_storage_sorter_circuit_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::TransientStorageSorter(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::TransientStorageSorter(x), + &mut artifacts_callback, ); // eip 4844 circuits are basic, but they do not need closed form input commitments @@ -1486,8 +1757,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::EIP4844Repack, eip_4844_circuits.clone(), *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::EIP4844Repack(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::EIP4844Repack(x), + &mut artifacts_callback, ); // All done! diff --git a/src/witness/postprocessing/mod.rs b/src/witness/postprocessing/mod.rs index 4fcb8d4f..2bd764df 100644 --- a/src/witness/postprocessing/mod.rs +++ b/src/witness/postprocessing/mod.rs @@ -2,6 +2,7 @@ use super::*; use crate::witness::utils::*; use crate::zkevm_circuits::eip_4844::input::EIP4844OutputData; +use boojum::gadgets::queue::full_state_queue::FullStateCircuitQueueRawWitness; use boojum::gadgets::traits::encodable::WitnessVarLengthEncodable; use circuit_definitions::aux_definitions::witness_oracle::VmWitnessOracle; use circuit_definitions::boojum::field::U64Representable; @@ -63,7 +64,12 @@ use circuit_definitions::zkevm_circuits::transient_storage_validity_by_grand_pro use circuit_definitions::zkevm_circuits::transient_storage_validity_by_grand_product::input::*; use circuit_definitions::Field; use crossbeam::atomic::AtomicCell; +use derivative::Derivative; use observable_witness::ObservableWitness; +use oracle::WitnessGenerationArtifact; +use zkevm_circuits::base_structures::memory_query::{MemoryQuery, MEMORY_QUERY_PACKED_WIDTH}; +use zkevm_circuits::base_structures::vm_state::FULL_SPONGE_QUEUE_STATE_WIDTH; +use zkevm_circuits::ram_permutation::input::RamPermutationCycleInputOutputWitness; use std::sync::Arc; @@ -465,19 +471,15 @@ where pub(crate) fn make_circuits< T: ClosedFormInputField, S: ZkSyncUniformSynthesisFunction, - CB: FnMut(ZkSyncUniformCircuitInstance), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), + WCB: Fn(ZkSyncUniformCircuitInstance) -> ZkSyncBaseLayerCircuit, + CB: FnMut(WitnessGenerationArtifact), >( geometry: u32, circuit_type: BaseLayerCircuitType, circuits_data: Vec, round_function: Poseidon2Goldilocks, - mut circuit_callback: CB, - recursion_queue_callback: &mut QSCB, + wrap_circuit: WCB, + artifacts_callback: &mut CB, ) -> ( FirstAndLastCircuitWitness>, Vec>, @@ -499,7 +501,9 @@ where let mut maker = CircuitMaker::new(geometry, round_function.clone()); for circuit_input in circuits_data.into_iter() { - circuit_callback(maker.process(circuit_input, circuit_type)); + artifacts_callback(WitnessGenerationArtifact::BaseLayerCircuit(wrap_circuit( + maker.process(circuit_input, circuit_type), + ))); } let ( @@ -507,14 +511,39 @@ where recursion_queue_simulator, circuits_compact_forms_witnesses, ) = maker.into_results(); - recursion_queue_callback( + artifacts_callback(WitnessGenerationArtifact::RecursionQueue(( circuit_type as u64, recursion_queue_simulator, circuits_compact_forms_witnesses.clone(), - ); + ))); ( first_and_last_observable_witnesses, circuits_compact_forms_witnesses, ) } + +#[derive(Derivative, serde::Serialize, serde::Deserialize)] +#[derivative(Clone, Debug, Default)] +#[serde(bound = "")] +pub struct RamPermutationQueuesWitness { + pub unsorted_queue_witness: FullStateCircuitQueueRawWitness< + F, + MemoryQuery, + FULL_SPONGE_QUEUE_STATE_WIDTH, + MEMORY_QUERY_PACKED_WIDTH, + >, + pub sorted_queue_witness: FullStateCircuitQueueRawWitness< + F, + MemoryQuery, + FULL_SPONGE_QUEUE_STATE_WIDTH, + MEMORY_QUERY_PACKED_WIDTH, + >, +} + +#[derive(Derivative, serde::Serialize, serde::Deserialize)] +#[derivative(Clone, Debug, Default)] +#[serde(bound = "")] +pub struct RamPermutationCircuitInstancePartialWitness { + pub closed_form_input: RamPermutationCycleInputOutputWitness, +}