proof-producer: enable bbf circuits

- bytecode
- read/write
- zkevm

.gitignore

@@ -13,6 +13,10 @@
 .*.lb
 *.csv
 
+#protobuf generated files
+*pb.h
+*pb.cc
+
 ## Intermediate documents:
 *.dvi
 *.xdv

CMakeLists.txt

@@ -8,7 +8,7 @@ elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
   add_compile_options (-fcolor-diagnostics)
 endif ()
 
-if(DEFINED CMAKE_BUILD_TYPE AND ${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+if(DEFINED CMAKE_BUILD_TYPE AND CMAKE_BUILD_TYPE STREQUAL "Debug")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -ggdb")
     set(BOOST_FORCEINLINE "OFF") # improves debugging traces
 endif()

crypto3/libs/blueprint/include/nil/blueprint/bbf/generic.hpp

@@ -36,10 +36,9 @@
 
 #include <nil/crypto3/zk/math/expression.hpp>
 #include <nil/crypto3/zk/snark/arithmetization/plonk/constraint_system.hpp>
+#include <nil/crypto3/zk/snark/arithmetization/plonk/assignment.hpp>
 // #include <nil/crypto3/zk/snark/arithmetization/plonk/copy_constraint.hpp> // NB: part of the previous include
 
-#include <nil/blueprint/blueprint/plonk/assignment.hpp>
-#include <nil/blueprint/blueprint/plonk/circuit.hpp>
 #include <nil/blueprint/component.hpp>
 //#include <nil/blueprint/manifest.hpp>
 #include <nil/blueprint/gate_id.hpp>
@@ -54,7 +53,7 @@ namespace nil {
 
             template<typename FieldType>
             class basic_context {
-                using assignment_type = assignment<crypto3::zk::snark::plonk_constraint_system<FieldType>>;
+                using assignment_type = nil::crypto3::zk::snark::plonk_assignment_table<FieldType>;
                 using assignment_description_type = nil::crypto3::zk::snark::plonk_table_description<FieldType>;
 
                 public:
@@ -177,8 +176,7 @@ namespace nil {
             class context<FieldType, GenerationStage::ASSIGNMENT> : public basic_context<FieldType> { // assignment-specific definition
             public:
                 using TYPE = typename FieldType::value_type;
-
-                using assignment_type = assignment<crypto3::zk::snark::plonk_constraint_system<FieldType>>;
+                using assignment_type = nil::crypto3::zk::snark::plonk_assignment_table<FieldType>;
                 using assignment_description_type = nil::crypto3::zk::snark::plonk_table_description<FieldType>;
                 using plonk_copy_constraint = crypto3::zk::snark::plonk_copy_constraint<FieldType>;
                 using lookup_input_constraints_type = std::vector<TYPE>;
@@ -292,7 +290,7 @@ namespace nil {
                 using basic_context<FieldType>::col_map;
                 using basic_context<FieldType>::add_rows_to_description;
 
-                using assignment_type = assignment<crypto3::zk::snark::plonk_constraint_system<FieldType>>;
+                using assignment_type = nil::crypto3::zk::snark::plonk_assignment_table<FieldType>;
                 using assignment_description_type = nil::crypto3::zk::snark::plonk_table_description<FieldType>;
 
                 using basic_context<FieldType>::row_shift;

crypto3/libs/blueprint/include/nil/blueprint/bbf/l1_wrapper.hpp

@@ -183,7 +183,7 @@ namespace nil {
             typename plonk_l1_wrapper<BlueprintFieldType, BBFType, ComponentStaticInfoArgs...>::result_type generate_circuit(
                 const plonk_l1_wrapper<BlueprintFieldType, BBFType, ComponentStaticInfoArgs...>  &component,
                 circuit<crypto3::zk::snark::plonk_constraint_system<BlueprintFieldType>>  &bp,
-                assignment<crypto3::zk::snark::plonk_constraint_system<BlueprintFieldType>>  &assignment,
+                nil::crypto3::zk::snark::plonk_assignment_table<BlueprintFieldType> &assignment,
                 const typename BBFType<BlueprintFieldType, nil::blueprint::bbf::GenerationStage::CONSTRAINTS>::input_type  &instance_input,
                 const std::size_t start_row_index,
                 ComponentStaticInfoArgs... component_static_info_args
@@ -298,4 +298,4 @@ namespace nil {
             }
         }
     }
-}
+}

crypto3/libs/blueprint/include/nil/blueprint/zkevm_bbf/bytecode.hpp

@@ -21,6 +21,8 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 //---------------------------------------------------------------------------//
+
+#pragma once
 #include <nil/blueprint/zkevm_bbf/types/hashed_buffers.hpp>
 #include <nil/blueprint/zkevm_bbf/subcomponents/bytecode_table.hpp>
 #include <nil/blueprint/zkevm_bbf/subcomponents/keccak_table.hpp>
@@ -173,4 +175,4 @@ namespace nil {
             };
         }
     }
-}
+}

crypto3/libs/blueprint/include/nil/blueprint/zkevm_bbf/copy.hpp

@@ -21,6 +21,7 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 //---------------------------------------------------------------------------//
+#pragma once
 
 #include <nil/blueprint/zkevm_bbf/subcomponents/keccak_table.hpp>
 #include <nil/blueprint/zkevm_bbf/subcomponents/bytecode_table.hpp>

crypto3/libs/blueprint/include/nil/blueprint/zkevm_bbf/subcomponents/bytecode_table.hpp

@@ -23,6 +23,7 @@
 //---------------------------------------------------------------------------//
 #pragma once
 
+#pragma once
 #include <nil/blueprint/zkevm_bbf/types/hashed_buffers.hpp>
 
 namespace nil {
@@ -119,4 +120,4 @@ namespace nil {
 	    };
         }
     }
-}
+}

crypto3/libs/blueprint/include/nil/blueprint/zkevm_bbf/subcomponents/copy_table.hpp

@@ -117,4 +117,4 @@ namespace nil {
             };
          }
     }
-}
+}

crypto3/libs/blueprint/include/nil/blueprint/zkevm_bbf/subcomponents/keccak_table.hpp

@@ -112,4 +112,4 @@ namespace nil {
             };
         } // namespace bbf
     } // namespace blueprint
-} // namespace nil
+} // namespace nil

crypto3/libs/blueprint/include/nil/blueprint/zkevm_bbf/types/rw_operation.hpp

@@ -108,14 +108,15 @@ namespace nil {
                 std::size_t rw_id,
                 bool is_write,
                 zkevm_word_type value,
-                zkevm_word_type value_prev
+                zkevm_word_type value_prev,
+                zkevm_word_type address = zkevm_word_type(0)
             ){
-                return rw_operation({rw_operation_type::storage, id, 0, 0, storage_key, rw_id, is_write, value, value_prev});
+                return rw_operation({rw_operation_type::storage, id, address, 0, storage_key, rw_id, is_write, value, value_prev});
             }
 
             rw_operation padding_operation(){
                 return rw_operation({rw_operation_type::padding, 0, 0, 0, 0, 0, 0, 0});
             }
         } // namespace bbf
     } // namespace blueprint
-} // namespace nil
+} // namespace nil

crypto3/libs/blueprint/include/nil/blueprint/zkevm_bbf/zkevm.hpp

@@ -39,6 +39,8 @@
 #include <nil/blueprint/zkevm_bbf/subcomponents/bytecode_table.hpp>
 #include <nil/blueprint/zkevm_bbf/subcomponents/rw_table.hpp>
 #include <nil/blueprint/zkevm_bbf/subcomponents/copy_table.hpp>
+#include <nil/blueprint/zkevm_bbf/types/zkevm_state.hpp>
+#include <nil/blueprint/zkevm_bbf/opcodes/zkevm_opcodes.hpp>
 
 #include <nil/blueprint/zkevm_bbf/opcodes/zkevm_opcodes.hpp>
 
@@ -170,7 +172,8 @@ namespace nil {
                             zkevm_opcode current_opcode = opcode_from_number(current_state.opcode);
 
                             if( opcode_impls.find(current_opcode) == opcode_impls.end() ){
-                                std::cout << "Opcode not found" << current_opcode << std::endl;
+                                std::cout << "Opcode not found " << current_opcode << " skip it" << std::endl;
+                                continue;
                             }
                             std::size_t current_opcode_bare_rows_amount = opcode_impls[current_opcode]->rows_amount();
                             std::size_t current_opcode_rows_amount = std::ceil(float(current_opcode_bare_rows_amount)/2) * 2;
@@ -534,4 +537,4 @@ namespace nil {
             };
         }
     }
-}
+}

parallel-crypto3/libs/parallel-zk/include/nil/crypto3/zk/snark/arithmetization/plonk/assignment.hpp

@@ -349,6 +349,12 @@ namespace nil {
                             rows_amount(), std::pow(2, std::ceil(std::log2(rows_amount()))));
                     }
 
+                    crypto3::zk::snark::plonk_table_description<FieldType> get_description() const {
+                        return crypto3::zk::snark::plonk_table_description<FieldType>(
+                            witnesses_amount(), public_inputs_amount(), constants_amount(), selectors_amount(),
+                            rows_amount(), std::pow(2, std::ceil(std::log2(rows_amount()))));
+                    }
+
                     template <typename InputVariableType>
                     const ColumnType& get_variable_value_without_rotation(const InputVariableType& var) const {
                         switch (var.type) {

parallel-crypto3/libs/parallel-zk/include/nil/crypto3/zk/snark/arithmetization/plonk/constraint.hpp

@@ -83,6 +83,23 @@ namespace nil {
                         }
                     }
 
+                    // Constructor for integral types.
+                    template<class NumericType>
+                    plonk_constraint(const NumericType& coeff,
+                            typename std::enable_if<std::is_integral<NumericType>::value, NumericType>::type* = nullptr)
+                        : math::expression<VariableType>(coeff) {
+                    }
+
+                    // Constructor for number<cpp_int_backend<...>>.
+                    template<class BackendType>
+                    plonk_constraint(const boost::multiprecision::number<BackendType> &coeff)
+                        : math::expression<VariableType>(coeff) {
+                    }
+
+                    plonk_constraint(const typename VariableType::assignment_type &coeff)
+                        : math::expression<VariableType>(coeff) {
+                    }
+
                     typename VariableType::assignment_type
                         evaluate(std::size_t row_index,
                                  const plonk_assignment_table<FieldType> &assignments) const {

proof-producer.nix

@@ -6,6 +6,7 @@
   boost,
   gdb,
   lldb,
+  protobuf,
   cmake_modules,
   enableDebugging,
   gtest,
@@ -20,7 +21,7 @@ in stdenv.mkDerivation {
   src = lib.sourceByRegex ./. ["^proof-producer(/.*)?$" "^crypto3(/.*)?$" "^parallel-crypto3(/.*)?$" "CMakeLists.txt"];
   hardeningDisable = [ "fortify" ];
 
-  nativeBuildInputs = [ cmake ninja pkg-config ] ++
+  nativeBuildInputs = [ cmake ninja pkg-config protobuf ] ++
                        (lib.optional (!stdenv.isDarwin) gdb) ++
                        (lib.optional (stdenv.isDarwin) lldb);
 

proof-producer/CMakeLists.txt

@@ -10,7 +10,7 @@
 
 cmake_minimum_required(VERSION 3.22 FATAL_ERROR)
 
-if(DEFINED CMAKE_BUILD_TYPE AND ${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+if(DEFINED CMAKE_BUILD_TYPE AND CMAKE_BUILD_TYPE STREQUAL "Debug")
     set(ZK_PLACEHOLDER_DEBUG_ENABLED TRUE)
 endif()
 

proof-producer/bin/proof-producer/include/nil/proof-generator/prover.hpp

@@ -26,6 +26,7 @@
 #include <random>
 #include <sstream>
 #include <optional>
+#include <chrono>
 
 #include <boost/log/trivial.hpp>
 
@@ -118,10 +119,11 @@ namespace nil {
                 VERIFY = 5,
                 GENERATE_AGGREGATED_CHALLENGE = 6,
                 GENERATE_PARTIAL_PROOF = 7,
-                COMPUTE_COMBINED_Q = 8,
-                GENERATE_AGGREGATED_FRI_PROOF = 9,
-                GENERATE_CONSISTENCY_CHECKS_PROOF = 10,
-                MERGE_PROOFS = 11
+                FAST_GENERATE_PARTIAL_PROOF = 8,
+                COMPUTE_COMBINED_Q = 9,
+                GENERATE_AGGREGATED_FRI_PROOF = 10,
+                GENERATE_CONSISTENCY_CHECKS_PROOF = 11,
+                MERGE_PROOFS = 12
             };
 
             ProverStage prover_stage_from_string(const std::string& stage) {
@@ -134,6 +136,7 @@ namespace nil {
                     {"verify", ProverStage::VERIFY},
                     {"generate-aggregated-challenge", ProverStage::GENERATE_AGGREGATED_CHALLENGE},
                     {"generate-partial-proof", ProverStage::GENERATE_PARTIAL_PROOF},
+                    {"fast-generate-partial-proof", ProverStage::FAST_GENERATE_PARTIAL_PROOF},
                     {"compute-combined-Q", ProverStage::COMPUTE_COMBINED_Q},
                     {"merge-proofs", ProverStage::MERGE_PROOFS},
                     {"aggregated-FRI", ProverStage::GENERATE_AGGREGATED_FRI_PROOF},
@@ -319,6 +322,7 @@ namespace nil {
                 BOOST_ASSERT(lpc_scheme_);
 
                 BOOST_LOG_TRIVIAL(info) << "Generating proof...";
+                auto start = std::chrono::high_resolution_clock::now();
                 auto prover = nil::crypto3::zk::snark::placeholder_prover<BlueprintField, PlaceholderParams>(
                         *public_preprocessed_data_,
                         *private_preprocessed_data_,
@@ -327,6 +331,8 @@ namespace nil {
                         std::move(*lpc_scheme_),
                         true);
                 Proof proof = prover.process();
+                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - start);
+                std::cout << "POOF GENERATE: " << duration.count() << "\n";
                 BOOST_LOG_TRIVIAL(info) << "Proof generated";
 
                 lpc_scheme_.emplace(prover.move_commitment_scheme()); // get back the commitment scheme used in prover
@@ -775,6 +781,7 @@ namespace nil {
                 create_lpc_scheme();
 
                 BOOST_LOG_TRIVIAL(info) << "Preprocessing public data";
+                auto start = std::chrono::high_resolution_clock::now();
                 public_preprocessed_data_.emplace(
                     nil::crypto3::zk::snark::placeholder_public_preprocessor<BlueprintField, PlaceholderParams>::
                         process(
@@ -785,16 +792,21 @@ namespace nil {
                             max_quotient_chunks_
                         )
                 );
+                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - start);
+                std::cout << "PREPROCESS: " << duration.count() << "\n";
                 return true;
             }
 
             bool preprocess_private_data() {
 
                 BOOST_LOG_TRIVIAL(info) << "Preprocessing private data";
+                auto start = std::chrono::high_resolution_clock::now();
                 private_preprocessed_data_.emplace(
                     nil::crypto3::zk::snark::placeholder_private_preprocessor<BlueprintField, PlaceholderParams>::
                         process(*constraint_system_, assignment_table_->move_private_table(), *table_description_)
                 );
+                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - start);
+                std::cout << "PREPROCESS PRIVATE DATA: " << duration.count() << "\n";
 
                 // This is the last stage of preprocessor, and the assignment table is not used after this function call.
                 assignment_table_.reset();
@@ -1108,11 +1120,14 @@ namespace nil {
             }
 
             bool setup_prover() {
+                auto start = std::chrono::high_resolution_clock::now();
                 const auto err = CircuitFactory<BlueprintField>::initialize_circuit(circuit_name_, constraint_system_, assignment_table_, table_description_);
                 if (err) {
                     BOOST_LOG_TRIVIAL(error) << "Can't initialize circuit " << circuit_name_ << ": " << err.value();
                     return false;
                 }
+                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - start);
+                std::cout << "PRESET: " << duration.count() << "\n";
 
                 return true;
             }

proof-producer/bin/proof-producer/src/arg_parser.hpp

@@ -65,12 +65,12 @@ namespace nil {
             boost::filesystem::path proof_of_work_output_file = "proof_of_work.dat";
             boost::log::trivial::severity_level log_level = boost::log::trivial::severity_level::info;
             CurvesVariant elliptic_curve_type = type_identity<nil::crypto3::algebra::curves::pallas>{};
-            HashesVariant hash_type = type_identity<nil::crypto3::hashes::keccak_1600<256>>{};
+            HashesVariant hash_type = type_identity</*nil::crypto3::hashes::keccak_1600<256>*/nil::crypto3::hashes::sha2<256>>{};
 
             std::size_t lambda = 9;
             std::size_t grind = 0;
-            std::size_t expand_factor = 2;
-            std::size_t max_quotient_chunks = 0;
+            std::size_t expand_factor = 0;//2;
+            std::size_t max_quotient_chunks = 20;//0;
         };
 
         std::optional<ProverOptions> parse_args(int argc, char* argv[]);

proof-producer/bin/proof-producer/src/main.cpp

@@ -76,6 +76,9 @@ int run_prover(const nil::proof_generator::ProverOptions& prover_options) {
                     if (!prover_options.assignment_table_file_path.empty() && prover_result) {
                         prover_result = prover.save_binary_assignment_table_to_file(prover_options.assignment_table_file_path);
                     }
+                    if (!prover_options.assignment_description_file_path.empty() && prover_result) {
+                        prover_result = prover.save_assignment_description(prover_options.assignment_description_file_path);
+                    }
                     break;
                 case nil::proof_generator::detail::ProverStage::PREPROCESS:
                     prover_result =
@@ -122,6 +125,20 @@ int run_prover(const nil::proof_generator::ProverOptions& prover_options) {
                             prover_options.theta_power_file_path) &&
                         prover.save_commitment_state_to_file(prover_options.updated_commitment_scheme_state_path);
                     break;
+                case nil::proof_generator::detail::ProverStage::FAST_GENERATE_PARTIAL_PROOF:
+                    // Preset, fill assignment table, preprocess
+                    prover_result =
+                        prover.setup_prover() &&
+                        prover.fill_assignment_table(prover_options.trace_file_path) &&
+                        prover.preprocess_public_data() &&
+                        prover.save_preprocessed_common_data_to_file(prover_options.preprocessed_common_data_path) &&
+                        prover.preprocess_private_data() &&
+                        prover.generate_partial_proof_to_file(
+                            prover_options.proof_file_path,
+                            prover_options.challenge_file_path,
+                            prover_options.theta_power_file_path) &&
+                        prover.save_commitment_state_to_file(prover_options.updated_commitment_scheme_state_path);
+                    break;
                 case nil::proof_generator::detail::ProverStage::VERIFY:
                     prover_result =
                         prover.read_circuit(prover_options.circuit_file_path) &&