add circuit data serialization

plonky2_poseidon2/Cargo.toml

@@ -11,8 +11,10 @@ anyhow     = { version = "1.0.89" }
 unroll = { version = "0.1.5", default-features = false }
 serde      = { version = "1.0.210" , features = ["rc"] }
 serde_json = { version = "1.0" }
-plonky2    = { version = "0.2.2" }
+plonky2    = { version = "0.2.2", default-features = true}
 plonky2_field = { version = "0.2.2", default-features = false }
+log = { version = "0.4.20", default-features = false }
+jemallocator = "0.5.4"
 
 [dev-dependencies]
 criterion = { version = "0.5.1", default-features = false }

plonky2_poseidon2/src/gate/mod.rs

@@ -1 +1 @@
-pub mod poseidon2;
+pub mod poseidon2;

plonky2_poseidon2/src/lib.rs

@@ -1,3 +1,4 @@
 pub mod gate;
 pub mod poseidon2_hash;
-pub mod config;
+pub mod config;
+pub mod serialization;

plonky2_poseidon2/src/serialization/gate_serialization.rs

@@ -0,0 +1,88 @@
+use std::time::Instant;
+use plonky2_field::extension::Extendable;
+use plonky2::gates::arithmetic_base::ArithmeticGate;
+use plonky2::gates::arithmetic_extension::ArithmeticExtensionGate;
+use plonky2::gates::base_sum::BaseSumGate;
+use plonky2::gates::constant::ConstantGate;
+use plonky2::gates::coset_interpolation::CosetInterpolationGate;
+use plonky2::gates::exponentiation::ExponentiationGate;
+use plonky2::gates::gate::GateRef;
+use plonky2::gates::lookup::LookupGate;
+use plonky2::gates::lookup_table::LookupTableGate;
+use plonky2::gates::multiplication_extension::MulExtensionGate;
+use plonky2::gates::noop::NoopGate;
+use plonky2::gates::poseidon::PoseidonGate;
+use crate::gate::poseidon2::Poseidon2Gate;
+use plonky2::gates::poseidon_mds::PoseidonMdsGate;
+use plonky2::gates::public_input::PublicInputGate;
+use plonky2::gates::random_access::RandomAccessGate;
+use plonky2::gates::reducing::ReducingGate;
+use plonky2::gates::reducing_extension::ReducingExtensionGate;
+use plonky2::hash::hash_types::RichField;
+use plonky2::{read_gate_impl, get_gate_tag_impl};
+use plonky2::plonk::circuit_data::{CircuitConfig, CommonCircuitData};
+use plonky2::util::serialization::{Buffer, GateSerializer, IoResult};
+use crate::poseidon2_hash::poseidon2::Poseidon2;
+use std::vec::Vec;
+use plonky2::iop::witness::PartialWitness;
+use plonky2::plonk::circuit_builder::CircuitBuilder;
+use plonky2_field::goldilocks_field::GoldilocksField;
+
+#[macro_export]
+/// The macros are re-implemented here because of import issue with plonky2
+/// in plonky2 `use std::vec::Vec;` // For macros was not set to public
+/// so here these are re-implemented
+macro_rules! impl_gate_serializer {
+    ($target:ty, $($gate_types:ty),+) => {
+        fn read_gate(
+            &self,
+            buf: &mut plonky2::util::serialization::Buffer,
+            common: &plonky2::plonk::circuit_data::CommonCircuitData<F, D>,
+        ) -> plonky2::util::serialization::IoResult<plonky2::gates::gate::GateRef<F, D>> {
+            let tag = plonky2::util::serialization::Read::read_u32(buf)?;
+            read_gate_impl!(buf, tag, common, $($gate_types),+)
+        }
+
+        fn write_gate(
+            &self,
+            buf: &mut Vec<u8>,
+            gate: &plonky2::gates::gate::GateRef<F, D>,
+            common: &plonky2::plonk::circuit_data::CommonCircuitData<F, D>,
+        ) -> plonky2::util::serialization::IoResult<()> {
+            let tag = get_gate_tag_impl!(gate, $($gate_types),+)?;
+
+            plonky2::util::serialization::Write::write_u32(buf, tag)?;
+            gate.0.serialize(buf, common)?;
+            Ok(())
+        }
+    };
+}
+
+/// A gate serializer that can be used to serialize all default gates supported
+/// by the `plonky2` library with the added Poseidon2 Gate
+#[derive(Debug)]
+pub struct DefaultGateSerializer;
+impl<F: RichField + Extendable<D> + Poseidon2, const D: usize> GateSerializer<F, D> for DefaultGateSerializer {
+    impl_gate_serializer! {
+            DefaultGateSerializer,
+            ArithmeticGate,
+            ArithmeticExtensionGate<D>,
+            BaseSumGate<2>,
+            ConstantGate,
+            CosetInterpolationGate<F, D>,
+            ExponentiationGate<F, D>,
+            LookupGate,
+            LookupTableGate,
+            MulExtensionGate<D>,
+            NoopGate,
+            PoseidonMdsGate<F, D>,
+            PoseidonGate<F, D>,
+            Poseidon2Gate<F, D>,
+            PublicInputGate,
+            RandomAccessGate<F, D>,
+            ReducingExtensionGate<D>,
+            ReducingGate<D>
+        }
+}
+
+

plonky2_poseidon2/src/serialization/generator_serialization.rs

@@ -0,0 +1,80 @@
+use std::marker::PhantomData;
+use plonky2_field::extension::Extendable;
+use plonky2::gates::arithmetic_base::{ArithmeticBaseGenerator, ArithmeticGate};
+use plonky2::gates::arithmetic_extension::{ArithmeticExtensionGate, ArithmeticExtensionGenerator};
+use plonky2::gates::base_sum::{BaseSplitGenerator, BaseSumGate};
+use plonky2::gates::constant::ConstantGate;
+use plonky2::gates::coset_interpolation::{CosetInterpolationGate, InterpolationGenerator};
+use plonky2::gates::exponentiation::{ExponentiationGate, ExponentiationGenerator};
+use plonky2::gates::gate::{AnyGate, Gate, GateRef};
+use plonky2::gates::lookup::{LookupGate, LookupGenerator};
+use plonky2::gates::lookup_table::{LookupTableGate, LookupTableGenerator};
+use plonky2::gates::multiplication_extension::{MulExtensionGate, MulExtensionGenerator};
+use plonky2::gates::noop::NoopGate;
+use plonky2::gates::poseidon::{PoseidonGate, PoseidonGenerator};
+use crate::gate::poseidon2::{Poseidon2Gate, Poseidon2Generator};
+use plonky2::gates::poseidon_mds::{PoseidonMdsGate, PoseidonMdsGenerator};
+use plonky2::gates::public_input::PublicInputGate;
+use plonky2::gates::random_access::{RandomAccessGate, RandomAccessGenerator};
+use plonky2::gates::reducing::{ReducingGate, ReducingGenerator};
+use plonky2::gates::reducing_extension::ReducingGenerator as ReducingExtensionGenerator;
+use plonky2::gates::reducing_extension::ReducingExtensionGate;
+use plonky2::hash::hash_types::RichField;
+use plonky2::{impl_gate_serializer, impl_generator_serializer, get_generator_tag_impl, read_generator_impl};
+use plonky2::read_gate_impl;
+use plonky2::get_gate_tag_impl;
+use plonky2::plonk::circuit_data::CommonCircuitData;
+use plonky2::util::serialization::{Buffer, GateSerializer, IoError, IoResult, Read, WitnessGeneratorSerializer, Write};
+use crate::poseidon2_hash::poseidon2::Poseidon2;
+use std::vec::Vec;
+use plonky2::gadgets::arithmetic::EqualityGenerator;
+use plonky2::gadgets::arithmetic_extension::QuotientGeneratorExtension;
+use plonky2::gadgets::range_check::LowHighGenerator;
+use plonky2::gadgets::split_base::BaseSumGenerator;
+use plonky2::gadgets::split_join::{SplitGenerator, WireSplitGenerator};
+use plonky2::iop::generator::{ConstantGenerator, CopyGenerator, NonzeroTestGenerator, RandomValueGenerator, SimpleGenerator, WitnessGeneratorRef};
+use plonky2::plonk::config::{AlgebraicHasher, GenericConfig};
+use plonky2::recursion::dummy_circuit::DummyProofGenerator;
+
+#[derive(Debug, Default)]
+pub struct DefaultGeneratorSerializer<C: GenericConfig<D>, const D: usize> {
+    pub _phantom: PhantomData<C>,
+}
+
+/// A generator serializer that can be used to serialize all default generators supported
+/// by the `plonky2` library with the added `Poseidon2Generator`
+impl<F, C, const D: usize> WitnessGeneratorSerializer<F, D> for DefaultGeneratorSerializer<C, D>
+    where
+        F: RichField + Extendable<D> + Poseidon2,
+        C: GenericConfig<D, F = F> + 'static,
+        C::Hasher: AlgebraicHasher<F>,
+{
+    impl_generator_serializer! {
+            DefaultGeneratorSerializer,
+            ArithmeticBaseGenerator<F, D>,
+            ArithmeticExtensionGenerator<F, D>,
+            BaseSplitGenerator<2>,
+            BaseSumGenerator<2>,
+            ConstantGenerator<F>,
+            CopyGenerator,
+            DummyProofGenerator<F, C, D>,
+            EqualityGenerator,
+            ExponentiationGenerator<F, D>,
+            InterpolationGenerator<F, D>,
+            LookupGenerator,
+            LookupTableGenerator,
+            LowHighGenerator,
+            MulExtensionGenerator<F, D>,
+            NonzeroTestGenerator,
+            PoseidonGenerator<F, D>,
+            Poseidon2Generator<F, D>,
+            PoseidonMdsGenerator<D>,
+            QuotientGeneratorExtension<D>,
+            RandomAccessGenerator<F, D>,
+            RandomValueGenerator,
+            ReducingGenerator<D>,
+            ReducingExtensionGenerator<D>,
+            SplitGenerator,
+            WireSplitGenerator
+        }
+}

plonky2_poseidon2/src/serialization/mod.rs

@@ -0,0 +1,5 @@
+pub mod gate_serialization;
+pub mod generator_serialization;
+
+pub use gate_serialization::DefaultGateSerializer;
+pub use generator_serialization::DefaultGeneratorSerializer;

proof-input/src/gen_input.rs

@@ -395,7 +395,7 @@ mod tests {
     use plonky2::iop::witness::PartialWitness;
     use plonky2::plonk::circuit_builder::CircuitBuilder;
     use codex_plonky2_circuits::circuits::params::CircuitParams;
-    use codex_plonky2_circuits::circuits::sample_cells::{MerklePath, SampleCircuit, SampleCircuitInput};
+    use codex_plonky2_circuits::circuits::sample_cells::SampleCircuit;
     use crate::params::{C, D, F};
 
     // Test sample cells (non-circuit)
@@ -410,20 +410,16 @@ mod tests {
     #[test]
     fn test_proof_in_circuit() -> anyhow::Result<()> {
         // get input
-        let params = TestParams::default();
+        let mut params = TestParams::default();
+        params.n_samples = 10;
         let circ_input = gen_testing_circuit_input::<F,D>(&params);
 
         // Create the circuit
         let config = CircuitConfig::standard_recursion_config();
         let mut builder = CircuitBuilder::<F, D>::new(config);
 
-        let circuit_params = CircuitParams {
-            max_depth: params.max_depth,
-            max_log2_n_slots: params.dataset_max_depth(),
-            block_tree_depth: params.bot_depth(),
-            n_field_elems_per_cell: params.n_field_elems_per_cell(),
-            n_samples: params.n_samples,
-        };
+        let mut circuit_params = CircuitParams::default();
+        circuit_params.n_samples = 10;
 
         // build the circuit
         let circ = SampleCircuit::new(circuit_params.clone());

proof-input/src/json.rs

@@ -1,7 +1,9 @@
 use anyhow::{anyhow, Error, Result};
 use serde::{Deserialize, Serialize};
 use std::fs::File;
+use std::{fs, io};
 use std::io::{BufReader, Write};
+use std::path::Path;
 use crate::gen_input::{DatasetTree, gen_testing_circuit_input};
 use plonky2::hash::hash_types::{HashOut, RichField};
 use plonky2::plonk::config::{GenericConfig, Hasher};
@@ -271,17 +273,27 @@ pub fn import_circ_input_from_json<F: RichField + Extendable<D> + Poseidon2, con
     Ok(circ_input)
 }
 
+/// Writes the provided bytes to the specified file path using `std::fs::write`.
+pub fn write_bytes_to_file<P: AsRef<Path>>(data: Vec<u8>, path: P) -> io::Result<()> {
+    fs::write(path, data)
+}
+
+/// Reads the contents of the specified file and returns them as a vector of bytes using `std::fs::read`.
+pub fn read_bytes_from_file<P: AsRef<Path>>(path: P) -> io::Result<Vec<u8>> {
+    fs::read(path)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::params::{C, D, F};
-    use std::fs;
     use std::time::Instant;
     use codex_plonky2_circuits::circuits::params::CircuitParams;
     use codex_plonky2_circuits::circuits::sample_cells::SampleCircuit;
     use plonky2::iop::witness::PartialWitness;
     use plonky2::plonk::circuit_builder::CircuitBuilder;
-    use plonky2::plonk::circuit_data::CircuitConfig;
+    use plonky2::plonk::circuit_data::{CircuitConfig, CircuitData};
+    use plonky2_poseidon2::serialization::{DefaultGateSerializer, DefaultGeneratorSerializer};
     use crate::gen_input::verify_circuit_input;
 
     // Test to generate the JSON file
@@ -342,13 +354,8 @@ mod tests {
         let config = CircuitConfig::standard_recursion_config();
         let mut builder = CircuitBuilder::<F, D>::new(config);
 
-        let circuit_params = CircuitParams {
-            max_depth: params.max_depth,
-            max_log2_n_slots: params.dataset_max_depth(),
-            block_tree_depth: params.bot_depth(),
-            n_field_elems_per_cell: params.n_field_elems_per_cell(),
-            n_samples: params.n_samples,
-        };
+        let circuit_params = CircuitParams::default();
+
         let circ = SampleCircuit::new(circuit_params.clone());
         let mut targets = circ.sample_slot_circuit(&mut builder);
 
@@ -399,4 +406,56 @@ mod tests {
 
         Ok(())
     }
+
+    // test out custom default gate and generator serializers to export/import circuit data
+    #[test]
+    fn test_circuit_data_serializer() -> anyhow::Result<()> {
+        let params = TestParams::default();
+
+        // Create the circuit
+        let config = CircuitConfig::standard_recursion_config();
+        let mut builder = CircuitBuilder::<F, D>::new(config);
+
+        let circuit_params = CircuitParams::default();
+        let circ = SampleCircuit::new(circuit_params.clone());
+        let mut targets = circ.sample_slot_circuit(&mut builder);
+
+        // Create a PartialWitness and assign
+        let mut pw = PartialWitness::new();
+
+        // gen circ input
+        let imported_circ_input: SampleCircuitInput<F, D> = gen_testing_circuit_input::<F,D>(&params);
+        circ.sample_slot_assign_witness(&mut pw, &mut targets, imported_circ_input);
+
+        // Build the circuit
+        let data = builder.build::<C>();
+        println!("circuit size = {:?}", data.common.degree_bits());
+
+        let gate_serializer = DefaultGateSerializer;
+        let generator_serializer =DefaultGeneratorSerializer::<C, D>::default();
+        let data_bytes = data.to_bytes(&gate_serializer, &generator_serializer).unwrap();
+
+        let file_path = "circ_data.bin";
+        // Write data to the file
+        write_bytes_to_file(data_bytes.clone(), file_path).unwrap();
+        println!("Data written to {}", file_path);
+
+        // Read data back from the file
+        let read_data = read_bytes_from_file(file_path).unwrap();
+        let data = CircuitData::<F,C,D>::from_bytes(&read_data, &gate_serializer, &generator_serializer).unwrap();
+
+        // Prove the circuit with the assigned witness
+        let start_time = Instant::now();
+        let proof_with_pis = data.prove(pw)?;
+        println!("prove_time = {:?}", start_time.elapsed());
+
+        // Verify the proof
+        let verifier_data = data.verifier_data();
+        assert!(
+            verifier_data.verify(proof_with_pis).is_ok(),
+            "Merkle proof verification failed"
+        );
+
+        Ok(())
+    }
 }

workflow/benches/sample_cells.rs

@@ -5,19 +5,22 @@ use plonky2::plonk::circuit_builder::CircuitBuilder;
 use plonky2::plonk::circuit_data::CircuitConfig;
 use plonky2::plonk::config::GenericConfig;
 
-use proof_input::json::import_circ_input_from_json;
-use codex_plonky2_circuits::circuits::sample_cells::{SampleCircuit, SampleCircuitInput};
+use codex_plonky2_circuits::circuits::sample_cells::SampleCircuit;
 use codex_plonky2_circuits::circuits::params::CircuitParams;
-use proof_input::params::{D, C, F, Params};
+use proof_input::gen_input::gen_testing_circuit_input;
+use proof_input::params::{D, C, F, Params, TestParams};
 
 /// Benchmark for building, proving, and verifying the Plonky2 circuit.
 fn bench_prove_verify(c: &mut Criterion) {
     // get default parameters
-    let circuit_params = CircuitParams::default();
+    let mut test_params = TestParams::default();
+    test_params.n_samples = 10;
 
-    // Import the circuit input from a JSON file
-    let circ_input: SampleCircuitInput<F, D> = import_circ_input_from_json("input.json").expect("Failed to import circuit input from JSON");
-    println!("Witness imported from input.json");
+    let mut circuit_params = CircuitParams::default();
+    circuit_params.n_samples = 10;
+
+    // gen the circuit input
+    let circ_input = gen_testing_circuit_input::<F,D>(&test_params);
 
     // Create the circuit configuration
     let config = CircuitConfig::standard_recursion_config();
@@ -54,16 +57,11 @@ fn bench_prove_verify(c: &mut Criterion) {
     println!("Build time: {:?}", build_duration);
     println!("Circuit size (degree bits): {:?}", data.common.degree_bits());
 
-    // Benchmark the Proving Phase
-    group.bench_function("Prove Circuit", |b| {
-        b.iter(|| {
-            let local_pw = pw.clone();
-            data.prove(local_pw).expect("Failed to prove circuit")
-        })
-    });
-
     // Generate the proof once for verification benchmarking
+    let prove_start = std::time::Instant::now();
     let proof_with_pis = data.prove(pw.clone()).expect("Failed to prove circuit");
+    let prove_duration = prove_start.elapsed();
+    println!("prove time: {:?}", prove_duration);
     let verifier_data = data.verifier_data();
 
     println!("Proof size: {} bytes", proof_with_pis.to_bytes().len());

workflow/src/bin/build_circ.rs

@@ -4,6 +4,8 @@ use plonky2::plonk::circuit_builder::CircuitBuilder;
 use anyhow::Result;
 use std::time::Instant;
 use codex_plonky2_circuits::circuits::sample_cells::SampleCircuit;
+use plonky2_poseidon2::serialization::{DefaultGateSerializer,DefaultGeneratorSerializer};
+use proof_input::json::write_bytes_to_file;
 use proof_input::params::Params;
 use proof_input::params::{D, C, F};
 
@@ -24,5 +26,14 @@ fn main() -> Result<()> {
     println!("Build time: {:?}", build_time.elapsed());
     println!("Circuit size (degree bits): {:?}", data.common.degree_bits());
 
+    let gate_serializer = DefaultGateSerializer;
+    let generator_serializer =DefaultGeneratorSerializer::<C, D>::default();
+    let data_bytes = data.to_bytes(&gate_serializer, &generator_serializer).unwrap();
+
+    let file_path = "circ_data.bin";
+    // Write data to the file
+    write_bytes_to_file(data_bytes.clone(), file_path).unwrap();
+    println!("Data written to {}", file_path);
+
     Ok(())
 }

workflow/src/bin/gen_input.rs

@@ -15,6 +15,7 @@ fn main() -> Result<()> {
     // export circuit parameters to json file
     let filename= "input.json";
     export_circ_input_to_json(circ_input, filename)?;
+    println!("proof input written to {}", filename);
 
     Ok(())
 }