update readme and refactor

codex-plonky2-circuits/README.md

@@ -11,46 +11,19 @@ This crate is an implementation of the [codex storage proofs circuits](https://g
 
 - [`sample_cells`](./src/circuits/sample_cells.rs) is the Plonky2 Circuit implementation for sampling cells in dataset merkle tree.
 
+- [`keyed_compress`](./src/circuits/keyed_compress.rs) is the compression function used in the construction (and reconstruction) of the Merkle tree root. The function takes 2 hash digest (4 Goldilocks field elements each) and a key, then outputs a single hash digest. 
+
+- [`sponge`](./src/circuits/sponge.rs) contains the hash function (with and without padding) used to hash cells and during sampling. 
+
 - [`params`](./src/circuits/params.rs) is the parameters used in the circuits.
 
 - [`utils`](./src/circuits/utils.rs) contains helper functions.
 
+## Documentation
+writeup coming soon...
+
 ## Usage
-TODO!
+see [`workflow`](../workflow) for how to use the circuits and run them. 
 
 ## Benchmarks
-In here we show the preliminary benchmarks of codex storage proofs circuits. 
+see [`BENCHMARKS.md`](../proof-input/BENCHMARKS.md)
-
-### Running benchmarks
-To run the benchmarks for safe merkle tree circuit, you can use the following command:
-
-```bash
-cargo bench --bench safe_circuit
-```
-To run the benchmarks for proving cells circuit, you can use the following command:
-Note: make sure to adjust the parameters as need in [`params`](./src/circuits/params.rs)
-
-```bash
-cargo bench --bench prove_cells
-```
-
-To run the benchmarks for sampling circuit, you can use the following command:
-Note: make sure to adjust the parameters as need in [`params`](./src/circuits/params.rs)
-
-```bash
-cargo bench --bench sample_cells
-```
-
-### Results
-Benchmark results for proving 10 cells (10 samples) in a Slot Merkle tree with max depth 16 with 
-the small tree (block tree) is of depth 5 so 32 cells in each block. Cell data size is 2048 bytes (256 field elements)
-
-| Operation | Time (ms) |
-|-----------|-----------|
-| **Build** | 34.4 ms   |
-| **Prove** | 50.3 ms   |
-| **Verify**| 2.4 ms    |
-
-Circuit size: 2<sup>10</sup> gates
-
-Proof size: 116,008 bytes

proof-input/BENCHMARKS.md

@@ -0,0 +1,41 @@
+## Benchmarks
+
+In here we show the preliminary benchmarks of codex storage proofs circuits.
+
+## Running Benchmarks
+
+To run the benchmarks for safe merkle tree circuit, you can use the following command:
+
+```bash
+cargo bench --bench safe_circuit
+```
+
+To run the benchmarks for sampling circuit, you can use the following command:
+Note: make sure to adjust the parameters as need in ....
+
+```bash
+cargo bench --bench sample_cells
+```
+
+The following operations were benchmarked:
+
+- **Build Circuit**: Time taken to construct the circuit for the specified params.
+- **Prove Circuit**: Time taken to generate a proof for the constructed circuit.
+- **Verify Circuit**: Time taken to verify the generated proof.
+
+#### Build Time
+
+
+#### Prove Time
+
+
+#### Verify Time
+
+
+#### Proof Size
+
+
+#### Peak Memory Usage
+
+
+### Remarks

proof-input/README.md

@@ -0,0 +1,23 @@
+# Input Generator for the Plonky2 Circuit
+WARNING: This is a work-in-progress prototype, and has not received careful code review. This implementation is NOT ready for production use.
+
+This crate generates input to the proof circuit based on the test parameters. The proof input generated can be ported into
+the [`plonky2 codex proof circuits`](../codex-plonky2-circuits). Currently only generates fake data for testing.
+
+## Code organization
+
+- [`gen_input`](./src/gen_input.rs) contains the necessary function to generate the proof input. 
+
+- [`json`](./src/json.rs) contains the serialization function to read and write the proof input from/to json files. 
+
+- [`params`](./src/params.rs) is the test parameters used to generate the input. The params include circuit params as well.
+
+- [`sponge`](./src/sponge.rs) contains the non-circuit version of hash function (with and without padding) used to hash cells and during sampling.
+
+- [`utils`](./src/utils.rs) contains helper functions.
+
+## Usage
+see [`workflow`](../workflow) for how to generate proof input.
+
+## Benchmarks
+see [`BENCHMARKS.md`](../proof-input/BENCHMARKS.md)

proof-input/src/gen_input.rs

@@ -363,7 +363,6 @@ impl<
                 mask_bits.clone()
             );
             let cell_index = bits_le_padded_to_usize(&cell_index_bits);
-            println!("sample cell index = {}", cell_index);
             let mut s_proof = slot.get_proof(cell_index);
             Self::pad_proof(&mut s_proof, self.params.max_depth);
             slot_proofs.push(s_proof);

proof-input/src/params.rs

@@ -5,6 +5,7 @@ use plonky2::plonk::config::{GenericConfig, PoseidonGoldilocksConfig};
 use std::env;
 use anyhow::{Result, Context};
 use codex_plonky2_circuits::circuits::params::CircuitParams;
+use plonky2_field::goldilocks_field::GoldilocksField;
 use plonky2_poseidon2::config::Poseidon2GoldilocksConfig;
 
 // test types
@@ -132,14 +133,12 @@ impl TestParams {
 
     // DATASET_DEPTH
     pub fn dataset_max_depth(&self) -> usize {
-        // self.max_slots.trailing_zeros() as usize
         ceiling_log2(self.max_slots)
     }
 
     // DATASET_DEPTH for test
     pub fn dataset_depth_test(&self) -> usize {
         ceiling_log2(self.n_slots)
-        // self.n_slots.trailing_zeros() as usize
     }
 
 }

workflow/Cargo.toml

@@ -18,4 +18,16 @@ proof-input = { path = "../proof-input" }
 
 [[bin]]
 name = "prove_and_verify"
-path = "src/main.rs"
+path = "src/bin/prove_and_verify.rs"
+
+[[bin]]
+name = "gen_input"
+path = "src/bin/gen_input.rs"
+
+[[bin]]
+name = "build_circ"
+path = "src/bin/build_circ.rs"
+
+[[bin]]
+name = "prove"
+path = "src/bin/prove.rs"

workflow/README.md

@@ -0,0 +1,121 @@
+# Workflow of the Storage Proof Circuits
+WARNING: This is a work-in-progress prototype, and has not received careful code review. This implementation is NOT ready for production use.
+
+This crate guides you through generating the circuit input, 
+exporting it to a JSON file, importing from a JSON file, 
+running the circuits to generate a proof, and finally verify the proof.
+
+This crate can be used to:
+
+- Generate circuit input from **fake data** with given params.
+- Build the plonky2 codex storage proof circuits. 
+- Generate a proof with given proof input in JSON file.
+- Verify the proof.
+
+## Code organization
+
+- [`gen_input`](./src/bin/gen_input.rs) contains the main function to generated input with the given params as environment variables. 
+
+- [`build_circ`](./src/bin/build_circ.rs) contains the main function to generated input with the given params as environment variables.
+
+- [`prove`](./src/bin/prove.rs) contains the main function to generated input with the given params as environment variables.
+
+- [`prove_and_verify`](./src/bin/prove_and_verify.rs) contains the main function to generated input with the given params as environment variables.
+
+
+## Usage
+
+### Prerequisites
+
+- **Rust Toolchain**: Ensure you have Rust installed. If not, install it from [rustup.rs](https://rustup.rs/).
+
+- **Rust nightly**:: This crate requires the Rust nightly compiler. To install the nightly toolchain, use `rustup`:
+
+```bash
+rustup install nightly
+```
+
+To ensure that the nightly toolchain is used when building this crate, you can set the override in the project directory:
+
+```bash
+rustup override set nightly
+```
+
+### Generate Circuit Input
+The steps to generate circuit input with **fake data** are the following:
+
+#### Step 1: Setting Up Parameters
+Parameters for generating the circuit input can be defined in [`params.sh`](./params.sh).
+You can customize the test parameters by setting the following environment variables:
+
+```bash
+export MAXDEPTH=32        # Maximum depth of the slot tree
+export MAXSLOTS=256       # Maximum number of slots
+export CELLSIZE=2048      # Cell size in bytes
+export BLOCKSIZE=65536    # Block size in bytes
+export NSAMPLES=5         # Number of samples to prove
+
+export ENTROPY=1234567    # External randomness
+export SEED=12345         # Seed for creating fake data
+
+export NSLOTS=11          # Number of slots in the dataset
+export SLOTINDEX=3        # Which slot to prove (0..NSLOTS-1)
+export NCELLS=512         # Number of cells in this slot
+```
+Alternatively, for testing you can just use the default parameters as follows:
+
+```rust
+use proof_input::params::TestParams;
+
+fn main() {
+    let params = TestParams::<F,D>::default();
+}
+```
+#### Step 2: Run the Script
+Once the params are set, you can run the script to generate the [`JSON file`](./input.json).
+
+```bash
+sudo bash ./gen_input.sh
+```
+
+### Build the Circuit
+To build the circuit and measure the time to build, you can simply run the script:
+```bash
+sudo bash ./build_circuit.sh
+```
+To see the source code of how to build the circuit, see [`build_circ`](./src/bin/build_circ.rs).
+
+### Generate the Proof
+After generating the circuit input (in a JSON file),
+you can run the circuits to generate the proofs.
+First make sure you have the [`JSON file`](./input.json), then follow the steps:
+
+#### Step 1: Setting Up Circuit Parameters
+Parameters for the circuit can be defined in [`circ_params.sh`](./circ_params.sh).
+You can customize the test parameters by setting the following environment variables:
+```bash
+export MAX_DEPTH=32        # maximum depth of the slot tree
+export MAX_LOG2_N_SLOTS=8 # Depth of the dataset tree = ceiling_log2(max_slots)
+export BLOCK_TREE_DEPTH=5 # depth of the mini tree (block tree)
+export N_FIELD_ELEMS_PER_CELL=272 # number of field elements per cell
+export N_SAMPLES=5 # number of samples to prove
+```
+
+#### Step 2: Run the Script
+Once the params are set, you can run the script to generate the proof.
+You can also see the time taken to generate the proof. 
+
+```bash
+sudo bash ./prove.sh
+```
+
+### Build, Prove, and Verify
+To automate the whole process, you can run the following script 
+the script builds the circuit, loads the JSON circuit input, generates the proof, and verifies it. 
+It also shows the time taken for each step. 
+Make sure that you generate the circuit input prior to this so that you have the [`JSON input file`](./input.json) and set the [`circ_params.sh`](./circ_params.sh).
+
+```bash
+sudo bash ./prove_and_verify.sh
+```
+To inspect the source code, see [`prove_and_verify`](./src/bin/prove_and_verify.rs).

workflow/build_circuit.sh

@@ -7,4 +7,4 @@ source ./params.sh
 cargo build --release
 
 # Run the Rust executable
-cargo run prove_and_verify
+cargo run --bin build_circ

workflow/circ_params.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+export MAX_DEPTH=32        # maximum depth of the slot tree
+export MAX_LOG2_N_SLOTS=8 # Depth of the dataset tree = ceiling_log2(max_slots)
+export BLOCK_TREE_DEPTH=5 # depth of the mini tree (block tree)
+export N_FIELD_ELEMS_PER_CELL=272 # number of field elements per cell
+export N_SAMPLES=5 # number of samples to prove

workflow/gen_input.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Source the parameters from params.sh
+source ./params.sh
+
+# Build
+cargo build --release
+
+# Run the Rust executable
+cargo run --bin gen_input

workflow/prove.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Source the parameters from params.sh
+source ./circ_params.sh
+
+# Build
+cargo build --release
+
+# Run the Rust executable
+cargo run --bin prove

workflow/prove_and_verify.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Source the parameters from params.sh
+source ./circ_params.sh
+
+# Build
+cargo build --release
+
+# Run the Rust executable
+cargo run --bin prove_and_verify

workflow/src/bin/build_circ.rs

@@ -0,0 +1,28 @@
+use plonky2::plonk::circuit_data::CircuitConfig;
+use plonky2::plonk::config::GenericConfig;
+use plonky2::plonk::circuit_builder::CircuitBuilder;
+use anyhow::Result;
+use std::time::Instant;
+use codex_plonky2_circuits::circuits::sample_cells::SampleCircuit;
+use proof_input::params::Params;
+use proof_input::params::{D, C, F};
+
+fn main() -> Result<()> {
+    // Load the parameters from environment variables
+    let params = Params::from_env()?;
+
+    // Create the circuit
+    let config = CircuitConfig::standard_recursion_config();
+    let mut builder = CircuitBuilder::<F, D>::new(config);
+    let circuit_params = params.circuit_params;
+    let circ = SampleCircuit::new(circuit_params);
+    let mut targets = circ.sample_slot_circuit(&mut builder);
+
+    // Build the circuit
+    let build_time = Instant::now();
+    let data = builder.build::<C>();
+    println!("Build time: {:?}", build_time.elapsed());
+    println!("Circuit size (degree bits): {:?}", data.common.degree_bits());
+
+    Ok(())
+}

workflow/src/bin/gen_input.rs

@@ -0,0 +1,20 @@
+use plonky2::plonk::config::GenericConfig;
+use anyhow::Result;
+use proof_input::json::export_circ_input_to_json;
+use proof_input::gen_input::gen_testing_circuit_input;
+use proof_input::params::Params;
+use proof_input::params::{D, F};
+
+fn main() -> Result<()> {
+    // Load the parameters from environment variables
+    let params = Params::from_env()?;
+
+    // generate circuit input with given parameters
+    let circ_input = gen_testing_circuit_input::<F,D>(&params.test);
+
+    // export circuit parameters to json file
+    let filename= "input.json";
+    export_circ_input_to_json(circ_input, filename)?;
+
+    Ok(())
+}

workflow/src/bin/prove.rs

@@ -0,0 +1,45 @@
+use plonky2::plonk::circuit_data::CircuitConfig;
+use plonky2::plonk::config::GenericConfig;
+use plonky2::iop::witness::PartialWitness;
+use plonky2::plonk::circuit_builder::CircuitBuilder;
+use anyhow::Result;
+use std::time::Instant;
+
+use proof_input::json::import_circ_input_from_json;
+use codex_plonky2_circuits::circuits::sample_cells::{SampleCircuit, SampleCircuitInput};
+use codex_plonky2_circuits::circuits::params::CircuitParams;
+use proof_input::params::{D, C, F};
+
+fn main() -> Result<()> {
+    // Load the parameters from environment variables
+    let circuit_params = CircuitParams::from_env()?;
+
+    // Read the witness from input.json
+    let circ_input: SampleCircuitInput<F, D> = import_circ_input_from_json("input.json")?;
+    println!("Witness imported from input.json");
+
+    // Create the circuit
+    let config = CircuitConfig::standard_recursion_config();
+    let mut builder = CircuitBuilder::<F, D>::new(config);
+    let circ = SampleCircuit::new(circuit_params);
+    let mut targets = circ.sample_slot_circuit(&mut builder);
+
+    // Create a PartialWitness and assign
+    let mut pw = PartialWitness::new();
+    circ.sample_slot_assign_witness(&mut pw, &mut targets, circ_input);
+
+    // Build the circuit
+    let build_time = Instant::now();
+    let data = builder.build::<C>();
+    println!("Build time: {:?}", build_time.elapsed());
+    println!("Circuit size (degree bits): {:?}", data.common.degree_bits());
+
+    // Prove the circuit with the assigned witness
+    let start_time = Instant::now();
+    let proof_with_pis = data.prove(pw)?;
+    println!("Proving time: {:?}", start_time.elapsed());
+
+    //TODO: write proof to json file
+
+    Ok(())
+}

workflow/src/bin/prove_and_verify.rs

@@ -8,29 +8,27 @@ use std::time::Instant;
 use proof_input::json::import_circ_input_from_json;
 use codex_plonky2_circuits::circuits::sample_cells::{SampleCircuit, SampleCircuitInput};
 use codex_plonky2_circuits::circuits::params::CircuitParams;
-use proof_input::params::Params;
 use proof_input::params::{D, C, F};
 
 fn main() -> Result<()> {
     // Load the parameters from environment variables
-    let params = Params::from_env()?;
+    let circuit_params = CircuitParams::from_env()?;
 
     // Read the witness from input.json
-    let witness: SampleCircuitInput<F, D> = import_circ_input_from_json("input.json")?;
+    let circ_input: SampleCircuitInput<F, D> = import_circ_input_from_json("input.json")?;
     println!("Witness imported from input.json");
 
     // Create the circuit
     let config = CircuitConfig::standard_recursion_config();
     let mut builder = CircuitBuilder::<F, D>::new(config);
 
-    let circuit_params = params.circuit_params;
     let circ = SampleCircuit::new(circuit_params);
     let mut targets = circ.sample_slot_circuit(&mut builder);
 
     // Create a PartialWitness and assign
     let mut pw = PartialWitness::new();
 
-    circ.sample_slot_assign_witness(&mut pw, &mut targets, witness);
+    circ.sample_slot_assign_witness(&mut pw, &mut targets, circ_input);
 
     // Build the circuit
     let build_time = Instant::now();