add rudimentary hash counting statistics

This commit is contained in:
Balazs Komuves 2025-01-30 14:20:56 +01:00
parent 23dff50624
commit f7b4e62f2e
No known key found for this signature in database
GPG Key ID: F63B7AEF18435562
14 changed files with 224 additions and 13 deletions

View File

@ -33,6 +33,8 @@ serde_json = { workspace = true }
static_assertions = { workspace = true }
unroll = { workspace = true }
web-time = { version = "1.0.0", optional = true }
strum = "0.26"
strum_macros = "0.26"
# Local dependencies
plonky2_field = { version = "1.0.0", path = "../field", default-features = false }

View File

@ -28,7 +28,8 @@ use plonky2::plonk::circuit_builder::CircuitBuilder;
use plonky2::plonk::circuit_data::{CircuitConfig, CommonCircuitData, VerifierOnlyCircuitData};
use plonky2::plonk::config::{AlgebraicHasher, GenericConfig, PoseidonGoldilocksConfig};
use plonky2::plonk::proof::{CompressedProofWithPublicInputs, ProofWithPublicInputs};
use plonky2::plonk::prover::{prove,prove_with_options,ProverOptions};
use plonky2::plonk::prover::{prove, prove_with_options, ProverOptions};
use plonky2::plonk::verifier::{VerifierOptions, HashStatisticsPrintLevel};
use plonky2::util::serialization::DefaultGateSerializer;
use plonky2::util::timing::TimingTree;
use plonky2_field::extension::Extendable;
@ -245,6 +246,7 @@ where
let prover_opts = ProverOptions {
export_witness: Some(format!("{}_witness.json",name)),
print_hash_statistics: HashStatisticsPrintLevel::Summary, // ::None,
};
let mut timing = TimingTree::new("prove", Level::Debug);
@ -261,7 +263,10 @@ where
fs::write(format!("recursion_{}_proof.json" , name), proof_serialized ).expect("Unable to write file");
*/
data.verify(proof.clone())?;
let verifier_opts = VerifierOptions {
print_hash_statistics: HashStatisticsPrintLevel::Summary,
};
data.verify_with_options(proof.clone(), &verifier_opts)?;
Ok((proof, data.verifier_only, data.common))
}

View File

@ -5,6 +5,7 @@ use plonky2::plonk::circuit_builder::CircuitBuilder;
use plonky2::plonk::circuit_data::CircuitConfig;
use plonky2::plonk::config::{GenericConfig, PoseidonGoldilocksConfig};
use plonky2::plonk::prover::ProverOptions;
use plonky2::plonk::verifier::{VerifierOptions, HashStatisticsPrintLevel};
/// An example of using Plonky2 to prove a statement of the form
/// "I know the 100th element of the Fibonacci sequence, starting with constants a and b."
@ -42,6 +43,7 @@ fn main() -> Result<()> {
let prover_opts = ProverOptions {
export_witness: Some(String::from("fibonacci_witness.json")),
print_hash_statistics: HashStatisticsPrintLevel::Info,
};
let proof = data.prove_with_options(pw, &prover_opts)?;
@ -51,5 +53,9 @@ fn main() -> Result<()> {
proof.public_inputs[0], proof.public_inputs[1], proof.public_inputs[2]
);
data.verify(proof)
let verifier_opts = VerifierOptions {
print_hash_statistics: HashStatisticsPrintLevel::Summary,
};
data.verify_with_options(proof, &verifier_opts)
}

View File

@ -10,6 +10,7 @@ use plonky2::plonk::circuit_builder::CircuitBuilder;
use plonky2::plonk::circuit_data::{CircuitConfig,CircuitData};
use plonky2::plonk::config::{GenericConfig, PoseidonGoldilocksConfig};
use plonky2::plonk::prover::ProverOptions;
use plonky2::plonk::verifier::{VerifierOptions, HashStatisticsPrintLevel};
//use plonky2::gadgets::lookup;
@ -142,6 +143,7 @@ fn main() -> Result<()> {
let prover_opts = ProverOptions {
export_witness: Some(String::from("lookup_witness.json")),
print_hash_statistics: HashStatisticsPrintLevel::None,
};
let proof = data.prove_with_options(pw, &prover_opts)?;

View File

@ -13,6 +13,7 @@ use plonky2::plonk::circuit_builder::CircuitBuilder;
use plonky2::plonk::circuit_data::{CircuitConfig,CircuitData};
use plonky2::plonk::config::{GenericConfig, PoseidonGoldilocksConfig};
use plonky2::plonk::prover::ProverOptions;
use plonky2::plonk::verifier::{VerifierOptions, HashStatisticsPrintLevel};
//use plonky2::gadgets::lookup;
@ -87,6 +88,7 @@ fn main() -> Result<()> {
let prover_opts = ProverOptions {
export_witness: Some(String::from("multi_lookup_witness.json")),
print_hash_statistics: HashStatisticsPrintLevel::None,
};
let proof = data.prove_with_options(pw, &prover_opts)?;

View File

@ -17,6 +17,7 @@ use crate::hash::hash_types::RichField;
use crate::hash::merkle_tree::MerkleTree;
use crate::iop::challenger::Challenger;
use crate::plonk::config::GenericConfig;
use crate::plonk::prover::ProverOptions;
use crate::timed;
use crate::util::reducing::ReducingFactor;
use crate::util::timing::TimingTree;
@ -181,6 +182,7 @@ impl<F: RichField + Extendable<D>, C: GenericConfig<D, F = F>, const D: usize>
final_poly_coeff_len: Option<usize>,
max_num_query_steps: Option<usize>,
timing: &mut TimingTree,
prover_options: &ProverOptions,
) -> FriProof<F, C::Hasher, D> {
assert!(D > 1, "Not implemented for D=1.");
let alpha = challenger.get_extension_challenge::<D>();
@ -231,6 +233,7 @@ impl<F: RichField + Extendable<D>, C: GenericConfig<D, F = F>, const D: usize>
final_poly_coeff_len,
max_num_query_steps,
timing,
prover_options,
);
fri_proof

View File

@ -11,11 +11,13 @@ use crate::field::polynomial::{PolynomialCoeffs, PolynomialValues};
use crate::fri::proof::{FriInitialTreeProof, FriProof, FriQueryRound, FriQueryStep};
use crate::fri::{FriConfig, FriParams};
use crate::hash::hash_types::{RichField, NUM_HASH_OUT_ELTS};
use crate::hash::hashing::PlonkyPermutation;
use crate::hash::hashing::*;
use crate::hash::merkle_tree::MerkleTree;
use crate::iop::challenger::Challenger;
use crate::plonk::config::GenericConfig;
use crate::plonk::plonk_common::reduce_with_powers;
use crate::plonk::prover::ProverOptions;
use crate::plonk::verifier::HashStatisticsPrintLevel;
use crate::timed;
use crate::util::reverse_index_bits_in_place;
use crate::util::timing::TimingTree;
@ -32,6 +34,7 @@ pub fn fri_proof<F: RichField + Extendable<D>, C: GenericConfig<D, F = F>, const
final_poly_coeff_len: Option<usize>,
max_num_query_steps: Option<usize>,
timing: &mut TimingTree,
prover_options: &ProverOptions,
) -> FriProof<F, C::Hasher, D> {
let n = lde_polynomial_values.len();
assert_eq!(lde_polynomial_coeffs.len(), n);
@ -50,6 +53,10 @@ pub fn fri_proof<F: RichField + Extendable<D>, C: GenericConfig<D, F = F>, const
)
);
if prover_options.print_hash_statistics >= HashStatisticsPrintLevel::Debug {
print_hash_counters("after commit phase");
}
// PoW phase
let pow_witness = timed!(
timing,
@ -57,6 +64,10 @@ pub fn fri_proof<F: RichField + Extendable<D>, C: GenericConfig<D, F = F>, const
fri_proof_of_work::<F, C, D>(challenger, &fri_params.config)
);
if prover_options.print_hash_statistics >= HashStatisticsPrintLevel::Debug {
print_hash_counters("after PoW grinding");
}
// Query phase
let query_round_proofs =
fri_prover_query_rounds::<F, C, D>(initial_merkle_trees, &trees, challenger, n, fri_params);
@ -180,6 +191,8 @@ pub(crate) fn fri_proof_of_work<
let witness_input_pos = challenger.input_buffer.len();
duplex_intermediate_state.set_from_iter(challenger.input_buffer.clone(), 0);
// println!("duplex_intermediate_state = {:?}", duplex_intermediate_state);
let pow_witness = (0..=F::NEG_ONE.to_canonical_u64())
.into_par_iter()
.find_any(|&candidate| {
@ -193,6 +206,8 @@ pub(crate) fn fri_proof_of_work<
.map(F::from_canonical_u64)
.expect("Proof of work failed. This is highly unlikely!");
// println!("pow_witness = {:?}",pow_witness);
// Recompute pow_response using our normal Challenger code, and make sure it matches.
challenger.observe_element(pow_witness);
let pow_response = challenger.get_challenge();

View File

@ -13,9 +13,11 @@ use crate::fri::{FriConfig, FriParams};
use crate::hash::hash_types::RichField;
use crate::hash::merkle_proofs::verify_merkle_proof_to_cap;
use crate::hash::merkle_tree::MerkleCap;
use crate::hash::hashing::*;
use crate::plonk::config::{GenericConfig, Hasher};
use crate::util::reducing::ReducingFactor;
use crate::util::{log2_strict, reverse_bits, reverse_index_bits_in_place};
use crate::plonk::verifier::{VerifierOptions, HashStatisticsPrintLevel};
/// Computes P'(x^arity) from {P(x*g^i)}_(i=0..arity), where g is a `arity`-th root of unity
/// and P' is the FRI reduced polynomial.
@ -70,6 +72,7 @@ pub fn verify_fri_proof<
initial_merkle_caps: &[MerkleCap<F, C::Hasher>],
proof: &FriProof<F, C::Hasher, D>,
params: &FriParams,
verify_options: &VerifierOptions,
) -> Result<()> {
validate_fri_proof_shape::<F, C, D>(proof, instance, params)?;
@ -87,10 +90,11 @@ pub fn verify_fri_proof<
let precomputed_reduced_evals =
PrecomputedReducedOpenings::from_os_and_alpha(openings, challenges.fri_alpha);
for (&x_index, round_proof) in challenges
for (round_counter,(&x_index, round_proof)) in challenges
.fri_query_indices
.iter()
.zip(&proof.query_round_proofs)
.enumerate()
{
fri_verifier_query_round::<F, C, D>(
instance,
@ -102,7 +106,14 @@ pub fn verify_fri_proof<
n,
round_proof,
params,
verify_options,
)?;
if verify_options.print_hash_statistics >= HashStatisticsPrintLevel::Debug {
let s = format!("after query round #{}",round_counter);
print_hash_counters(&s);
}
}
Ok(())
@ -175,6 +186,7 @@ fn fri_verifier_query_round<
n: usize,
round_proof: &FriQueryRound<F, C::Hasher, D>,
params: &FriParams,
verify_options: &VerifierOptions,
) -> Result<()> {
fri_verify_initial_proof::<F, C::Hasher>(
x_index,
@ -197,6 +209,11 @@ fn fri_verifier_query_round<
params,
);
if verify_options.print_hash_statistics >= HashStatisticsPrintLevel::Debug {
let s = format!("after combine_initial");
print_hash_counters(&s);
}
for (i, &arity_bits) in params.reduction_arity_bits.iter().enumerate() {
let arity = 1 << arity_bits;
let evals = &round_proof.steps[i].evals;
@ -224,6 +241,11 @@ fn fri_verifier_query_round<
&round_proof.steps[i].merkle_proof,
)?;
if verify_options.print_hash_statistics >= HashStatisticsPrintLevel::Debug {
let s = format!("after folding step #{}",i);
print_hash_counters(&s);
}
// Update the point x to x^arity.
subgroup_x = subgroup_x.exp_power_of_2(arity_bits);

View File

@ -10,6 +10,71 @@ use crate::iop::target::Target;
use crate::plonk::circuit_builder::CircuitBuilder;
use crate::plonk::config::AlgebraicHasher;
use strum_macros::FromRepr;
use std::sync::atomic::{AtomicUsize,Ordering};
//------------------------------------------------------------------------------
// hash usage statistics
#[derive(Copy, Clone, Debug, FromRepr)]
pub enum HashUsage {
Permutation,
Compress,
// Sponge,
// Duplex,
}
impl HashUsage {
pub fn from_usize(v: usize) -> HashUsage {
HashUsage::from_repr(v.try_into().unwrap()).unwrap()
}
}
const ATOMIC_ORDER: Ordering = Ordering::SeqCst;
#[derive(Debug)]
pub struct HashCounter {
current_usage: AtomicUsize,
count_permute: AtomicUsize,
count_compress: AtomicUsize,
}
static the_hash_counters: HashCounter = HashCounter {
current_usage: AtomicUsize::new(HashUsage::Permutation as usize),
count_permute: AtomicUsize::new(0),
count_compress: AtomicUsize::new(0),
};
pub fn reset_hash_counters() {
the_hash_counters.current_usage .store(HashUsage::Permutation as usize, ATOMIC_ORDER);
the_hash_counters.count_permute .store(0, ATOMIC_ORDER );
the_hash_counters.count_compress.store(0, ATOMIC_ORDER );
}
pub fn set_current_hash_usage(what: HashUsage) {
the_hash_counters.current_usage.store(what as usize, ATOMIC_ORDER);
}
pub fn increment_given_hash_counter(which: HashUsage) {
let _ = match which {
HashUsage::Permutation => the_hash_counters.count_permute .fetch_add(1, ATOMIC_ORDER),
HashUsage::Compress => the_hash_counters.count_compress.fetch_add(1, ATOMIC_ORDER),
};
}
pub fn increment_current_hash_counter() {
let cur = the_hash_counters.current_usage.load(ATOMIC_ORDER);
increment_given_hash_counter(HashUsage::from_usize(cur));
}
pub fn print_hash_counters(msg: &str) {
let nperms = the_hash_counters.count_permute.load(ATOMIC_ORDER);
println!("hash statistic ({})",msg);
println!(" - number of permutations = {}",nperms);
}
//------------------------------------------------------------------------------
impl<F: RichField + Extendable<D>, const D: usize> CircuitBuilder<F, D> {
pub fn hash_or_noop<H: AlgebraicHasher<F>>(&mut self, inputs: Vec<Target>) -> HashOutTarget {
let zero = self.zero();

View File

@ -15,6 +15,7 @@ use crate::gates::poseidon::PoseidonGate;
use crate::gates::poseidon_mds::PoseidonMdsGate;
use crate::hash::hash_types::{HashOut, RichField};
use crate::hash::hashing::{compress, hash_n_to_hash_no_pad, PlonkyPermutation};
use crate::hash::hashing::{HashUsage, increment_given_hash_counter};
use crate::iop::ext_target::ExtensionTarget;
use crate::iop::target::{BoolTarget, Target};
use crate::plonk::circuit_builder::CircuitBuilder;
@ -862,6 +863,7 @@ impl<T: Copy + Debug + Default + Eq + Permuter + Send + Sync> PlonkyPermutation<
fn permute(&mut self) {
self.state = T::permute(self.state);
increment_given_hash_counter(HashUsage::Permutation);
}
fn squeeze(&self) -> &[T] {

View File

@ -46,8 +46,8 @@ use crate::plonk::circuit_builder::CircuitBuilder;
use crate::plonk::config::{GenericConfig, Hasher};
use crate::plonk::plonk_common::PlonkOracle;
use crate::plonk::proof::{CompressedProofWithPublicInputs, ProofWithPublicInputs};
use crate::plonk::prover::{prove,prove_with_options,ProverOptions};
use crate::plonk::verifier::verify;
use crate::plonk::prover::{prove, prove_with_options, ProverOptions};
use crate::plonk::verifier::{verify, verify_with_options, VerifierOptions};
use crate::util::serialization::{
Buffer, GateSerializer, IoResult, Read, WitnessGeneratorSerializer, Write,
};
@ -213,6 +213,10 @@ impl<F: RichField + Extendable<D>, C: GenericConfig<D, F = F>, const D: usize>
verify::<F, C, D>(proof_with_pis, &self.verifier_only, &self.common)
}
pub fn verify_with_options(&self, proof_with_pis: ProofWithPublicInputs<F, C, D>, verifier_options: &VerifierOptions) -> Result<()> {
verify_with_options::<F, C, D>(proof_with_pis, &self.verifier_only, &self.common, verifier_options)
}
pub fn verify_compressed(
&self,
compressed_proof_with_pis: CompressedProofWithPublicInputs<F, C, D>,

View File

@ -26,7 +26,7 @@ use crate::iop::ext_target::ExtensionTarget;
use crate::iop::target::Target;
use crate::plonk::circuit_data::{CommonCircuitData, VerifierOnlyCircuitData};
use crate::plonk::config::{GenericConfig, Hasher};
use crate::plonk::verifier::verify_with_challenges;
use crate::plonk::verifier::{verify_with_challenges, DEFAULT_VERIFIER_OPTIONS};
use crate::util::serialization::{Buffer, Read, Write};
#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)]
@ -227,6 +227,7 @@ impl<F: RichField + Extendable<D>, C: GenericConfig<D, F = F>, const D: usize>
challenges,
verifier_data,
common_data,
&DEFAULT_VERIFIER_OPTIONS,
)
}

View File

@ -23,6 +23,7 @@ use crate::gates::lookup::LookupGate;
use crate::gates::lookup_table::LookupTableGate;
use crate::gates::selectors::{LookupSelectors};
use crate::hash::hash_types::RichField;
use crate::hash::hashing::*;
use crate::iop::challenger::Challenger;
use crate::iop::generator::generate_partial_witness;
use crate::iop::target::Target;
@ -33,6 +34,7 @@ use crate::plonk::config::{GenericConfig, Hasher};
use crate::plonk::plonk_common::PlonkOracle;
use crate::plonk::proof::{OpeningSet, Proof, ProofWithPublicInputs};
use crate::plonk::vanishing_poly::{eval_vanishing_poly_base_batch, get_lut_poly};
use crate::plonk::verifier::HashStatisticsPrintLevel;
use crate::plonk::vars::EvaluationVarsBaseBatch;
use crate::timed;
use crate::util::partial_products::{partial_products_and_z_gx, quotient_chunk_products};
@ -121,10 +123,12 @@ pub fn set_lookup_wires<
#[derive(Debug,Clone)]
pub struct ProverOptions {
pub export_witness: Option<String>, // export the full witness into the given file
pub print_hash_statistics: HashStatisticsPrintLevel,
}
pub const DEFAULT_PROVER_OPTIONS: ProverOptions = ProverOptions {
export_witness: None,
export_witness: None,
print_hash_statistics: HashStatisticsPrintLevel::None,
};
// things we want to export to be used by third party tooling
@ -223,14 +227,23 @@ pub fn prove_with_options<F: RichField + Extendable<D>, C: GenericConfig<D, F =
where
C::Hasher: Hasher<F>,
C::InnerHasher: Hasher<F>,
{
reset_hash_counters();
let partition_witness = timed!(
timing,
&format!("run {} generators", prover_data.generators.len()),
generate_partial_witness(inputs, prover_data, common_data)?
);
prove_with_partition_witness(prover_data, common_data, partition_witness, timing, prover_options)
let result = prove_with_partition_witness(prover_data, common_data, partition_witness, timing, prover_options);
if prover_options.print_hash_statistics >= HashStatisticsPrintLevel::Summary {
print_hash_counters("prover total");
}
result
}
pub fn prove_with_partition_witness<
@ -288,6 +301,10 @@ where
)
);
if prover_options.print_hash_statistics >= HashStatisticsPrintLevel::Info {
print_hash_counters("after wires commitment");
}
// export witness etc for third party tooling
match &prover_options.export_witness {
None => (),
@ -366,6 +383,10 @@ where
)
);
if prover_options.print_hash_statistics >= HashStatisticsPrintLevel::Info {
print_hash_counters("after partial product and lookup commitment");
}
challenger.observe_cap::<C::Hasher>(&partial_products_zs_and_lookup_commitment.merkle_tree.cap);
let alphas = challenger.get_n_challenges(num_challenges);
@ -414,6 +435,10 @@ where
)
);
if prover_options.print_hash_statistics >= HashStatisticsPrintLevel::Info {
print_hash_counters("after quotient poly commitment");
}
challenger.observe_cap::<C::Hasher>(&quotient_polys_commitment.merkle_tree.cap);
let zeta = challenger.get_extension_challenge::<D>();
@ -436,12 +461,16 @@ where
&wires_commitment,
&partial_products_zs_and_lookup_commitment,
&quotient_polys_commitment,
common_data
common_data,
)
);
challenger.observe_openings(&openings.to_fri_openings());
let instance = common_data.get_fri_instance(zeta);
if prover_options.print_hash_statistics >= HashStatisticsPrintLevel::Info {
print_hash_counters("before FRI opening proof");
}
let opening_proof = timed!(
timing,
"compute opening proofs",
@ -458,6 +487,7 @@ where
None,
None,
timing,
prover_options,
)
);

View File

@ -6,6 +6,7 @@ use crate::field::extension::Extendable;
use crate::field::types::Field;
use crate::fri::verifier::verify_fri_proof;
use crate::hash::hash_types::RichField;
use crate::hash::hashing::*;
use crate::plonk::circuit_data::{CommonCircuitData, VerifierOnlyCircuitData};
use crate::plonk::config::{GenericConfig, Hasher};
use crate::plonk::plonk_common::reduce_with_powers;
@ -14,27 +15,76 @@ use crate::plonk::validate_shape::validate_proof_with_pis_shape;
use crate::plonk::vanishing_poly::eval_vanishing_poly;
use crate::plonk::vars::EvaluationVars;
// debugging features in the verifier
#[derive(Debug,Clone,PartialEq,PartialOrd)]
pub enum HashStatisticsPrintLevel {
None,
Summary,
Info,
Debug,
}
#[derive(Debug,Clone)]
pub struct VerifierOptions {
pub print_hash_statistics: HashStatisticsPrintLevel,
}
pub const DEFAULT_VERIFIER_OPTIONS: VerifierOptions = VerifierOptions {
print_hash_statistics: HashStatisticsPrintLevel::None,
};
pub(crate) fn verify<F: RichField + Extendable<D>, C: GenericConfig<D, F = F>, const D: usize>(
proof_with_pis: ProofWithPublicInputs<F, C, D>,
verifier_data: &VerifierOnlyCircuitData<C, D>,
common_data: &CommonCircuitData<F, D>,
) -> Result<()> {
verify_with_options(
proof_with_pis,
verifier_data,
common_data,
&DEFAULT_VERIFIER_OPTIONS,
)
}
pub(crate) fn verify_with_options<F: RichField + Extendable<D>, C: GenericConfig<D, F = F>, const D: usize>(
proof_with_pis: ProofWithPublicInputs<F, C, D>,
verifier_data: &VerifierOnlyCircuitData<C, D>,
common_data: &CommonCircuitData<F, D>,
verifier_options: &VerifierOptions,
) -> Result<()> {
reset_hash_counters();
validate_proof_with_pis_shape(&proof_with_pis, common_data)?;
let public_inputs_hash = proof_with_pis.get_public_inputs_hash();
if verifier_options.print_hash_statistics >= HashStatisticsPrintLevel::Info {
print_hash_counters("after PI");
}
let challenges = proof_with_pis.get_challenges(
public_inputs_hash,
&verifier_data.circuit_digest,
common_data,
)?;
verify_with_challenges::<F, C, D>(
if verifier_options.print_hash_statistics >= HashStatisticsPrintLevel::Info {
print_hash_counters("after challenges");
}
let result = verify_with_challenges::<F, C, D>(
proof_with_pis.proof,
public_inputs_hash,
challenges,
verifier_data,
common_data,
)
verifier_options
);
if verifier_options.print_hash_statistics >= HashStatisticsPrintLevel::Summary {
print_hash_counters("verify total");
}
result
}
pub(crate) fn verify_with_challenges<
@ -47,6 +97,7 @@ pub(crate) fn verify_with_challenges<
challenges: ProofChallenges<F, D>,
verifier_data: &VerifierOnlyCircuitData<C, D>,
common_data: &CommonCircuitData<F, D>,
verifier_options: &VerifierOptions,
) -> Result<()> {
let local_constants = &proof.openings.constants;
let local_wires = &proof.openings.wires;
@ -112,6 +163,7 @@ pub(crate) fn verify_with_challenges<
merkle_caps,
&proof.opening_proof,
&common_data.fri_params,
&verifier_options,
)?;
Ok(())