diff --git a/benches/encoder.rs b/benches/encoder.rs index b90b2a4..278244b 100644 --- a/benches/encoder.rs +++ b/benches/encoder.rs @@ -1,5 +1,7 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use dynamic_data_experiments::{byte_data::{Params,Data}, encoder::{RSEncoder, BLSEncoder}}; +use dynamic_data_experiments::{byte_data::{Params,Data}, encoder::{G8Encoder, BLSEncoder}}; +use dynamic_data_experiments::encoder::BLSFieldEncoder; +use dynamic_data_experiments::field_matrix::Matrix; use dynamic_data_experiments::traits::{DataMatrix, Encoder}; fn bench_rs_encode(c: &mut Criterion) { @@ -13,11 +15,11 @@ fn bench_rs_encode(c: &mut Criterion) { // generate a random data matrix once let data = Data::new_random(params.clone()); - c.bench_function("RSEncoder::encode", |b| { + c.bench_function("G8Encoder::encode", |b| { b.iter(|| { // clone data for each iteration to avoid mutating the original let mut d = black_box(data.clone()); - RSEncoder::encode(&mut d).expect("encode failed"); + G8Encoder::encode(&mut d).expect("encode failed"); }); }); } @@ -41,5 +43,24 @@ fn bench_bls_encode(c: &mut Criterion) { }); } -criterion_group!(benches, bench_rs_encode, bench_bls_encode); +fn bench_bls_field_encode(c: &mut Criterion) { + // test parameters + let k = 100; + let p = 100; + let n = k + p; + let m = 200; + let params = Params { k, n, m }; + + // generate a random data matrix once + let data = Matrix::new_random(params.clone()); + + c.bench_function("BLSFieldEncoder::encode", |b| { + b.iter(|| { + let mut d = black_box(data.clone()); + BLSFieldEncoder::encode(&mut d).expect("encode failed"); + }); + }); +} + +criterion_group!(benches, bench_rs_encode, bench_bls_encode, bench_bls_field_encode); criterion_main!(benches); diff --git a/src/byte_data.rs b/src/byte_data.rs index 5bd6acd..e4143bf 100644 --- a/src/byte_data.rs +++ b/src/byte_data.rs @@ -11,6 +11,44 @@ pub struct Params{ pub n: usize, pub m: usize, } + +impl Params{ + pub fn check_bounds(&self, r: usize, c: usize) -> anyhow::Result<()>{ + assert!( + r < self.n, + "row index {} out of bounds; must be < {}", + r, + self.n + ); + assert!( + c < self.m, + "col index {} out of bounds; must be < {}", + c, + self.m + ); + Ok(()) + } + pub fn check_rows(&self, r: usize) -> anyhow::Result<()>{ + assert!( + r < self.n, + "row index {} out of bounds; must be < {}", + r, + self.n + ); + Ok(()) + } + + pub fn check_cols(&self, c: usize) -> anyhow::Result<()>{ + assert!( + c < self.m, + "col index {} out of bounds; must be < {}", + c, + self.m + ); + Ok(()) + } +} + /// data struct contains shards matrix with dimensions `n`*`m` /// the matrix contains n rows, k of which are source data and the rest p = (n-k) are parity #[derive(Clone, Debug)] @@ -20,21 +58,11 @@ pub struct Data{ } impl Data{ - pub fn get_row(&self, idx: usize) -> &Vec{ - &self.matrix[idx] - } pub fn get_row_mut(&mut self, idx: usize) -> &mut Vec{ &mut self.matrix[idx] } - pub fn get_col(&self, idx: usize) -> Vec<&T> { - self.matrix - .iter() - .map(|row| &row[idx]) - .collect() - } - pub fn get_col_mut(&mut self, idx: usize) -> Vec<&mut T> { self.matrix .iter_mut() @@ -67,9 +95,35 @@ impl DataMatrix for Data { } } + fn get(&self, r: usize, c: usize) -> anyhow::Result { + self.params.check_bounds(r,c)?; + Ok(self.matrix[r][c].clone()) + } + + fn get_row(&self, r: usize) -> anyhow::Result> { + self.params.check_rows(r)?; + + Ok(self.matrix[r].to_vec()) + } + + fn get_col(&self, c: usize) -> anyhow::Result> { + self.params.check_cols(c)?; + + Ok(self.matrix + .iter() + .map(|row| row[c].clone()) + .collect()) + } + + fn set(&mut self, r: usize, c: usize, elem: u8) -> anyhow::Result<()> { + self.params.check_bounds(r,c)?; + self.matrix[r][c] = elem; + Ok(()) + } + /// Update col `c` in shards. /// given `new_col` will replace the column `c` or `shards[0..k][c]` - fn update_col(&mut self, c: usize, new_col: &[u8]) { + fn update_col(&mut self, c: usize, new_col: &[u8]) -> anyhow::Result<()>{ // sanity checks assert!( new_col.len() == self.params.k, @@ -77,17 +131,13 @@ impl DataMatrix for Data { new_col.len(), self.params.k ); - assert!( - c < self.params.m, - "col index {} out of bounds; must be < {}", - c, - self.params.m - ); + self.params.check_cols(c)?; // write into each of the k data row at position c for i in 0..self.params.k { self.matrix[i][c] = new_col[i]; } + Ok(()) } /// Print all shards diff --git a/src/encoder.rs b/src/encoder.rs index 36dc838..b3b9a99 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -7,16 +7,16 @@ use ark_poly::{DenseUVPolynomial, GeneralEvaluationDomain, Polynomial}; use ark_poly::univariate::DensePolynomial; use reed_solomon_erasure::galois_8::ReedSolomon; use crate::byte_data::{Data, Params}; -use crate::traits::Encoder; +use crate::traits::{DataMatrix, Encoder}; use ark_poly::domain::EvaluationDomain; use crate::field_matrix::Matrix; // ------------- RS Encoder ------------ -pub struct RSEncoder{ +pub struct G8Encoder{ phantom_data: PhantomData } -impl RSEncoder{ +impl G8Encoder{ pub fn new() -> Self{ Self{ phantom_data: PhantomData::default() @@ -24,7 +24,7 @@ impl RSEncoder{ } } -impl Encoder for RSEncoder { +impl Encoder for G8Encoder { type Params = Params; type DataMatrix = Data; @@ -53,7 +53,7 @@ impl Encoder for RSEncoder { Ok(()) } - fn encode_col(data: &mut Data, c: usize) -> Result>{ + fn encode_col(data: &mut Data, c: usize) -> Result<()>{ // bounds check if c >= data.params.m { return Err(anyhow!("col index {} out of bounds (< {})", c, data.params.m)); @@ -66,7 +66,7 @@ impl Encoder for RSEncoder { // Build the column: data = existing byte, parity = zero let mut temp: Vec> = (0..n) .map(|i| { - let byte = data.matrix[i][c]; + let byte = data.get(i,c).unwrap(); if i < k { vec![byte] } else { @@ -80,17 +80,15 @@ impl Encoder for RSEncoder { let rse = ReedSolomon::new(k, p)?; rse.encode(&mut refs)?; - // Write back parity and collect full col - let mut full_col = Vec::with_capacity(n); + // Write back parity for i in 0..n { let b = refs[i][0]; if i >= k { - data.matrix[i][c] = b; + data.set(i,c, b)?; } - full_col.push(b); } - Ok(full_col) + Ok(()) } fn reconstruct(params: Params, matrix_opts: &mut Vec>>) -> Result<()>{ @@ -133,7 +131,7 @@ impl Encoder for BLSEncoder { Ok(()) } - fn encode_col(data: &mut Self::DataMatrix, c: usize) -> Result> { + fn encode_col(data: &mut Self::DataMatrix, c: usize) -> Result<()> { let n = data.params.n.clone(); let k = data.params.k.clone(); let mut col = data.get_col_mut(c); @@ -141,14 +139,13 @@ impl Encoder for BLSEncoder { let poly_poly = UniPoly381::from_coefficients_slice(&col_f); let domain: GeneralEvaluationDomain = EvaluationDomain::::new(n).unwrap(); - let mut new_col: Vec = vec![]; + // let mut new_col: Vec = vec![]; for i in k..n{ let eval = poly_poly.evaluate(&domain.element(i)); - new_col.push(eval.0.0[0] as u8); *col[i] = eval.0.0[0] as u8; } - Ok(new_col) + Ok(()) } fn reconstruct(_params: Params, _matrix_opts: &mut Vec>>) -> Result<()> { @@ -158,7 +155,11 @@ impl Encoder for BLSEncoder { // --------- BLS Encoder over FieldMatrix ---------------- -impl Encoder for BLSEncoder { +pub struct BLSFieldEncoder{ + phantom_data: PhantomData +} + +impl Encoder for BLSFieldEncoder{ type Params = Params; type DataMatrix = Matrix; @@ -169,23 +170,18 @@ impl Encoder for BLSEncoder { Ok(()) } - fn encode_col(data: &mut Matrix, c: usize) -> Result> { - // let n = data.params.n.clone(); - // let k = data.params.k.clone(); - // let mut col = data.get_col_mut(c); - // let col_f: Vec = col.iter().map(|i| ::from_le_bytes_mod_order(&i.clone().to_le_bytes())).collect(); - // let poly_poly = UniPoly381::from_coefficients_slice(&col_f); - // let domain: GeneralEvaluationDomain = EvaluationDomain::::new(n).unwrap(); - // - // let mut new_col: Vec = vec![]; - // for i in k..n{ - // let eval = poly_poly.evaluate(&domain.element(i)); - // new_col.push(eval.0.0[0] as u8); - // *col[i] = eval.0.0[0] as u8; - // } - // - // Ok(new_col) - todo!() + fn encode_col(data: &mut Matrix, c: usize) -> Result<()> { + let n = data.params.n.clone(); + let k = data.params.k.clone(); + let col: Vec = data.get_col(c)?; + let poly = UniPoly381::from_coefficients_slice(&col); + let domain: GeneralEvaluationDomain = EvaluationDomain::::new(n).unwrap(); + + for i in k..n{ + let eval = poly.evaluate(&domain.element(i)); + data.set(i,c,eval)?; + } + Ok(()) } fn reconstruct(_params: Params, _matrix_opts: &mut Vec>>) -> Result<()> { diff --git a/src/field_matrix.rs b/src/field_matrix.rs index 5cb0d45..bf5e7c2 100644 --- a/src/field_matrix.rs +++ b/src/field_matrix.rs @@ -1,7 +1,7 @@ use ark_ff::Field; -use ark_std::rand::Rng; -use ark_std::UniformRand; +use ark_std::{test_rng}; use crate::byte_data::{Data, Params}; +use crate::traits::DataMatrix; /// a Field matrix with `row` number of rows and `cols` number of columns @@ -12,34 +12,8 @@ pub struct Matrix { } impl Matrix { - /// Creates a new matrix from given field data. - pub fn new(params: Params, elms: Vec>) -> Self { - assert!(elms.len() == params.n, "number of rows must match"); - for row in &elms { - assert!(row.len() == params.m, "each row must have `m` elements"); - } - Matrix { params, elms } - } - /// Generates a random matrix with given dimensions, uses given rng for randomness. - pub fn new_random(params: Params, rng: &mut R) -> Self - where - F: UniformRand, - { - let rows = params.n; - let cols = params.m; - let mut data = Vec::with_capacity(rows); - for _ in 0..rows { - let mut row = Vec::with_capacity(cols); - for _ in 0..cols { - row.push(F::rand(rng)); - } - data.push(row); - } - Matrix { params, elms: data } - } - - /// Creates a new matrix from given data struct + /// Creates a new matrix from given u8 data struct pub fn from_data(data: &Data) -> Self{ let rows = data.params.n; let cols = data.params.m; @@ -55,23 +29,10 @@ impl Matrix { Matrix { params: data.params.clone(), elms:field_data } } - /// get the row at 0 Vec{ - assert!(idx < self.params.n, "Row index out of bounds"); - self.elms[idx].to_vec() - } - /// get mut the row at 0 &mut Vec{ - assert!(idx < self.params.n, "Row index out of bounds"); - &mut self.elms[idx] - } - - pub fn get_col(&self, idx: usize) -> Vec<&F> { - self.elms - .iter() - .map(|row| &row[idx]) - .collect() + pub fn get_row_mut(&mut self, idx: usize) -> anyhow::Result<&mut Vec>{ + self.params.check_rows(idx)?; + Ok(&mut self.elms[idx]) } pub fn get_col_mut(&mut self, idx: usize) -> Vec<&mut F> { @@ -80,9 +41,54 @@ impl Matrix { .map(|row| &mut row[idx]) .collect() } +} + +impl DataMatrix for Matrix { + type Params = Params; + + /// Generates a random matrix with given dimensions, uses given rng for randomness. + fn new_random(params: Params) -> Self + { + let mut rng = test_rng(); + let rows = params.n; + let cols = params.m; + let mut data = Vec::with_capacity(rows); + for _ in 0..rows { + let mut row = Vec::with_capacity(cols); + for _ in 0..cols { + row.push(F::rand(&mut rng)); + } + data.push(row); + } + Matrix { params, elms: data } + } + + fn get(&self, r: usize, c: usize) -> anyhow::Result { + self.params.check_bounds(r,c)?; + Ok(self.elms[r][c].clone()) + } + + fn set(&mut self, r: usize, c: usize, elem: F) -> anyhow::Result<()>{ + self.params.check_bounds(r,c)?; + Ok(self.elms[r][c] = elem) + } + + /// get the row at 0 anyhow::Result>{ + self.params.check_rows(idx)?; + Ok(self.elms[idx].to_vec()) + } + + fn get_col(&self, idx: usize) -> anyhow::Result> { + self.params.check_cols(idx)?; + Ok(self.elms + .iter() + .map(|row| row[idx].clone()) + .collect()) + } /// Print matrix - pub fn pretty_print(&self) { + fn pretty_print(&self) { for (i, shard) in self.elms.iter().enumerate() { print!("row {:>2}: ", i); for &b in shard { @@ -91,6 +97,24 @@ impl Matrix { println!(); } } + + fn update_col(&mut self, c: usize, new_col: &[F]) -> anyhow::Result<()> { + self.params.check_cols(c)?; + + // ensure the provided column has exactly `k` entries + assert!( + new_col.len() == self.params.k, + "new_col length ({}) must equal k ({})", + new_col.len(), + self.params.k + ); + + for (r, val) in new_col.iter().enumerate() { + self.elms[r][c] = val.clone(); + } + + Ok(()) + } } diff --git a/src/kzg.rs b/src/kzg.rs index 3d43fa1..9b94c1d 100644 --- a/src/kzg.rs +++ b/src/kzg.rs @@ -18,7 +18,7 @@ use ark_poly_commit::marlin_pc::MarlinKZG10; use ark_poly_commit::sonic_pc::UniversalParams; use crate::byte_data::Params; use crate::field_matrix::Matrix; -use crate::traits::PolynomialCommitmentScheme; +use crate::traits::{PolynomialCommitmentScheme, DataMatrix}; use ark_crypto_primitives::sponge::poseidon::{PoseidonSponge, PoseidonConfig}; use ark_poly_commit::kzg10::Proof; @@ -67,7 +67,8 @@ impl PolynomialCommitmentScheme for KZGPolyComm { let mut row_polynomials = vec![]; let timer = start_timer!(|| format!("Poly evaluations and interpolation for {} rows", degree)); for i in 0..matrix.params.n{ - let poly_evals = Evaluations::from_vec_and_domain(matrix.row(i), srs.ploycommit_domain.clone()); + let row = matrix.get_row(i)?.to_vec(); + let poly_evals = Evaluations::from_vec_and_domain(row, srs.ploycommit_domain.clone()); let row_poly = poly_evals.interpolate(); let label = String::from(format!("row_poly_{}", i)); let labeled_poly = LabeledPolynomial::new( diff --git a/src/test.rs b/src/test.rs index d130d30..1b8989e 100644 --- a/src/test.rs +++ b/src/test.rs @@ -5,7 +5,7 @@ mod tests { use crate::kzg::{F, KZGPolyComm}; use crate::field_matrix::Matrix; use ark_poly_commit::{Polynomial}; - use crate::encoder::{BLSEncoder, RSEncoder}; + use crate::encoder::{BLSEncoder, BLSFieldEncoder, G8Encoder}; use crate::traits::{DataMatrix, Encoder, PolynomialCommitmentScheme}; #[test] @@ -31,7 +31,7 @@ mod tests { let original: Vec> = data.matrix[..k].to_vec(); // encode - RSEncoder::encode(&mut data).expect("encode failed"); + G8Encoder::encode(&mut data).expect("encode failed"); println!("data after encoding:"); data.pretty_print(); @@ -44,7 +44,7 @@ mod tests { matrix_opts[k] = None; // reconstruct missing rows - RSEncoder::reconstruct(data.params.clone(), &mut matrix_opts).expect("reconstruction should succeed"); + G8Encoder::reconstruct(data.params.clone(), &mut matrix_opts).expect("reconstruction should succeed"); // verify reconstruction for data shards for i in 0..k { @@ -54,7 +54,7 @@ mod tests { } #[test] - fn test_bls_encodeer() { + fn test_bls_encoder() { // test parameters let k = 4; let p = 4; @@ -98,6 +98,54 @@ mod tests { // } } + #[test] + fn test_bls_field_encoder() { + // test parameters + let k = 4; + let p = 4; + let n = k + p; + let m = 8; + + // generate Data with random content + let params = Params { + k, + n, + m, + }; + let data = Data::new_random(params); + println!("data #row ={}", data.matrix.len()); + println!("data #col ={}", data.matrix[0].len()); + println!("data before encoding:"); + data.pretty_print(); + // original data matrix + let mut original = Matrix::from_data(&data); + let original_copy = Matrix::from_data(&data); + println!("data as Field elements:"); + original.pretty_print(); + + // encode + BLSFieldEncoder::encode(&mut original).expect("encode failed"); + println!("data after encoding:"); + original.pretty_print(); + + // verify data matrix unchanged + assert_eq!(original.elms[..k], original_copy.elms[..k]); + + // simulate loss of one data and one parity rows + let mut matrix_opts: Vec<_> = data.matrix.iter().cloned().map(Some).collect(); + matrix_opts[1] = None; + matrix_opts[k] = None; + + // TODO: reconstruct missing rows + // BLSEncoder::reconstruct(data.params.clone(), &mut matrix_opts).expect("reconstruction should succeed"); + + // verify reconstruction for data shards + // for i in 0..k { + // let recovered = matrix_opts[i].clone().unwrap(); + // assert_eq!(recovered, &original[i][..]); + // } + } + #[test] fn test_commit_rows() { // dimensions: 8 rows (4 parity), 8 columns @@ -112,7 +160,7 @@ mod tests { m, }; let mut data = Data::new_random(params.clone()); - RSEncoder::encode(&mut data).expect("encode failed"); + G8Encoder::encode(&mut data).expect("encode failed"); // make a random n×m matrix let matrix = Matrix::from_data(&data); @@ -136,7 +184,7 @@ mod tests { // check that each polynomial really interpolates its original rows for (i, poly) in row_polys.iter().enumerate() { - let row = matrix.row(i); + let row = matrix.get_row(i).unwrap(); // evaluate poly at each domain point and collect let evals: Vec<_> = srs .ploycommit_domain @@ -161,7 +209,7 @@ mod tests { m, }; let mut data = Data::new_random(params.clone()); - RSEncoder::encode(&mut data).expect("encode failed"); + G8Encoder::encode(&mut data).expect("encode failed"); // make a random n×m matrix let matrix = Matrix::from_data(&data); @@ -178,7 +226,7 @@ mod tests { for col in 0..m { let proof = KZGPolyComm::open(&kzg_comm, &srs, row, col) .expect("open should succeed"); - let expected: F = matrix.row(row)[col].clone(); + let expected: F = matrix.elms[row][col].clone(); assert!( KZGPolyComm::verify(&kzg_comm, &srs, row, col, expected, &proof) @@ -206,7 +254,7 @@ mod tests { }; // snapshot of original let mut data = Data::new_random(params); - RSEncoder::encode(&mut data).expect("encode failed"); + G8Encoder::encode(&mut data).expect("encode failed"); println!("original data:"); data.pretty_print(); @@ -230,7 +278,7 @@ mod tests { ); } - let _coded_row = RSEncoder::encode_col(&mut data, c).unwrap(); + let _coded_row = G8Encoder::encode_col(&mut data, c).unwrap(); println!("data after encoding update:"); data.pretty_print(); } @@ -250,7 +298,7 @@ mod tests { }; // snapshot of original let mut data = Data::new_random(params.clone()); - RSEncoder::encode(&mut data).expect("encode failed"); + G8Encoder::encode(&mut data).expect("encode failed"); // Build a matrix where entry (i,j) = i * m + j let mut matrix = Matrix::::from_data(&data); @@ -266,7 +314,7 @@ mod tests { // a row to update let row_idx = 1; - let old_row = matrix.row(row_idx); + let old_row = matrix.get_row(row_idx)?; // a new row by adding a constant to each element let new_row: Vec<_> = old_row.iter() @@ -275,7 +323,7 @@ mod tests { // Apply the change to the in-memory matrix { - let row_slice = matrix.row_mut(row_idx); + let row_slice = matrix.get_row_mut(row_idx)?; for (j, val) in new_row.iter().enumerate() { row_slice[j] = *val; } @@ -291,7 +339,7 @@ mod tests { .elements() .map(|x| poly.polynomial().evaluate(&x)) .collect(); - assert_eq!(evals, matrix.row(i)); + assert_eq!(evals, matrix.get_row(i)?); } // === new fresh commit on updated matrix === diff --git a/src/traits.rs b/src/traits.rs index 9e8e8cc..3c3bea2 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -4,7 +4,11 @@ use crate::byte_data::Params; pub trait DataMatrix{ type Params; fn new_random(params: Self::Params) -> Self; - fn update_col(&mut self, c: usize, new_col: &[T]); + fn get(&self, r: usize, c: usize) -> Result; + fn get_row(&self, r: usize) -> Result>; + fn get_col(&self, c: usize) -> Result>; + fn set(&mut self, r: usize, c: usize, elem: T) -> Result<()>; + fn update_col(&mut self, c: usize, new_col: &[T]) -> Result<()>; fn pretty_print(&self); } @@ -17,7 +21,7 @@ pub trait Encoder{ /// encode in place the input data matrix fn encode(data: &mut Self::DataMatrix) -> Result<()>; /// encode a single column in place - fn encode_col(data: &mut Self::DataMatrix, c: usize) -> Result>; + fn encode_col(data: &mut Self::DataMatrix, c: usize) -> Result<()>; /// reconstruct in place fn reconstruct(params: Params, matrix_opts: &mut Vec>>) -> Result<()>; }