logos-storage/dynamic-data-experiments
1
0
Fork 0
mirror of https://github.com/logos-storage/dynamic-data-experiments.git synced 2026-01-02 05:03:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

impl BLS encoder

Browse Source
This commit is contained in:
M Alghazwi 2025-06-26 18:10:44 +02:00
parent 5e75e1e654
commit ea58b0ed15
7 changed files with 262 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
benches/encoder.rs
View File

@ -1,5 +1,7 @@
use criterion::{black_box, criterion_group, criterion_main, Criterion};
use dynamic_data_experiments::{byte_data::{Params,Data}, encoder::{RSEncoder, BLSEncoder}};
use dynamic_data_experiments::{byte_data::{Params,Data}, encoder::{G8Encoder, BLSEncoder}};
use dynamic_data_experiments::encoder::BLSFieldEncoder;
use dynamic_data_experiments::field_matrix::Matrix;
use dynamic_data_experiments::traits::{DataMatrix, Encoder};
fn bench_rs_encode(c: &mut Criterion) {
@ -13,11 +15,11 @@ fn bench_rs_encode(c: &mut Criterion) {
// generate a random data matrix once
let data = Data::new_random(params.clone());
c.bench_function("RSEncoder::encode", |b| {
c.bench_function("G8Encoder::encode", |b| {
b.iter(|| {
// clone data for each iteration to avoid mutating the original
let mut d = black_box(data.clone());
RSEncoder::encode(&mut d).expect("encode failed");
G8Encoder::encode(&mut d).expect("encode failed");
});
});
}
@ -41,5 +43,24 @@ fn bench_bls_encode(c: &mut Criterion) {
});
}
criterion_group!(benches, bench_rs_encode, bench_bls_encode);
fn bench_bls_field_encode(c: &mut Criterion) {
// test parameters
let k = 100;
let p = 100;
let n = k + p;
let m = 200;
let params = Params { k, n, m };
// generate a random data matrix once
let data = Matrix::new_random(params.clone());
c.bench_function("BLSFieldEncoder::encode", |b| {
b.iter(|| {
let mut d = black_box(data.clone());
BLSFieldEncoder::encode(&mut d).expect("encode failed");
});
});
}
criterion_group!(benches, bench_rs_encode, bench_bls_encode, bench_bls_field_encode);
criterion_main!(benches);

84
src/byte_data.rs
View File

@ -11,6 +11,44 @@ pub struct Params{
pub n: usize,
pub m: usize,
}
impl Params{
pub fn check_bounds(&self, r: usize, c: usize) -> anyhow::Result<()>{
assert!(
r < self.n,
"row index {} out of bounds; must be < {}",
r,
self.n
);
assert!(
c < self.m,
"col index {} out of bounds; must be < {}",
c,
self.m
);
Ok(())
}
pub fn check_rows(&self, r: usize) -> anyhow::Result<()>{
assert!(
r < self.n,
"row index {} out of bounds; must be < {}",
r,
self.n
);
Ok(())
}
pub fn check_cols(&self, c: usize) -> anyhow::Result<()>{
assert!(
c < self.m,
"col index {} out of bounds; must be < {}",
c,
self.m
);
Ok(())
}
}
/// data struct contains shards matrix with dimensions `n`*`m`
/// the matrix contains n rows, k of which are source data and the rest p = (n-k) are parity
#[derive(Clone, Debug)]
@ -20,21 +58,11 @@ pub struct Data<T>{
}
impl<T> Data<T>{
pub fn get_row(&self, idx: usize) -> &Vec<T>{
&self.matrix[idx]
}
pub fn get_row_mut(&mut self, idx: usize) -> &mut Vec<T>{
&mut self.matrix[idx]
}
pub fn get_col(&self, idx: usize) -> Vec<&T> {
self.matrix
.iter()
.map(|row| &row[idx])
.collect()
}
pub fn get_col_mut(&mut self, idx: usize) -> Vec<&mut T> {
self.matrix
.iter_mut()
@ -67,9 +95,35 @@ impl DataMatrix<u8> for Data<u8> {
}
}
fn get(&self, r: usize, c: usize) -> anyhow::Result<u8> {
self.params.check_bounds(r,c)?;
Ok(self.matrix[r][c].clone())
}
fn get_row(&self, r: usize) -> anyhow::Result<Vec<u8>> {
self.params.check_rows(r)?;
Ok(self.matrix[r].to_vec())
}
fn get_col(&self, c: usize) -> anyhow::Result<Vec<u8>> {
self.params.check_cols(c)?;
Ok(self.matrix
.iter()
.map(|row| row[c].clone())
.collect())
}
fn set(&mut self, r: usize, c: usize, elem: u8) -> anyhow::Result<()> {
self.params.check_bounds(r,c)?;
self.matrix[r][c] = elem;
Ok(())
}
/// Update col `c` in shards.
/// given `new_col` will replace the column `c` or `shards[0..k][c]`
fn update_col(&mut self, c: usize, new_col: &[u8]) {
fn update_col(&mut self, c: usize, new_col: &[u8]) -> anyhow::Result<()>{
// sanity checks
assert!(
new_col.len() == self.params.k,
@ -77,17 +131,13 @@ impl DataMatrix<u8> for Data<u8> {
new_col.len(),
self.params.k
);
assert!(
c < self.params.m,
"col index {} out of bounds; must be < {}",
c,
self.params.m
);
self.params.check_cols(c)?;
// write into each of the k data row at position c
for i in 0..self.params.k {
self.matrix[i][c] = new_col[i];
}
Ok(())
}
/// Print all shards

62
src/encoder.rs
View File

@ -7,16 +7,16 @@ use ark_poly::{DenseUVPolynomial, GeneralEvaluationDomain, Polynomial};
use ark_poly::univariate::DensePolynomial;
use reed_solomon_erasure::galois_8::ReedSolomon;
use crate::byte_data::{Data, Params};
use crate::traits::Encoder;
use crate::traits::{DataMatrix, Encoder};
use ark_poly::domain::EvaluationDomain;
use crate::field_matrix::Matrix;
// ------------- RS Encoder ------------
pub struct RSEncoder<T>{
pub struct G8Encoder<T>{
phantom_data: PhantomData<T>
}
impl RSEncoder<u8>{
impl G8Encoder<u8>{
pub fn new() -> Self{
Self{
phantom_data: PhantomData::default()
@ -24,7 +24,7 @@ impl RSEncoder<u8>{
}
}
impl Encoder<u8> for RSEncoder<u8> {
impl Encoder<u8> for G8Encoder<u8> {
type Params = Params;
type DataMatrix<T> = Data<u8>;
@ -53,7 +53,7 @@ impl Encoder<u8> for RSEncoder<u8> {
Ok(())
}
fn encode_col(data: &mut Data<u8>, c: usize) -> Result<Vec<u8>>{
fn encode_col(data: &mut Data<u8>, c: usize) -> Result<()>{
// bounds check
if c >= data.params.m {
return Err(anyhow!("col index {} out of bounds (< {})", c, data.params.m));
@ -66,7 +66,7 @@ impl Encoder<u8> for RSEncoder<u8> {
// Build the column: data = existing byte, parity = zero
let mut temp: Vec<Vec<u8>> = (0..n)
.map(|i| {
let byte = data.matrix[i][c];
let byte = data.get(i,c).unwrap();
if i < k {
vec![byte]
} else {
@ -80,17 +80,15 @@ impl Encoder<u8> for RSEncoder<u8> {
let rse = ReedSolomon::new(k, p)?;
rse.encode(&mut refs)?;
// Write back parity and collect full col
let mut full_col = Vec::with_capacity(n);
// Write back parity
for i in 0..n {
let b = refs[i][0];
if i >= k {
data.matrix[i][c] = b;
data.set(i,c, b)?;
}
full_col.push(b);
}
Ok(full_col)
Ok(())
}
fn reconstruct(params: Params, matrix_opts: &mut Vec<Option<Vec<u8>>>) -> Result<()>{
@ -133,7 +131,7 @@ impl Encoder<u8> for BLSEncoder<u8> {
Ok(())
}
fn encode_col(data: &mut Self::DataMatrix<u8>, c: usize) -> Result<Vec<u8>> {
fn encode_col(data: &mut Self::DataMatrix<u8>, c: usize) -> Result<()> {
let n = data.params.n.clone();
let k = data.params.k.clone();
let mut col = data.get_col_mut(c);
@ -141,14 +139,13 @@ impl Encoder<u8> for BLSEncoder<u8> {
let poly_poly = UniPoly381::from_coefficients_slice(&col_f);
let domain: GeneralEvaluationDomain<F> = EvaluationDomain::<F>::new(n).unwrap();
let mut new_col: Vec<u8> = vec![];
// let mut new_col: Vec<u8> = vec![];
for i in k..n{
let eval = poly_poly.evaluate(&domain.element(i));
new_col.push(eval.0.0[0] as u8);
*col[i] = eval.0.0[0] as u8;
}
Ok(new_col)
Ok(())
}
fn reconstruct(_params: Params, _matrix_opts: &mut Vec<Option<Vec<u8>>>) -> Result<()> {
@ -158,7 +155,11 @@ impl Encoder<u8> for BLSEncoder<u8> {
// --------- BLS Encoder over FieldMatrix ----------------
impl Encoder<F> for BLSEncoder<F> {
pub struct BLSFieldEncoder<T>{
phantom_data: PhantomData<T>
}
impl Encoder<F> for BLSFieldEncoder<F>{
type Params = Params;
type DataMatrix<T> = Matrix<F>;
@ -169,23 +170,18 @@ impl Encoder<F> for BLSEncoder<F> {
Ok(())
}
fn encode_col(data: &mut Matrix<F>, c: usize) -> Result<Vec<F>> {
// let n = data.params.n.clone();
// let k = data.params.k.clone();
// let mut col = data.get_col_mut(c);
// let col_f: Vec<F> = col.iter().map(|i| <F as PrimeField>::from_le_bytes_mod_order(&i.clone().to_le_bytes())).collect();
// let poly_poly = UniPoly381::from_coefficients_slice(&col_f);
// let domain: GeneralEvaluationDomain<F> = EvaluationDomain::<F>::new(n).unwrap();
//
// let mut new_col: Vec<u8> = vec![];
// for i in k..n{
// let eval = poly_poly.evaluate(&domain.element(i));
// new_col.push(eval.0.0[0] as u8);
// *col[i] = eval.0.0[0] as u8;
// }
//
// Ok(new_col)
todo!()
fn encode_col(data: &mut Matrix<F>, c: usize) -> Result<()> {
let n = data.params.n.clone();
let k = data.params.k.clone();
let col: Vec<F> = data.get_col(c)?;
let poly = UniPoly381::from_coefficients_slice(&col);
let domain: GeneralEvaluationDomain<F> = EvaluationDomain::<F>::new(n).unwrap();
for i in k..n{
let eval = poly.evaluate(&domain.element(i));
data.set(i,c,eval)?;
}
Ok(())
}
fn reconstruct(_params: Params, _matrix_opts: &mut Vec<Option<Vec<F>>>) -> Result<()> {

116
src/field_matrix.rs
View File

@ -1,7 +1,7 @@
use ark_ff::Field;
use ark_std::rand::Rng;
use ark_std::UniformRand;
use ark_std::{test_rng};
use crate::byte_data::{Data, Params};
use crate::traits::DataMatrix;
/// a Field matrix with `row` number of rows and `cols` number of columns
@ -12,34 +12,8 @@ pub struct Matrix<F: Field + Clone> {
}
impl<F: Field + Clone> Matrix<F> {
/// Creates a new matrix from given field data.
pub fn new(params: Params, elms: Vec<Vec<F>>) -> Self {
assert!(elms.len() == params.n, "number of rows must match");
for row in &elms {
assert!(row.len() == params.m, "each row must have `m` elements");
}
Matrix { params, elms }
}
/// Generates a random matrix with given dimensions, uses given rng for randomness.
pub fn new_random<R: Rng + ?Sized>(params: Params, rng: &mut R) -> Self
where
F: UniformRand,
{
let rows = params.n;
let cols = params.m;
let mut data = Vec::with_capacity(rows);
for _ in 0..rows {
let mut row = Vec::with_capacity(cols);
for _ in 0..cols {
row.push(F::rand(rng));
}
data.push(row);
}
Matrix { params, elms: data }
}
/// Creates a new matrix from given data struct
/// Creates a new matrix from given u8 data struct
pub fn from_data(data: &Data<u8>) -> Self{
let rows = data.params.n;
let cols = data.params.m;
@ -55,23 +29,10 @@ impl<F: Field + Clone> Matrix<F> {
Matrix { params: data.params.clone(), elms:field_data }
}
/// get the row at 0<idx<n
pub fn row(&self, idx: usize) -> Vec<F>{
assert!(idx < self.params.n, "Row index out of bounds");
self.elms[idx].to_vec()
}
/// get mut the row at 0<idx<n
pub fn row_mut(&mut self, idx: usize) -> &mut Vec<F>{
assert!(idx < self.params.n, "Row index out of bounds");
&mut self.elms[idx]
}
pub fn get_col(&self, idx: usize) -> Vec<&F> {
self.elms
.iter()
.map(|row| &row[idx])
.collect()
pub fn get_row_mut(&mut self, idx: usize) -> anyhow::Result<&mut Vec<F>>{
self.params.check_rows(idx)?;
Ok(&mut self.elms[idx])
}
pub fn get_col_mut(&mut self, idx: usize) -> Vec<&mut F> {
@ -80,9 +41,54 @@ impl<F: Field + Clone> Matrix<F> {
.map(|row| &mut row[idx])
.collect()
}
}
impl<F: Field + Clone> DataMatrix<F> for Matrix<F> {
type Params = Params;
/// Generates a random matrix with given dimensions, uses given rng for randomness.
fn new_random(params: Params) -> Self
{
let mut rng = test_rng();
let rows = params.n;
let cols = params.m;
let mut data = Vec::with_capacity(rows);
for _ in 0..rows {
let mut row = Vec::with_capacity(cols);
for _ in 0..cols {
row.push(F::rand(&mut rng));
}
data.push(row);
}
Matrix { params, elms: data }
}
fn get(&self, r: usize, c: usize) -> anyhow::Result<F> {
self.params.check_bounds(r,c)?;
Ok(self.elms[r][c].clone())
}
fn set(&mut self, r: usize, c: usize, elem: F) -> anyhow::Result<()>{
self.params.check_bounds(r,c)?;
Ok(self.elms[r][c] = elem)
}
/// get the row at 0<idx<n
fn get_row(&self, idx: usize) -> anyhow::Result<Vec<F>>{
self.params.check_rows(idx)?;
Ok(self.elms[idx].to_vec())
}
fn get_col(&self, idx: usize) -> anyhow::Result<Vec<F>> {
self.params.check_cols(idx)?;
Ok(self.elms
.iter()
.map(|row| row[idx].clone())
.collect())
}
/// Print matrix
pub fn pretty_print(&self) {
fn pretty_print(&self) {
for (i, shard) in self.elms.iter().enumerate() {
print!("row {:>2}: ", i);
for &b in shard {
@ -91,6 +97,24 @@ impl<F: Field + Clone> Matrix<F> {
println!();
}
}
fn update_col(&mut self, c: usize, new_col: &[F]) -> anyhow::Result<()> {
self.params.check_cols(c)?;
// ensure the provided column has exactly `k` entries
assert!(
new_col.len() == self.params.k,
"new_col length ({}) must equal k ({})",
new_col.len(),
self.params.k
);
for (r, val) in new_col.iter().enumerate() {
self.elms[r][c] = val.clone();
}
Ok(())
}
}

5
src/kzg.rs
View File

@ -18,7 +18,7 @@ use ark_poly_commit::marlin_pc::MarlinKZG10;
use ark_poly_commit::sonic_pc::UniversalParams;
use crate::byte_data::Params;
use crate::field_matrix::Matrix;
use crate::traits::PolynomialCommitmentScheme;
use crate::traits::{PolynomialCommitmentScheme, DataMatrix};
use ark_crypto_primitives::sponge::poseidon::{PoseidonSponge, PoseidonConfig};
use ark_poly_commit::kzg10::Proof;
@ -67,7 +67,8 @@ impl PolynomialCommitmentScheme for KZGPolyComm {
let mut row_polynomials = vec![];
let timer = start_timer!(|| format!("Poly evaluations and interpolation for {} rows", degree));
for i in 0..matrix.params.n{
let poly_evals = Evaluations::from_vec_and_domain(matrix.row(i), srs.ploycommit_domain.clone());
let row = matrix.get_row(i)?.to_vec();
let poly_evals = Evaluations::from_vec_and_domain(row, srs.ploycommit_domain.clone());
let row_poly = poly_evals.interpolate();
let label = String::from(format!("row_poly_{}", i));
let labeled_poly = LabeledPolynomial::new(

76
src/test.rs
View File

@ -5,7 +5,7 @@ mod tests {
use crate::kzg::{F, KZGPolyComm};
use crate::field_matrix::Matrix;
use ark_poly_commit::{Polynomial};
use crate::encoder::{BLSEncoder, RSEncoder};
use crate::encoder::{BLSEncoder, BLSFieldEncoder, G8Encoder};
use crate::traits::{DataMatrix, Encoder, PolynomialCommitmentScheme};
#[test]
@ -31,7 +31,7 @@ mod tests {
let original: Vec<Vec<u8>> = data.matrix[..k].to_vec();
// encode
RSEncoder::encode(&mut data).expect("encode failed");
G8Encoder::encode(&mut data).expect("encode failed");
println!("data after encoding:");
data.pretty_print();
@ -44,7 +44,7 @@ mod tests {
matrix_opts[k] = None;
// reconstruct missing rows
RSEncoder::reconstruct(data.params.clone(), &mut matrix_opts).expect("reconstruction should succeed");
G8Encoder::reconstruct(data.params.clone(), &mut matrix_opts).expect("reconstruction should succeed");
// verify reconstruction for data shards
for i in 0..k {
@ -54,7 +54,7 @@ mod tests {
}
#[test]
fn test_bls_encodeer() {
fn test_bls_encoder() {
// test parameters
let k = 4;
let p = 4;
@ -98,6 +98,54 @@ mod tests {
// }
}
#[test]
fn test_bls_field_encoder() {
// test parameters
let k = 4;
let p = 4;
let n = k + p;
let m = 8;
// generate Data with random content
let params = Params {
k,
n,
m,
};
let data = Data::new_random(params);
println!("data #row ={}", data.matrix.len());
println!("data #col ={}", data.matrix[0].len());
println!("data before encoding:");
data.pretty_print();
// original data matrix
let mut original = Matrix::from_data(&data);
let original_copy = Matrix::from_data(&data);
println!("data as Field elements:");
original.pretty_print();
// encode
BLSFieldEncoder::encode(&mut original).expect("encode failed");
println!("data after encoding:");
original.pretty_print();
// verify data matrix unchanged
assert_eq!(original.elms[..k], original_copy.elms[..k]);
// simulate loss of one data and one parity rows
let mut matrix_opts: Vec<_> = data.matrix.iter().cloned().map(Some).collect();
matrix_opts[1] = None;
matrix_opts[k] = None;
// TODO: reconstruct missing rows
// BLSEncoder::reconstruct(data.params.clone(), &mut matrix_opts).expect("reconstruction should succeed");
// verify reconstruction for data shards
// for i in 0..k {
// let recovered = matrix_opts[i].clone().unwrap();
// assert_eq!(recovered, &original[i][..]);
// }
}
#[test]
fn test_commit_rows() {
// dimensions: 8 rows (4 parity), 8 columns
@ -112,7 +160,7 @@ mod tests {
m,
};
let mut data = Data::new_random(params.clone());
RSEncoder::encode(&mut data).expect("encode failed");
G8Encoder::encode(&mut data).expect("encode failed");
// make a random n×m matrix
let matrix = Matrix::from_data(&data);
@ -136,7 +184,7 @@ mod tests {
// check that each polynomial really interpolates its original rows
for (i, poly) in row_polys.iter().enumerate() {
let row = matrix.row(i);
let row = matrix.get_row(i).unwrap();
// evaluate poly at each domain point and collect
let evals: Vec<_> = srs
.ploycommit_domain
@ -161,7 +209,7 @@ mod tests {
m,
};
let mut data = Data::new_random(params.clone());
RSEncoder::encode(&mut data).expect("encode failed");
G8Encoder::encode(&mut data).expect("encode failed");
// make a random n×m matrix
let matrix = Matrix::from_data(&data);
@ -178,7 +226,7 @@ mod tests {
for col in 0..m {
let proof = KZGPolyComm::open(&kzg_comm, &srs, row, col)
.expect("open should succeed");
let expected: F = matrix.row(row)[col].clone();
let expected: F = matrix.elms[row][col].clone();
assert!(
KZGPolyComm::verify(&kzg_comm, &srs, row, col, expected, &proof)
@ -206,7 +254,7 @@ mod tests {
};
// snapshot of original
let mut data = Data::new_random(params);
RSEncoder::encode(&mut data).expect("encode failed");
G8Encoder::encode(&mut data).expect("encode failed");
println!("original data:");
data.pretty_print();
@ -230,7 +278,7 @@ mod tests {
);
}
let _coded_row = RSEncoder::encode_col(&mut data, c).unwrap();
let _coded_row = G8Encoder::encode_col(&mut data, c).unwrap();
println!("data after encoding update:");
data.pretty_print();
}
@ -250,7 +298,7 @@ mod tests {
};
// snapshot of original
let mut data = Data::new_random(params.clone());
RSEncoder::encode(&mut data).expect("encode failed");
G8Encoder::encode(&mut data).expect("encode failed");
// Build a matrix where entry (i,j) = i * m + j
let mut matrix = Matrix::<F>::from_data(&data);
@ -266,7 +314,7 @@ mod tests {
// a row to update
let row_idx = 1;
let old_row = matrix.row(row_idx);
let old_row = matrix.get_row(row_idx)?;
// a new row by adding a constant to each element
let new_row: Vec<_> = old_row.iter()
@ -275,7 +323,7 @@ mod tests {
// Apply the change to the in-memory matrix
{
let row_slice = matrix.row_mut(row_idx);
let row_slice = matrix.get_row_mut(row_idx)?;
for (j, val) in new_row.iter().enumerate() {
row_slice[j] = *val;
}
@ -291,7 +339,7 @@ mod tests {
.elements()
.map(|x| poly.polynomial().evaluate(&x))
.collect();
assert_eq!(evals, matrix.row(i));
assert_eq!(evals, matrix.get_row(i)?);
}
// === new fresh commit on updated matrix ===

8
src/traits.rs
View File

@ -4,7 +4,11 @@ use crate::byte_data::Params;
pub trait DataMatrix<T>{
type Params;
fn new_random(params: Self::Params) -> Self;
fn update_col(&mut self, c: usize, new_col: &[T]);
fn get(&self, r: usize, c: usize) -> Result<T>;
fn get_row(&self, r: usize) -> Result<Vec<T>>;
fn get_col(&self, c: usize) -> Result<Vec<T>>;
fn set(&mut self, r: usize, c: usize, elem: T) -> Result<()>;
fn update_col(&mut self, c: usize, new_col: &[T]) -> Result<()>;
fn pretty_print(&self);
}
@ -17,7 +21,7 @@ pub trait Encoder<T>{
/// encode in place the input data matrix
fn encode(data: &mut Self::DataMatrix<T>) -> Result<()>;
/// encode a single column in place
fn encode_col(data: &mut Self::DataMatrix<T>, c: usize) -> Result<Vec<T>>;
fn encode_col(data: &mut Self::DataMatrix<T>, c: usize) -> Result<()>;
/// reconstruct in place
fn reconstruct(params: Params, matrix_opts: &mut Vec<Option<Vec<T>>>) -> Result<()>;
}