mirror of
https://github.com/logos-storage/dynamic-data-experiments.git
synced 2026-01-02 13:13:08 +00:00
update encoder
This commit is contained in:
parent
092e6efb2a
commit
22916acf93
@ -1,27 +1,42 @@
|
||||
use std::marker::PhantomData;
|
||||
use anyhow::{anyhow, Result};
|
||||
use reed_solomon_erasure::galois_8::ReedSolomon;
|
||||
use crate::byte_data::Data;
|
||||
use crate::byte_data::{Data, Params};
|
||||
use crate::traits::Encoder;
|
||||
|
||||
pub struct RSEncoder<T>{
|
||||
phantom_data: PhantomData<T>
|
||||
}
|
||||
|
||||
impl RSEncoder<u8>{
|
||||
pub fn new() -> Self{
|
||||
Self{
|
||||
phantom_data: PhantomData::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Encoder<u8> for RSEncoder<u8> {
|
||||
type Params = Params;
|
||||
type DataMatrix<T> = Data<u8>;
|
||||
|
||||
impl Encoder<u8> for Data<u8> {
|
||||
/// encode the columns of the data matrix in place
|
||||
fn encode(&mut self) -> Result<()> {
|
||||
let n = self.params.n;
|
||||
assert!(self.params.k < n, "k must be less than total shards");
|
||||
let p = n - self.params.k;
|
||||
fn encode(data: &mut Data<u8>) -> Result<()> {
|
||||
let n = data.params.n;
|
||||
assert!(data.params.k < n, "k must be less than total shards");
|
||||
let p = n - data.params.k;
|
||||
|
||||
// ensure all shards are same length
|
||||
let shard_size = self.matrix[0].len();
|
||||
for shard in &self.matrix[1..] {
|
||||
assert_eq!(shard.len(), shard_size, "all shards must have equal length");
|
||||
// ensure all rows are same length
|
||||
let row_size = data.matrix[0].len();
|
||||
for row in &data.matrix[1..] {
|
||||
assert_eq!(row.len(), row_size, "all rows must have equal length");
|
||||
}
|
||||
|
||||
// build the encoder
|
||||
let rse = ReedSolomon::new(self.params.k, p)?;
|
||||
let rse = ReedSolomon::new(data.params.k, p)?;
|
||||
|
||||
// prepare mutable slice references for in-place encode
|
||||
let mut shards_refs: Vec<&mut [u8]> = self.matrix.iter_mut()
|
||||
let mut shards_refs: Vec<&mut [u8]> = data.matrix.iter_mut()
|
||||
.map(|v| v.as_mut_slice())
|
||||
.collect();
|
||||
|
||||
@ -30,20 +45,20 @@ impl Encoder<u8> for Data<u8> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn encode_col(&mut self, c: usize) -> Result<Vec<u8>>{
|
||||
fn encode_col(data: &mut Data<u8>, c: usize) -> Result<Vec<u8>>{
|
||||
// bounds check
|
||||
if c >= self.params.m {
|
||||
return Err(anyhow!("shard index {} out of bounds (< {})", c, self.params.m));
|
||||
if c >= data.params.m {
|
||||
return Err(anyhow!("col index {} out of bounds (< {})", c, data.params.m));
|
||||
}
|
||||
|
||||
let n = self.params.n;
|
||||
let k = self.params.k;
|
||||
let n = data.params.n;
|
||||
let k = data.params.k;
|
||||
let p = n - k;
|
||||
|
||||
// Build the column: data = existing byte, parity = zero
|
||||
let mut temp: Vec<Vec<u8>> = (0..n)
|
||||
.map(|i| {
|
||||
let byte = self.matrix[i][c];
|
||||
let byte = data.matrix[i][c];
|
||||
if i < k {
|
||||
vec![byte]
|
||||
} else {
|
||||
@ -62,7 +77,7 @@ impl Encoder<u8> for Data<u8> {
|
||||
for i in 0..n {
|
||||
let b = refs[i][0];
|
||||
if i >= k {
|
||||
self.matrix[i][c] = b;
|
||||
data.matrix[i][c] = b;
|
||||
}
|
||||
full_col.push(b);
|
||||
}
|
||||
@ -70,7 +85,13 @@ impl Encoder<u8> for Data<u8> {
|
||||
Ok(full_col)
|
||||
}
|
||||
|
||||
fn reconstruct(&mut self) -> Result<()>{
|
||||
todo!()
|
||||
fn reconstruct(params: Params, matrix_opts: &mut Vec<Option<Vec<u8>>>) -> Result<()>{
|
||||
let n = params.n;
|
||||
let k = params.k;
|
||||
let p = n - k;
|
||||
let rse = ReedSolomon::new(k, p).unwrap();
|
||||
// reconstruct missing rows
|
||||
rse.reconstruct(matrix_opts)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
16
src/test.rs
16
src/test.rs
@ -6,6 +6,7 @@ mod tests {
|
||||
use crate::kzg::{F, KZGPolyComm};
|
||||
use crate::field_matrix::Matrix;
|
||||
use ark_poly_commit::{Polynomial};
|
||||
use crate::encoder::RSEncoder;
|
||||
use crate::traits::{DataMatrix, Encoder, PolynomialCommitmentScheme};
|
||||
|
||||
#[test]
|
||||
@ -31,7 +32,7 @@ mod tests {
|
||||
let original: Vec<Vec<u8>> = data.matrix[..k].to_vec();
|
||||
|
||||
// encode
|
||||
data.encode().expect("encode failed");
|
||||
RSEncoder::encode(&mut data).expect("encode failed");
|
||||
println!("data after encoding:");
|
||||
data.pretty_print();
|
||||
|
||||
@ -39,13 +40,12 @@ mod tests {
|
||||
assert_eq!(data.matrix[..k], original[..]);
|
||||
|
||||
// simulate loss of one data and one parity rows
|
||||
let rse = ReedSolomon::new(k, p).unwrap();
|
||||
let mut matrix_opts: Vec<_> = data.matrix.iter().cloned().map(Some).collect();
|
||||
matrix_opts[1] = None;
|
||||
matrix_opts[k] = None;
|
||||
|
||||
// reconstruct missing rows
|
||||
rse.reconstruct(&mut matrix_opts).expect("reconstruct failed");
|
||||
RSEncoder::reconstruct(data.params.clone(), &mut matrix_opts).expect("reconstruction should succeed");
|
||||
|
||||
// verify reconstruction for data shards
|
||||
for i in 0..k {
|
||||
@ -68,7 +68,7 @@ mod tests {
|
||||
m,
|
||||
};
|
||||
let mut data = Data::new_random(params.clone());
|
||||
data.encode().expect("encode failed");
|
||||
RSEncoder::encode(&mut data).expect("encode failed");
|
||||
|
||||
// make a random n×m matrix
|
||||
let matrix = Matrix::from_data(&data);
|
||||
@ -117,7 +117,7 @@ mod tests {
|
||||
m,
|
||||
};
|
||||
let mut data = Data::new_random(params.clone());
|
||||
data.encode().expect("encode failed");
|
||||
RSEncoder::encode(&mut data).expect("encode failed");
|
||||
|
||||
// make a random n×m matrix
|
||||
let matrix = Matrix::from_data(&data);
|
||||
@ -162,7 +162,7 @@ mod tests {
|
||||
};
|
||||
// snapshot of original
|
||||
let mut data = Data::new_random(params);
|
||||
data.encode().expect("encode failed");
|
||||
RSEncoder::encode(&mut data).expect("encode failed");
|
||||
println!("original data:");
|
||||
data.pretty_print();
|
||||
|
||||
@ -186,7 +186,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
let _coded_row = data.encode_col(c).unwrap();
|
||||
let _coded_row = RSEncoder::encode_col(&mut data, c).unwrap();
|
||||
println!("data after encoding update:");
|
||||
data.pretty_print();
|
||||
}
|
||||
@ -206,7 +206,7 @@ mod tests {
|
||||
};
|
||||
// snapshot of original
|
||||
let mut data = Data::new_random(params.clone());
|
||||
data.encode().expect("encode failed");
|
||||
RSEncoder::encode(&mut data).expect("encode failed");
|
||||
|
||||
// Build a matrix where entry (i,j) = i * m + j
|
||||
let mut matrix = Matrix::<F>::from_data(&data);
|
||||
|
||||
@ -1,21 +1,25 @@
|
||||
use anyhow::Result;
|
||||
use crate::byte_data::Params;
|
||||
|
||||
pub trait DataMatrix<T>{
|
||||
type Params;
|
||||
fn new_random(_: Self::Params) -> Self;
|
||||
fn new_random(params: Self::Params) -> Self;
|
||||
fn update_col(&mut self, c: usize, new_col: &[T]);
|
||||
fn pretty_print(&self);
|
||||
}
|
||||
|
||||
/// Encoder trait
|
||||
pub trait Encoder<T>{
|
||||
type Params;
|
||||
/// data matrix type to encode
|
||||
type DataMatrix<U>;
|
||||
|
||||
/// encode in place the input data matrix
|
||||
fn encode(&mut self) -> Result<()>;
|
||||
fn encode(data: &mut Self::DataMatrix<T>) -> Result<()>;
|
||||
/// encode a single column in place
|
||||
fn encode_col(&mut self, c: usize) -> Result<Vec<T>>;
|
||||
fn encode_col(data: &mut Self::DataMatrix<T>, c: usize) -> Result<Vec<T>>;
|
||||
/// reconstruct in place
|
||||
fn reconstruct(&mut self) -> Result<()>;
|
||||
fn reconstruct(params: Params, matrix_opts: &mut Vec<Option<Vec<T>>>) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Polynomial Commitment scheme (e.g. KZG) trait
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user