mirror of
https://github.com/logos-storage/dynamic-data-experiments.git
synced 2026-01-08 16:13:12 +00:00
update encoder
This commit is contained in:
parent
092e6efb2a
commit
22916acf93
@ -1,27 +1,42 @@
|
|||||||
|
use std::marker::PhantomData;
|
||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
use reed_solomon_erasure::galois_8::ReedSolomon;
|
use reed_solomon_erasure::galois_8::ReedSolomon;
|
||||||
use crate::byte_data::Data;
|
use crate::byte_data::{Data, Params};
|
||||||
use crate::traits::Encoder;
|
use crate::traits::Encoder;
|
||||||
|
|
||||||
|
pub struct RSEncoder<T>{
|
||||||
|
phantom_data: PhantomData<T>
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RSEncoder<u8>{
|
||||||
|
pub fn new() -> Self{
|
||||||
|
Self{
|
||||||
|
phantom_data: PhantomData::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Encoder<u8> for RSEncoder<u8> {
|
||||||
|
type Params = Params;
|
||||||
|
type DataMatrix<T> = Data<u8>;
|
||||||
|
|
||||||
impl Encoder<u8> for Data<u8> {
|
|
||||||
/// encode the columns of the data matrix in place
|
/// encode the columns of the data matrix in place
|
||||||
fn encode(&mut self) -> Result<()> {
|
fn encode(data: &mut Data<u8>) -> Result<()> {
|
||||||
let n = self.params.n;
|
let n = data.params.n;
|
||||||
assert!(self.params.k < n, "k must be less than total shards");
|
assert!(data.params.k < n, "k must be less than total shards");
|
||||||
let p = n - self.params.k;
|
let p = n - data.params.k;
|
||||||
|
|
||||||
// ensure all shards are same length
|
// ensure all rows are same length
|
||||||
let shard_size = self.matrix[0].len();
|
let row_size = data.matrix[0].len();
|
||||||
for shard in &self.matrix[1..] {
|
for row in &data.matrix[1..] {
|
||||||
assert_eq!(shard.len(), shard_size, "all shards must have equal length");
|
assert_eq!(row.len(), row_size, "all rows must have equal length");
|
||||||
}
|
}
|
||||||
|
|
||||||
// build the encoder
|
// build the encoder
|
||||||
let rse = ReedSolomon::new(self.params.k, p)?;
|
let rse = ReedSolomon::new(data.params.k, p)?;
|
||||||
|
|
||||||
// prepare mutable slice references for in-place encode
|
// prepare mutable slice references for in-place encode
|
||||||
let mut shards_refs: Vec<&mut [u8]> = self.matrix.iter_mut()
|
let mut shards_refs: Vec<&mut [u8]> = data.matrix.iter_mut()
|
||||||
.map(|v| v.as_mut_slice())
|
.map(|v| v.as_mut_slice())
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
@ -30,20 +45,20 @@ impl Encoder<u8> for Data<u8> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn encode_col(&mut self, c: usize) -> Result<Vec<u8>>{
|
fn encode_col(data: &mut Data<u8>, c: usize) -> Result<Vec<u8>>{
|
||||||
// bounds check
|
// bounds check
|
||||||
if c >= self.params.m {
|
if c >= data.params.m {
|
||||||
return Err(anyhow!("shard index {} out of bounds (< {})", c, self.params.m));
|
return Err(anyhow!("col index {} out of bounds (< {})", c, data.params.m));
|
||||||
}
|
}
|
||||||
|
|
||||||
let n = self.params.n;
|
let n = data.params.n;
|
||||||
let k = self.params.k;
|
let k = data.params.k;
|
||||||
let p = n - k;
|
let p = n - k;
|
||||||
|
|
||||||
// Build the column: data = existing byte, parity = zero
|
// Build the column: data = existing byte, parity = zero
|
||||||
let mut temp: Vec<Vec<u8>> = (0..n)
|
let mut temp: Vec<Vec<u8>> = (0..n)
|
||||||
.map(|i| {
|
.map(|i| {
|
||||||
let byte = self.matrix[i][c];
|
let byte = data.matrix[i][c];
|
||||||
if i < k {
|
if i < k {
|
||||||
vec![byte]
|
vec![byte]
|
||||||
} else {
|
} else {
|
||||||
@ -62,7 +77,7 @@ impl Encoder<u8> for Data<u8> {
|
|||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
let b = refs[i][0];
|
let b = refs[i][0];
|
||||||
if i >= k {
|
if i >= k {
|
||||||
self.matrix[i][c] = b;
|
data.matrix[i][c] = b;
|
||||||
}
|
}
|
||||||
full_col.push(b);
|
full_col.push(b);
|
||||||
}
|
}
|
||||||
@ -70,7 +85,13 @@ impl Encoder<u8> for Data<u8> {
|
|||||||
Ok(full_col)
|
Ok(full_col)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reconstruct(&mut self) -> Result<()>{
|
fn reconstruct(params: Params, matrix_opts: &mut Vec<Option<Vec<u8>>>) -> Result<()>{
|
||||||
todo!()
|
let n = params.n;
|
||||||
|
let k = params.k;
|
||||||
|
let p = n - k;
|
||||||
|
let rse = ReedSolomon::new(k, p).unwrap();
|
||||||
|
// reconstruct missing rows
|
||||||
|
rse.reconstruct(matrix_opts)?;
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
16
src/test.rs
16
src/test.rs
@ -6,6 +6,7 @@ mod tests {
|
|||||||
use crate::kzg::{F, KZGPolyComm};
|
use crate::kzg::{F, KZGPolyComm};
|
||||||
use crate::field_matrix::Matrix;
|
use crate::field_matrix::Matrix;
|
||||||
use ark_poly_commit::{Polynomial};
|
use ark_poly_commit::{Polynomial};
|
||||||
|
use crate::encoder::RSEncoder;
|
||||||
use crate::traits::{DataMatrix, Encoder, PolynomialCommitmentScheme};
|
use crate::traits::{DataMatrix, Encoder, PolynomialCommitmentScheme};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -31,7 +32,7 @@ mod tests {
|
|||||||
let original: Vec<Vec<u8>> = data.matrix[..k].to_vec();
|
let original: Vec<Vec<u8>> = data.matrix[..k].to_vec();
|
||||||
|
|
||||||
// encode
|
// encode
|
||||||
data.encode().expect("encode failed");
|
RSEncoder::encode(&mut data).expect("encode failed");
|
||||||
println!("data after encoding:");
|
println!("data after encoding:");
|
||||||
data.pretty_print();
|
data.pretty_print();
|
||||||
|
|
||||||
@ -39,13 +40,12 @@ mod tests {
|
|||||||
assert_eq!(data.matrix[..k], original[..]);
|
assert_eq!(data.matrix[..k], original[..]);
|
||||||
|
|
||||||
// simulate loss of one data and one parity rows
|
// simulate loss of one data and one parity rows
|
||||||
let rse = ReedSolomon::new(k, p).unwrap();
|
|
||||||
let mut matrix_opts: Vec<_> = data.matrix.iter().cloned().map(Some).collect();
|
let mut matrix_opts: Vec<_> = data.matrix.iter().cloned().map(Some).collect();
|
||||||
matrix_opts[1] = None;
|
matrix_opts[1] = None;
|
||||||
matrix_opts[k] = None;
|
matrix_opts[k] = None;
|
||||||
|
|
||||||
// reconstruct missing rows
|
// reconstruct missing rows
|
||||||
rse.reconstruct(&mut matrix_opts).expect("reconstruct failed");
|
RSEncoder::reconstruct(data.params.clone(), &mut matrix_opts).expect("reconstruction should succeed");
|
||||||
|
|
||||||
// verify reconstruction for data shards
|
// verify reconstruction for data shards
|
||||||
for i in 0..k {
|
for i in 0..k {
|
||||||
@ -68,7 +68,7 @@ mod tests {
|
|||||||
m,
|
m,
|
||||||
};
|
};
|
||||||
let mut data = Data::new_random(params.clone());
|
let mut data = Data::new_random(params.clone());
|
||||||
data.encode().expect("encode failed");
|
RSEncoder::encode(&mut data).expect("encode failed");
|
||||||
|
|
||||||
// make a random n×m matrix
|
// make a random n×m matrix
|
||||||
let matrix = Matrix::from_data(&data);
|
let matrix = Matrix::from_data(&data);
|
||||||
@ -117,7 +117,7 @@ mod tests {
|
|||||||
m,
|
m,
|
||||||
};
|
};
|
||||||
let mut data = Data::new_random(params.clone());
|
let mut data = Data::new_random(params.clone());
|
||||||
data.encode().expect("encode failed");
|
RSEncoder::encode(&mut data).expect("encode failed");
|
||||||
|
|
||||||
// make a random n×m matrix
|
// make a random n×m matrix
|
||||||
let matrix = Matrix::from_data(&data);
|
let matrix = Matrix::from_data(&data);
|
||||||
@ -162,7 +162,7 @@ mod tests {
|
|||||||
};
|
};
|
||||||
// snapshot of original
|
// snapshot of original
|
||||||
let mut data = Data::new_random(params);
|
let mut data = Data::new_random(params);
|
||||||
data.encode().expect("encode failed");
|
RSEncoder::encode(&mut data).expect("encode failed");
|
||||||
println!("original data:");
|
println!("original data:");
|
||||||
data.pretty_print();
|
data.pretty_print();
|
||||||
|
|
||||||
@ -186,7 +186,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let _coded_row = data.encode_col(c).unwrap();
|
let _coded_row = RSEncoder::encode_col(&mut data, c).unwrap();
|
||||||
println!("data after encoding update:");
|
println!("data after encoding update:");
|
||||||
data.pretty_print();
|
data.pretty_print();
|
||||||
}
|
}
|
||||||
@ -206,7 +206,7 @@ mod tests {
|
|||||||
};
|
};
|
||||||
// snapshot of original
|
// snapshot of original
|
||||||
let mut data = Data::new_random(params.clone());
|
let mut data = Data::new_random(params.clone());
|
||||||
data.encode().expect("encode failed");
|
RSEncoder::encode(&mut data).expect("encode failed");
|
||||||
|
|
||||||
// Build a matrix where entry (i,j) = i * m + j
|
// Build a matrix where entry (i,j) = i * m + j
|
||||||
let mut matrix = Matrix::<F>::from_data(&data);
|
let mut matrix = Matrix::<F>::from_data(&data);
|
||||||
|
|||||||
@ -1,21 +1,25 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
use crate::byte_data::Params;
|
||||||
|
|
||||||
pub trait DataMatrix<T>{
|
pub trait DataMatrix<T>{
|
||||||
type Params;
|
type Params;
|
||||||
fn new_random(_: Self::Params) -> Self;
|
fn new_random(params: Self::Params) -> Self;
|
||||||
fn update_col(&mut self, c: usize, new_col: &[T]);
|
fn update_col(&mut self, c: usize, new_col: &[T]);
|
||||||
fn pretty_print(&self);
|
fn pretty_print(&self);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Encoder trait
|
/// Encoder trait
|
||||||
pub trait Encoder<T>{
|
pub trait Encoder<T>{
|
||||||
|
type Params;
|
||||||
|
/// data matrix type to encode
|
||||||
|
type DataMatrix<U>;
|
||||||
|
|
||||||
/// encode in place the input data matrix
|
/// encode in place the input data matrix
|
||||||
fn encode(&mut self) -> Result<()>;
|
fn encode(data: &mut Self::DataMatrix<T>) -> Result<()>;
|
||||||
/// encode a single column in place
|
/// encode a single column in place
|
||||||
fn encode_col(&mut self, c: usize) -> Result<Vec<T>>;
|
fn encode_col(data: &mut Self::DataMatrix<T>, c: usize) -> Result<Vec<T>>;
|
||||||
/// reconstruct in place
|
/// reconstruct in place
|
||||||
fn reconstruct(&mut self) -> Result<()>;
|
fn reconstruct(params: Params, matrix_opts: &mut Vec<Option<Vec<T>>>) -> Result<()>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Polynomial Commitment scheme (e.g. KZG) trait
|
/// Polynomial Commitment scheme (e.g. KZG) trait
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user