From ee85d8737d3cb27d50a7e1dd486b4379412eee8e Mon Sep 17 00:00:00 2001 From: Daniel Sanchez Date: Wed, 17 Apr 2024 13:03:25 +0200 Subject: [PATCH] DA: KZG+RS core (#632) * Removed old kzg rs modules * Added new kzgrs core module * Implemented bytes_to_polynomial and tests * Use coefficient form * Refactor evaluations into method * Use domain elements instead of roots of unity in tests * Fix encoding and test * Clippy happy * Add comments * Implement polynomial commitment * Implement proof generation * Sketch fn signature for verification * implement proof verification * Implemented verification and tests * Return evaluations from bytes_to_polynomial as well * Use modular le bytes * Implement rs encode/decode * Implement decoding tests * Implement decode using lagrange * Cleanup imports --- Cargo.toml | 4 +- nodes/nomos-node/src/api.rs | 2 - nodes/nomos-node/src/lib.rs | 1 - nodes/nomos-node/src/main.rs | 2 +- nomos-cli/src/api/da.rs | 5 +- nomos-core/src/da/attestation/mod.rs | 3 - nomos-core/src/da/certificate/mod.rs | 3 - nomos-core/src/da/certificate/select.rs | 5 +- nomos-core/src/da/certificate_metadata.rs | 2 - nomos-da/full-replication/src/lib.rs | 1 - nomos-da/kzg/Cargo.toml | 17 -- nomos-da/kzg/benches/nomos_kzg.rs | 36 ---- nomos-da/kzg/src/dynamic_kzg.rs | 245 ---------------------- nomos-da/kzg/src/lib.rs | 82 -------- nomos-da/kzg/src/types.rs | 63 ------ nomos-da/kzgrs/Cargo.toml | 22 ++ nomos-da/kzgrs/src/common.rs | 120 +++++++++++ nomos-da/kzgrs/src/kzg.rs | 131 ++++++++++++ nomos-da/kzgrs/src/lib.rs | 3 + nomos-da/kzgrs/src/rs.rs | 133 ++++++++++++ nomos-da/reed-solomon/Cargo.toml | 9 - nomos-da/reed-solomon/src/lib.rs | 73 ------- 22 files changed, 413 insertions(+), 549 deletions(-) delete mode 100644 nomos-da/kzg/Cargo.toml delete mode 100644 nomos-da/kzg/benches/nomos_kzg.rs delete mode 100644 nomos-da/kzg/src/dynamic_kzg.rs delete mode 100644 nomos-da/kzg/src/lib.rs delete mode 100644 nomos-da/kzg/src/types.rs create mode 100644 nomos-da/kzgrs/Cargo.toml create mode 100644 nomos-da/kzgrs/src/common.rs create mode 100644 nomos-da/kzgrs/src/kzg.rs create mode 100644 nomos-da/kzgrs/src/lib.rs create mode 100644 nomos-da/kzgrs/src/rs.rs delete mode 100644 nomos-da/reed-solomon/Cargo.toml delete mode 100644 nomos-da/reed-solomon/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index ee8cb169..b1ae0542 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,8 +11,6 @@ members = [ "nomos-services/metrics", "nomos-services/data-availability", "nomos-services/system-sig", - "nomos-da/reed-solomon", - "nomos-da/kzg", "nomos-da/full-replication", # TODO: add it again and reimplement full replication # "nomos-cli", @@ -22,6 +20,6 @@ members = [ "consensus/carnot-engine", "consensus/cryptarchia-engine", "ledger/cryptarchia-ledger", - "tests", + "tests", "nomos-da/kzgrs", ] resolver = "2" diff --git a/nodes/nomos-node/src/api.rs b/nodes/nomos-node/src/api.rs index aeffa4af..5d53cd51 100644 --- a/nodes/nomos-node/src/api.rs +++ b/nodes/nomos-node/src/api.rs @@ -19,7 +19,6 @@ use tower_http::{ use utoipa::OpenApi; use utoipa_swagger_ui::SwaggerUi; -use full_replication::{Blob, Certificate}; use nomos_core::{header::HeaderId, tx::Transaction}; use nomos_mempool::{ network::adapters::libp2p::Libp2pAdapter as MempoolNetworkAdapter, @@ -32,7 +31,6 @@ use nomos_api::{ http::{cl, consensus, libp2p, mempool, metrics, storage}, Backend, }; -use nomos_core::da::certificate; /// Configuration for the Http Server #[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] diff --git a/nodes/nomos-node/src/lib.rs b/nodes/nomos-node/src/lib.rs index d1ea0cab..9ebfc0e4 100644 --- a/nodes/nomos-node/src/lib.rs +++ b/nodes/nomos-node/src/lib.rs @@ -4,7 +4,6 @@ mod tx; use color_eyre::eyre::Result; use full_replication::Certificate; -use full_replication::{AbsoluteNumber, Attestation, Blob, FullReplication}; #[cfg(feature = "metrics")] use metrics::{backend::map::MapMetricsBackend, types::MetricsData, MetricsService}; diff --git a/nodes/nomos-node/src/main.rs b/nodes/nomos-node/src/main.rs index 13a5f00d..0855d1a9 100644 --- a/nodes/nomos-node/src/main.rs +++ b/nodes/nomos-node/src/main.rs @@ -1,4 +1,4 @@ -use full_replication::{Blob, Certificate}; +use full_replication::Certificate; #[cfg(feature = "metrics")] use nomos_metrics::MetricsSettings; use nomos_node::{ diff --git a/nomos-cli/src/api/da.rs b/nomos-cli/src/api/da.rs index 42d956c6..8b137891 100644 --- a/nomos-cli/src/api/da.rs +++ b/nomos-cli/src/api/da.rs @@ -1,4 +1 @@ -use super::CLIENT; -use full_replication::Certificate; -use nomos_core::da::certificate; -use reqwest::Url; + diff --git a/nomos-core/src/da/attestation/mod.rs b/nomos-core/src/da/attestation/mod.rs index eadd31db..fa79467a 100644 --- a/nomos-core/src/da/attestation/mod.rs +++ b/nomos-core/src/da/attestation/mod.rs @@ -1,6 +1,3 @@ -use bytes::Bytes; -use std::hash::Hash; - pub trait Attestation { type Signature; fn attestation_signature(&self) -> Self::Signature; diff --git a/nomos-core/src/da/certificate/mod.rs b/nomos-core/src/da/certificate/mod.rs index 86e1e5b2..1247e02e 100644 --- a/nomos-core/src/da/certificate/mod.rs +++ b/nomos-core/src/da/certificate/mod.rs @@ -1,8 +1,5 @@ pub mod select; -use bytes::Bytes; -use std::hash::Hash; - pub trait Certificate { type VerificationParameters; type Signature; diff --git a/nomos-core/src/da/certificate/select.rs b/nomos-core/src/da/certificate/select.rs index 163d8e77..1690a215 100644 --- a/nomos-core/src/da/certificate/select.rs +++ b/nomos-core/src/da/certificate/select.rs @@ -31,9 +31,6 @@ impl BlobCertificateSelect for FillSize impl Iterator + 'i { - utils::select::select_from_till_fill_size::( - |blob| SIZE, - certificates, - ) + utils::select::select_from_till_fill_size::(|_| SIZE, certificates) } } diff --git a/nomos-core/src/da/certificate_metadata.rs b/nomos-core/src/da/certificate_metadata.rs index 512a75b8..8474935c 100644 --- a/nomos-core/src/da/certificate_metadata.rs +++ b/nomos-core/src/da/certificate_metadata.rs @@ -1,5 +1,3 @@ -use super::certificate::Certificate; - pub trait CertificateExtension { type Extension; fn extension(&self) -> Self::Extension; diff --git a/nomos-da/full-replication/src/lib.rs b/nomos-da/full-replication/src/lib.rs index 0ace160f..003bc380 100644 --- a/nomos-da/full-replication/src/lib.rs +++ b/nomos-da/full-replication/src/lib.rs @@ -13,7 +13,6 @@ use blake2::{ }; use bytes::Bytes; use nomos_core::da::certificate_metadata::CertificateExtension; -use nomos_core::wire; use serde::{Deserialize, Serialize}; /// Re-export the types for OpenAPI diff --git a/nomos-da/kzg/Cargo.toml b/nomos-da/kzg/Cargo.toml deleted file mode 100644 index 46363f19..00000000 --- a/nomos-da/kzg/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "nomos-kzg" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -kzg = { git = "https://github.com/sifraitech/rust-kzg.git", rev = "222a61df62ef26e91448ad2c934ab8b408e45a61", package = "rust-kzg-blst", features = ["parallel"] } -kzg_traits = { git = "https://github.com/sifraitech/rust-kzg.git", rev = "222a61df62ef26e91448ad2c934ab8b408e45a61", package = "kzg" } - -[dev-dependencies] -criterion = "0.5.1" - -[[bench]] -name = "nomos_kzg" -harness = false diff --git a/nomos-da/kzg/benches/nomos_kzg.rs b/nomos-da/kzg/benches/nomos_kzg.rs deleted file mode 100644 index 094cdfac..00000000 --- a/nomos-da/kzg/benches/nomos_kzg.rs +++ /dev/null @@ -1,36 +0,0 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use kzg::{types::kzg_settings::FsKZGSettings, utils::generate_trusted_setup}; -use kzg_traits::{FFTSettings, KZGSettings}; -use nomos_kzg::{Blob, KzgSettings}; - -fn nomos_dynamic_vs_external(c: &mut Criterion) { - let (g1s, g2s) = generate_trusted_setup(4096, [0; 32]); - let fft_settings = kzg::types::fft_settings::FsFFTSettings::new(8).unwrap(); - let settings = FsKZGSettings::new(&g1s, &g2s, 4096, &fft_settings).unwrap(); - let kzg_settings = KzgSettings { - settings: settings.clone(), - bytes_per_field_element: 32, - }; - let data = [5; 4096 * 32]; - let blob = Blob::from_bytes(&data, &kzg_settings).unwrap(); - - let mut group = c.benchmark_group("KZG Commitment Benchmarks"); - - group.bench_function("nomos blob commitment", |b| { - b.iter(|| nomos_kzg::compute_commitment(black_box(&data), black_box(&kzg_settings))) - }); - - group.bench_function("external blob commitment", |b| { - b.iter(|| { - kzg::eip_4844::blob_to_kzg_commitment_rust( - black_box(&blob.inner()), - black_box(&settings), - ) - }) - }); - - group.finish(); -} - -criterion_group!(benches, nomos_dynamic_vs_external); -criterion_main!(benches); diff --git a/nomos-da/kzg/src/dynamic_kzg.rs b/nomos-da/kzg/src/dynamic_kzg.rs deleted file mode 100644 index 26363683..00000000 --- a/nomos-da/kzg/src/dynamic_kzg.rs +++ /dev/null @@ -1,245 +0,0 @@ -//! Custom variant of rust-kzg that supports dynamic sized blobs. -//! https://github.com/sifraitech/rust-kzg -//! Some types were changed to fit our API for comfort. -//! Blob related constants were removed and we use a config based approach. - -use crate::types::{Blob, Commitment, KzgSettings, Proof}; -use kzg::kzg_proofs::g1_linear_combination; -use kzg::types::fr::FsFr; -use kzg::types::g1::FsG1; -use kzg::types::kzg_settings::FsKZGSettings; -use kzg::types::poly::FsPoly; -use kzg_traits::eip_4844::{ - bytes_of_uint64, hash, hash_to_bls_field, verify_kzg_proof_rust, CHALLENGE_INPUT_SIZE, - FIAT_SHAMIR_PROTOCOL_DOMAIN, -}; -use kzg_traits::{Fr, Poly, G1}; - -pub fn blob_to_kzg_commitment( - blob: &Blob, - s: &FsKZGSettings, - field_elements_per_blob: usize, -) -> FsG1 { - let mut out = FsG1::default(); - g1_linear_combination(&mut out, &s.secret_g1, &blob.inner, field_elements_per_blob); - out -} - -pub fn compute_blob_kzg_proof( - blob: &Blob, - commitment: &Commitment, - settings: &KzgSettings, -) -> Result { - if !commitment.0.is_valid() { - return Err("Invalid commitment".to_string()); - } - - let evaluation_challenge_fr = compute_challenge(blob, &commitment.0, settings); - let (proof, _) = compute_kzg_proof(blob, &evaluation_challenge_fr, settings); - Ok(proof) -} - -pub fn verify_blob_kzg_proof( - blob: &Blob, - commitment: &Commitment, - proof: &Proof, - settings: &KzgSettings, -) -> Result { - if !commitment.0.is_valid() { - return Err("Invalid commitment".to_string()); - } - if !proof.0.is_valid() { - return Err("Invalid proof".to_string()); - } - - let polynomial = blob_to_polynomial(&blob.inner); - let evaluation_challenge_fr = compute_challenge(blob, &commitment.0, settings); - let y_fr = evaluate_polynomial_in_evaluation_form_rust( - &polynomial, - &evaluation_challenge_fr, - &settings.settings, - ); - verify_kzg_proof_rust( - &commitment.0, - &evaluation_challenge_fr, - &y_fr, - &proof.0, - &settings.settings, - ) -} - -fn compute_challenge(blob: &Blob, commitment: &FsG1, settings: &KzgSettings) -> FsFr { - let mut bytes: Vec = vec![0; CHALLENGE_INPUT_SIZE]; - - // Copy domain separator - bytes[..16].copy_from_slice(&FIAT_SHAMIR_PROTOCOL_DOMAIN); - bytes_of_uint64(&mut bytes[16..24], blob.len() as u64); - // Set all other bytes of this 16-byte (little-endian) field to zero - bytes_of_uint64(&mut bytes[24..32], 0); - - // Copy blob - for i in 0..blob.len() { - let v = blob.inner[i].to_bytes(); - bytes[(32 + i * settings.bytes_per_field_element) - ..(32 + (i + 1) * settings.bytes_per_field_element)] - .copy_from_slice(&v); - } - - // Copy commitment - let v = commitment.to_bytes(); - for i in 0..v.len() { - bytes[32 + settings.bytes_per_field_element * blob.len() + i] = v[i]; - } - - // Now let's create the challenge! - let eval_challenge = hash(&bytes); - hash_to_bls_field(&eval_challenge) -} - -fn compute_kzg_proof(blob: &Blob, z: &FsFr, s: &KzgSettings) -> (FsG1, FsFr) { - let polynomial = blob_to_polynomial(blob.inner.as_slice()); - let poly_len = polynomial.coeffs.len(); - let y = evaluate_polynomial_in_evaluation_form_rust(&polynomial, z, &s.settings); - - let mut tmp: FsFr; - let roots_of_unity: &Vec = &s.settings.fs.roots_of_unity; - - let mut m: usize = 0; - let mut q: FsPoly = FsPoly::new(poly_len); - - let mut inverses_in: Vec = vec![FsFr::default(); poly_len]; - let mut inverses: Vec = vec![FsFr::default(); poly_len]; - - for i in 0..poly_len { - if z.equals(&roots_of_unity[i]) { - // We are asked to compute a KZG proof inside the domain - m = i + 1; - inverses_in[i] = FsFr::one(); - continue; - } - // (p_i - y) / (ω_i - z) - q.coeffs[i] = polynomial.coeffs[i].sub(&y); - inverses_in[i] = roots_of_unity[i].sub(z); - } - - fr_batch_inv(&mut inverses, &inverses_in, poly_len); - - for (i, inverse) in inverses.iter().enumerate().take(poly_len) { - q.coeffs[i] = q.coeffs[i].mul(inverse); - } - - if m != 0 { - // ω_{m-1} == z - m -= 1; - q.coeffs[m] = FsFr::zero(); - for i in 0..poly_len { - if i == m { - continue; - } - // Build denominator: z * (z - ω_i) - tmp = z.sub(&roots_of_unity[i]); - inverses_in[i] = tmp.mul(z); - } - - fr_batch_inv(&mut inverses, &inverses_in, poly_len); - - for i in 0..poly_len { - if i == m { - continue; - } - // Build numerator: ω_i * (p_i - y) - tmp = polynomial.coeffs[i].sub(&y); - tmp = tmp.mul(&roots_of_unity[i]); - // Do the division: (p_i - y) * ω_i / (z * (z - ω_i)) - tmp = tmp.mul(&inverses[i]); - q.coeffs[m] = q.coeffs[m].add(&tmp); - } - } - - let proof = g1_lincomb(&s.settings.secret_g1, &q.coeffs, poly_len); - (proof, y) -} - -fn evaluate_polynomial_in_evaluation_form_rust(p: &FsPoly, x: &FsFr, s: &FsKZGSettings) -> FsFr { - let poly_len = p.coeffs.len(); - let roots_of_unity: &Vec = &s.fs.roots_of_unity; - let mut inverses_in: Vec = vec![FsFr::default(); poly_len]; - let mut inverses: Vec = vec![FsFr::default(); poly_len]; - - for i in 0..poly_len { - if x.equals(&roots_of_unity[i]) { - return p.get_coeff_at(i); - } - inverses_in[i] = x.sub(&roots_of_unity[i]); - } - - fr_batch_inv(&mut inverses, &inverses_in, poly_len); - - let mut tmp: FsFr; - let mut out = FsFr::zero(); - - for i in 0..poly_len { - tmp = inverses[i].mul(&roots_of_unity[i]); - tmp = tmp.mul(&p.coeffs[i]); - out = out.add(&tmp); - } - - tmp = FsFr::from_u64(poly_len as u64); - out = out.div(&tmp).unwrap(); - tmp = x.pow(poly_len); - tmp = tmp.sub(&FsFr::one()); - out = out.mul(&tmp); - out -} - -fn fr_batch_inv(out: &mut [FsFr], a: &[FsFr], len: usize) { - assert!(len > 0); - - let mut accumulator = FsFr::one(); - - for i in 0..len { - out[i] = accumulator; - accumulator = accumulator.mul(&a[i]); - } - - accumulator = accumulator.eucl_inverse(); - - for i in (0..len).rev() { - out[i] = out[i].mul(&accumulator); - accumulator = accumulator.mul(&a[i]); - } -} - -fn g1_lincomb(points: &[FsG1], scalars: &[FsFr], length: usize) -> FsG1 { - let mut out = FsG1::default(); - g1_linear_combination(&mut out, points, scalars, length); - out -} - -fn blob_to_polynomial(blob: &[FsFr]) -> FsPoly { - let mut p: FsPoly = FsPoly::new(blob.len()); - p.coeffs = blob.to_vec(); - p -} - -#[cfg(test)] -mod test { - use super::*; - use kzg::utils::generate_trusted_setup; - use kzg_traits::{eip_4844::blob_to_kzg_commitment_rust, FFTSettings, KZGSettings}; - - #[test] - fn test_blob_to_kzg_commitment() { - let (g1s, g2s) = generate_trusted_setup(4096, [0; 32]); - let fft_settings = kzg::types::fft_settings::FsFFTSettings::new(8).unwrap(); - let settings = FsKZGSettings::new(&g1s, &g2s, 4096, &fft_settings).unwrap(); - let kzg_settings = KzgSettings { - settings, - bytes_per_field_element: 32, - }; - let blob = Blob::from_bytes(&[5; 4096 * 32], &kzg_settings).unwrap(); - let commitment = blob_to_kzg_commitment(&blob, &kzg_settings.settings, 4096); - let commitment2 = blob_to_kzg_commitment_rust(&blob.inner, &kzg_settings.settings).unwrap(); - assert_eq!(commitment, commitment2); - } -} diff --git a/nomos-da/kzg/src/lib.rs b/nomos-da/kzg/src/lib.rs deleted file mode 100644 index 00527293..00000000 --- a/nomos-da/kzg/src/lib.rs +++ /dev/null @@ -1,82 +0,0 @@ -mod dynamic_kzg; -mod types; - -pub use crate::types::{Blob, Commitment, KzgSettings, Proof}; -pub use dynamic_kzg::{blob_to_kzg_commitment, compute_blob_kzg_proof, verify_blob_kzg_proof}; -use std::error::Error; - -pub const BYTES_PER_PROOF: usize = 48; -pub const BYTES_PER_COMMITMENT: usize = 48; - -/// Compute a kzg commitment for the given data. -/// It works for arbitrary data, but the data must be a multiple of **32 bytes**. -/// The data is interpreted as a sequence of field elements. Each consisting of **32 bytes**. -pub fn compute_commitment( - data: &[u8], - settings: &KzgSettings, -) -> Result> { - let blob = Blob::from_bytes(data, settings)?; - Ok(Commitment(blob_to_kzg_commitment( - &blob, - &settings.settings, - data.len() / settings.bytes_per_field_element, - ))) -} - -/// Compute a kzg proof for each field element in the given data. -/// It works for arbitrary data, but the data must be a multiple of **32 bytes**. -/// The data is interpreted as a sequence of field elements. Each consisting of **32 bytes**. -pub fn compute_proofs( - data: &[u8], - commitment: &Commitment, - settings: &KzgSettings, -) -> Result, Box> { - let blobs = data - .chunks(settings.bytes_per_field_element) - .map(|b| Blob::from_bytes(b, settings)); - let mut res = Vec::new(); - for blob in blobs { - let blob = blob?; - res.push(Proof(compute_blob_kzg_proof(&blob, commitment, settings)?)) - } - Ok(res) -} - -/// Verify a kzg proof for the given blob. -/// It works for arbitrary data, but the data must be a multiple of **32 bytes**. -/// The data is interpreted as a sequence of field elements. Each consisting of **32 bytes**. -pub fn verify_blob( - blob: &[u8], - proof: &Proof, - commitment: &Commitment, - settings: &KzgSettings, -) -> Result> { - let blob = Blob::from_bytes(blob, settings)?; - verify_blob_kzg_proof(&blob, commitment, proof, settings).map_err(|e| e.into()) -} - -#[cfg(test)] -mod test { - use super::*; - use kzg::types::kzg_settings::FsKZGSettings; - use kzg::utils::generate_trusted_setup; - use kzg_traits::{FFTSettings, KZGSettings}; - - #[test] - fn test_compute_and_verify() -> Result<(), Box> { - let (g1s, g2s) = generate_trusted_setup(4096, [0; 32]); - let fft_settings = kzg::types::fft_settings::FsFFTSettings::new(8).unwrap(); - let settings = FsKZGSettings::new(&g1s, &g2s, 4096, &fft_settings).unwrap(); - let kzg_settings = KzgSettings { - settings, - bytes_per_field_element: 32, - }; - let blob = vec![0; 4096]; - let commitment = compute_commitment(&blob, &kzg_settings)?; - let proofs = compute_proofs(&blob, &commitment, &kzg_settings)?; - for proof in proofs { - assert!(verify_blob(&blob, &proof, &commitment, &kzg_settings)?); - } - Ok(()) - } -} diff --git a/nomos-da/kzg/src/types.rs b/nomos-da/kzg/src/types.rs deleted file mode 100644 index 3d6054ff..00000000 --- a/nomos-da/kzg/src/types.rs +++ /dev/null @@ -1,63 +0,0 @@ -use crate::{BYTES_PER_COMMITMENT, BYTES_PER_PROOF}; -use kzg::types::fr::FsFr; -use kzg::types::g1::FsG1; -use kzg::types::kzg_settings::FsKZGSettings; -use kzg_traits::{Fr, G1}; -use std::error::Error; - -/// A wrapper around the KZG settings that also stores the number of bytes per field element. -pub struct KzgSettings { - pub settings: FsKZGSettings, - pub bytes_per_field_element: usize, -} - -/// A KZG commitment. -pub struct Commitment(pub(crate) FsG1); - -/// A KZG proof. -pub struct Proof(pub(crate) FsG1); - -/// A blob of data. -pub struct Blob { - pub(crate) inner: Vec, -} - -impl Commitment { - pub fn as_bytes_owned(&self) -> [u8; BYTES_PER_COMMITMENT] { - self.0.to_bytes() - } -} - -impl Proof { - pub fn as_bytes_owned(&self) -> [u8; BYTES_PER_PROOF] { - self.0.to_bytes() - } -} - -impl Blob { - pub fn from_bytes(data: &[u8], settings: &KzgSettings) -> Result> { - let mut inner = Vec::with_capacity(data.len() / settings.bytes_per_field_element); - for chunk in data.chunks(settings.bytes_per_field_element) { - if chunk.len() < settings.bytes_per_field_element { - let mut padded_chunk = vec![0; settings.bytes_per_field_element]; - padded_chunk[..chunk.len()].copy_from_slice(chunk); - inner.push(FsFr::from_bytes(&padded_chunk)?); - } else { - inner.push(FsFr::from_bytes(chunk)?); - } - } - Ok(Self { inner }) - } - - pub fn len(&self) -> usize { - self.inner.len() - } - - pub fn is_empty(&self) -> bool { - self.inner.is_empty() - } - - pub fn inner(&self) -> Vec { - self.inner.clone() - } -} diff --git a/nomos-da/kzgrs/Cargo.toml b/nomos-da/kzgrs/Cargo.toml new file mode 100644 index 00000000..de81d8e2 --- /dev/null +++ b/nomos-da/kzgrs/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "kzgrs" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +once_cell = "1.19" +ark-ec = "0.4.2" +ark-bls12-381 = { version = "0.4.0" } +ark-bls12-381-ext = "0.4.1" +ark-ff = { version = "0.4.2" } +ark-poly = "0.4.2" +ark-poly-commit = "0.4.0" +ark-serialize = { version = "0.4" } +num-bigint = "0.4.4" +thiserror = "1.0.58" +num-traits = "0.2.18" + +[dev-dependencies] +rand = "0.8.5" \ No newline at end of file diff --git a/nomos-da/kzgrs/src/common.rs b/nomos-da/kzgrs/src/common.rs new file mode 100644 index 00000000..9f6636d9 --- /dev/null +++ b/nomos-da/kzgrs/src/common.rs @@ -0,0 +1,120 @@ +// std +// crates +use ark_bls12_381::fr::Fr; +use ark_ff::{BigInteger256, PrimeField, Zero}; +use ark_poly::domain::general::GeneralEvaluationDomain; +use ark_poly::evaluations::univariate::Evaluations; +use ark_poly::univariate::DensePolynomial; +use thiserror::Error; +// internal + +#[derive(Error, Debug)] +pub enum KzgRsError { + #[error("Data isn't properly padded, data len must match modulus {expected_modulus} but it is {current_size}")] + UnpaddedDataError { + expected_modulus: usize, + current_size: usize, + }, + #[error("ChunkSize should be < 32 (bytes), got {0}")] + ChunkSizeTooBig(usize), + #[error(transparent)] + PolyCommitError(#[from] ark_poly_commit::Error), +} + +/// Transform chunks of bytes (of size `CHUNK_SIZE`) into `Fr` which are considered evaluations of a +/// polynomial. +pub fn bytes_to_evaluations( + data: &[u8], + domain: GeneralEvaluationDomain, +) -> Evaluations { + assert!((data.len() % CHUNK_SIZE).is_zero()); + Evaluations::from_vec_and_domain( + data.chunks(CHUNK_SIZE) + .map(|e| { + // use little endian for convenience as shortening 1 byte (<32 supported) + // do not matter in this endianness + let bint: BigInteger256 = Fr::from_le_bytes_mod_order(e) + .try_into() + .expect("Bytes size should fit for an 256 bits integer"); + Fr::new(bint) + }) + .collect(), + domain, + ) +} + +/// Transform chunks of bytes (of size `CHUNK_SIZE`) into `Fr` which are considered evaluations of a +/// polynomial. Then use FFT to transform that polynomial into coefficient form. +/// `CHUNK_SIZE` needs to be 31 (bytes) or less, otherwise it cannot be encoded. +/// The input data need to be padded, so it fits in a len modulus of `CHUNK_SIZE`. +/// Returns the polynomial in evaluation form and in coefficient form +pub fn bytes_to_polynomial( + data: &[u8], + domain: GeneralEvaluationDomain, +) -> Result<(Evaluations, DensePolynomial), KzgRsError> { + if CHUNK_SIZE >= 32 { + return Err(KzgRsError::ChunkSizeTooBig(CHUNK_SIZE)); + } + if data.len() % CHUNK_SIZE != 0 { + return Err(KzgRsError::UnpaddedDataError { + expected_modulus: CHUNK_SIZE, + current_size: data.len(), + }); + } + let evals = bytes_to_evaluations::(data, domain); + let coefficients = evals.interpolate_by_ref(); + Ok((evals, coefficients)) +} + +#[cfg(test)] +mod test { + use super::{bytes_to_evaluations, bytes_to_polynomial, KzgRsError}; + use ark_bls12_381::fr::Fr; + use ark_ff::{BigInteger, PrimeField}; + use ark_poly::{EvaluationDomain, GeneralEvaluationDomain, Polynomial}; + use once_cell::sync::Lazy; + use rand::{thread_rng, Fill}; + + const CHUNK_SIZE: usize = 31; + static DOMAIN: Lazy> = + Lazy::new(|| GeneralEvaluationDomain::new(128).unwrap()); + #[test] + fn encode_random_polynomial() { + const N: usize = 100; + let mut bytes: [u8; CHUNK_SIZE * N] = [0; CHUNK_SIZE * N]; + let mut rng = thread_rng(); + bytes.try_fill(&mut rng).unwrap(); + let evals = bytes_to_evaluations::<31>(&bytes, *DOMAIN); + let (_, poly) = bytes_to_polynomial::<31>(&bytes, *DOMAIN).unwrap(); + for i in 0..100 { + let eval_point = DOMAIN.element(i); + let point = poly.evaluate(&eval_point); + // check point is the same + assert_eq!(evals[i as usize], point); + // check point bytes are the same + assert_eq!( + &bytes[CHUNK_SIZE * i..CHUNK_SIZE * i + CHUNK_SIZE], + &point.into_bigint().to_bytes_le()[..CHUNK_SIZE] + ) + } + } + + #[test] + fn encode_chunk_size_too_big() { + assert!(matches!( + bytes_to_polynomial::<32>(&[], *DOMAIN), + Err(KzgRsError::ChunkSizeTooBig(32)) + )); + } + + #[test] + fn encode_not_padded_data() { + assert!(matches!( + bytes_to_polynomial::<31>(&[0; 12], *DOMAIN), + Err(KzgRsError::UnpaddedDataError { + expected_modulus: 31, + current_size: 12 + }) + )); + } +} diff --git a/nomos-da/kzgrs/src/kzg.rs b/nomos-da/kzgrs/src/kzg.rs new file mode 100644 index 00000000..d64ddfdc --- /dev/null +++ b/nomos-da/kzgrs/src/kzg.rs @@ -0,0 +1,131 @@ +use crate::common::KzgRsError; +use ark_bls12_381::{Bls12_381, Fr}; +use ark_ec::pairing::Pairing; +use ark_poly::univariate::DensePolynomial; +use ark_poly::{DenseUVPolynomial, EvaluationDomain, GeneralEvaluationDomain, Polynomial}; +use ark_poly_commit::kzg10::{Commitment, Powers, Proof, UniversalParams, KZG10}; +use num_traits::One; +use std::borrow::Cow; +use std::ops::{Mul, Neg}; + +/// Commit to a polynomial where each of the evaluations are over `w(i)` for the degree +/// of the polynomial being omega (`w`) the root of unity (2^x). +pub fn commit_polynomial( + polynomial: &DensePolynomial, + global_parameters: &UniversalParams, +) -> Result, KzgRsError> { + let roots_of_unity = Powers { + powers_of_g: Cow::Borrowed(&global_parameters.powers_of_g), + powers_of_gamma_g: Cow::Owned(vec![]), + }; + KZG10::commit(&roots_of_unity, polynomial, None, None) + .map_err(KzgRsError::PolyCommitError) + .map(|(commitment, _)| commitment) +} + +/// Compute a witness polynomial in that satisfies `witness(x) = (f(x)-v)/(x-u)` +pub fn generate_element_proof( + element_index: usize, + polynomial: &DensePolynomial, + global_parameters: &UniversalParams, + domain: &GeneralEvaluationDomain, +) -> Result, KzgRsError> { + let u = domain.element(element_index); + let v = polynomial.evaluate(&u); + let f_x_v = polynomial + &DensePolynomial::::from_coefficients_vec(vec![-v]); + let x_u = DensePolynomial::::from_coefficients_vec(vec![-u, Fr::one()]); + let witness_polynomial: DensePolynomial<_> = &f_x_v / &x_u; + let proof = commit_polynomial(&witness_polynomial, global_parameters)?; + let proof = Proof { + w: proof.0, + random_v: None, + }; + Ok(proof) +} + +/// Verify proof for a single element +pub fn verify_element_proof( + element_index: usize, + element: &Fr, + commitment: &Commitment, + proof: &Proof, + domain: &GeneralEvaluationDomain, + global_parameters: &UniversalParams, +) -> bool { + let u = domain.element(element_index); + let v = element; + let commitment_check_g1 = commitment.0 + global_parameters.powers_of_g[0].mul(v).neg(); + let proof_check_g2 = global_parameters.beta_h + global_parameters.h.mul(u).neg(); + let lhs = Bls12_381::pairing(commitment_check_g1, global_parameters.h); + let rhs = Bls12_381::pairing(proof.w, proof_check_g2); + lhs == rhs +} + +#[cfg(test)] +mod test { + use crate::common::{bytes_to_evaluations, bytes_to_polynomial}; + use crate::kzg::{commit_polynomial, generate_element_proof, verify_element_proof}; + use ark_bls12_381::{Bls12_381, Fr}; + use ark_poly::univariate::DensePolynomial; + use ark_poly::{DenseUVPolynomial, EvaluationDomain, GeneralEvaluationDomain}; + use ark_poly_commit::kzg10::{UniversalParams, KZG10}; + use once_cell::sync::Lazy; + use rand::{thread_rng, Fill}; + + const COEFFICIENTS_SIZE: usize = 16; + static GLOBAL_PARAMETERS: Lazy> = Lazy::new(|| { + let mut rng = rand::thread_rng(); + KZG10::>::setup( + crate::kzg::test::COEFFICIENTS_SIZE - 1, + true, + &mut rng, + ) + .unwrap() + }); + + static DOMAIN: Lazy> = + Lazy::new(|| GeneralEvaluationDomain::new(COEFFICIENTS_SIZE).unwrap()); + #[test] + fn test_poly_commit() { + let poly = DensePolynomial::from_coefficients_vec((0..10).map(|i| Fr::from(i)).collect()); + assert!(matches!( + commit_polynomial(&poly, &GLOBAL_PARAMETERS), + Ok(_) + )); + } + + #[test] + fn generate_proof_and_validate() { + let mut bytes: [u8; 310] = [0; 310]; + let mut rng = thread_rng(); + bytes.try_fill(&mut rng).unwrap(); + let evaluations = bytes_to_evaluations::<31>(&bytes, *DOMAIN).evals; + let (_, poly) = bytes_to_polynomial::<31>(&bytes, *DOMAIN).unwrap(); + let commitment = commit_polynomial(&poly, &GLOBAL_PARAMETERS).unwrap(); + let proofs: Vec<_> = (0..10) + .map(|i| generate_element_proof(i, &poly, &GLOBAL_PARAMETERS, &DOMAIN).unwrap()) + .collect(); + for (i, (element, proof)) in evaluations.iter().zip(proofs.iter()).enumerate() { + // verifying works + assert!(verify_element_proof( + i, + element, + &commitment, + proof, + &DOMAIN, + &GLOBAL_PARAMETERS + )); + // verification fails for other items + for ii in i + 1..10 { + assert!(!verify_element_proof( + ii, + element, + &commitment, + proof, + &DOMAIN, + &GLOBAL_PARAMETERS + )); + } + } + } +} diff --git a/nomos-da/kzgrs/src/lib.rs b/nomos-da/kzgrs/src/lib.rs new file mode 100644 index 00000000..130b2090 --- /dev/null +++ b/nomos-da/kzgrs/src/lib.rs @@ -0,0 +1,3 @@ +pub mod common; +pub mod kzg; +pub mod rs; diff --git a/nomos-da/kzgrs/src/rs.rs b/nomos-da/kzgrs/src/rs.rs new file mode 100644 index 00000000..902f04dd --- /dev/null +++ b/nomos-da/kzgrs/src/rs.rs @@ -0,0 +1,133 @@ +use ark_bls12_381::Fr; +use ark_ff::{BigInteger, Field, PrimeField}; +use ark_poly::univariate::DensePolynomial; +use ark_poly::{ + DenseUVPolynomial, EvaluationDomain, Evaluations, GeneralEvaluationDomain, Polynomial, +}; +use num_traits::Zero; +use std::ops::{Mul, Neg}; + +/// Extend a polynomial over some factor `polynomial.len()*factor and return the original points +/// plus the extra ones. +/// `factor` need to be `>1` +pub fn encode( + polynomial: &DensePolynomial, + evaluations: &Evaluations, + factor: usize, + domain: &GeneralEvaluationDomain, +) -> Evaluations { + assert!(factor > 1); + Evaluations::from_vec_and_domain( + (0..evaluations.evals.len() * factor) + .map(|i| polynomial.evaluate(&domain.element(i))) + .collect(), + *domain, + ) +} + +/// Interpolate points into a polynomial, then evaluate the polynomial in the original evaluations +/// to recover the original data. +/// `domain` need to be the same domain of the original `evaluations` and `polynomial` used for encoding. +pub fn decode( + original_chunks_len: usize, + points: &[Option], + domain: &GeneralEvaluationDomain, +) -> Evaluations { + let (points, roots_of_unity): (Vec, Vec) = points + .iter() + .enumerate() + .flat_map(|(i, e)| { + if let Some(e) = e { + Some((*e, domain.element(i))) + } else { + None + } + }) + .unzip(); + let coeffs = lagrange_interpolate(&points, &roots_of_unity); + Evaluations::from_vec_and_domain( + (0..original_chunks_len) + .map(|i| coeffs.evaluate(&domain.element(i))) + .collect(), + *domain, + ) +} + +/// Interpolate a set of points using lagrange interpolation and roots of unity +/// Warning!! Be aware that the mapping between points and roots of unity is the intended: +/// A polynomial `f(x)` is derived for `w_x` (root) mapping to p_x. `[(w_1, p_1)..(w_n, p_n)]` even +/// if points are missing it is important to keep the mapping integrity. +pub fn lagrange_interpolate(points: &[Fr], roots_of_unity: &[Fr]) -> DensePolynomial { + assert_eq!(points.len(), roots_of_unity.len()); + let mut result = DensePolynomial::from_coefficients_vec(vec![Fr::zero()]); + for i in 0..roots_of_unity.len() { + let mut summand = DensePolynomial::from_coefficients_vec(vec![points[i]]); + for j in 0..points.len() { + if i != j { + let weight_adjustment = + (roots_of_unity[i] - roots_of_unity[j]) + .inverse() + .expect( + "Roots of unity are/should not repeated. If this panics it means we have no coefficients enough in the evaluation domain" + ); + summand = summand.naive_mul(&DensePolynomial::from_coefficients_vec(vec![ + weight_adjustment.mul(roots_of_unity[j]).neg(), + weight_adjustment, + ])) + } + } + result = result + summand; + } + result +} + +/// Reconstruct bytes from the polynomial evaluation points using original chunk size and a set of points +pub fn points_to_bytes(points: &[Fr]) -> Vec { + fn point_to_buff(p: &Fr) -> impl Iterator { + p.into_bigint().to_bytes_le().into_iter().take(CHUNK_SIZE) + } + points + .iter() + .map(point_to_buff::) + .flatten() + .collect() +} + +#[cfg(test)] +mod test { + use crate::common::bytes_to_polynomial; + use crate::rs::{decode, encode, points_to_bytes}; + use ark_bls12_381::Fr; + use ark_poly::{EvaluationDomain, GeneralEvaluationDomain}; + use once_cell::sync::Lazy; + use rand::{thread_rng, Fill}; + + const COEFFICIENTS_SIZE: usize = 32; + static DOMAIN: Lazy> = + Lazy::new(|| GeneralEvaluationDomain::new(COEFFICIENTS_SIZE).unwrap()); + + #[test] + fn test_encode_decode() { + let mut bytes: [u8; 310] = [0; 310]; + let mut rng = thread_rng(); + bytes.try_fill(&mut rng).unwrap(); + + let (evals, poly) = bytes_to_polynomial::<31>(&bytes, *DOMAIN).unwrap(); + + let encoded = encode(&poly, &evals, 2, &DOMAIN); + let mut encoded: Vec> = encoded.evals.into_iter().map(Some).collect(); + + let decoded = decode(10, &encoded, &DOMAIN); + let decoded_bytes = points_to_bytes::<31>(&decoded.evals); + assert_eq!(decoded_bytes, bytes); + + // check with missing pieces + + for i in (1..encoded.len()).step_by(2) { + encoded[i] = None; + } + + let decoded_bytes = points_to_bytes::<31>(&decoded.evals); + assert_eq!(decoded_bytes, bytes); + } +} diff --git a/nomos-da/reed-solomon/Cargo.toml b/nomos-da/reed-solomon/Cargo.toml deleted file mode 100644 index 843614dc..00000000 --- a/nomos-da/reed-solomon/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "reed-solomon" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -reed-solomon-erasure = "6.0" \ No newline at end of file diff --git a/nomos-da/reed-solomon/src/lib.rs b/nomos-da/reed-solomon/src/lib.rs deleted file mode 100644 index c2289fdd..00000000 --- a/nomos-da/reed-solomon/src/lib.rs +++ /dev/null @@ -1,73 +0,0 @@ -use reed_solomon_erasure::{galois_8::ReedSolomon, Error}; - -/// Reed Sololomon encode the elements with a custom parity ratio -/// # Arguments -/// * `parity_ratio` - Ratio of parity elements over original elements size -/// * `elements` - Elements to encode -pub fn encode_elements(parity_ratio: usize, elements: &[u8]) -> Result, Error> { - let mut encoded = vec![vec![0]; elements.len() * (parity_ratio + 1)]; - for (i, &e) in elements.iter().enumerate() { - // review bytes encoding - encoded[i] = e.to_be_bytes().to_vec(); - } - let encoder = ReedSolomon::new(elements.len(), elements.len() * parity_ratio)?; - encoder.encode(&mut encoded)?; - Ok(encoded.into_iter().flatten().collect()) -} - -/// Reed solomon decode the elements with a custom parity ratio -/// # Arguments -/// * `original_size` - Original size of encoded elements -/// * `parity_ratio` - Ratio of parity elements over original elements size (must be the same as the one used for encoding) -/// * `elements` - Elements to decode -pub fn decode_from_elements( - original_size: usize, - parity_ratio: usize, - elements: &[Option], -) -> Result, Error> { - let mut elements: Vec<_> = elements - .iter() - .map(|e| e.map(|n| n.to_be_bytes().to_vec())) - .collect(); - let decoder = ReedSolomon::new(original_size, parity_ratio * original_size)?; - decoder.reconstruct(&mut elements)?; - Ok(elements - .into_iter() - .filter_map(|e: Option>| e.map(|n| u8::from_be_bytes(n.try_into().unwrap()))) - .collect()) -} - -#[cfg(test)] -mod test { - use reed_solomon_erasure::Error; - - #[test] - fn encode_with_ratio() { - let elements = vec![1, 2, 3, 4, 5, 6, 7, 8]; - let encoded = super::encode_elements(1, &elements).unwrap(); - // check intended size - assert_eq!(encoded.len(), 16); - // check elements - assert_eq!(&encoded[0..8], &elements); - } - - #[test] - fn decode_with_ratio() { - let elements = vec![1, 2, 3, 4, 5, 6, 7, 8]; - let encoded = super::encode_elements(1, &elements).unwrap(); - let mut encoded: Vec<_> = encoded.into_iter().map(Some).collect(); - encoded[4..12].copy_from_slice(&[None; 8]); - let decoded = super::decode_from_elements(8, 1, &encoded).unwrap(); - assert_eq!(decoded[0..8], elements); - } - - #[test] - fn decode_fails_with_insufficient_shards() { - let elements = vec![1, 2, 3, 4, 5, 6, 7, 8]; - let encoded = super::encode_elements(1, &elements).unwrap(); - let mut encoded: Vec<_> = encoded.into_iter().map(Some).collect(); - encoded[7..].copy_from_slice(&[None; 9]); - let decoded = super::decode_from_elements(8, 1, &encoded); - assert!(matches!(decoded, Err(Error::TooFewShardsPresent))); - } -}