status-im
/
c-kzg-4844
mirror of https://github.com/status-im/c-kzg-4844.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support Rust `no_std` environments (#347)

This commit is contained in:
DaniPopes 2023-09-06 00:06:03 +02:00 committed by GitHub
parent 5b55a54d5e
commit 551e2f90d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 156 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
.github/workflows/rust-tests.yml vendored
View File

@ -8,6 +8,20 @@ on:
- main
jobs:
feature-checks:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- uses: dtolnay/rust-toolchain@stable
- uses: taiki-e/install-action@cargo-hack
- uses: Swatinem/rust-cache@v2
- name: cargo hack
working-directory: bindings/rust
run: cargo hack check --feature-powerset --depth 2
tests:
runs-on: ${{ matrix.os }}
strategy:
@ -20,14 +34,14 @@ jobs:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Get latest version of stable rust
run: rustup update stable
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Build and Test (minimal preset)
working-directory: bindings/rust
run: cargo test --all --release --features="minimal-spec" --tests
run: cargo test --features minimal-spec
- name: Build and Test (mainnet preset)
working-directory: bindings/rust
run: cargo test --all --release --tests
run: cargo test --features mainnet-spec
- name: Benchmark
working-directory: bindings/rust
run: cargo bench

19
bindings/rust/Cargo.toml
View File

@ -10,8 +10,9 @@ license = "Apache-2.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features]
default = ["mainnet-spec"]
default = ["std", "mainnet-spec"]
std = ["hex/std", "libc/std", "serde?/std"]
serde = ["dep:serde"]
mainnet-spec = []
minimal-spec = []
@ -21,20 +22,24 @@ minimal-spec = []
no-threads = []
[dependencies]
hex = "0.4.2"
libc = "0.2"
serde = { version = "1.0", features = ["derive"] }
blst = "0.3.11"
hex = { version = "0.4.2", default-features = false, features = ["alloc"] }
libc = { version = "0.2", default-features = false }
serde = { version = "1.0", optional = true, default-features = false, features = [
"alloc",
"derive",
] }
blst = { version = "0.3.11", default-features = false }
[dev-dependencies]
criterion = "0.5.1"
glob = "0.3.1"
rand = "0.8.5"
serde = { version = "1.0", features = ["derive"] }
serde_yaml = "0.9.17"
serde_json = "1.0.105"
[build-dependencies]
bindgen = { git = "https://github.com/rust-lang/rust-bindgen" , rev = "0de11f0a521611ac8738b7b01d19dddaf3899e66" }
bindgen = { git = "https://github.com/rust-lang/rust-bindgen", rev = "0de11f0a521611ac8738b7b01d19dddaf3899e66" }
cc = "1.0"
[target.'cfg(target_env = "msvc")'.build-dependencies]

27
bindings/rust/benches/kzg_benches.rs
View File

@ -1,8 +1,7 @@
use std::path::PathBuf;
use c_kzg::*;
use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput};
use rand::{rngs::ThreadRng, Rng};
use std::path::Path;
use std::sync::Arc;
fn generate_random_field_element(rng: &mut ThreadRng) -> Bytes32 {
@ -26,7 +25,7 @@ fn generate_random_blob(rng: &mut ThreadRng) -> Blob {
pub fn criterion_benchmark(c: &mut Criterion) {
let max_count: usize = 64;
let mut rng = rand::thread_rng();
let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt");
let trusted_setup_file = Path::new("../../src/trusted_setup.txt");
assert!(trusted_setup_file.exists());
let kzg_settings = Arc::new(KzgSettings::load_trusted_setup_file(trusted_setup_file).unwrap());
@ -86,29 +85,21 @@ pub fn criterion_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("verify_blob_kzg_proof_batch");
for count in [1, 2, 4, 8, 16, 32, 64] {
assert!(count <= max_count);
group.throughput(Throughput::Elements(count as u64));
group.bench_with_input(BenchmarkId::from_parameter(count), &count, |b, &count| {
b.iter_batched_ref(
|| {
let blobs_subset = blobs.clone().into_iter().take(count).collect::<Vec<Blob>>();
let commitments_subset = commitments
.clone()
.into_iter()
.take(count)
.collect::<Vec<Bytes48>>();
let proofs_subset = proofs
.clone()
.into_iter()
.take(count)
.collect::<Vec<Bytes48>>();
let blobs_subset = blobs[..count].to_vec();
let commitments_subset = commitments[..count].to_vec();
let proofs_subset = proofs[..count].to_vec();
(blobs_subset, commitments_subset, proofs_subset)
},
|(blobs_subset, commitments_subset, proofs_subset)| {
KzgProof::verify_blob_kzg_proof_batch(
&blobs_subset,
&commitments_subset,
&proofs_subset,
blobs_subset,
commitments_subset,
proofs_subset,
&kzg_settings,
)
.unwrap();

131
bindings/rust/src/bindings/mod.rs
View File

@ -2,7 +2,9 @@
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
mod serde_helpers;
#[cfg(feature = "serde")]
mod serde;
#[cfg(test)]
mod test_formats;
include!(concat!(env!("OUT_DIR"), "/generated.rs"));
@ -31,9 +33,17 @@ use {
ckzg_min_verify_kzg_proof as verify_kzg_proof,
};
use std::ffi::CString;
use std::mem::MaybeUninit;
use std::path::PathBuf;
use alloc::string::String;
use alloc::vec::Vec;
use core::ffi::CStr;
use core::fmt;
use core::mem::MaybeUninit;
use core::ops::{Deref, DerefMut};
#[cfg(feature = "std")]
use alloc::ffi::CString;
#[cfg(feature = "std")]
use std::path::Path;
pub const BYTES_PER_G1_POINT: usize = 48;
pub const BYTES_PER_G2_POINT: usize = 96;
@ -78,6 +88,23 @@ pub enum Error {
CError(C_KZG_RET),
}
#[cfg(feature = "std")]
impl std::error::Error for Error {}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidBytesLength(s)
| Self::InvalidHexFormat(s)
| Self::InvalidKzgProof(s)
| Self::InvalidKzgCommitment(s)
| Self::InvalidTrustedSetup(s)
| Self::MismatchLength(s) => f.write_str(s),
Self::CError(s) => fmt::Debug::fmt(s, f),
}
}
}
/// Converts a hex string (with or without the 0x prefix) to bytes.
pub fn hex_to_bytes(hex_str: &str) -> Result<Vec<u8>, Error> {
let trimmed_str = hex_str.strip_prefix("0x").unwrap_or(hex_str);
@ -90,8 +117,8 @@ impl KZGSettings {
/// Initializes a trusted setup from `FIELD_ELEMENTS_PER_BLOB` g1 points
/// and 65 g2 points in byte format.
pub fn load_trusted_setup(
g1_bytes: Vec<[u8; BYTES_PER_G1_POINT]>,
g2_bytes: Vec<[u8; BYTES_PER_G2_POINT]>,
g1_bytes: &[[u8; BYTES_PER_G1_POINT]],
g2_bytes: &[[u8; BYTES_PER_G2_POINT]],
) -> Result<Self, Error> {
if g1_bytes.len() != FIELD_ELEMENTS_PER_BLOB {
return Err(Error::InvalidTrustedSetup(format!(
@ -111,17 +138,16 @@ impl KZGSettings {
unsafe {
let res = load_trusted_setup(
kzg_settings.as_mut_ptr(),
g1_bytes.as_ptr() as *const u8,
g1_bytes.as_ptr().cast(),
g1_bytes.len(),
g2_bytes.as_ptr() as *const u8,
g2_bytes.as_ptr().cast(),
g2_bytes.len(),
);
if let C_KZG_RET::C_KZG_OK = res {
Ok(kzg_settings.assume_init())
} else {
Err(Error::InvalidTrustedSetup(format!(
"Invalid trusted setup: {:?}",
res
"Invalid trusted setup: {res:?}",
)))
}
}
@ -133,10 +159,8 @@ impl KZGSettings {
/// 65 # This is fixed and is used for providing multiproofs up to 64 field elements.
/// FIELD_ELEMENT_PER_BLOB g1 byte values
/// 65 g2 byte values
pub fn load_trusted_setup_file(file_path: PathBuf) -> Result<Self, Error> {
// SAFETY: vec![b'r'] has no 0 bytes.
let mode = unsafe { CString::from_vec_unchecked(vec![b'r']) };
#[cfg(feature = "std")]
pub fn load_trusted_setup_file(file_path: &Path) -> Result<Self, Error> {
#[cfg(unix)]
let file_path_bytes = {
use std::os::unix::prelude::OsStrExt;
@ -144,27 +168,52 @@ impl KZGSettings {
};
#[cfg(windows)]
let file_path_bytes = {
file_path
.as_os_str()
.to_str()
.ok_or(Error::InvalidTrustedSetup(format!(
"Unsupported non unicode file path"
)))?
.as_bytes()
};
let file_path_bytes = file_path
.as_os_str()
.to_str()
.ok_or_else(|| Error::InvalidTrustedSetup("Unsupported non unicode file path".into()))?
.as_bytes();
let file_path = CString::new(file_path_bytes)
.map_err(|e| Error::InvalidTrustedSetup(format!("Invalid trusted setup file: {e}")))?;
Self::load_trusted_setup_file_inner(&file_path)
}
/// Loads the trusted setup parameters from a file. The file format is as follows:
///
/// FIELD_ELEMENTS_PER_BLOB
/// 65 # This is fixed and is used for providing multiproofs up to 64 field elements.
/// FIELD_ELEMENT_PER_BLOB g1 byte values
/// 65 g2 byte values
#[cfg(not(feature = "std"))]
pub fn load_trusted_setup_file(file_path: &CStr) -> Result<Self, Error> {
Self::load_trusted_setup_file_inner(file_path)
}
/// Loads the trusted setup parameters from a file.
///
/// Same as [`load_trusted_setup_file`](Self::load_trusted_setup_file)
#[cfg_attr(not(feature = "std"), doc = ", but takes a `CStr` instead of a `Path`")]
/// .
pub fn load_trusted_setup_file_inner(file_path: &CStr) -> Result<Self, Error> {
// SAFETY: `b"r\0"` is a valid null-terminated string.
const MODE: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"r\0") };
// SAFETY:
// - .as_ptr(): pointer is not dangling because file_path has not been dropped.
// Usage or ptr: File will not be written to it by the c code.
let file_ptr = unsafe { libc::fopen(file_path.as_ptr(), mode.as_ptr()) };
let file_ptr = unsafe { libc::fopen(file_path.as_ptr(), MODE.as_ptr()) };
if file_ptr.is_null() {
let e = std::io::Error::last_os_error();
#[cfg(not(feature = "std"))]
return Err(Error::InvalidTrustedSetup(format!(
"Failed to open trusted setup file {e}"
"Failed to open trusted setup file {file_path:?}"
)));
#[cfg(feature = "std")]
return Err(Error::InvalidTrustedSetup(format!(
"Failed to open trusted setup file {file_path:?}: {}",
std::io::Error::last_os_error()
)));
}
let mut kzg_settings = MaybeUninit::<KZGSettings>::uninit();
@ -174,15 +223,13 @@ impl KZGSettings {
Ok(kzg_settings.assume_init())
} else {
Err(Error::InvalidTrustedSetup(format!(
"Invalid trusted setup: {:?}",
res
"Invalid trusted setup: {res:?}"
)))
}
};
// We don't really care if this succeeds.
// We don't really care if this fails.
let _unchecked_close_result = unsafe { libc::fclose(file_ptr) };
drop(file_path);
result
}
@ -473,8 +520,6 @@ impl From<[u8; 48]> for Bytes48 {
}
}
use std::ops::{Deref, DerefMut};
impl Deref for Bytes32 {
type Target = [u8; 32];
fn deref(&self) -> &Self::Target {
@ -534,11 +579,11 @@ unsafe impl Sync for KZGSettings {}
unsafe impl Send for KZGSettings {}
#[cfg(test)]
#[allow(unused_imports, dead_code)]
mod tests {
use super::*;
use rand::{rngs::ThreadRng, Rng};
use std::fs;
use std::{fs, path::PathBuf};
use test_formats::{
blob_to_kzg_commitment_test, compute_blob_kzg_proof, compute_kzg_proof,
verify_blob_kzg_proof, verify_blob_kzg_proof_batch, verify_kzg_proof,
@ -555,7 +600,7 @@ mod tests {
arr.into()
}
fn test_simple(trusted_setup_file: PathBuf) {
fn test_simple(trusted_setup_file: &Path) {
let mut rng = rand::thread_rng();
assert!(trusted_setup_file.exists());
let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
@ -610,9 +655,9 @@ mod tests {
#[test]
fn test_end_to_end() {
let trusted_setup_file = if cfg!(feature = "minimal-spec") {
PathBuf::from("../../src/trusted_setup_4.txt")
Path::new("../../src/trusted_setup_4.txt")
} else {
PathBuf::from("../../src/trusted_setup.txt")
Path::new("../../src/trusted_setup.txt")
};
test_simple(trusted_setup_file);
}
@ -627,7 +672,7 @@ mod tests {
#[cfg(not(feature = "minimal-spec"))]
#[test]
fn test_blob_to_kzg_commitment() {
let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt");
let trusted_setup_file = Path::new("../../src/trusted_setup.txt");
assert!(trusted_setup_file.exists());
let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
let test_files: Vec<PathBuf> = glob::glob(BLOB_TO_KZG_COMMITMENT_TESTS)
@ -654,7 +699,7 @@ mod tests {
#[cfg(not(feature = "minimal-spec"))]
#[test]
fn test_compute_kzg_proof() {
let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt");
let trusted_setup_file = Path::new("../../src/trusted_setup.txt");
assert!(trusted_setup_file.exists());
let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
let test_files: Vec<PathBuf> = glob::glob(COMPUTE_KZG_PROOF_TESTS)
@ -684,7 +729,7 @@ mod tests {
#[cfg(not(feature = "minimal-spec"))]
#[test]
fn test_compute_blob_kzg_proof() {
let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt");
let trusted_setup_file = Path::new("../../src/trusted_setup.txt");
assert!(trusted_setup_file.exists());
let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
let test_files: Vec<PathBuf> = glob::glob(COMPUTE_BLOB_KZG_PROOF_TESTS)
@ -712,7 +757,7 @@ mod tests {
#[cfg(not(feature = "minimal-spec"))]
#[test]
fn test_verify_kzg_proof() {
let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt");
let trusted_setup_file = Path::new("../../src/trusted_setup.txt");
assert!(trusted_setup_file.exists());
let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
let test_files: Vec<PathBuf> = glob::glob(VERIFY_KZG_PROOF_TESTS)
@ -744,7 +789,7 @@ mod tests {
#[cfg(not(feature = "minimal-spec"))]
#[test]
fn test_verify_blob_kzg_proof() {
let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt");
let trusted_setup_file = Path::new("../../src/trusted_setup.txt");
assert!(trusted_setup_file.exists());
let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
let test_files: Vec<PathBuf> = glob::glob(VERIFY_BLOB_KZG_PROOF_TESTS)
@ -775,7 +820,7 @@ mod tests {
#[cfg(not(feature = "minimal-spec"))]
#[test]
fn test_verify_blob_kzg_proof_batch() {
let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt");
let trusted_setup_file = Path::new("../../src/trusted_setup.txt");
assert!(trusted_setup_file.exists());
let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
let test_files: Vec<PathBuf> = glob::glob(VERIFY_BLOB_KZG_PROOF_BATCH_TESTS)

62
bindings/rust/src/bindings/serde_helpers.rs → bindings/rust/src/bindings/serde.rs
View File

@ -1,6 +1,9 @@
//! Serde serialization and deserialization for the basic types in this crate.
use crate::{Blob, Bytes32, Bytes48};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use alloc::string::String;
use alloc::vec::Vec;
use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer};
/// Serialize a byte vec as a hex string with 0x prefix
pub fn serialize_bytes<S, T>(x: T, s: S) -> Result<S::Ok, S::Error>
@ -11,6 +14,12 @@ where
s.serialize_str(&format!("0x{}", hex::encode(x.as_ref())))
}
fn deserialize_hex<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Vec<u8>, D::Error> {
let s = String::deserialize(deserializer)?;
let hex_bytes = s.strip_prefix("0x").unwrap_or(&s);
hex::decode(hex_bytes).map_err(Error::custom)
}
impl Serialize for Blob {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
@ -21,44 +30,20 @@ impl Serialize for Blob {
}
impl<'de> Deserialize<'de> for Blob {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = String::deserialize(deserializer)?;
let bytes_res = match value.strip_prefix("0x") {
Some(value) => hex::decode(value),
None => hex::decode(&value),
};
let bytes = bytes_res.map_err(|e| serde::de::Error::custom(e.to_string()))?;
Blob::from_bytes(bytes.as_slice()).map_err(|e| serde::de::Error::custom(format!("{:?}", e)))
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
Blob::from_bytes(&deserialize_hex(deserializer)?).map_err(Error::custom)
}
}
impl Serialize for Bytes48 {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
serialize_bytes(self.bytes, serializer)
}
}
impl<'de> Deserialize<'de> for Bytes48 {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = String::deserialize(deserializer)?;
let bytes_res = match value.strip_prefix("0x") {
Some(value) => hex::decode(value),
None => hex::decode(&value),
};
let bytes = bytes_res.map_err(|e| serde::de::Error::custom(e.to_string()))?;
Bytes48::from_bytes(bytes.as_slice())
.map_err(|e| serde::de::Error::custom(format!("{:?}", e)))
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
Bytes48::from_bytes(&deserialize_hex(deserializer)?).map_err(Error::custom)
}
}
@ -76,15 +61,7 @@ impl<'de> Deserialize<'de> for Bytes32 {
where
D: Deserializer<'de>,
{
let value = String::deserialize(deserializer)?;
let bytes_res = match value.strip_prefix("0x") {
Some(value) => hex::decode(value),
None => hex::decode(&value),
};
let bytes = bytes_res.map_err(|e| serde::de::Error::custom(e.to_string()))?;
Bytes32::from_bytes(bytes.as_slice())
.map_err(|e| serde::de::Error::custom(format!("{:?}", e)))
Bytes32::from_bytes(&deserialize_hex(deserializer)?).map_err(Error::custom)
}
}
@ -92,7 +69,6 @@ impl<'de> Deserialize<'de> for Bytes32 {
mod tests {
use super::super::*;
use rand::{rngs::ThreadRng, Rng};
use std::path::PathBuf;
fn generate_random_blob(rng: &mut ThreadRng) -> Blob {
let mut arr = [0u8; BYTES_PER_BLOB];
@ -105,11 +81,11 @@ mod tests {
arr.into()
}
fn trusted_setup_file() -> PathBuf {
fn trusted_setup_file() -> &'static Path {
if cfg!(feature = "minimal-spec") {
PathBuf::from("../../src/trusted_setup_4.txt")
Path::new("../../src/trusted_setup_4.txt")
} else {
PathBuf::from("../../src/trusted_setup.txt")
Path::new("../../src/trusted_setup.txt")
}
}

7
bindings/rust/src/bindings/test_formats/verify_blob_kzg_proof_batch.rs
View File

@ -1,6 +1,8 @@
#![allow(dead_code)]
use crate::{Blob, Bytes48, Error};
use alloc::string::String;
use alloc::vec::Vec;
use serde::Deserialize;
#[derive(Deserialize)]
@ -12,11 +14,12 @@ pub struct Input {
impl Input {
pub fn get_blobs(&self) -> Result<Vec<Blob>, Error> {
let mut v: Vec<Blob> = Vec::new();
// TODO: `iter.map.collect` overflows the stack
let mut v = Vec::with_capacity(self.blobs.len());
for blob in &self.blobs {
v.push(Blob::from_hex(blob)?);
}
return Ok(v);
Ok(v)
}
pub fn get_commitments(&self) -> Result<Vec<Bytes48>, Error> {

5
bindings/rust/src/lib.rs
View File

@ -1,3 +1,8 @@
#![cfg_attr(not(feature = "std"), no_std)]
#[macro_use]
extern crate alloc;
// This `extern crate` invocation tells `rustc` that we actually need the symbols from `blst`.
// Without it, the compiler won't link to `blst` when compiling this crate.
// See: https://kornel.ski/rust-sys-crate#linking