From 551e2f90d8a630ed51d9b4e6e0ace8839db5307e Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Wed, 6 Sep 2023 00:06:03 +0200 Subject: [PATCH] Support Rust `no_std` environments (#347) --- .github/workflows/rust-tests.yml | 22 ++- bindings/rust/Cargo.toml | 19 ++- bindings/rust/benches/kzg_benches.rs | 27 ++-- bindings/rust/src/bindings/mod.rs | 131 ++++++++++++------ .../bindings/{serde_helpers.rs => serde.rs} | 62 +++------ .../verify_blob_kzg_proof_batch.rs | 7 +- bindings/rust/src/lib.rs | 5 + 7 files changed, 156 insertions(+), 117 deletions(-) rename bindings/rust/src/bindings/{serde_helpers.rs => serde.rs} (72%) diff --git a/.github/workflows/rust-tests.yml b/.github/workflows/rust-tests.yml index 5dec120..b5808ef 100644 --- a/.github/workflows/rust-tests.yml +++ b/.github/workflows/rust-tests.yml @@ -8,6 +8,20 @@ on: - main jobs: + feature-checks: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + - uses: dtolnay/rust-toolchain@stable + - uses: taiki-e/install-action@cargo-hack + - uses: Swatinem/rust-cache@v2 + - name: cargo hack + working-directory: bindings/rust + run: cargo hack check --feature-powerset --depth 2 + tests: runs-on: ${{ matrix.os }} strategy: @@ -20,14 +34,14 @@ jobs: - uses: actions/checkout@v3 with: submodules: recursive - - name: Get latest version of stable rust - run: rustup update stable + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 - name: Build and Test (minimal preset) working-directory: bindings/rust - run: cargo test --all --release --features="minimal-spec" --tests + run: cargo test --features minimal-spec - name: Build and Test (mainnet preset) working-directory: bindings/rust - run: cargo test --all --release --tests + run: cargo test --features mainnet-spec - name: Benchmark working-directory: bindings/rust run: cargo bench diff --git a/bindings/rust/Cargo.toml b/bindings/rust/Cargo.toml index 176acd3..f9ff2e6 100644 --- a/bindings/rust/Cargo.toml +++ b/bindings/rust/Cargo.toml @@ -10,8 +10,9 @@ license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] - -default = ["mainnet-spec"] +default = ["std", "mainnet-spec"] +std = ["hex/std", "libc/std", "serde?/std"] +serde = ["dep:serde"] mainnet-spec = [] minimal-spec = [] @@ -21,20 +22,24 @@ minimal-spec = [] no-threads = [] [dependencies] -hex = "0.4.2" -libc = "0.2" -serde = { version = "1.0", features = ["derive"] } -blst = "0.3.11" +hex = { version = "0.4.2", default-features = false, features = ["alloc"] } +libc = { version = "0.2", default-features = false } +serde = { version = "1.0", optional = true, default-features = false, features = [ + "alloc", + "derive", +] } +blst = { version = "0.3.11", default-features = false } [dev-dependencies] criterion = "0.5.1" glob = "0.3.1" rand = "0.8.5" +serde = { version = "1.0", features = ["derive"] } serde_yaml = "0.9.17" serde_json = "1.0.105" [build-dependencies] -bindgen = { git = "https://github.com/rust-lang/rust-bindgen" , rev = "0de11f0a521611ac8738b7b01d19dddaf3899e66" } +bindgen = { git = "https://github.com/rust-lang/rust-bindgen", rev = "0de11f0a521611ac8738b7b01d19dddaf3899e66" } cc = "1.0" [target.'cfg(target_env = "msvc")'.build-dependencies] diff --git a/bindings/rust/benches/kzg_benches.rs b/bindings/rust/benches/kzg_benches.rs index 6117b87..ad5656a 100644 --- a/bindings/rust/benches/kzg_benches.rs +++ b/bindings/rust/benches/kzg_benches.rs @@ -1,8 +1,7 @@ -use std::path::PathBuf; - use c_kzg::*; use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput}; use rand::{rngs::ThreadRng, Rng}; +use std::path::Path; use std::sync::Arc; fn generate_random_field_element(rng: &mut ThreadRng) -> Bytes32 { @@ -26,7 +25,7 @@ fn generate_random_blob(rng: &mut ThreadRng) -> Blob { pub fn criterion_benchmark(c: &mut Criterion) { let max_count: usize = 64; let mut rng = rand::thread_rng(); - let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt"); + let trusted_setup_file = Path::new("../../src/trusted_setup.txt"); assert!(trusted_setup_file.exists()); let kzg_settings = Arc::new(KzgSettings::load_trusted_setup_file(trusted_setup_file).unwrap()); @@ -86,29 +85,21 @@ pub fn criterion_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("verify_blob_kzg_proof_batch"); for count in [1, 2, 4, 8, 16, 32, 64] { + assert!(count <= max_count); group.throughput(Throughput::Elements(count as u64)); group.bench_with_input(BenchmarkId::from_parameter(count), &count, |b, &count| { b.iter_batched_ref( || { - let blobs_subset = blobs.clone().into_iter().take(count).collect::>(); - let commitments_subset = commitments - .clone() - .into_iter() - .take(count) - .collect::>(); - let proofs_subset = proofs - .clone() - .into_iter() - .take(count) - .collect::>(); - + let blobs_subset = blobs[..count].to_vec(); + let commitments_subset = commitments[..count].to_vec(); + let proofs_subset = proofs[..count].to_vec(); (blobs_subset, commitments_subset, proofs_subset) }, |(blobs_subset, commitments_subset, proofs_subset)| { KzgProof::verify_blob_kzg_proof_batch( - &blobs_subset, - &commitments_subset, - &proofs_subset, + blobs_subset, + commitments_subset, + proofs_subset, &kzg_settings, ) .unwrap(); diff --git a/bindings/rust/src/bindings/mod.rs b/bindings/rust/src/bindings/mod.rs index 5d29392..f32d8d0 100644 --- a/bindings/rust/src/bindings/mod.rs +++ b/bindings/rust/src/bindings/mod.rs @@ -2,7 +2,9 @@ #![allow(non_camel_case_types)] #![allow(non_snake_case)] -mod serde_helpers; +#[cfg(feature = "serde")] +mod serde; +#[cfg(test)] mod test_formats; include!(concat!(env!("OUT_DIR"), "/generated.rs")); @@ -31,9 +33,17 @@ use { ckzg_min_verify_kzg_proof as verify_kzg_proof, }; -use std::ffi::CString; -use std::mem::MaybeUninit; -use std::path::PathBuf; +use alloc::string::String; +use alloc::vec::Vec; +use core::ffi::CStr; +use core::fmt; +use core::mem::MaybeUninit; +use core::ops::{Deref, DerefMut}; + +#[cfg(feature = "std")] +use alloc::ffi::CString; +#[cfg(feature = "std")] +use std::path::Path; pub const BYTES_PER_G1_POINT: usize = 48; pub const BYTES_PER_G2_POINT: usize = 96; @@ -78,6 +88,23 @@ pub enum Error { CError(C_KZG_RET), } +#[cfg(feature = "std")] +impl std::error::Error for Error {} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidBytesLength(s) + | Self::InvalidHexFormat(s) + | Self::InvalidKzgProof(s) + | Self::InvalidKzgCommitment(s) + | Self::InvalidTrustedSetup(s) + | Self::MismatchLength(s) => f.write_str(s), + Self::CError(s) => fmt::Debug::fmt(s, f), + } + } +} + /// Converts a hex string (with or without the 0x prefix) to bytes. pub fn hex_to_bytes(hex_str: &str) -> Result, Error> { let trimmed_str = hex_str.strip_prefix("0x").unwrap_or(hex_str); @@ -90,8 +117,8 @@ impl KZGSettings { /// Initializes a trusted setup from `FIELD_ELEMENTS_PER_BLOB` g1 points /// and 65 g2 points in byte format. pub fn load_trusted_setup( - g1_bytes: Vec<[u8; BYTES_PER_G1_POINT]>, - g2_bytes: Vec<[u8; BYTES_PER_G2_POINT]>, + g1_bytes: &[[u8; BYTES_PER_G1_POINT]], + g2_bytes: &[[u8; BYTES_PER_G2_POINT]], ) -> Result { if g1_bytes.len() != FIELD_ELEMENTS_PER_BLOB { return Err(Error::InvalidTrustedSetup(format!( @@ -111,17 +138,16 @@ impl KZGSettings { unsafe { let res = load_trusted_setup( kzg_settings.as_mut_ptr(), - g1_bytes.as_ptr() as *const u8, + g1_bytes.as_ptr().cast(), g1_bytes.len(), - g2_bytes.as_ptr() as *const u8, + g2_bytes.as_ptr().cast(), g2_bytes.len(), ); if let C_KZG_RET::C_KZG_OK = res { Ok(kzg_settings.assume_init()) } else { Err(Error::InvalidTrustedSetup(format!( - "Invalid trusted setup: {:?}", - res + "Invalid trusted setup: {res:?}", ))) } } @@ -133,10 +159,8 @@ impl KZGSettings { /// 65 # This is fixed and is used for providing multiproofs up to 64 field elements. /// FIELD_ELEMENT_PER_BLOB g1 byte values /// 65 g2 byte values - pub fn load_trusted_setup_file(file_path: PathBuf) -> Result { - // SAFETY: vec![b'r'] has no 0 bytes. - let mode = unsafe { CString::from_vec_unchecked(vec![b'r']) }; - + #[cfg(feature = "std")] + pub fn load_trusted_setup_file(file_path: &Path) -> Result { #[cfg(unix)] let file_path_bytes = { use std::os::unix::prelude::OsStrExt; @@ -144,27 +168,52 @@ impl KZGSettings { }; #[cfg(windows)] - let file_path_bytes = { - file_path - .as_os_str() - .to_str() - .ok_or(Error::InvalidTrustedSetup(format!( - "Unsupported non unicode file path" - )))? - .as_bytes() - }; + let file_path_bytes = file_path + .as_os_str() + .to_str() + .ok_or_else(|| Error::InvalidTrustedSetup("Unsupported non unicode file path".into()))? + .as_bytes(); let file_path = CString::new(file_path_bytes) .map_err(|e| Error::InvalidTrustedSetup(format!("Invalid trusted setup file: {e}")))?; + Self::load_trusted_setup_file_inner(&file_path) + } + + /// Loads the trusted setup parameters from a file. The file format is as follows: + /// + /// FIELD_ELEMENTS_PER_BLOB + /// 65 # This is fixed and is used for providing multiproofs up to 64 field elements. + /// FIELD_ELEMENT_PER_BLOB g1 byte values + /// 65 g2 byte values + #[cfg(not(feature = "std"))] + pub fn load_trusted_setup_file(file_path: &CStr) -> Result { + Self::load_trusted_setup_file_inner(file_path) + } + + /// Loads the trusted setup parameters from a file. + /// + /// Same as [`load_trusted_setup_file`](Self::load_trusted_setup_file) + #[cfg_attr(not(feature = "std"), doc = ", but takes a `CStr` instead of a `Path`")] + /// . + pub fn load_trusted_setup_file_inner(file_path: &CStr) -> Result { + // SAFETY: `b"r\0"` is a valid null-terminated string. + const MODE: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"r\0") }; + // SAFETY: // - .as_ptr(): pointer is not dangling because file_path has not been dropped. // Usage or ptr: File will not be written to it by the c code. - let file_ptr = unsafe { libc::fopen(file_path.as_ptr(), mode.as_ptr()) }; + let file_ptr = unsafe { libc::fopen(file_path.as_ptr(), MODE.as_ptr()) }; if file_ptr.is_null() { - let e = std::io::Error::last_os_error(); + #[cfg(not(feature = "std"))] return Err(Error::InvalidTrustedSetup(format!( - "Failed to open trusted setup file {e}" + "Failed to open trusted setup file {file_path:?}" + ))); + + #[cfg(feature = "std")] + return Err(Error::InvalidTrustedSetup(format!( + "Failed to open trusted setup file {file_path:?}: {}", + std::io::Error::last_os_error() ))); } let mut kzg_settings = MaybeUninit::::uninit(); @@ -174,15 +223,13 @@ impl KZGSettings { Ok(kzg_settings.assume_init()) } else { Err(Error::InvalidTrustedSetup(format!( - "Invalid trusted setup: {:?}", - res + "Invalid trusted setup: {res:?}" ))) } }; - // We don't really care if this succeeds. + // We don't really care if this fails. let _unchecked_close_result = unsafe { libc::fclose(file_ptr) }; - drop(file_path); result } @@ -473,8 +520,6 @@ impl From<[u8; 48]> for Bytes48 { } } -use std::ops::{Deref, DerefMut}; - impl Deref for Bytes32 { type Target = [u8; 32]; fn deref(&self) -> &Self::Target { @@ -534,11 +579,11 @@ unsafe impl Sync for KZGSettings {} unsafe impl Send for KZGSettings {} #[cfg(test)] +#[allow(unused_imports, dead_code)] mod tests { use super::*; use rand::{rngs::ThreadRng, Rng}; - use std::fs; - + use std::{fs, path::PathBuf}; use test_formats::{ blob_to_kzg_commitment_test, compute_blob_kzg_proof, compute_kzg_proof, verify_blob_kzg_proof, verify_blob_kzg_proof_batch, verify_kzg_proof, @@ -555,7 +600,7 @@ mod tests { arr.into() } - fn test_simple(trusted_setup_file: PathBuf) { + fn test_simple(trusted_setup_file: &Path) { let mut rng = rand::thread_rng(); assert!(trusted_setup_file.exists()); let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap(); @@ -610,9 +655,9 @@ mod tests { #[test] fn test_end_to_end() { let trusted_setup_file = if cfg!(feature = "minimal-spec") { - PathBuf::from("../../src/trusted_setup_4.txt") + Path::new("../../src/trusted_setup_4.txt") } else { - PathBuf::from("../../src/trusted_setup.txt") + Path::new("../../src/trusted_setup.txt") }; test_simple(trusted_setup_file); } @@ -627,7 +672,7 @@ mod tests { #[cfg(not(feature = "minimal-spec"))] #[test] fn test_blob_to_kzg_commitment() { - let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt"); + let trusted_setup_file = Path::new("../../src/trusted_setup.txt"); assert!(trusted_setup_file.exists()); let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap(); let test_files: Vec = glob::glob(BLOB_TO_KZG_COMMITMENT_TESTS) @@ -654,7 +699,7 @@ mod tests { #[cfg(not(feature = "minimal-spec"))] #[test] fn test_compute_kzg_proof() { - let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt"); + let trusted_setup_file = Path::new("../../src/trusted_setup.txt"); assert!(trusted_setup_file.exists()); let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap(); let test_files: Vec = glob::glob(COMPUTE_KZG_PROOF_TESTS) @@ -684,7 +729,7 @@ mod tests { #[cfg(not(feature = "minimal-spec"))] #[test] fn test_compute_blob_kzg_proof() { - let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt"); + let trusted_setup_file = Path::new("../../src/trusted_setup.txt"); assert!(trusted_setup_file.exists()); let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap(); let test_files: Vec = glob::glob(COMPUTE_BLOB_KZG_PROOF_TESTS) @@ -712,7 +757,7 @@ mod tests { #[cfg(not(feature = "minimal-spec"))] #[test] fn test_verify_kzg_proof() { - let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt"); + let trusted_setup_file = Path::new("../../src/trusted_setup.txt"); assert!(trusted_setup_file.exists()); let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap(); let test_files: Vec = glob::glob(VERIFY_KZG_PROOF_TESTS) @@ -744,7 +789,7 @@ mod tests { #[cfg(not(feature = "minimal-spec"))] #[test] fn test_verify_blob_kzg_proof() { - let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt"); + let trusted_setup_file = Path::new("../../src/trusted_setup.txt"); assert!(trusted_setup_file.exists()); let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap(); let test_files: Vec = glob::glob(VERIFY_BLOB_KZG_PROOF_TESTS) @@ -775,7 +820,7 @@ mod tests { #[cfg(not(feature = "minimal-spec"))] #[test] fn test_verify_blob_kzg_proof_batch() { - let trusted_setup_file = PathBuf::from("../../src/trusted_setup.txt"); + let trusted_setup_file = Path::new("../../src/trusted_setup.txt"); assert!(trusted_setup_file.exists()); let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap(); let test_files: Vec = glob::glob(VERIFY_BLOB_KZG_PROOF_BATCH_TESTS) diff --git a/bindings/rust/src/bindings/serde_helpers.rs b/bindings/rust/src/bindings/serde.rs similarity index 72% rename from bindings/rust/src/bindings/serde_helpers.rs rename to bindings/rust/src/bindings/serde.rs index c056568..daa86d8 100644 --- a/bindings/rust/src/bindings/serde_helpers.rs +++ b/bindings/rust/src/bindings/serde.rs @@ -1,6 +1,9 @@ //! Serde serialization and deserialization for the basic types in this crate. + use crate::{Blob, Bytes32, Bytes48}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use alloc::string::String; +use alloc::vec::Vec; +use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer}; /// Serialize a byte vec as a hex string with 0x prefix pub fn serialize_bytes(x: T, s: S) -> Result @@ -11,6 +14,12 @@ where s.serialize_str(&format!("0x{}", hex::encode(x.as_ref()))) } +fn deserialize_hex<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + let s = String::deserialize(deserializer)?; + let hex_bytes = s.strip_prefix("0x").unwrap_or(&s); + hex::decode(hex_bytes).map_err(Error::custom) +} + impl Serialize for Blob { fn serialize(&self, serializer: S) -> Result where @@ -21,44 +30,20 @@ impl Serialize for Blob { } impl<'de> Deserialize<'de> for Blob { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - let bytes_res = match value.strip_prefix("0x") { - Some(value) => hex::decode(value), - None => hex::decode(&value), - }; - - let bytes = bytes_res.map_err(|e| serde::de::Error::custom(e.to_string()))?; - Blob::from_bytes(bytes.as_slice()).map_err(|e| serde::de::Error::custom(format!("{:?}", e))) + fn deserialize>(deserializer: D) -> Result { + Blob::from_bytes(&deserialize_hex(deserializer)?).map_err(Error::custom) } } impl Serialize for Bytes48 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { + fn serialize(&self, serializer: S) -> Result { serialize_bytes(self.bytes, serializer) } } impl<'de> Deserialize<'de> for Bytes48 { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - let bytes_res = match value.strip_prefix("0x") { - Some(value) => hex::decode(value), - None => hex::decode(&value), - }; - - let bytes = bytes_res.map_err(|e| serde::de::Error::custom(e.to_string()))?; - Bytes48::from_bytes(bytes.as_slice()) - .map_err(|e| serde::de::Error::custom(format!("{:?}", e))) + fn deserialize>(deserializer: D) -> Result { + Bytes48::from_bytes(&deserialize_hex(deserializer)?).map_err(Error::custom) } } @@ -76,15 +61,7 @@ impl<'de> Deserialize<'de> for Bytes32 { where D: Deserializer<'de>, { - let value = String::deserialize(deserializer)?; - let bytes_res = match value.strip_prefix("0x") { - Some(value) => hex::decode(value), - None => hex::decode(&value), - }; - - let bytes = bytes_res.map_err(|e| serde::de::Error::custom(e.to_string()))?; - Bytes32::from_bytes(bytes.as_slice()) - .map_err(|e| serde::de::Error::custom(format!("{:?}", e))) + Bytes32::from_bytes(&deserialize_hex(deserializer)?).map_err(Error::custom) } } @@ -92,7 +69,6 @@ impl<'de> Deserialize<'de> for Bytes32 { mod tests { use super::super::*; use rand::{rngs::ThreadRng, Rng}; - use std::path::PathBuf; fn generate_random_blob(rng: &mut ThreadRng) -> Blob { let mut arr = [0u8; BYTES_PER_BLOB]; @@ -105,11 +81,11 @@ mod tests { arr.into() } - fn trusted_setup_file() -> PathBuf { + fn trusted_setup_file() -> &'static Path { if cfg!(feature = "minimal-spec") { - PathBuf::from("../../src/trusted_setup_4.txt") + Path::new("../../src/trusted_setup_4.txt") } else { - PathBuf::from("../../src/trusted_setup.txt") + Path::new("../../src/trusted_setup.txt") } } diff --git a/bindings/rust/src/bindings/test_formats/verify_blob_kzg_proof_batch.rs b/bindings/rust/src/bindings/test_formats/verify_blob_kzg_proof_batch.rs index 3a30a0b..98e1966 100644 --- a/bindings/rust/src/bindings/test_formats/verify_blob_kzg_proof_batch.rs +++ b/bindings/rust/src/bindings/test_formats/verify_blob_kzg_proof_batch.rs @@ -1,6 +1,8 @@ #![allow(dead_code)] use crate::{Blob, Bytes48, Error}; +use alloc::string::String; +use alloc::vec::Vec; use serde::Deserialize; #[derive(Deserialize)] @@ -12,11 +14,12 @@ pub struct Input { impl Input { pub fn get_blobs(&self) -> Result, Error> { - let mut v: Vec = Vec::new(); + // TODO: `iter.map.collect` overflows the stack + let mut v = Vec::with_capacity(self.blobs.len()); for blob in &self.blobs { v.push(Blob::from_hex(blob)?); } - return Ok(v); + Ok(v) } pub fn get_commitments(&self) -> Result, Error> { diff --git a/bindings/rust/src/lib.rs b/bindings/rust/src/lib.rs index 99297e5..7c54cb5 100644 --- a/bindings/rust/src/lib.rs +++ b/bindings/rust/src/lib.rs @@ -1,3 +1,8 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +#[macro_use] +extern crate alloc; + // This `extern crate` invocation tells `rustc` that we actually need the symbols from `blst`. // Without it, the compiler won't link to `blst` when compiling this crate. // See: https://kornel.ski/rust-sys-crate#linking