Update rust bindings to support YAML ref tests (#164)

bindings/rust/Cargo.lock

@@ -42,10 +42,12 @@ name = "c-kzg"
 version = "0.1.0"
 dependencies = [
  "criterion",
+ "glob",
  "hex",
  "libc",
  "rand",
- "serde_json",
+ "serde",
+ "serde_yaml",
 ]
 
 [[package]]
@@ -204,6 +206,12 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "glob"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
+
 [[package]]
 name = "half"
 version = "1.8.2"
@@ -512,6 +520,19 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_yaml"
+version = "0.9.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fb06d4b6cdaef0e0c51fa881acb721bed3c924cfaa71d9c94a3b771dfdf6567"
+dependencies = [
+ "indexmap",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
 [[package]]
 name = "syn"
 version = "1.0.107"
@@ -545,6 +566,12 @@ version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
 
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc7ed8ba44ca06be78ea1ad2c3682a43349126c8818054231ee6f4748012aed2"
+
 [[package]]
 name = "walkdir"
 version = "2.3.2"

bindings/rust/Cargo.toml

@@ -11,13 +11,15 @@ mainnet-spec = []
 minimal-spec = []
 
 [dependencies]
-libc = "0.2"
 hex = "0.4.2"
+libc = "0.2"
+serde = { version = "1.0", features = ["derive"] }
 
 [dev-dependencies]
-rand = "0.8.5"
-serde_json = "1.0.89"
 criterion = "0.4"
+glob = "0.3.1"
+rand = "0.8.5"
+serde_yaml = "0.9.17"
 
 [[bench]]
 name = "kzg_benches"

bindings/rust/benches/kzg_benches.rs

@@ -103,7 +103,8 @@ pub fn criterion_benchmark(c: &mut Criterion) {
                         .take(count)
                         .collect::<Vec<Bytes48>>(),
                     &kzg_settings,
-                ).unwrap();
+                )
+                .unwrap();
             })
         });
     }

bindings/rust/src/lib.rs

@@ -2,6 +2,8 @@
 #![allow(non_camel_case_types)]
 #![allow(non_snake_case)]
 
+mod test_formats;
+
 include!("bindings.rs");
 
 use libc::fopen;
@@ -368,6 +370,11 @@ mod tests {
     use rand::{rngs::ThreadRng, Rng};
     use std::fs;
 
+    use crate::test_formats::{
+        blob_to_kzg_commitment_test, compute_blob_kzg_proof, compute_kzg_proof,
+        verify_blob_kzg_proof, verify_blob_kzg_proof_batch, verify_kzg_proof,
+    };
+
     fn generate_random_blob(rng: &mut ThreadRng) -> Blob {
         let mut arr = [0u8; BYTES_PER_BLOB];
         rng.fill(&mut arr[..]);
@@ -440,35 +447,12 @@ mod tests {
         test_simple(trusted_setup_file);
     }
 
-    fn get_blob(path: PathBuf) -> Blob {
+    const BLOB_TO_KZG_COMMITMENT_TESTS: &str = "../../tests/blob_to_kzg_commitment/*/*/*";
-        let input_str = fs::read_to_string(path).unwrap();
+    const COMPUTE_KZG_PROOF_TESTS: &str = "../../tests/compute_kzg_proof/*/*/*";
-        let input_bytes = hex::decode(input_str.as_bytes()).unwrap();
+    const COMPUTE_BLOB_KZG_PROOF_TESTS: &str = "../../tests/compute_blob_kzg_proof/*/*/*";
-        Blob::from_bytes(input_bytes.as_slice()).unwrap()
+    const VERIFY_KZG_PROOF_TESTS: &str = "../../tests/verify_kzg_proof/*/*/*";
-    }
+    const VERIFY_BLOB_KZG_PROOF_TESTS: &str = "../../tests/verify_blob_kzg_proof/*/*/*";
-
+    const VERIFY_BLOB_KZG_PROOF_BATCH_TESTS: &str = "../../tests/verify_blob_kzg_proof_batch/*/*/*";
-    fn get_bytes32(path: PathBuf) -> Bytes32 {
-        let input_str = fs::read_to_string(path).unwrap();
-        let input_bytes = hex::decode(input_str.as_bytes()).unwrap();
-        Bytes32::from_bytes(input_bytes.as_slice()).unwrap()
-    }
-
-    fn get_bytes48(path: PathBuf) -> Bytes48 {
-        let input_str = fs::read_to_string(path).unwrap();
-        let input_bytes = hex::decode(input_str.as_bytes()).unwrap();
-        Bytes48::from_bytes(input_bytes.as_slice()).unwrap()
-    }
-
-    fn get_boolean(path: PathBuf) -> bool {
-        let input_str = fs::read_to_string(path).unwrap();
-        input_str.contains("true")
-    }
-
-    const BLOB_TO_KZG_COMMITMENT_TESTS: &str = "../../tests/blob_to_kzg_commitment/";
-    const COMPUTE_KZG_PROOF_TESTS: &str = "../../tests/compute_kzg_proof/";
-    const COMPUTE_BLOB_KZG_PROOF_TESTS: &str = "../../tests/compute_blob_kzg_proof/";
-    const VERIFY_KZG_PROOF_TESTS: &str = "../../tests/verify_kzg_proof/";
-    const VERIFY_BLOB_KZG_PROOF_TESTS: &str = "../../tests/verify_blob_kzg_proof/";
-    const VERIFY_BLOB_KZG_PROOF_BATCH_TESTS: &str = "../../tests/verify_blob_kzg_proof_batch/";
 
     #[cfg(not(feature = "minimal-spec"))]
     #[test]
@@ -477,18 +461,18 @@ mod tests {
         assert!(trusted_setup_file.exists());
         let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
 
-        let tests = fs::read_dir(BLOB_TO_KZG_COMMITMENT_TESTS)
+        let tests = glob::glob(BLOB_TO_KZG_COMMITMENT_TESTS)
             .unwrap()
-            .map(|t| t.unwrap().path());
+            .map(|t| t.unwrap());
-        for test in tests {
+        for test_file in tests {
-            let blob = get_blob(test.join("blob.txt"));
+            let yaml_data = fs::read_to_string(test_file).unwrap();
-            let res = KZGCommitment::blob_to_kzg_commitment(blob, &kzg_settings);
+            let test: blob_to_kzg_commitment_test::Test = serde_yaml::from_str(&yaml_data).unwrap();
+            let res = KZGCommitment::blob_to_kzg_commitment(test.input.get_blob(), &kzg_settings);
 
             if res.is_ok() {
-                let expectedCommitment = get_bytes48(test.join("commitment.txt"));
+                assert_eq!(res.unwrap().bytes, test.get_output().unwrap().bytes)
-                assert_eq!(res.unwrap().bytes, expectedCommitment.bytes)
             } else {
-                assert!(!test.join("commitment.txt").exists());
+                assert!(test.get_output().is_none())
             }
         }
     }
@@ -500,19 +484,22 @@ mod tests {
         assert!(trusted_setup_file.exists());
         let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
 
-        let tests = fs::read_dir(COMPUTE_KZG_PROOF_TESTS)
+        let tests = glob::glob(COMPUTE_KZG_PROOF_TESTS)
             .unwrap()
-            .map(|t| t.unwrap().path());
+            .map(|t| t.unwrap());
-        for test in tests {
+        for test_file in tests {
-            let blob = get_blob(test.join("blob.txt"));
+            let yaml_data = fs::read_to_string(test_file).unwrap();
-            let input_point = get_bytes32(test.join("input_point.txt"));
+            let test: compute_kzg_proof::Test = serde_yaml::from_str(&yaml_data).unwrap();
-            let res = KZGProof::compute_kzg_proof(blob, input_point, &kzg_settings);
+            let res = KZGProof::compute_kzg_proof(
+                test.input.get_blob(),
+                test.input.get_z(),
+                &kzg_settings,
+            );
 
             if res.is_ok() {
-                let expected_proof = get_bytes48(test.join("proof.txt"));
+                assert_eq!(res.unwrap().bytes, test.get_output().unwrap().bytes)
-                assert_eq!(res.unwrap().bytes, expected_proof.bytes)
             } else {
-                assert!(!test.join("proof.txt").exists());
+                assert!(test.get_output().is_none())
             }
         }
     }
@@ -524,18 +511,18 @@ mod tests {
         assert!(trusted_setup_file.exists());
         let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
 
-        let tests = fs::read_dir(COMPUTE_BLOB_KZG_PROOF_TESTS)
+        let tests = glob::glob(COMPUTE_BLOB_KZG_PROOF_TESTS)
             .unwrap()
-            .map(|t| t.unwrap().path());
+            .map(|t| t.unwrap());
-        for test in tests {
+        for test_file in tests {
-            let blob = get_blob(test.join("blob.txt"));
+            let yaml_data = fs::read_to_string(test_file).unwrap();
-            let res = KZGProof::compute_blob_kzg_proof(blob, &kzg_settings);
+            let test: compute_blob_kzg_proof::Test = serde_yaml::from_str(&yaml_data).unwrap();
+            let res = KZGProof::compute_blob_kzg_proof(test.input.get_blob(), &kzg_settings);
 
             if res.is_ok() {
-                let expected_proof = get_bytes48(test.join("proof.txt"));
+                assert_eq!(res.unwrap().bytes, test.get_output().unwrap().bytes)
-                assert_eq!(res.unwrap().bytes, expected_proof.bytes)
             } else {
-                assert!(!test.join("proof.txt").exists());
+                assert!(test.get_output().is_none())
             }
         }
     }
@@ -547,27 +534,24 @@ mod tests {
         assert!(trusted_setup_file.exists());
         let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
 
-        let tests = fs::read_dir(VERIFY_KZG_PROOF_TESTS)
+        let tests = glob::glob(VERIFY_KZG_PROOF_TESTS)
             .unwrap()
-            .map(|t| t.unwrap().path());
+            .map(|t| t.unwrap());
-        for test in tests {
+        for test_file in tests {
-            let commitment = get_bytes48(test.join("commitment.txt"));
+            let yaml_data = fs::read_to_string(test_file).unwrap();
-            let input_point = get_bytes32(test.join("input_point.txt"));
+            let test: verify_kzg_proof::Test = serde_yaml::from_str(&yaml_data).unwrap();
-            let claimed_value = get_bytes32(test.join("claimed_value.txt"));
-            let proof = get_bytes48(test.join("proof.txt"));
             let res = KZGProof::verify_kzg_proof(
-                commitment,
+                test.input.get_commitment(),
-                input_point,
+                test.input.get_z(),
-                claimed_value,
+                test.input.get_y(),
-                proof,
+                test.input.get_proof(),
                 &kzg_settings,
             );
 
             if res.is_ok() {
-                let expected_ok = get_boolean(test.join("ok.txt"));
+                assert_eq!(res.unwrap(), test.get_output().unwrap())
-                assert_eq!(res.unwrap(), expected_ok)
             } else {
-                assert!(!test.join("ok.txt").exists());
+                assert!(test.get_output().is_none())
             }
         }
     }
@@ -579,20 +563,23 @@ mod tests {
         assert!(trusted_setup_file.exists());
         let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
 
-        let tests = fs::read_dir(VERIFY_BLOB_KZG_PROOF_TESTS)
+        let tests = glob::glob(VERIFY_BLOB_KZG_PROOF_TESTS)
             .unwrap()
-            .map(|t| t.unwrap().path());
+            .map(|t| t.unwrap());
-        for test in tests {
+        for test_file in tests {
-            let blob = get_blob(test.join("blob.txt"));
+            let yaml_data = fs::read_to_string(test_file).unwrap();
-            let commitment = get_bytes48(test.join("commitment.txt"));
+            let test: verify_blob_kzg_proof::Test = serde_yaml::from_str(&yaml_data).unwrap();
-            let proof = get_bytes48(test.join("proof.txt"));
+            let res = KZGProof::verify_blob_kzg_proof(
-            let res = KZGProof::verify_blob_kzg_proof(blob, commitment, proof, &kzg_settings);
+                test.input.get_blob(),
+                test.input.get_commitment(),
+                test.input.get_proof(),
+                &kzg_settings,
+            );
 
             if res.is_ok() {
-                let expected_ok = get_boolean(test.join("ok.txt"));
+                assert_eq!(res.unwrap(), test.get_output().unwrap())
-                assert_eq!(res.unwrap(), expected_ok)
             } else {
-                assert!(!test.join("ok.txt").exists());
+                assert!(test.get_output().is_none())
             }
         }
     }
@@ -604,52 +591,23 @@ mod tests {
         assert!(trusted_setup_file.exists());
         let kzg_settings = KZGSettings::load_trusted_setup_file(trusted_setup_file).unwrap();
 
-        let tests = fs::read_dir(VERIFY_BLOB_KZG_PROOF_BATCH_TESTS)
+        let tests = glob::glob(VERIFY_BLOB_KZG_PROOF_BATCH_TESTS)
             .unwrap()
-            .map(|t| t.unwrap().path());
+            .map(|t| t.unwrap());
-        for test in tests {
+        for test_file in tests {
-            let mut blobFiles = fs::read_dir(test.join("blobs"))
+            let yaml_data = fs::read_to_string(test_file).unwrap();
-                .unwrap()
+            let test: verify_blob_kzg_proof_batch::Test = serde_yaml::from_str(&yaml_data).unwrap();
-                .map(|entry| entry.unwrap())
-                .collect::<Vec<_>>();
-            blobFiles.sort_by_key(|dir| dir.path());
-            let blobs = blobFiles
-                .iter()
-                .map(|blobFile| get_blob(blobFile.path()))
-                .collect::<Vec<Blob>>();
-
-            let mut commitmentFiles = fs::read_dir(test.join("commitments"))
-                .unwrap()
-                .map(|entry| entry.unwrap())
-                .collect::<Vec<_>>();
-            commitmentFiles.sort_by_key(|dir| dir.path());
-            let commitments = commitmentFiles
-                .iter()
-                .map(|commitmentFile| get_bytes48(commitmentFile.path()))
-                .collect::<Vec<Bytes48>>();
-
-            let mut proof_files = fs::read_dir(test.join("proofs"))
-                .unwrap()
-                .map(|entry| entry.unwrap())
-                .collect::<Vec<_>>();
-            proof_files.sort_by_key(|dir| dir.path());
-            let proofs = proof_files
-                .iter()
-                .map(|proof_file| get_bytes48(proof_file.path()))
-                .collect::<Vec<Bytes48>>();
-
             let res = KZGProof::verify_blob_kzg_proof_batch(
-                blobs.as_slice(),
+                test.input.get_blobs().as_slice(),
-                commitments.as_slice(),
+                test.input.get_commitments().as_slice(),
-                proofs.as_slice(),
+                test.input.get_proofs().as_slice(),
                 &kzg_settings,
             );
 
             if res.is_ok() {
-                let expectedOk = get_boolean(test.join("ok.txt"));
+                assert_eq!(res.unwrap(), test.get_output().unwrap())
-                assert_eq!(res.unwrap(), expectedOk)
             } else {
-                assert!(!test.join("ok.txt").exists());
+                assert!(test.get_output().is_none())
             }
         }
     }

bindings/rust/src/test_formats/blob_to_kzg_commitment_test.rs

@@ -0,0 +1,37 @@
+#![allow(dead_code)]
+
+use crate::Blob;
+use crate::Bytes48;
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+pub struct Input<'a> {
+    blob: &'a str,
+}
+
+impl Input<'_> {
+    pub fn get_blob(&self) -> Blob {
+        Blob::from_bytes(&hex::decode(self.blob.replace("0x", "")).unwrap()).unwrap()
+    }
+}
+
+#[derive(Deserialize)]
+pub struct Test<'a> {
+    #[serde(borrow)]
+    pub input: Input<'a>,
+    #[serde(borrow)]
+    output: Option<&'a str>,
+}
+
+impl Test<'_> {
+    pub fn get_output(&self) -> Option<Bytes48> {
+        if self.output.is_some() {
+            Some(
+                Bytes48::from_bytes(&hex::decode(self.output.unwrap().replace("0x", "")).unwrap())
+                    .unwrap(),
+            )
+        } else {
+            None
+        }
+    }
+}

bindings/rust/src/test_formats/compute_blob_kzg_proof.rs

@@ -0,0 +1,37 @@
+#![allow(dead_code)]
+
+use crate::Blob;
+use crate::Bytes48;
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+pub struct Input<'a> {
+    blob: &'a str,
+}
+
+impl Input<'_> {
+    pub fn get_blob(&self) -> Blob {
+        Blob::from_bytes(&hex::decode(self.blob.replace("0x", "")).unwrap()).unwrap()
+    }
+}
+
+#[derive(Deserialize)]
+pub struct Test<'a> {
+    #[serde(borrow)]
+    pub input: Input<'a>,
+    #[serde(borrow)]
+    output: Option<&'a str>,
+}
+
+impl Test<'_> {
+    pub fn get_output(&self) -> Option<Bytes48> {
+        if self.output.is_some() {
+            Some(
+                Bytes48::from_bytes(&hex::decode(self.output.unwrap().replace("0x", "")).unwrap())
+                    .unwrap(),
+            )
+        } else {
+            None
+        }
+    }
+}

bindings/rust/src/test_formats/compute_kzg_proof.rs

@@ -0,0 +1,43 @@
+#![allow(dead_code)]
+
+use crate::Blob;
+use crate::Bytes32;
+use crate::Bytes48;
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+pub struct Input<'a> {
+    blob: &'a str,
+    z: &'a str,
+}
+
+impl Input<'_> {
+    pub fn get_blob(&self) -> Blob {
+        Blob::from_bytes(&hex::decode(self.blob.replace("0x", "")).unwrap()).unwrap()
+    }
+
+    pub fn get_z(&self) -> Bytes32 {
+        Bytes32::from_bytes(&hex::decode(self.z.replace("0x", "")).unwrap()).unwrap()
+    }
+}
+
+#[derive(Deserialize)]
+pub struct Test<'a> {
+    #[serde(borrow)]
+    pub input: Input<'a>,
+    #[serde(borrow)]
+    output: Option<&'a str>,
+}
+
+impl Test<'_> {
+    pub fn get_output(&self) -> Option<Bytes48> {
+        if self.output.is_some() {
+            Some(
+                Bytes48::from_bytes(&hex::decode(self.output.unwrap().replace("0x", "")).unwrap())
+                    .unwrap(),
+            )
+        } else {
+            None
+        }
+    }
+}

bindings/rust/src/test_formats/mod.rs

@@ -0,0 +1,6 @@
+pub mod blob_to_kzg_commitment_test;
+pub mod compute_blob_kzg_proof;
+pub mod compute_kzg_proof;
+pub mod verify_blob_kzg_proof;
+pub mod verify_blob_kzg_proof_batch;
+pub mod verify_kzg_proof;

bindings/rust/src/test_formats/verify_blob_kzg_proof.rs

@@ -0,0 +1,39 @@
+#![allow(dead_code)]
+
+use crate::Blob;
+use crate::Bytes48;
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+pub struct Input<'a> {
+    blob: &'a str,
+    commitment: &'a str,
+    proof: &'a str,
+}
+
+impl Input<'_> {
+    pub fn get_blob(&self) -> Blob {
+        Blob::from_bytes(&hex::decode(self.blob.replace("0x", "")).unwrap()).unwrap()
+    }
+
+    pub fn get_commitment(&self) -> Bytes48 {
+        Bytes48::from_bytes(&hex::decode(self.commitment.replace("0x", "")).unwrap()).unwrap()
+    }
+
+    pub fn get_proof(&self) -> Bytes48 {
+        Bytes48::from_bytes(&hex::decode(self.proof.replace("0x", "")).unwrap()).unwrap()
+    }
+}
+
+#[derive(Deserialize)]
+pub struct Test<'a> {
+    #[serde(borrow)]
+    pub input: Input<'a>,
+    output: Option<bool>,
+}
+
+impl Test<'_> {
+    pub fn get_output(&self) -> Option<bool> {
+        self.output
+    }
+}

bindings/rust/src/test_formats/verify_blob_kzg_proof_batch.rs

@@ -0,0 +1,50 @@
+#![allow(dead_code)]
+
+use crate::Blob;
+use crate::Bytes48;
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+pub struct Input {
+    blobs: Vec<String>,
+    commitments: Vec<String>,
+    proofs: Vec<String>,
+}
+
+impl Input {
+    pub fn get_blobs(&self) -> Vec<Blob> {
+        self.blobs
+            .iter()
+            .map(|f| hex::decode(f.replace("0x", "")).unwrap())
+            .map(|bytes| Blob::from_bytes(bytes.as_slice()).unwrap())
+            .collect::<Vec<Blob>>()
+    }
+
+    pub fn get_commitments(&self) -> Vec<Bytes48> {
+        self.commitments
+            .iter()
+            .map(|f| hex::decode(f.replace("0x", "")).unwrap())
+            .map(|bytes| Bytes48::from_bytes(bytes.as_slice()).unwrap())
+            .collect::<Vec<Bytes48>>()
+    }
+
+    pub fn get_proofs(&self) -> Vec<Bytes48> {
+        self.proofs
+            .iter()
+            .map(|f| hex::decode(f.replace("0x", "")).unwrap())
+            .map(|bytes| Bytes48::from_bytes(bytes.as_slice()).unwrap())
+            .collect::<Vec<Bytes48>>()
+    }
+}
+
+#[derive(Deserialize)]
+pub struct Test {
+    pub input: Input,
+    output: Option<bool>,
+}
+
+impl Test {
+    pub fn get_output(&self) -> Option<bool> {
+        self.output
+    }
+}

bindings/rust/src/test_formats/verify_kzg_proof.rs

@@ -0,0 +1,44 @@
+#![allow(dead_code)]
+
+use crate::Bytes32;
+use crate::Bytes48;
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+pub struct Input<'a> {
+    commitment: &'a str,
+    z: &'a str,
+    y: &'a str,
+    proof: &'a str,
+}
+
+impl Input<'_> {
+    pub fn get_commitment(&self) -> Bytes48 {
+        Bytes48::from_bytes(&hex::decode(self.commitment.replace("0x", "")).unwrap()).unwrap()
+    }
+
+    pub fn get_z(&self) -> Bytes32 {
+        Bytes32::from_bytes(&hex::decode(self.z.replace("0x", "")).unwrap()).unwrap()
+    }
+
+    pub fn get_y(&self) -> Bytes32 {
+        Bytes32::from_bytes(&hex::decode(self.y.replace("0x", "")).unwrap()).unwrap()
+    }
+
+    pub fn get_proof(&self) -> Bytes48 {
+        Bytes48::from_bytes(&hex::decode(self.proof.replace("0x", "")).unwrap()).unwrap()
+    }
+}
+
+#[derive(Deserialize)]
+pub struct Test<'a> {
+    #[serde(borrow)]
+    pub input: Input<'a>,
+    output: Option<bool>,
+}
+
+impl Test<'_> {
+    pub fn get_output(&self) -> Option<bool> {
+        self.output
+    }
+}