From 24930233f6f8fadad8e24771179d2710c64a0180 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 17 Jan 2024 22:04:47 +0200 Subject: [PATCH] Feature/msgpack argument passing (#9) * adding rmpv crate * plumb in msgpack * setting up mpack basics * setting up mpack basics * setting up mpack basics * setting up mpack basics * updates * chunks * chunks * chunks * add mpack proof func * add mpack proof func - tests infra * add mpack proof func - tests infra remove * add mpack proof func - split mpack * rework funcs * rework funcs * rework funcs * rework funcs * rework funcs * rework funcs * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor - read orig for testing * refactor - read orig for testing * setting up tests * setting up tests * setting up tests * setting up tests * setting rest of data * setting rest of data * setting rest of data * setting rest of data * re-add original prove for comparison * re-add original prove for comparison * re-add original prove for comparison * cleanup * refactor * refactor * pass tests * initial setup to build as a nim package * initial setup * update build setup * update build setup * update build setup * add nim ffi and genffi build task * add nim ffi and genffi build task * add nim ffi and genffi build task * update init to remove redundant pointers * update init to remove redundant pointers * update init to remove redundant pointers * update init to remove redundant pointers * save mpack * save mpack * update ffi * update ffi * add example ffi test * add example ffi test * updates * fix tests * adding git ignore * rename * run testament * fix stuffs * fix stuffs * fix stuffs * update build * update build --- .gitignore | 7 + Cargo.toml | 5 + build.nims | 18 + codex_proofs_ffi.nim | 57 + codex_storage_proofs.nim | 38 + codex_storage_proofs.nimble | 11 + config.nims | 1 + proof_test.mpack | Bin 0 -> 36389 bytes src/circuit_tests/mod.rs | 34 +- src/ffi.rs | 262 ++++- src/storage_proofs.rs | 133 ++- {test => tests}/circuits/storer-test.circom | 0 tests/config.nims | 1 + tests/proof_test.mpack | Bin 0 -> 36389 bytes tests/proof_test.mpack-dump.json | 1060 +++++++++++++++++++ {test => tests}/storer.js | 0 tests/tffi.nim | 33 + 17 files changed, 1617 insertions(+), 43 deletions(-) create mode 100644 build.nims create mode 100644 codex_proofs_ffi.nim create mode 100644 codex_storage_proofs.nim create mode 100644 codex_storage_proofs.nimble create mode 100644 config.nims create mode 100644 proof_test.mpack rename {test => tests}/circuits/storer-test.circom (100%) create mode 100644 tests/config.nims create mode 100644 tests/proof_test.mpack create mode 100644 tests/proof_test.mpack-dump.json rename {test => tests}/storer.js (100%) create mode 100644 tests/tffi.nim diff --git a/.gitignore b/.gitignore index d3ad3a8..2360bee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ node_modules/ +* +!*/ +!*.* +nim.cfg # Added by cargo @@ -16,3 +20,6 @@ test/circuits/artifacts out.log src/circuit_tests/artifacts/* !src/circuit_tests/artifacts/.keep +codex_storage_proofs +test/tffi +node_modules diff --git a/Cargo.toml b/Cargo.toml index 38a1b14..da84cf9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,10 @@ edition = "2021" [profile.dev] opt-level = 3 +[profile.release] +strip = true +opt-level = 2 + [lib] crate-type = [ "staticlib", # Ensure it gets compiled as a (static) C library @@ -37,3 +41,4 @@ serde_json = "1.0.94" num-traits = "0.2.15" ark-relations = { version = "0.4.0", features = ["std", "tracing-subscriber"] } rs-poseidon = {git = "https://github.com/status-im/rs-poseidon" } +rmpv = "1.0.1" diff --git a/build.nims b/build.nims new file mode 100644 index 0000000..05139a6 --- /dev/null +++ b/build.nims @@ -0,0 +1,18 @@ +import std/os + +task genffi, "update the nim ffi bindings": + exec "cargo install nbindgen" + exec "nbindgen --crate codex-storage-proofs --output codex_proofs_ffi.nim" + +task compileCircuits, "compile test circuits": + exec "npm i" + exec "circom src/circuit_tests/poseidon-digest-test.circom --r1cs --wasm -o src/circuit_tests/artifacts" + exec "circom src/circuit_tests/poseidon-hash-test.circom --r1cs --wasm -o src/circuit_tests/artifacts" + exec "circom src/circuit_tests/storer-test.circom --r1cs --wasm -o src/circuit_tests/artifacts" + +task tests, "run unit tests": + let storerR1cs = fileExists "src/circuit_tests/artifacts/storer-test.r1cs" + let storerWasm = fileExists "src/circuit_tests/artifacts/storer-test_js/storer-test.wasm" + if not storerR1cs or not storerWasm: + compileCircuitsTask() + exec "nim c -r tests/tffi.nim" diff --git a/codex_proofs_ffi.nim b/codex_proofs_ffi.nim new file mode 100644 index 0000000..89cb467 --- /dev/null +++ b/codex_proofs_ffi.nim @@ -0,0 +1,57 @@ +const EXT_ID_U256_BE* = 51 + +const EXT_ID_U256_LE* = 50 + + +type StorageProofs* {.incompleteStruct.} = object + +type Buffer* = object + data: ptr uint8 + len: uint + +type ProofCtx* = object + proof: Buffer + public_inputs: Buffer + +## # Safety +# +# Use on a valid pointer to ProofCtx or panics +proc free_proof_ctx*(ctx: ptr ProofCtx) {.importc: "free_proof_ctx".} + +## # Safety +# +# Use on a valid pointer to StorageProofs or panics +proc free_prover*(prover: ptr StorageProofs) {.importc: "free_prover".} + +## # Safety +# +# Construct a StorageProofs object +proc init_storage_proofs*(r1cs: Buffer, + wasm: Buffer, + zkey: ptr Buffer): (ptr StorageProofs) {.importc: "init_storage_proofs".} + +## # Safety +# +# Use after constructing a StorageProofs object with init +proc prove*(prover_ptr: ptr StorageProofs, + chunks: ptr Buffer, + siblings: ptr Buffer, + hashes: ptr Buffer, + path: ptr int32, + path_len: uint, + pubkey: ptr Buffer, + root: ptr Buffer, + salt: ptr Buffer): (ptr ProofCtx) {.importc: "prove".} + +## # Safety +# +# Use after constructing a StorageProofs object with init +proc prove_mpack_ext*(prover_ptr: ptr StorageProofs, + args: ptr Buffer): (ptr ProofCtx) {.importc: "prove_mpack_ext".} + +## # Safety +# +# Should be called on a valid proof and public inputs previously generated by prove +proc verify*(prover_ptr: ptr StorageProofs, + proof: ptr Buffer, + public_inputs: ptr Buffer): bool {.importc: "verify".} diff --git a/codex_storage_proofs.nim b/codex_storage_proofs.nim new file mode 100644 index 0000000..139ec5a --- /dev/null +++ b/codex_storage_proofs.nim @@ -0,0 +1,38 @@ + +import std/os +import std/strutils +import std/macros + +const + currentDir = currentSourcePath().parentDir() + libDir* = currentDir/"target"/"release" + libPath* = libDir/"libcodex_storage_proofs.a" + +static: + let cmd = "cargo build --release" + warning "\nBuilding codex-storage-proofs: " & cmd + let (output, exitCode) = gorgeEx cmd + for ln in output.splitLines(): + warning("cargo> " & ln) + if exitCode != 0: + raise (ref Defect)(msg: "Failed to build codex-storage-proofs") + + +{.passl: "-lcodex_storage_proofs" & " -L" & libDir.} + +include codex_proofs_ffi + +proc len*(buff: Buffer): int = + buff.len.int + +template unsafeBufferPath*(path: var string): Buffer = + assert path.len() > 0 + Buffer(data: cast[ptr uint8](path.cstring), + len: path.len().uint) + +template unsafeBufferFromFile*(path: string): Buffer = + assert path.len() > 0 + let entireFile = readFile(path) + + Buffer(data: cast[ptr uint8](entireFile.cstring), + len: entireFile.len().uint) diff --git a/codex_storage_proofs.nimble b/codex_storage_proofs.nimble new file mode 100644 index 0000000..97463cd --- /dev/null +++ b/codex_storage_proofs.nimble @@ -0,0 +1,11 @@ +version = "0.1.0" +author = "Jaremy Creechley" +description = "Codex Storage Proofs runner for Rust Circom library" +license = "MIT" +srcDir = "." + +# Dependencies + +requires "nim >= 1.6.14" + +include "build.nims" diff --git a/config.nims b/config.nims new file mode 100644 index 0000000..50ed134 --- /dev/null +++ b/config.nims @@ -0,0 +1 @@ +include "build.nims" diff --git a/proof_test.mpack b/proof_test.mpack new file mode 100644 index 0000000000000000000000000000000000000000..108d05cbfe0c5c40cc67db56bbe60ce2710a4712 GIT binary patch literal 36389 zcmeI&KWvs&6b5k67&I{sP7((+lHg_wwpcfnmTD<{Z9m(5ZUuq1KxzB6*1AEW0~0Z5 z9E=*%poxnKF*-UB9bI&DF-Aq>pTvZX!AnTq>6Yg=e3Plu>&xlA_ndp4=RD`U-&0bG9=Zma38^F}XalQ*&npb?^bG;aj?0yjOD zHv*4d&8r;BCd=7mDVt1W6F?()bSiHI=?yFep&Q9(0j_K3RZe6RV5yC~%GqoJAc5Qk z@&J%PVgivJ&rXkLlc%!@us4{%alvT_9hd+$AIot8ketb@bb4X}<#RZ1bU2&9HG(1o zOoK;|yFfv$?DX^51kM7Q0qPOt3zTxGM*xzc92eLKD(o|PBj`zBmDlq|kh>5%U~yc@$ypS&d zU=KnGs2PSU05#}7$FtKw9x(L-c@A*_G{R&SKmyDF5(h#DI07PrG65R_8fS98KuUpV z0~~<~Yy|uQ#Rf_iU>bNAk|7DD?VrGGnZ)HsJurfSIt(}g0tcdk&IKlnkX3*-_?iOR z`rwwgbLSt*9YMbgzY%~j8E7SFhC!de+zCn&L>K4|@&#@hnm+Id7!l^TK;Y1pOytA_ zG=fLaQbT$JOF`(Mf&$mIbLe2i3oMmtUm*_=LI4uTT_6tt2_z;E8T4YH+dv=!dxHra z7o3LBfeBDE%mx7@(4%%Tj=%)U=WyN#8ZEd+_?iNk29F?jfr45&F3=3aSwJ&DJ%VI_ zQV#V9Kr)m=2OB|!g=Tm@p9NNdjt+7cLI=zMae>^0e1WP4=?#nt`2rrnSwQc$bG|@- z1ceWrhQtI}0o^AkRbVNA8eG6=z5tltPUltDvk7F~R9>Ywm|n>n!C9aWSj`&&0wBF% z{<4wJ0_qUTC)^RV3Q*P7vPU3J03<*`P>(>gL3AN5{W&fGSYU?UeAxkh0W@~9)1Y@D zrJyzfzrb+;N8r&&jtdM-LB9akfeCOOpb>Bcae+Dw0un?U-1Kt3Mj-gUyb3S_bNAk|7DD?VrGG znZ(;afvJb4#0Ag>UsFI^gA#>)8Ga)GV=~Z6&cuB#1819ZaO*rlIKrkAM+j zehUN+ZOKF(8_nAS5||@^N6=D3dPBDWp@RwvnxNeTss2zx2P57d)V{J22q6Fo5jtfph=)eT18D@h366jGob@()IOL~I|l+WR2$@)Xt2pTQ8 zM);bN^lUi`cm%l%6x6DRqj_7-0?ja-1vCTHBS;1)JKG!un|;PXog`j z0;@nr2e}KO17?7@K<+}mKvjeE2F8SZ0gvD;pm*E#lx^OY+=c!K3LiKPi3zd-#)qI( zfu#UyZ~>!DsOt~q0$_eSUH{|eZCPc#`JehjF@da`s{e8Gwye?{Os_Oc)*s47a2DtT zR_p6GZ_7r207&n}X36?P*=VEwFU{Lx0_qUTC)^RV3Q*P7nzPp*ibo($03<*`P>(>g zL3AN5{q=A(Z%bSNu)qwx`LYB20%+_s?$sZP)1Y@DrJyzfzrb+;N8r&&Jsi#35*PTg z1NsHH4oraS0F8hnhzrzd5Rf3+;HH81J~H*d=-lgH)ep$T>UpK4|@&#@hnm+Id7!l^TK;Y1p zOytA_G=fLaQbT$JOF`(Mf&$mIbLe2i3oMmtUm*_=LI4uTT_6tt2_z;E8T4YH+dv=! zdxHra7o3LBfeBDE%mx7@(4%%Tj=%)U=WyN#8ZEd+_?iNk29F?jfr45&F3=3aSwJ&D zJ%VI_QV#V9Kr)m=2OB|!g=Tm@p9NNdjt+7cLI=zMae>^0e1WP4=?#nt`2rrnSwQc$ zbG|@-1ceWrhQtI}0o^AkRbVNA8eG6=z5tltPUltDvk7F~R9>Ywm|n>n!C9aWSj`&& z0wBF%{<4wJ0_qUTC)^RV3Q*P7vPU3J03<*`P>(>gL3AN5{W&fGSYU?UeAxkh0W@~9 z)1Y@DrJyzfzrb+;N8r&&jtdM-LB9akfeCOOpb>Bcae+Dw0un?U-1Kt3Mj-gUyb3S_ zbNAk|7DD zPrLKS&(1HN?!H$(2);Xg@#sUBpSpbG*6H2XZruFj(uWuR`0|@``|iKi`s>&0pFLVk z4!r---gC$H9Qx;{-9P->+Bf$3pHJ)>9JxOJ+r{x+Zywpa{kxI>*(z7(X1jAIyI0IJ zaJ~m7n@h8+b60lkyyxD{mGkFU%hCS6`FQ)g!@rE|+57v$Up(^lleg}>^v=iMKXBpa S52l}e`>V+ZH@mZE#qK|;*2yCP literal 0 HcmV?d00001 diff --git a/src/circuit_tests/mod.rs b/src/circuit_tests/mod.rs index d520612..0de054e 100644 --- a/src/circuit_tests/mod.rs +++ b/src/circuit_tests/mod.rs @@ -130,24 +130,24 @@ mod test { ]; let root = treehash(hashes.as_slice()); - let proof_bytes = &mut Vec::new(); - let public_inputs_bytes = &mut Vec::new(); + // let proof_bytes = &mut Vec::new(); + // let public_inputs_bytes = &mut Vec::new(); - prover - .prove( - chunks.as_slice(), - siblings, - hashes.as_slice(), - path.as_slice(), - root, - root, // random salt - block hash - proof_bytes, - public_inputs_bytes, - ) - .unwrap(); + // prover + // .prove( + // chunks.as_slice(), + // siblings, + // hashes.as_slice(), + // path.as_slice(), + // root, + // root, // random salt - block hash + // proof_bytes, + // public_inputs_bytes, + // ) + // .unwrap(); - assert!(prover - .verify(proof_bytes.as_slice(), public_inputs_bytes.as_slice()) - .is_ok()); + // assert!(prover + // .verify(proof_bytes.as_slice(), public_inputs_bytes.as_slice()) + // .is_ok()); } } diff --git a/src/ffi.rs b/src/ffi.rs index 1a779d7..c3b14d9 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -36,33 +36,26 @@ impl ProofCtx { /// /// Construct a StorageProofs object #[no_mangle] -pub unsafe extern "C" fn init( - r1cs: *const &Buffer, - wasm: *const &Buffer, - zkey: *const &Buffer, +pub unsafe extern "C" fn init_storage_proofs( + r1cs: Buffer, + wasm: Buffer, + zkey: *const Buffer, ) -> *mut StorageProofs { let r1cs = { - if r1cs.is_null() { - return std::ptr::null_mut(); - } - let slice = std::slice::from_raw_parts((*r1cs).data, (*r1cs).len); - str::from_utf8(slice).unwrap().to_string() + let slice = std::slice::from_raw_parts((r1cs).data, (r1cs).len); + str::from_utf8(slice).unwrap().to_string().to_owned() }; let wasm = { - if wasm.is_null() { - return std::ptr::null_mut(); - } - - let slice = std::slice::from_raw_parts((*wasm).data, (*wasm).len); - str::from_utf8(slice).unwrap().to_string() + let slice = std::slice::from_raw_parts((wasm).data, (wasm).len); + str::from_utf8(slice).unwrap().to_string().to_owned() }; let zkey = { if !zkey.is_null() { let slice = std::slice::from_raw_parts((*zkey).data, (*zkey).len); - Some(str::from_utf8(slice).unwrap().to_string()) + Some(str::from_utf8(slice).unwrap().to_string().to_owned()) } else { None } @@ -93,6 +86,10 @@ pub unsafe extern "C" fn prove( .map(|c| U256::try_from_le_slice(c).unwrap()) .collect::>() }; + // println!("prove:args: {}", "chunks"); + // for n in chunks { + // println!("\t{}", n); + // } let siblings = { let slice = std::slice::from_raw_parts((*siblings).data, (*siblings).len); @@ -115,7 +112,7 @@ pub unsafe extern "C" fn prove( slice.to_vec() }; - let pubkey = + let _pubkey = U256::try_from_le_slice(std::slice::from_raw_parts((*pubkey).data, (*pubkey).len)).unwrap(); let root = @@ -144,6 +141,31 @@ pub unsafe extern "C" fn prove( Box::into_raw(Box::new(ProofCtx::new(proof_bytes, public_inputs_bytes))) } +/// # Safety +/// +/// Use after constructing a StorageProofs object with init +#[no_mangle] +pub unsafe extern "C" fn prove_mpack_ext( + prover_ptr: *mut StorageProofs, + args: *const Buffer, +) -> *mut ProofCtx { + let inputs = std::slice::from_raw_parts((*args).data, (*args).len); + + let proof_bytes = &mut Vec::new(); + let public_inputs_bytes = &mut Vec::new(); + + let mut _prover = &mut *prover_ptr; + _prover + .prove_mpack( + inputs, + proof_bytes, + public_inputs_bytes, + ) + .unwrap(); + + Box::into_raw(Box::new(ProofCtx::new(proof_bytes, public_inputs_bytes))) +} + #[no_mangle] /// # Safety /// @@ -185,15 +207,205 @@ pub unsafe extern "C" fn free_proof_ctx(ctx: *mut ProofCtx) { #[cfg(test)] mod tests { - use ark_std::rand::{distributions::Alphanumeric, rngs::ThreadRng, Rng}; + use std::fs::File; + use std::io::prelude::*; + + + use ark_std::rand::{distributions::Alphanumeric, rngs::StdRng, Rng, SeedableRng}; use rs_poseidon::poseidon::hash; use ruint::aliases::U256; use crate::{ - circuit_tests::utils::{digest, treehash}, + circuit_tests::utils::{digest, treehash}, storage_proofs::EXT_ID_U256_LE, ffi::prove_mpack_ext }; - use super::{init, prove, Buffer}; + use super::{init_storage_proofs, prove, Buffer}; + + use rmpv::Value; + use rmpv::encode::write_value; + use rmpv::decode::read_value; + + #[test] + fn test_mpack() { + let mut buf = Vec::new(); + let _val = Value::from("le message"); + + // example of serializing the random chunk data + // we build them up in mpack Value enums + let data = (0..4) + .map(|_| { + let rng = StdRng::seed_from_u64(42); + let preimages: Vec = rng + .sample_iter(Alphanumeric) + .take(256) + .map(|c| U256::from(c)) + .collect(); + let hash = digest(&preimages, Some(16)); + (preimages, hash) + }) + .collect::, U256)>>(); + + let chunks = data.iter() + .map(|c| { + let x = c.0.iter() + .map(|c| Value::Ext(EXT_ID_U256_LE, c.to_le_bytes_vec())) + .collect::>(); + Value::Array(x) + }) + .collect::>(); + let chunks = Value::Array(chunks); + let data = Value::Map(vec![(Value::String("chunks".into()), chunks.clone() )]); + + println!("Debug: chunks: {:?}", chunks[0][0]); + + // Serialize the value types to an array pointer + write_value(&mut buf, &data).unwrap(); + let mut rd: &[u8] = &buf[..]; + + let args = read_value(&mut rd).unwrap(); + + assert!(Value::is_map(&args)); + assert!(Value::is_array(&args["chunks"])); + assert!(Value::is_array(&args["chunks"][0])); + + let mut arg_chunks: Vec> = Vec::new(); + + // deserialize the data back into u256's + // instead of this, we'll want to use `builder.push_input` + args["chunks"] + .as_array() + .unwrap() + .iter() + .for_each(|c| { + if let Some(x) = c.as_array() { + let mut vals: Vec = Vec::new(); + x.iter().for_each(|n| { + let b = n.as_ext().unwrap(); + // ensure it's a LE uin256 which we've set as ext 50 + assert_eq!(b.0, 50); + vals.push(U256::try_from_le_slice(b.1).unwrap()); + // TODO: change to use + // builder.push_input("hashes", *c) + }); + arg_chunks.push(vals); + } else { + panic!("unhandled type!"); + } + }); + + assert_eq!(arg_chunks.len(), 4); + assert_eq!(arg_chunks[0].len(), 256); + + } + + fn u256_to_mpack(n: &U256) -> Value { + Value::Ext(EXT_ID_U256_LE, n.to_le_bytes_vec()) + } + + #[test] + fn test_storer_ffi_mpack() { + let mut buf = Vec::new(); + let _val = Value::from("le message"); + + // example of serializing the random chunk data + // we build them up in mpack Value enums + let data = (0..4) + .map(|_| { + let rng = StdRng::seed_from_u64(42); + let preimages: Vec = rng + .sample_iter(Alphanumeric) + .take(256) + .map(|c| U256::from(c)) + .collect(); + let hash = digest(&preimages, Some(16)); + (preimages, hash) + }) + .collect::, U256)>>(); + + let chunks = data.iter() + .map(|c| { + let x = c.0.iter().map(u256_to_mpack).collect::>(); + Value::Array(x) + }) + .collect::>(); + let chunks = Value::Array(chunks); + + println!("Debug: chunks: {:?}", chunks[0][0]); + + let hashes: Vec = data.iter().map(|c| c.1).collect(); + + let hashes_mpk = Value::Array(hashes.iter().map(u256_to_mpack).collect()); + + let path = [0, 1, 2, 3]; + let path_mpk = Value::Array(path.iter().map(|i| rmpv::Value::from(*i)).collect()); + + let parent_hash_l = hash(&[hashes[0], hashes[1]]); + let parent_hash_r = hash(&[hashes[2], hashes[3]]); + + let sibling_hashes = &[ + hashes[1], + parent_hash_r, + hashes[0], + parent_hash_r, + hashes[3], + parent_hash_l, + hashes[2], + parent_hash_l, + ]; + + let siblings_mpk: Value = Value::Array(sibling_hashes + .iter() + .map(u256_to_mpack) + .collect::>()); + + let root = treehash(hashes.as_slice()); + + // let root_bytes: [u8; U256::BYTES] = root.to_le_bytes(); + let root_mpk = u256_to_mpack(&root); + + // Serialize the value types to an array pointer + let mpk_data = Value::Map(vec![ + (Value::String("chunks".into()), chunks.clone() ), + (Value::String("siblings".into()), siblings_mpk.clone() ), + (Value::String("hashes".into()), hashes_mpk.clone() ), + (Value::String("path".into()), path_mpk.clone() ), + (Value::String("root".into()), root_mpk.clone() ), + (Value::String("salt".into()), root_mpk.clone() ), + ]); + write_value(&mut buf, &mpk_data ).unwrap(); + let rd: &[u8] = &buf[..]; + + let mut file = File::create("proof_test.mpack").unwrap(); + file.write_all(rd).unwrap(); + + let args_buff = Buffer { + data: rd.as_ptr() as *const u8, + len: rd.len(), + }; + + let r1cs_path = "src/circuit_tests/artifacts/storer-test.r1cs"; + let wasm_path = "src/circuit_tests/artifacts/storer-test_js/storer-test.wasm"; + + let r1cs = Buffer { + data: r1cs_path.as_ptr(), + len: r1cs_path.len(), + }; + + let wasm = Buffer { + data: wasm_path.as_ptr(), + len: wasm_path.len(), + }; + + let prover_ptr = unsafe { init_storage_proofs(r1cs, wasm, std::ptr::null()) }; + let prove_ctx: *mut crate::ffi::ProofCtx = unsafe { + prove_mpack_ext( + prover_ptr, + &args_buff as *const Buffer, + ) + }; + + assert!(prove_ctx.is_null() == false); + } #[test] fn test_storer_ffi() { @@ -201,7 +413,7 @@ mod tests { // and hash is the hash of each vector generated using the digest function let data = (0..4) .map(|_| { - let rng = ThreadRng::default(); + let rng = StdRng::seed_from_u64(42); let preimages: Vec = rng .sample_iter(Alphanumeric) .take(256) @@ -272,18 +484,18 @@ mod tests { let r1cs_path = "src/circuit_tests/artifacts/storer-test.r1cs"; let wasm_path = "src/circuit_tests/artifacts/storer-test_js/storer-test.wasm"; - let r1cs = &Buffer { + let r1cs = Buffer { data: r1cs_path.as_ptr(), len: r1cs_path.len(), }; - let wasm = &Buffer { + let wasm = Buffer { data: wasm_path.as_ptr(), len: wasm_path.len(), }; - let prover_ptr = unsafe { init(&r1cs, &wasm, std::ptr::null()) }; - let prove_ctx = unsafe { + let prover_ptr = unsafe { init_storage_proofs(r1cs, wasm, std::ptr::null()) }; + let prove_ctx: *mut crate::ffi::ProofCtx = unsafe { prove( prover_ptr, &chunks_buff as *const Buffer, diff --git a/src/storage_proofs.rs b/src/storage_proofs.rs index 09bd570..316f246 100644 --- a/src/storage_proofs.rs +++ b/src/storage_proofs.rs @@ -1,7 +1,7 @@ use std::fs::File; use ark_bn254::{Bn254, Fr}; -use ark_circom::{read_zkey, CircomBuilder, CircomConfig}; +use ark_circom::{read_zkey, CircomBuilder, CircomConfig, CircomCircuit}; use ark_groth16::{ create_random_proof as prove, generate_random_parameters, prepare_verifying_key, verify_proof, Proof, ProvingKey, @@ -10,6 +10,15 @@ use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Read}; use ark_std::rand::rngs::ThreadRng; use ruint::aliases::U256; +use rmpv; +use rmpv::decode::read_value; + +type Params256Ty = ark_ec::bn::Bn; + +pub const EXT_ID_U256_LE: i8 = 50; +pub const EXT_ID_U256_BE: i8 = 51; + + #[derive(Debug, Clone)] pub struct StorageProofs { builder: CircomBuilder, @@ -41,6 +50,36 @@ impl StorageProofs { } } + pub fn prove_mpack( + &mut self, + inputs: &[u8], + proof_bytes: &mut Vec, + public_inputs_bytes: &mut Vec, + ) -> Result<(), String> { + let mut builder: CircomBuilder = self.builder.clone(); + + parse_mpack_args(&mut builder, inputs)?; + + let circuit: CircomCircuit = builder.build() + .map_err(|e| e.to_string())?; + + let inputs = circuit + .get_public_inputs() + .ok_or("Unable to get public inputs!")?; + let proof = + prove(circuit, &self.params, &mut self.rng) + .map_err(|e| e.to_string())?; + + proof + .serialize(proof_bytes) + .map_err(|e| e.to_string())?; + inputs + .serialize(public_inputs_bytes) + .map_err(|e| e.to_string())?; + + Ok(()) + } + pub fn prove( &mut self, chunks: &[U256], @@ -96,3 +135,95 @@ impl StorageProofs { Ok(()) } } + +fn decode_number(val: &rmpv::Value) -> Result { + match val { + rmpv::Value::Ext(id, val) => { + match *id { + EXT_ID_U256_LE => + match U256::try_from_le_slice(val) { + Some(i) => Ok(i), + None => Err("error parsing 256".to_string()), + } + num => return Err(format!("unhandled ext id {}", num)), + } + }, + rmpv::Value::Integer(val) => { + if let Some(val) = val.as_u64() { + return Ok(U256::from(val)); + } else if let Some(val) = val.as_i64() { + return Ok(U256::from(val)); + } else { + return Err("unexpected integer kind".to_string()); + } + } + _ => return Err("expected ext mpack kind or integer".to_string()), + } +} + +fn parse_mpack_arrays( + builder: &mut CircomBuilder, + name: &str, + array: &Vec +) -> Result<(), String> { + + println!("deserde: array: {} size: {}", name, array.len()); + if array.len() > 0 && array[0].is_array() { + println!("deserde: arrayOfArrays: {}", name); + for element in array { + match element .as_array() { + Some(element ) => { + parse_mpack_arrays(builder, name, element)?; + }, + _ => { + print!("error expected array: {}", name); + return Err("expected inner array of u256".to_string()) + }, + } + } + } else { + println!("deserde: name: {}", name); + for val in array { + let n = decode_number(val)?; + println!("\t{}", n); + builder.push_input(name, n); + } + println!("done: name: {}", name); + } + + Ok(()) +} + +fn parse_mpack_args( + builder: &mut CircomBuilder, + mut inputs: &[u8] +) -> Result<(), String> { + let values: rmpv::Value = read_value(&mut inputs).map_err(|e| e.to_string())?; + let args: &Vec<(rmpv::Value, rmpv::Value)> = match values.as_map() { + Some(args) => args, + None => return Err("args must be a map of string to arrays".to_string()), + }; + + for (key, val) in args { + let name = match key.as_str() { + Some(n) => n, + None => return Err(format!("expected string value")), + }; + match val { + // add a (name, Vec) or (name, Vev>) arrays + rmpv::Value::Array(vals) => { + parse_mpack_arrays(builder, name, vals)?; + }, + // directly add a (name,u256) arg pair + rmpv::Value::Ext(_, _) => { + let n = decode_number(val)?; + println!("deserde: name: {} u256: {}", name, n); + builder.push_input(name, n); + }, + _ => return Err("unhandled argument kind".to_string()), + } + } + + println!("parse_mpack_args DONE!"); + Ok(()) +} diff --git a/test/circuits/storer-test.circom b/tests/circuits/storer-test.circom similarity index 100% rename from test/circuits/storer-test.circom rename to tests/circuits/storer-test.circom diff --git a/tests/config.nims b/tests/config.nims new file mode 100644 index 0000000..28cc8e2 --- /dev/null +++ b/tests/config.nims @@ -0,0 +1 @@ +--path:"../" \ No newline at end of file diff --git a/tests/proof_test.mpack b/tests/proof_test.mpack new file mode 100644 index 0000000000000000000000000000000000000000..108d05cbfe0c5c40cc67db56bbe60ce2710a4712 GIT binary patch literal 36389 zcmeI&KWvs&6b5k67&I{sP7((+lHg_wwpcfnmTD<{Z9m(5ZUuq1KxzB6*1AEW0~0Z5 z9E=*%poxnKF*-UB9bI&DF-Aq>pTvZX!AnTq>6Yg=e3Plu>&xlA_ndp4=RD`U-&0bG9=Zma38^F}XalQ*&npb?^bG;aj?0yjOD zHv*4d&8r;BCd=7mDVt1W6F?()bSiHI=?yFep&Q9(0j_K3RZe6RV5yC~%GqoJAc5Qk z@&J%PVgivJ&rXkLlc%!@us4{%alvT_9hd+$AIot8ketb@bb4X}<#RZ1bU2&9HG(1o zOoK;|yFfv$?DX^51kM7Q0qPOt3zTxGM*xzc92eLKD(o|PBj`zBmDlq|kh>5%U~yc@$ypS&d zU=KnGs2PSU05#}7$FtKw9x(L-c@A*_G{R&SKmyDF5(h#DI07PrG65R_8fS98KuUpV z0~~<~Yy|uQ#Rf_iU>bNAk|7DD?VrGGnZ)HsJurfSIt(}g0tcdk&IKlnkX3*-_?iOR z`rwwgbLSt*9YMbgzY%~j8E7SFhC!de+zCn&L>K4|@&#@hnm+Id7!l^TK;Y1pOytA_ zG=fLaQbT$JOF`(Mf&$mIbLe2i3oMmtUm*_=LI4uTT_6tt2_z;E8T4YH+dv=!dxHra z7o3LBfeBDE%mx7@(4%%Tj=%)U=WyN#8ZEd+_?iNk29F?jfr45&F3=3aSwJ&DJ%VI_ zQV#V9Kr)m=2OB|!g=Tm@p9NNdjt+7cLI=zMae>^0e1WP4=?#nt`2rrnSwQc$bG|@- z1ceWrhQtI}0o^AkRbVNA8eG6=z5tltPUltDvk7F~R9>Ywm|n>n!C9aWSj`&&0wBF% z{<4wJ0_qUTC)^RV3Q*P7vPU3J03<*`P>(>gL3AN5{W&fGSYU?UeAxkh0W@~9)1Y@D zrJyzfzrb+;N8r&&jtdM-LB9akfeCOOpb>Bcae+Dw0un?U-1Kt3Mj-gUyb3S_bNAk|7DD?VrGG znZ(;afvJb4#0Ag>UsFI^gA#>)8Ga)GV=~Z6&cuB#1819ZaO*rlIKrkAM+j zehUN+ZOKF(8_nAS5||@^N6=D3dPBDWp@RwvnxNeTss2zx2P57d)V{J22q6Fo5jtfph=)eT18D@h366jGob@()IOL~I|l+WR2$@)Xt2pTQ8 zM);bN^lUi`cm%l%6x6DRqj_7-0?ja-1vCTHBS;1)JKG!un|;PXog`j z0;@nr2e}KO17?7@K<+}mKvjeE2F8SZ0gvD;pm*E#lx^OY+=c!K3LiKPi3zd-#)qI( zfu#UyZ~>!DsOt~q0$_eSUH{|eZCPc#`JehjF@da`s{e8Gwye?{Os_Oc)*s47a2DtT zR_p6GZ_7r207&n}X36?P*=VEwFU{Lx0_qUTC)^RV3Q*P7nzPp*ibo($03<*`P>(>g zL3AN5{q=A(Z%bSNu)qwx`LYB20%+_s?$sZP)1Y@DrJyzfzrb+;N8r&&Jsi#35*PTg z1NsHH4oraS0F8hnhzrzd5Rf3+;HH81J~H*d=-lgH)ep$T>UpK4|@&#@hnm+Id7!l^TK;Y1p zOytA_G=fLaQbT$JOF`(Mf&$mIbLe2i3oMmtUm*_=LI4uTT_6tt2_z;E8T4YH+dv=! zdxHra7o3LBfeBDE%mx7@(4%%Tj=%)U=WyN#8ZEd+_?iNk29F?jfr45&F3=3aSwJ&D zJ%VI_QV#V9Kr)m=2OB|!g=Tm@p9NNdjt+7cLI=zMae>^0e1WP4=?#nt`2rrnSwQc$ zbG|@-1ceWrhQtI}0o^AkRbVNA8eG6=z5tltPUltDvk7F~R9>Ywm|n>n!C9aWSj`&& z0wBF%{<4wJ0_qUTC)^RV3Q*P7vPU3J03<*`P>(>gL3AN5{W&fGSYU?UeAxkh0W@~9 z)1Y@DrJyzfzrb+;N8r&&jtdM-LB9akfeCOOpb>Bcae+Dw0un?U-1Kt3Mj-gUyb3S_ zbNAk|7DD zPrLKS&(1HN?!H$(2);Xg@#sUBpSpbG*6H2XZruFj(uWuR`0|@``|iKi`s>&0pFLVk z4!r---gC$H9Qx;{-9P->+Bf$3pHJ)>9JxOJ+r{x+Zywpa{kxI>*(z7(X1jAIyI0IJ zaJ~m7n@h8+b60lkyyxD{mGkFU%hCS6`FQ)g!@rE|+57v$Up(^lleg}>^v=iMKXBpa S52l}e`>V+ZH@mZE#qK|;*2yCP literal 0 HcmV?d00001 diff --git a/tests/proof_test.mpack-dump.json b/tests/proof_test.mpack-dump.json new file mode 100644 index 0000000..5a38271 --- /dev/null +++ b/tests/proof_test.mpack-dump.json @@ -0,0 +1,1060 @@ +{ + "chunks": [ + [ + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + + ], + [ + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + + ], + [ + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + + ], + [ + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + + ] + ], + "siblings": [ + , + , + , + , + , + , + , + + ], + "hashes": [ + , + , + , + + ], + "path": [ + 0, + 1, + 2, + 3 + ], + "root": , + "salt": +} diff --git a/test/storer.js b/tests/storer.js similarity index 100% rename from test/storer.js rename to tests/storer.js diff --git a/tests/tffi.nim b/tests/tffi.nim new file mode 100644 index 0000000..18c5769 --- /dev/null +++ b/tests/tffi.nim @@ -0,0 +1,33 @@ + +import std/os +import unittest2 +import codex_storage_proofs + +suite "storage proofs ffi": + test "basic ffi circuit": + var + r1csPath = "src/circuit_tests/artifacts/storer-test.r1cs".absolutePath() + wasmPath = "src/circuit_tests/artifacts/storer-test_js/storer-test.wasm".absolutePath() + + if not r1csPath.fileExists(): + raise newException(ValueError, "missing expected r1cs file: " & r1csPath) + if not wasmPath.fileExists(): + raise newException(ValueError, "missing expected wasm file: " & wasmPath) + + let + r1cs_buff = unsafeBufferPath(r1csPath) + wasm_buff = unsafeBufferPath(wasmPath) + + let storage_ctx = init_storage_proofs(r1cs_buff, wasm_buff, nil) + + echo "storage_ctx: ", storage_ctx.repr + check storage_ctx != nil + + var + mpack_arg_path = "tests/proof_test.mpack" + proofBuff = unsafeBufferFromFile(mpack_arg_path) + echo "proofArgs:size: ", proofBuff.len() + let res = prove_mpack_ext(storage_ctx, addr proofBuff) + + echo "result: ", res.repr + check res != nil