From 5a0ed0e5971fd596b7e79ef4127671bfdbccd2a2 Mon Sep 17 00:00:00 2001 From: Sanaz Taheri Boshrooyeh <35961250+staheri14@users.noreply.github.com> Date: Wed, 24 Mar 2021 10:26:56 -0700 Subject: [PATCH] Adds Merkle tree Nim bindings (#430) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * entirely replaces the prior rln header, the var variables are changed to ptr * updates the unittest of key_gen * adds test for update_next_member * updates membershipKeyGen internals and prototype * adds createRLNInstance * adds helpers methods * adds generateKeyPairBuffer * cleans up the test and adds comments * renames merkleTreeDepth to d * fixes a buf re decoding the keys into sk and pk * adds getSKPK proc * unifies key gen helper procs, adds todos * comments out the createRLNInstance * refactors the code based on the updated createRLNInstance interface * adds the test for the verify proc * fixes a variable name and replaces random key gen with the real key gen * tests a simple hash * adds get_root method * fixes the data pointer issue and adds the proof breakdown * adds rln * adds unit tests for Merkle tree * deletes unnecessary comments * updates createRLNInstance to return bool indicating the success of call * updates create RLN Instance interface * minor * clean up * removes unused imports * adds documentation * adds comments * adds byteutils * removes extra spaces * updates rln submodule * deletes genSKPK * fixes a bug in membershipKeyGen * unsafeAddr to addr * Update waku/v2/protocol/waku_rln_relay/rln.nim Co-authored-by: Oskar Thorén * clean up Co-authored-by: Oskar Thorén --- tests/v2/test_waku_rln_relay.nim | 170 ++++++++++++++++-- vendor/rln | 2 +- waku/v2/node/wakunode2.nim | 12 +- waku/v2/protocol/waku_rln_relay/rln.nim | 64 ++----- .../waku_rln_relay/waku_rln_relay_utils.nim | 49 ++--- 5 files changed, 215 insertions(+), 82 deletions(-) diff --git a/tests/v2/test_waku_rln_relay.nim b/tests/v2/test_waku_rln_relay.nim index fe44a4ee9..6b1c52ec1 100644 --- a/tests/v2/test_waku_rln_relay.nim +++ b/tests/v2/test_waku_rln_relay.nim @@ -180,8 +180,15 @@ procSuite "Waku rln relay": ethAccountAddress = accounts[9] await web3.close() + # create an RLN instance + var + ctx = RLN[Bn256]() + ctxPtr = addr(ctx) + ctxPtrPtr = addr(ctxPtr) + doAssert(createRLNInstance(32, ctxPtrPtr)) + # generate the membership keys - let membershipKeyPair = membershipKeyGen() + let membershipKeyPair = membershipKeyGen(ctxPtrPtr[]) check: membershipKeyPair.isSome @@ -219,8 +226,9 @@ procSuite "Waku rln relay": await node.stop() + suite "Waku rln relay": - test "Keygen Nim Wrappers": + test "key_gen Nim Wrappers": var merkleDepth: csize_t = 32 # parameters.key contains the parameters related to the Poseidon hasher @@ -231,25 +239,27 @@ suite "Waku rln relay": parameters = readFile("waku/v2/protocol/waku_rln_relay/parameters.key") pbytes = parameters.toBytes() len : csize_t = uint(pbytes.len) - parametersBuffer = Buffer(`ptr`: unsafeAddr(pbytes[0]), len: len) + parametersBuffer = Buffer(`ptr`: addr(pbytes[0]), len: len) check: # check the parameters.key is not empty pbytes.len != 0 # ctx holds the information that is going to be used for the key generation var - obj = RLNBn256() - objPtr = unsafeAddr(obj) - ctx = objPtr - let res = newCircuitFromParams(merkleDepth, unsafeAddr parametersBuffer, ctx) + obj = RLN[Bn256]() + objPtr = addr(obj) + objptrptr = addr(objPtr) + ctx = objptrptr + let res = new_circuit_from_params(merkleDepth, addr parametersBuffer, ctx) check: # check whether the circuit parameters are generated successfully res == true # keysBufferPtr will hold the generated key pairs i.e., secret and public keys var - keysBufferPtr : Buffer - done = keyGen(ctx, keysBufferPtr) + keysBuffer : Buffer + keysBufferPtr = addr(keysBuffer) + done = key_gen(ctx[], keysBufferPtr) check: # check whether the keys are generated successfully done == true @@ -262,7 +272,14 @@ suite "Waku rln relay": debug "generated keys: ", generatedKeys test "membership Key Gen": - var key = membershipKeyGen() + # create an RLN instance + var + ctx = RLN[Bn256]() + ctxPtr = addr(ctx) + ctxPtrPtr = addr(ctxPtr) + doAssert(createRLNInstance(32, ctxPtrPtr)) + + var key = membershipKeyGen(ctxPtrPtr[]) var empty : array[32,byte] check: key.isSome @@ -271,4 +288,135 @@ suite "Waku rln relay": key.get().secretKey != empty key.get().publicKey != empty - debug "the generated membership key pair: ", key \ No newline at end of file + debug "the generated membership key pair: ", key + + test "get_root Nim binding": + # create an RLN instance which also includes an empty Merkle tree + var + ctx = RLN[Bn256]() + ctxPtr = addr(ctx) + ctxPtrPtr = addr(ctxPtr) + doAssert(createRLNInstance(32, ctxPtrPtr)) + + # read the Merkle Tree root + var + root1 {.noinit.} : Buffer = Buffer() + rootPtr1 = addr(root1) + get_root_successful1 = get_root(ctxPtrPtr[], rootPtr1) + doAssert(get_root_successful1) + doAssert(root1.len == 32) + + # read the Merkle Tree root + var + root2 {.noinit.} : Buffer = Buffer() + rootPtr2 = addr(root2) + get_root_successful2 = get_root(ctxPtrPtr[], rootPtr2) + doAssert(get_root_successful2) + doAssert(root2.len == 32) + + var rootValue1 = cast[ptr array[32,byte]] (root1.`ptr`) + let rootHex1 = rootValue1[].toHex + + var rootValue2 = cast[ptr array[32,byte]] (root2.`ptr`) + let rootHex2 = rootValue2[].toHex + + # the two roots must be identical + doAssert(rootHex1 == rootHex2) + + test "update_next_member Nim Wrapper": + # create an RLN instance which also includes an empty Merkle tree + var + ctx = RLN[Bn256]() + ctxPtr = addr(ctx) + ctxPtrPtr = addr(ctxPtr) + doAssert(createRLNInstance(32, ctxPtrPtr)) + + # generate a key pair + var keypair = membershipKeyGen(ctxPtrPtr[]) + doAssert(keypair.isSome()) + var pkBuffer = Buffer(`ptr`: addr(keypair.get().publicKey[0]), len: 32) + let pkBufferPtr = addr pkBuffer + + # add the member to the tree + var member_is_added = update_next_member(ctxPtrPtr[], pkBufferPtr) + check: + member_is_added == true + + test "delete_member Nim wrapper": + # create an RLN instance which also includes an empty Merkle tree + var + ctx = RLN[Bn256]() + ctxPtr = addr(ctx) + ctxPtrPtr = addr(ctxPtr) + doAssert(createRLNInstance(32, ctxPtrPtr)) + + # delete the first member + var deleted_member_index = uint(0) + let deletion_success = delete_member(ctxPtrPtr[], deleted_member_index) + doAssert(deletion_success) + + test "Merkle tree consistency check between deletion and insertion": + # create an RLN instance + var + ctx = RLN[Bn256]() + ctxPtr = addr(ctx) + ctxPtrPtr = addr(ctxPtr) + doAssert(createRLNInstance(32, ctxPtrPtr)) + + # read the Merkle Tree root + var + root1 {.noinit.} : Buffer = Buffer() + rootPtr1 = addr(root1) + get_root_successful1 = get_root(ctxPtrPtr[], rootPtr1) + doAssert(get_root_successful1) + doAssert(root1.len == 32) + + # generate a key pair + var keypair = membershipKeyGen(ctxPtrPtr[]) + doAssert(keypair.isSome()) + var pkBuffer = Buffer(`ptr`: addr(keypair.get().publicKey[0]), len: 32) + let pkBufferPtr = addr pkBuffer + + # add the member to the tree + var member_is_added = update_next_member(ctxPtrPtr[], pkBufferPtr) + doAssert(member_is_added) + + # read the Merkle Tree root after insertion + var + root2 {.noinit.} : Buffer = Buffer() + rootPtr2 = addr(root2) + get_root_successful2 = get_root(ctxPtrPtr[], rootPtr2) + doAssert(get_root_successful2) + doAssert(root2.len == 32) + + # delete the first member + var deleted_member_index = uint(0) + let deletion_success = delete_member(ctxPtrPtr[], deleted_member_index) + doAssert(deletion_success) + + # read the Merkle Tree root after the deletion + var + root3 {.noinit.} : Buffer = Buffer() + rootPtr3 = addr(root3) + get_root_successful3 = get_root(ctxPtrPtr[], rootPtr3) + doAssert(get_root_successful3) + doAssert(root3.len == 32) + + var rootValue1 = cast[ptr array[32,byte]] (root1.`ptr`) + let rootHex1 = rootValue1[].toHex + debug "The initial root", rootHex1 + + var rootValue2 = cast[ptr array[32,byte]] (root2.`ptr`) + let rootHex2 = rootValue2[].toHex + debug "The root after insertion", rootHex2 + + var rootValue3 = cast[ptr array[32,byte]] (root3.`ptr`) + let rootHex3 = rootValue3[].toHex + debug "The root after deletion", rootHex3 + + # the root must change after the insertion + doAssert(not(rootHex1 == rootHex2)) + + ## The initial root of the tree (empty tree) must be identical to + ## the root of the tree after one insertion followed by a deletion + doAssert(rootHex1 == rootHex3) \ No newline at end of file diff --git a/vendor/rln b/vendor/rln index a80f5d013..628fd7422 160000 --- a/vendor/rln +++ b/vendor/rln @@ -1 +1 @@ -Subproject commit a80f5d013eb092ff18bd1d946c57565e2cdc65da +Subproject commit 628fd742237f49347469dfcbbe57fd591d6629b1 diff --git a/waku/v2/node/wakunode2.nim b/waku/v2/node/wakunode2.nim index 626fc2c2c..379ab38c6 100644 --- a/waku/v2/node/wakunode2.nim +++ b/waku/v2/node/wakunode2.nim @@ -15,7 +15,7 @@ import ../protocol/waku_store/waku_store, ../protocol/waku_swap/waku_swap, ../protocol/waku_filter/waku_filter, - ../protocol/waku_rln_relay/waku_rln_relay_utils, + ../protocol/waku_rln_relay/[rln,waku_rln_relay_utils], ../utils/peers, ./message_store/message_store, ../utils/requests, @@ -327,13 +327,21 @@ proc mountStore*(node: WakuNode, store: MessageStore = nil) = node.subscriptions.subscribe(WakuStoreCodec, node.wakuStore.subscription()) proc mountRlnRelay*(node: WakuNode, ethClientAddress: Option[string] = none(string), ethAccountAddress: Option[Address] = none(Address), membershipContractAddress: Option[Address] = none(Address)) {.async.} = + # TODO return a bool value to indicate the success of the call # check whether inputs are provided doAssert(ethClientAddress.isSome()) doAssert(ethAccountAddress.isSome()) doAssert(membershipContractAddress.isSome()) + # create an RLN instance + var + ctx = RLN[Bn256]() + ctxPtr = addr(ctx) + ctxPtrPtr = addr(ctxPtr) + doAssert(createRLNInstance(32, ctxPtrPtr)) + # generate the membership keys - let membershipKeyPair = membershipKeyGen() + let membershipKeyPair = membershipKeyGen(ctxPtrPtr[]) # check whether keys are generated doAssert(membershipKeyPair.isSome()) debug "the membership key for the rln relay is generated" diff --git a/waku/v2/protocol/waku_rln_relay/rln.nim b/waku/v2/protocol/waku_rln_relay/rln.nim index 0458a0a51..635f3379b 100644 --- a/waku/v2/protocol/waku_rln_relay/rln.nim +++ b/waku/v2/protocol/waku_rln_relay/rln.nim @@ -2,9 +2,6 @@ import os -# librln.dylib is the rln library taken from https://github.com/kilic/rln (originally implemented in rust with an exposed C API) -# contains the key generation and other relevant functions - const libPath = "vendor/rln/target/debug/" when defined(Windows): @@ -14,58 +11,31 @@ elif defined(Linux): elif defined(MacOsX): const libName* = libPath / "librln.dylib" -# Data types ----------------------------- - -# pub struct Buffer { -# pub ptr: *const u8, -# pub len: usize, -# } - -type - Buffer* = object - `ptr`*: pointer - len*: csize_t - RLNBn256* = object - -# Procedures ------------------------------ # all the following procedures are Nim wrappers for the functions defined in libName {.push dynlib: libName.} -# pub extern "C" fn new_circuit_from_params( -# merkle_depth: usize, -# index: usize, -# parameters_buffer: *const Buffer, -# ctx: *mut *mut RLN, -# ) -> bool -proc newCircuitFromParams*(merkle_depth: csize_t, parameters_buffer: ptr Buffer, ctx: var ptr RLNBn256): bool{.importc: "new_circuit_from_params".} +type RLN*[E] {.incompleteStruct.} = object +type Bn256* = pointer -# pub extern "C" fn key_gen(ctx: *const RLN, keypair_buffer: *mut Buffer) -> bool -proc keyGen*(ctx: ptr RLNBn256, keypair_buffer: var Buffer): bool {.importc: "key_gen".} +## Buffer struct is taken from +# https://github.com/celo-org/celo-threshold-bls-rs/blob/master/crates/threshold-bls-ffi/src/ffi.rs +type Buffer* = object + `ptr`*: ptr uint8 + len*: uint +proc key_gen*(ctx: ptr RLN[Bn256], keypair_buffer: ptr Buffer): bool {.importc: "key_gen".} -# pub extern "C" fn hash( -# ctx: *const RLN, -# inputs_buffer: *const Buffer, -# input_len: *const usize, -# output_buffer: *mut Buffer, -# ) -> bool -proc hash*(ctx: ptr RLNBn256, inputs_buffer:ptr Buffer, input_len: ptr csize_t, output_buffer: ptr Buffer ) {.importc: "hash".} #TODO not tested yet +proc new_circuit_from_params*(merkle_depth: uint, + parameters_buffer: ptr Buffer, + ctx: ptr (ptr RLN[Bn256])): bool {.importc: "new_circuit_from_params".} -# pub extern "C" fn verify( -# ctx: *const RLN, -# proof_buffer: *const Buffer, -# public_inputs_buffer: *const Buffer, -# result_ptr: *mut u32, -# ) -> bool +#------------------------------Merkle Tree operations ----------------------------------------- +proc update_next_member*(ctx: ptr RLN[Bn256], + input_buffer: ptr Buffer): bool {.importc: "update_next_member".} +proc delete_member*(ctx: ptr RLN[Bn256], index: uint): bool {.importc: "delete_member".} - -# pub extern "C" fn generate_proof( -# ctx: *const RLN, -# input_buffer: *const Buffer, -# output_buffer: *mut Buffer, -# ) -> bool +proc get_root*(ctx: ptr RLN[Bn256], output_buffer: ptr Buffer): bool {.importc: "get_root".} +#---------------------------------------------------------------------------------------------- {.pop.} - - diff --git a/waku/v2/protocol/waku_rln_relay/waku_rln_relay_utils.nim b/waku/v2/protocol/waku_rln_relay/waku_rln_relay_utils.nim index 25fd270a7..a77856c70 100644 --- a/waku/v2/protocol/waku_rln_relay/waku_rln_relay_utils.nim +++ b/waku/v2/protocol/waku_rln_relay/waku_rln_relay_utils.nim @@ -31,10 +31,12 @@ contract(MembershipContract): # TODO define a return type of bool for register method to signify a successful registration proc register(pubkey: Uint256) # external payable -proc membershipKeyGen*(): Option[MembershipKeyPair] = - # generates a MembershipKeyPair that can be used for the registration into the rln membership contract - var - merkleDepth: csize_t = 32 +proc createRLNInstance*(d: int, ctxPtrPtr: ptr (ptr RLN[Bn256])): bool = + ## generates an instance of RLN + ## An RLN instance supports both zkSNARKs logics and Merkle tree data structure and operations + ## d indicates the depth of Merkle tree + var + merkleDepth: csize_t = uint(d) # parameters.key contains the parameters related to the Poseidon hasher # to generate this file, clone this repo https://github.com/kilic/rln # and run the following command in the root directory of the cloned project @@ -43,29 +45,29 @@ proc membershipKeyGen*(): Option[MembershipKeyPair] = parameters = readFile("waku/v2/protocol/waku_rln_relay/parameters.key") pbytes = parameters.toBytes() len : csize_t = uint(pbytes.len) - parametersBuffer = Buffer(`ptr`: unsafeAddr(pbytes[0]), len: len) + parametersBuffer = Buffer(`ptr`: addr(pbytes[0]), len: len) # check the parameters.key is not empty - if(pbytes.len == 0): + if (pbytes.len == 0): debug "error in parameters.key" - return none(MembershipKeyPair) - - # ctx holds the information that is going to be used for the key generation - var - obj = RLNBn256() - objPtr = unsafeAddr(obj) - ctx = objPtr - let res = newCircuitFromParams(merkleDepth, unsafeAddr parametersBuffer, ctx) - + return false + + # create an instance of RLN + let res = new_circuit_from_params(merkleDepth, addr parametersBuffer, ctxPtrPtr) # check whether the circuit parameters are generated successfully - if(res == false): + if (res == false): debug "error in parameters generation" - return none(MembershipKeyPair) + return false + return true + +proc membershipKeyGen*(ctxPtr: ptr RLN[Bn256]): Option[MembershipKeyPair] = + ## generates a MembershipKeyPair that can be used for the registration into the rln membership contract # keysBufferPtr will hold the generated key pairs i.e., secret and public keys var - keysBufferPtr : Buffer - done = keyGen(ctx, keysBufferPtr) + keysBuffer : Buffer + keysBufferPtr = addr(keysBuffer) + done = key_gen(ctxPtr, keysBufferPtr) # check whether the keys are generated successfully if(done == false): @@ -78,9 +80,14 @@ proc membershipKeyGen*(): Option[MembershipKeyPair] = debug "the generated keys are invalid" return none(MembershipKeyPair) + # TODO define a separate proc to decode the generated keys to the secret and public components + var + secret: array[32, byte] + public: array[32, byte] + for (i,x) in secret.mpairs: x = generatedKeys[i] + for (i,x) in public.mpairs: x = generatedKeys[i+32] + var - secret = cast[array[32, byte]](generatedKeys[0..31]) - public = cast[array[32, byte]](generatedKeys[31..^1]) keypair = MembershipKeyPair(secretKey: secret, publicKey: public) return some(keypair)