feat(rln-relay-v2): update C FFI api's and serde (#2385)

* feat(rln-relay-v2): integrate new ffi bindings, serde

* chore: remove ExtendedRateLimitProof, add comments

* fix: typo

waku/waku_rln_relay/constants.nim

@@ -1,6 +1,9 @@
 import
   stint
 
+import
+  ./protocol_types
+
 import
   ../waku_keystore
 
@@ -15,7 +18,8 @@ const
 # inputs of the membership contract constructor
 # TODO may be able to make these constants private and put them inside the waku_rln_relay_utils
 const
-  MembershipFee* = 1000000000000000.u256
+  # in wei
+  MembershipFee* = 0.u256
   #  the current implementation of the rln lib supports a circuit for Merkle tree with depth 20
   MerkleTreeDepth* = 20
   EthClient* = "http://127.0.0.1:8540"
@@ -29,6 +33,14 @@ const
 const
   DefaultRlnTreePath* = "rln_tree.db"
 
+when defined(rln_v2):
+  const
+    # pre-processed "rln/waku-rln-relay/v2.0.0" to array[32, byte]
+    DefaultRlnIdentifier*: RlnIdentifier = [114, 108, 110, 47, 119, 97, 107, 117, 
+                                            45, 114, 108, 110, 45, 114, 101, 108, 
+                                            97, 121, 47, 118, 50, 46, 48, 46, 
+                                            48, 0, 0, 0, 0, 0, 0, 0]
+
 # temporary variables to test waku-rln-relay performance in the static group mode
 const
   StaticGroupSize* = 10000

waku/waku_rln_relay/conversion_utils.nim

@@ -53,6 +53,24 @@ proc encodeLengthPrefix*(input: openArray[byte]): seq[byte] =
     output = concat(@len, @input)
   return output
 
+when defined(rln_v2):
+  proc serialize*(idSecretHash: IdentitySecretHash, 
+                  memIndex: MembershipIndex, 
+                  userMessageLimit: UserMessageLimit,
+                  messageId: MessageId,
+                  externalNullifier: ExternalNullifier,
+                  msg: openArray[byte]): seq[byte] =
+    ## a private proc to convert RateLimitProof and the data to a byte seq
+    ## this conversion is used in the proofGen proc
+    ## the serialization is done as instructed in  https://github.com/kilic/rln/blob/7ac74183f8b69b399e3bc96c1ae8ab61c026dc43/src/public.rs#L146
+    ## [ id_key<32> | id_index<8> | epoch<32> | signal_len<8> | signal<var> ]
+    let memIndexBytes = toBytes(uint64(memIndex), Endianness.littleEndian)
+    let userMessageLimitBytes = toBytes(uint64(userMessageLimit), Endianness.littleEndian)
+    let messageIdBytes = toBytes(uint64(messageId), Endianness.littleEndian)
+    let lenPrefMsg = encodeLengthPrefix(msg)
+    let output = concat(@idSecretHash, @memIndexBytes, @userMessageLimitBytes, @messageIdBytes, @externalNullifier, lenPrefMsg)
+    return output
+else:
   proc serialize*(idSecretHash: IdentitySecretHash, 
                 memIndex: MembershipIndex, 
                 epoch: Epoch,
@@ -72,6 +90,15 @@ proc serialize*(proof: RateLimitProof, data: openArray[byte]): seq[byte] =
   ## this conversion is used in the proof verification proc
   ## [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> | signal_len<8> | signal<var> ]
   let lenPrefMsg = encodeLengthPrefix(@data)
+  when defined(rln_v2):
+    var proofBytes = concat(@(proof.proof),
+                            @(proof.merkleRoot),
+                            @(proof.externalNullifier),
+                            @(proof.shareX),
+                            @(proof.shareY),
+                            @(proof.nullifier),
+                            lenPrefMsg)
+  else:
     var proofBytes = concat(@(proof.proof),
                             @(proof.merkleRoot),
                             @(proof.epoch),

waku/waku_rln_relay/group_manager/group_manager_base.nim

@@ -135,6 +135,17 @@ template slideRootQueue*(g: GroupManager): untyped =
       discard rootBuffer.slideRootQueue(root)
   rootBuffer
 
+when defined(rln_v2):
+  method verifyProof*(g: GroupManager,
+                      input: openArray[byte],
+                      proof: RateLimitProof): GroupManagerResult[bool] {.base,gcsafe,raises:[].} =
+    ## verifies the proof against the input and the current merkle root
+    ## TODO: verify the external nullifier with provided RateLimitProof
+    let proofVerifyRes = g.rlnInstance.proofVerify(input, RateLimitProof(proof), g.validRoots.items().toSeq())
+    if proofVerifyRes.isErr():
+      return err("proof verification failed: " & $proofVerifyRes.error())
+    return ok(proofVerifyRes.value())
+else:
   method verifyProof*(g: GroupManager,
                     input: openArray[byte],
                     proof: RateLimitProof): GroupManagerResult[bool] {.base,gcsafe,raises:[].} =
@@ -144,6 +155,7 @@ method verifyProof*(g: GroupManager,
       return err("proof verification failed: " & $proofVerifyRes.error())
     return ok(proofVerifyRes.value())
 
+
 method generateProof*(g: GroupManager,
                       data: openArray[byte],
                       epoch: Epoch): GroupManagerResult[RateLimitProof] {.base,gcsafe,raises:[].} =

waku/waku_rln_relay/protocol_types.nim

@@ -31,6 +31,12 @@ type
   RlnIdentifier* = array[32, byte]
   ZKSNARK* = array[128, byte]
 
+when defined(rln_v2):
+  type
+    UserMessageLimit* = uint64
+    MessageId* = uint64
+    ExternalNullifier* = array[32, byte]
+
 # Custom data types defined for waku rln relay -------------------------
 type RateLimitProof* = object
   ## RateLimitProof holds the public inputs to rln circuit as
@@ -39,8 +45,6 @@ type RateLimitProof* = object
   proof*: ZKSNARK
   ## the root of Merkle tree used for the generation of the `proof`
   merkleRoot*: MerkleNode
-  ## the epoch used for the generation of the `proof`
-  epoch*: Epoch
   ## shareX and shareY are shares of user's identity key
   ## these shares are created using Shamir secret sharing scheme
   ## see details in https://hackmd.io/tMTLMYmTR5eynw2lwK9n1w?view#Linear-Equation-amp-SSS
@@ -49,8 +53,13 @@ type RateLimitProof* = object
   ## nullifier enables linking two messages published during the same epoch
   ## see details in https://hackmd.io/tMTLMYmTR5eynw2lwK9n1w?view#Nullifiers
   nullifier*: Nullifier
+  ## the epoch used for the generation of the `proof`
+  epoch*: Epoch
   ## Application specific RLN Identifier
   rlnIdentifier*: RlnIdentifier
+  when defined(rln_v2):
+    ## the external nullifier used for the generation of the `proof` (derived from poseidon([epoch, rln_identifier]))
+    externalNullifier*: ExternalNullifier
 
 type ProofMetadata* = object
   nullifier*: Nullifier
@@ -69,6 +78,7 @@ type
 # Protobufs enc and init
 proc init*(T: type RateLimitProof, buffer: seq[byte]): ProtoResult[T] =
   var nsp: RateLimitProof
+
   let pb = initProtoBuffer(buffer)
 
   var proof: seq[byte]
@@ -101,6 +111,7 @@ proc init*(T: type RateLimitProof, buffer: seq[byte]): ProtoResult[T] =
 
   return ok(nsp)
 
+
 proc encode*(nsp: RateLimitProof): ProtoBuffer =
   var output = initProtoBuffer()
 
@@ -113,7 +124,6 @@ proc encode*(nsp: RateLimitProof): ProtoBuffer =
   output.write3(7, nsp.rlnIdentifier)
 
   output.finish3()
-
   return output
 
 type

waku/waku_rln_relay/rln/rln_interface.nim

@@ -103,6 +103,10 @@ proc seeded_key_gen*(ctx: ptr RLN, input_buffer: ptr Buffer, output_buffer: ptr
 proc generate_proof*(ctx: ptr RLN,
                      input_buffer: ptr Buffer,
                      output_buffer: ptr Buffer): bool {.importc: "generate_rln_proof".}
+## rln-v2
+## input_buffer has to be serialized as [ identity_secret<32> | identity_index<8> | user_message_limit<32> | message_id<32> | external_nullifier<32> | signal_len<8> | signal<var> ]
+## output_buffer holds the proof data and should be parsed as [ proof<128> | root<32> | external_nullifier<32> | share_x<32> | share_y<32> | nullifier<32> ]
+## rln-v1
 ## input_buffer has to be serialized as [ id_key<32> | id_index<8> | epoch<32> | signal_len<8> | signal<var> ]
 ## output_buffer holds the proof data and should be parsed as [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> ]
 ## integers wrapped in <> indicate value sizes in bytes
@@ -111,7 +115,10 @@ proc generate_proof*(ctx: ptr RLN,
 proc verify*(ctx: ptr RLN,
              proof_buffer: ptr Buffer,
              proof_is_valid_ptr: ptr bool): bool {.importc: "verify_rln_proof".}
-## proof_buffer has to be serialized as [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> | signal_len<8> | signal<var> ]
+## rln-v2
+## proof_buffer has to be serialized as [ proof<128> | root<32> | external_nullifier<32> | share_x<32> | share_y<32> | nullifier<32> | signal_len<8> | signal<var> ]
+## rln-v1
+## ## proof_buffer has to be serialized as [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> | signal_len<8> | signal<var> ]
 ## the return bool value indicates the success or failure of the call to the verify function
 ## the verification of the zk proof is available in proof_is_valid_ptr, where a value of true indicates success and false a failure
 
@@ -119,6 +126,9 @@ proc verify_with_roots*(ctx: ptr RLN,
                        proof_buffer: ptr Buffer,
                        roots_buffer: ptr Buffer,
                        proof_is_valid_ptr: ptr bool): bool {.importc: "verify_with_roots".}
+## rln-v2
+## proof_buffer has to be serialized as [ proof<128> | root<32> | external_nullifier<32> | share_x<32> | share_y<32> | nullifier<32> | signal_len<8> | signal<var> ]
+## rln-v1
 ## proof_buffer has to be serialized as [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> | signal_len<8> | signal<var> ]
 ## roots_buffer contains the concatenation of 32 bytes long serializations in little endian of root values
 ## the return bool value indicates the success or failure of the call to the verify function
@@ -128,6 +138,9 @@ proc zk_prove*(ctx: ptr RLN,
                input_buffer: ptr Buffer,
                output_buffer: ptr Buffer): bool {.importc: "prove".}
 ## Computes the zkSNARK proof and stores it in output_buffer for input values stored in input_buffer
+## rln-v2
+## input_buffer is serialized as input_data as [ identity_secret<32> | user_message_limit<32> | message_id<32> | path_elements<Vec<32>> | identity_path_index<Vec<1>> | x<32> | external_nullifier<32> ]
+## rln-v1
 ## input_buffer is serialized as input_data as [ id_key<32> | path_elements<Vec<32>> | identity_path_index<Vec<1>> | x<32> | epoch<32> | rln_identifier<32> ]
 ## output_buffer holds the proof data and should be parsed as [ proof<128> ]
 ## path_elements and indentity_path elements serialize a merkle proof for id_key and are vectors of elements of 32 and 1 bytes, respectively (not. Vec<>).

waku/waku_rln_relay/rln/wrappers.nim

@@ -159,8 +159,17 @@ proc poseidon*(data: seq[seq[byte]]): RlnRelayResult[array[32, byte]] =
 
   return ok(output)
 
+when defined(rln_v2):
   # TODO: collocate this proc with the definition of the RateLimitProof
   # and the ProofMetadata types
+  proc extractMetadata*(proof: RateLimitProof): RlnRelayResult[ProofMetadata] =
+    return ok(ProofMetadata(
+      nullifier: proof.nullifier,
+      shareX: proof.shareX,
+      shareY: proof.shareY,
+      externalNullifier: externalNullifierRes.get()
+    ))
+else:
   proc extractMetadata*(proof: RateLimitProof): RlnRelayResult[ProofMetadata] =
     let externalNullifierRes = poseidon(@[@(proof.epoch),
                                           @(proof.rlnIdentifier)])
@@ -173,6 +182,75 @@ proc extractMetadata*(proof: RateLimitProof): RlnRelayResult[ProofMetadata] =
       externalNullifier: externalNullifierRes.get()
     ))
 
+when defined(rln_v2):
+  proc proofGen*(rlnInstance: ptr RLN, 
+                 data: openArray[byte],
+                 membership: IdentityCredential,
+                 userMessageLimit: UserMessageLimit,
+                 messageId: MessageId,
+                 index: MembershipIndex,
+                 epoch: Epoch): RateLimitProofResult =
+
+    # obtain the external nullifier
+    let externalNullifierRes = poseidon(@[@(epoch),
+                                          @(DefaultRlnIdentifier)])
+
+    if externalNullifierRes.isErr():
+      return err("could not construct the external nullifier")
+
+    # serialize inputs
+    let serializedInputs = serialize(idSecretHash = membership.idSecretHash,
+                                    memIndex = index,
+                                    userMessageLimit = userMessageLimit,
+                                    messageId = messageId,
+                                    externalNullifier = externalNullifierRes.get(),
+                                    msg = data)
+    var inputBuffer = toBuffer(serializedInputs)
+
+    debug "input buffer ", inputBuffer= repr(inputBuffer)
+
+    # generate the proof
+    var proof: Buffer
+    let proofIsSuccessful = generate_proof(rlnInstance, addr inputBuffer, addr proof)
+    # check whether the generate_proof call is done successfully
+    if not proofIsSuccessful:
+      return err("could not generate the proof")
+
+    var proofValue = cast[ptr array[320, byte]] (proof.`ptr`)
+    let proofBytes: array[320, byte] = proofValue[]
+    debug "proof content", proofHex = proofValue[].toHex
+
+    ## parse the proof as [ proof<128> | root<32> | external_nullifier<32> | share_x<32> | share_y<32> | nullifier<32> ]
+
+    let
+      proofOffset = 128
+      rootOffset = proofOffset + 32
+      externalNullifierOffset = rootOffset + 32
+      shareXOffset = externalNullifierOffset + 32
+      shareYOffset = shareXOffset + 32
+      nullifierOffset = shareYOffset + 32
+
+    var
+      zkproof: ZKSNARK
+      proofRoot, shareX, shareY: MerkleNode
+      externalNullifier: ExternalNullifier
+      nullifier: Nullifier
+
+    discard zkproof.copyFrom(proofBytes[0..proofOffset-1])
+    discard proofRoot.copyFrom(proofBytes[proofOffset..rootOffset-1])
+    discard externalNullifier.copyFrom(proofBytes[rootOffset..externalNullifierOffset-1])
+    discard shareX.copyFrom(proofBytes[externalNullifierOffset..shareXOffset-1])
+    discard shareY.copyFrom(proofBytes[shareXOffset..shareYOffset-1])
+    discard nullifier.copyFrom(proofBytes[shareYOffset..nullifierOffset-1])
+
+    let output = RateLimitProof(proof: zkproof,
+                                merkleRoot: proofRoot,
+                                externalNullifier: externalNullifier,
+                                shareX: shareX,
+                                shareY: shareY,
+                                nullifier: nullifier)
+    return ok(output)
+else:
   proc proofGen*(rlnInstance: ptr RLN, data: openArray[byte],
     memKeys: IdentityCredential, memIndex: MembershipIndex,
     epoch: Epoch): RateLimitProofResult =