diff --git a/.gitignore b/.gitignore index 488f534d1..d4792a976 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,6 @@ /metrics/prometheus /metrics/waku-sim-all-nodes-grafana-dashboard.json -rln *.log package-lock.json package.json @@ -50,4 +49,4 @@ nimbus-build-system.paths *.db-wal *.sqlite3 *.sqlite3-shm -*.sqlite3-wal \ No newline at end of file +*.sqlite3-wal diff --git a/tests/v2/test_rln_group_manager_onchain.nim b/tests/v2/test_rln_group_manager_onchain.nim new file mode 100644 index 000000000..fe100f074 --- /dev/null +++ b/tests/v2/test_rln_group_manager_onchain.nim @@ -0,0 +1,352 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import + testutils/unittests, + stew/results, + options, + ../../waku/v2/protocol/waku_rln_relay/protocol_types, + ../../waku/v2/protocol/waku_rln_relay/constants, + ../../waku/v2/protocol/waku_rln_relay/contract, + ../../waku/v2/protocol/waku_rln_relay/rln, + ../../waku/v2/protocol/waku_rln_relay/conversion_utils, + ../../waku/v2/protocol/waku_rln_relay/group_manager/on_chain/group_manager + +import + std/[osproc, streams, strutils, sequtils], + chronos, chronicles, stint, web3, json, + stew/shims/net as stewNet, + libp2p/crypto/crypto, + eth/keys, + ../test_helpers, + ./test_utils + +from posix import kill, SIGINT + +proc generateCredentials(rlnInstance: ptr RLN): IdentityCredential = + let credRes = membershipKeyGen(rlnInstance) + return credRes.get() + +proc generateCredentials(rlnInstance: ptr RLN, n: int): seq[IdentityCredential] = + var credentials: seq[IdentityCredential] + for i in 0 ..< n: + credentials.add(generateCredentials(rlnInstance)) + return credentials + +# a util function used for testing purposes +# it deploys membership contract on Ganache (or any Eth client available on EthClient address) +# must be edited if used for a different contract than membership contract +proc uploadRLNContract*(ethClientAddress: string): Future[Address] {.async.} = + let web3 = await newWeb3(ethClientAddress) + debug "web3 connected to", ethClientAddress + + # fetch the list of registered accounts + let accounts = await web3.provider.eth_accounts() + web3.defaultAccount = accounts[1] + let add = web3.defaultAccount + debug "contract deployer account address ", add + + let balance = await web3.provider.eth_getBalance(web3.defaultAccount, "latest") + debug "Initial account balance: ", balance + + # deploy the poseidon hash contract and gets its address + let + hasherReceipt = await web3.deployContract(PoseidonHasherCode) + hasherAddress = hasherReceipt.contractAddress.get + debug "hasher address: ", hasherAddress + + + # encode membership contract inputs to 32 bytes zero-padded + let + membershipFeeEncoded = encode(MembershipFee).data + depthEncoded = encode(MerkleTreeDepth.u256).data + hasherAddressEncoded = encode(hasherAddress).data + # this is the contract constructor input + contractInput = membershipFeeEncoded & depthEncoded & hasherAddressEncoded + + + debug "encoded membership fee: ", membershipFeeEncoded + debug "encoded depth: ", depthEncoded + debug "encoded hasher address: ", hasherAddressEncoded + debug "encoded contract input:", contractInput + + # deploy membership contract with its constructor inputs + let receipt = await web3.deployContract(MembershipContractCode, + contractInput = contractInput) + let contractAddress = receipt.contractAddress.get + debug "Address of the deployed membership contract: ", contractAddress + + let newBalance = await web3.provider.eth_getBalance(web3.defaultAccount, "latest") + debug "Account balance after the contract deployment: ", newBalance + + await web3.close() + debug "disconnected from ", ethClientAddress + + return contractAddress + + +proc createEthAccount(): Future[(keys.PrivateKey, Address)] {.async.} = + let theRNG = keys.newRng() + + let web3 = await newWeb3(EthClient) + let accounts = await web3.provider.eth_accounts() + let gasPrice = int(await web3.provider.eth_gasPrice()) + web3.defaultAccount = accounts[0] + + let pk = keys.PrivateKey.random(theRNG[]) + let acc = Address(toCanonicalAddress(pk.toPublicKey())) + + var tx:EthSend + tx.source = accounts[0] + tx.value = some(ethToWei(10.u256)) + tx.to = some(acc) + tx.gasPrice = some(gasPrice) + + # Send 10 eth to acc + discard await web3.send(tx) + let balance = await web3.provider.eth_getBalance(acc, "latest") + assert(balance == ethToWei(10.u256)) + + return (pk, acc) + +# Runs Ganache daemon +proc runGanache(): Process = + # We run directly "node node_modules/ganache/dist/node/cli.js" rather than using "npx ganache", so that the daemon does not spawn in a new child process. + # In this way, we can directly send a SIGINT signal to the corresponding PID to gracefully terminate Ganache without dealing with multiple processes. + # Passed options are + # --port Port to listen on. + # --miner.blockGasLimit Sets the block gas limit in WEI. + # --wallet.defaultBalance The default account balance, specified in ether. + # See ganache documentation https://www.npmjs.com/package/ganache for more details + try: + let runGanache = startProcess("npx", args = ["--yes", "ganache", "--port", "8540", "--miner.blockGasLimit", "300000000000000", "--wallet.defaultBalance", "10000"], options = {poUsePath}) + let ganachePID = runGanache.processID + + # We read stdout from Ganache to see when daemon is ready + var ganacheStartLog: string + var cmdline: string + while true: + try: + if runGanache.outputstream.readLine(cmdline): + ganacheStartLog.add(cmdline) + if cmdline.contains("Listening on 127.0.0.1:8540"): + break + except: + break + debug "Ganache daemon is running and ready", pid=ganachePID, startLog=ganacheStartLog + return runGanache + except: + error "Ganache daemon run failed" + + +# Stops Ganache daemon +proc stopGanache(runGanache: Process) {.used.} = + + let ganachePID = runGanache.processID + + # We gracefully terminate Ganache daemon by sending a SIGINT signal to the runGanache PID to trigger RPC server termination and clean-up + let returnCodeSIGINT = kill(ganachePID.int32, SIGINT) + debug "Sent SIGINT to Ganache", ganachePID=ganachePID, returnCode=returnCodeSIGINT + + # We wait the daemon to exit + try: + let returnCodeExit = runGanache.waitForExit() + debug "Ganache daemon terminated", returnCode=returnCodeExit + debug "Ganache daemon run log", log=runGanache.outputstream.readAll() + except: + error "Ganache daemon termination failed" + +proc setup(): Future[OnchainGroupManager] {.async.} = + let rlnInstanceRes = createRlnInstance() + require: + rlnInstanceRes.isOk() + + let rlnInstance = rlnInstanceRes.get() + + let contractAddress = await uploadRLNContract(EthClient) + # connect to the eth client + let web3 = await newWeb3(EthClient) + + let accounts = await web3.provider.eth_accounts() + web3.defaultAccount = accounts[1] + + let (pk, _) = await createEthAccount() + + let onchainConfig = OnchainGroupManagerConfig(ethClientUrl: EthClient, + ethContractAddress: $contractAddress, + ethPrivateKey: some($pk)) + + let manager {.used.} = OnchainGroupManager(config: onchainConfig, + rlnInstance: rlnInstance) + + return manager + +suite "Onchain group manager": + # We run Ganache + let runGanache {.used.} = runGanache() + + asyncTest "should initialize successfully": + let manager = await setup() + await manager.init() + + check: + manager.config.ethRpc.isSome() + manager.config.rlnContract.isSome() + manager.config.membershipFee.isSome() + manager.initialized + + asyncTest "startGroupSync: should start group sync": + let manager = await setup() + + await manager.init() + await manager.startGroupSync() + + asyncTest "startGroupSync: should guard against uninitialized state": + let manager = await setup() + + expect(ValueError): + await manager.startGroupSync() + + asyncTest "startGroupSync: should sync to the state of the group": + let manager = await setup() + let credentials = generateCredentials(manager.rlnInstance) + + manager.idCredentials = some(credentials) + await manager.init() + + let merkleRootBeforeRes = manager.rlnInstance.getMerkleRoot() + require: + merkleRootBeforeRes.isOk() + let merkleRootBefore = merkleRootBeforeRes.get() + + let future = newFuture[void]("startGroupSync") + + proc generateCallback(fut: Future[void], idCommitment: IDCommitment): OnRegisterCallback = + proc callback(registrations: seq[Membership]): Future[void] {.async.} = + require: + registrations.len == 1 + registrations[0].idCommitment == idCommitment + registrations[0].index == 0 + fut.complete() + return callback + + manager.onRegister(generateCallback(future, credentials.idCommitment)) + await manager.startGroupSync() + + await future + + let merkleRootAfterRes = manager.rlnInstance.getMerkleRoot() + require: + merkleRootAfterRes.isOk() + let merkleRootAfter = merkleRootAfterRes.get() + + check: + merkleRootBefore != merkleRootAfter + + asyncTest "startGroupSync: should fetch history correctly": + let manager = await setup() + let credentials = generateCredentials(manager.rlnInstance, 5) + await manager.init() + + let merkleRootBeforeRes = manager.rlnInstance.getMerkleRoot() + require: + merkleRootBeforeRes.isOk() + let merkleRootBefore = merkleRootBeforeRes.get() + + var futures = [newFuture[void](), newFuture[void](), newFuture[void](), newFuture[void](), newFuture[void]()] + + proc generateCallback(futs: array[0..4, Future[system.void]], credentials: seq[IdentityCredential]): OnRegisterCallback = + var futureIndex = 0 + proc callback(registrations: seq[Membership]): Future[void] {.async.} = + require: + registrations.len == 1 + registrations[0].idCommitment == credentials[futureIndex].idCommitment + registrations[0].index == MembershipIndex(futureIndex) + futs[futureIndex].complete() + futureIndex += 1 + return callback + + manager.onRegister(generateCallback(futures, credentials)) + await manager.startGroupSync() + + for i in 0 ..< credentials.len(): + await manager.register(credentials[i]) + + await allFutures(futures) + + let merkleRootAfterRes = manager.rlnInstance.getMerkleRoot() + require: + merkleRootAfterRes.isOk() + let merkleRootAfter = merkleRootAfterRes.get() + + check: + merkleRootBefore != merkleRootAfter + + asyncTest "register: should guard against uninitialized state": + let manager = await setup() + let dummyCommitment = default(IDCommitment) + + expect(ValueError): + await manager.register(dummyCommitment) + + asyncTest "register: should register successfully": + let manager = await setup() + await manager.init() + await manager.startGroupSync() + + let idCommitment = generateCredentials(manager.rlnInstance).idCommitment + let merkleRootBeforeRes = manager.rlnInstance.getMerkleRoot() + require: + merkleRootBeforeRes.isOk() + let merkleRootBefore = merkleRootBeforeRes.get() + await manager.register(idCommitment) + let merkleRootAfterRes = manager.rlnInstance.getMerkleRoot() + require: + merkleRootAfterRes.isOk() + let merkleRootAfter = merkleRootAfterRes.get() + check: + merkleRootAfter.inHex() != merkleRootBefore.inHex() + manager.latestIndex == 1 + + asyncTest "register: callback is called": + let manager = await setup() + + var callbackCalled = false + let idCommitment = generateCredentials(manager.rlnInstance).idCommitment + + let fut = newFuture[void]() + + proc callback(registrations: seq[Membership]): Future[void] {.async.} = + require: + registrations.len == 1 + registrations[0].idCommitment == idCommitment + registrations[0].index == 0 + callbackCalled = true + fut.complete() + + manager.onRegister(callback) + await manager.init() + await manager.startGroupSync() + + await manager.register(idCommitment) + + await fut + check: + callbackCalled + + asyncTest "withdraw: should guard against uninitialized state": + let manager = await setup() + let idSecretHash = generateCredentials(manager.rlnInstance).idSecretHash + + expect(ValueError): + await manager.withdraw(idSecretHash) + + ################################ + ## Terminating/removing Ganache + ################################ + + # We stop Ganache daemon + stopGanache(runGanache) + diff --git a/tests/v2/test_rln_group_manager_static.nim b/tests/v2/test_rln_group_manager_static.nim new file mode 100644 index 000000000..d80841b64 --- /dev/null +++ b/tests/v2/test_rln_group_manager_static.nim @@ -0,0 +1,183 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import + testutils/unittests, + stew/results, + options, + ../../waku/v2/protocol/waku_rln_relay/protocol_types, + ../../waku/v2/protocol/waku_rln_relay/rln, + ../../waku/v2/protocol/waku_rln_relay/conversion_utils, + ../../waku/v2/protocol/waku_rln_relay/group_manager/static/group_manager + +import + stew/shims/net, + chronos, + libp2p/crypto/crypto, + eth/keys, + discovery/dnsdisc/builder + +proc generateCredentials(rlnInstance: ptr RLN): IdentityCredential = + let credRes = membershipKeyGen(rlnInstance) + return credRes.get() + +proc generateCredentials(rlnInstance: ptr RLN, n: int): seq[IdentityCredential] = + var credentials: seq[IdentityCredential] + for i in 0 ..< n: + credentials.add(generateCredentials(rlnInstance)) + return credentials + +suite "Static group manager": + setup: + let rlnInstanceRes = createRlnInstance() + require: + rlnInstanceRes.isOk() + + let rlnInstance = rlnInstanceRes.get() + let credentials = generateCredentials(rlnInstance, 10) + + let staticConfig = StaticGroupManagerConfig(groupSize: 10, + membershipIndex: 5, + groupKeys: credentials) + + let manager {.used.} = StaticGroupManager(config: staticConfig, + rlnInstance: rlnInstance) + + asyncTest "should initialize successfully": + let merkleRootBeforeRes = manager.rlnInstance.getMerkleRoot() + require: + merkleRootBeforeRes.isOk() + let merkleRootBefore = merkleRootBeforeRes.get() + + await manager.init() + let merkleRootAfterRes = manager.rlnInstance.getMerkleRoot() + require: + merkleRootAfterRes.isOk() + let merkleRootAfter = merkleRootAfterRes.get() + check: + manager.idCredentials.isSome() + manager.config.groupKeys.len == 10 + manager.config.groupSize == 10 + manager.config.membershipIndex == 5 + manager.config.groupKeys[5] == manager.idCredentials.get() + manager.latestIndex == 9 + merkleRootAfter.inHex() != merkleRootBefore.inHex() + + asyncTest "startGroupSync: should start group sync": + await manager.init() + await manager.startGroupSync() + + asyncTest "startGroupSync: should guard against uninitialized state": + let staticConfig = StaticGroupManagerConfig(groupSize: 0, + membershipIndex: 0, + groupKeys: @[]) + + let manager = StaticGroupManager(config: staticConfig, + rlnInstance: rlnInstance) + + expect(ValueError): + await manager.startGroupSync() + + asyncTest "register: should guard against uninitialized state": + let staticConfig = StaticGroupManagerConfig(groupSize: 0, + membershipIndex: 0, + groupKeys: @[]) + + let manager = StaticGroupManager(config: staticConfig, + rlnInstance: rlnInstance) + + let dummyCommitment = default(IDCommitment) + + expect(ValueError): + await manager.register(dummyCommitment) + + asyncTest "register: should register successfully": + await manager.init() + await manager.startGroupSync() + + let idCommitment = generateCredentials(manager.rlnInstance).idCommitment + let merkleRootBeforeRes = manager.rlnInstance.getMerkleRoot() + require: + merkleRootBeforeRes.isOk() + let merkleRootBefore = merkleRootBeforeRes.get() + await manager.register(idCommitment) + let merkleRootAfterRes = manager.rlnInstance.getMerkleRoot() + require: + merkleRootAfterRes.isOk() + let merkleRootAfter = merkleRootAfterRes.get() + check: + merkleRootAfter.inHex() != merkleRootBefore.inHex() + manager.latestIndex == 10 + + asyncTest "register: callback is called": + var callbackCalled = false + let idCommitment = generateCredentials(manager.rlnInstance).idCommitment + + let fut = newFuture[void]() + + proc callback(registrations: seq[Membership]): Future[void] {.async.} = + require: + registrations.len == 1 + registrations[0].idCommitment == idCommitment + registrations[0].index == 10 + callbackCalled = true + fut.complete() + + manager.onRegister(callback) + await manager.init() + await manager.startGroupSync() + + await manager.register(idCommitment) + + await fut + check: + callbackCalled + + asyncTest "withdraw: should guard against uninitialized state": + let idSecretHash = credentials[0].idSecretHash + + expect(ValueError): + await manager.withdraw(idSecretHash) + + asyncTest "withdraw: should withdraw successfully": + await manager.init() + await manager.startGroupSync() + + let idSecretHash = credentials[0].idSecretHash + let merkleRootBeforeRes = manager.rlnInstance.getMerkleRoot() + require: + merkleRootBeforeRes.isOk() + let merkleRootBefore = merkleRootBeforeRes.get() + await manager.withdraw(idSecretHash) + let merkleRootAfterRes = manager.rlnInstance.getMerkleRoot() + require: + merkleRootAfterRes.isOk() + let merkleRootAfter = merkleRootAfterRes.get() + check: + merkleRootAfter.inHex() != merkleRootBefore.inHex() + + asyncTest "withdraw: callback is called": + var callbackCalled = false + let idSecretHash = credentials[0].idSecretHash + let idCommitment = credentials[0].idCommitment + let fut = newFuture[void]() + + proc callback(withdrawals: seq[Membership]): Future[void] {.async.} = + require: + withdrawals.len == 1 + withdrawals[0].idCommitment == idCommitment + withdrawals[0].index == 0 + callbackCalled = true + fut.complete() + + manager.onWithdraw(callback) + await manager.init() + await manager.startGroupSync() + + await manager.withdraw(idSecretHash) + + await fut + check: + callbackCalled diff --git a/waku/v2/protocol/waku_rln_relay.nim b/waku/v2/protocol/waku_rln_relay.nim index 53c632f5b..47b79a524 100644 --- a/waku/v2/protocol/waku_rln_relay.nim +++ b/waku/v2/protocol/waku_rln_relay.nim @@ -3,6 +3,7 @@ import ./waku_rln_relay/constants, ./waku_rln_relay/protocol_types, ./waku_rln_relay/protocol_metrics, + ./waku_rln_relay/conversion_utils, ./waku_rln_relay/utils, ./waku_rln_relay/contract @@ -11,5 +12,6 @@ export constants, protocol_types, protocol_metrics, + conversion_utils, utils, contract diff --git a/waku/v2/protocol/waku_rln_relay/conversion_utils.nim b/waku/v2/protocol/waku_rln_relay/conversion_utils.nim new file mode 100644 index 000000000..5fa412c29 --- /dev/null +++ b/waku/v2/protocol/waku_rln_relay/conversion_utils.nim @@ -0,0 +1,161 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import + std/[sequtils], + web3, + chronicles, + stew/[arrayops, results, endians2], + stint +import + ./protocol_types +import + ../../utils/keyfile + +export + web3, + chronicles, + stint + +logScope: + topics = "waku rln_relay conversion_utils" + +proc toUInt256*(idCommitment: IDCommitment): UInt256 = + let pk = UInt256.fromBytesLE(idCommitment) + return pk + +proc toIDCommitment*(idCommitmentUint: UInt256): IDCommitment = + let pk = IDCommitment(idCommitmentUint.toBytesLE()) + return pk + +proc inHex*(value: array[32, byte]): string = + var valueHex = (UInt256.fromBytesLE(value)).toHex() + # We pad leading zeroes + while valueHex.len < value.len * 2: + valueHex = "0" & valueHex + return valueHex + +proc toMembershipIndex*(v: UInt256): MembershipIndex = + let membershipIndex: MembershipIndex = cast[MembershipIndex](v) + return membershipIndex + +proc appendLength*(input: openArray[byte]): seq[byte] = + ## returns length prefixed version of the input + ## with the following format [len<8>|input] + ## len: 8-byte value that represents the number of bytes in the `input` + ## len is serialized in little-endian + ## input: the supplied `input` + let + # the length should be serialized in little-endian + len = toBytes(uint64(input.len), Endianness.littleEndian) + output = concat(@len, @input) + return output + +proc serialize*(idSecretHash: IdentitySecretHash, memIndex: MembershipIndex, epoch: Epoch, + msg: openArray[byte]): seq[byte] = + ## a private proc to convert RateLimitProof and the data to a byte seq + ## this conversion is used in the proofGen proc + ## the serialization is done as instructed in https://github.com/kilic/rln/blob/7ac74183f8b69b399e3bc96c1ae8ab61c026dc43/src/public.rs#L146 + ## [ id_key<32> | id_index<8> | epoch<32> | signal_len<8> | signal ] + let memIndexBytes = toBytes(uint64(memIndex), Endianness.littleEndian) + let lenPrefMsg = appendLength(msg) + let output = concat(@idSecretHash, @memIndexBytes, @epoch, lenPrefMsg) + return output + + +proc serialize*(proof: RateLimitProof, data: openArray[byte]): seq[byte] = + ## a private proc to convert RateLimitProof and data to a byte seq + ## this conversion is used in the proof verification proc + ## [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> | signal_len<8> | signal ] + let lenPrefMsg = appendLength(@data) + var proofBytes = concat(@(proof.proof), + @(proof.merkleRoot), + @(proof.epoch), + @(proof.shareX), + @(proof.shareY), + @(proof.nullifier), + @(proof.rlnIdentifier), + lenPrefMsg) + + return proofBytes + +# Serializes a sequence of MerkleNodes +proc serialize*(roots: seq[MerkleNode]): seq[byte] = + var rootsBytes: seq[byte] = @[] + for root in roots: + rootsBytes = concat(rootsBytes, @root) + return rootsBytes + +proc serializeIdCommitments*(idComms: seq[IDCommitment]): seq[byte] = + ## serializes a seq of IDCommitments to a byte seq + ## the serialization is based on https://github.com/status-im/nwaku/blob/37bd29fbc37ce5cf636734e7dd410b1ed27b88c8/waku/v2/protocol/waku_rln_relay/rln.nim#L142 + ## the order of serialization is |id_commitment_len<8>|id_commitment| + var idCommsBytes = newSeq[byte]() + + # serialize the idComms, with its length prefixed + let len = toBytes(uint64(idComms.len), Endianness.littleEndian) + idCommsBytes.add(len) + + for idComm in idComms: + idCommsBytes = concat(idCommsBytes, @idComm) + + return idCommsBytes + +# Converts a sequence of tuples containing 4 string (i.e. identity trapdoor, nullifier, secret hash and commitment) to an IndentityCredential +proc toIdentityCredentials*(groupKeys: seq[(string, string, string, string)]): RlnRelayResult[seq[ + IdentityCredential]] = + ## groupKeys is sequence of membership key tuples in the form of (identity key, identity commitment) all in the hexadecimal format + ## the toIdentityCredentials proc populates a sequence of IdentityCredentials using the supplied groupKeys + ## Returns an error if the conversion fails + + var groupIdCredentials = newSeq[IdentityCredential]() + + for i in 0..groupKeys.len-1: + try: + let + idTrapdoor = hexToUint[IdentityTrapdoor.len*8](groupKeys[i][0]).toBytesLE() + idNullifier = hexToUint[IdentityNullifier.len*8](groupKeys[i][1]).toBytesLE() + idSecretHash = hexToUint[IdentitySecretHash.len*8](groupKeys[i][2]).toBytesLE() + idCommitment = hexToUint[IDCommitment.len*8](groupKeys[i][3]).toBytesLE() + groupIdCredentials.add(IdentityCredential(idTrapdoor: idTrapdoor, idNullifier: idNullifier, idSecretHash: idSecretHash, + idCommitment: idCommitment)) + except ValueError as err: + warn "could not convert the group key to bytes", err = err.msg + return err("could not convert the group key to bytes: " & err.msg) + return ok(groupIdCredentials) + +# Converts a sequence of tuples containing 2 string (i.e. identity secret hash and commitment) to an IndentityCredential +proc toIdentityCredentials*(groupKeys: seq[(string, string)]): RlnRelayResult[seq[ + IdentityCredential]] = + ## groupKeys is sequence of membership key tuples in the form of (identity key, identity commitment) all in the hexadecimal format + ## the toIdentityCredentials proc populates a sequence of IdentityCredentials using the supplied groupKeys + ## Returns an error if the conversion fails + + var groupIdCredentials = newSeq[IdentityCredential]() + + for i in 0..groupKeys.len-1: + try: + let + idSecretHash = hexToUint[IdentitySecretHash.len*8](groupKeys[i][0]).toBytesLE() + idCommitment = hexToUint[IDCommitment.len*8](groupKeys[i][1]).toBytesLE() + groupIdCredentials.add(IdentityCredential(idSecretHash: idSecretHash, + idCommitment: idCommitment)) + except ValueError as err: + warn "could not convert the group key to bytes", err = err.msg + return err("could not convert the group key to bytes: " & err.msg) + return ok(groupIdCredentials) + +proc toEpoch*(t: uint64): Epoch = + ## converts `t` to `Epoch` in little-endian order + let bytes = toBytes(t, Endianness.littleEndian) + debug "bytes", bytes = bytes + var epoch: Epoch + discard epoch.copyFrom(bytes) + return epoch + +proc fromEpoch*(epoch: Epoch): uint64 = + ## decodes bytes of `epoch` (in little-endian) to uint64 + let t = fromBytesLE(uint64, array[32, byte](epoch)) + return t diff --git a/waku/v2/protocol/waku_rln_relay/group_manager.nim b/waku/v2/protocol/waku_rln_relay/group_manager.nim new file mode 100644 index 000000000..1e1a17b48 --- /dev/null +++ b/waku/v2/protocol/waku_rln_relay/group_manager.nim @@ -0,0 +1,6 @@ +import + group_manager/[static, on_chain] + +export + static, + on_chain \ No newline at end of file diff --git a/waku/v2/protocol/waku_rln_relay/group_manager/group_manager_base.nim b/waku/v2/protocol/waku_rln_relay/group_manager/group_manager_base.nim new file mode 100644 index 000000000..12518e962 --- /dev/null +++ b/waku/v2/protocol/waku_rln_relay/group_manager/group_manager_base.nim @@ -0,0 +1,84 @@ +import + ../protocol_types +import + options, + chronos, + stew/results + +export + options, + chronos, + results, + protocol_types + +# This module contains the GroupManager interface +# The GroupManager is responsible for managing the group state +# It should be used to register new members, and withdraw existing members +# It should also be used to sync the group state with the rest of the group members + +type Membership* = object + idCommitment*: IDCommitment + index*: MembershipIndex + +type OnRegisterCallback* = proc (registrations: seq[Membership]): Future[void] {.gcsafe.} +type OnWithdrawCallback* = proc (withdrawals: seq[Membership]): Future[void] {.gcsafe.} + +type GroupManagerResult*[T] = Result[T, string] + +type + GroupManager*[Config] = ref object of RootObj + idCredentials*: Option[IdentityCredential] + registerCb*: Option[OnRegisterCallback] + withdrawCb*: Option[OnWithdrawCallback] + config*: Config + rlnInstance*: ptr RLN + initialized*: bool + latestIndex*: MembershipIndex + +# This proc is used to initialize the group manager +# Any initialization logic should be implemented here +method init*(g: GroupManager): Future[void] {.base,gcsafe.} = + return err("init proc for " & $g.kind & " is not implemented yet") + +# This proc is used to start the group sync process +# It should be used to sync the group state with the rest of the group members +method startGroupSync*(g: GroupManager): Future[void] {.base,gcsafe.} = + return err("startGroupSync proc for " & $g.kind & " is not implemented yet") + +# This proc is used to register a new identity commitment into the merkle tree +# The user may or may not have the identity secret to this commitment +# It should be used when detecting new members in the group, and syncing the group state +method register*(g: GroupManager, idCommitment: IDCommitment): Future[void] {.base,gcsafe.} = + return err("register proc for " & $g.kind & " is not implemented yet") + +# This proc is used to register a new identity commitment into the merkle tree +# The user should have the identity secret to this commitment +# It should be used when the user wants to join the group +method register*(g: GroupManager, credentials: IdentityCredential): Future[void] {.base,gcsafe.} = + return err("register proc for " & $g.kind & " is not implemented yet") + +# This proc is used to register a batch of new identity commitments into the merkle tree +# The user may or may not have the identity secret to these commitments +# It should be used when detecting a batch of new members in the group, and syncing the group state +method registerBatch*(g: GroupManager, idCommitments: seq[IDCommitment]): Future[void] {.base,gcsafe.} = + return err("registerBatch proc for " & $g.kind & " is not implemented yet") + +# This proc is used to set a callback that will be called when a new identity commitment is registered +# The callback may be called multiple times, and should be used to for any post processing +method onRegister*(g: GroupManager, cb: OnRegisterCallback) {.base,gcsafe.} = + g.registerCb = some(cb) + +# This proc is used to withdraw/remove an identity commitment from the merkle tree +# The user should have the identity secret hash to this commitment, by either deriving it, or owning it +method withdraw*(g: GroupManager, identitySecretHash: IdentitySecretHash): Future[void] {.base,gcsafe.} = + return err("withdraw proc for " & $g.kind & " is not implemented yet") + +# This proc is used to withdraw/remove a batch of identity commitments from the merkle tree +# The user should have the identity secret hash to these commitments, by either deriving them, or owning them +method withdrawBatch*(g: GroupManager, identitySecretHashes: seq[IdentitySecretHash]): Future[void] {.base,gcsafe.} = + return err("withdrawBatch proc for " & $g.kind & " is not implemented yet") + +# This proc is used to set a callback that will be called when an identity commitment is withdrawn +# The callback may be called multiple times, and should be used to for any post processing +method onWithdraw*(g: GroupManager, cb: OnWithdrawCallback) {.base,gcsafe.} = + g.withdrawCb = some(cb) diff --git a/waku/v2/protocol/waku_rln_relay/group_manager/on_chain.nim b/waku/v2/protocol/waku_rln_relay/group_manager/on_chain.nim new file mode 100644 index 000000000..e6a900f97 --- /dev/null +++ b/waku/v2/protocol/waku_rln_relay/group_manager/on_chain.nim @@ -0,0 +1,3 @@ +import on_chain/group_manager + +export group_manager \ No newline at end of file diff --git a/waku/v2/protocol/waku_rln_relay/group_manager/on_chain/group_manager.nim b/waku/v2/protocol/waku_rln_relay/group_manager/on_chain/group_manager.nim new file mode 100644 index 000000000..371987913 --- /dev/null +++ b/waku/v2/protocol/waku_rln_relay/group_manager/on_chain/group_manager.nim @@ -0,0 +1,328 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import + web3, + web3/ethtypes, + eth/keys as keys, + chronicles, + stint, + json, + std/tables, + stew/[byteutils, arrayops], + sequtils +import + ../../rln, + ../../conversion_utils, + ../group_manager_base + +from strutils import parseHexInt + +export group_manager_base + +logScope: + topics = "waku rln_relay onchain_group_manager" + +# membership contract interface +contract(RlnContract): + proc register(pubkey: Uint256) {.payable.} # external payable + proc MemberRegistered(pubkey: Uint256, index: Uint256) {.event.} + proc MEMBERSHIP_DEPOSIT(): Uint256 + # TODO the following are to be supported + # proc registerBatch(pubkeys: seq[Uint256]) # external payable + # proc withdraw(secret: Uint256, pubkeyIndex: Uint256, receiver: Address) + # proc withdrawBatch( secrets: seq[Uint256], pubkeyIndex: seq[Uint256], receiver: seq[Address]) + +type + RlnContractWithSender = Sender[RlnContract] + OnchainGroupManagerConfig* = object + ethClientUrl*: string + ethPrivateKey*: Option[string] + ethContractAddress*: string + ethRpc*: Option[Web3] + rlnContract*: Option[RlnContractWithSender] + membershipFee*: Option[Uint256] + membershipIndex*: Option[MembershipIndex] + latestProcessedBlock*: Option[BlockNumber] + + OnchainGroupManager* = ref object of GroupManager[OnchainGroupManagerConfig] + +template initializedGuard*(g: OnchainGroupManager): untyped = + if not g.initialized: + raise newException(ValueError, "OnchainGroupManager is not initialized") + +proc register*(g: OnchainGroupManager, idCommitment: IDCommitment): Future[void] {.async.} = + initializedGuard(g) + + let memberInserted = g.rlnInstance.insertMember(idCommitment) + if not memberInserted: + raise newException(ValueError,"member insertion failed") + + if g.registerCb.isSome(): + await g.registerCb.get()(@[Membership(idCommitment: idCommitment, index: g.latestIndex)]) + + g.latestIndex += 1 + + return + +proc registerBatch*(g: OnchainGroupManager, idCommitments: seq[IDCommitment]): Future[void] {.async.} = + initializedGuard(g) + + let membersInserted = g.rlnInstance.insertMembers(g.latestIndex, idCommitments) + if not membersInserted: + raise newException(ValueError, "Failed to insert members into the merkle tree") + + if g.registerCb.isSome(): + var membersSeq = newSeq[Membership]() + for i in 0 ..< idCommitments.len(): + var index = g.latestIndex + MembershipIndex(i) + debug "registering member", idCommitment = idCommitments[i], index = index, latestIndex = g.latestIndex + let member = Membership(idCommitment: idCommitments[i], index: index) + membersSeq.add(member) + await g.registerCb.get()(membersSeq) + + g.latestIndex += MembershipIndex(idCommitments.len()) + + return + +proc register*(g: OnchainGroupManager, identityCredentials: IdentityCredential): Future[void] {.async.} = + initializedGuard(g) + + let ethRpc = g.config.ethRpc.get() + let rlnContract = g.config.rlnContract.get() + let membershipFee = g.config.membershipFee.get() + + let gasPrice = int(await ethRpc.provider.eth_gasPrice()) * 2 + let idCommitment = identityCredentials.idCommitment.toUInt256() + + var txHash: TxHash + try: # send the registration transaction and check if any error occurs + txHash = await rlnContract.register(idCommitment).send(value = membershipFee, + gasPrice = gasPrice) + except ValueError as e: + raise newException(ValueError, "could not register the member: " & e.msg) + + let tsReceipt = await ethRpc.getMinedTransactionReceipt(txHash) + + # the receipt topic holds the hash of signature of the raised events + # TODO: make this robust. search within the event list for the event + let firstTopic = tsReceipt.logs[0].topics[0] + # the hash of the signature of MemberRegistered(uint256,uint256) event is equal to the following hex value + if firstTopic[0..65] != "0x5a92c2530f207992057b9c3e544108ffce3beda4a63719f316967c49bf6159d2": + raise newException(ValueError, "unexpected event signature") + + # the arguments of the raised event i.e., MemberRegistered are encoded inside the data field + # data = pk encoded as 256 bits || index encoded as 256 bits + let arguments = tsReceipt.logs[0].data + debug "tx log data", arguments=arguments + let + argumentsBytes = arguments.hexToSeqByte() + # In TX log data, uints are encoded in big endian + eventIndex = UInt256.fromBytesBE(argumentsBytes[32..^1]) + + g.config.membershipIndex = some(eventIndex.toMembershipIndex()) + + # don't handle member insertion into the tree here, it will be handled by the event listener + return + +proc withdraw*(g: OnchainGroupManager, idCommitment: IDCommitment): Future[void] {.async.} = + initializedGuard(g) + + # TODO: after slashing is enabled on the contract + +proc withdrawBatch*(g: OnchainGroupManager, idCommitments: seq[IDCommitment]): Future[void] {.async.} = + initializedGuard(g) + + # TODO: after slashing is enabled on the contract + +proc parseEvent*(event: type MemberRegistered, + log: JsonNode): GroupManagerResult[Membership] = + ## parses the `data` parameter of the `MemberRegistered` event `log` + ## returns an error if it cannot parse the `data` parameter + var idComm: UInt256 + var index: UInt256 + var data: string + # Remove the 0x prefix + try: + data = strip0xPrefix(log["data"].getStr()) + except CatchableError: + return err("failed to parse the data field of the MemberRegistered event: " & getCurrentExceptionMsg()) + var offset = 0 + try: + # Parse the idComm + offset += decode(data, offset, idComm) + # Parse the index + offset += decode(data, offset, index) + return ok(Membership(idCommitment: idComm.toIDCommitment(), index: index.toMembershipIndex())) + except: + return err("failed to parse the data field of the MemberRegistered event") + +type BlockTable* = OrderedTable[BlockNumber, seq[Membership]] + +proc getEvents*(g: OnchainGroupManager, fromBlock: BlockNumber, toBlock: Option[BlockNumber] = none(BlockNumber)): Future[BlockTable] {.async.} = + initializedGuard(g) + + let ethRpc = g.config.ethRpc.get() + let rlnContract = g.config.rlnContract.get() + + var normalizedToBlock: BlockNumber + if toBlock.isSome(): + var value = toBlock.get() + if value == 0: + # set to latest block + value = cast[BlockNumber](await ethRpc.provider.eth_blockNumber()) + normalizedToBlock = value + else: + normalizedToBlock = fromBlock + + var blockTable = default(BlockTable) + let events = await rlnContract.getJsonLogs(MemberRegistered, fromBlock = some(fromBlock.blockId()), toBlock = some(normalizedToBlock.blockId())) + if events.len == 0: + debug "no events found" + return blockTable + + for event in events: + let blockNumber = parseHexInt(event["blockNumber"].getStr()).uint + let parsedEventRes = parseEvent(MemberRegistered, event) + if parsedEventRes.isErr(): + error "failed to parse the MemberRegistered event", error=parsedEventRes.error() + raise newException(ValueError, "failed to parse the MemberRegistered event") + let parsedEvent = parsedEventRes.get() + + if blockTable.hasKey(blockNumber): + blockTable[blockNumber].add(parsedEvent) + else: + blockTable[blockNumber] = @[parsedEvent] + + return blockTable + +proc seedBlockTableIntoTree*(g: OnchainGroupManager, blockTable: BlockTable): Future[void] {.async.} = + initializedGuard(g) + + for blockNumber, members in blockTable.pairs(): + let latestIndex = g.latestIndex + let startingIndex = members[0].index + try: + await g.registerBatch(members.mapIt(it.idCommitment)) + except: + error "failed to insert members into the tree" + raise newException(ValueError, "failed to insert members into the tree") + debug "new members added to the Merkle tree", commitments=members.mapIt(it.idCommitment.inHex()) , startingIndex=startingIndex + let lastIndex = startingIndex + members.len.uint - 1 + let indexGap = startingIndex - latestIndex + if not (toSeq(startingIndex..lastIndex) == members.mapIt(it.index)): + raise newException(ValueError, "membership indices are not sequential") + if indexGap != 1.uint and lastIndex != latestIndex: + warn "membership index gap, may have lost connection", lastIndex, currIndex=latestIndex, indexGap = indexGap + g.config.latestProcessedBlock = some(blockNumber) + + return + +proc getEventsAndSeedIntoTree*(g: OnchainGroupManager, fromBlock: BlockNumber, toBlock: Option[BlockNumber] = none(BlockNumber)): Future[void] {.async.} = + initializedGuard(g) + + let events = await g.getEvents(fromBlock, toBlock) + await g.seedBlockTableIntoTree(events) + return + +proc getNewHeadCallback*(g: OnchainGroupManager): BlockHeaderHandler = + proc newHeadCallback(blockheader: BlockHeader) {.gcsafe.} = + let latestBlock = blockheader.number.uint + debug "block received", blockNumber = latestBlock + # get logs from the last block + try: + asyncSpawn g.getEventsAndSeedIntoTree(latestBlock) + except CatchableError: + warn "failed to handle log: ", error=getCurrentExceptionMsg() + return newHeadCallback + +proc newHeadErrCallback(error: CatchableError) = + warn "failed to get new head", error=error.msg + +proc startListeningToEvents*(g: OnchainGroupManager): Future[void] {.async.} = + initializedGuard(g) + + let ethRpc = g.config.ethRpc.get() + let newHeadCallback = g.getNewHeadCallback() + try: + discard await ethRpc.subscribeForBlockHeaders(newHeadCallback, newHeadErrCallback) + except: + raise newException(ValueError, "failed to subscribe to block headers: " & getCurrentExceptionMsg()) + +proc startOnchainSync*(g: OnchainGroupManager, fromBlock: BlockNumber = BlockNumber(0)): Future[void] {.async.} = + initializedGuard(g) + + try: + await g.getEventsAndSeedIntoTree(fromBlock, some(fromBlock)) + except: + raise newException(ValueError, "failed to get the history/reconcile missed blocks: " & getCurrentExceptionMsg()) + + # listen to blockheaders and contract events + try: + await g.startListeningToEvents() + except: + raise newException(ValueError, "failed to start listening to events: " & getCurrentExceptionMsg()) + +proc startGroupSync*(g: OnchainGroupManager): Future[void] {.async.} = + initializedGuard(g) + # Get archive history + try: + await startOnchainSync(g) + except: + raise newException(ValueError, "failed to start onchain sync service: " & getCurrentExceptionMsg()) + + if g.config.ethPrivateKey.isSome() and g.idCredentials.isSome(): + debug "registering commitment on contract" + await g.register(g.idCredentials.get()) + + return + +proc onRegister*(g: OnchainGroupManager, cb: OnRegisterCallback) {.gcsafe.} = + g.registerCb = some(cb) + +proc onWithdraw*(g: OnchainGroupManager, cb: OnWithdrawCallback) {.gcsafe.} = + g.withdrawCb = some(cb) + +proc init*(g: OnchainGroupManager): Future[void] {.async.} = + var ethRpc: Web3 + var contract: RlnContractWithSender + # check if the Ethereum client is reachable + try: + ethRpc = await newWeb3(g.config.ethClientUrl) + except: + raise newException(ValueError, "could not connect to the Ethereum client") + + let contractAddress = web3.fromHex(web3.Address, g.config.ethContractAddress) + contract = ethRpc.contractSender(RlnContract, contractAddress) + + # check if the contract exists by calling a static function + var membershipFee: Uint256 + try: + membershipFee = await contract.MEMBERSHIP_DEPOSIT().call() + except: + raise newException(ValueError, "could not get the membership deposit") + + if g.config.ethPrivateKey.isSome(): + let pk = string(g.config.ethPrivateKey.get()) + let pkParseRes = keys.PrivateKey.fromHex(pk) + if pkParseRes.isErr(): + raise newException(ValueError, "could not parse the private key") + ethRpc.privateKey = some(pkParseRes.get()) + + g.config.ethRpc = some(ethRpc) + g.config.rlnContract = some(contract) + g.config.membershipFee = some(membershipFee) + + + ethRpc.ondisconnect = proc() = + error "Ethereum client disconnected" + let fromBlock = g.config.latestProcessedBlock.get() + info "reconnecting with the Ethereum client, and restarting group sync", fromBlock = fromBlock + try: + asyncSpawn g.startOnchainSync(fromBlock) + except: + error "failed to restart group sync" + + g.initialized = true diff --git a/waku/v2/protocol/waku_rln_relay/group_manager/static.nim b/waku/v2/protocol/waku_rln_relay/group_manager/static.nim new file mode 100644 index 000000000..64a0f8ec3 --- /dev/null +++ b/waku/v2/protocol/waku_rln_relay/group_manager/static.nim @@ -0,0 +1,3 @@ +import static/group_manager + +export group_manager \ No newline at end of file diff --git a/waku/v2/protocol/waku_rln_relay/group_manager/static/group_manager.nim b/waku/v2/protocol/waku_rln_relay/group_manager/static/group_manager.nim new file mode 100644 index 000000000..0de9e619d --- /dev/null +++ b/waku/v2/protocol/waku_rln_relay/group_manager/static/group_manager.nim @@ -0,0 +1,110 @@ +import + ../group_manager_base, + ../../rln, + std/sequtils + +export + group_manager_base + +type + StaticGroupManagerConfig* = object + groupKeys*: seq[IdentityCredential] + groupSize*: uint + membershipIndex*: MembershipIndex + + StaticGroupManager* = ref object of GroupManager[StaticGroupManagerConfig] + +template initializedGuard*(g: StaticGroupManager): untyped = + if not g.initialized: + raise newException(ValueError, "StaticGroupManager is not initialized") + +proc init*(g: StaticGroupManager): Future[void] {.async,gcsafe.} = + let + groupSize = g.config.groupSize + groupKeys = g.config.groupKeys + membershipIndex = g.config.membershipIndex + + if membershipIndex < MembershipIndex(0) or membershipIndex >= MembershipIndex(groupSize): + raise newException(ValueError, "Invalid membership index. Must be within 0 and " & $(groupSize - 1) & "but was " & $membershipIndex) + g.idCredentials = some(groupKeys[membershipIndex]) + + # Seed the received commitments into the merkle tree + let idCommitments = groupKeys.mapIt(it.idCommitment) + let membersInserted = g.rlnInstance.insertMembers(g.latestIndex, idCommitments) + if not membersInserted: + raise newException(ValueError, "Failed to insert members into the merkle tree") + + g.latestIndex += MembershipIndex(idCommitments.len() - 1) + + g.initialized = true + + return + +proc startGroupSync*(g: StaticGroupManager): Future[void] = + initializedGuard(g) + var retFuture = newFuture[void]("StaticGroupManager.sta rtGroupSync") + # No-op + retFuture.complete() + return retFuture + +proc register*(g: StaticGroupManager, idCommitment: IDCommitment): Future[void] {.async.} = + initializedGuard(g) + + let memberInserted = g.rlnInstance.insertMember(idCommitment) + if not memberInserted: + raise newException(ValueError, "Failed to insert member into the merkle tree") + + g.latestIndex += 1 + + if g.registerCb.isSome(): + await g.registerCb.get()(@[Membership(idCommitment: idCommitment, index: g.latestIndex)]) + return + +proc registerBatch*(g: StaticGroupManager, idCommitments: seq[IDCommitment]): Future[void] {.async.} = + initializedGuard(g) + + let membersInserted = g.rlnInstance.insertMembers(g.latestIndex + 1, idCommitments) + if not membersInserted: + raise newException(ValueError, "Failed to insert members into the merkle tree") + + if g.registerCb.isSome(): + var memberSeq = newSeq[Membership]() + for i in 0.. | identity_nullifier<32> | identity_secret_hash<32> | id_commitment<32> | -## identity secret hash is the poseidon hash of [identity_trapdoor, identity_nullifier] -## id commitment is the poseidon hash of the identity secret hash -## the return bool value indicates the success or failure of the operation - -proc seeded_key_gen*(ctx: ptr RLN, input_buffer: ptr Buffer, output_buffer: ptr Buffer): bool {.importc: "seeded_extended_key_gen".} -## generates identity trapdoor, identity nullifier, identity secret hash and id commitment tuple serialized inside output_buffer as | identity_trapdoor<32> | identity_nullifier<32> | identity_secret_hash<32> | id_commitment<32> | using ChaCha20 -## seeded with an arbitrary long seed serialized in input_buffer -## The input seed provided by the user is hashed using Keccak256 before being passed to ChaCha20 as seed. -## identity secret hash is the poseidon hash of [identity_trapdoor, identity_nullifier] -## id commitment is the poseidon hash of the identity secret hash -## the return bool value indicates the success or failure of the operation - -proc generate_proof*(ctx: ptr RLN, - input_buffer: ptr Buffer, - output_buffer: ptr Buffer): bool {.importc: "generate_rln_proof".} -## input_buffer has to be serialized as [ id_key<32> | id_index<8> | epoch<32> | signal_len<8> | signal ] -## output_buffer holds the proof data and should be parsed as [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> ] -## integers wrapped in <> indicate value sizes in bytes -## the return bool value indicates the success or failure of the operation - -proc verify*(ctx: ptr RLN, - proof_buffer: ptr Buffer, - proof_is_valid_ptr: ptr bool): bool {.importc: "verify_rln_proof".} -## proof_buffer has to be serialized as [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> | signal_len<8> | signal ] -## the return bool value indicates the success or failure of the call to the verify function -## the verification of the zk proof is available in proof_is_valid_ptr, where a value of true indicates success and false a failure - -proc verify_with_roots*(ctx: ptr RLN, - proof_buffer: ptr Buffer, - roots_buffer: ptr Buffer, - proof_is_valid_ptr: ptr bool): bool {.importc: "verify_with_roots".} -## proof_buffer has to be serialized as [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> | signal_len<8> | signal ] -## roots_buffer contains the concatenation of 32 bytes long serializations in little endian of root values -## the return bool value indicates the success or failure of the call to the verify function -## the verification of the zk proof is available in proof_is_valid_ptr, where a value of true indicates success and false a failure - -proc zk_prove*(ctx: ptr RLN, - input_buffer: ptr Buffer, - output_buffer: ptr Buffer): bool {.importc: "prove".} -## Computes the zkSNARK proof and stores it in output_buffer for input values stored in input_buffer -## input_buffer is serialized as input_data as [ id_key<32> | path_elements> | identity_path_index> | x<32> | epoch<32> | rln_identifier<32> ] -## output_buffer holds the proof data and should be parsed as [ proof<128> ] -## path_elements and indentity_path elements serialize a merkle proof for id_key and are vectors of elements of 32 and 1 bytes, respectively (not. Vec<>). -## x is the x coordinate of the Shamir's secret share for which the proof is computed -## epoch is the input epoch (equivalently, the nullifier) -## the return bool value indicates the success or failure of the operation - -proc zk_verify*(ctx: ptr RLN, - proof_buffer: ptr Buffer, - proof_is_valid_ptr: ptr bool): bool {.importc: "verify".} -## Verifies the zkSNARK proof passed in proof_buffer -## input_buffer is serialized as input_data as [ proof<128> ] -## the verification of the zk proof is available in proof_is_valid_ptr, where a value of true indicates success and false a failure -## the return bool value indicates the success or failure of the operation - -#---------------------------------------------------------------------------------------------- - -#-------------------------------- Common procedures ------------------------------------------- -proc new_circuit*(tree_height: uint, input_buffer: ptr Buffer, ctx: ptr (ptr RLN)): bool {.importc: "new".} -## creates an instance of rln object as defined by the zerokit RLN lib -## tree_height represent the depth of the Merkle tree -## input_buffer contains a serialization of the path where the circuit resources can be found (.r1cs, .wasm, .zkey and optionally the verification_key.json) -## ctx holds the final created rln object -## the return bool value indicates the success or failure of the operation - -proc new_circuit_from_data*(tree_height: uint, circom_buffer: ptr Buffer, zkey_buffer: ptr Buffer, vk_buffer: ptr Buffer, ctx: ptr (ptr RLN)): bool {.importc: "new_with_params".} -## creates an instance of rln object as defined by the zerokit RLN lib by passing the required inputs as byte arrays -## tree_height represent the depth of the Merkle tree -## circom_buffer contains the bytes read from the Circom .wasm circuit -## zkey_buffer contains the bytes read from the .zkey proving key -## vk_buffer contains the bytes read from the verification_key.json -## ctx holds the final created rln object -## the return bool value indicates the success or failure of the operation - -proc hash*(ctx: ptr RLN, - input_buffer: ptr Buffer, - output_buffer: ptr Buffer): bool {.importc: "hash".} -## it hashes (sha256) the plain text supplied in inputs_buffer and then maps it to a field element -## this proc is used to map arbitrary signals to field element for the sake of proof generation -## inputs_buffer holds the hash input as a byte seq -## the hash output is generated and populated inside output_buffer -## the output_buffer contains 32 bytes hash output +export + rln_interface, + wrappers diff --git a/waku/v2/protocol/waku_rln_relay/rln/rln_interface.nim b/waku/v2/protocol/waku_rln_relay/rln/rln_interface.nim new file mode 100644 index 000000000..b3c175a5c --- /dev/null +++ b/waku/v2/protocol/waku_rln_relay/rln/rln_interface.nim @@ -0,0 +1,161 @@ +## Nim wrappers for the functions defined in librln +import + ../protocol_types + + +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + + +## Buffer struct is taken from +# https://github.com/celo-org/celo-threshold-bls-rs/blob/master/crates/threshold-bls-ffi/src/ffi.rs +type Buffer* = object + `ptr`*: ptr uint8 + len*: uint + +proc toBuffer*(x: openArray[byte]): Buffer = + ## converts the input to a Buffer object + ## the Buffer object is used to communicate data with the rln lib + var temp = @x + let baseAddr = cast[pointer](x) + let output = Buffer(`ptr`: cast[ptr uint8](baseAddr), len: uint(temp.len)) + return output + + +###################################################################### +## RLN Zerokit module APIs +###################################################################### + +#------------------------------ Merkle Tree operations ----------------------------------------- +proc update_next_member*(ctx: ptr RLN, input_buffer: ptr Buffer): bool {.importc: "set_next_leaf".} +## adds an element in the merkle tree to the next available position +## input_buffer points to the id commitment byte seq +## the return bool value indicates the success or failure of the operation + +proc delete_member*(ctx: ptr RLN, index: uint): bool {.importc: "delete_leaf".} +## index is the position of the id commitment key to be deleted from the tree +## the deleted id commitment key is replaced with a zero leaf +## the return bool value indicates the success or failure of the operation + +proc get_root*(ctx: ptr RLN, output_buffer: ptr Buffer): bool {.importc: "get_root".} +## get_root populates the passed pointer output_buffer with the current tree root +## the output_buffer holds the Merkle tree root of size 32 bytes +## the return bool value indicates the success or failure of the operation + +proc get_merkle_proof*(ctx: ptr RLN, index: uint, output_buffer: ptr Buffer): bool {.importc: "get_proof".} +## populates the passed pointer output_buffer with the merkle proof for the leaf at position index in the tree stored by ctx +## the output_buffer holds a serialized Merkle proof (vector of 32 bytes nodes) +## the return bool value indicates the success or failure of the operation + +proc set_leaf*(ctx: ptr RLN, index: uint, input_buffer: ptr Buffer): bool {.importc: "set_leaf".} +## sets the leaf at position index in the tree stored by ctx to the value passed by input_buffer +## the input_buffer holds a serialized leaf of 32 bytes +## the return bool value indicates the success or failure of the operation + +proc init_tree_with_leaves*(ctx: ptr RLN, input_buffer: ptr Buffer): bool {.importc: "init_tree_with_leaves".} +## sets multiple leaves in the tree stored by ctx to the value passed by input_buffer +## the input_buffer holds a serialized vector of leaves (32 bytes each) +## the input_buffer size is prefixed by a 8 bytes integer indicating the number of leaves +## leaves are set one after each other starting from index 0 +## the return bool value indicates the success or failure of the operation + +proc set_leaves_from*(ctx: ptr RLN, index: uint, input_buffer: ptr Buffer): bool {.importc: "set_leaves_from".} +## sets multiple leaves in the tree stored by ctx to the value passed by input_buffer +## the input_buffer holds a serialized vector of leaves (32 bytes each) +## the input_buffer size is prefixed by a 8 bytes integer indicating the number of leaves +## leaves are set one after each other starting from index `index` +## the return bool value indicates the success or failure of the operation + +proc reset_tree*(ctx: ptr RLN, tree_height: uint): bool {.importc: "set_tree".} +## resets the tree stored by ctx to the the empty tree (all leaves set to 0) of height tree_height +## the return bool value indicates the success or failure of the operation + +#---------------------------------------------------------------------------------------------- + +#-------------------------------- zkSNARKs operations ----------------------------------------- +proc key_gen*(ctx: ptr RLN, output_buffer: ptr Buffer): bool {.importc: "extended_key_gen".} +## generates identity trapdoor, identity nullifier, identity secret hash and id commitment tuple serialized inside output_buffer as | identity_trapdoor<32> | identity_nullifier<32> | identity_secret_hash<32> | id_commitment<32> | +## identity secret hash is the poseidon hash of [identity_trapdoor, identity_nullifier] +## id commitment is the poseidon hash of the identity secret hash +## the return bool value indicates the success or failure of the operation + +proc seeded_key_gen*(ctx: ptr RLN, input_buffer: ptr Buffer, output_buffer: ptr Buffer): bool {.importc: "seeded_extended_key_gen".} +## generates identity trapdoor, identity nullifier, identity secret hash and id commitment tuple serialized inside output_buffer as | identity_trapdoor<32> | identity_nullifier<32> | identity_secret_hash<32> | id_commitment<32> | using ChaCha20 +## seeded with an arbitrary long seed serialized in input_buffer +## The input seed provided by the user is hashed using Keccak256 before being passed to ChaCha20 as seed. +## identity secret hash is the poseidon hash of [identity_trapdoor, identity_nullifier] +## id commitment is the poseidon hash of the identity secret hash +## the return bool value indicates the success or failure of the operation + +proc generate_proof*(ctx: ptr RLN, + input_buffer: ptr Buffer, + output_buffer: ptr Buffer): bool {.importc: "generate_rln_proof".} +## input_buffer has to be serialized as [ id_key<32> | id_index<8> | epoch<32> | signal_len<8> | signal ] +## output_buffer holds the proof data and should be parsed as [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> ] +## integers wrapped in <> indicate value sizes in bytes +## the return bool value indicates the success or failure of the operation + +proc verify*(ctx: ptr RLN, + proof_buffer: ptr Buffer, + proof_is_valid_ptr: ptr bool): bool {.importc: "verify_rln_proof".} +## proof_buffer has to be serialized as [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> | signal_len<8> | signal ] +## the return bool value indicates the success or failure of the call to the verify function +## the verification of the zk proof is available in proof_is_valid_ptr, where a value of true indicates success and false a failure + +proc verify_with_roots*(ctx: ptr RLN, + proof_buffer: ptr Buffer, + roots_buffer: ptr Buffer, + proof_is_valid_ptr: ptr bool): bool {.importc: "verify_with_roots".} +## proof_buffer has to be serialized as [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> | signal_len<8> | signal ] +## roots_buffer contains the concatenation of 32 bytes long serializations in little endian of root values +## the return bool value indicates the success or failure of the call to the verify function +## the verification of the zk proof is available in proof_is_valid_ptr, where a value of true indicates success and false a failure + +proc zk_prove*(ctx: ptr RLN, + input_buffer: ptr Buffer, + output_buffer: ptr Buffer): bool {.importc: "prove".} +## Computes the zkSNARK proof and stores it in output_buffer for input values stored in input_buffer +## input_buffer is serialized as input_data as [ id_key<32> | path_elements> | identity_path_index> | x<32> | epoch<32> | rln_identifier<32> ] +## output_buffer holds the proof data and should be parsed as [ proof<128> ] +## path_elements and indentity_path elements serialize a merkle proof for id_key and are vectors of elements of 32 and 1 bytes, respectively (not. Vec<>). +## x is the x coordinate of the Shamir's secret share for which the proof is computed +## epoch is the input epoch (equivalently, the nullifier) +## the return bool value indicates the success or failure of the operation + +proc zk_verify*(ctx: ptr RLN, + proof_buffer: ptr Buffer, + proof_is_valid_ptr: ptr bool): bool {.importc: "verify".} +## Verifies the zkSNARK proof passed in proof_buffer +## input_buffer is serialized as input_data as [ proof<128> ] +## the verification of the zk proof is available in proof_is_valid_ptr, where a value of true indicates success and false a failure +## the return bool value indicates the success or failure of the operation + +#---------------------------------------------------------------------------------------------- + +#-------------------------------- Common procedures ------------------------------------------- +proc new_circuit*(tree_height: uint, input_buffer: ptr Buffer, ctx: ptr (ptr RLN)): bool {.importc: "new".} +## creates an instance of rln object as defined by the zerokit RLN lib +## tree_height represent the depth of the Merkle tree +## input_buffer contains a serialization of the path where the circuit resources can be found (.r1cs, .wasm, .zkey and optionally the verification_key.json) +## ctx holds the final created rln object +## the return bool value indicates the success or failure of the operation + +proc new_circuit_from_data*(tree_height: uint, circom_buffer: ptr Buffer, zkey_buffer: ptr Buffer, vk_buffer: ptr Buffer, ctx: ptr (ptr RLN)): bool {.importc: "new_with_params".} +## creates an instance of rln object as defined by the zerokit RLN lib by passing the required inputs as byte arrays +## tree_height represent the depth of the Merkle tree +## circom_buffer contains the bytes read from the Circom .wasm circuit +## zkey_buffer contains the bytes read from the .zkey proving key +## vk_buffer contains the bytes read from the verification_key.json +## ctx holds the final created rln object +## the return bool value indicates the success or failure of the operation + +proc hash*(ctx: ptr RLN, + input_buffer: ptr Buffer, + output_buffer: ptr Buffer): bool {.importc: "hash".} +## it hashes (sha256) the plain text supplied in inputs_buffer and then maps it to a field element +## this proc is used to map arbitrary signals to field element for the sake of proof generation +## inputs_buffer holds the hash input as a byte seq +## the hash output is generated and populated inside output_buffer +## the output_buffer contains 32 bytes hash output diff --git a/waku/v2/protocol/waku_rln_relay/rln/wrappers.nim b/waku/v2/protocol/waku_rln_relay/rln/wrappers.nim new file mode 100644 index 000000000..d80837a86 --- /dev/null +++ b/waku/v2/protocol/waku_rln_relay/rln/wrappers.nim @@ -0,0 +1,235 @@ +import + chronicles, + options, + stew/[arrayops, results], + nimcrypto/utils + +import + ./rln_interface, + ../conversion_utils, + ../protocol_types, + ../protocol_metrics, + ../constants +import + ../../../utils/time + +logScope: + topics = "waku rln_relay ffi" + +proc membershipKeyGen*(ctxPtr: ptr RLN): RlnRelayResult[IdentityCredential] = + ## generates a IdentityCredential that can be used for the registration into the rln membership contract + ## Returns an error if the key generation fails + + # keysBufferPtr will hold the generated identity tuple i.e., trapdoor, nullifier, secret hash and commitment + var + keysBuffer: Buffer + keysBufferPtr = addr(keysBuffer) + done = key_gen(ctxPtr, keysBufferPtr) + + # check whether the keys are generated successfully + if(done == false): + return err("error in key generation") + + var generatedKeys = cast[ptr array[4*32, byte]](keysBufferPtr.`ptr`)[] + # the public and secret keys together are 64 bytes + if (generatedKeys.len != 4*32): + return err("generated keys are of invalid length") + + # TODO define a separate proc to decode the generated keys to the secret and public components + var + idTrapdoor: array[32, byte] + idNullifier: array[32, byte] + idSecretHash: array[32, byte] + idCommitment: array[32, byte] + for (i, x) in idTrapdoor.mpairs: x = generatedKeys[i+0*32] + for (i, x) in idNullifier.mpairs: x = generatedKeys[i+1*32] + for (i, x) in idSecretHash.mpairs: x = generatedKeys[i+2*32] + for (i, x) in idCommitment.mpairs: x = generatedKeys[i+3*32] + + var + identityCredential = IdentityCredential(idTrapdoor: idTrapdoor, idNullifier: idNullifier, idSecretHash: idSecretHash, idCommitment: idCommitment) + + return ok(identityCredential) + +proc createRLNInstanceLocal*(d: int = MerkleTreeDepth): RLNResult = + ## generates an instance of RLN + ## An RLN instance supports both zkSNARKs logics and Merkle tree data structure and operations + ## d indicates the depth of Merkle tree + ## Returns an error if the instance creation fails + var + rlnInstance: ptr RLN + merkleDepth: csize_t = uint(d) + resourcesPathBuffer = RlnResourceFolder.toOpenArrayByte(0, RlnResourceFolder.high).toBuffer() + + # create an instance of RLN + let res = new_circuit(merkleDepth, addr resourcesPathBuffer, addr rlnInstance) + # check whether the circuit parameters are generated successfully + if (res == false): + debug "error in parameters generation" + return err("error in parameters generation") + return ok(rlnInstance) + +proc createRLNInstance*(d: int = MerkleTreeDepth): RLNResult = + ## Wraps the rln instance creation for metrics + ## Returns an error if the instance creation fails + var res: RLNResult + waku_rln_instance_creation_duration_seconds.nanosecondTime: + res = createRLNInstanceLocal(d) + return res + +proc hash*(rlnInstance: ptr RLN, data: openArray[byte]): MerkleNode = + ## a thin layer on top of the Nim wrapper of the Poseidon hasher + debug "hash input", hashhex = data.toHex() + var lenPrefData = appendLength(data) + var + hashInputBuffer = lenPrefData.toBuffer() + outputBuffer: Buffer # will holds the hash output + + debug "hash input buffer length", bufflen = hashInputBuffer.len + let + hashSuccess = hash(rlnInstance, addr hashInputBuffer, addr outputBuffer) + + # check whether the hash call is done successfully + if not hashSuccess: + debug "error in hash" + return default(MerkleNode) + + let + output = cast[ptr MerkleNode](outputBuffer.`ptr`)[] + + return output + +proc proofGen*(rlnInstance: ptr RLN, data: openArray[byte], + memKeys: IdentityCredential, memIndex: MembershipIndex, + epoch: Epoch): RateLimitProofResult = + + # serialize inputs + let serializedInputs = serialize(idSecretHash = memKeys.idSecretHash, + memIndex = memIndex, + epoch = epoch, + msg = data) + var inputBuffer = toBuffer(serializedInputs) + + debug "input buffer ", inputBuffer= repr(inputBuffer) + + # generate the proof + var proof: Buffer + let proofIsSuccessful = generate_proof(rlnInstance, addr inputBuffer, addr proof) + # check whether the generate_proof call is done successfully + if not proofIsSuccessful: + return err("could not generate the proof") + + var proofValue = cast[ptr array[320, byte]] (proof.`ptr`) + let proofBytes: array[320, byte] = proofValue[] + debug "proof content", proofHex = proofValue[].toHex + + ## parse the proof as [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> ] + + let + proofOffset = 128 + rootOffset = proofOffset + 32 + epochOffset = rootOffset + 32 + shareXOffset = epochOffset + 32 + shareYOffset = shareXOffset + 32 + nullifierOffset = shareYOffset + 32 + rlnIdentifierOffset = nullifierOffset + 32 + + var + zkproof: ZKSNARK + proofRoot, shareX, shareY: MerkleNode + epoch: Epoch + nullifier: Nullifier + rlnIdentifier: RlnIdentifier + + discard zkproof.copyFrom(proofBytes[0..proofOffset-1]) + discard proofRoot.copyFrom(proofBytes[proofOffset..rootOffset-1]) + discard epoch.copyFrom(proofBytes[rootOffset..epochOffset-1]) + discard shareX.copyFrom(proofBytes[epochOffset..shareXOffset-1]) + discard shareY.copyFrom(proofBytes[shareXOffset..shareYOffset-1]) + discard nullifier.copyFrom(proofBytes[shareYOffset..nullifierOffset-1]) + discard rlnIdentifier.copyFrom(proofBytes[nullifierOffset..rlnIdentifierOffset-1]) + + let output = RateLimitProof(proof: zkproof, + merkleRoot: proofRoot, + epoch: epoch, + shareX: shareX, + shareY: shareY, + nullifier: nullifier, + rlnIdentifier: rlnIdentifier) + + return ok(output) + +# validRoots should contain a sequence of roots in the acceptable windows. +# As default, it is set to an empty sequence of roots. This implies that the validity check for the proof's root is skipped +proc proofVerify*(rlnInstance: ptr RLN, + data: openArray[byte], + proof: RateLimitProof, + validRoots: seq[MerkleNode] = @[]): RlnRelayResult[bool] = + ## verifies the proof, returns an error if the proof verification fails + ## returns true if the proof is valid + var + proofBytes = serialize(proof, data) + proofBuffer = proofBytes.toBuffer() + validProof: bool + rootsBytes = serialize(validRoots) + rootsBuffer = rootsBytes.toBuffer() + + trace "serialized proof", proof = proofBytes.toHex() + + let verifyIsSuccessful = verify_with_roots(rlnInstance, addr proofBuffer, addr rootsBuffer, addr validProof) + if not verifyIsSuccessful: + # something went wrong in verification call + warn "could not verify validity of the proof", proof=proof + return err("could not verify the proof") + + if not validProof: + return ok(false) + else: + return ok(true) + +proc insertMember*(rlnInstance: ptr RLN, idComm: IDCommitment): bool = + ## inserts a member to the tree + ## returns true if the member is inserted successfully + ## returns false if the member could not be inserted + var pkBuffer = toBuffer(idComm) + let pkBufferPtr = addr pkBuffer + + # add the member to the tree + let memberAdded = update_next_member(rlnInstance, pkBufferPtr) + return memberAdded + + +proc insertMembers*(rlnInstance: ptr RLN, + index: MembershipIndex, + idComms: seq[IDCommitment]): bool = + ## Insert multiple members i.e., identity commitments + ## returns true if the insertion is successful + ## returns false if any of the insertions fails + ## Note: This proc is atomic, i.e., if any of the insertions fails, all the previous insertions are rolled back + + # serialize the idComms + let idCommsBytes = serializeIdCommitments(idComms) + + var idCommsBuffer = idCommsBytes.toBuffer() + let idCommsBufferPtr = addr idCommsBuffer + # add the member to the tree + let membersAdded = set_leaves_from(rlnInstance, index, idCommsBufferPtr) + return membersAdded + +proc removeMember*(rlnInstance: ptr RLN, index: MembershipIndex): bool = + let deletion_success = delete_member(rlnInstance, index) + return deletion_success + +proc getMerkleRoot*(rlnInstance: ptr RLN): MerkleNodeResult = + # read the Merkle Tree root after insertion + var + root {.noinit.}: Buffer = Buffer() + rootPtr = addr(root) + getRootSuccessful = getRoot(rlnInstance, rootPtr) + if not getRootSuccessful: + return err("could not get the root") + if not root.len == 32: + return err("wrong output size") + + var rootValue = cast[ptr MerkleNode] (root.`ptr`)[] + return ok(rootValue) diff --git a/waku/v2/protocol/waku_rln_relay/utils.nim b/waku/v2/protocol/waku_rln_relay/utils.nim index f62e1c365..3725f4067 100644 --- a/waku/v2/protocol/waku_rln_relay/utils.nim +++ b/waku/v2/protocol/waku_rln_relay/utils.nim @@ -14,9 +14,10 @@ import libp2p/protocols/pubsub/rpc/messages, libp2p/protocols/pubsub/pubsub, stew/results, - stew/[byteutils, arrayops, endians2] + stew/[byteutils, arrayops] import ./rln, + ./conversion_utils, ./constants, ./protocol_types, ./protocol_metrics @@ -57,94 +58,23 @@ contract(MembershipContract): # proc withdraw(secret: Uint256, pubkeyIndex: Uint256, receiver: Address) # proc withdrawBatch( secrets: seq[Uint256], pubkeyIndex: seq[Uint256], receiver: seq[Address]) -proc toBuffer*(x: openArray[byte]): Buffer = - ## converts the input to a Buffer object - ## the Buffer object is used to communicate data with the rln lib - var temp = @x - let baseAddr = cast[pointer](x) - let output = Buffer(`ptr`: cast[ptr uint8](baseAddr), len: uint(temp.len)) - return output - -proc createRLNInstanceLocal(d: int = MerkleTreeDepth): RLNResult = - ## generates an instance of RLN - ## An RLN instance supports both zkSNARKs logics and Merkle tree data structure and operations - ## d indicates the depth of Merkle tree - ## Returns an error if the instance creation fails - var - rlnInstance: ptr RLN - merkleDepth: csize_t = uint(d) - resourcesPathBuffer = RlnResourceFolder.toOpenArrayByte(0, RlnResourceFolder.high).toBuffer() - - # create an instance of RLN - let res = new_circuit(merkleDepth, addr resourcesPathBuffer, addr rlnInstance) - # check whether the circuit parameters are generated successfully - if (res == false): - debug "error in parameters generation" - return err("error in parameters generation") - return ok(rlnInstance) - -proc membershipKeyGen*(ctxPtr: ptr RLN): RlnRelayResult[IdentityCredential] = - ## generates a IdentityCredential that can be used for the registration into the rln membership contract - ## Returns an error if the key generation fails - - # keysBufferPtr will hold the generated identity tuple i.e., trapdoor, nullifier, secret hash and commitment - var - keysBuffer: Buffer - keysBufferPtr = addr(keysBuffer) - done = key_gen(ctxPtr, keysBufferPtr) - - # check whether the keys are generated successfully - if(done == false): - return err("error in key generation") - - var generatedKeys = cast[ptr array[4*32, byte]](keysBufferPtr.`ptr`)[] - # the public and secret keys together are 64 bytes - if (generatedKeys.len != 4*32): - return err("generated keys are of invalid length") - - # TODO define a separate proc to decode the generated keys to the secret and public components - var - idTrapdoor: array[32, byte] - idNullifier: array[32, byte] - idSecretHash: array[32, byte] - idCommitment: array[32, byte] - for (i, x) in idTrapdoor.mpairs: x = generatedKeys[i+0*32] - for (i, x) in idNullifier.mpairs: x = generatedKeys[i+1*32] - for (i, x) in idSecretHash.mpairs: x = generatedKeys[i+2*32] - for (i, x) in idCommitment.mpairs: x = generatedKeys[i+3*32] - - var - identityCredential = IdentityCredential(idTrapdoor: idTrapdoor, idNullifier: idNullifier, idSecretHash: idSecretHash, idCommitment: idCommitment) - - return ok(identityCredential) - -proc createRLNInstance*(d: int = MerkleTreeDepth): RLNResult = - ## Wraps the rln instance creation for metrics - ## Returns an error if the instance creation fails - var res: RLNResult - waku_rln_instance_creation_duration_seconds.nanosecondTime: - res = createRLNInstanceLocal(d) - return res - -proc toUInt256*(idCommitment: IDCommitment): UInt256 = - let pk = UInt256.fromBytesLE(idCommitment) - return pk - -proc toIDCommitment*(idCommitmentUint: UInt256): IDCommitment = - let pk = IDCommitment(idCommitmentUint.toBytesLE()) - return pk - -proc inHex*(value: IdentityTrapdoor or IdentityNullifier or IdentitySecretHash or IDCommitment or MerkleNode or Nullifier or Epoch or RlnIdentifier): string = - var valueHex = (UInt256.fromBytesLE(value)).toHex() +proc inHex*(value: + IdentityTrapdoor or + IdentityNullifier or + IdentitySecretHash or + IDCommitment or + MerkleNode or + Nullifier or + Epoch or + RlnIdentifier + ): string = + var valueHex = UInt256.fromBytesLE(value) + valueHex = valueHex.toHex() # We pad leading zeroes while valueHex.len < value.len * 2: valueHex = "0" & valueHex return valueHex -proc toMembershipIndex(v: UInt256): MembershipIndex = - let membershipIndex: MembershipIndex = cast[MembershipIndex](v) - return membershipIndex - proc register*(idComm: IDCommitment, ethAccountAddress: Option[Address], ethAccountPrivKey: keys.PrivateKey, ethClientAddress: string, membershipContractAddress: Address, registrationHandler: Option[RegistrationHandler] = none(RegistrationHandler)): Future[Result[MembershipIndex, string]] {.async.} = # TODO may need to also get eth Account Private Key as PrivateKey ## registers the idComm into the membership contract whose address is in rlnPeer.membershipContractAddress @@ -216,216 +146,6 @@ proc register*(rlnPeer: WakuRLNRelay, registrationHandler: Option[RegistrationHa return err(regResult.error()) return ok(true) -proc appendLength*(input: openArray[byte]): seq[byte] = - ## returns length prefixed version of the input - ## with the following format [len<8>|input] - ## len: 8-byte value that represents the number of bytes in the `input` - ## len is serialized in little-endian - ## input: the supplied `input` - let - # the length should be serialized in little-endian - len = toBytes(uint64(input.len), Endianness.littleEndian) - output = concat(@len, @input) - return output - -proc hash*(rlnInstance: ptr RLN, data: openArray[byte]): MerkleNode = - ## a thin layer on top of the Nim wrapper of the Poseidon hasher - debug "hash input", hashhex = data.toHex() - var lenPrefData = appendLength(data) - var - hashInputBuffer = lenPrefData.toBuffer() - outputBuffer: Buffer # will holds the hash output - - debug "hash input buffer length", bufflen = hashInputBuffer.len - let - hashSuccess = hash(rlnInstance, addr hashInputBuffer, addr outputBuffer) - output = cast[ptr MerkleNode](outputBuffer.`ptr`)[] - - return output - -proc serialize(idSecretHash: IdentitySecretHash, memIndex: MembershipIndex, epoch: Epoch, - msg: openArray[byte]): seq[byte] = - ## a private proc to convert RateLimitProof and the data to a byte seq - ## this conversion is used in the proofGen proc - ## the serialization is done as instructed in https://github.com/kilic/rln/blob/7ac74183f8b69b399e3bc96c1ae8ab61c026dc43/src/public.rs#L146 - ## [ id_key<32> | id_index<8> | epoch<32> | signal_len<8> | signal ] - let memIndexBytes = toBytes(uint64(memIndex), Endianness.littleEndian) - let lenPrefMsg = appendLength(msg) - let output = concat(@idSecretHash, @memIndexBytes, @epoch, lenPrefMsg) - return output - -proc proofGen*(rlnInstance: ptr RLN, data: openArray[byte], - memKeys: IdentityCredential, memIndex: MembershipIndex, - epoch: Epoch): RateLimitProofResult = - - # serialize inputs - let serializedInputs = serialize(idSecretHash = memKeys.idSecretHash, - memIndex = memIndex, - epoch = epoch, - msg = data) - var inputBuffer = toBuffer(serializedInputs) - - debug "input buffer ", inputBuffer= repr(inputBuffer) - - # generate the proof - var proof: Buffer - let proofIsSuccessful = generate_proof(rlnInstance, addr inputBuffer, addr proof) - # check whether the generate_proof call is done successfully - if not proofIsSuccessful: - return err("could not generate the proof") - - var proofValue = cast[ptr array[320, byte]] (proof.`ptr`) - let proofBytes: array[320, byte] = proofValue[] - debug "proof content", proofHex = proofValue[].toHex - - ## parse the proof as [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> ] - - let - proofOffset = 128 - rootOffset = proofOffset + 32 - epochOffset = rootOffset + 32 - shareXOffset = epochOffset + 32 - shareYOffset = shareXOffset + 32 - nullifierOffset = shareYOffset + 32 - rlnIdentifierOffset = nullifierOffset + 32 - - var - zkproof: ZKSNARK - proofRoot, shareX, shareY: MerkleNode - epoch: Epoch - nullifier: Nullifier - rlnIdentifier: RlnIdentifier - - discard zkproof.copyFrom(proofBytes[0..proofOffset-1]) - discard proofRoot.copyFrom(proofBytes[proofOffset..rootOffset-1]) - discard epoch.copyFrom(proofBytes[rootOffset..epochOffset-1]) - discard shareX.copyFrom(proofBytes[epochOffset..shareXOffset-1]) - discard shareY.copyFrom(proofBytes[shareXOffset..shareYOffset-1]) - discard nullifier.copyFrom(proofBytes[shareYOffset..nullifierOffset-1]) - discard rlnIdentifier.copyFrom(proofBytes[nullifierOffset..rlnIdentifierOffset-1]) - - let output = RateLimitProof(proof: zkproof, - merkleRoot: proofRoot, - epoch: epoch, - shareX: shareX, - shareY: shareY, - nullifier: nullifier, - rlnIdentifier: rlnIdentifier) - - return ok(output) - -proc serialize(proof: RateLimitProof, data: openArray[byte]): seq[byte] = - ## a private proc to convert RateLimitProof and data to a byte seq - ## this conversion is used in the proof verification proc - ## [ proof<128> | root<32> | epoch<32> | share_x<32> | share_y<32> | nullifier<32> | rln_identifier<32> | signal_len<8> | signal ] - let lenPrefMsg = appendLength(@data) - var proofBytes = concat(@(proof.proof), - @(proof.merkleRoot), - @(proof.epoch), - @(proof.shareX), - @(proof.shareY), - @(proof.nullifier), - @(proof.rlnIdentifier), - lenPrefMsg) - - return proofBytes - -# Serializes a sequence of MerkleNodes -proc serialize(roots: seq[MerkleNode]): seq[byte] = - var rootsBytes: seq[byte] = @[] - for root in roots: - rootsBytes = concat(rootsBytes, @root) - return rootsBytes - -# validRoots should contain a sequence of roots in the acceptable windows. -# As default, it is set to an empty sequence of roots. This implies that the validity check for the proof's root is skipped -proc proofVerify*(rlnInstance: ptr RLN, - data: openArray[byte], - proof: RateLimitProof, - validRoots: seq[MerkleNode] = @[]): RlnRelayResult[bool] = - ## verifies the proof, returns an error if the proof verification fails - ## returns true if the proof is valid - var - proofBytes = serialize(proof, data) - proofBuffer = proofBytes.toBuffer() - validProof: bool - rootsBytes = serialize(validRoots) - rootsBuffer = rootsBytes.toBuffer() - - trace "serialized proof", proof = proofBytes.toHex() - - let verifyIsSuccessful = verify_with_roots(rlnInstance, addr proofBuffer, addr rootsBuffer, addr validProof) - if not verifyIsSuccessful: - # something went wrong in verification call - warn "could not verify validity of the proof", proof=proof - return err("could not verify the proof") - - if not validProof: - return ok(false) - else: - return ok(true) - -proc insertMember*(rlnInstance: ptr RLN, idComm: IDCommitment): bool = - ## inserts a member to the tree - ## returns true if the member is inserted successfully - ## returns false if the member could not be inserted - var pkBuffer = toBuffer(idComm) - let pkBufferPtr = addr pkBuffer - - # add the member to the tree - let memberAdded = update_next_member(rlnInstance, pkBufferPtr) - return memberAdded - -proc serializeIdCommitments*(idComms: seq[IDCommitment]): seq[byte] = - ## serializes a seq of IDCommitments to a byte seq - ## the serialization is based on https://github.com/status-im/nwaku/blob/37bd29fbc37ce5cf636734e7dd410b1ed27b88c8/waku/v2/protocol/waku_rln_relay/rln.nim#L142 - ## the order of serialization is |id_commitment_len<8>|id_commitment| - var idCommsBytes = newSeq[byte]() - - # serialize the idComms, with its length prefixed - let len = toBytes(uint64(idComms.len), Endianness.littleEndian) - idCommsBytes.add(len) - - for idComm in idComms: - idCommsBytes = concat(idCommsBytes, @idComm) - - return idCommsBytes - -proc insertMembers*(rlnInstance: ptr RLN, - index: MembershipIndex, - idComms: seq[IDCommitment]): bool = - ## Insert multiple members i.e., identity commitments - ## returns true if the insertion is successful - ## returns false if any of the insertions fails - ## Note: This proc is atomic, i.e., if any of the insertions fails, all the previous insertions are rolled back - - # serialize the idComms - let idCommsBytes = serializeIdCommitments(idComms) - - var idCommsBuffer = idCommsBytes.toBuffer() - let idCommsBufferPtr = addr idCommsBuffer - # add the member to the tree - let membersAdded = set_leaves_from(rlnInstance, index, idCommsBufferPtr) - return membersAdded - -proc removeMember*(rlnInstance: ptr RLN, index: MembershipIndex): bool = - let deletion_success = delete_member(rlnInstance, index) - return deletion_success - -proc getMerkleRoot*(rlnInstance: ptr RLN): MerkleNodeResult = - # read the Merkle Tree root after insertion - var - root {.noinit.}: Buffer = Buffer() - rootPtr = addr(root) - getRootSuccessful = getRoot(rlnInstance, rootPtr) - if not getRootSuccessful: - return err("could not get the root") - if not root.len == 32: - return err("wrong output size") - - var rootValue = cast[ptr MerkleNode] (root.`ptr`)[] - return ok(rootValue) - proc updateValidRootQueue*(wakuRlnRelay: WakuRLNRelay, root: MerkleNode): void = ## updates the valid Merkle root queue with the latest root and pops the oldest one when the capacity of `AcceptableRootWindowSize` is reached let overflowCount = wakuRlnRelay.validMerkleRoots.len() - AcceptableRootWindowSize @@ -468,50 +188,6 @@ proc validateRoot*(wakuRlnRelay: WakuRLNRelay, root: MerkleNode): bool = ## Validate against the window of roots stored in wakuRlnRelay.validMerkleRoots return root in wakuRlnRelay.validMerkleRoots -# Converts a sequence of tuples containing 4 string (i.e. identity trapdoor, nullifier, secret hash and commitment) to an IndentityCredential -proc toIdentityCredentials*(groupKeys: seq[(string, string, string, string)]): RlnRelayResult[seq[ - IdentityCredential]] = - ## groupKeys is sequence of membership key tuples in the form of (identity key, identity commitment) all in the hexadecimal format - ## the toIdentityCredentials proc populates a sequence of IdentityCredentials using the supplied groupKeys - ## Returns an error if the conversion fails - - var groupIdCredentials = newSeq[IdentityCredential]() - - for i in 0..groupKeys.len-1: - try: - let - idTrapdoor = hexToUint[IdentityTrapdoor.len*8](groupKeys[i][0]).toBytesLE() - idNullifier = hexToUint[IdentityNullifier.len*8](groupKeys[i][1]).toBytesLE() - idSecretHash = hexToUint[IdentitySecretHash.len*8](groupKeys[i][2]).toBytesLE() - idCommitment = hexToUint[IDCommitment.len*8](groupKeys[i][3]).toBytesLE() - groupIdCredentials.add(IdentityCredential(idTrapdoor: idTrapdoor, idNullifier: idNullifier, idSecretHash: idSecretHash, - idCommitment: idCommitment)) - except ValueError as err: - warn "could not convert the group key to bytes", err = err.msg - return err("could not convert the group key to bytes: " & err.msg) - return ok(groupIdCredentials) - -# Converts a sequence of tuples containing 2 string (i.e. identity secret hash and commitment) to an IndentityCredential -proc toIdentityCredentials*(groupKeys: seq[(string, string)]): RlnRelayResult[seq[ - IdentityCredential]] = - ## groupKeys is sequence of membership key tuples in the form of (identity key, identity commitment) all in the hexadecimal format - ## the toIdentityCredentials proc populates a sequence of IdentityCredentials using the supplied groupKeys - ## Returns an error if the conversion fails - - var groupIdCredentials = newSeq[IdentityCredential]() - - for i in 0..groupKeys.len-1: - try: - let - idSecretHash = hexToUint[IdentitySecretHash.len*8](groupKeys[i][0]).toBytesLE() - idCommitment = hexToUint[IDCommitment.len*8](groupKeys[i][1]).toBytesLE() - groupIdCredentials.add(IdentityCredential(idSecretHash: idSecretHash, - idCommitment: idCommitment)) - except ValueError as err: - warn "could not convert the group key to bytes", err = err.msg - return err("could not convert the group key to bytes: " & err.msg) - return ok(groupIdCredentials) - proc calcMerkleRoot*(list: seq[IDCommitment]): RlnRelayResult[string] = ## returns the root of the Merkle tree that is computed from the supplied list ## the root is in hexadecimal format @@ -604,6 +280,12 @@ proc rlnRelayStaticSetUp*(rlnRelayMembershipIndex: MembershipIndex): RlnRelayRes return ok((groupIDCommitmentsOpt, groupIdCredentialsOpt, memIndexOpt)) +proc calcEpoch*(t: float64): Epoch = + ## gets time `t` as `flaot64` with subseconds resolution in the fractional part + ## and returns its corresponding rln `Epoch` value + let e = uint64(t/EpochUnitSeconds) + return toEpoch(e) + proc hasDuplicate*(rlnPeer: WakuRLNRelay, msg: WakuMessage): RlnRelayResult[bool] = ## returns true if there is another message in the `nullifierLog` of the `rlnPeer` with the same ## epoch and nullifier as `msg`'s epoch and nullifier but different Shamir secret shares @@ -680,25 +362,6 @@ proc updateLog*(rlnPeer: WakuRLNRelay, msg: WakuMessage): RlnRelayResult[bool] = except KeyError as e: return err("the epoch was not found") -proc toEpoch*(t: uint64): Epoch = - ## converts `t` to `Epoch` in little-endian order - let bytes = toBytes(t, Endianness.littleEndian) - debug "bytes", bytes = bytes - var epoch: Epoch - discard epoch.copyFrom(bytes) - return epoch - -proc fromEpoch*(epoch: Epoch): uint64 = - ## decodes bytes of `epoch` (in little-endian) to uint64 - let t = fromBytesLE(uint64, array[32, byte](epoch)) - return t - -proc calcEpoch*(t: float64): Epoch = - ## gets time `t` as `flaot64` with subseconds resolution in the fractional part - ## and returns its corresponding rln `Epoch` value - let e = uint64(t/EpochUnitSeconds) - return toEpoch(e) - proc getCurrentEpoch*(): Epoch = ## gets the current rln Epoch time return calcEpoch(epochTime()) @@ -1000,8 +663,8 @@ proc handleGroupUpdates*(rlnPeer: WakuRLNRelay) {.async, gcsafe.} = proc addRLNRelayValidator*(wakuRlnRelay: WakuRLNRelay, wakuRelay: WakuRelay, - pubsubTopic: PubsubTopic, - contentTopic: ContentTopic, + pubsubTopic: PubsubTopic, + contentTopic: ContentTopic, spamHandler: Option[SpamHandler] = none(SpamHandler)) = ## this procedure is a thin wrapper for the pubsub addValidator method ## it sets a validator for the waku messages published on the supplied pubsubTopic and contentTopic @@ -1220,7 +883,7 @@ proc mount(wakuRelay: WakuRelay, registrationHandler: Option[RegistrationHandler] = none(RegistrationHandler) ): Future[RlnRelayResult[WakuRlnRelay]] {.async.} = - if not conf.rlnRelayDynamic: + if not conf.rlnRelayDynamic: info " setting up waku-rln-relay in off-chain mode... " # set up rln relay inputs let staticSetupRes = rlnRelayStaticSetUp(MembershipIndex(conf.rlnRelayMembershipIndex)) @@ -1233,11 +896,11 @@ proc mount(wakuRelay: WakuRelay, else: # mount rlnrelay in off-chain mode with a static group of users let mountRes = mountRlnRelayStatic(wakuRelay, - group = groupOpt.get(), + group = groupOpt.get(), memIdCredential = idCredentialOpt.get(), - memIndex = memIndexOpt.get(), + memIndex = memIndexOpt.get(), pubsubTopic = conf.rlnRelayPubsubTopic, - contentTopic = conf.rlnRelayContentTopic, + contentTopic = conf.rlnRelayContentTopic, spamHandler = spamHandler) if mountRes.isErr(): @@ -1325,26 +988,26 @@ proc mount(wakuRelay: WakuRelay, if credentials.isSome(): # mount rln-relay in on-chain mode, with credentials that were read or generated rlnRelayRes = await mountRlnRelayDynamic(wakuRelay, - memContractAddr = ethMemContractAddress, + memContractAddr = ethMemContractAddress, ethClientAddr = ethClientAddr, - ethAccountAddress = ethAccountAddressOpt, - ethAccountPrivKeyOpt = ethAccountPrivKeyOpt, + ethAccountAddress = ethAccountAddressOpt, + ethAccountPrivKeyOpt = ethAccountPrivKeyOpt, pubsubTopic = conf.rlnRelayPubsubTopic, - contentTopic = conf.rlnRelayContentTopic, - spamHandler = spamHandler, + contentTopic = conf.rlnRelayContentTopic, + spamHandler = spamHandler, registrationHandler = registrationHandler, memIdCredential = some(credentials.get().identityCredential), - memIndex = some(credentials.get().rlnIndex)) + memIndex = some(credentials.get().rlnIndex)) else: # mount rln-relay in on-chain mode, with the provided private key rlnRelayRes = await mountRlnRelayDynamic(wakuRelay, - memContractAddr = ethMemContractAddress, + memContractAddr = ethMemContractAddress, ethClientAddr = ethClientAddr, - ethAccountAddress = ethAccountAddressOpt, - ethAccountPrivKeyOpt = ethAccountPrivKeyOpt, + ethAccountAddress = ethAccountAddressOpt, + ethAccountPrivKeyOpt = ethAccountPrivKeyOpt, pubsubTopic = conf.rlnRelayPubsubTopic, - contentTopic = conf.rlnRelayContentTopic, - spamHandler = spamHandler, + contentTopic = conf.rlnRelayContentTopic, + spamHandler = spamHandler, registrationHandler = registrationHandler) persistCredentials = true @@ -1354,13 +1017,13 @@ proc mount(wakuRelay: WakuRelay, # a new credential will be generated during the mount process but will not be persisted info "no need to persist or use a persisted rln-relay credential" rlnRelayRes = await mountRlnRelayDynamic(wakuRelay, - memContractAddr = ethMemContractAddress, + memContractAddr = ethMemContractAddress, ethClientAddr = ethClientAddr, - ethAccountAddress = ethAccountAddressOpt, - ethAccountPrivKeyOpt = ethAccountPrivKeyOpt, + ethAccountAddress = ethAccountAddressOpt, + ethAccountPrivKeyOpt = ethAccountPrivKeyOpt, pubsubTopic = conf.rlnRelayPubsubTopic, - contentTopic = conf.rlnRelayContentTopic, - spamHandler = spamHandler, + contentTopic = conf.rlnRelayContentTopic, + spamHandler = spamHandler, registrationHandler = registrationHandler) if rlnRelayRes.isErr(): @@ -1368,7 +1031,7 @@ proc mount(wakuRelay: WakuRelay, let wakuRlnRelay = rlnRelayRes.get() if persistCredentials: # persist rln credential - credentials = some(RlnMembershipCredentials(rlnIndex: wakuRlnRelay.membershipIndex, + credentials = some(RlnMembershipCredentials(rlnIndex: wakuRlnRelay.membershipIndex, identityCredential: wakuRlnRelay.identityCredential)) if writeRlnCredentials(rlnRelayCredPath, credentials.get(), conf.rlnRelayCredentialsPassword).isErr(): return err("error in storing rln credentials") @@ -1393,11 +1056,11 @@ proc new*(T: type WakuRlnRelay, return err("The relay protocol does not support the configured pubsub topic") debug "rln-relay input validation passed" - waku_rln_relay_mounting_duration_seconds.nanosecondTime: + waku_rln_relay_mounting_duration_seconds.nanosecondTime: let rlnRelayRes = await mount( wakuRelay, conf, spamHandler, registrationHandler ) - return rlnRelayRes \ No newline at end of file + return rlnRelayRes