Merge 16e17b4a9d76ef496ccd57c61d26e603d4c6af78 into 64a0ed7d967454d9c3b345023719e6ca5d73f129

tests/waku_rln_relay/test_rln_group_manager_onchain.nim

@@ -97,7 +97,8 @@ suite "Onchain group manager":
     check:
       merkleRootBefore != merkleRootAfter
 
-  test "trackRootChanges: should fetch history correctly":
+  test "trackRootChanges: should fetch history correctly: fetch single root()":
+    # basic check for the soon to be deprecated root contract function, is replaced by getRecentRoots()
     # TODO: We can't use `trackRootChanges()` directly in this test because its current implementation
     #       relies on a busy loop rather than event-based monitoring. but that busy loop fetch root every 5 seconds
     #       so we can't use it in this test. 
@@ -107,7 +108,8 @@ suite "Onchain group manager":
     (waitFor manager.init()).isOkOr:
       raiseAssert $error
 
-    let merkleRootBefore = waitFor manager.fetchMerkleRoot()
+    let merkleRootBefore = (waitFor manager.fetchMerkleRoot()).valueOr:
+      raiseAssert "Failed to fetch merkle root before: " & error
 
     for i in 0 ..< credentials.len():
       info "Registering credential", index = i, credential = credentials[i]
@@ -115,12 +117,82 @@ suite "Onchain group manager":
         assert false, "Failed to register credential " & $i & ": " & error
       discard waitFor manager.updateRoots()
 
-    let merkleRootAfter = waitFor manager.fetchMerkleRoot()
+    let merkleRootAfter = (waitFor manager.fetchMerkleRoot()).valueOr:
+      raiseAssert "Failed to fetch merkle root after: " & error
 
     check:
       merkleRootBefore != merkleRootAfter
       manager.validRoots.len() == credentialCount
 
+  test "trackRootChanges: should fetch history correctly: fetch root cache":
+    # Verify that the group_manager list of valid roots is updated correctly from the recent roots
+    # cache as new credentials are registered.
+    # TODO: We can't use `trackRootChanges()` directly in this test because its current implementation
+    #       relies on a busy loop rather than event-based monitoring. but that busy loop fetch root every 5 seconds
+    #       so we can't use it in this test.
+
+    const credentialCount = 5
+    let credentials = generateCredentials(credentialCount)
+    (waitFor manager.init()).isOkOr:
+      raiseAssert $error
+
+    let merkleRootCacheBefore = (waitFor manager.fetchMerkleRootsCache()).valueOr:
+      raiseAssert "Failed to fetch merkle root cache before: " & error
+
+    check:
+      merkleRootCacheBefore.len == 5 * 32
+      merkleRootCacheBefore.allIt(it == 0'u8)
+      manager.validRoots.len() == 0
+
+    for i in 0 ..< credentials.len():
+      info "Registering credential", index = i, credential = credentials[i]
+      (waitFor manager.register(credentials[i], UserMessageLimit(20))).isOkOr:
+        assert false, "Failed to register credential " & $i & ": " & error
+      discard waitFor manager.updateRecentRoots()
+
+    let merkleRootCacheAfter = (waitFor manager.fetchMerkleRootsCache()).valueOr:
+      raiseAssert "Failed to fetch merkle root cache after: " & error
+
+    check:
+      merkleRootCacheAfter.len == 5 * 32
+      not merkleRootCacheAfter.allIt(it == 0'u8)
+      manager.validRoots.len() == credentialCount
+      manager.validRoots.items().toSeq().allIt(it != default(MerkleNode))
+
+  test "trackRootChanges: oldest roots are evicted once the window is exceeded":
+    const
+      initialCount = 5
+      additionalCount = 6
+    let credentials = generateCredentials(initialCount + additionalCount)
+    (waitFor manager.init()).isOkOr:
+      raiseAssert $error
+
+    # Register the first 5 credentials and snapshot the 3 oldest roots.
+    for i in 0 ..< initialCount:
+      (waitFor manager.register(credentials[i], UserMessageLimit(20))).isOkOr:
+        assert false, "Failed to register credential " & $i & ": " & error
+      discard waitFor manager.updateRecentRoots()
+
+    check manager.validRoots.len() >= 3
+    let firstThreeBefore =
+      @[manager.validRoots[0], manager.validRoots[1], manager.validRoots[2]]
+
+    # Register the remaining credentials, pushing the deque past AcceptableRootWindowSize.
+    for i in initialCount ..< credentials.len():
+      (waitFor manager.register(credentials[i], UserMessageLimit(20))).isOkOr:
+        assert false, "Failed to register credential " & $i & ": " & error
+      discard waitFor manager.updateRecentRoots()
+
+    let rootsAfter = manager.validRoots.items().toSeq()
+
+    # 51 registrations into a window of 50 evicts exactly the single oldest root,
+    # so only the first of the original three is gone; the other two remain.
+    check:
+      manager.validRoots.len() == AcceptableRootWindowSize
+      firstThreeBefore[0] notin rootsAfter
+      firstThreeBefore[1] in rootsAfter
+      firstThreeBefore[2] in rootsAfter
+
   test "register: should guard against uninitialized state":
     let dummyCommitment = default(IDCommitment)
 
@@ -214,7 +286,7 @@ suite "Onchain group manager":
 
     waitFor fut
 
-    let rootUpdated = waitFor manager.updateRoots()
+    let rootUpdated = waitFor manager.updateRecentRoots()
 
     if rootUpdated:
       let proofResult = waitFor manager.fetchMerkleProofElements()
@@ -296,7 +368,7 @@ suite "Onchain group manager":
       assert false, "error returned when calling register: " & error
     waitFor fut
 
-    let rootUpdated = waitFor manager.updateRoots()
+    let rootUpdated = waitFor manager.updateRecentRoots()
 
     if rootUpdated:
       let proofResult = waitFor manager.fetchMerkleProofElements()
@@ -333,7 +405,7 @@ suite "Onchain group manager":
 
     let messageBytes = "Hello".toBytes()
 
-    let rootUpdated = waitFor manager.updateRoots()
+    let rootUpdated = waitFor manager.updateRecentRoots()
 
     manager.merkleProofCache = newSeq[byte](640)
     for i in 0 ..< 640:
@@ -362,7 +434,7 @@ suite "Onchain group manager":
       verified == false
 
   test "root queue should be updated correctly":
-    const credentialCount = 12
+    const credentialCount = 9
     let credentials = generateCredentials(credentialCount)
     (waitFor manager.init()).isOkOr:
       raiseAssert $error
@@ -391,7 +463,7 @@ suite "Onchain group manager":
     for i in 0 ..< credentials.len():
       (waitFor manager.register(credentials[i], UserMessageLimit(20))).isOkOr:
         assert false, "Failed to register credential " & $i & ": " & error
-      discard waitFor manager.updateRoots()
+      discard waitFor manager.updateRecentRoots()
 
     waitFor allFutures(futures)
 
@@ -436,7 +508,7 @@ suite "Onchain group manager":
     (waitFor manager.register(credentials, UserMessageLimit(20))).isOkOr:
       assert false, "register failed: " & error
 
-    discard waitFor manager.updateRoots()
+    discard waitFor manager.updateRecentRoots()
     let roots = manager.validRoots.items().toSeq()
     require:
       roots.len > 0

waku/waku_rln_relay/group_manager/on_chain/group_manager.nim

@@ -62,10 +62,10 @@ proc fetchMerkleProofElements*(
   let response = await sendEthCallWithParams(
     ethRpc = g.ethRpc.get(),
     functionSignature = methodSig,
-    params = paddedParam,
     fromAddress = g.ethRpc.get().defaultAccount,
     toAddress = fromHex(Address, g.ethContractAddress),
     chainId = g.chainId,
+    params = paddedParam,
   )
 
   return response
@@ -73,14 +73,32 @@ proc fetchMerkleProofElements*(
 proc fetchMerkleRoot*(
     g: OnchainGroupManager
 ): Future[Result[UInt256, string]] {.async.} =
-  let merkleRoot = await sendEthCallWithoutParams(
+  try:
-    ethRpc = g.ethRpc.get(),
+    let merkleRoot = await sendEthCallWithoutParams(
-    functionSignature = "root()",
+      ethRpc = g.ethRpc.get(),
-    fromAddress = g.ethRpc.get().defaultAccount,
+      functionSignature = "root()",
-    toAddress = fromHex(Address, g.ethContractAddress),
+      fromAddress = g.ethRpc.get().defaultAccount,
-    chainId = g.chainId,
+      toAddress = fromHex(Address, g.ethContractAddress),
-  )
+      chainId = g.chainId,
-  return merkleRoot
+    )
+    return merkleRoot
+  except CatchableError:
+    error "Failed to fetch Merkle root", error = getCurrentExceptionMsg()
+    return err("Failed to fetch merkle root: " & getCurrentExceptionMsg())
+
+proc fetchMerkleRootsCache*(
+    g: OnchainGroupManager
+): Future[Result[seq[byte], string]] {.async.} =
+  let
+    # using sendEthCallWithParams to get return type of seq[bytes] for getRecentRoots() function which returns an array of bytes32
+    merkleRoots = await sendEthCallWithParams(
+      ethRpc = g.ethRpc.get(),
+      functionSignature = "getRecentRoots()",
+      fromAddress = g.ethRpc.get().defaultAccount,
+      toAddress = fromHex(Address, g.ethContractAddress),
+      chainId = g.chainId,
+    )
+  return merkleRoots
 
 proc fetchNextFreeIndex*(
     g: OnchainGroupManager
@@ -102,10 +120,10 @@ proc fetchMembershipStatus*(
     await sendEthCallWithParams(
       ethRpc = g.ethRpc.get(),
       functionSignature = "isInMembershipSet(uint256)",
-      params = params,
       fromAddress = g.ethRpc.get().defaultAccount,
       toAddress = fromHex(Address, g.ethContractAddress),
       chainId = g.chainId,
+      params = params,
     )
   ).valueOr:
     return err("Failed to check membership: " & error)
@@ -148,14 +166,69 @@ proc updateRoots*(g: OnchainGroupManager): Future[bool] {.async.} =
 
   return false
 
+proc updateRecentRoots*(g: OnchainGroupManager): Future[bool] {.async.} =
+  let bytes = (await g.fetchMerkleRoot()).valueOr:
+    error "Failed to fetch current Merkle root", error = error
+    return false
+
+  if (bytes.len mod 32) != 0:
+    error "Invalid recent roots payload length", length = bytes.len
+    return false
+
+  let chunkCount = bytes.len div 32
+  if chunkCount != 5:
+    warn "Unexpected number of recent roots returned; proceeding anyway",
+      count = chunkCount
+
+  # Parse 32-byte chunks (contract returns newest-first) into MerkleNode values,
+  # reversing to oldest-first and skipping zero roots.
+  var newRootsDequeOrder: seq[MerkleNode] = @[]
+  for startIdx in countdown(bytes.len - 32, 0, 32):
+    let u = UInt256.fromBytesBE(bytes.toOpenArray(startIdx, startIdx + 31))
+    if u.isZero:
+      continue
+    newRootsDequeOrder.add(UInt256ToField(u))
+
+  if newRootsDequeOrder.len == 0:
+    debug "no non-zero recent roots to add; skipping update"
+    return false
+
+  # Determine overlap with existing tail so we only append truly new roots
+  var overlap = min(g.validRoots.len, newRootsDequeOrder.len)
+  var matchLen = 0
+  # Find the largest n (<= overlap) such that last n of validRoots == first n of newRootsDequeOrder
+  for n in countdown(overlap, 1):
+    var ok = true
+    let startIdx = g.validRoots.len - n
+    for i in 0 ..< n:
+      if g.validRoots[startIdx + i] != newRootsDequeOrder[i]:
+        ok = false
+        break
+    if ok:
+      matchLen = n
+      break
+
+  let toAdd = newRootsDequeOrder[matchLen ..< newRootsDequeOrder.len]
+  if toAdd.len == 0:
+    return false
+
+  # Append new roots to the tail; trim happens below if we exceed the window.
+  toAdd.mapIt(g.validRoots.addLast(it))
+  debug "appended recent roots", count = toAdd.len, roots = toAdd
+
+  while g.validRoots.len > AcceptableRootWindowSize:
+    trace "removing old merkle root", root = g.validRoots[0]
+    discard g.validRoots.popFirst()
+
+  return true
+
 proc trackRootChanges*(g: OnchainGroupManager): Future[Result[void, string]] {.async.} =
   ?checkInitialized(g)
 
-  const rpcDelay = 5.seconds
+  const rpcDelay = 10.seconds
 
   while true:
-    await sleepAsync(rpcDelay)
+    let rootUpdated = await g.updateRecentRoots()
-    let rootUpdated = await g.updateRoots()
 
     if rootUpdated:
       ## The membership set on-chain has changed (some new members have joined or some members have left)
@@ -174,6 +247,7 @@ proc trackRootChanges*(g: OnchainGroupManager): Future[Result[void, string]] {.a
 
       let memberCount = cast[int64](nextFreeIndex)
       waku_rln_number_registered_memberships.set(float64(memberCount))
+    await sleepAsync(rpcDelay)
 
 method register*(
     g: OnchainGroupManager, rateCommitment: RateCommitment
@@ -393,8 +467,11 @@ method generateProof*(
     external_nullifier: extNullifier,
   )
 
-  let output = generateRlnProofWithWitness(g.rlnInstance, witness, epoch, rlnIdentifier).valueOr:
+  waku_rln_proof_generation_duration_seconds.nanosecondTime:
-    return err("Failed to generate proof: " & error)
+    let output = generateRlnProofWithWitness(
+      g.rlnInstance, witness, epoch, rlnIdentifier
+    ).valueOr:
+      return err("Failed to generate proof: " & error)
 
   info "Proof generated successfully", proof = output
 

waku/waku_rln_relay/group_manager/on_chain/rpc_wrapper.nim

@@ -76,10 +76,10 @@ proc sendEthCallWithoutParams*(
 proc sendEthCallWithParams*(
     ethRpc: Web3,
     functionSignature: string,
-    params: seq[byte],
     fromAddress: Address,
     toAddress: Address,
     chainId: UInt256,
+    params: seq[byte] = @[],
 ): Future[Result[seq[byte], string]] {.async.} =
   ## Workaround for web3 chainId=null issue with parameterized contract calls
   let functionHash =