From 855818f5457cff4b2575e465c1fbd2e78502103a Mon Sep 17 00:00:00 2001 From: stubbsta Date: Mon, 13 Jul 2026 11:52:49 +0200 Subject: [PATCH] Move message publish retries out of the kernel on stale RLN proof On a stale-merkle-proof rejection the kernel and REST layers no longer republish internally. They schedule a detached merkle proof refresh and fail fast with a distinguishable, retry-worthy error, leaving the retry decision to the caller. - add GroupManager.scheduleMerkleProofRefresh: drops the cached path and asyncSpawns the refetch (join-shielded, so a cancelled caller never cancels the shared refetch); base is a no-op. - v3 lightpush (node.lightpushPublish): on an RLN-related 420/504, call scheduleMerkleProofRefresh and return immediately, normalized to 504 OUT_OF_RLN_PROOF with RlnProofRefreshScheduledMsg. The internal republish and its bounded timeout are gone. - send service: lightpushPublishToAny now routes through node.lightpushPublish so every NextRoundRetry round regenerates the proof against the current cache; the existing OUT_OF_RLN_PROOF -> NextRoundRetry mapping is the retry. - REST lightpush: pass the 504 through unchanged (no internal retry). - REST relay: attachRlnProofValidateWithRetry -> attachRlnProofAndValidate; validate once, on RLN-invalid schedule the refresh and return 503 with the marker instead of regenerating and revalidating in-request. - rename RlnRefreshRetryTimeout -> RlnMerkleProofRefreshTimeout; it now bounds only the legacy lightpush refresh, which keeps its one-shot internal retry (no wire error taxonomy to delegate to callers). Not touched: legacy lightpush retry behavior, the proof.nim attachRLNProof/checkAndGenerateRLNProof split, and the merkle path refetch/coalescing internals of ensureFreshMerkleProofPath. Co-Authored-By: Claude Fable 5 --- logos_delivery/waku/api/publish.nim | 9 +- .../waku/node/waku_node/lightpush.nim | 43 +++---- .../waku/rest_api/endpoint/relay/handlers.nim | 54 ++++----- logos_delivery/waku/rln/constants.nim | 15 ++- .../rln/group_manager/group_manager_base.nim | 8 ++ .../group_manager/on_chain/group_manager.nim | 13 ++ tests/node/test_wakunode_lightpush.nim | 114 ++++++++++++------ .../test_rln_group_manager_onchain.nim | 24 ++++ tests/wakunode_rest/test_rest_relay.nim | 70 ++++++++--- 9 files changed, 235 insertions(+), 115 deletions(-) diff --git a/logos_delivery/waku/api/publish.nim b/logos_delivery/waku/api/publish.nim index e45eff942..f9f3229f6 100644 --- a/logos_delivery/waku/api/publish.nim +++ b/logos_delivery/waku/api/publish.nim @@ -16,6 +16,7 @@ import logos_delivery/waku/[ waku_core, node/waku_node, + node/waku_node/lightpush, node/peer_manager, waku_relay/protocol, rln, @@ -50,11 +51,13 @@ proc lightpushPeerAvailable*(self: Waku, shard: PubsubTopic): bool = proc lightpushPublishToAny*( self: Waku, shard: PubsubTopic, message: WakuMessage ): Future[WakuLightPushResult] {.async.} = - ## Selects a lightpush service peer for `shard` and publishes `message`. - ## Returns SERVICE_NOT_AVAILABLE when no peer is available. + ## Selects a lightpush service peer for `shard` and publishes `message` + ## through the node's lightpush flow, which attaches an RLN proof per + ## attempt when RLN is mounted. Returns SERVICE_NOT_AVAILABLE when no peer + ## is available. let peer = self.node.peerManager.selectPeer(WakuLightPushCodec, some(shard)).valueOr: return lightpushResultServiceUnavailable("no lightpush peer available for shard") try: - return await self.node.wakuLightpushClient.publish(some(shard), message, peer) + return await self.node.lightpushPublish(some(shard), message, some(peer)) except CatchableError as e: return lightpushResultInternalError(e.msg) diff --git a/logos_delivery/waku/node/waku_node/lightpush.nim b/logos_delivery/waku/node/waku_node/lightpush.nim index d5ee5903a..878777f8e 100644 --- a/logos_delivery/waku/node/waku_node/lightpush.nim +++ b/logos_delivery/waku/node/waku_node/lightpush.nim @@ -116,7 +116,7 @@ proc runRlnRefreshRetry( ): Future[legacy_lightpush_protocol.WakuLightPushResult[string]] {.async, gcsafe.} = ## Force-refreshes the RLN merkle proof path and retries the publish once. ## Only the refresh (on-chain refetch + proof regeneration) is bounded by - ## RlnRefreshRetryTimeout — a hanging RPC cannot stall the caller + ## RlnMerkleProofRefreshTimeout — a hanging RPC cannot stall the caller ## indefinitely and `fallback` (the original rejection) is returned instead. ## The retried publish itself runs unbounded, matching the first attempt. info "legacy lightpush send rejected as RLN-invalid; " & @@ -124,7 +124,7 @@ proc runRlnRefreshRetry( rln.get().groupManager.invalidateMerkleProofCache() let refreshFut = attachRLNProof(rln.get(), msgWithProof) - if not (await refreshFut.withTimeout(RlnRefreshRetryTimeout)): + if not (await refreshFut.withTimeout(RlnMerkleProofRefreshTimeout)): warn "legacy lightpush RLN proof refresh timed out; returning original error" return fallback let retryMsg = refreshFut.read().valueOr: @@ -345,12 +345,11 @@ proc lightpushPublish*( let firstResult = await lightpushPublishHandler(node, pubsubForPublish, msgWithProof, toPeer, mixify) - # A publish error can indicate a stale Merkle proof path; refresh it and - # retry the publish once. Gate only on unambiguously RLN-related failures: - # 504 (OUT_OF_RLN_PROOF) is always RLN-specific; 420 (INVALID_MESSAGE) is - # also returned for non-RLN rejections (e.g. oversized messages), so require - # the error description to contain RlnValidatorErrorMsg — matching the legacy - # lightpush path — to avoid unbounded on-chain RPCs on non-RLN errors. + # A publish rejection can indicate a stale Merkle proof path. Gate only on + # unambiguously RLN-related failures: 504 (OUT_OF_RLN_PROOF) is always + # RLN-specific; 420 (INVALID_MESSAGE) is also returned for non-RLN + # rejections (e.g. oversized messages), so require the error description to + # contain RlnValidatorErrorMsg. if firstResult.isOk() or rln.isNone(): return firstResult let isRlnRelatedFailure = @@ -361,20 +360,16 @@ proc lightpushPublish*( if not isRlnRelatedFailure: return firstResult - info "lightpush send rejected; refreshing merkle proof and retrying once", + # Schedule a merkle proof refresh and surface the rejection immediately, + # normalized to 504 (OUT_OF_RLN_PROOF) with RlnProofRefreshScheduledMsg so + # callers can tell "stale proof, retry the publish" from a permanent + # rejection. A retried publish regenerates its proof against the refreshed + # cache, or coalesces onto the still-running refetch. + info "lightpush send rejected as RLN-invalid; scheduling merkle proof refresh", statusCode = $firstResult.error.code - rln.get().groupManager.invalidateMerkleProofCache() - - # Only the refresh (on-chain refetch + proof regeneration) is bounded — a - # hanging RPC cannot stall the caller indefinitely and the original - # rejection is returned instead. The retried publish itself runs unbounded, - # matching the first attempt. - let refreshFut = attachRLNProof(rln.get(), msgWithProof) - if not (await refreshFut.withTimeout(RlnRefreshRetryTimeout)): - warn "lightpush RLN proof refresh timed out; returning original error", - statusCode = $firstResult.error.code - return firstResult - let retryMsg = refreshFut.read().valueOr: - return lighpushErrorResult(LightPushErrorCode.OUT_OF_RLN_PROOF, error) - - return await lightpushPublishHandler(node, pubsubForPublish, retryMsg, toPeer, mixify) + rln.get().groupManager.scheduleMerkleProofRefresh() + return lighpushErrorResult( + LightPushErrorCode.OUT_OF_RLN_PROOF, + RlnProofRefreshScheduledMsg & ": " & + firstResult.error.desc.get($firstResult.error.code), + ) diff --git a/logos_delivery/waku/rest_api/endpoint/relay/handlers.nim b/logos_delivery/waku/rest_api/endpoint/relay/handlers.nim index 6832e8697..0976877f4 100644 --- a/logos_delivery/waku/rest_api/endpoint/relay/handlers.nim +++ b/logos_delivery/waku/rest_api/endpoint/relay/handlers.nim @@ -55,18 +55,20 @@ type RlnPublishErrorKind = enum ProofGenFailed ## Local proof generation failed — server-side (500). ValidationRejected ## Validator rejected the message — client-side (400). + StaleProofSuspected ## Stale merkle path; refresh scheduled — retry (503). RlnPublishError = object kind: RlnPublishErrorKind desc: string -proc attachRlnProofValidateWithRetry( +proc attachRlnProofAndValidate( rln: Rln, wakuRelay: WakuRelay, pubsubTopic: PubsubTopic, message: WakuMessage ): Future[Result[WakuMessage, RlnPublishError]] {.async.} = - ## Attaches an RLN proof to `message`, validates it via `wakuRelay`, and if - ## the validator rejects it as RLN-invalid (error contains RlnValidatorErrorMsg) - ## force-refreshes the merkle proof path and retries once. Returns the message - ## with its final proof on success. Callers invoke only when RLN is mounted. + ## Attaches an RLN proof to `message` and validates it via `wakuRelay`. + ## If the validator rejects it as RLN-invalid (error contains + ## RlnValidatorErrorMsg), schedules a background merkle proof refresh and + ## fails early with StaleProofSuspected — the caller decides whether to + ## retry. Callers invoke only when RLN is mounted. var msg = message msg.proof = ( await rln.generateRLNProof(msg.toRLNSignal(), float64(getTime().toUnix())) @@ -77,28 +79,20 @@ proc attachRlnProofValidateWithRetry( ) ) - let firstValidateResult = await wakuRelay.validateMessage(pubsubTopic, msg) - if firstValidateResult.isOk(): + let validateResult = await wakuRelay.validateMessage(pubsubTopic, msg) + if validateResult.isOk(): return ok(msg) - if not firstValidateResult.error.contains(RlnValidatorErrorMsg): - return - err(RlnPublishError(kind: ValidationRejected, desc: firstValidateResult.error)) + if not validateResult.error.contains(RlnValidatorErrorMsg): + return err(RlnPublishError(kind: ValidationRejected, desc: validateResult.error)) - info "relay publish rejected as RLN-invalid; refreshing merkle proof and retrying once" - rln.groupManager.invalidateMerkleProofCache() - msg.proof = ( - await rln.generateRLNProof(msg.toRLNSignal(), float64(getTime().toUnix())) - ).valueOr: - return err( - RlnPublishError( - kind: ProofGenFailed, desc: "error appending RLN proof on retry: " & $error - ) + info "relay publish rejected as RLN-invalid; scheduling merkle proof refresh" + rln.groupManager.scheduleMerkleProofRefresh() + return err( + RlnPublishError( + kind: StaleProofSuspected, + desc: RlnProofRefreshScheduledMsg & ": " & validateResult.error, ) - - (await wakuRelay.validateMessage(pubsubTopic, msg)).isOkOr: - return err(RlnPublishError(kind: ValidationRejected, desc: error)) - - return ok(msg) + ) proc installRelayApiHandlers*( router: var RestRouter, node: WakuNode, cache: MessageCache @@ -217,15 +211,15 @@ proc installRelayApiHandlers*( if not node.rln.isNil(): message = ( - await attachRlnProofValidateWithRetry( - node.rln, node.wakuRelay, pubsubTopic, message - ) + await attachRlnProofAndValidate(node.rln, node.wakuRelay, pubsubTopic, message) ).valueOr: case error.kind of ProofGenFailed: return RestApiResponse.internalServerError("Failed to publish: " & error.desc) of ValidationRejected: return RestApiResponse.badRequest("Failed to publish: " & error.desc) + of StaleProofSuspected: + return RestApiResponse.serviceUnavailable("Failed to publish: " & error.desc) else: (await node.wakuRelay.validateMessage(pubsubTopic, message)).isOkOr: return RestApiResponse.badRequest("Failed to publish: " & error) @@ -347,15 +341,15 @@ proc installRelayApiHandlers*( if not node.rln.isNil(): message = ( - await attachRlnProofValidateWithRetry( - node.rln, node.wakuRelay, pubsubTopic, message - ) + await attachRlnProofAndValidate(node.rln, node.wakuRelay, pubsubTopic, message) ).valueOr: case error.kind of ProofGenFailed: return RestApiResponse.internalServerError("Failed to publish: " & error.desc) of ValidationRejected: return RestApiResponse.badRequest("Failed to publish: " & error.desc) + of StaleProofSuspected: + return RestApiResponse.serviceUnavailable("Failed to publish: " & error.desc) else: (await node.wakuRelay.validateMessage(pubsubTopic, message)).isOkOr: return RestApiResponse.badRequest("Failed to publish: " & error) diff --git a/logos_delivery/waku/rln/constants.nim b/logos_delivery/waku/rln/constants.nim index 028c548c7..eabc7da54 100644 --- a/logos_delivery/waku/rln/constants.nim +++ b/logos_delivery/waku/rln/constants.nim @@ -20,12 +20,19 @@ const RlnCredentialsFilename* = "rlnCredentials.txt" # RLN Validator message rejection Error string, is used to trigger proof refresh and publish retry in the lightpush client const RlnValidatorErrorMsg* = "RLN validation failed" -# Upper bound on the reactive merkle proof refresh after a stale-RLN +# Returned as the OUT_OF_RLN_PROOF error description when a stale merkle proof +# is suspected and a background refresh has been scheduled. Signals to callers +# that retrying the publish is worthwhile: the retry regenerates the proof +# against the refreshed cache. +const RlnProofRefreshScheduledMsg* = + "stale RLN proof suspected; refresh scheduled, retry the publish" + +# Upper bound on a reactive merkle proof refresh after a stale-RLN # rejection: fetching a fresh merkle path from the RLN contract (eth_call # round-trip) plus proof regeneration. Without a bound a hanging RPC endpoint -# could stall the caller indefinitely. The retried publish is not covered — -# it runs unbounded, matching the first attempt. -const RlnRefreshRetryTimeout* = 5.seconds +# could stall the caller indefinitely. Only the refresh is covered — a +# subsequent publish retry runs unbounded, matching the first attempt. +const RlnMerkleProofRefreshTimeout* = 5.seconds # inputs of the membership contract constructor # TODO may be able to make these constants private and put them inside the waku_rln_utils diff --git a/logos_delivery/waku/rln/group_manager/group_manager_base.nim b/logos_delivery/waku/rln/group_manager/group_manager_base.nim index d52e8e2ef..c113e06ef 100644 --- a/logos_delivery/waku/rln/group_manager/group_manager_base.nim +++ b/logos_delivery/waku/rln/group_manager/group_manager_base.nim @@ -148,5 +148,13 @@ method invalidateMerkleProofCache*(g: GroupManager) {.base, gcsafe, raises: [].} ## cache and refetches from chain. Base implementation is a no-op. discard +method scheduleMerkleProofRefresh*(g: GroupManager) {.base, gcsafe, raises: [].} = + ## Drops the cached merkle proof path and starts a detached background + ## refetch. Callers invoke this when a publish is rejected on a stale + ## membership cache but must not wait for the refetch themselves: the next + ## proof-gen either finds the refreshed cache or coalesces onto the + ## in-flight refetch. Base implementation is a no-op. + discard + method isReady*(g: GroupManager): Future[bool] {.base, async.} = return true diff --git a/logos_delivery/waku/rln/group_manager/on_chain/group_manager.nim b/logos_delivery/waku/rln/group_manager/on_chain/group_manager.nim index a746068f2..b28f894eb 100644 --- a/logos_delivery/waku/rln/group_manager/on_chain/group_manager.nim +++ b/logos_delivery/waku/rln/group_manager/on_chain/group_manager.nim @@ -309,6 +309,19 @@ method invalidateMerkleProofCache*(g: OnchainGroupManager) {.gcsafe, raises: []. ## 420 INVALID_MESSAGE or 504 OUT_OF_RLN_PROOF reply). g.merkleProofCache = @[] +method scheduleMerkleProofRefresh*(g: OnchainGroupManager) {.gcsafe, raises: [].} = + ## Drops the cached merkle proof path and starts a detached refetch so the + ## cache is warm again by the time a caller retries the publish. A refetch + ## failure is logged and left for the next `ensureFreshMerkleProofPath` call + ## to repair. + g.merkleProofCache = @[] + + proc refresh() {.async.} = + (await g.ensureFreshMerkleProofPath()).isOkOr: + warn "merkle proof refresh failed", error = error + + asyncSpawn refresh() + method register*( g: OnchainGroupManager, rateCommitment: RateCommitment ): Future[Result[void, string]] {.async.} = diff --git a/tests/node/test_wakunode_lightpush.nim b/tests/node/test_wakunode_lightpush.nim index cc8ea255d..f13cbcaab 100644 --- a/tests/node/test_wakunode_lightpush.nim +++ b/tests/node/test_wakunode_lightpush.nim @@ -1,7 +1,7 @@ {.used.} import - std/[options, tempfiles, osproc], + std/[options, tempfiles, osproc, strutils], testutils/unittests, chronos, std/strformat, @@ -212,12 +212,62 @@ suite "RLN Proofs as a Lightpush Service": # The tests below drive `server.lightpushPublish(...)` directly against the # server node. Because `server.wakuLightPush` is mounted (and no lightpush # client is), lightpushPublish takes the self-request path — it still runs - # the full client-side flow (proof gen, retry-on-420/504, one-retry cap), - # but the request lands in the local pushHandler instead of going over the + # the full client-side flow (proof gen, RLN-rejection classification), but + # the request lands in the local pushHandler instead of going over the # wire. Swapping in a stub pushHandler lets each test control exactly what - # attempt N sees. + # each attempt sees. - asyncTest "retry fires on 420 (INVALID_MESSAGE) and second attempt succeeds": + asyncTest "RLN-related 420 surfaces as 504 and schedules a merkle proof refresh": + var callCount = 0 + let stub: PushMessageHandler = proc( + pubsubTopic: PubsubTopic, message: WakuMessage + ): Future[WakuLightPushResult] {.async.} = + inc callCount + return + lighpushErrorResult(LightPushErrorCode.INVALID_MESSAGE, RlnValidatorErrorMsg) + server.wakuLightPush.pushHandler = stub + + let manager = cast[OnchainGroupManager](server.rln.groupManager) + let goodCache = manager.merkleProofCache + check goodCache.len > 0 + + let response = await server.lightpushPublish(some(pubsubTopic), message) + + check: + # The rejection is surfaced immediately — no internal republish — and + # normalized to 504 so callers can key their retry on it. + callCount == 1 + response.isErr() + response.error.code == LightPushErrorCode.OUT_OF_RLN_PROOF + response.error.desc.get("").contains(RlnProofRefreshScheduledMsg) + + # The refresh runs detached from the publish call: the cache was + # dropped and a refetch is in flight. Once it settles the path is + # fresh again. + let inFlight = manager.proofPathRefreshInFlightFut + if not inFlight.isNil(): + await inFlight.join() + check manager.merkleProofCache == goodCache + + asyncTest "504 (OUT_OF_RLN_PROOF) from the service passes through without republish": + var callCount = 0 + let stub: PushMessageHandler = proc( + pubsubTopic: PubsubTopic, message: WakuMessage + ): Future[WakuLightPushResult] {.async.} = + inc callCount + return lighpushErrorResult( + LightPushErrorCode.OUT_OF_RLN_PROOF, "simulated out-of-proof" + ) + server.wakuLightPush.pushHandler = stub + + let response = await server.lightpushPublish(some(pubsubTopic), message) + + check: + callCount == 1 + response.isErr() + response.error.code == LightPushErrorCode.OUT_OF_RLN_PROOF + + asyncTest "caller retry after 504 lands on the refreshed merkle path": var callCount = 0 let stub: PushMessageHandler = proc( pubsubTopic: PubsubTopic, message: WakuMessage @@ -230,34 +280,39 @@ suite "RLN Proofs as a Lightpush Service": return lightpushSuccessResult(1) server.wakuLightPush.pushHandler = stub - let response = await server.lightpushPublish(some(pubsubTopic), message) + let firstResponse = await server.lightpushPublish(some(pubsubTopic), message) + + check: + firstResponse.isErr() + firstResponse.error.code == LightPushErrorCode.OUT_OF_RLN_PROOF + + # Model the caller-level retry (send service next round, REST handler): + # a second publish regenerates the proof against the refreshed cache. + let secondResponse = await server.lightpushPublish(some(pubsubTopic), message) check: callCount == 2 - response.isOk() - response.get() == 1 + secondResponse.isOk() + secondResponse.get() == 1 - asyncTest "retry fires on 504 (OUT_OF_RLN_PROOF) and second attempt succeeds": + asyncTest "420 without the RLN error text passes through unchanged": var callCount = 0 let stub: PushMessageHandler = proc( pubsubTopic: PubsubTopic, message: WakuMessage ): Future[WakuLightPushResult] {.async.} = inc callCount - if callCount == 1: - return lighpushErrorResult( - LightPushErrorCode.OUT_OF_RLN_PROOF, "simulated out-of-proof" - ) - return lightpushSuccessResult(1) + return + lighpushErrorResult(LightPushErrorCode.INVALID_MESSAGE, "unrelated rejection") server.wakuLightPush.pushHandler = stub let response = await server.lightpushPublish(some(pubsubTopic), message) check: - callCount == 2 - response.isOk() - response.get() == 1 + callCount == 1 + response.isErr() + response.error.code == LightPushErrorCode.INVALID_MESSAGE - asyncTest "no retry on error codes outside 420/504": + asyncTest "error codes outside 420/504 pass through unchanged": var callCount = 0 let stub: PushMessageHandler = proc( pubsubTopic: PubsubTopic, message: WakuMessage @@ -275,26 +330,9 @@ suite "RLN Proofs as a Lightpush Service": response.isErr() response.error.code == LightPushErrorCode.INTERNAL_SERVER_ERROR - asyncTest "retry cap: two consecutive 420s surface the second error": - var callCount = 0 - let stub: PushMessageHandler = proc( - pubsubTopic: PubsubTopic, message: WakuMessage - ): Future[WakuLightPushResult] {.async.} = - inc callCount - return - lighpushErrorResult(LightPushErrorCode.INVALID_MESSAGE, RlnValidatorErrorMsg) - server.wakuLightPush.pushHandler = stub - - let response = await server.lightpushPublish(some(pubsubTopic), message) - - check: - callCount == 2 - response.isErr() - response.error.code == LightPushErrorCode.INVALID_MESSAGE - - asyncTest "no retry when node.rln is nil": - # Detach RLN so the retry branch short-circuits on rln.isNone() even - # for a 420. Restore before teardown so server.stop() sees the same + asyncTest "rejection passes through unchanged when node.rln is nil": + # Detach RLN so the RLN-rejection branch short-circuits on rln.isNone() + # even for a 420. Restore before teardown so server.stop() sees the same # object graph it was constructed with. let savedRln = server.rln server.rln = nil diff --git a/tests/waku_rln_relay/test_rln_group_manager_onchain.nim b/tests/waku_rln_relay/test_rln_group_manager_onchain.nim index c3e38262b..f1de851f1 100644 --- a/tests/waku_rln_relay/test_rln_group_manager_onchain.nim +++ b/tests/waku_rln_relay/test_rln_group_manager_onchain.nim @@ -618,6 +618,30 @@ suite "Onchain group manager": manager.merkleProofCache.len > 0 manager.proofPathRefreshInFlightFut == inFlight + test "scheduleMerkleProofRefresh: drops the cache and refetches in the background": + (waitFor manager.init()).isOkOr: + raiseAssert $error + + let credentials = generateCredentials() + (waitFor manager.register(credentials, UserMessageLimit(20))).isOkOr: + assert false, "register failed: " & error + + # Prime the cache, then corrupt it to model the stale path a publish + # rejection points at. + let goodCache = (waitFor manager.fetchMerkleProofElements()).valueOr: + raiseAssert "failed to prime path: " & error + manager.merkleProofCache = newSeq[byte](goodCache.len) + manager.proofPathRefreshInFlightFut = nil + + manager.scheduleMerkleProofRefresh() + + # The call returns without waiting; the refetch is already in flight. + let inFlight = manager.proofPathRefreshInFlightFut + check inFlight != nil + + waitFor inFlight.join() + check manager.merkleProofCache == goodCache + test "verifyProof: should verify valid proof": let credentials = generateCredentials() (waitFor manager.init()).isOkOr: diff --git a/tests/wakunode_rest/test_rest_relay.nim b/tests/wakunode_rest/test_rest_relay.nim index 95f8b8370..b59dc463d 100644 --- a/tests/wakunode_rest/test_rest_relay.nim +++ b/tests/wakunode_rest/test_rest_relay.nim @@ -1,7 +1,7 @@ {.used.} import - std/[sequtils, strformat, tempfiles, osproc, options], + std/[sequtils, strformat, strutils, tempfiles, osproc, options], stew/byteutils, testutils/unittests, presto, @@ -794,11 +794,12 @@ suite "Waku v2 Rest API - Relay": await restServer.closeWait() await node.stop() - asyncTest "Stale RLN proof triggers force-refresh and retry - POST /relay/v1/messages/{topic}": + asyncTest "Stale RLN proof returns 503 and schedules a refresh - POST /relay/v1/messages/{topic}": ## When the cached Merkle proof path is stale the handler generates a proof ## whose root the local RLN validator rejects. The handler must detect the - ## RlnValidatorErrorMsg, force-refresh the cached path, regenerate the proof - ## with the correct root, and succeed — returning 200 OK. + ## RlnValidatorErrorMsg, schedule a background merkle proof refresh, and + ## fail early with 503 + RlnProofRefreshScheduledMsg. A client retry then + ## succeeds against the refreshed path. let node = testWakuNode() (await node.mountRelay()).isOkOr: assert false, "Failed to mount relay" @@ -826,7 +827,7 @@ suite "Waku v2 Rest API - Relay": # Corrupt the cache with zeros so the first generateRLNProof call produces a # proof with a Merkle root that is not in the valid-roots window. - # validateMessage will return RlnValidatorErrorMsg, triggering the retry. + # validateMessage will return RlnValidatorErrorMsg. manager.merkleProofCache = newSeq[byte](goodCache.len) var restPort = Port(0) @@ -855,21 +856,40 @@ suite "Waku v2 Rest API - Relay": ), ) - # Handler force-refreshed the path and retried successfully + # The handler fails early with the retry signal; the refresh runs detached. check: - response.status == 200 + response.status == 503 $response.contentType == $MIMETYPE_TEXT - response.data == "OK" - manager.merkleProofCache == goodCache # force-refresh restored the correct path + response.data.contains(RlnProofRefreshScheduledMsg) + + let inFlight = manager.proofPathRefreshInFlightFut + if not inFlight.isNil(): + await inFlight.join() + check manager.merkleProofCache == goodCache # refresh restored the correct path + + # A client retry now succeeds against the refreshed path. + let retryResponse = await client.relayPostMessagesV1( + DefaultPubsubTopic, + RelayWakuMessage( + payload: base64.encode("TEST-PAYLOAD"), + contentTopic: some(DefaultContentTopic), + timestamp: some(now()), + ), + ) + + check: + retryResponse.status == 200 + retryResponse.data == "OK" await restServer.stop() await restServer.closeWait() await node.stop() - asyncTest "Stale RLN proof triggers force-refresh and retry - POST /relay/v1/auto/messages/{topic}": - ## Same reactive retry as the static-sharding handler, exercised via the - ## auto-sharding endpoint. A relay-only mesh node is connected so that - ## node.publish() has a gossipsub peer and can return success. + asyncTest "Stale RLN proof returns 503 and schedules a refresh - POST /relay/v1/auto/messages/{topic}": + ## Same fail-fast behavior as the static-sharding handler, exercised via + ## the auto-sharding endpoint. A relay-only mesh node is connected so that + ## node.publish() has a gossipsub peer and the client retry can return + ## success. # Relay-only mesh node — no RLN needed, just provides a gossipsub peer. let meshNode = testWakuNode() @@ -942,11 +962,29 @@ suite "Waku v2 Rest API - Relay": ) ) + # The handler fails early with the retry signal; the refresh runs detached. check: - response.status == 200 + response.status == 503 $response.contentType == $MIMETYPE_TEXT - response.data == "OK" - manager.merkleProofCache == goodCache + response.data.contains(RlnProofRefreshScheduledMsg) + + let inFlight = manager.proofPathRefreshInFlightFut + if not inFlight.isNil(): + await inFlight.join() + check manager.merkleProofCache == goodCache + + # A client retry now succeeds against the refreshed path. + let retryResponse = await client.relayPostAutoMessagesV1( + RelayWakuMessage( + payload: base64.encode("TEST-PAYLOAD"), + contentTopic: some(DefaultContentTopic), + timestamp: some(now()), + ) + ) + + check: + retryResponse.status == 200 + retryResponse.data == "OK" await restServer.stop() await restServer.closeWait()