From 3323325526bfa4898ca0c5c289638585c341af22 Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Tue, 16 Dec 2025 02:52:20 +0100 Subject: [PATCH 1/5] chore: extend RequestBroker with supporting native and external types and added possibility to define non-async (aka sync) requests for simplicity and performance (#3665) * chore: extend RequestBroker with supporting native and external types and added possibility to define non-async (aka sync) requests for simplicity and performance * Adapt gcsafe pragma for RequestBroker sync requests and provider signatures as requirement --------- Co-authored-by: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> --- tests/common/test_request_broker.nim | 328 ++++++++++++++++++- waku/common/broker/request_broker.nim | 438 ++++++++++++++++++++------ 2 files changed, 651 insertions(+), 115 deletions(-) diff --git a/tests/common/test_request_broker.nim b/tests/common/test_request_broker.nim index 2ffd9cbf8..a534216dc 100644 --- a/tests/common/test_request_broker.nim +++ b/tests/common/test_request_broker.nim @@ -6,6 +6,10 @@ import std/strutils import waku/common/broker/request_broker +## --------------------------------------------------------------------------- +## Async-mode brokers + tests +## --------------------------------------------------------------------------- + RequestBroker: type SimpleResponse = object value*: string @@ -31,11 +35,14 @@ RequestBroker: suffix: string ): Future[Result[DualResponse, string]] {.async.} -RequestBroker: +RequestBroker(async): type ImplicitResponse = ref object note*: string -suite "RequestBroker macro": +static: + doAssert typeof(SimpleResponse.request()) is Future[Result[SimpleResponse, string]] + +suite "RequestBroker macro (async mode)": test "serves zero-argument providers": check SimpleResponse .setProvider( @@ -52,7 +59,7 @@ suite "RequestBroker macro": test "zero-argument request errors when unset": let res = waitFor SimpleResponse.request() - check res.isErr + check res.isErr() check res.error.contains("no zero-arg provider") test "serves input-based providers": @@ -78,7 +85,6 @@ suite "RequestBroker macro": .setProvider( proc(key: string, subKey: int): Future[Result[KeyedResponse, string]] {.async.} = raise newException(ValueError, "simulated failure") - ok(KeyedResponse(key: key, payload: "")) ) .isOk() @@ -90,7 +96,7 @@ suite "RequestBroker macro": test "input request errors when unset": let res = waitFor KeyedResponse.request("foo", 2) - check res.isErr + check res.isErr() check res.error.contains("input signature") test "supports both provider types simultaneously": @@ -109,11 +115,11 @@ suite "RequestBroker macro": .isOk() let noInput = waitFor DualResponse.request() - check noInput.isOk + check noInput.isOk() check noInput.value.note == "base" let withInput = waitFor DualResponse.request("-extra") - check withInput.isOk + check withInput.isOk() check withInput.value.note == "base-extra" check withInput.value.count == 6 @@ -129,7 +135,7 @@ suite "RequestBroker macro": DualResponse.clearProvider() let res = waitFor DualResponse.request() - check res.isErr + check res.isErr() test "implicit zero-argument provider works by default": check ImplicitResponse @@ -140,14 +146,14 @@ suite "RequestBroker macro": .isOk() let res = waitFor ImplicitResponse.request() - check res.isOk + check res.isOk() ImplicitResponse.clearProvider() check res.value.note == "auto" test "implicit zero-argument request errors when unset": let res = waitFor ImplicitResponse.request() - check res.isErr + check res.isErr() check res.error.contains("no zero-arg provider") test "no provider override": @@ -171,7 +177,7 @@ suite "RequestBroker macro": check DualResponse.setProvider(overrideProc).isErr() let noInput = waitFor DualResponse.request() - check noInput.isOk + check noInput.isOk() check noInput.value.note == "base" let stillResponse = waitFor DualResponse.request(" still works") @@ -191,8 +197,306 @@ suite "RequestBroker macro": check DualResponse.setProvider(overrideProc).isOk() let nowSuccWithOverride = waitFor DualResponse.request() - check nowSuccWithOverride.isOk + check nowSuccWithOverride.isOk() check nowSuccWithOverride.value.note == "something else" check nowSuccWithOverride.value.count == 1 DualResponse.clearProvider() + +## --------------------------------------------------------------------------- +## Sync-mode brokers + tests +## --------------------------------------------------------------------------- + +RequestBroker(sync): + type SimpleResponseSync = object + value*: string + + proc signatureFetch*(): Result[SimpleResponseSync, string] + +RequestBroker(sync): + type KeyedResponseSync = object + key*: string + payload*: string + + proc signatureFetchWithKey*( + key: string, subKey: int + ): Result[KeyedResponseSync, string] + +RequestBroker(sync): + type DualResponseSync = object + note*: string + count*: int + + proc signatureNoInput*(): Result[DualResponseSync, string] + proc signatureWithInput*(suffix: string): Result[DualResponseSync, string] + +RequestBroker(sync): + type ImplicitResponseSync = ref object + note*: string + +static: + doAssert typeof(SimpleResponseSync.request()) is Result[SimpleResponseSync, string] + doAssert not ( + typeof(SimpleResponseSync.request()) is Future[Result[SimpleResponseSync, string]] + ) + doAssert typeof(KeyedResponseSync.request("topic", 1)) is + Result[KeyedResponseSync, string] + +suite "RequestBroker macro (sync mode)": + test "serves zero-argument providers (sync)": + check SimpleResponseSync + .setProvider( + proc(): Result[SimpleResponseSync, string] = + ok(SimpleResponseSync(value: "hi")) + ) + .isOk() + + let res = SimpleResponseSync.request() + check res.isOk() + check res.value.value == "hi" + + SimpleResponseSync.clearProvider() + + test "zero-argument request errors when unset (sync)": + let res = SimpleResponseSync.request() + check res.isErr() + check res.error.contains("no zero-arg provider") + + test "serves input-based providers (sync)": + var seen: seq[string] = @[] + check KeyedResponseSync + .setProvider( + proc(key: string, subKey: int): Result[KeyedResponseSync, string] = + seen.add(key) + ok(KeyedResponseSync(key: key, payload: key & "-payload+" & $subKey)) + ) + .isOk() + + let res = KeyedResponseSync.request("topic", 1) + check res.isOk() + check res.value.key == "topic" + check res.value.payload == "topic-payload+1" + check seen == @["topic"] + + KeyedResponseSync.clearProvider() + + test "catches provider exception (sync)": + check KeyedResponseSync + .setProvider( + proc(key: string, subKey: int): Result[KeyedResponseSync, string] = + raise newException(ValueError, "simulated failure") + ) + .isOk() + + let res = KeyedResponseSync.request("neglected", 11) + check res.isErr() + check res.error.contains("simulated failure") + + KeyedResponseSync.clearProvider() + + test "input request errors when unset (sync)": + let res = KeyedResponseSync.request("foo", 2) + check res.isErr() + check res.error.contains("input signature") + + test "supports both provider types simultaneously (sync)": + check DualResponseSync + .setProvider( + proc(): Result[DualResponseSync, string] = + ok(DualResponseSync(note: "base", count: 1)) + ) + .isOk() + + check DualResponseSync + .setProvider( + proc(suffix: string): Result[DualResponseSync, string] = + ok(DualResponseSync(note: "base" & suffix, count: suffix.len)) + ) + .isOk() + + let noInput = DualResponseSync.request() + check noInput.isOk() + check noInput.value.note == "base" + + let withInput = DualResponseSync.request("-extra") + check withInput.isOk() + check withInput.value.note == "base-extra" + check withInput.value.count == 6 + + DualResponseSync.clearProvider() + + test "clearProvider resets both entries (sync)": + check DualResponseSync + .setProvider( + proc(): Result[DualResponseSync, string] = + ok(DualResponseSync(note: "temp", count: 0)) + ) + .isOk() + DualResponseSync.clearProvider() + + let res = DualResponseSync.request() + check res.isErr() + + test "implicit zero-argument provider works by default (sync)": + check ImplicitResponseSync + .setProvider( + proc(): Result[ImplicitResponseSync, string] = + ok(ImplicitResponseSync(note: "auto")) + ) + .isOk() + + let res = ImplicitResponseSync.request() + check res.isOk() + + ImplicitResponseSync.clearProvider() + check res.value.note == "auto" + + test "implicit zero-argument request errors when unset (sync)": + let res = ImplicitResponseSync.request() + check res.isErr() + check res.error.contains("no zero-arg provider") + + test "implicit zero-argument provider raises error (sync)": + check ImplicitResponseSync + .setProvider( + proc(): Result[ImplicitResponseSync, string] = + raise newException(ValueError, "simulated failure") + ) + .isOk() + + let res = ImplicitResponseSync.request() + check res.isErr() + check res.error.contains("simulated failure") + + ImplicitResponseSync.clearProvider() + +## --------------------------------------------------------------------------- +## POD / external type brokers + tests (distinct/alias behavior) +## --------------------------------------------------------------------------- + +type ExternalDefinedTypeAsync = object + label*: string + +type ExternalDefinedTypeSync = object + label*: string + +type ExternalDefinedTypeShared = object + label*: string + +RequestBroker: + type PodResponse = int + + proc signatureFetch*(): Future[Result[PodResponse, string]] {.async.} + +RequestBroker: + type ExternalAliasedResponse = ExternalDefinedTypeAsync + + proc signatureFetch*(): Future[Result[ExternalAliasedResponse, string]] {.async.} + +RequestBroker(sync): + type ExternalAliasedResponseSync = ExternalDefinedTypeSync + + proc signatureFetch*(): Result[ExternalAliasedResponseSync, string] + +RequestBroker(sync): + type DistinctStringResponseA = distinct string + +RequestBroker(sync): + type DistinctStringResponseB = distinct string + +RequestBroker(sync): + type ExternalDistinctResponseA = distinct ExternalDefinedTypeShared + +RequestBroker(sync): + type ExternalDistinctResponseB = distinct ExternalDefinedTypeShared + +suite "RequestBroker macro (POD/external types)": + test "supports non-object response types (async)": + check PodResponse + .setProvider( + proc(): Future[Result[PodResponse, string]] {.async.} = + ok(PodResponse(123)) + ) + .isOk() + + let res = waitFor PodResponse.request() + check res.isOk() + check int(res.value) == 123 + + PodResponse.clearProvider() + + test "supports aliased external types (async)": + check ExternalAliasedResponse + .setProvider( + proc(): Future[Result[ExternalAliasedResponse, string]] {.async.} = + ok(ExternalAliasedResponse(ExternalDefinedTypeAsync(label: "ext"))) + ) + .isOk() + + let res = waitFor ExternalAliasedResponse.request() + check res.isOk() + check ExternalDefinedTypeAsync(res.value).label == "ext" + + ExternalAliasedResponse.clearProvider() + + test "supports aliased external types (sync)": + check ExternalAliasedResponseSync + .setProvider( + proc(): Result[ExternalAliasedResponseSync, string] = + ok(ExternalAliasedResponseSync(ExternalDefinedTypeSync(label: "ext"))) + ) + .isOk() + + let res = ExternalAliasedResponseSync.request() + check res.isOk() + check ExternalDefinedTypeSync(res.value).label == "ext" + + ExternalAliasedResponseSync.clearProvider() + + test "distinct response types avoid overload ambiguity (sync)": + check DistinctStringResponseA + .setProvider( + proc(): Result[DistinctStringResponseA, string] = + ok(DistinctStringResponseA("a")) + ) + .isOk() + + check DistinctStringResponseB + .setProvider( + proc(): Result[DistinctStringResponseB, string] = + ok(DistinctStringResponseB("b")) + ) + .isOk() + + check ExternalDistinctResponseA + .setProvider( + proc(): Result[ExternalDistinctResponseA, string] = + ok(ExternalDistinctResponseA(ExternalDefinedTypeShared(label: "ea"))) + ) + .isOk() + + check ExternalDistinctResponseB + .setProvider( + proc(): Result[ExternalDistinctResponseB, string] = + ok(ExternalDistinctResponseB(ExternalDefinedTypeShared(label: "eb"))) + ) + .isOk() + + let resA = DistinctStringResponseA.request() + let resB = DistinctStringResponseB.request() + check resA.isOk() + check resB.isOk() + check string(resA.value) == "a" + check string(resB.value) == "b" + + let resEA = ExternalDistinctResponseA.request() + let resEB = ExternalDistinctResponseB.request() + check resEA.isOk() + check resEB.isOk() + check ExternalDefinedTypeShared(resEA.value).label == "ea" + check ExternalDefinedTypeShared(resEB.value).label == "eb" + + DistinctStringResponseA.clearProvider() + DistinctStringResponseB.clearProvider() + ExternalDistinctResponseA.clearProvider() + ExternalDistinctResponseB.clearProvider() diff --git a/waku/common/broker/request_broker.nim b/waku/common/broker/request_broker.nim index a8a6651d7..dece77381 100644 --- a/waku/common/broker/request_broker.nim +++ b/waku/common/broker/request_broker.nim @@ -6,8 +6,15 @@ ## Worth considering using it in a single provider, many requester scenario. ## ## Provides a declarative way to define an immutable value type together with a -## thread-local broker that can register an asynchronous provider, dispatch typed -## requests and clear provider. +## thread-local broker that can register an asynchronous or synchronous provider, +## dispatch typed requests and clear provider. +## +## For consideration use `sync` mode RequestBroker when you need to provide simple value(s) +## where there is no long-running async operation involved. +## Typically it act as a accessor for the local state of generic setting. +## +## `async` mode is better to be used when you request date that may involve some long IO operation +## or action. ## ## Usage: ## Declare your desired request type inside a `RequestBroker` macro, add any number of fields. @@ -24,6 +31,56 @@ ## proc signature*(arg1: ArgType, arg2: AnotherArgType): Future[Result[TypeName, string]] ## ## ``` +## +## Sync mode (no `async` / `Future`) can be generated with: +## +## ```nim +## RequestBroker(sync): +## type TypeName = object +## field1*: FieldType +## +## proc signature*(): Result[TypeName, string] +## proc signature*(arg1: ArgType): Result[TypeName, string] +## ``` +## +## Note: When the request type is declared as a native type / alias / externally-defined +## type (i.e. not an inline `object` / `ref object` definition), RequestBroker +## will wrap it in `distinct` automatically unless you already used `distinct`. +## This avoids overload ambiguity when multiple brokers share the same +## underlying base type (Nim overload resolution does not consider return type). +## +## This means that for non-object request types you typically: +## - construct values with an explicit cast/constructor, e.g. `MyType("x")` +## - unwrap with a cast when needed, e.g. `string(myVal)` or `BaseType(myVal)` +## +## Example (native response type): +## ```nim +## RequestBroker(sync): +## type MyCount = int # exported as: `distinct int` +## +## MyCount.setProvider(proc(): Result[MyCount, string] = ok(MyCount(42))) +## let res = MyCount.request() +## if res.isOk(): +## let raw = int(res.get()) +## ``` +## +## Example (externally-defined type): +## ```nim +## type External = object +## label*: string +## +## RequestBroker: +## type MyExternal = External # exported as: `distinct External` +## +## MyExternal.setProvider( +## proc(): Future[Result[MyExternal, string]] {.async.} = +## ok(MyExternal(External(label: "hi"))) +## ) +## let res = await MyExternal.request() +## if res.isOk(): +## let base = External(res.get()) +## echo base.label +## ``` ## The 'TypeName' object defines the requestable data (but also can be seen as request for action with return value). ## The 'signature' proc defines the provider(s) signature, that is enforced at compile time. ## One signature can be with no arguments, another with any number of arguments - where the input arguments are @@ -31,12 +88,12 @@ ## ## After this, you can register a provider anywhere in your code with ## `TypeName.setProvider(...)`, which returns error if already having a provider. -## Providers are async procs or lambdas that take no arguments and return a Future[Result[TypeName, string]]. +## Providers are async procs/lambdas in default mode and sync procs in sync mode. ## Only one provider can be registered at a time per signature type (zero arg and/or multi arg). ## ## Requests can be made from anywhere with no direct dependency on the provider by ## calling `TypeName.request()` - with arguments respecting the signature(s). -## This will asynchronously call the registered provider and return a Future[Result[TypeName, string]]. +## In async mode, this returns a Future[Result[TypeName, string]]. In sync mode, it returns Result[TypeName, string]. ## ## Whenever you no want to process requests (or your object instance that provides the request goes out of scope), ## you can remove it from the broker with `TypeName.clearProvider()`. @@ -49,10 +106,10 @@ ## text*: string ## ## ## Define the request and provider signature, that is enforced at compile time. -## proc signature*(): Future[Result[Greeting, string]] +## proc signature*(): Future[Result[Greeting, string]] {.async.} ## ## ## Also possible to define signature with arbitrary input arguments. -## proc signature*(lang: string): Future[Result[Greeting, string]] +## proc signature*(lang: string): Future[Result[Greeting, string]] {.async.} ## ## ... ## Greeting.setProvider( @@ -60,6 +117,23 @@ ## ok(Greeting(text: "hello")) ## ) ## let res = await Greeting.request() +## +## +## ... +## # using native type as response for a synchronous request. +## RequestBroker(sync): +## type NeedThatInfo = string +## +##... +## NeedThatInfo.setProvider( +## proc(): Result[NeedThatInfo, string] = +## ok("this is the info you wanted") +## ) +## let res = NeedThatInfo.request().valueOr: +## echo "not ok due to: " & error +## NeedThatInfo(":-(") +## +## echo string(res) ## ``` ## If no `signature` proc is declared, a zero-argument form is generated ## automatically, so the caller only needs to provide the type definition. @@ -77,7 +151,11 @@ proc errorFuture[T](message: string): Future[Result[T, string]] {.inline.} = fut.complete(err(Result[T, string], message)) fut -proc isReturnTypeValid(returnType, typeIdent: NimNode): bool = +type RequestBrokerMode = enum + rbAsync + rbSync + +proc isAsyncReturnTypeValid(returnType, typeIdent: NimNode): bool = ## Accept Future[Result[TypeIdent, string]] as the contract. if returnType.kind != nnkBracketExpr or returnType.len != 2: return false @@ -92,6 +170,23 @@ proc isReturnTypeValid(returnType, typeIdent: NimNode): bool = return false inner[2].kind == nnkIdent and inner[2].eqIdent("string") +proc isSyncReturnTypeValid(returnType, typeIdent: NimNode): bool = + ## Accept Result[TypeIdent, string] as the contract. + if returnType.kind != nnkBracketExpr or returnType.len != 3: + return false + if returnType[0].kind != nnkIdent or not returnType[0].eqIdent("Result"): + return false + if returnType[1].kind != nnkIdent or not returnType[1].eqIdent($typeIdent): + return false + returnType[2].kind == nnkIdent and returnType[2].eqIdent("string") + +proc isReturnTypeValid(returnType, typeIdent: NimNode, mode: RequestBrokerMode): bool = + case mode + of rbAsync: + isAsyncReturnTypeValid(returnType, typeIdent) + of rbSync: + isSyncReturnTypeValid(returnType, typeIdent) + proc cloneParams(params: seq[NimNode]): seq[NimNode] = ## Deep copy parameter definitions so they can be inserted in multiple places. result = @[] @@ -109,73 +204,122 @@ proc collectParamNames(params: seq[NimNode]): seq[NimNode] = continue result.add(ident($nameNode)) -proc makeProcType(returnType: NimNode, params: seq[NimNode]): NimNode = +proc makeProcType( + returnType: NimNode, params: seq[NimNode], mode: RequestBrokerMode +): NimNode = var formal = newTree(nnkFormalParams) formal.add(returnType) for param in params: formal.add(param) - let pragmas = newTree(nnkPragma, ident("async")) - newTree(nnkProcTy, formal, pragmas) + case mode + of rbAsync: + let pragmas = newTree(nnkPragma, ident("async")) + newTree(nnkProcTy, formal, pragmas) + of rbSync: + let raisesPragma = newTree( + nnkExprColonExpr, ident("raises"), newTree(nnkBracket, ident("CatchableError")) + ) + let pragmas = newTree(nnkPragma, raisesPragma, ident("gcsafe")) + newTree(nnkProcTy, formal, pragmas) -macro RequestBroker*(body: untyped): untyped = +proc parseMode(modeNode: NimNode): RequestBrokerMode = + ## Parses the mode selector for the 2-argument macro overload. + ## Supported spellings: `sync` / `async` (case-insensitive). + let raw = ($modeNode).strip().toLowerAscii() + case raw + of "sync": + rbSync + of "async": + rbAsync + else: + error("RequestBroker mode must be `sync` or `async` (default is async)", modeNode) + +proc ensureDistinctType(rhs: NimNode): NimNode = + ## For PODs / aliases / externally-defined types, wrap in `distinct` unless + ## it's already distinct. + if rhs.kind == nnkDistinctTy: + return copyNimTree(rhs) + newTree(nnkDistinctTy, copyNimTree(rhs)) + +proc generateRequestBroker(body: NimNode, mode: RequestBrokerMode): NimNode = when defined(requestBrokerDebug): echo body.treeRepr + echo "RequestBroker mode: ", $mode var typeIdent: NimNode = nil var objectDef: NimNode = nil - var isRefObject = false for stmt in body: if stmt.kind == nnkTypeSection: for def in stmt: if def.kind != nnkTypeDef: continue + if not typeIdent.isNil(): + error("Only one type may be declared inside RequestBroker", def) + + typeIdent = baseTypeIdent(def[0]) let rhs = def[2] - var objectType: NimNode + + ## Support inline object types (fields are auto-exported) + ## AND non-object types / aliases (e.g. `string`, `int`, `OtherType`). case rhs.kind of nnkObjectTy: - objectType = rhs + let recList = rhs[2] + if recList.kind != nnkRecList: + error("RequestBroker object must declare a standard field list", rhs) + var exportedRecList = newTree(nnkRecList) + for field in recList: + case field.kind + of nnkIdentDefs: + ensureFieldDef(field) + var cloned = copyNimTree(field) + for i in 0 ..< cloned.len - 2: + cloned[i] = exportIdentNode(cloned[i]) + exportedRecList.add(cloned) + of nnkEmpty: + discard + else: + error( + "RequestBroker object definition only supports simple field declarations", + field, + ) + objectDef = newTree( + nnkObjectTy, copyNimTree(rhs[0]), copyNimTree(rhs[1]), exportedRecList + ) of nnkRefTy: - isRefObject = true - if rhs.len != 1 or rhs[0].kind != nnkObjectTy: - error( - "RequestBroker ref object must wrap a concrete object definition", rhs + if rhs.len != 1: + error("RequestBroker ref type must have a single base", rhs) + if rhs[0].kind == nnkObjectTy: + let obj = rhs[0] + let recList = obj[2] + if recList.kind != nnkRecList: + error("RequestBroker object must declare a standard field list", obj) + var exportedRecList = newTree(nnkRecList) + for field in recList: + case field.kind + of nnkIdentDefs: + ensureFieldDef(field) + var cloned = copyNimTree(field) + for i in 0 ..< cloned.len - 2: + cloned[i] = exportIdentNode(cloned[i]) + exportedRecList.add(cloned) + of nnkEmpty: + discard + else: + error( + "RequestBroker object definition only supports simple field declarations", + field, + ) + let exportedObjectType = newTree( + nnkObjectTy, copyNimTree(obj[0]), copyNimTree(obj[1]), exportedRecList ) - objectType = rhs[0] - else: - continue - if not typeIdent.isNil(): - error("Only one object type may be declared inside RequestBroker", def) - typeIdent = baseTypeIdent(def[0]) - let recList = objectType[2] - if recList.kind != nnkRecList: - error("RequestBroker object must declare a standard field list", objectType) - var exportedRecList = newTree(nnkRecList) - for field in recList: - case field.kind - of nnkIdentDefs: - ensureFieldDef(field) - var cloned = copyNimTree(field) - for i in 0 ..< cloned.len - 2: - cloned[i] = exportIdentNode(cloned[i]) - exportedRecList.add(cloned) - of nnkEmpty: - discard + objectDef = newTree(nnkRefTy, exportedObjectType) else: - error( - "RequestBroker object definition only supports simple field declarations", - field, - ) - let exportedObjectType = newTree( - nnkObjectTy, - copyNimTree(objectType[0]), - copyNimTree(objectType[1]), - exportedRecList, - ) - if isRefObject: - objectDef = newTree(nnkRefTy, exportedObjectType) + ## `ref SomeType` (SomeType can be defined elsewhere) + objectDef = ensureDistinctType(rhs) else: - objectDef = exportedObjectType + ## Non-object type / alias (e.g. `string`, `int`, `SomeExternalType`). + objectDef = ensureDistinctType(rhs) if typeIdent.isNil(): - error("RequestBroker body must declare exactly one object type", body) + error("RequestBroker body must declare exactly one type", body) when defined(requestBrokerDebug): echo "RequestBroker generating type: ", $typeIdent @@ -183,7 +327,6 @@ macro RequestBroker*(body: untyped): untyped = let exportedTypeIdent = postfix(copyNimTree(typeIdent), "*") let typeDisplayName = sanitizeIdentName(typeIdent) let typeNameLit = newLit(typeDisplayName) - let isRefObjectLit = newLit(isRefObject) var zeroArgSig: NimNode = nil var zeroArgProviderName: NimNode = nil var zeroArgFieldName: NimNode = nil @@ -211,10 +354,14 @@ macro RequestBroker*(body: untyped): untyped = if params.len == 0: error("Signature must declare a return type", stmt) let returnType = params[0] - if not isReturnTypeValid(returnType, typeIdent): - error( - "Signature must return Future[Result[`" & $typeIdent & "`, string]]", stmt - ) + if not isReturnTypeValid(returnType, typeIdent, mode): + case mode + of rbAsync: + error( + "Signature must return Future[Result[`" & $typeIdent & "`, string]]", stmt + ) + of rbSync: + error("Signature must return Result[`" & $typeIdent & "`, string]", stmt) let paramCount = params.len - 1 if paramCount == 0: if zeroArgSig != nil: @@ -258,14 +405,20 @@ macro RequestBroker*(body: untyped): untyped = var typeSection = newTree(nnkTypeSection) typeSection.add(newTree(nnkTypeDef, exportedTypeIdent, newEmptyNode(), objectDef)) - let returnType = quote: - Future[Result[`typeIdent`, string]] + let returnType = + case mode + of rbAsync: + quote: + Future[Result[`typeIdent`, string]] + of rbSync: + quote: + Result[`typeIdent`, string] if not zeroArgSig.isNil(): - let procType = makeProcType(returnType, @[]) + let procType = makeProcType(returnType, @[], mode) typeSection.add(newTree(nnkTypeDef, zeroArgProviderName, newEmptyNode(), procType)) if not argSig.isNil(): - let procType = makeProcType(returnType, cloneParams(argParams)) + let procType = makeProcType(returnType, cloneParams(argParams), mode) typeSection.add(newTree(nnkTypeDef, argProviderName, newEmptyNode(), procType)) var brokerRecList = newTree(nnkRecList) @@ -316,33 +469,69 @@ macro RequestBroker*(body: untyped): untyped = quote do: `accessProcIdent`().`zeroArgFieldName` = nil ) - result.add( - quote do: - proc request*( - _: typedesc[`typeIdent`] - ): Future[Result[`typeIdent`, string]] {.async: (raises: []).} = - let provider = `accessProcIdent`().`zeroArgFieldName` - if provider.isNil(): - return err( - "RequestBroker(" & `typeNameLit` & "): no zero-arg provider registered" - ) - let catchedRes = catch: - await provider() + case mode + of rbAsync: + result.add( + quote do: + proc request*( + _: typedesc[`typeIdent`] + ): Future[Result[`typeIdent`, string]] {.async: (raises: []).} = + let provider = `accessProcIdent`().`zeroArgFieldName` + if provider.isNil(): + return err( + "RequestBroker(" & `typeNameLit` & "): no zero-arg provider registered" + ) + let catchedRes = catch: + await provider() - if catchedRes.isErr(): - return err("Request failed:" & catchedRes.error.msg) + if catchedRes.isErr(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & + catchedRes.error.msg + ) - let providerRes = catchedRes.get() - when `isRefObjectLit`: + let providerRes = catchedRes.get() if providerRes.isOk(): let resultValue = providerRes.get() - if resultValue.isNil(): - return err( - "RequestBroker(" & `typeNameLit` & "): provider returned nil result" - ) - return providerRes + when compiles(resultValue.isNil()): + if resultValue.isNil(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider returned nil result" + ) + return providerRes - ) + ) + of rbSync: + result.add( + quote do: + proc request*( + _: typedesc[`typeIdent`] + ): Result[`typeIdent`, string] {.gcsafe, raises: [].} = + let provider = `accessProcIdent`().`zeroArgFieldName` + if provider.isNil(): + return err( + "RequestBroker(" & `typeNameLit` & "): no zero-arg provider registered" + ) + + var providerRes: Result[`typeIdent`, string] + try: + providerRes = provider() + except CatchableError as e: + return err( + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & + e.msg + ) + + if providerRes.isOk(): + let resultValue = providerRes.get() + when compiles(resultValue.isNil()): + if resultValue.isNil(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider returned nil result" + ) + return providerRes + + ) if not argSig.isNil(): result.add( quote do: @@ -363,10 +552,7 @@ macro RequestBroker*(body: untyped): untyped = let argNameIdents = collectParamNames(requestParamDefs) let providerSym = genSym(nskLet, "provider") var formalParams = newTree(nnkFormalParams) - formalParams.add( - quote do: - Future[Result[`typeIdent`, string]] - ) + formalParams.add(copyNimTree(returnType)) formalParams.add( newTree( nnkIdentDefs, @@ -378,8 +564,14 @@ macro RequestBroker*(body: untyped): untyped = for paramDef in requestParamDefs: formalParams.add(paramDef) - let requestPragmas = quote: - {.async: (raises: []), gcsafe.} + let requestPragmas = + case mode + of rbAsync: + quote: + {.async: (raises: []).} + of rbSync: + quote: + {.gcsafe, raises: [].} var providerCall = newCall(providerSym) for argName in argNameIdents: providerCall.add(argName) @@ -396,23 +588,49 @@ macro RequestBroker*(body: untyped): untyped = "): no provider registered for input signature" ) ) - requestBody.add( - quote do: - let catchedRes = catch: - await `providerCall` - if catchedRes.isErr(): - return err("Request failed:" & catchedRes.error.msg) - let providerRes = catchedRes.get() - when `isRefObjectLit`: + case mode + of rbAsync: + requestBody.add( + quote do: + let catchedRes = catch: + await `providerCall` + if catchedRes.isErr(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & + catchedRes.error.msg + ) + + let providerRes = catchedRes.get() if providerRes.isOk(): let resultValue = providerRes.get() - if resultValue.isNil(): - return err( - "RequestBroker(" & `typeNameLit` & "): provider returned nil result" - ) - return providerRes - ) + when compiles(resultValue.isNil()): + if resultValue.isNil(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider returned nil result" + ) + return providerRes + ) + of rbSync: + requestBody.add( + quote do: + var providerRes: Result[`typeIdent`, string] + try: + providerRes = `providerCall` + except CatchableError as e: + return err( + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & e.msg + ) + + if providerRes.isOk(): + let resultValue = providerRes.get() + when compiles(resultValue.isNil()): + if resultValue.isNil(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider returned nil result" + ) + return providerRes + ) # requestBody.add(providerCall) result.add( newTree( @@ -436,3 +654,17 @@ macro RequestBroker*(body: untyped): untyped = when defined(requestBrokerDebug): echo result.repr + + return result + +macro RequestBroker*(body: untyped): untyped = + ## Default (async) mode. + generateRequestBroker(body, rbAsync) + +macro RequestBroker*(mode: untyped, body: untyped): untyped = + ## Explicit mode selector. + ## Example: + ## RequestBroker(sync): + ## type Foo = object + ## proc signature*(): Result[Foo, string] + generateRequestBroker(body, parseMode(mode)) From bc5059083ec0af6bfb91aa98cb546758ad52e6db Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Tue, 16 Dec 2025 13:49:03 -0300 Subject: [PATCH 2/5] chore: pin logos-messaging-interop-tests to `SMOKE_TEST_STABLE` (#3667) * pin to interop-tests SMOKE_TEST_STABLE --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c94577f0..2b12a5109 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,7 +143,7 @@ jobs: nwaku-nwaku-interop-tests: needs: build-docker-image - uses: logos-messaging/logos-messaging-interop-tests/.github/workflows/nim_waku_PR.yml@SMOKE_TEST_0.0.1 + uses: logos-messaging/logos-messaging-interop-tests/.github/workflows/nim_waku_PR.yml@SMOKE_TEST_STABLE with: node_nwaku: ${{ needs.build-docker-image.outputs.image }} From 7c24a15459a1892ffa17421b981f3f3dcf652523 Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Thu, 18 Dec 2025 00:07:29 +0100 Subject: [PATCH 3/5] simple cleanup rm unused DiscoveryManager from waku.nim (#3671) --- waku/factory/waku.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/waku/factory/waku.nim b/waku/factory/waku.nim index bed8a9137..c0380ccc9 100644 --- a/waku/factory/waku.nim +++ b/waku/factory/waku.nim @@ -13,7 +13,6 @@ import libp2p/services/autorelayservice, libp2p/services/hpservice, libp2p/peerid, - libp2p/discovery/discoverymngr, libp2p/discovery/rendezvousinterface, eth/keys, eth/p2p/discoveryv5/enr, @@ -63,7 +62,6 @@ type Waku* = ref object dynamicBootstrapNodes*: seq[RemotePeerInfo] dnsRetryLoopHandle: Future[void] networkConnLoopHandle: Future[void] - discoveryMngr: DiscoveryManager node*: WakuNode From 2d40cb9d62ba24eac9f58da1be3a3c6eb4357253 Mon Sep 17 00:00:00 2001 From: Arseniy Klempner Date: Wed, 17 Dec 2025 18:51:10 -0800 Subject: [PATCH 4/5] fix: hash inputs for external nullifier, remove length prefix for sha256 (#3660) * fix: hash inputs for external nullifier, remove length prefix for sha256 * feat: use nimcrypto keccak instead of sha256 ffi * feat: wrapper function to generate external nullifier --- tests/waku_rln_relay/test_waku_rln_relay.nim | 47 ------------------- .../group_manager/on_chain/group_manager.nim | 7 ++- waku/waku_rln_relay/rln/wrappers.nim | 34 +++++--------- 3 files changed, 16 insertions(+), 72 deletions(-) diff --git a/tests/waku_rln_relay/test_waku_rln_relay.nim b/tests/waku_rln_relay/test_waku_rln_relay.nim index ea3a5ca62..3430657ad 100644 --- a/tests/waku_rln_relay/test_waku_rln_relay.nim +++ b/tests/waku_rln_relay/test_waku_rln_relay.nim @@ -70,53 +70,6 @@ suite "Waku rln relay": info "the generated identity credential: ", idCredential - test "hash Nim Wrappers": - # create an RLN instance - let rlnInstance = createRLNInstanceWrapper() - require: - rlnInstance.isOk() - - # prepare the input - let - msg = "Hello".toBytes() - hashInput = encodeLengthPrefix(msg) - hashInputBuffer = toBuffer(hashInput) - - # prepare other inputs to the hash function - let outputBuffer = default(Buffer) - - let hashSuccess = sha256(unsafeAddr hashInputBuffer, unsafeAddr outputBuffer, true) - require: - hashSuccess - let outputArr = cast[ptr array[32, byte]](outputBuffer.`ptr`)[] - - check: - "1e32b3ab545c07c8b4a7ab1ca4f46bc31e4fdc29ac3b240ef1d54b4017a26e4c" == - outputArr.inHex() - - let - hashOutput = cast[ptr array[32, byte]](outputBuffer.`ptr`)[] - hashOutputHex = hashOutput.toHex() - - info "hash output", hashOutputHex - - test "sha256 hash utils": - # create an RLN instance - let rlnInstance = createRLNInstanceWrapper() - require: - rlnInstance.isOk() - let rln = rlnInstance.get() - - # prepare the input - let msg = "Hello".toBytes() - - let hashRes = sha256(msg) - - check: - hashRes.isOk() - "1e32b3ab545c07c8b4a7ab1ca4f46bc31e4fdc29ac3b240ef1d54b4017a26e4c" == - hashRes.get().inHex() - test "poseidon hash utils": # create an RLN instance let rlnInstance = createRLNInstanceWrapper() diff --git a/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim b/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim index bdb272c1f..2ce7d4423 100644 --- a/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim +++ b/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim @@ -379,7 +379,7 @@ method generateProof*( let x = keccak.keccak256.digest(data) - let extNullifier = poseidon(@[@(epoch), @(rlnIdentifier)]).valueOr: + let extNullifier = generateExternalNullifier(epoch, rlnIdentifier).valueOr: return err("Failed to compute external nullifier: " & error) let witness = RLNWitnessInput( @@ -457,10 +457,9 @@ method verifyProof*( var normalizedProof = proof - normalizedProof.externalNullifier = poseidon( - @[@(proof.epoch), @(proof.rlnIdentifier)] - ).valueOr: + let externalNullifier = generateExternalNullifier(proof.epoch, proof.rlnIdentifier).valueOr: return err("Failed to compute external nullifier: " & error) + normalizedProof.externalNullifier = externalNullifier let proofBytes = serialize(normalizedProof, input) let proofBuffer = proofBytes.toBuffer() diff --git a/waku/waku_rln_relay/rln/wrappers.nim b/waku/waku_rln_relay/rln/wrappers.nim index d1dec2b38..1b2b0270f 100644 --- a/waku/waku_rln_relay/rln/wrappers.nim +++ b/waku/waku_rln_relay/rln/wrappers.nim @@ -6,7 +6,8 @@ import stew/[arrayops, byteutils, endians2], stint, results, - std/[sequtils, strutils, tables] + std/[sequtils, strutils, tables], + nimcrypto/keccak as keccak import ./rln_interface, ../conversion_utils, ../protocol_types, ../protocol_metrics import ../../waku_core, ../../waku_keystore @@ -119,24 +120,6 @@ proc createRLNInstance*(): RLNResult = res = createRLNInstanceLocal() return res -proc sha256*(data: openArray[byte]): RlnRelayResult[MerkleNode] = - ## a thin layer on top of the Nim wrapper of the sha256 hasher - var lenPrefData = encodeLengthPrefix(data) - var - hashInputBuffer = lenPrefData.toBuffer() - outputBuffer: Buffer # will holds the hash output - - trace "sha256 hash input buffer length", bufflen = hashInputBuffer.len - let hashSuccess = sha256(addr hashInputBuffer, addr outputBuffer, true) - - # check whether the hash call is done successfully - if not hashSuccess: - return err("error in sha256 hash") - - let output = cast[ptr MerkleNode](outputBuffer.`ptr`)[] - - return ok(output) - proc poseidon*(data: seq[seq[byte]]): RlnRelayResult[array[32, byte]] = ## a thin layer on top of the Nim wrapper of the poseidon hasher var inputBytes = serialize(data) @@ -180,9 +163,18 @@ proc toLeaves*(rateCommitments: seq[RateCommitment]): RlnRelayResult[seq[seq[byt leaves.add(leaf) return ok(leaves) +proc generateExternalNullifier*( + epoch: Epoch, rlnIdentifier: RlnIdentifier +): RlnRelayResult[ExternalNullifier] = + let epochHash = keccak.keccak256.digest(@(epoch)) + let rlnIdentifierHash = keccak.keccak256.digest(@(rlnIdentifier)) + let externalNullifier = poseidon(@[@(epochHash), @(rlnIdentifierHash)]).valueOr: + return err("Failed to compute external nullifier: " & error) + return ok(externalNullifier) + proc extractMetadata*(proof: RateLimitProof): RlnRelayResult[ProofMetadata] = - let externalNullifier = poseidon(@[@(proof.epoch), @(proof.rlnIdentifier)]).valueOr: - return err("could not construct the external nullifier") + let externalNullifier = generateExternalNullifier(proof.epoch, proof.rlnIdentifier).valueOr: + return err("Failed to compute external nullifier: " & error) return ok( ProofMetadata( nullifier: proof.nullifier, From 834eea945d05b4092466f3953467b28467e6b24c Mon Sep 17 00:00:00 2001 From: Tanya S <120410716+stubbsta@users.noreply.github.com> Date: Fri, 19 Dec 2025 10:55:53 +0200 Subject: [PATCH 5/5] chore: pin rln dependencies to specific version (#3649) * Add foundry version in makefile and install scripts * revert to older verison of Anvil for rln tests and anvil_install fix * pin pnpm version to be installed as rln dep * source pnpm after new install * Add to github path * use npm to install pnpm for rln ci * Update foundry and pnpm versions in Makefile --- Makefile | 6 ++- scripts/install_anvil.sh | 47 ++++++++++++++++++++--- scripts/install_pnpm.sh | 35 +++++++++++++++-- scripts/install_rln_tests_dependencies.sh | 8 ++-- 4 files changed, 84 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 44f1c6495..35c107d2d 100644 --- a/Makefile +++ b/Makefile @@ -119,6 +119,10 @@ endif ################## .PHONY: deps libbacktrace +FOUNDRY_VERSION := 1.5.0 +PNPM_VERSION := 10.23.0 + + rustup: ifeq (, $(shell which cargo)) # Install Rustup if it's not installed @@ -128,7 +132,7 @@ ifeq (, $(shell which cargo)) endif rln-deps: rustup - ./scripts/install_rln_tests_dependencies.sh + ./scripts/install_rln_tests_dependencies.sh $(FOUNDRY_VERSION) $(PNPM_VERSION) deps: | deps-common nat-libs waku.nims diff --git a/scripts/install_anvil.sh b/scripts/install_anvil.sh index 1bf4bd7b1..c573ac31c 100755 --- a/scripts/install_anvil.sh +++ b/scripts/install_anvil.sh @@ -2,14 +2,51 @@ # Install Anvil -if ! command -v anvil &> /dev/null; then +REQUIRED_FOUNDRY_VERSION="$1" + +if command -v anvil &> /dev/null; then + # Foundry is already installed; check the current version. + CURRENT_FOUNDRY_VERSION=$(anvil --version 2>/dev/null | awk '{print $2}') + + if [ -n "$CURRENT_FOUNDRY_VERSION" ]; then + # Compare CURRENT_FOUNDRY_VERSION < REQUIRED_FOUNDRY_VERSION using sort -V + lower_version=$(printf '%s\n%s\n' "$CURRENT_FOUNDRY_VERSION" "$REQUIRED_FOUNDRY_VERSION" | sort -V | head -n1) + + if [ "$lower_version" != "$REQUIRED_FOUNDRY_VERSION" ]; then + echo "Anvil is already installed with version $CURRENT_FOUNDRY_VERSION, which is older than the required $REQUIRED_FOUNDRY_VERSION. Please update Foundry manually if needed." + fi + fi +else BASE_DIR="${XDG_CONFIG_HOME:-$HOME}" FOUNDRY_DIR="${FOUNDRY_DIR:-"$BASE_DIR/.foundry"}" FOUNDRY_BIN_DIR="$FOUNDRY_DIR/bin" + echo "Installing Foundry..." curl -L https://foundry.paradigm.xyz | bash - # Extract the source path from the download result - echo "foundryup_path: $FOUNDRY_BIN_DIR" - # run foundryup - $FOUNDRY_BIN_DIR/foundryup + + # Add Foundry to PATH for this script session + export PATH="$FOUNDRY_BIN_DIR:$PATH" + + # Verify foundryup is available + if ! command -v foundryup >/dev/null 2>&1; then + echo "Error: foundryup installation failed or not found in $FOUNDRY_BIN_DIR" + exit 1 + fi + + # Run foundryup to install the required version + if [ -n "$REQUIRED_FOUNDRY_VERSION" ]; then + echo "Installing Foundry tools version $REQUIRED_FOUNDRY_VERSION..." + foundryup --install "$REQUIRED_FOUNDRY_VERSION" + else + echo "Installing latest Foundry tools..." + foundryup + fi + + # Verify anvil was installed + if ! command -v anvil >/dev/null 2>&1; then + echo "Error: anvil installation failed" + exit 1 + fi + + echo "Anvil successfully installed: $(anvil --version)" fi \ No newline at end of file diff --git a/scripts/install_pnpm.sh b/scripts/install_pnpm.sh index 34ba47b07..fcfc82ccd 100755 --- a/scripts/install_pnpm.sh +++ b/scripts/install_pnpm.sh @@ -1,8 +1,37 @@ #!/usr/bin/env bash # Install pnpm -if ! command -v pnpm &> /dev/null; then - echo "pnpm is not installed, installing it now..." - npm i pnpm --global + +REQUIRED_PNPM_VERSION="$1" + +if command -v pnpm &> /dev/null; then + # pnpm is already installed; check the current version. + CURRENT_PNPM_VERSION=$(pnpm --version 2>/dev/null) + + if [ -n "$CURRENT_PNPM_VERSION" ]; then + # Compare CURRENT_PNPM_VERSION < REQUIRED_PNPM_VERSION using sort -V + lower_version=$(printf '%s\n%s\n' "$CURRENT_PNPM_VERSION" "$REQUIRED_PNPM_VERSION" | sort -V | head -n1) + + if [ "$lower_version" != "$REQUIRED_PNPM_VERSION" ]; then + echo "pnpm is already installed with version $CURRENT_PNPM_VERSION, which is older than the required $REQUIRED_PNPM_VERSION. Please update pnpm manually if needed." + fi + fi +else + # Install pnpm using npm + if [ -n "$REQUIRED_PNPM_VERSION" ]; then + echo "Installing pnpm version $REQUIRED_PNPM_VERSION..." + npm install -g pnpm@$REQUIRED_PNPM_VERSION + else + echo "Installing latest pnpm..." + npm install -g pnpm + fi + + # Verify pnpm was installed + if ! command -v pnpm >/dev/null 2>&1; then + echo "Error: pnpm installation failed" + exit 1 + fi + + echo "pnpm successfully installed: $(pnpm --version)" fi diff --git a/scripts/install_rln_tests_dependencies.sh b/scripts/install_rln_tests_dependencies.sh index e19e0ef3c..c8c083b54 100755 --- a/scripts/install_rln_tests_dependencies.sh +++ b/scripts/install_rln_tests_dependencies.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash # Install Anvil -./scripts/install_anvil.sh +FOUNDRY_VERSION="$1" +./scripts/install_anvil.sh "$FOUNDRY_VERSION" -#Install pnpm -./scripts/install_pnpm.sh \ No newline at end of file +# Install pnpm +PNPM_VERSION="$2" +./scripts/install_pnpm.sh "$PNPM_VERSION" \ No newline at end of file