From 356f0ccc1b6e4c2064ea28854c9cbfb26ab926a2 Mon Sep 17 00:00:00 2001 From: Ivan Folgueira Bande Date: Tue, 2 Sep 2025 23:47:08 +0200 Subject: [PATCH] working simplification --- ffi.nim | 4 +- ffi/ffi_context.nim | 8 +- ffi/ffi_thread_request.nim | 48 +++++----- ffi/ffi_types.nim | 5 ++ ffi/internal/ffi_macro.nim | 176 +++++++++++++++++++++---------------- 5 files changed, 136 insertions(+), 105 deletions(-) diff --git a/ffi.nim b/ffi.nim index 50d8326..ce1c7ec 100644 --- a/ffi.nim +++ b/ffi.nim @@ -1,5 +1,5 @@ import - ffi/[alloc, ffi_types, ffi_context, ffi_thread_request], - ffi/internal/[ffi_library, ffi_macro] + ffi/internal/[ffi_library, ffi_macro], + ffi/[alloc, ffi_types, ffi_context, ffi_thread_request] export alloc, ffi_library, ffi_macro, ffi_types, ffi_context, ffi_thread_request diff --git a/ffi/ffi_context.nim b/ffi/ffi_context.nim index 8af9820..782a156 100644 --- a/ffi/ffi_context.nim +++ b/ffi/ffi_context.nim @@ -4,7 +4,7 @@ import std/[options, atomics, os, net, locks] import chronicles, chronos, chronos/threadsync, taskpools/channels_spsc_single, results -import ./ffi_types, ./ffi_thread_request +import ./ffi_types, ./ffi_thread_request, ./internal/ffi_macro type FFIContext* = object ffiThread: Thread[(ptr FFIContext)] @@ -20,6 +20,7 @@ type FFIContext* = object eventCallback*: pointer eventUserdata*: pointer running: Atomic[bool] # To control when the threads are running + registeredRequests: Table[string, FFIRequestProc] const git_version* {.strdefine.} = "n/a" @@ -131,7 +132,9 @@ proc ffiThreadBody[TT](ctx: ptr FFIContext) {.thread.} = continue ## Handle the request - asyncSpawn FFIThreadRequest.process(request, addr ffiHandler) + asyncSpawn FFIThreadRequest.process( + request, addr ffiHandler, addr ctx.registeredRequests + ) let fireRes = ctx.reqReceivedSignal.fireSync() if fireRes.isErr(): @@ -148,6 +151,7 @@ proc createFFIContext*[T](tt: typedesc[T]): Result[ptr FFIContext, string] = ctx.reqReceivedSignal = ThreadSignalPtr.new().valueOr: return err("couldn't create reqReceivedSignal ThreadSignalPtr") ctx.lock.initLock() + ctx.registeredRequests = ffi_macro.registeredRequests ctx.running.store(true) diff --git a/ffi/ffi_thread_request.nim b/ffi/ffi_thread_request.nim index 0bc9e91..ac52348 100644 --- a/ffi/ffi_thread_request.nim +++ b/ffi/ffi_thread_request.nim @@ -2,39 +2,34 @@ ## The requests are created by the main thread and processed by ## the Waku Thread. -import std/macros -import std/json, results +import std/[json, macros], results, tables import chronos, chronos/threadsync -import ./ffi_types, ./internal/ffi_macro +import ./ffi_types, ./internal/ffi_macro, ./alloc, ./ffi_context import waku/factory/waku -import ../../../library/waku_thread_requests/requests/peer_manager_request -# type FFIRequestProcessor* = -# concept -# proc process[R]( -# T: type FFIThreadRequest, request: ptr FFIThreadRequest, reqHandler: ptr R -# ) +# import ../../../library/waku_thread_requests/requests/peer_manager_request type FFIThreadRequest* = object callback: FFICallBack userData: pointer - reqId: uint + reqId: cstring reqContent*: pointer proc init*( T: typedesc[FFIThreadRequest], callback: FFICallBack, userData: pointer, - reqId: uint, + reqId: cstring, reqContent: pointer, ): ptr type T = var ret = createShared(FFIThreadRequest) ret[].callback = callback ret[].userData = userData - ret[].reqId = reqId + ret[].reqId = reqId.alloc() ret[].reqContent = reqContent return ret proc deleteRequest(request: ptr FFIThreadRequest) = + deallocShared(request[].reqId) deallocShared(request) proc handleRes[T: string | void]( @@ -63,21 +58,20 @@ proc handleRes[T: string | void]( ) return -proc nilProcess(reqId: uint): Future[Result[string, string]] {.async.} = +proc nilProcess(reqId: cstring): Future[Result[string, string]] {.async.} = return err("This request type is not implemented: " & $reqId) -ffiGenerateProcess() +proc process*[R]( + T: type FFIThreadRequest, + request: ptr FFIThreadRequest, + reqHandler: ptr R, + registeredRequests: ptr Table[string, FFIRequestProc], +) {.async.} = + let reqId = $request[].reqId -# dumpAstGen: -# proc process*[R]( -# T: type FFIThreadRequest, request: ptr FFIThreadRequest, reqHandler: ptr R -# ) {.async.} = -# # reqHandler represents the object that actually processes the request. -# let retFut = -# case request[].reqId -# of 1: -# cast[ptr PeerManagementRequest](request[].reqContent).process(reqHandler) -# else: -# nilProcess(request[].reqId) - -# handleRes(await retFut, request) + let retFut = + if not registeredRequests[].contains(reqId): + nilProcess(request[].reqId) + else: + registeredRequests[][reqId](request[].reqContent, reqHandler) + handleRes(await retFut, request) diff --git a/ffi/ffi_types.nim b/ffi/ffi_types.nim index f57a230..9bd91a7 100644 --- a/ffi/ffi_types.nim +++ b/ffi/ffi_types.nim @@ -1,3 +1,5 @@ +import chronos + ################################################################################ ### Exported types @@ -15,6 +17,9 @@ const RET_MISSING_CALLBACK*: cint = 2 ################################################################################ ### FFI utils +type FFIRequestProc* = + proc(request: pointer, reqHandler: pointer): Future[Result[string, string]] {.async.} + template foreignThreadGc*(body: untyped) = when declared(setupForeignThreadGc): setupForeignThreadGc() diff --git a/ffi/internal/ffi_macro.nim b/ffi/internal/ffi_macro.nim index e01fbcd..246204a 100644 --- a/ffi/internal/ffi_macro.nim +++ b/ffi/internal/ffi_macro.nim @@ -1,38 +1,10 @@ import std/[macros, tables] import chronos -import ../ffi_thread_request, ../ffi_types +import ../ffi_types -var registeredRequests {.compileTime.}: Table[uint, NimNode] - -proc fnv1aHash32*(s: string): uint = - const - FNV_offset_basis: uint = 2166136261'u32.uint - FNV_prime: uint = 16777619'u32.uint - var hash: uint = FNV_offset_basis - for c in s: - hash = hash xor uint(ord(c)) - hash = hash * FNV_prime - return hash - -proc generateProc(p: NimNode, args: varargs[NimNode]): NimNode = - let wrapperPragmas = - nnkPragma.newTree(ident("dynlib"), ident("exportc"), ident("cdecl")) - let origBody = p.body - let wrapperBody = quote: - initializeLibrary() - `origBody` - - let origParams = p.params - let wrapperProc = newProc( - name = p.name, params = @[origParams], pragmas = wrapperPragmas, body = wrapperBody - ) - - var res = newStmtList() - res.add(wrapperProc) - return res +var registeredRequests* {.threadvar.}: Table[string, FFIRequestProc] proc buildFfiNewReqProc(reqTypeName, body: NimNode): NimNode = - # Standard FFI params var formalParams = newSeq[NimNode]() var procNode: NimNode @@ -104,7 +76,7 @@ proc buildFfiNewReqProc(reqTypeName, body: NimNode): NimNode = quote do: let typeStr = $T var ret = - FFIThreadRequest.init(callback, userData, fnv1aHash32(typeStr), `reqObjIdent`) + FFIThreadRequest.init(callback, userData, typeStr.cstring, `reqObjIdent`) return ret ) @@ -116,8 +88,6 @@ proc buildFfiNewReqProc(reqTypeName, body: NimNode): NimNode = pragmas = newEmptyNode(), ) - echo result.repr - proc buildFfiDeleteReqProc(reqTypeName: NimNode): NimNode = ## Generates something like: ## proc ffiDeleteReq(self: ptr CreateNodeRequest) = @@ -151,7 +121,9 @@ proc buildProcessFFIRequestProc(reqTypeName, reqHandler, body: NimNode): NimNode ## appCallbacks: AppCallbacks; ## waku: ptr Waku) ... - # Require: ident: ptr SomeType + ## Builds: + ## proc processFFIRequest*(request: pointer; waku: ptr Waku) ... + if reqHandler.kind != nnkExprColonExpr: error( "Second argument must be a typed parameter, e.g., waku: ptr Waku. Found: " & @@ -165,58 +137,125 @@ proc buildProcessFFIRequestProc(reqTypeName, reqHandler, body: NimNode): NimNode var procNode = body if procNode.kind == nnkStmtList and procNode.len == 1: procNode = procNode[0] - if procNode.kind != nnkLambda: error "registerFFI expects a lambda definition. Found: " & $procNode.kind - var formalParams = newSeq[NimNode]() - - # First param: T: typedesc[reqTypeName] let typedescParam = newIdentDefs(ident("T"), nnkBracketExpr.newTree(ident("typedesc"), reqTypeName)) - formalParams.add(typedescParam) - # Add original lambda params + # Build formal params: (returnType, request: pointer, waku: ptr Waku) let procParams = procNode[3] - for p in procParams[1 .. ^1]: - formalParams.add(p) + var formalParams: seq[NimNode] = @[] + formalParams.add(procParams[0]) # return type + formalParams.add(typedescParam) + formalParams.add(newIdentDefs(ident("request"), ident("pointer"))) + formalParams.add(newIdentDefs(reqHandler[0], rhs)) # e.g. waku: ptr Waku - # Add pointer handler param at the end - formalParams.add( - newIdentDefs(reqHandler[0], rhs) # e.g., waku: ptr Waku - ) - - # Return type first - formalParams = @[procParams[0]] & formalParams - - # Pragmas - let pragmas = - if procNode.len >= 5: - procNode[4] - else: - newEmptyNode() - - # Body + # Inject cast/unpack/defer into the body let bodyNode = if procNode.body.kind == nnkStmtList: procNode.body else: newStmtList(procNode.body) - # Build proc + let newBody = newStmtList() + let reqIdent = ident("req") + + newBody.add quote do: + let `reqIdent`: ptr `reqTypeName` = cast[ptr `reqTypeName`](request) + defer: + ffiDeleteReq(`reqIdent`) + + # automatically unpack fields into locals + for p in procParams[1 ..^ 1]: + let fieldName = p[0] # Ident + + newBody.add quote do: + let `fieldName` = `reqIdent`[].`fieldName` + + # Append user's lambda body + newBody.add(bodyNode) + result = newProc( name = postfix(ident("processFFIRequest"), "*"), params = formalParams, - body = bodyNode, + body = newBody, procType = nnkProcDef, - pragmas = pragmas, + pragmas = + if procNode.len >= 5: + procNode[4] + else: + newEmptyNode(), ) - echo result.repr + +proc addNewRequestToRegistry(reqTypeName, reqHandler: NimNode): NimNode = + ## Adds a new request to the registeredRequests table. + ## The key is the hash of the request type name, and the value is the NimNode + ## representing the request type. + + # Build: request[].reqContent + let reqContent = + newDotExpr(newTree(nnkDerefExpr, ident("request")), ident("reqContent")) + + # Build Future[Result[string, string]] return type + let returnType = nnkBracketExpr.newTree( + ident("Future"), + nnkBracketExpr.newTree(ident("Result"), ident("string"), ident("string")), + ) + + # Extract the type from reqHandler (generic: ptr Waku, ptr Foo, ptr Bar, etc.) + let rhsType = + if reqHandler.kind == nnkExprColonExpr: + reqHandler[1] # Use the explicit type + else: + error "Second argument must be a typed parameter, e.g. waku: ptr Waku" + + # Build: cast[ptr Waku](reqHandler) or cast[ptr Foo](reqHandler) dynamically + let castedHandler = newTree( + nnkCast, + rhsType, # The type, e.g. ptr Waku + ident("reqHandler"), # The expression to cast + ) + + let callExpr = newCall( + newDotExpr(reqTypeName, ident("processFFIRequest")), ident("request"), castedHandler + ) + + var newBody = newStmtList() + newBody.add( + quote do: + return await `callExpr` + ) + + # Build: + # proc(request: pointer, reqHandler: pointer): + # Future[Result[string, string]] {.async.} = + # CreateNodeRequest.processFFIRequest(request, reqHandler) + let asyncProc = newProc( + name = newEmptyNode(), # anonymous proc + params = + @[ + returnType, + newIdentDefs(ident("request"), ident("pointer")), + newIdentDefs(ident("reqHandler"), ident("pointer")), + ], + body = newBody, + pragmas = nnkPragma.newTree(ident("async")), + ) + + let reqTypeNameStr = $reqTypeName + + # Use a string literal instead of reqTypeNameStr + let key = newLit($reqTypeName) + # Generate: registeredRequests["CreateNodeRequest"] = + result = + newAssignment(newTree(nnkBracketExpr, ident("registeredRequests"), key), asyncProc) macro registerFFI*(reqTypeName, reqHandler, body: untyped): untyped = let ffiNewReqProc = buildFfiNewReqProc(reqTypeName, body) let processProc = buildProcessFFIRequestProc(reqTypeName, reqHandler, body) - result = newStmtList(ffiNewReqProc, processProc) + let addNewReqToReg = addNewRequestToRegistry(reqTypeName, reqHandler) + result = newStmtList(ffiNewReqProc, processProc, addNewReqToReg) macro ffiGenerateProcess*(): untyped = ## Generates a case statement that handles all the possible registered FFI requests @@ -226,15 +265,6 @@ macro ffiGenerateProcess*(): untyped = var caseStmt = newTree(nnkCaseStmt, newDotExpr(newTree(nnkBracketExpr, reqParam), ident"reqId")) - for caseKey, typeIdent in registeredRequests.pairs: - let castExpr = newTree( - nnkCast, - newTree(nnkPtrTy, typeIdent), - newDotExpr(newTree(nnkBracketExpr, reqParam), ident"reqContent"), - ) - let callExpr = newCall(ident"process", castExpr, reqHandlerParam) - caseStmt.add newTree(nnkOfBranch, newLit(caseKey), nnkStmtList.newTree(callExpr)) - caseStmt.add newTree( nnkElse, nnkStmtList.newTree( @@ -254,5 +284,3 @@ macro ffiGenerateProcess*(): untyped = ) {.async.} = let `retFutSym` = `caseStmt` handleRes(await `retFutSym`, `reqParam`) - - echo result.repr