From c0013b3a8fcaf8ebe443a57259647e0619404718 Mon Sep 17 00:00:00 2001 From: Ivan FB Date: Sat, 13 Jun 2026 20:51:11 +0200 Subject: [PATCH] feat(host): registry + in-flight table for {.ffiHost.} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First increment of typed host callbacks (roadmap #1): the data-structure layer, independent of the FFI thread and the macro so it can be unit-tested in isolation. - FFIHostRegistry: wire-name -> (host fn ptr, userData). A missing entry is a normal outcome (the imported proc errors), never a crash — never-crash policy. nil fn unregisters. - FFIPendingTable: monotonic token -> the chronos Future an awaiting {.ffiHost.} proc is blocked on. completePending drops unknown/double completions; failAllPending errors every outstanding future on teardown so no awaiting handler is abandoned. Both lock-guarded so a host thread and the FFI thread can touch them concurrently; futures are only ever completed on the FFI thread. 6 unit tests pass under orc and refc. Co-Authored-By: Claude Opus 4.8 --- ffi.nim | 8 +- ffi/ffi_host.nim | 173 +++++++++++++++++++++++++++++ tests/unit/test_host_callbacks.nim | 107 ++++++++++++++++++ 3 files changed, 284 insertions(+), 4 deletions(-) create mode 100644 ffi/ffi_host.nim create mode 100644 tests/unit/test_host_callbacks.nim diff --git a/ffi.nim b/ffi.nim index f399e3b..876623c 100644 --- a/ffi.nim +++ b/ffi.nim @@ -3,12 +3,12 @@ import chronos, chronicles import ffi/internal/[ffi_library, ffi_macro], ffi/[ - alloc, ffi_types, ffi_events, ffi_context, ffi_context_pool, ffi_thread_request, - cbor_serial, + alloc, ffi_types, ffi_events, ffi_host, ffi_context, ffi_context_pool, + ffi_thread_request, cbor_serial, ] export atomics, tables export chronos, chronicles export - atomics, alloc, ffi_library, ffi_macro, ffi_types, ffi_events, ffi_context, - ffi_context_pool, ffi_thread_request, cbor_serial + atomics, alloc, ffi_library, ffi_macro, ffi_types, ffi_events, ffi_host, + ffi_context, ffi_context_pool, ffi_thread_request, cbor_serial diff --git a/ffi/ffi_host.nim b/ffi/ffi_host.nim new file mode 100644 index 0000000..b08964f --- /dev/null +++ b/ffi/ffi_host.nim @@ -0,0 +1,173 @@ +## Registry + in-flight table backing typed host callbacks (`{.ffiHost.}`). +## +## This is the data-structure layer of roadmap item #1 (see +## docs/design-host-callbacks.md). It owns two per-context concerns and nothing +## else — the FFI-thread completion bridge and the `{.ffiHost.}` macro land in +## later increments and build on these primitives: +## +## 1. `FFIHostRegistry` — maps a wire name (e.g. "fetch_profile") to the host's +## registered function pointer + userData. A missing entry is a normal, +## non-fatal outcome (the imported proc resolves to an error), never a crash. +## 2. `FFIPendingTable` — maps a monotonic `token` to the chronos `Future` an +## awaiting `{.ffiHost.}` proc is blocked on. The host answers later (on any +## thread) by `token`; the FFI thread drains and completes the future. +## +## Both structures are lock-guarded so a host thread (registering / completing) +## and the FFI thread (looking up / completing) can touch them concurrently. +## Futures themselves are only ever completed on the FFI thread — `complete*` +## here is called from the loop drain, not from the host thread directly. + +import std/[locks, tables] +import chronos +import ./ffi_types + +# --------------------------------------------------------------------------- +# Host function pointer +# --------------------------------------------------------------------------- + +type FFIHostFn* = proc( + token: uint64, req: ptr cchar, reqLen: csize_t, userData: pointer +) {.cdecl, gcsafe, raises: [].} + ## A host-implemented function. `req`/`reqLen` carry the marshaled request + ## (valid only for the duration of the call — the host copies what it needs). + ## The host answers asynchronously via `_host_complete(ctx, token, …)`. + +type HostResult* = object + ## The raw outcome the host delivered for one token: a return code plus the + ## response bytes (native POD or CBOR — decoded by the awaiting proc). + ret*: cint + bytes*: seq[byte] + +proc okResult*(bytes: seq[byte]): HostResult = + return HostResult(ret: RET_OK, bytes: bytes) + +proc errResult*(msg: string): HostResult = + var b = newSeq[byte](msg.len) + if msg.len > 0: + copyMem(addr b[0], unsafeAddr msg[0], msg.len) + return HostResult(ret: RET_ERR, bytes: b) + +# --------------------------------------------------------------------------- +# Host function registry +# --------------------------------------------------------------------------- + +type + HostFnEntry = object + fn: FFIHostFn + userData: pointer + + FFIHostRegistry* = object + lock: Lock + fns: Table[string, HostFnEntry] + +proc initHostRegistry*(reg: var FFIHostRegistry) = + ## Call exactly once on the owning thread before sharing. The embedded `Lock` + ## wraps a platform primitive that cannot be safely double-initialised. + reg.lock.initLock() + reg.fns = initTable[string, HostFnEntry]() + +proc deinitHostRegistry*(reg: var FFIHostRegistry) = + ## Mirror of `initHostRegistry`; call once, after all other threads have + ## stopped using the registry. Resets the GC-managed table so pool slot reuse + ## on another thread doesn't run a destructor against this thread's heap. + reg.lock.deinitLock() + reg.fns = default(Table[string, HostFnEntry]) + +proc registerHostFn*( + reg: var FFIHostRegistry, name: string, fn: FFIHostFn, userData: pointer +): bool {.raises: [].} = + ## Registers (or replaces) the host implementation for `name`. Returns false + ## if `fn` is nil — the only documented failure — so a foreign caller can + ## treat a nil pointer as "unregister intent" without crashing. + if fn.isNil(): + withLock reg.lock: + reg.fns.del(name) + return false + withLock reg.lock: + reg.fns[name] = HostFnEntry(fn: fn, userData: userData) + return true + +proc lookupHostFn*( + reg: var FFIHostRegistry, name: string +): tuple[fn: FFIHostFn, userData: pointer, found: bool] {.raises: [].} = + ## Returns the registered `(fn, userData)` for `name`, or `found == false` when + ## no host implementation exists — the awaiting proc turns that into an error. + var entry: HostFnEntry + var got = false + withLock reg.lock: + if reg.fns.hasKey(name): + entry = reg.fns.getOrDefault(name) + got = true + return (entry.fn, entry.userData, got) + +proc clearHostFns*(reg: var FFIHostRegistry) {.raises: [].} = + withLock reg.lock: + reg.fns.clear() + +# --------------------------------------------------------------------------- +# In-flight completion table +# --------------------------------------------------------------------------- + +type FFIPendingTable* = object + lock: Lock + nextToken: uint64 ## Monotonic; 0 is reserved as "invalid", tokens start at 1. + pending: Table[uint64, Future[HostResult]] + +proc initPendingTable*(tbl: var FFIPendingTable) = + tbl.lock.initLock() + tbl.nextToken = 0'u64 + tbl.pending = initTable[uint64, Future[HostResult]]() + +proc deinitPendingTable*(tbl: var FFIPendingTable) = + tbl.lock.deinitLock() + tbl.pending = default(Table[uint64, Future[HostResult]]) + tbl.nextToken = 0'u64 + +proc newPending*( + tbl: var FFIPendingTable +): tuple[token: uint64, fut: Future[HostResult]] = + ## Allocates a token and registers a fresh, uncompleted future under it. The + ## `{.ffiHost.}` proc awaits the returned future; the host answers by token. + let fut = newFuture[HostResult]("ffiHostCall") + var assigned: uint64 = 0 + withLock tbl.lock: + tbl.nextToken.inc() + assigned = tbl.nextToken + tbl.pending[assigned] = fut + return (assigned, fut) + +proc completePending*( + tbl: var FFIPendingTable, token: uint64, res: HostResult +): bool = + ## Completes and removes the future for `token`. Returns false for an unknown + ## or already-completed token — a late / double completion is dropped, not a + ## crash. MUST be called on the FFI (event-loop) thread: it touches the + ## chronos future. + var fut: Future[HostResult] = nil + withLock tbl.lock: + if tbl.pending.hasKey(token): + fut = tbl.pending.getOrDefault(token) + tbl.pending.del(token) + if fut.isNil() or fut.finished(): + return false + fut.complete(res) + return true + +proc failAllPending*(tbl: var FFIPendingTable, msg: string) = + ## Completes every outstanding future with an error and clears the table — + ## used on context teardown so no awaiting handler is abandoned. FFI thread + ## only. + var futs: seq[Future[HostResult]] = @[] + withLock tbl.lock: + for _, fut in tbl.pending: + futs.add(fut) + tbl.pending.clear() + for fut in futs: + if not fut.isNil() and not fut.finished(): + fut.complete(errResult(msg)) + +proc pendingCount*(tbl: var FFIPendingTable): int {.raises: [].} = + var n = 0 + withLock tbl.lock: + n = tbl.pending.len + return n diff --git a/tests/unit/test_host_callbacks.nim b/tests/unit/test_host_callbacks.nim new file mode 100644 index 0000000..5b5a0df --- /dev/null +++ b/tests/unit/test_host_callbacks.nim @@ -0,0 +1,107 @@ +## Unit tests for the host-callback primitives (`FFIHostRegistry` / +## `FFIPendingTable`) that back `{.ffiHost.}` — roadmap item #1, increment 1 +## (see docs/design-host-callbacks.md). +## +## These exercise the data structures directly: no FFI thread, no macro, no +## completion bridge. They pin down registration, lookup, token allocation, and +## future completion semantics in isolation. + +import std/locks +import unittest2 +import chronos +import ffi + +# A host fn does nothing here — we only assert it round-trips through the +# registry. `userData` carries a tag we read back to prove identity. +proc noopHostFn( + token: uint64, req: ptr cchar, reqLen: csize_t, userData: pointer +) {.cdecl, gcsafe, raises: [].} = + discard + +proc bytesToStr(b: seq[byte]): string = + var s = newString(b.len) + if b.len > 0: + copyMem(addr s[0], unsafeAddr b[0], b.len) + return s + +suite "FFIHostRegistry": + test "register, lookup, replace, and remove": + var reg: FFIHostRegistry + initHostRegistry(reg) + defer: + deinitHostRegistry(reg) + + var tag = 42 + check registerHostFn(reg, "fetch_profile", noopHostFn, addr tag) + + let hit = lookupHostFn(reg, "fetch_profile") + check hit.found + check hit.userData == addr tag + check not hit.fn.isNil() + + # missing name -> found == false (never a crash) + check not lookupHostFn(reg, "does_not_exist").found + + # nil fn unregisters and reports false + check not registerHostFn(reg, "fetch_profile", nil, nil) + check not lookupHostFn(reg, "fetch_profile").found + + test "clear drops every registration": + var reg: FFIHostRegistry + initHostRegistry(reg) + defer: + deinitHostRegistry(reg) + check registerHostFn(reg, "a", noopHostFn, nil) + check registerHostFn(reg, "b", noopHostFn, nil) + clearHostFns(reg) + check not lookupHostFn(reg, "a").found + check not lookupHostFn(reg, "b").found + +suite "FFIPendingTable": + test "tokens are monotonic and start at 1": + var tbl: FFIPendingTable + initPendingTable(tbl) + defer: + deinitPendingTable(tbl) + let a = newPending(tbl) + let b = newPending(tbl) + check a.token == 1'u64 + check b.token == 2'u64 + check tbl.pendingCount == 2 + + test "completePending resolves the awaiting future and removes it": + var tbl: FFIPendingTable + initPendingTable(tbl) + defer: + deinitPendingTable(tbl) + let p = newPending(tbl) + check completePending(tbl, p.token, okResult(@[byte 1, 2, 3])) + check p.fut.finished() + check waitFor(p.fut).ret == RET_OK + check waitFor(p.fut).bytes == @[byte 1, 2, 3] + check tbl.pendingCount == 0 + + test "unknown or double completion is dropped, not fatal": + var tbl: FFIPendingTable + initPendingTable(tbl) + defer: + deinitPendingTable(tbl) + check not completePending(tbl, 999'u64, okResult(@[])) + let p = newPending(tbl) + check completePending(tbl, p.token, okResult(@[])) + check not completePending(tbl, p.token, okResult(@[])) # second time: dropped + + test "failAllPending errors every outstanding future": + var tbl: FFIPendingTable + initPendingTable(tbl) + defer: + deinitPendingTable(tbl) + let p1 = newPending(tbl) + let p2 = newPending(tbl) + failAllPending(tbl, "context shutting down") + check p1.fut.finished() + check p2.fut.finished() + let r = waitFor(p1.fut) + check r.ret == RET_ERR + check bytesToStr(r.bytes) == "context shutting down" + check tbl.pendingCount == 0