diff --git a/CHANGELOG.md b/CHANGELOG.md index 713eda8..497f2e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,9 +25,14 @@ All notable changes to this project are documented in this file. `{.ffi.}` / `{.ffiCtor.}` / `{.ffiDtor.}` / `{.ffiRaw.}` / `{.ffiEvent.}` inherits, and each annotation can override it with an `"abi = c"` / `"abi = cbor"` spec (e.g. `{.ffi: "abi = cbor".}`). `declareLibrary` is now - required before any FFI annotation. `c` is parsed and recorded but gated - until its codec lands; only `cbor` currently generates working bindings + required before any FFI annotation ([#78](https://github.com/logos-messaging/nim-ffi/issues/78)). +- `c` (flat C-struct) ABI **codec**: every `{.ffi: "abi = c".}` type gets a + `_CWire` companion plus `cwirePack` / `cwireUnpack` / `cwireFree`. This + first slice covers the flat path — POD scalars and `string` (as `cstring`); + composite fields follow. The `c` proc-dispatch path and the foreign (C++ / + Rust) generators are still pending, so `c` remains rejected on + proc/ctor/dtor/event annotations for now. - Queue-overflow handling: when the bounded event queue is full, the library sets a sticky "stuck" flag, logs an error, fires `not_responding` from the event thread, and rejects subsequent diff --git a/ffi.nim b/ffi.nim index 20f766c..4ef72bd 100644 --- a/ffi.nim +++ b/ffi.nim @@ -1,7 +1,7 @@ import std/[atomics, tables] import chronos, chronicles import - ffi/internal/[ffi_library, ffi_macro], + ffi/internal/[ffi_library, ffi_macro, c_wire], ffi/[ alloc, ffi_types, ffi_events, ffi_handles, ffi_context, ffi_context_pool, ffi_thread_request, cbor_serial, @@ -11,4 +11,4 @@ export atomics, tables export chronos, chronicles export atomics, alloc, ffi_library, ffi_macro, ffi_types, ffi_events, ffi_handles, - ffi_context, ffi_context_pool, ffi_thread_request, cbor_serial + ffi_context, ffi_context_pool, ffi_thread_request, cbor_serial, c_wire diff --git a/ffi/codegen/meta.nim b/ffi/codegen/meta.nim index c316724..d62b14b 100644 --- a/ffi/codegen/meta.nim +++ b/ffi/codegen/meta.nim @@ -5,8 +5,8 @@ import std/strutils type ABIFormat* {.pure.} = enum - ## Wire format for an FFI payload. `Cbor` is wired end-to-end; `C` (flat - ## C-struct) is recognized but gated by the macros until its codegen lands. + ## Wire format for an FFI payload. Only `Cbor` is wired end-to-end; `C` + ## (flat C-struct) has a type codec but no proc-dispatch path yet. Cbor = "cbor" C = "c" @@ -63,8 +63,8 @@ var libraryDeclared* {.compileTime.}: bool = false var currentDefaultABIFormat* {.compileTime.}: ABIFormat = ABIFormat.Cbor proc abiCodegenImplemented*(fmt: ABIFormat): bool = - ## Whether `fmt` has a working end-to-end proc-dispatch path. Only `Cbor` does - ## today; this is the single seam a future PR flips when `c` dispatch lands. + ## Whether `fmt` has a working proc-dispatch path. Only `Cbor` does today; the + ## seam a future PR flips once the `c` dispatch path is wired. fmt == ABIFormat.Cbor proc parseABIFormatName*(name: string): tuple[ok: bool, fmt: ABIFormat] = diff --git a/ffi/internal/c_macro_helpers.nim b/ffi/internal/c_macro_helpers.nim new file mode 100644 index 0000000..00e728e --- /dev/null +++ b/ffi/internal/c_macro_helpers.nim @@ -0,0 +1,209 @@ +## Compile-time helpers used by `ffi_macro.nim` for the `c` (flat C-struct) ABI. +## For each `{.ffi: "abi = c".}` object T, emits a `T_CWire` companion (`string` +## → `cstring`, POD unchanged) plus `cwirePack` / `cwireUnpack` / `cwireFree`. + +import std/macros +import ../codegen/meta + +var emittedCWireTypes {.compileTime.}: seq[string] + +proc isCWireEmitted(typeName: string): bool {.compileTime.} = + ## Indexed scan: works around a Nim 2.2 compile-time VM quirk where `for x in + ## seq` over a freshly-mutated `{.compileTime.}` seq goes stale. + for i in 0 ..< emittedCWireTypes.len: + if emittedCWireTypes[i] == typeName: + return true + false + +proc markCWireEmitted(typeName: string) {.compileTime.} = + if not isCWireEmitted(typeName): + emittedCWireTypes.add(typeName) + +proc cwireTypeName(userTypeName: string): string = + ## Companion-type naming convention; stable so generated tests reach in by name. + userTypeName & "_CWire" + +proc isStringType(t: NimNode): bool = + t.kind == nnkIdent and ($t == "string" or $t == "cstring") + +const cWireSupportedTypes = [ + "int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16", "uint32", + "uint64", "float", "float32", "float64", "bool", "char", "byte", "string", "cstring", +] + +proc assertCWireFieldSupported( + typeName, fieldName: string, fieldType: NimNode +) {.compileTime.} = + ## Gate the `abi = c` whitelist: the flat C codec handles only fixed-width + ## scalars, `bool`, `char`/`byte`, and `string`. Composite types (`seq[T]`, + ## `Opt[T]`, nested objects, ...) must use `abi = cbor`, which supports every + ## type. Future PRs widen this list as the codec grows. + if fieldType.kind == nnkIdent and $fieldType in cWireSupportedTypes: + return + error( + "ffi 'abi = c' type '" & typeName & "': field '" & fieldName & "' of type '" & + repr(fieldType) & + "' is not supported by the flat C codec (supported: scalars, bool, char, " & + "string). Use 'abi = cbor' for composite types." + ) + +proc wireFieldType(userType: NimNode): NimNode = + ## Map a user field type AST → its wire-form AST: `string` → `cstring`, + ## everything else unchanged. + if isStringType(userType): + return ident("cstring") + userType + +proc wireFieldsFor(fieldName: string, fieldType: NimNode): seq[NimNode] = + ## IdentDefs for `fieldName: fieldType` in the wire object. A seq because + ## composite fields later split into two physical fields. + @[newIdentDefs(ident(fieldName), wireFieldType(fieldType), newEmptyNode())] + +proc buildCWireTypeDef( + userTypeName: string, fieldNames: seq[string], fieldTypes: seq[NimNode] +): NimNode = + ## Build the bare `nnkTypeDef` (no enclosing TypeSection) for the wire + ## companion of `userTypeName`. + let wireName = ident(cwireTypeName(userTypeName)) + var fields: seq[NimNode] = @[] + for i in 0 ..< fieldNames.len: + for fd in wireFieldsFor(fieldNames[i], fieldTypes[i]): + fields.add(fd) + let recList = + if fields.len > 0: + newTree(nnkRecList, fields) + else: + newTree( + nnkRecList, newIdentDefs(ident("_placeholder"), ident("uint8"), newEmptyNode()) + ) + let objTy = newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), recList) + newTree(nnkTypeDef, postfix(wireName, "*"), newEmptyNode(), objTy) + +proc emitPackStmt(dstObj, srcObj, fieldNameIdent, userType: NimNode): seq[NimNode] = + ## Populate `dstObj.` from `srcObj.`: cstring allocation for + ## strings, direct copy for POD. + let srcAccess = newDotExpr(srcObj, fieldNameIdent) + let dstAccess = newDotExpr(dstObj, fieldNameIdent) + if isStringType(userType): + return @[newAssignment(dstAccess, newCall(ident("cwireAllocStr"), srcAccess))] + @[newAssignment(dstAccess, srcAccess)] + +proc emitUnpackStmt( + resultObj, srcObj, fieldNameIdent, userType: NimNode +): seq[NimNode] = + ## Fill `resultObj.` from `srcObj.`: `$cstring` copies into + ## Nim-managed memory, POD is a direct copy. + let srcAccess = newDotExpr(srcObj, fieldNameIdent) + let dstAccess = newDotExpr(resultObj, fieldNameIdent) + if isStringType(userType): + return @[newAssignment(dstAccess, newCall(ident("$"), srcAccess))] + @[newAssignment(dstAccess, srcAccess)] + +proc emitFreeStmt(dstObj, fieldNameIdent, userType: NimNode): seq[NimNode] = + ## Release `dstObj.`: free the cstring for strings, nothing for POD. + let dstAccess = newDotExpr(dstObj, fieldNameIdent) + if isStringType(userType): + return @[newCall(ident("cwireFreeStr"), dstAccess)] + @[] + +proc buildCWireProcs( + userTypeName: string, fieldNames: seq[string], fieldTypes: seq[NimNode] +): seq[NimNode] = + ## Generate cwirePack / cwireUnpack / cwireFree procs for `userTypeName`. All + ## three are public (`*`) so the macro-expanded code can call them. + let userName = ident(userTypeName) + let wireName = ident(cwireTypeName(userTypeName)) + + let packDst = ident("dst") + let packSrc = ident("src") + var packBody = newStmtList() + for i in 0 ..< fieldNames.len: + let fIdent = ident(fieldNames[i]) + for s in emitPackStmt(packDst, packSrc, fIdent, fieldTypes[i]): + packBody.add(s) + if fieldNames.len == 0: + packBody.add quote do: + discard + let packProc = newProc( + name = postfix(ident("cwirePack"), "*"), + params = @[ + newEmptyNode(), + newIdentDefs(packDst, nnkVarTy.newTree(wireName)), + newIdentDefs(packSrc, userName), + ], + body = packBody, + ) + + let unpSrc = ident("src") + let unpRes = ident("res") + var unpBody = newStmtList() + unpBody.add quote do: + var `unpRes`: `userName` + for i in 0 ..< fieldNames.len: + let fIdent = ident(fieldNames[i]) + for s in emitUnpackStmt(unpRes, unpSrc, fIdent, fieldTypes[i]): + unpBody.add(s) + unpBody.add quote do: + return `unpRes` + let unpProc = newProc( + name = postfix(ident("cwireUnpack"), "*"), + params = @[userName, newIdentDefs(unpSrc, wireName)], + body = unpBody, + ) + + let freeDst = ident("dst") + var freeBody = newStmtList() + for i in 0 ..< fieldNames.len: + let fIdent = ident(fieldNames[i]) + for s in emitFreeStmt(freeDst, fIdent, fieldTypes[i]): + freeBody.add(s) + if freeBody.len == 0: + freeBody.add quote do: + discard + let freeProc = newProc( + name = postfix(ident("cwireFree"), "*"), + params = @[newEmptyNode(), newIdentDefs(freeDst, nnkVarTy.newTree(wireName))], + body = freeBody, + ) + + @[packProc, unpProc, freeProc] + +proc fieldInfoForType( + typeName: string +): tuple[names: seq[string], types: seq[NimNode]] {.compileTime.} = + ## Look up an ffi type's fields from the compile-time registry and parse each + ## field's recorded type back into a NimNode AST. + for typeMeta in ffiTypeRegistry: + if typeMeta.name != typeName: + continue + var names: seq[string] = @[] + var types: seq[NimNode] = @[] + for f in typeMeta.fields: + names.add(f.name) + types.add(parseExpr(f.typeName)) + return (names, types) + error("fieldInfoForType: ffi type '" & typeName & "' not in registry") + +proc ensureCWireFor(typeName: string, sink: NimNode) {.compileTime.} = + ## Idempotent: if `typeName`'s cwire companion has not yet been emitted, + ## append its TypeSection and conversion procs to `sink` and mark it emitted. + if isCWireEmitted(typeName): + return + markCWireEmitted(typeName) + let info = fieldInfoForType(typeName) + for i in 0 ..< info.names.len: + assertCWireFieldSupported(typeName, info.names[i], info.types[i]) + let section = newNimNode(nnkTypeSection) + section.add(buildCWireTypeDef(typeName, info.names, info.types)) + sink.add(section) + for p in buildCWireProcs(typeName, info.names, info.types): + sink.add(p) + +proc flushCWireCompanions*(): NimNode {.compileTime.} = + ## Emit the `_CWire` companion + conversion procs for every registered + ## `abi = c` type. Called by `genBindings()` (a type-pragma macro can't). + let sink = newStmtList() + for typeMeta in ffiTypeRegistry: + if typeMeta.abiFormat == ABIFormat.C: + ensureCWireFor(typeMeta.name, sink) + sink diff --git a/ffi/internal/c_wire.nim b/ffi/internal/c_wire.nim new file mode 100644 index 0000000..973da35 --- /dev/null +++ b/ffi/internal/c_wire.nim @@ -0,0 +1,17 @@ +## Runtime helpers for the macro-generated `*_CWire` companion types: only the +## `cstring` fields need allocation, packed on pack and released on free. + +import ../alloc + +proc cwireAllocStr*(s: string): cstring {.inline.} = + ## NUL-terminated `malloc` copy of `s` (see `ffi/alloc.nim`); pair with + ## `cwireFreeStr`. Empty input still yields a valid buffer, never NULL. + alloc.alloc(s) + +proc cwireFreeStr*(s: var cstring) {.inline.} = + ## Idempotent free for a `cwireAllocStr` cstring; `nil` is a no-op. Taken by + ## `var` and reset to `nil` after release so a repeated call can't double-free. + if s.isNil(): + return + alloc.dealloc(s) + s = nil diff --git a/ffi/internal/ffi_library.nim b/ffi/internal/ffi_library.nim index b820f24..5eadba6 100644 --- a/ffi/internal/ffi_library.nim +++ b/ffi/internal/ffi_library.nim @@ -26,10 +26,8 @@ macro declareLibraryBase*(libraryName: static[string]): untyped = ## Generate {.passc: "-fPIC".} res.add nnkPragma.newTree(nnkExprColonExpr.newTree(ident"passc", newLit("-fPIC"))) - # The soname / install_name only make sense for an actual shared library and - # break a plain-executable link (fatally so on macOS: `-install_name` requires - # `-dynamiclib`). Emit them only when building `--app:lib` so that declaring a - # library is safe in unit tests that compile the FFI code as an executable. + # soname / install_name only apply to a shared library and break an executable + # link (fatally on macOS), so emit them only under `--app:lib`. if compileOption("app", "lib"): when defined(linux): ## Generates {.passl: "-Wl,-soname,libwaku.so".} (considering libraryName=="waku", for example) @@ -131,9 +129,8 @@ macro declareLibrary*( ## the `ctx: ptr FFIContext[libType]` parameter. See ## `examples/timer/timer.nim` for a working call site. ## - ## `defaultABIFormat` selects the wire format inherited by every `{.ffi.}`, - ## `{.ffiEvent.}`, `{.ffiCtor.}`, ... in this library — `"cbor"` (default) or - ## `"c"`. Individual annotations override it with an `"abi = ..."` spec. + ## `defaultABIFormat` (`"cbor"` default, or `"c"`) is the wire format every + ## annotation inherits unless it overrides with an `"abi = ..."` spec. currentLibType = $libType # so handle-receiver `.ffi.` procs can resolve the pool let (abiOk, abiFmt) = parseABIFormatName(defaultABIFormat) diff --git a/ffi/internal/ffi_macro.nim b/ffi/internal/ffi_macro.nim index 137c4b5..200c8da 100644 --- a/ffi/internal/ffi_macro.nim +++ b/ffi/internal/ffi_macro.nim @@ -2,6 +2,7 @@ import std/[macros, tables, strutils] import chronos import ../ffi_types import ../codegen/[meta, string_helpers] +import ./c_macro_helpers when defined(ffiGenBindings): import ../codegen/rust import ../codegen/cpp @@ -41,6 +42,12 @@ proc gateABIFormat(fmt: ABIFormat, where: string) {.compileTime.} = $fmt ) +proc gateFFITypeABIFormat(fmt: ABIFormat, where: string) {.compileTime.} = + ## Type annotations only register metadata. `cbor` uses the generic CBOR + ## overloads, while `c` emits its flat `_CWire` companion from `genBindings()`. + case fmt + of ABIFormat.Cbor, ABIFormat.C: discard + proc isPtr(typ: NimNode): bool = ## True iff `typ` is a `ptr T` type expression — i.e. an `nnkPtrTy` AST node. ## Used by the binding-generator metadata path to flag pointer-typed params @@ -739,16 +746,15 @@ macro ffi*(args: varargs[untyped]): untyped = ## proc mylib_send*(w: MyLib, cfg: SendConfig): Future[Result[string, string]] {.ffi.} = ## return ok("done") - # The annotated node is always the last vararg; any leading args are ABI - # override specs (`"abi = ..."`). + # Annotated node is the last vararg; leading args are `"abi = ..."` specs. let prc = args[^1] let abiFormat = resolveABIFormat(args[0 ..^ 2]) - # A `{.ffi.}` value type is a passive data definition (it can stand alone, so - # it does not require a declared library). `cbor` serialization rides the - # generic overloads; `c` is recognized but gated until its codec lands. + # A value type stands alone (no library required). Its `c` companion is + # emitted later by `genBindings()`, since a type-pragma macro can only return + # a TypeDef; `cbor` rides the generic overloads. Both abis are valid here. if prc.kind == nnkTypeDef: - gateABIFormat(abiFormat, "`.ffi.` type") + gateFFITypeABIFormat(abiFormat, "`.ffi.` type") var cleanTypeDef = prc.copyNimTree() if cleanTypeDef[0].kind == nnkPragmaExpr: cleanTypeDef[0] = cleanTypeDef[0][0] @@ -1655,7 +1661,9 @@ macro genBindings*( ## Supported languages (-d:targetLang): "rust" (default), "cpp". ## Output path and nim source path default to -d:ffiOutputDir and ## -d:ffiSrcPath, or can be passed as explicit arguments. - ## This macro is a no-op unless -d:ffiGenBindings is set. + ## Foreign-binding file emission is a no-op unless -d:ffiGenBindings is set; + ## the `abi = c` `_CWire` companions are emitted unconditionally (runtime + ## code, not generated files). ## ## Example (all via compile flags): ## genBindings() @@ -1691,4 +1699,7 @@ macro genBindings*( "genBindings: unknown targetLang '" & lang & "'. Use 'rust', 'cpp', or 'cddl'." ) - return newEmptyNode() + let cwireCompanions = flushCWireCompanions() + when defined(ffiDumpMacros): + echo cwireCompanions.repr + cwireCompanions diff --git a/tests/unit/test_c_wire.nim b/tests/unit/test_c_wire.nim new file mode 100644 index 0000000..4ecc0f3 --- /dev/null +++ b/tests/unit/test_c_wire.nim @@ -0,0 +1,54 @@ +## Round-trip correctness for the `c` (flat C-struct) ABI codec — flat path. +## +## Each `{.ffi: "abi = c".}` type gets a `_CWire` companion plus +## `cwirePack` / `cwireUnpack` / `cwireFree`. This asserts +## `cwireUnpack(cwirePack(x)) == x` for the flat scalar+string field shapes, +## including the empty-string edge case the cstring encoding must handle. +## +## `genBindings()` flushes the cwire companions for every abi=c type declared +## above it (a type-pragma macro can't splice them in at the type site). + +import unittest2 +import ffi + +type Flat {.ffi: "abi = c".} = object + name: string + label: string + count: int + size: uint32 + ratio: float64 + flag: bool + +genBindings() + +proc roundTrip(o: Flat): Flat = + ## Pack into the flat wire struct, copy back out, then release the wire + ## allocations — the exact lifecycle a boundary crossing would use. + var wire: Flat_CWire + cwirePack(wire, o) + let back = cwireUnpack(wire) + cwireFree(wire) + return back + +suite "c-ABI cwire round-trip (flat)": + test "populated scalars and strings survive pack/unpack/free": + let o = Flat( + name: "hello", + label: "world", + count: -42, + size: 4_000_000_000'u32, + ratio: 3.5, + flag: true, + ) + let back = roundTrip(o) + check back == o + # Distinct strings must not alias one another. + check back.name == "hello" + check back.label == "world" + + test "empty strings survive pack/unpack/free": + let o = Flat(name: "", label: "", count: 0, size: 0, ratio: 0.0, flag: false) + let back = roundTrip(o) + check back == o + check back.name == "" + check back.label == ""