build(wasm)!: bump the edge to nim-ffi 0.3.0, matching master

Closes the last drift from master's 4a85db1b: the edge was still building
against a vendored, threads-off fork of nim-ffi 0.1.3 while everything else had
moved to 0.3.0.

wasm-deps/ffi is re-vendored from the pinned 0.3.0 with a much smaller patch
than the 0.1.3 one: rather than gating ~20 call sites (and re-gating them on
every bump), ffi_singlethread.nim supplies API-compatible no-ops for
ThreadSignalPtr and Thread, which are what --threads:off actually forbids. The
upstream lifecycle code then compiles untouched. The only behavioural change is
sendRequestToFFIThread, which runs the handler on the caller's chronos loop
instead of enqueuing it for a worker that is never started.

edge_lib.nim moves to the 0.3.0 surface: declareLibrary now takes the library
type, contexts come from the generated <LibType>FFIPool, and genBindings()
closes the file. The five request procs use `{.ffiRaw: "abi = c".}` — the one
annotation that keeps the explicit (ctx, callback, userData, ...cstring) C
signature the browser already calls, so edge_new / edge_lightpush_publish /
edge_filter_subscribe / edge_store_* / edge_stop keep their symbols and arity.

Two things 0.3.0 changes that ARE visible, hence the `!`:

- Request replies are CBOR text strings, not raw bytes. `abi = c` does not
  prevent it: ffiRaw expands to registerReqFFI, which pins the codec to CBOR.
  Short replies (a hash, "") looked fine; only storeQuery's JSON.parse caught
  it. Hosts must strip the 1-5 byte header — the demos and ld-edge.js do, and
  fall through for unframed replies so they still work against a 0.1.x binary.
- FFIContext lost eventCallback/eventUserData in favour of a listener registry.
  logosdeliveryedge_set_event_callback keeps its exported signature and stores
  into a module-level slot instead, so events stay unframed and the JS is
  unchanged. A browser edge node has exactly one context.

Request params must be `string`, not `cstring`: the request is CBOR-encoded, so
a cstring field encodes the pointer and the multiaddr arrives empty.

Verified against the Status staging fleet with lion-signet's selftest — 20
passed, 0 failed, 0 skipped: connect, lightpush v3, store query + paging,
byte-identical payload round trip, history decrypt+verify, ns timestamps as
strings, offline invite recovery, clean teardown.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012qDYE5r2t2dMry5XpWWySu
This commit is contained in:
Ivan FB
2026-08-08 04:23:13 +02:00
co-authored by Claude Opus 5
parent 511716a7cd
commit a9fc67df3f
58 changed files with 14226 additions and 737 deletions
+19 -1
View File
@@ -128,13 +128,31 @@
const enc = new TextEncoder(), dec = new TextDecoder();
const cstr = (s) => { const b = enc.encode(s + "\0"); const p = M._malloc(b.length); M.HEAPU8.set(b, p); return p; };
const decode = (ptr, len) => dec.decode(M.HEAPU8.slice(ptr, ptr + len));
// nim-ffi 0.3.0 frames request replies as a CBOR text string (major type
// 3), even under `abi = c` — the ffiRaw macro expands to registerReqFFI,
// which pins the codec to CBOR. Strip the 1-5 byte header. Anything that
// isn't a well-formed text string (e.g. a raw 0.1.x reply, or an event,
// which the library sends unframed) passes through untouched.
const decodeReply = (ptr, len) => {
if (!ptr || !len) return "";
const b = M.HEAPU8.slice(ptr, ptr + len), ib = b[0], ai = ib & 0x1f;
if ((ib >> 5) !== 3) return dec.decode(b);
let head, size;
if (ai < 24) { head = 1; size = ai; }
else if (ai === 24) { head = 2; size = b[1]; }
else if (ai === 25) { head = 3; size = (b[1] << 8) | b[2]; }
else if (ai === 26) { head = 5; size = ((b[1] << 24) | (b[2] << 16) | (b[3] << 8) | b[4]) >>> 0; }
else return dec.decode(b);
if (head + size !== b.length) return dec.decode(b);
return dec.decode(b.subarray(head, head + size));
};
// Every FFI request goes through here, so it is also where the pump learns
// that there is work outstanding.
const reqCb = () => {
let resolve; const p = new Promise((r) => (resolve = r));
pump.enter();
const fn = M.addFunction((ret, msg, len) => {
M.removeFunction(fn); pump.leave(); resolve({ ret, msg: decode(msg, len) });
M.removeFunction(fn); pump.leave(); resolve({ ret, msg: decodeReply(msg, len) });
}, "viiii");
return { fn, p };
};
+56 -17
View File
@@ -28,27 +28,59 @@ import
logos_delivery/waku/waku_store/common,
logos_delivery/waku/common/paging
declareLibrary("logosdeliveryedge")
declareLibrary("logosdeliveryedge", EdgeNode, defaultABIFormat = "c")
# --- event callback wiring (filter push messages) ----------------------------
var eventCallbackLock: Lock
#
# nim-ffi 0.3.0 dropped `eventCallback` / `eventUserData` from FFIContext in
# favour of a listener registry reached through `<lib>_add_event_listener`. We
# keep the single-callback surface instead: it is the ABI ld-edge.js already
# speaks, and a browser edge node has exactly one context, so a module-level
# slot is equivalent to a per-context one.
var
eventCallbackLock: Lock
gEventCallback: FFICallBack
gEventUserData: pointer
initLock(eventCallbackLock)
proc logosdeliveryedge_set_event_callback(
ctx: ptr FFIContext[EdgeNode], callback: FFICallBack, userData: pointer
) {.exportc, cdecl.} =
if isNil(ctx):
echo "error: invalid context in logosdeliveryedge_set_event_callback"
return
## `ctx` is unused — kept in the signature so the exported C symbol is
## unchanged for existing callers.
eventCallbackLock.acquire()
defer:
eventCallbackLock.release()
ctx[].eventCallback = cast[pointer](callback)
ctx[].eventUserData = userData
gEventCallback = callback
gEventUserData = userData
proc emitEdgeEvent(eventName: string, payload: string) =
## Hands `payload` to the registered callback verbatim, matching what
## nim-ffi 0.1.x's `callEventCallback` put on the wire: RET_OK plus the raw
## JSON bytes (NOT NUL-terminated), which is what ld-edge.js parses.
eventCallbackLock.acquire()
let
cb = gEventCallback
ud = gEventUserData
eventCallbackLock.release()
if cb.isNil:
chronicles.error "no event callback registered", event = eventName
return
try:
if payload.len == 0:
cb(RET_OK, nil, 0.csize_t, ud)
else:
cb(RET_OK, unsafeAddr payload[0], payload.len.csize_t, ud)
except Exception, CatchableError:
chronicles.error "event callback raised",
event = eventName, error = getCurrentExceptionMsg()
# --- create node -------------------------------------------------------------
registerReqFFI(CreateEdgeNodeRequest, ctx: ptr FFIContext[EdgeNode]):
proc(serviceNode: cstring): Future[Result[string, string]] {.async.} =
# `string`, not `cstring`: 0.3.0 packs the request into a CBOR blob, and a
# cstring field would encode the pointer rather than the text (the multiaddr
# then arrives empty). The C entry point converts at the boundary.
proc(serviceNode: string): Future[Result[string, string]] {.async.} =
echo "[edge] creating edge node…"
let rng = crypto.newRng()
let privKey = crypto.PrivateKey.random(PKScheme.Secp256k1, rng).valueOr:
@@ -78,13 +110,15 @@ proc edge_new(
if isNil(callback):
echo "error: missing callback in edge_new"
return nil
var ctx = ffi.createFFIContext[EdgeNode]().valueOr:
# 0.3.0 acquires from a fixed per-library pool that declareLibrary emits as
# <LibType>FFIPool, rather than allocating a fresh context per call.
var ctx = ffi.createFFIContext(EdgeNodeFFIPool).valueOr:
let msg = "Error in createFFIContext: " & $error
callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData)
return nil
ctx.userData = userData
ffi.sendRequestToFFIThread(
ctx, CreateEdgeNodeRequest.ffiNewReq(callback, userData, serviceNode)
ctx, CreateEdgeNodeRequest.ffiNewReq(callback, userData, $serviceNode)
).isOkOr:
let msg = "error in sendRequestToFFIThread: " & $error
callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData)
@@ -100,7 +134,7 @@ proc edge_lightpush_publish(
contentTopic: cstring,
payload: cstring,
metaB64: cstring,
) {.ffi.} =
) {.ffiRaw: "abi = c".} =
## Build a WakuMessage from a content topic + UTF-8 payload and lightpush it. `metaB64` is
## an optional base64 app-defined `meta` field (<=64 bytes) — e.g. a message signature.
let metaBytes =
@@ -128,11 +162,12 @@ proc edge_filter_subscribe(
userData: pointer,
pubsubTopic: cstring,
contentTopics: cstring,
) {.ffi.} =
) {.ffiRaw: "abi = c".} =
proc onPush(pubsubTopic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} =
echo "[edge] filter push received on ", msg.contentTopic, " (",
msg.payload.len, " bytes)"
callEventCallback(ctx, "onReceivedMessage"):
emitEdgeEvent(
"onReceivedMessage",
$(
%*{
"pubsubTopic": string(pubsubTopic),
@@ -140,7 +175,8 @@ proc edge_filter_subscribe(
"payload": string.fromBytes(msg.payload),
"meta": base64.encode(msg.meta),
}
)
),
)
echo "[edge] filter subscribe → ", $contentTopics, " on ", $pubsubTopic
(
@@ -164,7 +200,7 @@ proc edge_store_connect(
callback: FFICallBack,
userData: pointer,
storeNode: cstring,
) {.ffi.} =
) {.ffiRaw: "abi = c".} =
## Dial a dedicated store peer. Only needed when the service node doesn't serve
## store itself (a bootstrap node typically doesn't).
echo "[edge] dialing store node ", $storeNode
@@ -185,7 +221,7 @@ proc edge_store_query(
pageSize: cstring,
forward: cstring,
cursorHex: cstring,
) {.ffi.} =
) {.ffiRaw: "abi = c".} =
## One page of history. `startNs`/`endNs`/`cursorHex` are optional ("" = unset);
## `forward` is "true"/"false". Returns
## {"messages":[{hash,contentTopic,payload,meta,timestamp}], "cursor":"…"}
@@ -252,7 +288,7 @@ proc edge_store_query(
# --- teardown ----------------------------------------------------------------
proc edge_stop(
ctx: ptr FFIContext[EdgeNode], callback: FFICallBack, userData: pointer
) {.ffi.} =
) {.ffiRaw: "abi = c".} =
## Stop the libp2p switch. Without this, "disconnect" in an app leaves the
## WebSocket to the service node open and the server still pushing filter
## messages into a dead callback, and a later reconnect builds a SECOND node.
@@ -264,6 +300,9 @@ proc edge_stop(
echo "[edge] switch stopped"
return ok("")
# Emits nim-ffi's dispatch wrappers; must follow every {.ffiRaw.} above.
genBindings()
# Build as a wasm MAIN module (not a -shared SIDE module): drop --nimMainPrefix
# (which made Nim treat this as a dynamic lib) and alias the NimMain symbol that
# declareLibrary's initializeLibrary importc's. Nim emits a `main` (the module
+7 -3
View File
@@ -1,10 +1,14 @@
import std/[atomics, tables]
import chronos, chronicles
import
ffi/internal/[ffi_library, ffi_macro],
ffi/[alloc, ffi_types, ffi_context, ffi_thread_request]
ffi/internal/[ffi_library, ffi_macro, ffi_export, c_wire],
ffi/[
alloc, ffi_types, ffi_events, ffi_handles, ffi_context, ffi_context_pool,
ffi_thread_request, cbor_serial,
]
export atomics, tables
export chronos, chronicles
export
atomics, alloc, ffi_library, ffi_macro, ffi_types, ffi_context, ffi_thread_request
atomics, alloc, ffi_library, ffi_macro, ffi_export, ffi_types, ffi_events,
ffi_handles, ffi_context, ffi_context_pool, ffi_thread_request, cbor_serial, c_wire
+302 -10
View File
@@ -1,22 +1,314 @@
# ffi.nimble
version = "0.1.3"
version = "0.3.0"
author = "Institute of Free Technology"
description = "FFI framework with custom header generation"
license = "MIT or Apache License 2.0"
packageName = "ffi"
packageName = "ffi"
requires "nim >= 2.2.4"
requires "nim >= 2.2.6"
requires "chronos"
requires "chronicles"
requires "taskpools"
requires "cbor_serialization == 0.3.0"
# Source files to include
# srcDir = "src"
# installFiles = @["src/ffi.nim", "mylib.h"]
const nimFlagsOrc = "--mm:orc -d:chronicles_log_level=WARN"
const nimFlagsRefc = "--mm:refc -d:chronicles_log_level=WARN"
# # 💡 Custom build step before installation
# before install:
# echo "Generating custom C header..."
# exec "nim r tools/gen_header.nim"
const timerSrc = "examples/timer/timer.nim"
const echoSrc = "examples/echo/echo.nim"
import std/[algorithm, os, strutils]
proc discoverUnitTests(): seq[string] =
# `listFiles` returns both .nim sources and any compiled binaries left in
# the dir from prior local runs — filter to .nim so we don't run a test
# twice (and don't try to `nim c -r` a stale binary).
var names: seq[string] = @[]
for path in listFiles(thisDir() / "tests/unit"):
if path.endsWith(".nim"):
let name = path.extractFilename.changeFileExt("")
if name.startsWith("test_"):
names.add(name)
names.sort()
return names
let unitTests = discoverUnitTests()
proc runOrQuit(cmd: string) =
# Workaround for newer nimble (shipping with Nim 2.2.10+) printing the
# OSError from a failed `exec` but exiting 0, which causes CI to report
# green on actual build/test failures. Echo the command first so the log
# makes clear which step failed.
try:
exec cmd
except OSError as e:
echo "command failed: ", cmd
echo "error: ", e.msg
quit(QuitFailure)
proc checkBindingsDiff(regenCmd: string, paths: openArray[string]) =
# On a diff, print a remediation hint instead of a bare diff wall. Re-quitting
# non-zero also dodges the nimble ≥2.2.10 exit-0-on-failure footgun (runOrQuit).
try:
exec "git diff --exit-code -- " & paths.join(" ")
except OSError:
echo "Checked-in bindings are stale. Run `" & regenCmd & "` and commit the result."
quit(QuitFailure)
proc sanFlags(san: string): string =
# Each --passC / --passL adds one literal flag to the C compiler / linker
# invocation — avoids any quoting ambiguity that arises from putting
# space-separated flags inside a single --passC argument.
#
# `asan-ubsan` enables LeakSanitizer too: ASan includes LSan, so leaks are
# reported when ASAN_OPTIONS=detect_leaks=1 (set by the sanitizer CI job).
case san
of "none", "":
""
of "asan-ubsan":
" --passC:-fsanitize=address,undefined" & " --passC:-fno-sanitize-recover=all" &
" --passC:-fno-omit-frame-pointer" & " --passC:-g" &
" --passL:-fsanitize=address,undefined"
of "tsan":
" --passC:-fsanitize=thread" & " --passC:-fno-omit-frame-pointer" & " --passC:-g" &
" --passC:-O1" & " --passL:-fsanitize=thread"
else:
raise newException(ValueError, "unknown NIM_FFI_SAN: " & san)
proc mmModes(): seq[string] =
## Memory-management modes to build under, selected by NIM_FFI_MM (empty = both).
case getEnv("NIM_FFI_MM", "")
of "orc":
@[nimFlagsOrc]
of "refc":
@[nimFlagsRefc]
else:
@[nimFlagsOrc, nimFlagsRefc]
proc applyTsanSuppressions() =
## Adds tsan.supp to TSAN_OPTIONS without clobbering options the CI job set.
let suppPath = thisDir() & "/tsan.supp"
let existing = getEnv("TSAN_OPTIONS")
if existing == "":
putEnv("TSAN_OPTIONS", "suppressions=" & suppPath)
elif "suppressions=" notin existing:
putEnv("TSAN_OPTIONS", existing & ":suppressions=" & suppPath)
proc genBindingsCmd(flags, src: string, langs = "rust", outDir = ""): string =
## One `nim c` that emits `langs` (comma-separated) from `src`. Output dir and
## embedded source path default to `<lang>_bindings/` next to `src`; `outDir`
## overrides every language. `--compileOnly` is enough because the binding
## files are written during macro expansion — nothing is linked.
var cmd =
"nim c " & flags & " -d:ffiGenBindings -d:targetLang=" & langs & " --compileOnly"
if outDir.len > 0:
cmd.add " -d:ffiOutputDir=" & outDir
cmd.add " " & src
cmd
proc removeStaleEchoLib() =
## CMake keys the shared `libecho.so` rebuild on echo.nim's mtime, not on
## `-d:ffiEchoAbiC`, so a stale lib from the other ABI is reused and segfaults.
## Every echo e2e task deletes it first to force a fresh rebuild.
for name in ["libecho.so", "libecho.dylib", "echo.dll"]:
let path = thisDir() / name
if fileExists(path):
rmFile(path)
task buildffi, "Compile the library":
exec "nim c " & nimFlagsOrc & " --app:lib --noMain ffi.nim"
task test, "Run all tests under --mm:orc and --mm:refc":
for flags in [nimFlagsOrc, nimFlagsRefc]:
for t in unitTests:
exec "nim c -r " & flags & " tests/unit/" & t & ".nim"
task test_alloc, "Run alloc unit tests under --mm:orc and --mm:refc":
exec "nim c -r " & nimFlagsOrc & " tests/unit/test_alloc.nim"
exec "nim c -r " & nimFlagsRefc & " tests/unit/test_alloc.nim"
task test_ffi, "Run FFI context integration tests under --mm:orc and --mm:refc":
exec "nim c -r " & nimFlagsOrc & " tests/unit/test_ffi_context.nim"
exec "nim c -r " & nimFlagsRefc & " tests/unit/test_ffi_context.nim"
task test_serial, "Run CBOR codec unit tests":
exec "nim c -r " & nimFlagsOrc & " tests/unit/test_serial.nim"
exec "nim c -r " & nimFlagsRefc & " tests/unit/test_serial.nim"
task bench_codec, "Microbenchmark: cbor vs c (cwire) wire-format codecs":
# Built with -d:danger so the numbers reflect optimized codegen, not the
# debug build. Not part of `test` — timing is a measurement, not a gate.
exec "nim c -r " & nimFlagsOrc & " -d:danger tests/bench/bench_codec.nim"
task bench_ffi_submit,
"Concurrent-submit stress + scaling gate for sendRequestToFFIThread":
# Honors NIM_FFI_SAN / NIM_FFI_MM like test_sanitized so CI drives it under
# asan-ubsan and tsan; FFI_SUBMIT_PER_THREAD sets per-thread volume.
let san = getEnv("NIM_FFI_SAN", "none")
let extra = sanFlags(san)
if san == "tsan":
applyTsanSuppressions()
for flags in mmModes():
exec "nim c -r " & flags & " -d:danger" & extra & " tests/bench/bench_ffi_submit.nim"
task test_cpp_e2e, "Build and run the C++ end-to-end tests for the timer example":
# Regenerate the C++ bindings so the suite always runs against fresh codegen.
runOrQuit "nimble genbindings_cpp"
runOrQuit "nimble genbindings_cpp_echo"
# Force a fresh CBOR libecho: a prior abi=c run leaves a same-named dylib that
# cmake would otherwise reuse, mismatching the CBOR bindings (segfault).
removeStaleEchoLib()
runOrQuit "cmake -S tests/e2e/cpp -B tests/e2e/cpp/build"
runOrQuit "cmake --build tests/e2e/cpp/build --config Debug"
# `-C Debug` is required on Windows multi-config generators because
# gtest_discover_tests(PRE_TEST) loads per-config include files; harmless on
# single-config generators (Make/Ninja) on Linux/macOS.
runOrQuit "ctest --test-dir tests/e2e/cpp/build --output-on-failure -C Debug"
task test_c_e2e, "Build and run the C end-to-end tests for the timer example":
# Regenerate the C bindings so the suite always runs against fresh codegen.
runOrQuit "nimble genbindings_c"
runOrQuit "cmake -S tests/e2e/c -B tests/e2e/c/build"
runOrQuit "cmake --build tests/e2e/c/build --config Debug"
runOrQuit "ctest --test-dir tests/e2e/c/build --output-on-failure -C Debug"
task test_c_abi_e2e, "Build and run the CBOR-free abi=c C end-to-end test (echo)":
runOrQuit "nimble genbindings_c_abi_echo"
removeStaleEchoLib()
runOrQuit "cmake -S tests/e2e/c_abi -B tests/e2e/c_abi/build"
runOrQuit "cmake --build tests/e2e/c_abi/build --config Debug"
runOrQuit "ctest --test-dir tests/e2e/c_abi/build --output-on-failure -C Debug"
task test_sanitized,
"Run all unit tests under a sanitizer (NIM_FFI_SAN) and mm (NIM_FFI_MM)":
let san = getEnv("NIM_FFI_SAN", "none")
let extra = sanFlags(san)
if san == "tsan":
applyTsanSuppressions()
for flags in mmModes():
for t in unitTests:
exec "nim c -r " & flags & extra & " tests/unit/" & t & ".nim"
task test_cpp_e2e_sanitized,
"Build and run the C++ e2e tests with a sanitizer (NIM_FFI_SAN) and mm (NIM_FFI_MM)":
let mm = getEnv("NIM_FFI_MM", "orc")
let san = getEnv("NIM_FFI_SAN", "none")
runOrQuit "nimble genbindings_cpp"
runOrQuit "nimble genbindings_cpp_echo"
# See test_cpp_e2e: force a fresh CBOR libecho so a prior abi=c dylib can't be
# reused against the CBOR bindings.
removeStaleEchoLib()
runOrQuit "cmake -S tests/e2e/cpp -B tests/e2e/cpp/build" & " -DNIM_FFI_MM=" & mm &
" -DNIM_FFI_SANITIZER=" & san
runOrQuit "cmake --build tests/e2e/cpp/build --config Debug -j"
runOrQuit "ctest --test-dir tests/e2e/cpp/build --output-on-failure -C Debug"
task test_c_e2e_sanitized,
"Build and run the C e2e tests with a sanitizer (NIM_FFI_SAN) and mm (NIM_FFI_MM)":
let mm = getEnv("NIM_FFI_MM", "orc")
let san = getEnv("NIM_FFI_SAN", "none")
runOrQuit "nimble genbindings_c"
runOrQuit "cmake -S tests/e2e/c -B tests/e2e/c/build" & " -DNIM_FFI_MM=" & mm &
" -DNIM_FFI_SANITIZER=" & san
runOrQuit "cmake --build tests/e2e/c/build --config Debug -j"
runOrQuit "ctest --test-dir tests/e2e/c/build --output-on-failure -C Debug"
task test_c_abi_e2e_sanitized,
"Build and run the abi=c C e2e test with a sanitizer (NIM_FFI_SAN)":
let san = getEnv("NIM_FFI_SAN", "none")
runOrQuit "nimble genbindings_c_abi_echo"
removeStaleEchoLib()
runOrQuit "cmake -S tests/e2e/c_abi -B tests/e2e/c_abi/build" & " -DNIM_FFI_SANITIZER=" &
san
runOrQuit "cmake --build tests/e2e/c_abi/build --config Debug -j"
runOrQuit "ctest --test-dir tests/e2e/c_abi/build --output-on-failure -C Debug"
task genbindings_example, "Generate Rust bindings for the timer example":
exec genBindingsCmd(nimFlagsOrc, timerSrc)
exec genBindingsCmd(nimFlagsRefc, timerSrc)
task genbindings_rust, "Generate Rust bindings for the timer example":
exec genBindingsCmd(nimFlagsOrc, timerSrc, "rust")
exec genBindingsCmd(nimFlagsRefc, timerSrc, "rust")
task genbindings_cddl, "Generate CDDL schema for the timer example":
exec genBindingsCmd(nimFlagsOrc, timerSrc, "cddl")
task genbindings_cpp, "Generate C++ bindings for the timer example":
exec genBindingsCmd(nimFlagsOrc, timerSrc, "cpp")
exec genBindingsCmd(nimFlagsRefc, timerSrc, "cpp")
task genbindings_cpp_echo, "Generate C++ bindings for the echo example":
exec genBindingsCmd(nimFlagsOrc, echoSrc, "cpp")
exec genBindingsCmd(nimFlagsRefc, echoSrc, "cpp")
task genbindings_c, "Generate C bindings for the timer example":
exec genBindingsCmd(nimFlagsOrc, timerSrc, "c")
exec genBindingsCmd(nimFlagsRefc, timerSrc, "c")
task genbindings_c_echo, "Generate C bindings for the echo example":
exec genBindingsCmd(nimFlagsOrc, echoSrc, "c")
exec genBindingsCmd(nimFlagsRefc, echoSrc, "c")
task genbindings_c_abi_echo, "Generate CBOR-free abi=c C bindings for the echo example":
# abiOut forces output beside the CBOR `c_bindings/` instead of overwriting it.
const abiOut = "examples/echo/c_abi_bindings"
const abiFlags = " -d:ffiEchoAbiC -d:ffiSrcPath=../echo.nim"
exec genBindingsCmd(nimFlagsOrc & abiFlags, echoSrc, "c", abiOut)
exec genBindingsCmd(nimFlagsRefc & abiFlags, echoSrc, "c", abiOut)
task check_bindings_rust, "Verify checked-in Rust bindings match Nim source":
runOrQuit "nimble genbindings_rust"
checkBindingsDiff(
"nimble genbindings_rust",
[
"examples/timer/rust_bindings/Cargo.toml",
"examples/timer/rust_bindings/build.rs", "examples/timer/rust_bindings/src",
],
)
task check_bindings_cpp, "Verify checked-in C++ bindings match Nim source":
runOrQuit "nimble genbindings_cpp"
runOrQuit "nimble genbindings_cpp_echo"
checkBindingsDiff(
"nimble genbindings_cpp && nimble genbindings_cpp_echo",
[
"examples/timer/cpp_bindings/my_timer.hpp",
"examples/timer/cpp_bindings/CMakeLists.txt",
"examples/echo/cpp_bindings/echo.hpp", "examples/echo/cpp_bindings/CMakeLists.txt",
],
)
task check_bindings_c, "Verify checked-in C bindings match Nim source":
runOrQuit "nimble genbindings_c"
runOrQuit "nimble genbindings_c_echo"
checkBindingsDiff(
"nimble genbindings_c && nimble genbindings_c_echo",
[
"examples/timer/c_bindings/my_timer.h",
"examples/timer/c_bindings/nim_ffi_prelude.h",
"examples/timer/c_bindings/nim_ffi_cbor.h",
"examples/timer/c_bindings/CMakeLists.txt", "examples/echo/c_bindings/echo.h",
"examples/echo/c_bindings/nim_ffi_prelude.h",
"examples/echo/c_bindings/nim_ffi_cbor.h",
"examples/echo/c_bindings/CMakeLists.txt",
],
)
task check_bindings_c_abi, "Verify checked-in abi=c C bindings match Nim source":
runOrQuit "nimble genbindings_c_abi_echo"
checkBindingsDiff(
"nimble genbindings_c_abi_echo",
[
"examples/echo/c_abi_bindings/echo.h",
"examples/echo/c_abi_bindings/CMakeLists.txt",
],
)
task check_bindings, "Verify all checked-in example bindings match Nim source":
exec "nimble check_bindings_rust"
exec "nimble check_bindings_cpp"
exec "nimble check_bindings_c"
exec "nimble check_bindings_c_abi"
+31 -15
View File
@@ -1,41 +1,57 @@
## Can be shared safely between threads
## Cross-thread allocation helpers backed by libc `malloc`/`free`.
## Avoids Nim `allocShared` whose TLS-owned MemRegion segfaults when freed from a
## thread other than the one that allocated (and may have since exited); libc is process-global.
import system/ansi_c
type SharedSeq*[T] = tuple[data: ptr UncheckedArray[T], len: int]
proc alloc*(str: cstring): cstring =
# Byte allocation from the given address.
# There should be the corresponding manual deallocation with deallocShared !
## Fresh null-terminated `c_malloc` copy of `str`; free with `dealloc(cstring)`.
if str.isNil():
var ret = cast[cstring](allocShared(1)) # Allocate memory for the null terminator
ret[0] = '\0' # Set the null terminator
var ret = cast[cstring](c_malloc(1))
ret[0] = '\0'
return ret
let ret = cast[cstring](allocShared(len(str) + 1))
let ret = cast[cstring](c_malloc(csize_t(len(str) + 1)))
copyMem(ret, str, len(str) + 1)
return ret
proc alloc*(str: string): cstring =
## Byte allocation from the given address.
## There should be the corresponding manual deallocation with deallocShared !
var ret = cast[cstring](allocShared(str.len + 1))
var ret = cast[cstring](c_malloc(csize_t(str.len + 1)))
let s = cast[seq[char]](str)
for i in 0 ..< str.len:
ret[i] = s[i]
ret[str.len] = '\0'
return ret
proc dealloc*(p: cstring) {.inline.} =
## Frees an `alloc(...)` buffer. Nil-safe.
if not p.isNil():
c_free(cast[pointer](p))
proc allocBox*(size: int): pointer =
## `c_malloc` block for a cross-thread callback box; free with `freeBox`.
c_malloc(csize_t(size))
proc freeBox*(p: pointer) =
if not p.isNil():
c_free(p)
proc allocSharedSeq*[T](s: seq[T]): SharedSeq[T] =
let data = allocShared(sizeof(T) * s.len)
if s.len != 0:
copyMem(data, unsafeAddr s[0], s.len)
if s.len == 0:
return (cast[ptr UncheckedArray[T]](nil), 0)
let data = c_malloc(csize_t(sizeof(T) * s.len))
copyMem(data, unsafeAddr s[0], sizeof(T) * s.len)
return (cast[ptr UncheckedArray[T]](data), s.len)
proc deallocSharedSeq*[T](s: var SharedSeq[T]) =
deallocShared(s.data)
if not s.data.isNil():
c_free(s.data)
s.len = 0
proc toSeq*[T](s: SharedSeq[T]): seq[T] =
## Creates a seq[T] from a SharedSeq[T]. No explicit dealloc is required
## as req[T] is a GC managed type.
var ret = newSeq[T]()
for i in 0 ..< s.len:
ret.add(s.data[i])
+46
View File
@@ -0,0 +1,46 @@
## `cbor_serialization` wrapper adapting its exception API to `Result[T, string]` for the FFI layer.
## `.ffi.` payloads (plain `object` and `ref T`) cross as value copies; raw `pointer`/`ptr T` are
## rejected at macro-expansion time (see `rejectRawPtrType`).
import system/ansi_c
import cbor_serialization, cbor_serialization/std/options, results
export cbor_serialization, options, results
const CborNullByte*: byte = 0xf6'u8
## CBOR `null` — wire sentinel for empty OK payloads.
proc cborEncode*[T](x: T): seq[byte] =
return Cbor.encode(x)
proc cborEncodeShared*[T](x: T): tuple[data: ptr UncheckedArray[byte], len: int] =
## Encodes `x` into a caller-owned `c_malloc` buffer (free via `cborFreeShared`).
## Empty payloads return `(nil, 0)` without allocating.
let bytes = Cbor.encode(x)
if bytes.len == 0:
return (nil, 0)
let buf = cast[ptr UncheckedArray[byte]](c_malloc(csize_t(bytes.len)))
copyMem(buf, unsafeAddr bytes[0], bytes.len)
return (buf, bytes.len)
proc cborFreeShared*(data: var ptr UncheckedArray[byte]) =
## Frees a `cborEncodeShared` buffer and nils the pointer. Nil-safe.
if not data.isNil():
c_free(data)
data = nil
proc cborDecode*[T](data: openArray[byte], _: typedesc[T]): Result[T, string] =
## Decode `data` into a `T`, mapping any exception to `Result.err`.
try:
let v = Cbor.decode(data, T)
return ok(v)
except CatchableError as exc:
return err(exc.msg)
proc cborDecodePtr*[T](
data: ptr UncheckedArray[byte], dataLen: int, _: typedesc[T]
): Result[T, string] =
## Convenience for ptr+len buffers.
if dataLen <= 0:
return cborDecode(default(seq[byte]), T)
cborDecode(toOpenArray(data, 0, dataLen - 1), T)
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,25 @@
## Helpers shared by the C/C++ binding generators (cpp.nim, c.nim).
import std/strutils
import ./meta, ./string_helpers
proc stripLibPrefix*(procName, libName: string): string =
## Drops the `<lib>_` prefix from an exported C symbol.
let prefix = libName & "_"
if procName.startsWith(prefix):
return procName[prefix.len .. ^1]
return procName
proc reqStructName*(p: FFIProcMeta): string =
## Per-proc wire envelope name: `<PascalCase(procName)>Req` (`...CtorReq` for ctors).
let camel = snakeToPascalCase(p.procName)
if p.kind == FFIKind.CTOR:
camel & "CtorReq"
else:
camel & "Req"
proc libTypeName*(ctors: seq[FFIProcMeta], libName: string): string =
## The library type name, from the first ctor or derived from `libName`.
if ctors.len > 0:
return ctors[0].libTypeName
capitalizeFirstLetter(libName)
+190
View File
@@ -0,0 +1,190 @@
## CDDL (RFC 8610) schema generator mirroring the CBOR wire format from
## ffi/cbor_serial.nim: types become rules, procs get request/response rules.
import std/[os, strutils, unicode]
import ./meta, ./string_helpers
proc innerOf(typeName, prefix: string): string =
if typeName.startsWith(prefix) and typeName.endsWith("]"):
return typeName[prefix.len .. ^2]
return ""
proc capitalizeFirstLetter(s: string): string =
if s.len == 0:
return s
return s.capitalize()
proc toCamelCase(s: string): string =
## "testlib_create" → "TestlibCreate"
var parts = s.split('_')
var res = ""
for p in parts:
res.add capitalizeFirstLetter(p)
return res
proc nimTypeToCddl*(typeName: string): string =
## Nim type name → CDDL equivalent; unknown names pass through as rule refs.
let t = typeName.strip()
let seqI = innerOf(t, "seq[")
if seqI.len > 0:
let inner = seqI.strip()
if inner == "byte" or inner == "uint8":
# seq[byte] rides the wire as a CBOR byte string.
return "bytes"
return "[* " & nimTypeToCddl(inner) & "]"
let arrI = innerOf(t, "array[")
if arrI.len > 0:
# Emit an unbounded array of the element type (CDDL lacks a fixed-length literal).
let commaIdx = arrI.find(',')
let elemT =
if commaIdx >= 0:
arrI[commaIdx + 1 .. ^1].strip()
else:
arrI
return "[* " & nimTypeToCddl(elemT) & "]"
let optI = innerOf(t, "Option[")
if optI.len > 0:
return nimTypeToCddl(optI) & " / nil"
let mayI = innerOf(t, "Maybe[")
if mayI.len > 0:
return nimTypeToCddl(mayI) & " / nil"
case t
of "bool": "bool"
of "int", "int64", "int32", "int16", "int8": "int"
of "uint", "uint64", "uint32", "uint16", "uint8", "byte": "uint"
of "string", "cstring": "tstr"
of "float", "float64": "float64"
of "float32": "float32"
of "pointer": "uint"
else: t
proc reqStructName(p: FFIProcMeta): string =
## Mirrors the Nim macro: <CamelCase(procName)>{Ctor}Req.
let camel = toCamelCase(p.procName)
if p.kind == FFIKind.CTOR:
camel & "CtorReq"
else:
camel & "Req"
proc emitMap(
fields: openArray[tuple[name: string, typeName: string, isPtr: bool]]
): string =
if fields.len == 0:
return "{ }"
var parts: seq[string] = @[]
for f in fields:
let cddlType =
if f.isPtr:
"uint"
else:
nimTypeToCddl(f.typeName)
parts.add(f.name & ": " & cddlType)
"{ " & parts.join(", ") & " }"
proc emitEnumAlternatives(t: FFITypeMeta): string =
## An enum rides as the CBOR text `$value` yields, so the rule is a choice of
## string literals.
var alts: seq[string] = @[]
for v in t.enumValues:
alts.add("\"" & v.wire & "\"")
alts.join(" / ")
proc emitObjectFields(t: FFITypeMeta): string =
var fields: seq[tuple[name: string, typeName: string, isPtr: bool]] = @[]
for f in t.fields:
fields.add((name: f.name, typeName: f.typeName, isPtr: false))
emitMap(fields)
proc emitReqFields(p: FFIProcMeta): string =
var fields: seq[tuple[name: string, typeName: string, isPtr: bool]] = @[]
for ep in p.extraParams:
fields.add((name: ep.name, typeName: ep.typeName, isPtr: ep.ridesAsPtr()))
emitMap(fields)
proc responseRule(p: FFIProcMeta): string =
## CDDL shape of the success payload; error payloads are raw UTF-8, absent here.
case p.kind
of FFIKind.CTOR:
# Ctor returns the FFI context address as a CBOR decimal string.
"tstr"
of FFIKind.DTOR:
# Dtor payload is a CBOR null sentinel.
"nil"
of FFIKind.FFI, FFIKind.STATIC:
if p.returnRidesAsPtr():
"uint"
else:
nimTypeToCddl(p.returnTypeName)
proc generateCddlSchema*(
procs: seq[FFIProcMeta],
types: seq[FFITypeMeta],
libName: string,
nimSrcRelPath: string,
): string =
var L: seq[string] = @[]
L.add("; CDDL schema for `" & libName & "` — auto-generated from " & nimSrcRelPath)
L.add("; Wire format: CBOR (RFC 8949). Errors return raw UTF-8 (not CBOR) and")
L.add("; are intentionally absent from this schema.")
L.add("")
if types.len > 0:
L.add(
"; ─── User-declared FFI types ──────────────────────────────────────"
)
for t in types:
let rule =
if t.isEnum():
emitEnumAlternatives(t)
else:
emitObjectFields(t)
L.add(t.name & " = " & rule)
L.add("")
# Per-proc request envelopes (one CBOR blob per request).
let nonDtor = block:
var r: seq[FFIProcMeta] = @[]
for p in procs:
if p.kind != FFIKind.DTOR:
r.add(p)
r
if nonDtor.len > 0:
L.add(
"; ─── Request envelopes (one CBOR blob per request) ────────────────"
)
for p in nonDtor:
L.add(reqStructName(p) & " = " & emitReqFields(p))
L.add("")
# Per-proc request/response rules.
L.add(
"; ─── Procs ─────────────────────────────────────────────────────────"
)
for p in procs:
let kindTag =
case p.kind
of FFIKind.CTOR: "ctor"
of FFIKind.DTOR: "dtor"
of FFIKind.FFI: "ffi"
of FFIKind.STATIC: "ffiStatic"
L.add("; " & p.procName & " (" & kindTag & ")")
L.add(renderDocComment(p.doc, "", "; "))
if p.kind != FFIKind.DTOR:
L.add(p.procName & "-request = " & reqStructName(p))
L.add(p.procName & "-response = " & responseRule(p))
L.add("")
return L.join("\n")
proc generateCddlBindings*(
procs: seq[FFIProcMeta],
types: seq[FFITypeMeta],
libName: string,
outputDir: string,
nimSrcRelPath: string,
) =
createDir(outputDir)
writeFile(
outputDir / (libName & ".cddl"),
generateCddlSchema(procs, types, libName, nimSrcRelPath),
)
+69
View File
@@ -0,0 +1,69 @@
## Literal rendering for `{.ffiConst.}` values, shared by the C/C++/Rust generators.
## The registry stores Nim's `$value`; each backend re-quotes it for its syntax.
import std/strutils
import ./types_ir
func cByteEscape(ch: char): string =
## 3-digit octal: C caps an octal escape at 3 digits, so a following digit
## can't be swallowed into it the way it can with `\x`.
return "\\" & toOct(ord(ch), 3)
func rustByteEscape(ch: char): string =
return "\\x" & toHex(ord(ch), 2)
func escapeLit(
s: string, byteEscape: proc(ch: char): string {.noSideEffect, nimcall.}
): string =
var escaped = ""
for ch in s:
case ch
of '"':
escaped.add("\\\"")
of '\\':
escaped.add("\\\\")
of '\n':
escaped.add("\\n")
of '\r':
escaped.add("\\r")
of '\t':
escaped.add("\\t")
else:
if ch < ' ' or ch == '\x7F':
escaped.add(byteEscape(ch))
else:
escaped.add(ch)
return escaped
func cEscapeStringLit*(s: string): string =
return escapeLit(s, cByteEscape)
func rustEscapeStringLit*(s: string): string =
return escapeLit(s, rustByteEscape)
func cConstValue*(t: FFIType, value: string): string =
## C/C++ literal. Every emission site is a typed declaration, so the declared
## type already fixes the width; only the two cases the type can't rescue get
## a suffix — `ULL` because a decimal above `INT64_MAX` fits no signed type,
## and `f` because a bare `1.5` is a double and narrowing it warns.
case t.kind
of ftStr:
return "\"" & cEscapeStringLit(value) & "\""
of ftScalar:
case t.scalar
of skU64:
return value & "ULL"
of skF32:
return value & "f"
else:
return value
else:
return value
func rustConstValue*(t: FFIType, value: string): string =
## Rust literal; the declared type annotation carries the width, so no suffix.
case t.kind
of ftStr:
return "\"" & rustEscapeStringLit(value) & "\""
else:
return value
+574
View File
@@ -0,0 +1,574 @@
## C++ binding generator: header-only binding + CMakeLists, CBOR over the wire.
import std/[os, strutils]
import ./meta, ./string_helpers, ./c_cpp_common, ./types_ir, ./consts
## Fixed 64-bit wire type for any Nim `ptr T` / `pointer`.
const CppPtrType* = "uint64_t"
## Trailing param of every call that can't inherit a ctx's `timeout_`.
const CppTimeoutParam = "std::chrono::milliseconds timeout = std::chrono::seconds{30}"
const
HeaderPreludeTpl = staticRead("templates/cpp/header_prelude.hpp.tpl")
ResultTpl = staticRead("templates/cpp/result.hpp.tpl")
CborHelpersTpl = staticRead("templates/cpp/cbor_helpers.hpp.tpl")
SyncCallHelperTpl = staticRead("templates/cpp/sync_call_helper.hpp.tpl")
ContextRuleOf5Tpl = staticRead("templates/cpp/context_rule_of_5.hpp.tpl")
CMakeListsTpl = staticRead("templates/cpp/CMakeLists.txt.tpl")
func cppScalar(s: ScalarKind): string =
case s
of skBool: "bool"
of skI8: "int8_t"
of skI16: "int16_t"
of skI32: "int32_t"
of skI64: "int64_t"
of skU8: "uint8_t"
of skU16: "uint16_t"
of skU32: "uint32_t"
of skU64: "uint64_t"
of skF32: "float"
of skF64: "double"
func cppSeq(elem: string): string =
"std::vector<" & elem & ">"
func cppOpt(elem: string): string =
"std::optional<" & elem & ">"
const cppMap = NativeTypeMap(
scalar: cppScalar,
str: "std::string",
bytes: "std::vector<uint8_t>",
ptrType: CppPtrType,
seqOf: cppSeq,
optOf: cppOpt,
) ## structName omitted: C++ uses the user type name verbatim
proc nimTypeToCpp*(typeName: string): string =
renderNative(cppMap, parseFFIType(typeName))
proc emitEnumCborCodec(lines: var seq[string], t: FFITypeMeta) =
## Appends the `enum class` plus its TinyCBOR codec pair. The wire form is the
## CBOR text `$value` yields on the Nim side, so the codec maps name ↔ value.
lines.add("enum class $1 {" % [t.name])
for v in t.enumValues:
lines.add(" $1 = $2," % [v.name, $v.ord])
lines.add("};")
lines.add("inline CborError encode_cbor(CborEncoder& e, const $1& v) {" % [t.name])
lines.add(" switch (v) {")
for v in t.enumValues:
lines.add(
" case $1::$2: return cbor_encode_text_stringz(&e, \"$3\");" %
[t.name, v.name, v.wire]
)
lines.add(" }")
lines.add(" return CborErrorImproperValue;")
lines.add("}")
lines.add("inline CborError decode_cbor(CborValue& it, $1& v) {" % [t.name])
lines.add(" std::string name;")
lines.add(" CborError err = decode_cbor(it, name);")
lines.add(" if (err) return err;")
for v in t.enumValues:
lines.add(
" if (name == \"$1\") { v = $2::$3; return CborNoError; }" %
[v.wire, t.name, v.name]
)
lines.add(" return CborErrorImproperValue;")
lines.add("}")
lines.add("")
proc emitStructCborCodec(
lines: var seq[string], structName: string, fields: seq[(string, string)]
) =
## Appends per-struct TinyCBOR encode_cbor + decode_cbor functions emitting a
## text-keyed CBOR map. The C++ type in `fields` is unused (overloads dispatch).
let n = fields.len
if n == 0:
lines.add(
"inline CborError encode_cbor(CborEncoder& e, const $1&) {" % [structName]
)
else:
lines.add(
"inline CborError encode_cbor(CborEncoder& e, const $1& v) {" % [structName]
)
lines.add(" CborEncoder m;")
lines.add(" CborError err = cbor_encoder_create_map(&e, &m, $1);" % [$n])
lines.add(" if (err) return err;")
for (name, _) in fields:
lines.add(
" err = cbor_encode_text_stringz(&m, \"$1\"); if (err) return err;" % [name]
)
lines.add(
" err = encode_cbor(m, v.$1); if (err) return err;" % [name]
)
lines.add(" return cbor_encoder_close_container(&e, &m);")
lines.add("}")
if n == 0:
lines.add("inline CborError decode_cbor(CborValue& it, $1&) {" % [structName])
lines.add(" if (!cbor_value_is_map(&it)) return CborErrorImproperValue;")
lines.add(" return cbor_value_advance(&it);")
lines.add("}")
return
lines.add("inline CborError decode_cbor(CborValue& it, $1& v) {" % [structName])
lines.add(" if (!cbor_value_is_map(&it)) return CborErrorImproperValue;")
lines.add(" CborValue field;")
lines.add(" CborError err;")
for (name, _) in fields:
lines.add(
" err = cbor_value_map_find_value(&it, \"$1\", &field); if (err) return err;" %
[name]
)
lines.add(" if (!cbor_value_is_valid(&field)) return CborErrorImproperValue;")
lines.add(" err = decode_cbor(field, v.$1); if (err) return err;" % [name])
lines.add(" return cbor_value_advance(&it);")
lines.add("}")
proc cppBracedInit(structName: string, fieldNames: seq[string]): string =
## C++ braced-init for a Req struct, e.g. `TimerEchoReq{message, count}`.
return structName & "{" & fieldNames.join(", ") & "}"
proc emitEventDispatcher(
lines: var seq[string], ctxTypeName, libName: string, events: seq[FFIEventMeta]
) =
## Emits the public per-event `addOn<X>Listener` / `removeEventListener` API.
## Callables are owned by `listeners_` (unique_ptr keyed by id); the raw
## pointer is the dylib's `user_data`, stable until removal.
if events.len == 0:
return
lines.add(
" // ── Event listener API ──────────────────────────────────"
)
lines.add(" struct ListenerHandle { std::uint64_t id = 0; };")
lines.add("")
for ev in events:
let methodName =
"addOn" & capitalizeFirstLetter(ev.nimProcName).substr(2) & "Listener"
lines.add(renderMemberDocComment(ev.doc))
lines.add(
" ListenerHandle $1(std::function<void(const $2&)> handler) {" %
[methodName, ev.payloadTypeName]
)
lines.add(
" auto owned = std::make_unique<TypedListener<$1>>(std::move(handler));" %
[ev.payloadTypeName]
)
lines.add(" auto* raw = owned.get();")
lines.add(" const auto id = $1_add_event_listener(" % [libName])
lines.add(
" ptr_, \"$1\", &$2::typedTrampoline<$3>, raw);" %
[ev.wireName, ctxTypeName, ev.payloadTypeName]
)
lines.add(" if (id == 0) return ListenerHandle{0};")
lines.add(" listeners_.emplace(id, std::move(owned));")
lines.add(" return ListenerHandle{id};")
lines.add(" }")
lines.add("")
lines.add(" bool removeEventListener(ListenerHandle handle) {")
lines.add(" if (handle.id == 0) return false;")
lines.add(
" const auto rc = $1_remove_event_listener(ptr_, handle.id);" % [libName]
)
lines.add(" listeners_.erase(handle.id);")
lines.add(" return rc == 0;")
lines.add(" }")
lines.add("")
proc emitEventTrampoline(lines: var seq[string], events: seq[FFIEventMeta]) =
## Private listener machinery for `emitEventDispatcher`: polymorphic
## `ListenerBase`, `TypedListener<T>` and the `typedTrampoline<T>` decoder.
if events.len == 0:
return
lines.add(" struct ListenerBase {")
lines.add(" virtual ~ListenerBase() = default;")
lines.add(" };")
lines.add("")
lines.add(" template <class T>")
lines.add(" struct TypedListener : ListenerBase {")
lines.add(" std::function<void(const T&)> fn;")
lines.add(
" explicit TypedListener(std::function<void(const T&)> f) : fn(std::move(f)) {}"
)
lines.add(" };")
lines.add("")
lines.add(" template <class T>")
lines.add(
" static void typedTrampoline(int ret, const char* msg, std::size_t len, void* ud) {"
)
lines.add(" if (!ud || ret != 0 || !msg || len == 0) return;")
lines.add(" auto* listener = static_cast<TypedListener<T>*>(ud);")
lines.add(" if (!listener->fn) return;")
lines.add(" CborParser parser; CborValue it;")
lines.add(
" if (cbor_parser_init(reinterpret_cast<const std::uint8_t*>(msg), len, 0, &parser, &it) != CborNoError) return;"
)
lines.add(" if (!cbor_value_is_map(&it)) return;")
lines.add(" CborValue payloadField;")
lines.add(
" if (cbor_value_map_find_value(&it, \"payload\", &payloadField) != CborNoError) return;"
)
lines.add(" T payload{};")
lines.add(" if (decode_cbor(payloadField, payload) != CborNoError) return;")
lines.add(" listener->fn(payload);")
lines.add(" }")
lines.add("")
proc generateCppHeader*(
procs: seq[FFIProcMeta],
types: seq[FFITypeMeta],
libName: string,
events: seq[FFIEventMeta] = @[],
consts: seq[FFIConstMeta] = @[],
): string =
var lines: seq[string] = @[]
lines.add(HeaderPreludeTpl)
if events.len > 0:
lines.add("#include <unordered_map>")
lines.add(ResultTpl)
# Generic CBOR overloads must precede the non-template struct codecs that call them (parse-time name lookup).
lines.add(CborHelpersTpl)
if consts.len > 0:
lines.add("// ============================================================")
lines.add("// Generated constants")
lines.add("// ============================================================")
lines.add("")
for c in consts:
let t = parseFFIType(c.typeName)
# A string const is a `const char*`, not std::string: constexpr can't own a heap value.
let cppType =
if t.kind == ftStr:
"const char*"
else:
nimTypeToCpp(c.typeName)
lines.add(
"constexpr $1 $2 = $3;" %
[cppType, identToUpperSnake(c.name), cConstValue(t, c.value)]
)
lines.add("")
# Enums first: a struct codec that takes one must see its overload already declared.
var structTypes: seq[FFITypeMeta] = @[]
for t in types:
if t.isEnum():
emitEnumCborCodec(lines, t)
else:
structTypes.add(t)
if structTypes.len > 0:
lines.add("// ============================================================")
lines.add("// User-declared FFI types")
lines.add("// ============================================================")
lines.add("")
for t in structTypes:
lines.add("struct $1 {" % [t.name])
for f in t.fields:
lines.add(" $1 $2;" % [nimTypeToCpp(f.typeName), f.name])
lines.add("};")
var fields: seq[(string, string)] = @[]
for f in t.fields:
fields.add((f.name, nimTypeToCpp(f.typeName)))
emitStructCborCodec(lines, t.name, fields)
lines.add("")
lines.add("// ============================================================")
lines.add("// Per-proc request envelopes (CBOR encoded on the wire)")
lines.add("// ============================================================")
lines.add("")
for p in procs:
if p.kind == FFIKind.DTOR:
continue
let reqName = reqStructName(p)
lines.add("struct $1 {" % [reqName])
for ep in p.extraParams:
let cppType =
if ep.ridesAsPtr():
CppPtrType
else:
nimTypeToCpp(ep.typeName)
lines.add(" $1 $2;" % [cppType, ep.name])
lines.add("};")
var fields: seq[(string, string)] = @[]
for ep in p.extraParams:
let cppType =
if ep.ridesAsPtr():
CppPtrType
else:
nimTypeToCpp(ep.typeName)
fields.add((ep.name, cppType))
emitStructCborCodec(lines, reqName, fields)
lines.add("")
lines.add("// ============================================================")
lines.add("// C FFI declarations")
lines.add("// ============================================================")
lines.add("")
lines.add("extern \"C\" {")
lines.add(
"typedef void (*FFICallback)(int ret, const char* msg, size_t len, void* user_data);"
)
lines.add("")
for p in procs:
lines.add(renderBlockDocComment(p.doc))
case p.kind
of FFIKind.FFI:
lines.add(
"int $1(void* ctx, FFICallback callback, void* user_data, const uint8_t* req_cbor, size_t req_cbor_len);" %
[p.procName]
)
of FFIKind.STATIC:
lines.add(
"int $1(FFICallback callback, void* user_data, const uint8_t* req_cbor, size_t req_cbor_len);" %
[p.procName]
)
of FFIKind.CTOR:
lines.add(
"void* $1(const uint8_t* req_cbor, size_t req_cbor_len, FFICallback callback, void* user_data);" %
[p.procName]
)
of FFIKind.DTOR:
lines.add("int $1(void* ctx);" % [p.procName])
# Listener-registration ABI is always exported.
lines.add(
"uint64_t $1_add_event_listener(void* ctx, const char* event_name, FFICallback callback, void* user_data);" %
[libName]
)
lines.add(
"int $1_remove_event_listener(void* ctx, uint64_t listener_id);" % [libName]
)
lines.add("} // extern \"C\"")
lines.add("")
lines.add(SyncCallHelperTpl)
let classified = classifyProcs(procs)
let ctors = classified.ctors
let ctxTypeName = libTypeName(ctors, libName) & "Ctx"
lines.add("// ============================================================")
lines.add("// High-level C++ context class")
lines.add("// ============================================================")
lines.add("")
lines.add("class $1 {" % [ctxTypeName])
lines.add("public:")
for ctor in ctors:
let reqName = reqStructName(ctor)
var ctorParams: seq[string] = @[]
var epNames: seq[string] = @[]
for ep in ctor.extraParams:
let cppType =
if ep.ridesAsPtr():
CppPtrType
else:
nimTypeToCpp(ep.typeName)
ctorParams.add("const $1& $2" % [cppType, ep.name])
epNames.add(ep.name)
let ctorParamsWithTimeout =
if ctorParams.len > 0:
ctorParams.join(", ") & ", " & CppTimeoutParam
else:
CppTimeoutParam
let reqInit = cppBracedInit(reqName, epNames)
# `create` yields the ctx via the callback's CBOR address (sync void* return discarded), owned as a unique_ptr since the class forbids copy/move.
let createRet = "Result<std::unique_ptr<$1>>" % [ctxTypeName]
lines.add(renderMemberDocComment(ctor.doc))
lines.add(" static $1 create($2) {" % [createRet, ctorParamsWithTimeout])
lines.add(" const auto ffi_req_ = $1;" % [reqInit])
lines.add(" auto ffi_enc_ = encodeCborFFI(ffi_req_);")
lines.add(
" if (ffi_enc_.isErr()) return $1::err(ffi_enc_.error());" % [createRet]
)
lines.add(" const auto& ffi_req_bytes_ = ffi_enc_.value();")
lines.add(" auto ffi_raw_ = ffi_call_([&](FFICallback cb, void* ud) {")
lines.add(
" (void)$1(ffi_req_bytes_.data(), ffi_req_bytes_.size(), cb, ud);" %
[ctor.procName]
)
lines.add(" return 0;")
lines.add(" }, timeout);")
lines.add(
" if (ffi_raw_.isErr()) return $1::err(ffi_raw_.error());" % [createRet]
)
lines.add(" auto ffi_addr_ = decodeCborFFI<std::string>(ffi_raw_.value());")
lines.add(
" if (ffi_addr_.isErr()) return $1::err(ffi_addr_.error());" % [createRet]
)
lines.add(" const auto& addr_str = ffi_addr_.value();")
# from_chars (not stoull) so a bad payload is an err() Result, not a throw.
lines.add(" std::uint64_t addr = 0;")
lines.add(" const char* addr_begin = addr_str.data();")
lines.add(" const char* addr_end = addr_begin + addr_str.size();")
lines.add(" const auto fc_ = std::from_chars(addr_begin, addr_end, addr);")
lines.add(" if (fc_.ec != std::errc() || fc_.ptr != addr_end) {")
lines.add(
" return $1::err(\"FFI create returned non-numeric address: \" + addr_str);" %
[createRet]
)
lines.add(" }")
# `new` (not make_unique) so the ctor can stay private.
lines.add(
" return $1::ok(std::unique_ptr<$2>(new $2(reinterpret_cast<void*>(static_cast<uintptr_t>(addr)), timeout)));" %
[createRet, ctxTypeName]
)
lines.add(" }")
lines.add("")
let captureList =
if epNames.len > 0:
epNames.join(", ") & ", timeout"
else:
"timeout"
let callList =
if epNames.len > 0:
epNames.join(", ") & ", timeout"
else:
"timeout"
lines.add(renderMemberDocComment(ctor.doc))
lines.add(
" static std::future<Result<std::unique_ptr<$1>>> createAsync($2) {" %
[ctxTypeName, ctorParamsWithTimeout]
)
lines.add(
" return std::async(std::launch::async, [$1]() { return create($2); });" %
[captureList, callList]
)
lines.add(" }")
lines.add("")
lines.add(
ContextRuleOf5Tpl.multiReplace(("{{CTX}}", ctxTypeName), ("{{LIB}}", libName))
)
emitEventDispatcher(lines, ctxTypeName, libName, events)
# A static has no ctx to inherit `timeout_` from, so it takes its own `timeout`.
for m in classified.replyProcs():
let isStatic = m.isStatic()
let methodName = stripLibPrefix(m.procName, libName)
let retCppType =
if m.returnRidesAsPtr():
CppPtrType
else:
nimTypeToCpp(m.returnTypeName)
let reqName = reqStructName(m)
var methParams: seq[string] = @[]
var methParamNames: seq[string] = @[]
for ep in m.extraParams:
let cppType =
if ep.ridesAsPtr():
CppPtrType
else:
nimTypeToCpp(ep.typeName)
methParams.add("const $1& $2" % [cppType, ep.name])
methParamNames.add(ep.name)
let methParamNamesStr = methParamNames.join(", ")
let methParamsStr =
if not isStatic:
methParams.join(", ")
elif methParams.len > 0:
methParams.join(", ") & ", " & CppTimeoutParam
else:
CppTimeoutParam
let reqInit = cppBracedInit(reqName, methParamNames)
let methRet = "Result<$1>" % [retCppType]
lines.add(renderMemberDocComment(m.doc))
let decl = if isStatic: " static $1 $2($3) {" else: " $1 $2($3) const {"
lines.add(decl % [methRet, methodName, methParamsStr])
lines.add(" const auto ffi_req_ = $1;" % [reqInit])
lines.add(" auto ffi_enc_ = encodeCborFFI(ffi_req_);")
lines.add(
" if (ffi_enc_.isErr()) return $1::err(ffi_enc_.error());" % [methRet]
)
lines.add(" const auto& ffi_req_bytes_ = ffi_enc_.value();")
lines.add(" auto ffi_raw_ = ffi_call_([&](FFICallback cb, void* ud) {")
let ctxArg = if isStatic: "" else: "ptr_, "
lines.add(
" return $1($2cb, ud, ffi_req_bytes_.data(), ffi_req_bytes_.size());" %
[m.procName, ctxArg]
)
lines.add(" }, $1);" % [if isStatic: "timeout" else: "timeout_"])
lines.add(
" if (ffi_raw_.isErr()) return $1::err(ffi_raw_.error());" % [methRet]
)
lines.add(" return decodeCborFFI<$1>(ffi_raw_.value());" % [retCppType])
lines.add(" }")
lines.add("")
# A method calls `this->methodName(...)` so a same-named param can't shadow
# the call target; a static has no `this` and forwards its own `timeout`.
let staticArgs =
if methParamNames.len > 0:
methParamNamesStr & ", timeout"
else:
"timeout"
let asyncArgs = if isStatic: staticArgs else: methParamNamesStr
let asyncCapture =
if isStatic:
staticArgs
elif methParamNamesStr.len > 0:
"this, " & methParamNamesStr
else:
"this"
let asyncDecl =
if isStatic:
" static std::future<$1> $2Async($3) {"
else:
" std::future<$1> $2Async($3) const {"
lines.add(renderMemberDocComment(m.doc))
lines.add(asyncDecl % [methRet, methodName, methParamsStr])
lines.add(
" return std::async(std::launch::async, [$1]() { return $2$3($4); });" %
[asyncCapture, (if isStatic: "" else: "this->"), methodName, asyncArgs]
)
lines.add(" }")
lines.add("")
lines.add("private:")
# Listener machinery must precede the `listeners_` member (its value type must be complete at declaration).
emitEventTrampoline(lines, events)
lines.add(" void* ptr_;")
lines.add(" std::chrono::milliseconds timeout_;")
if events.len > 0:
lines.add(
" std::unordered_map<std::uint64_t, std::unique_ptr<ListenerBase>> listeners_;"
)
lines.add(
" explicit $1(void* p, std::chrono::milliseconds t) : ptr_(p), timeout_(t) {}" %
[ctxTypeName]
)
lines.add("};")
lines.add("")
return lines.join("\n")
proc generateCppCMakeLists*(libName: string, nimSrcRelPath: string): string =
let src = nimSrcRelPath.replace("\\", "/")
return CMakeListsTpl.multiReplace(("{{LIB}}", libName), ("{{SRC}}", src))
proc generateCppBindings*(
procs: seq[FFIProcMeta],
types: seq[FFITypeMeta],
libName: string,
outputDir: string,
nimSrcRelPath: string,
events: seq[FFIEventMeta] = @[],
consts: seq[FFIConstMeta] = @[],
) =
createDir(outputDir)
writeFile(
outputDir / (libName & ".hpp"),
generateCppHeader(procs, types, libName, events, consts),
)
writeFile(outputDir / "CMakeLists.txt", generateCppCMakeLists(libName, nimSrcRelPath))
+204
View File
@@ -0,0 +1,204 @@
## Compile-time metadata types for FFI binding generation, populated by the
## {.ffiCtor.}/{.ffi.} macros and consumed by codegen.
import std/[strutils, options]
type
ABIFormat* {.pure.} = enum
## FFI payload wire format. `Cbor` is wired end-to-end; `C` has a type codec
## but no proc-dispatch path yet.
Cbor = "cbor"
C = "c"
FFIParamMeta* = object
name*: string
typeName*: string
isPtr*: bool
isHandle*: bool # {.ffiHandle.} type, wire form uint64
FFIKind* {.pure.} = enum
FFI
CTOR
DTOR
STATIC ## `{.ffiStatic.}`: context-independent, its wrapper takes no `ctx`
FFIProcMeta* = object
procName*: string
libName*: string
kind*: FFIKind
libTypeName*: string
doc*: string
extraParams*: seq[FFIParamMeta] # all params except the lib param
returnTypeName*: string
returnIsPtr*: bool
returnIsHandle*: bool
abiFormat*: ABIFormat
scalarFastPath*: bool
## `abi = c` proc with an all-scalar signature: uses the CBOR-free fast
## path, and binds only in the `abi = c` C header (see `bindableProcs`).
FFIFieldMeta* = object
name*: string
typeName*: string
FFIEnumValueMeta* = object
## One `{.ffi.}` enum value. `wire` is what `$value` yields — the symbol name,
## or the associated string if the enum declares one — which is exactly what
## cbor_serialization puts on the wire.
name*: string
wire*: string
ord*: int
FFITypeMeta* = object
name*: string
fields*: seq[FFIFieldMeta]
abiFormat*: ABIFormat
enumValues*: seq[FFIEnumValueMeta] ## non-empty iff the type is an enum
FFIConstMeta* = object
## A `{.ffiConst.}` value. `value` is the compile-time-evaluated result of
## `$theConst`, re-rendered as a literal by each backend.
name*: string
typeName*: string
value*: string
FFIEventMeta* = object
## Library-initiated event from `{.ffiEvent: "wire_name".}`; `wireName` is
## the verbatim CBOR `eventType` the foreign side dispatches on.
wireName*: string
nimProcName*: string
libName*: string
payloadTypeName*: string
abiFormat*: ABIFormat
doc*: string
var ffiProcRegistry* {.compileTime.}: seq[FFIProcMeta]
var ffiTypeRegistry* {.compileTime.}: seq[FFITypeMeta]
var ffiEventRegistry* {.compileTime.}: seq[FFIEventMeta]
var ffiConstRegistry* {.compileTime.}: seq[FFIConstMeta]
var currentLibName* {.compileTime.}: string
# Set by `declareLibrary`; the FFI annotations require it.
var libraryDeclared* {.compileTime.}: bool = false
# Set by `genBindings()`. Annotations expanded after it register too late to be emitted, so the macros check this and fail loudly instead of dropping silently.
var genBindingsEmitted* {.compileTime.}: bool = false
# Library-wide default ABI, inherited by each annotation unless it overrides.
var currentDefaultABIFormat* {.compileTime.}: ABIFormat = ABIFormat.Cbor
proc abiCodegenImplemented*(fmt: ABIFormat): bool =
## Whether `fmt` has a working proc-dispatch path (both Cbor and C do).
fmt in {ABIFormat.Cbor, ABIFormat.C}
proc overrideKey*(override: string): string =
## Lowercased key of a `key = value` pragma override, e.g. `"abi = c"` → `"abi"`.
override.split('=')[0].strip().toLowerAscii()
proc parseABIFormatName*(name: string): tuple[ok: bool, fmt: ABIFormat] =
## Bare format name ("c"/"cbor", case-insensitive) → ABIFormat; else ok=false.
case name.strip().toLowerAscii()
of "cbor":
(true, ABIFormat.Cbor)
of "c":
(true, ABIFormat.C)
else:
(false, ABIFormat.Cbor)
proc parseAbiSpec*(override: string): tuple[ok: bool, fmt: ABIFormat, err: string] =
## Parse an `"abi = <format>"` override; on bad grammar returns ok=false + err.
let parts = override.split('=')
if parts.len != 2:
return (
false,
ABIFormat.Cbor,
"invalid ABI override: '" & override & "'; expected `abi = c` or `abi = cbor`",
)
if parts[0].strip().toLowerAscii() != "abi":
return (
false,
ABIFormat.Cbor,
"invalid ABI override: '" & override & "'; expected `abi = c` or `abi = cbor`",
)
let (ok, fmt) = parseABIFormatName(parts[1])
if not ok:
return (
false,
ABIFormat.Cbor,
"unknown ABI format: '" & parts[1].strip() & "'; valid values are `c` and `cbor`",
)
(true, fmt, "")
# Lib type name (set by declareLibrary) so handle-receiver procs resolve the pool.
var currentLibType* {.compileTime.}: string
# Names of types marked `{.ffiHandle.}` (wire form uint64).
var ffiHandleTypeNames* {.compileTime.}: seq[string]
proc isFFIHandleTypeName*(name: string): bool {.compileTime.} =
name in ffiHandleTypeNames
func isEnum*(t: FFITypeMeta): bool =
return t.enumValues.len > 0
# Names of `{.ffi.}` enum types; the `abi = c` wire path has to reject them.
var ffiEnumTypeNames* {.compileTime.}: seq[string]
proc isFFIEnumTypeName*(name: string): bool {.compileTime.} =
name in ffiEnumTypeNames
func isStatic*(p: FFIProcMeta): bool =
p.kind == FFIKind.STATIC
type ClassifiedProcs* = object
ctors*: seq[FFIProcMeta]
methods*: seq[FFIProcMeta]
statics*: seq[FFIProcMeta]
dtor*: Option[FFIProcMeta]
func classifyProcs*(procs: seq[FFIProcMeta]): ClassifiedProcs =
## Splits the registry into constructors, methods, statics and the first destructor.
var c: ClassifiedProcs
for p in procs:
case p.kind
of FFIKind.CTOR:
c.ctors.add(p)
of FFIKind.FFI:
c.methods.add(p)
of FFIKind.STATIC:
c.statics.add(p)
of FFIKind.DTOR:
if c.dtor.isNone():
c.dtor = some(p)
c
func dtorProcName*(c: ClassifiedProcs): string =
## The destructor's proc name, or "" when the library has no destructor.
if c.dtor.isSome():
c.dtor.get().procName
else:
""
func replyProcs*(c: ClassifiedProcs): seq[FFIProcMeta] =
## Procs that reply with a decoded value: methods and statics.
c.methods & c.statics
proc ridesAsPtr*(ep: FFIParamMeta): bool =
## True if the param crosses the wire as an opaque uint64 (raw ptr or handle).
ep.isPtr or ep.isHandle
proc returnRidesAsPtr*(p: FFIProcMeta): bool =
## True if the return crosses the wire as an opaque uint64 (raw ptr or handle).
p.returnIsPtr or p.returnIsHandle
# Target language(s), override with -d:targetLang=cpp; comma-separated list allowed.
const targetLang* {.strdefine.} = "rust"
# Output dir override (-d:ffiOutputDir); empty derives `<lang>_bindings/` by src.
const ffiOutputDir* {.strdefine.} = ""
# Nim src path override relative to outputDir (-d:ffiSrcPath); empty derives it.
const ffiSrcPath* {.strdefine.} = ""
# When true, targets without scalar codegen silently omit scalar-only `abi = c` procs rather than failing the build. Off by default so the drop is loud; see genBindings().
const ffiAllowScalarSkip* {.booldefine.} = false
+825
View File
@@ -0,0 +1,825 @@
## Rust binding generator: emits a complete Rust crate using CBOR (ciborium).
import std/[os, strutils]
import ./meta, ./string_helpers, ./types_ir, ./consts
## Wire-format Rust type for any Nim `ptr T`/`pointer`; fixed 64-bit for a
## host-independent CBOR payload size (mirrors CppPtrType).
const RustPtrType* = "u64"
func rustScalar(s: ScalarKind): string =
case s
of skBool: "bool"
of skI8: "i8"
of skI16: "i16"
of skI32: "i32"
of skI64: "i64"
of skU8: "u8"
of skU16: "u16"
of skU32: "u32"
of skU64: "u64"
of skF32: "f32"
of skF64: "f64"
func rustSeq(elem: string): string =
"Vec<" & elem & ">"
func rustOpt(elem: string): string =
"Option<" & elem & ">"
const rustMap = NativeTypeMap(
scalar: rustScalar,
str: "String",
# serde encodes a plain Vec<u8> as a CBOR integer array, and Nim rejects that
# array. ByteBuf gives the CBOR byte string that Nim decodes.
bytes: "serde_bytes::ByteBuf",
ptrType: RustPtrType,
seqOf: rustSeq,
optOf: rustOpt,
structName: capitalizeFirstLetter,
)
proc nimTypeToRust*(typeName: string): string =
## Maps Nim type names to Rust type names, including generics.
renderNative(rustMap, parseFFIType(typeName))
proc deriveLibName*(procs: seq[FFIProcMeta]): string =
## Common prefix before the first `_` in proc names, e.g. "timer_create" → "timer".
if currentLibName.len > 0:
return currentLibName
if procs.len == 0:
return "unknown"
let first = procs[0].procName
let parts = first.split('_')
if parts.len > 0:
return parts[0]
return "unknown"
proc stripLibPrefix*(procName: string, libName: string): string =
## Strips the library prefix, e.g. ("timer_echo", "timer") → "echo".
let prefix = libName & "_"
if procName.startsWith(prefix):
return procName[prefix.len .. ^1]
return procName
proc reqStructName(p: FFIProcMeta): string =
## Mirrors the Nim macro: <CamelCase(procName)>Req or CtorReq for ctors.
let camel = snakeToPascalCase(p.procName)
if p.kind == FFIKind.CTOR:
camel & "CtorReq"
else:
camel & "Req"
func typeUsesBytes(typeName: string): bool =
## True if `typeName` is a `seq[byte]` at any depth of Seq or Option.
var t = parseFFIType(typeName)
while t.kind in {ftSeq, ftOpt}:
t = t.elem
t.kind == ftBytes
func needsSerdeBytes*(types: seq[FFITypeMeta], procs: seq[FFIProcMeta]): bool =
## True if a field, a parameter or a return type maps to `serde_bytes::ByteBuf`.
## `types` holds every struct. Thus a scan of the fields also finds the bytes
## in a nested struct.
for t in types:
for f in t.fields:
if typeUsesBytes(f.typeName):
return true
for p in procs:
for ep in p.extraParams:
if typeUsesBytes(ep.typeName):
return true
if p.returnTypeName.len > 0 and typeUsesBytes(p.returnTypeName):
return true
false
proc generateCargoToml*(libName: string, needsBytes = false): string =
# flume: callback channel (recv_timeout + recv_async), default-features off. tokio: only the async timeout.
# Add serde_bytes only when a `seq[byte]` goes on the wire as a CBOR byte string.
let serdeBytesDep = if needsBytes: "\nserde_bytes = \"0.11\"" else: ""
return
"""[package]
name = "$1"
version = "0.1.0"
edition = "2021"
[dependencies]
serde = { version = "1", features = ["derive"] }$2
ciborium = "0.2"
flume = { version = "0.11", default-features = false, features = ["async"] }
tokio = { version = "1", features = ["sync", "time"] }
[dev-dependencies]
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time"] }
""" %
[libName, serdeBytesDep]
proc generateBuildRs*(libName: string, nimSrcRelPath: string): string =
## Generates build.rs that compiles the Nim library; nimSrcRelPath is relative
## to the crate directory.
let escapedSrc = nimSrcRelPath.replace("\\", "\\\\")
return
"""use std::path::PathBuf;
use std::process::Command;
fn main() {
let manifest = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap());
let nim_src = manifest.join("$1");
let nim_src = nim_src.canonicalize().unwrap_or(manifest.join("$1"));
// Walk up to find the nim-ffi repo root (directory containing nim_src's library)
// The repo root is where nim c should be run from (contains config.nims).
// We assume nim_src lives somewhere under repo_root.
// Derive repo_root as the ancestor that contains the .nimble file or config.nims.
let mut repo_root = nim_src.clone();
loop {
repo_root = match repo_root.parent() {
Some(p) => p.to_path_buf(),
None => break,
};
if repo_root.join("config.nims").exists() || repo_root.join("ffi.nimble").exists() {
break;
}
}
#[cfg(target_os = "macos")]
let lib_ext = "dylib";
#[cfg(target_os = "linux")]
let lib_ext = "so";
let out_lib = repo_root.join(format!("lib$2.{lib_ext}"));
let mut cmd = Command::new("nim");
cmd.arg("c")
.arg("--mm:orc")
.arg("-d:chronicles_log_level=WARN")
.arg("--app:lib")
.arg("--noMain")
.arg(format!("--nimMainPrefix:lib$2"))
.arg(format!("-o:{}", out_lib.display()));
cmd.arg(&nim_src).current_dir(&repo_root);
let status = cmd.status().expect("failed to run nim compiler");
assert!(status.success(), "Nim compilation failed");
println!("cargo:rustc-link-search={}", repo_root.display());
println!("cargo:rustc-link-lib=$2");
println!("cargo:rerun-if-changed={}", nim_src.display());
}
""" %
[escapedSrc, libName]
proc generateLibRs*(): string =
return """mod ffi;
mod types;
mod api;
pub use types::*;
pub use api::*;
"""
proc generateFFIRs*(procs: seq[FFIProcMeta]): string =
## Generates ffi.rs with extern "C" declarations; each proc takes one CBOR
## buffer (ptr+len) as its request payload.
var lines: seq[string] = @[]
lines.add("use std::os::raw::{c_char, c_int, c_void};")
lines.add("")
lines.add("pub type FFICallback = unsafe extern \"C\" fn(")
lines.add(" ret: c_int,")
lines.add(" msg: *const c_char,")
lines.add(" len: usize,")
lines.add(" user_data: *mut c_void,")
lines.add(");")
lines.add("")
var libNames: seq[string] = @[]
for p in procs:
if p.libName notin libNames:
libNames.add(p.libName)
var linkLibName = ""
if libNames.len > 0 and libNames[0].len > 0:
linkLibName = libNames[0]
else:
if procs.len > 0:
let parts = procs[0].procName.split('_')
if parts.len > 0:
linkLibName = parts[0]
lines.add("#[link(name = \"$1\")]" % [linkLibName])
lines.add("extern \"C\" {")
for p in procs:
var params: seq[string] = @[]
lines.add(renderMemberDocComment(p.doc))
case p.kind
of FFIKind.FFI, FFIKind.STATIC:
if not p.isStatic():
params.add("ctx: *mut c_void")
params.add("callback: FFICallback")
params.add("user_data: *mut c_void")
params.add("req_cbor: *const u8")
params.add("req_cbor_len: usize")
lines.add(" pub fn $1($2) -> c_int;" % [p.procName, params.join(", ")])
of FFIKind.CTOR:
# Ctor: no ctx; returns the freshly-allocated handle.
params.add("req_cbor: *const u8")
params.add("req_cbor_len: usize")
params.add("callback: FFICallback")
params.add("user_data: *mut c_void")
lines.add(" pub fn $1($2) -> *mut c_void;" % [p.procName, params.join(", ")])
of FFIKind.DTOR:
params.add("ctx: *mut c_void")
lines.add(" pub fn $1($2) -> c_int;" % [p.procName, params.join(", ")])
# Listener-registration ABI, always present in the dylib.
lines.add(
" pub fn $1_add_event_listener(ctx: *mut c_void, event_name: *const c_char, callback: FFICallback, user_data: *mut c_void) -> u64;" %
[linkLibName]
)
lines.add(
" pub fn $1_remove_event_listener(ctx: *mut c_void, listener_id: u64) -> c_int;" %
[linkLibName]
)
lines.add("}")
return lines.join("\n") & "\n"
func rustConstType(typeName: string): string =
## `&str` rather than `String`: a `pub const` can't own a heap value. The
## 'static lifetime is implied, and spelling it out trips clippy.
let t = parseFFIType(typeName)
if t.kind == ftStr:
return "&str"
return renderNative(rustMap, t)
proc generateTypesRs*(
types: seq[FFITypeMeta], procs: seq[FFIProcMeta], consts: seq[FFIConstMeta] = @[]
): string =
## Generates types.rs: Rust structs for user FFI types and each per-proc Req.
var lines: seq[string] = @[]
lines.add("use serde::{Deserialize, Serialize};")
lines.add("")
for c in consts:
let t = parseFFIType(c.typeName)
lines.add(
"pub const $1: $2 = $3;" % [
identToUpperSnake(c.name), rustConstType(c.typeName), rustConstValue(t, c.value)
]
)
if consts.len > 0:
lines.add("")
for t in types:
if not t.isEnum():
continue
lines.add("#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]")
lines.add("pub enum $1 {" % [t.name])
for v in t.enumValues:
let variant = capitalizeFirstLetter(v.name)
# serde carries the same text form Nim's cbor_serialization writes.
if variant != v.wire:
lines.add(" #[serde(rename = \"$1\")]" % [v.wire])
lines.add(" $1," % [variant])
lines.add("}")
lines.add("")
for t in types:
if t.isEnum():
continue
lines.add("#[derive(Debug, Clone, Serialize, Deserialize)]")
lines.add("pub struct $1 {" % [t.name])
for f in t.fields:
let snakeName = camelToSnakeCase(f.name)
let rustType = nimTypeToRust(f.typeName)
# serde rename when camelCase differs from snake_case.
if snakeName != f.name:
lines.add(" #[serde(rename = \"$1\")]" % [f.name])
lines.add(" pub $1: $2," % [snakeName, rustType])
lines.add("}")
lines.add("")
# Per-proc Req structs: the unit of CBOR encoding sent across the boundary.
for p in procs:
if p.kind == FFIKind.DTOR:
continue
let reqName = reqStructName(p)
lines.add("#[derive(Debug, Clone, Serialize, Deserialize)]")
if p.extraParams.len == 0:
lines.add("pub struct $1 {}" % [reqName])
else:
lines.add("pub struct $1 {" % [reqName])
for ep in p.extraParams:
let snake = camelToSnakeCase(ep.name)
let rustType =
if ep.ridesAsPtr():
RustPtrType
else:
nimTypeToRust(ep.typeName)
if snake != ep.name:
lines.add(" #[serde(rename = \"$1\")]" % [ep.name])
lines.add(" pub $1: $2," % [snake, rustType])
lines.add("}")
lines.add("")
return lines.join("\n")
proc generateApiRs*(
procs: seq[FFIProcMeta], libName: string, events: seq[FFIEventMeta] = @[]
): string =
## Generates api.rs with a blocking and a tokio-async high-level API.
## Requests/responses are CBOR (ciborium); errors are raw UTF-8 strings.
var lines: seq[string] = @[]
let classified = classifyProcs(procs)
let ctors = classified.ctors
let dtorProcName = classified.dtorProcName
var libTypeName = ""
if ctors.len > 0:
libTypeName = ctors[0].libTypeName
else:
libTypeName = capitalizeFirstLetter(libName)
let ctxTypeName = libTypeName & "Ctx"
lines.add("use std::os::raw::{c_char, c_int, c_void};")
lines.add("use std::slice;")
lines.add("use std::time::Duration;")
lines.add("use serde::de::DeserializeOwned;")
lines.add("use serde::Serialize;")
lines.add("use super::ffi;")
lines.add("use super::types::*;")
lines.add("")
lines.add("fn encode_cbor<T: Serialize>(value: &T) -> Result<Vec<u8>, String> {")
lines.add(" let mut buf = Vec::new();")
lines.add(
" ciborium::ser::into_writer(value, &mut buf).map_err(|e| e.to_string())?;"
)
lines.add(" Ok(buf)")
lines.add("}")
lines.add("")
lines.add("fn decode_cbor<T: DeserializeOwned>(bytes: &[u8]) -> Result<T, String> {")
lines.add(" ciborium::de::from_reader(bytes).map_err(|e| e.to_string())")
lines.add("}")
lines.add("")
# FFI trampoline: user_data owns a Box<flume::Sender>; a late callback sends into a closed receiver, which is harmless.
lines.add("type FFIResult = Result<Vec<u8>, String>;")
lines.add("type FFISender = flume::Sender<FFIResult>;")
lines.add("")
lines.add("// Reconstruct the (ret, msg, len) tuple delivered by the C callback")
lines.add(
"// into a Result<Vec<u8>, String>: payload on success, UTF-8 message on error."
)
lines.add(
"// `from_utf8_lossy` accepts non-UTF-8 error bytes by inserting U+FFFD; the"
)
lines.add(
"// alternative would be to dispatch a separate Err for invalid UTF-8, but the"
)
lines.add("// codegen contract is that Nim handlers emit `string` error payloads, so")
lines.add("// invalid UTF-8 here would be a Nim-side bug.")
lines.add(
"unsafe fn ffi_payload(ret: c_int, msg: *const c_char, len: usize) -> FFIResult {"
)
lines.add(" let bytes = if msg.is_null() || len == 0 {")
lines.add(" Vec::new()")
lines.add(" } else {")
lines.add(" slice::from_raw_parts(msg as *const u8, len).to_vec()")
lines.add(" };")
lines.add(" if ret == NIMFFI_RET_OK { Ok(bytes) }")
lines.add(" else { Err(String::from_utf8_lossy(&bytes).into_owned()) }")
lines.add("}")
lines.add("")
lines.add("// nim-ffi result-callback status codes (mirror ffi/ffi_types.nim).")
lines.add("const NIMFFI_RET_OK: c_int = 0;")
lines.add("const NIMFFI_RET_MISSING_CALLBACK: c_int = 2;")
lines.add("const NIMFFI_RET_STALE_WARN: c_int = 3;")
lines.add("")
lines.add("unsafe extern \"C\" fn on_result(")
lines.add(" ret: c_int,")
lines.add(" msg: *const c_char,")
lines.add(" len: usize,")
lines.add(" user_data: *mut c_void,")
lines.add(") {")
lines.add(
" // NIMFFI_RET_STALE_WARN (3) is a non-terminal progress ping: the request"
)
lines.add(
" // is still running. This wrapper only delivers the final result, so ignore"
)
lines.add(
" // it WITHOUT reclaiming the box — a terminal callback still owns the Sender."
)
lines.add(" if ret == NIMFFI_RET_STALE_WARN { return; }")
lines.add("")
lines.add(" // Take ownership of the boxed Sender — dropping it at end of scope")
lines.add(" // releases the only outstanding handle.")
lines.add(" let tx = Box::from_raw(user_data as *mut FFISender);")
lines.add("")
lines.add(
" // `tx.send` returns Err only if the awaiting future was dropped (and with it"
)
lines.add(
" // the Receiver): e.g. tokio::time::timeout elapsed, a tokio::select! branch"
)
lines.add(
" // lost the race, or the future was dropped before being awaited. This cannot"
)
lines.add(
" // happen with the current rust_client demo but may occur in arbitrary"
)
lines.add(" // downstream consumers, so we discard the Err safely.")
lines.add(
" // Given that this is invoked from a Nim thread, we can't propagate the error by panicking or"
)
lines.add(
" // returning a Result. Furthermore, an API dev may intentionally set a timeout in the await,"
)
lines.add(
" // in which case is also fine to discard the send error in this case because the API user will"
)
lines.add(" // handle the timeout expiry in their own code.")
lines.add(
" // The important part is to ensure that the callback doesn't panic or block indefinitely if the"
)
lines.add(" // receiver is gone.")
lines.add(" let _ = tx.send(ffi_payload(ret, msg, len));")
lines.add("}")
lines.add("")
lines.add("fn ffi_call_sync<F>(timeout: Duration, f: F) -> FFIResult")
lines.add("where")
lines.add(" F: FnOnce(ffi::FFICallback, *mut c_void) -> c_int,")
lines.add("{")
lines.add(" let (tx, rx) = flume::bounded::<FFIResult>(1);")
lines.add(" let raw = Box::into_raw(Box::new(tx)) as *mut c_void;")
lines.add(" let ret = f(on_result, raw);")
lines.add(" if ret == NIMFFI_RET_MISSING_CALLBACK {")
lines.add(" // Callback will never fire; reclaim the box to avoid a leak.")
lines.add(" drop(unsafe { Box::from_raw(raw as *mut FFISender) });")
lines.add(" return Err(\"RET_MISSING_CALLBACK (internal error)\".into());")
lines.add(" }")
lines.add(" match rx.recv_timeout(timeout) {")
lines.add(" Ok(payload) => payload,")
lines.add(" Err(flume::RecvTimeoutError::Timeout) =>")
lines.add(" Err(format!(\"timed out after {:?}\", timeout)),")
lines.add(" Err(flume::RecvTimeoutError::Disconnected) =>")
lines.add(
" Err(\"callback channel disconnected before delivery\".into()),"
)
lines.add(" }")
lines.add("}")
lines.add("")
lines.add("async fn ffi_call_async<F>(timeout: Duration, f: F) -> FFIResult")
lines.add("where")
lines.add(" F: FnOnce(ffi::FFICallback, *mut c_void) -> c_int,")
lines.add("{")
lines.add(" let (tx, rx) = flume::bounded::<FFIResult>(1);")
lines.add(" let raw = Box::into_raw(Box::new(tx)) as *mut c_void;")
lines.add(" let ret = f(on_result, raw);")
lines.add(" if ret == NIMFFI_RET_MISSING_CALLBACK {")
lines.add(" drop(unsafe { Box::from_raw(raw as *mut FFISender) });")
lines.add(" return Err(\"RET_MISSING_CALLBACK (internal error)\".into());")
lines.add(" }")
lines.add(" match tokio::time::timeout(timeout, rx.recv_async()).await {")
lines.add(" Ok(Ok(payload)) => payload,")
lines.add(
" Ok(Err(_)) => Err(\"callback channel disconnected before delivery\".into()),"
)
lines.add(" Err(_) => Err(format!(\"timed out after {:?}\", timeout)),")
lines.add(" }")
lines.add("}")
lines.add("")
# Per-listener handler boxes + extern "C" trampolines: the Box is kept alive in `listeners`, its raw pointer is the per-event `user_data`.
if events.len > 0:
for ev in events:
let handlerStruct = capitalizeFirstLetter(ev.nimProcName) & "Handler"
let trampolineName = camelToSnakeCase(ev.nimProcName) & "_trampoline"
lines.add("struct $1 {" % [handlerStruct])
lines.add(" f: Box<dyn Fn(&$1) + Send + Sync>," % [ev.payloadTypeName])
lines.add("}")
lines.add("")
lines.add("unsafe extern \"C\" fn $1(" % [trampolineName])
lines.add(" ret: c_int, msg: *const c_char, len: usize, ud: *mut c_void,")
lines.add(") {")
lines.add(" if ud.is_null() || ret != 0 || msg.is_null() || len == 0 {")
lines.add(" return;")
lines.add(" }")
lines.add(" let h = &*(ud as *const $1);" % [handlerStruct])
lines.add(" let bytes = slice::from_raw_parts(msg as *const u8, len);")
lines.add(" #[derive(serde::Deserialize)]")
lines.add(" struct Envelope { payload: $1 }" % [ev.payloadTypeName])
lines.add(
" if let Ok(env) = ciborium::de::from_reader::<Envelope, _>(bytes) {"
)
lines.add(" (h.f)(&env.payload);")
lines.add(" }")
lines.add("}")
lines.add("")
# Public handle returned by every add_…_listener call.
lines.add("#[derive(Debug, Clone, Copy)]")
lines.add("pub struct ListenerHandle { pub id: u64 }")
lines.add("")
lines.add("/// High-level context for `$1`." % [libTypeName])
lines.add("pub struct $1 {" % [ctxTypeName])
lines.add(" ptr: *mut c_void,")
lines.add(" timeout: Duration,")
if events.len > 0:
# Keeps each handler box alive while its listener id is live on the Nim side.
lines.add(
" listeners: std::sync::Mutex<std::collections::HashMap<u64, Box<dyn std::any::Any + Send>>>,"
)
lines.add("}")
lines.add("")
# SAFETY block applies to both impls below.
lines.add(
"// SAFETY: The `ptr` field points to an FFIContext owned by the Nim runtime."
)
lines.add("// Every call through the generated FFI proc goes through")
lines.add(
"// `sendRequestToFFIThread` on the Nim side, which only enqueues the request"
)
lines.add("// onto a mutex-guarded MPSC queue (sound from any number of threads) and")
lines.add(
"// wakes the single FFI thread that dispatches every handler. The context is"
)
lines.add(
"// thus never mutated non-atomically from the caller's thread. The Nim-side"
)
lines.add("// reentrancy guard (`onFFIThread` threadvar) prevents handlers from")
lines.add("// re-entering the dispatcher. These invariants make it sound to mark the")
lines.add("// wrapper as Send + Sync.")
lines.add("unsafe impl Send for $1 {}" % [ctxTypeName])
lines.add("unsafe impl Sync for $1 {}" % [ctxTypeName])
lines.add("")
# Drop tears down the Nim runtime when the ctx goes out of scope; without it, forgetting the ctx leaks the entire runtime (FFI thread, watchdog, chronos).
if dtorProcName.len > 0:
lines.add("impl Drop for $1 {" % [ctxTypeName])
lines.add(" fn drop(&mut self) {")
lines.add(" if !self.ptr.is_null() {")
lines.add(" unsafe { ffi::$1(self.ptr); }" % [dtorProcName])
lines.add(" self.ptr = std::ptr::null_mut();")
lines.add(" }")
# `listeners` drops after this body; the dylib has joined its threads by then, so no callback is mid-flight against the raw pointers we handed it.
lines.add(" }")
lines.add("}")
lines.add("")
lines.add("impl $1 {" % [ctxTypeName])
for ctor in ctors:
let reqName = reqStructName(ctor)
var paramsList: seq[string] = @[]
var fieldInits: seq[string] = @[]
for ep in ctor.extraParams:
let snake = camelToSnakeCase(ep.name)
let rustType =
if ep.ridesAsPtr():
RustPtrType
else:
nimTypeToRust(ep.typeName)
paramsList.add("$1: $2" % [snake, rustType])
fieldInits.add(snake)
# `create` and `new_async` take an explicit `timeout: Duration` that flows into `self.timeout` so subsequent method calls inherit it.
let ctorParamsStr =
if paramsList.len > 0:
paramsList.join(", ") & ", timeout: Duration"
else:
"timeout: Duration"
let reqLit =
if fieldInits.len > 0:
reqName & " { " & fieldInits.join(", ") & " }"
else:
reqName & " {}"
lines.add(renderMemberDocComment(ctor.doc))
lines.add(" pub fn create($1) -> Result<Self, String> {" % [ctorParamsStr])
lines.add(" let req = $1;" % [reqLit])
lines.add(" let req_bytes = encode_cbor(&req)?;")
# Ctor also fires the callback carrying the payload, so discard the synchronous *mut c_void and yield RET_OK to wait on the callback.
lines.add(" let raw_bytes = ffi_call_sync(timeout, |cb, ud| unsafe {")
lines.add(
" let _ = ffi::$1(req_bytes.as_ptr(), req_bytes.len(), cb, ud);" %
[ctor.procName]
)
lines.add(" 0")
lines.add(" })?;")
# Ctor success payload is a CBOR text string holding the ctx address.
lines.add(" let addr_str: String = decode_cbor(&raw_bytes)?;")
lines.add(
" let addr: usize = addr_str.parse().map_err(|e: std::num::ParseIntError| e.to_string())?;"
)
if events.len > 0:
lines.add(
" Ok(Self { ptr: addr as *mut c_void, timeout, listeners: std::sync::Mutex::new(std::collections::HashMap::new()) })"
)
else:
lines.add(" Ok(Self { ptr: addr as *mut c_void, timeout })")
lines.add(" }")
lines.add("")
lines.add(renderMemberDocComment(ctor.doc))
lines.add(
" pub async fn new_async($1) -> Result<Self, String> {" % [ctorParamsStr]
)
lines.add(" let req = $1;" % [reqLit])
lines.add(" let req_bytes = encode_cbor(&req)?;")
# See `create`: discard the ctor's synchronous return; the callback delivers the ctx address.
lines.add(" let raw_bytes = ffi_call_async(timeout, move |cb, ud| unsafe {")
lines.add(
" let _ = ffi::$1(req_bytes.as_ptr(), req_bytes.len(), cb, ud);" %
[ctor.procName]
)
lines.add(" 0")
lines.add(" }).await?;")
lines.add(" let addr_str: String = decode_cbor(&raw_bytes)?;")
lines.add(
" let addr: usize = addr_str.parse().map_err(|e: std::num::ParseIntError| e.to_string())?;"
)
if events.len > 0:
lines.add(
" Ok(Self { ptr: addr as *mut c_void, timeout, listeners: std::sync::Mutex::new(std::collections::HashMap::new()) })"
)
else:
lines.add(" Ok(Self { ptr: addr as *mut c_void, timeout })")
lines.add(" }")
lines.add("")
if events.len > 0:
# Shared by every public `add_*_listener`: caller owns the concrete-typed box, erased to `dyn Any + Send` only on hand-off.
lines.add(" fn add_listener_inner(")
lines.add(" &self,")
lines.add(" event_name: *const c_char,")
lines.add(" callback: ffi::FFICallback,")
lines.add(" raw: *mut c_void,")
lines.add(" owned: Box<dyn std::any::Any + Send>,")
lines.add(" ) -> ListenerHandle {")
lines.add(" let id = unsafe {")
lines.add(
" ffi::$1_add_event_listener(self.ptr, event_name, callback, raw)" %
[libName]
)
lines.add(" };")
lines.add(" if id != 0 {")
lines.add(" self.listeners.lock().unwrap().insert(id, owned);")
lines.add(" }")
lines.add(" ListenerHandle { id }")
lines.add(" }")
lines.add("")
for ev in events:
let methodName = "add_" & camelToSnakeCase(ev.nimProcName) & "_listener"
let handlerStruct = capitalizeFirstLetter(ev.nimProcName) & "Handler"
let trampolineName = camelToSnakeCase(ev.nimProcName) & "_trampoline"
lines.add(renderMemberDocComment(ev.doc))
lines.add(
" /// Register a typed listener for `$1`. The returned handle can be" %
[ev.wireName]
)
lines.add(" /// passed to `remove_event_listener` to unregister.")
lines.add(" pub fn $1<F>(&self, handler: F) -> ListenerHandle" % [methodName])
lines.add(" where F: Fn(&$1) + Send + Sync + 'static," % [ev.payloadTypeName])
lines.add(" {")
lines.add(
" let owned: Box<$1> = Box::new($1 { f: Box::new(handler) });" %
[handlerStruct]
)
lines.add(
" let raw = &*owned as *const $1 as *mut c_void;" % [handlerStruct]
)
lines.add(
" self.add_listener_inner(b\"$1\\0\".as_ptr() as *const c_char, $2, raw, owned)" %
[ev.wireName, trampolineName]
)
lines.add(" }")
lines.add("")
# Remove by handle; drops the Box after the C ABI confirms unregistration.
lines.add(" /// Remove a previously-registered listener by handle. Returns true")
lines.add(" /// if the listener existed and was removed; false otherwise.")
lines.add(
" pub fn remove_event_listener(&self, handle: ListenerHandle) -> bool {"
)
lines.add(" if handle.id == 0 { return false; }")
lines.add(" let rc = unsafe {")
lines.add(
" ffi::$1_remove_event_listener(self.ptr, handle.id)" % [libName]
)
lines.add(" };")
lines.add(" self.listeners.lock().unwrap().remove(&handle.id);")
lines.add(" rc == 0")
lines.add(" }")
lines.add("")
# A static is an associated fn: no `&self` to read `timeout` from, so it takes one.
for m in classified.replyProcs():
let isStatic = m.isStatic()
let methodName = stripLibPrefix(m.procName, libName)
let retRustType = nimTypeToRust(m.returnTypeName)
let reqName = reqStructName(m)
var paramsList: seq[string] = @[]
var fieldInits: seq[string] = @[]
for ep in m.extraParams:
let snake = camelToSnakeCase(ep.name)
let rustType =
if ep.ridesAsPtr():
RustPtrType
else:
nimTypeToRust(ep.typeName)
paramsList.add("$1: $2" % [snake, rustType])
fieldInits.add(snake)
if isStatic:
paramsList.add("timeout: Duration")
let paramsStr =
if isStatic:
paramsList.join(", ")
elif paramsList.len > 0:
"&self, " & paramsList.join(", ")
else:
"&self"
let reqLit =
if fieldInits.len > 0:
reqName & " { " & fieldInits.join(", ") & " }"
else:
reqName & " {}"
let retTypeForApi = if m.returnRidesAsPtr(): RustPtrType else: retRustType
let timeoutExpr = if isStatic: "timeout" else: "self.timeout"
let ctxArg = if isStatic: "" else: "self.ptr, "
lines.add(renderMemberDocComment(m.doc))
lines.add(
" pub fn $1($2) -> Result<$3, String> {" %
[methodName, paramsStr, retTypeForApi]
)
lines.add(" let req = $1;" % [reqLit])
lines.add(" let req_bytes = encode_cbor(&req)?;")
lines.add(
" let raw_bytes = ffi_call_sync($1, |cb, ud| unsafe {" % [timeoutExpr]
)
lines.add(
" ffi::$1($2cb, ud, req_bytes.as_ptr(), req_bytes.len())" %
[m.procName, ctxArg]
)
lines.add(" })?;")
lines.add(" decode_cbor::<$1>(&raw_bytes)" % [retTypeForApi])
lines.add(" }")
lines.add("")
# async method: ptr cast to usize (Copy + Send) keeps the move closure and returned future Send for multi-threaded tokio runtimes.
lines.add(renderMemberDocComment(m.doc))
lines.add(
" pub async fn $1_async($2) -> Result<$3, String> {" %
[methodName, paramsStr, retTypeForApi]
)
lines.add(" let req = $1;" % [reqLit])
lines.add(" let req_bytes = encode_cbor(&req)?;")
if not isStatic:
lines.add(" let ptr = self.ptr as usize;")
lines.add(
" let raw_bytes = ffi_call_async($1, move |cb, ud| unsafe {" % [
timeoutExpr
]
)
lines.add(
" ffi::$1($2cb, ud, req_bytes.as_ptr(), req_bytes.len())" %
[m.procName, if isStatic: "" else: "ptr as *mut c_void, "]
)
lines.add(" }).await?;")
lines.add(" decode_cbor::<$1>(&raw_bytes)" % [retTypeForApi])
lines.add(" }")
lines.add("")
lines.add("}")
return lines.join("\n") & "\n"
proc generateRustCrate*(
procs: seq[FFIProcMeta],
types: seq[FFITypeMeta],
libName: string,
outputDir: string,
nimSrcRelPath: string,
events: seq[FFIEventMeta] = @[],
consts: seq[FFIConstMeta] = @[],
) =
## Generates a complete Rust crate in outputDir.
createDir(outputDir)
createDir(outputDir / "src")
writeFile(
outputDir / "Cargo.toml", generateCargoToml(libName, needsSerdeBytes(types, procs))
)
writeFile(outputDir / "build.rs", generateBuildRs(libName, nimSrcRelPath))
writeFile(outputDir / "src" / "lib.rs", generateLibRs())
writeFile(outputDir / "src" / "ffi.rs", generateFFIRs(procs))
writeFile(outputDir / "src" / "types.rs", generateTypesRs(types, procs, consts))
writeFile(outputDir / "src" / "api.rs", generateApiRs(procs, libName, events))
@@ -0,0 +1,96 @@
## Unicode-aware identifier casing and doc-comment rendering, shared by codegen
## and the FFI macro.
import std/[strutils, unicode]
func docLines(doc: string): seq[string] =
## `doc` split into lines, trailing blank ones dropped.
if doc.strip().len == 0:
return @[]
var lines = doc.splitLines()
while lines.len > 0 and lines[^1].strip().len == 0:
lines.setLen(lines.len - 1)
return lines
func renderDocComment*(doc, indent, prefix: string): seq[string] =
## `doc` as one `prefix`-led line comment per source line, at `indent`.
var rendered: seq[string] = @[]
for line in docLines(doc):
# A trailing `\` would splice the next generated line into a `//` comment.
rendered.add(
indent & (prefix & line).strip(leading = false, chars = Whitespace + {'\\'})
)
return rendered
func renderMemberDocComment*(doc: string): seq[string] =
## `///` at the indent C++ class members and Rust `impl` items sit at.
return doc.renderDocComment(" ", "/// ")
func escapeBlockComment(line: string): string =
## `*/` would close the comment early and splice the rest in as code.
return line.replace("*/", "* /")
func renderBlockDocComment*(doc: string, indent = ""): seq[string] =
## `doc` as a `/** ... */` block at `indent`; one-liners stay on one line.
let lines = docLines(doc)
if lines.len == 0:
return @[]
if lines.len == 1:
return @[indent & "/** " & escapeBlockComment(lines[0].strip()) & " */"]
var rendered = @[indent & "/**"]
for line in lines:
rendered.add((indent & " * " & escapeBlockComment(line)).strip(leading = false))
rendered.add(indent & " */")
return rendered
proc toLower*(s: string): string =
## Unicode-aware lowercase for an entire string.
var buf = ""
for r in runes(s):
buf.add($r.toLower())
return buf
proc camelToSnakeCase*(s: string): string =
## camelCase → snake_case, e.g. "delayMs" → "delay_ms".
var snake = ""
var first = true
for r in runes(s):
if r.isUpper() and not first:
snake.add('_')
snake.add($r.toLower())
first = false
return snake
func capitalizeFirstLetter*(s: string): string =
## Returns `s` with its first rune uppercased, rest unchanged.
if s.len == 0:
return s
var runesSeq = toRunes(s)
runesSeq[0] = runesSeq[0].toUpper()
return $runesSeq
func identToUpperSnake*(s: string): string =
## Nim identifier → UPPER_SNAKE, keeping acronym runs intact: "maxPeers" and
## "MAX_PEERS" both give "MAX_PEERS", "httpTTL" gives "HTTP_TTL".
var upper = ""
let rs = toRunes(s)
for i, r in rs:
if r == Rune('_'):
if upper.len > 0 and upper[^1] != '_':
upper.add('_')
continue
let startsWord =
i > 0 and r.isUpper() and
(not rs[i - 1].isUpper() or (i + 1 < rs.len and rs[i + 1].isLower()))
if startsWord and upper.len > 0 and upper[^1] != '_':
upper.add('_')
upper.add($r.toUpper())
return upper
proc snakeToPascalCase*(s: string): string =
## snake_case → PascalCase, e.g. "hello_world" → "HelloWorld".
let parts = s.split('_')
var pascal = ""
for p in parts:
pascal.add capitalizeFirstLetter(p)
return pascal
@@ -0,0 +1,47 @@
cmake_minimum_required(VERSION 3.14)
project({{LIB}}_c_bindings C)
set(CMAKE_C_STANDARD 11)
set(CMAKE_C_STANDARD_REQUIRED ON)
# ── Locate the repository root (contains ffi.nimble) ─────────────────────────
set(_search_dir "${CMAKE_CURRENT_SOURCE_DIR}")
set(REPO_ROOT "")
foreach(_i RANGE 10)
if(EXISTS "${_search_dir}/ffi.nimble")
set(REPO_ROOT "${_search_dir}")
break()
endif()
get_filename_component(_search_dir "${_search_dir}" DIRECTORY)
endforeach()
if("${REPO_ROOT}" STREQUAL "")
message(FATAL_ERROR "Cannot find repo root (no ffi.nimble in any ancestor)")
endif()
# Build the Nim dylib + vendored TinyCBOR (shared with the C++ backend).
set(NIM_FFI_LIB {{LIB}})
set(NIM_FFI_SRC {{SRC}})
include("${REPO_ROOT}/ffi/codegen/templates/nim_ffi_lib.cmake")
find_package(Threads REQUIRED)
add_library({{LIB}}_headers INTERFACE)
target_include_directories({{LIB}}_headers INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}")
target_link_libraries({{LIB}}_headers INTERFACE {{LIB}} tinycbor Threads::Threads)
# The generated header is async (no blocking helper), but consumer code that
# waits on a result callback typically uses nanosleep / pthreads, which need a
# POSIX feature level that strict `-std=c11` hides. Define it for consumers.
target_compile_definitions({{LIB}}_headers INTERFACE _POSIX_C_SOURCE=200809L)
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/main.c")
add_executable({{LIB}}_example main.c)
target_link_libraries({{LIB}}_example PRIVATE {{LIB}}_headers)
add_dependencies({{LIB}}_example {{LIB}}_nim_lib)
if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
add_custom_command(TARGET {{LIB}}_example POST_BUILD
COMMAND "${CMAKE_COMMAND}" -E copy_if_different
"${{{LIB}}_RUNTIME_LIB}"
"$<TARGET_FILE_DIR:{{LIB}}_example>"
COMMENT "Staging {{LIB}}.dll next to {{LIB}}_example.exe")
endif()
endif()
@@ -0,0 +1,72 @@
cmake_minimum_required(VERSION 3.14)
project({{LIB}}_c_abi_bindings C)
set(CMAKE_C_STANDARD 11)
set(CMAKE_C_STANDARD_REQUIRED ON)
# The CBOR-free `abi = c` binding links no TinyCBOR — the generated header
# structs are the ABI. Only the Nim dylib is built.
set(_search_dir "${CMAKE_CURRENT_SOURCE_DIR}")
set(REPO_ROOT "")
foreach(_i RANGE 10)
if(EXISTS "${_search_dir}/ffi.nimble")
set(REPO_ROOT "${_search_dir}")
break()
endif()
get_filename_component(_search_dir "${_search_dir}" DIRECTORY)
endforeach()
if("${REPO_ROOT}" STREQUAL "")
message(FATAL_ERROR "Cannot find repo root (no ffi.nimble in any ancestor)")
endif()
# Extra `nim c` arguments (e.g. a `-d:` that flips a shared example source to
# `abi = c`). A library that declares `defaultABIFormat = "c"` needs none.
set(NIM_FFI_EXTRA_ARGS "" CACHE STRING "Extra nim c args when building the dylib")
find_program(NIM_EXECUTABLE nim REQUIRED)
if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
set(NIM_LIB_FILE "${REPO_ROOT}/lib{{LIB}}.dylib")
elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
set(NIM_LIB_FILE "${REPO_ROOT}/{{LIB}}.dll")
else()
set(NIM_LIB_FILE "${REPO_ROOT}/lib{{LIB}}.so")
endif()
get_filename_component(NIM_SRC "${CMAKE_CURRENT_SOURCE_DIR}/{{SRC}}" ABSOLUTE)
add_custom_command(
OUTPUT "${NIM_LIB_FILE}"
COMMAND "${NIM_EXECUTABLE}" c
--mm:orc
-d:chronicles_log_level=WARN
--app:lib
--noMain
"--nimMainPrefix:lib{{LIB}}"
${NIM_FFI_EXTRA_ARGS}
"-o:${NIM_LIB_FILE}"
"${NIM_SRC}"
WORKING_DIRECTORY "${REPO_ROOT}"
DEPENDS "${NIM_SRC}"
COMMENT "Compiling Nim library lib{{LIB}} (abi = c)"
VERBATIM
)
add_custom_target({{LIB}}_nim_lib ALL DEPENDS "${NIM_LIB_FILE}")
add_library({{LIB}} SHARED IMPORTED GLOBAL)
set_target_properties({{LIB}} PROPERTIES IMPORTED_LOCATION "${NIM_LIB_FILE}")
add_dependencies({{LIB}} {{LIB}}_nim_lib)
find_package(Threads REQUIRED)
add_library({{LIB}}_headers INTERFACE)
target_include_directories({{LIB}}_headers INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}")
target_link_libraries({{LIB}}_headers INTERFACE {{LIB}} Threads::Threads)
target_compile_definitions({{LIB}}_headers INTERFACE _POSIX_C_SOURCE=200809L)
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/main.c")
add_executable({{LIB}}_example main.c)
target_link_libraries({{LIB}}_example PRIVATE {{LIB}}_headers)
add_dependencies({{LIB}}_example {{LIB}}_nim_lib)
endif()
@@ -0,0 +1,352 @@
#ifndef NIM_FFI_CBOR_HELPERS_H_INCLUDED
#define NIM_FFI_CBOR_HELPERS_H_INCLUDED
/* Leaf CBOR codecs (scalars, text strings, byte strings) plus the buffer
* drivers. The per-struct / per-container codecs in the library header call
* into these by name (C has no overloading, so each leaf gets a distinct
* nimffi_enc_* / nimffi_dec_* symbol). Guarded so two nim-ffi headers can
* share a translation unit. */
#include "nim_ffi_prelude.h"
#ifdef __cplusplus
extern "C" {
#endif
/* Result delivery callback exported by the Nim dylib: `ret` is 0 on success
* (then `msg`/`len` carry the CBOR response) or non-zero on failure (then
* `msg`/`len` carry the error text, which is NOT NUL-terminated). */
typedef void (*FFICallback)(int ret, const char* msg, size_t len, void* user_data);
/* Return / callback status codes. NIMFFI_RET_OK (0) is success; any non-zero
* value handed to a result callback's `err_code` (or returned by a submit call)
* is a failure. NIMFFI_RET_MISSING_CALLBACK is a special case from the Nim
* dispatcher: the callback will never fire, so the request path must report the
* failure itself.
*
* NIMFFI_RET_STALE_WARN is the one NON-terminal code: nim-ffi delivers it every
* ~5s while a handler is still running (with `msg`/`len` carrying the elapsed
* milliseconds as decimal text), then still ends with a terminal RET_OK/RET_ERR.
* A caller that only wants the final answer must ignore it, not treat it as an
* error. */
#define NIMFFI_RET_OK 0
#define NIMFFI_RET_ERROR 1
#define NIMFFI_RET_MISSING_CALLBACK 2
#define NIMFFI_RET_STALE_WARN 3
/* ── leaf encoders ─────────────────────────────────────────────────────── */
static inline CborError nimffi_enc_bool(CborEncoder* e, const bool* v) {
return cbor_encode_boolean(e, *v);
}
static inline CborError nimffi_enc_i64(CborEncoder* e, const int64_t* v) {
return cbor_encode_int(e, *v);
}
static inline CborError nimffi_enc_i32(CborEncoder* e, const int32_t* v) {
return cbor_encode_int(e, (int64_t)*v);
}
static inline CborError nimffi_enc_i16(CborEncoder* e, const int16_t* v) {
return cbor_encode_int(e, (int64_t)*v);
}
static inline CborError nimffi_enc_i8(CborEncoder* e, const int8_t* v) {
return cbor_encode_int(e, (int64_t)*v);
}
static inline CborError nimffi_enc_u64(CborEncoder* e, const uint64_t* v) {
return cbor_encode_uint(e, *v);
}
static inline CborError nimffi_enc_u32(CborEncoder* e, const uint32_t* v) {
return cbor_encode_uint(e, (uint64_t)*v);
}
static inline CborError nimffi_enc_u16(CborEncoder* e, const uint16_t* v) {
return cbor_encode_uint(e, (uint64_t)*v);
}
static inline CborError nimffi_enc_u8(CborEncoder* e, const uint8_t* v) {
return cbor_encode_uint(e, (uint64_t)*v);
}
static inline CborError nimffi_enc_f64(CborEncoder* e, const double* v) {
return cbor_encode_double(e, *v);
}
static inline CborError nimffi_enc_f32(CborEncoder* e, const float* v) {
return cbor_encode_float(e, *v);
}
static inline CborError nimffi_enc_str(CborEncoder* e, const NimFfiStr* v) {
return cbor_encode_text_string(e, v->data ? v->data : "", v->len);
}
static inline CborError nimffi_enc_bytes(CborEncoder* e, const NimFfiBytes* v) {
return cbor_encode_byte_string(e, v->data, v->len);
}
/* ── leaf decoders ─────────────────────────────────────────────────────── */
/* After reading a leaf, the parser must advance past it; both steps
* short-circuit on the same CborError, so they travel together. */
static inline CborError nimffi_advance_if_ok(CborValue* it, CborError err) {
if (err) {
return err;
}
return cbor_value_advance(it);
}
static inline CborError nimffi_dec_bool(CborValue* it, bool* out) {
if (!cbor_value_is_boolean(it)) {
return CborErrorImproperValue;
}
return nimffi_advance_if_ok(it, cbor_value_get_boolean(it, out));
}
static inline CborError nimffi_dec_i64(CborValue* it, int64_t* out) {
if (!cbor_value_is_integer(it)) {
return CborErrorImproperValue;
}
return nimffi_advance_if_ok(it, cbor_value_get_int64_checked(it, out));
}
static inline CborError nimffi_dec_i32(CborValue* it, int32_t* out) {
int64_t tmp = 0;
CborError err = nimffi_dec_i64(it, &tmp);
if (err) {
return err;
}
if (tmp < INT32_MIN || tmp > INT32_MAX) {
return CborErrorDataTooLarge;
}
*out = (int32_t)tmp;
return CborNoError;
}
static inline CborError nimffi_dec_i16(CborValue* it, int16_t* out) {
int64_t tmp = 0;
CborError err = nimffi_dec_i64(it, &tmp);
if (err) {
return err;
}
if (tmp < INT16_MIN || tmp > INT16_MAX) {
return CborErrorDataTooLarge;
}
*out = (int16_t)tmp;
return CborNoError;
}
static inline CborError nimffi_dec_i8(CborValue* it, int8_t* out) {
int64_t tmp = 0;
CborError err = nimffi_dec_i64(it, &tmp);
if (err) {
return err;
}
if (tmp < INT8_MIN || tmp > INT8_MAX) {
return CborErrorDataTooLarge;
}
*out = (int8_t)tmp;
return CborNoError;
}
static inline CborError nimffi_dec_u64(CborValue* it, uint64_t* out) {
if (!cbor_value_is_unsigned_integer(it)) {
return CborErrorImproperValue;
}
return nimffi_advance_if_ok(it, cbor_value_get_uint64(it, out));
}
static inline CborError nimffi_dec_u32(CborValue* it, uint32_t* out) {
uint64_t tmp = 0;
CborError err = nimffi_dec_u64(it, &tmp);
if (err) {
return err;
}
if (tmp > UINT32_MAX) {
return CborErrorDataTooLarge;
}
*out = (uint32_t)tmp;
return CborNoError;
}
static inline CborError nimffi_dec_u16(CborValue* it, uint16_t* out) {
uint64_t tmp = 0;
CborError err = nimffi_dec_u64(it, &tmp);
if (err) {
return err;
}
if (tmp > UINT16_MAX) {
return CborErrorDataTooLarge;
}
*out = (uint16_t)tmp;
return CborNoError;
}
static inline CborError nimffi_dec_u8(CborValue* it, uint8_t* out) {
uint64_t tmp = 0;
CborError err = nimffi_dec_u64(it, &tmp);
if (err) {
return err;
}
if (tmp > UINT8_MAX) {
return CborErrorDataTooLarge;
}
*out = (uint8_t)tmp;
return CborNoError;
}
static inline CborError nimffi_dec_f64(CborValue* it, double* out) {
if (cbor_value_is_double(it)) {
return nimffi_advance_if_ok(it, cbor_value_get_double(it, out));
}
if (cbor_value_is_float(it)) {
float f = 0.0f;
CborError err = cbor_value_get_float(it, &f);
if (err) {
return err;
}
*out = (double)f;
return cbor_value_advance(it);
}
return CborErrorImproperValue;
}
static inline CborError nimffi_dec_f32(CborValue* it, float* out) {
if (cbor_value_is_float(it)) {
return nimffi_advance_if_ok(it, cbor_value_get_float(it, out));
}
if (cbor_value_is_double(it)) {
double d = 0.0;
CborError err = cbor_value_get_double(it, &d);
if (err) {
return err;
}
*out = (float)d;
return cbor_value_advance(it);
}
return CborErrorImproperValue;
}
static inline CborError nimffi_dec_str(CborValue* it, NimFfiStr* out) {
if (!cbor_value_is_text_string(it)) {
return CborErrorImproperValue;
}
size_t len = 0;
CborError err = cbor_value_get_string_length(it, &len);
if (err) {
return err;
}
if (len == SIZE_MAX) { /* len + 1 would wrap to a 0-byte allocation */
return CborErrorDataTooLarge;
}
/* one extra byte so a NUL-free payload is a valid C string */
out->data = (char*)malloc(len + 1);
if (!out->data) {
return CborErrorOutOfMemory;
}
out->len = len;
size_t copied = len;
err = cbor_value_copy_text_string(it, out->data, &copied, NULL);
if (err) {
free(out->data);
out->data = NULL;
out->len = 0;
return err;
}
out->data[len] = '\0';
return cbor_value_advance(it);
}
static inline CborError nimffi_dec_bytes(CborValue* it, NimFfiBytes* out) {
if (!cbor_value_is_byte_string(it)) {
return CborErrorImproperValue;
}
size_t len = 0;
CborError err = cbor_value_get_string_length(it, &len);
if (err) {
return err;
}
out->data = (uint8_t*)malloc(len ? len : 1);
if (!out->data) {
return CborErrorOutOfMemory;
}
out->len = len;
size_t copied = len;
err = cbor_value_copy_byte_string(it, out->data, &copied, NULL);
if (err) {
free(out->data);
out->data = NULL;
out->len = 0;
return err;
}
return cbor_value_advance(it);
}
/* ── buffer drivers ────────────────────────────────────────────────────── */
typedef CborError (*nimffi_enc_fn)(CborEncoder*, const void*);
typedef CborError (*nimffi_dec_fn)(CborValue*, void*);
static inline char* nimffi_dup_cstr(const char* s) {
size_t n = strlen(s) + 1;
char* p = (char*)malloc(n);
if (p) {
memcpy(p, s, n);
}
return p;
}
/* NUL-terminated copy of a length-delimited (not NUL-terminated) byte run,
* for turning the FFICallback's raw error `msg`/`len` into a C string; NULL if
* it can't. */
static inline char* nimffi_dup_cstr_n(const char* s, size_t n) {
if (n == SIZE_MAX) {
return NULL;
}
char* p = (char*)malloc(n + 1);
if (p) {
if (n > 0) {
memcpy(p, s, n);
}
p[n] = '\0';
}
return p;
}
/* Encode `val` with `fn` into a freshly malloc'd buffer, doubling on overflow.
* Returns 0 and sets out/outlen on success; -1 and *err (heap) on failure. */
static inline int nimffi_encode_to_buf(
nimffi_enc_fn fn, const void* val,
uint8_t** out, size_t* outlen, char** err) {
size_t cap = 4096;
uint8_t* buf = (uint8_t*)malloc(cap);
if (!buf) {
if (err) *err = nimffi_dup_cstr("out of memory");
return -1;
}
for (;;) {
CborEncoder enc;
cbor_encoder_init(&enc, buf, cap, 0);
CborError e = fn(&enc, val);
if (e == CborNoError) {
*outlen = cbor_encoder_get_buffer_size(&enc, buf);
*out = buf;
return 0;
}
if (e == CborErrorOutOfMemory) {
size_t extra = cbor_encoder_get_extra_bytes_needed(&enc);
cap += extra > 0 ? extra : cap;
uint8_t* grown = (uint8_t*)realloc(buf, cap);
if (!grown) {
free(buf);
if (err) *err = nimffi_dup_cstr("out of memory");
return -1;
}
buf = grown;
continue;
}
free(buf);
if (err) *err = nimffi_dup_cstr(cbor_error_string(e));
return -1;
}
}
/* Decode a CBOR buffer into `out` with `fn`. Returns 0 on success; -1 and
* *err (heap) on failure. */
static inline int nimffi_decode_from_buf(
nimffi_dec_fn fn, const uint8_t* buf, size_t len,
void* out, char** err) {
CborParser parser;
CborValue it;
CborError e = cbor_parser_init(buf, len, 0, &parser, &it);
if (e != CborNoError) {
if (err) *err = nimffi_dup_cstr(cbor_error_string(e));
return -1;
}
e = fn(&it, out);
if (e != CborNoError) {
if (err) *err = nimffi_dup_cstr(cbor_error_string(e));
return -1;
}
return 0;
}
#ifdef __cplusplus
}
#endif
#endif /* NIM_FFI_CBOR_HELPERS_H_INCLUDED */
@@ -0,0 +1,88 @@
#ifndef NIM_FFI_PRELUDE_H_INCLUDED
#define NIM_FFI_PRELUDE_H_INCLUDED
/* Generated C binding for a nim-ffi library. Requests/responses travel as
* CBOR (encoded with vendored TinyCBOR on this side, matching the Nim-side
* cbor_serial codec on the wire — both ends speak RFC 8949).
*
* The API is asynchronous: every method/constructor takes a result callback
* and returns immediately. The callback fires exactly once — synchronously on
* a submit-time failure, otherwise from the Nim dispatch thread when the reply
* arrives.
*
* Memory ownership contract:
* - Request-side strings/sequences are *borrowed*: the binding only reads
* them while encoding, so a string literal wrapped with nimffi_str() is
* fine and is never freed by the binding.
* - Response values and error strings passed into a result callback are
* *owned by the binding* and valid only for the duration of that callback;
* the binding reclaims them once the callback returns. The caller never
* frees them. (The generated <lib>_free_<Type>() helpers are internal — the
* trampolines use them to reclaim decoded payloads.)
* - A context handle delivered to a constructor callback is the exception:
* ownership transfers to the caller, who releases it with
* <lib>_ctx_destroy(). It is a lifecycle handle, not returned data.
*
* Trust boundary: the decoders assume the CBOR they parse was produced by the
* paired Nim library. They reject malformed input rather than trusting it, but
* they are not hardened against a hostile peer feeding crafted payloads through
* the raw nimffi_decode_from_buf entry point.
*/
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <tinycbor/cbor.h>
#ifdef __cplusplus
extern "C" {
#endif
/* Owned, length-delimited UTF-8 text (Nim `string`/`cstring`). On the request
* side `data` may point at borrowed storage (see nimffi_str); on the response
* side it is heap-allocated and freed by nimffi_free_str. Always NUL-padded by
* one byte after decode so `data` is usable as a C string when it has no
* embedded NULs. */
typedef struct {
char* data;
size_t len;
} NimFfiStr;
/* Owned, length-delimited byte buffer (Nim `seq[byte]`). */
typedef struct {
uint8_t* data;
size_t len;
} NimFfiBytes;
/* Wrap a borrowed C string for use as a request field. The returned view is
* not owned by the binding and must outlive the call that encodes it. */
static inline NimFfiStr nimffi_str(const char* s) {
NimFfiStr v;
v.data = (char*)s;
v.len = s ? strlen(s) : 0;
return v;
}
static inline void nimffi_free_str(NimFfiStr* v) {
if (!v || !v->data) {
return;
}
free(v->data);
v->data = NULL;
v->len = 0;
}
static inline void nimffi_free_bytes(NimFfiBytes* v) {
if (!v || !v->data) {
return;
}
free(v->data);
v->data = NULL;
v->len = 0;
}
#ifdef __cplusplus
}
#endif
#endif /* NIM_FFI_PRELUDE_H_INCLUDED */
@@ -0,0 +1,50 @@
cmake_minimum_required(VERSION 3.14)
project({{LIB}}_cpp_bindings CXX C)
# The generated bindings target C++20: designated initializers and other
# C++20 constructs are used throughout the emitted code.
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
# MSVC defaults __cplusplus to 199711L regardless of the active /std:c++XX
# level — the generated header's C++20 guard would then misfire. /Zc:__cplusplus
# makes MSVC report the actual standard. Harmless on every other compiler.
if(MSVC)
add_compile_options(/Zc:__cplusplus)
endif()
# ── Locate the repository root (contains ffi.nimble) ─────────────────────────
set(_search_dir "${CMAKE_CURRENT_SOURCE_DIR}")
set(REPO_ROOT "")
foreach(_i RANGE 10)
if(EXISTS "${_search_dir}/ffi.nimble")
set(REPO_ROOT "${_search_dir}")
break()
endif()
get_filename_component(_search_dir "${_search_dir}" DIRECTORY)
endforeach()
if("${REPO_ROOT}" STREQUAL "")
message(FATAL_ERROR "Cannot find repo root (no ffi.nimble in any ancestor)")
endif()
# Build the Nim dylib + vendored TinyCBOR (shared with the C backend).
set(NIM_FFI_LIB {{LIB}})
set(NIM_FFI_SRC {{SRC}})
include("${REPO_ROOT}/ffi/codegen/templates/nim_ffi_lib.cmake")
add_library({{LIB}}_headers INTERFACE)
target_include_directories({{LIB}}_headers INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}")
target_link_libraries({{LIB}}_headers INTERFACE {{LIB}} tinycbor)
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/main.cpp")
add_executable({{LIB}}_example main.cpp)
target_link_libraries({{LIB}}_example PRIVATE {{LIB}}_headers)
add_dependencies({{LIB}}_example {{LIB}}_nim_lib)
if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
add_custom_command(TARGET {{LIB}}_example POST_BUILD
COMMAND "${CMAKE_COMMAND}" -E copy_if_different
"${{{LIB}}_RUNTIME_LIB}"
"$<TARGET_FILE_DIR:{{LIB}}_example>"
COMMENT "Staging {{LIB}}.dll next to {{LIB}}_example.exe")
endif()
endif()
@@ -0,0 +1,190 @@
// ── encode_cbor overloads (primitives + containers) ─────────────────────
// Per-struct encode_cbor / decode_cbor are emitted by cpp.nim next to each
// generated struct; these helpers cover the leaf types they defer into.
// Guarded so two nim-ffi headers can share a translation unit.
#ifndef NIM_FFI_CBOR_HELPERS_HPP_INCLUDED
#define NIM_FFI_CBOR_HELPERS_HPP_INCLUDED
inline CborError encode_cbor(CborEncoder& e, bool v) {
return cbor_encode_boolean(&e, v);
}
inline CborError encode_cbor(CborEncoder& e, int64_t v) {
return cbor_encode_int(&e, v);
}
inline CborError encode_cbor(CborEncoder& e, int32_t v) {
return cbor_encode_int(&e, static_cast<int64_t>(v));
}
inline CborError encode_cbor(CborEncoder& e, uint64_t v) {
return cbor_encode_uint(&e, v);
}
inline CborError encode_cbor(CborEncoder& e, double v) {
return cbor_encode_double(&e, v);
}
inline CborError encode_cbor(CborEncoder& e, const std::string& v) {
return cbor_encode_text_string(&e, v.data(), v.size());
}
template<typename T>
inline CborError encode_cbor(CborEncoder& e, const std::vector<T>& v) {
CborEncoder arr;
CborError err = cbor_encoder_create_array(&e, &arr, v.size());
if (err) return err;
for (const auto& item : v) {
err = encode_cbor(arr, item);
if (err) return err;
}
return cbor_encoder_close_container(&e, &arr);
}
// `seq[byte]` rides the wire as a CBOR byte string (major type 2), matching
// Nim's cbor_serialization. This non-template overload beats the std::vector<T>
// template in overload resolution, so std::vector<std::uint8_t> fields use it
// automatically.
inline CborError encode_cbor(CborEncoder& e, const std::vector<std::uint8_t>& v) {
return cbor_encode_byte_string(&e, v.data(), v.size());
}
template<typename T>
inline CborError encode_cbor(CborEncoder& e, const std::optional<T>& v) {
if (!v) return cbor_encode_null(&e);
return encode_cbor(e, *v);
}
// ── decode_cbor overloads ───────────────────────────────────────────────
// After reading a leaf value, the parser must advance past it; both steps
// short-circuit on the same CborError, so they always travel together.
inline CborError advance_if_ok(CborValue& it, CborError err) {
if (err) return err;
return cbor_value_advance(&it);
}
inline CborError decode_cbor(CborValue& it, bool& out) {
if (!cbor_value_is_boolean(&it)) return CborErrorImproperValue;
return advance_if_ok(it, cbor_value_get_boolean(&it, &out));
}
inline CborError decode_cbor(CborValue& it, int64_t& out) {
if (!cbor_value_is_integer(&it)) return CborErrorImproperValue;
return advance_if_ok(it, cbor_value_get_int64_checked(&it, &out));
}
inline CborError decode_cbor(CborValue& it, int32_t& out) {
int64_t tmp = 0;
CborError err = decode_cbor(it, tmp);
if (err) return err;
out = static_cast<int32_t>(tmp);
return CborNoError;
}
inline CborError decode_cbor(CborValue& it, uint64_t& out) {
if (!cbor_value_is_unsigned_integer(&it)) return CborErrorImproperValue;
return advance_if_ok(it, cbor_value_get_uint64(&it, &out));
}
inline CborError decode_cbor(CborValue& it, double& out) {
if (cbor_value_is_double(&it)) {
return advance_if_ok(it, cbor_value_get_double(&it, &out));
}
if (cbor_value_is_float(&it)) {
float f = 0.0f;
CborError err = cbor_value_get_float(&it, &f);
if (err) return err;
out = static_cast<double>(f);
return cbor_value_advance(&it);
}
return CborErrorImproperValue;
}
inline CborError decode_cbor(CborValue& it, std::string& out) {
if (!cbor_value_is_text_string(&it)) return CborErrorImproperValue;
size_t len = 0;
CborError err = cbor_value_get_string_length(&it, &len);
if (err) return err;
out.resize(len);
return advance_if_ok(
it, cbor_value_copy_text_string(&it, out.empty() ? nullptr : &out[0], &len, nullptr));
}
template<typename T>
inline CborError decode_cbor(CborValue& it, std::vector<T>& out) {
if (!cbor_value_is_array(&it)) return CborErrorImproperValue;
size_t len = 0;
CborError err = cbor_value_get_array_length(&it, &len);
if (err) return err;
out.clear();
out.resize(len);
CborValue inner;
err = cbor_value_enter_container(&it, &inner);
if (err) return err;
for (size_t i = 0; i < len; ++i) {
err = decode_cbor(inner, out[i]);
if (err) return err;
}
return cbor_value_leave_container(&it, &inner);
}
// Counterpart to the byte-string encoder above: decode a CBOR byte string
// (major type 2) back into std::vector<std::uint8_t>.
inline CborError decode_cbor(CborValue& it, std::vector<std::uint8_t>& out) {
if (!cbor_value_is_byte_string(&it)) return CborErrorImproperValue;
size_t len = 0;
CborError err = cbor_value_get_string_length(&it, &len);
if (err) return err;
out.resize(len);
return advance_if_ok(
it, cbor_value_copy_byte_string(&it, out.empty() ? nullptr : out.data(), &len, nullptr));
}
template<typename T>
inline CborError decode_cbor(CborValue& it, std::optional<T>& out) {
if (cbor_value_is_null(&it)) {
out = std::nullopt;
return cbor_value_advance(&it);
}
T tmp{};
CborError err = decode_cbor(it, tmp);
if (err) return err;
out = std::move(tmp);
return CborNoError;
}
// ── Public entry points ─────────────────────────────────────────────────
template<typename T>
inline Result<std::vector<std::uint8_t>> encodeCborFFI(const T& value) {
// Start with a generous 4 KiB buffer; double on overflow until it fits.
std::vector<std::uint8_t> buf(4096);
while (true) {
CborEncoder enc;
cbor_encoder_init(&enc, buf.data(), buf.size(), 0);
CborError err = encode_cbor(enc, value);
if (err == CborNoError) {
const size_t used = cbor_encoder_get_buffer_size(&enc, buf.data());
buf.resize(used);
return Result<std::vector<std::uint8_t>>::ok(std::move(buf));
}
if (err == CborErrorOutOfMemory) {
const size_t extra = cbor_encoder_get_extra_bytes_needed(&enc);
buf.resize(buf.size() + (extra > 0 ? extra : buf.size()));
continue;
}
return Result<std::vector<std::uint8_t>>::err(
std::string("FFI CBOR encode failed: ") + cbor_error_string(err));
}
}
template<typename T>
inline Result<T> decodeCborFFI(const std::vector<std::uint8_t>& bytes) {
CborParser parser;
CborValue it;
CborError err = cbor_parser_init(bytes.data(), bytes.size(), 0, &parser, &it);
if (err != CborNoError) {
return Result<T>::err(std::string("FFI CBOR parse init failed: ") +
cbor_error_string(err));
}
T out{};
err = decode_cbor(it, out);
if (err != CborNoError) {
return Result<T>::err(std::string("FFI CBOR decode failed: ") +
cbor_error_string(err));
}
return Result<T>::ok(std::move(out));
}
#endif // NIM_FFI_CBOR_HELPERS_HPP_INCLUDED
@@ -0,0 +1,20 @@
// Special-member policy: this class owns a {{LIB}} context, which in
// turn owns the library's worker thread(s) and internal state. Moving
// such an object out from under a caller silently tears that state
// down and is easy to misuse (e.g. storing in a container that
// relocates its elements). It also has no clean analogue in the other
// binding languages we generate. So copies and moves are both
// deleted; ownership is transferred via {{CTX}}::create returning a
// std::unique_ptr<{{CTX}}>. The destructor still releases the
// context.
~{{CTX}}() {
if (ptr_) {
{{LIB}}_destroy(ptr_);
ptr_ = nullptr;
}
}
{{CTX}}(const {{CTX}}&) = delete;
{{CTX}}& operator=(const {{CTX}}&) = delete;
{{CTX}}({{CTX}}&&) = delete;
{{CTX}}& operator=({{CTX}}&&) = delete;
@@ -0,0 +1,40 @@
#pragma once
// Generated bindings require C++20 (designated initializers and other
// C++20 constructs are used throughout the emitted code).
// MSVC keeps __cplusplus at 199711L unless /Zc:__cplusplus is passed,
// so consult _MSVC_LANG when present (it always reflects the active
// /std:c++XX level).
#if defined(_MSVC_LANG)
# if _MSVC_LANG < 202002L
# error "nim-ffi generated headers require C++20 or later (use /std:c++20)"
# endif
#elif !defined(__cplusplus) || __cplusplus < 202002L
# error "nim-ffi generated headers require C++20 or later"
#endif
#include <string>
#include <cstdint>
#include <chrono>
#include <charconv>
#include <mutex>
#include <condition_variable>
#include <memory>
#include <functional>
#include <future>
#include <vector>
#include <optional>
#include <type_traits>
#include <cstring>
#include <cassert>
extern "C" {
#include <tinycbor/cbor.h>
}
// nim-ffi result-callback status codes (mirror ffi/ffi_types.nim and the C
// header). Guarded so a translation unit that also pulls in the C header keeps
// a single definition.
#ifndef NIMFFI_RET_OK
#define NIMFFI_RET_OK 0
#define NIMFFI_RET_ERR 1
#define NIMFFI_RET_MISSING_CALLBACK 2
#define NIMFFI_RET_STALE_WARN 3
#endif
@@ -0,0 +1,61 @@
// ============================================================
// Result<T> — exception-free error channel
// ============================================================
// The generated bindings never throw: every fallible entry point (create,
// instance methods, and their *Async futures) returns a Result<T>. Callers
// branch on isOk()/isErr() (or the explicit bool conversion) and read
// value()/error(). This mirrors the Nim side's Result[T, string] and keeps
// us off C++23's std::expected.
#ifndef NIM_FFI_RESULT_HPP_INCLUDED
#define NIM_FFI_RESULT_HPP_INCLUDED
template <typename T>
class Result {
std::optional<T> value_;
std::string error_;
public:
static Result<T> ok(T value) {
Result<T> r;
r.value_ = std::move(value);
return r;
}
static Result<T> err(std::string message) {
Result<T> r;
r.error_ = std::move(message);
return r;
}
bool isOk() const { return value_.has_value(); }
bool isErr() const { return !value_.has_value(); }
explicit operator bool() const { return isOk(); }
const T& value() const { assert(value_.has_value() && "Result::value() called on err Result — check isOk() first"); return *value_; }
T& value() { assert(value_.has_value() && "Result::value() called on err Result — check isOk() first"); return *value_; }
const T& operator*() const { assert(value_.has_value() && "Result::operator*() called on err Result — check isOk() first"); return *value_; }
const T* operator->() const { assert(value_.has_value() && "Result::operator->() called on err Result — check isOk() first"); return &*value_; }
T&& take() { assert(value_.has_value() && "Result::take() called on err Result — check isOk() first"); return std::move(*value_); }
const std::string& error() const { assert(!value_.has_value() && "Result::error() called on ok Result — check isErr() first"); return error_; }
};
template <>
class Result<void> {
bool ok_ = true;
std::string error_;
public:
static Result<void> ok() {
Result<void> r;
r.ok_ = true;
return r;
}
static Result<void> err(std::string message) {
Result<void> r;
r.ok_ = false;
r.error_ = std::move(message);
return r;
}
Result() = default;
bool isOk() const { return ok_; }
bool isErr() const { return !ok_; }
explicit operator bool() const { return isOk(); }
const std::string& error() const { assert(!ok_ && "Result<void>::error() called on ok Result — check isErr() first"); return error_; }
};
#endif // NIM_FFI_RESULT_HPP_INCLUDED
@@ -0,0 +1,66 @@
// ============================================================
// Synchronous call helper
// ============================================================
// Guarded so two nim-ffi headers can share a translation unit.
#ifndef NIM_FFI_SYNC_CALL_HELPER_HPP_INCLUDED
#define NIM_FFI_SYNC_CALL_HELPER_HPP_INCLUDED
namespace {
struct FFICallState_ {
std::mutex mtx;
std::condition_variable cv;
bool done{false};
bool ok{false};
std::vector<std::uint8_t> bytes;
std::string err;
};
inline void ffi_cb_(int ret, const char* msg, size_t len, void* ud) {
// NIMFFI_RET_STALE_WARN (3) is a non-terminal progress ping: the request is
// still running. This blocking wrapper only reports the final result, so
// ignore it WITHOUT touching `ud` a terminal callback still owns the
// shared handle and will free it.
if (ret == NIMFFI_RET_STALE_WARN) return;
// ffi_call_ heap-allocated a shared_ptr and passed its address as ud;
// take ownership here so it's freed on every exit path.
std::unique_ptr<std::shared_ptr<FFICallState_>> handle(
static_cast<std::shared_ptr<FFICallState_>*>(ud));
FFICallState_& s = **handle;
std::lock_guard<std::mutex> lock(s.mtx);
s.ok = (ret == NIMFFI_RET_OK);
if (msg && len > 0) {
const auto* p = reinterpret_cast<const std::uint8_t*>(msg);
if (s.ok) s.bytes.assign(p, p + len);
else s.err.assign(msg, len);
}
s.done = true;
s.cv.notify_one();
}
inline Result<std::vector<std::uint8_t>> ffi_call_(
std::function<int(FFICallback, void*)> f,
std::chrono::milliseconds timeout) {
using Bytes = std::vector<std::uint8_t>;
auto state = std::make_shared<FFICallState_>();
auto* cb_ref = new std::shared_ptr<FFICallState_>(state);
const int ret = f(ffi_cb_, cb_ref);
if (ret == NIMFFI_RET_MISSING_CALLBACK) {
delete cb_ref;
return Result<Bytes>::err("RET_MISSING_CALLBACK (internal error)");
}
std::unique_lock<std::mutex> lock(state->mtx);
const bool fired = state->cv.wait_for(lock, timeout, [&]{ return state->done; });
if (!fired)
return Result<Bytes>::err("FFI call timed out after " +
std::to_string(timeout.count()) + "ms");
if (!state->ok)
return Result<Bytes>::err(state->err);
return Result<Bytes>::ok(std::move(state->bytes));
}
} // anonymous namespace
#endif // NIM_FFI_SYNC_CALL_HELPER_HPP_INCLUDED
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2017 Intel Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
@@ -0,0 +1,724 @@
/****************************************************************************
**
** Copyright (C) 2021 Intel Corporation
**
** Permission is hereby granted, free of charge, to any person obtaining a copy
** of this software and associated documentation files (the "Software"), to deal
** in the Software without restriction, including without limitation the rights
** to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
** copies of the Software, and to permit persons to whom the Software is
** furnished to do so, subject to the following conditions:
**
** The above copyright notice and this permission notice shall be included in
** all copies or substantial portions of the Software.
**
** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
** OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
** THE SOFTWARE.
**
****************************************************************************/
#ifndef CBOR_H
#define CBOR_H
#ifndef assert
#include <assert.h>
#endif
#include <limits.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include <stdio.h>
#include "tinycbor-version.h"
#define TINYCBOR_VERSION ((TINYCBOR_VERSION_MAJOR << 16) | (TINYCBOR_VERSION_MINOR << 8) | TINYCBOR_VERSION_PATCH)
#ifdef __cplusplus
extern "C" {
#else
#include <stdbool.h>
#endif
#ifndef SIZE_MAX
/* Some systems fail to define SIZE_MAX in <stdint.h>, even though C99 requires it...
* Conversion from signed to unsigned is defined in 6.3.1.3 (Signed and unsigned integers) p2,
* which says: "the value is converted by repeatedly adding or subtracting one more than the
* maximum value that can be represented in the new type until the value is in the range of the
* new type."
* So -1 gets converted to size_t by adding SIZE_MAX + 1, which results in SIZE_MAX.
*/
# define SIZE_MAX ((size_t)-1)
#endif
#ifndef CBOR_API
# define CBOR_API
#endif
#ifndef CBOR_PRIVATE_API
# define CBOR_PRIVATE_API
#endif
#ifndef CBOR_INLINE_API
# if defined(__cplusplus)
# define CBOR_INLINE inline
# define CBOR_INLINE_API inline
# else
# define CBOR_INLINE_API static CBOR_INLINE
# if defined(_MSC_VER)
# define CBOR_INLINE __inline
# elif defined(__GNUC__)
# define CBOR_INLINE __inline__
# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
# define CBOR_INLINE inline
# else
# define CBOR_INLINE
# endif
# endif
#endif
typedef enum CborType {
CborIntegerType = 0x00,
CborByteStringType = 0x40,
CborTextStringType = 0x60,
CborArrayType = 0x80,
CborMapType = 0xa0,
CborTagType = 0xc0,
CborSimpleType = 0xe0,
CborBooleanType = 0xf5,
CborNullType = 0xf6,
CborUndefinedType = 0xf7,
CborHalfFloatType = 0xf9,
CborFloatType = 0xfa,
CborDoubleType = 0xfb,
CborInvalidType = 0xff /* equivalent to the break byte, so it will never be used */
} CborType;
typedef uint64_t CborTag;
typedef enum CborKnownTags {
CborDateTimeStringTag = 0,
CborUnixTime_tTag = 1,
CborPositiveBignumTag = 2,
CborNegativeBignumTag = 3,
CborDecimalTag = 4,
CborBigfloatTag = 5,
CborCOSE_Encrypt0Tag = 16,
CborCOSE_Mac0Tag = 17,
CborCOSE_Sign1Tag = 18,
CborExpectedBase64urlTag = 21,
CborExpectedBase64Tag = 22,
CborExpectedBase16Tag = 23,
CborEncodedCborTag = 24,
CborUrlTag = 32,
CborBase64urlTag = 33,
CborBase64Tag = 34,
CborRegularExpressionTag = 35,
CborMimeMessageTag = 36,
CborCOSE_EncryptTag = 96,
CborCOSE_MacTag = 97,
CborCOSE_SignTag = 98,
CborSignatureTag = 55799
} CborKnownTags;
/* #define the constants so we can check with #ifdef */
#define CborDateTimeStringTag CborDateTimeStringTag
#define CborUnixTime_tTag CborUnixTime_tTag
#define CborPositiveBignumTag CborPositiveBignumTag
#define CborNegativeBignumTag CborNegativeBignumTag
#define CborDecimalTag CborDecimalTag
#define CborBigfloatTag CborBigfloatTag
#define CborCOSE_Encrypt0Tag CborCOSE_Encrypt0Tag
#define CborCOSE_Mac0Tag CborCOSE_Mac0Tag
#define CborCOSE_Sign1Tag CborCOSE_Sign1Tag
#define CborExpectedBase64urlTag CborExpectedBase64urlTag
#define CborExpectedBase64Tag CborExpectedBase64Tag
#define CborExpectedBase16Tag CborExpectedBase16Tag
#define CborEncodedCborTag CborEncodedCborTag
#define CborUrlTag CborUrlTag
#define CborBase64urlTag CborBase64urlTag
#define CborBase64Tag CborBase64Tag
#define CborRegularExpressionTag CborRegularExpressionTag
#define CborMimeMessageTag CborMimeMessageTag
#define CborCOSE_EncryptTag CborCOSE_EncryptTag
#define CborCOSE_MacTag CborCOSE_MacTag
#define CborCOSE_SignTag CborCOSE_SignTag
#define CborSignatureTag CborSignatureTag
/* Error API */
typedef enum CborError {
CborNoError = 0,
/* errors in all modes */
CborUnknownError,
CborErrorUnknownLength, /* request for length in array, map, or string with indeterminate length */
CborErrorAdvancePastEOF,
CborErrorIO,
/* parser errors streaming errors */
CborErrorGarbageAtEnd = 256,
CborErrorUnexpectedEOF,
CborErrorUnexpectedBreak,
CborErrorUnknownType, /* can only happen in major type 7 */
CborErrorIllegalType, /* type not allowed here */
CborErrorIllegalNumber,
CborErrorIllegalSimpleType, /* types of value less than 32 encoded in two bytes */
CborErrorNoMoreStringChunks,
/* parser errors in strict mode parsing only */
CborErrorUnknownSimpleType = 512,
CborErrorUnknownTag,
CborErrorInappropriateTagForType,
CborErrorDuplicateObjectKeys,
CborErrorInvalidUtf8TextString,
CborErrorExcludedType,
CborErrorExcludedValue,
CborErrorImproperValue,
CborErrorOverlongEncoding,
CborErrorMapKeyNotString,
CborErrorMapNotSorted,
CborErrorMapKeysNotUnique,
/* encoder errors */
CborErrorTooManyItems = 768,
CborErrorTooFewItems,
/* internal implementation errors */
CborErrorDataTooLarge = 1024,
CborErrorNestingTooDeep,
CborErrorUnsupportedType,
CborErrorUnimplementedValidation,
/* errors in converting to JSON */
CborErrorJsonObjectKeyIsAggregate = 1280,
CborErrorJsonObjectKeyNotString,
CborErrorJsonNotImplemented,
CborErrorOutOfMemory = (int) (~0U / 2 + 1),
CborErrorInternalError = (int) (~0U / 2) /* INT_MAX on two's complement machines */
} CborError;
CBOR_API const char *cbor_error_string(CborError error);
/* Encoder API */
typedef enum CborEncoderAppendType
{
CborEncoderAppendCborData = 0,
CborEncoderAppendStringData = 1
} CborEncoderAppendType;
typedef CborError (*CborEncoderWriteFunction)(void *, const void *, size_t, CborEncoderAppendType);
enum CborEncoderFlags
{
CborIteratorFlag_WriterFunction = 0x01,
CborIteratorFlag_ContainerIsMap_ = 0x20
};
struct CborEncoder
{
union {
uint8_t *ptr;
ptrdiff_t bytes_needed;
CborEncoderWriteFunction writer;
} data;
uint8_t *end;
size_t remaining;
int flags;
};
typedef struct CborEncoder CborEncoder;
static const size_t CborIndefiniteLength = SIZE_MAX;
#ifndef CBOR_NO_ENCODER_API
CBOR_API void cbor_encoder_init(CborEncoder *encoder, uint8_t *buffer, size_t size, int flags);
CBOR_API void cbor_encoder_init_writer(CborEncoder *encoder, CborEncoderWriteFunction writer, void *);
CBOR_API CborError cbor_encode_uint(CborEncoder *encoder, uint64_t value);
CBOR_API CborError cbor_encode_int(CborEncoder *encoder, int64_t value);
CBOR_API CborError cbor_encode_negative_int(CborEncoder *encoder, uint64_t absolute_value);
CBOR_API CborError cbor_encode_simple_value(CborEncoder *encoder, uint8_t value);
CBOR_API CborError cbor_encode_tag(CborEncoder *encoder, CborTag tag);
CBOR_API CborError cbor_encode_text_string(CborEncoder *encoder, const char *string, size_t length);
CBOR_INLINE_API CborError cbor_encode_text_stringz(CborEncoder *encoder, const char *string)
{ return cbor_encode_text_string(encoder, string, strlen(string)); }
CBOR_API CborError cbor_encode_byte_string(CborEncoder *encoder, const uint8_t *string, size_t length);
CBOR_API CborError cbor_encode_floating_point(CborEncoder *encoder, CborType fpType, const void *value);
CBOR_INLINE_API CborError cbor_encode_boolean(CborEncoder *encoder, bool value)
{ return cbor_encode_simple_value(encoder, (int)value - 1 + (CborBooleanType & 0x1f)); }
CBOR_INLINE_API CborError cbor_encode_null(CborEncoder *encoder)
{ return cbor_encode_simple_value(encoder, CborNullType & 0x1f); }
CBOR_INLINE_API CborError cbor_encode_undefined(CborEncoder *encoder)
{ return cbor_encode_simple_value(encoder, CborUndefinedType & 0x1f); }
CBOR_INLINE_API CborError cbor_encode_half_float(CborEncoder *encoder, const void *value)
{ return cbor_encode_floating_point(encoder, CborHalfFloatType, value); }
CBOR_API CborError cbor_encode_float_as_half_float(CborEncoder *encoder, float value);
CBOR_INLINE_API CborError cbor_encode_float(CborEncoder *encoder, float value)
{ return cbor_encode_floating_point(encoder, CborFloatType, &value); }
CBOR_INLINE_API CborError cbor_encode_double(CborEncoder *encoder, double value)
{ return cbor_encode_floating_point(encoder, CborDoubleType, &value); }
CBOR_API CborError cbor_encoder_create_array(CborEncoder *parentEncoder, CborEncoder *arrayEncoder, size_t length);
CBOR_API CborError cbor_encoder_create_map(CborEncoder *parentEncoder, CborEncoder *mapEncoder, size_t length);
CBOR_API CborError cbor_encoder_close_container(CborEncoder *parentEncoder, const CborEncoder *containerEncoder);
CBOR_API CborError cbor_encoder_close_container_checked(CborEncoder *parentEncoder, const CborEncoder *containerEncoder);
CBOR_INLINE_API uint8_t *_cbor_encoder_get_buffer_pointer(const CborEncoder *encoder)
{
return encoder->data.ptr;
}
CBOR_INLINE_API size_t cbor_encoder_get_buffer_size(const CborEncoder *encoder, const uint8_t *buffer)
{
return (size_t)(encoder->data.ptr - buffer);
}
CBOR_INLINE_API size_t cbor_encoder_get_extra_bytes_needed(const CborEncoder *encoder)
{
return encoder->end ? 0 : (size_t)encoder->data.bytes_needed;
}
#endif /* CBOR_NO_ENCODER_API */
/* Parser API */
enum CborParserGlobalFlags
{
CborParserFlag_ExternalSource = 0x01
};
enum CborParserIteratorFlags
{
/* used for all types, but not during string chunk iteration
* (values are static-asserted, don't change) */
CborIteratorFlag_IntegerValueIs64Bit = 0x01,
CborIteratorFlag_IntegerValueTooLarge = 0x02,
/* used only for CborIntegerType */
CborIteratorFlag_NegativeInteger = 0x04,
/* used only during string iteration */
CborIteratorFlag_BeforeFirstStringChunk = 0x04,
CborIteratorFlag_IteratingStringChunks = 0x08,
/* used for arrays, maps and strings, including during chunk iteration */
CborIteratorFlag_UnknownLength = 0x10,
/* used for maps, but must be kept for all types
* (ContainerIsMap value must be CborMapType - CborArrayType) */
CborIteratorFlag_ContainerIsMap = 0x20,
CborIteratorFlag_NextIsMapKey = 0x40
};
struct CborValue;
struct CborParserOperations
{
bool (*can_read_bytes)(void *token, size_t len);
void *(*read_bytes)(void *token, void *dst, size_t offset, size_t len);
void (*advance_bytes)(void *token, size_t len);
CborError (*transfer_string)(void *token, const void **userptr, size_t offset, size_t len);
};
struct CborParser
{
union {
const uint8_t *end;
const struct CborParserOperations *ops;
} source;
enum CborParserGlobalFlags flags;
};
typedef struct CborParser CborParser;
struct CborValue
{
const CborParser *parser;
union {
const uint8_t *ptr;
void *token;
} source;
uint32_t remaining;
uint16_t extra;
uint8_t type;
uint8_t flags;
};
typedef struct CborValue CborValue;
#ifndef CBOR_NO_PARSER_API
CBOR_API CborError cbor_parser_init(const uint8_t *buffer, size_t size, uint32_t flags, CborParser *parser, CborValue *it);
CBOR_API CborError cbor_parser_init_reader(const struct CborParserOperations *ops, CborParser *parser, CborValue *it, void *token);
CBOR_API CborError cbor_value_validate_basic(const CborValue *it);
CBOR_INLINE_API bool cbor_value_at_end(const CborValue *it)
{ return it->remaining == 0; }
CBOR_INLINE_API const uint8_t *cbor_value_get_next_byte(const CborValue *it)
{ return it->source.ptr; }
CBOR_API CborError cbor_value_reparse(CborValue *it);
CBOR_API CborError cbor_value_advance_fixed(CborValue *it);
CBOR_API CborError cbor_value_advance(CborValue *it);
CBOR_INLINE_API bool cbor_value_is_container(const CborValue *it)
{ return it->type == CborArrayType || it->type == CborMapType; }
CBOR_API CborError cbor_value_enter_container(const CborValue *it, CborValue *recursed);
CBOR_API CborError cbor_value_leave_container(CborValue *it, const CborValue *recursed);
CBOR_PRIVATE_API uint64_t _cbor_value_decode_int64_internal(const CborValue *value);
CBOR_INLINE_API uint64_t _cbor_value_extract_int64_helper(const CborValue *value)
{
return value->flags & CborIteratorFlag_IntegerValueTooLarge ?
_cbor_value_decode_int64_internal(value) : value->extra;
}
CBOR_INLINE_API bool cbor_value_is_valid(const CborValue *value)
{ return value && value->type != CborInvalidType; }
CBOR_INLINE_API CborType cbor_value_get_type(const CborValue *value)
{ return (CborType)value->type; }
/* Null & undefined type */
CBOR_INLINE_API bool cbor_value_is_null(const CborValue *value)
{ return value->type == CborNullType; }
CBOR_INLINE_API bool cbor_value_is_undefined(const CborValue *value)
{ return value->type == CborUndefinedType; }
/* Booleans */
CBOR_INLINE_API bool cbor_value_is_boolean(const CborValue *value)
{ return value->type == CborBooleanType; }
CBOR_INLINE_API CborError cbor_value_get_boolean(const CborValue *value, bool *result)
{
assert(cbor_value_is_boolean(value));
*result = !!value->extra;
return CborNoError;
}
/* Simple types */
CBOR_INLINE_API bool cbor_value_is_simple_type(const CborValue *value)
{ return value->type == CborSimpleType; }
CBOR_INLINE_API CborError cbor_value_get_simple_type(const CborValue *value, uint8_t *result)
{
assert(cbor_value_is_simple_type(value));
*result = (uint8_t)value->extra;
return CborNoError;
}
/* Integers */
CBOR_INLINE_API bool cbor_value_is_integer(const CborValue *value)
{ return value->type == CborIntegerType; }
CBOR_INLINE_API bool cbor_value_is_unsigned_integer(const CborValue *value)
{ return cbor_value_is_integer(value) && (value->flags & CborIteratorFlag_NegativeInteger) == 0; }
CBOR_INLINE_API bool cbor_value_is_negative_integer(const CborValue *value)
{ return cbor_value_is_integer(value) && (value->flags & CborIteratorFlag_NegativeInteger); }
CBOR_INLINE_API CborError cbor_value_get_raw_integer(const CborValue *value, uint64_t *result)
{
assert(cbor_value_is_integer(value));
*result = _cbor_value_extract_int64_helper(value);
return CborNoError;
}
CBOR_INLINE_API CborError cbor_value_get_uint64(const CborValue *value, uint64_t *result)
{
assert(cbor_value_is_unsigned_integer(value));
*result = _cbor_value_extract_int64_helper(value);
return CborNoError;
}
CBOR_INLINE_API CborError cbor_value_get_int64(const CborValue *value, int64_t *result)
{
assert(cbor_value_is_integer(value));
*result = (int64_t) _cbor_value_extract_int64_helper(value);
if (value->flags & CborIteratorFlag_NegativeInteger)
*result = -*result - 1;
return CborNoError;
}
CBOR_INLINE_API CborError cbor_value_get_int(const CborValue *value, int *result)
{
assert(cbor_value_is_integer(value));
*result = (int) _cbor_value_extract_int64_helper(value);
if (value->flags & CborIteratorFlag_NegativeInteger)
*result = -*result - 1;
return CborNoError;
}
CBOR_API CborError cbor_value_get_int64_checked(const CborValue *value, int64_t *result);
CBOR_API CborError cbor_value_get_int_checked(const CborValue *value, int *result);
CBOR_INLINE_API bool cbor_value_is_length_known(const CborValue *value)
{ return (value->flags & CborIteratorFlag_UnknownLength) == 0; }
/* Tags */
CBOR_INLINE_API bool cbor_value_is_tag(const CborValue *value)
{ return value->type == CborTagType; }
CBOR_INLINE_API CborError cbor_value_get_tag(const CborValue *value, CborTag *result)
{
assert(cbor_value_is_tag(value));
*result = _cbor_value_extract_int64_helper(value);
return CborNoError;
}
CBOR_API CborError cbor_value_skip_tag(CborValue *it);
/* Strings */
CBOR_INLINE_API bool cbor_value_is_byte_string(const CborValue *value)
{ return value->type == CborByteStringType; }
CBOR_INLINE_API bool cbor_value_is_text_string(const CborValue *value)
{ return value->type == CborTextStringType; }
CBOR_INLINE_API CborError cbor_value_get_string_length(const CborValue *value, size_t *length)
{
uint64_t v;
assert(cbor_value_is_byte_string(value) || cbor_value_is_text_string(value));
if (!cbor_value_is_length_known(value))
return CborErrorUnknownLength;
v = _cbor_value_extract_int64_helper(value);
*length = (size_t)v;
if (*length != v)
return CborErrorDataTooLarge;
return CborNoError;
}
CBOR_PRIVATE_API CborError _cbor_value_copy_string(const CborValue *value, void *buffer,
size_t *buflen, CborValue *next);
CBOR_PRIVATE_API CborError _cbor_value_dup_string(const CborValue *value, void **buffer,
size_t *buflen, CborValue *next);
CBOR_API CborError cbor_value_calculate_string_length(const CborValue *value, size_t *length);
CBOR_INLINE_API CborError cbor_value_copy_text_string(const CborValue *value, char *buffer,
size_t *buflen, CborValue *next)
{
assert(cbor_value_is_text_string(value));
return _cbor_value_copy_string(value, buffer, buflen, next);
}
CBOR_INLINE_API CborError cbor_value_copy_byte_string(const CborValue *value, uint8_t *buffer,
size_t *buflen, CborValue *next)
{
assert(cbor_value_is_byte_string(value));
return _cbor_value_copy_string(value, buffer, buflen, next);
}
CBOR_INLINE_API CborError cbor_value_dup_text_string(const CborValue *value, char **buffer,
size_t *buflen, CborValue *next)
{
assert(cbor_value_is_text_string(value));
return _cbor_value_dup_string(value, (void **)buffer, buflen, next);
}
CBOR_INLINE_API CborError cbor_value_dup_byte_string(const CborValue *value, uint8_t **buffer,
size_t *buflen, CborValue *next)
{
assert(cbor_value_is_byte_string(value));
return _cbor_value_dup_string(value, (void **)buffer, buflen, next);
}
CBOR_PRIVATE_API CborError _cbor_value_get_string_chunk_size(const CborValue *value, size_t *len);
CBOR_INLINE_API CborError cbor_value_get_string_chunk_size(const CborValue *value, size_t *len)
{
assert(value->flags & CborIteratorFlag_IteratingStringChunks);
return _cbor_value_get_string_chunk_size(value, len);
}
CBOR_INLINE_API bool cbor_value_string_iteration_at_end(const CborValue *value)
{
size_t dummy;
return cbor_value_get_string_chunk_size(value, &dummy) == CborErrorNoMoreStringChunks;
}
CBOR_PRIVATE_API CborError _cbor_value_begin_string_iteration(CborValue *value);
CBOR_INLINE_API CborError cbor_value_begin_string_iteration(CborValue *value)
{
assert(cbor_value_is_text_string(value) || cbor_value_is_byte_string(value));
assert(!(value->flags & CborIteratorFlag_IteratingStringChunks));
return _cbor_value_begin_string_iteration(value);
}
CBOR_PRIVATE_API CborError _cbor_value_finish_string_iteration(CborValue *value);
CBOR_INLINE_API CborError cbor_value_finish_string_iteration(CborValue *value)
{
assert(cbor_value_string_iteration_at_end(value));
return _cbor_value_finish_string_iteration(value);
}
CBOR_PRIVATE_API CborError _cbor_value_get_string_chunk(const CborValue *value, const void **bufferptr,
size_t *len, CborValue *next);
CBOR_INLINE_API CborError cbor_value_get_text_string_chunk(const CborValue *value, const char **bufferptr,
size_t *len, CborValue *next)
{
assert(cbor_value_is_text_string(value));
return _cbor_value_get_string_chunk(value, (const void **)bufferptr, len, next);
}
CBOR_INLINE_API CborError cbor_value_get_byte_string_chunk(const CborValue *value, const uint8_t **bufferptr,
size_t *len, CborValue *next)
{
assert(cbor_value_is_byte_string(value));
return _cbor_value_get_string_chunk(value, (const void **)bufferptr, len, next);
}
CBOR_API CborError cbor_value_text_string_equals(const CborValue *value, const char *string, bool *result);
/* Maps and arrays */
CBOR_INLINE_API bool cbor_value_is_array(const CborValue *value)
{ return value->type == CborArrayType; }
CBOR_INLINE_API bool cbor_value_is_map(const CborValue *value)
{ return value->type == CborMapType; }
CBOR_INLINE_API CborError cbor_value_get_array_length(const CborValue *value, size_t *length)
{
uint64_t v;
assert(cbor_value_is_array(value));
if (!cbor_value_is_length_known(value))
return CborErrorUnknownLength;
v = _cbor_value_extract_int64_helper(value);
*length = (size_t)v;
if (*length != v)
return CborErrorDataTooLarge;
return CborNoError;
}
CBOR_INLINE_API CborError cbor_value_get_map_length(const CborValue *value, size_t *length)
{
uint64_t v;
assert(cbor_value_is_map(value));
if (!cbor_value_is_length_known(value))
return CborErrorUnknownLength;
v = _cbor_value_extract_int64_helper(value);
*length = (size_t)v;
if (*length != v)
return CborErrorDataTooLarge;
return CborNoError;
}
CBOR_API CborError cbor_value_map_find_value(const CborValue *map, const char *string, CborValue *element);
/* Floating point */
CBOR_INLINE_API bool cbor_value_is_half_float(const CborValue *value)
{ return value->type == CborHalfFloatType; }
CBOR_API CborError cbor_value_get_half_float_as_float(const CborValue *value, float *result);
CBOR_INLINE_API CborError cbor_value_get_half_float(const CborValue *value, void *result)
{
assert(cbor_value_is_half_float(value));
assert((value->flags & CborIteratorFlag_IntegerValueTooLarge) == 0);
/* size has already been computed */
memcpy(result, &value->extra, sizeof(value->extra));
return CborNoError;
}
CBOR_INLINE_API bool cbor_value_is_float(const CborValue *value)
{ return value->type == CborFloatType; }
CBOR_INLINE_API CborError cbor_value_get_float(const CborValue *value, float *result)
{
uint32_t data;
assert(cbor_value_is_float(value));
assert(value->flags & CborIteratorFlag_IntegerValueTooLarge);
data = (uint32_t)_cbor_value_decode_int64_internal(value);
memcpy(result, &data, sizeof(*result));
return CborNoError;
}
CBOR_INLINE_API bool cbor_value_is_double(const CborValue *value)
{ return value->type == CborDoubleType; }
CBOR_INLINE_API CborError cbor_value_get_double(const CborValue *value, double *result)
{
uint64_t data;
assert(cbor_value_is_double(value));
assert(value->flags & CborIteratorFlag_IntegerValueTooLarge);
data = _cbor_value_decode_int64_internal(value);
memcpy(result, &data, sizeof(*result));
return CborNoError;
}
/* Validation API */
#ifndef CBOR_NO_VALIDATION_API
enum CborValidationFlags {
/* Bit mapping:
* bits 0-7 (8 bits): canonical format
* bits 8-11 (4 bits): canonical format & strict mode
* bits 12-20 (8 bits): strict mode
* bits 21-31 (10 bits): other
*/
CborValidateShortestIntegrals = 0x0001,
CborValidateShortestFloatingPoint = 0x0002,
CborValidateShortestNumbers = CborValidateShortestIntegrals | CborValidateShortestFloatingPoint,
CborValidateNoIndeterminateLength = 0x0100,
CborValidateMapIsSorted = 0x0200 | CborValidateNoIndeterminateLength,
CborValidateCanonicalFormat = 0x0fff,
CborValidateMapKeysAreUnique = 0x1000 | CborValidateMapIsSorted,
CborValidateTagUse = 0x2000,
CborValidateUtf8 = 0x4000,
CborValidateStrictMode = 0xfff00,
CborValidateMapKeysAreString = 0x100000,
CborValidateNoUndefined = 0x200000,
CborValidateNoTags = 0x400000,
CborValidateFiniteFloatingPoint = 0x800000,
/* unused = 0x1000000, */
/* unused = 0x2000000, */
CborValidateNoUnknownSimpleTypesSA = 0x4000000,
CborValidateNoUnknownSimpleTypes = 0x8000000 | CborValidateNoUnknownSimpleTypesSA,
CborValidateNoUnknownTagsSA = 0x10000000,
CborValidateNoUnknownTagsSR = 0x20000000 | CborValidateNoUnknownTagsSA,
CborValidateNoUnknownTags = 0x40000000 | CborValidateNoUnknownTagsSR,
CborValidateCompleteData = (int)0x80000000,
CborValidateStrictest = (int)~0U,
CborValidateBasic = 0
};
CBOR_API CborError cbor_value_validate(const CborValue *it, uint32_t flags);
#endif /* CBOR_NO_VALIDATION_API */
/* Human-readable (dump) API */
#ifndef CBOR_NO_PRETTY_API
enum CborPrettyFlags {
CborPrettyNumericEncodingIndicators = 0x01,
CborPrettyTextualEncodingIndicators = 0,
CborPrettyIndicateIndeterminateLength = 0x02,
CborPrettyIndicateIndetermineLength = CborPrettyIndicateIndeterminateLength, /* deprecated */
CborPrettyIndicateOverlongNumbers = 0x04,
CborPrettyShowStringFragments = 0x100,
CborPrettyMergeStringFragments = 0,
CborPrettyDefaultFlags = CborPrettyIndicateIndeterminateLength
};
typedef CborError (*CborStreamFunction)(void *token, const char *fmt, ...)
#ifdef __GNUC__
__attribute__((__format__(printf, 2, 3)))
#endif
;
CBOR_API CborError cbor_value_to_pretty_stream(CborStreamFunction streamFunction, void *token, CborValue *value, int flags);
/* The following API requires a hosted C implementation (uses FILE*) */
#if !defined(__STDC_HOSTED__) || __STDC_HOSTED__-0 == 1
CBOR_API CborError cbor_value_to_pretty_advance_flags(FILE *out, CborValue *value, int flags);
CBOR_API CborError cbor_value_to_pretty_advance(FILE *out, CborValue *value);
CBOR_INLINE_API CborError cbor_value_to_pretty(FILE *out, const CborValue *value)
{
CborValue copy = *value;
return cbor_value_to_pretty_advance_flags(out, &copy, CborPrettyDefaultFlags);
}
#endif /* __STDC_HOSTED__ check */
#endif /* CBOR_NO_PRETTY_API */
#endif /* CBOR_NO_PARSER_API */
#ifdef __cplusplus
}
#endif
#endif /* CBOR_H */
@@ -0,0 +1,689 @@
/****************************************************************************
**
** Copyright (C) 2021 Intel Corporation
**
** Permission is hereby granted, free of charge, to any person obtaining a copy
** of this software and associated documentation files (the "Software"), to deal
** in the Software without restriction, including without limitation the rights
** to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
** copies of the Software, and to permit persons to whom the Software is
** furnished to do so, subject to the following conditions:
**
** The above copyright notice and this permission notice shall be included in
** all copies or substantial portions of the Software.
**
** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
** OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
** THE SOFTWARE.
**
****************************************************************************/
#ifndef _BSD_SOURCE
#define _BSD_SOURCE 1
#endif
#ifndef _DEFAULT_SOURCE
#define _DEFAULT_SOURCE 1
#endif
#ifndef __STDC_LIMIT_MACROS
# define __STDC_LIMIT_MACROS 1
#endif
#include "cbor.h"
#include "cborinternal_p.h"
#include "compilersupport_p.h"
#include <stdlib.h>
#include <string.h>
/**
* \defgroup CborEncoding Encoding to CBOR
* \brief Group of functions used to encode data to CBOR.
*
* CborEncoder is used to encode data into a CBOR stream. The outermost
* CborEncoder is initialized by calling cbor_encoder_init(), with the buffer
* where the CBOR stream will be stored. The outermost CborEncoder is usually
* used to encode exactly one item, most often an array or map. It is possible
* to encode more than one item, but care must then be taken on the decoder
* side to ensure the state is reset after each item was decoded.
*
* Nested CborEncoder objects are created using cbor_encoder_create_array() and
* cbor_encoder_create_map(), later closed with cbor_encoder_close_container()
* or cbor_encoder_close_container_checked(). The pairs of creation and closing
* must be exactly matched and their parameters are always the same.
*
* CborEncoder writes directly to the user-supplied buffer, without extra
* buffering. CborEncoder does not allocate memory and CborEncoder objects are
* usually created on the stack of the encoding functions.
*
* The example below initializes a CborEncoder object with a buffer and encodes
* a single integer.
*
* \code
* uint8_t buf[16];
* CborEncoder encoder;
* cbor_encoder_init(&encoder, buf, sizeof(buf), 0);
* cbor_encode_int(&encoder, some_value);
* \endcode
*
* As explained before, usually the outermost CborEncoder object is used to add
* one array or map, which in turn contains multiple elements. The example
* below creates a CBOR map with one element: a key "foo" and a boolean value.
*
* \code
* uint8_t buf[16];
* CborEncoder encoder, mapEncoder;
* cbor_encoder_init(&encoder, buf, sizeof(buf), 0);
* cbor_encoder_create_map(&encoder, &mapEncoder, 1);
* cbor_encode_text_stringz(&mapEncoder, "foo");
* cbor_encode_boolean(&mapEncoder, some_value);
* cbor_encoder_close_container(&encoder, &mapEncoder);
* \endcode
*
* <h3 class="groupheader">Error checking and buffer size</h3>
*
* All functions operating on CborEncoder return a condition of type CborError.
* If the encoding was successful, they return CborNoError. Some functions do
* extra checking on the input provided and may return some other error
* conditions (for example, cbor_encode_simple_value() checks that the type is
* of the correct type).
*
* In addition, all functions check whether the buffer has enough bytes to
* encode the item being appended. If that is not possible, they return
* CborErrorOutOfMemory.
*
* It is possible to continue with the encoding of data past the first function
* that returns CborErrorOutOfMemory. CborEncoder functions will not overrun
* the buffer, but will instead count how many more bytes are needed to
* complete the encoding. At the end, you can obtain that count by calling
* cbor_encoder_get_extra_bytes_needed().
*
* \section1 Finalizing the encoding
*
* Once all items have been appended and the containers have all been properly
* closed, the user-supplied buffer will contain the CBOR stream and may be
* immediately used. To obtain the size of the buffer, call
* cbor_encoder_get_buffer_size() with the original buffer pointer.
*
* The example below illustrates how one can encode an item with error checking
* and then pass on the buffer for network sending.
*
* \code
* uint8_t buf[16];
* CborError err;
* CborEncoder encoder, mapEncoder;
* cbor_encoder_init(&encoder, buf, sizeof(buf), 0);
* err = cbor_encoder_create_map(&encoder, &mapEncoder, 1);
* if (err)
* return err;
* err = cbor_encode_text_stringz(&mapEncoder, "foo");
* if (err)
* return err;
* err = cbor_encode_boolean(&mapEncoder, some_value);
* if (err)
* return err;
* err = cbor_encoder_close_container_checked(&encoder, &mapEncoder);
* if (err)
* return err;
*
* size_t len = cbor_encoder_get_buffer_size(&encoder, buf);
* send_payload(buf, len);
* return CborNoError;
* \endcode
*
* Finally, the example below expands on the one above and also
* deals with dynamically growing the buffer if the initial allocation wasn't
* big enough. Note the two places where the error checking was replaced with
* an cbor_assertion, showing where the author assumes no error can occur.
*
* \code
* uint8_t *encode_string_array(const char **strings, int n, size_t *bufsize)
* {
* CborError err;
* CborEncoder encoder, arrayEncoder;
* size_t size = 256;
* uint8_t *buf = NULL;
*
* while (1) {
* int i;
* size_t more_bytes;
* uint8_t *nbuf = realloc(buf, size);
* if (nbuf == NULL)
* goto error;
* buf = nbuf;
*
* cbor_encoder_init(&encoder, buf, size, 0);
* err = cbor_encoder_create_array(&encoder, &arrayEncoder, n);
* cbor_assert(!err); // can't fail, the buffer is always big enough
*
* for (i = 0; i < n; ++i) {
* err = cbor_encode_text_stringz(&arrayEncoder, strings[i]);
* if (err && err != CborErrorOutOfMemory)
* goto error;
* }
*
* err = cbor_encoder_close_container_checked(&encoder, &arrayEncoder);
* cbor_assert(!err); // shouldn't fail!
*
* more_bytes = cbor_encoder_get_extra_bytes_needed(encoder);
* if (more_size) {
* // buffer wasn't big enough, try again
* size += more_bytes;
* continue;
* }
*
* *bufsize = cbor_encoder_get_buffer_size(encoder, buf);
* return buf;
* }
* error:
* free(buf);
* return NULL;
* }
* \endcode
*/
/**
* \addtogroup CborEncoding
* @{
*/
/**
* \struct CborEncoder
* Structure used to encode to CBOR.
*/
/**
* Initializes a CborEncoder structure \a encoder by pointing it to buffer \a
* buffer of size \a size. The \a flags field is currently unused and must be
* zero.
*/
void cbor_encoder_init(CborEncoder *encoder, uint8_t *buffer, size_t size, int flags)
{
encoder->data.ptr = buffer;
encoder->end = buffer + size;
encoder->remaining = 2;
encoder->flags = flags;
}
void cbor_encoder_init_writer(CborEncoder *encoder, CborEncoderWriteFunction writer, void *token)
{
#ifdef CBOR_ENCODER_WRITE_FUNCTION
(void) writer;
#else
encoder->data.writer = writer;
#endif
encoder->end = (uint8_t *)token;
encoder->remaining = 2;
encoder->flags = CborIteratorFlag_WriterFunction;
}
static inline void put16(void *where, uint16_t v)
{
uint16_t v_be = cbor_htons(v);
memcpy(where, &v_be, sizeof(v_be));
}
/* Note: Since this is currently only used in situations where OOM is the only
* valid error, we KNOW this to be true. Thus, this function now returns just 'true',
* but if in the future, any function starts returning a non-OOM error, this will need
* to be changed to the test. At the moment, this is done to prevent more branches
* being created in the tinycbor output */
static inline bool isOomError(CborError err)
{
if (CBOR_ENCODER_WRITER_CONTROL < 0)
return true;
/* CborErrorOutOfMemory is the only negative error code, intentionally
* so we can write the test like this */
return (int)err < 0;
}
static inline void put32(void *where, uint32_t v)
{
uint32_t v_be = cbor_htonl(v);
memcpy(where, &v_be, sizeof(v_be));
}
static inline void put64(void *where, uint64_t v)
{
uint64_t v_be = cbor_htonll(v);
memcpy(where, &v_be, sizeof(v_be));
}
static inline bool would_overflow(CborEncoder *encoder, size_t len)
{
ptrdiff_t remaining = (ptrdiff_t)encoder->end;
remaining -= remaining ? (ptrdiff_t)encoder->data.ptr : encoder->data.bytes_needed;
remaining -= (ptrdiff_t)len;
return unlikely(remaining < 0);
}
static inline void advance_ptr(CborEncoder *encoder, size_t n)
{
if (encoder->end)
encoder->data.ptr += n;
else
encoder->data.bytes_needed += n;
}
static inline CborError append_to_buffer(CborEncoder *encoder, const void *data, size_t len,
CborEncoderAppendType appendType)
{
if (CBOR_ENCODER_WRITER_CONTROL >= 0) {
if (encoder->flags & CborIteratorFlag_WriterFunction || CBOR_ENCODER_WRITER_CONTROL != 0) {
# ifdef CBOR_ENCODER_WRITE_FUNCTION
return CBOR_ENCODER_WRITE_FUNCTION(encoder->end, data, len, appendType);
# else
return encoder->data.writer(encoder->end, data, len, appendType);
# endif
}
}
#if CBOR_ENCODER_WRITER_CONTROL <= 0
if (would_overflow(encoder, len)) {
if (encoder->end != NULL) {
len -= encoder->end - encoder->data.ptr;
encoder->end = NULL;
encoder->data.bytes_needed = 0;
}
advance_ptr(encoder, len);
return CborErrorOutOfMemory;
}
memcpy(encoder->data.ptr, data, len);
encoder->data.ptr += len;
#endif
return CborNoError;
}
static inline CborError append_byte_to_buffer(CborEncoder *encoder, uint8_t byte)
{
return append_to_buffer(encoder, &byte, 1, CborEncoderAppendCborData);
}
static inline CborError encode_number_no_update(CborEncoder *encoder, uint64_t ui, uint8_t shiftedMajorType)
{
/* Little-endian would have been so much more convenient here:
* We could just write at the beginning of buf but append_to_buffer
* only the necessary bytes.
* Since it has to be big endian, do it the other way around:
* write from the end. */
uint64_t buf[2];
uint8_t *const bufend = (uint8_t *)buf + sizeof(buf);
uint8_t *bufstart = bufend - 1;
put64(buf + 1, ui); /* we probably have a bunch of zeros in the beginning */
if (ui < Value8Bit) {
*bufstart += shiftedMajorType;
} else {
uint8_t more = 0;
if (ui > 0xffU)
++more;
if (ui > 0xffffU)
++more;
if (ui > 0xffffffffU)
++more;
bufstart -= (size_t)1 << more;
*bufstart = shiftedMajorType + Value8Bit + more;
}
return append_to_buffer(encoder, bufstart, bufend - bufstart, CborEncoderAppendCborData);
}
static inline void saturated_decrement(CborEncoder *encoder)
{
if (encoder->remaining)
--encoder->remaining;
}
static inline CborError encode_number(CborEncoder *encoder, uint64_t ui, uint8_t shiftedMajorType)
{
saturated_decrement(encoder);
return encode_number_no_update(encoder, ui, shiftedMajorType);
}
/**
* Appends the unsigned 64-bit integer \a value to the CBOR stream provided by
* \a encoder.
*
* \sa cbor_encode_negative_int, cbor_encode_int
*/
CborError cbor_encode_uint(CborEncoder *encoder, uint64_t value)
{
return encode_number(encoder, value, UnsignedIntegerType << MajorTypeShift);
}
/**
* Appends the negative 64-bit integer whose absolute value is \a
* absolute_value to the CBOR stream provided by \a encoder.
*
* If the value \a absolute_value is zero, this function encodes -2^64.
*
* \sa cbor_encode_uint, cbor_encode_int
*/
CborError cbor_encode_negative_int(CborEncoder *encoder, uint64_t absolute_value)
{
return encode_number(encoder, absolute_value - 1, NegativeIntegerType << MajorTypeShift);
}
/**
* Appends the signed 64-bit integer \a value to the CBOR stream provided by
* \a encoder.
*
* \sa cbor_encode_negative_int, cbor_encode_uint
*/
CborError cbor_encode_int(CborEncoder *encoder, int64_t value)
{
/* adapted from code in RFC 7049 appendix C (pseudocode) */
uint64_t ui = value >> 63; /* extend sign to whole length */
uint8_t majorType = ui & 0x20; /* extract major type */
ui ^= value; /* complement negatives */
return encode_number(encoder, ui, majorType);
}
/**
* Appends the CBOR Simple Type of value \a value to the CBOR stream provided by
* \a encoder.
*
* This function may return error CborErrorIllegalSimpleType if the \a value
* variable contains a number that is not a valid simple type.
*/
CborError cbor_encode_simple_value(CborEncoder *encoder, uint8_t value)
{
#ifndef CBOR_ENCODER_NO_CHECK_USER
/* check if this is a valid simple type */
if (value >= HalfPrecisionFloat && value <= Break)
return CborErrorIllegalSimpleType;
#endif
return encode_number(encoder, value, SimpleTypesType << MajorTypeShift);
}
/**
* Appends the floating-point value of type \a fpType and pointed to by \a
* value to the CBOR stream provided by \a encoder. The value of \a fpType must
* be one of CborHalfFloatType, CborFloatType or CborDoubleType, otherwise the
* behavior of this function is undefined.
*
* This function is useful for code that needs to pass through floating point
* values but does not wish to have the actual floating-point code.
*
* \sa cbor_encode_half_float, cbor_encode_float_as_half_float, cbor_encode_float, cbor_encode_double
*/
CborError cbor_encode_floating_point(CborEncoder *encoder, CborType fpType, const void *value)
{
unsigned size;
uint8_t buf[1 + sizeof(uint64_t)];
cbor_assert(fpType == CborHalfFloatType || fpType == CborFloatType || fpType == CborDoubleType);
buf[0] = fpType;
size = 2U << (fpType - CborHalfFloatType);
if (size == 8)
put64(buf + 1, *(const uint64_t*)value);
else if (size == 4)
put32(buf + 1, *(const uint32_t*)value);
else
put16(buf + 1, *(const uint16_t*)value);
saturated_decrement(encoder);
return append_to_buffer(encoder, buf, size + 1, CborEncoderAppendCborData);
}
/**
* Appends the CBOR tag \a tag to the CBOR stream provided by \a encoder.
*
* \sa CborTag
*/
CborError cbor_encode_tag(CborEncoder *encoder, CborTag tag)
{
/* tags don't count towards the number of elements in an array or map */
return encode_number_no_update(encoder, tag, TagType << MajorTypeShift);
}
static CborError encode_string(CborEncoder *encoder, size_t length, uint8_t shiftedMajorType, const void *string)
{
CborError err = encode_number(encoder, length, shiftedMajorType);
if (err && !isOomError(err))
return err;
return append_to_buffer(encoder, string, length, CborEncoderAppendStringData);
}
/**
* \fn CborError cbor_encode_text_stringz(CborEncoder *encoder, const char *string)
*
* Appends the null-terminated text string \a string to the CBOR stream
* provided by \a encoder. CBOR requires that \a string be valid UTF-8, but
* TinyCBOR makes no verification of correctness. The terminating null is not
* included in the stream.
*
* \sa cbor_encode_text_string, cbor_encode_byte_string
*/
/**
* Appends the byte string \a string of length \a length to the CBOR stream
* provided by \a encoder. CBOR byte strings are arbitrary raw data.
*
* \sa cbor_encode_text_stringz, cbor_encode_text_string
*/
CborError cbor_encode_byte_string(CborEncoder *encoder, const uint8_t *string, size_t length)
{
return encode_string(encoder, length, ByteStringType << MajorTypeShift, string);
}
/**
* Appends the text string \a string of length \a length to the CBOR stream
* provided by \a encoder. CBOR requires that \a string be valid UTF-8, but
* TinyCBOR makes no verification of correctness.
*
* \sa CborError cbor_encode_text_stringz, cbor_encode_byte_string
*/
CborError cbor_encode_text_string(CborEncoder *encoder, const char *string, size_t length)
{
return encode_string(encoder, length, TextStringType << MajorTypeShift, string);
}
#ifdef __GNUC__
__attribute__((noinline))
#endif
static CborError create_container(CborEncoder *encoder, CborEncoder *container, size_t length, uint8_t shiftedMajorType)
{
CborError err;
container->data.ptr = encoder->data.ptr;
container->end = encoder->end;
saturated_decrement(encoder);
container->remaining = length + 1; /* overflow ok on CborIndefiniteLength */
cbor_static_assert((int)CborIteratorFlag_ContainerIsMap_ == (int)CborIteratorFlag_ContainerIsMap);
cbor_static_assert(((MapType << MajorTypeShift) & CborIteratorFlag_ContainerIsMap) == CborIteratorFlag_ContainerIsMap);
cbor_static_assert(((ArrayType << MajorTypeShift) & CborIteratorFlag_ContainerIsMap) == 0);
container->flags = shiftedMajorType & CborIteratorFlag_ContainerIsMap;
if (CBOR_ENCODER_WRITER_CONTROL == 0)
container->flags |= encoder->flags & CborIteratorFlag_WriterFunction;
if (length == CborIndefiniteLength) {
container->flags |= CborIteratorFlag_UnknownLength;
err = append_byte_to_buffer(container, shiftedMajorType + IndefiniteLength);
} else {
if (shiftedMajorType & CborIteratorFlag_ContainerIsMap)
container->remaining += length;
err = encode_number_no_update(container, length, shiftedMajorType);
}
return err;
}
/**
* Creates a CBOR array in the CBOR stream provided by \a parentEncoder and
* initializes \a arrayEncoder so that items can be added to the array using
* the CborEncoder functions. The array must be terminated by calling either
* cbor_encoder_close_container() or cbor_encoder_close_container_checked()
* with the same \a encoder and \a arrayEncoder parameters.
*
* The number of items inserted into the array must be exactly \a length items,
* otherwise the stream is invalid. If the number of items is not known when
* creating the array, the constant \ref CborIndefiniteLength may be passed as
* length instead, and an indefinite length array is created.
*
* \sa cbor_encoder_create_map
*/
CborError cbor_encoder_create_array(CborEncoder *parentEncoder, CborEncoder *arrayEncoder, size_t length)
{
return create_container(parentEncoder, arrayEncoder, length, ArrayType << MajorTypeShift);
}
/**
* Creates a CBOR map in the CBOR stream provided by \a parentEncoder and
* initializes \a mapEncoder so that items can be added to the map using
* the CborEncoder functions. The map must be terminated by calling either
* cbor_encoder_close_container() or cbor_encoder_close_container_checked()
* with the same \a encoder and \a mapEncoder parameters.
*
* The number of pair of items inserted into the map must be exactly \a length
* items, otherwise the stream is invalid. If the number is not known
* when creating the map, the constant \ref CborIndefiniteLength may be passed as
* length instead, and an indefinite length map is created.
*
* \b{Implementation limitation:} TinyCBOR cannot encode more than SIZE_MAX/2
* key-value pairs in the stream. If the length \a length is larger than this
* value (and is not \ref CborIndefiniteLength), this function returns error
* CborErrorDataTooLarge.
*
* \sa cbor_encoder_create_array
*/
CborError cbor_encoder_create_map(CborEncoder *parentEncoder, CborEncoder *mapEncoder, size_t length)
{
if (length != CborIndefiniteLength && length > SIZE_MAX / 2)
return CborErrorDataTooLarge;
return create_container(parentEncoder, mapEncoder, length, MapType << MajorTypeShift);
}
/**
* Closes the CBOR container (array or map) provided by \a containerEncoder and
* updates the CBOR stream provided by \a encoder. Both parameters must be the
* same as were passed to cbor_encoder_create_array() or
* cbor_encoder_create_map().
*
* Since version 0.5, this function verifies that the number of items (or pairs
* of items, in the case of a map) was correct. It is no longer necessary to call
* cbor_encoder_close_container_checked() instead.
*
* \sa cbor_encoder_create_array(), cbor_encoder_create_map()
*/
CborError cbor_encoder_close_container(CborEncoder *parentEncoder, const CborEncoder *containerEncoder)
{
// synchronise buffer state with that of the container
parentEncoder->end = containerEncoder->end;
parentEncoder->data = containerEncoder->data;
if (containerEncoder->flags & CborIteratorFlag_UnknownLength)
return append_byte_to_buffer(parentEncoder, BreakByte);
if (containerEncoder->remaining != 1)
return containerEncoder->remaining == 0 ? CborErrorTooManyItems : CborErrorTooFewItems;
if (!parentEncoder->end)
return CborErrorOutOfMemory; /* keep the state */
return CborNoError;
}
/**
* \fn CborError cbor_encode_boolean(CborEncoder *encoder, bool value)
*
* Appends the boolean value \a value to the CBOR stream provided by \a encoder.
*/
/**
* \fn CborError cbor_encode_null(CborEncoder *encoder)
*
* Appends the CBOR type representing a null value to the CBOR stream provided
* by \a encoder.
*
* \sa cbor_encode_undefined()
*/
/**
* \fn CborError cbor_encode_undefined(CborEncoder *encoder)
*
* Appends the CBOR type representing an undefined value to the CBOR stream
* provided by \a encoder.
*
* \sa cbor_encode_null()
*/
/**
* \fn CborError cbor_encode_half_float(CborEncoder *encoder, const void *value)
*
* Appends the IEEE 754 half-precision (16-bit) floating point value pointed to
* by \a value to the CBOR stream provided by \a encoder.
*
* \sa cbor_encode_floating_point(), cbor_encode_float(), cbor_encode_double()
*/
/**
* \fn CborError cbor_encode_float_as_half_float(CborEncoder *encoder, float value)
*
* Convert the IEEE 754 single-precision (32-bit) floating point value \a value
* to the IEEE 754 half-precision (16-bit) floating point value and append it
* to the CBOR stream provided by \a encoder.
* The \a value should be in the range of the IEEE 754 half-precision floating point type,
* INFINITY, -INFINITY, or NAN, otherwise the behavior of this function is undefined.
*
* \sa cbor_encode_floating_point(), cbor_encode_float(), cbor_encode_double()
*/
/**
* \fn CborError cbor_encode_float(CborEncoder *encoder, float value)
*
* Appends the IEEE 754 single-precision (32-bit) floating point value \a value
* to the CBOR stream provided by \a encoder.
*
* \sa cbor_encode_floating_point(), cbor_encode_half_float(), cbor_encode_float_as_half_float(), cbor_encode_double()
*/
/**
* \fn CborError cbor_encode_double(CborEncoder *encoder, double value)
*
* Appends the IEEE 754 double-precision (64-bit) floating point value \a value
* to the CBOR stream provided by \a encoder.
*
* \sa cbor_encode_floating_point(), cbor_encode_half_float(), cbor_encode_float_as_half_float(), cbor_encode_float()
*/
/**
* \fn size_t cbor_encoder_get_buffer_size(const CborEncoder *encoder, const uint8_t *buffer)
*
* Returns the total size of the buffer starting at \a buffer after the
* encoding finished without errors. The \a encoder and \a buffer arguments
* must be the same as supplied to cbor_encoder_init().
*
* If the encoding process had errors, the return value of this function is
* meaningless. If the only errors were CborErrorOutOfMemory, instead use
* cbor_encoder_get_extra_bytes_needed() to find out by how much to grow the
* buffer before encoding again.
*
* See \ref CborEncoding for an example of using this function.
*
* \sa cbor_encoder_init(), cbor_encoder_get_extra_bytes_needed(), CborEncoding
*/
/**
* \fn size_t cbor_encoder_get_extra_bytes_needed(const CborEncoder *encoder)
*
* Returns how many more bytes the original buffer supplied to
* cbor_encoder_init() needs to be extended by so that no CborErrorOutOfMemory
* condition will happen for the encoding. If the buffer was big enough, this
* function returns 0. The \a encoder must be the original argument as passed
* to cbor_encoder_init().
*
* This function is usually called after an encoding sequence ended with one or
* more CborErrorOutOfMemory errors, but no other error. If any other error
* happened, the return value of this function is meaningless.
*
* See \ref CborEncoding for an example of using this function.
*
* \sa cbor_encoder_init(), cbor_encoder_get_buffer_size(), CborEncoding
*/
/** @} */
@@ -0,0 +1,57 @@
/****************************************************************************
**
** Copyright (C) 2015 Intel Corporation
**
** Permission is hereby granted, free of charge, to any person obtaining a copy
** of this software and associated documentation files (the "Software"), to deal
** in the Software without restriction, including without limitation the rights
** to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
** copies of the Software, and to permit persons to whom the Software is
** furnished to do so, subject to the following conditions:
**
** The above copyright notice and this permission notice shall be included in
** all copies or substantial portions of the Software.
**
** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
** OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
** THE SOFTWARE.
**
****************************************************************************/
#define _BSD_SOURCE 1
#define _DEFAULT_SOURCE 1
#ifndef __STDC_LIMIT_MACROS
# define __STDC_LIMIT_MACROS 1
#endif
#include "cbor.h"
/**
* \addtogroup CborEncoding
* @{
*/
/**
* @deprecated
*
* Closes the CBOR container (array or map) provided by \a containerEncoder and
* updates the CBOR stream provided by \a encoder. Both parameters must be the
* same as were passed to cbor_encoder_create_array() or
* cbor_encoder_create_map().
*
* Prior to version 0.5, cbor_encoder_close_container() did not check the
* number of items added. Since that version, it does and now
* cbor_encoder_close_container_checked() is no longer needed.
*
* \sa cbor_encoder_create_array(), cbor_encoder_create_map()
*/
CborError cbor_encoder_close_container_checked(CborEncoder *encoder, const CborEncoder *containerEncoder)
{
return cbor_encoder_close_container(encoder, containerEncoder);
}
/** @} */
@@ -0,0 +1,188 @@
/****************************************************************************
**
** Copyright (C) 2021 Intel Corporation
**
** Permission is hereby granted, free of charge, to any person obtaining a copy
** of this software and associated documentation files (the "Software"), to deal
** in the Software without restriction, including without limitation the rights
** to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
** copies of the Software, and to permit persons to whom the Software is
** furnished to do so, subject to the following conditions:
**
** The above copyright notice and this permission notice shall be included in
** all copies or substantial portions of the Software.
**
** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
** OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
** THE SOFTWARE.
**
****************************************************************************/
#include "cbor.h"
#ifndef _
# define _(msg) msg
#endif
/**
* \enum CborError
* \ingroup CborGlobals
* The CborError enum contains the possible error values used by the CBOR encoder and decoder.
*
* TinyCBOR functions report success by returning CborNoError, or one error
* condition by returning one of the values below. One exception is the
* out-of-memory condition (CborErrorOutOfMemory), which the functions for \ref
* CborEncoding may report in bit-wise OR with other conditions.
*
* This technique allows code to determine whether the only error condition was
* a lack of buffer space, which may not be a fatal condition if the buffer can
* be resized. Additionally, the functions for \ref CborEncoding may continue
* to be used even after CborErrorOutOfMemory is returned, and instead they
* will simply calculate the extra space needed.
*
* \value CborNoError No error occurred
* \omitvalue CborUnknownError
* \value CborErrorUnknownLength Request for the length of an array, map or string whose length is not provided in the CBOR stream
* \value CborErrorAdvancePastEOF Not enough data in the stream to decode item (decoding would advance past end of stream)
* \value CborErrorIO An I/O error occurred, probably due to an out-of-memory situation
* \value CborErrorGarbageAtEnd Bytes exist past the end of the CBOR stream
* \value CborErrorUnexpectedEOF End of stream reached unexpectedly
* \value CborErrorUnexpectedBreak A CBOR break byte was found where not expected
* \value CborErrorUnknownType An unknown type (future extension to CBOR) was found in the stream
* \value CborErrorIllegalType An invalid type was found while parsing a chunked CBOR string
* \value CborErrorIllegalNumber An illegal initial byte (encoding unspecified additional information) was found
* \value CborErrorIllegalSimpleType An illegal encoding of a CBOR Simple Type of value less than 32 was found
* \omitvalue CborErrorUnknownSimpleType
* \omitvalue CborErrorUnknownTag
* \omitvalue CborErrorInappropriateTagForType
* \omitvalue CborErrorDuplicateObjectKeys
* \value CborErrorInvalidUtf8TextString Illegal UTF-8 encoding found while parsing CBOR Text String
* \value CborErrorTooManyItems Too many items were added to CBOR map or array of pre-determined length
* \value CborErrorTooFewItems Too few items were added to CBOR map or array of pre-determined length
* \value CborErrorDataTooLarge Data item size exceeds TinyCBOR's implementation limits
* \value CborErrorNestingTooDeep Data item nesting exceeds TinyCBOR's implementation limits
* \omitvalue CborErrorUnsupportedType
* \value CborErrorJsonObjectKeyIsAggregate Conversion to JSON failed because the key in a map is a CBOR map or array
* \value CborErrorJsonObjectKeyNotString Conversion to JSON failed because the key in a map is not a text string
* \value CborErrorOutOfMemory During CBOR encoding, the buffer provided is insufficient for encoding the data item;
* in other situations, TinyCBOR failed to allocate memory
* \value CborErrorInternalError An internal error occurred in TinyCBOR
*/
/**
* \ingroup CborGlobals
* Returns the error string corresponding to the CBOR error condition \a error.
*/
const char *cbor_error_string(CborError error)
{
switch (error) {
case CborNoError:
return "";
case CborUnknownError:
return _("unknown error");
case CborErrorOutOfMemory:
return _("out of memory/need more memory");
case CborErrorUnknownLength:
return _("unknown length (attempted to get the length of a map/array/string of indeterminate length");
case CborErrorAdvancePastEOF:
return _("attempted to advance past EOF");
case CborErrorIO:
return _("I/O error");
case CborErrorGarbageAtEnd:
return _("garbage after the end of the content");
case CborErrorUnexpectedEOF:
return _("unexpected end of data");
case CborErrorUnexpectedBreak:
return _("unexpected 'break' byte");
case CborErrorUnknownType:
return _("illegal byte (encodes future extension type)");
case CborErrorIllegalType:
return _("mismatched string type in chunked string");
case CborErrorIllegalNumber:
return _("illegal initial byte (encodes unspecified additional information)");
case CborErrorIllegalSimpleType:
return _("illegal encoding of simple type smaller than 32");
case CborErrorNoMoreStringChunks:
return _("no more byte or text strings available");
case CborErrorUnknownSimpleType:
return _("unknown simple type");
case CborErrorUnknownTag:
return _("unknown tag");
case CborErrorInappropriateTagForType:
return _("inappropriate tag for type");
case CborErrorDuplicateObjectKeys:
return _("duplicate keys in object");
case CborErrorInvalidUtf8TextString:
return _("invalid UTF-8 content in string");
case CborErrorExcludedType:
return _("excluded type found");
case CborErrorExcludedValue:
return _("excluded value found");
case CborErrorImproperValue:
case CborErrorOverlongEncoding:
return _("value encoded in non-canonical form");
case CborErrorMapKeyNotString:
case CborErrorJsonObjectKeyNotString:
return _("key in map is not a string");
case CborErrorMapNotSorted:
return _("map is not sorted");
case CborErrorMapKeysNotUnique:
return _("map keys are not unique");
case CborErrorTooManyItems:
return _("too many items added to encoder");
case CborErrorTooFewItems:
return _("too few items added to encoder");
case CborErrorDataTooLarge:
return _("internal error: data too large");
case CborErrorNestingTooDeep:
return _("internal error: too many nested containers found in recursive function");
case CborErrorUnsupportedType:
return _("unsupported type");
case CborErrorUnimplementedValidation:
return _("validation not implemented for the current parser state");
case CborErrorJsonObjectKeyIsAggregate:
return _("conversion to JSON failed: key in object is an array or map");
case CborErrorJsonNotImplemented:
return _("conversion to JSON failed: open_memstream unavailable");
case CborErrorInternalError:
return _("internal error");
}
return cbor_error_string(CborUnknownError);
}
@@ -0,0 +1,316 @@
/****************************************************************************
**
** Copyright (C) 2021 Intel Corporation
**
** Permission is hereby granted, free of charge, to any person obtaining a copy
** of this software and associated documentation files (the "Software"), to deal
** in the Software without restriction, including without limitation the rights
** to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
** copies of the Software, and to permit persons to whom the Software is
** furnished to do so, subject to the following conditions:
**
** The above copyright notice and this permission notice shall be included in
** all copies or substantial portions of the Software.
**
** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
** OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
** THE SOFTWARE.
**
****************************************************************************/
#ifndef CBORINTERNAL_P_H
#define CBORINTERNAL_P_H
#include "compilersupport_p.h"
#ifndef CBOR_NO_FLOATING_POINT
# include <float.h>
# include <math.h>
#else
# ifndef CBOR_NO_HALF_FLOAT_TYPE
# define CBOR_NO_HALF_FLOAT_TYPE 1
# endif
#endif
#ifndef CBOR_NO_HALF_FLOAT_TYPE
# if defined(__F16C__) || defined(__AVX2__)
# include <immintrin.h>
static inline unsigned short encode_half(float val)
{
__m128i m = _mm_cvtps_ph(_mm_set_ss(val), _MM_FROUND_CUR_DIRECTION);
return _mm_extract_epi16(m, 0);
}
static inline float decode_half(unsigned short half)
{
__m128i m = _mm_cvtsi32_si128(half);
return _mm_cvtss_f32(_mm_cvtph_ps(m));
}
# else
/* software implementation of float-to-fp16 conversions */
static inline unsigned short encode_half(double val)
{
uint64_t v;
int sign, exp, mant;
memcpy(&v, &val, sizeof(v));
sign = v >> 63 << 15;
exp = (v >> 52) & 0x7ff;
mant = v << 12 >> 12 >> (53-11); /* keep only the 11 most significant bits of the mantissa */
exp -= 1023;
if (exp == 1024) {
/* infinity or NaN */
exp = 16;
mant >>= 1;
} else if (exp >= 16) {
/* overflow, as largest number */
exp = 15;
mant = 1023;
} else if (exp >= -14) {
/* regular normal */
} else if (exp >= -24) {
/* subnormal */
mant |= 1024;
mant >>= -(exp + 14);
exp = -15;
} else {
/* underflow, make zero */
return 0;
}
/* safe cast here as bit operations above guarantee not to overflow */
return (unsigned short)(sign | ((exp + 15) << 10) | mant);
}
/* this function was copied & adapted from RFC 7049 Appendix D */
static inline double decode_half(unsigned short half)
{
int exp = (half >> 10) & 0x1f;
int mant = half & 0x3ff;
double val;
if (exp == 0) val = ldexp(mant, -24);
else if (exp != 31) val = ldexp(mant + 1024, exp - 25);
else val = mant == 0 ? INFINITY : NAN;
return half & 0x8000 ? -val : val;
}
# endif
#endif /* CBOR_NO_HALF_FLOAT_TYPE */
#ifndef CBOR_INTERNAL_API
# define CBOR_INTERNAL_API
#endif
#ifndef CBOR_PARSER_MAX_RECURSIONS
# define CBOR_PARSER_MAX_RECURSIONS 1024
#endif
#ifndef CBOR_ENCODER_WRITER_CONTROL
# define CBOR_ENCODER_WRITER_CONTROL 0
#endif
#ifndef CBOR_PARSER_READER_CONTROL
# define CBOR_PARSER_READER_CONTROL 0
#endif
/*
* CBOR Major types
* Encoded in the high 3 bits of the descriptor byte
* See http://tools.ietf.org/html/rfc7049#section-2.1
*/
typedef enum CborMajorTypes {
UnsignedIntegerType = 0U,
NegativeIntegerType = 1U,
ByteStringType = 2U,
TextStringType = 3U,
ArrayType = 4U,
MapType = 5U, /* a.k.a. object */
TagType = 6U,
SimpleTypesType = 7U
} CborMajorTypes;
/*
* CBOR simple and floating point types
* Encoded in the low 8 bits of the descriptor byte when the
* Major Type is 7.
*/
typedef enum CborSimpleTypes {
FalseValue = 20,
TrueValue = 21,
NullValue = 22,
UndefinedValue = 23,
SimpleTypeInNextByte = 24, /* not really a simple type */
HalfPrecisionFloat = 25, /* ditto */
SinglePrecisionFloat = 26, /* ditto */
DoublePrecisionFloat = 27, /* ditto */
Break = 31
} CborSimpleTypes;
enum {
SmallValueBitLength = 5U,
SmallValueMask = (1U << SmallValueBitLength) - 1, /* 31 */
Value8Bit = 24U,
Value16Bit = 25U,
Value32Bit = 26U,
Value64Bit = 27U,
IndefiniteLength = 31U,
MajorTypeShift = SmallValueBitLength,
MajorTypeMask = (int) (~0U << MajorTypeShift),
BreakByte = (unsigned)Break | (SimpleTypesType << MajorTypeShift)
};
static inline void copy_current_position(CborValue *dst, const CborValue *src)
{
/* This "if" is here for pedantry only: the two branches should perform
* the same memory operation. */
if (src->parser->flags & CborParserFlag_ExternalSource)
dst->source.token = src->source.token;
else
dst->source.ptr = src->source.ptr;
}
static inline bool can_read_bytes(const CborValue *it, size_t n)
{
if (CBOR_PARSER_READER_CONTROL >= 0) {
if (it->parser->flags & CborParserFlag_ExternalSource || CBOR_PARSER_READER_CONTROL != 0) {
#ifdef CBOR_PARSER_CAN_READ_BYTES_FUNCTION
return CBOR_PARSER_CAN_READ_BYTES_FUNCTION(it->source.token, n);
#else
return it->parser->source.ops->can_read_bytes(it->source.token, n);
#endif
}
}
/* Convert the pointer subtraction to size_t since end >= ptr
* (this prevents issues with (ptrdiff_t)n becoming negative).
*/
return (size_t)(it->parser->source.end - it->source.ptr) >= n;
}
static inline void advance_bytes(CborValue *it, size_t n)
{
if (CBOR_PARSER_READER_CONTROL >= 0) {
if (it->parser->flags & CborParserFlag_ExternalSource || CBOR_PARSER_READER_CONTROL != 0) {
#ifdef CBOR_PARSER_ADVANCE_BYTES_FUNCTION
CBOR_PARSER_ADVANCE_BYTES_FUNCTION(it->source.token, n);
#else
it->parser->source.ops->advance_bytes(it->source.token, n);
#endif
return;
}
}
it->source.ptr += n;
}
static inline CborError transfer_string(CborValue *it, const void **ptr, size_t offset, size_t len)
{
if (CBOR_PARSER_READER_CONTROL >= 0) {
if (it->parser->flags & CborParserFlag_ExternalSource || CBOR_PARSER_READER_CONTROL != 0) {
#ifdef CBOR_PARSER_TRANSFER_STRING_FUNCTION
return CBOR_PARSER_TRANSFER_STRING_FUNCTION(it->source.token, ptr, offset, len);
#else
return it->parser->source.ops->transfer_string(it->source.token, ptr, offset, len);
#endif
}
}
it->source.ptr += offset;
if (can_read_bytes(it, len)) {
*CONST_CAST(const void **, ptr) = it->source.ptr;
it->source.ptr += len;
return CborNoError;
}
return CborErrorUnexpectedEOF;
}
static inline void *read_bytes_unchecked(const CborValue *it, void *dst, size_t offset, size_t n)
{
if (CBOR_PARSER_READER_CONTROL >= 0) {
if (it->parser->flags & CborParserFlag_ExternalSource || CBOR_PARSER_READER_CONTROL != 0) {
#ifdef CBOR_PARSER_READ_BYTES_FUNCTION
return CBOR_PARSER_READ_BYTES_FUNCTION(it->source.token, dst, offset, n);
#else
return it->parser->source.ops->read_bytes(it->source.token, dst, offset, n);
#endif
}
}
return memcpy(dst, it->source.ptr + offset, n);
}
#ifdef __GNUC__
__attribute__((warn_unused_result))
#endif
static inline void *read_bytes(const CborValue *it, void *dst, size_t offset, size_t n)
{
if (can_read_bytes(it, offset + n))
return read_bytes_unchecked(it, dst, offset, n);
return NULL;
}
static inline uint16_t read_uint8(const CborValue *it, size_t offset)
{
uint8_t result;
read_bytes_unchecked(it, &result, offset, sizeof(result));
return result;
}
static inline uint16_t read_uint16(const CborValue *it, size_t offset)
{
uint16_t result;
read_bytes_unchecked(it, &result, offset, sizeof(result));
return cbor_ntohs(result);
}
static inline uint32_t read_uint32(const CborValue *it, size_t offset)
{
uint32_t result;
read_bytes_unchecked(it, &result, offset, sizeof(result));
return cbor_ntohl(result);
}
static inline uint64_t read_uint64(const CborValue *it, size_t offset)
{
uint64_t result;
read_bytes_unchecked(it, &result, offset, sizeof(result));
return cbor_ntohll(result);
}
static inline CborError extract_number_checked(const CborValue *it, uint64_t *value, size_t *bytesUsed)
{
uint8_t descriptor;
size_t bytesNeeded = 0;
/* We've already verified that there's at least one byte to be read */
read_bytes_unchecked(it, &descriptor, 0, 1);
descriptor &= SmallValueMask;
if (descriptor < Value8Bit) {
*value = descriptor;
} else if (unlikely(descriptor > Value64Bit)) {
return CborErrorIllegalNumber;
} else {
bytesNeeded = (size_t)(1 << (descriptor - Value8Bit));
if (!can_read_bytes(it, 1 + bytesNeeded))
return CborErrorUnexpectedEOF;
if (descriptor <= Value16Bit) {
if (descriptor == Value16Bit)
*value = read_uint16(it, 1);
else
*value = read_uint8(it, 1);
} else {
if (descriptor == Value32Bit)
*value = read_uint32(it, 1);
else
*value = read_uint64(it, 1);
}
}
if (bytesUsed)
*bytesUsed = bytesNeeded;
return CborNoError;
}
#endif /* CBORINTERNAL_P_H */
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,119 @@
/****************************************************************************
**
** Copyright (C) 2016 Intel Corporation
**
** Permission is hereby granted, free of charge, to any person obtaining a copy
** of this software and associated documentation files (the "Software"), to deal
** in the Software without restriction, including without limitation the rights
** to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
** copies of the Software, and to permit persons to whom the Software is
** furnished to do so, subject to the following conditions:
**
** The above copyright notice and this permission notice shall be included in
** all copies or substantial portions of the Software.
**
** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
** OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
** THE SOFTWARE.
**
****************************************************************************/
#ifndef _BSD_SOURCE
#define _BSD_SOURCE 1
#endif
#ifndef _DEFAULT_SOURCE
#define _DEFAULT_SOURCE 1
#endif
#ifndef __STDC_LIMIT_MACROS
# define __STDC_LIMIT_MACROS 1
#endif
#include "cbor.h"
#include "compilersupport_p.h"
#include <stdlib.h>
/**
* \fn CborError cbor_value_dup_text_string(const CborValue *value, char **buffer, size_t *buflen, CborValue *next)
*
* Allocates memory for the string pointed by \a value and copies it into this
* buffer. The pointer to the buffer is stored in \a buffer and the number of
* bytes copied is stored in \a buflen (those variables must not be NULL).
*
* If the iterator \a value does not point to a text string, the behaviour is
* undefined, so checking with \ref cbor_value_get_type or \ref
* cbor_value_is_text_string is recommended.
*
* If \c malloc returns a NULL pointer, this function will return error
* condition \ref CborErrorOutOfMemory.
*
* On success, \c{*buffer} will contain a valid pointer that must be freed by
* calling \c{free()}. This is the case even for zero-length strings.
*
* The \a next pointer, if not null, will be updated to point to the next item
* after this string. If \a value points to the last item, then \a next will be
* invalid.
*
* This function may not run in constant time (it will run in O(n) time on the
* number of chunks). It requires constant memory (O(1)) in addition to the
* malloc'ed block.
*
* \note This function does not perform UTF-8 validation on the incoming text
* string.
*
* \sa cbor_value_get_text_string_chunk(), cbor_value_copy_text_string(), cbor_value_dup_byte_string()
*/
/**
* \fn CborError cbor_value_dup_byte_string(const CborValue *value, uint8_t **buffer, size_t *buflen, CborValue *next)
*
* Allocates memory for the string pointed by \a value and copies it into this
* buffer. The pointer to the buffer is stored in \a buffer and the number of
* bytes copied is stored in \a buflen (those variables must not be NULL).
*
* If the iterator \a value does not point to a byte string, the behaviour is
* undefined, so checking with \ref cbor_value_get_type or \ref
* cbor_value_is_byte_string is recommended.
*
* If \c malloc returns a NULL pointer, this function will return error
* condition \ref CborErrorOutOfMemory.
*
* On success, \c{*buffer} will contain a valid pointer that must be freed by
* calling \c{free()}. This is the case even for zero-length strings.
*
* The \a next pointer, if not null, will be updated to point to the next item
* after this string. If \a value points to the last item, then \a next will be
* invalid.
*
* This function may not run in constant time (it will run in O(n) time on the
* number of chunks). It requires constant memory (O(1)) in addition to the
* malloc'ed block.
*
* \sa cbor_value_get_text_string_chunk(), cbor_value_copy_byte_string(), cbor_value_dup_text_string()
*/
CborError _cbor_value_dup_string(const CborValue *value, void **buffer, size_t *buflen, CborValue *next)
{
CborError err;
cbor_assert(buffer);
cbor_assert(buflen);
*buflen = SIZE_MAX;
err = _cbor_value_copy_string(value, NULL, buflen, NULL);
if (err)
return err;
++*buflen;
*buffer = malloc(*buflen);
if (!*buffer) {
/* out of memory */
return CborErrorOutOfMemory;
}
err = _cbor_value_copy_string(value, *buffer, buflen, next);
if (err) {
free(*buffer);
return err;
}
return CborNoError;
}
@@ -0,0 +1,205 @@
/****************************************************************************
**
** Copyright (C) 2017 Intel Corporation
**
** Permission is hereby granted, free of charge, to any person obtaining a copy
** of this software and associated documentation files (the "Software"), to deal
** in the Software without restriction, including without limitation the rights
** to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
** copies of the Software, and to permit persons to whom the Software is
** furnished to do so, subject to the following conditions:
**
** The above copyright notice and this permission notice shall be included in
** all copies or substantial portions of the Software.
**
** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
** OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
** THE SOFTWARE.
**
****************************************************************************/
#ifndef COMPILERSUPPORT_H
#define COMPILERSUPPORT_H
#include "cbor.h"
#ifndef _BSD_SOURCE
# define _BSD_SOURCE
#endif
#ifndef _DEFAULT_SOURCE
# define _DEFAULT_SOURCE
#endif
#ifndef assert
# include <assert.h>
#endif
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#ifndef __cplusplus
# include <stdbool.h>
#endif
#if __STDC_VERSION__ >= 201112L || (defined(__cplusplus) && __cplusplus >= 201103L) || (defined(__cpp_static_assert) && __cpp_static_assert >= 200410)
# define cbor_static_assert(x) static_assert(x, #x)
#elif !defined(__cplusplus) && defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 406) && (__STDC_VERSION__ > 199901L)
# define cbor_static_assert(x) _Static_assert(x, #x)
#else
# define cbor_static_assert(x) ((void)sizeof(char[2*!!(x) - 1]))
#endif
#if __STDC_VERSION__ >= 199901L || defined(__cplusplus)
/* inline is a keyword */
#else
/* use the definition from cbor.h */
# define inline CBOR_INLINE
#endif
#ifdef NDEBUG
# define cbor_assert(cond) do { if (!(cond)) unreachable(); } while (0)
#else
# define cbor_assert(cond) assert(cond)
#endif
#ifndef STRINGIFY
#define STRINGIFY(x) STRINGIFY2(x)
#endif
#define STRINGIFY2(x) #x
#if !defined(UINT32_MAX) || !defined(INT64_MAX)
/* C89? We can define UINT32_MAX portably, but not INT64_MAX */
# error "Your system has stdint.h but that doesn't define UINT32_MAX or INT64_MAX"
#endif
#ifndef DBL_DECIMAL_DIG
/* DBL_DECIMAL_DIG is C11 */
# define DBL_DECIMAL_DIG 17
#endif
#define DBL_DECIMAL_DIG_STR STRINGIFY(DBL_DECIMAL_DIG)
#if defined(__GNUC__) && defined(__i386__) && !defined(__iamcu__)
# define CBOR_INTERNAL_API_CC __attribute__((regparm(3)))
#elif defined(_MSC_VER) && defined(_M_IX86)
# define CBOR_INTERNAL_API_CC __fastcall
#else
# define CBOR_INTERNAL_API_CC
#endif
#ifndef __has_builtin
# define __has_builtin(x) 0
#endif
#if (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) || \
(__has_builtin(__builtin_bswap64) && __has_builtin(__builtin_bswap32))
# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
# define cbor_ntohll __builtin_bswap64
# define cbor_htonll __builtin_bswap64
# define cbor_ntohl __builtin_bswap32
# define cbor_htonl __builtin_bswap32
# ifdef __INTEL_COMPILER
# define cbor_ntohs _bswap16
# define cbor_htons _bswap16
# elif (__GNUC__ * 100 + __GNUC_MINOR__ >= 608) || __has_builtin(__builtin_bswap16)
# define cbor_ntohs __builtin_bswap16
# define cbor_htons __builtin_bswap16
# else
# define cbor_ntohs(x) (((uint16_t)(x) >> 8) | ((uint16_t)(x) << 8))
# define cbor_htons cbor_ntohs
# endif
# else
# define cbor_ntohll
# define cbor_htonll
# define cbor_ntohl
# define cbor_htonl
# define cbor_ntohs
# define cbor_htons
# endif
#elif defined(__sun)
# include <sys/byteorder.h>
#elif defined(_MSC_VER)
/* MSVC, which implies Windows, which implies little-endian and sizeof(long) == 4 */
# include <stdlib.h>
# define cbor_ntohll _byteswap_uint64
# define cbor_htonll _byteswap_uint64
# define cbor_ntohl _byteswap_ulong
# define cbor_htonl _byteswap_ulong
# define cbor_ntohs _byteswap_ushort
# define cbor_htons _byteswap_ushort
#endif
#ifndef cbor_ntohs
# include <arpa/inet.h>
# define cbor_ntohs ntohs
# define cbor_htons htons
#endif
#ifndef cbor_ntohl
# include <arpa/inet.h>
# define cbor_ntohl ntohl
# define cbor_htonl htonl
#endif
#ifndef cbor_ntohll
# define cbor_ntohll ntohll
# define cbor_htonll htonll
/* ntohll isn't usually defined */
# ifndef ntohll
# if (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || \
(defined(__BYTE_ORDER) && defined(__BIG_ENDIAN) && __BYTE_ORDER == __BIG_ENDIAN) || \
(defined(BYTE_ORDER) && defined(BIG_ENDIAN) && BYTE_ORDER == BIG_ENDIAN) || \
(defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || (defined(__BIG_ENDIAN__) && !defined(__LITTLE_ENDIAN__)) || \
defined(__ARMEB__) || defined(__MIPSEB__) || defined(__s390__) || defined(__sparc__)
# define ntohll
# define htonll
# elif (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) || \
(defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && __BYTE_ORDER == __LITTLE_ENDIAN) || \
(defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && BYTE_ORDER == LITTLE_ENDIAN) || \
defined(_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN__) || defined(__ARMEL__) || defined(__MIPSEL__) || \
defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64)
# define ntohll(x) ((ntohl((uint32_t)(x)) * UINT64_C(0x100000000)) + (ntohl((x) >> 32)))
# define htonll ntohll
# else
# error "Unable to determine byte order!"
# endif
# endif
#endif
#ifdef __cplusplus
# define CONST_CAST(t, v) const_cast<t>(v)
#else
/* C-style const_cast without triggering a warning with -Wcast-qual */
# define CONST_CAST(t, v) (t)(uintptr_t)(v)
#endif
#ifdef __GNUC__
#ifndef likely
# define likely(x) __builtin_expect(!!(x), 1)
#endif
#ifndef unlikely
# define unlikely(x) __builtin_expect(!!(x), 0)
#endif
# define unreachable() __builtin_unreachable()
#elif defined(_MSC_VER)
# define likely(x) (x)
# define unlikely(x) (x)
# define unreachable() __assume(0)
#else
# define likely(x) (x)
# define unlikely(x) (x)
# define unreachable() do {} while (0)
#endif
static inline bool add_check_overflow(size_t v1, size_t v2, size_t *r)
{
#if ((defined(__GNUC__) && (__GNUC__ >= 5)) && !defined(__INTEL_COMPILER)) || __has_builtin(__builtin_add_overflow)
return __builtin_add_overflow(v1, v2, r);
#else
/* unsigned additions are well-defined */
*r = v1 + v2;
return v1 > v1 + v2;
#endif
}
#endif /* COMPILERSUPPORT_H */
@@ -0,0 +1,3 @@
#define TINYCBOR_VERSION_MAJOR 0
#define TINYCBOR_VERSION_MINOR 6
#define TINYCBOR_VERSION_PATCH 0
@@ -0,0 +1,104 @@
/****************************************************************************
**
** Copyright (C) 2017 Intel Corporation
**
** Permission is hereby granted, free of charge, to any person obtaining a copy
** of this software and associated documentation files (the "Software"), to deal
** in the Software without restriction, including without limitation the rights
** to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
** copies of the Software, and to permit persons to whom the Software is
** furnished to do so, subject to the following conditions:
**
** The above copyright notice and this permission notice shall be included in
** all copies or substantial portions of the Software.
**
** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
** OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
** THE SOFTWARE.
**
****************************************************************************/
#ifndef CBOR_UTF8_H
#define CBOR_UTF8_H
#include "compilersupport_p.h"
#include <stdint.h>
static inline uint32_t get_utf8(const uint8_t **buffer, const uint8_t *end)
{
int charsNeeded;
uint32_t uc, min_uc;
uint8_t b;
ptrdiff_t n = end - *buffer;
if (n == 0)
return ~0U;
uc = *(*buffer)++;
if (uc < 0x80) {
/* single-byte UTF-8 */
return uc;
}
/* multi-byte UTF-8, decode it */
if (unlikely(uc <= 0xC1))
return ~0U;
if (uc < 0xE0) {
/* two-byte UTF-8 */
charsNeeded = 2;
min_uc = 0x80;
uc &= 0x1f;
} else if (uc < 0xF0) {
/* three-byte UTF-8 */
charsNeeded = 3;
min_uc = 0x800;
uc &= 0x0f;
} else if (uc < 0xF5) {
/* four-byte UTF-8 */
charsNeeded = 4;
min_uc = 0x10000;
uc &= 0x07;
} else {
return ~0U;
}
if (n < charsNeeded)
return ~0U;
/* first continuation character */
b = *(*buffer)++;
if ((b & 0xc0) != 0x80)
return ~0U;
uc <<= 6;
uc |= b & 0x3f;
if (charsNeeded > 2) {
/* second continuation character */
b = *(*buffer)++;
if ((b & 0xc0) != 0x80)
return ~0U;
uc <<= 6;
uc |= b & 0x3f;
if (charsNeeded > 3) {
/* third continuation character */
b = *(*buffer)++;
if ((b & 0xc0) != 0x80)
return ~0U;
uc <<= 6;
uc |= b & 0x3f;
}
}
/* overlong sequence? surrogate pair? out or range? */
if (uc < min_uc || uc - 0xd800U < 2048U || uc > 0x10ffff)
return ~0U;
return uc;
}
#endif /* CBOR_UTF8_H */
@@ -0,0 +1,87 @@
# Shared CMake logic for nim-ffi generated bindings. Builds the Nim library as
# a shared object and the vendored TinyCBOR as a static library, and exposes
# them as the imported target `${NIM_FFI_LIB}` (+ `${NIM_FFI_LIB}_nim_lib`) and
# the `tinycbor` target. Included by the per-language generated CMakeLists,
# which set REPO_ROOT, NIM_FFI_LIB (library name) and NIM_FFI_SRC (path to the
# .nim root, relative to the including CMakeLists) before including this file.
get_filename_component(NIM_SRC
"${CMAKE_CURRENT_SOURCE_DIR}/${NIM_FFI_SRC}"
ABSOLUTE)
find_program(NIM_EXECUTABLE nim REQUIRED)
if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
set(NIM_LIB_FILE "${REPO_ROOT}/lib${NIM_FFI_LIB}.dylib")
elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
set(NIM_LIB_FILE "${REPO_ROOT}/${NIM_FFI_LIB}.dll")
set(NIM_IMPLIB_FILE "${REPO_ROOT}/${NIM_FFI_LIB}.lib")
else()
set(NIM_LIB_FILE "${REPO_ROOT}/lib${NIM_FFI_LIB}.so")
endif()
# On Windows the default Nim toolchain (mingw gcc) doesn't emit an import
# library unless told to; without it MSVC consumers can't resolve any exported
# symbol at link time.
set(NIM_IMPLIB_PASSL "")
if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
set(NIM_IMPLIB_PASSL "--passL:-Wl,--out-implib,${NIM_IMPLIB_FILE}")
endif()
add_custom_command(
OUTPUT "${NIM_LIB_FILE}"
COMMAND "${NIM_EXECUTABLE}" c
--mm:orc
-d:chronicles_log_level=WARN
--app:lib
--noMain
"--nimMainPrefix:lib${NIM_FFI_LIB}"
${NIM_IMPLIB_PASSL}
"-o:${NIM_LIB_FILE}"
"${NIM_SRC}"
WORKING_DIRECTORY "${REPO_ROOT}"
DEPENDS "${NIM_SRC}"
BYPRODUCTS "${NIM_IMPLIB_FILE}"
COMMENT "Compiling Nim library lib${NIM_FFI_LIB}"
VERBATIM
)
add_custom_target(${NIM_FFI_LIB}_nim_lib ALL DEPENDS "${NIM_LIB_FILE}")
# On Windows an IMPORTED SHARED target needs IMPORTED_IMPLIB, but the Visual
# Studio multi-config generator did not pick it up and emitted
# `${NIM_FFI_LIB}-NOTFOUND.obj`. Side-step the IMPORTED machinery there by
# exposing the import library through a plain INTERFACE library.
if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
add_library(${NIM_FFI_LIB} INTERFACE)
target_link_libraries(${NIM_FFI_LIB} INTERFACE "${NIM_IMPLIB_FILE}")
else()
add_library(${NIM_FFI_LIB} SHARED IMPORTED GLOBAL)
set_target_properties(${NIM_FFI_LIB} PROPERTIES IMPORTED_LOCATION "${NIM_LIB_FILE}")
endif()
add_dependencies(${NIM_FFI_LIB} ${NIM_FFI_LIB}_nim_lib)
# Absolute path to the runtime library (DLL/dylib/so). Exposed via the cache so
# consumers in other directories can stage the DLL next to their executable on
# Windows.
set(${NIM_FFI_LIB}_RUNTIME_LIB "${NIM_LIB_FILE}" CACHE INTERNAL
"Absolute path to the ${NIM_FFI_LIB} runtime library")
# ── TinyCBOR (vendored at ffi/codegen/templates/cpp/vendor/tinycbor) ─────────
# The C and C++ backends share one vendored TinyCBOR copy. Guarded so two
# sibling bindings dirs in one parent project don't redefine the target.
set(TINYCBOR_SRC_DIR "${REPO_ROOT}/ffi/codegen/templates/cpp/vendor")
if(NOT TARGET tinycbor)
add_library(tinycbor STATIC
"${TINYCBOR_SRC_DIR}/tinycbor/cborencoder.c"
"${TINYCBOR_SRC_DIR}/tinycbor/cborencoder_close_container_checked.c"
"${TINYCBOR_SRC_DIR}/tinycbor/cborparser.c"
"${TINYCBOR_SRC_DIR}/tinycbor/cborparser_dup_string.c"
"${TINYCBOR_SRC_DIR}/tinycbor/cborerrorstrings.c"
)
target_include_directories(tinycbor PUBLIC
"${TINYCBOR_SRC_DIR}" # consumer uses #include <tinycbor/cbor.h>
"${TINYCBOR_SRC_DIR}/tinycbor" # internal _p.h includes resolve here
)
set_property(TARGET tinycbor PROPERTY C_STANDARD 99)
set_property(TARGET tinycbor PROPERTY POSITION_INDEPENDENT_CODE ON)
endif()
+129
View File
@@ -0,0 +1,129 @@
## Structured type model shared by the C / C++ / Rust binding generators:
## `parseFFIType` parses a Nim type string, `renderNative` walks it per backend.
import std/[strutils, options]
type
ScalarKind* {.pure.} = enum
skBool
skI8
skI16
skI32
skI64
skU8
skU16
skU32
skU64
skF32
skF64
FFITypeKind* {.pure.} = enum
ftScalar
ftStr
ftBytes
ftSeq
ftOpt
ftPtr
ftStruct
FFIType* = ref object
case kind*: FFITypeKind
of ftScalar:
scalar*: ScalarKind
of ftSeq, ftOpt:
elem*: FFIType
of ftStruct:
name*: string
else:
discard
NativeTypeMap* = object
## Per-backend type names; `structName` nil ⇒ user type name passes through.
scalar*: proc(s: ScalarKind): string {.noSideEffect, nimcall.}
str*: string
bytes*: string
ptrType*: string
seqOf*: proc(elem: string): string {.noSideEffect, nimcall.}
optOf*: proc(elem: string): string {.noSideEffect, nimcall.}
structName*: proc(name: string): string {.noSideEffect, nimcall.}
func genericInnerType*(typeName, prefix: string): string =
## Inner type of `Prefix[Inner]`, e.g. ("seq[int]", "seq[") → "int"; "" if no match.
if typeName.startsWith(prefix) and typeName.endsWith("]"):
return typeName[prefix.len .. ^2]
return ""
func scalarKind(t: string): Option[ScalarKind] =
case t
of "bool":
some(skBool)
of "int8":
some(skI8)
of "int16":
some(skI16)
of "int32":
some(skI32)
of "int", "int64":
some(skI64)
of "uint8", "byte":
some(skU8)
of "uint16":
some(skU16)
of "uint32":
some(skU32)
of "uint", "uint64":
some(skU64)
of "float32":
some(skF32)
of "float", "float64":
some(skF64)
else:
none(ScalarKind)
func parseFFIType*(typeName: string): FFIType =
## Nim type string → shared `FFIType`: ptr/pointer, seq[byte]→bytes, seq/Option/Maybe,
## scalars, string, else struct.
let t = typeName.strip()
if t.startsWith("ptr ") or t == "pointer":
return FFIType(kind: ftPtr)
let seqInner = genericInnerType(t, "seq[")
if seqInner.len > 0:
let inner = seqInner.strip()
if inner == "byte" or inner == "uint8":
return FFIType(kind: ftBytes)
return FFIType(kind: ftSeq, elem: parseFFIType(inner))
var optInner = genericInnerType(t, "Option[")
if optInner.len == 0:
optInner = genericInnerType(t, "Maybe[")
if optInner.len > 0:
return FFIType(kind: ftOpt, elem: parseFFIType(optInner.strip()))
let sc = scalarKind(t)
if sc.isSome():
return FFIType(kind: ftScalar, scalar: sc.get())
if t == "string" or t == "cstring":
return FFIType(kind: ftStr)
FFIType(kind: ftStruct, name: t)
func renderNative*(m: NativeTypeMap, t: FFIType): string =
## Recursively walks `t` into a native type string for backend `m`.
case t.kind
of ftScalar:
m.scalar(t.scalar)
of ftStr:
m.str
of ftBytes:
m.bytes
of ftPtr:
m.ptrType
of ftSeq:
m.seqOf(renderNative(m, t.elem))
of ftOpt:
m.optOf(renderNative(m, t.elem))
of ftStruct:
if m.structName.isNil():
t.name
else:
m.structName(t.name)
+134
View File
@@ -0,0 +1,134 @@
## Event-thread body and FFI-thread liveness monitoring. Included from
## `ffi_context.nim`. Drains queued events into listeners and emits
## NotResponding/Responding on FFI-heartbeat stall/recovery.
type
NotRespondingEvent* = object
RespondingEvent* = object
const
NotRespondingEventName* = "not_responding"
RespondingEventName* = "responding"
proc dispatchToListeners[T](
ctx: ptr FFIContext[T], eventName: string, data: pointer, dataLen: int
) =
## Holds reg.lock across snapshot + invocation so concurrent add/remove blocks
## until dispatch returns.
withLock ctx[].eventRegistry.lock:
let listeners = ctx[].eventRegistry.byEvent.getOrDefault(eventName)
if listeners.len == 0:
chronicles.debug "no listener registered", event = eventName
return
foreignThreadGc:
try:
notifyListeners(listeners, RET_OK, data, dataLen)
except Exception, CatchableError:
notifyListenersErr(
listeners,
"Exception dispatching " & eventName & ": " & getCurrentExceptionMsg(),
)
proc emitLivenessEvent[T, P](ctx: ptr FFIContext[T], name: string, payload: P) =
## Dispatches directly to listeners, bypassing the (possibly wedged) queue.
let event =
try:
EventEnvelope[P](eventType: name, payload: payload).cborEncode()
except CatchableError as e:
chronicles.error "liveness event encode failed", name = name, err = e.msg
return
let dataPtr: pointer =
if event.len > 0:
cast[pointer](unsafeAddr event[0])
else:
cast[pointer](emptyListenerPayload)
ctx.dispatchToListeners(name, dataPtr, event.len)
proc onNotResponding*(ctx: ptr FFIContext) =
emitLivenessEvent(ctx, NotRespondingEventName, NotRespondingEvent())
proc onResponding*(ctx: ptr FFIContext) =
## Fired once when the heartbeat resumes after a NotRespondingEvent.
emitLivenessEvent(ctx, RespondingEventName, RespondingEvent())
proc dispatchQueuedEvent[T](ctx: ptr FFIContext[T], qe: QueuedEvent) =
## Reads the borrowed slab payload; `commitDequeue` frees any heap fallback.
ctx.dispatchToListeners($qe.name, qe.data, qe.dataLen)
proc drainOneEvent[T](ctx: ptr FFIContext[T]): bool =
## Peek → dispatch → commit; slot stays pinned across dispatch, `defer` commits
## even if a listener raises. False when the queue is empty.
let opt = ctx.eventQueue.peekEvent()
if opt.isNone():
return false
defer:
ctx.eventQueue.commitDequeue()
ctx.dispatchQueuedEvent(opt.get())
true
proc drainEventQueue[T](ctx: ptr FFIContext[T]) =
while ctx.drainOneEvent():
discard
type HeartbeatMonitor = object
startedAt: Moment
lastChange: Moment
lastValue: int64
notifiedStale: bool
proc init(T: type HeartbeatMonitor, ctx: ptr FFIContext): T =
let now = Moment.now()
T(
startedAt: now,
lastChange: now,
lastValue: ctx.ffiHeartbeat.load(),
notifiedStale: false,
)
proc check[T](hb: var HeartbeatMonitor, ctx: ptr FFIContext[T]) =
## Fires onNotResponding/onResponding on stall/recovery; each latches once per episode.
if Moment.now() - hb.startedAt <= FFIHeartbeatStartDelay:
return
let cur = ctx.ffiHeartbeat.load()
if cur != hb.lastValue:
if hb.notifiedStale:
onResponding(ctx)
hb.lastValue = cur
hb.lastChange = Moment.now()
hb.notifiedStale = false
elif not hb.notifiedStale and Moment.now() - hb.lastChange > FFIHeartbeatStaleThreshold:
onNotResponding(ctx)
hb.notifiedStale = true
proc eventRun[T](ctx: ptr FFIContext[T]) {.async.} =
var hb = HeartbeatMonitor.init(ctx)
var notifiedStuck = false # latched forever — eventQueueStuck is sticky terminal.
# Keep draining after `running` flips false until the FFI thread exits, so events from an async {.ffiDtor.} teardown are still dispatched.
while ctx.running.load() or not ctx.ffiThreadExited.load():
discard await ctx.eventQueueSignal.wait().withTimeout(EventThreadTickInterval)
ctx.drainEventQueue()
# Liveness only while running; skip during the teardown drain.
if ctx.running.load():
# Fire after drain so reg.lock is free (FFI thread would deadlock here).
if not notifiedStuck and ctx.eventQueueStuck.load():
onNotResponding(ctx)
notifiedStuck = true
hb.check(ctx)
# Catch anything enqueued between the last drain and the FFI thread's exit.
ctx.drainEventQueue()
proc eventThreadBody[T](ctx: ptr FFIContext[T]) {.thread.} =
## Drains the event queue and runs the FFI-thread heartbeat check.
defer:
let fireRes = ctx.eventThreadExitSignal.fireSync()
if fireRes.isErr():
error "failed to fire eventThreadExitSignal", err = fireRes.error
try:
waitFor eventRun(ctx)
except CatchableError as e:
error "event thread exited with exception", error = e.msg
+8 -7
View File
@@ -1,11 +1,12 @@
## Compile-time selection of the execution transport.
##
## Default (threaded): each FFIContext spawns an FFI worker thread + a watchdog
## thread and hands requests over a chronos ThreadSignalPtr + SPSC channel.
## Those rely on OS threads + eventfd-style signalling, absent in a baseline
## WebAssembly sandbox.
## Default (threaded): each FFIContext owns an FFI worker thread and an event
## thread, woken over chronos ThreadSignalPtr with requests carried on a queue
## bank. Those need OS threads and eventfd-style signalling, neither of which
## exists in a baseline WebAssembly sandbox.
##
## `singleThreaded` collapses the worker onto the calling thread: a request runs
## inline to completion. Auto-selected for Emscripten/WASM; forceable anywhere
## with `-d:ffiSingleThreaded`.
## `singleThreaded` collapses the workers onto the calling thread: a request is
## spawned on the caller's chronos loop and driven by the host through
## `ffi_poll()`. Auto-selected for Emscripten/WASM; forceable anywhere with
## `-d:ffiSingleThreaded`.
const singleThreaded* = defined(ffiSingleThreaded) or defined(emscripten)
+242 -271
View File
@@ -1,302 +1,273 @@
{.pragma: exported, exportc, cdecl, raises: [].}
{.pragma: callback, cdecl, raises: [], gcsafe.}
## FFIContext type plus lifecycle (init / signal-stop / join / destroy).
{.passc: "-fPIC".}
import std/[options, atomics, os, net, locks, json, tables]
import std/[atomics, locks, options, sequtils, tables]
import chronicles, chronos, results
import ./ffi_config
when not singleThreaded:
# ThreadSignalPtr requires threads enabled; the SPSC channel only carries
# requests across the worker-thread boundary. Neither exists inline.
import chronos/threadsync, taskpools/channels_spsc_single
import ./ffi_types, ./ffi_thread_request, ./internal/ffi_macro, ./logging
when singleThreaded:
# chronos/threadsync is a {.fatal.} under --threads:off, and so is
# system.Thread. ffi_singlethread supplies API-compatible no-ops so the
# lifecycle code below compiles unchanged. See ffi_config.nim.
import ./ffi_singlethread
else:
import chronos/threadsync
import
./ffi_types,
./ffi_events,
./ffi_handles,
./ffi_thread_request,
./ffi_request_queue,
./logging,
./cbor_serial
export ffi_events, ffi_handles
type CtxLifecycle* {.pure.} = enum
## State machine guarding a pooled FFI context (Atomic on FFIContext).
## Active -> RecyclePending when the ffiDtor requests recycle
## RecyclePending -> Recycling FFI loop claimed it, draining handlers
## Recycling -> Active createFFIContext reuses the slot
Active
RecyclePending
Recycling
type FFIContext*[T] = object
myLib*: ptr T
# main library object (e.g., Waku, LibP2P, SDS, the one to be exposed as a library)
when not singleThreaded:
ffiThread: Thread[(ptr FFIContext[T])]
# represents the main FFI thread in charge of attending API consumer actions
watchdogThread: Thread[(ptr FFIContext[T])]
# monitors the FFI thread and notifies the FFI API consumer if it hangs
reqChannel: ChannelSPSCSingle[ptr FFIThreadRequest]
reqSignal: ThreadSignalPtr # to notify the FFI Thread that a new request is sent
reqReceivedSignal: ThreadSignalPtr
# to signal main thread, interfacing with the FFI thread, that FFI thread received the request
else:
myLibStorage: T
# Threaded mode roots the library object on the FFI worker thread's stack
# (`ffiReqHandler`). With no worker thread we keep that backing store in
# the context instead, GC-rooted via the holder in createFFIContext.
lock: Lock
myLib*: ptr T # main library object (Waku, LibP2P, SDS, …)
myLibRefd*: bool
# refc only: true once myLib[] (a ref) has been GC_ref'd to root it against
# the cycle collector. Balanced by GC_unref in freeLib.
myLibOwned*: bool
# true once a ctor stored a createShared'd lib into myLib (vs the worker's
# stack fallback). freeLib only frees/destroys owned libs.
inUse*: Atomic[bool]
# Whether this pooled context is claimed. The recycle handler clears it on
# the FFI thread so the slot returns to the pool without recreating threads.
lifecycle*: Atomic[CtxLifecycle]
recycleDoneSignal: ThreadSignalPtr
# fired by the recycle handler once the lib is freed, just before it releases
# the slot; the synchronous recycleFFIContext caller waits on it.
libReady*: Atomic[bool]
# False until a {.ffiCtor.} stores the library. Before that, `myLib` points
# at the default fallback of the FFI thread. For a `ref` type that fallback
# is nil.
ffiThread: Thread[(ptr FFIContext[T])]
eventThread: Thread[(ptr FFIContext[T])]
reqQueueBank: RequestQueueBank
reqSignal: ThreadSignalPtr
stopSignal: ThreadSignalPtr
threadExitSignal: ThreadSignalPtr
eventQueueSignal: ThreadSignalPtr
eventThreadExitSignal: ThreadSignalPtr
userData*: pointer
eventCallback*: pointer
eventUserdata*: pointer
running: Atomic[bool] # To control when the threads are running
eventRegistry*: FFIEventRegistry
handles*: FFIHandleRegistry
eventQueue*: EventQueue
ffiHeartbeat*: Atomic[int64]
eventQueueStuck*: Atomic[bool]
ffiThreadExited*: Atomic[bool]
# set once FFI thread (incl. async {.ffiDtor.}) is done; event thread drains until then
running: Atomic[bool]
registeredRequests: ptr Table[cstring, FFIRequestProc]
# Pointer to with the registered requests at compile time
staleWarnInterval*: Duration
var onFFIThread* {.threadvar.}: bool
const git_version* {.strdefine.} = "n/a"
template callEventCallback*(ctx: ptr FFIContext, eventName: string, body: untyped) =
if isNil(ctx[].eventCallback):
chronicles.error eventName & " - eventCallback is nil"
return
const RecycleTimeoutMs* {.intdefine: "ffiRecycleTimeoutMs".} = 1500
## Bounds one drain round of the recycle handler. The handler runs at most two
## rounds: it waits for the in-flight handlers, then cancels them and waits
## again. Override with `-d:ffiRecycleTimeoutMs=<ms>`.
const RecycleTimeout* = RecycleTimeoutMs.milliseconds
foreignThreadGc:
try:
let event = body
cast[FFICallBack](ctx[].eventCallback)(
RET_OK, unsafeAddr event[0], cast[csize_t](len(event)), ctx[].eventUserData
)
except Exception, CatchableError:
let msg =
"Exception " & eventName & " when calling 'eventCallBack': " &
getCurrentExceptionMsg()
cast[FFICallBack](ctx[].eventCallback)(
RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), ctx[].eventUserData
)
const
RecycleWaitTimeout* = 2 * RecycleTimeout + 2.seconds
## Caller-side bound for synchronous recycle. It covers both drain rounds
## plus slack, so it only fires when the worker itself is wedged.
EventThreadTickInterval* = 1.seconds
FFIHeartbeatStartDelay* = 10.seconds
FFIHeartbeatStaleThreshold* = 1.seconds
when not singleThreaded:
proc sendRequestToFFIThread*(
ctx: ptr FFIContext, ffiRequest: ptr FFIThreadRequest, timeout = InfiniteDuration
): Result[void, string] =
ctx.lock.acquire()
# This lock is only necessary while we use a SP Channel and while the signalling
# between threads assumes that there aren't concurrent requests.
# Rearchitecting the signaling + migrating to a MP Channel will allow us to receive
# requests concurrently and spare us the need of locks
defer:
ctx.lock.release()
const StaleWarnIntervalMs* {.intdefine: "ffiStaleWarnIntervalMs".} = 5000
## `RET_STALE_WARN` cadence; handlers are never timed out.
const StaleWarnInterval* = StaleWarnIntervalMs.milliseconds
## Sending the request
let sentOk = ctx.reqChannel.trySend(ffiRequest)
if not sentOk:
return err("Couldn't send a request to the ffi thread")
type FFITeardownProc*[T] = proc(lib: ptr T): Future[void] {.async.}
let fireSyncRes = ctx.reqSignal.fireSync()
if fireSyncRes.isErr():
return err("failed fireSync: " & $fireSyncRes.error)
proc ffiTeardownHook*[T](): var FFITeardownProc[T] =
## Per-library teardown slot (one `{.global.}` per `T`), awaited by the FFI thread before exit.
## Runtime slot not an overload: an overload would bind the no-op default before the dtor is visible.
var hook {.global.}: FFITeardownProc[T]
hook
if fireSyncRes.get() == false:
return err("Couldn't fireSync in time")
include ./event_thread
include ./ffi_thread
## wait until the FFI working thread properly received the request
let res = ctx.reqReceivedSignal.waitSync(timeout)
if res.isErr():
return err("Couldn't receive reqReceivedSignal signal")
template closeAndNil(field: untyped) =
if not field.isNil():
?field.close()
field = nil
## Notice that in case of "ok", the deallocShared(req) is performed by the FFI Thread in the
## process proc.
return ok()
proc deinitContextResources*[T](ctx: ptr FFIContext[T]): Result[void, string] =
## Mirror of `initContextResources`. Threads MUST be joined first; fields nil'd after close.
deinitRequestQueue(ctx[].reqQueueBank)
deinitEventRegistry(ctx[].eventRegistry)
deinitHandleRegistry(ctx[].handles)
deinitEventQueue(ctx[].eventQueue)
when defined(gcRefc):
# ThreadSignalPtr.close() under refc hangs via signal-handler re-entry; the
# recycle pool makes full destroy rare, so the leaked fd stays bounded.
discard
else:
closeAndNil(ctx.reqSignal)
closeAndNil(ctx.stopSignal)
closeAndNil(ctx.threadExitSignal)
closeAndNil(ctx.eventQueueSignal)
closeAndNil(ctx.eventThreadExitSignal)
closeAndNil(ctx.recycleDoneSignal)
ok()
type Foo = object
registerReqFFI(WatchdogReq, foo: ptr Foo):
proc(): Future[Result[string, string]] {.async.} =
return ok("FFI thread is not blocked")
template newSignalOrErr(field: untyped, name: string) =
field = ThreadSignalPtr.new().valueOr:
return err("couldn't create ThreadSignalPtr: " & name & ": " & $error)
type JsonNotRespondingEvent = object
eventType: string
proc initContextResources*[T](ctx: ptr FFIContext[T]): Result[void, string] =
## On failure, deferred cleanup closes partial state; caller releases the slot.
# Nil first so deferred cleanup can't double-close a reused pool slot.
ctx.reqSignal = nil
ctx.stopSignal = nil
ctx.threadExitSignal = nil
ctx.eventQueueSignal = nil
ctx.eventThreadExitSignal = nil
ctx.recycleDoneSignal = nil
ctx.myLibOwned = false
ctx.myLibRefd = false
ctx.lifecycle.store(CtxLifecycle.Active)
initRequestQueue(ctx[].reqQueueBank)
initEventRegistry(ctx[].eventRegistry)
initHandleRegistry(ctx[].handles)
initEventQueue(ctx[].eventQueue)
ctx.ffiHeartbeat.store(0)
ctx.libReady.store(false)
ctx.eventQueueStuck.store(false)
ctx.ffiThreadExited.store(false)
ctx.staleWarnInterval = StaleWarnInterval
proc init(T: type JsonNotRespondingEvent): T =
return JsonNotRespondingEvent(eventType: "not_responding")
var success = false
defer:
if not success:
# `ctx` is a pool slot the caller owns; close what was opened, never free it.
ctx.deinitContextResources().isOkOr:
error "failed to clean up resources after createFFIContext failure",
error = error
proc `$`(event: JsonNotRespondingEvent): string =
$(%*event)
newSignalOrErr(ctx.reqSignal, "reqSignal")
newSignalOrErr(ctx.stopSignal, "stopSignal")
newSignalOrErr(ctx.threadExitSignal, "threadExitSignal")
newSignalOrErr(ctx.eventQueueSignal, "eventQueueSignal")
newSignalOrErr(ctx.eventThreadExitSignal, "eventThreadExitSignal")
newSignalOrErr(ctx.recycleDoneSignal, "recycleDoneSignal")
proc onNotResponding*(ctx: ptr FFIContext) =
callEventCallback(ctx, "onNotResponding"):
$JsonNotRespondingEvent.init()
ctx.registeredRequests = addr ffi_types.registeredRequests
when not singleThreaded:
proc watchdogThreadBody(ctx: ptr FFIContext) {.thread.} =
## Watchdog thread that monitors the FFI thread and notifies the library user if it hangs.
## This thread never blocks.
ctx.running.store(true)
let watchdogRun = proc(ctx: ptr FFIContext) {.async.} =
const WatchdogStartDelay = 10.seconds
const WatchdogTimeinterval = 1.seconds
const WatchdogTimeout = 20.seconds
try:
createThread(ctx.ffiThread, ffiThreadBody[T], ctx)
except ValueError, ResourceExhaustedError:
return err("failed to create the FFI thread: " & getCurrentExceptionMsg())
# Give time for the node to be created and up before sending watchdog requests
await sleepAsync(WatchdogStartDelay)
while true:
await sleepAsync(WatchdogTimeinterval)
if ctx.running.load == false:
debug "Watchdog thread exiting because FFIContext is not running"
break
let callback = proc(
callerRet: cint, msg: ptr cchar, len: csize_t, userData: pointer
) {.cdecl, gcsafe, raises: [].} =
discard ## Don't do anything. Just respecting the callback signature.
const nilUserData = nil
trace "Sending watchdog request to FFI thread"
sendRequestToFFIThread(ctx, WatchdogReq.ffiNewReq(callback, nilUserData), WatchdogTimeout).isOkOr:
error "Failed to send watchdog request to FFI thread", error = $error
onNotResponding(ctx)
waitFor watchdogRun(ctx)
proc processRequest[T](
request: ptr FFIThreadRequest, ctx: ptr FFIContext[T]
) {.async.} =
## Invoked within the FFI thread to process a request coming from the FFI API consumer thread.
let reqId = $request[].reqId
## The reqId determines which proc will handle the request.
## The registeredRequests represents a table defined at compile time.
## Then, registeredRequests == Table[reqId, proc-handling-the-request-asynchronously]
let retFut =
if not ctx[].registeredRequests[].contains(reqId):
## That shouldn't happen because only registered requests should be sent to the FFI thread.
nilProcess(request[].reqId)
else:
ctx[].registeredRequests[][reqId](request[].reqContent, ctx)
handleRes(await retFut, request)
when not singleThreaded:
proc ffiThreadBody[T](ctx: ptr FFIContext[T]) {.thread.} =
## FFI thread body that attends library user API requests
logging.setupLog(logging.LogLevel.DEBUG, logging.LogFormat.TEXT)
let ffiRun = proc(ctx: ptr FFIContext[T]) {.async.} =
var ffiReqHandler: T
## Holds the main library object, i.e., in charge of handling the ffi requests.
## e.g., Waku, LibP2P, SDS, etc.
while true:
await ctx.reqSignal.wait()
if ctx.running.load == false:
break
## Wait for a request from the ffi consumer thread
var request: ptr FFIThreadRequest
let recvOk = ctx.reqChannel.tryRecv(request)
if not recvOk:
chronicles.error "ffi thread could not receive a request"
continue
ctx.myLib = addr ffiReqHandler
## Handle the request
asyncSpawn processRequest(request, ctx)
let fireRes = ctx.reqReceivedSignal.fireSync()
if fireRes.isErr():
error "could not fireSync back to requester thread", error = fireRes.error
waitFor ffiRun(ctx)
when singleThreaded:
type SingleThreadedHolder[T] = ref object of RootObj
## GC-traced cell so the library object stored in `ctx.myLibStorage` (a `ref`
## for e.g. Waku) stays scanned. Kept alive in `gSingleThreadedRoots`.
## `of RootObj` so holders can be stored uniformly as `RootRef`.
ctx: FFIContext[T]
var gSingleThreadedRoots {.threadvar.}: seq[RootRef]
proc sendRequestToFFIThread*(
ctx: ptr FFIContext, ffiRequest: ptr FFIThreadRequest, timeout = InfiniteDuration
): Result[void, string] =
## Single-threaded transport. `processRequest` fires the callback and frees
## the request via `handleRes`.
when defined(emscripten):
# Browser: handlers await the network (WebSocket). Blocking with `waitFor`
# would starve the JS event loop and deadlock. Fire-and-forget instead; the
# host drives chronos via `ffi_poll()` and the callback fires on completion.
asyncSpawn processRequest(ffiRequest, ctx)
poll() # kick the handler up to its first await
else:
try:
waitFor processRequest(ffiRequest, ctx)
except CatchableError as e:
return err("processRequest failed: " & e.msg)
return ok()
proc ffiPoll*() {.exportc: "ffi_poll", cdecl.} =
## Advance chronos one step. The browser host calls this from its event loop
## (setTimeout / requestAnimationFrame) so async handlers progress without
## blocking the JS thread; callbacks fire as work completes.
poll()
proc createFFIContext*[T](): Result[ptr FFIContext[T], string] =
## No worker/watchdog threads. The context lives inside a GC-rooted holder so
## `myLibStorage` (the library `ref`) is scanned; `myLib` points at it.
let holder = SingleThreadedHolder[T]()
gSingleThreadedRoots.add(holder)
let ctx = addr holder.ctx
ctx.lock.initLock()
ctx.registeredRequests = addr ffi_types.registeredRequests
ctx.running.store(true)
ctx.myLib = addr ctx.myLibStorage
return ok(ctx)
proc destroyFFIContext*[T](ctx: ptr FFIContext[T]): Result[void, string] =
try:
createThread(ctx.eventThread, eventThreadBody[T], ctx)
except ValueError, ResourceExhaustedError:
# Join ffiThread before deferred cleanup closes signals it's waiting on.
ctx.running.store(false)
ctx.lock.deinitLock()
# Drop the GC root so the holder (and its library object) can be collected.
for i in 0 ..< gSingleThreadedRoots.len:
let h = cast[SingleThreadedHolder[T]](gSingleThreadedRoots[i])
if cast[pointer](addr h.ctx) == cast[pointer](ctx):
gSingleThreadedRoots.del(i)
break
return ok()
else:
proc createFFIContext*[T](): Result[ptr FFIContext[T], string] =
## This proc is called from the main thread and it creates
## the FFI working thread.
var ctx = createShared(FFIContext[T], 1)
ctx.reqSignal = ThreadSignalPtr.new().valueOr:
return err("couldn't create reqSignal ThreadSignalPtr")
ctx.reqReceivedSignal = ThreadSignalPtr.new().valueOr:
return err("couldn't create reqReceivedSignal ThreadSignalPtr")
ctx.lock.initLock()
ctx.registeredRequests = addr ffi_types.registeredRequests
ctx.running.store(true)
try:
createThread(ctx.ffiThread, ffiThreadBody[T], ctx)
except ValueError, ResourceExhaustedError:
freeShared(ctx)
return err("failed to create the FFI thread: " & getCurrentExceptionMsg())
try:
createThread(ctx.watchdogThread, watchdogThreadBody, ctx)
except ValueError, ResourceExhaustedError:
freeShared(ctx)
return err("failed to create the watchdog thread: " & getCurrentExceptionMsg())
return ok(ctx)
proc destroyFFIContext*[T](ctx: ptr FFIContext[T]): Result[void, string] =
ctx.running.store(false)
let signaledOnTime = ctx.reqSignal.fireSync().valueOr:
return err("error in destroyFFIContext: " & $error)
if not signaledOnTime:
return err("failed to signal reqSignal on time in destroyFFIContext")
let fireRes = ctx.reqSignal.fireSync()
if fireRes.isErr():
error "failed to signal ffiThread during event-thread cleanup",
error = fireRes.error
joinThread(ctx.ffiThread)
joinThread(ctx.watchdogThread)
ctx.lock.deinitLock()
?ctx.reqSignal.close()
?ctx.reqReceivedSignal.close()
freeShared(ctx)
return err("failed to create the event thread: " & getCurrentExceptionMsg())
return ok()
success = true
ok()
template checkParams*(ctx: ptr FFIContext, callback: FFICallBack, userData: pointer) =
if not isNil(ctx):
ctx[].userData = userData
proc fireOrErr(sig: ThreadSignalPtr, name: string): Result[void, string] =
let fired = sig.fireSync().valueOr:
return err("error signaling: " & name & ": " & $error)
if not fired:
return err("failed to signal: " & name & " on time")
ok()
if isNil(callback):
return RET_MISSING_CALLBACK
proc waitExitOrErr(
sig: ThreadSignalPtr, name: string, timeout: Duration
): Result[void, string] =
let exited = sig.waitSync(timeout).valueOr:
return err("error waiting for exit: " & name & ": " & $error)
if not exited:
return err("did not exit in time: " & name & " (leaking ctx to avoid hang)")
ok()
proc signalStop*[T](ctx: ptr FFIContext[T]): Result[void, string] =
# Skip onNotResponding on error: it takes reg.lock a stuck listener may hold (deadlock risk).
ctx.running.store(false)
?ctx.reqSignal.fireOrErr("reqSignal")
?ctx.stopSignal.fireOrErr("stopSignal")
ctx.eventQueueSignal.fireOrErr("eventQueueSignal").isOkOr:
error "failed to signal eventQueueSignal in signalStop", error = error
ok()
proc tryClaim*[T](ctx: ptr FFIContext[T]): bool =
## Atomically claim a free pooled context (false -> true).
var expected = false
ctx.inUse.compareExchange(expected, true)
proc releaseClaim*[T](ctx: ptr FFIContext[T]) =
ctx.inUse.store(false)
proc isInUse*[T](ctx: ptr FFIContext[T]): bool =
ctx.inUse.load()
proc markAsActive*[T](ctx: ptr FFIContext[T]) =
## Reused context: its worker threads are still alive; re-arm for requests.
ctx.lifecycle.store(CtxLifecycle.Active)
proc requestRecycle*[T](ctx: ptr FFIContext[T]): Result[void, string] =
## Ask the FFI thread to drain, free the lib and release the slot, WITHOUT
## stopping its worker/event threads, so the next createFFIContext reuses them.
## Synchronous: waits on recycleDoneSignal. No fd churn -> no select() limit.
var expected = CtxLifecycle.Active
if not ctx.lifecycle.compareExchange(expected, CtxLifecycle.RecyclePending):
return err("requestRecycle: context is not Active (already recycling)")
# A recycle that timed out can fire late. The CAS makes this the only recycle
# in flight, so drop that stale fire before the wait below can answer to it.
discard ctx.recycleDoneSignal.waitSync(ZeroDuration)
let fired = ctx.reqSignal.fireSync().valueOr:
return err("requestRecycle: failed to signal the FFI thread: " & $error)
if not fired:
return err("requestRecycle: failed to signal the FFI thread in time")
let done = ctx.recycleDoneSignal.waitSync(RecycleWaitTimeout).valueOr:
return err("requestRecycle: failed waiting for recycle: " & $error)
if not done:
return err("requestRecycle: recycle did not complete in time")
ok()
## Per-thread exit wait before stopAndJoinThreads leaks ctx rather than hanging; async
## `{.ffiDtor.}` teardown can outlast the default. Override `-d:ffiThreadExitTimeoutMs=<ms>`.
const ThreadExitTimeoutMs* {.intdefine: "ffiThreadExitTimeoutMs".} = 1500
const ThreadExitTimeout* = ThreadExitTimeoutMs.milliseconds
proc stopAndJoinThreads*[T](ctx: ptr FFIContext[T]): Result[void, string] =
## On timeout, returns err and skips remaining joins (leaves threads live); caller cleans up.
ctx.signalStop().isOkOr:
return err("signalStop failed: " & $error)
?ctx.threadExitSignal.waitExitOrErr("FFI thread", ThreadExitTimeout)
joinThread(ctx.ffiThread)
?ctx.eventThreadExitSignal.waitExitOrErr("event thread", ThreadExitTimeout)
joinThread(ctx.eventThread)
ok()
+142
View File
@@ -0,0 +1,142 @@
import std/[atomics, sysatomics]
import results
import ./ffi_context
const MaxFFIContexts* = 32
type
StaticCtxState = enum
## Lifecycle of the pool's `{.ffiStatic.}` context; see `staticFFIContext`.
StaticCtxNone
StaticCtxCreating
StaticCtxDestroying
StaticCtxReady
FFIContextPool*[T] = object
## Fixed pool of FFI contexts, plus the one `{.ffiStatic.}` context. Each
## slot's worker + event threads and signal fds are built once (on first
## use) and reused across create/recycle cycles — recycle keeps them alive,
## so repeated create/destroy does not churn fds. Bounds ThreadSignalPtr fds
## at MaxFFIContexts * (signals per ctx).
contexts: array[MaxFFIContexts, FFIContext[T]]
initialized: array[MaxFFIContexts, Atomic[bool]]
staticCtx: Atomic[pointer]
staticState: Atomic[StaticCtxState]
proc releaseSlot[T](pool: var FFIContextPool[T], ctx: ptr FFIContext[T]) =
## Full-teardown release: the slot must be rebuilt before it serves again.
for i in 0 ..< MaxFFIContexts:
if pool.contexts[i].addr == ctx:
pool.initialized[i].store(false)
break
ctx.releaseClaim()
proc createFFIContext*[T](
pool: var FFIContextPool[T]
): Result[ptr FFIContext[T], string] =
## Acquires a context from the fixed pool. A slot's worker is built once on
## first use and reused (markAsActive) on every later acquisition.
for i in 0 ..< MaxFFIContexts:
let ctx = pool.contexts[i].addr
if not ctx.tryClaim():
continue
if pool.initialized[i].load():
# Reused slot: a prior recycle drained and released it; worker still alive.
ctx.markAsActive()
return ok(ctx)
initContextResources(ctx).isOkOr:
ctx.releaseClaim()
return err("createFFIContext: initContextResources failed: " & $error)
pool.initialized[i].store(true)
return ok(ctx)
err("FFI context pool exhausted (max " & $MaxFFIContexts & " contexts)")
proc isStaticCtx[T](pool: var FFIContextPool[T], ctx: ptr FFIContext[T]): bool =
## True while `ctx` is the pool's static context, including mid-teardown.
# `staticCtx` is cleared only once the slot is released, so matching on the
# pointer covers `Destroying` too.
pool.staticCtx.load() == cast[pointer](ctx)
proc recycleFFIContext*[T](
pool: var FFIContextPool[T], ctx: ptr FFIContext[T]
): Result[void, string] =
## Normal teardown: drains in-flight handlers, frees the lib and returns the
## slot to the pool WITHOUT stopping its threads, so a later createFFIContext
## reuses them. Synchronous (waits for the FFI thread to finish draining).
# Recycling it would release the slot while `staticState` still points at it.
if pool.isStaticCtx(ctx):
return err("recycleFFIContext(pool): the {.ffiStatic.} context outlives every ctx")
ctx.requestRecycle()
proc destroyFFIContext*[T](
pool: var FFIContextPool[T], ctx: ptr FFIContext[T]
): Result[void, string] =
## Full teardown: stops/joins the threads and frees resources, marking the slot
## uninitialised so a later createFFIContext rebuilds it; normal cleanup uses
## recycleFFIContext. On thread-exit timeout the slot is leaked; closing
## live-thread resources is unsafe.
# Destroying it would release the slot while `staticState` still points at it.
if pool.isStaticCtx(ctx):
return err("destroyFFIContext(pool): the {.ffiStatic.} context outlives every ctx")
ctx.stopAndJoinThreads().isOkOr:
return err("destroyFFIContext(pool): " & $error)
let deinitRes = ctx.deinitContextResources()
pool.releaseSlot(ctx)
deinitRes.isOkOr:
return err("destroyFFIContext(pool): " & $error)
ok()
proc staticFFIContext*[T](
pool: var FFIContextPool[T]
): Result[ptr FFIContext[T], string] =
## The pool's `{.ffiStatic.}` context, created on first use: a static proc has
## no ctx of its own, but its handler still needs an FFI thread.
# Holds its slot until `destroyStaticFFIContext`, so `pool` must outlive its
# threads: only call this on the global `declareLibrary` emits. `myLib` stays
# the zero value. A failed create resets to `StaticCtxNone` so waiters retry.
while true:
case pool.staticState.load()
of StaticCtxReady:
return ok(cast[ptr FFIContext[T]](pool.staticCtx.load()))
of StaticCtxCreating, StaticCtxDestroying:
cpuRelax()
of StaticCtxNone:
var expected = StaticCtxNone
if not pool.staticState.compareExchange(expected, StaticCtxCreating):
continue
let ctx = pool.createFFIContext().valueOr:
pool.staticState.store(StaticCtxNone)
return err("staticFFIContext: " & error)
pool.staticCtx.store(cast[pointer](ctx))
pool.staticState.store(StaticCtxReady)
return ok(ctx)
proc destroyStaticFFIContext*[T](pool: var FFIContextPool[T]): Result[void, string] =
## Teardown counterpart to `staticFFIContext`: stops the static context's
## threads and frees its slot. A no-op when there is no static context.
# Claiming `Ready -> Destroying` serialises concurrent teardowns; it does not
# make teardown safe against a static call already in flight.
var expected = StaticCtxReady
if not pool.staticState.compareExchange(expected, StaticCtxDestroying):
return ok()
let ctx = cast[ptr FFIContext[T]](pool.staticCtx.load())
ctx.stopAndJoinThreads().isOkOr:
# Threads are still live: leak the slot rather than free resources under them.
pool.staticState.store(StaticCtxReady)
return err("destroyStaticFFIContext: " & $error)
let deinitRes = ctx.deinitContextResources()
pool.releaseSlot(ctx)
pool.staticCtx.store(nil)
pool.staticState.store(StaticCtxNone)
deinitRes.isOkOr:
return err("destroyStaticFFIContext: " & $error)
ok()
proc isValidCtx*[T](pool: var FFIContextPool[T], ctx: pointer): bool =
## Rejects nil / dangling pointers at the API boundary.
if ctx.isNil():
return false
for i in 0 ..< MaxFFIContexts:
if cast[pointer](pool.contexts[i].addr) == ctx:
return pool.contexts[i].addr.isInUse()
false
+330
View File
@@ -0,0 +1,330 @@
## Per-context event registry + bounded SPSC queue. FFI thread enqueues, event
## thread drains; payloads use c_malloc so they survive cross-thread heap reuse.
{.pragma: callback, cdecl, raises: [], gcsafe.}
import system/ansi_c
import std/[atomics, locks, sequtils, options, tables]
import chronicles
import ./ffi_types, ./cbor_serial, ./alloc
type EventEnvelope*[T] = object ## CBOR wire shape: { eventType: tstr, payload: <T> }.
eventType*: string
payload*: T
type
FFIEventListener* = object
id*: uint64
callback*: FFICallBack
userData*: pointer
FFIEventRegistry* = object
lock*: Lock
nextId*: uint64 # 0 is reserved as "invalid"; ids start at 1.
byEvent*: Table[string, seq[FFIEventListener]]
proc initEventRegistry*(reg: var FFIEventRegistry) =
## Run once on the owning thread before sharing (re-initLock is UB).
reg.lock.initLock()
reg.nextId = 0'u64
reg.byEvent = initTable[string, seq[FFIEventListener]]()
proc deinitEventRegistry*(reg: var FFIEventRegistry) =
## Mirror of `initEventRegistry`; resets GC fields so slot reuse sees no dtor.
reg.lock.deinitLock()
reg.byEvent = default(Table[string, seq[FFIEventListener]])
reg.nextId = 0'u64
proc clearListeners*(reg: var FFIEventRegistry) {.raises: [].} =
## Removes all listeners. The pool calls this when it recycles a context. The
## lock stays in place, because the event thread uses it across recycles.
withLock reg.lock:
reg.byEvent.clear()
reg.nextId = 0'u64
proc addEventListener*(
reg: var FFIEventRegistry,
eventName: string,
callback: FFICallBack,
userData: pointer,
): uint64 {.raises: [].} =
## Returns the listener id (>0), or 0 if `callback` is nil.
if callback.isNil():
return 0
var assigned: uint64 = 0
withLock reg.lock:
reg.nextId.inc()
assigned = reg.nextId
let listener =
FFIEventListener(id: assigned, callback: callback, userData: userData)
reg.byEvent.mgetOrPut(eventName, @[]).add(listener)
assigned
proc removeEventListener*(reg: var FFIEventRegistry, id: uint64): bool {.raises: [].} =
## Safe from inside a dispatch; the in-flight snapshot still delivers once.
if id == 0'u64:
return false
var removed = false
withLock reg.lock:
var
pruneKey = ""
prune = false
for key, listeners in reg.byEvent.mpairs:
let before = listeners.len
listeners.keepItIf(it.id != id)
if listeners.len < before:
removed = true
if listeners.len == 0:
pruneKey = key
prune = true
break
if prune:
reg.byEvent.del(pruneKey)
removed
proc removeAllEventListeners*(reg: var FFIEventRegistry) {.raises: [].} =
## Does not reset the id counter.
withLock reg.lock:
reg.byEvent.clear()
proc snapshotListeners*(
reg: var FFIEventRegistry, eventName: string
): seq[FFIEventListener] {.raises: [].} =
## Lock held only across the copy so re-entrant add/remove can't deadlock.
var listeners: seq[FFIEventListener] = @[]
withLock reg.lock:
for l in reg.byEvent.getOrDefault(eventName):
listeners.add(l)
listeners
const EventQueueCapacity* {.intdefine.} = 1024
## Sustained backlog here means a listener is wedged. Override `-d:EventQueueCapacity=N`.
const MaxEventPayloadBytes* {.intdefine.} = 512
## Per-slot payload slab; larger payloads take a one-off c_malloc freed on
## commit. Override `-d:MaxEventPayloadBytes=N`.
const MaxEventNameBytes* {.intdefine.} = 64
## Per-slot name slab (incl. NUL); longer names take the heap fallback.
## Override `-d:MaxEventNameBytes=N`.
const emptyListenerPayload*: cstring = ""
## Non-nil zero-length stand-in for empty payloads/names (nil would be UB for
## consumers doing memcpy even at len 0).
type
QueuedEvent* = object
# `name`/`data` point into reused per-slot buffers, or a one-off c_malloc marked by `*HeapOwned` when oversize; both c_malloc'd so they outlive the FFI thread's heap.
name*: cstring
nameHeapOwned*: bool
data*: ptr UncheckedArray[byte]
dataLen*: int
dataHeapOwned*: bool
EventQueue* = object # SPSC ring; plain lock since ops are short and uncontended.
lock*: Lock
head*: int
tail*: int
count*: int
buf*: array[EventQueueCapacity, QueuedEvent]
slab*: array[EventQueueCapacity, ptr UncheckedArray[byte]]
nameSlab*: array[EventQueueCapacity, ptr UncheckedArray[byte]]
proc allocSlot(nbytes: int): ptr UncheckedArray[byte] {.raises: [].} =
if nbytes <= 0:
return nil
cast[ptr UncheckedArray[byte]](c_malloc(csize_t(nbytes)))
proc initEventQueue*(q: var EventQueue) {.raises: [].} =
q.lock.initLock()
q.head = 0
q.tail = 0
q.count = 0
for i in 0 ..< EventQueueCapacity:
q.buf[i] = QueuedEvent()
q.slab[i] = allocSlot(MaxEventPayloadBytes)
q.nameSlab[i] = allocSlot(MaxEventNameBytes)
proc releaseEvent*(qe: QueuedEvent) {.raises: [], gcsafe.} =
## Frees only heap-fallback buffers; reused slot buffers persist.
if qe.nameHeapOwned and not qe.name.isNil():
c_free(cast[pointer](qe.name))
if qe.dataHeapOwned and not qe.data.isNil():
c_free(qe.data)
proc deinitEventQueue*(q: var EventQueue) {.raises: [].} =
## Both producer and consumer must have stopped.
for i in 0 ..< EventQueueCapacity:
releaseEvent(q.buf[i])
q.buf[i] = QueuedEvent()
if not q.slab[i].isNil():
c_free(q.slab[i])
q.slab[i] = nil
if not q.nameSlab[i].isNil():
c_free(q.nameSlab[i])
q.nameSlab[i] = nil
q.head = 0
q.tail = 0
q.count = 0
q.lock.deinitLock()
proc copyIntoSlot(
slot: ptr UncheckedArray[byte], slotCap, nbytes: int, src: pointer
): tuple[buf: ptr UncheckedArray[byte], heap: bool, ok: bool] {.raises: [].} =
## Copies into `slot` when it fits, else a one-off c_malloc; `ok=false` only on
## alloc failure.
if nbytes <= 0:
return (nil, false, true)
if nbytes <= slotCap and not slot.isNil():
copyMem(slot, src, nbytes)
return (slot, false, true)
let heapBuf = cast[ptr UncheckedArray[byte]](c_malloc(csize_t(nbytes)))
if heapBuf.isNil():
return (nil, false, false)
copyMem(heapBuf, src, nbytes)
(heapBuf, true, true)
proc tryEnqueueEvent*(
q: var EventQueue, name: cstring, src: pointer, dataLen: int
): bool {.raises: [], gcsafe.} =
## Copies `name` (NUL included) and payload into the tail slot's reused buffers
## or a heap fallback; false when the ring is full or a fallback alloc fails.
withLock q.lock:
if q.count >= EventQueueCapacity:
return false
let slot = q.tail
# Include the NUL so the stored copy stays a valid cstring.
let nameBytes =
if name.isNil():
0
else:
name.len + 1
let nameRes =
copyIntoSlot(q.nameSlab[slot], MaxEventNameBytes, nameBytes, cast[pointer](name))
if not nameRes.ok:
return false
let dataRes = copyIntoSlot(q.slab[slot], MaxEventPayloadBytes, dataLen, src)
if not dataRes.ok:
if nameRes.heap:
c_free(nameRes.buf)
return false
let nameCStr =
if nameRes.buf.isNil():
emptyListenerPayload
else:
cast[cstring](nameRes.buf)
q.buf[slot] = QueuedEvent(
name: nameCStr,
nameHeapOwned: nameRes.heap,
data: dataRes.buf,
dataLen: dataLen,
dataHeapOwned: dataRes.heap,
)
q.tail = (q.tail + 1) mod EventQueueCapacity
q.count.inc()
true
proc peekEvent*(q: var EventQueue): Option[QueuedEvent] {.raises: [], gcsafe.} =
## Returns the head without advancing (slot stays pinned so the producer can't
## reuse it mid-read); pair each non-none peek with a `commitDequeue`.
withLock q.lock:
if q.count == 0:
return none(QueuedEvent)
return some(q.buf[q.head])
proc commitDequeue*(q: var EventQueue) {.raises: [], gcsafe.} =
## Retires the dispatched head slot: frees any heap fallback and frees the slot.
withLock q.lock:
if q.count == 0:
return
releaseEvent(q.buf[q.head])
q.buf[q.head] = QueuedEvent()
q.head = (q.head + 1) mod EventQueueCapacity
q.count.dec()
proc eventQueueLen*(q: var EventQueue): int {.raises: [], gcsafe.} =
withLock q.lock:
return q.count
proc notifyListeners*(
listeners: seq[FFIEventListener], retCode: cint, data: pointer, dataLen: int
) =
## Empty payloads use `emptyListenerPayload` so consumers never see a nil ptr.
let n = max(dataLen, 0)
let dataPtr =
if n > 0 and not data.isNil():
cast[ptr cchar](data)
else:
cast[ptr cchar](emptyListenerPayload)
for listener in listeners:
listener.callback(retCode, dataPtr, cast[csize_t](n), listener.userData)
proc notifyListenersErr*(listeners: seq[FFIEventListener], msg: string) =
let p =
if msg.len > 0:
cast[pointer](unsafeAddr msg[0])
else:
cast[pointer](emptyListenerPayload)
notifyListeners(listeners, RET_ERR, p, msg.len)
var ffiCurrentEventRegistry* {.threadvar.}: ptr FFIEventRegistry
var ffiCurrentEventQueue* {.threadvar.}: ptr EventQueue
# Installed by the FFI thread so dispatch templates need no `ctx`.
var ffiCurrentEventQueueStuck* {.threadvar.}: ptr Atomic[bool]
# Sticky overflow flag; FFI request entry point reads it to reject.
var ffiCurrentNotifyEventEnqueued* {.threadvar.}: proc() {.gcsafe, raises: [].}
# Wake hook so this module needn't depend on chronos; nil-safe.
template enqueueOrMarkStuck(eventName: string, src: pointer, dataLen: int) =
## Enqueues into the reused slot buffers; on queue-full sets the sticky stuck
## flag and wakes the event thread (firing onNotResponding here could deadlock).
block enqueueBlock:
let q = ffiCurrentEventQueue
if q.isNil():
chronicles.error "event queue not set on this thread", event = eventName
break enqueueBlock
if not q[].tryEnqueueEvent(cstring(eventName), src, dataLen):
chronicles.error "event queue full; library marked stuck",
event = eventName, capacity = EventQueueCapacity
if not ffiCurrentEventQueueStuck.isNil():
ffiCurrentEventQueueStuck[].store(true)
if not ffiCurrentNotifyEventEnqueued.isNil():
ffiCurrentNotifyEventEnqueued()
break enqueueBlock
if not ffiCurrentNotifyEventEnqueued.isNil():
ffiCurrentNotifyEventEnqueued()
template dispatchFFIEvent*(eventName: string, body: untyped) =
## `body` yields string/seq[byte]. FFI thread only: enqueues; event thread fans out.
block:
let evtName: string = eventName
let bodyVal = body
let dataLen = bodyVal.len
let src: pointer =
if dataLen > 0:
unsafeAddr bodyVal[0]
else:
nil
enqueueOrMarkStuck(evtName, src, dataLen)
template dispatchFFIEventCbor*(eventName: string, eventPayload: typed) =
## Typed CBOR variant; param is `eventPayload` to avoid clobbering
## `EventEnvelope.payload` substitution.
block:
let evtName: string = eventName
let encoded = cborEncode(
EventEnvelope[typeof(eventPayload)](eventType: evtName, payload: eventPayload)
)
let src: pointer =
if encoded.len > 0:
unsafeAddr encoded[0]
else:
nil
enqueueOrMarkStuck(evtName, src, encoded.len)
+60
View File
@@ -0,0 +1,60 @@
## Per-context registry of live `{.ffiHandle.}` objects; only the `uint64` id crosses the
## boundary. Ids are monotonic, never recycled (0 = null). FFI-thread-only, so no locking.
import std/tables
import results
import ./cbor_serial
type
FFIHandleRoot* = ref object of RootObj ## Base of every `{.ffiHandle.}` type.
FFIHandleEntry = object
obj: FFIHandleRoot
typeName: string
FFIHandleRegistry* = object
nextId*: uint64
byHandle*: Table[uint64, FFIHandleEntry]
proc initHandleRegistry*(reg: var FFIHandleRegistry) =
reg.nextId = 0'u64
reg.byHandle = initTable[uint64, FFIHandleEntry]()
proc deinitHandleRegistry*(reg: var FFIHandleRegistry) =
reg.byHandle = default(Table[uint64, FFIHandleEntry])
reg.nextId = 0'u64
proc register*(
reg: var FFIHandleRegistry, obj: FFIHandleRoot, typeName: string
): uint64 =
reg.nextId.inc()
reg.byHandle[reg.nextId] = FFIHandleEntry(obj: obj, typeName: typeName)
reg.nextId
proc lookup*(
reg: var FFIHandleRegistry, handle: uint64, typeName: string
): Result[FFIHandleRoot, string] =
## Live ref for `handle`; err if absent or registered under another type.
let entry = reg.byHandle.getOrDefault(handle)
if entry.obj.isNil():
return err("no ffiHandle with id " & $handle)
if entry.typeName != typeName:
return err(
"ffiHandle " & $handle & " has type '" & entry.typeName & "', expected '" &
typeName & "'"
)
ok(entry.obj)
proc release*(reg: var FFIHandleRegistry, handle: uint64): bool {.discardable.} =
if not reg.byHandle.hasKey(handle):
return false
reg.byHandle.del(handle)
return true
proc releaseAll*(reg: var FFIHandleRegistry) =
## Must run on the FFI thread that allocated the refs.
reg.byHandle.clear()
proc encodeHandle*(id: uint64): seq[byte] =
## Single ABI seam for the handle-id wire format.
cborEncode(id)
+92
View File
@@ -0,0 +1,92 @@
## Sharded, mutex-guarded MPSC ingress for `ptr FFIThreadRequest`: N intrusive
## FIFOs (one per producer) spread lock contention; the request is its own node
## so enqueue never touches a Nim GC heap. Unbounded — submit never blocks.
import std/[atomics, locks]
import ./ffi_thread_request
const
RequestQueueCount* = 16
## Independent ingress queues; ≥ concurrent producer count keeps collisions low.
QueuePadBytes = 192
## Pads each queue past a cache line (128B on Apple silicon) to avoid false
## sharing between adjacent queues.
static:
# `myQueueIndex` masks with `and`, so the count must be a power of two.
doAssert (RequestQueueCount and (RequestQueueCount - 1)) == 0,
"RequestQueueCount must be a power of two"
type
RequestQueue = object
lock: Lock
head: ptr FFIThreadRequest ## consumer pops here (oldest)
tail: ptr FFIThreadRequest ## producers append here (newest)
pad: array[QueuePadBytes, byte]
RequestQueueBank* = object
queues: array[RequestQueueCount, RequestQueue]
var gRequestQueue {.threadvar.}: int
var gRequestQueueAssigned {.threadvar.}: bool
var gRequestQueueCounter: Atomic[int]
## Round-robins producers onto distinct queues on first use so they fill evenly.
proc myQueueIndex(): int {.raises: [].} =
if not gRequestQueueAssigned:
gRequestQueue = gRequestQueueCounter.fetchAdd(1)
gRequestQueueAssigned = true
return gRequestQueue and (RequestQueueCount - 1)
proc initRequestQueue*(bank: var RequestQueueBank) {.raises: [].} =
for queue in bank.queues.mitems:
queue.lock.initLock()
queue.head = nil
queue.tail = nil
proc deinitRequestQueue*(bank: var RequestQueueBank) {.raises: [].} =
## Both producers and consumer must have stopped. Frees any still-queued request
## (e.g. one raced in after the final drain) so a teardown race leaks nothing.
for queue in bank.queues.mitems:
var request = queue.head
while not request.isNil():
let nextRequest = request[].next
deleteRequest(request)
request = nextRequest
queue.head = nil
queue.tail = nil
queue.lock.deinitLock()
proc pushRequest*(
bank: var RequestQueueBank, request: ptr FFIThreadRequest
): bool {.raises: [].} =
## Append `request` to this thread's queue (takes ownership). True only when the
## queue was empty — the one push that must wake the sleeping consumer.
request[].next = nil
let idx = myQueueIndex()
withLock bank.queues[idx].lock:
let wasEmpty = bank.queues[idx].tail.isNil()
if bank.queues[idx].tail.isNil():
bank.queues[idx].head = request
else:
bank.queues[idx].tail[].next = request
bank.queues[idx].tail = request
return wasEmpty
proc mergeQueues*(bank: var RequestQueueBank): ptr FFIThreadRequest {.raises: [].} =
## Single-consumer: splice every queue into one chain and reset them. Caller owns
## the chain and must read each `next` before dispatch (dispatch frees the request).
var head: ptr FFIThreadRequest = nil
var tail: ptr FFIThreadRequest = nil
for queue in bank.queues.mitems:
withLock queue.lock:
let h = queue.head
if not h.isNil():
if head.isNil():
head = h
else:
tail[].next = h
tail = queue.tail
queue.head = nil
queue.tail = nil
return head
+73
View File
@@ -0,0 +1,73 @@
## Threads-off stand-ins for the two primitives nim-ffi's context is built on.
##
## `chronos/threadsync` is a hard `{.fatal.}` under `--threads:off`, and
## `system.Thread` / `createThread` / `joinThread` do not exist there either --
## so the FFIContext object cannot even be *declared*, let alone used.
##
## Rather than gate all ~20 use sites (and re-gate them on every nim-ffi bump),
## this module supplies API-compatible no-ops. The upstream lifecycle code then
## compiles unchanged: it "creates" threads that do not exist and "fires"
## signals nobody waits on. That is sound only because the single-threaded
## transport never enqueues -- `sendRequestToFFIThread` runs the handler inline
## on the caller's chronos loop -- so no worker is needed to drain anything.
##
## Keep this in step with chronos' ThreadSignalPtr surface when bumping nim-ffi;
## a missing proc shows up as a plain "undeclared field" at compile time.
{.push raises: [].}
import chronos, results
type
ThreadSignalPtr* = ptr object
## No-op stand-in for chronos' cross-thread signal. Pointer-shaped, not an
## object: the lifecycle code assigns `nil` to these fields on teardown and
## nil-checks them before use, so a value type does not typecheck.
Thread*[T] = object ## No-op stand-in for system.Thread.
started: bool
var dummySignal: int
## Address handed out by `new` so signals read as non-nil (nil means
## "not initialised" upstream). Never dereferenced.
proc new*(T: typedesc[ThreadSignalPtr]): Result[ThreadSignalPtr, string] =
ok(cast[ThreadSignalPtr](addr dummySignal))
proc close*(signal: ThreadSignalPtr): Result[void, string] =
ok()
proc fireSync*(
signal: ThreadSignalPtr, timeout = InfiniteDuration
): Result[bool, string] =
## Nothing waits on these threads-off, so a "fire" is a successful no-op.
ok(true)
proc wait*(
signal: ThreadSignalPtr
): Future[void] {.async: (raises: [CancelledError]).} =
## Never completes on its own. The only callers are the worker loops, which
## are never started threads-off, so this is unreachable rather than a hang.
await sleepAsync(InfiniteDuration)
proc waitSync*(
signal: ThreadSignalPtr, timeout = InfiniteDuration
): Result[bool, string] =
## Reports "signalled" immediately. Callers use this to block until a worker
## acknowledges something; with no worker there is nothing to wait for, and
## returning false would stall shutdown on a signal that can never arrive.
ok(true)
proc createThread*[T](
thread: var Thread[T], body: proc(arg: T) {.thread, nimcall.}, arg: T
) =
## Deliberately does not run `body`: the worker loops block on signals that
## never fire. Requests are dispatched inline instead.
thread.started = true
proc joinThread*[T](thread: Thread[T]) =
discard
proc running*[T](thread: Thread[T]): bool =
false
+295
View File
@@ -0,0 +1,295 @@
## FFI-thread body and request submission API. Included from `ffi_context.nim`.
## Dispatches `FFIThreadRequest`s from `reqQueueBank` and advances
## `ctx.ffiHeartbeat` so the event thread can spot a wedged FFI thread.
proc sendRequestToFFIThreadQueued(
ctx: ptr FFIContext, ffiRequest: ptr FFIThreadRequest
): Result[void, string] =
if ctx.eventQueueStuck.load():
deleteRequest(ffiRequest)
return err("event queue stuck - library cannot accept new requests")
if onFFIThread:
# A handler re-dispatching onto its own FFI thread would deadlock; reject.
deleteRequest(ffiRequest)
return err(
"reentrant ffi call: a handler invoked sendRequestToFFIThread on its own context"
)
if ctx.lifecycle.load() != CtxLifecycle.Active:
deleteRequest(ffiRequest)
return err("FFI context is not accepting requests (being recycled)")
# Wake only when the push found the queue empty: waking per submit kills scaling, and a skipped wake just waits the consumer's 100ms poll.
let shouldWake = ctx.reqQueueBank.pushRequest(ffiRequest)
# A failed wake is non-fatal (poll-drain still dispatches); erroring here would double-fire the callback for a request that still completes.
if shouldWake:
ctx.reqSignal.fireSync().isOkOr:
error "failed to wake FFI thread after enqueue (request still queued)",
error = error
ok()
when not singleThreaded:
proc sendRequestToFFIThread*(
ctx: ptr FFIContext, ffiRequest: ptr FFIThreadRequest
): Result[void, string] =
sendRequestToFFIThreadQueued(ctx, ffiRequest)
proc awaitWithStaleWarnings(
retFut: Future[Result[seq[byte], string]],
request: ptr FFIThreadRequest,
interval: Duration,
reqId: string,
): Future[Result[seq[byte], string]] {.async.} =
## Pings RET_STALE_WARN every `interval` while the handler runs, then returns
## its real result. Never cancels the handler: a hard-cancel mid-call could
## leave the underlying library partially applied.
let intervalMs = interval.milliseconds
if intervalMs <= 0:
return await retFut
var elapsed = 0'i64
while not retFut.finished():
let timer = sleepAsync(interval)
# `race` doesn't cancel the loser, so the handler keeps running.
discard await race(retFut, timer)
if retFut.finished():
if not timer.finished():
await timer.cancelAndWait()
break
elapsed += intervalMs
warn "ffi request still in flight; caller notified via RET_STALE_WARN",
reqId = reqId, elapsedMs = elapsed
fireStaleWarn(request, elapsed)
return await retFut
proc processRequest[T](
request: ptr FFIThreadRequest, ctx: ptr FFIContext[T]
) {.async.} =
## Processes one request on the FFI thread.
let reqId = $request[].reqId
let reqIdCs = reqId.cstring # keeps reqId alive
let retFut =
if not ctx[].registeredRequests[].contains(reqIdCs):
nilProcess(request[].reqId)
else:
ctx[].registeredRequests[][reqIdCs](cast[pointer](request), ctx)
# One try over warn-loop + handler so a shutdown-drain cancel still reaches the response-and-free below.
let res =
try:
await awaitWithStaleWarnings(retFut, request, ctx.staleWarnInterval, reqId)
except CatchableError as e:
Result[seq[byte], string].err(
"Error in processRequest for " & reqId & ": " & e.msg
)
try:
handleRes(res, request)
except Exception as e:
error "Unexpected exception in handleRes", error = e.msg
when singleThreaded:
proc sendRequestToFFIThread*(
ctx: ptr FFIContext, ffiRequest: ptr FFIThreadRequest
): Result[void, string] =
## Single-threaded transport: nothing drains reqQueueBank, so run the
## handler on the caller's chronos loop instead of enqueuing it.
##
## Fire-and-forget rather than `waitFor`: handlers await the network, and
## blocking here would starve the browser's event loop and deadlock. The
## host drives progress with `ffi_poll()`; `processRequest` still fires the
## caller's callback and frees the request through `handleRes`, exactly as
## the threaded path does.
if ctx.lifecycle.load() != CtxLifecycle.Active:
deleteRequest(ffiRequest)
return err("FFI context is not accepting requests (being recycled)")
asyncSpawn processRequest(ffiRequest, ctx)
poll() # advance the handler to its first await
return ok()
proc ffiPoll*() {.exportc: "ffi_poll", cdecl.} =
## One chronos iteration. The browser host calls this from its event loop so
## handlers progress without blocking JS. One call == one iteration, so the
## host must pump hard while a request is in flight -- see
## docs/wasm-edge-node.md.
poll()
proc freeLib[T](ctx: ptr FFIContext[T]) {.gcsafe.} =
## Releases the library object the ctor stored in ctx.myLib. Only owned libs
## (createShared'd by a ctor) are freed; the worker's stack fallback is not.
# A reused slot skips initContextResources, so the recycle path clears this.
ctx.libReady.store(false)
if not ctx.myLibOwned or ctx.myLib.isNil():
ctx.myLib = nil
return
when not defined(gcRefc):
try:
{.cast(gcsafe).}:
`=destroy`(ctx.myLib[])
except Exception as e:
error "destroying the library on recycle raised; freeing it anyway", error = e.msg
else:
when T is ref:
if ctx.myLibRefd:
GC_unref(ctx.myLib[])
ctx.myLibRefd = false
freeShared(ctx.myLib)
ctx.myLib = nil
ctx.myLibOwned = false
const RecycledReason =
"FFI context was recycled before this request ran; the caller is gone"
proc rejectQueuedRequests[T](ctx: ptr FFIContext[T]) =
## Fails every queued request instead of dispatching it. A request that a
## destroyed context left behind still carries that host's `userData`, which
## the host has freed; running it would answer a dead callback, and running it
## after the slot is reused would run it against the library of the next owner.
var request = ctx.reqQueueBank.mergeQueues()
while not request.isNil():
let nextRequest = request[].next # read before handleRes frees it
try:
handleRes(Result[seq[byte], string].err(RecycledReason), request)
except Exception as e:
error "rejecting a queued request raised", error = e.msg
request = nextRequest
proc recycleContext[T](
ctx: ptr FFIContext[T], ongoing: ptr seq[Future[void]]
) {.async.} =
## Drain in-flight handlers, free the lib, clear listeners and release the
## slot — all WITHOUT stopping the worker/event threads, so the next
## createFFIContext reuses them (no fd churn). Then fire recycleDoneSignal.
ongoing[].keepItIf(not it.finished())
var drained = ongoing[].len == 0
if not drained:
drained = await allFutures(ongoing[]).withTimeout(RecycleTimeout)
if not drained:
for fut in ongoing[]:
fut.cancelSoon()
drained = await allFutures(ongoing[]).withTimeout(RecycleTimeout)
freeLib(ctx)
clearListeners(ctx[].eventRegistry)
rejectQueuedRequests(ctx)
ongoing[].setLen(0)
# Fire before the release: a thread that claims the freed slot first would
# otherwise take this fire as the answer to its own recycle.
let fireRes = ctx.recycleDoneSignal.fireSync()
if fireRes.isErr():
error "failed to fire recycleDoneSignal", err = fireRes.error
ctx.releaseClaim()
var ffiEventQueueSignalPtr {.threadvar.}: ThreadSignalPtr
# Stashed so the hook has no closure env.
proc ffiNotifyEventEnqueuedHook() {.gcsafe, raises: [].} =
if not ffiEventQueueSignalPtr.isNil():
let res = ffiEventQueueSignalPtr.fireSync()
if res.isErr():
error "failed to fire eventQueueSignal after enqueue", err = res.error
proc proveAlive(ctx: ptr FFIContext) =
## Advance the heartbeat the event thread polls; only movement matters, not value.
ctx.ffiHeartbeat.atomicInc()
proc ffiThreadBody[T](ctx: ptr FFIContext[T]) {.thread.} =
ffiCurrentEventRegistry = addr ctx[].eventRegistry
ffiCurrentEventQueue = addr ctx[].eventQueue
ffiCurrentEventQueueStuck = addr ctx[].eventQueueStuck
ffiEventQueueSignalPtr = ctx.eventQueueSignal
ffiCurrentNotifyEventEnqueued = ffiNotifyEventEnqueuedHook
onFFIThread = true
logging.setupLog(logging.LogLevel.DEBUG, logging.LogFormat.TEXT)
defer:
onFFIThread = false
# Free handle refs on the thread that allocated them (refc heap is thread-local).
ctx[].handles.releaseAll()
# Let the event thread stop draining and exit; wake it so it notices now.
ctx.ffiThreadExited.store(true)
ctx.eventQueueSignal.fireSync().isOkOr:
error "failed to wake event thread on FFI thread exit", err = error
# Unblocks destroyFFIContext's bounded wait.
let fireRes = ctx.threadExitSignal.fireSync()
if fireRes.isErr():
error "failed to fire threadExitSignal on FFI thread exit", err = fireRes.error
let ffiRun = proc(ctx: ptr FFIContext[T]) {.async.} =
var ffiReqHandler: T # main library object (Waku, LibP2P, SDS, …)
# Tracked so shutdown can drain them; abandoning a future leaks its request.
var pending: seq[Future[void]] = @[]
proc cleanFinishedRequests() =
var i = 0
while i < pending.len:
if not pending[i].finished():
inc i
continue
pending.del(i)
proc processQueue() =
## Drain fully: one wake can stand for many submits.
while true:
var request = ctx.reqQueueBank.mergeQueues()
if request.isNil():
break
while not request.isNil():
let nextRequest = request[].next # read before processRequest frees it
# Tick per dispatch so a backlog can't flatline the heartbeat mid-drain.
ctx.proveAlive()
if ctx.myLib.isNil():
# Must stay inside the closure: keeps `ffiReqHandler` alive across awaits.
ctx.myLib = addr ffiReqHandler
pending.add processRequest(request, ctx)
request = nextRequest
while ctx.running.load():
ctx.proveAlive()
# Recycle requested by the ffiDtor: drain + free lib + release the slot,
# keeping this thread alive for the next createFFIContext to reuse.
var expected = CtxLifecycle.RecyclePending
if ctx.lifecycle.compareExchange(expected, CtxLifecycle.Recycling):
await recycleContext(ctx, addr pending)
continue
# A submit that read `Active` just before the recycle can still land here.
# Fail it rather than run it against the library of the next owner.
if ctx.lifecycle.load() != CtxLifecycle.Active:
rejectQueuedRequests(ctx)
discard await ctx.reqSignal.wait().withTimeout(chronos.milliseconds(100))
continue
cleanFinishedRequests()
# Block until a submit signals us, or at most 100ms.
discard await ctx.reqSignal.wait().withTimeout(chronos.milliseconds(100))
processQueue()
# Drain once more for requests enqueued just before `running` flipped.
processQueue()
cleanFinishedRequests()
if pending.len > 0:
try:
await allFutures(pending)
except CatchableError as e:
error "draining pending FFI requests on shutdown raised", error = e.msg
# Run the library's async {.ffiDtor.} shutdown before join if one exists and a request populated `myLib`; exceptions logged, never propagated.
let teardown = ffiTeardownHook[T]()
if not teardown.isNil() and not ctx.myLib.isNil():
try:
await teardown(ctx.myLib)
except CatchableError as e:
error "library teardown raised on shutdown", error = e.msg
waitFor ffiRun(ctx)
+210 -37
View File
@@ -1,64 +1,237 @@
## This file contains the base message request type that will be handled.
## The requests are created by the main thread and processed by
## the FFI Thread.
## Request blob passed main→FFI thread. Uses libc malloc/free (not Nim
## allocShared) so a producer thread exiting before the FFI thread frees can't
## dangle into reclaimed per-thread ORC TLS.
import std/[json, macros], results, tables
import system/ansi_c
import results
import chronos
import ./ffi_config
when not singleThreaded:
import chronos/threadsync # ThreadSignalPtr requires threads enabled
import ./ffi_types, ./internal/ffi_macro, ./alloc
import ./ffi_types, ./alloc, ./cbor_serial
const EmptyErrorMarker = "unknown error"
## RET_ERR fallback message; keeps the callback msg ptr non-nil.
const MaxScalarArgs* = 8
## Inline scalar fast-path capacity; more params can't use it (compile-time checked).
type FFIThreadRequest* = object
callback: FFICallBack
userData: pointer
reqId*: cstring
reqContent*: pointer
callback*: FFICallBack
userData*: pointer
reqId*: cstring ## Req type name used to look up the handler.
data*: ptr UncheckedArray[byte]
## Owned request payload: CBOR-encoded, or a packed `_CWire` struct on the
## `abi = c` path. Nil on the scalar fast path.
dataLen*: int
rawReply*: bool
## CBOR-free request (scalar fast path or `abi = c`): the reply is raw bytes,
## so a 0-length one is a real empty string, not a CBOR "no value".
scalarArgs*: array[MaxScalarArgs, uint64]
## Inlined scalar args (no per-call c_malloc); a plain array keeps
## `deleteRequest` unaliased.
next*: ptr FFIThreadRequest
## Intrusive queue link; request doubles as its own node so enqueue needs no
## ORC-heap alloc.
responded*: bool
## De-dupes the callback across timeout/completion; both on FFI thread, no race.
func ffiPackScalar*[T](x: T): uint64 =
## Bit-cast one scalar into a uint64 request slot. Reverse with `ffiUnpackScalar`.
when T is SomeFloat:
cast[uint64](float64(x))
elif T is bool:
uint64(ord(x))
elif T is SomeSignedInt:
cast[uint64](int64(x))
else:
uint64(x)
func ffiUnpackScalar*[T](u: uint64, _: typedesc[T]): T =
## Inverse of `ffiPackScalar`.
when T is SomeFloat:
T(cast[float64](u))
elif T is bool:
u != 0'u64
elif T is SomeSignedInt:
T(cast[int64](u))
else:
T(u)
proc allocBaseRequest(
callback: FFICallBack, userData: pointer, reqId: cstring
): ptr FFIThreadRequest =
## c_malloc the envelope and set routing fields; payload set by a helper below.
var ret = cast[ptr FFIThreadRequest](c_malloc(csize_t(sizeof(FFIThreadRequest))))
ret[].callback = callback
ret[].userData = userData
ret[].reqId = reqId.alloc()
ret[].data = nil
ret[].dataLen = 0
ret[].rawReply = false
ret[].next = nil
ret[].responded = false
return ret
proc copySharedPayload(req: ptr FFIThreadRequest, data: ptr byte, dataLen: int) =
## c_malloc a fresh buffer and copy `dataLen` bytes in; empty payload is a no-op.
if dataLen > 0 and not data.isNil():
req[].data = cast[ptr UncheckedArray[byte]](c_malloc(csize_t(dataLen)))
copyMem(req[].data, data, dataLen)
req[].dataLen = dataLen
proc adoptOwnedSharedPayload(
req: ptr FFIThreadRequest, data: ptr UncheckedArray[byte], dataLen: int
) =
## Embed an already-c_malloc'd buffer without copying; frees a zero-length
## non-nil buffer so it doesn't leak.
if dataLen > 0 and not data.isNil():
req[].data = data
req[].dataLen = dataLen
elif not data.isNil():
c_free(data)
proc initFromPtr*(
T: typedesc[FFIThreadRequest],
callback: FFICallBack,
userData: pointer,
reqId: cstring,
data: ptr byte,
dataLen: int,
): ptr type T =
## Copies raw ptr+len into a fresh buffer owned by the returned request.
var ret = allocBaseRequest(callback, userData, reqId)
copySharedPayload(ret, data, dataLen)
return ret
proc init*(
T: typedesc[FFIThreadRequest],
callback: FFICallBack,
userData: pointer,
reqId: cstring,
reqContent: pointer,
data: openArray[byte],
): ptr type T =
var ret = createShared(FFIThreadRequest)
ret[].callback = callback
ret[].userData = userData
ret[].reqId = reqId.alloc()
ret[].reqContent = reqContent
## Like `initFromPtr` but from a Nim openArray.
let dataPtr =
if data.len > 0:
cast[ptr byte](unsafeAddr data[0])
else:
nil
initFromPtr(T, callback, userData, reqId, dataPtr, data.len)
proc initFromOwnedShared*(
T: typedesc[FFIThreadRequest],
callback: FFICallBack,
userData: pointer,
reqId: cstring,
data: ptr UncheckedArray[byte],
dataLen: int,
rawReply: bool = false,
): ptr type T =
## Adopts an already-c_malloc'd buffer (no copy); `deleteRequest` c_frees it.
## Pass `(nil, 0)` for an empty payload. Set `rawReply` when the handler answers
## with raw (non-CBOR) bytes, so an empty reply reads as a real empty value.
var ret = allocBaseRequest(callback, userData, reqId)
adoptOwnedSharedPayload(ret, data, dataLen)
ret[].rawReply = rawReply
return ret
proc deleteRequest(request: ptr FFIThreadRequest) =
deallocShared(request[].reqId)
deallocShared(request)
proc initScalar*(
T: typedesc[FFIThreadRequest],
callback: FFICallBack,
userData: pointer,
reqId: cstring,
args: varargs[uint64],
): ptr type T =
## Scalar-fast-path request: packed args ride inline, no payload c_malloc.
doAssert args.len <= MaxScalarArgs,
"initScalar: " & $args.len & " scalar args exceed MaxScalarArgs (" & $MaxScalarArgs &
")"
var ret = allocBaseRequest(callback, userData, reqId)
ret[].rawReply = true
for i in 0 ..< args.len:
ret[].scalarArgs[i] = args[i]
ret
proc handleRes*[T: string | void](
res: Result[T, string], request: ptr FFIThreadRequest
) =
## Handles the Result responses, which can either be Result[string, string] or
## Result[void, string].
func ffiRawRetBytes*[T](x: T): seq[byte] =
## CBOR-free handler result as raw bytes: string/cstring ride as UTF-8, other
## scalars as the 8-byte native image of `ffiPackScalar(x)`.
when T is string:
var b = newSeq[byte](x.len)
if x.len > 0:
copyMem(addr b[0], unsafeAddr x[0], x.len)
b
elif T is cstring:
let n = x.len
var b = newSeq[byte](n)
if n > 0:
copyMem(addr b[0], cast[pointer](x), n)
b
else:
let u = ffiPackScalar(x)
var b = newSeq[byte](sizeof(uint64))
copyMem(addr b[0], unsafeAddr u, sizeof(uint64))
b
defer:
deleteRequest(request)
proc deleteRequest*(request: ptr FFIThreadRequest) =
if not request[].data.isNil:
c_free(request[].data)
if not request[].reqId.isNil:
c_free(cast[pointer](request[].reqId))
c_free(request)
proc fireCallback*(res: Result[seq[byte], string], request: ptr FFIThreadRequest) =
## Answers the foreign callback at most once (timeout and completion both call
## it). Does NOT free the request; `handleRes` does.
if request[].responded:
return
request[].responded = true
if res.isErr():
foreignThreadGc:
let msg = "ffi error: handleRes fireSyncRes error: " & $res.error
let msg = if res.error.len > 0: res.error else: EmptyErrorMarker
request[].callback(
RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), request[].userData
RET_ERR, unsafeAddr msg[0], cast[csize_t](msg.len), request[].userData
)
return
foreignThreadGc:
var msg: cstring = ""
when T is string:
msg = res.get().cstring()
let bytes = res.get()
if bytes.len > 0:
request[].callback(
RET_OK,
cast[ptr cchar](unsafeAddr bytes[0]),
cast[csize_t](bytes.len),
request[].userData,
)
elif request[].rawReply:
# A CBOR-free 0-byte return is a real empty string, not CBOR "no value".
var empty: byte
request[].callback(
RET_OK, cast[ptr cchar](addr empty), 0.csize_t, request[].userData
)
else:
# Always hand the callback a real buffer; CBOR null marks "no value".
var sentinel = CborNullByte
request[].callback(
RET_OK, cast[ptr cchar](addr sentinel), 1.csize_t, request[].userData
)
proc fireStaleWarn*(request: ptr FFIThreadRequest, elapsedMs: int64) =
## In-flight ping; leaves `responded` unset and may fire many times — the
## terminal RET_OK/RET_ERR is still owed.
if request[].responded:
return
foreignThreadGc:
let msg = $elapsedMs
request[].callback(
RET_OK, unsafeAddr msg[0], cast[csize_t](len(msg)), request[].userData
RET_STALE_WARN,
cast[ptr cchar](unsafeAddr msg[0]),
cast[csize_t](msg.len),
request[].userData,
)
return
proc nilProcess*(reqId: cstring): Future[Result[string, string]] {.async.} =
proc handleRes*(res: Result[seq[byte], string], request: ptr FFIThreadRequest) =
## Terminal step: delivers the response and frees the request exactly once.
defer:
deleteRequest(request)
fireCallback(res, request)
proc nilProcess*(reqId: cstring): Future[Result[seq[byte], string]] {.async.} =
return err("This request type is not implemented: " & $reqId)
+10 -17
View File
@@ -1,25 +1,23 @@
import std/tables
import chronos
################################################################################
### Exported types
type FFICallBack* = proc(
callerRet: cint, msg: ptr cchar, len: csize_t, userData: pointer
) {.cdecl, gcsafe, raises: [].}
## Result-delivery callback. `RET_OK`/`RET_ERR` fire once and end the request;
## `RET_STALE_WARN` may fire repeatedly before them.
const RET_OK*: cint = 0
const RET_ERR*: cint = 1
const RET_MISSING_CALLBACK*: cint = 2
const RET_STALE_WARN*: cint = 3
## Non-terminal: request still in flight, fires every `StaleWarnInterval` with
## `msg` = elapsed ms as decimal ASCII, always followed by a terminal code.
### End of exported types
################################################################################
################################################################################
### FFI utils
type FFIRequestProc* =
proc(request: pointer, reqHandler: pointer): Future[Result[string, string]] {.async.}
type FFIRequestProc* = proc(
request: pointer, reqHandler: pointer
): Future[Result[seq[byte], string]] {.async.}
## OK payload is a CBOR-encoded response body; errors are plain UTF-8.
template foreignThreadGc*(body: untyped) =
when declared(setupForeignThreadGc):
@@ -30,10 +28,5 @@ template foreignThreadGc*(body: untyped) =
when declared(tearDownForeignThreadGc):
tearDownForeignThreadGc()
## Registered requests table populated at compile time and never updated at run time.
## The key represents the request type name as cstring, e.g., "CreateNodeRequest".
## The value is a proc that handles the request asynchronously.
## Compile-time-populated table: request type name (cstring) -> async handler.
var registeredRequests*: Table[cstring, FFIRequestProc]
### End of FFI utils
################################################################################
@@ -0,0 +1,950 @@
## Compile-time helpers for the `abi = c` C-struct ABI: for each `{.ffi: "abi = c".}`
## object T, emits a `T_CWire` companion plus `cwirePack`/`cwireUnpack`/`cwireFree`.
## A `seq` may only be a top-level field (no single-field wire form to nest).
import std/macros
import ../codegen/meta
const
cwireItemsSuffix = "_items"
cwireLenSuffix = "_len"
var emittedCWireTypes {.compileTime.}: seq[string]
proc isCWireEmitted(typeName: string): bool {.compileTime.} =
# Indexed scan: `for x in seq` over a freshly-mutated compileTime seq goes stale on the Nim 2.2 VM.
for i in 0 ..< emittedCWireTypes.len:
if emittedCWireTypes[i] == typeName:
return true
false
proc markCWireEmitted(typeName: string) {.compileTime.} =
if not isCWireEmitted(typeName):
emittedCWireTypes.add(typeName)
proc cwireTypeName*(userTypeName: string): string =
userTypeName & "_CWire"
proc seqItemsField(obj, field: NimNode): NimNode =
newDotExpr(obj, ident($field & cwireItemsSuffix))
proc seqLenField(obj, field: NimNode): NimNode =
newDotExpr(obj, ident($field & cwireLenSuffix))
proc isStringType*(t: NimNode): bool =
t.kind == nnkIdent and ($t == "string" or $t == "cstring")
proc isBracketOf(t: NimNode, heads: openArray[string]): bool =
t.kind == nnkBracketExpr and t.len >= 2 and t[0].kind == nnkIdent and $t[0] in heads
proc isSeqType(t: NimNode): bool =
isBracketOf(t, ["seq"])
proc isOptionType(t: NimNode): bool =
isBracketOf(t, ["Option", "Maybe"])
proc isArrayType(t: NimNode): bool =
t.kind == nnkBracketExpr and t.len == 3 and t[0].kind == nnkIdent and $t[0] == "array"
proc isTupleType(t: NimNode): bool =
t.kind == nnkTupleTy
proc tupleComponents(t: NimNode): seq[tuple[name: string, typ: NimNode]] =
## Flatten a named tuple into `(name, type)` pairs, one per name.
var comps: seq[tuple[name: string, typ: NimNode]] = @[]
for defs in t:
if defs.kind != nnkIdentDefs:
error("cwire: only named tuples are supported: " & t.repr)
let typ = defs[^2]
for i in 0 ..< defs.len - 2:
comps.add((name: $defs[i], typ: typ))
comps
proc isKnownFFIType(name: string): bool {.compileTime.} =
for typeMeta in ffiTypeRegistry:
if typeMeta.name == name and not typeMeta.isEnum():
return true
false
proc isNestedFFIType(t: NimNode): bool =
## Enums are excluded: they are registered types but have no `_CWire`
## companion, and treating one as a nested struct would silently drop its value.
t.kind == nnkIdent and isKnownFFIType($t)
proc rejectEnumOnCWire(t: NimNode) {.compileTime.} =
if t.kind == nnkIdent and isFFIEnumTypeName($t):
error(
"cwire: `abi = c` does not support enum types yet, but " & $t &
" crosses the boundary here; use the CBOR ABI for this proc or type"
)
proc cwireNeedsFree(t: NimNode): bool =
## Whether the wire form of `t` owns allocations `cwireFree` must release.
if isStringType(t) or isNestedFFIType(t) or isOptionType(t) or isSeqType(t):
return true
if isArrayType(t):
return cwireNeedsFree(t[2])
if isTupleType(t):
for c in tupleComponents(t):
if cwireNeedsFree(c.typ):
return true
return false
false
proc rejectNestedSeq(t: NimNode) =
error(
"cwire: `seq` has no single-field wire form, so it can't nest inside " &
"another container (use it only as a top-level field): " & t.repr
)
proc wireValueType(t: NimNode): NimNode =
## Single-field wire form of value type `t`; `seq` has none, so it errors here.
rejectEnumOnCWire(t)
if isStringType(t):
return ident("cstring")
if isNestedFFIType(t):
return ident(cwireTypeName($t))
if isOptionType(t):
return nnkPtrTy.newTree(wireValueType(t[1]))
if isArrayType(t):
return
nnkBracketExpr.newTree(ident("array"), t[1].copyNimTree(), wireValueType(t[2]))
if isTupleType(t):
let wireTup = nnkTupleTy.newTree()
for c in tupleComponents(t):
wireTup.add(newIdentDefs(ident(c.name), wireValueType(c.typ)))
return wireTup
if isSeqType(t):
rejectNestedSeq(t)
t
proc wireFieldsFor(fieldName: string, fieldType: NimNode): seq[NimNode] =
## IdentDefs for one field; `seq[T]` splits into `_items` + `_len`.
if isSeqType(fieldType):
let elemWire = wireValueType(fieldType[1])
let itemsField = newIdentDefs(
ident(fieldName & cwireItemsSuffix),
nnkPtrTy.newTree(nnkBracketExpr.newTree(ident("UncheckedArray"), elemWire)),
newEmptyNode(),
)
let lenField =
newIdentDefs(ident(fieldName & cwireLenSuffix), ident("int"), newEmptyNode())
return @[itemsField, lenField]
@[newIdentDefs(ident(fieldName), wireValueType(fieldType), newEmptyNode())]
proc buildCWireTypeDef(
userTypeName: string, fieldNames: seq[string], fieldTypes: seq[NimNode]
): NimNode =
## Build the bare `nnkTypeDef` for the wire companion of `userTypeName`.
let wireName = ident(cwireTypeName(userTypeName))
var fields: seq[NimNode] = @[]
for i in 0 ..< fieldNames.len:
for fd in wireFieldsFor(fieldNames[i], fieldTypes[i]):
fields.add(fd)
let recList =
if fields.len > 0:
newTree(nnkRecList, fields)
else:
newTree(
nnkRecList, newIdentDefs(ident("_placeholder"), ident("uint8"), newEmptyNode())
)
let objTy = newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), recList)
newTree(nnkTypeDef, postfix(wireName, "*"), newEmptyNode(), objTy)
proc emitOptionPack(dstAccess, srcAccess, userType: NimNode): NimNode
proc emitOptionUnpack(dstAccess, srcAccess, userType: NimNode): NimNode
proc emitOptionFree(dstAccess, userType: NimNode): NimNode
proc emitArrayPack(dstAccess, srcAccess, arrType: NimNode): NimNode
proc emitArrayUnpack(dstAccess, srcAccess, arrType: NimNode): NimNode
proc emitArrayFree(dstAccess, arrType: NimNode): NimNode
proc emitTuplePack(dstAccess, srcAccess, tupType: NimNode): NimNode
proc emitTupleUnpack(dstAccess, srcAccess, tupType: NimNode): NimNode
proc emitTupleFree(dstAccess, tupType: NimNode): NimNode
proc emitElemPack(dstElem, srcElem, elemType: NimNode): NimNode =
## Pack one value; recurses through nested ffi/Option/array/tuple, POD copied.
if isStringType(elemType):
return newAssignment(dstElem, newCall(ident("cwireAllocStr"), srcElem))
if isNestedFFIType(elemType):
return newCall(ident("cwirePack"), dstElem, srcElem)
if isOptionType(elemType):
return emitOptionPack(dstElem, srcElem, elemType)
if isArrayType(elemType):
return emitArrayPack(dstElem, srcElem, elemType)
if isTupleType(elemType):
return emitTuplePack(dstElem, srcElem, elemType)
if isSeqType(elemType):
rejectNestedSeq(elemType)
newAssignment(dstElem, srcElem)
proc emitElemUnpack(dstElem, srcElem, elemType: NimNode): NimNode =
## Inverse of `emitElemPack`: copy one value back into Nim memory.
if isStringType(elemType):
return newAssignment(dstElem, newCall(ident("$"), srcElem))
if isNestedFFIType(elemType):
return newAssignment(dstElem, newCall(ident("cwireUnpack"), srcElem))
if isOptionType(elemType):
return emitOptionUnpack(dstElem, srcElem, elemType)
if isArrayType(elemType):
return emitArrayUnpack(dstElem, srcElem, elemType)
if isTupleType(elemType):
return emitTupleUnpack(dstElem, srcElem, elemType)
if isSeqType(elemType):
rejectNestedSeq(elemType)
newAssignment(dstElem, srcElem)
proc emitElemFree(elemAccess, elemType: NimNode): NimNode =
## Free one value, or `nnkEmpty` for POD.
if isStringType(elemType):
return newCall(ident("cwireFreeStr"), elemAccess)
if isNestedFFIType(elemType):
return newCall(ident("cwireFree"), elemAccess)
if isOptionType(elemType):
return emitOptionFree(elemAccess, elemType)
if isArrayType(elemType):
return emitArrayFree(elemAccess, elemType)
if isTupleType(elemType):
return emitTupleFree(elemAccess, elemType)
if isSeqType(elemType):
rejectNestedSeq(elemType)
newEmptyNode()
proc maybeStmt(n: NimNode): NimNode =
## `n` as a one-statement list, empty list when `nnkEmpty`.
if n.kind == nnkEmpty:
return newStmtList()
newStmtList(n)
proc indexLoop(access, idx, body: NimNode): NimNode =
## `for <idx> in low(access) .. high(access): body` (covers non-0-based ranges).
nnkForStmt.newTree(
idx,
nnkInfix.newTree(
ident(".."), newCall(ident("low"), access), newCall(ident("high"), access)
),
newStmtList(body),
)
proc emitArrayPack(dstAccess, srcAccess, arrType: NimNode): NimNode =
## Pack a fixed `array[N, T]` element-by-element into the inline wire array.
let idx = genSym(nskForVar, "i")
let body = emitElemPack(
nnkBracketExpr.newTree(dstAccess, idx),
nnkBracketExpr.newTree(srcAccess, idx),
arrType[2],
)
indexLoop(srcAccess, idx, body)
proc emitArrayUnpack(dstAccess, srcAccess, arrType: NimNode): NimNode =
## Inverse of `emitArrayPack`.
let idx = genSym(nskForVar, "i")
let body = emitElemUnpack(
nnkBracketExpr.newTree(dstAccess, idx),
nnkBracketExpr.newTree(srcAccess, idx),
arrType[2],
)
indexLoop(srcAccess, idx, body)
proc emitArrayFree(dstAccess, arrType: NimNode): NimNode =
## Free each array element; `nnkEmpty` when the element owns nothing.
if not cwireNeedsFree(arrType[2]):
return newEmptyNode()
let idx = genSym(nskForVar, "i")
let body = emitElemFree(nnkBracketExpr.newTree(dstAccess, idx), arrType[2])
indexLoop(dstAccess, idx, body)
proc emitTuplePack(dstAccess, srcAccess, tupType: NimNode): NimNode =
## Pack each named tuple component into the matching wire component.
let body = newStmtList()
for c in tupleComponents(tupType):
let nm = ident(c.name)
body.add(emitElemPack(newDotExpr(dstAccess, nm), newDotExpr(srcAccess, nm), c.typ))
body
proc emitTupleUnpack(dstAccess, srcAccess, tupType: NimNode): NimNode =
## Inverse of `emitTuplePack`.
let body = newStmtList()
for c in tupleComponents(tupType):
let nm = ident(c.name)
body.add(
emitElemUnpack(newDotExpr(dstAccess, nm), newDotExpr(srcAccess, nm), c.typ)
)
body
proc emitTupleFree(dstAccess, tupType: NimNode): NimNode =
## Free each tuple component that owns allocations.
if not cwireNeedsFree(tupType):
return newEmptyNode()
let body = newStmtList()
for c in tupleComponents(tupType):
body.add(maybeStmt(emitElemFree(newDotExpr(dstAccess, ident(c.name)), c.typ)))
body
proc emitSeqPack(dstObj, srcAccess, fieldNameIdent, userType: NimNode): NimNode =
## Pack a seq into an `allocShared` `UncheckedArray`; empty = nil items + 0 len.
let elemType = userType[1]
let wireElem = wireValueType(elemType)
let items = seqItemsField(dstObj, fieldNameIdent)
let count = seqLenField(dstObj, fieldNameIdent)
let bufType =
nnkPtrTy.newTree(nnkBracketExpr.newTree(ident("UncheckedArray"), wireElem))
let idx = genSym(nskForVar, "i")
let elemPack = emitElemPack(
nnkBracketExpr.newTree(items, idx), nnkBracketExpr.newTree(srcAccess, idx), elemType
)
let forLoop = nnkForStmt.newTree(
idx,
nnkInfix.newTree(
ident("..<"), newLit(0), newCall(newDotExpr(srcAccess, ident("len")))
),
newStmtList(elemPack),
)
quote:
if `srcAccess`.len() == 0:
`items` = nil
`count` = 0
else:
`items` = cast[`bufType`](cwireAllocBuf(sizeof(`wireElem`) * `srcAccess`.len()))
`forLoop`
`count` = `srcAccess`.len()
proc emitOptionPack(dstAccess, srcAccess, userType: NimNode): NimNode =
## Pack an Option into a `ptr`: some → `cwireAllocBuf` box, none → nil. Payload
## read into a local once so a composite inner type isn't re-`get()` per element.
let innerType = userType[1]
let wireInner = wireValueType(innerType)
let bufType = nnkPtrTy.newTree(wireInner)
let innerVal = genSym(nskLet, "innerVal")
let elemPack = emitElemPack(nnkBracketExpr.newTree(dstAccess), innerVal, innerType)
quote:
if `srcAccess`.isSome():
`dstAccess` = cast[`bufType`](cwireAllocBuf(sizeof(`wireInner`)))
let `innerVal` = `srcAccess`.get()
`elemPack`
else:
`dstAccess` = nil
proc emitPackStmt(dstObj, srcObj, fieldNameIdent, userType: NimNode): seq[NimNode] =
## Populate `dstObj.<field>` from `srcObj.<field>`.
let srcAccess = newDotExpr(srcObj, fieldNameIdent)
let dstAccess = newDotExpr(dstObj, fieldNameIdent)
if isSeqType(userType):
return @[emitSeqPack(dstObj, srcAccess, fieldNameIdent, userType)]
@[emitElemPack(dstAccess, srcAccess, userType)]
proc emitSeqUnpack(dstAccess, srcObj, fieldNameIdent, userType: NimNode): NimNode =
## Rebuild a Nim seq from the `_items`/`_len` wire pair.
let elemType = userType[1]
let items = seqItemsField(srcObj, fieldNameIdent)
let count = seqLenField(srcObj, fieldNameIdent)
let elemVar = genSym(nskVar, "elem")
let idx = genSym(nskForVar, "i")
let elemUnpack = emitElemUnpack(elemVar, nnkBracketExpr.newTree(items, idx), elemType)
quote:
`dstAccess` = @[]
for `idx` in 0 ..< `count`:
var `elemVar`: `elemType`
`elemUnpack`
`dstAccess`.add(`elemVar`)
proc emitOptionUnpack(dstAccess, srcAccess, userType: NimNode): NimNode =
## Rebuild an Option from a wire `ptr`: nil → none, else unpack the pointee.
let innerType = userType[1]
let elemVar = genSym(nskVar, "innerVal")
let elemUnpack = emitElemUnpack(elemVar, nnkBracketExpr.newTree(srcAccess), innerType)
quote:
if `srcAccess`.isNil():
`dstAccess` = none(`innerType`)
else:
var `elemVar`: `innerType`
`elemUnpack`
`dstAccess` = some(`elemVar`)
proc emitUnpackStmt(
resultObj, srcObj, fieldNameIdent, userType: NimNode
): seq[NimNode] =
## Fill `resultObj.<field>` from `srcObj.<field>`.
let srcAccess = newDotExpr(srcObj, fieldNameIdent)
let dstAccess = newDotExpr(resultObj, fieldNameIdent)
if isSeqType(userType):
return @[emitSeqUnpack(dstAccess, srcObj, fieldNameIdent, userType)]
@[emitElemUnpack(dstAccess, srcAccess, userType)]
proc emitSeqFree(dstObj, fieldNameIdent, userType: NimNode): NimNode =
## Free a seq field: each element (skipped for POD), then the shared buffer.
let elemType = userType[1]
let items = seqItemsField(dstObj, fieldNameIdent)
let count = seqLenField(dstObj, fieldNameIdent)
let idx = genSym(nskForVar, "i")
let elemFree = emitElemFree(nnkBracketExpr.newTree(items, idx), elemType)
let freeLoop =
if elemFree.kind == nnkEmpty:
newStmtList()
else:
newStmtList(
nnkForStmt.newTree(
idx, nnkInfix.newTree(ident("..<"), newLit(0), count), newStmtList(elemFree)
)
)
quote:
if not `items`.isNil():
`freeLoop`
cwireFreeBuf(`items`)
`items` = nil
`count` = 0
proc emitOptionFree(dstAccess, userType: NimNode): NimNode =
## Free an Option field: the pointee (skipped for POD), then the box.
let innerType = userType[1]
let freeInner = maybeStmt(emitElemFree(nnkBracketExpr.newTree(dstAccess), innerType))
quote:
if not `dstAccess`.isNil():
`freeInner`
cwireFreeBuf(`dstAccess`)
`dstAccess` = nil
proc emitFreeStmt(dstObj, fieldNameIdent, userType: NimNode): seq[NimNode] =
## Release `dstObj.<field>`: free cstrings/arrays/pointers; POD frees nothing.
let dstAccess = newDotExpr(dstObj, fieldNameIdent)
if isSeqType(userType):
return @[emitSeqFree(dstObj, fieldNameIdent, userType)]
let elemFree = emitElemFree(dstAccess, userType)
if elemFree.kind == nnkEmpty:
return @[]
@[elemFree]
proc buildCWireProcs(
userTypeName: string, fieldNames: seq[string], fieldTypes: seq[NimNode]
): seq[NimNode] =
## Generate public cwirePack / cwireUnpack / cwireFree procs for `userTypeName`.
let userName = ident(userTypeName)
let wireName = ident(cwireTypeName(userTypeName))
let packDst = ident("dst")
let packSrc = ident("src")
var packBody = newStmtList()
for i in 0 ..< fieldNames.len:
let fIdent = ident(fieldNames[i])
for s in emitPackStmt(packDst, packSrc, fIdent, fieldTypes[i]):
packBody.add(s)
if fieldNames.len == 0:
packBody.add quote do:
discard
let packProc = newProc(
name = postfix(ident("cwirePack"), "*"),
params = @[
newEmptyNode(),
newIdentDefs(packDst, nnkVarTy.newTree(wireName)),
newIdentDefs(packSrc, userName),
],
body = packBody,
)
let unpSrc = ident("src")
let unpRes = ident("res")
var unpBody = newStmtList()
unpBody.add quote do:
var `unpRes`: `userName`
for i in 0 ..< fieldNames.len:
let fIdent = ident(fieldNames[i])
for s in emitUnpackStmt(unpRes, unpSrc, fIdent, fieldTypes[i]):
unpBody.add(s)
unpBody.add quote do:
return `unpRes`
let unpProc = newProc(
name = postfix(ident("cwireUnpack"), "*"),
params = @[userName, newIdentDefs(unpSrc, wireName)],
body = unpBody,
)
let freeDst = ident("dst")
var freeBody = newStmtList()
for i in 0 ..< fieldNames.len:
let fIdent = ident(fieldNames[i])
for s in emitFreeStmt(freeDst, fIdent, fieldTypes[i]):
freeBody.add(s)
if freeBody.len == 0:
freeBody.add quote do:
discard
let freeProc = newProc(
name = postfix(ident("cwireFree"), "*"),
params = @[newEmptyNode(), newIdentDefs(freeDst, nnkVarTy.newTree(wireName))],
body = freeBody,
)
@[packProc, unpProc, freeProc]
proc fieldInfoForType(
typeName: string
): tuple[names: seq[string], types: seq[NimNode]] {.compileTime.} =
## Look up an ffi type's fields from the registry, parsing each recorded type.
for typeMeta in ffiTypeRegistry:
if typeMeta.name != typeName:
continue
var names: seq[string] = @[]
var types: seq[NimNode] = @[]
for f in typeMeta.fields:
names.add(f.name)
types.add(parseExpr(f.typeName))
return (names, types)
error("fieldInfoForType: ffi type '" & typeName & "' not in registry")
proc collectNestedFFITypes(
fieldTypes: seq[NimNode], deps: var seq[string]
) {.compileTime.} =
## Append (deduped) nested ffi type names in `fieldTypes`, recursing through
## `seq`/`Option`/`array`/`tuple`.
for t in fieldTypes:
if isNestedFFIType(t):
let n = $t
if n notin deps:
deps.add(n)
elif isSeqType(t) or isOptionType(t):
collectNestedFFITypes(@[t[1]], deps)
elif isArrayType(t):
collectNestedFFITypes(@[t[2]], deps)
elif isTupleType(t):
for c in tupleComponents(t):
collectNestedFFITypes(@[c.typ], deps)
proc ensureCWireFor(typeName: string, sink: NimNode) {.compileTime.} =
## Idempotent: append `typeName`'s cwire companion + procs to `sink` if not yet
## emitted. Nested ffi deps are ensured first so the AST is self-contained.
if isCWireEmitted(typeName):
return
let info = fieldInfoForType(typeName)
var deps: seq[string] = @[]
collectNestedFFITypes(info.types, deps)
for dep in deps:
ensureCWireFor(dep, sink)
markCWireEmitted(typeName)
let section = newNimNode(nnkTypeSection)
section.add(buildCWireTypeDef(typeName, info.names, info.types))
sink.add(section)
for p in buildCWireProcs(typeName, info.names, info.types):
sink.add(p)
proc flushCWireCompanions*(): NimNode {.compileTime.} =
## Emit the `_CWire` companion + procs for every registered `abi = c` type.
let sink = newStmtList()
for typeMeta in ffiTypeRegistry:
if typeMeta.abiFormat == ABIFormat.C:
ensureCWireFor(typeMeta.name, sink)
sink
## abi = c proc dispatch. The foreign surface is CBOR-free (the `_CWire` structs are
## the C ABI) but transport reuses the CBOR request path internally. Emitted at
## `genBindings()` time (after `flushCWireCompanions`) so the companions are in scope.
type
CAbiKind = enum
cakMethod
cakCtor
cakStatic
CAbiSpec = object
kind: CAbiKind
exportName: string ## snake_case C symbol, e.g. "echo_shout"
libType: NimNode ## library value type, e.g. `Echo`
envelope: NimNode ## per-proc Req type, e.g. `EchoShoutReq`
paramNames: seq[string] ## envelope field names (the extra params)
paramTypes: seq[NimNode] ## envelope field types
respType: NimNode ## method result T; empty for a ctor
handler: NimNode
## FFI-thread handler, deferred here so it lands after the `_CWire`
## companions it packs/unpacks through.
var cAbiSpecs {.compileTime.}: seq[CAbiSpec]
proc copyTypes(types: seq[NimNode]): seq[NimNode] {.compileTime.} =
var res: seq[NimNode] = @[]
for t in types:
res.add(t.copyNimTree())
res
proc registerCAbiProc*(
isStatic: bool,
exportName: string,
libType, envelope: NimNode,
paramNames: seq[string],
paramTypes: seq[NimNode],
respType, handler: NimNode,
) {.compileTime.} =
## Record an `abi = c` method (or `{.ffiStatic.}` proc) for `flushCAbiDispatch`.
## Nodes are `copyNimTree` frozen: reusing the Req section's originals (bound to
## `nnkSym`) would ICE.
cAbiSpecs.add(
CAbiSpec(
kind: if isStatic: cakStatic else: cakMethod,
exportName: exportName,
libType: libType.copyNimTree(),
envelope: envelope.copyNimTree(),
paramNames: paramNames,
paramTypes: copyTypes(paramTypes),
respType: respType.copyNimTree(),
handler: handler.copyNimTree(),
)
)
proc registerCAbiCtor*(
exportName: string,
libType, envelope: NimNode,
paramNames: seq[string],
paramTypes: seq[NimNode],
handler: NimNode,
) {.compileTime.} =
## Record an `abi = c` ctor for `flushCAbiDispatch`; see `registerCAbiProc`
## for why nodes are `copyNimTree` frozen.
cAbiSpecs.add(
CAbiSpec(
kind: cakCtor,
exportName: exportName,
libType: libType.copyNimTree(),
envelope: envelope.copyNimTree(),
paramNames: paramNames,
paramTypes: copyTypes(paramTypes),
respType: newEmptyNode(),
handler: handler.copyNimTree(),
)
)
proc cdeclReplyPragma(): NimNode =
nnkPragma.newTree(
ident("cdecl"),
ident("gcsafe"),
nnkExprColonExpr.newTree(ident("raises"), nnkBracket.newTree()),
)
proc cAbiCbType(replyType: NimNode): NimNode =
## The caller's typed reply callback proc type.
let fp = nnkFormalParams.newTree(
newEmptyNode(),
newIdentDefs(ident("err"), ident("cint")),
newIdentDefs(ident("reply"), replyType),
newIdentDefs(ident("errMsg"), ident("cstring")),
newIdentDefs(ident("ud"), ident("pointer")),
)
nnkProcTy.newTree(fp, cdeclReplyPragma())
proc boxTypeDef(boxName, cbType: NimNode): NimNode =
## Box object holding the caller's callback + user data across the thread hop.
let recList = nnkRecList.newTree(
newIdentDefs(ident("fn"), cbType), newIdentDefs(ident("ud"), ident("pointer"))
)
let objTy = nnkObjectTy.newTree(newEmptyNode(), newEmptyNode(), recList)
nnkTypeSection.newTree(nnkTypeDef.newTree(boxName, newEmptyNode(), objTy))
proc replyTrampProc(trampName, body: NimNode): NimNode =
## `FFICallBack`-shaped proc: runs on the FFI thread, converts the reply, frees the box.
newProc(
name = trampName,
params = @[
newEmptyNode(),
newIdentDefs(ident("ret"), ident("cint")),
newIdentDefs(ident("msg"), nnkPtrTy.newTree(ident("cchar"))),
newIdentDefs(ident("len"), ident("csize_t")),
newIdentDefs(ident("ud"), ident("pointer")),
],
body = body,
pragmas = cdeclReplyPragma(),
)
proc objectTrampBody(boxName, respWire: NimNode): NimNode =
## Reply trampoline for an object return: the payload is already the packed
## `_CWire` image, so hand its address straight to the caller and release the
## buffers it owns. `reply` is nil only on error.
quote:
let box = cast[ptr `boxName`](ud)
if box.isNil():
return
if ret == RET_STALE_WARN:
# Non-terminal progress signal: keep the box, don't read the payload.
return
defer:
freeBox(box)
if box.fn.isNil():
return
try:
if ret != RET_OK:
var em = newString(int(len))
if int(len) > 0:
copyMem(addr em[0], msg, int(len))
box.fn(ret, nil, em.cstring, box.ud)
return
if msg.isNil() or int(len) != sizeof(`respWire`):
box.fn(RET_ERR, nil, "abi = c reply: unexpected payload size".cstring, box.ud)
return
var wire = cast[ptr `respWire`](msg)[]
box.fn(RET_OK, addr wire, "".cstring, box.ud)
cwireFree(wire)
except CatchableError as e:
box.fn(RET_ERR, nil, e.msg.cstring, box.ud)
proc stringTrampBody(boxName: NimNode): NimNode =
## Reply trampoline for a `string` return (and the ctor's address string): the
## payload is raw length-delimited UTF-8, so copy it into a NUL-terminated
## `cstring`. Whichever of reply/error is unused rides as empty, safe to deref.
quote:
let box = cast[ptr `boxName`](ud)
if box.isNil():
return
if ret == RET_STALE_WARN:
# Non-terminal progress signal: keep the box, don't read the payload.
return
defer:
freeBox(box)
if box.fn.isNil():
return
try:
var payload = newString(int(len))
if int(len) > 0 and not msg.isNil():
copyMem(addr payload[0], msg, int(len))
if ret != RET_OK:
box.fn(ret, "".cstring, payload.cstring, box.ud)
return
box.fn(RET_OK, payload.cstring, "".cstring, box.ud)
except CatchableError as e:
box.fn(RET_ERR, "".cstring, e.msg.cstring, box.ud)
proc ctxBindingGuard(
poolIdent, emptyReply, ctxIdent: NimNode, isStatic: bool
): NimNode {.compileTime.} =
## Prologue that binds `ctxIdent`: a method validates the ctx it was handed, a
## static resolves the library's shared one.
#
# Any call can be a host thread's first entry. The body allocates via the GC
# on the calling thread, so register it first; initializeLibrary is idempotent.
# Raw AST: `when declared` of an undeclared symbol inside `quote` ICEs.
let initGuard = nnkWhenStmt.newTree(
nnkElifBranch.newTree(
newCall(ident("declared"), ident("initializeLibrary")),
newStmtList(newCall(ident("initializeLibrary"))),
)
)
if not isStatic:
let methodGuard = quote:
if onReply.isNil():
return RET_MISSING_CALLBACK
if not `poolIdent`.isValidCtx(cast[pointer](`ctxIdent`)):
onReply(
RET_ERR, `emptyReply`, "ctx is not a valid FFI context".cstring, userData
)
return RET_ERR
methodGuard.insert(0, initGuard)
return methodGuard
let guard = quote:
if onReply.isNil():
return RET_MISSING_CALLBACK
let `ctxIdent` = `poolIdent`.staticFFIContext().valueOr:
let errStr = "ffiStatic: " & error
onReply(RET_ERR, `emptyReply`, errStr.cstring, userData)
return RET_ERR
guard.insert(0, initGuard)
guard
proc exportedProc(
spec: CAbiSpec,
boxName, envWire, trampName, poolIdent, cbType: NimNode,
isStatic: bool,
): NimNode =
# `cwireUnpack`/`cwirePack` alloc on the calling thread; `ctxBindingGuard`
# registered it. No teardown: it would free the heap of a host thread still
# calling in. A host thread that exits leaks its heap; accepted.
let envName = spec.envelope
let ctxIdent = ident("ctx")
# String reply: empty non-nil cstring on error; object reply: nil ptr gated by err_code.
let emptyReply =
if isStringType(spec.respType):
newDotExpr(newLit(""), ident("cstring"))
else:
newNilLit()
let body = quote:
var ownedWire: `envWire`
cwirePack(ownedWire, cwireUnpack(req[]))
let ownedCopy = cwireOwnedCopy(ownedWire)
if ownedCopy.isNil():
cwireFree(ownedWire)
onReply(RET_ERR, `emptyReply`, "out of memory".cstring, userData)
return RET_ERR
let reqBuf = cast[ptr UncheckedArray[byte]](ownedCopy)
let box = cast[ptr `boxName`](allocBox(sizeof(`boxName`)))
box.fn = onReply
box.ud = userData
let typeStr = $`envName`
let reqPtr = FFIThreadRequest.initFromOwnedShared(
`trampName`, box, typeStr.cstring, reqBuf, sizeof(`envWire`), rawReply = true
)
let sendRes =
try:
ffi_context.sendRequestToFFIThread(`ctxIdent`, reqPtr)
except Exception as e:
Result[void, string].err("sendRequestToFFIThread exception: " & e.msg)
if sendRes.isErr():
# A rejected send already `deleteRequest`ed the struct copy, which frees only
# the struct itself; `ownedWire` still aliases its field buffers, so free them
# here — on success the FFI thread's unpack does it instead.
cwireFree(ownedWire)
onReply(RET_ERR, `emptyReply`, sendRes.error.cstring, userData)
return RET_ERR
return RET_OK
let fullBody = ctxBindingGuard(poolIdent, emptyReply, ctxIdent, isStatic)
for stmt in body:
fullBody.add(stmt)
var params = @[
ident("cint"),
newIdentDefs(ident("onReply"), cbType),
newIdentDefs(ident("userData"), ident("pointer")),
newIdentDefs(ident("req"), nnkPtrTy.newTree(envWire)),
]
if not isStatic:
let libFFICtx =
nnkPtrTy.newTree(nnkBracketExpr.newTree(ident("FFIContext"), spec.libType))
params.insert(newIdentDefs(ctxIdent, libFFICtx), 1)
newProc(
name = ident($envName & "CAbiExport"),
params = params,
body = fullBody,
pragmas = nnkPragma.newTree(
ident("dynlib"),
nnkExprColonExpr.newTree(ident("exportc"), newStrLitNode(spec.exportName)),
ident("cdecl"),
nnkExprColonExpr.newTree(ident("raises"), nnkBracket.newTree()),
),
)
proc exportedCtorProc(
spec: CAbiSpec, boxName, envWire, trampName, poolIdent, cbType: NimNode
): NimNode =
let envName = spec.envelope
# No `foreignThreadGc` (see exportedMethodProc). initGuard is built as raw AST because a `when declared` over an undeclared symbol inside `quote` ICEs.
let initGuard = nnkWhenStmt.newTree(
nnkElifBranch.newTree(
newCall(ident("declared"), ident("initializeLibrary")),
newStmtList(newCall(ident("initializeLibrary"))),
)
)
let body = quote:
let ctxRes = `poolIdent`.createFFIContext()
if ctxRes.isErr():
if not onCreated.isNil():
onCreated(
RET_ERR,
"".cstring,
("ffiCtor: failed to create FFIContext: " & $ctxRes.error).cstring,
userData,
)
return nil
let ctx = ctxRes.get()
var ownedWire: `envWire`
cwirePack(ownedWire, cwireUnpack(req[]))
let ownedCopy = cwireOwnedCopy(ownedWire)
if ownedCopy.isNil():
cwireFree(ownedWire)
if not onCreated.isNil():
onCreated(RET_ERR, "".cstring, "out of memory".cstring, userData)
return nil
let reqBuf = cast[ptr UncheckedArray[byte]](ownedCopy)
let box = cast[ptr `boxName`](allocBox(sizeof(`boxName`)))
box.fn = onCreated
box.ud = userData
let typeStr = $`envName`
let reqPtr = FFIThreadRequest.initFromOwnedShared(
`trampName`, box, typeStr.cstring, reqBuf, sizeof(`envWire`), rawReply = true
)
let sendRes =
try:
ctx.sendRequestToFFIThread(reqPtr)
except Exception as e:
Result[void, string].err("sendRequestToFFIThread exception: " & e.msg)
if sendRes.isErr():
# See exportedMethodProc: the rejected send freed the struct copy, not the
# field buffers `ownedWire` still aliases.
cwireFree(ownedWire)
if not onCreated.isNil():
onCreated(RET_ERR, "".cstring, sendRes.error.cstring, userData)
return nil
return cast[pointer](ctx)
body.insert(0, initGuard)
newProc(
name = ident($envName & "CAbiExport"),
params = @[
ident("pointer"),
newIdentDefs(ident("req"), nnkPtrTy.newTree(envWire)),
newIdentDefs(ident("onCreated"), cbType),
newIdentDefs(ident("userData"), ident("pointer")),
],
body = body,
pragmas = nnkPragma.newTree(
ident("dynlib"),
nnkExprColonExpr.newTree(ident("exportc"), newStrLitNode(spec.exportName)),
ident("cdecl"),
nnkExprColonExpr.newTree(ident("raises"), nnkBracket.newTree()),
),
)
proc ensureCWireForFields(
sink: NimNode, typeName: string, names: seq[string], types: seq[NimNode]
) {.compileTime.} =
## Emit the `_CWire` companion + procs for a synthetic per-proc Req envelope
## (not a user `{.ffi.}` type, so not in `ffiTypeRegistry`).
if isCWireEmitted(typeName):
return
var deps: seq[string] = @[]
collectNestedFFITypes(types, deps)
for dep in deps:
ensureCWireFor(dep, sink)
markCWireEmitted(typeName)
let section = newNimNode(nnkTypeSection)
section.add(buildCWireTypeDef(typeName, names, types))
sink.add(section)
for p in buildCWireProcs(typeName, names, types):
sink.add(p)
proc flushCAbiDispatch*(): NimNode {.compileTime.} =
## Emit the exported wrappers + reply trampolines for every registered
## `abi = c` proc. Runs after `flushCWireCompanions`.
let sink = newStmtList()
for spec in cAbiSpecs:
let envName = spec.envelope
ensureCWireForFields(sink, $envName, spec.paramNames, spec.paramTypes)
sink.add(spec.handler)
let envWire = ident(cwireTypeName($envName))
let boxName = ident($envName & "CBox")
let trampName = ident($envName & "CReply")
let poolIdent = ident($spec.libType & "FFIPool")
case spec.kind
of cakCtor:
let cbType = cAbiCbType(ident("cstring"))
sink.add(boxTypeDef(boxName, cbType))
sink.add(replyTrampProc(trampName, stringTrampBody(boxName)))
sink.add(exportedCtorProc(spec, boxName, envWire, trampName, poolIdent, cbType))
of cakMethod, cakStatic:
let isStatic = spec.kind == cakStatic
let rt = spec.respType
if isStringType(rt):
let cbType = cAbiCbType(ident("cstring"))
sink.add(boxTypeDef(boxName, cbType))
sink.add(replyTrampProc(trampName, stringTrampBody(boxName)))
sink.add(
exportedProc(spec, boxName, envWire, trampName, poolIdent, cbType, isStatic)
)
# `isKnownFFIType`, not just `nnkIdent`: a bare `int` is an ident too, and
# would otherwise reach for a `int_CWire` companion that is never emitted.
elif rt.kind == nnkIdent and isKnownFFIType($rt):
let respWire = ident(cwireTypeName($rt))
let cbType = cAbiCbType(nnkPtrTy.newTree(respWire))
sink.add(boxTypeDef(boxName, cbType))
sink.add(replyTrampProc(trampName, objectTrampBody(boxName, respWire)))
sink.add(
exportedProc(spec, boxName, envWire, trampName, poolIdent, cbType, isStatic)
)
else:
error(
"abi = c: unsupported response type for proc '" & spec.exportName & "': " &
rt.repr & " — reply with a `string` or an `{.ffi.}` object type. " &
"A scalar return is wired only for an all-scalar `{.ffi.}` method."
)
sink
+44
View File
@@ -0,0 +1,44 @@
## Memory helpers for the macro-generated `*_CWire` types — the flat C-ABI mirror
## of a Nim object, where strings and seq/Option payloads live in separate buffers
## the struct only points at. These procs allocate and free those buffers, and copy
## the struct across the hop to the FFI thread.
import ../alloc
proc cwireAllocBuf*(size: int): pointer =
## Buffer for a wire seq/Option payload. libc `malloc` rather than `allocShared`
## so one thread can allocate and a different thread can free (see ../alloc).
alloc.allocBox(size)
proc cwireFreeBuf*(p: pointer) =
## Frees a `cwireAllocBuf` buffer; does nothing if `p` is nil.
alloc.freeBox(p)
proc cwireAllocStr*(s: string): cstring {.inline.} =
## NUL-terminated copy of `s` for a wire string field; free with `cwireFreeStr`.
alloc.alloc(s)
proc cwireFreeStr*(s: var cstring) {.inline.} =
## Frees a wire string field and nils it, so freeing twice is harmless.
if s.isNil():
return
alloc.dealloc(s)
s = nil
func cwireStructBytes*[W](wire: W): seq[byte] =
## The struct's raw bytes, to hand a reply back over the FFI-thread hop. Copies
## the pointers, not what they point at, so `wire`'s buffers must stay alive
## until the receiver `cwireFree`s them.
var b = newSeq[byte](sizeof(W))
copyMem(addr b[0], unsafeAddr wire, sizeof(W))
b
proc cwireOwnedCopy*[W](wire: W): ptr W =
## The same shallow copy, into `malloc` memory the FFI thread adopts and frees;
## nil if the allocation fails. `copyMem` because assigning into raw `malloc`
## bytes would run ORC's copy hooks over uninitialised memory.
let p = cast[ptr W](alloc.allocBox(sizeof(W)))
if p.isNil():
return nil
copyMem(p, unsafeAddr wire, sizeof(W))
return p
@@ -0,0 +1,29 @@
## Compile-time pieces that more than one `{.ffi.}` codegen path shares.
import std/macros
func unwrapPostfix*(n: NimNode): NimNode =
## Strips the `*` of an exported name, so a name node reads the same whether
## the writer exported it or not.
return
if n.kind == nnkPostfix:
n[1]
else:
n
func procIdent*(prc: NimNode): NimNode =
return unwrapPostfix(prc[0])
proc buildLibReadyGuard*(
ctxHandlerName, libTypeName: NimNode
): NimNode {.compileTime.} =
## Rejects a request that reaches the FFI thread before the ctor stores a
## library. The guard applies only to a `ref` type. For an `object` type the
## fallback is a usable zero value, but for a `ref` type it is nil. The guard
## runs in the handler, behind the ctor in the queue. Thus a host can send a
## call before it waits for the create callback.
quote:
when `libTypeName` is ref:
if not `ctxHandlerName`[].libReady.load():
return
err("library is not initialized: the constructor failed or has not run yet")
+169
View File
@@ -0,0 +1,169 @@
## Simple synchronous C export for a nim-ffi library.
##
## `{.ffi.}` and `{.ffiCtor.}` give the async path. That path uses a context
## handle and encodes the data with CBOR. It fits a library that keeps state
## across many calls. `{.ffiExport.}` covers the other common case: a few simple
## lifecycle entry points. The host loads them with `dlopen` and `dlsym`, then
## calls them synchronously. There is no context, no callback and no CBOR. The
## return value of the function crosses the ABI directly.
##
## Write native Nim types. `ffiExport` maps them to the C ABI:
## int / int64 -> C long long
## int32 / bool -> C int
## uint / uint64 -> C unsigned long long
## float -> C double
## string -> C const char* (valid until the same thread calls again)
## (no return) -> C void
## A return type with no mapping is a compile error.
## The `const char*` buffer belongs to the calling thread. Copy the bytes before
## that thread calls another `{.ffiExport.}` proc that returns a string.
## `ffiExport` also injects the `initializeLibrary()` call of the library. The Nim
## runtime therefore starts on the first call, and the host never calls NimMain.
## The wrapper catches every exception of the body, prints it and returns the
## zero value, because an exception must not cross the C ABI.
##
## declareLibraryBase("myLib") # emits initializeLibrary()
## proc my_start(): int {.ffiExport.} = 0 # -> long long my_start(void)
## proc my_alive(): uint64 {.ffiExport.} = beats # -> unsigned long long my_alive(void)
## proc my_error(): string {.ffiExport.} = lastErr # -> const char* my_error(void)
##
## Build the shared library with `--noMain --nimMainPrefix:libmyLib`. A proc with
## `{.ffiExport.}` takes no arguments. For a call with arguments, use `{.ffi.}`.
import std/macros
import ./ffi_route
import ./ffi_codegen_common
const passthroughCTypes = [
"cint", "cuint", "clong", "culong", "clonglong", "culonglong", "cfloat", "cdouble",
"cstring", "pointer",
]
proc cReturnType(t: NimNode, exportName: string): NimNode =
## Maps the native Nim return type to the C ABI type that crosses the boundary.
## A type with no mapping is an error: emitting the Nim type as-is would export
## a symbol whose ABI no host can call.
if t.kind == nnkEmpty:
return t
if t.kind == nnkIdent:
case $t
of "int", "int64":
# `int` is pointer-wide, so `cint` would truncate it on a 64-bit host.
return ident("clonglong")
of "int8", "int16", "int32", "bool":
return ident("cint")
of "uint", "uint64":
return ident("culonglong")
of "uint8", "uint16", "uint32":
return ident("cuint")
of "float", "float64":
return ident("cdouble")
of "float32":
return ident("cfloat")
of "string":
return ident("cstring")
else:
if $t in passthroughCTypes:
return t
error(
"`.ffiExport.` proc " & exportName & " returns " & t.repr &
", which has no C ABI mapping. Return a scalar, a bool, a string, a C type, " &
"or nothing. For a richer return type, use `{.ffi.}`."
)
proc withoutFFIPragmas(pragmas: NimNode): NimNode =
## Drops only the pragma that routed the proc here, so a `raises` or `gcsafe`
## the writer asked for still applies to the body.
if pragmas.kind != nnkPragma:
return newEmptyNode()
var kept = nnkPragma.newTree()
for p in pragmas:
let name =
if p.kind in {nnkExprColonExpr, nnkCall}:
p[0]
else:
p
if name.kind == nnkIdent and $name in ["ffi", "ffiExport"]:
continue
kept.add(p)
return
if kept.len == 0:
newEmptyNode()
else:
kept
proc buildFFIExportProc*(prc: NimNode): NimNode {.compileTime.} =
## Emits the synchronous C export. `{.ffi.}` and `{.ffiExport.}` share it.
prc.expectKind({nnkProcDef, nnkFuncDef})
let exportName = $procIdent(prc)
let params = prc.params
let nativeRet = params[0]
let cRet = cReturnType(nativeRet, exportName)
# The user body becomes a private impl proc that the exported wrapper calls.
let implName = genSym(nskProc, exportName & "Impl")
var impl = copyNimTree(prc)
impl[0] = implName
impl[4] = withoutFFIPragmas(prc[4])
let wrapName = ident(exportName)
let boot = quote:
when declared(initializeLibrary):
initializeLibrary()
# A Nim exception that unwinds through a cdecl frame into the host is
# undefined behaviour, so every wrapper catches and reports instead.
let raiseNote = newLit("error: " & exportName & " raised: ")
var res = newStmtList(impl)
if nativeRet.kind == nnkIdent and $nativeRet == "string":
# One buffer per calling thread: a process-wide buffer would let one thread
# free the bytes another thread is still reading.
let buf = genSym(nskVar, exportName & "Buf")
res.add quote do:
var `buf` {.threadvar.}: pointer
proc `wrapName`(): cstring {.exportc: `exportName`, cdecl, dynlib, raises: [].} =
`boot`
var s = ""
try:
s = `implName`()
except CatchableError as e:
echo `raiseNote`, e.msg
if not `buf`.isNil():
deallocShared(`buf`)
`buf` = allocShared(s.len + 1)
if s.len > 0:
copyMem(`buf`, unsafeAddr s[0], s.len)
cast[ptr UncheckedArray[char]](`buf`)[s.len] = '\0'
return cast[cstring](`buf`)
elif nativeRet.kind == nnkEmpty:
res.add quote do:
proc `wrapName`() {.exportc: `exportName`, cdecl, dynlib, raises: [].} =
`boot`
try:
`implName`()
except CatchableError as e:
echo `raiseNote`, e.msg
else:
res.add quote do:
proc `wrapName`(): `cRet` {.exportc: `exportName`, cdecl, dynlib, raises: [].} =
`boot`
try:
return `cRet`(`implName`())
except CatchableError as e:
echo `raiseNote`, e.msg
return `cRet`(0)
return res
macro ffiExport*(prc: untyped): untyped =
## Marks a proc that takes no arguments as a simple synchronous C export. The
## macro maps the native Nim return type to the C ABI and starts the Nim
## runtime. `{.ffi.}` reaches the same path from the shape alone. See the
## module doc.
prc.expectKind({nnkProcDef, nnkFuncDef})
assertFFIPath(prc, fpExport)
return buildFFIExportProc(prc)
+184 -26
View File
@@ -1,9 +1,55 @@
import std/[macros, atomics], strformat, chronicles, chronos
import
std/[macros, atomics, sysatomics, compilesettings], strformat, chronicles, chronos
import strutils
import ../codegen/meta
func nimMainPrefixOnCmdLine(cmdLine: string): tuple[found: bool, value: string] =
## Last `--nimMainPrefix:X` on the command line (style-insensitive match, `:`
## or `=`); absence isn't proof it was never set (config.nims may not surface).
var found = false
var value = ""
for tok in cmdLine.splitWhitespace():
let body = tok.strip(trailing = false, chars = {'-'})
let sep = body.find({':', '='})
if sep < 0:
continue
if body[0 ..< sep].toLowerAscii().replace("_", "") == "nimmainprefix":
found = true
value = body[sep + 1 .. ^1]
(found, value)
proc validateNimMainPrefix(libraryName: string) {.compileTime.} =
## The init symbol is importc'd as `lib{libraryName}NimMain`, so the build must
## pass `--nimMainPrefix:lib{libraryName}`; a mismatch errors, absence only
## hints (config.nims may set it) and only under `--app:lib`.
let expectedPrefix = "lib" & libraryName
let (prefixFound, prefixValue) =
nimMainPrefixOnCmdLine(querySetting(SingleValueSetting.commandLine))
if prefixFound and prefixValue != expectedPrefix:
error(
"declareLibrary(\"" & libraryName &
"\"): the Nim runtime init symbol is importc'd as " & expectedPrefix &
"NimMain, so the build needs --nimMainPrefix:" & expectedPrefix &
", but the command line passes --nimMainPrefix:" & prefixValue &
". Change the flag to --nimMainPrefix:" & expectedPrefix &
" (it must be \"lib\" followed by the declareLibrary name)."
)
elif not prefixFound and compileOption("app", "lib"):
hint(
"declareLibrary(\"" & libraryName & "\"): pass --nimMainPrefix:" & expectedPrefix &
" so the Nim runtime init symbol " & expectedPrefix &
"NimMain resolves; without it the build may fail with an undefined-symbol" &
" link error (ignore this hint if the prefix is set in config.nims)."
)
macro declareLibraryBase*(libraryName: static[string]): untyped =
currentLibName = libraryName
validateNimMainPrefix(libraryName)
macro declareLibrary*(libraryName: static[string]): untyped =
var res = newStmtList()
## Generate {.pragma: exported, exportc, cdecl, raises: [].}
# {.pragma: exported, exportc, cdecl, raises: [].}
res.add nnkPragma.newTree(
nnkExprColonExpr.newTree(ident"pragma", ident"exported"),
ident"exportc",
@@ -11,7 +57,7 @@ macro declareLibrary*(libraryName: static[string]): untyped =
nnkExprColonExpr.newTree(ident"raises", nnkBracket.newTree()),
)
## Generate {.pragma: callback, cdecl, raises: [], gcsafe.}
# {.pragma: callback, cdecl, raises: [], gcsafe.}
res.add nnkPragma.newTree(
nnkExprColonExpr.newTree(ident"pragma", ident"callback"),
ident"cdecl",
@@ -19,22 +65,26 @@ macro declareLibrary*(libraryName: static[string]): untyped =
ident"gcsafe",
)
## Generate {.passc: "-fPIC".}
# {.passc: "-fPIC".}
res.add nnkPragma.newTree(nnkExprColonExpr.newTree(ident"passc", newLit("-fPIC")))
when defined(linux) and not defined(emscripten):
# NB: under emscripten (--os:linux) a `-Wl,-soname` makes emcc build a wasm
# SIDE module, which breaks EXPORTED_FUNCTIONS/malloc. The wasm/edge build is
# a MAIN module, so skip the soname there.
## Generates {.passl: "-Wl,-soname,libwaku.so".} (considering libraryName=="waku", for example)
let soName = fmt"-Wl,-soname,lib{libraryName}.so"
res.add(
newNimNode(nnkPragma).add(
nnkExprColonExpr.newTree(ident"passl", newStrLitNode(soName))
# soname / install_name only apply to a shared library and break an executable link (fatally on macOS), so emit them only under `--app:lib`.
if compileOption("app", "lib"):
when defined(linux):
let soName = fmt"-Wl,-soname,lib{libraryName}.so"
res.add(
newNimNode(nnkPragma).add(
nnkExprColonExpr.newTree(ident"passl", newStrLitNode(soName))
)
)
)
## proc lib{libraryName}NimMain() {.importc.}
elif defined(macosx):
let installName = fmt"-install_name @rpath/lib{libraryName}.dylib"
res.add(
newNimNode(nnkPragma).add(
nnkExprColonExpr.newTree(ident"passl", newStrLitNode(installName))
)
)
# proc lib{libraryName}NimMain() {.importc.}
let libNimMainName = ident(fmt"lib{libraryName}NimMain")
let importcPragma = nnkPragma.newTree(ident"importc")
let procDef = newProc(
@@ -45,14 +95,14 @@ macro declareLibrary*(libraryName: static[string]): untyped =
)
res.add(procDef)
# Create: var initialized: Atomic[bool]
let atomicType = nnkBracketExpr.newTree(ident("Atomic"), ident("bool"))
# initState: 0=not started, 1=in progress, 2=done. Atomic (not a bool) so a racing caller can't skip past the gate mid-init (else Windows WSAStartup fails).
let atomicType = nnkBracketExpr.newTree(ident("Atomic"), ident("int"))
let varStmt = nnkVarSection.newTree(
nnkIdentDefs.newTree(ident("initialized"), atomicType, newEmptyNode())
nnkIdentDefs.newTree(ident("initState"), atomicType, newEmptyNode())
)
res.add(varStmt)
## Android chronicles redirection
# Android chronicles redirection
let chroniclesBlock = quote:
when defined(android) and compiles(defaultChroniclesStream.outputs[0].writer):
defaultChroniclesStream.outputs[0].writer = proc(
@@ -66,12 +116,16 @@ macro declareLibrary*(libraryName: static[string]): untyped =
let initializeLibraryProc = quote:
proc `procName`*() {.exported.} =
if not initialized.exchange(true):
## Every Nim library needs to call `<yourprefix>NimMain` once exactly,
## to initialize the Nim runtime.
## Being `<yourprefix>` the value given in the optional
## compilation flag --nimMainPrefix:yourprefix
## Calls `<prefix>NimMain` exactly once to init the Nim runtime. Concurrent
## callers must block until it returns (its chronos globalInit runs
## WSAStartup on Windows; racing past yields "WSAStartup failed" later).
var expected: int = 0
if initState.compareExchange(expected, 1):
`nimMainName`()
initState.store(2)
else:
while initState.load() != 2:
cpuRelax()
when declared(setupForeignThreadGc):
setupForeignThreadGc()
when declared(nimGC_setStackBottom):
@@ -82,3 +136,107 @@ macro declareLibrary*(libraryName: static[string]): untyped =
res.add(initializeLibraryProc)
return res
macro declareLibrary*(
libraryName: static[string],
libType: untyped,
defaultABIFormat: static[string] = "cbor",
): untyped =
## Declares a library and emits the C-exported event ABI (`_add_event_listener` /
## `_remove_event_listener`) on its `FFIContext`. `defaultABIFormat` (`"cbor"`/`"c"`)
## is inherited unless an annotation overrides via `"abi = ..."`.
currentLibType = $libType # so handle-receiver `.ffi.` procs can resolve the pool
let (abiOk, abiFmt) = parseABIFormatName(defaultABIFormat)
if not abiOk:
error(
"declareLibrary: unknown defaultABIFormat '" & defaultABIFormat &
"'; valid values are \"c\" and \"cbor\""
)
currentDefaultABIFormat = abiFmt
libraryDeclared = true
var stmts = newStmtList()
stmts.add(newCall(ident("declareLibraryBase"), newStrLitNode(libraryName)))
# The pool the generated wrappers validate against.
let poolIdent = ident($libType & "FFIPool")
stmts.add quote do:
when not declared(`poolIdent`):
var `poolIdent`*: FFIContextPool[`libType`]
let ctxType = nnkPtrTy.newTree(nnkBracketExpr.newTree(ident("FFIContext"), libType))
let cdeclExportPragma = newTree(
nnkPragma,
ident("dynlib"),
ident("exportc"),
ident("cdecl"),
newTree(nnkExprColonExpr, ident("raises"), newTree(nnkBracket)),
)
# {libraryName}_add_event_listener
let addName = libraryName & "_add_event_listener"
let addErr = "error: invalid context in " & addName
let addBody = quote:
# This code runs on the foreign caller thread. That thread can differ from
# the thread of an earlier entry point. If the GC of the thread is not
# ready, the first Nim allocation ($eventName, the registry Table and seq)
# faults. Therefore initialize the GC here.
when declared(initializeLibrary):
initializeLibrary()
var ret: uint64 = 0
if isNil(ctx):
echo `addErr`
return ret
let evtName =
if eventName.isNil():
""
else:
$eventName
ret = addEventListener(ctx[].eventRegistry, evtName, callback, userData)
return ret
stmts.add(
newProc(
name = ident(addName),
params = @[
ident("uint64"),
newIdentDefs(ident("ctx"), ctxType),
newIdentDefs(ident("eventName"), ident("cstring")),
newIdentDefs(ident("callback"), ident("FFICallBack")),
newIdentDefs(ident("userData"), ident("pointer")),
],
body = addBody,
pragmas = cdeclExportPragma,
)
)
# Param is `listenerId`, not `id`: `id` collides with chronos's `futures.id` template under quote injection and the captured symbol wins.
let removeName = libraryName & "_remove_event_listener"
let removeErr = "error: invalid context in " & removeName
let removeBody = quote:
when declared(initializeLibrary):
initializeLibrary()
var ret: cint = 1
if isNil(ctx):
echo `removeErr`
return ret
if removeEventListener(ctx[].eventRegistry, listenerId):
ret = 0
return ret
stmts.add(
newProc(
name = ident(removeName),
params = @[
ident("cint"),
newIdentDefs(ident("ctx"), ctxType),
newIdentDefs(ident("listenerId"), ident("uint64")),
],
body = removeBody,
pragmas = cdeclExportPragma,
)
)
return stmts
File diff suppressed because it is too large Load Diff
+93
View File
@@ -0,0 +1,93 @@
## Picks the FFI path of a proc from the shape of its signature.
##
## `{.ffi.}`, `{.ffiStatic.}`, `{.ffiExport.}`, `{.ffiDtor.}` and `{.ffiEvent.}`
## own five disjoint shapes, so one router serves all five. Each shape that the
## router claims fails to compile under any other pragma today, so the router
## only turns a compile error into the meaning the writer intended.
##
## `{.ffiCtor.}` stays explicit, because its shape is not free. A ctor differs
## from a static call by one token: the type inside `Result`. A static call that
## returns the library type builds today and exports a working C symbol. A
## router would silently give it the ctor ABI instead.
import std/macros
import ../codegen/meta
import ./ffi_codegen_common
type FFIPath* = enum
fpMethod ## A library or handle receiver, and an async result.
fpStatic ## No receiver, and an async result.
fpExport ## No arguments, and a synchronous result.
fpDtor ## A library receiver, and no result.
fpEvent ## A payload parameter, and no result.
func pathPragma*(path: FFIPath): string =
case path
of fpMethod: "`.ffi.`"
of fpStatic: "`.ffiStatic.`"
of fpExport: "`.ffiExport.`"
of fpDtor: "`.ffiDtor.`"
of fpEvent: "`.ffiEvent.`"
func pathShape*(path: FFIPath): string =
case path
of fpMethod:
"the first parameter is the library type or an {.ffiHandle.} type, and the " &
"return type is Future[Result[T, string]]"
of fpStatic:
"there is no library receiver, and the return type is Future[Result[T, string]]"
of fpExport:
"there are no parameters, and the return type is a plain Nim type"
of fpDtor:
"there is one library parameter, and the return type is nothing or Future[void]"
of fpEvent:
"there is a payload parameter that is not the library type, and there is no result"
func isFuture(t: NimNode): bool =
return
t.kind == nnkBracketExpr and t.len == 2 and t[0].kind == nnkIdent and
$t[0] == "Future"
func isFutureVoid(t: NimNode): bool =
return isFuture(t) and t[1].kind == nnkIdent and $t[1] == "void"
proc isLibReceiver(t: NimNode): bool {.compileTime.} =
## The receiver is the type that `declareLibrary` recorded, or a handle type.
if t.kind != nnkIdent:
return false
return ($t == currentLibType and currentLibType.len > 0) or isFFIHandleTypeName($t)
proc routeFFIProc*(prc: NimNode): FFIPath {.compileTime.} =
## Reads the receiver and the return type, then names the path.
let params = prc.params
let ret = params[0]
let hasReceiver = params.len > 1 and isLibReceiver(params[1][1])
if hasReceiver:
return if ret.kind == nnkEmpty or isFutureVoid(ret): fpDtor else: fpMethod
if params.len == 1 and not isFuture(ret):
return fpExport
# A static call always returns Future[Result[T, string]], so a payload
# parameter with no result can only be an event.
if params.len > 1 and ret.kind == nnkEmpty:
return fpEvent
return fpStatic
proc assertFFIPath*(prc: NimNode, want: FFIPath) {.compileTime.} =
## Guards an explicit pragma against a signature that routes elsewhere.
let got = routeFFIProc(prc)
if got == want:
return
let name = $procIdent(prc)
# A receiver is the one mismatch a caller can read straight off the signature.
if want == fpStatic and got == fpMethod:
error(
"`.ffiStatic.` proc " & name & " takes " & prc.params[1][1].repr &
" as its first parameter, which is the library type or an {.ffiHandle.} type. " &
"A receiver belongs to a context. Make it an `{.ffi.}` method instead."
)
error(
pathPragma(want) & " proc " & name & " has the shape of a " & pathPragma(got) &
" proc. Use " & pathPragma(got) & " here, or make sure that " & pathShape(want) &
"."
)
+154
View File
@@ -0,0 +1,154 @@
## CBOR-free scalar fast path for all-scalar `{.ffi: "abi = c".}` methods.
import std/macros
import ../codegen/meta
import ./ffi_codegen_common
const scalarPodTypeNames = [
"int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16", "uint32",
"uint64", "byte", "float", "float32", "float64", "bool",
]
## Fixed-width POD scalars that survive the async hop by value; `cstring`/
## `string` are excluded as params (they alias caller memory read after return).
func isScalarParamTypeName*(name: string): bool =
name in scalarPodTypeNames
func isScalarReturnTypeName*(name: string): bool =
## Unlike params, a `string`/`cstring` return is fine: the bytes ride back raw.
name in scalarPodTypeNames or name == "string" or name == "cstring"
func isScalarOnly*(p: FFIProcMeta): bool =
## True iff every wire param and return of `p` is scalar. Handles and raw
## pointers are excluded.
if p.kind != FFIKind.FFI:
return false
if p.returnIsPtr or p.returnIsHandle:
return false
if not isScalarReturnTypeName(p.returnTypeName):
return false
for ep in p.extraParams:
if ep.isPtr or ep.isHandle or not isScalarParamTypeName(ep.typeName):
return false
true
func bindableProcs*(procs: seq[FFIProcMeta]): seq[FFIProcMeta] =
## Procs the CBOR-speaking generators emit for; scalar-fast-path procs are
## dropped (their inline-scalar export doesn't match the CBOR codegen shape).
## The `abi = c` C header binds the full registry instead.
var kept: seq[FFIProcMeta] = @[]
for p in procs:
if not p.scalarFastPath:
kept.add(p)
kept
proc buildScalarPath*(
helperProc, ctxGuard, reqPtrIdent, sendAndReply: NimNode,
userProcName, cExportProcName: NimNode,
cExportName: string,
ctxType: NimNode,
camelName: string,
extraParamNames: seq[string],
extraParamTypes: seq[NimNode],
procMeta: FFIProcMeta,
): NimNode {.compileTime.} =
## Emits the scalar-fast-path codegen for one `.ffi.` proc; the caller supplies
## the generic dispatch pieces, this owns the inline pack/unpack/raw-bytes wiring.
let scalarReqKey = camelName & "Req"
let reqIdent = genSym(nskLet, "ffiReq")
let ctxHandlerName = genSym(nskLet, "ffiCtxHandler")
let handlerBody = newStmtList()
handlerBody.add quote do:
let `reqIdent` = cast[ptr FFIThreadRequest](request)
let `ctxHandlerName` = cast[`ctxType`](reqHandler)
# ctxType is `ptr FFIContext[LibType]`; the guard needs the library type.
handlerBody.add(buildLibReadyGuard(ctxHandlerName, ctxType[0][1]))
let helperCall = newTree(nnkCall, userProcName)
let ctxMyLib = newDotExpr(newTree(nnkDerefExpr, ctxHandlerName), ident("myLib"))
helperCall.add(newTree(nnkDerefExpr, ctxMyLib))
for i in 0 ..< extraParamNames.len:
let argIdent = ident(extraParamNames[i])
let slot = nnkBracketExpr.newTree(
newDotExpr(newTree(nnkDerefExpr, reqIdent), ident("scalarArgs")), newLit(i)
)
handlerBody.add(
newLetStmt(argIdent, newCall(ident("ffiUnpackScalar"), slot, extraParamTypes[i]))
)
helperCall.add(argIdent)
let retValIdent = genSym(nskLet, "retVal")
handlerBody.add quote do:
let `retValIdent` = (await `helperCall`).valueOr:
return err(error)
return ok(ffiRawRetBytes(`retValIdent`))
let seqByteResult = nnkBracketExpr.newTree(
ident("Future"),
nnkBracketExpr.newTree(
ident("Result"),
nnkBracketExpr.newTree(ident("seq"), ident("byte")),
ident("string"),
),
)
let handlerProc = newProc(
name = newEmptyNode(),
params = @[
seqByteResult,
newIdentDefs(ident("request"), ident("pointer")),
newIdentDefs(ident("reqHandler"), ident("pointer")),
],
body = handlerBody,
pragmas = nnkPragma.newTree(ident("async")),
)
let registerAssign = newAssignment(
nnkBracketExpr.newTree(ident("registeredRequests"), newLit(scalarReqKey)),
handlerProc,
)
var scalarParams = @[
ident("cint"),
newIdentDefs(ident("ctx"), ctxType),
newIdentDefs(ident("callback"), ident("FFICallBack")),
newIdentDefs(ident("userData"), ident("pointer")),
]
for i in 0 ..< extraParamNames.len:
scalarParams.add(newIdentDefs(ident(extraParamNames[i]), extraParamTypes[i]))
let ffiBody = newStmtList()
ffiBody.add ctxGuard
let initScalarCall = newTree(
nnkCall,
newDotExpr(ident("FFIThreadRequest"), ident("initScalar")),
ident("callback"),
ident("userData"),
newDotExpr(newLit(scalarReqKey), ident("cstring")),
)
for i in 0 ..< extraParamNames.len:
initScalarCall.add(newCall(ident("ffiPackScalar"), ident(extraParamNames[i])))
ffiBody.add newLetStmt(reqPtrIdent, initScalarCall)
ffiBody.add sendAndReply
let ffiProc = newProc(
name = postfix(cExportProcName, "*"),
params = scalarParams,
body = ffiBody,
pragmas = newTree(
nnkPragma,
ident("dynlib"),
newTree(nnkExprColonExpr, ident("exportc"), newStrLitNode(cExportName)),
ident("cdecl"),
newTree(nnkExprColonExpr, ident("raises"), newTree(nnkBracket)),
),
)
# Registered so metadata stays introspectable; `bindableProcs` drops it later.
var scalarMeta = procMeta
scalarMeta.scalarFastPath = true
ffiProcRegistry.add(scalarMeta)
newStmtList(helperProc, registerAssign, ffiProc)
+8 -16
View File
@@ -1,6 +1,4 @@
## This code has been copied and addapted from `status-im/nimbu-eth2` project.
## Link: https://github.com/status-im/nimbus-eth2/blob/c585b0a5b1ae4d55af38ad7f4715ad455e791552/beacon_chain/nimbus_binary_common.nim
## This is also copied in logos-messaging-nim repository (2025-12-10)
## Adapted from status-im/nimbus-eth2 nimbus_binary_common.nim.
import
std/[typetraits, os, strutils, syncio],
chronicles,
@@ -15,11 +13,8 @@ type LogFormat* = enum
TEXT
JSON
## Utils
proc stripAnsi(v: string): string =
## Copied from: https://github.com/status-im/nimbus-eth2/blob/stable/beacon_chain/nimbus_binary_common.nim#L41
## Silly chronicles, colors is a compile-time property
## chronicles colors are a compile-time property, so strip ANSI at runtime.
var
res = newStringOfCap(v.len)
i: int
@@ -31,14 +26,14 @@ proc stripAnsi(v: string): string =
x = i + 1
found = false
while x < v.len: # look for [..m
while x < v.len:
let c2 = v[x]
if x == i + 1:
if c2 != '[':
break
else:
if c2 in {'0' .. '9'} + {';'}:
discard # keep looking
discard
elif c2 == 'm':
i = x + 1
found = true
@@ -47,7 +42,7 @@ proc stripAnsi(v: string): string =
break
inc x
if found: # skip adding c
if found:
continue
res.add c
inc i
@@ -58,13 +53,11 @@ proc writeAndFlush(f: syncio.File, s: LogOutputStr) =
try:
f.write(s)
f.flushFile()
except CatchableError:
except IOError:
logLoggingFailure(cstring(s), getCurrentException())
## Setup
proc setupLogLevel(level: LogLevel) =
# TODO: Support per topic level configuratio
# TODO: Support per topic level configuration
topics_registry.setLogLevel(level)
proc setupLogFormat(format: LogFormat, color = true) =
@@ -94,12 +87,11 @@ proc setupLogFormat(format: LogFormat, color = true) =
.}
proc setupLog*(level: LogLevel, format: LogFormat) =
## Logging setup
# Adhere to NO_COLOR initiative: https://no-color.org/
let color =
try:
not parseBool(os.getEnv("NO_COLOR", "false"))
except CatchableError:
except ValueError:
true
setupLogLevel(level)
+54 -9
View File
@@ -3,21 +3,66 @@
"metaData": {
"url": "https://github.com/logos-messaging/nim-ffi",
"downloadMethod": "git",
"vcsRevision": "06111de155253b34e47ed2aaed1d61d08d62cc1b",
"vcsRevision": "53515de17af0ef3e88b2aec9675b8163dddc14ae",
"files": [
"/ffi.nim",
"/ffi/ffi_types.nim",
"/ffi.nimble",
"/ffi/ffi_thread_request.nim",
"/ffi/alloc.nim",
"/ffi/logging.nim",
"/ffi/codegen/templates/c/header_prelude.h.tpl",
"/ffi/codegen/templates/cpp/vendor/tinycbor/LICENSE",
"/ffi/internal/ffi_library.nim",
"/ffi/codegen/templates/cpp/vendor/tinycbor/cborerrorstrings.c",
"/ffi/codegen/templates/cpp/CMakeLists.txt.tpl",
"/ffi/internal/ffi_export.nim",
"/ffi/ffi_handles.nim",
"/ffi/ffi_request_queue.nim",
"/ffi/codegen/c_cpp_common.nim",
"/ffi/ffi_types.nim",
"/ffi/codegen/templates/cpp/vendor/tinycbor/cborparser_dup_string.c",
"/ffi/event_thread.nim",
"/ffi/codegen/cpp.nim",
"/ffi/codegen/cddl.nim",
"/ffi/internal/c_macro_helpers.nim",
"/ffi/codegen/templates/cpp/vendor/tinycbor/tinycbor-version.h",
"/ffi/internal/ffi_codegen_common.nim",
"/ffi/codegen/templates/cpp/vendor/tinycbor/cborinternal_p.h",
"/ffi/cbor_serial.nim",
"/ffi/codegen/templates/nim_ffi_lib.cmake",
"/ffi/codegen/templates/cpp/vendor/tinycbor/cborparser.c",
"/ffi/codegen/templates/c/cbor_helpers.h.tpl",
"/ffi.nim",
"/ffi/codegen/templates/cpp/result.hpp.tpl",
"/ffi/alloc.nim",
"/ffi/codegen/rust.nim",
"/ffi/codegen/string_helpers.nim",
"/ffi/ffi_thread_request.nim",
"/ffi/codegen/templates/cpp/vendor/tinycbor/utf8_p.h",
"/ffi/ffi_thread.nim",
"/ffi/codegen/meta.nim",
"/ffi/codegen/consts.nim",
"/ffi/codegen/templates/cpp/sync_call_helper.hpp.tpl",
"/ffi/codegen/types_ir.nim",
"/ffi/codegen/templates/c/CMakeLists_abi.txt.tpl",
"/ffi/codegen/templates/cpp/vendor/tinycbor/cbor.h",
"/ffi/codegen/templates/cpp/vendor/tinycbor/cborencoder_close_container_checked.c",
"/ffi/ffi_context.nim",
"/ffi/codegen/templates/cpp/vendor/tinycbor/cborencoder.c",
"/ffi.nimble",
"/ffi/internal/ffi_scalar.nim",
"/ffi/codegen/templates/c/CMakeLists.txt.tpl",
"/ffi/codegen/templates/cpp/cbor_helpers.hpp.tpl",
"/ffi/ffi_events.nim",
"/ffi/codegen/c.nim",
"/ffi/internal/ffi_macro.nim",
"/ffi/ffi_context.nim"
"/ffi/codegen/templates/cpp/header_prelude.hpp.tpl",
"/ffi/codegen/templates/cpp/vendor/tinycbor/compilersupport_p.h",
"/ffi/codegen/templates/cpp/context_rule_of_5.hpp.tpl",
"/ffi/ffi_context_pool.nim",
"/ffi/internal/c_wire.nim",
"/ffi/logging.nim",
"/ffi/internal/ffi_route.nim"
],
"binaries": [],
"specialVersions": [
"0.1.3"
"0.3.0",
"#53515de17af0ef3e88b2aec9675b8163dddc14ae"
]
}
}