From b3531ea0315f5a99a14009fb2b5f95dcd4973dc3 Mon Sep 17 00:00:00 2001 From: Ivan FB Date: Fri, 7 Aug 2026 21:13:20 +0200 Subject: [PATCH] build(wasm): commit wasm-deps, the browser edge build's own dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These were never tracked, yet nothing in library/edge compiles without them: - edge_builders.nim — a standalone copy of libp2p's SwitchBuilder with QUIC, autotls and ws-transport stripped out. libp2p/builders pulls lsquic and boringssl, neither of which builds for wasm, and Nim resolves that import to the real package regardless of --path overrides, so bypassing it needs a separate module rather than a flag. - the patched ffi + shim headers the emscripten build compiles against. Leaving them untracked meant the browser edge node was one `rm -rf` from being unrecoverable, and that a fresh clone could never reproduce the artifact. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01M321nwYww2xHYUsVyXZBxi --- wasm-deps/brokers/brokers.nim | 4 + wasm-deps/brokers/brokers.nimble | 1343 +++++++++++++ wasm-deps/brokers/brokers/api_library.nim | 1783 +++++++++++++++++ wasm-deps/brokers/brokers/broker_context.nim | 166 ++ .../brokers/brokers/broker_implement.nim | 268 +++ .../brokers/brokers/broker_interface.nim | 243 +++ wasm-deps/brokers/brokers/event_broker.nim | 628 ++++++ .../brokers/internal/api_cbor_codec.nim | 250 +++ .../brokers/internal/api_cbor_courier.nim | 345 ++++ .../brokers/internal/api_cbor_descriptor.nim | 117 ++ .../internal/api_cbor_event_courier.nim | 171 ++ .../internal/api_cbor_subs_registry.nim | 403 ++++ .../brokers/internal/api_cbor_tuple.nim | 85 + .../internal/api_codegen_cbor_cddl.nim | 244 +++ .../brokers/internal/api_codegen_cbor_go.nim | 865 ++++++++ .../brokers/internal/api_codegen_cbor_h.nim | 203 ++ .../brokers/internal/api_codegen_cbor_hpp.nim | 1662 +++++++++++++++ .../brokers/internal/api_codegen_cbor_py.nim | 1153 +++++++++++ .../internal/api_codegen_cbor_rust.nim | 1043 ++++++++++ .../brokers/internal/api_codegen_cmake.nim | 213 ++ .../brokers/brokers/internal/api_common.nim | 259 +++ .../internal/api_event_broker_cbor.nim | 105 + .../brokers/brokers/internal/api_outdir.nim | 38 + .../internal/api_request_broker_cbor.nim | 628 ++++++ .../brokers/brokers/internal/api_schema.nim | 232 +++ .../brokers/internal/api_type_resolver.nim | 455 +++++ .../brokers/brokers/internal/broker_debug.nim | 111 + .../brokers/internal/helper/broker_utils.nim | 547 +++++ .../brokers/internal/mt_broker_common.nim | 324 +++ .../brokers/brokers/internal/mt_codec.nim | 307 +++ .../brokers/brokers/internal/mt_config.nim | 596 ++++++ .../brokers/internal/mt_event_broker.nim | 935 +++++++++ .../brokers/brokers/internal/mt_queue.nim | 592 ++++++ .../brokers/internal/mt_request_broker.nim | 1660 +++++++++++++++ .../brokers/brokers/multi_request_broker.nim | 746 +++++++ wasm-deps/brokers/brokers/request_broker.nim | 1033 ++++++++++ wasm-deps/brokers/nimblemeta.json | 51 + wasm-deps/edge_builders.nim | 547 +++++ wasm-deps/ffi/ffi.nim | 10 + wasm-deps/ffi/ffi.nimble | 22 + wasm-deps/ffi/ffi/alloc.nim | 42 + wasm-deps/ffi/ffi/ffi_config.nim | 11 + wasm-deps/ffi/ffi/ffi_context.nim | 302 +++ wasm-deps/ffi/ffi/ffi_thread_request.nim | 64 + wasm-deps/ffi/ffi/ffi_types.nim | 39 + wasm-deps/ffi/ffi/internal/ffi_library.nim | 84 + wasm-deps/ffi/ffi/internal/ffi_macro.nim | 549 +++++ wasm-deps/ffi/ffi/logging.nim | 106 + wasm-deps/ffi/nimblemeta.json | 23 + wasm-deps/shim-include/sys/queue.h | 909 +++++++++ 50 files changed, 22516 insertions(+) create mode 100644 wasm-deps/brokers/brokers.nim create mode 100644 wasm-deps/brokers/brokers.nimble create mode 100644 wasm-deps/brokers/brokers/api_library.nim create mode 100644 wasm-deps/brokers/brokers/broker_context.nim create mode 100644 wasm-deps/brokers/brokers/broker_implement.nim create mode 100644 wasm-deps/brokers/brokers/broker_interface.nim create mode 100644 wasm-deps/brokers/brokers/event_broker.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_cbor_codec.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_cbor_courier.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_cbor_descriptor.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_cbor_event_courier.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_cbor_subs_registry.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_cbor_tuple.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_codegen_cbor_cddl.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_codegen_cbor_go.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_codegen_cbor_h.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_codegen_cbor_hpp.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_codegen_cbor_py.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_codegen_cbor_rust.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_codegen_cmake.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_common.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_event_broker_cbor.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_outdir.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_request_broker_cbor.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_schema.nim create mode 100644 wasm-deps/brokers/brokers/internal/api_type_resolver.nim create mode 100644 wasm-deps/brokers/brokers/internal/broker_debug.nim create mode 100644 wasm-deps/brokers/brokers/internal/helper/broker_utils.nim create mode 100644 wasm-deps/brokers/brokers/internal/mt_broker_common.nim create mode 100644 wasm-deps/brokers/brokers/internal/mt_codec.nim create mode 100644 wasm-deps/brokers/brokers/internal/mt_config.nim create mode 100644 wasm-deps/brokers/brokers/internal/mt_event_broker.nim create mode 100644 wasm-deps/brokers/brokers/internal/mt_queue.nim create mode 100644 wasm-deps/brokers/brokers/internal/mt_request_broker.nim create mode 100644 wasm-deps/brokers/brokers/multi_request_broker.nim create mode 100644 wasm-deps/brokers/brokers/request_broker.nim create mode 100644 wasm-deps/brokers/nimblemeta.json create mode 100644 wasm-deps/edge_builders.nim create mode 100644 wasm-deps/ffi/ffi.nim create mode 100644 wasm-deps/ffi/ffi.nimble create mode 100644 wasm-deps/ffi/ffi/alloc.nim create mode 100644 wasm-deps/ffi/ffi/ffi_config.nim create mode 100644 wasm-deps/ffi/ffi/ffi_context.nim create mode 100644 wasm-deps/ffi/ffi/ffi_thread_request.nim create mode 100644 wasm-deps/ffi/ffi/ffi_types.nim create mode 100644 wasm-deps/ffi/ffi/internal/ffi_library.nim create mode 100644 wasm-deps/ffi/ffi/internal/ffi_macro.nim create mode 100644 wasm-deps/ffi/ffi/logging.nim create mode 100644 wasm-deps/ffi/nimblemeta.json create mode 100644 wasm-deps/shim-include/sys/queue.h diff --git a/wasm-deps/brokers/brokers.nim b/wasm-deps/brokers/brokers.nim new file mode 100644 index 000000000..29256d79f --- /dev/null +++ b/wasm-deps/brokers/brokers.nim @@ -0,0 +1,4 @@ +import + brokers/ + [event_broker, request_broker, multi_request_broker, broker_context, api_library] +export event_broker, request_broker, multi_request_broker, broker_context, api_library diff --git a/wasm-deps/brokers/brokers.nimble b/wasm-deps/brokers/brokers.nimble new file mode 100644 index 000000000..785fe2317 --- /dev/null +++ b/wasm-deps/brokers/brokers.nimble @@ -0,0 +1,1343 @@ +import std/[os, strutils] + +# Package +version = "3.1.1" +author = "Nagy Zoltan Peter" +description = + "Type-safe, decoupled messaging patterns for Nim / single thread, cross-thread and FFI API support!" +license = "MIT" +skipDirs = @["tests", "examples", "tools"] + +# Dependencies +requires "nim >= 2.2.4" +requires "chronos >= 4.0.0" +requires "results >= 0.5.0" +requires "chronicles >= 0.10.0" +requires "testutils >= 0.5.0" +requires "cbor_serialization >= 0.3.0" + +proc quoteArg(arg: string): string = + if defined(windows): + result = '"' & arg.replace("\"", "\"\"") & '"' + else: + result = '"' & arg.replace("\"", "\\\"") & '"' + +proc compileVariantSuffix(env: string): string = + let normalized = env.toLowerAscii() + let memoryManager = if normalized.contains("--mm:refc"): "refc" else: "orc" + let buildMode = if normalized.contains("-d:release"): "release" else: "debug" + + memoryManager & "_" & buildMode + +proc findPythonExe(): string = + result = findExe("python3") + if result.len == 0: + result = findExe("python") + if result.len == 0: + quit "Python interpreter not found. Install python3 or add it to PATH." + +proc nimMainPrefixFlag(prefix: string): string = + ## Returns "--nimMainPrefix:" on POSIX and "" on Windows. + ## + ## --nimMainPrefix is a POSIX-only concern. On POSIX, dlopen with + ## RTLD_GLOBAL merges all shared-object exports into a single flat namespace, + ## so two Nim .so files that both define NimMain collide; the prefix renames + ## them (e.g. fooNimMain, barNimMain) to prevent that. + ## + ## On Windows the PE loader resolves every import as "DLL!Symbol", giving + ## each DLL its own isolated namespace — foo.dll!NimMain and bar.dll!NimMain + ## never clash, so the prefix is unnecessary. + ## + ## Using --nimMainPrefix on Windows also triggers a Nim codegen bug: + ## the C generator forward-declares the prefixed NimMain without + ## __declspec(dllexport) and then defines it with N_LIB_EXPORT, which both + ## clang and GCC reject as a hard error (err_attribute_dll_redeclaration). + when defined(windows): + result = "" + else: + result = " --nimMainPrefix:" & prefix + +proc nimWindowsCcFlag(): string = + ## On Windows we standardize FFI builds on clang. Nim's default cc on + ## Windows is MinGW gcc (msvcrt.dll), but the C/C++ side is built by + ## cmake via Visual Studio + MSVC by default (ucrt.dll). Mixing two CRTs + ## across the DLL boundary causes heap/stdio/TLS mismatches that surface + ## as random crashes at process teardown. Forcing clang on the Nim side + ## keeps both halves on a single CRT (MinGW msvcrt or release UCRT, + ## depending on which clang the runner ships). + when defined(windows): " --cc:clang" else: "" + +proc nimWindowsImplibFlag(outDir, libName: string): string = + ## Force lld to emit an import library at a known path next to the DLL. + ## + ## clang in gnu-driver mode (the only mode available when the runner + ## ships MinGW-bundled clang under external/mingw-amd64/bin) defaults + ## to ld.lld in gnu-mode, which does NOT auto-emit any import library. + ## The cmake consumer then fails with "ninja: error: '.lib' + ## missing and no known rule to make it" because our IMPORTED_IMPLIB + ## cmake property points at that path. + ## + ## Passing `-Wl,--out-implib=` makes lld write a gnu-format + ## import library at the requested path. The file extension is purely + ## conventional — we keep `.lib` so that the cmake IMPORTED_IMPLIB + ## paths stay uniform across asan (MSVC-format .lib) and non-asan + ## (gnu-format .lib) builds. Both formats are accepted by clang+lld + ## consumers in gnu-driver mode. + when defined(windows): + " --passL:-Wl,--out-implib=" & outDir & "/" & libName & ".lib" + else: + "" + +proc skipRefcOnWindows(opt, label: string): bool = + ## Returns true (and prints a skip notice) when `opt` requests --mm:refc on + ## Windows. See README → "Platform Support" + "Known Limitations" for the + ## reasoning: chronos' Win32 RegisterWaitForSingleObject path fires its + ## completion on a thread-pool thread that the refc stop-the-world GC + ## cannot suspend, leading to use-after-free on + ## ThreadSignalPtr/Channel-driven workloads. This affects every layer that + ## relies on the cross-thread signal infrastructure: MT brokers, the FFI + ## API runtime and all FFI tests. ORC's atomic refcounting has no STW + ## phase, so the same code is safe under --mm:orc. + ## + ## TEMPORARILY DISABLED — refc-on-Windows is forced through CI so we + ## can observe whether the channel-dispatch refactor + Round-2 CBOR + ## work has actually closed the failure mode. Restore the body when + ## the experiment ends (or wrap the experiment in a kill switch). + discard opt + discard label + # when defined(windows): + # if "--mm:refc" in opt or "refc" == opt: + # echo "Skipping " & label & " (" & opt & + # ") on Windows: refc + chronos thread-pool callback is unsafe — use --mm:orc." + # return true + false + +proc memoryManagerMatrix(): seq[string] = + ## Returns the set of `--mm:` values the wrapper / example tasks + ## should iterate over. + ## + ## - If `MM` is set in the environment, honour the explicit choice + ## (e.g. `MM=refc nimble runTypeMapTestLibPy`) — run that one only. + ## - Otherwise, run both `orc` and `refc` so the parity matrix is + ## exercised end-to-end under both memory managers. + ## + ## TEMPORARILY: Windows runs the same orc+refc default — see the + ## `skipRefcOnWindows` doc-block for the (commented-out) historical + ## reason refc was excluded. Restore the branch below when the + ## experiment ends. + if existsEnv("MM"): + @[getEnv("MM")] + # elif defined(windows): + # @["orc"] + else: + @["orc", "refc"] + +proc setMM(mm: string) = + ## Helper for the matrix loops: pins `MM` for the duration of one + ## library rebuild + foreign-side run. The matrix loop reassigns + ## per iteration; the env var also leaks back to the caller's shell, + ## but that's the same behaviour the existing single-shot tasks had. + putEnv("MM", mm) + +proc cmakeWindowsConfigureExtras(): string = + ## On Windows we drive cmake with Ninja + clang/clang++ + lld, pin the + ## release UCRT and select RelWithDebInfo. The default Visual Studio + ## generator ignores CMAKE_*_COMPILER for the toolset and pulls in MSVC + ## link.exe + the debug UCRT for Debug configs — both incompatible with + ## the clang-built Nim DLLs. + when defined(windows): + " -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++" & + " -DCMAKE_LINKER_TYPE=LLD -DCMAKE_BUILD_TYPE=RelWithDebInfo" & + " -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDLL" + else: + "" + +# FFI build of mylib.nim, emitting into nimlib/build/. Drives the +# C++ example through the generated mylib.h / mylib.hpp. +proc buildFfiExampleFlags( + generatePy = false, generateRust = false, generateGo = false +): string = + result = + "-d:BrokerFfiApi --threads:on --app:lib --path:. --outdir:examples/ffiapi/nimlib/build" + result.add(nimMainPrefixFlag("mylib")) + result.add(nimWindowsCcFlag()) + result.add(nimWindowsImplibFlag("examples/ffiapi/nimlib/build", "mylib")) + if existsEnv("MM"): + result.add(" --mm:" & getEnv("MM")) + else: + result.add(" --mm:orc") + if generatePy or existsEnv("GEN_PY"): + result.add(" -d:BrokerFfiApiGenPy") + if generateRust or existsEnv("GEN_RUST"): + result.add(" -d:BrokerFfiApiGenRust") + if generateGo or existsEnv("GEN_GO"): + result.add(" -d:BrokerFfiApiGenGo") + +proc buildFfiExampleLibrary( + generatePy = false, generateRust = false, generateGo = false +) = + exec "nim c " & buildFfiExampleFlags(generatePy, generateRust, generateGo) & + " examples/ffiapi/nimlib/mylib.nim" + +proc buildTorpedoExampleFlags( + generatePy = false, generateRust = false, generateGo = false +): string = + result = + "-d:BrokerFfiApi --threads:on --app:lib --path:. --outdir:examples/torpedo/nimlib/build" + result.add(nimMainPrefixFlag("torpedolib")) + result.add(nimWindowsCcFlag()) + result.add(nimWindowsImplibFlag("examples/torpedo/nimlib/build", "torpedolib")) + if existsEnv("MM"): + result.add(" --mm:" & getEnv("MM")) + else: + result.add(" --mm:orc") + if generatePy or existsEnv("GEN_PY"): + result.add(" -d:BrokerFfiApiGenPy") + if generateRust or existsEnv("GEN_RUST"): + result.add(" -d:BrokerFfiApiGenRust") + if generateGo or existsEnv("GEN_GO"): + result.add(" -d:BrokerFfiApiGenGo") + +proc buildTorpedoExampleLibrary( + generatePy = false, generateRust = false, generateGo = false +) = + exec "nim c " & buildTorpedoExampleFlags(generatePy, generateRust, generateGo) & + " examples/torpedo/nimlib/torpedolib.nim" + +proc ffiExamplesBuildDir(): string = + "examples/ffiapi/cmake-build" + +proc buildFfiCmakeTarget(target = "") = + let cmakeDir = "examples/ffiapi" + let buildDir = ffiExamplesBuildDir() + mkDir(buildDir) + exec "cmake -S " & cmakeDir & " -B " & buildDir & cmakeWindowsConfigureExtras() + if target.len == 0: + exec "cmake --build " & buildDir + else: + exec "cmake --build " & buildDir & " --target " & target + +proc ffiExampleExecutablePath(exampleDir: string): string = + when defined(windows): + joinPath(exampleDir, "build", "example.exe") + else: + joinPath(exampleDir, "build", "example") + +proc torpedoCmakeBuildDir(): string = + "examples/torpedo/cmake-build" + +proc buildTorpedoCmakeTarget(target = "") = + let cmakeDir = "examples/torpedo" + let buildDir = torpedoCmakeBuildDir() + mkDir(buildDir) + exec "cmake -S " & cmakeDir & " -B " & buildDir & cmakeWindowsConfigureExtras() + if target.len == 0: + exec "cmake --build " & buildDir + else: + exec "cmake --build " & buildDir & " --target " & target + +proc torpedoExecutablePath(): string = + when defined(windows): + joinPath("examples", "torpedo", "cpp_example", "build", "torpedo.exe") + else: + joinPath("examples", "torpedo", "cpp_example", "build", "torpedo") + +proc test(env, path: string) = + let outputPath = + joinPath("build", path & "_" & compileVariantSuffix(env)).addFileExt(ExeExt) + let label = path & " [" & env & "]" + exec "nim c " & env & " --path:. --out:" & quoteArg(outputPath) & " test/" & path & + ".nim" + echo "=== RUN " & label & " ===" + # Use exec (live stdout+stderr) instead of gorgeEx so a SIGSEGV / runtime + # abort that fires before the buffered output is flushed still surfaces + # its Nim stack trace to the CI log. gorgeEx captured stdout only and + # printed it after the binary exited, which loses any backtrace the Nim + # runtime writes to stderr at crash time. + exec quoteArg(outputPath) + echo "=== PASS " & label & " ===" + +proc isExcludedNimPath(path: string): bool = + let normalized = path.replace('\\', '/') + normalized == "nimbledeps" or normalized == "vendor" or normalized == "doc" or + normalized == "build" or normalized == ".venv" or normalized == ".git" or + normalized.startsWith("nimbledeps/") or normalized.startsWith("vendor/") or + normalized.startsWith("doc/") or normalized.startsWith("build/") or + normalized.startsWith(".venv/") or normalized.startsWith(".git/") or + normalized.startsWith("./nimbledeps/") or normalized.startsWith("./vendor/") or + normalized.startsWith("./doc/") or normalized.startsWith("./build/") or + normalized.startsWith("./.venv/") or normalized.startsWith("./.git/") + +proc isNphFile(path: string): bool = + path.endsWith(".nim") or path.endsWith(".nimble") + +proc addUniqueNimFiles(files: var seq[string], output: string) = + for line in output.splitLines(): + let path = line.strip() + if path.len > 0 and isNphFile(path) and not isExcludedNimPath(path) and + path notin files: + files.add(path) + +proc changedNimFiles(): seq[string] = + for command in [ + "git diff --name-only --diff-filter=ACMR --", + "git diff --cached --name-only --diff-filter=ACMR --", + "git ls-files --others --exclude-standard -- '*.nim' '*.nimble'", + ]: + let (output, exitCode) = gorgeEx(command) + if exitCode != 0: + quit "Unable to determine modified files from git" + + result.addUniqueNimFiles(output) + +proc collectNimFiles(dir: string, files: var seq[string]) = + for kind, path in walkDir(dir, relative = true): + let fullPath = + if dir == ".": + path + else: + joinPath(dir, path) + let normalized = fullPath.replace('\\', '/') + case kind + of pcDir: + if not isExcludedNimPath(normalized): + collectNimFiles(normalized, files) + of pcFile, pcLinkToFile: + if isNphFile(normalized) and not isExcludedNimPath(normalized): + files.add(normalized) + else: + discard + +proc allNimFiles(): seq[string] = + collectNimFiles(".", result) + +proc installNphIfNeeded() = + if findExe("nph").len == 0: + echo "Installing nph formatter" + exec "nimble install -y nph" + +proc runNph(files: seq[string], emptyMessage: string) = + installNphIfNeeded() + + if files.len == 0: + echo emptyMessage + else: + for file in files: + exec "nph " & quoteArg(file) + +task fetchVendor, "Initialize/update vendored third-party dependencies (git submodules)": + ## Fetches the third-party C/C++ dependencies required by the FFI + ## builds (currently jsoncons under vendor/jsoncons). Safe to run repeatedly. + if not dirExists(".git"): + quit "fetchVendor must be run from a git checkout (no .git directory found)." + exec "git submodule update --init --recursive vendor" + +task test, "Run all single and multi-threaded broker tests": + let tests = [ + "test_event_broker", "test_request_broker", "test_request_broker_sugar", + "test_request_broker_sync_void", "test_multi_request_broker", "test_broker_oop", + "test_broker_lifecycle", + ] + for f in tests: + for opt in [ + "-d:nimUnittestOutputLevel:VERBOSE --mm:orc", + "-d:nimUnittestOutputLevel:VERBOSE --mm:refc", + "-d:nimUnittestOutputLevel:VERBOSE -d:release -d:gcAssert -d:sysAssert --mm:orc", + "-d:nimUnittestOutputLevel:VERBOSE -d:release -d:gcAssert -d:sysAssert --mm:refc", + ]: + test opt, f + + let mtTests = [ + "test_multi_thread_request_broker", "test_multi_thread_event_broker", + "test_multi_thread_broker_configs", "test_mt_large_payload", + ] + for f in mtTests: + for opt in [ + "-d:nimUnittestOutputLevel:VERBOSE --mm:orc --threads:on", + "-d:nimUnittestOutputLevel:VERBOSE --mm:refc --threads:on", + "-d:nimUnittestOutputLevel:VERBOSE -d:release --mm:orc --threads:on", + "-d:nimUnittestOutputLevel:VERBOSE -d:release --mm:refc --threads:on", + ]: + if skipRefcOnWindows(opt, f): + continue + test opt, f + +task testSugarRejects, "Compile-fail tests: each test/reject/*.nim must NOT compile": + let rejects = + ["reject_mismatch", "reject_mixedname", "reject_dupzero", "reject_badret"] + for f in rejects: + let (outp, code) = gorgeEx( + "nim c --hints:off --path:. --outdir:build/reject test/reject/" & f & ".nim" + ) + if code == 0: + echo outp + quit("REJECT TEST FAILED: " & f & " compiled but must not", 1) + echo " reject OK (correctly rejected): " & f + # API-mode rejects (reduced-A): cross-interface apiName collisions only + # manifest under -d:BrokerFfiApi --threads:on. + let apiRejects = ["reject_iface_apicollision"] + for f in apiRejects: + let (outp, code) = gorgeEx( + "nim c --hints:off -d:BrokerFfiApi --threads:on --path:. --outdir:build/reject test/reject/" & + f & ".nim" + ) + if code == 0: + echo outp + quit("REJECT TEST FAILED: " & f & " compiled but must not", 1) + echo " reject OK (correctly rejected): " & f + echo "all sugar-reject tests passed" + +task runFfiBenchEventStress, + "Build benchlib + the Part D-4/D-5 event dispatch stress drivers and run them": + # Build the benchlib shared library into test/ffibench/build/. + exec "nim c -d:BrokerFfiApi --threads:on --app:lib --path:. " & + "--outdir:test/ffibench/build --mm:orc " & + "--nimMainPrefix:benchlib test/ffibench/benchlib.nim" + # Configure + build the five event-stress drivers via the existing CMake project. + mkDir("test/ffibench/cmake-build") + exec "cmake -S test/ffibench -B test/ffibench/cmake-build" + exec "cmake --build test/ffibench/cmake-build " & + "--target stress_event_mixed_audience " & "--target stress_event_no_foreign " & + "--target stress_event_no_nim " & "--target stress_event_shutdown " & + "--target stress_event_slow_callback" + # Run each in sequence; non-zero exit propagates through `exec`. + exec "test/ffibench/build/stress_event_mixed_audience" + exec "test/ffibench/build/stress_event_no_foreign" + exec "test/ffibench/build/stress_event_no_nim" + exec "test/ffibench/build/stress_event_shutdown" + exec "test/ffibench/build/stress_event_slow_callback" + +proc runFfiBenchEventStressAsanFor(mm: string) = + ## Build benchlib + the Part D-4/D-5 drivers with AddressSanitizer + ## under the requested memory manager (`orc` or `refc`) and run all + ## five drivers. The Nim library carries the sanitizer instrumentation + ## via -fsanitize=address + -d:useMalloc; the CMake project picks up + ## ASAN via -DASAN=ON. Pass = no driver exits non-zero AND no + ## sanitizer report aborts the process. + let buildDir = "test/ffibench/build_asan" + let cmakeDir = "test/ffibench/cmake-build-asan" + exec "nim c -d:BrokerFfiApi --threads:on --app:lib --path:. " & "--outdir:" & buildDir & + " --mm:" & mm & " " & "--nimMainPrefix:benchlib -d:useMalloc " & + "--passC:-fsanitize=address --passC:-fno-omit-frame-pointer " & + "--passL:-fsanitize=address --debugger:native " & "test/ffibench/benchlib.nim" + mkDir(cmakeDir) + let absBuildDir = thisDir() & "/" & buildDir + exec "cmake -S test/ffibench -B " & cmakeDir & " -DASAN=ON -DBENCH_DIR=" & + quoteArg(absBuildDir) + exec "cmake --build " & cmakeDir & " --target stress_event_mixed_audience " & + "--target stress_event_no_foreign " & "--target stress_event_no_nim " & + "--target stress_event_shutdown " & "--target stress_event_slow_callback" + exec buildDir & "/stress_event_mixed_audience" + exec buildDir & "/stress_event_no_foreign" + exec buildDir & "/stress_event_no_nim" + exec buildDir & "/stress_event_shutdown" + exec buildDir & "/stress_event_slow_callback" + +task runFfiBenchEventStressAsan, + "Run the Part D-4/D-5 event dispatch stress drivers under AddressSanitizer (orc + refc)": + runFfiBenchEventStressAsanFor("orc") + runFfiBenchEventStressAsanFor("refc") + +proc buildBenchLibWithMM(mm: string, release: bool) = + ## Shared helper used by the FFI perftest task. Compiles the + ## benchlib shared library with the requested memory manager and + ## build mode into test/ffibench/build/. The C++ driver picks up + ## whatever lib is sitting there. + var flags = + "-d:BrokerFfiApi --threads:on --app:lib --path:. --outdir:test/ffibench/build --mm:" & + mm & " --nimMainPrefix:benchlib" + if release: + flags.add(" -d:release") + exec "nim c " & flags & " test/ffibench/benchlib.nim" + +task perftestFfi, "FFI perftest from C++ (5×500×512B; orc + refc × debug + release)": + ## Companion to `nimble perftest` on the FFI side. Mirrors the same + ## 5 × 500 × 512 B shape via test/ffibench/perf_driver.cpp so the + ## numbers line up directly against the Nim-direct baseline printed + ## by perf_test_multi_thread_*_broker.nim. + mkDir("test/ffibench/cmake-build") + for mm in memoryManagerMatrix(): + for releaseTag in ["debug", "release"]: + let release = releaseTag == "release" + echo "\n=== perftestFfi: --mm:" & mm & " (" & releaseTag & ") ===" + buildBenchLibWithMM(mm, release) + # Reconfigure cmake — the cached lib has the same mtime if mm + # toggled, so re-invoking cmake -B ensures the linker sees the + # fresh dylib mtime via the regenerated build.ninja. + exec "cmake -S test/ffibench -B test/ffibench/cmake-build " & + (if release: "-DCMAKE_BUILD_TYPE=Release" else: "") + exec "cmake --build test/ffibench/cmake-build --target perf_driver" + exec "test/ffibench/build/perf_driver" + +task runFfiBenchEvent, "Build benchlib (release/orc) + bench_event_driver and run it": + ## Part D-6 — captures the per-emit cost across four scenarios: + ## (a) no foreign subs, no nim listeners — atomic-counter fast path + ## (b) 1 foreign subscriber — full courier path + ## (c) M foreign subscribers — encode-amortize-fanout + ## (d) K same-thread Nim listeners — Lane 1 cost in isolation + ## Output is CSV on stdout; numbers are captured in doc/bench_baseline.md. + exec "nim c -d:release -d:BrokerFfiApi --threads:on --app:lib --path:. " & + "--outdir:test/ffibench/build --mm:orc " & + "--nimMainPrefix:benchlib test/ffibench/benchlib.nim" + mkDir("test/ffibench/cmake-build") + exec "cmake -S test/ffibench -B test/ffibench/cmake-build " & + "-DCMAKE_BUILD_TYPE=Release" + exec "cmake --build test/ffibench/cmake-build --target bench_event_driver" + exec "test/ffibench/build/bench_event_driver" + +task perftest, "Run performance and stress tests": + let mtTests = + ["perf_test_multi_thread_request_broker", "perf_test_multi_thread_event_broker"] + for f in mtTests: + for opt in [ + "-d:nimUnittestOutputLevel:VERBOSE --mm:orc --threads:on", + "-d:nimUnittestOutputLevel:VERBOSE --mm:refc --threads:on", + "-d:nimUnittestOutputLevel:VERBOSE -d:release --mm:orc --threads:on", + "-d:nimUnittestOutputLevel:VERBOSE -d:release --mm:refc --threads:on", + ]: + if skipRefcOnWindows(opt, f): + continue + test opt, f + +task testApi, "Run codec unit tests + library init integration tests": + # Codec round-trip tests (no FFI flags needed). + let codecTests = ["test_api_codec"] + for f in codecTests: + for opt in [ + "-d:nimUnittestOutputLevel:VERBOSE --mm:orc", + "-d:nimUnittestOutputLevel:VERBOSE --mm:refc", + "-d:nimUnittestOutputLevel:VERBOSE -d:release --mm:orc", + "-d:nimUnittestOutputLevel:VERBOSE -d:release --mm:refc", + ]: + test opt, f + + # Library-init integration tests need the FFI runtime. + # Each test uses a different --nimMainPrefix to keep their generated + # NimMain symbols distinct. + let apiTests = [ + ("test_api_library_init", "apitest"), + ("test_api_event_teardown_isolation", "cbevt"), + ("test_api_discovery", "apidisc"), + ("test_broker_interface_api", "brokerifaceapi"), + ("test_broker_interface_mt", "brokerifacemt"), + ("typemappingtestlib/test_typemappingtestlib", "typemappingtestlib"), + ] + for (f, prefix) in apiTests: + for opt in [ + "-d:nimUnittestOutputLevel:VERBOSE -d:BrokerFfiApi --mm:orc --threads:on", + "-d:nimUnittestOutputLevel:VERBOSE -d:BrokerFfiApi --mm:refc --threads:on", + "-d:nimUnittestOutputLevel:VERBOSE -d:BrokerFfiApi -d:release --mm:orc --threads:on", + "-d:nimUnittestOutputLevel:VERBOSE -d:BrokerFfiApi -d:release --mm:refc --threads:on", + ]: + if skipRefcOnWindows(opt, f): + continue + let extraOpt = nimMainPrefixFlag(prefix) + test opt & extraOpt, f + +proc findCargoExe(): string = + ## Returns the cargo invocation token. Resolving the symlink (rustup + ## multi-call binary on most installs) loses the dispatch hint, so we + ## return the bare name and rely on PATH lookup at exec time. + if findExe("cargo").len == 0: + quit "Cargo (Rust toolchain) not found. Install rustup or add cargo to PATH." + result = "cargo" + +task runFfiExampleRust, + "Build the FFI example library + Rust crate and run the Rust example (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runFfiExampleRust: --mm:" & mm & " ===" + setMM(mm) + buildFfiExampleLibrary(generateRust = true) + exec quoteArg(findCargoExe()) & + " run --manifest-path examples/ffiapi/rust_example/Cargo.toml" + +proc findGoExe(): string = + ## Returns the `go` toolchain invocation token. Like cargo via rustup, + ## we rely on PATH lookup at exec time so the user's installed Go is used. + if findExe("go").len == 0: + quit "Go toolchain not found. Install Go 1.21+ or add `go` to PATH." + result = "go" + +proc writeFfiGoModFor(buildDir: string) = + ## The generated Go module is emitted into either `nimlib/build/mylib_go` + ## or `nimlib/build/mylib_go`. Go can't conditionally pick a + ## `replace` target by build tag, so we rewrite the example's go.mod + ## per mode before invoking the Go toolchain. + let modPath = "examples/ffiapi/go_example/go.mod" + var contents = "// Generated by nim-brokers test harness — do not edit.\n" + contents.add("module github.com/status-im/nim-brokers/examples/ffiapi/go_example\n\n") + contents.add("go 1.21\n\n") + contents.add("require mylib v0.0.0\n") + if buildDir == "build": + contents.add("require github.com/fxamacker/cbor/v2 v2.7.0\n") + contents.add("\nreplace mylib => ../nimlib/" & buildDir & "/mylib_go\n") + writeFile(modPath, contents) + # Sync go.sum + transitive deps for the cbor case. + if buildDir == "build": + withDir "examples/ffiapi/go_example": + exec quoteArg(findGoExe()) & " mod tidy" + +task buildFfiExampleGo, "Build the FFI API example library + generated Go wrapper": + buildFfiExampleLibrary(generateGo = true) + +task runFfiExampleGo, + "Build the FFI API example library + run the Go example (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runFfiExampleGo: --mm:" & mm & " ===" + setMM(mm) + buildFfiExampleLibrary(generateGo = true) + writeFfiGoModFor("build") + withDir "examples/ffiapi/go_example": + exec quoteArg(findGoExe()) & " run ." + +# --------------------------------------------------------------------------- +# FFI build of mylib.nim + the same cpp_example/main.cpp. +# --------------------------------------------------------------------------- + +task buildFfiExample, "Build FFI API example library (into nimlib/build)": + buildFfiExampleLibrary() + +task buildFfiExampleCpp, + "Build FFI API example — C++ application against the library (via CMake)": + buildFfiExampleLibrary() + buildFfiCmakeTarget("example_cpp") + +task runFfiExampleCpp, + "Build and run the C++ FFI example application against the library (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runFfiExampleCpp: --mm:" & mm & " ===" + setMM(mm) + buildFfiExampleLibrary() + buildFfiCmakeTarget("example_cpp") + exec quoteArg(ffiExampleExecutablePath("examples/ffiapi/cpp_example")) + +task runFfiExamplePy, + "Build the FFI example library + Python wrapper and run python_example/main.py (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runFfiExamplePy: --mm:" & mm & " ===" + setMM(mm) + buildFfiExampleLibrary(true) + putEnv("MYLIB_BUILD_DIR", "build") + exec quoteArg(findPythonExe()) & " " & + quoteArg("examples/ffiapi/python_example/main.py") + +# --------------------------------------------------------------------------- +# hierlib — the OOP interface-model FFI example (BrokerInterface(API) + +# BrokerImplement + bindToContext). Same C ABI as the flat mylib example. +# --------------------------------------------------------------------------- + +proc buildHierExampleLibrary( + generatePy = false, generateRust = false, generateGo = false +) = + var flags = + "-d:BrokerFfiApi --threads:on --app:lib --path:. " & + "--outdir:examples/ffiapi/hierlib/nimlib/build" + flags.add(nimMainPrefixFlag("hierlib")) + flags.add(nimWindowsCcFlag()) + flags.add(nimWindowsImplibFlag("examples/ffiapi/hierlib/nimlib/build", "hierlib")) + if existsEnv("MM"): + flags.add(" --mm:" & getEnv("MM")) + else: + flags.add(" --mm:orc") + if generatePy or existsEnv("GEN_PY"): + flags.add(" -d:BrokerFfiApiGenPy") + if generateRust or existsEnv("GEN_RUST"): + flags.add(" -d:BrokerFfiApiGenRust") + if generateGo or existsEnv("GEN_GO"): + flags.add(" -d:BrokerFfiApiGenGo") + exec "nim c " & flags & " examples/ffiapi/hierlib/nimlib/hierlib.nim" + +proc buildHierCmakeTarget(target = "") = + let cmakeDir = "examples/ffiapi/hierlib" + let buildDir = cmakeDir & "/cmake-build" + mkDir(buildDir) + exec "cmake -S " & cmakeDir & " -B " & buildDir & cmakeWindowsConfigureExtras() + if target.len == 0: + exec "cmake --build " & buildDir + else: + exec "cmake --build " & buildDir & " --target " & target + +task buildHierExample, "Build the hierlib interface-model FFI example library": + buildHierExampleLibrary() + +task runHierExampleCpp, "Build hierlib + the C++ example and run it (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runHierExampleCpp: --mm:" & mm & " ===" + setMM(mm) + buildHierExampleLibrary() + buildHierCmakeTarget("hier_cpp") + exec quoteArg(ffiExampleExecutablePath("examples/ffiapi/hierlib/cpp_example")) + +task runHierExampleRust, + "Build hierlib + Rust crate and run the Rust example (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runHierExampleRust: --mm:" & mm & " ===" + setMM(mm) + buildHierExampleLibrary(generateRust = true) + exec quoteArg(findCargoExe()) & + " run --manifest-path examples/ffiapi/hierlib/rust_example/Cargo.toml" + +task runHierExampleGo, "Build hierlib + Go module and run the Go example (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runHierExampleGo: --mm:" & mm & " ===" + setMM(mm) + buildHierExampleLibrary(generateGo = true) + withDir "examples/ffiapi/hierlib/go_example": + exec quoteArg(findGoExe()) & " mod tidy" + exec quoteArg(findGoExe()) & " run ." + +# Persistence example: two-layer interfaces (IPersistence -> IBackend) with a +# factory selecting File/Memory backends, requests + events at both levels, and +# per-instance routing. The entry module is IPersistenceLib.nim but the +# registered library name (and generated header/dylib) is "persistence", so the +# dylib output name is forced to lib here. +proc persistenceLibOutFlag(): string = + let dir = "examples/persistence/nimlib/build" + when defined(windows): + " --out:" & (dir / "persistence.dll") + elif defined(macosx): + " --out:" & (dir / "libpersistence.dylib") + else: + " --out:" & (dir / "libpersistence.so") + +proc buildPersistenceExampleLibrary( + generatePy = false, generateRust = false, generateGo = false +) = + var flags = + "-d:BrokerFfiApi --threads:on --app:lib --path:. " & + "--outdir:examples/persistence/nimlib/build" + flags.add(nimMainPrefixFlag("persistence")) + flags.add(nimWindowsCcFlag()) + flags.add(nimWindowsImplibFlag("examples/persistence/nimlib/build", "persistence")) + if existsEnv("MM"): + flags.add(" --mm:" & getEnv("MM")) + else: + flags.add(" --mm:orc") + if generatePy or existsEnv("GEN_PY"): + flags.add(" -d:BrokerFfiApiGenPy") + if generateRust or existsEnv("GEN_RUST"): + flags.add(" -d:BrokerFfiApiGenRust") + if generateGo or existsEnv("GEN_GO"): + flags.add(" -d:BrokerFfiApiGenGo") + flags.add(persistenceLibOutFlag()) + exec "nim c " & flags & " examples/persistence/nimlib/IPersistenceLib.nim" + +proc buildPersistenceCmakeTarget(target = "") = + let cmakeDir = "examples/persistence" + let buildDir = cmakeDir & "/cmake-build" + mkDir(buildDir) + exec "cmake -S " & cmakeDir & " -B " & buildDir & cmakeWindowsConfigureExtras() + if target.len == 0: + exec "cmake --build " & buildDir + else: + exec "cmake --build " & buildDir & " --target " & target + +task buildPersistenceExample, + "Build the persistence interface-model FFI example library": + buildPersistenceExampleLibrary() + +task runPersistenceExampleCpp, + "Build persistence + the C++ example and run it (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runPersistenceExampleCpp: --mm:" & mm & " ===" + setMM(mm) + buildPersistenceExampleLibrary() + buildPersistenceCmakeTarget("persistence_cpp") + exec quoteArg(ffiExampleExecutablePath("examples/persistence/cpp_example")) + +task runPersistenceExamplePy, + "Build persistence + Python wrapper and run persistence/python_example/main.py (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runPersistenceExamplePy: --mm:" & mm & " ===" + setMM(mm) + buildPersistenceExampleLibrary(generatePy = true) + exec quoteArg(findPythonExe()) & " " & + quoteArg("examples/persistence/python_example/main.py") + +task runPersistenceExampleRust, + "Build persistence + Rust crate and run the Rust example (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runPersistenceExampleRust: --mm:" & mm & " ===" + setMM(mm) + buildPersistenceExampleLibrary(generateRust = true) + exec quoteArg(findCargoExe()) & + " run --manifest-path examples/persistence/rust_example/Cargo.toml" + +task runPersistenceExampleGo, + "Build persistence + Go wrapper and run the Go example (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runPersistenceExampleGo: --mm:" & mm & " ===" + setMM(mm) + buildPersistenceExampleLibrary(generateGo = true) + withDir "examples/persistence/go_example": + exec quoteArg(findGoExe()) & " mod tidy" + exec quoteArg(findGoExe()) & " run ." + +task runPersistenceExampleNim, "Build and run the pure-Nim persistence example": + let mm = + if existsEnv("MM"): + getEnv("MM") + else: + "orc" + var flags = "--threads:on --path:. --outdir:build --mm:" & mm + exec "nim c -r " & flags & " examples/persistence/nim_example/main.nim" + +task runHierExamplePy, + "Build hierlib + Python wrapper and run hierlib/python_example/main.py (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runHierExamplePy: --mm:" & mm & " ===" + setMM(mm) + buildHierExampleLibrary(true) + exec quoteArg(findPythonExe()) & " " & + quoteArg("examples/ffiapi/hierlib/python_example/main.py") + +# FFI build of the typemapping test library: compiles +# test/typemappingtestlib/typemappingtestlib.nim with -d:BrokerFfiApi +# into build/ and drives test_typemappingtestlib.{cpp,py} against +# that build. +proc buildTypeMapTestLib( + genPy: bool = false, genRust: bool = false, genGo: bool = false +) = + let mm = + if existsEnv("MM"): + getEnv("MM") + else: + "orc" + let release = existsEnv("RELEASE") + var flags = + "-d:BrokerFfiApi --threads:on --app:lib --mm:" & mm & + " --path:. --outdir:test/typemappingtestlib/build" + flags.add(nimMainPrefixFlag("typemappingtestlib")) + flags.add(nimWindowsCcFlag()) + flags.add(nimWindowsImplibFlag("test/typemappingtestlib/build", "typemappingtestlib")) + if release: + flags.add(" -d:release") + if genPy: + flags.add(" -d:BrokerFfiApiGenPy") + if genRust or existsEnv("GEN_RUST"): + flags.add(" -d:BrokerFfiApiGenRust") + if genGo or existsEnv("GEN_GO"): + flags.add(" -d:BrokerFfiApiGenGo") + exec "nim c " & flags & " test/typemappingtestlib/typemappingtestlib.nim" + +task buildTypeMapTestLib, "Build the type-mapping parity test library": + buildTypeMapTestLib() + +proc typeMapTestLibCmakeDir(): string = + "test/typemappingtestlib/cmake-build" + +task runTypeMapTestLibCpp, + "Build the parity library + run the C++ parity test against it (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runTypeMapTestLibCpp: --mm:" & mm & " ===" + setMM(mm) + buildTypeMapTestLib() + let cmakeDir = typeMapTestLibCmakeDir() + let srcDir = "test/typemappingtestlib" + exec "cmake -S " & quoteArg(srcDir) & " -B " & quoteArg(cmakeDir) & + cmakeWindowsConfigureExtras() + exec "cmake --build " & quoteArg(cmakeDir) + exec quoteArg("test/typemappingtestlib/build/test_typemappingtestlib") + +task runTypeMapTestLibRust, + "Build the parity library + Rust wrapper and run the Rust parity test (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runTypeMapTestLibRust: --mm:" & mm & " ===" + setMM(mm) + buildTypeMapTestLib(genRust = true) + exec quoteArg(findCargoExe()) & + " run --manifest-path test/typemappingtestlib/rust_test/Cargo.toml" + +proc writeTypeMapGoModFor(buildDir: string) = + let modPath = "test/typemappingtestlib/go_test/go.mod" + var contents = "// Generated by nim-brokers test harness — do not edit.\n" + contents.add( + "module github.com/status-im/nim-brokers/test/typemappingtestlib/go_test\n\n" + ) + contents.add("go 1.21\n\n") + contents.add("require typemappingtestlib v0.0.0\n") + if buildDir == "build": + contents.add("require github.com/fxamacker/cbor/v2 v2.7.0\n") + contents.add( + "\nreplace typemappingtestlib => ../" & buildDir & "/typemappingtestlib_go\n" + ) + writeFile(modPath, contents) + if buildDir == "build": + withDir "test/typemappingtestlib/go_test": + exec quoteArg(findGoExe()) & " mod tidy" + +task runTypeMapTestLibGo, + "Build the parity library + Go wrapper and run the Go parity test (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runTypeMapTestLibGo: --mm:" & mm & " ===" + setMM(mm) + buildTypeMapTestLib(genGo = true) + writeTypeMapGoModFor("build") + withDir "test/typemappingtestlib/go_test": + exec quoteArg(findGoExe()) & " run ." + +task runTypeMapTestLibPy, + "Build the parity library + Python wrapper and run the Python parity test (orc + refc)": + for mm in memoryManagerMatrix(): + echo "\n=== runTypeMapTestLibPy: --mm:" & mm & " ===" + setMM(mm) + buildTypeMapTestLib(true) + # The test_typemappingtestlib.py driver runs against the FFI + # build; selection is via TYPEMAP_BUILD_DIR which points at the + # build output that holds the matching generated .py wrapper. + putEnv("TYPEMAP_BUILD_DIR", "build") + exec quoteArg(findPythonExe()) & " " & + quoteArg("test/typemappingtestlib/test_typemappingtestlib.py") + +# --------------------------------------------------------------------------- +# Sanitizer support — ASan(+UBSan), ASan+LSan(+UBSan) on Linux, and TSan. +# --------------------------------------------------------------------------- +# Modes (the `mode` string threaded through the helpers below): +# "asan" — AddressSanitizer + UndefinedBehaviorSanitizer. detect_leaks=0. +# Cross-platform (macOS/Linux/Windows). +# "asanleak" — asan + LeakSanitizer (detect_leaks=1). LSan is Linux-only; on +# macOS/Windows this degrades to plain ASan+UBSan (with a notice), +# since LSan is unsupported there. +# "tsan" — ThreadSanitizer. Mutually exclusive with ASan, so it is always a +# SEPARATE build. Built with `--tlsEmulation:off` so TSan observes +# real TLS accesses (the MT brokers lean heavily on threadvars: +# gBrokerThreadSignal, mtThreadIdMarker, the per-thread pollers). +# +# UBSan is folded into the asan modes (same build, near-zero cost). `function` +# and `vptr` checks are disabled: the FFI C ABI casts cdecl callback pointers +# (trips -fsanitize=function) and carries no C++ RTTI (vptr). Everything else — +# alignment, signed-overflow, null deref, bad enum/bool, shift UB — stays on. + +proc sanitizerSuppPath(name: string): string = + thisDir() / "tools" / "sanitizers" / name + +proc sanitizerCompileFlags(mode: string): string = + case mode + of "tsan": + result = "-fsanitize=thread -fno-omit-frame-pointer -g" + else: # asan / asanleak + result = + "-fsanitize=address -fsanitize=undefined " & + "-fno-sanitize=function,vptr -fno-omit-frame-pointer -g" + when defined(windows): + # Windows ASAN symbolizes via PDB (CodeView), not DWARF. + result.add(" -gcodeview") + +proc sanitizerLinkFlags(mode: string, sharedLib: bool = false): string = + case mode + of "tsan": + result = "-fsanitize=thread -g" + when defined(linux): + if sharedLib: + result.add(" -shared-libtsan") + else: # asan / asanleak + result = "-fsanitize=address -fsanitize=undefined -g" + when defined(linux): + # The Nim .so links the shared sanitizer runtime; a foreign exe linked + # with the static runtime otherwise can't satisfy the .so's dep. + if sharedLib: + result.add(" -shared-libasan") + when defined(windows): + # Tell lld to emit a PDB so ASAN frames carry function/line info. + result.add(" -Wl,/debug") + +proc linuxSharedRuntimeOnPath(printName: string) = + ## Put the directory holding the named clang_rt shared runtime on + ## LD_LIBRARY_PATH so a Nim .so linked with the *shared* sanitizer runtime + ## loads cleanly. No-op if clang can't resolve it (static-runtime build). + when defined(linux): + let (so, rc) = gorgeEx("clang -print-file-name=" & printName) + let trimmed = so.strip() + if rc == 0 and trimmed.len > 0 and trimmed != printName: + let dir = parentDir(trimmed) + let cur = getEnv("LD_LIBRARY_PATH") + putEnv( + "LD_LIBRARY_PATH", + if cur.len == 0: + dir + else: + dir & ":" & cur, + ) + +proc setSanitizerEnv(mode: string) = + putEnv("MallocNanoZone", "0") + if not existsEnv("ASAN_SYMBOLIZER_PATH"): + let llvmSym = findExe("llvm-symbolizer") + if llvmSym.len > 0: + putEnv("ASAN_SYMBOLIZER_PATH", llvmSym) + case mode + of "tsan": + var opts = + "symbolize=1:halt_on_error=1:second_deadlock_stack=1:history_size=4:exitcode=66" + let supp = sanitizerSuppPath("tsan.supp") + if fileExists(supp): + opts.add(":suppressions=" & supp) + putEnv("TSAN_OPTIONS", opts) + linuxSharedRuntimeOnPath("libclang_rt.tsan-x86_64.so") + else: # asan / asanleak + var leaks = mode == "asanleak" + when not defined(linux): + if leaks: + echo "note: LeakSanitizer is Linux-only; running plain ASan+UBSan here" + leaks = false + let detect = if leaks: "detect_leaks=1" else: "detect_leaks=0" + putEnv( + "ASAN_OPTIONS", + detect & + ":symbolize=1:print_stacktrace=1:halt_on_error=1:abort_on_error=0:strict_string_checks=1", + ) + var ubopts = "print_stacktrace=1:halt_on_error=1" + let usupp = sanitizerSuppPath("ubsan.supp") + if fileExists(usupp): + ubopts.add(":suppressions=" & usupp) + putEnv("UBSAN_OPTIONS", ubopts) + if leaks: + let lsupp = sanitizerSuppPath("lsan.supp") + if fileExists(lsupp): + putEnv("LSAN_OPTIONS", "suppressions=" & lsupp) + linuxSharedRuntimeOnPath("libclang_rt.asan-x86_64.so") + +# Back-compat shims (pre-existing call sites + external dispatch references). +proc setAsanEnv() = + setSanitizerEnv("asan") + +proc asanCompileFlags(): string = + sanitizerCompileFlags("asan") + +proc asanLinkFlags(sharedLib: bool = false): string = + sanitizerLinkFlags("asan", sharedLib) + +proc testSan(mode, mm, path: string, extra = "") = + ## Build `test/.nim` under the requested sanitizer `mode` + memory + ## manager and run it. `extra` carries per-test compile flags (e.g. the FFI + ## `-d:BrokerFfiApi --nimMainPrefix:` set). + let outputPath = joinPath("build", path & "_" & mode & "_" & mm).addFileExt(ExeExt) + let label = path & " [" & mode & ", clang, mm:" & mm & ", debug]" + # -d:noSignalHandler: disable Nim's SIGSEGV handler so the sanitizer's own + # handler fires on faults. Without it, Nim prints a traceback and exits + # before the sanitizer can report the underlying error. + # -d:useMalloc routes Nim's heaps (incl. the shared heap backing the FFI + # registry/courier buffers) through the system allocator so the sanitizer + # actually sees those allocations. For TSan it is REQUIRED: Nim's native + # MemRegion allocator shares internal free-list metadata across threads in a + # way TSan can't see, producing false-positive races on alloc/dealloc. + var flags = + "--cc:clang --debugger:native -d:nimUnittestOutputLevel:VERBOSE " & + "-d:noSignalHandler -d:useMalloc --threads:on --mm:" & mm + if mode == "tsan": + flags.add(" --tlsEmulation:off") + flags.add( + " --passC:" & quoteArg(sanitizerCompileFlags(mode)) & " --passL:" & + quoteArg(sanitizerLinkFlags(mode)) & " --path:. --out:" & quoteArg(outputPath) + ) + if extra.len > 0: + flags.add(" " & extra) + exec "nim c " & flags & " test/" & path & ".nim" + setSanitizerEnv(mode) + echo "=== RUN " & label & " ===" + exec quoteArg(outputPath) + echo "=== PASS " & label & " ===" + +proc testAsan(mm: string, path: string) = + testSan("asan", mm, path) + +task testMtEventBrokerAsanOrc, + "Run multi-thread event broker tests under AddressSanitizer (clang, orc, debug)": + testAsan("orc", "test_multi_thread_event_broker") + +task testMtEventBrokerAsanRefc, + "Run multi-thread event broker tests under AddressSanitizer (clang, refc, debug)": + if skipRefcOnWindows("refc", "testMtEventBrokerAsanRefc"): + return + testAsan("refc", "test_multi_thread_event_broker") + +task testMtRequestBrokerAsanOrc, + "Run multi-thread request broker tests under AddressSanitizer (clang, orc, debug)": + testAsan("orc", "test_multi_thread_request_broker") + +task testMtRequestBrokerAsanRefc, + "Run multi-thread request broker tests under AddressSanitizer (clang, refc, debug)": + if skipRefcOnWindows("refc", "testMtRequestBrokerAsanRefc"): + return + testAsan("refc", "test_multi_thread_request_broker") + +task testMtBrokerConfigsAsanOrc, + "Run multi-thread broker config showcase under AddressSanitizer (clang, orc, debug)": + testAsan("orc", "test_multi_thread_broker_configs") + +task testMtBrokerConfigsAsanRefc, + "Run multi-thread broker config showcase under AddressSanitizer (clang, refc, debug)": + if skipRefcOnWindows("refc", "testMtBrokerConfigsAsanRefc"): + return + testAsan("refc", "test_multi_thread_broker_configs") + +# --------------------------------------------------------------------------- +# ThreadSanitizer variants of the multi-thread broker tests. TSan is the +# most relevant sanitizer for these: it validates the Channel[T] / shared +# ThreadSignalPtr / Lock-protected bucket registry / Atomic happens-before +# the MT (and FFI) lanes rely on. +# --------------------------------------------------------------------------- +task testMtEventBrokerTsanOrc, + "Run multi-thread event broker tests under ThreadSanitizer (clang, orc, debug)": + testSan("tsan", "orc", "test_multi_thread_event_broker") + +task testMtEventBrokerTsanRefc, + "Run multi-thread event broker tests under ThreadSanitizer (clang, refc, debug)": + if skipRefcOnWindows("refc", "testMtEventBrokerTsanRefc"): + return + testSan("tsan", "refc", "test_multi_thread_event_broker") + +task testMtRequestBrokerTsanOrc, + "Run multi-thread request broker tests under ThreadSanitizer (clang, orc, debug)": + testSan("tsan", "orc", "test_multi_thread_request_broker") + +task testMtRequestBrokerTsanRefc, + "Run multi-thread request broker tests under ThreadSanitizer (clang, refc, debug)": + if skipRefcOnWindows("refc", "testMtRequestBrokerTsanRefc"): + return + testSan("tsan", "refc", "test_multi_thread_request_broker") + +task testMtBrokerConfigsTsanOrc, + "Run multi-thread broker config showcase under ThreadSanitizer (clang, orc, debug)": + testSan("tsan", "orc", "test_multi_thread_broker_configs") + +task testMtBrokerConfigsTsanRefc, + "Run multi-thread broker config showcase under ThreadSanitizer (clang, refc, debug)": + if skipRefcOnWindows("refc", "testMtBrokerConfigsTsanRefc"): + return + testSan("tsan", "refc", "test_multi_thread_broker_configs") + +# --------------------------------------------------------------------------- +# Sanitizer coverage for the FFI event-teardown isolation regression test +# (the cross-context subs-count fix). FFI build needs -d:BrokerFfiApi and a +# distinct --nimMainPrefix. +# --------------------------------------------------------------------------- +const teardownTestExtra = "-d:BrokerFfiApi --nimMainPrefix:cbevt" + +task testApiTeardownAsanOrc, + "Run FFI event-teardown isolation test under ASan+UBSan (clang, orc, debug)": + testSan("asan", "orc", "test_api_event_teardown_isolation", teardownTestExtra) + +task testApiTeardownAsanRefc, + "Run FFI event-teardown isolation test under ASan+UBSan (clang, refc, debug)": + if skipRefcOnWindows("refc", "testApiTeardownAsanRefc"): + return + testSan("asan", "refc", "test_api_event_teardown_isolation", teardownTestExtra) + +task testApiTeardownTsanOrc, + "Run FFI event-teardown isolation test under ThreadSanitizer (clang, orc, debug)": + testSan("tsan", "orc", "test_api_event_teardown_isolation", teardownTestExtra) + +task testApiTeardownTsanRefc, + "Run FFI event-teardown isolation test under ThreadSanitizer (clang, refc, debug)": + if skipRefcOnWindows("refc", "testApiTeardownTsanRefc"): + return + testSan("tsan", "refc", "test_api_event_teardown_isolation", teardownTestExtra) + +# --------------------------------------------------------------------------- +# Persistence C++ example under sanitizers — builds the Nim library AND the +# C++ consumer with matching instrumentation, then runs the consumer. +# `mode` ∈ {asan, asanleak, tsan}. Driven across orc+refc by the tasks below. +# --------------------------------------------------------------------------- +proc runSanitizedPersistenceCpp(mode, mm: string) = + let libBuild = "examples/persistence/nimlib/build" + let cmakeDir = "examples/persistence" + let buildDir = cmakeDir & "/cmake-build-" & mode + var libFlags = + "-d:BrokerFfiApi --threads:on --app:lib --path:. --cc:clang --debugger:native " & + "-d:noSignalHandler -d:useMalloc --mm:" & mm & " --outdir:" & libBuild + if mode == "tsan": + libFlags.add(" --tlsEmulation:off") + libFlags.add(nimMainPrefixFlag("persistence")) + libFlags.add( + " --passC:" & quoteArg(sanitizerCompileFlags(mode)) & " --passL:" & + quoteArg(sanitizerLinkFlags(mode, sharedLib = true)) + ) + libFlags.add(persistenceLibOutFlag()) + exec "nim c " & libFlags & " examples/persistence/nimlib/IPersistenceLib.nim" + mkDir(buildDir) + exec "cmake -S " & cmakeDir & " -B " & buildDir & + " -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_FLAGS=" & + quoteArg(sanitizerCompileFlags(mode)) & " -DCMAKE_EXE_LINKER_FLAGS=" & + quoteArg(sanitizerLinkFlags(mode)) + exec "cmake --build " & buildDir & " --target persistence_cpp" + setSanitizerEnv(mode) + let label = "persistence cpp [" & mode & ", clang, mm:" & mm & "]" + echo "=== RUN " & label & " ===" + exec quoteArg(ffiExampleExecutablePath("examples/persistence/cpp_example")) + echo "=== PASS " & label & " ===" + +task sanitizePersistenceCppAsan, + "Build+run the persistence C++ example under ASan+UBSan (orc + refc)": + for mm in memoryManagerMatrix(): + runSanitizedPersistenceCpp("asan", mm) + +task sanitizePersistenceCppTsan, + "Build+run the persistence C++ example under ThreadSanitizer (orc + refc)": + for mm in memoryManagerMatrix(): + runSanitizedPersistenceCpp("tsan", mm) + +# ---------------------------------------------------------------------------- +# probeWinTlsUninit — minimal repro for LIMITATION.md §2.1 +# ---------------------------------------------------------------------------- +# Demonstrates that a Win32 RegisterWaitForSingleObject completion callback +# allocating Nim memory crashes under --mm:refc on Windows (uninitialized TLS +# on the NT thread-pool wait thread) and passes under --mm:orc. No chronos, +# no brokers — see test/probe_win_tls_uninit.nim for full discussion. +# +# Expected exit codes: +# * Non-Windows hosts → 77 (skip) +# * Windows + orc → 0 +# * Windows + refc → non-zero (crash); task asserts this and +# exits 0 to signal "hypothesis reproduced". +proc runProbeWinTlsUninit(mm: string) = + let outBin = "build" / ("probe_win_tls_uninit_" & mm) + let outBinExe = + when defined(windows): + outBin & ".exe" + else: + outBin + mkDir "build" + # `--out:` with a path overrides `--outdir:`, so put the path directly on + # `--out:` to land the binary under build/. + exec "nim c --threads:on --mm:" & mm & " -d:release " & "--out:" & quoteArg(outBin) & + " " & quoteArg("test/probe_win_tls_uninit.nim") + # Run the probe with live stdout+stderr (exec) so a refc crash's Nim/OS + # backtrace lands in the CI log. exec raises OSError on non-zero exit; we + # use that to distinguish "exit 0" from "exit non-zero / crash". + var exitedNonZero = false + try: + exec quoteArg(outBinExe) + except OSError: + exitedNonZero = true + when defined(windows): + if mm == "orc": + if exitedNonZero: + echo "::error::probeWinTlsUninit/orc: probe must succeed under ORC" + quit(1) + echo "probeWinTlsUninit/orc: PASS" + else: + # refc on Windows: we *expect* the probe to crash. If it exits 0, the + # §2.1 hypothesis no longer reproduces and the doc needs revisiting. + if not exitedNonZero: + echo "::warning::probeWinTlsUninit/refc exited 0 — §2.1 hypothesis " & + "no longer reproduces. Review doc/LIMITATION.md §2.1." + quit(1) + echo "probeWinTlsUninit/refc: hypothesis reproduced (probe crashed " & + "as expected)" + else: + # Non-Windows hosts: probe exits 77 (skip). exec sees that as non-zero + # → OSError → exitedNonZero=true is the success path here. + if not exitedNonZero: + echo "::error::probeWinTlsUninit on non-Windows: expected skip exit" + quit(1) + echo "probeWinTlsUninit: skipped (non-Windows host)" + +task probeWinTlsUninitOrc, + "Run the §2.1 TLS-uninit probe under --mm:orc (must pass on Windows)": + runProbeWinTlsUninit("orc") + +task probeWinTlsUninitRefc, + "Run the §2.1 TLS-uninit probe under --mm:refc (expected to crash on Windows)": + runProbeWinTlsUninit("refc") + +task runTorpedoExampleRust, "Build the Torpedo Duel FFI library ": + buildTorpedoExampleLibrary(generateRust = true) + exec quoteArg(findCargoExe()) & + " run --manifest-path examples/torpedo/rust_example/Cargo.toml" + +proc writeTorpedoGoModFor(buildDir: string) = + let modPath = "examples/torpedo/go_example/go.mod" + var contents = "// Generated by nim-brokers test harness — do not edit.\n" + contents.add( + "module github.com/status-im/nim-brokers/examples/torpedo/go_example\n\n" + ) + contents.add("go 1.21\n\n") + contents.add("require torpedolib v0.0.0\n") + if buildDir == "build": + contents.add("require github.com/fxamacker/cbor/v2 v2.7.0\n") + contents.add("\nreplace torpedolib => ../nimlib/" & buildDir & "/torpedolib_go\n") + writeFile(modPath, contents) + if buildDir == "build": + withDir "examples/torpedo/go_example": + exec quoteArg(findGoExe()) & " mod tidy" + +task runTorpedoExampleGo, "Build the Torpedo Duel FFI library + run the Go example": + buildTorpedoExampleLibrary(generateGo = true) + writeTorpedoGoModFor("build") + withDir "examples/torpedo/go_example": + exec quoteArg(findGoExe()) & " run ." + +# FFI build of the torpedo example. Same torpedolib.nim source + +# same cpp_example/main.cpp, compiled against the FFI codegen output. +task buildTorpedoExample, "Build the torpedo FFI example library (into nimlib/build)": + buildTorpedoExampleLibrary() + +task buildTorpedoExampleCpp, + "Build the Torpedo Duel C++ application against the library (via CMake)": + buildTorpedoExampleLibrary() + buildTorpedoCmakeTarget("torpedo_cpp") + +task runTorpedoExampleCpp, + "Build and run the Torpedo Duel C++ text UI example against the library": + buildTorpedoExampleLibrary() + buildTorpedoCmakeTarget("torpedo_cpp") + exec quoteArg(torpedoExecutablePath()) + +task runTorpedoExamplePy, + "Build the torpedo library + Python wrapper and run the SAME python_example/main.py against it": + buildTorpedoExampleLibrary(true) + putEnv("TORPEDOLIB_BUILD_DIR", "build") + exec quoteArg(findPythonExe()) & " " & + quoteArg("examples/torpedo/python_example/main.py") + +task nph, "Install nph if needed and format modified Nim files": + runNph(changedNimFiles(), "No modified .nim or .nimble files to format") + +task nphall, "Install nph if needed and format all Nim files in the project": + runNph(allNimFiles(), "No .nim or .nimble files found to format") + +task alltests, + "Run every test suite: test, testApi, runFfiExampleCpp, runFfiExamplePy, runTypeMapTestLibCpp, runTypeMapTestLibPy": + exec "nimble test" + exec "nimble runFfiExampleCpp" + exec "nimble runFfiExamplePy" + exec "nimble testApi" + exec "nimble runTypeMapTestLibCpp" + exec "nimble runTypeMapTestLibPy" + +task allAsan, "Run all tests under ASan+UBSan (clang, orc/refc, debug)": + exec "nimble testMtEventBrokerAsanOrc" + exec "nimble testMtEventBrokerAsanRefc" + exec "nimble testMtRequestBrokerAsanOrc" + exec "nimble testMtRequestBrokerAsanRefc" + exec "nimble testMtBrokerConfigsAsanOrc" + exec "nimble testMtBrokerConfigsAsanRefc" + exec "nimble testApiTeardownAsanOrc" + exec "nimble testApiTeardownAsanRefc" + +task allTsan, + "Run all multi-thread + FFI-teardown tests under ThreadSanitizer (orc/refc)": + exec "nimble testMtEventBrokerTsanOrc" + exec "nimble testMtEventBrokerTsanRefc" + exec "nimble testMtRequestBrokerTsanOrc" + exec "nimble testMtRequestBrokerTsanRefc" + exec "nimble testMtBrokerConfigsTsanOrc" + exec "nimble testMtBrokerConfigsTsanRefc" + exec "nimble testApiTeardownTsanOrc" + exec "nimble testApiTeardownTsanRefc" + +task allAsanLeak, "Run all tests under ASan+UBSan+LSan (Linux leak detection; orc/refc)": + # LSan is Linux-only; on macOS/Windows these degrade to plain ASan+UBSan. + testSan("asanleak", "orc", "test_multi_thread_event_broker") + testSan("asanleak", "refc", "test_multi_thread_event_broker") + testSan("asanleak", "orc", "test_multi_thread_request_broker") + testSan("asanleak", "refc", "test_multi_thread_request_broker") + testSan("asanleak", "orc", "test_multi_thread_broker_configs") + testSan("asanleak", "refc", "test_multi_thread_broker_configs") + testSan("asanleak", "orc", "test_api_event_teardown_isolation", teardownTestExtra) + testSan("asanleak", "refc", "test_api_event_teardown_isolation", teardownTestExtra) + +task allSan, "Run the full sanitizer matrix: ASan+UBSan, then ThreadSanitizer": + exec "nimble allAsan" + exec "nimble allTsan" diff --git a/wasm-deps/brokers/brokers/api_library.nim b/wasm-deps/brokers/brokers/api_library.nim new file mode 100644 index 000000000..30e017ccf --- /dev/null +++ b/wasm-deps/brokers/brokers/api_library.nim @@ -0,0 +1,1783 @@ +## API Library Registration +## ------------------------ +## Provides the `registerBrokerLibrary` macro that generates: +## 1. Library context lifecycle management (createContext/shutdown C exports) +## 2. Compile-time validation of mandatory InitializeRequest/ShutdownRequest types +## 3. C header file generation from accumulated broker declarations +## 4. Memory management helpers (free_string) +## 5. Delivery thread creation (hosts event listeners, calls C callbacks) +## 6. Aggregate event listener provider (dispatches by typeId) +## 7. Aggregate cleanup (removes all listeners on shutdown) +## +## Usage: +## ```nim +## registerBrokerLibrary: +## name: "mylib" +## initializeRequest: InitializeRequest +## shutdownRequest: ShutdownRequest +## refType: MyLibObject # optional +## ``` +## +## The `registerBrokerLibrary` macro MUST appear after all `EventBroker(API)` +## and `RequestBroker(API)` declarations in the source file. + +{.push raises: [].} + +import std/[atomics, locks, macros, os, strutils, tables] +import chronos, chronicles +import results +import ./broker_context, ./internal/api_common +import ./internal/helper/broker_utils +import ./internal/api_codegen_cbor_h +import ./internal/api_codegen_cbor_hpp +import ./internal/api_codegen_cbor_py +import ./internal/api_codegen_cbor_rust +import ./internal/api_codegen_cbor_go +import ./internal/api_codegen_cbor_cddl +import ./internal/api_codegen_cmake +import ./internal/api_cbor_descriptor +import ./internal/api_cbor_subs_registry +import ./internal/api_cbor_tuple +import ./internal/api_cbor_courier +import ./internal/api_cbor_event_courier +import ./internal/mt_broker_common +import ./internal/broker_debug + +export api_cbor_descriptor, api_cbor_subs_registry, api_cbor_tuple, api_cbor_courier +export api_cbor_event_courier +export mt_broker_common + +export results, chronos, chronicles, broker_context, api_common + +# --------------------------------------------------------------------------- +# Macro helpers +# --------------------------------------------------------------------------- + +proc parseLibraryConfig( + body: NimNode +): tuple[ + name: string, + version: string, + initializeRequest: NimNode, + shutdownRequest: NimNode, + refType: NimNode, + mainClass: string, +] {.compileTime.} = + var name = "" + var version = "0.1.0" + var initializeReq: NimNode = nil + var shutdownReq: NimNode = nil + var refTy: NimNode = nil + var mainClass = "" + + for stmt in body: + if stmt.kind == nnkCall and stmt.len == 2: + let key = $stmt[0] + let value = stmt[1] + case key.toLowerAscii() + of "name": + if value.kind == nnkStmtList and value.len == 1: + name = value[0].strVal + elif value.kind == nnkStrLit: + name = value.strVal + else: + error("name must be a string literal", value) + of "version": + var v = value + if v.kind == nnkStmtList and v.len == 1: + v = v[0] + if v.kind == nnkStrLit: + version = v.strVal + else: + error("version must be a string literal", v) + of "initializerequest": + if value.kind == nnkStmtList and value.len == 1: + initializeReq = value[0] + else: + initializeReq = value + of "shutdownrequest", "destroyrequest": + if value.kind == nnkStmtList and value.len == 1: + shutdownReq = value[0] + else: + shutdownReq = value + of "reftype": + if value.kind == nnkStmtList and value.len == 1: + refTy = value[0] + else: + refTy = value + of "mainclass": + # reduced-A (A1): designates the main `BrokerInterface(API)` facade for + # a multi-interface library. Other (API) interfaces are auto-discovered + # from the compile-time registry and emitted as their own sub-wrappers. + var v = value + if v.kind == nnkStmtList and v.len == 1: + v = v[0] + case v.kind + of nnkIdent, nnkSym: + mainClass = $v + of nnkStrLit: + mainClass = v.strVal + else: + error("mainClass must be an interface type name", v) + else: + error("Unknown registerBrokerLibrary key: " & key, stmt) + else: + error("registerBrokerLibrary expects key: value pairs", stmt) + + if name.len == 0: + error("registerBrokerLibrary requires a 'name' field", body) + if initializeReq.isNil(): + error("registerBrokerLibrary requires a 'initializeRequest' field", body) + if shutdownReq.isNil(): + error( + "registerBrokerLibrary requires a 'shutdownRequest' field (the legacy 'destroyRequest' alias is still accepted)", + body, + ) + if mainClass.len > 0 and not isApiInterface(mainClass): + error( + "registerBrokerLibrary: mainClass '" & mainClass & + "' is not a registered BrokerInterface(API). Declare it with " & + "`BrokerInterface(API, " & mainClass & "): ...` before registerBrokerLibrary.", + body, + ) + + ( + name: name, + version: version, + initializeRequest: initializeReq, + shutdownRequest: shutdownReq, + refType: refTy, + mainClass: mainClass, + ) + +proc parseTypeExpr( + exprText: string, context: NimNode +): NimNode {.compileTime, raises: [].} = + try: + parseExpr(exprText) + except ValueError as exc: + error( + "Failed to parse generated type expression '" & exprText & "': " & exc.msg, + context, + ) + +# --------------------------------------------------------------------------- +# Macro +# --------------------------------------------------------------------------- + +proc registerBrokerLibraryCborImpl( + body: NimNode, + config: + tuple[ + name: string, + version: string, + initializeRequest: NimNode, + shutdownRequest: NimNode, + refType: NimNode, + mainClass: string, + ], +): NimNode + +proc registerBrokerLibraryImpl(body: NimNode): NimNode = + let config = parseLibraryConfig(body) + registerBrokerLibraryCborImpl(body, config) + +# --------------------------------------------------------------------------- +# CBOR-mode library codegen. +# +# Emits the small fixed C ABI surface (initialize / createContext / shutdown +# / allocBuffer / freeBuffer / call) plus a per-library dispatch case +# statement that routes apiName strings to the adapter procs registered by +# `RequestBroker(API)` expansions. Buffer ownership: every void* crossing +# the ABI is allocated by Nim and freed by Nim. Threading: a dedicated +# processing thread runs `setupProviders(ctx)` and the chronos event loop +# that drives the request providers; foreign threads invoke `_call` +# and use `waitFor` to drive a momentary chronos loop on the calling +# thread, which dispatches across the MT broker channel to the processing +# thread. Events are not yet wired (Phase 3). +# --------------------------------------------------------------------------- + +proc registerBrokerLibraryCborImpl( + body: NimNode, + config: + tuple[ + name: string, + version: string, + initializeRequest: NimNode, + shutdownRequest: NimNode, + refType: NimNode, + mainClass: string, + ], +): NimNode = + let libName = config.name + gApiLibraryName = libName + + let initializeReqIdent = config.initializeRequest + let shutdownReqIdent = config.shutdownRequest + + # Identifiers + let initFuncName = libName & "_initialize" + let initFuncNameLit = newLit(initFuncName) + let initFuncIdent = ident(initFuncName) + let createContextFuncName = libName & "_createContext" + let createContextFuncNameLit = newLit(createContextFuncName) + let createContextFuncIdent = ident(createContextFuncName) + let shutdownFuncName = libName & "_shutdown" + let shutdownFuncNameLit = newLit(shutdownFuncName) + let shutdownFuncIdent = ident(shutdownFuncName) + let callFuncName = libName & "_call" + let callFuncNameLit = newLit(callFuncName) + let callFuncIdent = ident(callFuncName) + let allocBufFuncName = libName & "_allocBuffer" + let allocBufFuncNameLit = newLit(allocBufFuncName) + let allocBufFuncIdent = ident(allocBufFuncName) + let freeBufFuncName = libName & "_freeBuffer" + let freeBufFuncNameLit = newLit(freeBufFuncName) + let freeBufFuncIdent = ident(freeBufFuncName) + + let nimMainIdent = ident(libName & "NimMain") + let nimInitFlagIdent = ident("g" & libName & "NimInit") + let gcRegFlagIdent = ident("g" & libName & "GcReg") + let ctxEntryIdent = ident(libName & "CborCtxEntry") + let procThreadArgIdent = ident(libName & "CborThreadArg") + let procThreadProcIdent = ident(libName & "CborProcessingThread") + let delivThreadProcIdent = ident(libName & "CborDeliveryThread") + let ctxsIdent = ident("g" & libName & "CborCtxs") + let ctxsLockIdent = ident("g" & libName & "CborCtxsLock") + let ctxsInitIdent = ident("g" & libName & "CborCtxsInit") + let dispatchProcIdent = ident(libName & "CborDispatch") + let libNameLit = newLit(libName) + + # Event subscription identifiers (used by the per-event installers and the + # subscribe / unsubscribe C exports). + let eventCallbackTypeIdent = ident(libName & "CborEventCallback") + # Phase 10: subscription state lives in a hand-rolled shared-heap registry + # (`api_cbor_subs_registry`). The previous codegen kept a GC'd + # `Table[(uint32, string), seq[Subscription]]` plus `Lock` here; that broke + # `--mm:refc` cross-thread delivery because subscribe/unsubscribe run on + # foreign caller threads while the listener fires on the processing thread. + let subsRegIdent = ident("g" & libName & "CborSubsReg") + let subsHandleIdent = ident("g" & libName & "CborNextSubHandle") + let subscribeFuncName = libName & "_subscribe" + let subscribeFuncNameLit = newLit(subscribeFuncName) + let subscribeFuncIdent = ident(subscribeFuncName) + let unsubscribeFuncName = libName & "_unsubscribe" + let unsubscribeFuncNameLit = newLit(unsubscribeFuncName) + let unsubscribeFuncIdent = ident(unsubscribeFuncName) + # reduced-A (A4): per-instance teardown export + processing-thread worker. + let releaseInstanceFuncName = libName & "_releaseInstance" + let releaseInstanceFuncNameLit = newLit(releaseInstanceFuncName) + let releaseInstanceFuncIdent = ident(releaseInstanceFuncName) + let releaseCtxProcName = libName & "CborReleaseCtx" + let releaseCtxProcIdent = ident(releaseCtxProcName) + let releaseApiNameLit = newLit("__release_instance") + let knownEventPredIdent = ident(libName & "CborIsKnownEvent") + let installAllListenersIdent = ident(libName & "CborInstallAllListeners") + # Part D-3: per-event helper that maps an event name to its global + # `Atomic[int]` foreign-subscriber count. Subscribe / unsubscribe + # use it to bump / decrement the counter so the emit-side fast-path + # can short-circuit (no CBOR encode, no courier enqueue) when zero + # foreign subscribers exist for an event. + let getEventSubsCountIdent = ident(libName & "CborEventSubsCountAtomicPtr") + # Part D-3: name → static cstring resolver. The eventCourierPoll + # extracts the event name from the in-ring message (`m.eventName`, + # a stack-local array after `tryDequeue`). Passing that pointer to + # foreign callbacks would dangle the moment the poll proc returns + # — callbacks legitimately store the eventName cstring (see the + # `gSlots[i].name = eventName` pattern in the typemappingtestlib + # test). This lookup returns a STATIC string-literal cstring, which + # is permanently valid, so the callback can store it freely. + let resolveEventNameCstrIdent = ident(libName & "CborResolveEventNameCstring") + + # Discovery / introspection identifiers (Phase 6). + let listApisFuncName = libName & "_listApis" + let listApisFuncNameLit = newLit(listApisFuncName) + let listApisFuncIdent = ident(listApisFuncName) + let getSchemaFuncName = libName & "_getSchema" + let getSchemaFuncNameLit = newLit(getSchemaFuncName) + let getSchemaFuncIdent = ident(getSchemaFuncName) + let descriptorIdent = ident("g" & libName & "CborDescriptor") + let apiListIdent = ident("g" & libName & "CborApiList") + let descriptorBuildIdent = ident(libName & "CborBuildDescriptor") + let apiListBuildIdent = ident(libName & "CborBuildApiList") + + # Hard cap on a single buffer to detect runaway encodes. + let bufSizeCap = newLit(64 * 1024 * 1024) + + result = newStmtList() + + # Compile-time validation of the mandatory request types. + result.add( + quote do: + when not compiles(typeof(`initializeReqIdent`)): + {. + error: + "registerBrokerLibrary: initializeRequest type '" & + astToStr(`initializeReqIdent`) & + "' is not defined. Ensure a RequestBroker(API) declaring this type " & + "appears before registerBrokerLibrary." + .} + when not compiles(typeof(`shutdownReqIdent`)): + {. + error: + "registerBrokerLibrary: shutdownRequest type '" & + astToStr(`shutdownReqIdent`) & + "' is not defined. Ensure a RequestBroker(API) declaring this type " & + "appears before registerBrokerLibrary." + .} + ) + + # Foreign-thread GC bootstrap (once per compilation unit). + result.add(emitEnsureForeignThreadGc()) + + # NimMain import on POSIX (Windows DllMain auto-runs it). + when not defined(windows): + let nimMainImportName = newLit(libName & "NimMain") + result.add( + quote do: + proc `nimMainIdent`() {.importc: `nimMainImportName`, cdecl.} + ) + + # ------------------------------------------------------------------ + # Build the dispatch case statement from accumulated request entries. + # Snapshot and clear the accumulator so a second registerBrokerLibrary in + # the same compilation unit (a future multi-library scenario) starts fresh. + # ------------------------------------------------------------------ + # Snapshot the registered request adapters and event entries. We + # deliberately do NOT clear the global accumulators here — Nim's + # compile-time VM aliases `let` copies of seqs back to the source, so + # resetting before reading would leave us with an empty list. A future + # multi-library-per-compilation scenario would need a different pattern + # (e.g., snapshot a length and slice from there next time). + let entries = gApiCborRequestEntries + let eventEntries = gApiCborEventEntries + + # The dispatch proc is async and returns just `seq[byte]`. To signal + # "unknown apiName" without raising or capturing a `var bool`, the + # convention is: empty seq + the calling `_call` checks against the + # known-name set (a separate non-async predicate proc). + let knownNamePredIdent = ident(libName & "CborIsKnownApiName") + + var caseStmt = nnkCaseStmt.newTree(ident("apiName")) + for entry in entries: + let nameLit = newLit(entry.apiName) + let adapterCall = newCall(ident(entry.adapterProc), ident("ctx"), ident("reqBuf")) + let branchBody = newStmtList( + nnkReturnStmt.newTree(nnkCommand.newTree(ident("await"), adapterCall)) + ) + caseStmt.add(nnkOfBranch.newTree(nameLit, branchBody)) + caseStmt.add( + nnkElse.newTree( + newStmtList(nnkReturnStmt.newTree(prefix(nnkBracket.newTree(), "@"))) + ) + ) + + let dispatchProc = nnkProcDef.newTree( + postfix(dispatchProcIdent, "*"), + newEmptyNode(), + newEmptyNode(), + nnkFormalParams.newTree( + nnkBracketExpr.newTree( + ident("Future"), nnkBracketExpr.newTree(ident("seq"), ident("byte")) + ), + newIdentDefs(ident("apiName"), ident("string")), + newIdentDefs(ident("ctx"), ident("BrokerContext")), + newIdentDefs(ident("reqBuf"), nnkBracketExpr.newTree(ident("seq"), ident("byte"))), + ), + nnkPragma.newTree( + newColonExpr( + ident("async"), + nnkTupleConstr.newTree(newColonExpr(ident("raises"), nnkBracket.newTree())), + ), + ident("gcsafe"), + ), + newEmptyNode(), + newStmtList(caseStmt), + ) + result.add(dispatchProc) + + # ------------------------------------------------------------------ + # reduced-A (A4): per-context teardown. `_releaseInstance(ctx)` and + # `_shutdown` route a reserved-apiName message to the processing thread + # which runs this proc: it clears the request providers and drops the event + # listeners keyed by `ctx`. Running here (the processing thread) is required + # because the MT broker buckets are keyed by the processing thread's id — + # clearing from a foreign thread would touch the wrong bucket. After this the + # Nim sub-instance is no longer pinned by its provider closures and the GC + # reclaims it (no FFI-side ownership). Idempotent: clearing an absent ctx is a + # no-op. `when compiles` guards keep it valid whether a broker is request/ + # event / single-thread / mt / API. + block: + var seenReq: seq[string] = @[] + var src = "proc " & releaseCtxProcName & "(ctx: BrokerContext) {.gcsafe.} =\n" + var body = "" + for entry in entries: + if entry.responseTypeName.len == 0 or entry.responseTypeName in seenReq: + continue + seenReq.add(entry.responseTypeName) + body.add( + " when compiles(" & entry.responseTypeName & ".clearProvider(ctx)):\n" & " " & + entry.responseTypeName & ".clearProvider(ctx)\n" + ) + for e in eventEntries: + body.add( + " when compiles(" & e.typeName & ".dropAllListeners(ctx)):\n" & + " when typeof(" & e.typeName & ".dropAllListeners(ctx)) is void:\n" & + " " & e.typeName & ".dropAllListeners(ctx)\n" & " else:\n" & + " discard " & e.typeName & ".dropAllListeners(ctx)\n" + ) + if body.len == 0: + body = " discard ctx\n" + src.add(body) + try: + result.add(parseStmt(src)) + except ValueError as exc: + error("reduced-A release-teardown codegen failed: " & exc.msg) + + # Companion predicate: foreign caller dispatch needs to distinguish + # "unknown name" from "known name with empty response". Predicate is a + # plain non-async proc so `_call` can call it directly. + var nameSet = newStmtList() + var nameCase = nnkCaseStmt.newTree(ident("apiName")) + for entry in entries: + nameCase.add( + nnkOfBranch.newTree( + newLit(entry.apiName), newStmtList(nnkReturnStmt.newTree(ident("true"))) + ) + ) + nameCase.add(nnkElse.newTree(newStmtList(nnkReturnStmt.newTree(ident("false"))))) + nameSet.add(nameCase) + let knownProc = nnkProcDef.newTree( + postfix(knownNamePredIdent, "*"), + newEmptyNode(), + newEmptyNode(), + nnkFormalParams.newTree( + ident("bool"), newIdentDefs(ident("apiName"), ident("string")) + ), + nnkPragma.newTree(ident("gcsafe")), + newEmptyNode(), + nameSet, + ) + result.add(knownProc) + + # ------------------------------------------------------------------ + # Event known-name predicate (companion to request side). + # ------------------------------------------------------------------ + block: + var eventCase = nnkCaseStmt.newTree(ident("eventName")) + for e in eventEntries: + eventCase.add( + nnkOfBranch.newTree( + newLit(e.apiName), newStmtList(nnkReturnStmt.newTree(ident("true"))) + ) + ) + eventCase.add(nnkElse.newTree(newStmtList(nnkReturnStmt.newTree(ident("false"))))) + let eventKnownProc = nnkProcDef.newTree( + postfix(knownEventPredIdent, "*"), + newEmptyNode(), + newEmptyNode(), + nnkFormalParams.newTree( + ident("bool"), newIdentDefs(ident("eventName"), ident("string")) + ), + nnkPragma.newTree(ident("gcsafe")), + newEmptyNode(), + newStmtList(eventCase), + ) + result.add(eventKnownProc) + + # ------------------------------------------------------------------ + # Per-library types and globals. + # ------------------------------------------------------------------ + result.add( + quote do: + type + `procThreadArgIdent` = object + ctx: BrokerContext + shutdownFlag: Atomic[int] + processingReady: Atomic[int] + deliveryReady: Atomic[int] + processingErrorMessage: cstring + deliveryErrorMessage: cstring + # Part C — buffer courier. `courier` is allocated in + # `_createContext` and freed in `_shutdown`. `courierSignal` is + # the processing thread's broker dispatch signal, published by + # the processing thread once its chronos loop is up so a foreign + # `_call` can wake it after enqueuing a request. + courier: ptr CborCourier + courierSignal: ThreadSignalPtr + # Part D-3 — event courier. Producer is the processing thread + # (per-event handler runs there now, encode-once-on-emit-thread); + # consumer is the delivery thread, which polls the ring and + # fans out foreign callbacks. `deliverySignal` is the delivery + # thread's broker dispatch signal so the processing thread can + # wake it after enqueuing an event. + eventCourier: ptr CborEventCourier + deliverySignal: ThreadSignalPtr + + `ctxEntryIdent` = object + ctx: BrokerContext + procThread: Thread[ptr `procThreadArgIdent`] + delivThread: Thread[ptr `procThreadArgIdent`] + arg: ptr `procThreadArgIdent` + active: bool + + `eventCallbackTypeIdent`* = proc( + ctx: uint32, + eventName: cstring, + payloadBuf: pointer, + payloadLen: int32, + userData: pointer, + ) {.cdecl, gcsafe, raises: [].} + + var `ctxsIdent`: seq[ptr `ctxEntryIdent`] + var `ctxsLockIdent`: Lock + var `ctxsInitIdent`: Atomic[int] + + # Subscription registry: shared-heap hash table from + # `api_cbor_subs_registry`. Lazily allocated in `_initialize`. + var `subsRegIdent`: ptr SubsRegistry + var `subsHandleIdent`: Atomic[uint64] + + var `nimInitFlagIdent`: Atomic[int] + var `gcRegFlagIdent` {.threadvar.}: bool + ) + + # ------------------------------------------------------------------ + # `_version` — static semver baked from `registerBrokerLibrary`. + # ------------------------------------------------------------------ + let cborVersionFuncName = libName & "_version" + let cborVersionFuncNameLit = newLit(cborVersionFuncName) + let cborVersionFuncIdent = ident(cborVersionFuncName) + let cborVersionConstIdent = ident("g" & libName & "VersionStr") + let cborVersionStrLit = newLit(config.version) + result.add( + quote do: + const `cborVersionConstIdent`: cstring = `cborVersionStrLit` + proc `cborVersionFuncIdent`*(): cstring {. + exportc: `cborVersionFuncNameLit`, cdecl, dynlib + .} = + `cborVersionConstIdent` + + ) + + # ------------------------------------------------------------------ + # `_initialize` — Nim runtime + GC setup. Idempotent. + # ------------------------------------------------------------------ + result.add( + quote do: + proc `initFuncIdent`*() {.exportc: `initFuncNameLit`, cdecl, dynlib.} = + # Step 1: one-time Nim runtime init. + while true: + case `nimInitFlagIdent`.load(moAcquire) + of 2: + break + of -1: + return + of 1: + sleep(1) + else: + var expected = 0 + if `nimInitFlagIdent`.compareExchange(expected, 1, moAcquire, moRelaxed): + when compileOption("app", "lib") and not defined(windows): + let initRes = catch: + `nimMainIdent`() + if initRes.isErr(): + error "Failed to initialize Nim runtime", + library = `libNameLit`, detail = initRes.error.msg + `nimInitFlagIdent`.store(-1, moRelease) + return + `nimInitFlagIdent`.store(2, moRelease) + break + + # Step 2: per-thread foreign GC registration. + when compileOption("app", "lib"): + if not `gcRegFlagIdent`: + when declared(setupForeignThreadGc): + setupForeignThreadGc() + `gcRegFlagIdent` = true + when declared(nimGC_setStackBottom): + var locals {.volatile, noinit.}: pointer + locals = addr(locals) + nimGC_setStackBottom(locals) + + # Step 3: lazy-init the global ctx registry and the subs map/lock. + var ctxsExpected = 0 + if `ctxsInitIdent`.compareExchange(ctxsExpected, 1, moAcquire, moRelaxed): + initLock(`ctxsLockIdent`) + `subsRegIdent` = subsRegistryNew() + `ctxsInitIdent`.store(2, moRelease) + else: + while `ctxsInitIdent`.load(moAcquire) != 2: + sleep(1) + + ) + + # ------------------------------------------------------------------ + # `_allocBuffer` / `_freeBuffer`. + # ------------------------------------------------------------------ + result.add( + quote do: + proc `allocBufFuncIdent`*( + size: int32 + ): pointer {.exportc: `allocBufFuncNameLit`, cdecl, dynlib.} = + ensureForeignThreadGc() + if size <= 0 or size.int > `bufSizeCap`: + return nil + allocShared0(size.int) + + proc `freeBufFuncIdent`*( + buf: pointer + ) {.exportc: `freeBufFuncNameLit`, cdecl, dynlib.} = + ensureForeignThreadGc() + if not buf.isNil: + deallocShared(buf) + + ) + + # ------------------------------------------------------------------ + # Part D-3 — per-event-type globals. + # + # Each event type gets: + # * `gCborSubsCount: Atomic[int]` — the lock-free + # emit-side fast-path discriminator. Bumped by `_subscribe` after + # the registry insertion succeeds, decremented by `_unsubscribe`. + # Read with `moAcquire` in the per-event handler; if zero, the + # handler returns immediately with NO CBOR encode and NO courier + # enqueue (the 90 % production case is "no foreign subscriber", + # and that case must pay nothing — see PartD plan §1, §5). + # ------------------------------------------------------------------ + var perEventGlobals = newStmtList() + for e in eventEntries: + let subsCountIdent = ident("g" & libName & "Cbor" & e.typeName & "SubsCount") + perEventGlobals.add( + quote do: + var `subsCountIdent`: Atomic[int] + ) + result.add(perEventGlobals) + + # ------------------------------------------------------------------ + # Part D-3 — name→atomic lookup. The subscribe / unsubscribe entry + # points receive a `cstring` event name and need to find the right + # per-event counter to bump / decrement. A generated case statement + # does the dispatch in O(1) string-equality with no Table allocation. + # Returns `nil` for unknown names — the subscribe path filters those + # via `knownEventPredIdent` BEFORE calling, so a nil here is a + # programming error (the case covers every registered event). + # ------------------------------------------------------------------ + block: + var lookupBranches = newStmtList() + let nameVar = ident("name") + var caseStmt = nnkCaseStmt.newTree(nameVar) + for e in eventEntries: + let subsCountIdent = ident("g" & libName & "Cbor" & e.typeName & "SubsCount") + caseStmt.add( + nnkOfBranch.newTree( + newLit(e.apiName), + newStmtList(nnkReturnStmt.newTree(nnkAddr.newTree(subsCountIdent))), + ) + ) + caseStmt.add(nnkElse.newTree(newStmtList(nnkReturnStmt.newTree(newNilLit())))) + lookupBranches.add(caseStmt) + let lookupProc = nnkProcDef.newTree( + postfix(getEventSubsCountIdent, "*"), + newEmptyNode(), + newEmptyNode(), + nnkFormalParams.newTree( + nnkPtrTy.newTree(nnkBracketExpr.newTree(ident("Atomic"), ident("int"))), + newIdentDefs(nameVar, ident("string")), + ), + nnkPragma.newTree( + ident("gcsafe"), nnkExprColonExpr.newTree(ident("raises"), nnkBracket.newTree()) + ), + newEmptyNode(), + lookupBranches, + ) + result.add(lookupProc) + + # Part D-3: companion name → static cstring resolver. Takes a cstring + # (the on-stack one from the courier message) and returns the + # equivalent **string-literal-backed** cstring — permanently valid, + # safe for the callback to store across the courier-poll boundary. + # `else` returns the input pointer as a degraded fallback (the + # subscribe path already filters unknown events via + # `knownEventPredIdent`, so this branch should be unreachable in + # practice). + block: + var resolverBranches = newStmtList() + let nameVar = ident("name") + var caseStmt = nnkCaseStmt.newTree(newCall(ident("$"), nameVar)) + for e in eventEntries: + caseStmt.add( + nnkOfBranch.newTree( + newLit(e.apiName), + newStmtList( + nnkReturnStmt.newTree(newDotExpr(newLit(e.apiName), ident("cstring"))) + ), + ) + ) + caseStmt.add(nnkElse.newTree(newStmtList(nnkReturnStmt.newTree(nameVar)))) + resolverBranches.add(caseStmt) + let resolverProc = nnkProcDef.newTree( + postfix(resolveEventNameCstrIdent, "*"), + newEmptyNode(), + newEmptyNode(), + nnkFormalParams.newTree(ident("cstring"), newIdentDefs(nameVar, ident("cstring"))), + nnkPragma.newTree( + ident("gcsafe"), nnkExprColonExpr.newTree(ident("raises"), nnkBracket.newTree()) + ), + newEmptyNode(), + resolverBranches, + ) + result.add(resolverProc) + + # ------------------------------------------------------------------ + # Per-event listener installers — Part D-3 rewrite. + # + # Each installer registers an MT-broker listener whose body is the + # FFI-lane emit-side dispatch: + # + # 1. `load(moAcquire)` the per-event subs-count atomic. + # If zero → return immediately. No encode, no allocation, no + # courier touch. This is the 90 % production hot path. + # 2. CBOR-encode the event payload **once** into a shared-heap + # buffer (emit-thread cost, paid only when subscribers exist). + # 3. Enqueue an `EventMsg` (eventName + ctx + buf + bufLen) into + # the per-context event courier ring; ownership of the buffer + # transfers to the consumer (delivery thread). Fire the + # delivery thread's broker dispatch signal. + # 4. Foreign-callback fanout happens on the delivery thread, NOT + # here. Slow / reentrant foreign callbacks therefore can't + # block the provider that emitted the event (Part D §1a). + # + # The installer also: + # * registers a `dropAllListeners` companion cleanup hook (PartD + # §8a) so when user Nim code calls `.dropAllListeners`, + # the foreign-subscriber registry is cleared in lock-step. + # + # Same-thread fast path: `installAllListenersIdent` is called on the + # PROCESSING thread (see processing-thread proc), so emit (processing + # thread) → MT broker same-thread direct asyncSpawn → handler runs + # on processing thread → does atomic check + encode + courier + # enqueue (no MT-slab marshal). The delivery thread then receives + # the opaque buffer via the courier ring. + # ------------------------------------------------------------------ + var installerNames: seq[string] = @[] + for e in eventEntries: + let eventTypeIdent = ident(e.typeName) + let installerIdent = ident(libName & "Cbor" & e.typeName & "Installer") + let eventNameLit = newLit(e.apiName) + let subsCountIdent = ident("g" & libName & "Cbor" & e.typeName & "SubsCount") + let dropAllHookProcTypeIdent = ident(e.typeName & "MtDropAllHook") + let setDropAllHookIdent = ident("setDropAll" & e.typeName & "Hook") + installerNames.add($installerIdent) + result.add( + quote do: + proc `installerIdent`*(ctx: BrokerContext): Result[void, string] = + # The arg is `arg: ptr ` — captured by the + # closure via the caller (`installAllListeners(ctx, arg)`). + # But since installers run inside the processing-thread proc + # where `arg` is in scope, we access it through a global + # bootstrap: the umbrella installer below threads the arg in. + proc handler( + evt: `eventTypeIdent` + ): Future[void] {.async: (raises: []), gcsafe.} = + # Part D-3 fast path: lock-free atomic discriminator. + if `subsCountIdent`.load(moAcquire) == 0: + return + # CBOR-encode the payload once into a shared-heap buffer. + # Ownership of `payloadBuf` transfers to the courier on + # successful enqueue; freed by the delivery-thread poller + # after the foreign-callback fanout completes. + var payloadBuf: pointer = nil + var payloadLen: int = 0 + let encRes = cborEncodeShared(evt, payloadBuf, payloadLen) + if encRes.isErr: + return + # Build the courier message. `eventName` is inlined as + # NUL-terminated ASCII so the message is pure POD (no GC). + var msg: CborEventMsg + let nameLit: cstring = `eventNameLit`.cstring + var ni = 0 + while ni < CborEventNameMax - 1 and nameLit[ni] != '\0': + msg.eventName[ni] = nameLit[ni] + inc ni + msg.eventName[ni] = '\0' + msg.ctx = uint32(ctx) + msg.buf = payloadBuf + msg.bufLen = int32(payloadLen) + # Locate the per-context event courier via the ctx table. + # (Kept simple by walking the small ctx list under the lock; + # in steady state there's one or a handful of ctxs.) The + # ctx list is a global `seq` (GC'd container) but we only + # read pointer fields out of it under the lock — the + # cast(gcsafe) annotation is required because the + # generated async handler is gcsafe by signature. + # reduced-A: route by classCtx (low16) so a SUB-INSTANCE emit + # (sub ctx shares the library classCtx, distinct instanceCtx) finds + # the owning library's event courier. `msg.ctx` carries the FULL + # ctx, so the delivery thread still snapshots subscribers by the + # exact emitting ctx — per-instance event routing stays exact. + let libCtxKey = uint32(ctx) and 0x0000FFFF'u32 + var courier: ptr CborEventCourier = nil + var sig: ThreadSignalPtr = nil + {.cast(gcsafe).}: + withLock `ctxsLockIdent`: + for i in 0 ..< `ctxsIdent`.len: + let e = `ctxsIdent`[i] + if (uint32(e.ctx) and 0x0000FFFF'u32) == libCtxKey and e.active: + courier = e.arg.eventCourier + sig = e.arg.deliverySignal + break + if courier.isNil: + # Ctx torn down between subscribe and emit — drop cleanly. + if not payloadBuf.isNil: + deallocShared(payloadBuf) + return + if not tryEnqueue(addr courier.ring, msg): + # Ring full — drop the event (fire-and-forget contract). + # The buffer never entered the ring so we own it. + if not payloadBuf.isNil: + deallocShared(payloadBuf) + return + if not sig.isNil: + fireBrokerSignal(sig) + + let listenRes = `eventTypeIdent`.listen(ctx, handler) + if listenRes.isErr: + return Result[void, string].err(listenRes.error) + + # Part D-3 §8a: register the dropAllListeners cleanup hook + # so that if user Nim code calls `.dropAllListeners(ctx)`, + # the foreign-subscriber registry for this `(ctx, eventName)` + # is cleared and the atomic counter is reset. Without this + # hook, foreign subs would orphan (the listener that reads + # them is gone, but `_subscribe` would still bump the count + # → wasted encodes; the SubNodes would only release at + # `_shutdown`'s `subsRegistryFreeForCtx`). + proc dropAllHook(brokerCtx: BrokerContext) {.gcsafe, raises: [].} = + # Decrement the shared per-event subs-count by the exact number of + # subs removed for THIS ctx — never reset to 0, which would silence + # sibling contexts/instances sharing the event name. + let removed = subsRegistryRemoveAllForKeyN( + `subsRegIdent`, uint32(brokerCtx), `eventNameLit`.cstring + ) + if removed > 0: + discard `subsCountIdent`.fetchSub(removed, moRelease) + + `eventTypeIdent`.`setDropAllHookIdent`(dropAllHook) + return Result[void, string].ok() + + ) + + # ------------------------------------------------------------------ + # Umbrella installer: called from the processing thread after + # setupProviders so listeners are live before any foreign caller can + # subscribe. + # ------------------------------------------------------------------ + var installerCalls = newStmtList() + for installerName in installerNames: + let installerIdent = ident(installerName) + installerCalls.add( + newTree(nnkPrefix, ident("?"), newCall(installerIdent, ident("ctx"))) + ) + installerCalls.add( + nnkReturnStmt.newTree( + newCall( + newDotExpr( + nnkBracketExpr.newTree(ident("Result"), ident("void"), ident("string")), + ident("ok"), + ) + ) + ) + ) + let installAllProc = nnkProcDef.newTree( + postfix(installAllListenersIdent, "*"), + newEmptyNode(), + newEmptyNode(), + nnkFormalParams.newTree( + nnkBracketExpr.newTree(ident("Result"), ident("void"), ident("string")), + newIdentDefs(ident("ctx"), ident("BrokerContext")), + ), + newEmptyNode(), + newEmptyNode(), + installerCalls, + ) + result.add(installAllProc) + + # ------------------------------------------------------------------ + # `_subscribe` / `_unsubscribe`. + # + # Subscribe handle convention: 0 == failure (unknown event, allocation + # error, nil callback for non-probe paths). 1 is reserved as the + # "supported" sentinel returned for probe calls (cb == nil). Real + # subscriptions start at 2 to avoid colliding with these reserved + # values. + # ------------------------------------------------------------------ + result.add( + quote do: + proc `subscribeFuncIdent`*( + ctx: uint32, + eventNameC: cstring, + cb: `eventCallbackTypeIdent`, + userData: pointer, + ): uint64 {.exportc: `subscribeFuncNameLit`, cdecl, dynlib.} = + ensureForeignThreadGc() + if eventNameC.isNil: + return 0'u64 + let name = $eventNameC + if not `knownEventPredIdent`(name): + return 0'u64 + if cb.isNil: + # Probe mode: caller wants to know whether the eventName is + # supported by this library version. 1 is a sentinel never + # returned for real subscriptions. + return 1'u64 + # Real subscription handle: skip 0 (failure) and 1 (probe). + let h = `subsHandleIdent`.fetchAdd(1, moRelaxed) + 2'u64 + # `eventNameC` is owned by the foreign caller; the registry copies it + # into shared heap on insertion, so we don't need to keep `name` + # alive past this call. + subsRegistryAdd(`subsRegIdent`, ctx, eventNameC, h, cast[pointer](cb), userData) + # Part D-3: bump the per-event subs-count atomic AFTER the + # registry insertion. moRelease pairs with the emit-side's + # moAcquire load — if the load sees > 0, the registry already + # contains this subscription (the snapshot the delivery thread + # takes will see it on the next emit, modulo single-window + # race that's documented as expected). + let counter = `getEventSubsCountIdent`(name) + if not counter.isNil: + discard counter[].fetchAdd(1, moRelease) + return h + + proc `unsubscribeFuncIdent`*( + ctx: uint32, eventNameC: cstring, handle: uint64 + ): int32 {.exportc: `unsubscribeFuncNameLit`, cdecl, dynlib.} = + ensureForeignThreadGc() + if eventNameC.isNil: + return -1'i32 + let name = $eventNameC + if handle == 0'u64: + # Drop every subscription for this (ctx, name). Decrement the + # shared per-event counter by the exact number removed — never + # reset to 0, which would silence other ctxs/instances sharing + # the event name (the counter is a process-global aggregate gate). + let removed = subsRegistryRemoveAllForKeyN(`subsRegIdent`, ctx, eventNameC) + if removed >= 0: + if removed > 0: + let counter = `getEventSubsCountIdent`(name) + if not counter.isNil: + discard counter[].fetchSub(removed, moRelease) + return 0'i32 + return removed + let res = subsRegistryRemoveOne(`subsRegIdent`, ctx, eventNameC, handle) + if res == 0: + # Part D-3: decrement after a successful removal. + let counter = `getEventSubsCountIdent`(name) + if not counter.isNil: + discard counter[].fetchSub(1, moRelease) + return res + + ) + + # ------------------------------------------------------------------ + # Delivery thread proc (one per ctx). Part D, phase D-3: pure event + # courier consumer. Polls `arg.eventCourier.ring` for opaque CBOR + # buffers produced by the processing thread, snapshots the + # foreign-subscriber list for `(ctx, eventName)`, fans out the + # synchronous foreign callbacks, then frees the buffer. + # + # This thread does NOT install MT EventBroker listeners (those moved + # back to the processing thread in D-3 to recover the same-thread + # fast path — the FFI lane forks at the per-event handler instead of + # at the MT broker dispatch layer). The thread still owns a chronos + # event loop (via `ensureBrokerDispatchStarted`) so the + # `eventCourierPoll` proc registered below runs whenever the + # delivery signal fires. + # ------------------------------------------------------------------ + result.add( + quote do: + proc `delivThreadProcIdent`(arg: ptr `procThreadArgIdent`) {.thread.} = + setThreadBrokerContext(arg.ctx) + + # Publish this thread's broker dispatch signal so the processing + # thread (the producer) can wake us after enqueuing an event. + arg.deliverySignal = getOrInitBrokerSignal() + + # Event-courier poller — drains the ring, fans out, frees. + # `subsRegistrySnapshot` allocates the snapshot on shared heap; + # we free it via `subsRegistrySnapshotFree` after the fanout. + proc eventCourierPoll(): int {.gcsafe, raises: [].} = + var didWork = 0 + while true: + var m: CborEventMsg + if not tryDequeue(addr arg.eventCourier.ring, m): + break + didWork = 1 + # The eventName was inlined NUL-terminated into the courier + # message; that storage is the poll proc's stack frame after + # `tryDequeue` and dies the moment this proc returns. Foreign + # callbacks legitimately store the eventName cstring across + # calls (the typemappingtestlib test does), so we resolve to + # a STATIC string-literal cstring via the per-library + # generated resolver before invoking the callback. + let stackNameC = cast[cstring](addr m.eventName[0]) + let nameC = `resolveEventNameCstrIdent`(stackNameC) + var snap: ptr UncheckedArray[SubSnapshot] = nil + var snapLen: int = 0 + subsRegistrySnapshot(`subsRegIdent`, m.ctx, nameC, snap, snapLen) + if snapLen > 0 and not m.buf.isNil: + for i in 0 ..< snapLen: + let cbPtr = snap[i].cb + if cbPtr.isNil: + continue + let cbTyped = cast[`eventCallbackTypeIdent`](cbPtr) + cbTyped(m.ctx, nameC, m.buf, m.bufLen, snap[i].userData) + if snapLen > 0: + subsRegistrySnapshotFree(snap) + if not m.buf.isNil: + deallocShared(m.buf) + didWork + + registerBrokerPoller(eventCourierPoll) + ensureBrokerDispatchStarted() + + arg.deliveryReady.store(1, moRelease) + + proc awaitShutdown(flag: ptr Atomic[int]) {.async: (raises: []).} = + while flag[].load(moAcquire) != 1: + let s = catch: + await sleepAsync(milliseconds(5)) + if s.isErr(): + discard + + waitFor awaitShutdown(addr arg.shutdownFlag) + # Drain + tear down the dispatch loop before this thread exits. + # Any in-flight foreign callback runs synchronously inside + # eventCourierPoll and returns before the loop exits. Buffers + # still queued in the courier ring at this point are freed by + # `drainAndFree(arg.eventCourier)` in `_shutdown`. + stopBrokerDispatchHere() + + ) + + # ------------------------------------------------------------------ + # Processing thread proc (one per ctx). Runs setupProviders then loops + # on a chronos event loop until shutdownFlag is set. + # ------------------------------------------------------------------ + result.add( + quote do: + proc `procThreadProcIdent`(arg: ptr `procThreadArgIdent`) {.thread.} = + setThreadBrokerContext(arg.ctx) + + when compiles(setupProviders(arg.ctx).isErr()): + let setupCatchRes = catch: + setupProviders(arg.ctx) + if setupCatchRes.isErr(): + arg.processingErrorMessage = + allocCStringCopy("setupProviders raised: " & setupCatchRes.error.msg) + arg.processingReady.store(-1, moRelease) + return + let setupRes = setupCatchRes.get() + if setupRes.isErr(): + arg.processingErrorMessage = allocCStringCopy(setupRes.error()) + arg.processingReady.store(-1, moRelease) + return + elif compiles(setupProviders(arg.ctx)): + let setupCatchRes = catch: + setupProviders(arg.ctx) + if setupCatchRes.isErr(): + arg.processingErrorMessage = + allocCStringCopy("setupProviders raised: " & setupCatchRes.error.msg) + arg.processingReady.store(-1, moRelease) + return + + # Part D phase D-3: event listener installation runs on the + # PROCESSING thread (back from the D-2 delivery-thread arm) so + # that emit (processing thread) → MT broker takes the + # same-thread direct asyncSpawn fast path → per-event handler + # runs here on the processing thread → atomic-check + CBOR + # encode + courier enqueue → delivery thread receives the + # opaque buffer through `arg.eventCourier.ring` and fans out + # foreign callbacks. Net: no MT-slab marshal in the FFI lane, + # foreign callbacks still run off the processing thread. + let installCatchRes = catch: + `installAllListenersIdent`(arg.ctx) + if installCatchRes.isErr(): + arg.processingErrorMessage = allocCStringCopy( + "event listener install raised: " & installCatchRes.error.msg + ) + arg.processingReady.store(-1, moRelease) + return + let installRes = installCatchRes.get() + if installRes.isErr(): + arg.processingErrorMessage = + allocCStringCopy("event listener install failed: " & installRes.error()) + arg.processingReady.store(-1, moRelease) + return + + # ---------------------------------------------------------------- + # Part C — buffer courier. The processing thread owns CBOR decode, + # the provider call, and CBOR encode. A foreign `_call` hands + # us a raw request buffer over `arg.courier.chan` and blocks on a + # response slot; we wake on the shared broker dispatch signal. + # ---------------------------------------------------------------- + proc handleCourierMsg(m: CborCallMsg) {.async: (raises: []), gcsafe.} = + # Copy the request bytes off the shared buffer, then free it — + # ownership of `m.reqBuf` transferred to us via the channel. + var nimReq = newSeq[byte](m.reqLen.int) + if m.reqLen > 0 and not m.reqBuf.isNil: + copyMem(addr nimReq[0], m.reqBuf, m.reqLen.int) + if not m.reqBuf.isNil: + deallocShared(m.reqBuf) + let apiName = $cast[cstring](addr m.apiName[0]) + var respBuf: pointer = nil + var respLen: int32 = 0 + var status: int32 = 0 + if apiName == `releaseApiNameLit`: + # reduced-A: per-context teardown control op (from + # `_releaseInstance`). Clears providers + listeners for the + # addressed ctx on this (processing) thread, then completes the slot. + `releaseCtxProcIdent`(BrokerContext(m.targetCtx)) + completeSlot(arg.courier, m.slotIdx.int, nil, 0'i32, 0'i32) + return + if not `knownNamePredIdent`(apiName): + let em = "unknown apiName: " & apiName + let b = allocShared0(em.len) + if em.len > 0: + copyMem(b, unsafeAddr em[0], em.len) + respBuf = b + respLen = int32(em.len) + status = -4'i32 + else: + # reduced-A: dispatch against the FULL ctx the caller addressed + # (sub-instance ctx for create-instance subs; == arg.ctx otherwise), + # so the broker provider keyed by the sub ctx is reached. Falls back + # to arg.ctx for legacy messages where targetCtx was never set (0). + let dispCtx = + if m.targetCtx != 0'u32: + BrokerContext(m.targetCtx) + else: + arg.ctx + let dispRes = catch: + await `dispatchProcIdent`(apiName, dispCtx, nimReq) + if dispRes.isErr(): + status = -10'i32 + else: + let respBytes = dispRes.get() + if respBytes.len > 0: + let b = allocShared0(respBytes.len) + copyMem(b, unsafeAddr respBytes[0], respBytes.len) + respBuf = b + respLen = int32(respBytes.len) + completeSlot(arg.courier, m.slotIdx.int, respBuf, respLen, status) + + # Drained by the shared `brokerDispatchLoop` whenever the dispatch + # signal fires. Each message is handled on its own spawned + # coroutine so a slow provider does not stall the drain. + proc courierPoll(): int {.gcsafe, raises: [].} = + var didWork = 0 + while true: + var m: CborCallMsg + if not tryDequeue(addr arg.courier.ring, m): + break + asyncSpawn handleCourierMsg(m) + didWork = 1 + didWork + + # Publish this thread's dispatch signal so a foreign `_call` + # can wake us, register the courier poller, and start the loop. + # Done BEFORE `processingReady = 1` so the first `_call` (which can + # only arrive after `createContext` returns) is always serviced. + arg.courierSignal = getOrInitBrokerSignal() + registerBrokerPoller(courierPoll) + ensureBrokerDispatchStarted() + + arg.processingReady.store(1, moRelease) + + proc awaitShutdown(flag: ptr Atomic[int]) {.async: (raises: []).} = + while flag[].load(moAcquire) != 1: + let s = catch: + await sleepAsync(milliseconds(5)) + if s.isErr(): + discard + + waitFor awaitShutdown(addr arg.shutdownFlag) + # Part C: tear down the dispatch-loop coroutine cleanly before the + # thread exits. `_shutdown` waits for `courier.inFlight` to reach 0 + # (while this thread is still handling) before it sets + # `shutdownFlag`, so no courier message is in flight here. + stopBrokerDispatchHere() + + ) + + # ------------------------------------------------------------------ + # `_createContext` — spawn delivery + processing threads, await ready. + # `_shutdown` — signal, join both, free. + # ------------------------------------------------------------------ + result.add( + quote do: + proc `createContextFuncIdent`*( + errOut: ptr cstring + ): uint32 {.exportc: `createContextFuncNameLit`, cdecl, dynlib.} = + `initFuncIdent`() + ensureForeignThreadGc() + + # Skip BrokerContext value 0 — `_createContext` reserves 0 + # as the failure return code visible to foreign callers. + var bctx = NewBrokerContext() + while uint32(bctx) == 0'u32: + bctx = NewBrokerContext() + # reduced-A: record this library's event-listener installer keyed by its + # classCtx so create-instance requests can install courier listeners for + # sub-instance ctxs (which share this classCtx). + registerApiCtxListenerInstaller(classCtx(bctx), `installAllListenersIdent`) + let arg = + cast[ptr `procThreadArgIdent`](allocShared0(sizeof(`procThreadArgIdent`))) + arg.ctx = bctx + arg.shutdownFlag.store(0, moRelaxed) + arg.processingReady.store(0, moRelaxed) + arg.deliveryReady.store(0, moRelaxed) + arg.processingErrorMessage = nil + arg.deliveryErrorMessage = nil + # Part C — courier: 64 response slots = ceiling on concurrent + # in-flight `_call`s; a call past that fails fast. + arg.courier = newCborCourier(64) + arg.courierSignal = nil + # Part D-3 — event courier: 256-slot ring (burst capacity, not + # concurrency bound; producer is fire-and-forget). A full ring + # drops the event with a diagnostic. Re-tunable after D-6 bench. + arg.eventCourier = newCborEventCourier(256) + arg.deliverySignal = nil + + let entry = cast[ptr `ctxEntryIdent`](allocShared0(sizeof(`ctxEntryIdent`))) + entry.ctx = bctx + entry.arg = arg + entry.active = true + + # Part D — spawn delivery thread BEFORE the processing thread so + # the delivery thread is live (and can receive cross-thread events) + # before any provider emits. + let delivCreateRes = catch: + createThread(entry.delivThread, `delivThreadProcIdent`, arg) + if delivCreateRes.isErr(): + if not errOut.isNil: + errOut[] = allocCStringCopy( + "Failed to spawn delivery thread: " & delivCreateRes.error.msg + ) + freeCborCourier(arg.courier) + drainAndFree(arg.eventCourier) + deallocShared(arg) + deallocShared(entry) + return 0'u32 + + # Poll for deliveryReady. + block: + var waitedMs = 0 + const timeoutMs = 5000 + var status = 0 + while waitedMs < timeoutMs: + status = arg.deliveryReady.load(moAcquire).int + if status != 0: + break + sleep(1) + inc waitedMs + if status != 1: + arg.shutdownFlag.store(1, moRelease) + joinThread(entry.delivThread) + if not errOut.isNil: + if not arg.deliveryErrorMessage.isNil: + errOut[] = arg.deliveryErrorMessage + arg.deliveryErrorMessage = nil + else: + errOut[] = allocCStringCopy("delivery thread did not become ready") + freeCborCourier(arg.courier) + deallocShared(arg) + deallocShared(entry) + return 0'u32 + + # Spawn processing thread. + let createRes = catch: + createThread(entry.procThread, `procThreadProcIdent`, arg) + if createRes.isErr(): + if not errOut.isNil: + errOut[] = allocCStringCopy( + "Failed to spawn processing thread: " & createRes.error.msg + ) + arg.shutdownFlag.store(1, moRelease) + joinThread(entry.delivThread) + freeCborCourier(arg.courier) + drainAndFree(arg.eventCourier) + deallocShared(arg) + deallocShared(entry) + return 0'u32 + + # Poll for processingReady (or failure). + var waitedMs = 0 + const timeoutMs = 5000 + var status = 0 + while waitedMs < timeoutMs: + status = arg.processingReady.load(moAcquire).int + if status != 0: + break + sleep(1) + inc waitedMs + + if status != 1: + arg.shutdownFlag.store(1, moRelease) + joinThread(entry.delivThread) + joinThread(entry.procThread) + if not errOut.isNil: + if not arg.processingErrorMessage.isNil: + errOut[] = arg.processingErrorMessage + arg.processingErrorMessage = nil + else: + errOut[] = allocCStringCopy("processing thread did not become ready") + freeCborCourier(arg.courier) + drainAndFree(arg.eventCourier) + deallocShared(arg) + deallocShared(entry) + return 0'u32 + + withLock `ctxsLockIdent`: + `ctxsIdent`.add(entry) + return uint32(bctx) + + proc `shutdownFuncIdent`*( + ctx: uint32 + ): int32 {.exportc: `shutdownFuncNameLit`, cdecl, dynlib.} = + ensureForeignThreadGc() + var entryToShutdown: ptr `ctxEntryIdent` = nil + withLock `ctxsLockIdent`: + for i in 0 ..< `ctxsIdent`.len: + let e = `ctxsIdent`[i] + if uint32(e.ctx) == ctx and e.active: + e.active = false + entryToShutdown = e + break + if entryToShutdown.isNil: + return -1'i32 + + # Part C — drain in-flight `_call`s BEFORE stopping the processing + # thread. `active` is already false (set under the lock above) so + # no new call enters; in-flight calls complete (the processing + # thread is still handling) and decrement `inFlight`. Only once + # inFlight reaches 0 is the channel guaranteed quiescent, so + # signalling shutdown + freeing the courier cannot race a `_call`. + # A bounded timeout guards against a hung provider (best-effort). + block: + let courier = entryToShutdown.arg.courier + if not courier.isNil: + var waitedMs = 0 + const drainTimeoutMs = 5000 + while courier.inFlight.load(moAcquire) > 0 and waitedMs < drainTimeoutMs: + sleep(1) + inc waitedMs + + entryToShutdown.arg.shutdownFlag.store(1, moRelease) + # Part D: join delivery thread first — it must finish any + # in-flight foreign callbacks before we tear down the processing + # thread (which owns the providers that emitted those events). + joinThread(entryToShutdown.delivThread) + joinThread(entryToShutdown.procThread) + # Free this lib's subscription state for the whole class after both + # threads are joined — no concurrent listener can be mid-snapshot. + # The sweep drains the lib ctx (instanceCtx 0) AND every still-alive + # sub-instance sharing its classCtx, decrementing each per-event + # global subs-count by the exact number removed so the shared gate + # stays a correct running sum for sibling lib contexts. + proc onFreed(name: cstring, count: int32) {.gcsafe, raises: [].} = + let counter = `getEventSubsCountIdent`($name) + if not counter.isNil and count > 0: + discard counter[].fetchSub(count, moRelease) + + subsRegistryFreeForClass(`subsRegIdent`, classCtx(BrokerContext(ctx)), onFreed) + if not entryToShutdown.arg.processingErrorMessage.isNil: + freeCString(entryToShutdown.arg.processingErrorMessage) + if not entryToShutdown.arg.deliveryErrorMessage.isNil: + freeCString(entryToShutdown.arg.deliveryErrorMessage) + # Part C — free the courier after both threads joined. + freeCborCourier(entryToShutdown.arg.courier) + # Part D-3 — free the event courier (drains any messages left + # in the ring, freeing their buffers) after both threads joined. + drainAndFree(entryToShutdown.arg.eventCourier) + deallocShared(entryToShutdown.arg) + + withLock `ctxsLockIdent`: + for i in 0 ..< `ctxsIdent`.len: + if `ctxsIdent`[i] == entryToShutdown: + `ctxsIdent`.del(i) + break + deallocShared(entryToShutdown) + return 0'i32 + + ) + + # ------------------------------------------------------------------ + # `_call` — string dispatch over the generated case statement. + # ------------------------------------------------------------------ + result.add( + quote do: + proc `callFuncIdent`*( + ctx: uint32, + apiNameC: cstring, + reqBuf: pointer, + reqLen: int32, + respBufOut: ptr pointer, + respLenOut: ptr int32, + ): int32 {.exportc: `callFuncNameLit`, cdecl, dynlib.} = + # Part C — buffer courier. This runs on the foreign caller's + # thread and does NO CBOR decode and NO chronos loop: it hands the + # raw request buffer to the processing thread and blocks on a + # response slot. `reqBuf` ownership transfers to the processing + # thread on a successful `send`; every error path frees it here. + ensureForeignThreadGc() + if respBufOut.isNil or respLenOut.isNil: + if not reqBuf.isNil: + deallocShared(reqBuf) + return -1'i32 + respBufOut[] = nil + respLenOut[] = 0 + if apiNameC.isNil: + if not reqBuf.isNil: + deallocShared(reqBuf) + return -2'i32 + if reqLen < 0 or reqLen.int > `bufSizeCap`: + if not reqBuf.isNil: + deallocShared(reqBuf) + return -3'i32 + let nameLen = apiNameC.len + if nameLen >= CborApiNameMax: + if not reqBuf.isNil: + deallocShared(reqBuf) + return -2'i32 + + # Resolve ctx -> courier. `inFlight` is bumped under the SAME lock + # `_shutdown` uses to flip `active`, so once shutdown has run no + # new call can enter; `_shutdown` then waits for inFlight -> 0. + # reduced-A: route by classCtx (low16). A library context is registered + # with instanceCtx 0; a sub-instance ctx shares the same classCtx but + # carries a distinct instanceCtx, so masking it off recovers the owning + # library context's courier. The full `ctx` is carried in the message + # (targetCtx) so the processing thread dispatches against the sub ctx. + let libCtxKey = ctx and 0x0000FFFF'u32 + var courier: ptr CborCourier = nil + var courierSig: ThreadSignalPtr = nil + withLock `ctxsLockIdent`: + for i in 0 ..< `ctxsIdent`.len: + let e = `ctxsIdent`[i] + if uint32(e.ctx) == libCtxKey and e.active: + courier = e.arg.courier + courierSig = e.arg.courierSignal + discard courier.inFlight.fetchAdd(1, moAcquireRelease) + break + if courier.isNil: + if not reqBuf.isNil: + deallocShared(reqBuf) + return -5'i32 + + let slotIdx = claimSlot(courier) + if slotIdx < 0: + discard courier.inFlight.fetchSub(1, moAcquireRelease) + if not reqBuf.isNil: + deallocShared(reqBuf) + return -6'i32 + + var msg: CborCallMsg + if nameLen > 0: + copyMem(addr msg.apiName[0], apiNameC, nameLen) + # `msg` is stack-zero-initialised, so apiName stays NUL-terminated. + msg.reqBuf = reqBuf + msg.reqLen = reqLen + msg.slotIdx = int32(slotIdx) + msg.targetCtx = ctx # full ctx (sub-instance routing, reduced-A) + # Ownership of reqBuf transfers into the ring here. Enqueue is + # backstopped by the slot claim above (ring.cap == slotCount), so + # a false return is a programming error rather than backpressure; + # we still handle it cleanly: undo the slot + inFlight, free + # reqBuf, return -6. + if not tryEnqueue(addr courier.ring, msg): + releaseSlot(courier, slotIdx) + discard courier.inFlight.fetchSub(1, moAcquireRelease) + if not reqBuf.isNil: + deallocShared(reqBuf) + return -6'i32 + if not courierSig.isNil: + discard courierSig.fireSync() + + let res = waitSlot(courier, slotIdx) + releaseSlot(courier, slotIdx) + respBufOut[] = res.respBuf + respLenOut[] = res.respLen + discard courier.inFlight.fetchSub(1, moAcquireRelease) + return res.status + + ) + + # ------------------------------------------------------------------ + # reduced-A (A4): `_releaseInstance(ctx)` — drop a sub-instance's + # providers + listeners. Routes a reserved-apiName control message through + # the same courier (by classCtx mask) so the teardown runs on the processing + # thread, then returns. The foreign sub-wrapper calls this from its RAII path + # (C++ dtor / Rust Drop / Go Close / Python close). Idempotent + safe on an + # already-released or unknown ctx (returns 0). The Nim sub-instance is freed + # by the GC once its providers are cleared — no FFI-side ownership. + # ------------------------------------------------------------------ + result.add( + quote do: + proc `releaseInstanceFuncIdent`*( + ctx: uint32 + ): int32 {.exportc: `releaseInstanceFuncNameLit`, cdecl, dynlib.} = + ensureForeignThreadGc() + let libCtxKey = ctx and 0x0000FFFF'u32 + var courier: ptr CborCourier = nil + var courierSig: ThreadSignalPtr = nil + withLock `ctxsLockIdent`: + for i in 0 ..< `ctxsIdent`.len: + let e = `ctxsIdent`[i] + if uint32(e.ctx) == libCtxKey and e.active: + courier = e.arg.courier + courierSig = e.arg.courierSignal + discard courier.inFlight.fetchAdd(1, moAcquireRelease) + break + if courier.isNil: + return 0'i32 # unknown/closed ctx: nothing to release. + let slotIdx = claimSlot(courier) + if slotIdx < 0: + discard courier.inFlight.fetchSub(1, moAcquireRelease) + return -6'i32 + var msg: CborCallMsg + const relName = `releaseApiNameLit` + copyMem(addr msg.apiName[0], cstring(relName), relName.len) + msg.reqBuf = nil + msg.reqLen = 0 + msg.slotIdx = int32(slotIdx) + msg.targetCtx = ctx + if not tryEnqueue(addr courier.ring, msg): + releaseSlot(courier, slotIdx) + discard courier.inFlight.fetchSub(1, moAcquireRelease) + return -6'i32 + if not courierSig.isNil: + discard courierSig.fireSync() + let res = waitSlot(courier, slotIdx) + releaseSlot(courier, slotIdx) + discard courier.inFlight.fetchSub(1, moAcquireRelease) + return res.status + + ) + + # ------------------------------------------------------------------ + # Generated artifacts: write the C header + CDDL schema next to the + # build output so foreign-language wrappers can pick them up via `-I`. + # ------------------------------------------------------------------ + let outDir = + detectOutputDir(when defined(BrokerFfiApiOutDir): BrokerFfiApiOutDir else: "") + var requestNames: seq[string] = @[] + for e in entries: + requestNames.add(e.apiName) + var eventNames: seq[string] = @[] + for e in eventEntries: + eventNames.add(e.apiName) + generateCborCHeaderFile(outDir, libName, config.version, requestNames, eventNames) + generateCborCppHeaderFile(outDir, libName, entries, eventEntries, config.mainClass) + when defined(BrokerFfiApiGenPy): + generateCborPyFile(outDir, libName, entries, eventEntries, config.mainClass) + when defined(BrokerFfiApiGenRust): + generateCborRustFile(outDir, libName, entries, eventEntries, config.mainClass) + when defined(BrokerFfiApiGenGo): + generateCborGoFile(outDir, libName, entries, eventEntries, config.mainClass) + + generateCMakePackageFiles( + outDir, libName, config.version, cborMode = true, hasCpp = true + ) + + # Emit the CDDL schema and capture its text for the runtime descriptor. + let cddlText = + generateCborCddlFile(outDir, libName, entries, eventEntries, gApiTypeRegistry) + + # ------------------------------------------------------------------ + # Discovery API (Phase 6): runtime descriptor + `_listApis` / + # `_getSchema` C exports. The descriptor is built lazily on the + # first call; subsequent calls re-use the cached value. + # ------------------------------------------------------------------ + let descriptorOnceIdent = ident("g" & libName & "CborDescriptorOnce") + let apiListOnceIdent = ident("g" & libName & "CborApiListOnce") + + # Build the descriptor population body as a Nim source string. Doing it + # this way keeps every string / int literal in straight Nim code rather + # than having to splice deeply nested AST through `quote do:`. + var buildSrc = "proc " & libName & "CborBuildDescriptor(): LibraryDescriptor =\n" + buildSrc.add(" result.libName = " & escape(libName) & "\n") + buildSrc.add(" result.cddl = " & escape(cddlText) & "\n") + buildSrc.add(" result.requests = @[\n") + for r in entries: + buildSrc.add(" ApiRequestInfo(apiName: " & escape(r.apiName) & ",\n") + let argsTypeRepr = + if r.argFields.len > 0: + upperCamel(r.apiName) & "Args" + else: + "" + buildSrc.add(" argsType: " & escape(argsTypeRepr) & ",\n") + buildSrc.add(" argFields: @[\n") + for (fname, ftype) in r.argFields: + buildSrc.add( + " ApiFieldInfo(name: " & escape(fname) & ", nimType: " & escape(ftype) & + "),\n" + ) + buildSrc.add(" ],\n") + buildSrc.add(" responseType: " & escape(r.responseTypeName) & "),\n") + buildSrc.add(" ]\n") + buildSrc.add(" result.events = @[\n") + for e in eventEntries: + buildSrc.add( + " ApiEventInfo(apiName: " & escape(e.apiName) & ", payloadType: " & + escape(e.typeName) & "),\n" + ) + buildSrc.add(" ]\n") + buildSrc.add(" result.types = @[\n") + for t in gApiTypeRegistry: + if t.name.endsWith("CborArgs"): + continue + let kindStr = + case t.kind + of atkObject: "object" + of atkEnum: "enum" + of atkAlias: "alias" + of atkDistinct: "distinct" + buildSrc.add( + " ApiTypeInfo(name: " & escape(t.name) & ", kind: " & escape(kindStr) & ",\n" + ) + buildSrc.add(" fields: @[\n") + for f in t.fields: + buildSrc.add( + " ApiFieldInfo(name: " & escape(f.name) & ", nimType: " & + escape(f.nimType) & "),\n" + ) + buildSrc.add(" ],\n") + buildSrc.add(" enumValues: @[\n") + for v in t.enumValues: + buildSrc.add( + " ApiEnumValueInfo(name: " & escape(v.name) & ", ordinal: " & $v.ordinal & + "),\n" + ) + buildSrc.add(" ],\n") + buildSrc.add(" underlyingType: " & escape(t.underlyingType) & "),\n") + buildSrc.add(" ]\n") + + # Build the lightweight ApiList in the same fashion. + buildSrc.add("\nproc " & libName & "CborBuildApiList(): ApiList =\n") + buildSrc.add(" result.libName = " & escape(libName) & "\n") + buildSrc.add(" result.requests = @[\n") + for r in entries: + buildSrc.add(" " & escape(r.apiName) & ",\n") + buildSrc.add(" ]\n") + buildSrc.add(" result.events = @[\n") + for e in eventEntries: + buildSrc.add(" " & escape(e.apiName) & ",\n") + buildSrc.add(" ]\n") + + try: + result.add(parseStmt(buildSrc)) + except ValueError as e: + error("CBOR FFI: failed to parse generated descriptor builder: " & e.msg) + + let ensureDescriptorIdent = ident(libName & "CborEnsureDescriptor") + let ensureApiListIdent = ident(libName & "CborEnsureApiList") + + # Cached singletons + lazy initialisers. + result.add( + quote do: + var `descriptorOnceIdent`: bool + var `descriptorIdent`: LibraryDescriptor + var `apiListOnceIdent`: bool + var `apiListIdent`: ApiList + + proc `ensureDescriptorIdent`() {.gcsafe.} = + {.cast(gcsafe).}: + if not `descriptorOnceIdent`: + `descriptorIdent` = `descriptorBuildIdent`() + `descriptorOnceIdent` = true + + proc `ensureApiListIdent`() {.gcsafe.} = + {.cast(gcsafe).}: + if not `apiListOnceIdent`: + `apiListIdent` = `apiListBuildIdent`() + `apiListOnceIdent` = true + + ) + + # `_listApis` — returns a JSON-encoded `ApiList` string. + result.add( + quote do: + proc `listApisFuncIdent`*( + respBufOut: ptr pointer, respLenOut: ptr int32 + ): int32 {.exportc: `listApisFuncNameLit`, cdecl, dynlib.} = + ensureForeignThreadGc() + if respBufOut.isNil or respLenOut.isNil: + return -1'i32 + respBufOut[] = nil + respLenOut[] = 0 + `ensureApiListIdent`() + var jsonStr: string + try: + jsonStr = toJsonString(`apiListIdent`) + except CatchableError: + return -10'i32 + if jsonStr.len > 0: + let buf = allocShared0(jsonStr.len) + copyMem(buf, addr jsonStr[0], jsonStr.len) + respBufOut[] = buf + respLenOut[] = int32(jsonStr.len) + return 0'i32 + + ) + + # `_getSchema` — returns a JSON-encoded `LibraryDescriptor` string. + result.add( + quote do: + proc `getSchemaFuncIdent`*( + respBufOut: ptr pointer, respLenOut: ptr int32 + ): int32 {.exportc: `getSchemaFuncNameLit`, cdecl, dynlib.} = + ensureForeignThreadGc() + if respBufOut.isNil or respLenOut.isNil: + return -1'i32 + respBufOut[] = nil + respLenOut[] = 0 + `ensureDescriptorIdent`() + var jsonStr: string + try: + jsonStr = toJsonString(`descriptorIdent`) + except CatchableError: + return -10'i32 + if jsonStr.len > 0: + let buf = allocShared0(jsonStr.len) + copyMem(buf, addr jsonStr[0], jsonStr.len) + respBufOut[] = buf + respLenOut[] = int32(jsonStr.len) + return 0'i32 + + ) + + when defined(brokerDebug): + writeBrokerDebug( + "BrokerLibrary", + libName, + result, + header = + $entries.len & " request adapters, " & $eventEntries.len & " event entries", + ) + when defined(brokerDebugStdout): + echo "[brokers/cbor] registerBrokerLibraryCborImpl emitted runtime for '" & libName & + "' with " & $entries.len & " request adapters and " & $eventEntries.len & + " event entries" + echo result.repr + +{.pop.} + +macro registerBrokerLibrary*(body: untyped): untyped = + ## Generates the full shared-library surface for a broker FFI library. + ## A no-op unless `-d:BrokerFfiApi` is set, so client code never needs + ## a `when defined(...)` guard around it. + when defined(BrokerFfiApi): + registerBrokerLibraryImpl(body) + else: + newStmtList() diff --git a/wasm-deps/brokers/brokers/broker_context.nim b/wasm-deps/brokers/brokers/broker_context.nim new file mode 100644 index 000000000..99b7681be --- /dev/null +++ b/wasm-deps/brokers/brokers/broker_context.nim @@ -0,0 +1,166 @@ +{.push raises: [].} + +import std/[strutils, concurrency/atomics], chronos + +type BrokerContext* = distinct uint32 + +func `==`*(a, b: BrokerContext): bool = + uint32(a) == uint32(b) + +func `!=`*(a, b: BrokerContext): bool = + uint32(a) != uint32(b) + +func `$`*(bc: BrokerContext): string = + toHex(uint32(bc), 8) + +# --------------------------------------------------------------------------- +# Context split — a BrokerContext packs two uint16 halves: +# bits [15:0] classCtx — which broker-object/interface scope ("global" +# context). 0 = reserved (nil/invalid), 1 = the +# default base scope, 2..0xFFFE = allocated, +# 0xFFFF = reserved guard. +# bits [31:16] instanceCtx — which instance of that scope. 0 = flat / +# class-level (no specific instance), 1..0xFFFF = +# OOP-owned instances. +# Bucket lookup remains keyed by the full uint32; the split is semantic. +# --------------------------------------------------------------------------- + +func classCtx*(bc: BrokerContext): uint16 = + uint16(uint32(bc) and 0xFFFF'u32) + +func instanceCtx*(bc: BrokerContext): uint16 = + uint16((uint32(bc) shr 16) and 0xFFFF'u32) + +func makeBrokerContext*(classCtx, instanceCtx: uint16): BrokerContext = + BrokerContext((uint32(instanceCtx) shl 16) or uint32(classCtx)) + +const DefaultBrokerContext* = makeBrokerContext(1'u16, 0'u16) + ## 0x0000_0001 — + ## the base "global" flat scope (classCtx 1, instance 0). Deliberately not + ## 0x0 so an unset/nil context is distinguishable from the default. + +# --------------------------------------------------------------------------- +# Thread-global broker context +# --------------------------------------------------------------------------- +# +# Each thread has its own BrokerContext value (threadvar). +# Defaults to DefaultBrokerContext until explicitly set via +# setThreadBrokerContext or initThreadBrokerContext. +# +# NOTE: Module-level threadvar assignments only execute on the main thread. +# Secondary threads get zero-initialized threadvars, so we use a flag to +# lazily initialize on first access. + +var globalBrokerContextLock {.threadvar.}: AsyncLock +globalBrokerContextLock = newAsyncLock() +var globalBrokerContextValue {.threadvar.}: BrokerContext +globalBrokerContextValue = DefaultBrokerContext +var globalBrokerContextInitialized {.threadvar.}: bool +globalBrokerContextInitialized = true # main thread is initialized + +proc threadGlobalBrokerContext*(): BrokerContext = + ## Returns the currently active broker context for this thread. + ## + ## Defaults to `DefaultBrokerContext` until explicitly set via + ## `setThreadBrokerContext` or `initThreadBrokerContext`. + ## Lock-free threadvar read — safe to call from anywhere. + if not globalBrokerContextInitialized: + globalBrokerContextValue = DefaultBrokerContext + globalBrokerContextInitialized = true + globalBrokerContextValue + +# Backward-compatible alias +template globalBrokerContext*(): BrokerContext = + threadGlobalBrokerContext() + +var gClassCtxCounter: Atomic[uint32] + +proc newClassCtx*(): uint16 = + ## Allocate a fresh, process-unique classCtx (the low-16 "global" scope id). + ## Shared by flat `NewBrokerContext` and the OOP interface-class registration + ## so every classCtx is unique. Starts at 2 (0 = nil, 1 = default scope). + let id = gClassCtxCounter.fetchAdd(1, moRelaxed) + 2'u32 + doAssert id < 0xFFFF'u32, "BrokerContext classCtx space exhausted (max 65534)" + uint16(id) + +proc NewBrokerContext*(): BrokerContext = + ## A flat "global" context: a fresh classCtx with instanceCtx 0. + makeBrokerContext(newClassCtx(), 0'u16) + +var gInstanceCtxCounter: Atomic[uint32] + +proc newInstanceCtx*(parentCtx: BrokerContext): BrokerContext = + ## Allocate a sub-instance context that SHARES `parentCtx`'s classCtx (so it + ## routes to the same library context — same processing/delivery thread and + ## courier) but carries a fresh, process-unique instanceCtx (high16). + ## + ## Used by create-instance FFI requests (reduced-A): a sub-interface instance + ## lives on the main library's processing thread, so it must share the library + ## classCtx. `_call` masks the instanceCtx off to find the courier, then + ## dispatches against the full sub ctx so the provider keyed by it is hit. + ## The counter is process-monotonic, so two sub-instances under the same + ## library never collide on instanceCtx. + let id = gInstanceCtxCounter.fetchAdd(1, moRelaxed) + 1'u32 + doAssert id < 0x1_0000'u32, "BrokerContext instanceCtx space exhausted (max 65535)" + makeBrokerContext(classCtx(parentCtx), uint16(id)) + +# --------------------------------------------------------------------------- +# Sync thread-context binding (usable from {.thread.} init, before event loop) +# --------------------------------------------------------------------------- + +proc setThreadBrokerContext*(ctx: BrokerContext) = + ## Installs an existing BrokerContext as this thread's global broker context. + ## + ## Use when the context was created elsewhere (e.g. on the main thread) + ## and this thread should adopt it. Readable via `threadGlobalBrokerContext()`. + ## + ## This is sync and thread-safe (writes only to this thread's threadvar). + globalBrokerContextValue = ctx + globalBrokerContextInitialized = true + +proc initThreadBrokerContext*(): BrokerContext = + ## Generates a new BrokerContext and installs it as this thread's + ## global broker context. Returns the new context so it can be + ## propagated to other threads for cross-thread broker access. + ## + ## Convenience for: `let ctx = NewBrokerContext(); setThreadBrokerContext(ctx)` + let ctx = NewBrokerContext() + setThreadBrokerContext(ctx) + return ctx + +# --------------------------------------------------------------------------- +# Async scoped context (backward compat) +# --------------------------------------------------------------------------- + +template lockGlobalBrokerContext*(brokerCtx: BrokerContext, body: untyped): untyped = + ## Runs `body` while holding the global broker context lock with the provided + ## `brokerCtx` installed as the globally accessible context. + ## + ## This template is intended for use from within `chronos` async procs. + block: + # Lazy init: threadvar is nil on secondary threads (module-level init + # only runs on the main thread). + if globalBrokerContextLock.isNil(): + globalBrokerContextLock = newAsyncLock() + await noCancel(globalBrokerContextLock.acquire()) + let previousBrokerCtx = globalBrokerContextValue + globalBrokerContextValue = brokerCtx + globalBrokerContextInitialized = true + try: + body + finally: + globalBrokerContextValue = previousBrokerCtx + try: + globalBrokerContextLock.release() + except AsyncLockError: + doAssert false, "globalBrokerContextLock.release(): lock not held" + +template lockNewGlobalBrokerContext*(body: untyped): untyped = + ## Runs `body` while holding the global broker context lock with a freshly + ## generated broker context installed as the global accessor. + ## + ## The previous global broker context (if any) is restored on exit. + lockGlobalBrokerContext(NewBrokerContext()): + body + +{.pop.} diff --git a/wasm-deps/brokers/brokers/broker_implement.nim b/wasm-deps/brokers/brokers/broker_implement.nim new file mode 100644 index 000000000..592021d9c --- /dev/null +++ b/wasm-deps/brokers/brokers/broker_implement.nim @@ -0,0 +1,268 @@ +## BrokerImplement — derived implementation of a BrokerInterface +## (doc/HIERARCHICAL_BROKERS_PLAN.md, phase P4). +## +## type MyServiceImpl = ref object of IMyService +## db: Database +## +## BrokerImplement MyServiceImpl of IMyService: +## proc init(db: Database) = ## optional; `self` is the new instance +## self.db = db +## method getHealth(self: MyServiceImpl): Future[Result[GetHealth, string]] = +## ok(GetHealth(...)) ## raw method overrides of the abstract base +## +## Generates: `MyServiceImpl.new(db = ...)` (allocates an instance brokerCtx and +## runs `init`), per-instance provider closures that dispatch each request to +## the overriding method (capturing `self`), and `close(self)` which clears +## those providers — breaking the instance<->closure cycle (mandatory under +## --mm:refc) and freeing the instance ctx for reuse. + +import std/[macros, strutils, atomics] +import chronos, results +import ./broker_context +import ./request_broker, ./event_broker +import ./internal/helper/broker_utils + +export chronos, results, broker_context, request_broker, event_broker + +proc canonPragma(async: bool): NimNode {.compileTime.} = + ## Canonical override pragma matching the BrokerInterface abstract base + ## (byte-identical async/raises/gcsafe is required for method dispatch). + let src = + if async: + "proc d() {.async: (raises: []), gcsafe.} = discard" + else: + "proc d() {.gcsafe, raises: [].} = discard" + parseStmt(src)[0][4] + +proc isAsyncRet(ret: NimNode): bool {.compileTime.} = + ret.kind == nnkBracketExpr and ret.len >= 1 and ret[0].kind == nnkIdent and + ret[0].eqIdent("Future") + +proc baseName(n: NimNode): NimNode {.compileTime.} = + if n.kind == nnkPostfix: + n[1] + else: + n + +macro BrokerImplement*(args: varargs[untyped]): untyped = + ## See module docs. Invoked as `BrokerImplement Impl of IFace: `. + if args.len < 2: + macros.error("BrokerImplement requires `Impl of IFace:` and a body") + let body = args[^1] + if body.kind != nnkStmtList: + macros.error("BrokerImplement body must be a `:` block") + let infix = args[0] + if infix.kind != nnkInfix or not infix[0].eqIdent("of"): + macros.error( + "BrokerImplement must be written `BrokerImplement Impl of IFace:`", infix + ) + let implName = infix[1] + let implStr = $implName + let ifaceStr = $infix[2] + + result = newStmtList() + + var initParams: seq[NimNode] = @[] # extra new() params (after the typedesc) + var initBody = newStmtList() + # (verb, brokerName, argParams, payloadRepr, async) + var methods: seq[(string, string, seq[NimNode], string, bool)] = @[] + + for stmt in body: + case stmt.kind + of nnkProcDef: + if not baseName(stmt[0]).eqIdent("init"): + macros.error( + "BrokerImplement only allows an `init` proc and `method` overrides", stmt + ) + let p = stmt.params + for i in 1 ..< p.len: # skip return type + initParams.add(copyNimTree(p[i])) + initBody = copyNimTree(stmt.body) + of nnkMethodDef: + let verb = $baseName(stmt[0]) + let p = stmt.params + let ret = p[0] + let async = isAsyncRet(ret) + let payload = extractResultOk(ret, async) + if payload.isNil: + macros.error( + "method `" & verb & "` must return " & + (if async: "Future[Result[T, string]]" else: "Result[T, string]"), + stmt, + ) + # Stamp the canonical override pragma and emit the method verbatim. + var m = copyNimTree(stmt) + m[4] = canonPragma(async) + result.add(m) + var margs: seq[NimNode] = @[] + for i in 2 ..< p.len: # skip return (0) and self (1) + margs.add(copyNimTree(p[i])) + methods.add((verb, capitalizeAscii(verb), margs, payload.repr.strip(), async)) + of nnkEmpty, nnkCommentStmt: + discard + else: + macros.error( + "BrokerImplement only allows an `init` proc and `method` overrides", stmt + ) + + # Compile-time fulfillment check: every request verb declared in the + # interface must have a corresponding method override in the implementation. + let ifaceVerbs = interfaceRequestVerbs(ifaceStr) + for (verb, typeName) in ifaceVerbs: + var found = false + for m in methods: + if m[0] == verb: + found = true + break + if not found: + macros.error( + "BrokerImplement " & implStr & ": missing method override for '" & verb & + "' (request type " & typeName & ") declared in " & ifaceStr + ) + + # Per-class context allocation state. + let classCtxVar = ident(implStr & "BrokerClassCtx") + let instCounter = ident(implStr & "BrokerInstCounter") + let setupName = ident(implStr & "SetupProviders") + result.add( + quote do: + # classCtx allocated once at module init (immutable -> race-free and + # gcsafe to read); per-instance instanceCtx from an atomic counter. + let `classCtxVar` = newClassCtx() + var `instCounter` {.global.}: Atomic[uint16] + ) + + # setupProviders — register a per-instance provider closure per request that + # dispatches to the overriding method (capturing `self`). + var setupSrc = "proc " & $setupName & "(self: " & implStr & ") {.gcsafe.} =\n" + if methods.len == 0: + setupSrc.add(" discard\n") + for (verb, brokerName, margs, payload, async) in methods: + var paramDecls = "" + var argNames = "" + for a in margs: + paramDecls.add((if paramDecls.len > 0: ", " else: "") & a.repr.strip()) + for j in 0 ..< a.len - 2: + argNames.add((if argNames.len > 0: ", " else: "") & $baseName(a[j])) + let ret = + if async: + "Future[Result[" & payload & ", string]]" + else: + "Result[" & payload & ", string]" + # Pragma must match the broker's generated provider proc type + # (request_broker `makeProcType`): plain `{.async.}` for async, + # `{.gcsafe, raises: [CatchableError].}` for sync. + let prag = if async: "{.async.}" else: "{.gcsafe, raises: [CatchableError].}" + let call = (if async: "await " else: "") & "self." & verb & "(" & argNames & ")" + setupSrc.add( + " discard " & brokerName & ".setProvider(self.brokerCtx, proc(" & paramDecls & + "): " & ret & " " & prag & " =\n " & call & ")\n" + ) + result.add(parseStmt(setupSrc)) + + # new() — allocate the instance, its brokerCtx, run init, wire providers. + var newFormal = nnkFormalParams.newTree(copyNimTree(implName)) + newFormal.add( + newIdentDefs( + ident("T"), nnkBracketExpr.newTree(ident("typedesc"), copyNimTree(implName)) + ) + ) + for p in initParams: + newFormal.add(copyNimTree(p)) + # Build new()'s body as ONE flat scope so `self` is visible to the spliced + # init body. `self` is interpolated as an explicit ident (quote would gensym + # a literal `let self`, breaking the user's `self.field` references). + let selfId = ident("self") + var newBody = newStmtList() + let pre = quote: + let `selfId` = `implName`() + `selfId`.brokerCtx = + makeBrokerContext(`classCtxVar`, `instCounter`.fetchAdd(1'u16, moRelaxed) + 1'u16) + for s in pre: + newBody.add(s) + for s in initBody: + newBody.add(copyNimTree(s)) + let post = quote: + `setupName`(`selfId`) + `selfId` + for s in post: + newBody.add(copyNimTree(s)) + # A0: new() is gcsafe — the create-instance FFI path constructs sub-instances + # in a gcsafe request method body (classCtx is an immutable `let`, instanceCtx + # an atomic, setupProviders is gcsafe). Requires the impl's `init` body to be + # gcsafe (trivial field writes always are). If a real in-process user needs a + # non-gcsafe init, add a separate non-gcsafe constructor rather than relaxing + # this. + result.add( + nnkProcDef.newTree( + postfix(ident("new"), "*"), + newEmptyNode(), + newEmptyNode(), + newFormal, + nnkPragma.newTree(ident("gcsafe")), + newEmptyNode(), + newBody, + ) + ) + + # bindToContext() — construct an instance that ADOPTS an externally-supplied + # brokerCtx (the FFI library context allocated by `_createContext`) + # instead of allocating its own. Lets a BrokerInterface(API) impl serve as the + # provider set for registerBrokerLibrary's `setupProviders(ctx)` (runs on the + # processing thread → gcsafe). Wires providers keyed by `ctx`. + var bindFormal = nnkFormalParams.newTree(copyNimTree(implName)) + bindFormal.add( + newIdentDefs( + ident("T"), nnkBracketExpr.newTree(ident("typedesc"), copyNimTree(implName)) + ) + ) + bindFormal.add(newIdentDefs(ident("ctx"), ident("BrokerContext"))) + for p in initParams: + bindFormal.add(copyNimTree(p)) + var bindBody = newStmtList() + let bindPre = quote: + let `selfId` = `implName`() + `selfId`.brokerCtx = ctx + for s in bindPre: + bindBody.add(s) + for s in initBody: + bindBody.add(copyNimTree(s)) + for s in post: + bindBody.add(copyNimTree(s)) + result.add( + nnkProcDef.newTree( + postfix(ident("bindToContext"), "*"), + newEmptyNode(), + newEmptyNode(), + bindFormal, + nnkPragma.newTree(ident("gcsafe")), + newEmptyNode(), + bindBody, + ) + ) + + # close() — clear this instance's providers (breaks the refc cycle) and free + # its ctx. Idempotent. + var closeSrc = "proc close*(self: " & implStr & ") =\n" + closeSrc.add(" if self.brokerCtx == DefaultBrokerContext: return\n") + for (verb, brokerName, margs, payload, async) in methods: + closeSrc.add(" " & brokerName & ".clearProvider(self.brokerCtx)\n") + # B2: also drop this instance's event listeners. The interface published its + # event types via the compile-time registry; guard with `when compiles` so it + # works whether the event broker is single-thread / mt / API. + for ev in interfaceEvents(ifaceStr): + # dropAllListeners clears the listener table synchronously (before its first + # await), so discarding the Future from sync close() still removes listeners; + # only the in-flight-cancel await is abandoned (matches teardown semantics). + closeSrc.add(" when compiles(" & ev & ".dropAllListeners(self.brokerCtx)):\n") + closeSrc.add( + " when typeof(" & ev & ".dropAllListeners(self.brokerCtx)) is void:\n" + ) + closeSrc.add(" " & ev & ".dropAllListeners(self.brokerCtx)\n") + closeSrc.add(" else:\n") + closeSrc.add(" discard " & ev & ".dropAllListeners(self.brokerCtx)\n") + closeSrc.add(" self.brokerCtx = DefaultBrokerContext\n") + result.add(parseStmt(closeSrc)) + + when defined(brokerDebug): + echo result.repr diff --git a/wasm-deps/brokers/brokers/broker_interface.nim b/wasm-deps/brokers/brokers/broker_interface.nim new file mode 100644 index 000000000..cdcf1f6eb --- /dev/null +++ b/wasm-deps/brokers/brokers/broker_interface.nim @@ -0,0 +1,243 @@ +## BrokerInterface — an abstract, OOP-style facade over a group of Event / +## Request brokers (see doc/HIERARCHICAL_BROKERS_PLAN.md, phase P3). +## +## A `BrokerInterface` block declares the *contract*: the events it can emit +## and the requests it answers. It generates: +## * a `ref object of RootObj` interface type carrying a hidden `brokerCtx`; +## * the underlying Event/Request brokers (re-emitted verbatim, or lowered to +## their `(API)` variants when the interface is declared `(API)`); +## * one abstract `{.base.}` `method` per request (pure-virtual — raises +## until a `BrokerImplement` derived type overrides it); +## * a generic instance-scoped event facade (`self.emit` / `self.listen` / +## `self.dropListener`) that injects `self.brokerCtx`. +## +## Invocation forms (note: `BrokerInterface(API) IFace:` does NOT parse in Nim — +## the `(API)` binds as a call; use the comma form instead): +## BrokerInterface IFace: ## or BrokerInterface(IFace): +## EventBroker: ... +## RequestBroker: ... +## BrokerInterface(API, IFace): ## (API) propagates to every sub-broker +## EventBroker: ... +## RequestBroker: ... +## +## Requests inside an interface use the proc-sugar form (a lowercase verb proc); +## the verb becomes the abstract method name a `BrokerImplement` overrides. + +import std/[macros, strutils] +import chronos, results +import ./broker_context +import ./request_broker, ./event_broker +import ./internal/helper/broker_utils + +export chronos, results, broker_context, request_broker, event_broker + +proc isApiArg(n: NimNode): bool = + n.kind == nnkIdent and n.eqIdent("API") + +proc brokerHeadName(stmt: NimNode): string = + ## The macro name a sub-block invokes (EventBroker / RequestBroker), or "". + if stmt.kind notin {nnkCall, nnkCommand}: + return "" + let head = stmt[0] + if head.kind == nnkIdent: + return $head + "" + +proc renderAbstractMethod( + ifaceName, verb, payloadRepr: string, argParams: seq[NimNode], async: bool +): string = + ## Render an abstract base method as Nim source (parsed back via parseStmt — + ## sidesteps fiddly pragma-AST construction for `async: (raises: [])`). + var params = "self: " & ifaceName + for p in argParams: + params.add(", " & p.repr.strip()) + let ret = + if async: + "Future[Result[" & payloadRepr & ", string]]" + else: + "Result[" & payloadRepr & ", string]" + let pragma = + if async: + "{.base, async: (raises: []), gcsafe.}" + else: + "{.base, gcsafe, raises: [].}" + result = + "method " & verb & "*(" & params & "): " & ret & " " & pragma & " =\n" & + " raiseAssert(\"" & ifaceName & "." & verb & " has no implementation\")\n" + +macro BrokerInterface*(args: varargs[untyped]): untyped = + ## See module docs. `args` is `[?, , ]` in any order for + ## the leading idents, with the `:` block as the final argument. + if args.len < 2: + macros.error("BrokerInterface requires an interface name and a `:` body block") + let body = args[^1] + if body.kind != nnkStmtList: + macros.error("BrokerInterface body must be a `:` block") + + var ifaceName: NimNode = nil + var isApi = false + for i in 0 ..< args.len - 1: + if isApiArg(args[i]): + isApi = true + elif args[i].kind == nnkIdent: + if ifaceName != nil: + macros.error( + "BrokerInterface: unexpected extra name `" & $args[i] & "`", args[i] + ) + ifaceName = args[i] + else: + macros.error("BrokerInterface: unexpected argument", args[i]) + if ifaceName.isNil: + macros.error("BrokerInterface requires an interface name", body) + + let ifaceNameStr = $ifaceName + result = newStmtList() + + # 1. Interface ref type with the hidden context. + result.add( + quote do: + type `ifaceName`* = ref object of RootObj + brokerCtx*: BrokerContext + + ) + + # 2. Walk the sub-blocks: re-emit each broker (lowered to `(API)` when the + # interface is `(API)`), and generate abstract methods for requests. + var eventNames: seq[string] = @[] + var requestTypes: seq[string] = @[] # sanitized request broker type names (A1) + var requestVerbs: seq[(string, string)] = @[] # (verb, sanitized type name) + for stmt in body: + let headName = brokerHeadName(stmt) + if headName notin ["EventBroker", "RequestBroker"]: + macros.error( + "BrokerInterface body may only contain `EventBroker:` / `RequestBroker:` blocks", + stmt, + ) + let innerBody = stmt[^1] + if innerBody.kind != nnkStmtList: + macros.error( + headName & " inside BrokerInterface must have a `:` body block", stmt + ) + let hasMode = stmt.len == 3 # nnkCall(Head, mode, body) + + # Re-emit the underlying broker. + if isApi: + if hasMode: + macros.error( + "BrokerInterface(API): sub-brokers must be plain `" & headName & + ":` (the API mode is applied automatically)", + stmt, + ) + result.add(newCall(ident(headName), ident("API"), copyNimTree(innerBody))) + else: + result.add(copyNimTree(stmt)) + + # Requests → abstract methods. + if headName == "RequestBroker": + let async = isApi or not (hasMode and stmt[1].eqIdent("sync")) + let sg = parseRequestSugar(innerBody, "BrokerInterface RequestBroker", async) + let payloadRepr = sg.payloadType.repr.strip() + # Record the request broker type name (matches CborRequestEntry. + # responseTypeName) so codegen can attribute the flat entry to this iface. + requestTypes.add(sanitizeIdentName(sg.typeIdent)) + requestVerbs.add((sg.verb, sanitizeIdentName(sg.typeIdent))) + if not sg.zeroArgProc.isNil: + result.add( + parseStmt( + renderAbstractMethod(ifaceNameStr, sg.verb, payloadRepr, @[], async) + ) + ) + if not sg.argProc.isNil: + result.add( + parseStmt( + renderAbstractMethod( + ifaceNameStr, sg.verb, payloadRepr, sg.argParams, async + ) + ) + ) + elif headName == "EventBroker": + # Record the event type so BrokerImplement.close() can drop listeners. + let evParsed = parseSingleTypeDef(innerBody, "BrokerInterface EventBroker") + eventNames.add($evParsed.typeIdent) + + # Publish this interface's event types for BrokerImplement teardown (B2). + registerInterfaceEvents(ifaceNameStr, eventNames) + + # Publish this interface's request verbs for BrokerImplement fulfillment check. + registerInterfaceVerbs(ifaceNameStr, requestVerbs) + + # A1: publish (API) interfaces to the compile-time registry so + # registerBrokerLibrary can designate a main class and partition the per- + # interface wrapper surface. Plain (non-API) interfaces are not FFI-exposed. + if isApi: + registerApiInterface(ifaceNameStr, requestTypes, eventNames) + + # 3. Generic instance-scoped event facade — forwards any event typedesc to + # the underlying ctx-based broker API using `self.brokerCtx`. + result.add( + quote do: + template emit*(self: `ifaceName`, t: typedesc, args: varargs[untyped]): untyped = + t.emit(self.brokerCtx, args) + + template listen*(self: `ifaceName`, t: typedesc, handler: untyped): untyped = + t.listen(self.brokerCtx, handler) + + template dropListener*(self: `ifaceName`, t: typedesc, handle: untyped): untyped = + t.dropListener(self.brokerCtx, handle) + + ) + + # 4. Factory / dependency-injection. A consumer depends only on the interface + # module; an implementer installs a constructor via `provideFactory` + # (last wins) and the consumer obtains an instance via `create`. The + # factory may close over outer config, or take a typed config at call + # time. In-process the factory returns the real impl (direct virtual + # dispatch); the cross-runtime proxy variant is wired in P6. + # NOTE (P6): factory storage is a process-global here; cross-thread FFI use + # will harden it (lock + shared) when registerBrokerLibrary lands. + let ifaceNameLit = newLit(ifaceNameStr) + let facVar = ident(ifaceNameStr & "BrokerFactory") + let facCfgVar = ident(ifaceNameStr & "BrokerFactoryCfg") + result.add( + quote do: + var `facVar` {.global.}: + proc(cfg: pointer): Result[`ifaceName`, string] {.raises: [].} + var `facCfgVar` {.global.}: string + + proc provideFactory*( + _: typedesc[`ifaceName`], f: proc(): Result[`ifaceName`, string] + ) = + `facCfgVar` = "" + `facVar` = proc(cfg: pointer): Result[`ifaceName`, string] {.raises: [].} = + try: + f() + except Exception as e: + err(`ifaceNameLit` & " factory raised: " & e.msg) + + proc provideFactory*[A]( + _: typedesc[`ifaceName`], f: proc(cfg: A): Result[`ifaceName`, string] + ) = + `facCfgVar` = $A + `facVar` = proc(cfg: pointer): Result[`ifaceName`, string] {.raises: [].} = + try: + f(cast[ptr A](cfg)[]) + except Exception as e: + err(`ifaceNameLit` & " factory raised: " & e.msg) + + proc create*(_: typedesc[`ifaceName`]): Result[`ifaceName`, string] = + if `facVar`.isNil: + return err("no factory provided for " & `ifaceNameLit`) + `facVar`(nil) + + proc create*[A](_: typedesc[`ifaceName`], cfg: A): Result[`ifaceName`, string] = + if `facVar`.isNil: + return err("no factory provided for " & `ifaceNameLit`) + if `facCfgVar` != $A: + return err( + `ifaceNameLit` & " factory config type mismatch (got " & $A & ", expected " & + `facCfgVar` & ")" + ) + var c = cfg + `facVar`(addr c) + + ) diff --git a/wasm-deps/brokers/brokers/event_broker.nim b/wasm-deps/brokers/brokers/event_broker.nim new file mode 100644 index 000000000..423ef81da --- /dev/null +++ b/wasm-deps/brokers/brokers/event_broker.nim @@ -0,0 +1,628 @@ +## EventBroker +## ------------------- +## EventBroker represents a reactive decoupling pattern, that +## allows event-driven development without +## need for direct dependencies in between emitters and listeners. +## Worth considering using it in a single or many emitters to many listeners scenario. +## +## Generates a standalone, type-safe event broker for the declared type. +## The macro exports the value type itself plus a broker companion that manages +## listeners via thread-local storage. +## +## Type definitions: +## - Inline `object` / `ref object` definitions are supported. +## - Native types, aliases, and externally-defined types are also supported. +## In that case, EventBroker will automatically wrap the declared RHS type in +## `distinct` unless you already used `distinct`. +## This keeps event types unique even when multiple brokers share the same +## underlying base type. +## +## Default vs. context aware use: +## Every generated broker is a thread-local global instance. This means EventBroker +## enables decoupled event exchange threadwise. +## +## Sometimes we use brokers inside a context (e.g. within a component that has many +## modules or subsystems). If you instantiate multiple such components in a single +## thread, and each component must have its own listener set for the same EventBroker +## type, you can use context-aware EventBroker. +## +## Context awareness is supported through the `BrokerContext` argument for +## `listen`, `emit`, `dropListener`, and `dropAllListeners`. +## Listener stores are kept separate per broker context. +## +## Default broker context is defined as `DefaultBrokerContext`. If you don't need +## context awareness, you can keep using the interfaces without the context +## argument, which operate on `DefaultBrokerContext`. +## +## Usage: +## Declare your desired event type inside an `EventBroker` macro, add any number of fields.: +## ```nim +## EventBroker: +## type TypeName = object +## field1*: FieldType +## field2*: AnotherFieldType +## ``` +## +## After this, you can register async listeners anywhere in your code with +## `TypeName.listen(...)`, which returns a handle to the registered listener. +## Listeners are async procs or lambdas that take a single argument of the event type. +## Any number of listeners can be registered in different modules. +## +## Events can be emitted from anywhere with no direct dependency on the listeners by +## calling `TypeName.emit(...)` with an instance of the event type. +## This will asynchronously notify all registered listeners with the emitted event. +## +## Whenever you no longer need a listener (or your object instance that listen to the event goes out of scope), +## you can remove it from the broker with the handle returned by `listen`. +## This is done by calling `TypeName.dropListener(handle)`. +## Alternatively, you can remove all registered listeners through `TypeName.dropAllListeners()`. +## +## +## Example: +## ```nim +## EventBroker: +## type GreetingEvent = object +## text*: string +## +## let handle = GreetingEvent.listen( +## proc(evt: GreetingEvent): Future[void] {.async.} = +## echo evt.text +## ) +## GreetingEvent.emit(text= "hi") +## GreetingEvent.dropListener(handle) +## ``` + +## Example (non-object event type): +## ```nim +## EventBroker: +## type CounterEvent = int # exported as: `distinct int` +## +## discard CounterEvent.listen( +## proc(evt: CounterEvent): Future[void] {.async.} = +## echo int(evt) +## ) +## CounterEvent.emit(CounterEvent(42)) +## ``` + +import std/[macros, strutils, tables] +import chronos, chronicles, results +import ./internal/helper/broker_utils, ./broker_context +import ./internal/broker_debug + +when compileOption("threads"): + import ./internal/mt_config, ./internal/mt_event_broker + export mt_config, mt_event_broker + +when compileOption("threads") and defined(BrokerFfiApi): + # Part A — native C-ABI codegen retired. See note in request_broker.nim. + import ./internal/api_event_broker_cbor + export api_event_broker_cbor + +export chronicles, results, chronos, broker_context + +type EventBrokerMode = enum + ebDefault + ebMultiThread + ebApi + +proc parseEventBrokerMode(modeNode: NimNode): EventBrokerMode = + let raw = ($modeNode).strip().toLowerAscii() + case raw + of "mt": + ebMultiThread + of "api": + ebApi + else: + error("Unknown EventBroker mode: " & $modeNode & ". Expected: mt or API", modeNode) + +proc generateEventBroker(body: NimNode): NimNode = + when defined(brokerDebug): + echo body.treeRepr + let parsed = parseSingleTypeDef(body, "EventBroker", collectFieldInfo = true) + let typeIdent = parsed.typeIdent + let objectDef = parsed.objectDef + let fieldNames = parsed.fieldNames + let fieldTypes = parsed.fieldTypes + let hasInlineFields = parsed.hasInlineFields + let isVoid = parsed.isVoid + ## Payload-less event (`type X = void`): the listener proc, the dispatch + ## task and `emit` all drop the event-value parameter. The parser lowers + ## `void` to a unique empty `object` so the broker still has a distinct + ## identity to name; `isVoid` just strips the now-meaningless value arg. + + let exportedTypeIdent = postfix(copyNimTree(typeIdent), "*") + let sanitized = sanitizeIdentName(typeIdent) + let typeNameLit = newLit($typeIdent) + let handlerProcIdent = ident(sanitized & "ListenerProc") + let listenerHandleIdent = ident(sanitized & "Listener") + let brokerTypeIdent = ident(sanitized & "Broker") + let exportedHandlerProcIdent = postfix(copyNimTree(handlerProcIdent), "*") + let exportedListenerHandleIdent = postfix(copyNimTree(listenerHandleIdent), "*") + let exportedBrokerTypeIdent = postfix(copyNimTree(brokerTypeIdent), "*") + let bucketTypeIdent = ident(sanitized & "CtxBucket") + let findBucketIdxIdent = ident(sanitized & "FindBucketIdx") + let getOrCreateBucketIdxIdent = ident(sanitized & "GetOrCreateBucketIdx") + let accessProcIdent = ident("access" & sanitized & "Broker") + let globalVarIdent = ident("g" & sanitized & "Broker") + let listenImplIdent = ident("register" & sanitized & "Listener") + let dropListenerImplIdent = ident("drop" & sanitized & "Listener") + let dropAllListenersImplIdent = ident("dropAll" & sanitized & "Listeners") + let emitImplIdent = ident("emit" & sanitized & "Value") + let listenerTaskIdent = ident("notify" & sanitized & "Listener") + let cancelInFlightIdent = ident("cancelInFlight" & sanitized) + let pruneInFlightIdent = ident("pruneInFlight" & sanitized) + + result = newStmtList() + + let handlerProcTy = + if isVoid: + quote: + proc(): Future[void] {.async: (raises: []), gcsafe.} + else: + quote: + proc(event: `typeIdent`): Future[void] {.async: (raises: []), gcsafe.} + + result.add( + quote do: + type + `exportedTypeIdent` = `objectDef` + `exportedListenerHandleIdent` = object + id*: uint64 + + `exportedHandlerProcIdent` = `handlerProcTy` + `bucketTypeIdent` = object + brokerCtx: BrokerContext + listeners: Table[uint64, `handlerProcIdent`] + nextId: uint64 + inFlight: seq[Future[void]] + + `exportedBrokerTypeIdent` = ref object + buckets: seq[`bucketTypeIdent`] + + ) + + result.add( + quote do: + var `globalVarIdent` {.threadvar.}: `brokerTypeIdent` + ) + + result.add( + quote do: + proc `accessProcIdent`(): `brokerTypeIdent` = + if `globalVarIdent`.isNil(): + new(`globalVarIdent`) + `globalVarIdent`.buckets = @[ + `bucketTypeIdent`( + brokerCtx: DefaultBrokerContext, + listeners: initTable[uint64, `handlerProcIdent`](), + nextId: 1'u64, + inFlight: @[], + ) + ] + `globalVarIdent` + + ) + + result.add( + quote do: + proc `findBucketIdxIdent`( + broker: `brokerTypeIdent`, brokerCtx: BrokerContext + ): int = + if brokerCtx == DefaultBrokerContext: + return 0 + for i in 1 ..< broker.buckets.len: + if broker.buckets[i].brokerCtx == brokerCtx: + return i + return -1 + + proc `getOrCreateBucketIdxIdent`( + broker: `brokerTypeIdent`, brokerCtx: BrokerContext + ): int = + let idx = `findBucketIdxIdent`(broker, brokerCtx) + if idx >= 0: + return idx + broker.buckets.add( + `bucketTypeIdent`( + brokerCtx: brokerCtx, + listeners: initTable[uint64, `handlerProcIdent`](), + nextId: 1'u64, + inFlight: @[], + ) + ) + return broker.buckets.high + + proc `listenImplIdent`( + brokerCtx: BrokerContext, handler: `handlerProcIdent` + ): Result[`listenerHandleIdent`, string] = + if handler.isNil(): + return err("Must provide a non-nil event handler") + var broker = `accessProcIdent`() + + let bucketIdx = `getOrCreateBucketIdxIdent`(broker, brokerCtx) + if broker.buckets[bucketIdx].nextId == 0'u64: + broker.buckets[bucketIdx].nextId = 1'u64 + + if broker.buckets[bucketIdx].nextId == high(uint64): + error "Cannot add more listeners: ID space exhausted", + nextId = $broker.buckets[bucketIdx].nextId + return err("Cannot add more listeners, listener ID space exhausted") + + let newId = broker.buckets[bucketIdx].nextId + inc broker.buckets[bucketIdx].nextId + broker.buckets[bucketIdx].listeners[newId] = handler + return ok(`listenerHandleIdent`(id: newId)) + + ) + + result.add( + quote do: + proc `cancelInFlightIdent`( + broker: `brokerTypeIdent`, bucketIdx: int + ) {.async: (raises: []).} = + ## Cancel all in-flight listener futures for the given bucket, + ## then clear the in-flight seq. Uses timeout to handle the + ## self-removal edge case (listener dropping itself inside its handler). + var pending: seq[Future[void]] = @[] + for fut in broker.buckets[bucketIdx].inFlight: + if not fut.finished(): + pending.add(fut.cancelAndWait()) + for fut in pending: + try: + discard await withTimeout(fut, chronos.seconds(5)) + except CancelledError: + # Expected when actively cancelling in-flight listener futures. + discard + except CatchableError as exc: + # Log unexpected errors during cancellation while still completing teardown. + error "Failed to cancel in-flight listener future", + bucketIdx = bucketIdx, errorMsg = exc.msg + broker.buckets[bucketIdx].inFlight.setLen(0) + + proc `pruneInFlightIdent`(broker: `brokerTypeIdent`, bucketIdx: int) = + ## Sync opportunistic cleanup of completed futures. + ## Called on each emit to prevent unbounded seq growth. + var j = 0 + while j < broker.buckets[bucketIdx].inFlight.len: + if broker.buckets[bucketIdx].inFlight[j].finished(): + let last = broker.buckets[bucketIdx].inFlight.len - 1 + broker.buckets[bucketIdx].inFlight[j] = + broker.buckets[bucketIdx].inFlight[last] + broker.buckets[bucketIdx].inFlight.setLen(last) # swap-delete, O(1) + else: + inc j + + ) + + result.add( + quote do: + proc `dropListenerImplIdent`( + brokerCtx: BrokerContext, handle: `listenerHandleIdent` + ) {.async: (raises: []).} = + if handle.id == 0'u64: + return + var broker = `accessProcIdent`() + + let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx) + if bucketIdx < 0: + return + + if broker.buckets[bucketIdx].listeners.len == 0: + return + + # Remove from table — prevents future dispatches + broker.buckets[bucketIdx].listeners.del(handle.id) + + # Cancel and wait for all in-flight futures (timeout-guarded) + await `cancelInFlightIdent`(broker, bucketIdx) + + if brokerCtx != DefaultBrokerContext and + broker.buckets[bucketIdx].listeners.len == 0: + broker.buckets.delete(bucketIdx) + + ) + + result.add( + quote do: + proc `dropAllListenersImplIdent`( + brokerCtx: BrokerContext + ) {.async: (raises: []).} = + var broker = `accessProcIdent`() + + let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx) + if bucketIdx < 0: + return + + # Clear listeners — prevents new dispatches + if broker.buckets[bucketIdx].listeners.len > 0: + broker.buckets[bucketIdx].listeners.clear() + + # Cancel and wait for all in-flight futures + await `cancelInFlightIdent`(broker, bucketIdx) + + if brokerCtx != DefaultBrokerContext: + broker.buckets.delete(bucketIdx) + + ) + + result.add( + quote do: + proc listen*( + _: typedesc[`typeIdent`], handler: `handlerProcIdent` + ): Result[`listenerHandleIdent`, string] = + return `listenImplIdent`(DefaultBrokerContext, handler) + + proc listen*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + handler: `handlerProcIdent`, + ): Result[`listenerHandleIdent`, string] = + return `listenImplIdent`(brokerCtx, handler) + + ) + + result.add( + quote do: + proc dropListener*( + _: typedesc[`typeIdent`], handle: `listenerHandleIdent` + ): Future[void] {.async: (raises: []).} = + await `dropListenerImplIdent`(DefaultBrokerContext, handle) + + proc dropListener*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + handle: `listenerHandleIdent`, + ): Future[void] {.async: (raises: []).} = + await `dropListenerImplIdent`(brokerCtx, handle) + + proc dropAllListeners*( + _: typedesc[`typeIdent`] + ): Future[void] {.async: (raises: []).} = + await `dropAllListenersImplIdent`(DefaultBrokerContext) + + proc dropAllListeners*( + _: typedesc[`typeIdent`], brokerCtx: BrokerContext + ): Future[void] {.async: (raises: []).} = + await `dropAllListenersImplIdent`(brokerCtx) + + ) + + if isVoid: + # Payload-less event: listener task, emitImpl and `emit` carry no + # event value. `emit` is only the typedesc form (`TypeName.emit()`), + # since a bare value-less `emit()` would be hopelessly ambiguous. + result.add( + quote do: + proc `listenerTaskIdent`( + callback: `handlerProcIdent` + ) {.async: (raises: []), gcsafe.} = + if callback.isNil(): + return + try: + await callback() + except Exception: + error "Failed to execute event listener", error = getCurrentExceptionMsg() + + proc `emitImplIdent`( + brokerCtx: BrokerContext + ): Future[void] {.async: (raises: []), gcsafe.} = + let broker = `accessProcIdent`() + let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx) + if bucketIdx < 0: + # nothing to do as nobody is listening + return + if broker.buckets[bucketIdx].listeners.len == 0: + return + + # Prune completed futures (sync — no yield point) + `pruneInFlightIdent`(broker, bucketIdx) + + var callbacks: seq[`handlerProcIdent`] = @[] + for cb in broker.buckets[bucketIdx].listeners.values: + callbacks.add(cb) + for cb in callbacks: + let fut = `listenerTaskIdent`(cb) + broker.buckets[bucketIdx].inFlight.add(fut) + + proc emit*(_: typedesc[`typeIdent`]) = + asyncSpawn `emitImplIdent`(DefaultBrokerContext) + + proc emit*(_: typedesc[`typeIdent`], brokerCtx: BrokerContext) = + asyncSpawn `emitImplIdent`(brokerCtx) + + ) + else: + result.add( + quote do: + proc `listenerTaskIdent`( + callback: `handlerProcIdent`, event: `typeIdent` + ) {.async: (raises: []), gcsafe.} = + if callback.isNil(): + return + try: + await callback(event) + except Exception: + error "Failed to execute event listener", error = getCurrentExceptionMsg() + + proc `emitImplIdent`( + brokerCtx: BrokerContext, event: `typeIdent` + ): Future[void] {.async: (raises: []), gcsafe.} = + when compiles(event.isNil()): + if event.isNil(): + error "Cannot emit uninitialized event object", eventType = `typeNameLit` + return + let broker = `accessProcIdent`() + let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx) + if bucketIdx < 0: + # nothing to do as nobody is listening + return + if broker.buckets[bucketIdx].listeners.len == 0: + return + + # Prune completed futures (sync — no yield point) + `pruneInFlightIdent`(broker, bucketIdx) + + var callbacks: seq[`handlerProcIdent`] = @[] + for cb in broker.buckets[bucketIdx].listeners.values: + callbacks.add(cb) + for cb in callbacks: + let fut = `listenerTaskIdent`(cb, event) + broker.buckets[bucketIdx].inFlight.add(fut) + + proc emit*(event: `typeIdent`) = + asyncSpawn `emitImplIdent`(DefaultBrokerContext, event) + + proc emit*(_: typedesc[`typeIdent`], event: `typeIdent`) = + asyncSpawn `emitImplIdent`(DefaultBrokerContext, event) + + proc emit*( + _: typedesc[`typeIdent`], brokerCtx: BrokerContext, event: `typeIdent` + ) = + asyncSpawn `emitImplIdent`(brokerCtx, event) + + ) + + if hasInlineFields: + # Typedesc emit constructor overloads for inline object/ref object types. + var emitCtorParams = newTree(nnkFormalParams, newEmptyNode()) + let typedescParamType = + newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)) + emitCtorParams.add( + newTree(nnkIdentDefs, ident("_"), typedescParamType, newEmptyNode()) + ) + for i in 0 ..< fieldNames.len: + emitCtorParams.add( + newTree( + nnkIdentDefs, + copyNimTree(fieldNames[i]), + copyNimTree(fieldTypes[i]), + newEmptyNode(), + ) + ) + + var emitCtorExpr = newTree(nnkObjConstr, copyNimTree(typeIdent)) + for i in 0 ..< fieldNames.len: + emitCtorExpr.add( + newTree( + nnkExprColonExpr, copyNimTree(fieldNames[i]), copyNimTree(fieldNames[i]) + ) + ) + + let emitCtorCallDefault = + newCall(copyNimTree(emitImplIdent), ident("DefaultBrokerContext"), emitCtorExpr) + let emitCtorBodyDefault = quote: + asyncSpawn `emitCtorCallDefault` + + let typedescEmitProcDefault = newTree( + nnkProcDef, + postfix(ident("emit"), "*"), + newEmptyNode(), + newEmptyNode(), + emitCtorParams, + newEmptyNode(), + newEmptyNode(), + emitCtorBodyDefault, + ) + result.add(typedescEmitProcDefault) + + var emitCtorParamsCtx = newTree(nnkFormalParams, newEmptyNode()) + emitCtorParamsCtx.add( + newTree(nnkIdentDefs, ident("_"), typedescParamType, newEmptyNode()) + ) + emitCtorParamsCtx.add( + newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode()) + ) + for i in 0 ..< fieldNames.len: + emitCtorParamsCtx.add( + newTree( + nnkIdentDefs, + copyNimTree(fieldNames[i]), + copyNimTree(fieldTypes[i]), + newEmptyNode(), + ) + ) + + let emitCtorCallCtx = + newCall(copyNimTree(emitImplIdent), ident("brokerCtx"), copyNimTree(emitCtorExpr)) + let emitCtorBodyCtx = quote: + asyncSpawn `emitCtorCallCtx` + + let typedescEmitProcCtx = newTree( + nnkProcDef, + postfix(ident("emit"), "*"), + newEmptyNode(), + newEmptyNode(), + emitCtorParamsCtx, + newEmptyNode(), + newEmptyNode(), + emitCtorBodyCtx, + ) + result.add(typedescEmitProcCtx) + + when defined(brokerDebug): + writeBrokerDebug("EventBroker", sanitized, result) + when defined(brokerDebugStdout): + echo result.repr + +macro EventBroker*(args: varargs[untyped]): untyped = + ## Single-thread default mode, or explicit mode selector with optional kwargs. + ## + ## Examples: + ## EventBroker: + ## type MyEvent = object + ## value*: int + ## + ## EventBroker(mt): + ## type MyEvent = object + ## value*: int + ## + ## EventBroker(mt, queueDepth = 1024, slabCapacity = 4096): + ## type MyEvent = object + ## value*: int + if args.len == 0: + macros.error("EventBroker requires a body block") + if args.len == 1: + return generateEventBroker(args[0]) + let mode = args[0] + let body = args[^1] + if body.kind notin {nnkStmtList, nnkTypeDef, nnkTypeSection}: + error( + "EventBroker(" & mode.repr & ") body must be a `:` block of type definitions (got " & + $body.kind & ")", + body, + ) + var kwargs: seq[NimNode] + for i in 1 ..< args.len - 1: + kwargs.add(args[i]) + let m = parseEventBrokerMode(mode) + let split = (kwargs: kwargs, body: body) + case m + of ebMultiThread: + when not compileOption("threads"): + macros.error("EventBroker(mt) requires --threads:on. " & + "Compile with `--threads:on` to use multi-thread EventBroker.") + else: + let cfg = parseMtEvtKwargs(split.kwargs) + generateMtEventBroker(body, cfg) + of ebApi: + when not compileOption("threads"): + macros.error("EventBroker(API) requires --threads:on. " & + "Compile with `--threads:on` to use API EventBroker.") + else: + when defined(BrokerFfiApi): + # Validate kwargs at the outer macro so errors point at the + # user's call site, then pass them through to the deferred + # codegen which re-parses them into an MtEvtCfg (the API + # broker rides the same MT lane internally, so the same + # capacity knobs apply). + discard parseMtEvtKwargs(split.kwargs) + generateApiCborEventBroker(body, split.kwargs) + else: + let cfg = parseMtEvtKwargs(split.kwargs) + generateMtEventBroker(body, cfg) + of ebDefault: + if split.kwargs.len > 0: + error( + "EventBroker(" & mode.repr & ") does not accept kwargs (kwargs are mt-only)", + split.kwargs[0], + ) + generateEventBroker(body) diff --git a/wasm-deps/brokers/brokers/internal/api_cbor_codec.nim b/wasm-deps/brokers/brokers/internal/api_cbor_codec.nim new file mode 100644 index 000000000..3b0dffbff --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_cbor_codec.nim @@ -0,0 +1,250 @@ +## API CBOR Codec +## --------------- +## CBOR encode/decode primitives for the CBOR FFI strategy. +## +## This module owns the `BrokerCbor` flavor (configured with strict-but- +## forward-compat settings), the `CborResponseEnvelope[T]` wire type that +## represents `Result[T, string]` on the wire, and the encode/decode helpers +## that wrap `nim-cbor-serialization`'s exception-raising API as +## `Result`-returning procs suitable for `raises: []` call sites. +## +## Design choices (see plan §4): +## - Response envelope is a CBOR map with two optional fields: +## `{ "ok": T }` for success, `{ "err": tstr }` for failure. +## The map form lets us extend the schema without breaking older wrappers. +## - Void responses use the `CborUnit` zero-field marker so the generic +## `CborResponseEnvelope[T]` type also covers `Result[void, string]`. +## - Encoding never raises — failures are surfaced as `Result.err`. Caller +## threads (often foreign threads via the FFI gate) cannot meaningfully +## handle a Nim `IOError` so all serialization exceptions are caught and +## stringified at this layer. +## +## All buffers exchanged with the FFI boundary live elsewhere +## (`api_common`'s shared-heap helpers); this module deals only in +## `seq[byte]` / `openArray[byte]`. + +{.push raises: [].} + +import std/[options, typetraits] +import results +import cbor_serialization +import cbor_serialization/[reader_impl, writer] +import cbor_serialization/std/options as cbor_options + +export results, cbor_serialization, cbor_options + +# --------------------------------------------------------------------------- +# Flavor +# --------------------------------------------------------------------------- + +createCborFlavor( + BrokerCbor, + automaticObjectSerialization = true, + automaticPrimitivesSerialization = true, + requireAllFields = true, + # Provider-side decode rejects malformed requests up front rather than + # silently zero-initialising missing fields. + omitOptionalFields = true, # Compactness: only populated Options hit the wire. + allowUnknownFields = true, + # Wrappers built against a newer schema can still talk to an older Nim + # library — unknown fields are dropped on decode rather than failing. + skipNullFields = false, +) + +# Encode enums as numeric ordinals so the wire format matches what +# Python's IntEnum and C++'s underlying enum class produce naturally. +# Without this override the upstream default is `EnumAsString`, which +# decodes fine on the Nim side but diverges from foreign-language +# wrappers that send enum values as ints. +enumRep(Cbor, BrokerCbor, EnumRepresentation.EnumAsNumber) + +# --------------------------------------------------------------------------- +# Distinct-type bridging +# +# nim-cbor-serialization 0.3.0 ships a generic writer for distinct types +# (`proc write*[T: distinct]` in writer.nim) but the matching reader is +# commented out upstream. We provide both halves here: +# - a generic `read[T: distinct]` that decodes into the underlying type +# and casts back, mirroring the writer's behaviour. +# - the flavor-level `defaultReader(distinct)` / `defaultWriter(distinct)` +# bindings so user-defined distinct types work out of the box on the +# `BrokerCbor` flavor without per-type registration boilerplate. +# --------------------------------------------------------------------------- + +proc read*[T: distinct]( + r: var CborReader, value: var T +) {.raises: [SerializationError, IOError].} = + mixin readValue + var underlying: distinctBase(T, recursive = false) + readValue(r, underlying) + value = T(underlying) + +BrokerCbor.defaultReader(distinct) + # Writer side is already bound by `defaultPrimitiveWriter` (see + # cbor_serialization/format.nim:99). Re-binding here causes + # `ambiguous call writeValue` at user call sites. + +# Enum reader override. +# +# With `enumRep = EnumAsNumber` (set above) the writer emits enum values +# as CBOR Unsigned ints, matching what Python's `IntEnum` and C++'s +# `enum class` underlying values produce on the wire. The upstream +# `read[T: enum]` only accepts CBOR strings (its private `parseEnum` +# helper hard-codes `allowNumericRepr = false`), so we provide a +# numeric-aware override at the flavor level: read an int via the +# already-bound `read[T: SomeInteger]`, range-check against the enum's +# low/high ordinals, then cast. +proc readValue*[T: enum]( + r: var (BrokerCbor.Reader), value: var T +) {.raises: [IOError, SerializationError].} = + mixin read + var i: int + read(r, i) + if i < ord(T.low) or i > ord(T.high): + raise + newException(CborReaderError, "CBOR enum value " & $i & " out of range for " & $T) + value = T(i) + +# --------------------------------------------------------------------------- +# Wire types +# --------------------------------------------------------------------------- + +type CborUnit* = object + ## Empty marker used as the payload of `Result[void, string]` envelopes. + ## Encodes as a zero-field CBOR map (`{}`). + +type CborResponseEnvelope*[T] = object + ## Wire representation of `Result[T, string]`. + ## + ## With the BrokerCbor flavor (`omitOptionalFields = true`), exactly one + ## of `ok` and `err` is populated on a well-formed envelope. Decode + ## validates this in `fromEnvelope`. + ok*: Option[T] + err*: Option[string] + +# --------------------------------------------------------------------------- +# Result <-> Envelope +# --------------------------------------------------------------------------- + +proc toEnvelope*[T](r: Result[T, string]): CborResponseEnvelope[T] = + if r.isOk(): + CborResponseEnvelope[T](ok: some(r.value), err: none(string)) + else: + CborResponseEnvelope[T](ok: none(T), err: some(r.error)) + +proc fromEnvelope*[T](e: CborResponseEnvelope[T]): Result[T, string] {.raises: [].} = + if e.ok.isSome() and e.err.isSome(): + return Result[T, string].err( + "malformed CBOR response envelope: both 'ok' and 'err' present" + ) + if e.ok.isSome(): + return Result[T, string].ok(e.ok.get()) + if e.err.isSome(): + return Result[T, string].err(e.err.get()) + Result[T, string].err( + "malformed CBOR response envelope: neither 'ok' nor 'err' present" + ) + +# --------------------------------------------------------------------------- +# Encode / Decode helpers +# --------------------------------------------------------------------------- + +template cborEncode*[T](value: T): Result[seq[byte], string] = + ## Encode `value` to CBOR using the BrokerCbor flavor. Wraps every encode + ## failure as `Result.err`; never raises. + ## + ## Implemented as a template so that `BrokerCbor`'s flavor-bound templates + ## (`init`, `writeValue`, `PreferredOutputType`) resolve at the user's + ## call site rather than inside a generic proc — the latter loses access + ## to the flavor's auto-generated object writers. + block: + var encRes: Result[seq[byte], string] + try: + let buf = BrokerCbor.encode(value) + encRes = Result[seq[byte], string].ok(buf) + except SerializationError as exc: + encRes = Result[seq[byte], string].err("cbor encode failed: " & exc.msg) + except IOError as exc: + encRes = Result[seq[byte], string].err("cbor encode IO failure: " & exc.msg) + except CatchableError as exc: + encRes = + Result[seq[byte], string].err("cbor encode unexpected failure: " & exc.msg) + encRes + +template cborEncodeShared*[T]( + value: T, bufOut: var pointer, lenOut: var int +): Result[void, string] = + ## Refc-safe variant of `cborEncode`: produces an `allocShared0`-owned + ## buffer and never lets the intermediate `seq[byte]` escape across thread + ## boundaries. + ## + ## On `ok` the caller owns `bufOut` (size `lenOut` bytes) and must + ## `deallocShared(bufOut)` once done. On empty input `bufOut` is `nil` and + ## `lenOut` is 0. Used by the CBOR FFI listener path: under `--mm:refc` a + ## `seq[byte]` produced on the delivery thread cannot be safely shared with + ## subscriber callbacks invoked synchronously, so we copy the bytes into + ## shared heap immediately and drop the seq. + ## + ## Same template-vs-generic-proc rationale as `cborEncode`. + block: + bufOut = nil + lenOut = 0 + var encShRes: Result[void, string] + try: + let buf = BrokerCbor.encode(value) + if buf.len > 0: + let p = allocShared0(buf.len) + copyMem(p, unsafeAddr buf[0], buf.len) + bufOut = p + lenOut = buf.len + encShRes = Result[void, string].ok() + except SerializationError as exc: + encShRes = Result[void, string].err("cbor encode failed: " & exc.msg) + except IOError as exc: + encShRes = Result[void, string].err("cbor encode IO failure: " & exc.msg) + except CatchableError as exc: + encShRes = Result[void, string].err("cbor encode unexpected failure: " & exc.msg) + encShRes + +template cborDecode*[T](buf: openArray[byte], _: typedesc[T]): Result[T, string] = + ## Decode a CBOR-encoded buffer into `T` using the BrokerCbor flavor. + ## Wraps every decode failure as `Result.err`; never raises. Same + ## template-vs-generic-proc rationale as `cborEncode`. + block: + var decRes: Result[T, string] + try: + let v = BrokerCbor.decode(buf, T) + decRes = Result[T, string].ok(v) + except SerializationError as exc: + decRes = Result[T, string].err("cbor decode failed: " & exc.msg) + except IOError as exc: + decRes = Result[T, string].err("cbor decode IO failure: " & exc.msg) + except CatchableError as exc: + decRes = Result[T, string].err("cbor decode unexpected failure: " & exc.msg) + decRes + +# --------------------------------------------------------------------------- +# Result envelope shortcuts +# --------------------------------------------------------------------------- + +template cborEncodeResultEnvelope*[T](r: Result[T, string]): Result[seq[byte], string] = + ## Encode `Result[T, string]` as a CBOR response envelope. + cborEncode(toEnvelope(r)) + +template cborDecodeResultEnvelope*[T]( + buf: openArray[byte], _: typedesc[T] +): Result[T, string] = + ## Decode a CBOR response envelope into `Result[T, string]`. + ## + ## Returns the inner `Result` on success, or a framework error string + ## (prefixed `cbor decode failed: ...`) on a CBOR-level failure. + block: + let envRes = cborDecode(buf, CborResponseEnvelope[T]) + var res: Result[T, string] + if envRes.isErr(): + res = Result[T, string].err(envRes.error) + else: + res = fromEnvelope(envRes.value) + res + +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_cbor_courier.nim b/wasm-deps/brokers/brokers/internal/api_cbor_courier.nim new file mode 100644 index 000000000..7d5cc6212 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_cbor_courier.nim @@ -0,0 +1,345 @@ +## api_cbor_courier — runtime support for the CBOR FFI "buffer courier". +## ===================================================================== +## Part C of the CBOR refactoring (doc/CBOR_Refactoring.md §6). +## +## A CBOR-mode `_call` runs on a foreign caller's thread. Instead of +## decoding CBOR and driving a momentary chronos loop on that foreign +## thread, it becomes a pure courier: +## +## 1. copy the API name into a fixed POD message, +## 2. hand the raw request buffer (by pointer, ownership transferred) +## to the processing thread over a `Channel`, +## 3. block on a per-call response slot until the processing thread +## writes the response back. +## +## The processing thread owns CBOR decode/encode and the provider call. +## +## This module is plain runtime code (NOT codegen) used by the generated +## library runtime in `api_library.nim`. It deliberately contains no Nim +## GC types on the cross-thread message path: `CborCallMsg` is pure POD, +## so a foreign thread can enqueue one with zero GC involvement. +## +## Memory model: +## - `reqBuf` — `allocShared0` by `_allocBuffer`; ownership moves +## into the `CborCallMsg`; the processing thread frees it exactly once +## after copying the bytes out. +## - `respBuf` — `allocShared0` on the processing thread; ownership +## returns to the `_call` thread via the slot; the foreign caller +## frees it via `_freeBuffer`. +## - Response slots use a `Lock`+`Cond` (zero OS handles) for the +## blocking handoff — no busy-poll, no per-slot `ThreadSignalPtr`. + +{.push raises: [].} + +import std/[atomics, locks] + +const CborApiNameMax* = 256 + ## Inline fixed-size buffer for the ASCII API name carried in a courier + ## message. Carrying the name itself (rather than an interned id) keeps + ## the message self-describing and avoids a separate id table that could + ## silently desync from the dispatch `case`. + +const CborMaxSlotSegments = 4 + ## Doubling the slot pool from `origSlotCount` to the 4× ceiling appends at + ## most two segments beyond the initial one (N → +N → +2N), so three are + ## ever live; 4 leaves a margin. + +type + CborCallMsg* = object + ## Pure-POD message a foreign `_call` thread hands to the processing + ## thread. No Nim `string`/`seq`/`ref` — safe to copy through a + ## `Channel` with zero GC involvement on the foreign thread. + apiName*: array[CborApiNameMax, char] ## NUL-terminated ASCII + reqBuf*: pointer ## allocShared0; ownership transfers to the processing thread + reqLen*: int32 + slotIdx*: int32 ## index of the response slot to complete + targetCtx*: uint32 + ## reduced-A: the FULL BrokerContext the foreign caller addressed. For a + ## main-context call this equals the library ctx; for a sub-instance call + ## it carries the sub ctx (same classCtx as the library, distinct + ## instanceCtx). The processing thread dispatches the adapter against this + ## so the provider keyed by the sub ctx is reached. + + CborRespSlot = object + lock: Lock + cond: Cond + inUse: Atomic[int] ## 0 free, 1 claimed — claimed via CAS + ready: int ## guarded by `lock`: 0 pending, 1 complete + respBuf: pointer ## allocShared0; ownership returns to the `_call` thread + respLen: int32 + status: int32 ## the int32 `_call` returns to the foreign caller + + CborCallRing* = object + ## Single-lock POD-element MPSC ring, allocated wholly in shared + ## heap. Replaces `system.Channel[CborCallMsg]` deliberately: that + ## channel allocates its message slots out of the sender thread's + ## per-thread Nim allocator, and once the sender thread exits its + ## TLS-tied allocator descriptor is freed by pthread cleanup. A + ## subsequent `close()` on the shutdown thread walks straight into + ## that dead descriptor (caught by ASAN on the stress_mt teardown + ## path). This ring uses `allocShared` for its storage — single + ## owner (the `CborCourier`), freed from the same thread that + ## allocated it, no per-thread allocator involvement. + buf: ptr UncheckedArray[CborCallMsg] + cap: int + head: int ## next index the consumer reads + tail: int ## next index a producer writes + count: int ## guarded by `lock` + lock: Lock + + CborSlotSegment = object + ## One append-only block of response slots. Existing segments are never + ## moved or freed until teardown, so a foreign thread blocked in + ## `waitSlot` on a slot's `Cond` keeps a stable address. This is the + ## reason the pool grows by *appending* segments rather than + ## reallocating one array: relocating a slot whose `Lock`/`Cond` a + ## blocked `_call` is waiting on is a use-after-free. + slots: ptr UncheckedArray[CborRespSlot] + base: int ## global index of `slots[0]` + len: int ## number of slots in this segment + + CborCourier* = object + ## One per library context. Lives in shared heap; created in + ## `_createContext`, freed in `_shutdown` after the processing thread + ## has joined and all in-flight `_call`s have drained. + ring*: CborCallRing + segs: array[CborMaxSlotSegments, CborSlotSegment] + nSegs: Atomic[int] + ## Live segment count. Published with `moRelease` after a new segment is + ## fully populated; the lock-free claim scan reads it with `moAcquire`. + ## Append-only — segments are never removed before teardown. + slotCount: int + ## Total live slots across all segments. Read/written only under + ## `ring.lock` (growth coordinates the slot pool and the ring together). + origSlotCount: int + ## Set once at construction; the growth ceiling is `4 * origSlotCount`. + inFlight*: Atomic[int] + ## Count of `_call`s that passed the active-check but have not yet + ## finished reading their slot. `_shutdown` waits for this to reach + ## zero — while the processing thread is still handling — before it + ## tells the processing thread to stop. + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + +proc newCborCourier*(slotCount: int): ptr CborCourier = + ## Allocate a courier with `slotCount` response slots. `slotCount` is the + ## *initial* ceiling on concurrent in-flight `_call`s; the request ring is + ## sized the same, so the slot pool gates the ring (a `_call` always claims + ## a slot before enqueuing). On exhaustion the pool and ring grow together + ## by doubling, up to a hard ceiling of `4 * slotCount` — see `claimSlot`. + let c = cast[ptr CborCourier](allocShared0(sizeof(CborCourier))) + c.ring.buf = + cast[ptr UncheckedArray[CborCallMsg]](allocShared0(slotCount * sizeof(CborCallMsg))) + c.ring.cap = slotCount + c.ring.head = 0 + c.ring.tail = 0 + c.ring.count = 0 + initLock(c.ring.lock) + c.origSlotCount = slotCount + c.slotCount = slotCount + let seg0 = cast[ptr UncheckedArray[CborRespSlot]](allocShared0( + slotCount * sizeof(CborRespSlot) + )) + for i in 0 ..< slotCount: + initLock(seg0[i].lock) + initCond(seg0[i].cond) + seg0[i].inUse.store(0, moRelaxed) + c.segs[0] = CborSlotSegment(slots: seg0, base: 0, len: slotCount) + c.nSegs.store(1, moRelease) + c + +proc freeCborCourier*(c: ptr CborCourier) = + ## Release a courier. MUST be called only after the processing thread + ## has joined and `inFlight` has reached zero — see `_shutdown`. + if c.isNil: + return + for s in 0 ..< c.nSegs.load(moAcquire): + let seg = addr c.segs[s] + for i in 0 ..< seg.len: + deinitCond(seg.slots[i].cond) + deinitLock(seg.slots[i].lock) + deallocShared(seg.slots) + deinitLock(c.ring.lock) + if not c.ring.buf.isNil: + deallocShared(c.ring.buf) + deallocShared(c) + +# --------------------------------------------------------------------------- +# Ring — MPSC over a fixed-size POD slot array. Single lock for both ends; +# the ring is not the contended path (per-call cost is dominated by the +# Cond handoff and the chronos coroutine spawn). +# --------------------------------------------------------------------------- + +proc growRingLocked(r: ptr CborCallRing, newCap: int): bool = + ## Grow the POD ring to `newCap` (> `r.cap`), linearising live elements. + ## Caller MUST hold `r.lock`. Safe because `CborCallMsg` is pure POD and no + ## thread holds a pointer into `buf` across the lock. + ## + ## Returns false — leaving the ring completely untouched — if the new buffer + ## cannot be allocated, so the caller can roll back the coordinated pool+ring + ## growth instead of dereferencing nil while holding the lock. + let newBuf = + cast[ptr UncheckedArray[CborCallMsg]](allocShared0(newCap * sizeof(CborCallMsg))) + if newBuf.isNil: + return false + for i in 0 ..< r.count: + newBuf[i] = r.buf[(r.head + i) mod r.cap] + deallocShared(r.buf) + r.buf = newBuf + r.head = 0 + r.tail = r.count + r.cap = newCap + true + +proc tryEnqueue*(r: ptr CborCallRing, msg: CborCallMsg): bool = + ## Multi-producer. Returns false on full. A `_call` always claims a + ## response slot before enqueuing and the ring is grown in step with the + ## slot pool (see `claimSlot`), so the ring cap always matches the live + ## slot count and a `false` here is a programming error, not backpressure. + acquire(r.lock) + if r.count >= r.cap: + release(r.lock) + return false + r.buf[r.tail] = msg + r.tail = (r.tail + 1) mod r.cap + inc r.count + release(r.lock) + true + +proc tryDequeue*(r: ptr CborCallRing, dst: var CborCallMsg): bool = + ## Single consumer. Returns false on empty. + acquire(r.lock) + if r.count == 0: + release(r.lock) + return false + dst = r.buf[r.head] + r.head = (r.head + 1) mod r.cap + dec r.count + release(r.lock) + true + +# --------------------------------------------------------------------------- +# Response slots +# --------------------------------------------------------------------------- + +proc slotAt(c: ptr CborCourier, idx: int): ptr CborRespSlot {.inline.} = + ## Map a global slot index to its slot in the owning segment. Segments are + ## append-only and never relocated, so a published index stays valid. + for s in 0 ..< c.nSegs.load(moAcquire): + let seg = addr c.segs[s] + if idx >= seg.base and idx < seg.base + seg.len: + return addr seg.slots[idx - seg.base] + nil + +proc initClaimedSlot(s: ptr CborRespSlot) {.inline.} = + acquire(s.lock) + s.ready = 0 + s.respBuf = nil + s.respLen = 0 + s.status = 0 + release(s.lock) + +proc tryClaimScan(c: ptr CborCourier): int = + ## Scan all live slots for a free one; CAS-claim and reset it. Returns the + ## global index, or -1 if none free. Lock-free over the published segments. + for sgi in 0 ..< c.nSegs.load(moAcquire): + let seg = addr c.segs[sgi] + for i in 0 ..< seg.len: + var expected = 0 + if seg.slots[i].inUse.compareExchange(expected, 1, moAcquire, moRelaxed): + initClaimedSlot(addr seg.slots[i]) + return seg.base + i + -1 + +proc claimSlot*(c: ptr CborCourier): int = + ## Claim a free response slot. Returns its index, or -1 only when the pool + ## is at its `4 * origSlotCount` ceiling and fully in-use. On exhaustion + ## below the ceiling the pool grows by appending a new segment (existing + ## slots are never moved) and the ring grows in step — both under + ## `ring.lock`. Growth is the rare slow path. + let fast = tryClaimScan(c) + if fast >= 0: + return fast + # Pool exhausted. Coordinate growth under the ring lock. + acquire(c.ring.lock) + # Re-scan under the lock: a concurrent release or a concurrent grow may + # have produced a usable slot since the lock-free scan above. + let again = tryClaimScan(c) + if again >= 0: + release(c.ring.lock) + return again + let curCount = c.slotCount + let newCount = min(curCount * 2, c.origSlotCount * 4) + let segIdx = c.nSegs.load(moAcquire) + if newCount == curCount or segIdx >= CborMaxSlotSegments: + release(c.ring.lock) # at the ceiling — retain the drop contract + return -1 + let addLen = newCount - curCount + let seg = + cast[ptr UncheckedArray[CborRespSlot]](allocShared0(addLen * sizeof(CborRespSlot))) + if seg.isNil: + # OOM allocating the new slot segment: nothing has been mutated yet, so + # release the lock and retain the refusal (drop) contract rather than + # crashing in initLock/initCond. + release(c.ring.lock) + return -1 + for i in 0 ..< addLen: + initLock(seg[i].lock) + initCond(seg[i].cond) + seg[i].inUse.store(0, moRelaxed) + # Grow the ring in step BEFORE committing any pool state. If the ring buffer + # can't be allocated, roll back the freshly-built segment (nothing has been + # published — slotCount/segs/nSegs are untouched and the ring is left intact) + # and retain the refusal contract. + if not growRingLocked(addr c.ring, newCount): + for i in 0 ..< addLen: + deinitCond(seg[i].cond) + deinitLock(seg[i].lock) + deallocShared(seg) + release(c.ring.lock) + return -1 + # Ring grown; the pool+ring growth is guaranteed to complete. Claim slot 0 of + # the new segment BEFORE publishing it, so no concurrent scanner can race us. + var expected = 0 + discard seg[0].inUse.compareExchange(expected, 1, moAcquire, moRelaxed) + initClaimedSlot(addr seg[0]) + c.segs[segIdx] = CborSlotSegment(slots: seg, base: curCount, len: addLen) + c.slotCount = newCount # ring cap already == newCount + c.nSegs.store(segIdx + 1, moRelease) # publish last + release(c.ring.lock) + curCount # global index of seg[0], already claimed + +proc releaseSlot*(c: ptr CborCourier, idx: int) = + ## Return a slot to the free pool. Call only after `waitSlot` returned. + slotAt(c, idx).inUse.store(0, moRelease) + +proc completeSlot*( + c: ptr CborCourier, idx: int, respBuf: pointer, respLen: int32, status: int32 +) = + ## Processing-thread side: publish a response and wake the waiting + ## `_call`. `respBuf` ownership passes to the `_call` thread. + let s = slotAt(c, idx) + acquire(s.lock) + s.respBuf = respBuf + s.respLen = respLen + s.status = status + s.ready = 1 + signal(s.cond) + release(s.lock) + +proc waitSlot*( + c: ptr CborCourier, idx: int +): tuple[respBuf: pointer, respLen: int32, status: int32] = + ## Foreign `_call` side: block until `completeSlot` publishes a response. + ## Zero-fd blocking handoff via `Cond` — no busy-poll. + let s = slotAt(c, idx) + acquire(s.lock) + while s.ready == 0: + wait(s.cond, s.lock) + result = (s.respBuf, s.respLen, s.status) + s.ready = 0 + release(s.lock) + +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_cbor_descriptor.nim b/wasm-deps/brokers/brokers/internal/api_cbor_descriptor.nim new file mode 100644 index 000000000..0cdab91cf --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_cbor_descriptor.nim @@ -0,0 +1,117 @@ +## Runtime schema-descriptor types for the CBOR FFI discovery API. +## +## `_listApis` and `_getSchema` return JSON-encoded views of these +## records so dynamic clients can introspect a library's surface without +## referring to the build-time generated headers. +## +## These types are hand-rolled (not produced by the broker macros) and are +## therefore part of the *stable* CBOR FFI v1 contract: changes to fields +## here are wire-breaking. Add new fields rather than rename / reorder. + +{.push raises: [].} + +import std/[json, options] +import ./api_cbor_codec + +export api_cbor_codec + +type + ApiFieldInfo* = object + name*: string + nimType*: string + + ApiEnumValueInfo* = object + name*: string + ordinal*: int + + ApiTypeInfo* = object + name*: string + kind*: string ## "object" / "enum" / "alias" / "distinct"; matches `ApiTypeKind`. + fields*: seq[ApiFieldInfo] + enumValues*: seq[ApiEnumValueInfo] + underlyingType*: string + + ApiRequestInfo* = object + apiName*: string + argsType*: string + ## Nim type name of the synthesised args struct; + ## empty string for zero-arg requests. + argFields*: seq[ApiFieldInfo] + responseType*: string + + ApiEventInfo* = object + apiName*: string + payloadType*: string + + ApiList* = object ## Lightweight payload returned by `_listApis`. + libName*: string + requests*: seq[string] + events*: seq[string] + + LibraryDescriptor* = object ## Full payload returned by `_getSchema`. + libName*: string + cddl*: string ## Verbatim contents of the generated `.cddl`. + requests*: seq[ApiRequestInfo] + events*: seq[ApiEventInfo] + types*: seq[ApiTypeInfo] + +{.pop.} + +# JSON serialisation lives outside `{.push raises: [].}` because std/json +# indexing can raise KeyError. + +proc toJson*(f: ApiFieldInfo): JsonNode = + %*{"name": f.name, "nimType": f.nimType} + +proc toJson*(v: ApiEnumValueInfo): JsonNode = + %*{"name": v.name, "ordinal": v.ordinal} + +proc toJson*(t: ApiTypeInfo): JsonNode = + result = %*{ + "name": t.name, + "kind": t.kind, + "fields": newJArray(), + "enumValues": newJArray(), + "underlyingType": t.underlyingType, + } + for f in t.fields: + result["fields"].add(f.toJson()) + for v in t.enumValues: + result["enumValues"].add(v.toJson()) + +proc toJson*(r: ApiRequestInfo): JsonNode = + result = %*{ + "apiName": r.apiName, + "argsType": r.argsType, + "argFields": newJArray(), + "responseType": r.responseType, + } + for f in r.argFields: + result["argFields"].add(f.toJson()) + +proc toJson*(e: ApiEventInfo): JsonNode = + %*{"apiName": e.apiName, "payloadType": e.payloadType} + +proc toJson*(a: ApiList): JsonNode = + %*{"libName": a.libName, "requests": a.requests, "events": a.events} + +proc toJson*(d: LibraryDescriptor): JsonNode = + result = %*{ + "libName": d.libName, + "cddl": d.cddl, + "requests": newJArray(), + "events": newJArray(), + "types": newJArray(), + } + for r in d.requests: + result["requests"].add(r.toJson()) + for e in d.events: + result["events"].add(e.toJson()) + for t in d.types: + result["types"].add(t.toJson()) + +proc toJsonString*(a: ApiList): string = + $a.toJson() + +proc toJsonString*(d: LibraryDescriptor): string = + $d.toJson() diff --git a/wasm-deps/brokers/brokers/internal/api_cbor_event_courier.nim b/wasm-deps/brokers/brokers/internal/api_cbor_event_courier.nim new file mode 100644 index 000000000..1b0d05190 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_cbor_event_courier.nim @@ -0,0 +1,171 @@ +## api_cbor_event_courier — fire-and-forget ring for CBOR FFI event delivery. +## ============================================================================ +## Part D-3 of the CBOR refactoring (doc/CBOR_Round2_PartD_EventCourier.md). +## +## A CBOR-mode event emitted by a provider on the processing thread needs to +## fan out to all foreign-callback subscribers without blocking the provider. +## The shape is the mirror image of `api_cbor_courier`: +## +## producer (processing thread) +## 1. CBOR-encode the event payload **once** into a shared-heap buffer, +## 2. enqueue an `EventMsg` carrying `(eventName, ctx, buf, bufLen)` +## — ownership of `buf` transfers to the consumer, +## 3. wake the delivery thread via its broker dispatch signal. +## +## consumer (delivery thread, via `registerBrokerPoller`) +## 1. dequeue messages from the ring, +## 2. snapshot the foreign-subscriber list for `(ctx, eventName)`, +## 3. invoke each foreign callback synchronously, +## 4. free the buffer. +## +## Differences from `api_cbor_courier`: +## - **No response slots, no `inFlight` counter** — events are +## fire-and-forget. Producers do not block, do not wait for a reply. +## - Ring is sized for **burst capacity** (default 256), not for +## concurrent in-flight count. A full ring drops the event with a +## diagnostic (logged by the caller) — appropriate for the +## fire-and-forget contract. +## - `eventName` is carried inline as a fixed-size NUL-terminated +## ASCII buffer (same convention as `CborCallMsg.apiName`) so the +## message stays POD — zero GC involvement on the producer side. +## +## This module is plain runtime code (NOT codegen) used by the generated +## library runtime in `api_library.nim`. + +{.push raises: [].} + +import std/locks + +const CborEventNameMax* = 256 + ## Inline fixed-size buffer for the ASCII event name carried in a + ## courier message. Same value as `CborApiNameMax` — every event name + ## the CBOR-mode subscribe surface accepts already fits within this + ## bound (the wrapper validates name length). + +type + CborEventMsg* = object + ## Pure-POD message handed from the processing thread (producer) to + ## the delivery thread (consumer). No Nim `string` / `seq` / `ref` + ## crosses the channel — the producer encoded the payload into a + ## shared-heap buffer and transfers ownership of it via `buf`. + eventName*: array[CborEventNameMax, char] ## NUL-terminated ASCII + ctx*: uint32 ## BrokerContext.uint32; identifies the per-ctx sub list + buf*: pointer + ## `allocShared0`; ownership transferred to the consumer. + ## The consumer frees this exactly once after the fan-out completes. + bufLen*: int32 + + CborEventRing* = object + ## Single-lock POD-element ring, allocated wholly in shared heap. + ## Same shape (and same rationale) as `CborCallRing` — + ## `system.Channel[T]` is avoided to keep the storage out of the + ## producer thread's per-thread Nim allocator (would leak/UAF when + ## the producer thread exits before the consumer fully drains). + buf: ptr UncheckedArray[CborEventMsg] + cap: int + origCap: int ## set once at construction; growth ceiling is `4 * origCap` + head: int ## next index the consumer reads + tail: int ## next index a producer writes + count: int ## guarded by `lock` + lock: Lock + + CborEventCourier* = object + ## One per library context. Lives in shared heap; created in + ## `_createContext`, freed in `_shutdown` **after both threads have + ## joined**. The teardown sequence drains any messages still in the + ## ring (freeing their `buf`s) before deallocating the ring storage. + ring*: CborEventRing + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + +proc newCborEventCourier*(ringCap: int): ptr CborEventCourier = + ## Allocate an event courier sized for `ringCap` outstanding events. + ## Producers that find the ring full drop the event (events are + ## fire-and-forget). Pick `ringCap` generously — there's no slot pool + ## gating it the way `CborCourier`'s slot count gates its ring. + let c = cast[ptr CborEventCourier](allocShared0(sizeof(CborEventCourier))) + c.ring.buf = + cast[ptr UncheckedArray[CborEventMsg]](allocShared0(ringCap * sizeof(CborEventMsg))) + c.ring.cap = ringCap + c.ring.origCap = ringCap + c.ring.head = 0 + c.ring.tail = 0 + c.ring.count = 0 + initLock(c.ring.lock) + c + +proc drainAndFree*(c: ptr CborEventCourier) = + ## Free any messages still in the ring (deallocating their `buf`), + ## then free the ring storage and the courier itself. MUST be called + ## only after both the producer and consumer threads have joined. + if c.isNil: + return + # Drain remaining messages — buffers must be freed exactly once. + acquire(c.ring.lock) + while c.ring.count > 0: + let m = c.ring.buf[c.ring.head] + if not m.buf.isNil: + deallocShared(m.buf) + c.ring.head = (c.ring.head + 1) mod c.ring.cap + dec c.ring.count + release(c.ring.lock) + deinitLock(c.ring.lock) + if not c.ring.buf.isNil: + deallocShared(c.ring.buf) + deallocShared(c) + +# --------------------------------------------------------------------------- +# Ring — single-lock MPSC over a fixed-size POD slot array. +# --------------------------------------------------------------------------- + +proc tryEnqueue*(r: ptr CborEventRing, msg: CborEventMsg): bool = + ## Multi-producer (though in practice the producer is the single + ## processing thread). Returns false on full — the caller is + ## responsible for freeing `msg.buf` in that case (the buffer never + ## entered the ring, so the ring never took ownership). + acquire(r.lock) + if r.count >= r.cap: + # Full: grow by doubling, up to a hard ceiling of `4 * origCap`. At the + # ceiling retain the fire-and-forget drop contract. + let newCap = min(r.cap * 2, r.origCap * 4) + if newCap == r.cap: + release(r.lock) + return false + let newBuf = cast[ptr UncheckedArray[CborEventMsg]](allocShared0( + newCap * sizeof(CborEventMsg) + )) + if newBuf.isNil: + # OOM: keep the existing buffer untouched and fall back to the drop + # contract (same as hitting the ceiling) rather than dereferencing nil. + release(r.lock) + return false + for i in 0 ..< r.count: + newBuf[i] = r.buf[(r.head + i) mod r.cap] + deallocShared(r.buf) + r.buf = newBuf + r.head = 0 + r.tail = r.count + r.cap = newCap + r.buf[r.tail] = msg + r.tail = (r.tail + 1) mod r.cap + inc r.count + release(r.lock) + true + +proc tryDequeue*(r: ptr CborEventRing, dst: var CborEventMsg): bool = + ## Single consumer (the delivery thread's event-courier poller). + ## Returns false on empty. Ownership of `dst.buf` transfers to the + ## caller — they must `deallocShared` it after the fan-out. + acquire(r.lock) + if r.count == 0: + release(r.lock) + return false + dst = r.buf[r.head] + r.head = (r.head + 1) mod r.cap + dec r.count + release(r.lock) + true + +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_cbor_subs_registry.nim b/wasm-deps/brokers/brokers/internal/api_cbor_subs_registry.nim new file mode 100644 index 000000000..940f3d690 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_cbor_subs_registry.nim @@ -0,0 +1,403 @@ +## CBOR Subscription Registry +## -------------------------- +## Refc-safe subscription book-keeping for the CBOR FFI listener path. +## +## The CBOR-mode listener delivery thread crosses GC boundaries with the +## subscriber registration path (subscribe/unsubscribe run on foreign caller +## threads via the C ABI). Under `--mm:orc` atomic refcounts make a plain +## `Table[(uint32, string), seq[Subscription]]` work; under `--mm:refc` the +## per-thread heaps + STW collector cannot safely see another thread's +## refcounted pointers, which used to gate the Phase 9F listener stress +## under macOS+Nim 2.2.4+refc+debug. +## +## This module replaces that GC'd container with a hand-rolled shared-heap +## hash table: +## - `BucketHead` (one per `(ctx, eventName)` key) and `SubNode` (one per +## subscription) are allocated via `allocShared0`. +## - The event-name key is stored as an owned `cstring` +## (`allocCStringCopy` at insertion, `freeCString` when the bucket goes +## away). +## - Bucket arrays are `ptr UncheckedArray[ptr BucketHead]`, never `seq`. +## +## All public procs are `{.gcsafe, raises: [].}` and acquire the registry's +## internal `Lock`. Snapshot copies `(cb, userData)` to a freshly-allocated +## shared buffer under the lock, so callbacks fan out unlocked against POD +## values that no concurrent unsubscriber can free. +## +## Callback type: stored as `pointer` so this module is generic across +## libraries. Callers cast back to the per-library `CborEventCallback` +## (a `cdecl, gcsafe, raises: []` proc type) at the call site. + +{.push raises: [].} + +import std/locks + +type + SubSnapshot* = object ## A POD copy of `(cb, userData)`. Callbacks fan out unlocked. + cb*: pointer + userData*: pointer + + SubNode = object + handle: uint64 + cb: pointer + userData: pointer + next: ptr SubNode + + BucketHead = object + ctx: uint32 + eventName: cstring # owned (allocCStringCopy) + eventNameLen: int + subsHead: ptr SubNode + subsCount: int + next: ptr BucketHead # collision chain + + SubsRegistry* = object + buckets: ptr UncheckedArray[ptr BucketHead] + bucketsLen: uint32 # always a power of two; mask = bucketsLen - 1 + entryCount: int # live BucketHead count, drives resize + lock: Lock + +const + InitialBuckets: uint32 = 32 + ResizeNumerator = 3 + ResizeDenominator = 4 # resize at load factor 0.75 + +# --------------------------------------------------------------------------- +# Internal helpers (no locking — caller must hold reg.lock) +# --------------------------------------------------------------------------- + +proc cstrLen(s: cstring): int {.inline, raises: [].} = + if s.isNil: + return 0 + var p = cast[ptr UncheckedArray[char]](s) + var i = 0 + while p[i] != '\0': + inc i + i + +proc cstrEq(a: cstring, aLen: int, b: cstring, bLen: int): bool {.inline.} = + if aLen != bLen: + return false + if aLen == 0: + return true + let ap = cast[ptr UncheckedArray[byte]](a) + let bp = cast[ptr UncheckedArray[byte]](b) + for i in 0 ..< aLen: + if ap[i] != bp[i]: + return false + true + +proc cstrAlloc(s: cstring, sLen: int): cstring {.inline, raises: [].} = + ## Local mirror of `allocCStringCopy(string)` for `cstring` input — avoids + ## pulling in `api_common` (and its chronos chain) here. + if sLen == 0: + return cast[cstring](nil) + let buf = cast[cstring](allocShared(sLen + 1)) + let src = cast[pointer](s) + copyMem(buf, src, sLen) + cast[ptr char](cast[int](buf) + sLen)[] = '\0' + buf + +proc cstrFree(s: cstring) {.inline.} = + if not s.isNil: + deallocShared(s) + +proc keyHash(ctx: uint32, name: cstring, nameLen: int): uint32 {.inline.} = + # FNV-1a-ish, seeded with ctx so two ctxs sharing a name spread across buckets. + var h: uint32 = 2166136261'u32 xor ctx + if nameLen > 0: + let p = cast[ptr UncheckedArray[byte]](name) + for i in 0 ..< nameLen: + h = h xor uint32(p[i]) + h = h * 16777619'u32 + h + +proc bucketIndex( + reg: ptr SubsRegistry, ctx: uint32, name: cstring, nameLen: int +): uint32 {.inline.} = + keyHash(ctx, name, nameLen) and (reg.bucketsLen - 1'u32) + +proc findBucket( + reg: ptr SubsRegistry, ctx: uint32, name: cstring, nameLen: int +): ptr BucketHead = + let idx = bucketIndex(reg, ctx, name, nameLen) + var b = reg.buckets[idx] + while not b.isNil: + if b.ctx == ctx and cstrEq(b.eventName, b.eventNameLen, name, nameLen): + return b + b = b.next + nil + +proc unlinkBucket(reg: ptr SubsRegistry, target: ptr BucketHead) = + let idx = bucketIndex(reg, target.ctx, target.eventName, target.eventNameLen) + var prev: ptr BucketHead = nil + var cur = reg.buckets[idx] + while not cur.isNil: + if cur == target: + if prev.isNil: + reg.buckets[idx] = cur.next + else: + prev.next = cur.next + return + prev = cur + cur = cur.next + +proc freeNodeChain(head: ptr SubNode) = + var cur = head + while not cur.isNil: + let nxt = cur.next + deallocShared(cur) + cur = nxt + +proc disposeBucket(b: ptr BucketHead) = + freeNodeChain(b.subsHead) + cstrFree(b.eventName) + deallocShared(b) + +proc resize(reg: ptr SubsRegistry, newLen: uint32) = + ## Double-or-larger rehash. Caller holds the lock. + let bytes = sizeof(ptr BucketHead) * int(newLen) + let newBuckets = cast[ptr UncheckedArray[ptr BucketHead]](allocShared0(bytes)) + let oldBuckets = reg.buckets + let oldLen = reg.bucketsLen + reg.buckets = newBuckets + reg.bucketsLen = newLen + for i in 0 ..< oldLen: + var cur = oldBuckets[i] + while not cur.isNil: + let nxt = cur.next + let idx = bucketIndex(reg, cur.ctx, cur.eventName, cur.eventNameLen) + cur.next = reg.buckets[idx] + reg.buckets[idx] = cur + cur = nxt + deallocShared(oldBuckets) + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +proc subsRegistryNew*(): ptr SubsRegistry {.gcsafe, raises: [].} = + let reg = cast[ptr SubsRegistry](allocShared0(sizeof(SubsRegistry))) + let bytes = sizeof(ptr BucketHead) * int(InitialBuckets) + reg.buckets = cast[ptr UncheckedArray[ptr BucketHead]](allocShared0(bytes)) + reg.bucketsLen = InitialBuckets + reg.entryCount = 0 + initLock(reg.lock) + reg + +proc subsRegistryFree*(reg: ptr SubsRegistry) {.gcsafe, raises: [].} = + ## Tear the entire registry down. Not normally called by the codegen — the + ## generated runtime currently leaks the registry at process exit, matching + ## the prior `Table` behaviour. Provided for completeness / tests. + if reg.isNil: + return + for i in 0 ..< reg.bucketsLen: + var cur = reg.buckets[i] + while not cur.isNil: + let nxt = cur.next + disposeBucket(cur) + cur = nxt + deallocShared(reg.buckets) + deinitLock(reg.lock) + deallocShared(reg) + +proc subsRegistryAdd*( + reg: ptr SubsRegistry, + ctx: uint32, + name: cstring, + handle: uint64, + cb: pointer, + userData: pointer, +) {.gcsafe, raises: [].} = + ## Idempotent on `handle`: if a node with the same handle already exists for + ## the key, the call is a no-op. Handles are minted by an atomic counter at + ## the codegen call site so this branch normally never fires; it exists to + ## keep the data structure self-consistent under bizarre caller bugs. + {.cast(gcsafe).}: + withLock reg.lock: + let nameLen = cstrLen(name) + var bucket = findBucket(reg, ctx, name, nameLen) + if bucket.isNil: + bucket = cast[ptr BucketHead](allocShared0(sizeof(BucketHead))) + bucket.ctx = ctx + bucket.eventName = cstrAlloc(name, nameLen) + bucket.eventNameLen = nameLen + bucket.subsHead = nil + bucket.subsCount = 0 + let idx = bucketIndex(reg, ctx, name, nameLen) + bucket.next = reg.buckets[idx] + reg.buckets[idx] = bucket + inc reg.entryCount + if reg.entryCount * ResizeDenominator > int(reg.bucketsLen) * ResizeNumerator: + resize(reg, reg.bucketsLen * 2'u32) + else: + var cur = bucket.subsHead + while not cur.isNil: + if cur.handle == handle: + return + cur = cur.next + let node = cast[ptr SubNode](allocShared0(sizeof(SubNode))) + node.handle = handle + node.cb = cb + node.userData = userData + node.next = bucket.subsHead + bucket.subsHead = node + inc bucket.subsCount + +proc subsRegistryRemoveOne*( + reg: ptr SubsRegistry, ctx: uint32, name: cstring, handle: uint64 +): int32 {.gcsafe, raises: [], discardable.} = + ## Returns: 0 ok, -2 key not found, -3 handle not found. + {.cast(gcsafe).}: + withLock reg.lock: + let nameLen = cstrLen(name) + let bucket = findBucket(reg, ctx, name, nameLen) + if bucket.isNil: + return -2'i32 + var prev: ptr SubNode = nil + var cur = bucket.subsHead + while not cur.isNil: + if cur.handle == handle: + if prev.isNil: + bucket.subsHead = cur.next + else: + prev.next = cur.next + deallocShared(cur) + dec bucket.subsCount + if bucket.subsCount == 0: + unlinkBucket(reg, bucket) + disposeBucket(bucket) + dec reg.entryCount + return 0'i32 + prev = cur + cur = cur.next + return -3'i32 + +proc subsRegistryRemoveAllForKey*( + reg: ptr SubsRegistry, ctx: uint32, name: cstring +): int32 {.gcsafe, raises: [], discardable.} = + ## Returns 0 if the key existed (and was dropped), -2 otherwise. + {.cast(gcsafe).}: + withLock reg.lock: + let nameLen = cstrLen(name) + let bucket = findBucket(reg, ctx, name, nameLen) + if bucket.isNil: + return -2'i32 + unlinkBucket(reg, bucket) + disposeBucket(bucket) + dec reg.entryCount + return 0'i32 + +proc subsRegistryRemoveAllForKeyN*( + reg: ptr SubsRegistry, ctx: uint32, name: cstring +): int32 {.gcsafe, raises: [].} = + ## Returns the number of subscriptions removed (>= 0), or -2 if the key was + ## not found. Teardown paths must decrement the shared per-event subs-count + ## by the exact number removed (not reset to 0) so a sibling context/instance + ## sharing the event name is not silenced. + {.cast(gcsafe).}: + withLock reg.lock: + let nameLen = cstrLen(name) + let bucket = findBucket(reg, ctx, name, nameLen) + if bucket.isNil: + return -2'i32 + let removed = int32(bucket.subsCount) + unlinkBucket(reg, bucket) + disposeBucket(bucket) + dec reg.entryCount + return removed + +proc subsRegistrySnapshot*( + reg: ptr SubsRegistry, + ctx: uint32, + name: cstring, + bufOut: var ptr UncheckedArray[SubSnapshot], + lenOut: var int, +) {.gcsafe, raises: [].} = + ## Allocates a shared-heap array of `(cb, userData)` for the bucket. Sets + ## `bufOut = nil`, `lenOut = 0` if there are no subscribers — callers should + ## then skip `subsRegistrySnapshotFree`. + bufOut = nil + lenOut = 0 + {.cast(gcsafe).}: + withLock reg.lock: + let nameLen = cstrLen(name) + let bucket = findBucket(reg, ctx, name, nameLen) + if bucket.isNil or bucket.subsCount == 0: + return + let n = bucket.subsCount + let bytes = sizeof(SubSnapshot) * n + let buf = cast[ptr UncheckedArray[SubSnapshot]](allocShared0(bytes)) + var cur = bucket.subsHead + var i = 0 + while not cur.isNil and i < n: + buf[i].cb = cur.cb + buf[i].userData = cur.userData + cur = cur.next + inc i + bufOut = buf + lenOut = i + +proc subsRegistrySnapshotFree*(buf: ptr UncheckedArray[SubSnapshot]) {.inline.} = + if not buf.isNil: + deallocShared(buf) + +proc subsRegistryFreeForCtx*( + reg: ptr SubsRegistry, ctx: uint32 +) {.gcsafe, raises: [].} = + ## Drops every bucket whose `ctx` matches. Called from `_shutdown(ctx)` + ## after the processing thread has been joined, so no concurrent delivery + ## can race with this teardown. + {.cast(gcsafe).}: + withLock reg.lock: + for i in 0 ..< reg.bucketsLen: + var prev: ptr BucketHead = nil + var cur = reg.buckets[i] + while not cur.isNil: + let nxt = cur.next + if cur.ctx == ctx: + if prev.isNil: + reg.buckets[i] = nxt + else: + prev.next = nxt + disposeBucket(cur) + dec reg.entryCount + else: + prev = cur + cur = nxt + +type SubsFreedCb* = proc(name: cstring, count: int32) {.gcsafe, raises: [].} + ## Invoked once per disposed bucket by `subsRegistryFreeForClass` with the + ## bucket's event name and live subscription count, so the caller can + ## decrement the matching per-event subs-count atomic. + +proc subsRegistryFreeForClass*( + reg: ptr SubsRegistry, classCtx: uint16, onFreed: SubsFreedCb +) {.gcsafe, raises: [].} = + ## Drops every bucket whose ctx low16 == `classCtx` — the lib ctx itself + ## (instanceCtx 0) plus every sub-instance sharing its classCtx. For each + ## disposed bucket with live subs, invokes `onFreed(eventName, subsCount)` + ## so the caller can decrement the shared per-event subs-count. Called from + ## `_shutdown(libCtx)` after both threads are joined, so no concurrent + ## delivery can race this teardown. + {.cast(gcsafe).}: + withLock reg.lock: + for i in 0 ..< reg.bucketsLen: + var prev: ptr BucketHead = nil + var cur = reg.buckets[i] + while not cur.isNil: + let nxt = cur.next + if (cur.ctx and 0x0000FFFF'u32) == uint32(classCtx): + if not onFreed.isNil and cur.subsCount > 0: + onFreed(cur.eventName, int32(cur.subsCount)) + if prev.isNil: + reg.buckets[i] = nxt + else: + prev.next = nxt + disposeBucket(cur) + dec reg.entryCount + else: + prev = cur + cur = nxt + +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_cbor_tuple.nim b/wasm-deps/brokers/brokers/internal/api_cbor_tuple.nim new file mode 100644 index 000000000..68fff3b66 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_cbor_tuple.nim @@ -0,0 +1,85 @@ +## api_cbor_tuple +## --------------- +## Map-shaped CBOR encoders/decoders for named Nim tuple aliases used +## across the FFI boundary. +## +## ## Why this exists +## +## `cbor_serialization` 0.3.0 emits Nim tuples as positional CBOR arrays +## (`writer.nim:423` — `proc write*[T: tuple]`) and decodes them +## symmetrically (`reader_impl.nim:144` — `proc read*[T: tuple]`). +## Wrapper-side codegen (Cpp / Py / Rust / Go) emits a tuple alias as +## a struct with NAMED fields and expects a CBOR map keyed by those +## names. Without alignment, a wrapper round-trip of `seq[TupleRow]` +## fails with "invalid type: sequence, expected map". +## +## This module provides a macro `bindCborTupleMap(T)` that emits a +## per-tuple `write` / `read` overload bound to the `BrokerCbor` flavor. +## The overloads encode/consume a CBOR map keyed by the Nim field +## names. Resolver code calls the macro for every named tuple alias +## that's auto-registered as part of the FFI surface. +## +## ## Limitation +## +## Only NAMED tuple aliases are supported (e.g. +## `type TupleRow = tuple[key: string, payload: string]`). Unnamed +## positional tuples (`tuple[int32, string]`) keep the library default +## (positional CBOR array) — wrappers receive synthesised field names +## (`first`, `second`, ...) on the struct side which would not match +## a positional CBOR shape; the tuple-as-struct codegen rejects > 9 +## positional elements anyway, so no wrapper currently emits structs +## from unnamed tuples. + +{.push raises: [].} + +import std/macros +import cbor_serialization +import cbor_serialization/[reader_impl, writer] + +import ./api_cbor_codec + +export api_cbor_codec + +macro bindCborTupleMap*(T: typed): untyped = + ## Emit `write` and `read` overloads for tuple type `T` that use the + ## CBOR map shape (field name → value) instead of the default + ## positional CBOR array. The overloads bind to `BrokerCbor.Writer` / + ## `BrokerCbor.Reader` so they take precedence over the generic + ## `write[T: tuple]` / `read[T: tuple]` from cbor_serialization. + let typeIdent = T + let writerSym = bindSym("CborWriter") + let readerSym = bindSym("CborReader") + let valueIdent = ident("value") + let writerIdent = ident("w") + let readerIdent = ident("r") + let keyIdent = ident("key") + + # Field names are extracted at proc body-instantiation time via + # `fieldPairs`, so the macro only needs to emit the proc skeletons — + # the proc body iterates the type's fields generically. + result = quote: + proc write*( + `writerIdent`: var `writerSym`, `valueIdent`: `typeIdent` + ) {.raises: [IOError].} = + var fieldsCount = 0 + for _, _ in fieldPairs(`valueIdent`): + inc fieldsCount + `writerIdent`.beginObject(fieldsCount) + for fieldName, fieldValue in fieldPairs(`valueIdent`): + `writerIdent`.writeField(fieldName, fieldValue) + `writerIdent`.endObject(stopCode = false) + + proc read*( + `readerIdent`: var `readerSym`, `valueIdent`: var `typeIdent` + ) {.raises: [SerializationError, IOError].} = + mixin readValue + `readerIdent`.parseObject(`keyIdent`): + var matched = false + for fieldName, fieldValue in fieldPairs(`valueIdent`): + if not matched and fieldName == `keyIdent`: + `readerIdent`.readValue(fieldValue) + matched = true + if not matched: + `readerIdent`.skipSingleValue() + +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_codegen_cbor_cddl.nim b/wasm-deps/brokers/brokers/internal/api_codegen_cbor_cddl.nim new file mode 100644 index 000000000..f617b78bf --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_codegen_cbor_cddl.nim @@ -0,0 +1,244 @@ +## CDDL emission for the CBOR FFI surface. +## +## Walks the per-library `CborRequestEntry` / `CborEventEntry` accumulators +## and the shared `gApiTypeRegistry` to produce a `.cddl` schema file +## next to the generated C/C++/Python wrappers. The schema is consumable by +## external CDDL tooling (`cddl validate`, `cuddle`, …) and is also embedded +## verbatim in the runtime discovery descriptor returned from +## `_getSchema`. +## +## CDDL mapping summary: +## bool -> bool +## int / intN -> int +## uint / uintN / byte -> uint +## float / floatN -> float +## string / cstring -> tstr +## seq[T] -> [* T-cddl] +## array[N, T] -> [N*N T-cddl] +## Option[T] -> T-cddl / null +## -> uint +## resolved underlying type +## -> rule reference (PascalCase name) +## +## The args type for a request is emitted inline as a synthetic +## `Args` rule. The response envelope shape is a single +## reusable rule `BrokerResultEnvelope` parameterised by inlining the +## payload type per request — CDDL has no generics, so we expand it. + +{.push raises: [].} + +import std/[macros, os, strutils] +import ./api_schema, ./api_common + +# --------------------------------------------------------------------------- +# Type-name helpers +# --------------------------------------------------------------------------- + +proc upperCamel*(s: string): string {.compileTime.} = + ## "device_updated" -> "DeviceUpdated"; "GetStatus" stays "GetStatus". + result = "" + var capNext = true + for ch in s: + if ch == '_' or ch == '-': + capNext = true + else: + if capNext: + result.add(ch.toUpperAscii()) + capNext = false + else: + result.add(ch) + +proc stripGenericPrefix(s: string, prefix: string): string {.compileTime.} = + ## Returns the inner of `prefix[...]`, e.g. `seq[int32]` -> `int32`. + ## Caller has already verified the prefix. + let inner = s[prefix.len + 1 .. ^2] + inner.strip() + +proc parseArrayParts(s: string): tuple[size: string, elem: string] {.compileTime.} = + ## Parse `array[N, T]` into (N, T). Returns ("", "") on malformed input. + if not s.toLowerAscii().startsWith("array["): + return ("", "") + let inner = s[6 .. ^2] + let comma = inner.find(',') + if comma < 0: + return ("", "") + (inner[0 ..< comma].strip(), inner[comma + 1 .. ^1].strip()) + +# --------------------------------------------------------------------------- +# Nim type -> CDDL fragment +# --------------------------------------------------------------------------- + +proc nimTypeToCddl*(nimType: string): string {.compileTime.} = + ## Maps a Nim type spelling to a CDDL fragment. Falls back to a rule + ## reference (the type name itself) for registered objects/enums; the + ## caller is responsible for emitting that rule elsewhere in the file. + let t = nimType.strip() + let lower = t.toLowerAscii() + + case lower + of "bool": + return "bool" + of "string", "cstring": + return "tstr" + of "char": + return "uint .size 1" + of "int", "int8", "int16", "int32", "int64": + return "int" + of "uint", "uint8", "uint16", "uint32", "uint64", "byte": + return "uint" + of "float", "float32", "float64": + return "float" + else: + discard + + if lower.startsWith("seq[") and lower.endsWith("]"): + return "[* " & nimTypeToCddl(stripGenericPrefix(t, "seq")) & "]" + + if lower.startsWith("option[") and lower.endsWith("]"): + return nimTypeToCddl(stripGenericPrefix(t, "option")) & " / null" + + if lower.startsWith("array["): + let (sz, elem) = parseArrayParts(t) + if sz.len > 0 and elem.len > 0: + return "[" & sz & "*" & sz & " " & nimTypeToCddl(elem) & "]" + + if isAliasOrDistinctRegistered(t): + return nimTypeToCddl(resolveUnderlyingType(t)) + + if isEnumRegistered(t): + return "uint" + + if isTypeRegistered(t): + return t + + # Unknown type — emit verbatim and let the CDDL consumer surface the + # missing rule. This preserves debuggability without aborting codegen + # for legitimate generic types we haven't taught the mapper about yet. + t + +# --------------------------------------------------------------------------- +# Type-rule emission +# --------------------------------------------------------------------------- + +proc emitObjectRule(entry: ApiTypeEntry): string {.compileTime.} = + result = entry.name & " = {\n" + for f in entry.fields: + result.add(" " & f.name & ": " & nimTypeToCddl(f.nimType) & ",\n") + result.add("}\n") + +proc emitEnumRule(entry: ApiTypeEntry): string {.compileTime.} = + result = "; enum " & entry.name & ":\n" + for v in entry.enumValues: + result.add("; " & v.name & " = " & $v.ordinal & "\n") + result.add(entry.name & " = uint\n") + +proc emitAliasRule(entry: ApiTypeEntry): string {.compileTime.} = + let kind = + case entry.kind + of atkAlias: "alias" + of atkDistinct: "distinct" + else: "alias" + result = "; " & kind & " of " & entry.underlyingType & "\n" + result.add(entry.name & " = " & nimTypeToCddl(entry.underlyingType) & "\n") + +proc emitTypeRule(entry: ApiTypeEntry): string {.compileTime.} = + case entry.kind + of atkObject: + emitObjectRule(entry) + of atkEnum: + emitEnumRule(entry) + of atkAlias, atkDistinct: + emitAliasRule(entry) + +# --------------------------------------------------------------------------- +# Args / envelope rule emission +# --------------------------------------------------------------------------- + +proc emitArgsRule( + ruleName: string, argFields: seq[(string, string)] +): string {.compileTime.} = + result = ruleName & " = {\n" + for (fname, ftype) in argFields: + result.add(" " & fname & ": " & nimTypeToCddl(ftype) & ",\n") + result.add("}\n") + +proc emitEnvelopeRule(ruleName: string, payloadCddl: string): string {.compileTime.} = + ## CBOR encoding produced by `omitOptionalFields = true`: a map with at + ## most one of `ok` / `err`, mutually exclusive. + result = ruleName & " = { ? ok: " & payloadCddl & ", ? err: tstr }\n" + +# --------------------------------------------------------------------------- +# File emission +# --------------------------------------------------------------------------- + +proc cddlPath(outDir, libName: string): string {.compileTime.} = + if outDir.len > 0: + outDir & "/" & libName & ".cddl" + else: + libName & ".cddl" + +proc generateCborCddl*( + libName: string, + requestEntries: seq[CborRequestEntry], + eventEntries: seq[CborEventEntry], + typeRegistry: seq[ApiTypeEntry], +): string {.compileTime.} = + ## Pure-string assembly so the same blob can be both written to disk and + ## embedded as a string literal in the generated runtime descriptor. + result = "; Generated by nim-brokers CBOR FFI codegen for '" & libName & "'.\n" + result.add("; Do not edit — regenerate by recompiling the library.\n\n") + + result.add("; ----- Shared types ----------------------------------------\n") + for entry in typeRegistry: + if entry.name.endsWith("CborArgs"): + # Synthetic args structs emitted per-request below. + continue + result.add(emitTypeRule(entry)) + result.add("\n") + + if requestEntries.len > 0: + result.add("; ----- Requests --------------------------------------------\n") + for r in requestEntries: + let argsRule = upperCamel(r.apiName) & "Args" + let respEnvRule = upperCamel(r.apiName) & "Response" + let payloadCddl = + if r.responseTypeName.len > 0: + nimTypeToCddl(r.responseTypeName) + else: + "{}" + + result.add("; apiName: \"" & r.apiName & "\"\n") + if r.argFields.len > 0: + result.add(emitArgsRule(argsRule, r.argFields)) + else: + result.add(argsRule & " = {}\n") + result.add(emitEnvelopeRule(respEnvRule, payloadCddl)) + result.add("\n") + + if eventEntries.len > 0: + result.add("; ----- Events ----------------------------------------------\n") + for e in eventEntries: + result.add("; eventName: \"" & e.apiName & "\"\n") + result.add( + upperCamel(e.apiName) & "Event = " & nimTypeToCddl(e.typeName) & "\n\n" + ) + +proc generateCborCddlFile*( + outDir: string, + libName: string, + requestEntries: seq[CborRequestEntry], + eventEntries: seq[CborEventEntry], + typeRegistry: seq[ApiTypeEntry], +): string {.compileTime, raises: [].} = + ## Writes `.cddl` and returns the file's contents so the caller + ## can embed the same string in the generated runtime discovery payload. + ensureGeneratedOutputDir(outDir) + let body = generateCborCddl(libName, requestEntries, eventEntries, typeRegistry) + let path = cddlPath(outDir, libName) + try: + writeFile(path, body) + except IOError: + error("Failed to write generated CDDL '" & path & "': " & getCurrentExceptionMsg()) + body + +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_codegen_cbor_go.nim b/wasm-deps/brokers/brokers/internal/api_codegen_cbor_go.nim new file mode 100644 index 000000000..49134b571 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_codegen_cbor_go.nim @@ -0,0 +1,865 @@ +## CBOR-mode Go wrapper code generation. +## +## Mirrors `api_codegen_cbor_rust.nim` but emits idiomatic Go with +## `(T, error)` returns. Uses `github.com/fxamacker/cbor/v2` for +## CBOR encoding/decoding (struct tags map Nim camelCase wire keys to +## Go-style PascalCase fields). +## +## Native and CBOR generations write to separate `` trees +## (`nimlib/build/` vs `nimlib/build_cbor/`), so each generated module +## directory contains exactly one wrapper. The filename is the same in +## both modes — `.go` and `_callbacks.c` — matching +## the C/C++/Rust convention where consumers pick build vs build_cbor +## via their build system, not via build-tag selection inside the +## module. + +{.push raises: [].} + +import std/[macros, strutils, tables] +import ./api_common, ./api_schema +import ./helper/broker_utils # reduced-A: per-interface partitioning + +# --------------------------------------------------------------------------- +# Nim → Go type mapping (registry-aware, used in CBOR mode) +# --------------------------------------------------------------------------- + +const goPrimMap = { + "bool": "bool", + "string": "string", + "char": "string", + "int": "int32", + "int8": "int8", + "int16": "int16", + "int32": "int32", + "int64": "int64", + "uint": "uint32", + "uint8": "uint8", + "uint16": "uint16", + "uint32": "uint32", + "uint64": "uint64", + "byte": "byte", + "float": "float64", + "float32": "float32", + "float64": "float64", +}.toTable + +proc isGoPrimitive(nimType: string): bool {.compileTime.} = + nimType.strip() in goPrimMap + +proc primGoHint(nimType: string): string {.compileTime.} = + goPrimMap.getOrDefault(nimType.strip(), "") + +proc unwrapBracket(s, head: string): string {.compileTime.} = + let t = s.strip() + t[head.len + 1 .. ^2].strip() + +proc parseArrayInner(s: string): string {.compileTime.} = + let inner = s.strip()[6 ..^ 2] + let comma = inner.find(',') + if comma < 0: + return "" + inner[comma + 1 .. ^1].strip() + +proc nimTypeToGoCborHint*(nimType: string): string {.compileTime.} = + ## Recursive Nim → Go type for CBOR mode. Returns "" when unmappable. + let t = nimType.strip() + let lower = t.toLowerAscii() + if isGoPrimitive(t): + return primGoHint(t) + if lower.startsWith("seq[") and lower.endsWith("]"): + let inner = nimTypeToGoCborHint(unwrapBracket(t, "seq")) + return + if inner.len > 0: + # Compact CBOR for seq[byte] uses a Go []byte (cbor lib auto-detects). + "[]" & inner + else: + "" + if lower.startsWith("array["): + let elem = parseArrayInner(t) + let inner = nimTypeToGoCborHint(elem) + return + if inner.len > 0: + "[]" & inner + else: + "" + if lower.startsWith("option[") and lower.endsWith("]"): + let inner = nimTypeToGoCborHint(unwrapBracket(t, "option")) + return + if inner.len > 0: + "*" & inner + else: + "" + if isTypeRegistered(t): + let entry = lookupTypeEntry(t) + case entry.kind + of atkObject, atkEnum: + return t + of atkAlias, atkDistinct: + # Recurse via outer mapper for distinct/alias-over-compound (e.g. + # `distinct seq[byte]` → `[]byte` rather than `""`). + return nimTypeToGoCborHint(resolveUnderlyingType(t)) + "" + +proc isGoCborMappable*(nimType: string): bool {.compileTime.} = + nimTypeToGoCborHint(nimType).len > 0 + +proc goExportedField*(name: string): string {.compileTime.} = + if name.len > 0 and name[0] >= 'a' and name[0] <= 'z': + chr(ord(name[0]) - 32) & name[1 ..^ 1] + else: + name + +const goReservedWords = [ + "break", "case", "chan", "const", "continue", "default", "defer", "else", + "fallthrough", "for", "func", "go", "goto", "if", "import", "interface", "map", + "package", "range", "return", "select", "struct", "switch", "type", "var", +] + +proc goSafeParam*(name: string): string {.compileTime.} = + ## Returns a Go-legal local identifier — appends `Arg` suffix when the + ## Nim parameter name collides with a Go reserved keyword (e.g. + ## `range` → `rangeArg`, `type` → `typeArg`). The CBOR wire field is + ## emitted from the original name, so wire compatibility is preserved. + if name in goReservedWords: + name & "Arg" + else: + name + +proc snakeToPascal(name: string): string {.compileTime.} = + ## Converts a snake_case identifier (CBOR apiName / event name) to + ## PascalCase for Go method exports. + result = "" + var capitalize = true + for ch in name: + if ch == '_' or ch == '-': + capitalize = true + elif capitalize: + result.add( + if ch >= 'a' and ch <= 'z': + chr(ord(ch) - 32) + else: + ch + ) + capitalize = false + else: + result.add(ch) + +proc goCborClassName(libName: string): string {.compileTime.} = + result = "" + var capitalize = true + for ch in libName: + if ch == '_' or ch == '-': + capitalize = true + elif capitalize: + result.add(chr(ord(ch) - 32 * ord(ch in {'a' .. 'z'}))) + capitalize = false + else: + result.add(ch) + +proc goCborPackageName(libName: string): string {.compileTime.} = + result = "" + for ch in libName: + if ch != '_' and ch != '-': + result.add( + if ch >= 'A' and ch <= 'Z': + chr(ord(ch) + 32) + else: + ch + ) + +# --------------------------------------------------------------------------- +# File emission +# --------------------------------------------------------------------------- + +{.pop.} + +proc goSubStructName(iface: string): string {.compileTime.} = + ## Wrapper struct name for a sub-interface: strip a leading `I` before an + ## uppercase letter (IWidget -> Widget), else use the name as-is. + if iface.len > 1 and iface[0] == 'I' and iface[1] in {'A' .. 'Z'}: + iface[1 ..^ 1] + else: + iface + +proc generateCborGoFile*( + outDir: string, + libName: string, + requestEntries: seq[CborRequestEntry], + eventEntries: seq[CborEventEntry], + mainClass: string = "", +) {.compileTime, raises: [].} = + ## Emits `/_go/{.go, _callbacks.c}`. + ## Same filenames as the native generator — only one wrapper exists per + ## build dir, so no build tags / no `_cbor` suffix. + ensureGeneratedOutputDir(outDir) + + # reduced-A: per-interface partition. Sub-interface names derived from the + # entries via interfaceOwningRequestType (NOT apiInterfaces() — the VM aliases + # a by-value seq return to an empty copy). + proc ownsReqMain(e: CborRequestEntry): bool {.compileTime.} = + if mainClass.len == 0: + return true + let o = interfaceOwningRequestType(e.responseTypeName) + o.len == 0 or o == mainClass + + proc ownsEvtMain(ev: CborEventEntry): bool {.compileTime.} = + if mainClass.len == 0: + return true + let o = interfaceOwningEventType(ev.typeName) + o.len == 0 or o == mainClass + + var subInterfaceNames: seq[string] = @[] + if mainClass.len > 0: + for e in requestEntries: + let o = interfaceOwningRequestType(e.responseTypeName) + if o.len > 0 and o != mainClass and o notin subInterfaceNames: + subInterfaceNames.add(o) + let modDir = + if outDir.len > 0: + outDir & "/" & libName & "_go" + else: + libName & "_go" + ensureGeneratedOutputDir(modDir) + + let pkgName = goCborPackageName(libName) + let className = goCborClassName(libName) + let p = libName & "_" + + # ---------------------- go.mod ---------------------- + # Always emit go.mod with the cbor dependency. (If a native-only build + # ran first and wrote go.mod without it, overwrite.) + var goMod = "// Generated by nim-brokers Go FFI codegen — do not edit.\n" + goMod.add("module " & libName & "\n\n") + goMod.add("go 1.21\n\n") + goMod.add("require github.com/fxamacker/cbor/v2 v2.7.0\n") + try: + writeFile(modDir & "/go.mod", goMod) + except IOError: + error("Failed to write go.mod: " & getCurrentExceptionMsg()) + + # ---------------------- .go ---------------------- + var g = "// Generated by nim-brokers CBOR FFI Go codegen — do not edit.\n" + g.add("//\n") + g.add( + "// CBOR-mode Go wrapper around the fixed 11-fn ABI declared by the `" & libName & + "` shared library.\n" + ) + g.add("//\n") + g.add("// Public surface mirrors the native build:\n") + g.add("// " & libName & ".Version()\n") + g.add("// " & libName & ".New() + lib.CreateContext()\n") + g.add("// (args) -> (T, error)\n") + g.add("// On(callback) -> uint64 / Off(handle uint64)\n") + g.add("//\n") + for e in requestEntries: + var sigParams = "" + for i, (n, t) in e.argFields.pairs: + if i > 0: + sigParams.add(", ") + let h = nimTypeToGoCborHint(t) + sigParams.add(goExportedField(n) & " " & (if h.len > 0: h else: "any")) + g.add( + "// " & snakeToPascal(e.apiName) & "(" & sigParams & ") (" & e.responseTypeName & + ", error)\n" + ) + for ev in eventEntries: + g.add("// On" & snakeToPascal(ev.apiName) & "(callback) uint64\n") + g.add("// Off" & snakeToPascal(ev.apiName) & "(handle uint64)\n") + g.add("\n") + + g.add("package " & pkgName & "\n\n") + + # cgo prelude + g.add("/*\n") + g.add("#cgo CFLAGS: -I${SRCDIR}/..\n") + g.add("#cgo LDFLAGS: -L${SRCDIR}/.. -l" & libName & "\n") + g.add("#cgo darwin LDFLAGS: -Wl,-rpath,${SRCDIR}/..\n") + g.add("#cgo linux LDFLAGS: -Wl,-rpath,${SRCDIR}/..\n") + g.add("#include \n") + g.add("#include \n") + g.add("#include \n") + g.add("#include \"" & libName & ".h\"\n") + g.add( + "uint64_t go_cbor_subscribe(uint32_t ctx, const char* name, void* user_data);\n" + ) + g.add("*/\n") + g.add("import \"C\"\n\n") + + g.add("import (\n") + g.add("\t\"errors\"\n") + g.add("\t\"runtime\"\n") + g.add("\t\"runtime/cgo\"\n") + g.add("\t\"sync\"\n") + g.add("\t\"unsafe\"\n") + g.add("\t\"github.com/fxamacker/cbor/v2\"\n") + g.add(")\n\n") + g.add("var _ = errors.New\n") + g.add("var _ = runtime.SetFinalizer\n") + g.add("var _ cgo.Handle\n") + g.add("var _ sync.Mutex\n") + g.add("var _ unsafe.Pointer\n") + g.add("var _ = cbor.Marshal\n\n") + + # Per-context cgo.Handle registry — same UAF-safe pattern as native: + # the closure stays alive across Off until Close() runs. + g.add("var cborHandleReg = struct {\n") + g.add("\tmu sync.Mutex\n") + g.add("\tperCtx map[uint32][]cgo.Handle\n") + g.add("}{perCtx: make(map[uint32][]cgo.Handle)}\n\n") + g.add("func registerCborHandle(ctx C.uint32_t, h cgo.Handle) {\n") + g.add("\tcborHandleReg.mu.Lock()\n") + g.add( + "\tcborHandleReg.perCtx[uint32(ctx)] = append(cborHandleReg.perCtx[uint32(ctx)], h)\n" + ) + g.add("\tcborHandleReg.mu.Unlock()\n") + g.add("}\n\n") + g.add("func dropCborHandlesForCtx(ctx C.uint32_t) {\n") + g.add("\tcborHandleReg.mu.Lock()\n") + g.add("\thandles := cborHandleReg.perCtx[uint32(ctx)]\n") + g.add("\tdelete(cborHandleReg.perCtx, uint32(ctx))\n") + g.add("\tcborHandleReg.mu.Unlock()\n") + g.add("\tfor _, h := range handles { h.Delete() }\n") + g.add("}\n\n") + + # ---- Generated payload types ------------------------------------------ + var enumNames: seq[string] = @[] + for entry in gApiTypeRegistry: + if entry.kind == atkEnum: + enumNames.add(entry.name) + var aliasNames: seq[string] = @[] + for entry in gApiTypeRegistry: + if entry.kind in {atkDistinct, atkAlias}: + aliasNames.add(entry.name) + var objectNames: seq[string] = @[] + for entry in gApiTypeRegistry: + if entry.kind == atkObject and not entry.name.endsWith("CborArgs"): + objectNames.add(entry.name) + + # A "scalar payload" is a primitive (non-object) broker type — `type X = + # int32` — registered as a distinct alias of its underlying primitive. + # Its CBOR wire value is a bare scalar; the Go surface uses the + # `type X = ` alias directly. Such a type has no object fields, so + # the event handler delivers the bare value rather than unpacked fields. + proc isScalarPayload(name: string): bool {.compileTime.} = + name.len > 0 and isTypeRegistered(name) and + lookupTypeEntry(name).kind in {atkAlias, atkDistinct} and + primGoHint(resolveUnderlyingType(name)).len > 0 + + if enumNames.len > 0 or aliasNames.len > 0 or objectNames.len > 0: + g.add("// -------- Generated payload types --------\n\n") + + for name in enumNames: + let entry = lookupTypeEntry(name) + g.add("type " & name & " int32\n\n") + g.add("const (\n") + if entry.enumValues.len == 0: + g.add("\t" & name & "_Unknown " & name & " = 0\n") + else: + for v in entry.enumValues: + g.add("\t" & name & "_" & v.name & " " & name & " = " & $v.ordinal & "\n") + g.add(")\n\n") + + for name in aliasNames: + let underlying = resolveUnderlyingType(name) + let goU = primGoHint(underlying) + if goU.len == 0: + g.add( + "// TODO: alias '" & name & "' resolves to '" & underlying & + "' (no Go primitive)\n\n" + ) + continue + g.add("type " & name & " = " & goU & "\n\n") + + for name in objectNames: + let entry = lookupTypeEntry(name) + g.add("type " & name & " struct {\n") + var anyField = false + for f in entry.fields: + let hint = nimTypeToGoCborHint(f.nimType) + if hint.len == 0: + g.add("\t// TODO: Nim type '" & f.nimType & "' not yet mappable\n") + continue + let fx = goExportedField(f.name) + g.add("\t" & fx & " " & hint & " `cbor:\"" & f.name & "\"`\n") + anyField = true + if not anyField: + g.add("\t_ struct{}\n") + g.add("}\n\n") + + # ---- Lib struct + event handler type ----------------------------------- + g.add("// -------- Event dispatch --------\n\n") + g.add("// cborEventHandler is what we anchor on the Go side via cgo.NewHandle.\n") + g.add("// Each subscription's user_data is the corresponding cgo.Handle, so\n") + g.add("// the trampoline retrieves and invokes exactly that one closure per\n") + g.add("// event emit — no global map, no fan-out, no cross-context leakage.\n") + g.add("type cborEventHandler func([]byte)\n\n") + + g.add("// -------- Lib struct --------\n\n") + g.add("type " & className & " struct {\n") + g.add("\tctx C.uint32_t\n") + g.add("\tmu sync.Mutex\n") + g.add("}\n\n") + + g.add("func Version() string {\n") + g.add("\treturn C.GoString(C." & p & "version())\n") + g.add("}\n\n") + + g.add("func New() *" & className & " {\n") + g.add("\tC." & p & "initialize()\n") + g.add("\tl := &" & className & "{}\n") + g.add("\truntime.SetFinalizer(l, func(x *" & className & ") { x.Close() })\n") + g.add("\treturn l\n") + g.add("}\n\n") + + g.add("func (l *" & className & ") CreateContext() error {\n") + g.add("\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n") + g.add("\tif l.ctx != 0 { return errors.New(\"context already created\") }\n") + g.add("\tvar errPtr *C.char\n") + g.add("\tctx := C." & p & "createContext(&errPtr)\n") + g.add("\tif ctx == 0 {\n") + g.add("\t\tmsg := \"createContext returned 0\"\n") + g.add( + "\t\tif errPtr != nil { msg = C.GoString(errPtr); C." & p & + "freeBuffer(unsafe.Pointer(errPtr)) }\n" + ) + g.add("\t\treturn errors.New(msg)\n") + g.add("\t}\n") + g.add("\tl.ctx = ctx\n") + g.add("\treturn nil\n") + g.add("}\n\n") + + g.add("func (l *" & className & ") ValidContext() bool { return l.ctx != 0 }\n") + g.add("func (l *" & className & ") Ctx() uint32 { return uint32(l.ctx) }\n\n") + + g.add("func (l *" & className & ") Close() {\n") + g.add("\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n") + g.add("\tif l.ctx != 0 {\n") + g.add("\t\tC." & p & "shutdown(l.ctx)\n") + g.add("\t\tdropCborHandlesForCtx(l.ctx)\n") + g.add("\t\tl.ctx = 0\n") + g.add("\t}\n") + g.add("}\n\n") + + # ---- Internal call helper ---------------------------------------------- + g.add("// internalCborCall encodes args via CBOR, copies into a library-\n") + g.add("// allocated buffer (the C ABI frees it), dispatches, and returns\n") + g.add("// the response bytes (caller-side copy of a library-owned buffer).\n") + g.add( + "func (l *" & className & + ") internalCborCall(apiName string, args interface{}) ([]byte, error) {\n" + ) + g.add( + "\tif l.ctx == 0 { return nil, errors.New(\"library context is not created\") }\n" + ) + g.add("\tvar inBytes []byte\n") + g.add("\tif args != nil {\n") + g.add("\t\tvar err error\n") + g.add("\t\tinBytes, err = cbor.Marshal(args)\n") + g.add("\t\tif err != nil { return nil, err }\n") + g.add("\t}\n") + g.add("\tcName := C.CString(apiName)\n") + g.add("\tdefer C.free(unsafe.Pointer(cName))\n") + g.add("\t// The library expects an `_allocBuffer`-allocated input\n") + g.add("\t// buffer that it can free. Copy the Go bytes into one.\n") + g.add("\tvar inPtr unsafe.Pointer\n") + g.add("\tif len(inBytes) > 0 {\n") + g.add("\t\tinPtr = C." & p & "allocBuffer(C.int32_t(len(inBytes)))\n") + g.add("\t\tif inPtr == nil { return nil, errors.New(\"allocBuffer failed\") }\n") + g.add("\t\tC.memcpy(inPtr, unsafe.Pointer(&inBytes[0]), C.size_t(len(inBytes)))\n") + g.add("\t}\n") + g.add("\tvar outBuf unsafe.Pointer\n") + g.add("\tvar outLen C.int32_t\n") + g.add( + "\trc := C." & p & + "call(l.ctx, cName, inPtr, C.int32_t(len(inBytes)), &outBuf, &outLen)\n" + ) + g.add("\tif rc != 0 {\n") + g.add("\t\tif outBuf != nil { C." & p & "freeBuffer(outBuf) }\n") + g.add("\t\treturn nil, errors.New(\"call returned non-zero\")\n") + g.add("\t}\n") + g.add("\tif outBuf == nil { return nil, nil }\n") + g.add("\tout := C.GoBytes(outBuf, C.int(outLen))\n") + g.add("\tC." & p & "freeBuffer(outBuf)\n") + g.add("\treturn out, nil\n") + g.add("}\n\n") + + # ---- Per-request methods ------------------------------------------------ + # Factored emitters reused by the main Lib and each sub-interface struct. + proc emitGoReqMethod(e: CborRequestEntry, recv: string): string {.compileTime.} = + let methodName = snakeToPascal(e.apiName) + let respType = e.responseTypeName + var argsStructFields = "" + var argsAssign = "" + var firstNonZero = false + for (n, t) in e.argFields: + let h = nimTypeToGoCborHint(t) + let hType = if h.len > 0: h else: "any" + let exN = goExportedField(n) + argsStructFields.add("\t\t" & exN & " " & hType & " `cbor:\"" & n & "\"`\n") + argsAssign.add("\t\t" & exN & ": " & goSafeParam(n) & ",\n") + firstNonZero = true + result.add("func (l *" & recv & ") " & methodName & "(") + var firstP = true + for (n, t) in e.argFields: + let h = nimTypeToGoCborHint(t) + let hType = if h.len > 0: h else: "any" + if not firstP: + result.add(", ") + result.add(goSafeParam(n) & " " & hType) + firstP = false + result.add(") (" & respType & ", error) {\n") + result.add("\tvar zeroResp " & respType & "\n") + if firstNonZero: + result.add("\targs := struct {\n") + result.add(argsStructFields) + result.add("\t}{\n") + result.add(argsAssign) + result.add("\t}\n") + result.add("\tout, err := l.internalCborCall(\"" & e.apiName & "\", args)\n") + else: + result.add("\tout, err := l.internalCborCall(\"" & e.apiName & "\", nil)\n") + result.add("\tif err != nil { return zeroResp, err }\n") + result.add("\tvar env struct {\n") + result.add("\t\tOk *" & respType & " `cbor:\"ok\"`\n") + result.add("\t\tErr *string `cbor:\"err\"`\n") + result.add("\t}\n") + result.add( + "\tif derr := cbor.Unmarshal(out, &env); derr != nil { return zeroResp, derr }\n" + ) + result.add("\tif env.Err != nil { return zeroResp, errors.New(*env.Err) }\n") + result.add("\tif env.Ok != nil { return *env.Ok, nil }\n") + result.add("\treturn zeroResp, errors.New(\"empty response envelope\")\n") + result.add("}\n\n") + + # reduced-A: a create-instance method returns the typed sub-wrapper. The wire + # ok value is a bare uint32 ctx; build &Sub{ctx} from it + a finalizer backstop. + proc emitGoInstanceMethod(e: CborRequestEntry, recv: string): string {.compileTime.} = + let methodName = snakeToPascal(e.apiName) + let sub = goSubStructName(e.returnsInterface) + var argsStructFields = "" + var argsAssign = "" + var firstNonZero = false + for (n, t) in e.argFields: + let h = nimTypeToGoCborHint(t) + let hType = if h.len > 0: h else: "any" + let exN = goExportedField(n) + argsStructFields.add("\t\t" & exN & " " & hType & " `cbor:\"" & n & "\"`\n") + argsAssign.add("\t\t" & exN & ": " & goSafeParam(n) & ",\n") + firstNonZero = true + result.add("func (l *" & recv & ") " & methodName & "(") + var firstP = true + for (n, t) in e.argFields: + let h = nimTypeToGoCborHint(t) + let hType = if h.len > 0: h else: "any" + if not firstP: + result.add(", ") + result.add(goSafeParam(n) & " " & hType) + firstP = false + result.add(") (*" & sub & ", error) {\n") + if firstNonZero: + result.add("\targs := struct {\n") + result.add(argsStructFields) + result.add("\t}{\n") + result.add(argsAssign) + result.add("\t}\n") + result.add("\tout, err := l.internalCborCall(\"" & e.apiName & "\", args)\n") + else: + result.add("\tout, err := l.internalCborCall(\"" & e.apiName & "\", nil)\n") + result.add("\tif err != nil { return nil, err }\n") + result.add("\tvar env struct {\n") + result.add("\t\tOk *uint32 `cbor:\"ok\"`\n") + result.add("\t\tErr *string `cbor:\"err\"`\n") + result.add("\t}\n") + result.add( + "\tif derr := cbor.Unmarshal(out, &env); derr != nil { return nil, derr }\n" + ) + result.add("\tif env.Err != nil { return nil, errors.New(*env.Err) }\n") + result.add( + "\tif env.Ok == nil { return nil, errors.New(\"empty response envelope\") }\n" + ) + result.add("\tw := &" & sub & "{ctx: C.uint32_t(*env.Ok)}\n") + result.add("\truntime.SetFinalizer(w, func(x *" & sub & ") { x.Close() })\n") + result.add("\treturn w, nil\n") + result.add("}\n\n") + + for e in requestEntries: + if not ownsReqMain(e): + continue + if e.returnsInterface.len > 0: + g.add(emitGoInstanceMethod(e, className)) + else: + g.add(emitGoReqMethod(e, className)) + + # ---- Single CBOR event trampoline + per-event On/Off --------------------- + if eventEntries.len > 0: + g.add("// -------- CBOR event trampoline --------\n\n") + g.add("//export goCborEventTrampoline\n") + g.add( + "func goCborEventTrampoline(ctx C.uint32_t, name *C.char, buf unsafe.Pointer, bufLen C.int32_t, ud unsafe.Pointer) {\n" + ) + g.add("\t_ = ctx\n") + g.add("\t_ = name\n") + g.add("\tif ud == nil { return }\n") + g.add("\tvar payload []byte\n") + g.add("\tif buf != nil && bufLen > 0 {\n") + g.add("\t\tpayload = C.GoBytes(buf, C.int(bufLen))\n") + g.add("\t}\n") + g.add("\th := cgo.Handle(uintptr(ud))\n") + g.add("\tcb, ok := h.Value().(cborEventHandler)\n") + g.add("\tif !ok { return }\n") + g.add("\tcb(payload)\n") + g.add("}\n\n") + + for ev in eventEntries: + if not ownsEvtMain(ev): + continue + let exName = snakeToPascal(ev.apiName) + let payloadType = ev.typeName + # Walk the payload struct fields to build an unpacked-field handler + # signature that matches the native build's `func(f1, f2, ...)` shape. + var fieldNames: seq[string] = @[] + var fieldGoTypes: seq[string] = @[] + var fieldExNames: seq[string] = @[] + var fieldsOk = true + let scalarEvt = isScalarPayload(payloadType) + if scalarEvt: + # Scalar payload: the decoded `p` IS the value — one bare arg. + fieldNames.add("value") + fieldGoTypes.add(primGoHint(resolveUnderlyingType(payloadType))) + fieldExNames.add("value") + elif isTypeRegistered(payloadType): + let entry = lookupTypeEntry(payloadType) + for f in entry.fields: + let h = nimTypeToGoCborHint(f.nimType) + if h.len == 0: + fieldsOk = false + break + fieldNames.add(f.name) + fieldGoTypes.add(h) + fieldExNames.add(goExportedField(f.name)) + else: + fieldsOk = false + + if not fieldsOk: + # Fall back to whole-struct callback if the payload has unmappable + # fields (no native equivalent — both modes share the same gap). + g.add( + "// TODO(go-codegen-cbor): event '" & payloadType & + "' has fields not yet mappable\n" + ) + g.add( + "func (l *" & className & ") On" & exName & "(cb func(" & payloadType & + ")) uint64 { _ = cb; return 0 }\n\n" + ) + else: + var sig = "" + for i in 0 ..< fieldNames.len: + if i > 0: + sig.add(", ") + sig.add(fieldNames[i] & " " & fieldGoTypes[i]) + g.add( + "func (l *" & className & ") On" & exName & "(cb func(" & sig & ")) uint64 {\n" + ) + g.add("\tif l.ctx == 0 { return 0 }\n") + g.add("\twrap := cborEventHandler(func(payload []byte) {\n") + g.add("\t\tvar p " & payloadType & "\n") + g.add("\t\tif derr := cbor.Unmarshal(payload, &p); derr != nil { return }\n") + g.add("\t\tcb(") + if scalarEvt: + # Scalar payload: `p` IS the value — pass it directly. + g.add("p") + else: + for i in 0 ..< fieldNames.len: + if i > 0: + g.add(", ") + g.add("p." & fieldExNames[i]) + g.add(")\n") + g.add("\t})\n") + g.add("\th := cgo.NewHandle(wrap)\n") + g.add("\tcName := C.CString(\"" & ev.apiName & "\")\n") + g.add("\tdefer C.free(unsafe.Pointer(cName))\n") + g.add( + "\thandle := uint64(C.go_cbor_subscribe(l.ctx, cName, unsafe.Pointer(h)))\n" + ) + g.add("\tif handle == 0 {\n") + g.add("\t\th.Delete()\n") + g.add("\t\treturn 0\n") + g.add("\t}\n") + g.add("\tregisterCborHandle(l.ctx, h)\n") + g.add("\treturn handle\n") + g.add("}\n\n") + + g.add("func (l *" & className & ") Off" & exName & "(handle uint64) {\n") + g.add("\tif l.ctx == 0 { return }\n") + g.add("\tcName := C.CString(\"" & ev.apiName & "\")\n") + g.add("\tdefer C.free(unsafe.Pointer(cName))\n") + g.add("\tC." & p & "unsubscribe(l.ctx, cName, C.uint64_t(handle))\n") + g.add("}\n\n") + + # reduced-A: sub-interface wrapper structs. Each shares the single C ABI: its + # methods call C._call(ctx, ...) which the library routes by classCtx to + # the same processing thread. Close() (+ finalizer backstop) calls + # C._releaseInstance, after which the Nim instance is GC-reclaimed. + for ifaceName in subInterfaceNames: + let sub = goSubStructName(ifaceName) + g.add( + "// -------- " & sub & " — sub-instance wrapper of " & ifaceName & + " --------\n\n" + ) + g.add("type " & sub & " struct {\n") + g.add("\tctx C.uint32_t\n") + g.add("\tmu sync.Mutex\n") + g.add("}\n\n") + g.add("func (w *" & sub & ") Ctx() uint32 { return uint32(w.ctx) }\n") + g.add("func (w *" & sub & ") Valid() bool { return w.ctx != 0 }\n\n") + g.add("func (w *" & sub & ") Close() {\n") + g.add("\tw.mu.Lock()\n\tdefer w.mu.Unlock()\n") + g.add("\tif w.ctx != 0 {\n") + g.add("\t\tC." & p & "releaseInstance(w.ctx)\n") + g.add("\t\tw.ctx = 0\n") + g.add("\t}\n") + g.add("}\n\n") + # internalCborCall (same shape as the Lib method, keyed by w.ctx). The + # receiver var is named `l` so the shared request-method emitter (which + # calls `l.internalCborCall`) works unchanged. + g.add( + "func (l *" & sub & + ") internalCborCall(apiName string, args interface{}) ([]byte, error) {\n" + ) + g.add("\tif l.ctx == 0 { return nil, errors.New(\"sub-instance is released\") }\n") + g.add("\tvar inBytes []byte\n") + g.add("\tif args != nil {\n") + g.add("\t\tvar err error\n") + g.add("\t\tinBytes, err = cbor.Marshal(args)\n") + g.add("\t\tif err != nil { return nil, err }\n") + g.add("\t}\n") + g.add("\tcName := C.CString(apiName)\n") + g.add("\tdefer C.free(unsafe.Pointer(cName))\n") + g.add("\tvar inPtr unsafe.Pointer\n") + g.add("\tif len(inBytes) > 0 {\n") + g.add("\t\tinPtr = C." & p & "allocBuffer(C.int32_t(len(inBytes)))\n") + g.add("\t\tif inPtr == nil { return nil, errors.New(\"allocBuffer failed\") }\n") + g.add("\t\tC.memcpy(inPtr, unsafe.Pointer(&inBytes[0]), C.size_t(len(inBytes)))\n") + g.add("\t}\n") + g.add("\tvar outBuf unsafe.Pointer\n") + g.add("\tvar outLen C.int32_t\n") + g.add( + "\trc := C." & p & + "call(l.ctx, cName, inPtr, C.int32_t(len(inBytes)), &outBuf, &outLen)\n" + ) + g.add("\tif rc != 0 {\n") + g.add("\t\tif outBuf != nil { C." & p & "freeBuffer(outBuf) }\n") + g.add("\t\treturn nil, errors.New(\"call returned non-zero\")\n") + g.add("\t}\n") + g.add("\tif outBuf == nil { return nil, nil }\n") + g.add("\tout := C.GoBytes(outBuf, C.int(outLen))\n") + g.add("\tC." & p & "freeBuffer(outBuf)\n") + g.add("\treturn out, nil\n") + g.add("}\n\n") + for e in requestEntries: + if interfaceOwningRequestType(e.responseTypeName) == ifaceName: + g.add(emitGoReqMethod(e, sub)) + # Sub-interface event methods (subscribe/unsubscribe keyed by l.ctx). + for ev in eventEntries: + if interfaceOwningEventType(ev.typeName) != ifaceName: + continue + let exName = snakeToPascal(ev.apiName) + let payloadType = ev.typeName + var fieldNames: seq[string] = @[] + var fieldGoTypes: seq[string] = @[] + var fieldExNames: seq[string] = @[] + var fieldsOk = true + let scalarEvt = isScalarPayload(payloadType) + if scalarEvt: + fieldNames.add("value") + fieldGoTypes.add(primGoHint(resolveUnderlyingType(payloadType))) + fieldExNames.add("value") + elif isTypeRegistered(payloadType): + let entry = lookupTypeEntry(payloadType) + for f in entry.fields: + let h = nimTypeToGoCborHint(f.nimType) + if h.len == 0: + fieldsOk = false + break + fieldNames.add(f.name) + fieldGoTypes.add(h) + fieldExNames.add(goExportedField(f.name)) + else: + fieldsOk = false + if not fieldsOk: + g.add( + "// TODO(go-codegen-cbor): event '" & payloadType & + "' has fields not yet mappable\n" + ) + g.add( + "func (l *" & sub & ") On" & exName & "(cb func(" & payloadType & + ")) uint64 { _ = cb; return 0 }\n\n" + ) + else: + var sig = "" + for i in 0 ..< fieldNames.len: + if i > 0: + sig.add(", ") + sig.add(fieldNames[i] & " " & fieldGoTypes[i]) + g.add("func (l *" & sub & ") On" & exName & "(cb func(" & sig & ")) uint64 {\n") + g.add("\tif l.ctx == 0 { return 0 }\n") + g.add("\twrap := cborEventHandler(func(payload []byte) {\n") + g.add("\t\tvar p " & payloadType & "\n") + g.add("\t\tif derr := cbor.Unmarshal(payload, &p); derr != nil { return }\n") + g.add("\t\tcb(") + if scalarEvt: + g.add("p") + else: + for i in 0 ..< fieldNames.len: + if i > 0: + g.add(", ") + g.add("p." & fieldExNames[i]) + g.add(")\n") + g.add("\t})\n") + g.add("\th := cgo.NewHandle(wrap)\n") + g.add("\tcName := C.CString(\"" & ev.apiName & "\")\n") + g.add("\tdefer C.free(unsafe.Pointer(cName))\n") + g.add( + "\thandle := uint64(C.go_cbor_subscribe(l.ctx, cName, unsafe.Pointer(h)))\n" + ) + g.add("\tif handle == 0 {\n") + g.add("\t\th.Delete()\n") + g.add("\t\treturn 0\n") + g.add("\t}\n") + g.add("\tregisterCborHandle(l.ctx, h)\n") + g.add("\treturn handle\n") + g.add("}\n\n") + g.add("func (l *" & sub & ") Off" & exName & "(handle uint64) {\n") + g.add("\tif l.ctx == 0 { return }\n") + g.add("\tcName := C.CString(\"" & ev.apiName & "\")\n") + g.add("\tdefer C.free(unsafe.Pointer(cName))\n") + g.add("\tC." & p & "unsubscribe(l.ctx, cName, C.uint64_t(handle))\n") + g.add("}\n\n") + + try: + writeFile(modDir & "/" & libName & ".go", g) + except IOError: + error("Failed to write CBOR Go file: " & getCurrentExceptionMsg()) + + # ---------------------- _callbacks.c ---------------------- + if eventEntries.len > 0: + var c = "// Generated by nim-brokers CBOR FFI Go codegen — do not edit.\n" + c.add("#include \n") + c.add("#include \n") + c.add("#include \"" & libName & ".h\"\n") + c.add("#include \"_cgo_export.h\"\n\n") + c.add( + "uint64_t go_cbor_subscribe(uint32_t ctx, const char* name, void* user_data) {\n" + ) + c.add( + " return " & p & "subscribe(ctx, name, (" & p & + "event_cb_t)goCborEventTrampoline, user_data);\n" + ) + c.add("}\n") + try: + writeFile(modDir & "/" & libName & "_callbacks.c", c) + except IOError: + error("Failed to write CBOR Go callbacks file: " & getCurrentExceptionMsg()) + +{.push raises: [].} +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_codegen_cbor_h.nim b/wasm-deps/brokers/brokers/internal/api_codegen_cbor_h.nim new file mode 100644 index 000000000..0dc9be644 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_codegen_cbor_h.nim @@ -0,0 +1,203 @@ +## Generated C header for the CBOR FFI surface. +## +## Unlike the native codegen path (which accumulates per-request structs +## into `gApiHeaderDeclarations`), the CBOR ABI is fixed: every library +## exposes the same eight functions plus one typedef. The only per-library +## variation is the symbol prefix and the documented sets of supported +## apiNames / eventNames, which we emit as comment blocks for human +## readers and language wrappers that aren't using the runtime discovery +## API. + +{.push raises: [].} + +import std/[macros, os, strutils] +import ./api_common + +# --------------------------------------------------------------------------- +# C header emission +# --------------------------------------------------------------------------- + +{.pop.} + +proc generateCborCHeaderFile*( + outDir: string, + libName: string, + version: string, + requestApiNames: seq[string], + eventApiNames: seq[string], +) {.compileTime, raises: [].} = + ## Writes the fixed-shape C header for a CBOR-mode library. + ensureGeneratedOutputDir(outDir) + + let guardName = libName.toUpperAscii().replace("-", "_") & "_H" + let headerPath = + if outDir.len > 0: + outDir & "/" & libName & ".h" + else: + libName & ".h" + let p = libName & "_" + + var h = "/* Generated by nim-brokers CBOR FFI codegen — do not edit. */\n" + h.add("#ifndef " & guardName & "\n") + h.add("#define " & guardName & "\n\n") + h.add("#include \n") + h.add("#include \n") + h.add("#include \n\n") + h.add("#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n") + + h.add("/* ----------------------------------------------------------------\n") + h.add(" * Library identity\n") + h.add(" * ---------------------------------------------------------------- */\n\n") + h.add( + "/* Returns a NUL-terminated semver string for this library build (\"" & version & + "\").\n" & " * The returned pointer is owned by the library — do NOT free. */\n" + ) + h.add("const char* " & p & "version(void);\n\n") + + h.add("/* ----------------------------------------------------------------\n") + h.add(" * Lifecycle\n") + h.add(" * ---------------------------------------------------------------- */\n\n") + h.add( + "/* Initialise the Nim runtime and per-thread foreign GC state.\n" & + " * Idempotent; safe to call from any thread before other entry points. */\n" + ) + h.add("void " & p & "initialize(void);\n\n") + h.add( + "/* Create a new context. Returns the context id (>0 on success), or\n" & + " * 0 on failure with *errOut populated by a Nim-allocated error\n" & + " * message that the caller MUST free with " & p & "freeBuffer. */\n" + ) + h.add("uint32_t " & p & "createContext(char** errOut);\n\n") + h.add( + "/* Tear down a context. Returns 0 on success, -1 if the context was\n" & + " * not found or already shut down. */\n" + ) + h.add("int32_t " & p & "shutdown(uint32_t ctx);\n\n") + h.add( + "/* reduced-A: release a sub-instance created by a create-instance request.\n" & + " * Drops that ctx's request providers + event listeners on the processing\n" & + " * thread; the Nim instance is then reclaimed by the GC. Idempotent and\n" & + " * safe on an unknown/already-released ctx. Returns 0 on success. */\n" + ) + h.add("int32_t " & p & "releaseInstance(uint32_t ctx);\n\n") + + h.add("/* ----------------------------------------------------------------\n") + h.add(" * Buffer ownership\n") + h.add(" *\n") + h.add(" * Every void* crossing this ABI is allocated by Nim and freed by\n") + h.add(" * Nim. Callers obtain inbound request buffers via " & p & "allocBuffer,\n") + h.add(" * fill them with CBOR, and pass them into " & p & "call (which frees\n") + h.add(" * them before returning). Outbound response and error buffers are\n") + h.add(" * allocated by the library and the caller frees them with\n") + h.add(" * " & p & "freeBuffer.\n") + h.add(" * ---------------------------------------------------------------- */\n\n") + h.add( + "/* Allocate a Nim-owned buffer of `size` bytes. Returns NULL on size\n" & + " * <= 0, size > 64 MiB, or allocation failure. */\n" + ) + h.add("void* " & p & "allocBuffer(int32_t size);\n\n") + h.add("/* Free a buffer previously returned by " & p & "allocBuffer or by an\n") + h.add(" * out-parameter from " & p & "call / " & p & "createContext. NULL is a\n") + h.add(" * no-op. */\n") + h.add("void " & p & "freeBuffer(void* buf);\n\n") + + h.add("/* ----------------------------------------------------------------\n") + h.add(" * Sync request gate\n") + h.add(" *\n") + h.add(" * Returns:\n") + h.add(" * 0 — success; *respBufOut holds the CBOR response envelope\n") + h.add(" * -1 — respBufOut or respLenOut is NULL\n") + h.add(" * -2 — apiName is NULL\n") + h.add(" * -3 — reqLen is negative or exceeds 64 MiB\n") + h.add(" * -4 — apiName is unknown; *respBufOut holds a UTF-8 message\n") + h.add(" * -10 — internal dispatch failure\n") + h.add(" * ---------------------------------------------------------------- */\n\n") + h.add( + "int32_t " & p & "call(uint32_t ctx,\n" & + " const char* apiName,\n" & + " const void* reqBuf, int32_t reqLen,\n" & + " void** respBufOut, int32_t* respLenOut);\n\n" + ) + + h.add("/* ----------------------------------------------------------------\n") + h.add(" * Event subscription\n") + h.add(" *\n") + h.add(" * Subscribe with cb == NULL probes whether the eventName is\n") + h.add(" * supported by this library version: returns 1 (sentinel) when\n") + h.add(" * supported, 0 when not. Real subscription handles are >= 2.\n") + h.add(" *\n") + h.add(" * Unsubscribe returns:\n") + h.add(" * 0 — success\n") + h.add(" * -1 — eventName is NULL\n") + h.add(" * -2 — no subscriptions registered for (ctx, eventName)\n") + h.add(" * -3 — handle not found in the subscription list\n") + h.add(" *\n") + h.add(" * Pass handle == 0 to remove every subscription for (ctx, eventName).\n") + h.add(" * ---------------------------------------------------------------- */\n\n") + h.add( + "typedef void (*" & p & "event_cb_t)(uint32_t ctx,\n" & + " const char* eventName,\n" & + " const void* payloadBuf,\n" & + " int32_t payloadLen,\n" & + " void* userData);\n\n" + ) + h.add( + "uint64_t " & p & "subscribe(uint32_t ctx,\n" & + " const char* eventName,\n" & + " " & p & "event_cb_t cb,\n" & + " void* userData);\n\n" + ) + h.add( + "int32_t " & p & "unsubscribe(uint32_t ctx,\n" & + " const char* eventName,\n" & + " uint64_t handle);\n\n" + ) + + h.add("/* ----------------------------------------------------------------\n") + h.add(" * Discovery API\n") + h.add(" *\n") + h.add(" * Both functions allocate the response with " & p & "allocBuffer; the\n") + h.add(" * caller frees it via " & p & "freeBuffer.\n") + h.add(" *\n") + h.add(" * " & p & "listApis returns a JSON-encoded ApiList string:\n") + h.add(" * {\"libName\": \"...\", \"requests\": [...], \"events\": [...]}\n") + h.add(" *\n") + h.add(" * " & p & "getSchema returns a JSON-encoded LibraryDescriptor string (full\n") + h.add(" * schema including the embedded CDDL text). See <" & libName & ".cddl>\n") + h.add(" * for the static schema.\n") + h.add(" *\n") + h.add(" * The response buffer is a UTF-8 JSON string (not null-terminated).\n") + h.add(" *\n") + h.add(" * Returns 0 on success, -1 if any out-pointer is NULL.\n") + h.add(" * ---------------------------------------------------------------- */\n\n") + h.add("int32_t " & p & "listApis(void** respBufOut, int32_t* respLenOut);\n\n") + h.add("int32_t " & p & "getSchema(void** respBufOut, int32_t* respLenOut);\n\n") + + if requestApiNames.len > 0 or eventApiNames.len > 0: + h.add("/* ----------------------------------------------------------------\n") + h.add(" * Documented apiNames\n") + h.add(" * ---------------------------------------------------------------- */\n\n") + if requestApiNames.len > 0: + h.add("/* Requests (pass these as `apiName` to " & p & "call):\n") + for n in requestApiNames: + h.add(" * \"" & n & "\"\n") + h.add(" */\n\n") + if eventApiNames.len > 0: + h.add("/* Events (pass these as `eventName` to " & p & "subscribe):\n") + for n in eventApiNames: + h.add(" * \"" & n & "\"\n") + h.add(" */\n\n") + + h.add("#ifdef __cplusplus\n}\n#endif\n\n") + h.add("#endif /* " & guardName & " */\n") + + try: + writeFile(headerPath, h) + except IOError: + error( + "Failed to write generated CBOR C header '" & headerPath & "': " & + getCurrentExceptionMsg() + ) + +{.push raises: [].} +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_codegen_cbor_hpp.nim b/wasm-deps/brokers/brokers/internal/api_codegen_cbor_hpp.nim new file mode 100644 index 000000000..2b647842b --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_codegen_cbor_hpp.nim @@ -0,0 +1,1662 @@ +## Generated C++ header-only wrapper for the CBOR FFI surface. +## +## The wrapper lives entirely in headers — no separate translation +## unit — so foreign C++ projects only need to link the Nim-built +## shared library and `#include` the generated `.hpp`. +## +## Phase 4c emits typed C++ structs + JSONCONS_ALL_MEMBER_TRAITS macros +## for each registered request response, request args, and event +## payload type, and per-request methods on the `Lib` class that +## CBOR-encode the args, dispatch through the C gate, and decode the +## response envelope into a `Result`. +## +## The wrapper requires: +## - C++20 (std::span, std::optional, structured bindings) +## - jsoncons + jsoncons_ext/cbor headers in the include path +## +## Currently supported field/parameter Nim types: bool, int/int8..int64, +## uint8..uint64, float32/float64, string. seq[T], array[N, T], Option[T], +## and nested objects are deferred to a later phase that exercises the +## full type-mapping matrix (Phase 7). + +{.push raises: [].} + +import std/[macros, os, strutils] +import ./api_common, ./api_schema +import ./helper/broker_utils # reduced-A: per-interface partitioning + +# --------------------------------------------------------------------------- +# Nim → C++ type mapping +# --------------------------------------------------------------------------- + +proc primCppType(nimType: string): string {.compileTime.} = + ## Direct primitive mapping. Empty string for non-primitives. + case nimType.strip() + of "bool": "bool" + of "string": "std::string" + of "int", "int64": "int64_t" + of "int8": "int8_t" + of "int16": "int16_t" + of "int32": "int32_t" + of "uint", "uint64": "uint64_t" + of "uint8", "byte": "uint8_t" + of "uint16": "uint16_t" + of "uint32": "uint32_t" + of "float32": "float" + of "float", "float64": "double" + of "char": "char" + else: "" + +proc unwrapBracket(s, head: string): string {.compileTime.} = + ## "seq[X]" + "seq" -> "X" + let t = s.strip() + t[head.len + 1 .. ^2].strip() + +proc parseArrayInner(s: string): string {.compileTime.} = + ## "array[N, T]" -> "T" + let inner = s.strip()[6 ..^ 2] + let comma = inner.find(',') + if comma < 0: + return "" + inner[comma + 1 .. ^1].strip() + +proc nimTypeToCppType*(nimType: string): string {.compileTime.} = + ## Recursive Nim → C++ type mapping. Returns "" for unmappable types + ## (callers emit a TODO and skip the affected typed surface). + let t = nimType.strip() + let lower = t.toLowerAscii() + let prim = primCppType(t) + if prim.len > 0: + return prim + if lower == "seq[byte]": + # `jsoncons::byte_string` is jsoncons' own byte-string container. It + # satisfies `is_basic_byte_string`, so jsoncons encodes/decodes it as + # a CBOR byte string (major type 2) — what the Nim cbor_serialization + # decoder expects for `seq[byte]`. A plain `std::vector` would + # ride the wire as a CBOR array (major type 4) and be rejected on + # INBOUND request params. `byte_string` is container-like (data/size/ + # begin/end/operator[]/push_back). + return "jsoncons::byte_string" + if lower.startsWith("seq[") and lower.endsWith("]"): + let inner = nimTypeToCppType(unwrapBracket(t, "seq")) + return + if inner.len > 0: + "std::vector<" & inner & ">" + else: + "" + if lower.startsWith("array["): + let elem = parseArrayInner(t) + let inner = nimTypeToCppType(elem) + # `std::vector` keeps the trait machinery uniform with seq[T] and + # matches what jsoncons decodes a CBOR array into. The Nim side + # range-checks the array length on decode. + return + if inner.len > 0: + "std::vector<" & inner & ">" + else: + "" + if lower.startsWith("option[") and lower.endsWith("]"): + let inner = nimTypeToCppType(unwrapBracket(t, "option")) + return + if inner.len > 0: + "std::optional<" & inner & ">" + else: + "" + if isTypeRegistered(t): + let entry = lookupTypeEntry(t) + case entry.kind + of atkObject: + return t + of atkEnum: + return t + of atkAlias, atkDistinct: + # Recurse through the outer mapper (not just `primCppType`) so an + # alias / distinct over a compound Nim type like `seq[byte]` maps to + # `std::vector` rather than falling through to "". + return nimTypeToCppType(resolveUnderlyingType(t)) + "" + +proc isCppMappable*(nimType: string): bool {.compileTime.} = + nimTypeToCppType(nimType).len > 0 + +# --------------------------------------------------------------------------- +# Identifier helpers +# --------------------------------------------------------------------------- + +proc snakeToLowerCamel*(s: string): string {.compileTime.} = + ## "get_status" -> "getStatus", "add_numbers" -> "addNumbers". + result = "" + var capitalize = false + for ch in s: + if ch == '_': + capitalize = true + elif capitalize: + result.add(toUpperAscii(ch)) + capitalize = false + else: + result.add(ch) + +# --------------------------------------------------------------------------- +# Event callback parameter mapping +# --------------------------------------------------------------------------- +# +# For event payload fields we deliver UNPACKED positional args to user +# callbacks (mirroring native FFI mode). The parameter type differs from +# the storage type for non-POD fields: +# string -> std::string_view (zero-copy view on decoded std::string) +# seq[T] -> std::span (view on decoded std::vector) +# array[N, T] -> std::span (CBOR decodes array into vector) +# nested obj -> const T& +# primitives -> by value +# +# `eventCallbackParamType` returns the C++ parameter type spelling. +# `eventCallbackArgExpr` returns the expression that pulls the arg out +# of the decoded payload struct (named `evt`). + +proc eventCallbackParamType*(nimType: string): string {.compileTime.} = + ## Returns the C++ parameter type for unpacked event callback args. + ## Empty string => the field's underlying Nim type isn't mappable. + let t = nimType.strip() + let lower = t.toLowerAscii() + let prim = primCppType(t) + if prim.len > 0: + if t == "string": + return "std::string_view" + return prim + if lower.startsWith("seq[") and lower.endsWith("]"): + let innerNim = unwrapBracket(t, "seq") + # seq[string] -> span (parity with native FFI; + # the trampoline materialises a temporary vector over + # the decoded vector). + if innerNim.strip() == "string": + return "std::span" + let inner = nimTypeToCppType(innerNim) + return + if inner.len > 0: + "std::span" + else: + "" + if lower.startsWith("array["): + let elem = parseArrayInner(t) + let inner = nimTypeToCppType(elem) + return + if inner.len > 0: + "std::span" + else: + "" + if lower.startsWith("option[") and lower.endsWith("]"): + let inner = nimTypeToCppType(unwrapBracket(t, "option")) + return + if inner.len > 0: + "std::optional<" & inner & ">" + else: + "" + if isTypeRegistered(t): + let entry = lookupTypeEntry(t) + case entry.kind + of atkObject: + return "const " & t & "&" + of atkEnum: + return t + of atkAlias, atkDistinct: + # Recurse through the outer mapper (not just `primCppType`) so an + # alias / distinct over a compound Nim type like `seq[byte]` maps to + # `std::vector` rather than falling through to "". + return nimTypeToCppType(resolveUnderlyingType(t)) + "" + +proc eventCallbackArgExpr*(fieldName, nimType: string): string {.compileTime.} = + ## Builds the expression that destructures `evt.` into the + ## callback param shape returned by `eventCallbackParamType`. + let t = nimType.strip() + let lower = t.toLowerAscii() + if t == "string": + return "std::string_view(evt." & fieldName & ")" + if lower.startsWith("seq[") and lower.endsWith("]"): + let innerNim = unwrapBracket(t, "seq") + # seq[string]: pass a span over the temporary vector + # built in the invoke preamble (see eventCallbackInvokeSetup). + if innerNim.strip() == "string": + return "std::span(" & fieldName & "_view)" + let inner = nimTypeToCppType(innerNim) + if inner.len > 0: + return "std::span(evt." & fieldName & ")" + return "evt." & fieldName + if lower.startsWith("array["): + let elem = parseArrayInner(t) + let inner = nimTypeToCppType(elem) + if inner.len > 0: + return "std::span(evt." & fieldName & ")" + return "evt." & fieldName + # Primitives, enums, distincts, options, nested objects: pass directly. + "evt." & fieldName + +proc eventCallbackInvokeSetup*(fieldName, nimType: string): string {.compileTime.} = + ## Returns setup statements emitted inside `invoke()` BEFORE the user + ## callback is called. Used to materialise non-owning views over the + ## decoded payload (e.g. seq[string] -> vector) so the + ## span the user receives stays valid for the call duration. + let t = nimType.strip() + let lower = t.toLowerAscii() + if lower.startsWith("seq[") and lower.endsWith("]") and + unwrapBracket(t, "seq").strip() == "string": + let viewVar = fieldName & "_view" + return + " std::vector " & viewVar & ";\n" & " " & viewVar & + ".reserve(evt." & fieldName & ".size());\n" & " for (const auto& s : evt." & + fieldName & ") " & viewVar & ".emplace_back(s);\n" + "" + +# --------------------------------------------------------------------------- +# Per-type emission +# --------------------------------------------------------------------------- + +proc emitCppStructFields*(h: var string, entry: ApiTypeEntry): bool {.compileTime.} = + ## Emit the body of a C++ struct mirroring `entry.fields`. Returns true + ## when every field was successfully mapped. False (with a TODO comment + ## emitted) means the wrapper should skip emitting this type's typed + ## method to avoid handing out a half-mapped surface. + result = true + for f in entry.fields: + let cppType = nimTypeToCppType(f.nimType) + if cppType.len == 0: + h.add(" // TODO: Nim type '" & f.nimType & "' not yet mappable to C++\n") + result = false + else: + h.add(" " & cppType & " " & f.name & "{};\n") + +proc emitMemberTraitsMacro*( + h: var string, qualifiedName: string, typeName: string, fieldNames: seq[string] +) {.compileTime.} = + ## Emit a JSONCONS member-traits macro for ``. + ## + ## When the registered struct contains any `Option[T]` field, switch + ## from the `_ALL_` flavour (every member required) to the `_N_` + ## flavour with `N = required.len`, listing required fields first + ## then optional ones. Without this split, decoding a payload where + ## an `Option` field is `none` (no key on the wire) fails with + ## `Key 'X' not found`. + ## + ## Per jsoncons docs, these macros generate partial specialisations + ## of `jsoncons::json_type_traits` and must be invoked at namespace + ## scope enclosing `jsoncons` — i.e. global scope, type fully + ## qualified. Empty structs are skipped; jsoncons handles those + ## implicitly when nested via Option fields. + if fieldNames.len == 0: + return + + var required: seq[string] = @[] + var optional: seq[string] = @[] + if isTypeRegistered(typeName): + let entry = lookupTypeEntry(typeName) + for n in fieldNames: + var isOption = false + for f in entry.fields: + if f.name == n: + if f.nimType.toLowerAscii().startsWith("option["): + isOption = true + break + if isOption: + optional.add(n) + else: + required.add(n) + else: + required = fieldNames + + if optional.len == 0: + h.add("JSONCONS_ALL_MEMBER_TRAITS(" & qualifiedName) + for n in required: + h.add(", " & n) + h.add(")\n") + else: + h.add("JSONCONS_N_MEMBER_TRAITS(" & qualifiedName & ", " & $required.len) + for n in required: + h.add(", " & n) + for n in optional: + h.add(", " & n) + h.add(")\n") + +proc emitEnvelopeTraits*(h: var string, qualifiedName: string) {.compileTime.} = + ## Emit the JSONCONS macro for `::Envelope`. Both fields + ## are optional on the wire (`omitOptionalFields = true` in the + ## BrokerCbor flavor), so the required-count is 0. + h.add("JSONCONS_N_MEMBER_TRAITS(" & qualifiedName & ", 0, ok, err)\n") + +# --------------------------------------------------------------------------- +# Header file emission +# --------------------------------------------------------------------------- + +{.pop.} + +proc cppSubClassName(iface: string): string {.compileTime.} = + ## Wrapper class name for a sub-interface: strip a leading `I` before an + ## uppercase letter (IWidget -> Widget), else use the name as-is. + if iface.len > 1 and iface[0] == 'I' and iface[1] in {'A' .. 'Z'}: + iface[1 ..^ 1] + else: + iface + +proc generateCborCppHeaderFile*( + outDir: string, + libName: string, + requestEntries: seq[CborRequestEntry], + eventEntries: seq[CborEventEntry], + mainClass: string = "", +) {.compileTime, raises: [].} = + ## Writes the C++ wrapper header (.hpp) for a CBOR-mode library. + ensureGeneratedOutputDir(outDir) + + # reduced-A: an entry belongs to the main class when no mainClass is + # designated (legacy single class), or it is flat, or its owning interface is + # the main class. Sub-interface names are derived from the entries directly + # (interfaceOwningRequestType), not apiInterfaces() — the compile-time VM + # aliases a by-value seq return to an empty copy. + proc ownsReqMain(e: CborRequestEntry): bool {.compileTime.} = + if mainClass.len == 0: + return true + let o = interfaceOwningRequestType(e.responseTypeName) + o.len == 0 or o == mainClass + + proc ownsEvtMain(ev: CborEventEntry): bool {.compileTime.} = + if mainClass.len == 0: + return true + let o = interfaceOwningEventType(ev.typeName) + o.len == 0 or o == mainClass + + var subInterfaceNames: seq[string] = @[] + var anyInstanceReturn = false + if mainClass.len > 0: + for e in requestEntries: + if e.returnsInterface.len > 0: + anyInstanceReturn = true + let o = interfaceOwningRequestType(e.responseTypeName) + if o.len > 0 and o != mainClass and o notin subInterfaceNames: + subInterfaceNames.add(o) + + let guardName = libName.toUpperAscii().replace("-", "_") & "_HPP" + let headerPath = + if outDir.len > 0: + outDir & "/" & libName & ".hpp" + else: + libName & ".hpp" + let p = libName & "_" + + # Derive C++ class name from libName the same way the native codegen + # does: snake_case / kebab-case → PascalCase. "mylib" -> "Mylib", + # "typemappingtestlib_cbor" -> "TypemappingtestlibCbor". Keeps the + # public C++ surface identical between native and CBOR builds so the + # same client `main.cpp` can compile against either. + var className = "" + var capitalize = true + for ch in libName: + if ch == '_' or ch == '-': + capitalize = true + elif capitalize: + className.add(chr(ord(ch) - 32 * ord(ch in {'a' .. 'z'}))) + capitalize = false + else: + className.add(ch) + + # Note: emittablePayloads is computed below from the actual struct + # emission, not from the response / event lookup, so we cover types + # transitively referenced from object fields (seq[Tag], etc.). + + var h = + "// Generated by nim-brokers CBOR FFI codegen — do not edit.\n" & "//\n" & + "// Header-only C++ wrapper around the C ABI declared in `" & libName & ".h`.\n" & + "// Requires C++20 and jsoncons + jsoncons_ext/cbor in the include path.\n" & + "#ifndef " & guardName & "\n" & "#define " & guardName & "\n\n" & "#include \"" & + libName & ".h\"\n\n" & "#include \n" & + "#include \n\n" & "#include \n" & + "#include \n" & "#include \n" & "#include \n" & + "#include \n" & "#include \n" & "#include \n" & + "#include \n" & "#include \n" & "#include \n" & + "#include \n\n" & "namespace " & libName & " {\n\n" + + # Result + h.add("template \n") + h.add("class Result {\n") + h.add(" std::optional value_;\n") + h.add(" std::string error_;\n") + h.add("public:\n") + h.add(" static Result ok(T value) {\n") + h.add(" Result r;\n") + h.add(" r.value_ = std::move(value);\n") + h.add(" return r;\n") + h.add(" }\n") + h.add(" static Result err(std::string message) {\n") + h.add(" Result r;\n") + h.add(" r.error_ = std::move(message);\n") + h.add(" return r;\n") + h.add(" }\n") + h.add(" bool isOk() const { return value_.has_value(); }\n") + h.add(" bool isErr() const { return !value_.has_value(); }\n") + h.add(" explicit operator bool() const { return isOk(); }\n") + h.add(" const T& value() const { return *value_; }\n") + h.add(" T& value() { return *value_; }\n") + h.add(" const T& operator*() const { return *value_; }\n") + h.add(" const T* operator->() const { return &*value_; }\n") + h.add(" T&& take() { return std::move(*value_); }\n") + h.add(" const std::string& error() const { return error_; }\n") + h.add("};\n\n") + h.add("template <>\n") + h.add("class Result {\n") + h.add(" bool ok_ = true;\n") + h.add(" std::string error_;\n") + h.add("public:\n") + h.add(" static Result ok() {\n") + h.add(" Result r;\n") + h.add(" r.ok_ = true;\n") + h.add(" return r;\n") + h.add(" }\n") + h.add(" static Result err(std::string message) {\n") + h.add(" Result r;\n") + h.add(" r.ok_ = false;\n") + h.add(" r.error_ = std::move(message);\n") + h.add(" return r;\n") + h.add(" }\n") + h.add(" Result() = default;\n") + h.add(" bool isOk() const { return ok_; }\n") + h.add(" bool isErr() const { return !ok_; }\n") + h.add(" explicit operator bool() const { return isOk(); }\n") + h.add(" const std::string& error() const { return error_; }\n") + h.add("};\n\n") + + # ---- All registered enums + distinct/alias aliases + structs ---- + # We walk gApiTypeRegistry directly so types referenced from fields + # (e.g. `seq[Tag]` inside an object) get emitted, not just the + # immediate request-response or event-payload types. + var enumNames: seq[string] = @[] + var aliasNames: seq[string] = @[] + var objectNames: seq[string] = @[] + for entry in gApiTypeRegistry: + if entry.name.endsWith("CborArgs"): + continue # synthetic args structs are emitted per-request below + case entry.kind + of atkEnum: + enumNames.add(entry.name) + of atkDistinct, atkAlias: + aliasNames.add(entry.name) + of atkObject: + objectNames.add(entry.name) + + # Enum class declarations — the underlying type is fixed at int32_t so + # the wire encoding (CBOR Unsigned, ordinal value) round-trips with + # the BrokerCbor flavor's `enumRep = EnumAsNumber`. + if enumNames.len > 0: + h.add("// ---- Enums ----\n\n") + for name in enumNames: + let entry = lookupTypeEntry(name) + h.add("enum class " & name & " : int32_t {\n") + if entry.enumValues.len == 0: + h.add("};\n\n") + continue + for v in entry.enumValues: + h.add(" " & v.name & " = " & $v.ordinal & ",\n") + h.add("};\n\n") + + # Distinct / alias — plain `using` aliases of the underlying primitive. + if aliasNames.len > 0: + h.add("// ---- Distinct / alias types ----\n\n") + for name in aliasNames: + let underlying = resolveUnderlyingType(name) + let prim = primCppType(underlying) + if prim.len == 0: + h.add( + "// TODO: alias '" & name & "' resolves to '" & underlying & + "' which has no C++ primitive mapping\n\n" + ) + continue + h.add("using " & name & " = " & prim & ";\n") + if aliasNames.len > 0: + h.add("\n") + + # Object structs — emit forward declarations first so cross-references + # (e.g. `std::vector` inside another struct) compile regardless + # of registry order. + if objectNames.len > 0: + h.add("// ---- Object payload structs ----\n\n") + for name in objectNames: + h.add("struct " & name & ";\n") + h.add("\n") + + # Captured (typeName, [fieldName...]) for global-scope JSONCONS macros. + var payloadFields: seq[(string, seq[string])] = @[] + for name in objectNames: + let entry = lookupTypeEntry(name) + h.add("struct " & name & " {\n") + var allMapped = true + for f in entry.fields: + let cppType = nimTypeToCppType(f.nimType) + if cppType.len == 0: + h.add(" // TODO: Nim type '" & f.nimType & "' not yet mappable\n") + allMapped = false + else: + h.add(" " & cppType & " " & f.name & "{};\n") + h.add("};\n") + if allMapped: + var fieldNames: seq[string] = @[] + for f in entry.fields: + fieldNames.add(f.name) + payloadFields.add((name, fieldNames)) + h.add("\n") + + # Names of object types we successfully emitted with full field + # coverage — used to gate request/event method emission below. + var emittablePayloads: seq[string] = @[] + for (name, _) in payloadFields: + emittablePayloads.add(name) + + # A "scalar payload" is a primitive (non-object) broker type — `type X = + # int32` — registered as a distinct alias of its underlying primitive. + # The CBOR wire value is a bare scalar; the C++ surface uses the `using X + # = ` alias directly (no struct). Such a type is an emittable + # request response / event payload even though it has no object fields. + proc isScalarPayload(name: string): bool {.compileTime.} = + name.len > 0 and isTypeRegistered(name) and + lookupTypeEntry(name).kind in {atkAlias, atkDistinct} and + primCppType(resolveUnderlyingType(name)).len > 0 + + # A "void payload" is a zero-field broker type — `type X = void` (lowered + # to an empty object). It has no value; the request envelope carries only + # the ok/err signal and the event callback no payload. jsoncons cannot + # (de)serialise a bare empty struct, so the payload slot uses the generic + # `jsoncons::json` (which round-trips the empty `{}` map) and the request + # method surfaces as `Result`. + proc isVoidPayload(name: string): bool {.compileTime.} = + name.len > 0 and isTypeRegistered(name) and lookupTypeEntry(name).kind == atkObject and + lookupTypeEntry(name).fields.len == 0 + + proc isEmittablePayload(name: string): bool {.compileTime.} = + name in emittablePayloads or isScalarPayload(name) + + # The C++ type used in the request/event payload slot: `void` surfaces a + # `Result`, scalar/object payloads use their own type. + proc payloadCppType(name: string): string {.compileTime.} = + if isVoidPayload(name): "void" else: name + + # Effective callback/struct fields for a payload type: an object's real + # fields, or a single synthetic `value` field for a scalar payload. + proc effectiveFields(name: string): seq[ApiFieldDef] {.compileTime.} = + if isScalarPayload(name): + return @[ApiFieldDef(name: "value", nimType: resolveUnderlyingType(name))] + lookupTypeEntry(name).fields + + # ---- Compute envelope / args metadata (no emission yet) ---- + var envelopeNames: seq[string] = @[] + for e in requestEntries: + if e.responseTypeName.len == 0: + continue + if not isEmittablePayload(e.responseTypeName): + continue + let envName = e.responseTypeName & "Envelope" + if envName notin envelopeNames: + envelopeNames.add(envName) + + var argsStructByApi: seq[(string, string)] = @[] # (apiName, argsStructName) + var argsMethodSupported: seq[(string, bool)] = @[] # (apiName, allMapped) + var argsFields: seq[(string, seq[string])] = @[] # (typeName, [fieldName]) + for e in requestEntries: + if e.argFields.len == 0: + argsMethodSupported.add((e.apiName, true)) + continue + var allMapped = true + for (n, t) in e.argFields: + if not isCppMappable(t): + allMapped = false + break + argsMethodSupported.add((e.apiName, allMapped)) + if not allMapped: + continue + let camelBase = snakeToLowerCamel(e.apiName) + var argsName = camelBase & "Args" + if argsName.len > 0: + argsName[0] = toUpperAscii(argsName[0]) + argsStructByApi.add((e.apiName, argsName)) + var fieldNames: seq[string] = @[] + for (n, t) in e.argFields: + fieldNames.add(n) + argsFields.add((argsName, fieldNames)) + + # Lookup helpers. + proc argsStructName(apiName: string): string {.compileTime.} = + for (n, s) in argsStructByApi: + if n == apiName: + return s + "" + + proc isMethodSupported(apiName: string): bool {.compileTime.} = + for (n, ok) in argsMethodSupported: + if n == apiName: + return ok + false + + # Pre-compute event eligibility: an event is "emittable" iff its payload + # struct was fully mapped AND every field has an unpacked-callback param + # mapping (string -> string_view, seq -> span, etc.). + var emittableEvents: seq[CborEventEntry] = @[] + for ev in eventEntries: + if ev.typeName.len == 0 or not isEmittablePayload(ev.typeName): + continue + var allOk = true + for f in effectiveFields(ev.typeName): + if eventCallbackParamType(f.nimType).len == 0: + allOk = false + break + if allOk: + emittableEvents.add(ev) + + # reduced-A: events owned by the main interface (the only ones the main Lib + # class carries dispatchers/methods for). Traits/forward-decls stay full set. + var mainEvents: seq[CborEventEntry] = @[] + for ev in emittableEvents: + if ownsEvtMain(ev): + mainEvents.add(ev) + + # Emittable events owned by a given sub-interface, and whether it has any. + # An event-bearing sub-wrapper carries EventDispatchers (which hold + # `owner_ = this`), so it must be NON-MOVABLE and is therefore created on the + # heap and returned as `Result>`. An event-free sub + # (e.g. a request-only IWidget) stays movable and is returned by value. + proc subEventsOf(ifaceName: string): seq[CborEventEntry] {.compileTime.} = + for ev in emittableEvents: + if interfaceOwningEventType(ev.typeName) == ifaceName: + result.add(ev) + + proc subHasEvents(ifaceName: string): bool {.compileTime.} = + subEventsOf(ifaceName).len > 0 + + # reduced-A: forward-declare each sub-interface wrapper class so the main + # class can name `Result` as a create-instance method return type (a + # non-defining declaration does not instantiate Result, so a forward + # declaration suffices; the full Sub class is emitted after detail::). + for ifaceName in subInterfaceNames: + h.add("class " & cppSubClassName(ifaceName) & ";\n") + if subInterfaceNames.len > 0: + h.add("\n") + # Shared envelope for create-instance responses (wire ok = uint32 ctx). + if anyInstanceReturn: + h.add("struct __InstanceCtxEnvelope {\n") + h.add(" std::optional ok;\n") + h.add(" std::optional err;\n") + h.add("};\n\n") + + # ================================================================== + # Section 0.5: detail:: forward declarations (so Lib can name them) + # ================================================================== + h.add("namespace detail {\n") + h.add("template \n") + h.add("class EventDispatcher;\n\n") + for ev in emittableEvents: + h.add("struct " & ev.typeName & "EventTraits;\n") + h.add("} // namespace detail\n\n") + + # ================================================================== + # Section 1: Lib class — declarations only (no detail:: dependencies) + # ================================================================== + h.add("class " & className & " {\n") + h.add(" public:\n") + h.add(" " & className & "();\n") + h.add(" ~" & className & "();\n") + h.add(" " & className & "(const " & className & "&) = delete;\n") + h.add(" " & className & "& operator=(const " & className & "&) = delete;\n") + h.add(" " & className & "(" & className & "&&) = delete;\n") + h.add(" " & className & "& operator=(" & className & "&&) = delete;\n\n") + h.add(" static std::string_view version() noexcept;\n\n") + h.add(" Result createContext();\n") + h.add(" bool validContext() const noexcept;\n") + h.add(" explicit operator bool() const noexcept;\n") + h.add(" void shutdown() noexcept;\n") + h.add(" uint32_t ctx() const noexcept;\n\n") + + # Per-request method declarations (main interface only). + for e in requestEntries: + if e.responseTypeName.len == 0: + continue + if not ownsReqMain(e): + continue + let methodName = snakeToLowerCamel(e.apiName) + var sigParams = "" + if e.argFields.len > 0: + var first = true + for (n, t) in e.argFields: + if not first: + sigParams.add(", ") + sigParams.add(nimTypeToCppType(t) & " " & n) + first = false + if e.returnsInterface.len > 0: + # reduced-A: create-instance method returns the typed sub-wrapper — + # by-value if event-free, or unique_ptr if the sub carries events + # (non-movable, heap-stable for its EventDispatchers). + let subN = cppSubClassName(e.returnsInterface) + let retT = + if subHasEvents(e.returnsInterface): + "Result>" + else: + "Result<" & subN & ">" + h.add(" " & retT & " " & methodName & "(" & sigParams & ");\n") + continue + if not isEmittablePayload(e.responseTypeName): + h.add( + " // TODO: '" & e.apiName & "' return type '" & e.responseTypeName & + "' is not yet emitted as a typed C++ struct.\n" + ) + continue + if not isMethodSupported(e.apiName): + h.add( + " // TODO: '" & e.apiName & + "' has parameters whose Nim types aren't yet mappable to C++.\n" + ) + continue + h.add( + " Result<" & payloadCppType(e.responseTypeName) & "> " & methodName & "(" & + sigParams & ");\n" + ) + h.add("\n") + + # Per-event Callback aliases + on/off declarations. The public alias is + # emitted as a fully-spelled `std::function<...>` (mirrors native FFI) + # so that Lib's public surface does NOT depend on the detail::Traits + # struct being complete at this point — only forward-declared. The + # EventDispatcher's internal `Callback` typedef (resolved from Traits + # later) produces the same std::function<> instantiation, so the types + # are interchangeable at call sites. + for ev in eventEntries: + if not ownsEvtMain(ev): + continue # sub-interface events are not in scope for this slice + if ev notin emittableEvents: + h.add( + " // TODO: event '" & ev.apiName & "' payload type '" & ev.typeName & + "' is not yet emitted as a typed C++ struct.\n" + ) + continue + let camelBase = snakeToLowerCamel(ev.apiName) + var pascal = camelBase + if pascal.len > 0: + pascal[0] = toUpperAscii(pascal[0]) + let callbackAlias = ev.typeName & "Callback" + let onName = "on" & pascal + let offName = "off" & pascal + h.add(" using " & callbackAlias & " = std::function;\n") + h.add(" uint64_t " & onName & "(" & callbackAlias & " fn) noexcept;\n") + h.add(" void " & offName & "(uint64_t handle = 0) noexcept;\n\n") + + # Discovery declarations. + h.add(" std::string listApis();\n") + h.add(" std::string getSchema();\n\n") + + # Private section: dispatcher members + ctx. + # `lastError` is intentionally a per-call local in each generated method — + # NOT a shared instance member. Sharing it across calls was a data race + # (concurrent `addRequest`s on one `Lib` instance both write the same + # `std::string`, leading to a double-free on the SSO/heap buffer — + # caught by ASAN under stress_shutdown). + h.add(" private:\n") + h.add(" uint32_t ctx_ = 0;\n") + for ev in mainEvents: + let dispatcherType = ev.typeName & "Dispatcher" + let dispatcherMember = ev.apiName & "Dispatcher_" + h.add( + " using " & dispatcherType & " = detail::EventDispatcher<" & className & + ", detail::" & ev.typeName & "EventTraits>;\n" + ) + h.add(" std::unique_ptr<" & dispatcherType & "> " & dispatcherMember & ";\n") + h.add("};\n\n") + + # ================================================================== + # Sub-interface request method helpers (compile-time procs). + # emitSubReqDecl → in-class signature (declaration only) + # emitSubReqImpl → out-of-class inline definition (for detail:: namespace) + # ================================================================== + proc subReqSigParams(e: CborRequestEntry): string {.compileTime.} = + if e.argFields.len > 0: + var first = true + for (n, t) in e.argFields: + if not first: + result.add(", ") + result.add(nimTypeToCppType(t) & " " & n) + first = false + + proc subReqRetType(e: CborRequestEntry): string {.compileTime.} = + "Result<" & payloadCppType(e.responseTypeName) & ">" + + proc emitSubReqDecl(e: CborRequestEntry): string {.compileTime.} = + if e.responseTypeName.len == 0: + return "" + if e.returnsInterface.len > 0: + return " // TODO: nested create-instance from a sub-interface unsupported.\n" + if not isEmittablePayload(e.responseTypeName): + return + " // TODO: '" & e.apiName & "' return type '" & e.responseTypeName & + "' not emittable.\n" + if not isMethodSupported(e.apiName): + return " // TODO: '" & e.apiName & "' has unmappable parameter types.\n" + let methodName = snakeToLowerCamel(e.apiName) + result.add( + " " & subReqRetType(e) & " " & methodName & "(" & subReqSigParams(e) & ");\n" + ) + + proc emitSubReqImpl(e: CborRequestEntry, sub: string): string {.compileTime.} = + if e.responseTypeName.len == 0: + return "" + if e.returnsInterface.len > 0 or not isEmittablePayload(e.responseTypeName) or + not isMethodSupported(e.apiName): + return "" + let methodName = snakeToLowerCamel(e.apiName) + let envName = e.responseTypeName & "Envelope" + let voidResp = isVoidPayload(e.responseTypeName) + let resTy = subReqRetType(e) + let okExpr = + if voidResp: + resTy & "::ok()" + else: + resTy & "::ok(std::move(*env.ok))" + let argsName = argsStructName(e.apiName) + result.add( + "inline " & resTy & " " & sub & "::" & methodName & "(" & subReqSigParams(e) & + ") {\n" + ) + if e.argFields.len > 0: + var argsAssign = "" + for (n, t) in e.argFields: + argsAssign.add(" args." & n & " = " & n & ";\n") + result.add(" " & argsName & " args;\n") + result.add(argsAssign) + result.add(" std::size_t cborLen = 0;\n") + result.add(" try { cborLen = detail::cborEncodedSize(args); }\n") + result.add(" catch (const std::exception& ex) {\n") + result.add( + " return " & resTy & + "::err(std::string(\"size pass failed: \") + ex.what());\n" + ) + result.add(" }\n") + result.add( + " void* inBuf = (cborLen > 0) ? " & p & + "allocBuffer(static_cast(cborLen)) : nullptr;\n" + ) + result.add( + " if (cborLen > 0 && !inBuf) return " & resTy & + "::err(\"allocBuffer failed\");\n" + ) + result.add(" try {\n") + result.add( + " if (cborLen > 0) detail::cborEncodeInto(args, static_cast(inBuf), cborLen);\n" + ) + result.add(" } catch (const std::exception& ex) {\n") + result.add(" if (inBuf) { " & p & "freeBuffer(inBuf); }\n") + result.add( + " return " & resTy & + "::err(std::string(\"encode pass failed: \") + ex.what());\n" + ) + result.add(" }\n") + result.add(" std::string lastError;\n") + result.add( + " auto [status, resp] = detail::rawCallOwned(ctx_, lastError, \"" & e.apiName & + "\", inBuf, cborLen);\n" + ) + else: + result.add(" std::string lastError;\n") + result.add( + " auto [status, resp] = detail::rawCallOwned(ctx_, lastError, \"" & e.apiName & + "\", nullptr, 0);\n" + ) + result.add(" if (status != 0) return " & resTy & "::err(lastError);\n") + result.add(" if (resp.empty()) return " & resTy & "::err(\"empty response\");\n") + result.add(" " & envName & " env;\n") + result.add(" try {\n") + result.add(" auto v = resp.view();\n") + result.add( + " env = jsoncons::cbor::decode_cbor<" & envName & ">(v.begin(), v.end());\n" + ) + result.add(" } catch (const std::exception& ex) {\n") + result.add( + " return " & resTy & "::err(std::string(\"decode failed: \") + ex.what());\n" + ) + result.add(" }\n") + result.add(" if (env.err.has_value()) return " & resTy & "::err(*env.err);\n") + result.add(" if (env.ok.has_value()) return " & okExpr & ";\n") + result.add(" return " & resTy & "::err(\"malformed response envelope\");\n") + result.add("}\n\n") + + # ================================================================== + # Section 2: Sub-interface class declarations (signatures only). + # Emitted in the library namespace right after the main Lib class so + # they live under , ahead of namespace detail. + # Method implementations are emitted later inside namespace detail. + # ================================================================== + for ifaceName in subInterfaceNames: + let sub = cppSubClassName(ifaceName) + let subEvts = subEventsOf(ifaceName) + let hasEvts = subEvts.len > 0 + h.add("// ---- " & sub & " — sub-instance wrapper of " & ifaceName & " ----\n") + h.add("class " & sub & " {\n") + h.add(" private:\n") + h.add(" uint32_t ctx_ = 0;\n") + for ev in subEvts: + let dispType = ev.typeName & "Dispatcher" + let dispMember = ev.apiName & "Dispatcher_" + h.add( + " using " & dispType & " = detail::EventDispatcher<" & sub & ", detail::" & + ev.typeName & "EventTraits>;\n" + ) + h.add(" std::unique_ptr<" & dispType & "> " & dispMember & ";\n") + h.add("\n public:\n") + # Ctor/dtor/copy/move — declared here, defined out-of-class in detail::. + h.add(" explicit " & sub & "(uint32_t ctx);\n") + h.add(" ~" & sub & "();\n") + h.add(" " & sub & "(const " & sub & "&) = delete;\n") + h.add(" " & sub & "& operator=(const " & sub & "&) = delete;\n") + if hasEvts: + h.add(" " & sub & "(" & sub & "&&) = delete;\n") + h.add(" " & sub & "& operator=(" & sub & "&&) = delete;\n") + else: + h.add(" " & sub & "(" & sub & "&& o) noexcept;\n") + h.add(" " & sub & "& operator=(" & sub & "&& o) noexcept;\n") + h.add(" uint32_t ctx() const noexcept { return ctx_; }\n") + h.add(" bool valid() const noexcept { return ctx_ != 0; }\n") + h.add(" explicit operator bool() const noexcept { return ctx_ != 0; }\n") + h.add(" void close() noexcept;\n") + # Request method declarations. + for e in requestEntries: + if interfaceOwningRequestType(e.responseTypeName) == ifaceName: + h.add(emitSubReqDecl(e)) + # Event callback aliases + on/off declarations. + for ev in subEvts: + let camelBase = snakeToLowerCamel(ev.apiName) + var pascal = camelBase + if pascal.len > 0: + pascal[0] = toUpperAscii(pascal[0]) + let cbAlias = ev.typeName & "Callback" + h.add(" using " & cbAlias & " = std::function;\n") + h.add(" uint64_t on" & pascal & "(" & cbAlias & " fn) noexcept;\n") + h.add(" void off" & pascal & "(uint64_t handle = 0) noexcept;\n") + h.add("};\n\n") + + # ================================================================== + # Section 3: Envelope + args structs (internal plumbing, after class) + # ================================================================== + var emittedEnvelopes: seq[string] = @[] + for e in requestEntries: + if e.responseTypeName.len == 0: + continue + if not isEmittablePayload(e.responseTypeName): + continue + if e.responseTypeName in emittedEnvelopes: + continue + emittedEnvelopes.add(e.responseTypeName) + let envName = e.responseTypeName & "Envelope" + # A `void` payload has no struct jsoncons can (de)serialise — the `ok` + # slot holds the generic `jsoncons::json` so the empty `{}` map sent on + # the wire still round-trips and `has_value()` reports success. + let okType = + if isVoidPayload(e.responseTypeName): "jsoncons::json" else: e.responseTypeName + h.add("struct " & envName & " {\n") + h.add(" std::optional<" & okType & "> ok;\n") + h.add(" std::optional err;\n") + h.add("};\n\n") + + for e in requestEntries: + if e.argFields.len == 0: + continue + if not isMethodSupported(e.apiName): + continue + let an = argsStructName(e.apiName) + if an.len == 0: + continue + h.add("struct " & an & " {\n") + for (n, t) in e.argFields: + h.add(" " & nimTypeToCppType(t) & " " & n & "{};\n") + h.add("};\n\n") + + # ================================================================== + # Section 4: JSONCONS macros (global scope) + # ================================================================== + h.add("} // namespace " & libName & "\n\n") + + if enumNames.len > 0: + h.add( + "namespace jsoncons {\n" & "template \n" & + "struct broker_int_enum_traits {\n" & " using value_type = E;\n" & + " using underlying = int32_t;\n" & + " static constexpr bool is_compatible = true;\n" & + " static bool is(const Json& j) noexcept {\n" & + " return j.template is();\n" & " }\n" & + " static value_type as(const Json& j) {\n" & + " return static_cast(j.template as());\n" & " }\n" & + " static Json to_json(value_type v) {\n" & + " return Json(static_cast(v));\n" & " }\n" & + " template \n" & + " static Json to_json(value_type v, const Allocator&) {\n" & + " return to_json(v);\n" & " }\n" & "};\n" + ) + for name in enumNames: + let q = libName & "::" & name + h.add( + "template \n" & "struct json_type_traits\n" & + " : public broker_int_enum_traits {};\n" + ) + h.add("} // namespace jsoncons\n\n") + + for (name, fields) in payloadFields: + emitMemberTraitsMacro(h, libName & "::" & name, name, fields) + for envName in envelopeNames: + emitEnvelopeTraits(h, libName & "::" & envName) + if anyInstanceReturn: + emitEnvelopeTraits(h, libName & "::__InstanceCtxEnvelope") + for (name, fields) in argsFields: + # Args structs aren't in the public registry under their `Args` + # synthesised name; the loop falls through to `required = fieldNames` + # and behaves identically to the previous _ALL_ emission. Pass the + # name anyway so future arg-side Option support flips on for free. + emitMemberTraitsMacro(h, libName & "::" & name, name, fields) + if payloadFields.len > 0 or envelopeNames.len > 0 or argsFields.len > 0: + h.add("\n") + + # ================================================================== + # Section 5: detail encode helpers (need JSONCONS traits above) + # ================================================================== + h.add("namespace " & libName & " {\n") + h.add("namespace detail {\n\n") + + # NimBuffer RAII wrapper for Nim-allocated buffers. + h.add("class NimBuffer {\n") + h.add(" public:\n") + h.add(" NimBuffer() = default;\n") + h.add(" NimBuffer(void* p, int32_t n) noexcept : p_(p), n_(n) {}\n") + h.add( + " NimBuffer(NimBuffer&& o) noexcept : p_(o.p_), n_(o.n_) { o.p_ = nullptr; o.n_ = 0; }\n" + ) + h.add(" NimBuffer& operator=(NimBuffer&& o) noexcept {\n") + h.add( + " if (this != &o) { reset(); p_ = o.p_; n_ = o.n_; o.p_ = nullptr; o.n_ = 0; }\n" + ) + h.add(" return *this;\n") + h.add(" }\n") + h.add(" NimBuffer(const NimBuffer&) = delete;\n") + h.add(" NimBuffer& operator=(const NimBuffer&) = delete;\n") + h.add(" ~NimBuffer() { reset(); }\n") + h.add(" void reset() noexcept {\n") + h.add(" if (p_) { " & p & "freeBuffer(p_); p_ = nullptr; n_ = 0; }\n") + h.add(" }\n") + h.add(" bool empty() const noexcept { return p_ == nullptr || n_ <= 0; }\n") + h.add(" std::span view() const noexcept {\n") + h.add(" return {static_cast(p_),\n") + h.add(" static_cast(n_ > 0 ? n_ : 0)};\n") + h.add(" }\n") + h.add(" private:\n") + h.add(" void* p_ = nullptr;\n") + h.add(" int32_t n_ = 0;\n") + h.add("};\n\n") + + h.add("struct CountingSink {\n") + h.add(" using value_type = std::uint8_t;\n") + h.add(" std::size_t* count;\n") + h.add(" explicit CountingSink(std::size_t& c) noexcept : count(&c) {}\n") + h.add(" void append(const std::uint8_t*, std::size_t n) noexcept { *count += n; }\n") + h.add(" void push_back(std::uint8_t) noexcept { ++*count; }\n") + h.add(" void flush() noexcept {}\n") + h.add("};\n\n") + h.add("struct SpanSink {\n") + h.add(" using value_type = std::uint8_t;\n") + h.add(" std::uint8_t* dst;\n") + h.add(" std::size_t cap;\n") + h.add(" std::size_t* pos;\n") + h.add(" SpanSink(std::uint8_t* d, std::size_t c, std::size_t& p) noexcept\n") + h.add(" : dst(d), cap(c), pos(&p) {}\n") + h.add(" void append(const std::uint8_t* s, std::size_t n) noexcept {\n") + h.add(" std::memcpy(dst + *pos, s, n); *pos += n;\n") + h.add(" }\n") + h.add(" void push_back(std::uint8_t b) noexcept { dst[(*pos)++] = b; }\n") + h.add(" void flush() noexcept {}\n") + h.add("};\n\n") + # jsoncons 1.7.0 moved encode_traits into the `reflect` sub-namespace and + # renamed the entry point from `encode(v, enc, ctx, ec)` to + # `try_encode(alloc_set, v, enc)` returning `write_result` + # (= `expected`). + h.add("template \n") + h.add("std::size_t cborEncodedSize(const T& v) {\n") + h.add(" std::size_t n = 0;\n") + h.add(" jsoncons::cbor::basic_cbor_encoder enc{CountingSink{n}};\n") + h.add( + " auto result = jsoncons::reflect::encode_traits::try_encode(\n" & + " jsoncons::make_alloc_set(), v, enc);\n" + ) + h.add( + " if (!result) throw std::system_error(result.error(), \"cbor counting pass\");\n" + ) + h.add(" enc.flush();\n") + h.add(" return n;\n") + h.add("}\n\n") + h.add("template \n") + h.add("void cborEncodeInto(const T& v, std::uint8_t* dst, std::size_t cap) {\n") + h.add(" std::size_t pos = 0;\n") + h.add( + " jsoncons::cbor::basic_cbor_encoder enc{SpanSink{dst, cap, pos}};\n" + ) + h.add( + " auto result = jsoncons::reflect::encode_traits::try_encode(\n" & + " jsoncons::make_alloc_set(), v, enc);\n" + ) + h.add( + " if (!result) throw std::system_error(result.error(), \"cbor write pass\");\n" + ) + h.add(" enc.flush();\n") + h.add("}\n\n") + # rawCall / rawCallOwned — free functions in detail namespace. + h.add("inline std::pair\n") + h.add( + "rawCall(uint32_t ctx, std::string& lastError,\n" & + " const char* apiName, const std::uint8_t* in, std::size_t inLen) {\n" + ) + h.add(" if (ctx == 0) {\n") + h.add(" lastError = \"library context is not initialised\";\n") + h.add(" return {-1, NimBuffer{}};\n") + h.add(" }\n") + h.add(" void* inBuf = nullptr;\n") + h.add(" if (inLen > 0) {\n") + h.add(" inBuf = " & p & "allocBuffer(static_cast(inLen));\n") + h.add( + " if (!inBuf) { lastError = \"allocBuffer failed\"; return {-1, NimBuffer{}}; }\n" + ) + h.add(" std::memcpy(inBuf, in, inLen);\n") + h.add(" }\n") + h.add(" void* respBuf = nullptr;\n") + h.add(" int32_t respLen = 0;\n") + h.add(" const int32_t status = " & p & "call(\n") + h.add( + " ctx, apiName, inBuf, static_cast(inLen), &respBuf, &respLen);\n" + ) + h.add(" NimBuffer resp{respBuf, respLen};\n") + h.add(" if (status != 0) {\n") + h.add(" if (status == -4 && !resp.empty()) {\n") + h.add(" auto v = resp.view();\n") + h.add(" lastError.assign(reinterpret_cast(v.data()), v.size());\n") + h.add(" } else {\n") + h.add(" lastError = std::string(\"framework error: \") +\n") + h.add(" std::to_string(status);\n") + h.add(" }\n") + h.add(" return {status, std::move(resp)};\n") + h.add(" }\n") + h.add(" return {0, std::move(resp)};\n") + h.add("}\n\n") + + h.add("inline std::pair\n") + h.add( + "rawCallOwned(uint32_t ctx, std::string& lastError,\n" & + " const char* apiName, void* nimInBuf, std::size_t inLen) {\n" + ) + h.add(" if (ctx == 0) {\n") + h.add(" if (nimInBuf) " & p & "freeBuffer(nimInBuf);\n") + h.add(" lastError = \"library context is not initialised\";\n") + h.add(" return {-1, NimBuffer{}};\n") + h.add(" }\n") + h.add(" void* respBuf = nullptr;\n") + h.add(" int32_t respLen = 0;\n") + h.add(" const int32_t status = " & p & "call(\n") + h.add( + " ctx, apiName, nimInBuf, static_cast(inLen), &respBuf, &respLen);\n" + ) + h.add(" NimBuffer resp{respBuf, respLen};\n") + h.add(" if (status != 0) {\n") + h.add(" if (status == -4 && !resp.empty()) {\n") + h.add(" auto v = resp.view();\n") + h.add(" lastError.assign(reinterpret_cast(v.data()), v.size());\n") + h.add(" } else {\n") + h.add(" lastError = std::string(\"framework error: \") +\n") + h.add(" std::to_string(status);\n") + h.add(" }\n") + h.add(" return {status, std::move(resp)};\n") + h.add(" }\n") + h.add(" return {0, std::move(resp)};\n") + h.add("}\n\n") + + # ---- EventDispatcher template (CBOR-flavored) ---- + # One C-level subscription per event type, lazily registered on first + # add() and unregistered when the last user callback is removed. User + # callbacks are stored in a local map and fanned out under a snapshot + # taken inside the trampoline. Mirrors native FFI EventDispatcher + # semantics without the variadic C-arg shape (CBOR cb shape is fixed). + h.add("template \n") + h.add("class EventDispatcher {\n") + h.add(" public:\n") + h.add(" using Callback = typename Traits::template Callback;\n") + h.add(" using EventStruct = typename Traits::EventStruct;\n\n") + h.add(" explicit EventDispatcher(Owner& owner) noexcept : owner_(&owner) {}\n") + h.add(" EventDispatcher(const EventDispatcher&) = delete;\n") + h.add(" EventDispatcher& operator=(const EventDispatcher&) = delete;\n") + h.add(" EventDispatcher(EventDispatcher&&) = delete;\n") + h.add(" EventDispatcher& operator=(EventDispatcher&&) = delete;\n") + h.add(" ~EventDispatcher() { clear(); }\n\n") + h.add(" uint64_t add(Callback fn) noexcept {\n") + h.add(" std::lock_guard lock(mutex_);\n") + h.add(" if (!owner_ || owner_->ctx() == 0 || !fn) return 0;\n") + h.add(" if (nativeHandle_ == 0) {\n") + h.add(" nativeHandle_ = Traits::registerWithC(\n") + h.add( + " owner_->ctx(), &EventDispatcher::trampoline, static_cast(this));\n" + ) + h.add(" if (nativeHandle_ == 0) return 0;\n") + h.add(" }\n") + h.add(" const uint64_t localHandle = nextLocalHandle_++;\n") + h.add(" try {\n") + h.add(" callbacks_.emplace(localHandle, std::move(fn));\n") + h.add(" return localHandle;\n") + h.add(" } catch (...) {\n") + h.add(" if (callbacks_.empty() && nativeHandle_ != 0) {\n") + h.add(" Traits::unregisterWithC(owner_->ctx(), nativeHandle_);\n") + h.add(" nativeHandle_ = 0;\n") + h.add(" }\n") + h.add(" return 0;\n") + h.add(" }\n") + h.add(" }\n\n") + h.add(" void remove(uint64_t localHandle) noexcept {\n") + h.add(" std::lock_guard lock(mutex_);\n") + h.add(" callbacks_.erase(localHandle);\n") + h.add(" if (callbacks_.empty() && nativeHandle_ != 0) {\n") + h.add(" if (owner_ && owner_->ctx() != 0)\n") + h.add(" Traits::unregisterWithC(owner_->ctx(), nativeHandle_);\n") + h.add(" nativeHandle_ = 0;\n") + h.add(" }\n") + h.add(" }\n\n") + h.add(" void clear() noexcept {\n") + h.add(" std::lock_guard lock(mutex_);\n") + h.add(" callbacks_.clear();\n") + h.add(" if (nativeHandle_ != 0) {\n") + h.add(" if (owner_ && owner_->ctx() != 0)\n") + h.add(" Traits::unregisterWithC(owner_->ctx(), nativeHandle_);\n") + h.add(" nativeHandle_ = 0;\n") + h.add(" }\n") + h.add(" }\n\n") + h.add(" private:\n") + h.add(" static void trampoline(uint32_t ctx, const char* /*eventName*/,\n") + h.add(" const void* payloadBuf, int32_t payloadLen,\n") + h.add(" void* userData) noexcept {\n") + h.add(" auto* self = static_cast(userData);\n") + h.add(" if (!self || !payloadBuf || payloadLen <= 0) return;\n") + h.add(" EventStruct evt;\n") + h.add(" try {\n") + h.add(" std::span v{\n") + h.add(" static_cast(payloadBuf),\n") + h.add(" static_cast(payloadLen)};\n") + h.add(" evt = jsoncons::cbor::decode_cbor(v.begin(), v.end());\n") + h.add(" } catch (...) { return; }\n") + h.add(" self->deliver(ctx, evt);\n") + h.add(" }\n\n") + h.add(" void deliver(uint32_t ctx, const EventStruct& evt) noexcept {\n") + h.add(" std::vector snapshot;\n") + h.add(" {\n") + h.add(" std::lock_guard lock(mutex_);\n") + h.add(" if (!owner_ || ctx != owner_->ctx()) return;\n") + h.add(" try {\n") + h.add(" snapshot.reserve(callbacks_.size());\n") + h.add( + " for (const auto& [id, fn] : callbacks_) if (fn) snapshot.push_back(fn);\n" + ) + h.add(" } catch (...) { return; }\n") + h.add(" }\n") + h.add(" for (const auto& fn : snapshot) Traits::invoke(fn, *owner_, evt);\n") + h.add(" }\n\n") + h.add(" Owner* owner_ = nullptr;\n") + h.add(" std::mutex mutex_;\n") + h.add(" std::unordered_map callbacks_;\n") + h.add(" uint64_t nativeHandle_ = 0;\n") + h.add(" uint64_t nextLocalHandle_ = 1;\n") + h.add("};\n\n") + + # ---- Per-event Traits structs ---- + for ev in emittableEvents: + let evFields = effectiveFields(ev.typeName) + let evScalar = isScalarPayload(ev.typeName) + # A `void` event has no struct jsoncons can decode — the payload-less + # `{}` map is decoded through the generic `jsoncons::json`. + let evStruct = if isVoidPayload(ev.typeName): "jsoncons::json" else: ev.typeName + h.add("struct " & ev.typeName & "EventTraits {\n") + h.add(" using EventStruct = " & evStruct & ";\n\n") + # Callback alias: Owner&, then unpacked args. + h.add(" template \n") + h.add(" using Callback = std::function;\n\n") + h.add( + " static uint64_t registerWithC(uint32_t ctx,\n" & + " void (*cb)(uint32_t, const char*, const void*, int32_t, void*),\n" & + " void* userData) noexcept {\n" + ) + h.add(" return " & p & "subscribe(ctx, \"" & ev.apiName & "\", cb, userData);\n") + h.add(" }\n\n") + h.add(" static void unregisterWithC(uint32_t ctx, uint64_t handle) noexcept {\n") + h.add(" " & p & "unsubscribe(ctx, \"" & ev.apiName & "\", handle);\n") + h.add(" }\n\n") + h.add(" template \n") + h.add( + " static void invoke(const Callback& fn, Owner& owner,\n" & + " const " & evStruct & "& evt) noexcept {\n" + ) + # Per-field setup statements (e.g. seq[string] -> vector) + # emitted BEFORE the try block so any temporary views the user callback + # observes stay alive for the entire call. + for f in evFields: + let setup = eventCallbackInvokeSetup(f.name, f.nimType) + if setup.len > 0: + h.add(setup) + h.add(" try {\n") + h.add(" fn(owner") + if evScalar: + # Scalar payload: the decoded `evt` IS the value — pass it directly + # (no `.value` member, EventStruct is the primitive alias itself). + h.add(", evt") + else: + for f in evFields: + h.add(", " & eventCallbackArgExpr(f.name, f.nimType)) + h.add(");\n") + h.add(" } catch (...) {}\n") + h.add(" }\n") + h.add("};\n\n") + + h.add("} // namespace detail\n\n") + + # ================================================================== + # Sub-interface out-of-class inline method implementations. + # Emitted in the library namespace (same as the class declaration) + # after detail:: is fully defined so EventDispatcher is complete. + # ================================================================== + for ifaceName in subInterfaceNames: + let sub = cppSubClassName(ifaceName) + let subEvts = subEventsOf(ifaceName) + let hasEvts = subEvts.len > 0 + h.add("// ---- " & sub & " implementations ----\n") + # Ctor + if hasEvts: + h.add("inline " & sub & "::" & sub & "(uint32_t ctx)\n") + h.add(" : ctx_(ctx)\n") + for ev in subEvts: + let dispType = ev.typeName & "Dispatcher" + let dispMember = ev.apiName & "Dispatcher_" + h.add(" , " & dispMember & "(std::make_unique<" & dispType & ">(*this))\n") + h.add("{}\n") + else: + h.add("inline " & sub & "::" & sub & "(uint32_t ctx) noexcept : ctx_(ctx) {}\n") + # Dtor + h.add("inline " & sub & "::~" & sub & "() { close(); }\n") + # Move ctor/assign (event-free only) + if not hasEvts: + h.add( + "inline " & sub & "::" & sub & "(" & sub & + "&& o) noexcept : ctx_(o.ctx_) { o.ctx_ = 0; }\n" + ) + h.add( + "inline " & sub & "& " & sub & "::operator=(" & sub & + "&& o) noexcept { if (this != &o) { close(); ctx_ = o.ctx_; o.ctx_ = 0; } return *this; }\n" + ) + # close() + h.add("inline void " & sub & "::close() noexcept {\n") + for ev in subEvts: + let dispMember = ev.apiName & "Dispatcher_" + h.add(" if (" & dispMember & ") " & dispMember & "->clear();\n") + h.add(" if (ctx_) { " & p & "releaseInstance(ctx_); ctx_ = 0; }\n") + h.add("}\n") + # Request method implementations. + for e in requestEntries: + if interfaceOwningRequestType(e.responseTypeName) == ifaceName: + h.add(emitSubReqImpl(e, sub)) + # Event on/off implementations. + for ev in subEvts: + let camelBase = snakeToLowerCamel(ev.apiName) + var pascal = camelBase + if pascal.len > 0: + pascal[0] = toUpperAscii(pascal[0]) + let cbAlias = ev.typeName & "Callback" + let dispMember = ev.apiName & "Dispatcher_" + h.add( + "inline uint64_t " & sub & "::on" & pascal & "(" & cbAlias & + " fn) noexcept { return " & dispMember & "->add(std::move(fn)); }\n" + ) + h.add( + "inline void " & sub & "::off" & pascal & + "(uint64_t handle) noexcept { if (handle == 0) " & dispMember & + "->clear(); else " & dispMember & "->remove(handle); }\n" + ) + h.add("\n") + + # ================================================================== + # Section 6: Out-of-class inline Lib method definitions + # ================================================================== + + # ---- Lifecycle ---- + # Constructor initializes one EventDispatcher per emittable event type. + h.add("inline " & className & "::" & className & "()") + if mainEvents.len > 0: + h.add("\n") + var first = true + for ev in mainEvents: + let dispatcherType = ev.typeName & "Dispatcher" + let dispatcherMember = ev.apiName & "Dispatcher_" + if first: + h.add(" : ") + first = false + else: + h.add(" , ") + h.add(dispatcherMember & "(std::make_unique<" & dispatcherType & ">(*this))\n") + h.add(" { " & p & "initialize(); }\n\n") + else: + h.add(" { " & p & "initialize(); }\n\n") + h.add("inline " & className & "::~" & className & "() { shutdown(); }\n\n") + h.add("inline std::string_view " & className & "::version() noexcept {\n") + h.add(" return " & p & "version();\n") + h.add("}\n\n") + h.add("inline Result " & className & "::createContext() {\n") + h.add(" if (ctx_)\n") + h.add(" return Result::err(\"Context already created\");\n") + h.add(" char* err = nullptr;\n") + h.add(" ctx_ = " & p & "createContext(&err);\n") + h.add(" if (ctx_ == 0) {\n") + h.add(" if (err != nullptr) {\n") + h.add(" std::string msg(err);\n") + h.add(" " & p & "freeBuffer(err);\n") + h.add(" return Result::err(std::move(msg));\n") + h.add(" }\n") + h.add(" return Result::err(\"createContext failed\");\n") + h.add(" }\n") + h.add(" return Result::ok();\n") + h.add("}\n\n") + h.add( + "inline bool " & className & + "::validContext() const noexcept { return ctx_ != 0; }\n" + ) + h.add( + "inline " & className & + "::operator bool() const noexcept { return validContext(); }\n" + ) + h.add("inline uint32_t " & className & "::ctx() const noexcept { return ctx_; }\n\n") + # shutdown clears all event dispatchers before calling C shutdown + h.add("inline void " & className & "::shutdown() noexcept {\n") + for ev in mainEvents: + let dispatcherMember = ev.apiName & "Dispatcher_" + h.add(" if (" & dispatcherMember & ") " & dispatcherMember & "->clear();\n") + h.add(" if (ctx_) { " & p & "shutdown(ctx_); ctx_ = 0; }\n") + h.add("}\n\n") + + # ---- Per-request method implementations ---- + for e in requestEntries: + if e.responseTypeName.len == 0: + continue + if not ownsReqMain(e): + continue + let isInstance = e.returnsInterface.len > 0 + if not isInstance and not isEmittablePayload(e.responseTypeName): + continue + if not isMethodSupported(e.apiName): + continue + let methodName = snakeToLowerCamel(e.apiName) + # reduced-A: a create-instance method decodes the shared uint-ctx envelope + # and constructs the typed sub-wrapper from the returned ctx. + let envName = + if isInstance: + "__InstanceCtxEnvelope" + else: + e.responseTypeName & "Envelope" + let voidResp = (not isInstance) and isVoidPayload(e.responseTypeName) + let subEv = isInstance and subHasEvents(e.returnsInterface) + let resTy = + if isInstance and subEv: + "Result>" + elif isInstance: + "Result<" & cppSubClassName(e.returnsInterface) & ">" + else: + "Result<" & payloadCppType(e.responseTypeName) & ">" + let okExpr = + if isInstance and subEv: + resTy & "::ok(std::make_unique<" & cppSubClassName(e.returnsInterface) & + ">(static_cast(*env.ok)))" + elif isInstance: + resTy & "::ok(" & cppSubClassName(e.returnsInterface) & + "(static_cast(*env.ok)))" + elif voidResp: + resTy & "::ok()" + else: + resTy & "::ok(std::move(*env.ok))" + var sigParams = "" + var argsAssign = "" + let argsName = argsStructName(e.apiName) + if e.argFields.len > 0: + var first = true + for (n, t) in e.argFields: + if not first: + sigParams.add(", ") + sigParams.add(nimTypeToCppType(t) & " " & n) + argsAssign.add(" args." & n & " = " & n & ";\n") + first = false + h.add( + "inline " & resTy & " " & className & "::" & methodName & "(" & sigParams & ") {\n" + ) + if e.argFields.len > 0: + h.add(" " & argsName & " args;\n") + h.add(argsAssign) + h.add(" std::size_t cborLen = 0;\n") + h.add(" try { cborLen = detail::cborEncodedSize(args); }\n") + h.add(" catch (const std::exception& ex) {\n") + h.add( + " return " & resTy & + "::err(std::string(\"size pass failed: \") + ex.what());\n" + ) + h.add(" }\n") + h.add( + " void* inBuf = (cborLen > 0) ? " & p & + "allocBuffer(static_cast(cborLen)) : nullptr;\n" + ) + h.add(" if (cborLen > 0 && !inBuf)\n") + h.add(" return " & resTy & "::err(\"allocBuffer failed\");\n") + h.add(" try {\n") + h.add( + " if (cborLen > 0) detail::cborEncodeInto(args, static_cast(inBuf), cborLen);\n" + ) + h.add(" } catch (const std::exception& ex) {\n") + h.add(" if (inBuf) { " & p & "freeBuffer(inBuf); }\n") + h.add( + " return " & resTy & + "::err(std::string(\"encode pass failed: \") + ex.what());\n" + ) + h.add(" }\n") + h.add(" std::string lastError;\n") + h.add( + " auto [status, resp] = detail::rawCallOwned(ctx_, lastError, \"" & e.apiName & + "\", inBuf, cborLen);\n" + ) + else: + h.add(" std::string lastError;\n") + h.add( + " auto [status, resp] = detail::rawCallOwned(ctx_, lastError, \"" & e.apiName & + "\", nullptr, 0);\n" + ) + h.add(" if (status != 0)\n") + h.add(" return " & resTy & "::err(lastError);\n") + h.add(" if (resp.empty())\n") + h.add(" return " & resTy & "::err(\"empty response\");\n") + h.add(" " & envName & " env;\n") + h.add(" try {\n") + h.add(" auto v = resp.view();\n") + h.add( + " env = jsoncons::cbor::decode_cbor<" & envName & ">(v.begin(), v.end());\n" + ) + h.add(" } catch (const std::exception& ex) {\n") + h.add( + " return " & resTy & "::err(std::string(\"decode failed: \") + ex.what());\n" + ) + h.add(" }\n") + h.add(" if (env.err.has_value())\n") + h.add(" return " & resTy & "::err(*env.err);\n") + h.add(" if (env.ok.has_value())\n") + h.add(" return " & okExpr & ";\n") + h.add(" return " & resTy & "::err(\"malformed response envelope\");\n") + h.add("}\n\n") + + # ---- Per-event on/off implementations (delegate to dispatcher) ---- + for ev in mainEvents: + let camelBase = snakeToLowerCamel(ev.apiName) + var pascal = camelBase + if pascal.len > 0: + pascal[0] = toUpperAscii(pascal[0]) + let callbackAlias = ev.typeName & "Callback" + let onName = "on" & pascal + let offName = "off" & pascal + let dispatcherMember = ev.apiName & "Dispatcher_" + + h.add( + "inline uint64_t " & className & "::" & onName & "(" & callbackAlias & + " fn) noexcept {\n" + ) + h.add(" return " & dispatcherMember & "->add(std::move(fn));\n") + h.add("}\n\n") + + h.add( + "inline void " & className & "::" & offName & "(uint64_t handle) noexcept {\n" + ) + h.add(" if (handle == 0) " & dispatcherMember & "->clear();\n") + h.add(" else " & dispatcherMember & "->remove(handle);\n") + h.add("}\n\n") + + # ---- Discovery implementations ---- + h.add("inline std::string " & className & "::listApis() {\n") + h.add(" void* buf = nullptr;\n") + h.add(" int32_t len = 0;\n") + h.add(" const int32_t status = " & p & "listApis(&buf, &len);\n") + h.add(" detail::NimBuffer nb{buf, len};\n") + # listApis/getSchema have no error channel back to the caller beyond an + # empty return — the old `lastError_ = ...` writes were dead state on a + # member that has no public accessor. Dropped: returning {} is enough. + h.add(" if (status != 0) return {};\n") + h.add(" if (nb.empty()) return {};\n") + h.add(" auto v = nb.view();\n") + h.add(" return std::string(reinterpret_cast(v.data()), v.size());\n") + h.add("}\n\n") + h.add("inline std::string " & className & "::getSchema() {\n") + h.add(" void* buf = nullptr;\n") + h.add(" int32_t len = 0;\n") + h.add(" const int32_t status = " & p & "getSchema(&buf, &len);\n") + h.add(" detail::NimBuffer nb{buf, len};\n") + h.add(" if (status != 0) return {};\n") + h.add(" if (nb.empty()) return {};\n") + h.add(" auto v = nb.view();\n") + h.add(" return std::string(reinterpret_cast(v.data()), v.size());\n") + h.add("}\n\n") + + h.add("} // namespace " & libName & "\n\n") + h.add("#endif // " & guardName & "\n") + + try: + writeFile(headerPath, h) + except IOError: + error( + "Failed to write generated CBOR C++ header '" & headerPath & "': " & + getCurrentExceptionMsg() + ) + +{.push raises: [].} +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_codegen_cbor_py.nim b/wasm-deps/brokers/brokers/internal/api_codegen_cbor_py.nim new file mode 100644 index 000000000..150b67f9f --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_codegen_cbor_py.nim @@ -0,0 +1,1153 @@ +## Generated Python wrapper for the CBOR FFI surface. +## +## The generated `.py` ships alongside the shared library and uses +## ctypes for the C ABI plus the `cbor2` package for encode/decode. +## Discovery endpoints (`list_apis`, `get_schema`) return JSON and are +## decoded via stdlib `json`. Foreign Python projects only need to +## install `cbor2` (pure-Python, widely packaged) — no other runtime +## dependencies. +## +## The wrapper emits typed `dataclass` definitions for each registered +## request response, request args, and event payload type, and per- +## request methods on the libname-PascalCase wrapper class that +## CBOR-encode the args, dispatch through the C gate, and decode the +## response envelope into a `Result` object. Per-event `on_(callback)` +## methods register a typed callable that receives the owning library +## instance plus the unpacked event payload fields; `off_(handle = 0)` +## removes a single registration (or, with handle 0, all of them). +## Trampolines are kept alive in a per-event handler map mirroring the +## C++ EventDispatcher's GC anchor. +## +## Type-matrix coverage (Phase 7D): +## - Primitives: bool, int/intN, uint/uintN/byte, float/floatN, string, +## char. +## - Enums (atkEnum) → Python IntEnum classes. +## - Distinct/Alias (atkDistinct/atkAlias) → Python type alias of the +## resolved underlying type (typing.NewType-style: simple `=` alias). +## - Registered objects → @dataclass with typed fields and a paired +## `_decode_` / `_encode_` helper. +## - Composite types: seq[T], array[N, T] (typed as List[]), +## including seq[byte] and seq[]; nested objects. +## Unmappable types still produce a TODO stub so the wrapper compiles. + +{.push raises: [].} + +import std/[macros, strutils, tables] +import ./api_common, ./api_schema +import ./helper/broker_utils # reduced-A: per-interface partitioning + +# --------------------------------------------------------------------------- +# Nim → Python type mapping (registry-aware) +# --------------------------------------------------------------------------- + +const pyPrimMap = { + "bool": "bool", + "string": "str", + "char": "str", + "int": "int", + "int8": "int", + "int16": "int", + "int32": "int", + "int64": "int", + "uint": "int", + "uint8": "int", + "uint16": "int", + "uint32": "int", + "uint64": "int", + "byte": "int", + "float": "float", + "float32": "float", + "float64": "float", +}.toTable + +proc isPrimitive(nimType: string): bool {.compileTime.} = + nimType.strip() in pyPrimMap + +proc primPyHint(nimType: string): string {.compileTime.} = + pyPrimMap.getOrDefault(nimType.strip(), "") + +proc unwrapBracket(s, head: string): string {.compileTime.} = + ## "seq[X]" + "seq" -> "X" + let t = s.strip() + t[head.len + 1 .. ^2].strip() + +proc parseArrayInner(s: string): string {.compileTime.} = + ## "array[N, T]" -> "T" + let inner = s.strip()[6 ..^ 2] + let comma = inner.find(',') + if comma < 0: + return "" + inner[comma + 1 .. ^1].strip() + +proc nimTypeToPyHint*(nimType: string): string {.compileTime.} = + ## Recursive Nim → Python type hint. Falls back to "" for types we + ## don't yet know how to map (the caller emits a TODO). + let t = nimType.strip() + let lower = t.toLowerAscii() + if isPrimitive(t): + return primPyHint(t) + if lower == "seq[byte]": + # CBOR major type 2 (byte string) decodes to Python `bytes`; mirror + # that on the wrapper side so `Option[seq[byte]]` inside an Option + # also resolves to `Optional[bytes]` rather than `Optional[List[int]]`. + return "bytes" + if lower.startsWith("seq[") and lower.endsWith("]"): + let inner = nimTypeToPyHint(unwrapBracket(t, "seq")) + return + if inner.len > 0: + "List[" & inner & "]" + else: + "List[Any]" + if lower.startsWith("array["): + let elem = parseArrayInner(t) + let inner = nimTypeToPyHint(elem) + return + if inner.len > 0: + "List[" & inner & "]" + else: + "List[Any]" + if lower.startsWith("option[") and lower.endsWith("]"): + let inner = nimTypeToPyHint(unwrapBracket(t, "option")) + return + if inner.len > 0: + "Optional[" & inner & "]" + else: + "Optional[Any]" + if isTypeRegistered(t): + let entry = lookupTypeEntry(t) + case entry.kind + of atkObject: + return t + of atkEnum: + return t + of atkAlias, atkDistinct: + # Recurse via outer mapper for distinct/alias-over-compound (e.g. + # `distinct seq[byte]` → `List[int]` rather than `""`). + return nimTypeToPyHint(resolveUnderlyingType(t)) + "" + +proc nimTypeToPyDefault*(nimType: string): string {.compileTime.} = + ## Default value literal for a dataclass field. Collections use + ## `field(default_factory=list)`; objects/enums use `None` (callers + ## construct lazily — dataclass init still needs a callable default). + let t = nimType.strip() + let lower = t.toLowerAscii() + case t + of "bool": + return "False" + of "string", "char": + return "\"\"" + of "int", "int8", "int16", "int32", "int64", "uint", "uint8", "byte", "uint16", + "uint32", "uint64": + return "0" + of "float32", "float", "float64": + return "0.0" + else: + discard + if lower == "seq[byte]": + return "b\"\"" + if lower.startsWith("seq[") or lower.startsWith("array["): + return "field(default_factory=list)" + if lower.startsWith("option["): + return "None" + if isTypeRegistered(t): + let entry = lookupTypeEntry(t) + case entry.kind + of atkObject: + return "field(default_factory=" & t & ")" + of atkEnum: + return t & "(0)" + of atkAlias, atkDistinct: + return nimTypeToPyDefault(resolveUnderlyingType(t)) + "None" + +proc isPyMappable*(nimType: string): bool {.compileTime.} = + nimTypeToPyHint(nimType).len > 0 + +# --------------------------------------------------------------------------- +# Per-type encoder / decoder expression builders. +# +# These produce the Python source for the `_encode_` / `_decode_` +# helpers (called from request method bodies and from event trampolines). +# Returning a string keeps the codegen flat; the wrapper file is plain +# Python anyway. +# --------------------------------------------------------------------------- + +proc pyDecodeExpr(nimType, src: string): string {.compileTime.} = + ## Python expression that decodes the value at `src` (a Python + ## expression yielding the raw cbor2 result) into the in-memory + ## representation of `nimType`. + let t = nimType.strip() + let lower = t.toLowerAscii() + if t == "bool": + return "bool(" & src & ") if isinstance(" & src & ", bool) else False" + if t == "string" or t == "char": + return "(" & src & " if isinstance(" & src & ", str) else \"\")" + if t in [ + "int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16", "uint32", + "uint64", "byte", + ]: + return "(int(" & src & ") if isinstance(" & src & ", int) else 0)" + if t in ["float", "float32", "float64"]: + return "(float(" & src & ") if isinstance(" & src & ", (int, float)) else 0.0)" + if lower == "seq[byte]": + # CBOR byte string → Python `bytes`. Accept the byte-string shape + # cbor2 produces directly, and tolerate the legacy list-of-int + # shape (e.g. when a sender emits major type 4 instead of 2). + return "(bytes(" & src & ") if " & src & " is not None else b\"\")" + if lower.startsWith("seq[") and lower.endsWith("]"): + let inner = unwrapBracket(t, "seq") + let raw = "(" & src & " or [])" + return "[" & pyDecodeExpr(inner, "_x") & " for _x in " & raw & "]" + if lower.startsWith("array["): + let elem = parseArrayInner(t) + let raw = "(" & src & " or [])" + return "[" & pyDecodeExpr(elem, "_x") & " for _x in " & raw & "]" + if lower.startsWith("option[") and lower.endsWith("]"): + let inner = unwrapBracket(t, "option") + return "(None if " & src & " is None else " & pyDecodeExpr(inner, src) & ")" + if isTypeRegistered(t): + let entry = lookupTypeEntry(t) + case entry.kind + of atkObject: + return "_decode_" & t & "(" & src & ")" + of atkEnum: + return "_decode_" & t & "(" & src & ")" + of atkAlias, atkDistinct: + return pyDecodeExpr(resolveUnderlyingType(t), src) + # Unknown — pass through; cbor2 already gave us something. + src + +proc pyEncodeExpr(nimType, src: string): string {.compileTime.} = + ## Python expression that encodes the value at `src` into the + ## CBOR-friendly representation expected by the Nim provider for + ## `nimType`. + let t = nimType.strip() + let lower = t.toLowerAscii() + if isPrimitive(t): + return src + if lower == "seq[byte]": + # cbor2 encodes Python `bytes` as CBOR byte string (major type 2), + # which is what the Nim provider expects. Tolerate list-of-int input + # by converting on the fly. + return + "(" & src & " if isinstance(" & src & ", (bytes, bytearray)) else bytes(" & src & + " or []))" + if lower.startsWith("seq[") and lower.endsWith("]"): + let inner = unwrapBracket(t, "seq") + return "[" & pyEncodeExpr(inner, "_x") & " for _x in (" & src & " or [])]" + if lower.startsWith("array["): + let elem = parseArrayInner(t) + return "[" & pyEncodeExpr(elem, "_x") & " for _x in (" & src & " or [])]" + if lower.startsWith("option[") and lower.endsWith("]"): + let inner = unwrapBracket(t, "option") + return "(None if " & src & " is None else " & pyEncodeExpr(inner, src) & ")" + if isTypeRegistered(t): + let entry = lookupTypeEntry(t) + case entry.kind + of atkObject: + return "_encode_" & t & "(" & src & ")" + of atkEnum: + return "int(" & src & ")" + of atkAlias, atkDistinct: + return pyEncodeExpr(resolveUnderlyingType(t), src) + src + +proc snakeToLowerCamel(s: string): string {.compileTime.} = + result = "" + var capitalize = false + for ch in s: + if ch == '_': + capitalize = true + elif capitalize: + result.add(toUpperAscii(ch)) + capitalize = false + else: + result.add(ch) + +proc snakeToUpperCamel(s: string): string {.compileTime.} = + result = snakeToLowerCamel(s) + if result.len > 0: + result[0] = toUpperAscii(result[0]) + +# --------------------------------------------------------------------------- +# File emission +# --------------------------------------------------------------------------- + +{.pop.} + +proc subClassName(iface: string): string {.compileTime.} = + ## Wrapper class name for a sub-interface: strip a leading `I` before an + ## uppercase letter (IWidget -> Widget), else use the name as-is. + if iface.len > 1 and iface[0] == 'I' and iface[1] in {'A' .. 'Z'}: + iface[1 ..^ 1] + else: + iface + +proc generateCborPyFile*( + outDir: string, + libName: string, + requestEntries: seq[CborRequestEntry], + eventEntries: seq[CborEventEntry], + mainClass: string = "", +) {.compileTime, raises: [].} = + ## Writes the Python wrapper module (.py) for a CBOR-mode library. + ensureGeneratedOutputDir(outDir) + + let pyPath = + if outDir.len > 0: + outDir & "/" & libName & ".py" + else: + libName & ".py" + let p = libName & "_" + + # Derive Python class name from libName (PascalCase) the same way the + # native Python codegen does — gives the same public class name in both + # builds so a single test source can drive either. + var className = "" + block: + var capitalize = true + for ch in libName: + if ch == '_' or ch == '-': + capitalize = true + elif capitalize: + className.add(chr(ord(ch) - 32 * ord(ch in {'a' .. 'z'}))) + capitalize = false + else: + className.add(ch) + + # Note: type emission below walks `gApiTypeRegistry` directly so we + # cover every referenced object/enum/distinct, not just request + # response or event payload types. The objectNames seq populated + # later is what request/event method emission filters against. + + var py = + "# Generated by nim-brokers CBOR FFI codegen — do not edit.\n" & "#\n" & + "# Python wrapper around the C ABI declared in `" & libName & ".h`.\n" & + "# Requires Python 3.8+ and the `cbor2` package (pip install cbor2).\n" & "\n" & + "from __future__ import annotations\n" & "\n" & "import ctypes\n" & "import json\n" & + "import os\n" & "import platform\n" & "from dataclasses import dataclass, field\n" & + "from enum import IntEnum\n" & + "from typing import Any, Callable, Dict, Generic, List, Optional, TypeVar\n" & "\n" & + "import cbor2\n" & "\n\n" + + # Public-API interface summary, emitted right below the imports so the + # file reads as a self-documenting overview before the implementation. + # Same shape as the native Python wrapper's leading block — every + # request method and every event subscribe / unsubscribe pair appears + # with its full signature so a reader can scan the public surface + # without diving into the body. + py.add( + "# ---------------------------------------------------------------------------\n" + ) + py.add("# Public API surface (auto-generated from broker declarations)\n") + py.add( + "# ---------------------------------------------------------------------------\n" + ) + py.add("# class " & className & ":\n") + py.add("# version() -> str (@staticmethod)\n") + py.add("# __enter__() -> " & className & "\n") + py.add("# __exit__(*_) -> None\n") + py.add("# create_context() -> Result[None]\n") + py.add("# valid_context() -> bool\n") + py.add("# __bool__() -> bool\n") + py.add("# shutdown() -> None\n") + py.add("# ctx -> int (property)\n") + py.add("#\n") + py.add("# Each request method returns Result[] (use .is_ok() / .value /\n") + py.add("# .error). Each event has on_(callback) -> handle and\n") + py.add("# off_(handle = 0) -> None.\n") + py.add("#\n") + for e in requestEntries: + var sigParams = "" + for i, (n, t) in e.argFields.pairs: + if i > 0: + sigParams.add(", ") + sigParams.add(n & ": " & nimTypeToPyHint(t)) + py.add( + "# " & e.apiName & "(" & sigParams & ") -> Result[" & e.responseTypeName & "]\n" + ) + for ev in eventEntries: + py.add("# on_" & ev.apiName & "(callback) -> int\n") + py.add("# off_" & ev.apiName & "(handle = 0) -> None\n") + py.add("\n") + + # Library loading. + py.add( + "# ---------------------------------------------------------------------------\n" + ) + py.add("# Shared library loading\n") + py.add( + "# ---------------------------------------------------------------------------\n\n" + ) + py.add("def _resolve_library_path() -> str:\n") + py.add(" here = os.path.dirname(os.path.abspath(__file__))\n") + py.add(" sysname = platform.system()\n") + py.add(" if sysname == \"Windows\":\n") + py.add(" candidate = \"" & libName & ".dll\"\n") + py.add(" elif sysname == \"Darwin\":\n") + py.add(" candidate = \"lib" & libName & ".dylib\"\n") + py.add(" else:\n") + py.add(" candidate = \"lib" & libName & ".so\"\n") + py.add(" return os.path.join(here, candidate)\n\n") + py.add("_LIB = ctypes.CDLL(_resolve_library_path())\n\n") + + # C function signatures. + py.add( + "# ---------------------------------------------------------------------------\n" + ) + py.add("# C ABI bindings\n") + py.add( + "# ---------------------------------------------------------------------------\n\n" + ) + py.add("_LIB." & p & "version.argtypes = []\n") + py.add("_LIB." & p & "version.restype = ctypes.c_char_p\n\n") + py.add("_LIB." & p & "initialize.argtypes = []\n") + py.add("_LIB." & p & "initialize.restype = None\n\n") + py.add("_LIB." & p & "createContext.argtypes = [ctypes.POINTER(ctypes.c_char_p)]\n") + py.add("_LIB." & p & "createContext.restype = ctypes.c_uint32\n\n") + py.add("_LIB." & p & "shutdown.argtypes = [ctypes.c_uint32]\n") + py.add("_LIB." & p & "shutdown.restype = ctypes.c_int32\n\n") + # reduced-A: per-instance teardown (used by sub-wrapper close()). + py.add("_LIB." & p & "releaseInstance.argtypes = [ctypes.c_uint32]\n") + py.add("_LIB." & p & "releaseInstance.restype = ctypes.c_int32\n\n") + py.add("_LIB." & p & "allocBuffer.argtypes = [ctypes.c_int32]\n") + py.add("_LIB." & p & "allocBuffer.restype = ctypes.c_void_p\n\n") + py.add("_LIB." & p & "freeBuffer.argtypes = [ctypes.c_void_p]\n") + py.add("_LIB." & p & "freeBuffer.restype = None\n\n") + py.add("_LIB." & p & "call.argtypes = [\n") + py.add(" ctypes.c_uint32,\n") + py.add(" ctypes.c_char_p,\n") + py.add(" ctypes.c_void_p,\n") + py.add(" ctypes.c_int32,\n") + py.add(" ctypes.POINTER(ctypes.c_void_p),\n") + py.add(" ctypes.POINTER(ctypes.c_int32),\n") + py.add("]\n") + py.add("_LIB." & p & "call.restype = ctypes.c_int32\n\n") + + # Event callback type used for both subscribe and the trampolines below. + py.add("EVENT_CB_T = ctypes.CFUNCTYPE(\n") + py.add(" None,\n") + py.add(" ctypes.c_uint32, # ctx\n") + py.add(" ctypes.c_char_p, # eventName\n") + py.add(" ctypes.c_void_p, # payloadBuf\n") + py.add(" ctypes.c_int32, # payloadLen\n") + py.add(" ctypes.c_void_p, # userData\n") + py.add(")\n\n") + py.add("_LIB." & p & "subscribe.argtypes = [\n") + py.add(" ctypes.c_uint32,\n") + py.add(" ctypes.c_char_p,\n") + py.add(" EVENT_CB_T,\n") + py.add(" ctypes.c_void_p,\n") + py.add("]\n") + py.add("_LIB." & p & "subscribe.restype = ctypes.c_uint64\n\n") + py.add( + "_LIB." & p & + "unsubscribe.argtypes = [ctypes.c_uint32, ctypes.c_char_p, ctypes.c_uint64]\n" + ) + py.add("_LIB." & p & "unsubscribe.restype = ctypes.c_int32\n\n") + + py.add( + "_LIB." & p & + "listApis.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_int32)]\n" + ) + py.add("_LIB." & p & "listApis.restype = ctypes.c_int32\n\n") + py.add( + "_LIB." & p & + "getSchema.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_int32)]\n" + ) + py.add("_LIB." & p & "getSchema.restype = ctypes.c_int32\n\n") + + # Result helper. + py.add( + "# ---------------------------------------------------------------------------\n" + ) + py.add("# Result[T]\n") + py.add( + "# ---------------------------------------------------------------------------\n\n" + ) + py.add("T = TypeVar(\"T\")\n\n") + py.add("@dataclass\n") + py.add("class Result(Generic[T]):\n") + py.add(" \"\"\"Mirror of Nim's Result[T, string] envelope on the wire.\n\n") + py.add(" Use the `ok()` / `err()` factories to construct, never call\n") + py.add(" the dataclass constructor directly.\n") + py.add(" \"\"\"\n\n") + py.add(" _ok: bool = False\n") + py.add(" value: Optional[T] = None\n") + py.add(" error: str = \"\"\n\n") + py.add(" @classmethod\n") + py.add(" def ok(cls, value: T) -> \"Result[T]\":\n") + py.add(" return cls(_ok=True, value=value)\n\n") + py.add(" @classmethod\n") + py.add(" def err(cls, msg: str) -> \"Result[T]\":\n") + py.add(" return cls(_ok=False, error=msg)\n\n") + py.add(" def is_ok(self) -> bool:\n") + py.add(" return self._ok\n\n") + py.add(" def is_err(self) -> bool:\n") + py.add(" return not self._ok\n\n") + + # ----- Enums (atkEnum) ------------------------------------------------- + var enumNames: seq[string] = @[] + for entry in gApiTypeRegistry: + if entry.kind == atkEnum: + enumNames.add(entry.name) + + # ----- Distinct / alias (atkDistinct, atkAlias) ------------------------ + var aliasNames: seq[string] = @[] + for entry in gApiTypeRegistry: + if entry.kind in {atkDistinct, atkAlias}: + aliasNames.add(entry.name) + + # ----- Object types (atkObject) — emit all registered, not just + # response/event payload types, so seq[Tag] etc. resolve. ------------- + var objectNames: seq[string] = @[] + for entry in gApiTypeRegistry: + if entry.kind == atkObject and not entry.name.endsWith("CborArgs"): + objectNames.add(entry.name) + + # A "scalar payload" is a primitive (non-object) broker type — `type X = + # int32` — registered as a distinct alias of its underlying primitive. + # Its CBOR wire value is a bare scalar; the Python surface uses the + # `X = ` alias directly. Such a type is an emittable request + # response / event payload despite having no object fields. + proc isScalarPayload(name: string): bool {.compileTime.} = + name.len > 0 and isTypeRegistered(name) and + lookupTypeEntry(name).kind in {atkAlias, atkDistinct} and + primPyHint(resolveUnderlyingType(name)).len > 0 + + proc isEmittablePayload(name: string): bool {.compileTime.} = + name in objectNames or isScalarPayload(name) + + if enumNames.len > 0 or aliasNames.len > 0 or objectNames.len > 0: + py.add( + "# ---------------------------------------------------------------------------\n" + ) + py.add("# Generated payload types\n") + py.add( + "# ---------------------------------------------------------------------------\n\n" + ) + + # Enums. + for name in enumNames: + let entry = lookupTypeEntry(name) + py.add("class " & name & "(IntEnum):\n") + if entry.enumValues.len == 0: + py.add(" pass\n") + else: + for v in entry.enumValues: + py.add(" " & v.name & " = " & $v.ordinal & "\n") + py.add("\n") + py.add("def _decode_" & name & "(data: Any) -> " & name & ":\n") + py.add(" if isinstance(data, int):\n") + py.add(" try:\n") + py.add(" return " & name & "(data)\n") + py.add(" except ValueError:\n") + py.add(" return " & name & "(0)\n") + py.add(" return " & name & "(0)\n\n") + + # Distinct / alias — Python alias of the underlying primitive plus + # passthrough decode/encode helpers (so callers can freely use the + # alias name in type hints). + for name in aliasNames: + let underlying = resolveUnderlyingType(name) + let pyU = primPyHint(underlying) + if pyU.len == 0: + py.add( + "# TODO: alias '" & name & "' resolves to '" & underlying & + "' which has no Python primitive mapping\n\n" + ) + continue + py.add(name & " = " & pyU & "\n") + py.add("def _decode_" & name & "(data: Any) -> " & pyU & ":\n") + py.add(" return " & pyDecodeExpr(underlying, "data") & "\n\n") + py.add("def _encode_" & name & "(v: Any) -> " & pyU & ":\n") + py.add(" return " & pyEncodeExpr(underlying, "v") & "\n\n") + + # Objects: forward-declare names so per-type _decode/_encode helpers + # can reference each other regardless of declaration order. Python + # is lenient with forward refs inside def bodies, so emitting in + # registry order is enough. + for name in objectNames: + let entry = lookupTypeEntry(name) + py.add("@dataclass\n") + py.add("class " & name & ":\n") + var anyField = false + for f in entry.fields: + let hint = nimTypeToPyHint(f.nimType) + if hint.len == 0: + py.add(" # TODO: Nim type '" & f.nimType & "' not yet mappable\n") + continue + py.add( + " " & f.name & ": " & hint & " = " & nimTypeToPyDefault(f.nimType) & "\n" + ) + anyField = true + if not anyField: + py.add(" pass\n") + py.add("\n") + + # Per-object _decode and _encode helpers. + for name in objectNames: + let entry = lookupTypeEntry(name) + py.add("def _decode_" & name & "(data: Any) -> " & name & ":\n") + py.add(" if not isinstance(data, dict):\n") + py.add(" return " & name & "()\n") + py.add(" return " & name & "(\n") + for f in entry.fields: + if not isPyMappable(f.nimType): + continue + let raw = "data.get(\"" & f.name & "\")" + py.add(" " & f.name & "=" & pyDecodeExpr(f.nimType, raw) & ",\n") + py.add(" )\n\n") + + py.add("def _encode_" & name & "(v: Any) -> Dict[str, Any]:\n") + py.add(" if isinstance(v, dict):\n") + py.add(" return v\n") + py.add(" return {\n") + for f in entry.fields: + if not isPyMappable(f.nimType): + continue + py.add( + " \"" & f.name & "\": " & pyEncodeExpr(f.nimType, "v." & f.name) & ",\n" + ) + py.add(" }\n\n") + + # Lib class. + py.add( + "# ---------------------------------------------------------------------------\n" + ) + py.add("# Lib class\n") + py.add( + "# ---------------------------------------------------------------------------\n\n" + ) + py.add("class " & className & ":\n") + py.add(" \"\"\"Pythonic wrapper around the " & libName & " shared library.\n\n") + py.add(" Usage::\n\n") + py.add(" with " & className & "() as lib:\n") + py.add(" init = lib.create_context()\n") + py.add(" assert init.is_ok(), init.error\n") + py.add(" r = lib.echo_request(\"ping\")\n") + py.add(" \"\"\"\n\n") + + py.add(" @staticmethod\n") + py.add(" def version() -> str:\n") + py.add( + " \"\"\"Return the static semver string baked into the " & libName & + " library.\"\"\"\n" + ) + py.add(" raw = _LIB." & p & "version()\n") + py.add(" return raw.decode(\"utf-8\") if raw else \"\"\n\n") + + # reduced-A: ownership predicates — an entry belongs to the main class when no + # mainClass is designated (legacy single-class), or it is flat (no owning + # interface), or its owning interface IS the main class. + proc ownsReqMain(e: CborRequestEntry): bool = + if mainClass.len == 0: + return true + let o = interfaceOwningRequestType(e.responseTypeName) + o.len == 0 or o == mainClass + + proc ownsEvtMain(ev: CborEventEntry): bool = + if mainClass.len == 0: + return true + let o = interfaceOwningEventType(ev.typeName) + o.len == 0 or o == mainClass + + py.add(" def __init__(self) -> None:\n") + py.add(" _LIB." & p & "initialize()\n") + py.add(" self._ctx: int = 0\n") + py.add(" # Per-event handler maps + GC-anchor for trampolines.\n") + + # Per-event handler maps, initialised in __init__ (main-class events only). + for ev in eventEntries: + if not isEmittablePayload(ev.typeName): + continue + if mainClass.len > 0 and not ownsEvtMain(ev): + continue + let mapName = "_" & ev.apiName & "_handlers" + py.add(" self." & mapName & ": Dict[int, Any] = {}\n") + py.add("\n") + + py.add(" def create_context(self) -> Result[None]:\n") + py.add(" \"\"\"Create the library context. Result[None] — Result.ok(None)\n") + py.add(" on success, Result.err(msg) on failure.\"\"\"\n") + py.add(" if self._ctx != 0:\n") + py.add(" return Result.err(\"Context already created\")\n") + py.add(" err = ctypes.c_char_p()\n") + py.add(" ctx = _LIB." & p & "createContext(ctypes.byref(err))\n") + py.add(" if ctx == 0:\n") + py.add( + " msg = err.value.decode(\"utf-8\", errors=\"replace\") if err.value else \"createContext returned 0\"\n" + ) + py.add(" if err.value:\n") + py.add(" _LIB." & p & "freeBuffer(err)\n") + py.add(" return Result.err(msg)\n") + py.add(" self._ctx = ctx\n") + py.add(" return Result.ok(None)\n\n") + + py.add(" def valid_context(self) -> bool:\n") + py.add(" return self._ctx != 0\n\n") + py.add(" def __bool__(self) -> bool:\n") + py.add(" return self.valid_context()\n\n") + py.add(" @property\n") + py.add(" def ctx(self) -> int:\n") + py.add(" return self._ctx\n\n") + py.add(" def shutdown(self) -> None:\n") + py.add( + " \"\"\"Tear down the library context. Safe to call multiple times.\"\"\"\n" + ) + py.add(" if self._ctx:\n") + py.add(" _LIB." & p & "shutdown(self._ctx)\n") + py.add(" self._ctx = 0\n") + for ev in eventEntries: + if not isEmittablePayload(ev.typeName): + continue + if mainClass.len > 0 and not ownsEvtMain(ev): + continue + let mapName = "_" & ev.apiName & "_handlers" + py.add(" self." & mapName & ".clear()\n") + py.add("\n") + py.add(" def __enter__(self) -> \"" & className & "\":\n") + py.add(" return self\n\n") + py.add(" def __exit__(self, exc_type, exc, tb) -> None:\n") + py.add(" self.shutdown()\n\n") + py.add(" def __del__(self) -> None:\n") + py.add(" try:\n") + py.add(" self.shutdown()\n") + py.add(" except Exception:\n") + py.add(" pass\n\n") + + # Discovery API helpers (Phase 6). + py.add(" def list_apis(self) -> Dict[str, Any]:\n") + py.add(" \"\"\"Return the decoded ApiList describing the library surface.\n") + py.add(" Returns a dict parsed from the JSON response.\n") + py.add(" \"\"\"\n") + py.add( + " return self._fetch_descriptor(_LIB." & p & "listApis, \"listApis\")\n\n" + ) + py.add(" def get_schema(self) -> Dict[str, Any]:\n") + py.add( + " \"\"\"Return the decoded LibraryDescriptor (schema + CDDL text).\"\"\"\n" + ) + py.add( + " return self._fetch_descriptor(_LIB." & p & "getSchema, \"getSchema\")\n\n" + ) + py.add(" def _fetch_descriptor(self, fn, label: str) -> Dict[str, Any]:\n") + py.add(" resp_buf = ctypes.c_void_p()\n") + py.add(" resp_len = ctypes.c_int32()\n") + py.add(" status = fn(ctypes.byref(resp_buf), ctypes.byref(resp_len))\n") + py.add(" if status != 0:\n") + py.add(" raise RuntimeError(f\"{label} framework error: {status}\")\n") + py.add(" if not resp_buf or resp_len.value <= 0:\n") + py.add(" return {}\n") + py.add(" try:\n") + py.add(" payload = ctypes.string_at(resp_buf, resp_len.value)\n") + py.add(" finally:\n") + py.add(" _LIB." & p & "freeBuffer(resp_buf)\n") + py.add(" return json.loads(payload.decode('utf-8'))\n\n") + + # Helper: do a sync call. + py.add( + " def _do_call(self, api_name: str, req_payload: bytes) -> Optional[Dict[str, Any]]:\n" + ) + py.add( + " \"\"\"Dispatch a CBOR request and return the decoded envelope dict,\n" + ) + py.add(" or None on framework error (which raises a RuntimeError).\n") + py.add(" \"\"\"\n") + py.add(" in_buf = None\n") + py.add(" if req_payload:\n") + py.add(" in_buf = _LIB." & p & "allocBuffer(len(req_payload))\n") + py.add(" if not in_buf:\n") + py.add(" raise RuntimeError(\"allocBuffer failed\")\n") + py.add(" ctypes.memmove(in_buf, req_payload, len(req_payload))\n") + py.add(" resp_buf = ctypes.c_void_p()\n") + py.add(" resp_len = ctypes.c_int32()\n") + py.add(" status = _LIB." & p & "call(\n") + py.add(" self._ctx,\n") + py.add(" api_name.encode(\"utf-8\"),\n") + py.add(" in_buf,\n") + py.add(" len(req_payload),\n") + py.add(" ctypes.byref(resp_buf),\n") + py.add(" ctypes.byref(resp_len),\n") + py.add(" )\n") + py.add(" out: bytes = b\"\"\n") + py.add(" if resp_buf and resp_len.value > 0:\n") + py.add(" out = ctypes.string_at(resp_buf, resp_len.value)\n") + py.add(" _LIB." & p & "freeBuffer(resp_buf)\n") + py.add(" if status != 0:\n") + py.add(" if status == -4 and out:\n") + py.add( + " raise RuntimeError(out.decode(\"utf-8\", errors=\"replace\"))\n" + ) + py.add(" raise RuntimeError(f\"framework error: {status}\")\n") + py.add(" return cbor2.loads(out) if out else None\n\n") + + # reduced-A: a request method's body, indented for a wrapper class. Handles + # both normal requests (decode the typed payload) and instance-returning + # requests (Ok value is a uint32 ctx → construct the typed sub-wrapper). + # Reused by the main class and each sub-interface class. + proc emitReqMethod(e: CborRequestEntry): string = + if e.responseTypeName.len == 0: + return "" + var argsMappable = true + for (n, t) in e.argFields: + if not isPyMappable(t): + argsMappable = false + break + if e.returnsInterface.len > 0: + if not argsMappable: + return " # TODO: '" & e.apiName & "' has unmappable parameter types.\n\n" + let sub = subClassName(e.returnsInterface) + var sigParams = "self" + var argsDictBuilder = "{}" + if e.argFields.len > 0: + var dictParts = "" + for i, (n, t) in e.argFields.pairs: + sigParams.add(", " & n & ": " & nimTypeToPyHint(t)) + if i > 0: + dictParts.add(", ") + dictParts.add("\"" & n & "\": " & pyEncodeExpr(t, n)) + argsDictBuilder = "{" & dictParts & "}" + result.add( + " def " & e.apiName & "(" & sigParams & ") -> Result[\"" & sub & "\"]:\n" + ) + result.add(" if self._ctx == 0:\n") + result.add(" return Result.err(\"Library context is not created\")\n") + if e.argFields.len > 0: + result.add(" req_payload = cbor2.dumps(" & argsDictBuilder & ")\n") + else: + result.add(" req_payload = b\"\"\n") + result.add( + " try:\n" & " envelope = self._do_call(\"" & e.apiName & + "\", req_payload)\n" & " except RuntimeError as exc:\n" & + " return Result.err(str(exc))\n" + ) + result.add( + " if envelope is None or not isinstance(envelope, dict):\n" & + " return Result.err(\"empty or malformed response envelope\")\n" & + " if envelope.get(\"err\") is not None:\n" & + " return Result.err(str(envelope[\"err\"]))\n" & + " return Result.ok(" & sub & "(int(envelope.get(\"ok\"))))\n\n" + ) + return result + if not isEmittablePayload(e.responseTypeName): + return + " # TODO: '" & e.apiName & "' return type '" & e.responseTypeName & + "' is not a registered object type.\n\n" + if not argsMappable: + return + " # TODO: '" & e.apiName & + "' has parameters whose Nim types aren't yet mappable to Python.\n\n" + let methodName = e.apiName + var sigParams = "self" + var argsDictBuilder = "{}" + if e.argFields.len > 0: + var dictParts = "" + for i, (n, t) in e.argFields.pairs: + sigParams.add(", " & n & ": " & nimTypeToPyHint(t)) + if i > 0: + dictParts.add(", ") + dictParts.add("\"" & n & "\": " & pyEncodeExpr(t, n)) + argsDictBuilder = "{" & dictParts & "}" + result.add( + " def " & methodName & "(" & sigParams & ") -> Result[" & e.responseTypeName & + "]:\n" + ) + result.add(" if self._ctx == 0:\n") + result.add(" return Result.err(\"Library context is not created\")\n") + if e.argFields.len > 0: + result.add(" req_payload = cbor2.dumps(" & argsDictBuilder & ")\n") + else: + result.add(" req_payload = b\"\"\n") + result.add( + " try:\n" & " envelope = self._do_call(\"" & e.apiName & + "\", req_payload)\n" & " except RuntimeError as exc:\n" & + " return Result.err(str(exc))\n" + ) + result.add( + " if envelope is None or not isinstance(envelope, dict):\n" & + " return Result.err(\"empty or malformed response envelope\")\n" & + " if envelope.get(\"err\") is not None:\n" & + " return Result.err(str(envelope[\"err\"]))\n" & + " return Result.ok(_decode_" & e.responseTypeName & + "(envelope.get(\"ok\")))\n\n" + ) + + # Per-request typed methods (main class). + for e in requestEntries: + if not ownsReqMain(e): + continue + py.add(emitReqMethod(e)) + + # Per-event subscribe / unsubscribe (main-class events only). + for ev in eventEntries: + if not ownsEvtMain(ev): + continue + if not isEmittablePayload(ev.typeName): + py.add( + " # TODO: event '" & ev.apiName & "' payload type '" & ev.typeName & + "' is not a registered object type.\n\n" + ) + continue + let mapName = "_" & ev.apiName & "_handlers" + let onName = "on_" & ev.apiName + let offName = "off_" & ev.apiName + + # Build per-field type hints + per-field destructure args. The user + # callback signature is `(, *unpacked_field_types) -> None` + # — parity with the C++ wrapper and the native-FFI Python wrapper. + var hintParts: seq[string] = @[className] + var destructureArgs: seq[string] = @["self"] + if isScalarPayload(ev.typeName): + # Scalar payload: the decoded `evt` IS the value — one bare arg. + hintParts.add(primPyHint(resolveUnderlyingType(ev.typeName))) + destructureArgs.add("evt") + else: + for f in lookupTypeEntry(ev.typeName).fields: + hintParts.add(nimTypeToPyHint(f.nimType)) + destructureArgs.add("evt." & f.name) + let pyCallableHint = "Callable[[" & hintParts.join(", ") & "], None]" + + py.add(" def " & onName & "(self, callback: " & pyCallableHint & ") -> int:\n") + py.add( + " \"\"\"Subscribe to '" & ev.apiName & + "' events. Returns a handle (>=2) on success, 0 on failure.\n" + ) + py.add(" The callback receives the owning library instance as its\n") + py.add(" first argument followed by the unpacked event payload\n") + py.add(" fields.\"\"\"\n") + py.add(" if self._ctx == 0:\n") + py.add(" return 0\n") + py.add(" def trampoline(\n") + py.add(" ctx: int, name: bytes, buf: int, buf_len: int, _ud: int\n") + py.add(" ) -> None:\n") + py.add(" if not buf or buf_len <= 0:\n") + py.add(" return\n") + py.add(" try:\n") + py.add(" payload = ctypes.string_at(buf, buf_len)\n") + py.add(" data = cbor2.loads(payload)\n") + py.add(" evt = _decode_" & ev.typeName & "(data)\n") + py.add(" callback(" & destructureArgs.join(", ") & ")\n") + py.add(" except Exception:\n") + py.add(" # Swallow handler errors so they don't escape\n") + py.add(" # back across the C ABI boundary.\n") + py.add(" pass\n") + py.add(" cb = EVENT_CB_T(trampoline)\n") + py.add( + " h = _LIB." & p & "subscribe(self._ctx, b\"" & ev.apiName & + "\", cb, None)\n" + ) + py.add(" if h == 0 or h == 1:\n") + py.add(" return h\n") + py.add(" # Hold a reference to both the CFUNCTYPE wrapper and the\n") + py.add(" # user callback — without this the Python GC would free\n") + py.add(" # the trampoline before the C side fires it.\n") + py.add(" self." & mapName & "[h] = (cb, callback)\n") + py.add(" return h\n\n") + + py.add(" def " & offName & "(self, handle: int = 0) -> None:\n") + py.add( + " \"\"\"Unsubscribe from '" & ev.apiName & + "' events. handle=0 removes all.\"\"\"\n" + ) + py.add(" if self._ctx == 0:\n") + py.add(" return\n") + py.add( + " _LIB." & p & "unsubscribe(self._ctx, b\"" & ev.apiName & "\", handle)\n" + ) + py.add(" if handle == 0:\n") + py.add(" self." & mapName & ".clear()\n") + py.add(" else:\n") + py.add(" self." & mapName & ".pop(handle, None)\n\n") + + # reduced-A: per-sub-interface wrapper classes. Each non-main + # BrokerInterface(API) with at least one request entry gets its own class. A + # sub-instance is created by a main create-instance method (returns Result of + # the sub class), bound to its routing ctx. The sub class shares the single C + # ABI: its _do_call uses self._ctx, which the library routes by classCtx to + # the same processing thread. close() calls _releaseInstance(ctx). + if mainClass.len > 0: + # Collect distinct non-main owning interfaces directly from the entries. + # (Deriving from interfaceOwningRequestType avoids returning the compile- + # time registry seq by value, which the Nim VM aliases to an empty copy.) + var subNames: seq[string] = @[] + for e in requestEntries: + let o = interfaceOwningRequestType(e.responseTypeName) + if o.len > 0 and o != mainClass and o notin subNames: + subNames.add(o) + for ifaceName in subNames: + var ifaceReqs: seq[CborRequestEntry] = @[] + for e in requestEntries: + if interfaceOwningRequestType(e.responseTypeName) == ifaceName: + ifaceReqs.add(e) + if ifaceReqs.len == 0: + continue + let sub = subClassName(ifaceName) + py.add( + "# ---------------------------------------------------------------------------\n" + ) + py.add( + "# " & sub & " — sub-instance wrapper (created via a " & mainClass & + " request)\n" + ) + py.add( + "# ---------------------------------------------------------------------------\n\n" + ) + py.add("class " & sub & ":\n") + py.add(" \"\"\"Sub-interface instance of " & ifaceName & ".\n\n") + py.add(" Lives on the library's processing thread; obtained from a main\n") + py.add(" create-instance method. Call close() (or use as a context\n") + py.add(" manager) to release it — drops its providers/listeners.\"\"\"\n\n") + py.add(" def __init__(self, ctx: int) -> None:\n") + py.add(" self._ctx: int = ctx\n") + # Initialize per-event handler maps for sub-interface events. + for ev in eventEntries: + if interfaceOwningEventType(ev.typeName) != ifaceName: + continue + if not isEmittablePayload(ev.typeName): + continue + let mapName = "_" & ev.apiName & "_handlers" + py.add(" self." & mapName & ": Dict[int, Any] = {}\n") + py.add("\n") + py.add(" @property\n") + py.add(" def ctx(self) -> int:\n") + py.add(" return self._ctx\n\n") + py.add(" def valid(self) -> bool:\n") + py.add(" return self._ctx != 0\n\n") + py.add(" def __bool__(self) -> bool:\n") + py.add(" return self._ctx != 0\n\n") + # _do_call (same body as the main class; routes by self._ctx). + py.add( + " def _do_call(self, api_name: str, req_payload: bytes) -> Optional[Dict[str, Any]]:\n" + ) + py.add(" in_buf = None\n") + py.add(" if req_payload:\n") + py.add(" in_buf = _LIB." & p & "allocBuffer(len(req_payload))\n") + py.add(" if not in_buf:\n") + py.add(" raise RuntimeError(\"allocBuffer failed\")\n") + py.add(" ctypes.memmove(in_buf, req_payload, len(req_payload))\n") + py.add(" resp_buf = ctypes.c_void_p()\n") + py.add(" resp_len = ctypes.c_int32()\n") + py.add(" status = _LIB." & p & "call(\n") + py.add(" self._ctx,\n") + py.add(" api_name.encode(\"utf-8\"),\n") + py.add(" in_buf,\n") + py.add(" len(req_payload),\n") + py.add(" ctypes.byref(resp_buf),\n") + py.add(" ctypes.byref(resp_len),\n") + py.add(" )\n") + py.add(" out: bytes = b\"\"\n") + py.add(" if resp_buf and resp_len.value > 0:\n") + py.add(" out = ctypes.string_at(resp_buf, resp_len.value)\n") + py.add(" _LIB." & p & "freeBuffer(resp_buf)\n") + py.add(" if status != 0:\n") + py.add(" if status == -4 and out:\n") + py.add( + " raise RuntimeError(out.decode(\"utf-8\", errors=\"replace\"))\n" + ) + py.add(" raise RuntimeError(f\"framework error: {status}\")\n") + py.add(" return cbor2.loads(out) if out else None\n\n") + for e in ifaceReqs: + py.add(emitReqMethod(e)) + # Sub-interface event methods (subscribe/unsubscribe keyed by self._ctx). + for ev in eventEntries: + if interfaceOwningEventType(ev.typeName) != ifaceName: + continue + if not isEmittablePayload(ev.typeName): + py.add( + " # TODO: event '" & ev.apiName & "' payload type '" & ev.typeName & + "' is not a registered object type.\n\n" + ) + continue + let mapName = "_" & ev.apiName & "_handlers" + let onName = "on_" & ev.apiName + let offName = "off_" & ev.apiName + var hintParts: seq[string] = @[sub] + var destructureArgs: seq[string] = @["self"] + if isScalarPayload(ev.typeName): + hintParts.add(primPyHint(resolveUnderlyingType(ev.typeName))) + destructureArgs.add("evt") + else: + for f in lookupTypeEntry(ev.typeName).fields: + hintParts.add(nimTypeToPyHint(f.nimType)) + destructureArgs.add("evt." & f.name) + let pyCallableHint = "Callable[[" & hintParts.join(", ") & "], None]" + py.add( + " def " & onName & "(self, callback: " & pyCallableHint & ") -> int:\n" + ) + py.add( + " \"\"\"Subscribe to '" & ev.apiName & + "' events. Returns a handle (>=2) on success, 0 on failure.\"\"\"\n" + ) + py.add(" if self._ctx == 0:\n") + py.add(" return 0\n") + py.add(" def trampoline(\n") + py.add(" ctx: int, name: bytes, buf: int, buf_len: int, _ud: int\n") + py.add(" ) -> None:\n") + py.add(" if not buf or buf_len <= 0:\n") + py.add(" return\n") + py.add(" try:\n") + py.add(" payload = ctypes.string_at(buf, buf_len)\n") + py.add(" data = cbor2.loads(payload)\n") + py.add(" evt = _decode_" & ev.typeName & "(data)\n") + py.add(" callback(" & destructureArgs.join(", ") & ")\n") + py.add(" except Exception:\n") + py.add(" pass\n") + py.add(" cb = EVENT_CB_T(trampoline)\n") + py.add( + " h = _LIB." & p & "subscribe(self._ctx, b\"" & ev.apiName & + "\", cb, None)\n" + ) + py.add(" if h == 0 or h == 1:\n") + py.add(" return h\n") + py.add(" self." & mapName & "[h] = (cb, callback)\n") + py.add(" return h\n\n") + py.add(" def " & offName & "(self, handle: int = 0) -> None:\n") + py.add( + " \"\"\"Unsubscribe from '" & ev.apiName & + "' events. handle=0 removes all.\"\"\"\n" + ) + py.add(" if self._ctx == 0:\n") + py.add(" return\n") + py.add( + " _LIB." & p & "unsubscribe(self._ctx, b\"" & ev.apiName & + "\", handle)\n" + ) + py.add(" if handle == 0:\n") + py.add(" self." & mapName & ".clear()\n") + py.add(" else:\n") + py.add(" self." & mapName & ".pop(handle, None)\n\n") + py.add(" def close(self) -> None:\n") + py.add(" \"\"\"Release this sub-instance (idempotent).\"\"\"\n") + py.add(" if self._ctx:\n") + py.add(" _LIB." & p & "releaseInstance(self._ctx)\n") + py.add(" self._ctx = 0\n") + for ev in eventEntries: + if interfaceOwningEventType(ev.typeName) != ifaceName: + continue + if not isEmittablePayload(ev.typeName): + continue + let mapName = "_" & ev.apiName & "_handlers" + py.add(" self." & mapName & ".clear()\n") + py.add("\n") + py.add(" def __enter__(self) -> \"" & sub & "\":\n") + py.add(" return self\n\n") + py.add(" def __exit__(self, exc_type, exc, tb) -> None:\n") + py.add(" self.close()\n\n") + py.add(" def __del__(self) -> None:\n") + py.add(" try:\n") + py.add(" self.close()\n") + py.add(" except Exception:\n") + py.add(" pass\n\n") + + try: + writeFile(pyPath, py) + except IOError: + error( + "Failed to write generated CBOR Python wrapper '" & pyPath & "': " & + getCurrentExceptionMsg() + ) + +{.push raises: [].} +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_codegen_cbor_rust.nim b/wasm-deps/brokers/brokers/internal/api_codegen_cbor_rust.nim new file mode 100644 index 000000000..62fe525d8 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_codegen_cbor_rust.nim @@ -0,0 +1,1043 @@ +## Generated Rust wrapper for the CBOR FFI surface. +## +## The generated `_rs/` Cargo crate ships alongside the shared library +## and uses `ciborium` + `serde` for CBOR encode/decode of typed payloads +## plus `serde_json` for the discovery endpoints (`list_apis`, `get_schema`) +## which return JSON. Foreign Rust projects only need the three crate +## dependencies — no other tooling. +## +## The wrapper emits typed `#[derive(Serialize, Deserialize)]` structs for +## each registered request response, request args, and event payload type, +## plus per-request methods on the libname-PascalCase wrapper that +## CBOR-encode the args, dispatch through the C ABI, and decode the +## response envelope into a `Result`. Per-event +## `on_(callback) -> u64` methods register a typed closure; the +## library holds a `Mutex>` per event keyed by handle so +## the trampoline can dispatch back to user code, mirroring the C++ +## `EventDispatcher` GC anchor. +## +## Type-matrix coverage: +## - Primitives: bool, int/intN, uint/uintN/byte, float/floatN, string, +## char. +## - Enums (atkEnum) → `#[repr(i32)]` Rust enums with `From` impls. +## - Distinct/Alias (atkDistinct/atkAlias) → Rust `pub type X = Y;` +## aliases of the resolved underlying type. +## - Registered objects → `#[derive(Serialize, Deserialize, Clone, Debug, +## Default)] pub struct` with typed fields. +## - Composite types: seq[T], array[N, T] (typed as Vec), +## including seq[byte] and seq[]; nested objects. +## Unmappable types still produce a TODO stub so the wrapper compiles. + +{.push raises: [].} + +import std/[macros, strutils, tables] +import ./api_common, ./api_schema +import ./helper/broker_utils # reduced-A: per-interface partitioning + +# --------------------------------------------------------------------------- +# Nim → Rust type mapping (registry-aware) +# --------------------------------------------------------------------------- + +const rustPrimMap = { + "bool": "bool", + "string": "String", + "char": "String", + "int": "i32", + "int8": "i8", + "int16": "i16", + "int32": "i32", + "int64": "i64", + "uint": "u32", + "uint8": "u8", + "uint16": "u16", + "uint32": "u32", + "uint64": "u64", + "byte": "u8", + "float": "f64", + "float32": "f32", + "float64": "f64", +}.toTable + +proc isRustPrimitive(nimType: string): bool {.compileTime.} = + nimType.strip() in rustPrimMap + +proc primRustHint(nimType: string): string {.compileTime.} = + rustPrimMap.getOrDefault(nimType.strip(), "") + +proc unwrapBracket(s, head: string): string {.compileTime.} = + let t = s.strip() + t[head.len + 1 .. ^2].strip() + +proc parseArrayInner(s: string): string {.compileTime.} = + let inner = s.strip()[6 ..^ 2] + let comma = inner.find(',') + if comma < 0: + return "" + inner[comma + 1 .. ^1].strip() + +proc nimTypeToRustHint*(nimType: string): string {.compileTime.} = + ## Recursive Nim → Rust type. Falls back to "" for types we can't yet map. + let t = nimType.strip() + let lower = t.toLowerAscii() + if isRustPrimitive(t): + return primRustHint(t) + if lower.startsWith("seq[") and lower.endsWith("]"): + let inner = nimTypeToRustHint(unwrapBracket(t, "seq")) + return + if inner.len > 0: + "Vec<" & inner & ">" + else: + "Vec" + if lower.startsWith("array["): + let elem = parseArrayInner(t) + let inner = nimTypeToRustHint(elem) + return + if inner.len > 0: + "Vec<" & inner & ">" + else: + "Vec" + if lower.startsWith("option[") and lower.endsWith("]"): + let inner = nimTypeToRustHint(unwrapBracket(t, "option")) + return + if inner.len > 0: + "Option<" & inner & ">" + else: + "Option" + if isTypeRegistered(t): + let entry = lookupTypeEntry(t) + case entry.kind + of atkObject, atkEnum: + return t + of atkAlias, atkDistinct: + # Recurse via outer mapper so distinct/alias over compound types + # (e.g. `distinct seq[byte]`) maps to `Vec` rather than the "" fallback. + return nimTypeToRustHint(resolveUnderlyingType(t)) + "" + +proc nimTypeToRustDefaultHint*(nimType: string): string {.compileTime.} = + ## Returns a Rust default expression for a struct field initializer + ## (used in `Default::default()` derivations — the generated structs + ## use `#[derive(Default)]` so this is mainly informational, but + ## emitted as part of TODO comments). + let t = nimType.strip() + let lower = t.toLowerAscii() + case t + of "bool": + return "false" + of "string", "char": + return "String::new()" + of "int", "int8", "int16", "int32", "int64", "uint", "uint8", "byte", "uint16", + "uint32", "uint64": + return "0" + of "float32", "float", "float64": + return "0.0" + else: + discard + if lower.startsWith("seq[") or lower.startsWith("array["): + return "Vec::new()" + if lower.startsWith("option["): + return "None" + if isTypeRegistered(t): + let entry = lookupTypeEntry(t) + case entry.kind + of atkObject: + return "Default::default()" + of atkEnum: + return t & "::default()" + of atkAlias, atkDistinct: + return nimTypeToRustDefaultHint(resolveUnderlyingType(t)) + "Default::default()" + +proc isRustMappable*(nimType: string): bool {.compileTime.} = + nimTypeToRustHint(nimType).len > 0 + +# --------------------------------------------------------------------------- +# File emission +# --------------------------------------------------------------------------- + +{.pop.} + +proc cborRustClassName(libName: string): string {.compileTime.} = + result = "" + var capitalize = true + for ch in libName: + if ch == '_' or ch == '-': + capitalize = true + elif capitalize: + result.add(chr(ord(ch) - 32 * ord(ch in {'a' .. 'z'}))) + capitalize = false + else: + result.add(ch) + +proc rustSubStructName(iface: string): string {.compileTime.} = + ## Wrapper struct name for a sub-interface: strip a leading `I` before an + ## uppercase letter (IWidget -> Widget), else use the name as-is. + if iface.len > 1 and iface[0] == 'I' and iface[1] in {'A' .. 'Z'}: + iface[1 ..^ 1] + else: + iface + +proc generateCborRustFile*( + outDir: string, + libName: string, + requestEntries: seq[CborRequestEntry], + eventEntries: seq[CborEventEntry], + mainClass: string = "", +) {.compileTime, raises: [].} = + ## Writes the Rust wrapper crate (Cargo.toml + src/lib.rs) for a + ## CBOR-mode library under `/_rs/`. + ensureGeneratedOutputDir(outDir) + + # reduced-A: per-interface partition. Sub-interface names are derived from the + # entries via interfaceOwningRequestType (NOT apiInterfaces() — the VM aliases + # a by-value seq return to an empty copy). + proc ownsReqMain(e: CborRequestEntry): bool {.compileTime.} = + if mainClass.len == 0: + return true + let o = interfaceOwningRequestType(e.responseTypeName) + o.len == 0 or o == mainClass + + proc ownsEvtMain(ev: CborEventEntry): bool {.compileTime.} = + if mainClass.len == 0: + return true + let o = interfaceOwningEventType(ev.typeName) + o.len == 0 or o == mainClass + + var subInterfaceNames: seq[string] = @[] + if mainClass.len > 0: + for e in requestEntries: + let o = interfaceOwningRequestType(e.responseTypeName) + if o.len > 0 and o != mainClass and o notin subInterfaceNames: + subInterfaceNames.add(o) + let crateDir = + if outDir.len > 0: + outDir & "/" & libName & "_rs" + else: + libName & "_rs" + let srcDir = crateDir & "/src" + ensureGeneratedOutputDir(crateDir) + ensureGeneratedOutputDir(srcDir) + + let className = cborRustClassName(libName) + let p = libName & "_" + + # ---------------------- Cargo.toml ---------------------- + var cargo = "# Generated by nim-brokers CBOR FFI Rust codegen — do not edit.\n" + cargo.add("[package]\n") + cargo.add("name = \"" & libName & "\"\n") + cargo.add("version = \"0.1.0\"\n") + cargo.add("edition = \"2021\"\n") + cargo.add("rust-version = \"1.75\"\n\n") + cargo.add("[lib]\n") + cargo.add("name = \"" & libName & "\"\n") + cargo.add("crate-type = [\"rlib\"]\n\n") + cargo.add("[dependencies]\n") + cargo.add("ciborium = \"0.2\"\n") + cargo.add("serde = { version = \"1\", features = [\"derive\"] }\n") + cargo.add("serde_bytes = \"0.11\"\n") + cargo.add("serde_json = \"1\"\n") + try: + writeFile(crateDir & "/Cargo.toml", cargo) + except IOError: + error( + "Failed to write generated CBOR Rust Cargo.toml '" & crateDir & "/Cargo.toml': " & + getCurrentExceptionMsg() + ) + + # ---------------------- src/lib.rs ---------------------- + var rs = "// Generated by nim-brokers CBOR FFI Rust codegen — do not edit.\n" + rs.add("//\n") + rs.add("// Rust wrapper around the C ABI declared in `" & libName & ".h`.\n") + rs.add("// Requires Rust 1.75+ and the `ciborium` + `serde` + `serde_json` crates.\n") + rs.add("//\n") + rs.add("// Public API surface (auto-generated from broker declarations):\n") + rs.add("// pub fn version() -> String (associated)\n") + rs.add("// pub fn new() -> Self\n") + rs.add("// pub fn create_context(&mut self) -> Result<()>\n") + rs.add("// pub fn valid_context(&self) -> bool\n") + rs.add("// pub fn shutdown(&mut self)\n") + rs.add("// pub fn ctx(&self) -> u32\n") + rs.add("//\n") + rs.add("// Each request method returns Result. Each event has\n") + rs.add("// on_(callback) -> u64 and off_(handle).\n") + rs.add("//\n") + for e in requestEntries: + var sigParams = "" + for i, (n, t) in e.argFields.pairs: + if i > 0: + sigParams.add(", ") + sigParams.add(n & ": " & nimTypeToRustHint(t)) + rs.add( + "// " & e.apiName & "(" & sigParams & ") -> Result<" & e.responseTypeName & ">\n" + ) + for ev in eventEntries: + rs.add("// on_" & ev.apiName & "(callback) -> u64\n") + rs.add("// off_" & ev.apiName & "(handle)\n") + rs.add("\n") + + rs.add("#![allow(non_camel_case_types)]\n") + rs.add("#![allow(non_snake_case)]\n") + rs.add("#![allow(non_upper_case_globals)]\n") + rs.add("#![allow(dead_code)]\n") + rs.add("#![allow(unused_imports)]\n") + rs.add("#![allow(clippy::missing_safety_doc)]\n\n") + + rs.add("use serde::{Deserialize, Serialize};\n") + rs.add("use std::collections::HashMap;\n") + rs.add("use std::ffi::{CStr, CString};\n") + rs.add("use std::os::raw::{c_char, c_int, c_void};\n") + rs.add("use std::sync::{Arc, Mutex, OnceLock};\n\n") + + # ---- extern "C" bindings --------------------------------------------- + rs.add("// -------- C ABI bindings --------\n\n") + rs.add("extern \"C\" {\n") + rs.add(" fn " & p & "version() -> *const c_char;\n") + rs.add(" fn " & p & "initialize();\n") + rs.add(" fn " & p & "createContext(err: *mut *const c_char) -> u32;\n") + rs.add(" fn " & p & "shutdown(ctx: u32) -> i32;\n") + rs.add(" fn " & p & "releaseInstance(ctx: u32) -> i32;\n") + rs.add(" fn " & p & "allocBuffer(size: i32) -> *mut c_void;\n") + rs.add(" fn " & p & "freeBuffer(p: *mut c_void);\n") + rs.add(" fn " & p & "call(\n") + rs.add(" ctx: u32,\n") + rs.add(" api_name: *const c_char,\n") + rs.add(" in_buf: *const c_void,\n") + rs.add(" in_len: i32,\n") + rs.add(" out_buf: *mut *mut c_void,\n") + rs.add(" out_len: *mut i32,\n") + rs.add(" ) -> i32;\n") + rs.add(" fn " & p & "subscribe(\n") + rs.add(" ctx: u32,\n") + rs.add(" event_name: *const c_char,\n") + rs.add(" cb: EventCb,\n") + rs.add(" user_data: *mut c_void,\n") + rs.add(" ) -> u64;\n") + rs.add( + " fn " & p & + "unsubscribe(ctx: u32, event_name: *const c_char, handle: u64) -> i32;\n" + ) + rs.add( + " fn " & p & "listApis(out_buf: *mut *mut c_void, out_len: *mut i32) -> i32;\n" + ) + rs.add( + " fn " & p & "getSchema(out_buf: *mut *mut c_void, out_len: *mut i32) -> i32;\n" + ) + rs.add("}\n\n") + + rs.add( + "pub type EventCb = unsafe extern \"C\" fn(ctx: u32, name: *const c_char, buf: *const c_void, buf_len: i32, ud: *mut c_void);\n\n" + ) + + # ---- Result envelope ------------------------------------------------- + rs.add("/// Mirror of Nim's `Result[T, string]` envelope on the wire.\n") + rs.add("#[derive(Debug, Clone)]\n") + rs.add("pub struct Result {\n") + rs.add(" inner: ::std::result::Result,\n") + rs.add("}\n\n") + rs.add("impl Result {\n") + rs.add(" pub fn ok(value: T) -> Self { Self { inner: Ok(value) } }\n") + rs.add( + " pub fn err>(msg: S) -> Self { Self { inner: Err(msg.into()) } }\n" + ) + rs.add(" pub fn is_ok(&self) -> bool { self.inner.is_ok() }\n") + rs.add(" pub fn is_err(&self) -> bool { self.inner.is_err() }\n") + rs.add(" pub fn value(&self) -> Option<&T> { self.inner.as_ref().ok() }\n") + rs.add( + " pub fn error(&self) -> Option<&str> { self.inner.as_ref().err().map(|s| s.as_str()) }\n" + ) + rs.add( + " pub fn into_result(self) -> ::std::result::Result { self.inner }\n" + ) + rs.add("}\n\n") + + # ---- Generated payload types ----------------------------------------- + var enumNames: seq[string] = @[] + for entry in gApiTypeRegistry: + if entry.kind == atkEnum: + enumNames.add(entry.name) + var aliasNames: seq[string] = @[] + for entry in gApiTypeRegistry: + if entry.kind in {atkDistinct, atkAlias}: + aliasNames.add(entry.name) + var objectNames: seq[string] = @[] + for entry in gApiTypeRegistry: + if entry.kind == atkObject and not entry.name.endsWith("CborArgs"): + objectNames.add(entry.name) + + # A "scalar payload" is a primitive (non-object) broker type — `type X = + # int32` — registered as a distinct alias of its underlying primitive. + # Its CBOR wire value is a bare scalar; the Rust surface uses the + # `pub type X = ` alias directly. Such a type is an emittable + # request response / event payload despite having no object fields. + proc isScalarPayload(name: string): bool {.compileTime.} = + name.len > 0 and isTypeRegistered(name) and + lookupTypeEntry(name).kind in {atkAlias, atkDistinct} and + primRustHint(resolveUnderlyingType(name)).len > 0 + + proc isEmittablePayload(name: string): bool {.compileTime.} = + name in objectNames or isScalarPayload(name) + + if enumNames.len > 0 or aliasNames.len > 0 or objectNames.len > 0: + rs.add("// -------- Generated payload types --------\n\n") + + # Enums. + for name in enumNames: + let entry = lookupTypeEntry(name) + rs.add("#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]\n") + rs.add("#[repr(i32)]\n") + rs.add("#[serde(into = \"i32\", from = \"i32\")]\n") + rs.add("pub enum " & name & " {\n") + if entry.enumValues.len == 0: + rs.add(" Unknown = 0,\n") + else: + for v in entry.enumValues: + rs.add(" " & v.name & " = " & $v.ordinal & ",\n") + rs.add("}\n\n") + rs.add("impl Default for " & name & " {\n") + if entry.enumValues.len == 0: + rs.add(" fn default() -> Self { " & name & "::Unknown }\n") + else: + rs.add( + " fn default() -> Self { " & name & "::" & entry.enumValues[0].name & " }\n" + ) + rs.add("}\n\n") + rs.add("impl From for " & name & " {\n") + rs.add(" fn from(v: i32) -> Self {\n") + rs.add(" match v {\n") + for v in entry.enumValues: + rs.add(" " & $v.ordinal & " => " & name & "::" & v.name & ",\n") + rs.add(" _ => Self::default(),\n") + rs.add(" }\n") + rs.add(" }\n") + rs.add("}\n\n") + rs.add("impl From<" & name & "> for i32 {\n") + rs.add(" fn from(v: " & name & ") -> Self { v as i32 }\n") + rs.add("}\n\n") + + # Distinct / alias. + for name in aliasNames: + let underlying = resolveUnderlyingType(name) + let pyU = primRustHint(underlying) + if pyU.len == 0: + rs.add( + "// TODO: alias '" & name & "' resolves to '" & underlying & + "' which has no Rust primitive mapping\n\n" + ) + continue + rs.add("pub type " & name & " = " & pyU & ";\n\n") + + # Objects. + for name in objectNames: + let entry = lookupTypeEntry(name) + rs.add("#[derive(Debug, Clone, Default, Serialize, Deserialize)]\n") + rs.add("pub struct " & name & " {\n") + var anyField = false + for f in entry.fields: + let hint = nimTypeToRustHint(f.nimType) + if hint.len == 0: + rs.add(" // TODO: Nim type '" & f.nimType & "' not yet mappable\n") + continue + # Map seq[byte] to serde_bytes::ByteBuf for compact CBOR encoding. + let useByteBuf = f.nimType.strip().toLowerAscii() == "seq[byte]" + if useByteBuf: + rs.add(" #[serde(with = \"serde_bytes\")]\n") + rs.add(" pub " & f.name & ": " & hint & ",\n") + anyField = true + if not anyField: + # Zero-field payload (a `void` broker type). `#[serde(skip)]` keeps the + # placeholder field off the wire so the struct round-trips the empty + # `{}` CBOR map a payload-less request / event carries. + rs.add(" #[serde(skip)]\n") + rs.add(" _phantom: (),\n") + rs.add("}\n\n") + + # ---- Lib struct ------------------------------------------------------ + rs.add("// -------- Lib struct --------\n\n") + + # CBOR event dispatch via user_data. Each on_X registration leaks a + # Box> via Box::into_raw so its pointer is stable for + # the C broker to hold as user_data. The shared trampoline retrieves + # and invokes exactly one closure per emit — no global map, no + # fan-out, no cross-context leakage. Holders are tracked per-ctx and + # dropped together on shutdown (the broker docs say in-flight + # callbacks complete after off returns, so eager Drop on off would + # UAF; a per-ctx-shutdown free is the safe upper bound). + rs.add("type CborEventHandler = Arc;\n\n") + rs.add("struct CborHolderEntry { ctx: u32, ptr: *mut c_void }\n") + rs.add("unsafe impl Send for CborHolderEntry {}\n") + rs.add("unsafe impl Sync for CborHolderEntry {}\n\n") + rs.add( + "static CBOR_EVENT_HOLDERS: OnceLock>> = OnceLock::new();\n" + ) + rs.add("fn cbor_event_holders() -> &'static Mutex> {\n") + rs.add(" CBOR_EVENT_HOLDERS.get_or_init(|| Mutex::new(Vec::new()))\n") + rs.add("}\n\n") + rs.add("fn drop_cbor_event_holders_for_ctx(ctx: u32) {\n") + rs.add(" let mut g = cbor_event_holders().lock().unwrap();\n") + rs.add(" let mut keep: Vec = Vec::with_capacity(g.len());\n") + rs.add(" for e in g.drain(..) {\n") + rs.add(" if e.ctx == ctx {\n") + rs.add( + " unsafe { drop(Box::from_raw(e.ptr as *mut CborEventHandler)); }\n" + ) + rs.add(" } else { keep.push(e); }\n") + rs.add(" }\n") + rs.add(" *g = keep;\n") + rs.add("}\n\n") + + rs.add( + "/// Pythonic / C++-equivalent wrapper around the `" & libName & "` library.\n" + ) + rs.add("pub struct " & className & " {\n") + rs.add(" ctx: u32,\n") + rs.add("}\n\n") + + rs.add("impl " & className & " {\n") + rs.add(" /// Static semver string baked into the shared library.\n") + rs.add(" pub fn version() -> String {\n") + rs.add(" unsafe {\n") + rs.add(" let p = " & p & "version();\n") + rs.add( + " if p.is_null() { String::new() } else { CStr::from_ptr(p).to_string_lossy().into_owned() }\n" + ) + rs.add(" }\n") + rs.add(" }\n\n") + + rs.add(" pub fn new() -> Self {\n") + rs.add(" unsafe { " & p & "initialize(); }\n") + rs.add(" Self { ctx: 0 }\n") + rs.add(" }\n\n") + + rs.add(" pub fn create_context(&mut self) -> Result<()> {\n") + rs.add( + " if self.ctx != 0 { return Result::err(\"Context already created\"); }\n" + ) + rs.add(" unsafe {\n") + rs.add(" let mut err: *const c_char = std::ptr::null();\n") + rs.add(" let ctx = " & p & "createContext(&mut err as *mut _);\n") + rs.add(" if ctx == 0 {\n") + rs.add(" let msg = if err.is_null() {\n") + rs.add(" String::from(\"createContext returned 0\")\n") + rs.add(" } else {\n") + rs.add( + " let s = CStr::from_ptr(err).to_string_lossy().into_owned();\n" + ) + rs.add(" " & p & "freeBuffer(err as *mut c_void);\n") + rs.add(" s\n") + rs.add(" };\n") + rs.add(" return Result::err(msg);\n") + rs.add(" }\n") + rs.add(" self.ctx = ctx;\n") + rs.add(" Result::ok(())\n") + rs.add(" }\n") + rs.add(" }\n\n") + + rs.add(" pub fn valid_context(&self) -> bool { self.ctx != 0 }\n") + rs.add(" pub fn ctx(&self) -> u32 { self.ctx }\n\n") + + rs.add(" pub fn shutdown(&mut self) {\n") + rs.add(" if self.ctx != 0 {\n") + rs.add(" unsafe { " & p & "shutdown(self.ctx); }\n") + rs.add(" // C broker has finished dispatching; safe to free closures.\n") + rs.add(" drop_cbor_event_holders_for_ctx(self.ctx);\n") + rs.add(" self.ctx = 0;\n") + rs.add(" }\n") + rs.add(" }\n\n") + + # Discovery helpers. + rs.add( + " pub fn list_apis(&self) -> ::std::result::Result {\n" + ) + rs.add(" unsafe { fetch_descriptor(" & p & "listApis, \"listApis\") }\n") + rs.add(" }\n\n") + rs.add( + " pub fn get_schema(&self) -> ::std::result::Result {\n" + ) + rs.add(" unsafe { fetch_descriptor(" & p & "getSchema, \"getSchema\") }\n") + rs.add(" }\n\n") + + # Internal call helper. + rs.add( + " fn do_call(&self, api_name: &str, req_payload: &[u8]) -> ::std::result::Result, String> {\n" + ) + rs.add( + " if self.ctx == 0 { return Err(\"Library context is not created\".into()); }\n" + ) + rs.add(" unsafe {\n") + rs.add( + " let cname = CString::new(api_name).map_err(|e| e.to_string())?;\n" + ) + rs.add(" let in_buf: *const c_void = if req_payload.is_empty() {\n") + rs.add(" std::ptr::null()\n") + rs.add(" } else {\n") + rs.add(" let p = " & p & "allocBuffer(req_payload.len() as i32);\n") + rs.add( + " if p.is_null() { return Err(\"allocBuffer failed\".into()); }\n" + ) + rs.add( + " std::ptr::copy_nonoverlapping(req_payload.as_ptr(), p as *mut u8, req_payload.len());\n" + ) + rs.add(" p as *const c_void\n") + rs.add(" };\n") + rs.add(" let mut out_buf: *mut c_void = std::ptr::null_mut();\n") + rs.add(" let mut out_len: i32 = 0;\n") + rs.add(" let status = " & p & "call(\n") + rs.add(" self.ctx,\n") + rs.add(" cname.as_ptr(),\n") + rs.add(" in_buf,\n") + rs.add(" req_payload.len() as i32,\n") + rs.add(" &mut out_buf as *mut _,\n") + rs.add(" &mut out_len as *mut _,\n") + rs.add(" );\n") + rs.add(" let mut out: Vec = Vec::new();\n") + rs.add(" if !out_buf.is_null() && out_len > 0 {\n") + rs.add( + " let slice = std::slice::from_raw_parts(out_buf as *const u8, out_len as usize);\n" + ) + rs.add(" out = slice.to_vec();\n") + rs.add(" " & p & "freeBuffer(out_buf);\n") + rs.add(" }\n") + rs.add(" if status != 0 {\n") + rs.add(" if status == -4 && !out.is_empty() {\n") + rs.add( + " return Err(String::from_utf8_lossy(&out).into_owned());\n" + ) + rs.add(" }\n") + rs.add(" return Err(format!(\"framework error: {}\", status));\n") + rs.add(" }\n") + rs.add(" Ok(out)\n") + rs.add(" }\n") + rs.add(" }\n\n") + + # Per-request methods. Factored into a reusable emitter so the main Lib impl + # and each sub-interface impl share identical bodies (reduced-A). + proc emitRustReqMethod(e: CborRequestEntry): string {.compileTime.} = + if e.responseTypeName.len == 0: + return "" + if not isEmittablePayload(e.responseTypeName): + return + " // TODO: '" & e.apiName & "' return type '" & e.responseTypeName & + "' is not a registered object type.\n\n" + for (n, t) in e.argFields: + if not isRustMappable(t): + return + " // TODO: '" & e.apiName & + "' has parameters whose Nim types aren't yet mappable to Rust.\n\n" + let methodName = e.apiName + var sigParams = "&self" + var argsStructDecl = "" + var argsStructInit = "" + if e.argFields.len > 0: + argsStructDecl.add(" #[derive(Serialize)]\n") + argsStructDecl.add(" struct __Args {\n") + for (n, t) in e.argFields: + sigParams.add(", " & n & ": " & nimTypeToRustHint(t)) + let lowered = t.toLowerAscii().strip() + if lowered == "seq[byte]": + argsStructDecl.add(" #[serde(with = \"serde_bytes\")]\n") + elif lowered == "option[seq[byte]]": + argsStructDecl.add( + " #[serde(with = \"::serde_bytes\", default, skip_serializing_if = \"Option::is_none\")]\n" + ) + argsStructDecl.add(" " & n & ": " & nimTypeToRustHint(t) & ",\n") + argsStructInit.add(" " & n & ",\n") + argsStructDecl.add(" }\n") + result.add( + " pub fn " & methodName & "(" & sigParams & ") -> Result<" & e.responseTypeName & + "> {\n" + ) + if e.argFields.len > 0: + result.add(argsStructDecl) + result.add(" let args = __Args {\n") + result.add(argsStructInit) + result.add(" };\n") + result.add(" let mut buf: Vec = Vec::new();\n") + result.add(" if let Err(e) = ciborium::into_writer(&args, &mut buf) {\n") + result.add(" return Result::err(format!(\"cbor encode: {}\", e));\n") + result.add(" }\n") + else: + result.add(" let buf: Vec = Vec::new();\n") + result.add(" let raw = match self.do_call(\"" & e.apiName & "\", &buf) {\n") + result.add(" Ok(v) => v,\n") + result.add(" Err(e) => return Result::err(e),\n") + result.add(" };\n") + result.add(" if raw.is_empty() {\n") + result.add(" return Result::err(\"empty response envelope\");\n") + result.add(" }\n") + result.add(" #[derive(Deserialize)]\n") + result.add( + " struct __Env { #[serde(default)] ok: Option<" & e.responseTypeName & + ">, #[serde(default)] err: Option }\n" + ) + result.add( + " let env: __Env = match ciborium::from_reader(raw.as_slice()) {\n" + ) + result.add(" Ok(v) => v,\n") + result.add( + " Err(e) => return Result::err(format!(\"cbor decode: {}\", e)),\n" + ) + result.add(" };\n") + result.add(" if let Some(msg) = env.err { return Result::err(msg); }\n") + result.add(" match env.ok {\n") + result.add(" Some(v) => Result::ok(v),\n") + result.add(" None => Result::err(\"missing ok in envelope\"),\n") + result.add(" }\n") + result.add(" }\n\n") + + # reduced-A: a create-instance method returns the typed sub-wrapper. The wire + # ok value is a bare u32 ctx; we construct `Sub { ctx }` from it (same module, + # so the private field is accessible). + proc emitRustInstanceMethod(e: CborRequestEntry): string {.compileTime.} = + for (n, t) in e.argFields: + if not isRustMappable(t): + return " // TODO: '" & e.apiName & "' has unmappable parameter types.\n\n" + let sub = rustSubStructName(e.returnsInterface) + var sigParams = "&self" + var argsStructInit = "" + var argsStructDecl = "" + if e.argFields.len > 0: + argsStructDecl.add(" #[derive(Serialize)]\n") + argsStructDecl.add(" struct __Args {\n") + for (n, t) in e.argFields: + sigParams.add(", " & n & ": " & nimTypeToRustHint(t)) + argsStructDecl.add(" " & n & ": " & nimTypeToRustHint(t) & ",\n") + argsStructInit.add(" " & n & ",\n") + argsStructDecl.add(" }\n") + result.add( + " pub fn " & e.apiName & "(" & sigParams & ") -> Result<" & sub & "> {\n" + ) + if e.argFields.len > 0: + result.add(argsStructDecl) + result.add(" let args = __Args {\n") + result.add(argsStructInit) + result.add(" };\n") + result.add(" let mut buf: Vec = Vec::new();\n") + result.add(" if let Err(e) = ciborium::into_writer(&args, &mut buf) {\n") + result.add(" return Result::err(format!(\"cbor encode: {}\", e));\n") + result.add(" }\n") + else: + result.add(" let buf: Vec = Vec::new();\n") + result.add(" let raw = match self.do_call(\"" & e.apiName & "\", &buf) {\n") + result.add(" Ok(v) => v,\n") + result.add(" Err(e) => return Result::err(e),\n") + result.add(" };\n") + result.add(" if raw.is_empty() {\n") + result.add(" return Result::err(\"empty response envelope\");\n") + result.add(" }\n") + result.add(" #[derive(Deserialize)]\n") + result.add( + " struct __Env { #[serde(default)] ok: Option, #[serde(default)] err: Option }\n" + ) + result.add( + " let env: __Env = match ciborium::from_reader(raw.as_slice()) {\n" + ) + result.add(" Ok(v) => v,\n") + result.add( + " Err(e) => return Result::err(format!(\"cbor decode: {}\", e)),\n" + ) + result.add(" };\n") + result.add(" if let Some(msg) = env.err { return Result::err(msg); }\n") + result.add(" match env.ok {\n") + result.add(" Some(v) => Result::ok(" & sub & " { ctx: v }),\n") + result.add(" None => Result::err(\"missing ok in envelope\"),\n") + result.add(" }\n") + result.add(" }\n\n") + + rs.add(" // ---- Request methods ----\n\n") + for e in requestEntries: + if e.responseTypeName.len == 0: + continue + if not ownsReqMain(e): + continue + if e.returnsInterface.len > 0: + rs.add(emitRustInstanceMethod(e)) + else: + rs.add(emitRustReqMethod(e)) + + # Per-event subscribe / unsubscribe. + rs.add(" // ---- Event registration ----\n\n") + for ev in eventEntries: + if not ownsEvtMain(ev): + continue + if not isEmittablePayload(ev.typeName): + rs.add( + " // TODO: event '" & ev.apiName & "' payload type '" & ev.typeName & + "' is not a registered object type.\n\n" + ) + continue + let onName = "on_" & ev.apiName + let offName = "off_" & ev.apiName + # Build per-field type hints + per-field destructure args. The user + # callback signature is `Fn(field1, field2, ...)` — parity with the + # native-mode wrapper so the same client code drives either build. + var hintParts: seq[string] = @[] + var destructureArgs: seq[string] = @[] + if isScalarPayload(ev.typeName): + # Scalar payload: the decoded `v` IS the value — one bare arg. + hintParts.add(primRustHint(resolveUnderlyingType(ev.typeName))) + destructureArgs.add("v") + else: + for f in lookupTypeEntry(ev.typeName).fields: + let hint = nimTypeToRustHint(f.nimType) + hintParts.add(if hint.len > 0: hint else: "::serde_json::Value") + destructureArgs.add("v." & f.name) + let fnBound = hintParts.join(", ") + rs.add( + " pub fn " & onName & "(&self, callback: F) -> u64 where F: Fn(" & fnBound & + ") + Send + Sync + 'static {\n" + ) + rs.add(" if self.ctx == 0 { return 0; }\n") + rs.add(" let wrapper: CborEventHandler = Arc::new(move |raw: &[u8]| {\n") + rs.add( + " if let Ok(v) = ciborium::from_reader::<" & ev.typeName & + ", _>(raw) {\n" + ) + rs.add(" callback(" & destructureArgs.join(", ") & ");\n") + rs.add(" }\n") + rs.add(" });\n") + rs.add( + " let raw: *mut c_void = Box::into_raw(Box::new(wrapper)) as *mut c_void;\n" + ) + rs.add( + " let cname = match CString::new(\"" & ev.apiName & + "\") { Ok(s) => s, Err(_) => { unsafe { drop(Box::from_raw(raw as *mut CborEventHandler)); } return 0 } };\n" + ) + rs.add( + " let h = unsafe { " & p & + "subscribe(self.ctx, cname.as_ptr(), cbor_trampoline, raw) };\n" + ) + rs.add(" if h == 0 {\n") + rs.add( + " unsafe { drop(Box::from_raw(raw as *mut CborEventHandler)); }\n" + ) + rs.add(" return 0;\n") + rs.add(" }\n") + rs.add( + " cbor_event_holders().lock().unwrap().push(CborHolderEntry { ctx: self.ctx, ptr: raw });\n" + ) + rs.add(" h\n") + rs.add(" }\n\n") + + rs.add(" pub fn " & offName & "(&self, handle: u64) {\n") + rs.add(" if self.ctx == 0 { return; }\n") + rs.add( + " let cname = match CString::new(\"" & ev.apiName & + "\") { Ok(s) => s, Err(_) => return };\n" + ) + rs.add( + " unsafe { " & p & "unsubscribe(self.ctx, cname.as_ptr(), handle); }\n" + ) + rs.add(" }\n\n") + + rs.add("}\n\n") + + rs.add("impl Default for " & className & " {\n") + rs.add(" fn default() -> Self { Self::new() }\n") + rs.add("}\n\n") + + rs.add("impl Drop for " & className & " {\n") + rs.add(" fn drop(&mut self) { self.shutdown(); }\n") + rs.add("}\n\n") + + # reduced-A: sub-interface wrapper structs. Each shares the single C ABI: its + # methods call _call(ctx, ...) which the library routes by classCtx to + # the same processing thread. Drop / close() calls _releaseInstance, after + # which the Nim instance is reclaimed by the GC (no FFI-side ownership). + for ifaceName in subInterfaceNames: + let sub = rustSubStructName(ifaceName) + rs.add( + "// -------- " & sub & " — sub-instance wrapper of " & ifaceName & " --------\n" + ) + rs.add("pub struct " & sub & " {\n") + rs.add(" ctx: u32,\n") + rs.add("}\n\n") + rs.add("impl " & sub & " {\n") + rs.add(" pub fn ctx(&self) -> u32 { self.ctx }\n") + rs.add(" pub fn valid(&self) -> bool { self.ctx != 0 }\n\n") + rs.add(" pub fn close(&mut self) {\n") + rs.add(" if self.ctx != 0 {\n") + rs.add(" unsafe { " & p & "releaseInstance(self.ctx); }\n") + rs.add(" self.ctx = 0;\n") + rs.add(" }\n") + rs.add(" }\n\n") + # Internal call helper (same shape as Lib::do_call, keyed by self.ctx). + rs.add( + " fn do_call(&self, api_name: &str, req_payload: &[u8]) -> ::std::result::Result, String> {\n" + ) + rs.add( + " if self.ctx == 0 { return Err(\"sub-instance is released\".into()); }\n" + ) + rs.add(" unsafe {\n") + rs.add( + " let cname = CString::new(api_name).map_err(|e| e.to_string())?;\n" + ) + rs.add(" let in_buf: *const c_void = if req_payload.is_empty() {\n") + rs.add(" std::ptr::null()\n") + rs.add(" } else {\n") + rs.add(" let p = " & p & "allocBuffer(req_payload.len() as i32);\n") + rs.add( + " if p.is_null() { return Err(\"allocBuffer failed\".into()); }\n" + ) + rs.add( + " std::ptr::copy_nonoverlapping(req_payload.as_ptr(), p as *mut u8, req_payload.len());\n" + ) + rs.add(" p as *const c_void\n") + rs.add(" };\n") + rs.add(" let mut out_buf: *mut c_void = std::ptr::null_mut();\n") + rs.add(" let mut out_len: i32 = 0;\n") + rs.add(" let status = " & p & "call(\n") + rs.add( + " self.ctx, cname.as_ptr(), in_buf, req_payload.len() as i32,\n" + ) + rs.add(" &mut out_buf as *mut _, &mut out_len as *mut _,\n") + rs.add(" );\n") + rs.add(" let mut out: Vec = Vec::new();\n") + rs.add(" if !out_buf.is_null() && out_len > 0 {\n") + rs.add( + " let slice = std::slice::from_raw_parts(out_buf as *const u8, out_len as usize);\n" + ) + rs.add(" out = slice.to_vec();\n") + rs.add(" " & p & "freeBuffer(out_buf);\n") + rs.add(" }\n") + rs.add(" if status != 0 {\n") + rs.add(" if status == -4 && !out.is_empty() {\n") + rs.add( + " return Err(String::from_utf8_lossy(&out).into_owned());\n" + ) + rs.add(" }\n") + rs.add(" return Err(format!(\"framework error: {}\", status));\n") + rs.add(" }\n") + rs.add(" Ok(out)\n") + rs.add(" }\n") + rs.add(" }\n\n") + for e in requestEntries: + if interfaceOwningRequestType(e.responseTypeName) == ifaceName: + rs.add(emitRustReqMethod(e)) + # Sub-interface event methods (subscribe/unsubscribe keyed by self.ctx). + for ev in eventEntries: + if interfaceOwningEventType(ev.typeName) != ifaceName: + continue + if not isEmittablePayload(ev.typeName): + rs.add( + " // TODO: event '" & ev.apiName & "' payload type '" & ev.typeName & + "' is not a registered object type.\n\n" + ) + continue + let onName = "on_" & ev.apiName + let offName = "off_" & ev.apiName + var hintParts: seq[string] = @[] + var destructureArgs: seq[string] = @[] + if isScalarPayload(ev.typeName): + hintParts.add(primRustHint(resolveUnderlyingType(ev.typeName))) + destructureArgs.add("v") + else: + for f in lookupTypeEntry(ev.typeName).fields: + let hint = nimTypeToRustHint(f.nimType) + hintParts.add(if hint.len > 0: hint else: "::serde_json::Value") + destructureArgs.add("v." & f.name) + let fnBound = hintParts.join(", ") + rs.add( + " pub fn " & onName & "(&self, callback: F) -> u64 where F: Fn(" & fnBound & + ") + Send + Sync + 'static {\n" + ) + rs.add(" if self.ctx == 0 { return 0; }\n") + rs.add(" let wrapper: CborEventHandler = Arc::new(move |raw: &[u8]| {\n") + rs.add( + " if let Ok(v) = ciborium::from_reader::<" & ev.typeName & + ", _>(raw) {\n" + ) + rs.add(" callback(" & destructureArgs.join(", ") & ");\n") + rs.add(" }\n") + rs.add(" });\n") + rs.add( + " let raw: *mut c_void = Box::into_raw(Box::new(wrapper)) as *mut c_void;\n" + ) + rs.add( + " let cname = match CString::new(\"" & ev.apiName & + "\") { Ok(s) => s, Err(_) => { unsafe { drop(Box::from_raw(raw as *mut CborEventHandler)); } return 0 } };\n" + ) + rs.add( + " let h = unsafe { " & p & + "subscribe(self.ctx, cname.as_ptr(), cbor_trampoline, raw) };\n" + ) + rs.add(" if h == 0 {\n") + rs.add( + " unsafe { drop(Box::from_raw(raw as *mut CborEventHandler)); }\n" + ) + rs.add(" return 0;\n") + rs.add(" }\n") + rs.add( + " cbor_event_holders().lock().unwrap().push(CborHolderEntry { ctx: self.ctx, ptr: raw });\n" + ) + rs.add(" h\n") + rs.add(" }\n\n") + rs.add(" pub fn " & offName & "(&self, handle: u64) {\n") + rs.add(" if self.ctx == 0 { return; }\n") + rs.add( + " let cname = match CString::new(\"" & ev.apiName & + "\") { Ok(s) => s, Err(_) => return };\n" + ) + rs.add( + " unsafe { " & p & "unsubscribe(self.ctx, cname.as_ptr(), handle); }\n" + ) + rs.add(" }\n\n") + rs.add("}\n\n") + rs.add("impl Drop for " & sub & " {\n") + rs.add(" fn drop(&mut self) { self.close(); }\n") + rs.add("}\n\n") + + # Trampoline: each subscription's user_data points at a leaked + # Box>. Clone the Arc cheaply (atomic refcount) so + # in-flight callbacks survive a concurrent off / shutdown that drops + # the holder. + rs.add( + "unsafe extern \"C\" fn cbor_trampoline(ctx: u32, name: *const c_char, buf: *const c_void, buf_len: i32, ud: *mut c_void) {\n" + ) + rs.add(" let _ = ctx;\n") + rs.add(" let _ = name;\n") + rs.add(" if ud.is_null() || buf.is_null() || buf_len <= 0 { return; }\n") + rs.add( + " let slice = std::slice::from_raw_parts(buf as *const u8, buf_len as usize);\n" + ) + rs.add( + " let arc: CborEventHandler = unsafe { (*(ud as *const CborEventHandler)).clone() };\n" + ) + rs.add(" arc(slice);\n") + rs.add("}\n\n") + + # Discovery descriptor helper. + rs.add("unsafe fn fetch_descriptor(\n") + rs.add(" f: unsafe extern \"C\" fn(*mut *mut c_void, *mut i32) -> i32,\n") + rs.add(" label: &str,\n") + rs.add(") -> ::std::result::Result {\n") + rs.add(" let mut buf: *mut c_void = std::ptr::null_mut();\n") + rs.add(" let mut len: i32 = 0;\n") + rs.add(" let status = f(&mut buf as *mut _, &mut len as *mut _);\n") + rs.add( + " if status != 0 { return Err(format!(\"{} framework error: {}\", label, status)); }\n" + ) + rs.add(" if buf.is_null() || len <= 0 { return Ok(serde_json::Value::Null); }\n") + rs.add( + " let slice = std::slice::from_raw_parts(buf as *const u8, len as usize);\n" + ) + rs.add(" let v: serde_json::Value = match serde_json::from_slice(slice) {\n") + rs.add(" Ok(v) => v,\n") + rs.add(" Err(e) => {\n") + rs.add(" " & p & "freeBuffer(buf);\n") + rs.add(" return Err(format!(\"json decode {}: {}\", label, e));\n") + rs.add(" }\n") + rs.add(" };\n") + rs.add(" " & p & "freeBuffer(buf);\n") + rs.add(" Ok(v)\n") + rs.add("}\n") + + try: + writeFile(srcDir & "/lib.rs", rs) + except IOError: + error( + "Failed to write generated CBOR Rust source '" & srcDir & "/lib.rs': " & + getCurrentExceptionMsg() + ) + +{.push raises: [].} +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_codegen_cmake.nim b/wasm-deps/brokers/brokers/internal/api_codegen_cmake.nim new file mode 100644 index 000000000..865c9d1e4 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_codegen_cmake.nim @@ -0,0 +1,213 @@ +## api_codegen_cmake +## ---------------- +## Emits a per-library CMake package next to the generated headers and shared +## library so consumers can do: +## +## find_package( CONFIG REQUIRED) +## target_link_libraries(myapp PRIVATE ::) # C consumers +## target_link_libraries(myapp PRIVATE ::_cpp) # C++ consumers +## +## Files written into `outDir`: +## Config.cmake — defines IMPORTED targets +## ConfigVersion.cmake — version compatibility (SameMajorVersion) +## +## The package is fully relocatable: it resolves the shared library and headers +## relative to its own location (`CMAKE_CURRENT_LIST_DIR`), which is the same +## directory the Nim build dropped them into. +## +## CBOR mode adds a header-only jsoncons dependency on the C++ INTERFACE +## target. Consumers can either install jsoncons system-wide or set +## `_JSONCONS_INCLUDE_DIR` before `find_package`. + +{.push raises: [].} + +import std/strutils +import ./api_outdir + +{.pop.} + +proc generateCMakePackageFiles*( + outDir: string, libName: string, version: string, cborMode: bool, hasCpp: bool +) {.compileTime, raises: [].} = + ## Emits Config.cmake and ConfigVersion.cmake into `outDir`. + ensureGeneratedOutputDir(outDir) + + let baseDir = + if outDir.len > 0: + outDir & "/" + else: + "" + let configPath = baseDir & libName & "Config.cmake" + let versionPath = baseDir & libName & "ConfigVersion.cmake" + + let upperName = libName.toUpperAscii().replace("-", "_") + let nsName = libName # same as IMPORTED namespace prefix + + # ---------------- ConfigVersion.cmake ---------------- + # Hand-rolled SameMajorVersion logic so consumers don't need to invoke + # CMakePackageConfigHelpers — the file is self-contained. + let semverParts = version.split('.') + let pkgMajor = + if semverParts.len >= 1: + semverParts[0] + else: + "0" + var versionFile = "" + versionFile.add( + "# Auto-generated by brokers/api_codegen_cmake.nim — do not edit.\n" + ) + versionFile.add("set(PACKAGE_VERSION \"" & version & "\")\n\n") + versionFile.add("if(PACKAGE_FIND_VERSION VERSION_EQUAL PACKAGE_VERSION)\n") + versionFile.add(" set(PACKAGE_VERSION_EXACT TRUE)\n") + versionFile.add("endif()\n\n") + versionFile.add( + "if(NOT PACKAGE_FIND_VERSION OR PACKAGE_FIND_VERSION VERSION_LESS_EQUAL PACKAGE_VERSION)\n" + ) + versionFile.add(" set(PACKAGE_VERSION_COMPATIBLE FALSE)\n") + versionFile.add( + " if(NOT PACKAGE_FIND_VERSION OR \"${PACKAGE_FIND_VERSION_MAJOR}\" STREQUAL \"" & + pkgMajor & "\")\n" + ) + versionFile.add(" set(PACKAGE_VERSION_COMPATIBLE TRUE)\n") + versionFile.add(" endif()\n") + versionFile.add("else()\n") + versionFile.add(" set(PACKAGE_VERSION_COMPATIBLE FALSE)\n") + versionFile.add("endif()\n") + + try: + writeFile(versionPath, versionFile) + except IOError: + discard # keep raises:[] — codegen errors shouldn't crash compilation + + # ---------------- Config.cmake ---------------- + var cfg = "" + cfg.add("# Auto-generated by brokers/api_codegen_cmake.nim — do not edit.\n") + cfg.add("# CMake package for the '" & libName & "' broker FFI library.\n") + cfg.add("#\n") + cfg.add("# Provides:\n") + cfg.add( + "# " & nsName & "::" & libName & " — IMPORTED SHARED library + C headers\n" + ) + if hasCpp: + cfg.add( + "# " & nsName & "::" & libName & + "_cpp — INTERFACE for C++ consumers (C++20)\n" + ) + if cborMode: + cfg.add( + "# (depends on jsoncons; set " & upperName & + "_JSONCONS_INCLUDE_DIR to override discovery)\n" + ) + cfg.add("\n") + + cfg.add("cmake_minimum_required(VERSION 3.16)\n\n") + + cfg.add( + "get_filename_component(_" & libName & + "_pkg_dir \"${CMAKE_CURRENT_LIST_DIR}\" ABSOLUTE)\n\n" + ) + + # Resolve platform-specific shared library filename. + cfg.add("if(WIN32)\n") + cfg.add(" set(_" & libName & "_shared_name \"" & libName & ".dll\")\n") + cfg.add(" set(_" & libName & "_import_name \"" & libName & ".lib\")\n") + cfg.add("elseif(APPLE)\n") + cfg.add(" set(_" & libName & "_shared_name \"lib" & libName & ".dylib\")\n") + cfg.add("else()\n") + cfg.add(" set(_" & libName & "_shared_name \"lib" & libName & ".so\")\n") + cfg.add("endif()\n\n") + + cfg.add( + "set(_" & libName & "_shared_path \"${_" & libName & "_pkg_dir}/${_" & libName & + "_shared_name}\")\n" + ) + cfg.add("if(NOT EXISTS \"${_" & libName & "_shared_path}\")\n") + cfg.add( + " message(FATAL_ERROR \"" & libName & ": shared library not found at '${_" & libName & + "_shared_path}'.\")\n" + ) + cfg.add("endif()\n\n") + + cfg.add( + "set(_" & libName & "_header_path \"${_" & libName & "_pkg_dir}/" & libName & + ".h\")\n" + ) + cfg.add("if(NOT EXISTS \"${_" & libName & "_header_path}\")\n") + cfg.add( + " message(FATAL_ERROR \"" & libName & ": C header not found at '${_" & libName & + "_header_path}'.\")\n" + ) + cfg.add("endif()\n\n") + + # IMPORTED SHARED target — the C-level surface. Carries headers + library. + cfg.add("if(NOT TARGET " & nsName & "::" & libName & ")\n") + cfg.add(" add_library(" & nsName & "::" & libName & " SHARED IMPORTED)\n") + cfg.add(" set_target_properties(" & nsName & "::" & libName & " PROPERTIES\n") + cfg.add(" IMPORTED_LOCATION \"${_" & libName & "_shared_path}\"\n") + cfg.add(" INTERFACE_INCLUDE_DIRECTORIES \"${_" & libName & "_pkg_dir}\"\n") + cfg.add(" )\n") + cfg.add(" if(WIN32)\n") + cfg.add( + " set(_" & libName & "_import_path \"${_" & libName & "_pkg_dir}/${_" & libName & + "_import_name}\")\n" + ) + cfg.add(" if(EXISTS \"${_" & libName & "_import_path}\")\n") + cfg.add( + " set_target_properties(" & nsName & "::" & libName & + " PROPERTIES IMPORTED_IMPLIB \"${_" & libName & "_import_path}\")\n" + ) + cfg.add(" endif()\n") + cfg.add(" endif()\n") + cfg.add("endif()\n\n") + + if hasCpp: + # INTERFACE target for C++ consumers — pulls in the C target, requires + # C++20, and (CBOR mode) wires jsoncons. + cfg.add("if(NOT TARGET " & nsName & "::" & libName & "_cpp)\n") + cfg.add(" add_library(" & nsName & "::" & libName & "_cpp INTERFACE IMPORTED)\n") + cfg.add( + " set_property(TARGET " & nsName & "::" & libName & + "_cpp PROPERTY INTERFACE_LINK_LIBRARIES " & nsName & "::" & libName & ")\n" + ) + cfg.add( + " set_property(TARGET " & nsName & "::" & libName & + "_cpp PROPERTY INTERFACE_COMPILE_FEATURES cxx_std_20)\n" + ) + + if cborMode: + cfg.add("\n") + cfg.add(" # jsoncons (header-only) is required by the CBOR-mode C++ wrapper.\n") + cfg.add(" if(NOT DEFINED " & upperName & "_JSONCONS_INCLUDE_DIR)\n") + cfg.add( + " find_path(" & upperName & "_JSONCONS_INCLUDE_DIR\n" & + " NAMES jsoncons/json.hpp\n" & " PATHS\n" & " \"${_" & libName & + "_pkg_dir}/../../vendor/jsoncons/include\"\n" & " \"${_" & libName & + "_pkg_dir}/../vendor/jsoncons/include\"\n" & " \"${_" & libName & + "_pkg_dir}/vendor/jsoncons/include\"\n" & + " DOC \"Path to the jsoncons header-only library (root containing 'jsoncons/json.hpp').\"\n" & + " )\n" + ) + cfg.add(" endif()\n") + cfg.add(" if(NOT " & upperName & "_JSONCONS_INCLUDE_DIR)\n") + cfg.add( + " message(FATAL_ERROR \"" & libName & + " (CBOR mode): jsoncons headers not found. Install jsoncons or set " & + upperName & "_JSONCONS_INCLUDE_DIR.\")\n" + ) + cfg.add(" endif()\n") + cfg.add( + " set_property(TARGET " & nsName & "::" & libName & + "_cpp APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES \"${" & upperName & + "_JSONCONS_INCLUDE_DIR}\")\n" + ) + cfg.add("endif()\n\n") + + cfg.add("set(" & libName & "_FOUND TRUE)\n") + cfg.add("set(" & libName & "_VERSION \"" & version & "\")\n") + cfg.add("set(" & libName & "_LIBRARY \"${_" & libName & "_shared_path}\")\n") + cfg.add("set(" & libName & "_INCLUDE_DIR \"${_" & libName & "_pkg_dir}\")\n") + + try: + writeFile(configPath, cfg) + except IOError: + discard diff --git a/wasm-deps/brokers/brokers/internal/api_common.nim b/wasm-deps/brokers/brokers/internal/api_common.nim new file mode 100644 index 000000000..8b52bebfe --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_common.nim @@ -0,0 +1,259 @@ +## API Common +## ---------- +## Shared utilities for FFI API broker code generation. +## +## After the native FFI codegen surface was retired (see +## `doc/CBOR_Refactoring.md`), this module is a thin coordination layer +## that: +## - Re-exports the type schema registry and FFI mode flag +## - Owns the legacy FFI struct registry bridge +## - Owns compile-time accumulators that are shared across broker macros +## (event counters, handler entries, cleanup proc names) +## - Provides runtime memory helpers for the FFI boundary +## +## This module is only used when compiling with `-d:BrokerFfiApi`. + +{.push raises: [].} + +import std/macros + +import ./api_schema +import ./api_outdir +import ./helper/broker_utils + +export api_schema +export api_outdir + +# --------------------------------------------------------------------------- +# Library name accumulator +# --------------------------------------------------------------------------- + +var gApiLibraryName* {.compileTime.}: string = "" + +# --------------------------------------------------------------------------- +# Compile-time accumulators for delivery thread event system +# --------------------------------------------------------------------------- + +var gApiEventTypeCounter* {.compileTime.}: int = 0 + ## Auto-incrementing type ID for EventBroker(API) types. + ## NOTE: Must be incremented directly (not via a helper proc) because the + ## Nim VM does not persist side effects from called compileTime procs. + +var gApiSharedBrokerGenerated* {.compileTime.}: bool = false + ## Flag: has the shared RegisterEventListenerResult RequestBroker been emitted? + +var gApiEventHandlerEntries* {.compileTime.}: seq[(int, string)] = + @[] ## Accumulates (typeId, handlerProcName) pairs for the aggregate provider. + +var gApiEventCleanupProcNames* {.compileTime.}: seq[string] = + @[] ## Accumulates cleanup proc names for delivery thread teardown. + +var gApiRequestCleanupProcNames* {.compileTime.}: seq[string] = + @[] ## Accumulates cleanup proc names for request provider teardown. + +var gApiEventProcessLoopShutdownProcNames* {.compileTime.}: seq[string] = + @[] ## Accumulates async processLoop shutdown proc names for delivery thread teardown. + +var gApiForeignGcHelperEmitted* {.compileTime.}: bool = false + ## Flag: has the ensureForeignThreadGc() helper been emitted? + ## Each broker codegen module checks this before emitting the helper + ## to avoid duplicate definitions. + +# --------------------------------------------------------------------------- +# CBOR-mode dispatch table accumulator +# --------------------------------------------------------------------------- + +type CborRequestEntry* = object + apiName*: string ## Wire name foreign callers pass to `_call`. + adapterProc*: string ## Identifier of the generated adapter proc. + responseTypeName*: string + ## Nim type name for the response payload + ## (e.g. "GetStatus"). Foreign-language wrapper codegen consumes this + ## to emit typed return signatures. Empty if not yet populated by an + ## older caller path. + argFields*: seq[(string, string)] + ## (paramName, nimType) pairs from + ## the request signature, in declaration order. Empty for zero-arg + ## requests. Wrapper codegen turns this into the typed method + ## signature and the args struct mirroring the synthetic Nim + ## `CborArgs` object. + returnsInterface*: string + ## reduced-A: name of the BrokerInterface(API) this request *creates and + ## returns an instance of* (e.g. "IWidget"), or "" for a normal request. + ## When set, the wire `ok` value is a bare uint32 (the sub-instance's + ## BrokerContext); wrapper codegen emits a method returning the typed + ## sub-wrapper class built from that ctx instead of a decoded payload. + +var gApiCborRequestEntries* {.compileTime.}: seq[CborRequestEntry] = @[] + ## Accumulated by `RequestBroker(API)` expansions. + ## `registerBrokerLibrary` drains this list to emit the per-library + ## `Table[string, CborApiAdapter]` and the `_call` dispatch. + +type CborEventEntry* = object + apiName*: string ## Wire eventName foreign callers pass to `_subscribe`. + typeName*: string ## Nim type identifier for the event payload. + +var gApiCborEventEntries* {.compileTime.}: seq[CborEventEntry] = @[] + ## Accumulated by `EventBroker(API)` expansions. + ## `registerBrokerLibrary` reads this list to generate per-event + ## listener installers and the `CborIsKnownEvent` predicate. As + ## with `gApiCborRequestEntries`, this list is read but not reset — + ## Nim's compile-time VM aliases `let` copies of seqs back to the + ## source. + +proc registerCborEventEntry*(apiName, typeName: string) {.compileTime.} = + ## Register an event for the next library's CBOR-mode subscribe surface. + for entry in gApiCborEventEntries: + if entry.apiName == apiName: + let ownerNew = interfaceOwningEventType(typeName) + let ownerOld = interfaceOwningEventType(entry.typeName) + let ifaceHint = + if ownerNew.len > 0 or ownerOld.len > 0: + " ('" & typeName & "' in interface " & + (if ownerNew.len > 0: ownerNew else: "") & " vs '" & entry.typeName & + "' in interface " & (if ownerOld.len > 0: ownerOld else: "") & ")" + else: + "" + error( + "CBOR FFI: duplicate event apiName '" & apiName & "' (already registered by '" & + entry.typeName & "')" & ifaceHint & ". " & + "Each EventBroker(API) must have a unique event type name." + ) + gApiCborEventEntries.add(CborEventEntry(apiName: apiName, typeName: typeName)) + +proc registerCborRequestEntry*( + apiName, adapterProc: string, + responseTypeName: string = "", + argFields: seq[(string, string)] = @[], + returnsInterface: string = "", +) {.compileTime.} = + ## Register a CBOR request adapter for the next library that calls + ## `registerBrokerLibrary`. Detects duplicate apiNames at compile time + ## so two requests can't shadow each other on the wire. + for entry in gApiCborRequestEntries: + if entry.apiName == apiName: + # reduced-A: name the owning interfaces when the collision spans two + # BrokerInterface(API) declarations (apiNames are globally unique across + # the whole library, not per interface). + let ownerNew = interfaceOwningRequestType(responseTypeName) + let ownerOld = interfaceOwningRequestType(entry.responseTypeName) + let ifaceHint = + if ownerNew.len > 0 or ownerOld.len > 0: + " ('" & responseTypeName & "' in interface " & + (if ownerNew.len > 0: ownerNew else: "") & " vs '" & + entry.responseTypeName & "' in interface " & + (if ownerOld.len > 0: ownerOld else: "") & ")" + else: + "" + error( + "CBOR FFI: duplicate request apiName '" & apiName & "' (already registered by '" & + entry.adapterProc & "')" & ifaceHint & ". " & + "Each RequestBroker(API) must have a unique response type name." + ) + gApiCborRequestEntries.add( + CborRequestEntry( + apiName: apiName, + adapterProc: adapterProc, + responseTypeName: responseTypeName, + argFields: argFields, + returnsInterface: returnsInterface, + ) + ) + +# --------------------------------------------------------------------------- +# Legacy FFI struct registry bridge +# --------------------------------------------------------------------------- + +var gApiFfiStructs* {.compileTime.}: seq[(string, seq[(string, string)])] = @[] + ## Legacy registry. Kept for backward compatibility with existing ApiType usage. + ## New code should use `gApiTypeRegistry` from `api_schema` instead. + +proc registerApiFfiStruct*( + typeName: string, fields: seq[(string, string)] +) {.compileTime.} = + ## Register a type in both the legacy and new registries. + gApiFfiStructs.add((typeName, fields)) + registerFromFieldTuples(typeName, fields) + +proc lookupFfiStruct*(typeName: string): seq[(string, string)] {.compileTime.} = + ## Look up type fields. Checks the new type registry first, then falls back + ## to the legacy registry for backward compatibility. + if isTypeRegistered(typeName): + return lookupTypeFields(typeName) + for (name, fields) in gApiFfiStructs: + if name == typeName: + return fields + error( + "Type '" & typeName & "' not registered. " & + "Define it as a plain Nim type before the broker macro, " & + "or declare it with `ApiType:` for explicit registration." + ) + +{.pop.} + +# --------------------------------------------------------------------------- +# Runtime memory helpers +# --------------------------------------------------------------------------- + +proc allocCStringCopy*(s: string): cstring = + ## Allocates a copy of a Nim string as a shared C string. + ## The caller frees it via the generated FFI free helpers, which may run on + ## a different thread than the allocation site under --mm:refc. + if s.len == 0: + return nil + let buf = cast[cstring](allocShared(s.len + 1)) + copyMem(buf, unsafeAddr s[0], s.len) + cast[ptr char](cast[int](buf) + s.len)[] = '\0' + buf + +proc freeCString*(s: cstring) = + ## Frees a C string previously allocated by allocCStringCopy. + if not s.isNil: + deallocShared(s) + +# --------------------------------------------------------------------------- +# Shared-memory string helpers for cross-thread event data +# --------------------------------------------------------------------------- + +proc allocSharedCString*(s: string): cstring = + ## Allocate a C string copy in shared memory (safe for cross-thread use). + allocCStringCopy(s) + +proc freeSharedCString*(s: cstring) = + ## Free a C string allocated by `allocSharedCString`. + freeCString(s) + +# --------------------------------------------------------------------------- +# Foreign thread GC helper — emitted once per compilation unit +# --------------------------------------------------------------------------- + +proc emitEnsureForeignThreadGc*(): NimNode {.compileTime.} = + ## Returns the AST for the per-thread foreign thread GC registration helper. + ## Call this from each broker codegen module; it emits the helper only once + ## per compilation unit (guarded by `gApiForeignGcHelperEmitted`). + if gApiForeignGcHelperEmitted: + return newStmtList() + + gApiForeignGcHelperEmitted = true + + let tvGcReg = genSym(nskVar, "gForeignGcRegistered") + let ensureIdent = ident("ensureForeignThreadGc") + + result = quote: + var `tvGcReg` {.threadvar.}: bool + + proc `ensureIdent`() {.inline.} = + when compileOption("app", "lib"): + if not `tvGcReg`: + when declared(setupForeignThreadGc): + # setupForeignThreadGc already registers the thread with the GC + # and sets the stack bottom on modern Nim (>= 1.6). Manually + # calling nimGC_setStackBottom on top of it can corrupt GC state. + setupForeignThreadGc() + else: + # Fallback for very old Nim versions that lack setupForeignThreadGc. + when declared(nimGC_setStackBottom): + var locals {.volatile, noinit.}: pointer + locals = addr(locals) + nimGC_setStackBottom(locals) + `tvGcReg` = true diff --git a/wasm-deps/brokers/brokers/internal/api_event_broker_cbor.nim b/wasm-deps/brokers/brokers/internal/api_event_broker_cbor.nim new file mode 100644 index 000000000..41c43d788 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_event_broker_cbor.nim @@ -0,0 +1,105 @@ +## API EventBroker — CBOR mode codegen +## ------------------------------------ +## Generates the CBOR-mode surface for `EventBroker(API)` declarations. +## +## For each declaration this module emits: +## +## 1. The underlying multi-thread EventBroker (via `generateMtEventBroker`). +## Internal cross-thread emit dispatch stays as typed `Channel[T]` +## traffic; CBOR encoding only happens at the moment we hand the event +## to a foreign C callback. +## +## 2. A compile-time entry in `gApiCborEventEntries` so the upcoming +## `registerBrokerLibrary` CBOR backend can wire the event into the +## library's subscribe surface and emit a per-event listener installer. +## +## Listener installation is intentionally NOT generated here — installers +## need access to the library's subscription map / lock / callback-type, +## which only exist at `registerBrokerLibrary` expansion time. We just +## record the event's wire name and Nim type identifier; the library macro +## materialises the installer with the right captures. +## +## Wire `eventName` is the snake_case form of the event's Nim type +## identifier — e.g. `DeviceUpdated` becomes `device_updated`. + +{.push raises: [].} + +import std/[macros, strutils] +import ./helper/broker_utils, ./mt_event_broker, ./mt_config, ./api_common, ./api_schema +import ./api_request_broker_cbor # for registerCborObjectType +import ./api_type_resolver +import ./broker_debug + +# `api_type_resolver` re-export: see note in `api_request_broker_cbor.nim` +# — `autoRegisterApiType` is emitted into user code by broker macros and +# must resolve at the user-library expansion site post-Part-A retirement +# of the native `api_event_broker` re-export chain. +export mt_event_broker, mt_config, api_common, api_type_resolver + +proc generateApiCborEventBrokerImpl(body: NimNode, cfg: MtEvtCfg): NimNode = + result = newStmtList() + + # 1. Emit the underlying MT event broker (single-thread emit/listen API + # visible to user code, MT-aware cross-thread dispatch under the hood). + # The capacity config flows in from the outer EventBroker(API, ...) + # kwargs — same knobs as EventBroker(mt). + result.add(generateMtEventBroker(copyNimTree(body), cfg)) + + # 2. Parse the event type identifier and register the entry. Capture + # field info so wrapper codegen can emit typed structs for the + # payload. + let parsed = parseSingleTypeDef( + body, "EventBroker", allowRefToNonObject = true, collectFieldInfo = true + ) + let typeIdent = parsed.typeIdent + let typeName = sanitizeIdentName(typeIdent) + let apiName = toSnakeCase(typeName) + if parsed.hasInlineFields: + registerCborObjectType(typeName, parsed.fieldNames, parsed.fieldTypes) + elif parsed.isVoid: + # `void` → a zero-field object: a payload-less event notification. + registerCborObjectType(typeName, @[], @[]) + else: + registerCborPrimitiveType(typeName, parsed) + registerCborEventEntry(apiName, typeName) + + when defined(brokerDebug): + writeBrokerDebug( + "EventBrokerApi", typeName, result, header = "eventName='" & apiName & "'" + ) + when defined(brokerDebugStdout): + echo "[brokers/cbor] EventBroker(API) for '" & typeName & "' (eventName='" & + apiName & "')" + echo result.repr + +{.pop.} + +macro generateApiCborEventBrokerDeferred*(args: varargs[untyped]): untyped = + ## Typed-phase deferred entry point; populates the registry first. + ## Args layout: [body, kw0, kw1, ...] — kwargs are forwarded as raw + ## `nnkExprEqExpr` nodes from `generateApiCborEventBroker` so we + ## re-parse them here into an MtEvtCfg. + if args.len == 0: + error("generateApiCborEventBrokerDeferred requires a body", args) + let body = args[0] + var kwargs: seq[NimNode] + for i in 1 ..< args.len: + kwargs.add(args[i]) + let cfg = parseMtEvtKwargs(kwargs) + generateApiCborEventBrokerImpl(body, cfg) + +{.push raises: [].} + +proc generateApiCborEventBroker*(body: NimNode, kwargs: seq[NimNode]): NimNode = + result = newStmtList() + + let externalIdents = discoverExternalTypes(body) + if externalIdents.len > 0: + result.add(emitAutoRegistrations(externalIdents)) + + let deferred = newCall(ident("generateApiCborEventBrokerDeferred"), copyNimTree(body)) + for kw in kwargs: + deferred.add(copyNimTree(kw)) + result.add(deferred) + +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_outdir.nim b/wasm-deps/brokers/brokers/internal/api_outdir.nim new file mode 100644 index 000000000..de7cdfa36 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_outdir.nim @@ -0,0 +1,38 @@ +## api_outdir +## ---------- +## Tiny compile-time helper for ensuring the generated-output directory +## exists. Extracted from the (now-retired) native `api_codegen_c.nim` so +## the CBOR codegen and the CMake package emitter can share it without +## pulling in any native-codegen module. + +import std/[os, macros, compilesettings] + +proc detectOutputDir*(overrideOutDir = ""): string {.compileTime.} = + ## Resolves the compiler output directory for generated artifacts. Returns + ## the override if supplied, otherwise consults `outDir` / `outFile` + ## query settings, falling back to the empty string. + if overrideOutDir.len > 0: + return overrideOutDir + + let configuredOutDir = querySetting(SingleValueSetting.outDir) + if configuredOutDir.len > 0: + return configuredOutDir + + let configuredOutFile = querySetting(SingleValueSetting.outFile) + if configuredOutFile.len > 0: + let candidateDir = splitFile(configuredOutFile).dir + if candidateDir.len > 0: + return candidateDir + + return "" + +proc ensureGeneratedOutputDir*(outDir: string) {.compileTime, raises: [].} = + if outDir.len == 0 or dirExists(outDir): + return + try: + createDir(outDir) + except CatchableError: + error( + "Failed to create generated output directory '" & outDir & "': " & + getCurrentExceptionMsg() + ) diff --git a/wasm-deps/brokers/brokers/internal/api_request_broker_cbor.nim b/wasm-deps/brokers/brokers/internal/api_request_broker_cbor.nim new file mode 100644 index 000000000..a99c6e633 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_request_broker_cbor.nim @@ -0,0 +1,628 @@ +## API RequestBroker — CBOR mode codegen +## -------------------------------------- +## Generates the CBOR-mode surface for `RequestBroker(API)` declarations. +## +## For each declaration this module emits: +## +## 1. The underlying multi-thread RequestBroker, exactly as the native path +## does — providers register and run on the processing thread the same +## way regardless of FFI mode. Internal cross-thread dispatch stays as +## typed `Channel[T]` traffic; CBOR encoding only happens at the C ABI +## boundary. +## +## 2. (When the signature has arguments) a synthetic per-request CBOR args +## object that mirrors the parameter list field-by-field. Decoding the +## foreign request buffer into this object gives us individual local +## variables to forward into the broker's `request` call. +## +## 3. A CBOR adapter proc with the canonical signature +## +## proc CborAdapter*(ctx: BrokerContext, reqBuf: seq[byte]): +## Future[seq[byte]] {.async: (raises: []).} +## +## The adapter decodes the request buffer (or ignores it for zero-arg +## requests), `await`s the typed broker call, and encodes the resulting +## `Result[T, string]` as a CBOR response envelope. +## +## 4. A compile-time entry in `gApiCborRequestEntries` so the upcoming +## `registerBrokerLibrary` CBOR backend can wire the adapter into the +## library's dispatch table. +## +## The wire `apiName` is the snake_case form of the response type name — +## e.g. `InitializeRequest` becomes `initialize_request`. Foreign wrappers +## are generated to use the same name so the C entry point sees a stable +## identifier per broker. + +{.push raises: [].} + +import std/[macros, strutils] +import + ./helper/broker_utils, + ./mt_request_broker, + ./mt_config, + ./api_common, + ./api_cbor_codec, + ./api_schema, + ./api_type_resolver, + ./broker_debug + +# `api_type_resolver` re-export: `autoRegisterApiType` is emitted into the +# user-library AST by the broker macros and must resolve at the user's +# expansion site. Previously this came in transitively via the native +# `api_request_broker` re-export chain (retired in Part A); re-export it +# explicitly here so user code never needs a direct +# `import brokers/internal/api_type_resolver`. +export mt_request_broker, mt_config, api_common, api_cbor_codec, api_type_resolver + +# --------------------------------------------------------------------------- +# Schema registration +# --------------------------------------------------------------------------- + +proc registerCborObjectType*( + typeName: string, fieldNames, fieldTypes: seq[NimNode] +) {.compileTime.} = + ## Register a parsed object type in `gApiTypeRegistry` so the C++ / + ## Python / etc. wrapper codegen can emit typed structs for it. + ## Idempotent — subsequent calls for the same type are a no-op so a + ## type that ends up registered through both the auto-resolver and a + ## broker macro doesn't double-list. + if isTypeRegistered(typeName): + return + var entry = ApiTypeEntry(name: typeName, kind: atkObject) + for i in 0 ..< fieldNames.len: + var fname = $fieldNames[i] + # `fieldNames` from parseSingleTypeDef carry the original AST, + # which for inline `object` types is a plain Ident (export marker + # already lifted by the parser). Strip a trailing '*' defensively. + if fname.endsWith("*"): + fname.setLen(fname.len - 1) + let ftype = fieldTypes[i].repr.strip() + entry.fields.add(ApiFieldDef(name: fname, nimType: ftype)) + registerTypeEntry(entry) + +proc registerCborPrimitiveType*( + typeName: string, parsed: ParsedBrokerType +) {.compileTime.} = + ## Register a primitive (non-object) broker type — `type X = int32` — as a + ## distinct alias of its underlying primitive. Wrapper codegen then emits a + ## `using X = ` alias and treats X as an emittable scalar payload (the + ## CBOR wire value is a bare scalar, not a map). A no-op for non-primitive + ## non-object types, which stay TODO-stubbed in the wrappers. + if isTypeRegistered(typeName): + return + if parsed.objectDef.kind == nnkDistinctTy and parsed.objectDef.len == 1 and + parsed.objectDef[0].kind == nnkIdent and isNimPrimitive($parsed.objectDef[0]) and + ($parsed.objectDef[0]).toLowerAscii() notin ["cstring"]: + # `string` is allowed (maps to the wrapper's native string type) so a POD / + # option-B `string`-payload request is emittable; `cstring` stays excluded + # (unsafe to marshal across the FFI/CBOR boundary). + registerTypeEntry(makeAliasEntry(typeName, $parsed.objectDef[0], atkDistinct)) + +# --------------------------------------------------------------------------- +# Adapter proc type — exposed so registerBrokerLibrary (CBOR mode) can +# materialise a uniform table of dispatchers. +# --------------------------------------------------------------------------- + +type CborApiAdapter* = proc(ctx: BrokerContext, reqBuf: seq[byte]): Future[seq[byte]] {. + async: (raises: []), gcsafe +.} + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +proc collectSignatures( + body: NimNode +): tuple[ + zeroArg: NimNode, + argSig: NimNode, + argParams: seq[NimNode], + zeroArgName: string, + argSigName: string, +] {.compileTime.} = + ## Walk the macro body and split the (at most two) `signature*` proc + ## declarations into the zero-arg and arg-based slots, mirroring + ## `mt_request_broker` and the native path's handling. + result.zeroArg = nil + result.argSig = nil + result.argParams = @[] + result.zeroArgName = "" + result.argSigName = "" + + for stmt in body: + if stmt.kind != nnkProcDef: + continue + let procName = stmt[0] + let procNameIdent = + case procName.kind + of nnkIdent: + procName + of nnkPostfix: + procName[1] + else: + procName + if not ($procNameIdent).startsWith("signature"): + error("Signature proc names must start with `signature`", procName) + + let params = stmt.params + let paramCount = params.len - 1 + if paramCount == 0: + result.zeroArg = stmt + result.zeroArgName = $procNameIdent + elif paramCount >= 1: + result.argSig = stmt + result.argSigName = $procNameIdent + for idx in 1 ..< params.len: + result.argParams.add(copyNimTree(params[idx])) + +proc snakeApiName(typeIdent: NimNode): string {.compileTime.} = + ## Wire `apiName` for a request: snake_case form of the response type + ## identifier. e.g. `InitializeRequest` -> `initialize_request`. + toSnakeCase(sanitizeIdentName(typeIdent)) + +proc emitArgsType( + argsTypeIdent: NimNode, argParams: seq[NimNode] +): NimNode {.compileTime.} = + ## Build `type * = object\n field1*: T1\n field2*: T2` + ## from the arg-based signature's parameter nodes. + ## + ## Each `argParams[i]` is an `nnkIdentDefs` node carrying one or more + ## names plus a type. We expand each name into its own field so an arg + ## like `(a, b: int32)` produces two separate object fields. + var recList = newNimNode(nnkRecList) + for paramDefs in argParams: + let lastIdx = paramDefs.len - 1 + let typeNode = paramDefs[lastIdx - 1] + for nameIdx in 0 ..< lastIdx - 1: + let nameNode = paramDefs[nameIdx] + let fieldIdent = + case nameNode.kind + of nnkIdent, nnkSym: + ident($nameNode) + of nnkPostfix: + ident($nameNode[1]) + of nnkPragmaExpr: + ident($nameNode[0]) + else: + ident($nameNode) + recList.add( + newTree( + nnkIdentDefs, postfix(fieldIdent, "*"), copyNimTree(typeNode), newEmptyNode() + ) + ) + + newTree( + nnkTypeSection, + newTree( + nnkTypeDef, + postfix(argsTypeIdent, "*"), + newEmptyNode(), + newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), recList), + ), + ) + +# --------------------------------------------------------------------------- +# Adapter emission +# --------------------------------------------------------------------------- + +proc emitZeroArgAdapter( + typeIdent: NimNode, payloadType: NimNode, adapterIdent: NimNode, isVoid: bool +): NimNode {.compileTime.} = + ## Adapter for a zero-argument request: ignore the input buffer, await + ## the broker call, encode the response envelope. `typeIdent` is the + ## dispatch tag; `payloadType` is the (decoupled) value type the request + ## resolves to and the envelope carries. + ## + ## For a `void` payload the public broker resolves to `Result[void, string]`, + ## which has no `Option[void]`-encodable envelope. We bridge it to the wire + ## unit type `CborUnit` (a zero-field map `{}`), matching the legacy + ## `type X = void` form bit-for-bit. + if isVoid: + quote: + proc `adapterIdent`*( + ctx: BrokerContext, reqBuf: seq[byte] + ): Future[seq[byte]] {.async: (raises: []), gcsafe.} = + discard reqBuf + let r = await `typeIdent`.request(ctx) + let unitR = + if r.isOk: + Result[CborUnit, string].ok(CborUnit()) + else: + Result[CborUnit, string].err(r.error) + let envBytes = cborEncodeResultEnvelope(unitR) + if envBytes.isOk: + return envBytes.value + let errEnv = cborEncodeResultEnvelope( + Result[CborUnit, string].err("response encode failed: " & envBytes.error) + ) + if errEnv.isOk: + return errEnv.value + return @[] + + else: + quote: + proc `adapterIdent`*( + ctx: BrokerContext, reqBuf: seq[byte] + ): Future[seq[byte]] {.async: (raises: []), gcsafe.} = + discard reqBuf + let r = await `typeIdent`.request(ctx) + let envBytes = cborEncodeResultEnvelope(r) + if envBytes.isOk: + return envBytes.value + let errEnv = cborEncodeResultEnvelope( + Result[`payloadType`, string].err("response encode failed: " & envBytes.error) + ) + if errEnv.isOk: + return errEnv.value + return @[] + +proc emitArgAdapter( + typeIdent: NimNode, + payloadType: NimNode, + adapterIdent: NimNode, + argsTypeIdent: NimNode, + argParams: seq[NimNode], + isVoid: bool, +): NimNode {.compileTime, raises: [ValueError].} = + ## Adapter for an arg-based request. Decodes the request buffer into the + ## synthesised `argsTypeIdent`, awaits the broker call with each field + ## unpacked positionally, and encodes the resulting envelope. + ## + ## The proc body is rendered as a Nim source string and parsed back via + ## `parseStmt`. This sidesteps the awkward interaction between `quote + ## do:`'s gensym'd proc parameters and pre-built call nodes — every + ## identifier in the rendered string lives in the same local scope, so + ## name resolution is straightforward. + var fieldNames: seq[string] = @[] + for paramDefs in argParams: + let lastIdx = paramDefs.len - 1 + for nameIdx in 0 ..< lastIdx - 1: + let nameNode = paramDefs[nameIdx] + let nameStr = + case nameNode.kind + of nnkIdent, nnkSym: + $nameNode + of nnkPostfix: + $nameNode[1] + of nnkPragmaExpr: + $nameNode[0] + else: + $nameNode + fieldNames.add(nameStr) + + var argList = "" + for f in fieldNames: + argList.add(", decoded." & f) + + let typeIdentName = $typeIdent + # A `void` payload resolves to `Result[void, string]` (no encodable + # envelope); bridge it to the wire unit type `CborUnit`, matching the + # legacy `type X = void` form. Every envelope on this path then carries + # `CborUnit`, and the awaited result is converted before encoding. + let envTypeName = + if isVoid: + "CborUnit" + else: + payloadType.repr.strip() + let argsTypeIdentName = $argsTypeIdent + let adapterIdentName = $adapterIdent + + let encodeRespSrc = + if isVoid: + " let unitR =\n" & " if r.isOk: Result[CborUnit, string].ok(CborUnit())\n" & + " else: Result[CborUnit, string].err(r.error)\n" & + " let envBytes = cborEncodeResultEnvelope(unitR)\n" + else: + " let envBytes = cborEncodeResultEnvelope(r)\n" + + let src = + "proc " & adapterIdentName & "*(\n" & " ctx: BrokerContext, reqBuf: seq[byte]\n" & + "): Future[seq[byte]] {.async: (raises: []), gcsafe.} =\n" & + " let decRes = cborDecode(reqBuf, " & argsTypeIdentName & ")\n" & + " if decRes.isErr:\n" & " let errEnv = cborEncodeResultEnvelope(\n" & + " Result[" & envTypeName & + ", string].err(\"request decode failed: \" & decRes.error)\n" & " )\n" & + " if errEnv.isOk:\n" & " return errEnv.value\n" & " return @[]\n" & + " let decoded = decRes.value\n" & " let r = await " & typeIdentName & + ".request(ctx" & argList & ")\n" & encodeRespSrc & " if envBytes.isOk:\n" & + " return envBytes.value\n" & " let errEnv = cborEncodeResultEnvelope(\n" & + " Result[" & envTypeName & + ", string].err(\"response encode failed: \" & envBytes.error)\n" & " )\n" & + " if errEnv.isOk:\n" & " return errEnv.value\n" & " return @[]\n" + + parseStmt(src) + +# --------------------------------------------------------------------------- +# reduced-A: create-instance adapters. When a request's Ok payload type is a +# registered BrokerInterface(API), the provider builds and returns a sub- +# interface ref. We do NOT CBOR-encode the ref; instead the adapter extracts +# the sub-instance's BrokerContext and encodes it as a bare `uint32` (the +# routing handle). The foreign wrapper decodes that ctx and constructs the +# typed sub-wrapper class. Adapter + provider both run on the processing +# thread (same-thread direct dispatch), so the ref never crosses a channel — +# safe under both --mm:refc and --mm:orc. +# --------------------------------------------------------------------------- + +proc emitZeroArgInstanceAdapter( + typeIdent, adapterIdent: NimNode +): NimNode {.compileTime, raises: [ValueError].} = + let src = + "proc " & $adapterIdent & "*(\n" & " ctx: BrokerContext, reqBuf: seq[byte]\n" & + "): Future[seq[byte]] {.async: (raises: []), gcsafe.} =\n" & " discard reqBuf\n" & + " let r = await " & $typeIdent & ".request(ctx)\n" & " if r.isOk:\n" & + " installApiListenersForCtx(r.value.brokerCtx)\n" & " let mapped =\n" & + " if r.isOk: Result[uint32, string].ok(uint32(r.value.brokerCtx))\n" & + " else: Result[uint32, string].err(r.error)\n" & + " let envBytes = cborEncodeResultEnvelope(mapped)\n" & " if envBytes.isOk:\n" & + " return envBytes.value\n" & " return @[]\n" + parseStmt(src) + +proc emitArgInstanceAdapter( + typeIdent, adapterIdent, argsTypeIdent: NimNode, argParams: seq[NimNode] +): NimNode {.compileTime, raises: [ValueError].} = + var fieldNames: seq[string] = @[] + for paramDefs in argParams: + let lastIdx = paramDefs.len - 1 + for nameIdx in 0 ..< lastIdx - 1: + let nameNode = paramDefs[nameIdx] + let nameStr = + case nameNode.kind + of nnkIdent, nnkSym: + $nameNode + of nnkPostfix: + $nameNode[1] + of nnkPragmaExpr: + $nameNode[0] + else: + $nameNode + fieldNames.add(nameStr) + var argList = "" + for f in fieldNames: + argList.add(", decoded." & f) + let src = + "proc " & $adapterIdent & "*(\n" & " ctx: BrokerContext, reqBuf: seq[byte]\n" & + "): Future[seq[byte]] {.async: (raises: []), gcsafe.} =\n" & + " let decRes = cborDecode(reqBuf, " & $argsTypeIdent & ")\n" & + " if decRes.isErr:\n" & " let errEnv = cborEncodeResultEnvelope(\n" & + " Result[uint32, string].err(\"request decode failed: \" & decRes.error))\n" & + " if errEnv.isOk:\n" & " return errEnv.value\n" & " return @[]\n" & + " let decoded = decRes.value\n" & " let r = await " & $typeIdent & ".request(ctx" & + argList & ")\n" & " if r.isOk:\n" & + " installApiListenersForCtx(r.value.brokerCtx)\n" & " let mapped =\n" & + " if r.isOk: Result[uint32, string].ok(uint32(r.value.brokerCtx))\n" & + " else: Result[uint32, string].err(r.error)\n" & + " let envBytes = cborEncodeResultEnvelope(mapped)\n" & " if envBytes.isOk:\n" & + " return envBytes.value\n" & " return @[]\n" + parseStmt(src) + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + +proc generateApiCborRequestBrokerImpl( + body: NimNode, cfg: MtReqCfg +): NimNode {.raises: [ValueError].} = + ## Deferred-phase codegen for `RequestBroker(API)` under CBOR mode. + ## Runs after the typed-phase `autoRegisterApiType` calls have populated + ## `gApiTypeRegistry` for any external types referenced in the broker + ## response object or signature parameters (enums, distinct types, + ## nested objects). Wrapper codegen consumes that registry to emit + ## typed dataclasses / encoders / decoders. + result = newStmtList() + + # 1. Emit the underlying MT broker (typed Nim<->Nim dispatch on the + # processing thread, identical to the native path). Capacity + # config flows in from the outer RequestBroker(API, ...) kwargs — + # same knobs as RequestBroker(mt). + result.add(generateMtRequestBroker(copyNimTree(body), cfg)) + + # 2. Determine dispatch tag, payload, signatures, and the schema parse, + # supporting both the legacy `signature*` form and the proc-sugar. + var hasSignatureProc = false + var hasOtherProc = false + for stmt in body: + if stmt.kind == nnkProcDef: + let nm = stmt[0] + let nmId = (if nm.kind == nnkPostfix: nm[1] else: nm) + if ($nmId).startsWith("signature"): + hasSignatureProc = true + else: + hasOtherProc = true + let isSugar = hasOtherProc and not hasSignatureProc + + var typeIdent: NimNode = nil + var payloadType: NimNode = nil + var parsed: ParsedBrokerType + var zeroArgPresent = false + var argPresent = false + var argParams: seq[NimNode] = @[] + # Wire apiName suffixes. Legacy form keeps its descriptive + # `signature` mechanism (backward-compatible). The new proc-sugar + # uses the finalized rule: zero-arg stays bare, arg-based gets `_arg`. + var zeroApiSuffix = "" + var argApiSuffix = "" + + proc legacySuffix(sigName: string): string = + if sigName.len <= "signature".len: + return "" + toSnakeCase(sigName["signature".len .. ^1]) + + if not isSugar: + parsed = parseSingleTypeDef( + body, "RequestBroker", allowRefToNonObject = true, collectFieldInfo = true + ) + typeIdent = parsed.typeIdent + payloadType = copyNimTree(typeIdent) + let sigs = collectSignatures(body) + zeroArgPresent = not sigs.zeroArg.isNil + argPresent = not sigs.argSig.isNil + argParams = sigs.argParams + if zeroArgPresent and argPresent: + let zs = legacySuffix(sigs.zeroArgName) + zeroApiSuffix = (if zs.len > 0: "_" & zs else: "_zero") + let asfx = legacySuffix(sigs.argSigName) + argApiSuffix = (if asfx.len > 0: "_" & asfx else: "_args") + else: + let sg = parseRequestSugar(body, "RequestBroker", async = true) + typeIdent = sg.typeIdent + payloadType = sg.payloadType + parsed = sg.parsed + zeroArgPresent = not sg.zeroArgProc.isNil + argPresent = not sg.argProc.isNil + argParams = sg.argParams + if zeroArgPresent and argPresent: + argApiSuffix = "_arg" # zero-arg stays bare + + let typeName = sanitizeIdentName(typeIdent) + let apiName = snakeApiName(typeIdent) + + # reduced-A: does this request CREATE AND RETURN a sub-interface instance? + # (Its Ok payload type is a registered BrokerInterface(API).) If so the wire + # carries the sub-instance's ctx as a bare uint32 — we skip type registration + # (the interface ref is never CBOR-encoded) and emit instance adapters below. + let payloadName = payloadType.repr.strip() + let returnsIface = (if isApiInterface(payloadName): payloadName else: "") + + # Register the payload type in the schema so wrapper codegen can emit + # typed structs / aliases. For the proc-sugar POD form this mirrors the + # legacy `type X = ` registration exactly (wire-identical). + if returnsIface.len > 0: + discard # instance-returning request: no payload type to register. + elif parsed.hasInlineFields: + registerCborObjectType(typeName, parsed.fieldNames, parsed.fieldTypes) + elif parsed.isVoid: + # `void` → a zero-field object: payload-less request, the response + # envelope carries only the ok/err signal. + registerCborObjectType(typeName, @[], @[]) + else: + registerCborPrimitiveType(typeName, parsed) + + # Materialise (paramName, nimType) pairs from the arg-based signature so + # foreign-language wrapper codegen can emit a typed call signature. + proc paramFields(argParams: seq[NimNode]): seq[(string, string)] {.compileTime.} = + for paramDefs in argParams: + let lastIdx = paramDefs.len - 1 + let typeNode = paramDefs[lastIdx - 1] + let typeStr = typeNode.repr.strip() + for nameIdx in 0 ..< lastIdx - 1: + let nameNode = paramDefs[nameIdx] + let nameStr = + case nameNode.kind + of nnkIdent, nnkSym: + $nameNode + of nnkPostfix: + $nameNode[1] + of nnkPragmaExpr: + $nameNode[0] + else: + $nameNode + result.add((nameStr, typeStr)) + + # 3. Emit adapters + register descriptors. Naming rule (replaces the old + # `_zero`/`_args`): single signature → bare apiName; both slots present → + # the zero-arg keeps the bare name, the arg-based gets the `_arg` suffix + # (`Arg` in the foreign wrappers). + if not zeroArgPresent and not argPresent: + # No explicit signature — treat as zero-arg, matching the native default. + let adapterIdent = ident(typeName & "CborAdapter") + if returnsIface.len > 0: + result.add(emitZeroArgInstanceAdapter(typeIdent, adapterIdent)) + else: + result.add( + emitZeroArgAdapter(typeIdent, payloadType, adapterIdent, parsed.isVoid) + ) + registerCborRequestEntry( + apiName, $adapterIdent, typeName, @[], returnsInterface = returnsIface + ) + return + + if zeroArgPresent: + let zeroAdapterTag = if argPresent: "Zero" else: "" + let adapterIdent = ident(typeName & "CborAdapter" & zeroAdapterTag) + if returnsIface.len > 0: + result.add(emitZeroArgInstanceAdapter(typeIdent, adapterIdent)) + else: + result.add( + emitZeroArgAdapter(typeIdent, payloadType, adapterIdent, parsed.isVoid) + ) + registerCborRequestEntry( + apiName & zeroApiSuffix, + $adapterIdent, + typeName, + @[], + returnsInterface = returnsIface, + ) + + if argPresent: + let argAdapterTag = if zeroArgPresent: "Args" else: "" + let adapterIdent = ident(typeName & "CborAdapter" & argAdapterTag) + let argsTypeIdent = ident(typeName & "CborArgs" & argAdapterTag) + result.add(emitArgsType(argsTypeIdent, argParams)) + if returnsIface.len > 0: + result.add( + emitArgInstanceAdapter(typeIdent, adapterIdent, argsTypeIdent, argParams) + ) + else: + result.add( + emitArgAdapter( + typeIdent, payloadType, adapterIdent, argsTypeIdent, argParams, parsed.isVoid + ) + ) + let fields = paramFields(argParams) + registerCborRequestEntry( + apiName & argApiSuffix, + $adapterIdent, + typeName, + fields, + returnsInterface = returnsIface, + ) + + when defined(brokerDebug): + writeBrokerDebug( + "RequestBrokerApi", typeName, result, header = "apiName='" & apiName & "'" + ) + when defined(brokerDebugStdout): + echo "[brokers/cbor] RequestBroker(API) for '" & typeName & "' (apiName='" & + apiName & "')" + echo result.repr + +{.pop.} + +macro generateApiCborRequestBrokerDeferred*(args: varargs[untyped]): untyped = + ## Typed-phase deferred codegen entry point. By the time this expands, + ## any preceding `autoRegisterApiType` calls have already populated + ## `gApiTypeRegistry`, so wrapper codegen can introspect external + ## enum / distinct / object types without falling back to TODO stubs. + ## + ## Args layout: [body, kw0, kw1, ...]. Kwargs are forwarded as raw + ## `nnkExprEqExpr` nodes from `generateApiCborRequestBroker` and + ## re-parsed here into an MtReqCfg. + if args.len == 0: + error("generateApiCborRequestBrokerDeferred requires a body", args) + let body = args[0] + var kwargs: seq[NimNode] + for i in 1 ..< args.len: + kwargs.add(args[i]) + let cfg = parseMtReqKwargs(kwargs) + generateApiCborRequestBrokerImpl(body, cfg) + +{.push raises: [].} + +proc generateApiCborRequestBroker*(body: NimNode, kwargs: seq[NimNode]): NimNode = + ## Two-phase entry point — mirrors the native + ## `generateApiRequestBroker` pattern. Kwargs are passed through the + ## deferred macro call as raw nodes so the typed-phase expansion sees + ## the original literal values for `parseMtReqKwargs`. + result = newStmtList() + + let externalIdents = discoverExternalTypes(body) + if externalIdents.len > 0: + result.add(emitAutoRegistrations(externalIdents)) + + let deferred = + newCall(ident("generateApiCborRequestBrokerDeferred"), copyNimTree(body)) + for kw in kwargs: + deferred.add(copyNimTree(kw)) + result.add(deferred) + +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_schema.nim b/wasm-deps/brokers/brokers/internal/api_schema.nim new file mode 100644 index 000000000..21c81b1b1 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_schema.nim @@ -0,0 +1,232 @@ +## api_schema +## ---------- +## Compile-time type registry for FFI API code generation. +## +## This module provides a language-neutral schema that broker macros populate +## and codegen modules consume. It supports objects, enums, type aliases, +## and distinct types as first-class citizens. +## +## The registry stores type information for types used across the FFI boundary, +## needed for encoding/decoding, C/C++ header generation, and nested type +## marshalling. + +{.push raises: [].} + +import std/[macros, strutils] + +type + ApiTypeKind* = enum ## Discriminator for registered types. + atkObject ## Plain or ref object with fields + atkEnum ## Nim enum type + atkAlias ## Type alias (e.g. `type Timestamp = int64`) + atkDistinct ## Distinct type (e.g. `type MyId = distinct int32`) + + ApiEnumValue* = object ## A single value in an enum type. + name*: string + ordinal*: int + + ApiFieldDef* = object ## A single field in a type definition. + name*: string + nimType*: string ## "int64", "string", "bool", "seq[DeviceInfo]", etc. + isSeq*: bool ## true when nimType starts with "seq[" + seqElementType*: string ## e.g. "DeviceInfo" when isSeq + isArray*: bool ## true when nimType is "array[N, T]" + arraySize*: int ## e.g. 3 for array[3, int32] + arrayElementType*: string ## e.g. "int32" for array[3, int32] + isCustomObject*: bool ## true when type resolves to an object (not primitive) + + ApiTypeEntry* = object ## A registered type in the FFI schema. + name*: string ## "DeviceInfo" + kind*: ApiTypeKind ## What kind of type this is + fields*: seq[ApiFieldDef] ## field definitions (for atkObject) + enumValues*: seq[ApiEnumValue] ## enum values (for atkEnum) + underlyingType*: string ## base type (for atkAlias/atkDistinct) + +# --------------------------------------------------------------------------- +# Compile-time type registry +# --------------------------------------------------------------------------- + +var gApiTypeRegistry* {.compileTime.}: seq[ApiTypeEntry] = @[] + ## All types registered for FFI code generation. + ## Populated by auto-resolution (api_type_resolver) or legacy ApiType macro. + ## Consumed by codegen modules when processing seq[T] fields. + +# --------------------------------------------------------------------------- +# Primitive type detection +# --------------------------------------------------------------------------- + +const nimPrimitiveTypes* = [ + "string", "cstring", "char", "bool", "int", "int8", "int16", "int32", "int64", "uint", + "uint8", "uint16", "uint32", "uint64", "float", "float32", "float64", "byte", +] + +proc isNimPrimitive*(typeName: string): bool {.compileTime.} = + ## Returns true if `typeName` is a built-in Nim primitive type. + typeName.toLowerAscii() in nimPrimitiveTypes + +# --------------------------------------------------------------------------- +# Registry operations +# --------------------------------------------------------------------------- + +proc isTypeRegistered*(name: string): bool {.compileTime.} = + ## Check if a type is already in the registry. + for entry in gApiTypeRegistry: + if entry.name == name: + return true + false + +proc lookupTypeEntry*(name: string): ApiTypeEntry {.compileTime.} = + ## Lookup a type entry by name. Returns the entry or triggers a compile error. + for entry in gApiTypeRegistry: + if entry.name == name: + return entry + error( + "Type '" & name & "' not registered in FFI schema. " & + "Define it as a plain Nim type before using it in a broker macro, " & + "or declare it with `ApiType:` for explicit registration." + ) + +proc lookupTypeFields*(name: string): seq[(string, string)] {.compileTime.} = + ## Backward-compatible lookup returning (fieldName, nimTypeName) tuples. + ## This is the drop-in replacement for the old `lookupFfiStruct()`. + let entry = lookupTypeEntry(name) + for field in entry.fields: + result.add((field.name, field.nimType)) + +proc registerTypeEntry*(entry: ApiTypeEntry) {.compileTime.} = + ## Register a type in the schema. Skips if already registered. + if not isTypeRegistered(entry.name): + gApiTypeRegistry.add(entry) + +# --------------------------------------------------------------------------- +# Query helpers for type kinds +# --------------------------------------------------------------------------- + +proc isEnumRegistered*(name: string): bool {.compileTime.} = + ## Returns true if the name is registered as an enum type. + for entry in gApiTypeRegistry: + if entry.name == name and entry.kind == atkEnum: + return true + false + +proc isAliasOrDistinctRegistered*(name: string): bool {.compileTime.} = + ## Returns true if the name is registered as an alias or distinct type. + for entry in gApiTypeRegistry: + if entry.name == name and entry.kind in {atkAlias, atkDistinct}: + return true + false + +proc resolveUnderlyingType*(name: string): string {.compileTime.} = + ## Follows alias/distinct chains to the final underlying type name. + ## Returns the name itself if not registered as alias/distinct. + var current = name + var depth = 0 + while depth < 20: # safety limit + var found = false + for entry in gApiTypeRegistry: + if entry.name == current and entry.kind in {atkAlias, atkDistinct}: + current = entry.underlyingType + found = true + break + if not found: + break + inc depth + current + +# --------------------------------------------------------------------------- +# Type node inspection helpers +# --------------------------------------------------------------------------- + +proc isSeqOfPrimitive*(nimType: NimNode): bool {.compileTime.} = + ## Returns true when nimType is `seq[T]` and T is a primitive type. + if nimType.kind == nnkBracketExpr and nimType.len == 2 and + ($nimType[0]).toLowerAscii() == "seq": + let elemName = $nimType[1] + return isNimPrimitive(elemName) + false + +proc isArrayType*(nimType: NimNode): bool {.compileTime.} = + ## Returns true if the type node represents `array[N, T]`. + nimType.kind == nnkBracketExpr and nimType.len == 3 and + ($nimType[0]).toLowerAscii() == "array" + +proc arraySize*(nimType: NimNode): int {.compileTime.} = + ## Extracts N from `array[N, T]`. Expects an int literal. + assert isArrayType(nimType) + if nimType[1].kind == nnkIntLit: + int(nimType[1].intVal) + else: + error("array size must be an integer literal for FFI codegen", nimType[1]) + +proc arrayElemTypeName*(nimType: NimNode): string {.compileTime.} = + ## Extracts the element type name from `array[N, T]`. + assert isArrayType(nimType) + $nimType[2] + +# --------------------------------------------------------------------------- +# Field construction helpers +# --------------------------------------------------------------------------- + +proc makeFieldDef*(name, nimType: string): ApiFieldDef {.compileTime.} = + ## Construct an ApiFieldDef from name and type strings. + result.name = name + result.nimType = nimType + let lower = nimType.toLowerAscii() + if lower.startsWith("seq[") and lower.endsWith("]"): + result.isSeq = true + result.seqElementType = nimType[4 ..^ 2] # strip "seq[" and "]" + elif lower.startsWith("array["): + # Parse "array[N, T]" format + let inner = nimType[6 ..^ 2] # strip "array[" and "]" + let commaPos = inner.find(',') + if commaPos >= 0: + result.isArray = true + try: + result.arraySize = parseInt(inner[0 ..< commaPos].strip()) + except ValueError: + result.arraySize = 0 + result.arrayElementType = inner[commaPos + 1 .. ^1].strip() + if not isNimPrimitive(nimType) and not result.isSeq and not result.isArray: + result.isCustomObject = true + +proc makeTypeEntry*( + name: string, fields: seq[ApiFieldDef], kind: ApiTypeKind = atkObject +): ApiTypeEntry {.compileTime.} = + ## Construct an ApiTypeEntry for an object type. + result.name = name + result.kind = kind + result.fields = fields + +proc makeEnumEntry*( + name: string, values: seq[ApiEnumValue] +): ApiTypeEntry {.compileTime.} = + ## Construct an ApiTypeEntry for an enum type. + result.name = name + result.kind = atkEnum + result.enumValues = values + +proc makeAliasEntry*( + name: string, underlyingType: string, kind: ApiTypeKind = atkAlias +): ApiTypeEntry {.compileTime.} = + ## Construct an ApiTypeEntry for an alias or distinct type. + result.name = name + result.kind = kind + result.underlyingType = underlyingType + +# --------------------------------------------------------------------------- +# Backward compatibility: bridge to old registration format +# --------------------------------------------------------------------------- + +proc registerFromFieldTuples*( + typeName: string, fields: seq[(string, string)] +) {.compileTime.} = + ## Register a type from (fieldName, nimTypeName) tuples. + ## Used by the legacy ApiType macro and during migration. + if isTypeRegistered(typeName): + return + var fieldDefs: seq[ApiFieldDef] = @[] + for (fname, ftype) in fields: + fieldDefs.add(makeFieldDef(fname, ftype)) + registerTypeEntry(makeTypeEntry(typeName, fieldDefs)) + +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/api_type_resolver.nim b/wasm-deps/brokers/brokers/internal/api_type_resolver.nim new file mode 100644 index 000000000..7fd8e62d5 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/api_type_resolver.nim @@ -0,0 +1,455 @@ +## api_type_resolver +## ----------------- +## Two-phase external type introspection for FFI API broker macros. +## +## When a broker macro encounters a reference to an external type (e.g. +## `seq[DeviceInfo]` where `DeviceInfo` is a plain Nim type defined outside +## the macro body), this module resolves its fields at compile time and +## registers it in the API schema. +## +## ## Mechanism +## +## Phase 1 (called from an `untyped` broker macro): +## `discoverExternalTypes(body)` scans the raw AST for type identifiers +## that are not Nim primitives. Returns ident nodes. +## +## Phase 2 (typed macro expansion): +## `autoRegisterApiType(T: typed)` receives a resolved type symbol, +## calls `getTypeImpl()` to extract its fields, recursively resolves +## nested object types, and registers everything in `gApiTypeRegistry`. +## +## ## Supported type kinds +## +## - `object` types — field introspection and CItem generation +## - `enum` types — value extraction and C enum generation +## - `distinct` types — base type resolution and C typedef generation +## - Type aliases — base type resolution and C typedef generation + +{.push raises: [].} + +import std/[macros, strutils] +import ./api_schema + +export api_schema + +# --------------------------------------------------------------------------- +# Phase 2: Typed macro that resolves a single external type +# --------------------------------------------------------------------------- + +proc resolveActualSym(T: NimNode): NimNode {.compileTime.} = + ## Get the actual type symbol regardless of how T was passed. + ## Handles both typedesc[X] (from typed parameter) and direct symbols + ## (from recursive calls within the typed phase). + let impl = getTypeImpl(T) + case impl.kind + of nnkBracketExpr: + # typedesc[X] -> return X + if impl.len >= 2: + impl[1] + else: + nil + of nnkObjectTy: + # Already resolved; T itself is the symbol + T + of nnkEnumTy: + T + of nnkDistinctTy: + T + of nnkSym: + T + else: + nil + +proc extractFieldsFromSym(sym: NimNode): seq[(string, string)] {.compileTime.} = + ## Extract (fieldName, fieldTypeName) from a resolved type symbol. + let typeImpl = getTypeImpl(sym) + let obj = + if typeImpl.kind == nnkObjectTy: + typeImpl + elif typeImpl.kind == nnkBracketExpr and typeImpl.len >= 2: + getTypeImpl(typeImpl[1]) + else: + nil + + if obj.isNil or obj.kind != nnkObjectTy: + return @[] + + let recList = obj[2] + if recList.kind != nnkRecList: + return @[] + + for field in recList: + if field.kind != nnkIdentDefs: + continue + let fieldType = field[field.len - 2] + if fieldType.kind == nnkEmpty: + continue + for i in 0 ..< field.len - 2: + if field[i].kind == nnkEmpty: + continue + result.add(($field[i], repr(fieldType))) + +proc extractEnumValues(sym: NimNode): seq[(string, int)] {.compileTime.} = + ## Walk nnkEnumTy children to get (name, ordinal) pairs. + let typeImpl = getTypeImpl(sym) + let enumTy = + if typeImpl.kind == nnkEnumTy: + typeImpl + elif typeImpl.kind == nnkBracketExpr and typeImpl.len >= 2: + getTypeImpl(typeImpl[1]) + else: + nil + + if enumTy.isNil or enumTy.kind != nnkEnumTy: + return @[] + + var ordinal = 0 + for i in 1 ..< enumTy.len: # skip first child (empty node) + let child = enumTy[i] + case child.kind + of nnkSym: + result.add(($child, ordinal)) + inc ordinal + of nnkEnumFieldDef: + let fieldName = $child[0] + let fieldVal = int(child[1].intVal) + result.add((fieldName, fieldVal)) + ordinal = fieldVal + 1 + else: + discard + +const tuplePositionalNames* = + ["first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth", "ninth"] + ## Synthesised field names for unnamed positional tuple elements. + ## Tuples with more than 9 positional elements are rejected by the FFI + ## generator — wrap them in a named `object` instead. + +proc extractFieldsFromTupleSym(sym: NimNode): seq[(string, string)] {.compileTime.} = + ## Extract `(fieldName, fieldTypeName)` pairs from a resolved tuple type + ## symbol. Named tuples like `tuple[key: Key, payload: seq[byte]]` use the + ## declared field names verbatim. Unnamed positional tuples up to 9 + ## elements receive synthesised names from `tuplePositionalNames`. + let typeImpl = getTypeImpl(sym) + let tupleTy = if typeImpl.kind == nnkTupleTy: typeImpl else: nil + if tupleTy.isNil: + return @[] + + var posIdx = 0 + for child in tupleTy: + if child.kind == nnkIdentDefs: + let typeNode = child[child.len - 2] + for i in 0 ..< child.len - 2: + let rawName = $child[i] + result.add((rawName, typeNode.repr.strip())) + else: + if posIdx >= tuplePositionalNames.len: + error( + "FFI tuple support is limited to 9 positional fields; got element " & + $(posIdx + 1) & " of tuple " & $sym & ". Wrap in a named object instead.", + sym, + ) + result.add((tuplePositionalNames[posIdx], child.repr.strip())) + inc posIdx + +proc collectNestedTypeNodesFromTuple(sym: NimNode): seq[NimNode] {.compileTime.} = + ## Tuple-shaped analogue of `collectNestedTypeNodes` — walks a resolved + ## tuple type's fields and returns NimNodes for any nested custom types + ## (object / enum / distinct / alias / seq[Custom] / array[N, Custom]) + ## that need recursive registration. + let typeImpl = getTypeImpl(sym) + let tupleTy = if typeImpl.kind == nnkTupleTy: typeImpl else: nil + if tupleTy.isNil: + return @[] + + proc handleFieldType(fieldType: NimNode, acc: var seq[NimNode]) = + if fieldType.kind == nnkSym and not isNimPrimitive($fieldType): + let innerImpl = getTypeImpl(fieldType) + if innerImpl.kind in {nnkObjectTy, nnkEnumTy, nnkDistinctTy, nnkTupleTy}: + acc.add(fieldType) + else: + let instName = $getTypeInst(fieldType) + if instName != $fieldType and not isNimPrimitive(instName): + acc.add(fieldType) + elif fieldType.kind == nnkBracketExpr and fieldType.len >= 2 and + $fieldType[0] == "seq": + let elemSym = fieldType[1] + if elemSym.kind == nnkSym and not isNimPrimitive($elemSym): + let elemImpl = getTypeImpl(elemSym) + if elemImpl.kind in {nnkObjectTy, nnkEnumTy, nnkTupleTy, nnkDistinctTy}: + acc.add(elemSym) + elif fieldType.kind == nnkBracketExpr and fieldType.len == 3 and + $fieldType[0] == "array": + let elemSym = fieldType[2] + if elemSym.kind == nnkSym and not isNimPrimitive($elemSym): + let elemImpl = getTypeImpl(elemSym) + if elemImpl.kind in {nnkObjectTy, nnkEnumTy, nnkTupleTy, nnkDistinctTy}: + acc.add(elemSym) + + for child in tupleTy: + if child.kind == nnkIdentDefs: + let typeNode = child[child.len - 2] + handleFieldType(typeNode, result) + else: + handleFieldType(child, result) + +proc resolveAliasBase(sym: NimNode): string {.compileTime.} = + ## Follows alias/distinct chains to the underlying primitive name. + let typeImpl = getTypeImpl(sym) + if typeImpl.kind == nnkDistinctTy: + let base = typeImpl[0] + # `$` panics on non-symbol nodes (e.g. nnkBracketExpr for + # `distinct seq[byte]`); `repr` accepts any AST shape and yields + # the same printable form for symbols. + return base.repr.strip() + # For aliases, getTypeInst gives us the target + let typeInst = getTypeInst(sym) + if typeInst.kind == nnkBracketExpr and typeInst.len >= 2: + return typeInst[1].repr.strip() + if typeInst.kind == nnkSym: + return $typeInst + return sym.repr.strip() + +proc collectNestedTypeNodes(sym: NimNode): seq[NimNode] {.compileTime.} = + ## Walk the fields of a resolved type symbol and return NimNodes for + ## any nested custom object types or seq[T] element types that need + ## recursive registration. + let typeImpl = getTypeImpl(sym) + let obj = + if typeImpl.kind == nnkObjectTy: + typeImpl + elif typeImpl.kind == nnkBracketExpr and typeImpl.len >= 2: + getTypeImpl(typeImpl[1]) + else: + nil + + if obj.isNil or obj.kind != nnkObjectTy: + return @[] + + let recList = obj[2] + if recList.kind != nnkRecList: + return @[] + + for field in recList: + if field.kind != nnkIdentDefs: + continue + let fieldType = field[field.len - 2] + if fieldType.kind == nnkEmpty: + continue + + # Direct custom object field (e.g. `address: Address`) + if fieldType.kind == nnkSym and not isNimPrimitive($fieldType): + let innerImpl = getTypeImpl(fieldType) + if innerImpl.kind == nnkObjectTy: + result.add(fieldType) + elif innerImpl.kind == nnkEnumTy: + result.add(fieldType) + elif innerImpl.kind == nnkDistinctTy: + result.add(fieldType) + elif innerImpl.kind == nnkTupleTy: + result.add(fieldType) + else: + # Could be an alias — check if it resolves to something different + let instName = $getTypeInst(fieldType) + if instName != $fieldType and not isNimPrimitive(instName): + result.add(fieldType) + + # seq[T] where T is a custom type (e.g. `devices: seq[DeviceInfo]`) + elif fieldType.kind == nnkBracketExpr and fieldType.len >= 2 and + $fieldType[0] == "seq": + let elemSym = fieldType[1] + if elemSym.kind == nnkSym and not isNimPrimitive($elemSym): + let elemImpl = getTypeImpl(elemSym) + if elemImpl.kind in {nnkObjectTy, nnkEnumTy, nnkTupleTy, nnkDistinctTy}: + result.add(elemSym) + + # array[N, T] where T is a custom type + elif fieldType.kind == nnkBracketExpr and fieldType.len == 3 and + $fieldType[0] == "array": + let elemSym = fieldType[2] + if elemSym.kind == nnkSym and not isNimPrimitive($elemSym): + let elemImpl = getTypeImpl(elemSym) + if elemImpl.kind in {nnkObjectTy, nnkEnumTy, nnkTupleTy, nnkDistinctTy}: + result.add(elemSym) + +macro autoRegisterApiType*(T: typed): untyped = + ## Phase 2: Receives a resolved type symbol, extracts fields, + ## recursively processes nested types, registers in the schema, + ## and generates CItem type + encode proc + C/C++/Python codegen. + ## + ## Handles object types (full CItem generation), enum types (C enum + ## generation), and alias/distinct types (C typedef generation). + result = newStmtList() + + let actualSym = resolveActualSym(T) + if actualSym.isNil: + return result + + let typeName = $actualSym + if isTypeRegistered(typeName) or isNimPrimitive(typeName): + return result + + let typeImpl = getTypeImpl(actualSym) + + # Check for enum types + block checkEnum: + let enumTy = + if typeImpl.kind == nnkEnumTy: + typeImpl + elif typeImpl.kind == nnkBracketExpr and typeImpl.len >= 2: + let inner = getTypeImpl(typeImpl[1]) + if inner.kind == nnkEnumTy: inner else: nil + else: + nil + if not enumTy.isNil: + let values = extractEnumValues(actualSym) + var apiValues: seq[ApiEnumValue] = @[] + for (name, ordinal) in values: + apiValues.add(ApiEnumValue(name: name, ordinal: ordinal)) + registerTypeEntry(makeEnumEntry(typeName, apiValues)) + return result + + # Check for distinct types + if typeImpl.kind == nnkDistinctTy: + let baseName = resolveAliasBase(actualSym) + registerTypeEntry(makeAliasEntry(typeName, baseName, atkDistinct)) + return result + + # Check for alias types (sym that resolves to another sym/primitive) + block checkAlias: + let typeInst = getTypeInst(actualSym) + if typeInst.kind == nnkBracketExpr and typeInst.len >= 2: + let targetName = $typeInst[1] + if targetName != typeName: + registerTypeEntry(makeAliasEntry(typeName, targetName, atkAlias)) + return result + + # Tuple types — register as a synthesised object so the CBOR codegen + # modules (which iterate `gApiTypeRegistry` for `atkObject` entries) + # pick the tuple up and emit struct definitions. Named tuples keep + # their declared field names; unnamed positional tuples up to 9 + # elements receive `first`..`ninth`. + # + # Note: we deliberately DO NOT call `generateApiType` here. That path + # emits a fixed-layout `CItem` for the native ABI which has no + # count-companion field for `seq[T]` members — so a tuple like + # `tuple[a: Key, b: seq[byte]]` cannot fit. Native-ABI tuple support + # belongs to a follow-up task; for now the CBOR-mode codegen runs off + # the schema entry alone, and the native codegen sees an object with + # a missing CItem and falls back to its own TODO emission for + # downstream wrappers (which is what existing native-uncovered shapes + # like `seq[Object]` already do). + if typeImpl.kind == nnkTupleTy: + let tupleFields = extractFieldsFromTupleSym(actualSym) + if tupleFields.len == 0: + return result + let nestedNodesT = collectNestedTypeNodesFromTuple(actualSym) + for nestedSym in nestedNodesT: + let nestedName = $nestedSym + if not isTypeRegistered(nestedName) and not isNimPrimitive(nestedName): + result.add(newCall(ident("autoRegisterApiType"), nestedSym)) + registerFromFieldTuples(typeName, tupleFields) + # Bind a map-shaped CBOR encoder/decoder so the wire matches the + # named struct that wrappers emit for the same tuple type. The + # default `write[T: tuple]` in cbor_serialization writes positional + # CBOR arrays which decode wrappers reject as "expected map". + result.add(newCall(ident("bindCborTupleMap"), actualSym)) + return result + + # Object types — existing behavior + let fields = extractFieldsFromSym(actualSym) + if fields.len == 0: + return result + + # Emit recursive calls for nested types (depth-first: dependencies first) + let nestedNodes = collectNestedTypeNodes(actualSym) + for nestedSym in nestedNodes: + let nestedName = $nestedSym + if not isTypeRegistered(nestedName) and not isNimPrimitive(nestedName): + result.add(newCall(ident("autoRegisterApiType"), nestedSym)) + + # Register this type in the schema; CBOR codegen reads gApiTypeRegistry. + registerFromFieldTuples(typeName, fields) + +# --------------------------------------------------------------------------- +# Phase 1: Scan untyped AST for external type references +# --------------------------------------------------------------------------- + +proc scanTypeNode(ft: NimNode, result: var seq[NimNode]) {.compileTime.} = + ## Check a single type node for external type references. + ## Adds ident nodes for `seq[T]`, `array[N, T]`, and plain custom types. + if ft.kind == nnkEmpty: + return + # seq[T] + if ft.kind == nnkBracketExpr and ft.len >= 2 and $ft[0] == "seq": + let elemName = $ft[1] + if not isNimPrimitive(elemName): + result.add(ft[1]) + # array[N, T] + elif ft.kind == nnkBracketExpr and ft.len == 3 and $ft[0] == "array": + let elemName = $ft[2] + if not isNimPrimitive(elemName): + result.add(ft[2]) + # Plain custom type + elif ft.kind == nnkIdent and not isNimPrimitive($ft): + result.add(ft) + +proc discoverExternalTypes*(body: NimNode): seq[NimNode] {.compileTime.} = + ## Scan an untyped macro body for references to external types. + ## Returns ident nodes for each type that needs resolution. + ## + ## Detects: + ## - `seq[T]` fields in type definitions where T is not a primitive + ## - `array[N, T]` fields where T is not a primitive + ## - Plain custom type fields (`field: CustomType`) + ## - Type aliases (`type MyEvent = ExternalType`) + ## - `seq[T]` and custom types in proc signature parameters + var seen: seq[string] = @[] + + for stmt in body: + if stmt.kind == nnkTypeSection: + for def in stmt: + if def.kind != nnkTypeDef: + continue + let rhs = def[2] + + if rhs.kind == nnkObjectTy: + # Inline object: scan fields + let recList = rhs[2] + if recList.kind != nnkRecList: + continue + for field in recList: + if field.kind != nnkIdentDefs: + continue + let ft = field[field.len - 2] + scanTypeNode(ft, result) + elif rhs.kind == nnkIdent: + # Type alias: `type MyEvent = ExternalType` + let aliasTarget = $rhs + if not isNimPrimitive(aliasTarget): + result.add(rhs) + elif stmt.kind == nnkProcDef: + # Scan proc signature parameters for external types + let params = stmt.params + for i in 1 ..< params.len: + let paramDef = params[i] + if paramDef.kind == nnkIdentDefs: + let ft = paramDef[paramDef.len - 2] + scanTypeNode(ft, result) + + # Deduplicate (keep first occurrence) + var deduped: seq[NimNode] = @[] + for node in result: + let name = $node + if name notin seen: + seen.add(name) + deduped.add(node) + result = deduped + +proc emitAutoRegistrations*(externalIdents: seq[NimNode]): NimNode {.compileTime.} = + ## Generate `autoRegisterApiType(Ident)` calls for discovered external types. + ## These compile as typed macro invocations, triggering Phase 2 resolution. + result = newStmtList() + for typeIdent in externalIdents: + result.add(newCall(ident("autoRegisterApiType"), typeIdent)) + +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/broker_debug.nim b/wasm-deps/brokers/brokers/internal/broker_debug.nim new file mode 100644 index 000000000..4f5bccb47 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/broker_debug.nim @@ -0,0 +1,111 @@ +## Broker macro debug-dump helper +## =============================== +## Active when client code compiles with `-d:brokerDebug`. The broker +## macros call `writeBrokerDebug(...)` instead of (or in addition to) +## `echo result.repr`, dumping the generated Nim AST — rendered back +## to Nim source — into per-broker files for offline examination. +## +## Output layout (default): +## +## build/broker_debug/ +## ├── InitializeRequest__RequestBrokerApi.gen.nim +## ├── ShutdownRequest__RequestBrokerApi.gen.nim +## ├── DeviceStatusChanged__EventBrokerApi.gen.nim +## ├── PerfData__RequestBrokerMt.gen.nim +## ├── … +## └── mylib__BrokerLibrary.gen.nim ← `registerBrokerLibrary` +## (FFI C-ABI surface + +## courier/lifecycle plumbing) +## +## Override the directory with `-d:brokerDebugDir=`. The +## directory is created on demand. Files are overwritten — stale +## entries from prior builds are NOT auto-cleaned (delete the dir +## before a build if you want a fresh snapshot). +## +## To preserve the historical "echo result.repr" behaviour alongside +## the file dump, add `-d:brokerDebugStdout`. By default the dump is +## file-only so the build log isn't drowned in generated Nim. +## +## The helper is a `{.compileTime.}` proc — it runs in the Nim VM +## during macro expansion, the same way the C++/Python/Rust/Go +## wrapper codegens write their output files. + +{.push raises: [].} + +import std/[macros, os, strutils] + +const brokerDebugDirOverride {.strdefine: "brokerDebugDir".}: string = "" + +proc brokerDebugDir*(): string {.compileTime.} = + ## Directory under which dump files are written. Override via + ## `-d:brokerDebugDir=`. + if brokerDebugDirOverride.len > 0: brokerDebugDirOverride else: "build/broker_debug" + +proc sanitizeFileNamePart(s: string): string {.compileTime.} = + ## Coerce `s` to a portable filename fragment. Conservative — + ## anything outside `[A-Za-z0-9_-]` becomes `_`. + result = newStringOfCap(s.len) + for c in s: + if c in {'a' .. 'z', 'A' .. 'Z', '0' .. '9', '_', '-'}: + result.add(c) + else: + result.add('_') + +proc writeBrokerDebug*( + role: string, typeName: string, generated: NimNode, header: string = "" +) {.compileTime.} = + ## Dump the macro-generated AST for one broker (or the + ## `registerBrokerLibrary` output) into a per-broker file under + ## `brokerDebugDir()`. + ## + ## - `role` — e.g. "RequestBrokerApi" / "EventBrokerMt" / + ## "BrokerLibrary". Used in the filename suffix and + ## in the file header. + ## - `typeName` — the broker type name (or library name for + ## `BrokerLibrary`). Used as the filename stem. + ## - `generated` — the macro's `result` NimNode; its `.repr` is + ## written verbatim after a small comment header. + ## - `header` — optional one-line context note (e.g. + ## "apiName='initialize_request'"). + ## + ## Errors are reported via `echo` and the proc returns; we do NOT + ## raise into the compilation. A failed dump is a diagnostic loss, + ## not a build failure. + let dir = brokerDebugDir() + try: + createDir(dir) + except OSError as e: + echo "[brokers/debug] createDir('", dir, "') failed: ", e.msg, " — skipping dump." + return + except IOError as e: + echo "[brokers/debug] createDir('", dir, "') failed: ", e.msg, " — skipping dump." + return + except CatchableError as e: + echo "[brokers/debug] createDir('", dir, "') failed: ", e.msg, " — skipping dump." + return + + let safeName = sanitizeFileNamePart(typeName) + let safeRole = sanitizeFileNamePart(role) + let path = dir & "/" & safeName & "__" & safeRole & ".gen.nim" + + var s = newStringOfCap(4096) + s.add("## Auto-generated by nim-brokers macro expansion under -d:brokerDebug.\n") + s.add("## DO NOT EDIT — this file reflects the AST the macro emits,\n") + s.add("## rendered back to Nim source for offline examination.\n") + s.add("##\n") + s.add("## Role: " & role & "\n") + s.add("## Type: " & typeName & "\n") + if header.len > 0: + s.add("## Notes: " & header & "\n") + s.add("##\n") + s.add("## Open in your editor or pipe through `nph` for nicer formatting.\n\n") + s.add(generated.repr) + if not s.endsWith("\n"): + s.add("\n") + + try: + writeFile(path, s) + except IOError as e: + echo "[brokers/debug] writeFile('", path, "') failed: ", e.msg + +{.pop.} diff --git a/wasm-deps/brokers/brokers/internal/helper/broker_utils.nim b/wasm-deps/brokers/brokers/internal/helper/broker_utils.nim new file mode 100644 index 000000000..0dcb922e4 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/helper/broker_utils.nim @@ -0,0 +1,547 @@ +import std/[macros, strutils] + +type ParsedBrokerType* = object + ## Result of parsing the single `type` definition inside a broker macro body. + ## + ## - `typeIdent`: base identifier for the declared type name + ## - `objectDef`: exported type definition RHS (inline object fields exported; + ## non-object types wrapped in `distinct` unless already distinct) + ## - `isRefObject`: true only for inline `ref object` definitions + ## - `hasInlineFields`: true for inline `object` / `ref object` + ## - `fieldNames`/`fieldTypes`: populated only when `collectFieldInfo = true` + typeIdent*: NimNode + objectDef*: NimNode + isRefObject*: bool + hasInlineFields*: bool + isVoid*: bool ## true when the declared RHS is the bare `void` type + fieldNames*: seq[NimNode] + fieldTypes*: seq[NimNode] + +proc toSnakeCase*(name: string): string {.compileTime.} = + ## Converts PascalCase / camelCase to snake_case. Shared between the + ## CBOR codegen surface and any kept compile-time helper that needs to + ## derive a wire name from a Nim identifier. + result = "" + for i, ch in name: + if ch in {'A' .. 'Z'}: + if i > 0 and name[i - 1] notin {'A' .. 'Z', '_'}: + result.add('_') + result.add(chr(ord(ch) + 32)) + else: + result.add(ch) + +proc sanitizeIdentName*(node: NimNode): string = + var raw = $node + var sanitizedName = newStringOfCap(raw.len) + for ch in raw: + case ch + of 'A' .. 'Z', 'a' .. 'z', '0' .. '9', '_': + sanitizedName.add(ch) + else: + sanitizedName.add('_') + sanitizedName + +proc ensureFieldDef*(node: NimNode) = + if node.kind != nnkIdentDefs or node.len < 3: + error("Expected field definition of the form `name: Type`", node) + let typeSlot = node.len - 2 + if node[typeSlot].kind == nnkEmpty: + error("Field `" & $node[0] & "` must declare a type", node) + +proc exportIdentNode*(node: NimNode): NimNode = + case node.kind + of nnkIdent: + postfix(copyNimTree(node), "*") + of nnkPostfix: + node + else: + error("Unsupported identifier form in field definition", node) + +proc baseTypeIdent*(defName: NimNode): NimNode = + case defName.kind + of nnkIdent: + defName + of nnkAccQuoted: + if defName.len != 1: + error("Unsupported quoted identifier", defName) + defName[0] + of nnkPostfix: + baseTypeIdent(defName[1]) + of nnkPragmaExpr: + baseTypeIdent(defName[0]) + else: + error("Unsupported type name in broker definition", defName) + +proc ensureDistinctType*(rhs: NimNode): NimNode = + ## For PODs / aliases / externally-defined types, wrap in `distinct` unless + ## it's already distinct. + if rhs.kind == nnkDistinctTy: + return copyNimTree(rhs) + newTree(nnkDistinctTy, copyNimTree(rhs)) + +proc cloneParams*(params: seq[NimNode]): seq[NimNode] = + ## Deep copy parameter definitions so they can be inserted in multiple places. + result = @[] + for param in params: + result.add(copyNimTree(param)) + +proc collectParamNames*(params: seq[NimNode]): seq[NimNode] = + ## Extract all identifier symbols declared across IdentDefs nodes. + result = @[] + for param in params: + assert param.kind == nnkIdentDefs + for i in 0 ..< param.len - 2: + let nameNode = param[i] + if nameNode.kind == nnkEmpty: + continue + result.add(ident($nameNode)) + +proc parseOneTypeDef( + def: NimNode, + macroName: string, + allowRefToNonObject = false, + collectFieldInfo = false, +): ParsedBrokerType = + ## Parse a single nnkTypeDef node into a ParsedBrokerType. + ## Internal helper used by both parseSingleTypeDef and parseTypeDefs. + var fieldNames: seq[NimNode] = @[] + var fieldTypes: seq[NimNode] = @[] + + let typeIdent = baseTypeIdent(def[0]) + let rhs = def[2] + var objectDef: NimNode + var isRefObject = false + var hasInlineFields = false + var isVoid = false + + case rhs.kind + of nnkObjectTy: + let recList = rhs[2] + if recList.kind != nnkRecList: + error(macroName & " object must declare a standard field list", rhs) + var exportedRecList = newTree(nnkRecList) + for field in recList: + case field.kind + of nnkIdentDefs: + ensureFieldDef(field) + if collectFieldInfo: + let fieldTypeNode = field[field.len - 2] + for i in 0 ..< field.len - 2: + let baseFieldIdent = baseTypeIdent(field[i]) + fieldNames.add(copyNimTree(baseFieldIdent)) + fieldTypes.add(copyNimTree(fieldTypeNode)) + var cloned = copyNimTree(field) + for i in 0 ..< cloned.len - 2: + cloned[i] = exportIdentNode(cloned[i]) + exportedRecList.add(cloned) + of nnkEmpty: + discard + else: + error( + macroName & " object definition only supports simple field declarations", + field, + ) + objectDef = + newTree(nnkObjectTy, copyNimTree(rhs[0]), copyNimTree(rhs[1]), exportedRecList) + isRefObject = false + hasInlineFields = true + of nnkRefTy: + if rhs.len != 1: + error(macroName & " ref type must have a single base", rhs) + if rhs[0].kind == nnkObjectTy: + let obj = rhs[0] + let recList = obj[2] + if recList.kind != nnkRecList: + error(macroName & " object must declare a standard field list", obj) + var exportedRecList = newTree(nnkRecList) + for field in recList: + case field.kind + of nnkIdentDefs: + ensureFieldDef(field) + if collectFieldInfo: + let fieldTypeNode = field[field.len - 2] + for i in 0 ..< field.len - 2: + let baseFieldIdent = baseTypeIdent(field[i]) + fieldNames.add(copyNimTree(baseFieldIdent)) + fieldTypes.add(copyNimTree(fieldTypeNode)) + var cloned = copyNimTree(field) + for i in 0 ..< cloned.len - 2: + cloned[i] = exportIdentNode(cloned[i]) + exportedRecList.add(cloned) + of nnkEmpty: + discard + else: + error( + macroName & " object definition only supports simple field declarations", + field, + ) + let exportedObjectType = + newTree(nnkObjectTy, copyNimTree(obj[0]), copyNimTree(obj[1]), exportedRecList) + objectDef = newTree(nnkRefTy, exportedObjectType) + isRefObject = true + hasInlineFields = true + elif allowRefToNonObject: + ## `ref SomeType` (SomeType can be defined elsewhere) + objectDef = ensureDistinctType(rhs) + isRefObject = false + hasInlineFields = false + else: + error(macroName & " ref object must wrap a concrete object definition", rhs) + elif rhs.kind == nnkIdent and rhs.eqIdent("void"): + ## `void` — a payload-less broker. The bare `void` type cannot name a + ## broker (every `void` broker would share `typedesc[void]`, colliding + ## the generated `request` / `setProvider` / `emit` overloads). It is + ## therefore lowered to a *unique* empty `object` — a unit type — so + ## each broker keeps a distinct identity. `isVoid` lets broker macros + ## drop the now-meaningless value parameter from handler / emit + ## signatures; the request payload is simply the zero-field object. + objectDef = + newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), newTree(nnkRecList)) + isRefObject = false + hasInlineFields = false + isVoid = true + else: + ## Non-object type / alias. + objectDef = ensureDistinctType(rhs) + isRefObject = false + hasInlineFields = false + + result = ParsedBrokerType( + typeIdent: typeIdent, + objectDef: objectDef, + isRefObject: isRefObject, + hasInlineFields: hasInlineFields, + isVoid: isVoid, + fieldNames: fieldNames, + fieldTypes: fieldTypes, + ) + +proc parseTypeDefs*( + body: NimNode, + macroName: string, + allowRefToNonObject = false, + collectFieldInfo = false, +): seq[ParsedBrokerType] = + ## Parses all `type` definitions from a broker macro body. + ## Returns them in declaration order. Supports multiple types in a single + ## broker block (e.g. supporting types + primary type). + ## + ## Callers are responsible for identifying which entry is the "primary" type + ## (typically the last one, or the one referenced in the signature return type). + result = @[] + for stmt in body: + if stmt.kind != nnkTypeSection: + continue + for def in stmt: + if def.kind != nnkTypeDef: + continue + result.add(parseOneTypeDef(def, macroName, allowRefToNonObject, collectFieldInfo)) + + if result.len == 0: + error(macroName & " body must declare at least one type", body) + +proc parseSingleTypeDef*( + body: NimNode, + macroName: string, + allowRefToNonObject = false, + collectFieldInfo = false, +): ParsedBrokerType = + ## Parses exactly one `type` definition from a broker macro body. + ## Backward-compatible wrapper around parseTypeDefs that enforces a single type. + ## + ## Supported RHS: + ## - inline `object` / `ref object` (fields are auto-exported) + ## - non-object types / aliases / externally-defined types (wrapped in `distinct`) + ## - optionally: `ref SomeType` when `allowRefToNonObject = true` + let defs = parseTypeDefs(body, macroName, allowRefToNonObject, collectFieldInfo) + if defs.len > 1: + error("Only one type may be declared inside " & macroName, body) + result = defs[0] + +# --------------------------------------------------------------------------- +# RequestBroker proc-style sugar (option B — payload decoupled from the +# dispatch tag). Shared by the single-thread, multi-thread, and API +# RequestBroker generators so the surface stays identical across flavors. +# --------------------------------------------------------------------------- + +type ParsedRequestSugar* = object + ## Result of parsing the new proc-style RequestBroker sugar. + ## - `typeIdent` : the dispatch-tag type (broker name). + ## - `objectDef` : RHS to declare for the tag (the object for the object + ## form, `distinct payload` for the POD form). + ## - `payloadType`: the value type returned by `request` (decoupled — raw + ## payload for POD, == typeIdent for the object form). + ## - `fieldTypes` : object field types (object form) for MT auto-config; + ## empty for POD. + ## - `zeroArgProc`/`argProc`: the parsed signature proc defs (nil if absent). + ## - `argParams` : IdentDefs of the arg-based signature. + typeIdent*: NimNode + objectDef*: NimNode + payloadType*: NimNode + fieldTypes*: seq[NimNode] + zeroArgProc*: NimNode + argProc*: NimNode + argParams*: seq[NimNode] + verb*: string + ## The (lowercase) signature verb — the BrokerInterface method name that + ## `BrokerImplement` overrides (e.g. `getHealth`). + parsed*: ParsedBrokerType + ## Full parse of the dispatch tag over the payload — drives the API/CBOR + ## schema registration identically to the legacy `type X = ...` path. + +proc extractResultOk*(returnType: NimNode, async: bool): NimNode = + ## Ok payload type T from `Future[Result[T, string]]` (async) or + ## `Result[T, string]` (sync). Returns nil if the shape is invalid (error + ## type must be `string`). FFI/in-process errors are pinned to `string`. + if async: + if returnType.kind != nnkBracketExpr or returnType.len != 2: + return nil + if returnType[0].kind != nnkIdent or not returnType[0].eqIdent("Future"): + return nil + let inner = returnType[1] + if inner.kind != nnkBracketExpr or inner.len != 3: + return nil + if inner[0].kind != nnkIdent or not inner[0].eqIdent("Result"): + return nil + if not (inner[2].kind == nnkIdent and inner[2].eqIdent("string")): + return nil + return inner[1] + else: + if returnType.kind != nnkBracketExpr or returnType.len != 3: + return nil + if returnType[0].kind != nnkIdent or not returnType[0].eqIdent("Result"): + return nil + if not (returnType[2].kind == nnkIdent and returnType[2].eqIdent("string")): + return nil + return returnType[1] + +proc sugarVerbIdent(p: NimNode): NimNode = + let nm = p[0] + if nm.kind == nnkPostfix: + nm[1] + else: + nm + +proc parseRequestSugar*( + body: NimNode, macroName: string, async: bool +): ParsedRequestSugar = + ## Parse the proc-style sugar form of a RequestBroker body (one broker per + ## block, two signature slots, payload decoupled from the dispatch tag). + var typeDecl: NimNode = nil + var procs: seq[NimNode] = @[] + for stmt in body: + case stmt.kind + of nnkProcDef: + procs.add(stmt) + of nnkTypeSection: + for d in stmt: + if d.kind == nnkTypeDef: + if typeDecl != nil: + error(macroName & " sugar allows a single payload type", d) + typeDecl = d + of nnkEmpty: + discard + else: + error("Unsupported statement inside " & macroName & " definition", stmt) + if procs.len == 0: + error(macroName & " requires at least one signature proc", body) + + var verb = "" + for p in procs: + if verb.len == 0: + verb = $sugarVerbIdent(p) + elif not sugarVerbIdent(p).eqIdent(verb): + error("All signatures in one " & macroName & " block must share the proc name", p) + let brokerName = capitalizeAscii(verb) + result.verb = verb + + if typeDecl != nil: + let parsedT = parseSingleTypeDef( + newTree(nnkStmtList, newTree(nnkTypeSection, typeDecl)), + macroName, + allowRefToNonObject = true, + collectFieldInfo = true, + ) + result.typeIdent = parsedT.typeIdent + result.objectDef = parsedT.objectDef + result.fieldTypes = parsedT.fieldTypes + result.parsed = parsedT + if not result.typeIdent.eqIdent(brokerName): + error( + "Signature `" & verb & "` must pair with type `" & brokerName & "` (got `" & + $result.typeIdent & "`)", + typeDecl, + ) + result.payloadType = copyNimTree(result.typeIdent) + else: + # POD form: the broker name is derived solely from the proc verb and is + # always Capitalized (it is a Nim type / dispatch tag). Warn when we had to + # capitalize a lowercase verb so the `Broker.request(...)` handle name is + # not a surprise; writing the proc Capitalized (`proc GetConfig(...)`) is + # accepted and silences this. + if verb.len > 0 and verb[0] in {'a' .. 'z'}: + warning( + "RequestBroker: broker name is `" & brokerName & "` (capitalized from proc `" & + verb & "`); call it as `" & brokerName & ".request(...)`. Write `proc " & + brokerName & "(...)` to name it explicitly and silence this warning.", + procs[0], + ) + result.typeIdent = ident(brokerName) + + for p in procs: + let params = p.params + if params.len == 0: + error("Signature must declare a return type", p) + let pl = extractResultOk(params[0], async) + if pl.isNil: + error( + "Signature must return " & + (if async: "Future[Result[T, string]]" else: "Result[T, string]"), + p, + ) + if result.payloadType.isNil: + result.payloadType = copyNimTree(pl) + elif result.payloadType.repr != pl.repr: + error( + "All signatures of broker `" & brokerName & "` must return the same payload type", + p, + ) + let paramCount = params.len - 1 + if paramCount == 0: + if not result.zeroArgProc.isNil: + error("Only one zero-argument signature is allowed", p) + result.zeroArgProc = p + else: + if not result.argProc.isNil: + error("Only one argument-based signature is allowed", p) + result.argProc = p + result.argParams = @[] + for idx in 1 ..< params.len: + let pd = params[idx] + if pd.kind != nnkIdentDefs: + error("Signature parameter must be a standard identifier declaration", pd) + if pd[pd.len - 2].kind == nnkEmpty: + error("Signature parameter must declare a type", pd) + result.argParams.add(copyNimTree(pd)) + + if typeDecl == nil: + # POD: synthesize `type = ` and parse it through the normal + # path so the dispatch-tag classification (primitive / void / distinct) and + # the API/CBOR schema registration match the legacy `type X = ...` form. + let synth = newTree( + nnkStmtList, + newTree( + nnkTypeSection, + newTree( + nnkTypeDef, + copyNimTree(result.typeIdent), + newEmptyNode(), + copyNimTree(result.payloadType), + ), + ), + ) + result.parsed = parseSingleTypeDef( + synth, macroName, allowRefToNonObject = true, collectFieldInfo = true + ) + result.objectDef = result.parsed.objectDef + +# --------------------------------------------------------------------------- +# Compile-time interface -> event-type registry. BrokerInterface records the +# event types it declares; BrokerImplement reads them so the generated +# `close()` can drop the instance's event listeners (the impl macro otherwise +# doesn't know the interface's events). +# --------------------------------------------------------------------------- + +var gInterfaceEvents {.compileTime.}: seq[(string, seq[string])] = @[] + +proc registerInterfaceEvents*(iface: string, events: seq[string]) {.compileTime.} = + for i in 0 ..< gInterfaceEvents.len: + if gInterfaceEvents[i][0] == iface: + gInterfaceEvents[i][1] = events + return + gInterfaceEvents.add((iface, events)) + +proc interfaceEvents*(iface: string): seq[string] {.compileTime.} = + for it in gInterfaceEvents: + if it[0] == iface: + return it[1] + @[] + +# --------------------------------------------------------------------------- +# Compile-time registry of interface request verbs. +# Records, per interface, the verb name and the associated request type name. +# Used by BrokerImplement to validate that all declared requests are overridden. +# --------------------------------------------------------------------------- + +var gInterfaceVerbs {.compileTime.}: seq[(string, seq[(string, string)])] = @[] + +proc registerInterfaceVerbs*( + iface: string, verbs: seq[(string, string)] +) {.compileTime.} = + for i in 0 ..< gInterfaceVerbs.len: + if gInterfaceVerbs[i][0] == iface: + gInterfaceVerbs[i] = (iface, verbs) + return + gInterfaceVerbs.add((iface, verbs)) + +proc interfaceRequestVerbs*(iface: string): seq[(string, string)] {.compileTime.} = + for it in gInterfaceVerbs: + if it[0] == iface: + return it[1] + @[] + +# --------------------------------------------------------------------------- +# Compile-time registry of `BrokerInterface(API)` interfaces (reduced-A, A1). +# Records, per interface, the sanitized request *type* names and event *type* +# names it owns. The flat CBOR request/event entry registries store these same +# type names (CborRequestEntry.responseTypeName / CborEventEntry.typeName), so +# wrapper codegen can partition the flat entry lists per interface by matching +# on type name — no need to replicate the snake/suffix apiName derivation here. +# --------------------------------------------------------------------------- + +type ApiInterfaceEntry* = object + name*: string + requestTypes*: seq[string] ## sanitized request broker type names + eventTypes*: seq[string] ## event payload type names + +var gApiInterfaces {.compileTime.}: seq[ApiInterfaceEntry] = @[] + +proc registerApiInterface*( + name: string, requestTypes, eventTypes: seq[string] +) {.compileTime.} = + for i in 0 ..< gApiInterfaces.len: + if gApiInterfaces[i].name == name: + gApiInterfaces[i].requestTypes = requestTypes + gApiInterfaces[i].eventTypes = eventTypes + return + gApiInterfaces.add( + ApiInterfaceEntry(name: name, requestTypes: requestTypes, eventTypes: eventTypes) + ) + +proc apiInterfaces*(): seq[ApiInterfaceEntry] {.compileTime.} = + gApiInterfaces + +proc isApiInterface*(name: string): bool {.compileTime.} = + for it in gApiInterfaces: + if it.name == name: + return true + false + +proc interfaceOwningRequestType*(typeName: string): string {.compileTime.} = + ## Comma-joined names of every interface that declared the request broker + ## `typeName` (more than one when two interfaces reuse the same type name — + ## itself the most common apiName collision), or "" if none. + var owners: seq[string] = @[] + for it in gApiInterfaces: + for rt in it.requestTypes: + if rt == typeName: + owners.add(it.name) + owners.join(", ") + +proc interfaceOwningEventType*(typeName: string): string {.compileTime.} = + var owners: seq[string] = @[] + for it in gApiInterfaces: + for et in it.eventTypes: + if et == typeName: + owners.add(it.name) + owners.join(", ") diff --git a/wasm-deps/brokers/brokers/internal/mt_broker_common.nim b/wasm-deps/brokers/brokers/internal/mt_broker_common.nim new file mode 100644 index 000000000..c02a6d4a8 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/mt_broker_common.nim @@ -0,0 +1,324 @@ +## Multi-Thread Broker Common +## -------------------------- +## Shared runtime helpers used by both mt_request_broker and mt_event_broker. +## These are not generated — they are used directly by generated code. + +{.push raises: [].} + +import chronos, chronos/threadsync +import std/atomics +import std/[os, locks] # `sleep`; `Lock` for the API listener-installer registry +import results +import ../broker_context +import ./mt_queue +export chronos, threadsync, atomics + +# --------------------------------------------------------------------------- +# reduced-A: per-classCtx event-listener installer registry. +# +# An EventBroker(API) event only reaches the foreign event courier if the +# library's `installAllListeners` has been called for the *emitting* ctx. The +# main library ctx is handled at createContext, but a SUB-INSTANCE (created via +# a create-instance request, sharing the library classCtx with a distinct +# instanceCtx) needs its listeners installed too. registerBrokerLibrary records +# its installer keyed by classCtx here; the create-instance adapter calls +# `installApiListenersForCtx(subCtx)` on the processing thread. +# +# Storage is a fixed POD array (the installer is a bare `nimcall` function +# pointer, the key a uint16) so it is safe to share across threads under both +# --mm:refc and --mm:orc — no GC'd container crosses the thread boundary. +# --------------------------------------------------------------------------- + +const maxApiCtxInstallers* = 64 + +type ApiCtxListenerInstaller* = + proc(ctx: BrokerContext): Result[void, string] {.nimcall.} + +var gApiCtxInstallers: + array[maxApiCtxInstallers, tuple[classCtx: uint16, fn: ApiCtxListenerInstaller]] +var gApiCtxInstallerCount: int +var gApiCtxInstallerLock: Lock +var gApiCtxInstallerLockInit: Atomic[int] + +proc ensureApiCtxInstallerLock() {.gcsafe.} = + var expected = 0 + if gApiCtxInstallerLockInit.compareExchange(expected, 1, moAcquire, moRelaxed): + {.cast(gcsafe).}: + initLock(gApiCtxInstallerLock) + gApiCtxInstallerLockInit.store(2, moRelease) + else: + while gApiCtxInstallerLockInit.load(moAcquire) != 2: + sleep(0) + +proc registerApiCtxListenerInstaller*( + classCtx: uint16, fn: ApiCtxListenerInstaller +) {.gcsafe.} = + ## Record (or replace) the listener installer for a library, keyed by its + ## classCtx. Called once per `createContext`. + ensureApiCtxInstallerLock() + {.cast(gcsafe).}: + withLock gApiCtxInstallerLock: + for i in 0 ..< gApiCtxInstallerCount: + if gApiCtxInstallers[i].classCtx == classCtx: + gApiCtxInstallers[i].fn = fn + return + if gApiCtxInstallerCount < maxApiCtxInstallers: + gApiCtxInstallers[gApiCtxInstallerCount] = (classCtx, fn) + inc gApiCtxInstallerCount + +proc installApiListenersForCtx*(ctx: BrokerContext) {.gcsafe.} = + ## Install the owning library's event-courier listeners for a sub-instance + ## ctx (looked up by classCtx). Best-effort: if no installer is registered + ## (e.g. a library with no events) or it fails, the sub-instance simply has no + ## event delivery. Runs on the processing thread. + ensureApiCtxInstallerLock() + var fn: ApiCtxListenerInstaller = nil + let cc = classCtx(ctx) + {.cast(gcsafe).}: + withLock gApiCtxInstallerLock: + for i in 0 ..< gApiCtxInstallerCount: + if gApiCtxInstallers[i].classCtx == cc: + fn = gApiCtxInstallers[i].fn + break + if not fn.isNil: + try: + {.cast(gcsafe).}: + discard fn(ctx) + except Exception: + discard + +# --------------------------------------------------------------------------- +# Thread identity +# --------------------------------------------------------------------------- + +var mtThreadIdMarker* {.threadvar.}: bool + ## Each thread gets its own copy; `addr mtThreadIdMarker` is a unique thread id. + +template currentMtThreadId*(): pointer = + addr mtThreadIdMarker + +# --------------------------------------------------------------------------- +# Thread generation — monotonically increasing, unique per thread incarnation. +# Under refc, threadvar addresses can be reused when threads exit and new +# ones are created. The generation counter disambiguates reused addresses. +# --------------------------------------------------------------------------- + +var gMtThreadGenCounter: Atomic[uint64] + +var mtThreadGen* {.threadvar.}: uint64 +var mtThreadGenInitialized {.threadvar.}: bool + +proc currentMtThreadGen*(): uint64 = + if not mtThreadGenInitialized: + mtThreadGen = gMtThreadGenCounter.fetchAdd(1, moRelaxed) + mtThreadGenInitialized = true + mtThreadGen + +# --------------------------------------------------------------------------- +# Blocking await for {.thread.} procs +# --------------------------------------------------------------------------- + +template blockingAwait*[T](f: Future[T]): T = + ## Blocking await for use inside non-async `{.thread.}` procs. + ## Use this instead of `await` (which conflicts with chronos's async-only + ## `await`) or call `waitFor` directly. + waitFor(f) + +# --------------------------------------------------------------------------- +# Per-thread shared signal + dispatcher +# --------------------------------------------------------------------------- +# Instead of one ThreadSignalPtr (2 fds on macOS, 1 on Linux) per broker +# type per thread, every broker type on the same thread shares a single +# ThreadSignalPtr. Fd count drops from O(broker_types × threads) to +# O(threads). +# +# Each broker type registers a poll proc (ThreadDispatchPollFn). The +# shared brokerDispatchLoop coroutine fires whenever ANY channel on this +# thread has a new message, then drains all registered poll procs. +# Poll proc return values: +# 0 — nothing to process; keep registered +# 1 — message processed; keep registered +# 2 — done (shutdown or one-shot complete); remove from dispatcher +# --------------------------------------------------------------------------- + +type ThreadDispatchPollFn* = proc(): int {.gcsafe, raises: [].} + +var gBrokerThreadSignal* {.threadvar.}: ThreadSignalPtr +var gBrokerThreadPollers* {.threadvar.}: seq[ThreadDispatchPollFn] +var gBrokerDispatchStarted* {.threadvar.}: bool +var gBrokerDispatchStopRequested* {.threadvar.}: bool + ## Set by stopBrokerDispatchHere() to ask the loop to exit on its next + ## drain pass. Used by FFI entry points so that transient foreign threads + ## (e.g. the C++ caller of _request_*/_shutdown) don't accumulate + ## a persistent suspended coroutine and its associated chronos/GC state + ## across calls. The flag is cleared by stopBrokerDispatchHere() after the + ## loop confirms exit. + +proc getOrInitBrokerSignal*(): ThreadSignalPtr = + ## Get (or lazily create) the per-thread signal shared by all broker types. + if gBrokerThreadSignal.isNil: + let res = ThreadSignalPtr.new() + if res.isErr(): + raiseAssert "BrokerDispatcher: failed to create thread signal: " & res.error + gBrokerThreadSignal = res.get() + gBrokerThreadSignal + +proc fireBrokerSignal*(signal: ThreadSignalPtr) {.gcsafe, raises: [].} = + ## Wake the target thread's broker dispatcher. Safe to call from any thread. + discard signal.fireSync() + +proc registerBrokerPoller*(fn: ThreadDispatchPollFn) = + ## Register a poll function with this thread's dispatcher. + ## Must be called from the owning thread. + gBrokerThreadPollers.add(fn) + +proc brokerDispatchLoop*(signal: ThreadSignalPtr) {.async: (raises: []).} = + ## Single dispatch loop per chronos thread. Drains all registered broker + ## channel pollers whenever the shared signal fires. + while true: + # Drain: keep polling until every channel is empty. + var anyWork = true + while anyWork: + anyWork = false + var i = 0 + while i < gBrokerThreadPollers.len: + let r = gBrokerThreadPollers[i]() + case r + of 2: + # Poller is done — remove it. + gBrokerThreadPollers.del(i) + of 1: + anyWork = true + inc i + else: + inc i + # FFI-caller teardown hook: an external caller (stopBrokerDispatchHere) + # asked the loop to exit. Drain pass is complete, exit cleanly. + if gBrokerDispatchStopRequested: + break + # Wait for next signal. + let waitRes = catch: + await signal.wait() + if waitRes.isErr(): + break + if gBrokerDispatchStopRequested: + break + # Dispatcher is exiting (e.g. thread shutting down). Close the per-thread + # signal so its OS handle (eventfd on Linux, pipe pair on macOS) is reclaimed + # instead of leaking on every createContext/processing-thread cycle. Reset + # the threadvar state so a future ensureBrokerDispatchStarted() on a reused + # threadvar address (refc) starts fresh. + let sig = gBrokerThreadSignal + gBrokerThreadSignal = nil + gBrokerDispatchStarted = false + if not sig.isNil: + let closeRes = sig.close() + if closeRes.isErr(): + discard + +# --------------------------------------------------------------------------- +# Pending-ring-free registry — synchronous deferred cleanup at thread exit +# --------------------------------------------------------------------------- +# When clearProvider(ctx) closes a request broker's ring on the provider +# thread, the corresponding poll fn (registered via brokerDispatchLoop's +# pollers seq) detects `ring.isClosed()` on its next iteration and needs to +# free the shared-memory (ring, slab, pool) triple — but only after a grace +# window long enough for any cross-thread sender that snapshotted those +# pointers under the previous globalLock state to finish its enqueue. +# +# Previous design: `asyncSpawn deferredFreeReqRing(...)` — start an async +# proc that does `await sleepAsync(50ms)` then frees. Two problems: +# +# 1. Allocating the sleepAsync Future inside an asyncSpawn started during +# `cleanupAllRequestsIdent` runs the refc allocator at a moment where +# the thread's gch state is fragile from teardown churn. Observed as a +# hard SEGV in rawAlloc on Linux refc + ASAN (PR #13). +# +# 2. drainAsyncOps only polls chronos for 1ms — the 50ms sleepAsync would +# never fire before the processing thread exits, so the buffers either +# leak or are freed by an orphaned coroutine racing thread teardown. +# +# Current design: the poll fn instead records the triple in a thread-local +# seq; the processing-thread proc drains the seq AFTER drainAsyncOps via a +# single synchronous `sleep(50)` followed by direct free calls. No chronos +# involvement in the cleanup path; the grace window applies once for the +# whole ctx instead of once per broker. +type PendingRingFree* = object + ring*: ptr VyukovMpscRing[uint32] + slab*: ptr PayloadSlab + pool*: ptr ResponseSlotPool + +var gPendingRingFrees* {.threadvar.}: seq[PendingRingFree] + +proc enqueuePendingRingFree*( + ring: ptr VyukovMpscRing[uint32], slab: ptr PayloadSlab, pool: ptr ResponseSlotPool +) {.gcsafe.} = + ## Called from a broker poll fn on the provider thread when its ring has + ## been closed by clearProvider(). The (ring, slab, pool) triple will be + ## freed by `drainPendingRingFrees()` at thread shutdown. + {.cast(gcsafe).}: + gPendingRingFrees.add(PendingRingFree(ring: ring, slab: slab, pool: pool)) + +proc drainPendingRingFrees*() {.gcsafe.} = + ## Drain the per-thread pending-ring-free registry synchronously. + ## Sleeps once for a 50ms grace window covering all queued frees, then + ## releases each (ring, slab, pool) triple. Must be called from the owning + ## thread AFTER any chronos work that may still touch the buffers has + ## completed (i.e. after `drainAsyncOps` in the processing-thread proc). + if gPendingRingFrees.len == 0: + return + # Single grace window: 50ms is enough for any sender that snapshotted + # pool/slab/ring pointers before clearProvider closed the ring to either + # complete its enqueue (which then fails on isClosed()) or abort. Without + # this, a stale sender deref'ing the about-to-be-freed slab/pool crashes. + sleep(50) + for entry in gPendingRingFrees: + if not entry.ring.isNil: + freeVyukovMpscRing(entry.ring) + if not entry.slab.isNil: + deinitPayloadSlab(entry.slab[]) + deallocShared(entry.slab) + if not entry.pool.isNil: + deinitResponseSlotPool(entry.pool[]) + deallocShared(entry.pool) + gPendingRingFrees.setLen(0) + +proc ensureBrokerDispatchStarted*() = + ## Start the per-thread dispatch loop if not already running. + ## Must be called from within a chronos async context. + if not gBrokerDispatchStarted: + gBrokerDispatchStarted = true + asyncSpawn brokerDispatchLoop(getOrInitBrokerSignal()) + +proc stopBrokerDispatchHere*() = + ## Tear down the per-thread brokerDispatchLoop on the calling thread. + ## + ## Intended for **FFI entry points** (procs exported with `cdecl, dynlib` + ## that run on a foreign caller's thread). The dispatch loop was designed + ## for chronos-loop-owning threads (processing/delivery threads), which + ## are torn down via joinThread. An FFI caller's thread instead lives for + ## the entire process and re-enters Nim per call; without teardown its + ## suspended `await signal.wait()` future, registered pollers seq, and + ## chronos pending-callback list accumulate across calls and eventually + ## drag the thread's refc ZCT/heap into corruption (PR #13). + ## + ## Safe to call from sync context (after `waitFor` returns). No-op if the + ## loop was never started on this thread. Drives chronos via an internal + ## `waitFor` until the loop's coroutine actually exits. + if not gBrokerDispatchStarted: + return + gBrokerDispatchStopRequested = true + let sig = gBrokerThreadSignal + if not sig.isNil: + discard sig.fireSync() + + proc awaitLoopExit() {.async: (raises: []).} = + let deadline = Moment.now() + chronos.seconds(2) + while gBrokerDispatchStarted and Moment.now() < deadline: + let sleepRes = catch: + await sleepAsync(milliseconds(1)) + if sleepRes.isErr(): + break + + waitFor awaitLoopExit() + gBrokerDispatchStopRequested = false diff --git a/wasm-deps/brokers/brokers/internal/mt_codec.nim b/wasm-deps/brokers/brokers/internal/mt_codec.nim new file mode 100644 index 000000000..5cd1238ec --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/mt_codec.nim @@ -0,0 +1,307 @@ +## Runtime marshal / unmarshal helpers for (mt) broker payloads. +## +## The broker macro emits two thin per-type wrappers +## (`MtMarshal` / `MtUnmarshal`) that call into the +## generic `mtMarshalValue` / `mtUnmarshalValue` defined here. Those +## generics use `when supportsCopyMem(T):` + Nim's `fieldPairs` to walk +## arbitrary payload types at compile time, recursing into: +## +## - **POD types** (scalars, enums, distinct-of-POD, fixed POD arrays, +## objects whose fields are all POD): single `copyMem(sizeof(T))`. +## `supportsCopyMem` correctly classifies all of these. +## - **`string`**: 4-byte little-endian length + bytes. +## - **`seq[U]`**: 4-byte length + per-element recursive marshal. +## - **`array[N, U]` where U is non-POD**: per-element recursive marshal. +## - **objects with non-POD fields**: `fieldPairs` walks each field +## recursively. +## +## Forbidden (caught at the call site by a compile-time `{.error.}`): +## `ref T`, `ptr T`, `pointer`, `cstring`, proc-typed fields. +## +## Strings and seqs allocate on the *consumer thread's GC heap* during +## unmarshal — no thread-local pointer ever crosses a broker boundary, +## which is the §2.6 fix in practice. + +{.push raises: [].} + +import std/[macros, typetraits] + +# Generic recursive primitives. The `pos` parameter is updated in place; +# the bool return is false on overflow / truncation / malformed input. + +proc mtMarshalValue*[T]( + buf: ptr UncheckedArray[byte], cap: int, value: T, pos: var int +): bool {.gcsafe.} + +proc mtUnmarshalValue*[T]( + buf: ptr UncheckedArray[byte], len: int, value: var T, pos: var int +): bool {.gcsafe.} + +# Sequence specialization — separate generic so the element type `U` +# is statically known (we need `newSeq[U]` on the unmarshal side). + +proc mtMarshalSeq*[U]( + buf: ptr UncheckedArray[byte], cap: int, value: openArray[U], pos: var int +): bool {.gcsafe.} = + mixin mtMarshalValue # allow user overloads for element type + if pos + 4 > cap: + return false + let sLen = uint32(value.len) + copyMem(addr buf[pos], unsafeAddr sLen, 4) + pos += 4 + when supportsCopyMem(U): + let totalBytes = int(sLen) * sizeof(U) + if pos + totalBytes > cap: + return false + if sLen > 0'u32: + copyMem(addr buf[pos], unsafeAddr value[0], totalBytes) + pos += totalBytes + return true + else: + for e in value: + if not mtMarshalValue(buf, cap, e, pos): + return false + return true + +proc mtUnmarshalSeq*[U]( + buf: ptr UncheckedArray[byte], len: int, value: var seq[U], pos: var int +): bool {.gcsafe.} = + mixin mtUnmarshalValue # allow user overloads for element type + if pos + 4 > len: + return false + var sLen: uint32 + copyMem(addr sLen, addr buf[pos], 4) + pos += 4 + when supportsCopyMem(U): + let totalBytes = int(sLen) * sizeof(U) + if pos + totalBytes > len: + return false + value = newSeq[U](int(sLen)) + if sLen > 0'u32: + copyMem(addr value[0], addr buf[pos], totalBytes) + pos += totalBytes + return true + else: + value = newSeq[U](int(sLen)) + for i in 0 ..< int(sLen): + if not mtUnmarshalValue(buf, len, value[i], pos): + return false + return true + +proc mtMarshalValue*[T]( + buf: ptr UncheckedArray[byte], cap: int, value: T, pos: var int +): bool {.gcsafe.} = + mixin mtMarshalValue # allow user overloads for field types + when T is ref: + when compiles(value.brokerCtx): + # reduced-A: a BrokerInterface ref is a same-thread routing handle (it + # carries a `brokerCtx`). Create-instance dispatch is same-thread (adapter + # + provider both on the processing thread), so the ref never actually + # travels between threads — we marshal its pointer bytewise purely to + # satisfy the response codec's instantiation. This is NOT general + # cross-thread ref support; arbitrary refs still hard-error below. + if pos + sizeof(pointer) > cap: + return false + copyMem(addr buf[pos], unsafeAddr value, sizeof(pointer)) + pos += sizeof(pointer) + return true + else: + {.error: "mt broker payload field type is unsupported (ref T): " & $T.} + # ptr / pointer / cstring fall through to the `supportsCopyMem` branch + # below and are marshaled bytewise. Caller is responsible for the + # lifetime of what they point to — typically used for shared structures + # like chronos' ThreadSignalPtr. + elif supportsCopyMem(T): + if pos + sizeof(T) > cap: + return false + copyMem(addr buf[pos], unsafeAddr value, sizeof(T)) + pos += sizeof(T) + return true + elif T is string: + let sLen = uint32(value.len) + if pos + 4 + int(sLen) > cap: + return false + copyMem(addr buf[pos], unsafeAddr sLen, 4) + pos += 4 + if sLen > 0'u32: + copyMem(addr buf[pos], unsafeAddr value[0], int(sLen)) + pos += int(sLen) + return true + elif T is seq: + return mtMarshalSeq(buf, cap, value, pos) + elif T is array: + # Non-POD array (e.g. array[N, string]); iterate per element. + for i in 0 ..< value.len: + if not mtMarshalValue(buf, cap, value[i], pos): + return false + return true + elif T is (object or tuple): + for _, fval in fieldPairs(value): + if not mtMarshalValue(buf, cap, fval, pos): + return false + return true + elif T is distinct: + # Unwrap to the underlying base and recurse. POD distincts (e.g. + # `distinct int32`) are caught by the `supportsCopyMem` branch above; + # this branch handles distincts whose base needs structural marshaling + # such as `distinct seq[byte]` or `distinct string`. + var base = distinctBase(value) + return mtMarshalValue(buf, cap, base, pos) + else: + {.error: "mt broker payload field type is unsupported by mtMarshalValue: " & $T.} + +proc mtUnmarshalValue*[T]( + buf: ptr UncheckedArray[byte], len: int, value: var T, pos: var int +): bool {.gcsafe.} = + mixin mtUnmarshalValue # allow user overloads for field types + when T is ref: + when compiles(value.brokerCtx): + # reduced-A: BrokerInterface ref — same-thread routing handle, see the + # marshal counterpart. Reads the pointer bytes back. Bypasses GC refcount + # (the instance stays pinned by its provider closures), valid only because + # the create-instance path is same-thread and transient. + if pos + sizeof(pointer) > len: + return false + copyMem(unsafeAddr value, addr buf[pos], sizeof(pointer)) + pos += sizeof(pointer) + return true + else: + {.error: "mt broker payload field type is unsupported (ref T): " & $T.} + # ptr / pointer / cstring fall through to the `supportsCopyMem` branch + # below and are marshaled bytewise. Caller is responsible for the + # lifetime of what they point to — typically used for shared structures + # like chronos' ThreadSignalPtr. + elif supportsCopyMem(T): + if pos + sizeof(T) > len: + return false + copyMem(unsafeAddr value, addr buf[pos], sizeof(T)) + pos += sizeof(T) + return true + elif T is string: + if pos + 4 > len: + return false + var sLen: uint32 + copyMem(addr sLen, addr buf[pos], 4) + pos += 4 + if pos + int(sLen) > len: + return false + value = newString(int(sLen)) + if sLen > 0'u32: + copyMem(addr value[0], addr buf[pos], int(sLen)) + pos += int(sLen) + return true + elif T is seq: + return mtUnmarshalSeq(buf, len, value, pos) + elif T is array: + for i in 0 ..< value.len: + if not mtUnmarshalValue(buf, len, value[i], pos): + return false + return true + elif T is (object or tuple): + for _, fval in fieldPairs(value): + if not mtUnmarshalValue(buf, len, fval, pos): + return false + return true + elif T is distinct: + type Base = distinctBase(T) + var base: Base + if not mtUnmarshalValue(buf, len, base, pos): + return false + value = T(base) + return true + else: + {.error: "mt broker payload field type is unsupported by mtUnmarshalValue: " & $T.} + +# --------------------------------------------------------------------------- +# Marshal-size companion — pure byte-count walk, no writes. +# +# Mirrors mtMarshalValue exactly so the heap-spill path (flexible-mt-dispatch +# Part 2) can size an exact `allocShared0` buffer in one pass when a payload +# overflows the fixed slab cell. MUST stay structurally in lockstep with +# mtMarshalValue: every branch that advances `pos` there adds the same count +# here. Returns the marshaled byte length. +# --------------------------------------------------------------------------- + +proc mtMarshalSizeValue*[T](value: T): int {.gcsafe.} + +proc mtMarshalSizeSeq*[U](value: openArray[U]): int {.gcsafe.} = + mixin mtMarshalSizeValue + result = 4 # length prefix + when supportsCopyMem(U): + result += value.len * sizeof(U) + else: + for e in value: + result += mtMarshalSizeValue(e) + +proc mtMarshalSizeValue*[T](value: T): int {.gcsafe.} = + mixin mtMarshalSizeValue + when T is ref: + when compiles(value.brokerCtx): + return sizeof(pointer) + else: + {.error: "mt broker payload field type is unsupported (ref T): " & $T.} + elif supportsCopyMem(T): + return sizeof(T) + elif T is string: + return 4 + value.len + elif T is seq: + return mtMarshalSizeSeq(value) + elif T is array: + result = 0 + for i in 0 ..< value.len: + result += mtMarshalSizeValue(value[i]) + elif T is (object or tuple): + result = 0 + for _, fval in fieldPairs(value): + result += mtMarshalSizeValue(fval) + elif T is distinct: + var base = distinctBase(value) + return mtMarshalSizeValue(base) + else: + {. + error: "mt broker payload field type is unsupported by mtMarshalSizeValue: " & $T + .} + +# --------------------------------------------------------------------------- +# Per-type wrapper proc generation (called from broker macros) +# --------------------------------------------------------------------------- + +proc genMtCodecProcs*( + marshalIdent, unmarshalIdent: NimNode, typeIdent: NimNode +): seq[NimNode] = + ## Emits per-type marshal/unmarshal/size wrappers that bottom out to the + ## generic primitives above. Three procs returned: + ## [marshalProc, unmarshalProc, sizeProc]. The size proc is named + ## `Size` and returns the exact marshaled byte length (used by + ## the heap-spill path to size an allocShared0 buffer in one pass). + let bufIdent = ident("buf") + let capIdent = ident("cap") + let lenIdent = ident("len") + let valueIdent = ident("value") + let dstIdent = ident("dst") + let posIdent = ident("pos") + let sizeIdent = ident($marshalIdent & "Size") + + let marshalProc = quote: + proc `marshalIdent`( + `bufIdent`: ptr UncheckedArray[byte], `capIdent`: int, `valueIdent`: `typeIdent` + ): int {.gcsafe, raises: [].} = + var `posIdent` = 0 + if mtMarshalValue(`bufIdent`, `capIdent`, `valueIdent`, `posIdent`): + return `posIdent` + return -1 + + let unmarshalProc = quote: + proc `unmarshalIdent`( + `bufIdent`: ptr UncheckedArray[byte], + `lenIdent`: int, + `dstIdent`: var `typeIdent`, + ): bool {.gcsafe, raises: [].} = + var `posIdent` = 0 + return mtUnmarshalValue(`bufIdent`, `lenIdent`, `dstIdent`, `posIdent`) + + let sizeProc = quote: + proc `sizeIdent`(`valueIdent`: `typeIdent`): int {.gcsafe, raises: [].} = + mtMarshalSizeValue(`valueIdent`) + + @[marshalProc, unmarshalProc, sizeProc] diff --git a/wasm-deps/brokers/brokers/internal/mt_config.nim b/wasm-deps/brokers/brokers/internal/mt_config.nim new file mode 100644 index 000000000..1cf8560f5 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/mt_config.nim @@ -0,0 +1,596 @@ +## Multi-thread broker configuration +## --------------------------------- +## Compile-time config records and macro-argument parsing for the +## multi-thread Event / Request brokers. +## +## The macro entry points accept optional kwargs: +## +## EventBroker(mt, queueDepth = 1024, slabCapacity = 4096): ... +## RequestBroker(mt, responseSlots = 64, maxResponseBytes = 4096): ... +## +## When no kwargs are supplied the existing module-level defaults in +## `mt_event_broker.nim` / `mt_request_broker.nim` are used unchanged. + +{.push raises: [].} +{.push warning[UnreachableCode]: off.} + +import std/[macros, strutils] + +type + MtEvtCfg* = object ## Resolved EventBroker(mt) capacity config. + queueDepth*: int ## ring slots per listener bucket (power-of-2) + slabCapacity*: int ## global slab cell count + maxPayloadBytes*: int ## per-cell payload bytes + maxDynamicPayloadBytes*: int + ## ceiling for an auto-spilled (heap) payload that exceeds the fixed cell. + ## Spill is always-on; this is a dev-chosen sanity cap, default high(uint32) + ## (effectively unbounded). A payload above it is dropped (OOM/DoS backstop). + freeListShards*: int ## sharded free-list partitions + # Provenance — for the compile-time printout. "default" / "kwarg" / + # "preset:" / "auto:". + queueDepthOrigin*: string + slabCapacityOrigin*: string + maxPayloadBytesOrigin*: string + maxDynamicPayloadBytesOrigin*: string + freeListShardsOrigin*: string + + MtReqCfg* = object ## Resolved RequestBroker(mt) capacity config. + queueDepth*: int + slabCapacity*: int + maxPayloadBytes*: int + maxDynamicPayloadBytes*: int + ## ceiling for an auto-spilled request OR response payload. See MtEvtCfg. + responseSlots*: int + maxResponseBytes*: int + freeListShards*: int + queueDepthOrigin*: string + slabCapacityOrigin*: string + maxPayloadBytesOrigin*: string + maxDynamicPayloadBytesOrigin*: string + responseSlotsOrigin*: string + maxResponseBytesOrigin*: string + freeListShardsOrigin*: string + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- + +const + # Default ceiling for heap-spilled payloads. high(uint32) ≈ 4 GiB — also the + # intrinsic cap, since the cell/slot spill-length fields are uint32. Spill is + # always-on; this only bounds how large a single spill may grow. + DefaultMtMaxDynamicPayloadBytes* = int(high(uint32)) + + DefaultMtEvtQueueDepth* = 256 + DefaultMtEvtSlabCapacity* = 1024 + DefaultMtEvtMaxPayloadBytes* = 1024 + DefaultMtEvtFreeListShards* = 4 + + DefaultMtReqQueueDepth* = 256 + DefaultMtReqSlabCapacity* = 64 + DefaultMtReqMaxPayloadBytes* = 1024 + DefaultMtReqResponseSlots* = 256 + DefaultMtReqMaxResponseBytes* = 64 * 1024 + DefaultMtReqFreeListShards* = 2 + +# --------------------------------------------------------------------------- +# Built-in presets +# --------------------------------------------------------------------------- +# +# Named shorthand for capacity profiles. Recognised in the macro as +# `preset = `. +# +# defaultBalanced same as omitting `preset` +# fastBurst bursty emit/request, small payload — wide ring/slab +# largePayload infrequent traffic with big payloads +# tinyFootprint rare traffic, embedded / memory-constrained +# +# Individual kwargs supplied alongside `preset =` override the preset's +# values (so you can pick a profile and tweak one field). + +type BuiltinPreset* = enum + bpDefaultBalanced = "defaultBalanced" + bpFastBurst = "fastBurst" + bpLargePayload = "largePayload" + bpTinyFootprint = "tinyFootprint" + +proc parseBuiltinPreset(name: string, n: NimNode): BuiltinPreset = + case name + of "defaultBalanced": + bpDefaultBalanced + of "fastBurst": + bpFastBurst + of "largePayload": + bpLargePayload + of "tinyFootprint": + bpTinyFootprint + else: + error( + "Unknown preset '" & name & + "'. Built-in presets: defaultBalanced, fastBurst, largePayload, " & + "tinyFootprint. (User-defined presets are not yet supported.)", + n, + ) + bpDefaultBalanced + +proc applyEvtPreset(cfg: var MtEvtCfg, p: BuiltinPreset) = + let tag = "preset:" & $p + case p + of bpDefaultBalanced: + discard # already the default + of bpFastBurst: + cfg.queueDepth = 4096 + cfg.slabCapacity = 8192 + cfg.maxPayloadBytes = 256 + cfg.freeListShards = 8 + of bpLargePayload: + cfg.queueDepth = 64 + cfg.slabCapacity = 128 + cfg.maxPayloadBytes = 64 * 1024 + cfg.freeListShards = 2 + of bpTinyFootprint: + cfg.queueDepth = 32 + cfg.slabCapacity = 32 + cfg.maxPayloadBytes = 256 + cfg.freeListShards = 1 + cfg.queueDepthOrigin = tag + cfg.slabCapacityOrigin = tag + cfg.maxPayloadBytesOrigin = tag + cfg.freeListShardsOrigin = tag + +proc applyReqPreset(cfg: var MtReqCfg, p: BuiltinPreset) = + let tag = "preset:" & $p + case p + of bpDefaultBalanced: + discard + of bpFastBurst: + cfg.queueDepth = 4096 + cfg.slabCapacity = 256 + cfg.maxPayloadBytes = 256 + cfg.responseSlots = 1024 + cfg.maxResponseBytes = 4 * 1024 + cfg.freeListShards = 4 + of bpLargePayload: + cfg.queueDepth = 64 + cfg.slabCapacity = 32 + cfg.maxPayloadBytes = 64 * 1024 + cfg.responseSlots = 64 + cfg.maxResponseBytes = 256 * 1024 + cfg.freeListShards = 2 + of bpTinyFootprint: + cfg.queueDepth = 16 + cfg.slabCapacity = 8 + cfg.maxPayloadBytes = 256 + cfg.responseSlots = 16 + cfg.maxResponseBytes = 1024 + cfg.freeListShards = 1 + cfg.queueDepthOrigin = tag + cfg.slabCapacityOrigin = tag + cfg.maxPayloadBytesOrigin = tag + cfg.responseSlotsOrigin = tag + cfg.maxResponseBytesOrigin = tag + cfg.freeListShardsOrigin = tag + +proc defaultMtEvtCfg*(): MtEvtCfg = + MtEvtCfg( + queueDepth: DefaultMtEvtQueueDepth, + slabCapacity: DefaultMtEvtSlabCapacity, + maxPayloadBytes: DefaultMtEvtMaxPayloadBytes, + maxDynamicPayloadBytes: DefaultMtMaxDynamicPayloadBytes, + freeListShards: DefaultMtEvtFreeListShards, + queueDepthOrigin: "default", + slabCapacityOrigin: "default", + maxPayloadBytesOrigin: "default", + maxDynamicPayloadBytesOrigin: "default", + freeListShardsOrigin: "default", + ) + +proc defaultMtReqCfg*(): MtReqCfg = + MtReqCfg( + queueDepth: DefaultMtReqQueueDepth, + slabCapacity: DefaultMtReqSlabCapacity, + maxPayloadBytes: DefaultMtReqMaxPayloadBytes, + maxDynamicPayloadBytes: DefaultMtMaxDynamicPayloadBytes, + responseSlots: DefaultMtReqResponseSlots, + maxResponseBytes: DefaultMtReqMaxResponseBytes, + freeListShards: DefaultMtReqFreeListShards, + queueDepthOrigin: "default", + slabCapacityOrigin: "default", + maxPayloadBytesOrigin: "default", + maxDynamicPayloadBytesOrigin: "default", + responseSlotsOrigin: "default", + maxResponseBytesOrigin: "default", + freeListShardsOrigin: "default", + ) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +proc isPow2(n: int): bool {.inline.} = + n > 0 and (n and (n - 1)) == 0 + +proc intValOrFail(n: NimNode, kw: string): int = + ## Extract a compile-time int from a kwarg RHS. Errors clearly on + ## non-int input. + case n.kind + of nnkIntLit, nnkInt8Lit, nnkInt16Lit, nnkInt32Lit, nnkInt64Lit, nnkUIntLit, + nnkUInt8Lit, nnkUInt16Lit, nnkUInt32Lit, nnkUInt64Lit: + int(n.intVal) + else: + error("broker kwarg '" & kw & "' expects an integer literal, got " & $n.kind, n) + 0 + +# --------------------------------------------------------------------------- +# Kwarg parsing — EventBroker(mt) +# --------------------------------------------------------------------------- + +const ValidEvtKwargs = [ + "queueDepth", "slabCapacity", "maxPayloadBytes", "maxDynamicPayloadBytes", + "freeListShards", +] + +proc applyEvtKwarg(cfg: var MtEvtCfg, kw: string, n: NimNode) = + case kw + of "queueDepth": + let v = intValOrFail(n, kw) + if not isPow2(v): + error("EventBroker kwarg 'queueDepth' must be power-of-2, got " & $v, n) + cfg.queueDepth = v + cfg.queueDepthOrigin = "kwarg" + of "slabCapacity": + let v = intValOrFail(n, kw) + if v <= 0: + error("EventBroker kwarg 'slabCapacity' must be > 0, got " & $v, n) + cfg.slabCapacity = v + cfg.slabCapacityOrigin = "kwarg" + of "maxPayloadBytes": + let v = intValOrFail(n, kw) + if v <= 0: + error("EventBroker kwarg 'maxPayloadBytes' must be > 0, got " & $v, n) + cfg.maxPayloadBytes = v + cfg.maxPayloadBytesOrigin = "kwarg" + of "maxDynamicPayloadBytes": + let v = intValOrFail(n, kw) + if v <= 0 or v > int(high(uint32)): + error( + "EventBroker kwarg 'maxDynamicPayloadBytes' must be in 1..high(uint32), got " & + $v, + n, + ) + cfg.maxDynamicPayloadBytes = v + cfg.maxDynamicPayloadBytesOrigin = "kwarg" + of "freeListShards": + let v = intValOrFail(n, kw) + if v <= 0 or v > 64: + error("EventBroker kwarg 'freeListShards' must be in 1..64, got " & $v, n) + cfg.freeListShards = v + cfg.freeListShardsOrigin = "kwarg" + else: + error( + "Unknown EventBroker(mt) kwarg '" & kw & "'. Valid: " & ValidEvtKwargs.join(", "), + n, + ) + +proc presetFromKwargRhs(rhs: NimNode): BuiltinPreset = + ## Extracts a built-in preset name from a kwarg RHS. Accepts identifier + ## form (`preset = fastBurst`). + if rhs.kind != nnkIdent: + error( + "preset value must be one of the built-in preset names " & + "(defaultBalanced, fastBurst, largePayload, tinyFootprint), got " & $rhs.kind & + " — " & rhs.repr, + rhs, + ) + parseBuiltinPreset($rhs, rhs) + +proc parseMtEvtKwargs*(kwargs: openArray[NimNode]): MtEvtCfg = + ## Parses kwarg nodes (everything between `mt` and the trailing body). + ## Each node must be of shape `nnkExprEqExpr` (`name = value`). + ## + ## Order of application: + ## 1. defaultMtEvtCfg() + ## 2. `preset = ` if present (overrides defaults) + ## 3. individual kwargs (override the preset) + result = defaultMtEvtCfg() + for n in kwargs: + if n.kind != nnkExprEqExpr: + error( + "EventBroker(mt) expects kwargs of the form 'name = value', got " & $n.kind & + " — " & n.repr, + n, + ) + let nameNode = n[0] + if nameNode.kind != nnkIdent: + error("EventBroker(mt) kwarg name must be an identifier", nameNode) + if $nameNode == "preset": + applyEvtPreset(result, presetFromKwargRhs(n[1])) + for n in kwargs: + let name = $n[0] + if name == "preset": + continue + applyEvtKwarg(result, name, n[1]) + +# --------------------------------------------------------------------------- +# Kwarg parsing — RequestBroker(mt) +# --------------------------------------------------------------------------- + +const ValidReqKwargs = [ + "queueDepth", "slabCapacity", "maxPayloadBytes", "maxDynamicPayloadBytes", + "responseSlots", "maxResponseBytes", "freeListShards", +] + +proc applyReqKwarg(cfg: var MtReqCfg, kw: string, n: NimNode) = + case kw + of "queueDepth": + let v = intValOrFail(n, kw) + if not isPow2(v): + error("RequestBroker kwarg 'queueDepth' must be power-of-2, got " & $v, n) + cfg.queueDepth = v + cfg.queueDepthOrigin = "kwarg" + of "slabCapacity": + let v = intValOrFail(n, kw) + if v <= 0: + error("RequestBroker kwarg 'slabCapacity' must be > 0, got " & $v, n) + cfg.slabCapacity = v + cfg.slabCapacityOrigin = "kwarg" + of "maxPayloadBytes": + let v = intValOrFail(n, kw) + if v <= 0: + error("RequestBroker kwarg 'maxPayloadBytes' must be > 0, got " & $v, n) + cfg.maxPayloadBytes = v + cfg.maxPayloadBytesOrigin = "kwarg" + of "maxDynamicPayloadBytes": + let v = intValOrFail(n, kw) + if v <= 0 or v > int(high(uint32)): + error( + "RequestBroker kwarg 'maxDynamicPayloadBytes' must be in 1..high(uint32), got " & + $v, + n, + ) + cfg.maxDynamicPayloadBytes = v + cfg.maxDynamicPayloadBytesOrigin = "kwarg" + of "responseSlots": + let v = intValOrFail(n, kw) + if v <= 0: + error("RequestBroker kwarg 'responseSlots' must be > 0, got " & $v, n) + cfg.responseSlots = v + cfg.responseSlotsOrigin = "kwarg" + of "maxResponseBytes": + let v = intValOrFail(n, kw) + if v <= 0: + error("RequestBroker kwarg 'maxResponseBytes' must be > 0, got " & $v, n) + cfg.maxResponseBytes = v + cfg.maxResponseBytesOrigin = "kwarg" + of "freeListShards": + let v = intValOrFail(n, kw) + if v <= 0 or v > 64: + error("RequestBroker kwarg 'freeListShards' must be in 1..64, got " & $v, n) + cfg.freeListShards = v + cfg.freeListShardsOrigin = "kwarg" + else: + error( + "Unknown RequestBroker(mt) kwarg '" & kw & "'. Valid: " & ValidReqKwargs.join( + ", " + ), + n, + ) + +# --------------------------------------------------------------------------- +# Type-driven default sizing +# --------------------------------------------------------------------------- +# +# Walks a Nim type AST at macro time and recommends a cell payload size. +# Triggered when the user did NOT provide an explicit `maxPayloadBytes` +# / `maxResponseBytes` kwarg, AND a preset did not set those fields. +# +# Sizing table (matches doc/MT_BROKER_REFACTOR_RETROSPECTIVE.md §8): +# +# scalar (bool/intN/uintN/floatN/byte/char/enum/distinct of scalar) 64 B +# string (or object whose largest field is string) 4 KB +# seq[string] / object containing seq[string] 16 KB +# seq[byte] / object containing seq[byte] 64 KB +# anything else (alias / external type / unknown ident) 8 KB + warning + +const + ScalarBytes* = 64 + StringBytes* = 4 * 1024 + SeqStringBytes* = 16 * 1024 + SeqByteBytes* = 64 * 1024 + UnclassifiableBytes* = 8 * 1024 + +proc classifyTypeSize*(t: NimNode): tuple[bytes: int, reason: string] = + ## Classifies a type AST into a recommended payload-cell size. + ## Caller decides what to do with "unclassifiable" (typically: use + ## the value + emit a warning so the user knows to override). + if t.kind == nnkIdent: + let name = $t + case name + of "bool", "char", "byte", "uint", "int", "uint8", "int8", "uint16", "int16", + "uint32", "int32", "uint64", "int64", "float", "float32", "float64": + (ScalarBytes, "scalar:" & name) + of "string": + (StringBytes, "string") + else: + # enum / distinct / alias / external object — can't tell at macro + # time without resolving the symbol. Fall back to safe size. + (UnclassifiableBytes, "unclassifiable:" & name) + elif t.kind == nnkBracketExpr and t.len >= 2 and + (t[0].kind == nnkIdent or t[0].kind == nnkDotExpr): + # Accept both bare (`Option[T]`) and qualified + # (`options.Option[T]`) outer names. For the dotted form we treat + # the rightmost ident as the bracket name, while also retaining + # the fully qualified form so the existing `options.Option` arm + # below still matches when the user writes the full path. + let outer = + if t[0].kind == nnkIdent: + $t[0] + else: + # nnkDotExpr: lhs.rhs — use rhs as the primary name. + if t[0].len >= 2 and t[0][1].kind == nnkIdent: + $t[0][1] + else: + t[0].repr + if outer == "seq": + let inner = t[1] + if inner.kind == nnkIdent: + let n = $inner + if n == "byte" or n == "uint8": + (SeqByteBytes, "seq[byte]") + elif n == "string": + (SeqStringBytes, "seq[string]") + else: + # seq[] — assume short list of small items. + (StringBytes, "seq[" & n & "]") + else: + (UnclassifiableBytes, "unclassifiable:" & t.repr) + elif outer == "array": + # array[N, T] — bounded; treat as the underlying T classification. + if t.len >= 3: + classifyTypeSize(t[2]) + else: + (UnclassifiableBytes, "unclassifiable:" & t.repr) + elif outer == "Option": + # Option[T] — wire size is bounded by T plus a one-byte CBOR + # tag (null marker vs concrete value). Recurse into the inner + # type and reuse its classification verbatim; the +1 byte sits + # comfortably inside whatever bucket T lands in. Without this + # special case Option[seq[byte]] would silently under-allocate + # (8 KB fallback < 64 KB seq[byte]), while Option[int64] would + # noisily over-allocate at 8 KB. + let inner = classifyTypeSize(t[1]) + (inner.bytes, "Option[" & inner.reason & "]") + else: + (UnclassifiableBytes, "unclassifiable:" & outer) + else: + (UnclassifiableBytes, "unclassifiable:" & $t.kind) + +proc classifyFieldsMax*( + fieldTypes: openArray[NimNode] +): tuple[bytes: int, reason: string] = + ## Returns the maximum-size classification across a collection of + ## field-type ASTs. Used to size the cell for an inline object. + var bestBytes = ScalarBytes + var bestReason = "scalar" + for ft in fieldTypes: + let c = classifyTypeSize(ft) + if c.bytes > bestBytes: + bestBytes = c.bytes + bestReason = c.reason + (bestBytes, bestReason) + +proc peelFutureResult*(t: NimNode): NimNode = + ## Walks `Future[Result[T, E]]` and returns T. Returns nil if the + ## shape doesn't match. + var cur = t + if cur.kind == nnkBracketExpr and cur.len >= 2 and cur[0].kind == nnkIdent and + $cur[0] == "Future": + cur = cur[1] + if cur.kind == nnkBracketExpr and cur.len >= 2 and cur[0].kind == nnkIdent and + ($cur[0] == "Result" or $cur[0] == "results.Result"): + return cur[1] + nil + + # --------------------------------------------------------------------------- + # Compile-time summary formatting + # --------------------------------------------------------------------------- + +proc fmtBytes(n: int): string = + if n >= 1024 * 1024: + $(n div (1024 * 1024)) & "." & + align($((n mod (1024 * 1024)) div (102 * 1024)), 1, '0') & " MB" + elif n >= 1024: + $(n div 1024) & "." & align($((n mod 1024) div 102), 1, '0') & " KB" + else: + $n & " B" + +# Approximate per-element bytes — match the runtime layouts in mt_queue.nim. +# Exact figures don't matter; this is a sizing-guidance number for the user. +const + RingSlotBytes = 24 # Slot[uint32] = idx u32 + seq u64 + pad + CellHeaderBytes = 32 # CellHeader (refcount, length, prev/next idx) + RespSlotHeaderBytes = 48 # ResponseSlot header + +proc alignUp8(n: int): int {.inline.} = + (n + 7) and (not 7) + +proc estEvtIdleBytes(cfg: MtEvtCfg): tuple[ring, slab, total: int] = + let ring = cfg.queueDepth * RingSlotBytes + let cellStride = alignUp8(CellHeaderBytes + cfg.maxPayloadBytes) + let slab = cfg.slabCapacity * cellStride + (ring, slab, ring + slab) + +proc estReqIdleBytes(cfg: MtReqCfg): tuple[ring, slab, respPool, total: int] = + let ring = cfg.queueDepth * RingSlotBytes + let cellStride = alignUp8(CellHeaderBytes + cfg.maxPayloadBytes) + let slab = cfg.slabCapacity * cellStride + let slotStride = alignUp8(RespSlotHeaderBytes + cfg.maxResponseBytes) + let respPool = cfg.responseSlots * slotStride + (ring, slab, respPool, ring + slab + respPool) + +proc fmtEvtCfgSummary*(typeName: string, cfg: MtEvtCfg): string = + let est = estEvtIdleBytes(cfg) + "[brokers] EventBroker(" & typeName & "): " & "queueDepth=" & $cfg.queueDepth & " [" & + cfg.queueDepthOrigin & "], " & "slabCapacity=" & $cfg.slabCapacity & " [" & + cfg.slabCapacityOrigin & "], " & "maxPayloadBytes=" & $cfg.maxPayloadBytes & " [" & + cfg.maxPayloadBytesOrigin & "], freeListShards=" & $cfg.freeListShards & " [" & + cfg.freeListShardsOrigin & "] — idle RAM: ring≈" & fmtBytes(est.ring) & + ", slab≈" & fmtBytes(est.slab) & ", total≈" & fmtBytes(est.total) + +proc fmtReqCfgSummary*(typeName: string, cfg: MtReqCfg): string = + let est = estReqIdleBytes(cfg) + "[brokers] RequestBroker(" & typeName & "): " & "queueDepth=" & $cfg.queueDepth & " [" & + cfg.queueDepthOrigin & "], " & "slabCapacity=" & $cfg.slabCapacity & " [" & + cfg.slabCapacityOrigin & "], " & "maxPayloadBytes=" & $cfg.maxPayloadBytes & " [" & + cfg.maxPayloadBytesOrigin & "], responseSlots=" & $cfg.responseSlots & " [" & + cfg.responseSlotsOrigin & "], maxResponseBytes=" & $cfg.maxResponseBytes & " [" & + cfg.maxResponseBytesOrigin & "], freeListShards=" & $cfg.freeListShards & " [" & + cfg.freeListShardsOrigin & "] — idle RAM: ring≈" & fmtBytes(est.ring) & + ", slab≈" & fmtBytes(est.slab) & ", respPool≈" & fmtBytes(est.respPool) & + ", total≈" & fmtBytes(est.total) + +proc parseMtReqKwargs*(kwargs: openArray[NimNode]): MtReqCfg = + ## See `parseMtEvtKwargs` for order-of-application rules. + result = defaultMtReqCfg() + for n in kwargs: + if n.kind != nnkExprEqExpr: + error( + "RequestBroker(mt) expects kwargs of the form 'name = value', got " & $n.kind & + " — " & n.repr, + n, + ) + let nameNode = n[0] + if nameNode.kind != nnkIdent: + error("RequestBroker(mt) kwarg name must be an identifier", nameNode) + if $nameNode == "preset": + applyReqPreset(result, presetFromKwargRhs(n[1])) + for n in kwargs: + let name = $n[0] + if name == "preset": + continue + applyReqKwarg(result, name, n[1]) + +# --------------------------------------------------------------------------- +# Splitting varargs into kwargs + body +# --------------------------------------------------------------------------- + +proc splitMtArgs*( + args: NimNode, what: string +): tuple[kwargs: seq[NimNode], body: NimNode] = + ## Splits a `varargs[untyped]` macro arg list into (kwarg nodes, body + ## stmt-list). The body is always the last element. Errors if no body. + if args.len == 0: + error(what & " requires a body block", args) + let bodyNode = args[args.len - 1] + if bodyNode.kind notin {nnkStmtList, nnkTypeDef, nnkTypeSection}: + error( + what & " body must be a `:` block of type definitions (got " & $bodyNode.kind & ")", + bodyNode, + ) + var kw = newSeqOfCap[NimNode](args.len - 1) + for i in 0 ..< args.len - 1: + kw.add(args[i]) + (kw, bodyNode) + +{.pop.} # warning[UnreachableCode] +{.pop.} # raises: [] diff --git a/wasm-deps/brokers/brokers/internal/mt_event_broker.nim b/wasm-deps/brokers/brokers/internal/mt_event_broker.nim new file mode 100644 index 000000000..5b3ff3779 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/mt_event_broker.nim @@ -0,0 +1,935 @@ +## Multi-Thread EventBroker +## ------------------------ +## Generates a multi-thread capable EventBroker where listeners can be +## registered on any thread and events can be emitted from any thread. +## Events are delivered to all registered listeners across all threads +## (broadcast fan-out). +## +## Same-thread emit→listener dispatch bypasses the ring and is delivered +## directly via `asyncSpawn`. Cross-thread delivery uses a lock-free +## Vyukov MPSC ring + a global per-broker-type slab with refcounted +## payload cells (so one emit shares one cell across N listener threads +## via atomic refcount, rather than N deep-copies). +## +## See `doc/REFACTOR_MT_QUEUE.md` for the full design; this file is the +## EventBroker integration of Phase 2+3 of that plan. +## +## §2.6 safety contract honored by construction (Invariant I0): +## - The bucket-owning thread (the listener thread) allocates its ring +## via `createShared` and frees it via `shutdown(ctx)` on the same +## thread. +## - The global event slab is allocated lazily, by whichever thread +## first calls `listen()` or `emit()`. That thread MUST outlive the +## slab. +## - Sender threads only ever touch atomics + memcpy + signal-fire — +## never the Nim allocator on the hot path. + +{.push raises: [].} + +import std/[macros, strutils, locks, tables, atomics] +import chronos, chronicles +import results +import + ./helper/broker_utils, + ../broker_context, + ./mt_broker_common, + ./mt_queue, + ./mt_codec, + ./mt_config, + ./broker_debug + +export results, chronos, broker_context, chronicles, mt_broker_common, mt_config + +# Ring-slot sentinel: a slot's payload `uint32` is normally a slab cell +# index, but this reserved value carries a "clear local tvHandlers" +# control signal instead. Shutdown is communicated via the ring's +# `closed` flag, not a sentinel, because `tryEnqueue` rejects when +# closed and we don't want shutdown to compete with that. +# +# The sentinel lives in the same namespace as cell indices and MUST be +# larger than any legal slab capacity (bounded by uint32 in practice). +const CtrlClearListeners*: uint32 = high(uint32) - 1 + +# Capacity defaults moved to `mt_config.nim`; they remain re-exported via +# the `mt_config` module so external code referencing +# `DefaultMtEvtQueueDepth` etc. still resolves. + +# --------------------------------------------------------------------------- +# Macro code generator +# --------------------------------------------------------------------------- + +proc generateMtEventBroker*( + body: NimNode, cfgIn: MtEvtCfg = defaultMtEvtCfg() +): NimNode = + when defined(brokerDebug): + echo body.treeRepr + echo "EventBroker mode: mt" + + let parsed = parseSingleTypeDef(body, "EventBroker", collectFieldInfo = true) + let typeIdent = parsed.typeIdent + let objectDef = parsed.objectDef + let fieldNames = parsed.fieldNames + let fieldTypes = parsed.fieldTypes + let hasInlineFields = parsed.hasInlineFields + + let exportedTypeIdent = postfix(copyNimTree(typeIdent), "*") + let typeDisplayName = sanitizeIdentName(typeIdent) + let typeNameLit = newLit(typeDisplayName) + + # Apply type-driven default for maxPayloadBytes when neither a kwarg + # nor a preset set it. Warn if the type is unclassifiable so the user + # knows to provide an explicit override. Void / zero-field bodies + # collapse to the scalar bucket: a payload-less notification only + # ships the CBOR envelope (a handful of bytes), and the conservative + # 1 KB default would otherwise pin a full megabyte slab per event + # type for no reason. + var cfg = cfgIn + if cfg.maxPayloadBytesOrigin == "default": + if fieldTypes.len > 0: + let cls = classifyFieldsMax(fieldTypes) + cfg.maxPayloadBytes = cls.bytes + cfg.maxPayloadBytesOrigin = "auto:" & cls.reason + if cls.reason.startsWith("unclassifiable"): + warning( + "[brokers] EventBroker(" & typeDisplayName & ") could not auto-size payload (" & + cls.reason & "); falling back to " & $cls.bytes & + " B. Override with `maxPayloadBytes = N`." + ) + else: + cfg.maxPayloadBytes = ScalarBytes + cfg.maxPayloadBytesOrigin = "auto:void" + + when not defined(brokerConfigSilent): + hint(fmtEvtCfgSummary(typeDisplayName, cfg)) + + # ── Identifier setup ────────────────────────────────────────────────── + let handlerProcIdent = ident(typeDisplayName & "ListenerProc") + let listenerHandleIdent = ident(typeDisplayName & "Listener") + let exportedHandlerProcIdent = postfix(copyNimTree(handlerProcIdent), "*") + let exportedListenerHandleIdent = postfix(copyNimTree(listenerHandleIdent), "*") + + let bucketName = ident(typeDisplayName & "MtEventBucket") + + let globalBucketsIdent = ident("g" & typeDisplayName & "MtBuckets") + let globalBucketCountIdent = ident("g" & typeDisplayName & "MtBucketCount") + let globalBucketCapIdent = ident("g" & typeDisplayName & "MtBucketCap") + let globalLockIdent = ident("g" & typeDisplayName & "MtLock") + let globalInitIdent = ident("g" & typeDisplayName & "MtInit") + + let globalSlabIdent = ident("g" & typeDisplayName & "MtSlab") + let globalSlabInitIdent = ident("g" & typeDisplayName & "MtSlabInit") + + let initProcIdent = ident("ensureInit" & typeDisplayName & "MtBroker") + let initSlabProcIdent = ident("ensureSlab" & typeDisplayName & "MtBroker") + let growProcIdent = ident("grow" & typeDisplayName & "MtBuckets") + let listenerTaskIdent = ident("notify" & typeDisplayName & "Listener") + let pollFnMakerIdent = ident("makePollFn" & typeDisplayName) + let clearListenersIdent = ident("clearListeners" & typeDisplayName) + let releaseCellIdent = ident("releaseCell" & typeDisplayName) + let shardHintIdent = ident("shardHint" & typeDisplayName) + + let marshalIdent = ident(typeDisplayName & "MtMarshal") + let unmarshalIdent = ident(typeDisplayName & "MtUnmarshal") + let marshalSizeIdent = ident(typeDisplayName & "MtMarshalSize") + + let tvListenerCtxIdent = ident("g" & typeDisplayName & "TvListenerCtxs") + let tvListenerHandlersIdent = ident("g" & typeDisplayName & "TvListenerHandlers") + let tvNextIdsIdent = ident("g" & typeDisplayName & "TvNextIds") + let tvListenerFutsIdent = ident("g" & typeDisplayName & "TvListenerFuts") + let tvShutdownFutsIdent = ident("g" & typeDisplayName & "TvShutdownFuts") + + let listenImplIdent = ident("listen" & typeDisplayName & "MtImpl") + let emitImplIdent = ident("emit" & typeDisplayName & "MtImpl") + let dropListenerImplIdent = ident("drop" & typeDisplayName & "MtListenerImpl") + let dropAllListenersImplIdent = ident("dropAll" & typeDisplayName & "MtListenersImpl") + # Part D-3: optional companion hook fired by `dropAllListenersImpl` + # after listener clearing completes. Used by the CBOR FFI library + # (`api_library.nim`) to clear the foreign-subscriber registry + + # reset the per-event atomic counter in lock-step with Nim-side + # listener drops. Single slot per type — the only intended user is + # the per-event installer registered at `_createContext` time. + let dropAllHookProcTypeIdent = ident(typeDisplayName & "MtDropAllHook") + let dropAllHookIdent = ident("g" & typeDisplayName & "MtDropAllHook") + let dropAllHookLockIdent = ident("g" & typeDisplayName & "MtDropAllHookLock") + let dropAllHookInitIdent = ident("g" & typeDisplayName & "MtDropAllHookInit") + let setDropAllHookIdent = ident("setDropAll" & typeDisplayName & "Hook") + let shutdownProcessLoopsForCtxIdent = + ident("shutdownProcessLoopsForCtx" & typeDisplayName) + + let queueDepthLit = newLit(cfg.queueDepth) + let slabCapacityLit = newLit(cfg.slabCapacity) + let payloadBytesLit = newLit(cfg.maxPayloadBytes) + let maxDynPayloadLit = newLit(cfg.maxDynamicPayloadBytes) + let freeListShardsLit = newLit(uint32(cfg.freeListShards)) + + result = newStmtList() + + # ── Type section ────────────────────────────────────────────────────── + result.add( + quote do: + type + `exportedTypeIdent` = `objectDef` + `exportedListenerHandleIdent` = object + id*: uint64 + threadId*: pointer ## Thread that registered this listener. + + `exportedHandlerProcIdent` = + proc(event: `typeIdent`): Future[void] {.async: (raises: []), gcsafe.} + + `dropAllHookProcTypeIdent` = + proc(brokerCtx: BrokerContext) {.gcsafe, raises: [].} + + `bucketName` = object + brokerCtx: BrokerContext + ring: ptr VyukovMpscRing[uint32] + listenerSignal: ThreadSignalPtr + threadId: pointer + threadGen: uint64 ## disambiguates reused threadvar addresses + active: bool + hasListeners: bool + + ) + + # ── Codec procs (marshal / unmarshal) ───────────────────────────────── + for procNode in genMtCodecProcs(marshalIdent, unmarshalIdent, typeIdent): + result.add(procNode) + + # ── Global shared state ─────────────────────────────────────────────── + result.add( + quote do: + var `globalBucketsIdent`: ptr UncheckedArray[`bucketName`] + var `globalBucketCountIdent`: int + var `globalBucketCapIdent`: int + var `globalLockIdent`: Lock + var `globalInitIdent`: Atomic[int] + ## 0 = uninitialised, 1 = initialising, 2 = ready. CAS(0→1) wins; + ## losers spin until 2. + var `globalSlabIdent`: PayloadSlab + var `globalSlabInitIdent`: Atomic[int] + # Part D-3 dropAllListeners hook. Single slot per event type; + # `dropAllHookIdent` is `nil` when no hook is registered. + var `dropAllHookIdent`: `dropAllHookProcTypeIdent` + var `dropAllHookLockIdent`: Lock + var `dropAllHookInitIdent`: Atomic[int] + ## same protocol as `globalInitIdent`, gating the global slab. + ) + + # ── Init helpers ────────────────────────────────────────────────────── + result.add( + quote do: + proc `initSlabProcIdent`() = + ## Lazy-init the global event slab on first listen() or emit(). + ## The caller's thread becomes the slab's owner (must outlive it). + if `globalSlabInitIdent`.load(moRelaxed) == 2: + return + var expected = 0 + if `globalSlabInitIdent`.compareExchange(expected, 1, moAcquire, moRelaxed): + initPayloadSlab( + `globalSlabIdent`, + capacity = uint32(`slabCapacityLit`), + payloadBytes = uint32(`payloadBytesLit`), + nShards = `freeListShardsLit`, + ) + `globalSlabInitIdent`.store(2, moRelease) + else: + while `globalSlabInitIdent`.load(moAcquire) != 2: + discard + + proc `initProcIdent`() = + if `globalInitIdent`.load(moRelaxed) == 2: + `initSlabProcIdent`() + return + var expected = 0 + if `globalInitIdent`.compareExchange(expected, 1, moAcquire, moRelaxed): + initLock(`globalLockIdent`) + `globalBucketCapIdent` = 4 + `globalBucketsIdent` = cast[ptr UncheckedArray[`bucketName`]](createShared( + `bucketName`, `globalBucketCapIdent` + )) + `globalBucketCountIdent` = 0 + # Part D-3 dropAllListeners hook storage init. Same one-shot + # CAS-init protocol as the main globals so concurrent callers + # see an initialised lock before any reader/writer touches it. + var hookExpected = 0 + if `dropAllHookInitIdent`.compareExchange( + hookExpected, 1, moAcquire, moRelaxed + ): + initLock(`dropAllHookLockIdent`) + `dropAllHookInitIdent`.store(2, moRelease) + else: + while `dropAllHookInitIdent`.load(moAcquire) != 2: + discard + `globalInitIdent`.store(2, moRelease) + else: + while `globalInitIdent`.load(moAcquire) != 2: + discard + `initSlabProcIdent`() + + ) + + # ── Grow helper ─────────────────────────────────────────────────────── + result.add( + quote do: + proc `growProcIdent`() = + ## Must be called under lock. + let newCap = `globalBucketCapIdent` * 2 + let newBuf = + cast[ptr UncheckedArray[`bucketName`]](createShared(`bucketName`, newCap)) + for i in 0 ..< `globalBucketCountIdent`: + newBuf[i] = `globalBucketsIdent`[i] + # Intentional leak of the old buffer: see mt_request_broker.nim. + `globalBucketsIdent` = newBuf + `globalBucketCapIdent` = newCap + + ) + + # ── Threadvar listener storage ──────────────────────────────────────── + result.add( + quote do: + var `tvListenerCtxIdent` {.threadvar.}: seq[BrokerContext] + var `tvListenerHandlersIdent` {.threadvar.}: + seq[Table[uint64, `handlerProcIdent`]] + var `tvNextIdsIdent` {.threadvar.}: seq[uint64] + var `tvListenerFutsIdent` {.threadvar.}: seq[(BrokerContext, Future[void])] + var `tvShutdownFutsIdent` {.threadvar.}: seq[(BrokerContext, Future[void])] + ) + + # ── Listener task ───────────────────────────────────────────────────── + result.add( + quote do: + proc `listenerTaskIdent`( + callback: `handlerProcIdent`, event: `typeIdent` + ): Future[void] {.async: (raises: []).} = + if callback.isNil(): + return + try: + await callback(event) + except CatchableError: + error "Failed to execute event listener", + eventType = `typeNameLit`, error = getCurrentExceptionMsg() + + ) + + # ── Local helpers used by both same-thread emit and cross-thread poll + result.add( + quote do: + proc `shardHintIdent`(): uint32 {.inline.} = + ## Hash of the calling thread's TLS marker → free-list shard. + cast[uint32](cast[uint](currentMtThreadId()) shr 4) + + proc `clearListenersIdent`(loopCtx: BrokerContext) {.gcsafe, raises: [].} = + {.cast(gcsafe).}: + for i in 0 ..< `tvListenerCtxIdent`.len: + if `tvListenerCtxIdent`[i] == loopCtx: + `tvListenerHandlersIdent`[i].clear() + `tvListenerCtxIdent`.del(i) + `tvListenerHandlersIdent`.del(i) + `tvNextIdsIdent`.del(i) + break + + proc `releaseCellIdent`(cellIdx: uint32) {.inline, gcsafe.} = + if `globalSlabIdent`.decRefAndCheck(cellIdx): + `globalSlabIdent`.release(cellIdx, `shardHintIdent`()) + + ) + + # ── Poll fn maker ───────────────────────────────────────────────────── + result.add( + quote do: + proc `pollFnMakerIdent`( + ring: ptr VyukovMpscRing[uint32], + loopCtx: BrokerContext, + shutdownFut: Future[void], + ): ThreadDispatchPollFn = + let capturedRing = ring + let capturedCtx = loopCtx + let capturedShutdownFut = shutdownFut + return proc(): int {.gcsafe, raises: [].} = + {.cast(gcsafe).}: + var cellIdx: uint32 + if not capturedRing.tryDequeue(cellIdx): + # Empty. If the ring has been closed by shutdown, this is + # the definitive "drained" point (no more producers can + # enqueue past `closed=true`). Complete the shutdown + # future and self-unregister. + if capturedRing.isClosed(): + if not capturedShutdownFut.finished: + capturedShutdownFut.complete() + return 2 + return 0 + case cellIdx + of CtrlClearListeners: + `clearListenersIdent`(capturedCtx) + return 1 + else: + # Normal cell: decode, dispatch, decRef. dataPtr/dataLen resolve + # the heap-spill buffer when the payload spilled, else the inline + # cell region. + var ev: `typeIdent` + let payloadPtr = `globalSlabIdent`.dataPtr(cellIdx) + let payloadLen = `globalSlabIdent`.dataLen(cellIdx) + let ok = + try: + `unmarshalIdent`(payloadPtr, payloadLen, ev) + except Exception: + false + if ok: + var idx = -1 + for i in 0 ..< `tvListenerCtxIdent`.len: + if `tvListenerCtxIdent`[i] == capturedCtx: + idx = i + break + if idx >= 0: + var callbacks: seq[`handlerProcIdent`] = @[] + for cb in `tvListenerHandlersIdent`[idx].values: + callbacks.add(cb) + for cb in callbacks: + let fut: Future[void] = `listenerTaskIdent`(cb, ev) + `tvListenerFutsIdent`.add((capturedCtx, fut)) + asyncSpawn fut + else: + error "Failed to unmarshal event payload", eventType = `typeNameLit` + `releaseCellIdent`(cellIdx) + return 1 + + ) + + # ── listen impl ────────────────────────────────────────────────────── + result.add( + quote do: + proc `listenImplIdent`( + brokerCtx: BrokerContext, handler: `handlerProcIdent` + ): Result[`listenerHandleIdent`, string] = + if handler.isNil(): + return err("Must provide a non-nil event handler") + `initProcIdent`() + + var tvIdx = -1 + for i in 0 ..< `tvListenerCtxIdent`.len: + if `tvListenerCtxIdent`[i] == brokerCtx: + tvIdx = i + break + if tvIdx < 0: + `tvListenerCtxIdent`.add(brokerCtx) + `tvListenerHandlersIdent`.add(initTable[uint64, `handlerProcIdent`]()) + `tvNextIdsIdent`.add(1'u64) + tvIdx = `tvListenerCtxIdent`.len - 1 + + if `tvNextIdsIdent`[tvIdx] == high(uint64): + return err("Cannot add more listeners: ID space exhausted") + let newId = `tvNextIdsIdent`[tvIdx] + `tvNextIdsIdent`[tvIdx] += 1 + `tvListenerHandlersIdent`[tvIdx][newId] = handler + + # Ensure a bucket + ring exists for (brokerCtx, this thread). + let myThreadId = currentMtThreadId() + let myThreadGen = currentMtThreadGen() + var bucketExists = false + var spawnRing: ptr VyukovMpscRing[uint32] + withLock(`globalLockIdent`): + for i in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[i].brokerCtx == brokerCtx and + `globalBucketsIdent`[i].threadId == myThreadId and + `globalBucketsIdent`[i].threadGen == myThreadGen: + `globalBucketsIdent`[i].hasListeners = true + `globalBucketsIdent`[i].active = true + bucketExists = true + break + if not bucketExists: + if `globalBucketCountIdent` >= `globalBucketCapIdent`: + `growProcIdent`() + let ring = newVyukovMpscRing[uint32](`queueDepthLit`) + let listenerSig = getOrInitBrokerSignal() + let idx = `globalBucketCountIdent` + `globalBucketsIdent`[idx] = `bucketName`( + brokerCtx: brokerCtx, + ring: ring, + listenerSignal: listenerSig, + threadId: myThreadId, + threadGen: myThreadGen, + active: true, + hasListeners: true, + ) + `globalBucketCountIdent` += 1 + spawnRing = ring + + if not bucketExists and not spawnRing.isNil: + let shutdownFut = + newFuture[void]("eventBroker." & `typeNameLit` & ".shutdown") + `tvShutdownFutsIdent`.add((brokerCtx, shutdownFut)) + registerBrokerPoller(`pollFnMakerIdent`(spawnRing, brokerCtx, shutdownFut)) + ensureBrokerDispatchStarted() + + return ok(`listenerHandleIdent`(id: newId, threadId: myThreadId)) + + ) + + # ── Public listen ───────────────────────────────────────────────────── + result.add( + quote do: + proc listen*( + _: typedesc[`typeIdent`], handler: `handlerProcIdent` + ): Result[`listenerHandleIdent`, string] = + return `listenImplIdent`(DefaultBrokerContext, handler) + + proc listen*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + handler: `handlerProcIdent`, + ): Result[`listenerHandleIdent`, string] = + return `listenImplIdent`(brokerCtx, handler) + + ) + + # ── emit impl ───────────────────────────────────────────────────────── + result.add( + quote do: + proc `emitImplIdent`( + brokerCtx: BrokerContext, event: `typeIdent` + ) {.async: (raises: []).} = + `initProcIdent`() + + when compiles(event.isNil()): + if event.isNil(): + error "Cannot emit uninitialized event object", eventType = `typeNameLit` + return + + type CrossTarget = object + ring: ptr VyukovMpscRing[uint32] + signal: ThreadSignalPtr + + var crossTargets: seq[CrossTarget] = @[] + var hasSameThread = false + let myThreadId = currentMtThreadId() + let myThreadGen = currentMtThreadGen() + + withLock(`globalLockIdent`): + for i in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[i].brokerCtx == brokerCtx and + `globalBucketsIdent`[i].active and `globalBucketsIdent`[i].hasListeners: + if `globalBucketsIdent`[i].threadId == myThreadId and + `globalBucketsIdent`[i].threadGen == myThreadGen: + hasSameThread = true + else: + crossTargets.add( + CrossTarget( + ring: `globalBucketsIdent`[i].ring, + signal: `globalBucketsIdent`[i].listenerSignal, + ) + ) + + # Same-thread fast path: bypass ring entirely. + if hasSameThread: + var idx = -1 + for i in 0 ..< `tvListenerCtxIdent`.len: + if `tvListenerCtxIdent`[i] == brokerCtx: + idx = i + break + if idx >= 0: + var callbacks: seq[`handlerProcIdent`] = @[] + for cb in `tvListenerHandlersIdent`[idx].values: + callbacks.add(cb) + for cb in callbacks: + let fut: Future[void] = `listenerTaskIdent`(cb, event) + `tvListenerFutsIdent`.add((brokerCtx, fut)) + asyncSpawn fut + + if crossTargets.len == 0: + return + + # Cross-thread fan-out via shared refcounted cell. + let shardHint = `shardHintIdent`() + let cellIdx = `globalSlabIdent`.claim(shardHint) + if cellIdx == EmptyIdx: + warn "event dropped: slab exhausted", + eventType = `typeNameLit`, targets = crossTargets.len + return + + let cell = `globalSlabIdent`.cellPtr(cellIdx) + let payloadPtr = `globalSlabIdent`.cellPayloadPtr(cellIdx) + let written = + try: + `marshalIdent`(payloadPtr, int(`globalSlabIdent`.cellPayloadCap), event) + except Exception: + -1 + if written >= 0: + # Fast path: payload fit the fixed cell. + cell.payloadSize = uint32(written) + else: + # Auto-spill: payload exceeded the cell — marshal into an exact-size + # heap buffer instead of dropping. Owned by the cell; freed at release. + let needed = + try: + `marshalSizeIdent`(event) + except Exception: + -1 + if needed < 0 or needed > `maxDynPayloadLit`: + error "event dropped: payload exceeds maxDynamicPayloadBytes", + eventType = `typeNameLit`, needed = needed, cap = `maxDynPayloadLit` + `globalSlabIdent`.release(cellIdx, shardHint) + return + let spillBuf = allocShared0(needed) + if spillBuf.isNil: + error "event dropped: spill allocation failed", + eventType = `typeNameLit`, needed = needed + `globalSlabIdent`.release(cellIdx, shardHint) + return + let w2 = + try: + `marshalIdent`(cast[ptr UncheckedArray[byte]](spillBuf), needed, event) + except Exception: + -1 + if w2 < 0: + deallocShared(spillBuf) + `globalSlabIdent`.release(cellIdx, shardHint) + return + `globalSlabIdent`.setOverflow(cellIdx, spillBuf, uint32(w2)) + cell.refcount.store(crossTargets.len, moRelease) + + for target in crossTargets: + if not target.ring.tryEnqueue(cellIdx): + warn "event dropped: listener queue full", eventType = `typeNameLit` + `releaseCellIdent`(cellIdx) + else: + fireBrokerSignal(target.signal) + + ) + + # ── Public emit ─────────────────────────────────────────────────────── + result.add( + quote do: + proc emit*(event: `typeIdent`) {.async: (raises: []).} = + await `emitImplIdent`(DefaultBrokerContext, event) + + proc emit*(_: typedesc[`typeIdent`], event: `typeIdent`) {.async: (raises: []).} = + await `emitImplIdent`(DefaultBrokerContext, event) + + proc emit*( + _: typedesc[`typeIdent`], brokerCtx: BrokerContext, event: `typeIdent` + ) {.async: (raises: []).} = + await `emitImplIdent`(brokerCtx, event) + + ) + + # ── Field-constructor emit overloads (for inline object types) ──────── + if hasInlineFields: + let typedescParamType = + newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)) + + let asyncPragma = newTree( + nnkPragma, + newTree( + nnkExprColonExpr, + ident("async"), + newTree( + nnkTupleConstr, + newTree(nnkExprColonExpr, ident("raises"), newTree(nnkBracket)), + ), + ), + ) + + var emitCtorParams = newTree(nnkFormalParams, newEmptyNode()) + emitCtorParams.add( + newTree(nnkIdentDefs, ident("_"), typedescParamType, newEmptyNode()) + ) + for i in 0 ..< fieldNames.len: + emitCtorParams.add( + newTree( + nnkIdentDefs, + copyNimTree(fieldNames[i]), + copyNimTree(fieldTypes[i]), + newEmptyNode(), + ) + ) + + var emitCtorExpr = newTree(nnkObjConstr, copyNimTree(typeIdent)) + for i in 0 ..< fieldNames.len: + emitCtorExpr.add( + newTree( + nnkExprColonExpr, copyNimTree(fieldNames[i]), copyNimTree(fieldNames[i]) + ) + ) + + let emitCtorCallDefault = + newCall(copyNimTree(emitImplIdent), ident("DefaultBrokerContext"), emitCtorExpr) + let emitCtorBodyDefault = quote: + await `emitCtorCallDefault` + + let typedescEmitProcDefault = newTree( + nnkProcDef, + postfix(ident("emit"), "*"), + newEmptyNode(), + newEmptyNode(), + emitCtorParams, + copyNimTree(asyncPragma), + newEmptyNode(), + emitCtorBodyDefault, + ) + result.add(typedescEmitProcDefault) + + var emitCtorParamsCtx = newTree(nnkFormalParams, newEmptyNode()) + emitCtorParamsCtx.add( + newTree(nnkIdentDefs, ident("_"), typedescParamType, newEmptyNode()) + ) + emitCtorParamsCtx.add( + newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode()) + ) + for i in 0 ..< fieldNames.len: + emitCtorParamsCtx.add( + newTree( + nnkIdentDefs, + copyNimTree(fieldNames[i]), + copyNimTree(fieldTypes[i]), + newEmptyNode(), + ) + ) + + let emitCtorCallCtx = + newCall(copyNimTree(emitImplIdent), ident("brokerCtx"), copyNimTree(emitCtorExpr)) + let emitCtorBodyCtx = quote: + await `emitCtorCallCtx` + + let typedescEmitProcCtx = newTree( + nnkProcDef, + postfix(ident("emit"), "*"), + newEmptyNode(), + newEmptyNode(), + emitCtorParamsCtx, + copyNimTree(asyncPragma), + newEmptyNode(), + emitCtorBodyCtx, + ) + result.add(typedescEmitProcCtx) + + # ── dropListener impl ───────────────────────────────────────────────── + result.add( + quote do: + proc `dropListenerImplIdent`( + brokerCtx: BrokerContext, handle: `listenerHandleIdent` + ) = + if handle.id == 0'u64: + return + if handle.threadId != currentMtThreadId(): + error "dropListener called from wrong thread", + eventType = `typeNameLit`, + handleThread = repr(handle.threadId), + currentThread = repr(currentMtThreadId()) + return + + var tvIdx = -1 + for i in 0 ..< `tvListenerCtxIdent`.len: + if `tvListenerCtxIdent`[i] == brokerCtx: + tvIdx = i + break + if tvIdx < 0: + return + + `tvListenerHandlersIdent`[tvIdx].del(handle.id) + + if `tvListenerHandlersIdent`[tvIdx].len == 0: + `tvListenerCtxIdent`.del(tvIdx) + `tvListenerHandlersIdent`.del(tvIdx) + `tvNextIdsIdent`.del(tvIdx) + + let myThreadId = currentMtThreadId() + let myThreadGen = currentMtThreadGen() + withLock(`globalLockIdent`): + for i in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[i].brokerCtx == brokerCtx and + `globalBucketsIdent`[i].threadId == myThreadId and + `globalBucketsIdent`[i].threadGen == myThreadGen: + `globalBucketsIdent`[i].hasListeners = false + break + + ) + + # ── dropAllListeners impl ───────────────────────────────────────────── + # Same-thread: clears tvHandlers immediately + flips flag under lock. + # Cross-thread: flips flag + pushes a CtrlClearListeners sentinel into + # the bucket's ring so the listener thread clears its tvHandlers on + # the next poll cycle. + result.add( + quote do: + proc `dropAllListenersImplIdent`(brokerCtx: BrokerContext) = + `initProcIdent`() + + let myThreadId = currentMtThreadId() + var crossRings: seq[(ptr VyukovMpscRing[uint32], ThreadSignalPtr)] = @[] + + withLock(`globalLockIdent`): + for i in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[i].brokerCtx == brokerCtx and + `globalBucketsIdent`[i].hasListeners: + `globalBucketsIdent`[i].hasListeners = false + if `globalBucketsIdent`[i].threadId != myThreadId: + crossRings.add( + (`globalBucketsIdent`[i].ring, `globalBucketsIdent`[i].listenerSignal) + ) + + # Same-thread tv clear. + var tvIdx = -1 + for i in 0 ..< `tvListenerCtxIdent`.len: + if `tvListenerCtxIdent`[i] == brokerCtx: + tvIdx = i + break + if tvIdx >= 0: + `tvListenerHandlersIdent`[tvIdx].clear() + `tvListenerCtxIdent`.del(tvIdx) + `tvListenerHandlersIdent`.del(tvIdx) + `tvNextIdsIdent`.del(tvIdx) + + # Cross-thread: send control sentinel. + for (ring, sig) in crossRings: + discard ring.tryEnqueue(CtrlClearListeners) + fireBrokerSignal(sig) + + # Part D-3: invoke the companion cleanup hook (if any) AFTER + # listener clearing. The CBOR FFI library registers this hook + # in its per-event installer to clear the foreign-subscriber + # registry + reset the per-event atomic counter, keeping + # `SubsRegistry` in lock-step with the MT EventBroker listener + # table on dropAllListeners. The hook runs unlocked on the + # caller's thread; it's the hook's responsibility to acquire + # whatever locks its data structures need. + var hookSnap: `dropAllHookProcTypeIdent` = nil + {.cast(gcsafe).}: + withLock(`dropAllHookLockIdent`): + hookSnap = `dropAllHookIdent` + if not hookSnap.isNil: + hookSnap(brokerCtx) + + ) + + # ── Public dropListener / dropAllListeners ──────────────────────────── + result.add( + quote do: + proc dropListener*(_: typedesc[`typeIdent`], handle: `listenerHandleIdent`) = + `dropListenerImplIdent`(DefaultBrokerContext, handle) + + proc dropListener*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + handle: `listenerHandleIdent`, + ) = + `dropListenerImplIdent`(brokerCtx, handle) + + proc dropAllListeners*(_: typedesc[`typeIdent`]) = + `dropAllListenersImplIdent`(DefaultBrokerContext) + + proc dropAllListeners*(_: typedesc[`typeIdent`], brokerCtx: BrokerContext) = + `dropAllListenersImplIdent`(brokerCtx) + + proc `setDropAllHookIdent`*( + _: typedesc[`typeIdent`], hook: `dropAllHookProcTypeIdent` + ) = + ## Part D-3: register a companion cleanup hook fired by + ## `dropAllListeners` (any overload) AFTER listener clearing + ## completes. Passing `nil` clears the slot. Single slot per + ## event type — the intended sole caller is the CBOR FFI + ## library's per-event installer. + `initProcIdent`() + {.cast(gcsafe).}: + withLock(`dropAllHookLockIdent`): + `dropAllHookIdent` = hook + + ) + + # ── shutdownProcessLoopsForCtx (internal; used by API teardown) ─────── + # Must run on the bucket-owning thread. Drains the bucket's ring, + # decRefs remaining cells, removes the bucket from the registry, and + # deallocs its ring. The owner thread is the only safe deallocator + # (Invariant I0). + result.add( + quote do: + proc `shutdownProcessLoopsForCtxIdent`( + ctx: BrokerContext + ) {.async: (raises: []).} = + let myThreadId = currentMtThreadId() + let myThreadGen = currentMtThreadGen() + var ringsToShutdown: seq[(ptr VyukovMpscRing[uint32], ThreadSignalPtr)] = @[] + withLock(`globalLockIdent`): + var i = 0 + while i < `globalBucketCountIdent`: + if `globalBucketsIdent`[i].brokerCtx == ctx and + `globalBucketsIdent`[i].threadId == myThreadId and + `globalBucketsIdent`[i].threadGen == myThreadGen and + `globalBucketsIdent`[i].active: + ringsToShutdown.add( + (`globalBucketsIdent`[i].ring, `globalBucketsIdent`[i].listenerSignal) + ) + for j in i ..< `globalBucketCountIdent` - 1: + `globalBucketsIdent`[j] = `globalBucketsIdent`[j + 1] + `globalBucketCountIdent` -= 1 + else: + inc i + + var shutdownFuts: seq[Future[void]] = @[] + var k = 0 + while k < `tvShutdownFutsIdent`.len: + if `tvShutdownFutsIdent`[k][0] == ctx: + shutdownFuts.add(`tvShutdownFutsIdent`[k][1]) + `tvShutdownFutsIdent`.del(k) + else: + inc k + + # Close each ring; the poll fn observes `closed && empty` and + # self-unregisters via return-code 2 + completes shutdownFut. + # Signal the dispatcher so the poll fn actually runs. + for (ring, sig) in ringsToShutdown: + ring.close() + fireBrokerSignal(sig) + + for fut in shutdownFuts: + if not fut.finished(): + try: + discard await withTimeout(fut, chronos.seconds(5)) + except CatchableError: + discard + + # Drain in-flight listener futures for this context. + var j = 0 + while j < `tvListenerFutsIdent`.len: + if `tvListenerFutsIdent`[j][0] == ctx: + let fut = `tvListenerFutsIdent`[j][1] + if not fut.finished(): + try: + discard await withTimeout(fut, chronos.seconds(5)) + except CatchableError: + discard + `tvListenerFutsIdent`.del(j) + else: + inc j + + # Grace window: an emit that captured ring pointers under lock + # before we removed the bucket may still be mid-`tryEnqueue`. + # The poll fn has already self-unregistered (return 2), and the + # ring is closed, so any new tryEnqueue gets rejected — but we + # need a brief delay before deallocShared so the in-flight + # callers can complete their access. 50ms matches the original + # `deferredFreeEventChan` window. + try: + await sleepAsync(chronos.milliseconds(50)) + except CatchableError: + discard + for (ring, _) in ringsToShutdown: + freeVyukovMpscRing(ring) + + ) + + # ── Public shutdown ─────────────────────────────────────────────────── + result.add( + quote do: + proc shutdown*(_: typedesc[`typeIdent`]): Future[void] {.async: (raises: []).} = + await `shutdownProcessLoopsForCtxIdent`(DefaultBrokerContext) + + proc shutdown*( + _: typedesc[`typeIdent`], brokerCtx: BrokerContext + ): Future[void] {.async: (raises: []).} = + await `shutdownProcessLoopsForCtxIdent`(brokerCtx) + + ) + + when defined(brokerDebug): + writeBrokerDebug("EventBrokerMt", typeDisplayName, result) + when defined(brokerDebugStdout): + echo result.repr diff --git a/wasm-deps/brokers/brokers/internal/mt_queue.nim b/wasm-deps/brokers/brokers/internal/mt_queue.nim new file mode 100644 index 000000000..8e704f445 --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/mt_queue.nim @@ -0,0 +1,592 @@ +## Multi-Thread Broker Queue Primitives +## ------------------------------------ +## Lock-free MPSC primitives used to replace `Channel[T]` in the (mt) +## brokers. Implements `doc/REFACTOR_MT_QUEUE.md` §3. +## +## Invariants enforced by *structural design*, not by runtime asserts: +## +## I0 every `createShared` / `deallocShared` runs on a persistent owner +## thread (bucket-owner for per-bucket structures, global-slab-owner +## for events). Hot path (claim / release / enqueue / dequeue) never +## calls any Nim allocator. +## CARVE-OUT (flexible-mt-dispatch): when a marshaled payload exceeds the +## fixed cell, the producer `allocShared0`s a heap-spill buffer and the +## consumer-side `release` `deallocShared`s it. So the spill path DOES +## allocate on the hot path — a deliberate trade so oversized payloads +## (>cell, e.g. >1 MiB) succeed instead of being dropped. The common +## fits-the-cell path is unchanged and allocator-free. Spill buffers are +## POD bytes with single producer→consumer ownership, same cross-thread +## contract as `storage` itself. +## I1 Senders only execute: atomic load / store / CAS, memcpy into +## pre-allocated cells, and `ThreadSignalPtr.fireSync()` (external). +## I2 The owner thread must outlive every structure it owns. +## +## Phase 1 deliverable: this module + its tests. No broker code calls it +## yet — that lands in Phases 2-4. + +{.push raises: [].} + +import std/atomics + +# --------------------------------------------------------------------------- +# Cache-line padding helpers +# --------------------------------------------------------------------------- + +const CacheLineBytes* = 64 + +type CacheLineGap = array[CacheLineBytes, byte] + +# --------------------------------------------------------------------------- +# ShardedFreeList — Treiber stack with ABA tagging, sharded by thread hash +# --------------------------------------------------------------------------- +# +# Head word layout (uint64): +# bits 0..31 index into the free-list's external `nextLinks` array +# bits 32..63 ABA tag (incremented on every successful CAS) +# +# A special INDEX value `EmptyIdx` means "this shard is empty". +# The free-list does NOT own the storage — callers manage capacity and +# the `nextLinks` array externally. This keeps the primitive composable. + +const EmptyIdx*: uint32 = high(uint32) + +template makeHead(idx, tag: uint32): uint64 = + (uint64(tag) shl 32) or uint64(idx) + +template headIdx(v: uint64): uint32 = + uint32(v and 0xFFFFFFFF'u64) + +template headTag(v: uint64): uint32 = + uint32(v shr 32) + +type + FreeListShard = object + head: Atomic[uint64] + gap: CacheLineGap + + ShardedFreeList* = object + nShardsMask: uint32 ## nShards - 1; nShards is power-of-2 + nShards: uint32 + shards: ptr UncheckedArray[FreeListShard] + nextLinks: ptr UncheckedArray[uint32] ## idx → next idx (or EmptyIdx) + +proc initShardedFreeList*( + fl: var ShardedFreeList, nShards: uint32, capacity: uint32 +) {.gcsafe.} = + ## Initialize a sharded free-list. `nShards` MUST be a power of two. + ## `nextLinks` is allocated as a parallel array of `capacity` indices, + ## all initialised to `EmptyIdx`. + doAssert nShards > 0 and (nShards and (nShards - 1)) == 0, + "nShards must be power of two" + fl.nShards = nShards + fl.nShardsMask = nShards - 1 + fl.shards = + cast[ptr UncheckedArray[FreeListShard]](createShared(FreeListShard, nShards.int)) + for i in 0 ..< nShards.int: + fl.shards[i].head.store(makeHead(EmptyIdx, 0), moRelaxed) + fl.nextLinks = cast[ptr UncheckedArray[uint32]](createShared(uint32, capacity.int)) + for i in 0 ..< capacity.int: + fl.nextLinks[i] = EmptyIdx + +proc deinitShardedFreeList*(fl: var ShardedFreeList) {.gcsafe.} = + if not fl.shards.isNil: + deallocShared(fl.shards) + fl.shards = nil + if not fl.nextLinks.isNil: + deallocShared(fl.nextLinks) + fl.nextLinks = nil + +proc push*(fl: var ShardedFreeList, idx: uint32, shardHint: uint32) {.gcsafe.} = + ## Push `idx` onto the free-list. `shardHint` selects the shard to push to. + let shardIdx = shardHint and fl.nShardsMask + let shard = addr fl.shards[shardIdx] + while true: + let oldHead = shard.head.load(moAcquire) + fl.nextLinks[idx] = headIdx(oldHead) + # Tag increments on every successful push to dodge ABA. + let newHead = makeHead(idx, headTag(oldHead) + 1) + var expected = oldHead + if shard.head.compareExchangeWeak(expected, newHead, moAcquireRelease, moAcquire): + return + +proc pop*(fl: var ShardedFreeList, shardHint: uint32): uint32 {.gcsafe.} = + ## Pop an index from the free-list. Tries `shardHint`'s shard first; + ## if empty, scans other shards. Returns `EmptyIdx` if all shards are empty. + let preferred = shardHint and fl.nShardsMask + for offset in 0'u32 ..< fl.nShards: + let shardIdx = (preferred + offset) and fl.nShardsMask + let shard = addr fl.shards[shardIdx] + while true: + let oldHead = shard.head.load(moAcquire) + let idx = headIdx(oldHead) + if idx == EmptyIdx: + break # try next shard + let nextIdx = fl.nextLinks[idx] + let newHead = makeHead(nextIdx, headTag(oldHead) + 1) + var expected = oldHead + if shard.head.compareExchangeWeak(expected, newHead, moAcquireRelease, moAcquire): + return idx + # CAS failed; loop and retry on this shard. + return EmptyIdx + +# --------------------------------------------------------------------------- +# VyukovMpscRing[T] — bounded MPSC ring with closed-flag handoff +# --------------------------------------------------------------------------- +# +# Atomic protocol: +# Producer (`tryEnqueue`): +# 1. closed-check (acquire) → if closed, return false. +# 2. pos = enqPos.load(relaxed) +# 3. loop: +# slot = &slots[pos & mask] +# seq = slot.seq.load(acquire) +# diff = seq - pos (as signed) +# if diff == 0: +# CAS enqPos: pos → pos+1 (acquireRelease on success, acquire on failure) +# if success: break (slot is claimed; must publish) +# else: pos was reloaded into `expected`; loop +# elif diff < 0: +# return false (full) +# else: +# another producer claimed this slot already; reload pos, loop +# 4. write slot.payload +# 5. slot.seq.store(pos+1, release) -- publish +# +# Consumer (`tryDequeue`, single-thread): +# 1. pos = deqPos +# 2. seq = slots[pos & mask].seq.load(acquire) +# 3. if seq != pos+1: return false (empty / not yet published) +# 4. read payload +# 5. slots[pos & mask].seq.store(pos + capacity, release) -- slot reusable +# 6. deqPos = pos + 1 +# +# Closed-flag handoff (`drain` called by owner): +# 1. closed.store(true, release) +# 2. spin-loop: tryDequeue all visible items; sleep if there's a gap +# (slot not yet published by an in-flight producer that already CAS'd). +# Exit when deqPos == enqPos. + +type + Slot*[T] = object + seq: Atomic[uint64] + payload*: T + + VyukovMpscRing*[T] = object + capacity*: uint64 + mask: uint64 + closed: Atomic[bool] + gap0: CacheLineGap + enqPos: Atomic[uint64] + gap1: CacheLineGap + deqPos: uint64 + gap2: CacheLineGap + slots: ptr UncheckedArray[Slot[T]] + +proc newVyukovMpscRing*[T](capacity: int): ptr VyukovMpscRing[T] {.gcsafe.} = + ## Allocate a ring of the given capacity (must be power-of-2). + ## Returns ownership; deinit via `freeVyukovMpscRing`. + doAssert capacity > 0 and (capacity and (capacity - 1)) == 0, + "capacity must be power-of-2" + result = cast[ptr VyukovMpscRing[T]](createShared(VyukovMpscRing[T], 1)) + result.capacity = uint64(capacity) + result.mask = uint64(capacity - 1) + result.closed.store(false, moRelaxed) + result.enqPos.store(0, moRelaxed) + result.deqPos = 0 + result.slots = cast[ptr UncheckedArray[Slot[T]]](createShared(Slot[T], capacity)) + for i in 0 ..< capacity: + result.slots[i].seq.store(uint64(i), moRelaxed) + +proc freeVyukovMpscRing*[T](ring: ptr VyukovMpscRing[T]) {.gcsafe.} = + ## Deallocate. Must be called on the owner thread, with the ring already + ## drained (caller responsibility). + if ring.isNil: + return + if not ring.slots.isNil: + deallocShared(ring.slots) + deallocShared(ring) + +proc isClosed*[T](ring: ptr VyukovMpscRing[T]): bool {.gcsafe.} = + ring.closed.load(moAcquire) + +proc close*[T](ring: ptr VyukovMpscRing[T]) {.gcsafe.} = + ring.closed.store(true, moRelease) + +proc tryEnqueue*[T](ring: ptr VyukovMpscRing[T], item: sink T): bool {.gcsafe.} = + ## Returns true if enqueued, false if full or closed. + ## Safe to call from any number of producer threads. + if ring.closed.load(moAcquire): + return false + var pos = ring.enqPos.load(moRelaxed) + while true: + let slot = addr ring.slots[pos and ring.mask] + let seqV = slot.seq.load(moAcquire) + let diff = cast[int64](seqV) - cast[int64](pos) + if diff == 0: + var expected = pos + if ring.enqPos.compareExchangeWeak(expected, pos + 1, moAcquireRelease, moAcquire): + # We own slot[pos]. Re-check closed for the "closed after our + # initial check but before CAS" race; if closed, we must still + # publish so the consumer can observe and drain it. The drain + # protocol counts on the slot being published. + slot.payload = item + slot.seq.store(pos + 1, moRelease) + return true + # CAS failed; `expected` now holds the latest enqPos; retry. + pos = expected + elif diff < 0: + # Full: slot.seq lags pos, which means the prior occupant hasn't + # been consumed yet. + return false + else: + # diff > 0: another producer is ahead of us; reload pos. + pos = ring.enqPos.load(moRelaxed) + +proc tryDequeue*[T](ring: ptr VyukovMpscRing[T], outItem: var T): bool {.gcsafe.} = + ## Returns true if an item was dequeued, false if empty. + ## MUST be called from a single consumer thread. + let pos = ring.deqPos + let slot = addr ring.slots[pos and ring.mask] + let seqV = slot.seq.load(moAcquire) + let diff = cast[int64](seqV) - cast[int64](pos + 1) + if diff != 0: + return false + outItem = slot.payload + slot.seq.store(pos + ring.capacity, moRelease) + ring.deqPos = pos + 1 + return true + +proc isEmpty*[T](ring: ptr VyukovMpscRing[T]): bool {.gcsafe.} = + ## Consumer-side observation; producers may concurrently enqueue, + ## so callers must treat the result as a hint unless they also hold + ## a guarantee that no producers are active. + ring.enqPos.load(moAcquire) == ring.deqPos + +# --------------------------------------------------------------------------- +# RefCountedCell + PayloadSlab — pre-allocated payload cells +# --------------------------------------------------------------------------- +# +# Cell layout (computed at runtime, since payload bytes are variable-size): +# CellHeader fields: refcount (Atomic[int]), payloadSize (uint32), +# overflowLen (uint32), overflow (pointer) +# then: payloadBytes[] (payloadCap bytes; from +# slab.cellPayloadCap), starting at sizeof(CellHeader). +# (cellStride is alignUp(sizeof(CellHeader) + payloadCap, 8), so the exact +# payload offset is always sizeof(CellHeader) regardless of field packing — +# do not assume a hard-coded offset, the header grew with the spill fields.) +# +# We address cells by index (uint32) so the free-list can ABA-tag indices +# rather than pointers. Pointer access is via `slab.cellPtr(idx)`. + +type + CellHeader* = object + refcount*: Atomic[int] + payloadSize*: uint32 + ## inline marshaled bytes used. uint32 (not uint16) so a configured cell + ## may exceed 64 KiB — broker messages can be >1 MiB. 0 when the payload + ## spilled to the heap (see `overflow`). + overflowLen*: uint32 ## spilled byte count; 0 when the payload fit inline. + overflow*: pointer + ## heap-spill buffer (`allocShared0`) when the marshaled payload exceeded + ## the fixed cell; `nil` on the inline fast path. Owned by the cell: freed + ## in `release` (refcount→0 chokepoint) and walked by `deinitPayloadSlab`. + ## POD bytes only — same cross-thread ownership contract as `storage`. + + PayloadSlab* = object + capacity: uint32 + cellPayloadCap*: uint32 ## bytes available for marshaled data per cell + cellStride: uint32 ## sizeof(CellHeader) + cellPayloadCap, aligned + storage: ptr UncheckedArray[byte] + freeList: ShardedFreeList + +proc cellHeaderSize(): uint32 {.compileTime.} = + uint32(sizeof(CellHeader)) + +proc alignUp(v, a: uint32): uint32 = + (v + a - 1) and not (a - 1) + +proc initPayloadSlab*( + slab: var PayloadSlab, capacity: uint32, payloadBytes: uint32, nShards: uint32 +) {.gcsafe.} = + ## Pre-allocates `capacity` cells, each with `payloadBytes` of payload + ## space. Uses `nShards` (must be power-of-2) for free-list contention. + ## All cells start on the free-list. + doAssert capacity > 0 + doAssert payloadBytes > 0 + slab.capacity = capacity + slab.cellPayloadCap = payloadBytes + slab.cellStride = alignUp(cellHeaderSize() + payloadBytes, 8'u32) + slab.storage = cast[ptr UncheckedArray[byte]](createShared( + byte, int(capacity) * int(slab.cellStride) + )) + initShardedFreeList(slab.freeList, nShards, capacity) + # Seed the free-list with every cell. + for i in 0 ..< capacity: + push(slab.freeList, i, i) + +proc cellPtr*(slab: PayloadSlab, idx: uint32): ptr CellHeader {.gcsafe.} = + ## Returns the header pointer for the cell at `idx`. The payload bytes + ## immediately follow the header (at `cast[ptr byte](header) +% + ## sizeof(CellHeader)`). + cast[ptr CellHeader](addr slab.storage[int(idx) * int(slab.cellStride)]) + +proc cellPayloadPtr*( + slab: PayloadSlab, idx: uint32 +): ptr UncheckedArray[byte] {.gcsafe.} = + cast[ptr UncheckedArray[byte]](cast[uint](addr slab.storage[ + int(idx) * int(slab.cellStride) + ]) + uint(sizeof(CellHeader))) + +proc deinitPayloadSlab*(slab: var PayloadSlab) {.gcsafe.} = + ## MUST be called on the owner thread after every outstanding cell has + ## been released (caller responsibility). Frees the slab's storage and + ## the free-list's internal arrays. Also walks every cell to free any + ## heap-spill buffer still attached — covers shutdown / clearProvider with + ## undelivered in-flight cells (a cell closed before delivery never passes + ## through `release`, so its spill would otherwise leak). + if not slab.storage.isNil: + for i in 0'u32 ..< slab.capacity: + let cell = slab.cellPtr(i) + if not cell.overflow.isNil: + deallocShared(cell.overflow) + cell.overflow = nil + cell.overflowLen = 0 + deinitShardedFreeList(slab.freeList) + if not slab.storage.isNil: + deallocShared(slab.storage) + slab.storage = nil + +proc setOverflow*( + slab: PayloadSlab, idx: uint32, buf: pointer, len: uint32 +) {.gcsafe.} = + ## Attach a heap-spill buffer to a cell (payload exceeded the inline cell). + ## The cell takes ownership; `release`/`deinitPayloadSlab` free it. + let cell = slab.cellPtr(idx) + cell.overflow = buf + cell.overflowLen = len + cell.payloadSize = 0 + +proc dataPtr*(slab: PayloadSlab, idx: uint32): ptr UncheckedArray[byte] {.gcsafe.} = + ## Pointer to the marshaled bytes for a cell — the heap-spill buffer when the + ## payload spilled, else the inline payload region. + let cell = slab.cellPtr(idx) + if not cell.overflow.isNil: + cast[ptr UncheckedArray[byte]](cell.overflow) + else: + slab.cellPayloadPtr(idx) + +proc dataLen*(slab: PayloadSlab, idx: uint32): int {.gcsafe.} = + ## Marshaled byte count for a cell (spill length or inline payloadSize). + let cell = slab.cellPtr(idx) + if not cell.overflow.isNil: + int(cell.overflowLen) + else: + int(cell.payloadSize) + +proc claim*(slab: var PayloadSlab, shardHint: uint32): uint32 {.gcsafe.} = + ## Returns a cell index or `EmptyIdx` if the slab is exhausted. + pop(slab.freeList, shardHint) + +proc release*(slab: var PayloadSlab, idx: uint32, shardHint: uint32) {.gcsafe.} = + ## Returns a cell to the free-list. Caller must ensure no other thread + ## still holds a reference (refcount == 0). This is the single chokepoint a + ## cell passes through on its way back to the free-list (all delivery / drop / + ## error paths funnel here once refcount hits 0), so any heap-spill buffer is + ## freed here exactly once. + let cell = slab.cellPtr(idx) + if not cell.overflow.isNil: + deallocShared(cell.overflow) + cell.overflow = nil + cell.overflowLen = 0 + push(slab.freeList, idx, shardHint) + +proc incRef*(slab: PayloadSlab, idx: uint32) {.gcsafe.} = + discard slab.cellPtr(idx).refcount.fetchAdd(1, moAcquireRelease) + +proc decRefAndCheck*(slab: PayloadSlab, idx: uint32): bool {.gcsafe.} = + ## Returns true if this decrement brought refcount to zero (caller should + ## then `release(idx)`). + let prev = slab.cellPtr(idx).refcount.fetchSub(1, moAcquireRelease) + prev == 1 + +# --------------------------------------------------------------------------- +# ResponseSlot[T] + ResponseSlotPool[T] — single-shot request reply +# --------------------------------------------------------------------------- +# +# State machine on the slot's `state` byte: +# Empty(0) ── requester claimed; provider hasn't written yet +# │ +# ├── (provider) CAS Empty→Ready, write payload, signal requester +# │ │ +# │ └── (requester) read payload; release slot +# │ +# └── (requester timeout) CAS Empty→Abandoned +# │ +# └── (provider) sees Abandoned; releases slot +# +# In both terminal cases the slot returns to the pool's free-list +# exactly once. + +type + ResponseState* {.pure.} = enum + Empty = 0'u8 + Writing = 1'u8 ## reserved by provider; bytes in flight + Ready = 2'u8 + Abandoned = 3'u8 + + ResponseSlotHeader = object + state: Atomic[uint8] + pad0: array[3, byte] ## align the uint32 payloadSize to a 4-byte boundary + payloadSize: uint32 + ## uint32 (not uint16) so a response slot may exceed 64 KiB. + ## state(1) + pad0(3) + payloadSize(4) = 8 bytes → 8-aligned. + overflowLen: uint32 ## spilled response byte count; 0 when the response fit inline. + pad1: uint32 ## keep the pointer that follows 8-aligned (overflowLen at +8) + overflow: pointer + ## heap-spill buffer for an oversized response; `nil` inline. Owned by the + ## slot: freed in `release` and walked by `deinitResponseSlotPool`. + + ResponseSlotPool* = object + capacity*: uint32 + slotPayloadCap*: uint32 + slotStride: uint32 + storage: ptr UncheckedArray[byte] + freeList: ShardedFreeList + +proc respSlotHeaderSize(): uint32 {.compileTime.} = + uint32(sizeof(ResponseSlotHeader)) + +proc slotHeaderPtr( + pool: ResponseSlotPool, idx: uint32 +): ptr ResponseSlotHeader {.gcsafe.} = + cast[ptr ResponseSlotHeader](addr pool.storage[int(idx) * int(pool.slotStride)]) + +proc slotPayloadPtr*( + pool: ResponseSlotPool, idx: uint32 +): ptr UncheckedArray[byte] {.gcsafe.} = + cast[ptr UncheckedArray[byte]](cast[uint](addr pool.storage[ + int(idx) * int(pool.slotStride) + ]) + uint(sizeof(ResponseSlotHeader))) + +proc initResponseSlotPool*( + pool: var ResponseSlotPool, + capacity: uint32, + maxPayloadBytes: uint32, + nShards: uint32, +) {.gcsafe.} = + pool.capacity = capacity + pool.slotPayloadCap = maxPayloadBytes + pool.slotStride = alignUp(respSlotHeaderSize() + maxPayloadBytes, 8'u32) + pool.storage = cast[ptr UncheckedArray[byte]](createShared( + byte, int(capacity) * int(pool.slotStride) + )) + initShardedFreeList(pool.freeList, nShards, capacity) + for i in 0 ..< capacity: + let hdr = pool.slotHeaderPtr(i) + hdr.state.store(uint8(ResponseState.Empty), moRelaxed) + hdr.payloadSize = 0 + push(pool.freeList, i, i) + +proc deinitResponseSlotPool*(pool: var ResponseSlotPool) {.gcsafe.} = + ## Walk every slot to free any heap-spill buffer still attached (shutdown + ## with an undelivered response), then free storage + free-list arrays. + if not pool.storage.isNil: + for i in 0'u32 ..< pool.capacity: + let hdr = pool.slotHeaderPtr(i) + if not hdr.overflow.isNil: + deallocShared(hdr.overflow) + hdr.overflow = nil + hdr.overflowLen = 0 + deinitShardedFreeList(pool.freeList) + if not pool.storage.isNil: + deallocShared(pool.storage) + pool.storage = nil + +proc claim*(pool: var ResponseSlotPool, shardHint: uint32): uint32 {.gcsafe.} = + let idx = pop(pool.freeList, shardHint) + if idx != EmptyIdx: + let hdr = pool.slotHeaderPtr(idx) + hdr.payloadSize = 0 + # release() already frees+nils any spill, but defend against a slot that + # reached the free-list without passing release (it should not). + if not hdr.overflow.isNil: + deallocShared(hdr.overflow) + hdr.overflow = nil + hdr.overflowLen = 0 + hdr.state.store(uint8(ResponseState.Empty), moRelease) + idx + +proc release*(pool: var ResponseSlotPool, idx: uint32, shardHint: uint32) {.gcsafe.} = + ## Single chokepoint a slot passes through back to the free-list (requester + ## after read, or provider on abandon). Free any heap-spill buffer here. + let hdr = pool.slotHeaderPtr(idx) + if not hdr.overflow.isNil: + deallocShared(hdr.overflow) + hdr.overflow = nil + hdr.overflowLen = 0 + push(pool.freeList, idx, shardHint) + +proc beginWrite*(pool: ResponseSlotPool, idx: uint32): bool {.gcsafe.} = + ## Provider: CAS Empty→Writing. Returns false if the requester abandoned + ## the slot first (caller should release without writing). + let hdr = pool.slotHeaderPtr(idx) + var expected = uint8(ResponseState.Empty) + hdr.state.compareExchange( + expected, uint8(ResponseState.Writing), moAcquireRelease, moAcquire + ) + +proc commitWrite*(pool: ResponseSlotPool, idx: uint32, payloadSize: uint32) {.gcsafe.} = + ## Provider: finalize after writing payload bytes. Stores size + flips + ## state to Ready (release-ordered, so the bytes-write is visible to + ## any acquire-loader on the state). + let hdr = pool.slotHeaderPtr(idx) + hdr.payloadSize = payloadSize + hdr.state.store(uint8(ResponseState.Ready), moRelease) + +proc commitWriteOverflow*( + pool: ResponseSlotPool, idx: uint32, buf: pointer, len: uint32 +) {.gcsafe.} = + ## Provider: finalize an oversized response that spilled to the heap. The + ## slot takes ownership of `buf` (freed in `release`/`deinitResponseSlotPool`). + ## Sets inline payloadSize = 0 and flips state to Ready (release-ordered so the + ## buffer pointer + the bytes it points to are visible to an acquire-loader). + let hdr = pool.slotHeaderPtr(idx) + hdr.overflow = buf + hdr.overflowLen = len + hdr.payloadSize = 0 + hdr.state.store(uint8(ResponseState.Ready), moRelease) + +proc respDataPtr*( + pool: ResponseSlotPool, idx: uint32 +): ptr UncheckedArray[byte] {.gcsafe.} = + ## Pointer to the marshaled response bytes — spill buffer when spilled, else + ## the inline slot payload region. + let hdr = pool.slotHeaderPtr(idx) + if not hdr.overflow.isNil: + cast[ptr UncheckedArray[byte]](hdr.overflow) + else: + pool.slotPayloadPtr(idx) + +proc respDataLen*(pool: ResponseSlotPool, idx: uint32): int {.gcsafe.} = + let hdr = pool.slotHeaderPtr(idx) + if not hdr.overflow.isNil: + int(hdr.overflowLen) + else: + int(hdr.payloadSize) + +proc abandon*(pool: ResponseSlotPool, idx: uint32): bool {.gcsafe.} = + ## Requester: CAS Empty→Abandoned. Returns true if abandonment took + ## effect (provider hadn't started writing yet). If false, requester + ## must still wait for state==Ready and consume normally — provider + ## is mid-write or already done. + let hdr = pool.slotHeaderPtr(idx) + var expected = uint8(ResponseState.Empty) + hdr.state.compareExchange( + expected, uint8(ResponseState.Abandoned), moAcquireRelease, moAcquire + ) + +proc readyState*(pool: ResponseSlotPool, idx: uint32): bool {.gcsafe.} = + pool.slotHeaderPtr(idx).state.load(moAcquire) == uint8(ResponseState.Ready) + +proc payloadSize*(pool: ResponseSlotPool, idx: uint32): uint32 {.gcsafe.} = + pool.slotHeaderPtr(idx).payloadSize diff --git a/wasm-deps/brokers/brokers/internal/mt_request_broker.nim b/wasm-deps/brokers/brokers/internal/mt_request_broker.nim new file mode 100644 index 000000000..ddcfb771f --- /dev/null +++ b/wasm-deps/brokers/brokers/internal/mt_request_broker.nim @@ -0,0 +1,1660 @@ +## Multi-Thread RequestBroker +## -------------------------- +## Generates a multi-thread capable RequestBroker where the provider runs +## on the thread that called `setProvider` (which must keep its chronos +## event loop running), and requests from other threads are routed via +## a lock-free Vyukov MPSC ring + per-bucket payload slab + response slot +## pool. +## +## Same-thread requests bypass the ring and call the provider directly. +## +## See `doc/REFACTOR_MT_QUEUE.md` for the full design; this file is the +## RequestBroker integration of Phase 4 of that plan. +## +## §2.6 safety contract honored by construction (Invariant I0): +## - The bucket-owning thread (the provider thread, the one that +## called `setProvider`) allocates its ring + request slab + +## response slot pool via `createShared`, and frees them via +## `clearProvider` on the same thread. +## - Sender threads only ever touch atomics + memcpy + signal-fire on +## the hot path — never the Nim allocator beyond `claim`/`release` +## of pre-allocated slab cells and response slots. + +{.push raises: [].} + +import std/[macros, strutils, locks, os, atomics] +import chronos, chronicles +import results +import ./helper/broker_utils, ../broker_context + +import ./mt_broker_common, ./mt_queue, ./mt_codec, ./mt_config +import ./broker_debug +export results, chronos, chronicles, broker_context, mt_broker_common, mt_config + +# Capacity defaults moved to `mt_config.nim` and re-exported via the +# `mt_config` module so existing references to `DefaultMtReq*` constants +# continue to resolve. + +# --------------------------------------------------------------------------- +# Macro code generator +# --------------------------------------------------------------------------- + +proc isAsyncReturnTypeValid(returnType, typeIdent: NimNode): bool = + if returnType.kind != nnkBracketExpr or returnType.len != 2: + return false + if returnType[0].kind != nnkIdent or not returnType[0].eqIdent("Future"): + return false + let inner = returnType[1] + if inner.kind != nnkBracketExpr or inner.len != 3: + return false + if inner[0].kind != nnkIdent or not inner[0].eqIdent("Result"): + return false + if inner[1].kind != nnkIdent or not inner[1].eqIdent($typeIdent): + return false + inner[2].kind == nnkIdent and inner[2].eqIdent("string") + +proc generateMtRequestBroker*( + body: NimNode, cfgIn: MtReqCfg = defaultMtReqCfg() +): NimNode = + when defined(brokerDebug): + echo body.treeRepr + echo "RequestBroker mode: mt" + + # Classify legacy (`proc signature*`) vs proc-sugar (lowercase verb procs). + # Mirrors request_broker.nim; MT is always async. + var hasSignatureProc = false + var hasOtherProc = false + for stmt in body: + if stmt.kind == nnkProcDef: + let nm = stmt[0] + let nmId = (if nm.kind == nnkPostfix: nm[1] else: nm) + if ($nmId).startsWith("signature"): + hasSignatureProc = true + else: + hasOtherProc = true + let isSugar = hasOtherProc and not hasSignatureProc + + var typeIdent: NimNode = nil + var objectDef: NimNode = nil + var payloadType: NimNode = nil + var responseFieldTypes: seq[NimNode] = @[] + var zeroArgSig: NimNode = nil + var zeroArgProviderName: NimNode = nil + var argSig: NimNode = nil + var argParams: seq[NimNode] = @[] + var argProviderName: NimNode = nil + + if not isSugar: + let parsed = parseSingleTypeDef( + body, "RequestBroker", allowRefToNonObject = true, collectFieldInfo = true + ) + typeIdent = parsed.typeIdent + objectDef = parsed.objectDef + responseFieldTypes = parsed.fieldTypes + payloadType = copyNimTree(typeIdent) # legacy: dispatch tag == payload + + for stmt in body: + case stmt.kind + of nnkProcDef: + let procName = stmt[0] + let procNameIdent = + case procName.kind + of nnkIdent: + procName + of nnkPostfix: + procName[1] + else: + procName + if not ($procNameIdent).startsWith("signature"): + error("Signature proc names must start with `signature`", procName) + let params = stmt.params + if params.len == 0: + error("Signature must declare a return type", stmt) + let returnType = params[0] + if not isAsyncReturnTypeValid(returnType, typeIdent): + error( + "MT RequestBroker signature must return Future[Result[`" & $typeIdent & + "`, string]]", + stmt, + ) + let paramCount = params.len - 1 + if paramCount == 0: + if zeroArgSig != nil: + error("Only one zero-argument signature is allowed", stmt) + zeroArgSig = stmt + zeroArgProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderNoArgs") + elif paramCount >= 1: + if argSig != nil: + error("Only one argument-based signature is allowed", stmt) + argSig = stmt + argParams = @[] + for idx in 1 ..< params.len: + let paramDef = params[idx] + if paramDef.kind != nnkIdentDefs: + error( + "Signature parameter must be a standard identifier declaration", + paramDef, + ) + let paramTypeNode = paramDef[paramDef.len - 2] + if paramTypeNode.kind == nnkEmpty: + error("Signature parameter must declare a type", paramDef) + argParams.add(copyNimTree(paramDef)) + argProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderWithArgs") + of nnkTypeSection, nnkEmpty: + discard + else: + error("Unsupported statement inside RequestBroker definition", stmt) + + if zeroArgSig.isNil() and argSig.isNil(): + zeroArgSig = newEmptyNode() + zeroArgProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderNoArgs") + else: + # ---- New proc-sugar form (option B / decoupled payload) ---- + let sg = parseRequestSugar(body, "RequestBroker", async = true) + typeIdent = sg.typeIdent + objectDef = sg.objectDef + payloadType = sg.payloadType + responseFieldTypes = sg.fieldTypes + if not sg.zeroArgProc.isNil: + zeroArgSig = sg.zeroArgProc + zeroArgProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderNoArgs") + if not sg.argProc.isNil: + argSig = sg.argProc + argParams = sg.argParams + argProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderWithArgs") + + let exportedTypeIdent = postfix(copyNimTree(typeIdent), "*") + let typeDisplayName = sanitizeIdentName(typeIdent) + let typeNameLit = newLit(typeDisplayName) + + let returnType = quote: + Future[Result[`payloadType`, string]] + + # ── Type-driven auto-defaults ─────────────────────────────────────── + # Void / zero-field response and zero-arg signatures collapse to the + # scalar bucket: nothing larger than the Result envelope's tag bytes + # ever traverses the wire. Leaving the conservative 64 KB response / + # 1 KB payload default in place would otherwise pin a 16 MB response + # pool per RequestBroker for what is effectively a notification. + var cfg = cfgIn + if cfg.maxResponseBytesOrigin == "default": + if responseFieldTypes.len > 0: + let cls = classifyFieldsMax(responseFieldTypes) + cfg.maxResponseBytes = cls.bytes + cfg.maxResponseBytesOrigin = "auto:" & cls.reason + if cls.reason.startsWith("unclassifiable"): + warning( + "[brokers] RequestBroker(" & typeDisplayName & + ") could not auto-size response (" & cls.reason & "); falling back to " & + $cls.bytes & " B. Override with `maxResponseBytes = N`." + ) + else: + cfg.maxResponseBytes = ScalarBytes + cfg.maxResponseBytesOrigin = "auto:void" + if cfg.maxPayloadBytesOrigin == "default": + if argParams.len > 0: + var argTypes = newSeqOfCap[NimNode](argParams.len) + for p in argParams: + argTypes.add(p[p.len - 2]) + let cls = classifyFieldsMax(argTypes) + cfg.maxPayloadBytes = cls.bytes + cfg.maxPayloadBytesOrigin = "auto:" & cls.reason + if cls.reason.startsWith("unclassifiable"): + warning( + "[brokers] RequestBroker(" & typeDisplayName & + ") could not auto-size request payload (" & cls.reason & + "); falling back to " & $cls.bytes & + " B. Override with `maxPayloadBytes = N`." + ) + else: + cfg.maxPayloadBytes = ScalarBytes + cfg.maxPayloadBytesOrigin = "auto:void" + + when not defined(brokerConfigSilent): + hint(fmtReqCfgSummary(typeDisplayName, cfg)) + + # ── Identifier setup ──────────────────────────────────────────────── + let requestMsgName = ident(typeDisplayName & "MtRequestMsg") + let bucketName = ident(typeDisplayName & "MtBucket") + + let globalBucketsIdent = ident("g" & typeDisplayName & "MtBuckets") + let globalBucketCountIdent = ident("g" & typeDisplayName & "MtBucketCount") + let globalBucketCapIdent = ident("g" & typeDisplayName & "MtBucketCap") + let globalLockIdent = ident("g" & typeDisplayName & "MtLock") + let globalInitIdent = ident("g" & typeDisplayName & "MtInit") + let timeoutVarIdent = ident("g" & typeDisplayName & "MtTimeout") + + let initProcIdent = ident("ensureInit" & typeDisplayName & "MtBroker") + let growProcIdent = ident("grow" & typeDisplayName & "MtBuckets") + let sendReplyIdent = ident("sendReply" & typeDisplayName) + let handleMsgIdent = ident("handleMsg" & typeDisplayName) + let pollFnMakerIdent = ident("makePollFn" & typeDisplayName) + let shardHintIdent = ident("shardHint" & typeDisplayName) + let marshalIdent = ident(typeDisplayName & "MtMarshal") + let unmarshalIdent = ident(typeDisplayName & "MtUnmarshal") + let marshalSizeIdent = ident(typeDisplayName & "MtMarshalSize") + let marshalRespIdent = ident(typeDisplayName & "MtMarshalResp") + let unmarshalRespIdent = ident(typeDisplayName & "MtUnmarshalResp") + let marshalRespSizeIdent = ident(typeDisplayName & "MtMarshalRespSize") + + let queueDepthLit = newLit(cfg.queueDepth) + let slabCapacityLit = newLit(cfg.slabCapacity) + let payloadBytesLit = newLit(cfg.maxPayloadBytes) + let maxDynPayloadLit = newLit(cfg.maxDynamicPayloadBytes) + let responseSlotsLit = newLit(cfg.responseSlots) + let responseBytesLit = newLit(cfg.maxResponseBytes) + let freeListShardsLit = newLit(uint32(cfg.freeListShards)) + + result = newStmtList() + + # ── Type section (typeIdent + provider proc types) ─────────────────── + var typeSection = newTree(nnkTypeSection) + typeSection.add(newTree(nnkTypeDef, exportedTypeIdent, newEmptyNode(), objectDef)) + + proc makeProcType(returnType: NimNode, params: seq[NimNode]): NimNode = + var formal = newTree(nnkFormalParams) + formal.add(returnType) + for param in params: + formal.add(param) + let pragmas = newTree(nnkPragma, ident("async")) + newTree(nnkProcTy, formal, pragmas) + + if not zeroArgSig.isNil(): + let procType = makeProcType(returnType, @[]) + typeSection.add(newTree(nnkTypeDef, zeroArgProviderName, newEmptyNode(), procType)) + if not argSig.isNil(): + let procType = makeProcType(returnType, cloneParams(argParams)) + typeSection.add(newTree(nnkTypeDef, argProviderName, newEmptyNode(), procType)) + + # Request message struct. Carries args inline plus a response-slot + # index (the per-bucket pool index where the provider writes the + # result) and the requester's signal pointer (so the provider can + # wake the requester's dispatcher after writing). + var msgRecList = newTree(nnkRecList) + msgRecList.add( + newTree(nnkIdentDefs, ident("requestKind"), ident("int"), newEmptyNode()) + ) + if not argSig.isNil(): + for paramDef in argParams: + for i in 0 ..< paramDef.len - 2: + let nameNode = paramDef[i] + if nameNode.kind != nnkEmpty: + let typeNode = paramDef[paramDef.len - 2] + msgRecList.add( + newTree( + nnkIdentDefs, ident($nameNode), copyNimTree(typeNode), newEmptyNode() + ) + ) + msgRecList.add( + newTree(nnkIdentDefs, ident("responseSlotIdx"), ident("uint32"), newEmptyNode()) + ) + msgRecList.add( + newTree( + nnkIdentDefs, ident("requesterSignal"), ident("ThreadSignalPtr"), newEmptyNode() + ) + ) + typeSection.add( + newTree( + nnkTypeDef, + requestMsgName, + newEmptyNode(), + newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), msgRecList), + ) + ) + + # Bucket struct. + let responseSlotPoolType = quote: + ptr ResponseSlotPool + let requestRingType = quote: + ptr VyukovMpscRing[uint32] + let requestSlabType = quote: + ptr PayloadSlab + + var bucketRecList = newTree(nnkRecList) + bucketRecList.add( + newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode()) + ) + bucketRecList.add( + newTree(nnkIdentDefs, ident("ring"), requestRingType, newEmptyNode()) + ) + bucketRecList.add( + newTree(nnkIdentDefs, ident("slab"), requestSlabType, newEmptyNode()) + ) + bucketRecList.add( + newTree( + nnkIdentDefs, ident("responseSlotPool"), responseSlotPoolType, newEmptyNode() + ) + ) + bucketRecList.add( + newTree( + nnkIdentDefs, ident("providerSignal"), ident("ThreadSignalPtr"), newEmptyNode() + ) + ) + bucketRecList.add( + newTree(nnkIdentDefs, ident("threadId"), ident("pointer"), newEmptyNode()) + ) + bucketRecList.add( + newTree(nnkIdentDefs, ident("threadGen"), ident("uint64"), newEmptyNode()) + ) + typeSection.add( + newTree( + nnkTypeDef, + bucketName, + newEmptyNode(), + newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), bucketRecList), + ) + ) + + result.add(typeSection) + + # ── Codec procs for ReqMsg ─────────────────────────────────────────── + for procNode in genMtCodecProcs(marshalIdent, unmarshalIdent, requestMsgName): + result.add(procNode) + + # ── Codec procs for Result[typeIdent, string] ─────────────────────── + # Custom — `Result` is a case object on `oResultPrivate`; the generic + # `fieldPairs`-based marshaler would touch the wrong-tag fields. We + # encode an explicit `isOk` byte followed by either the value (T) or + # the error (string), recursively via mtMarshalValue/Unmarshal. + result.add( + quote do: + proc `marshalRespIdent`( + buf: ptr UncheckedArray[byte], cap: int, res: Result[`payloadType`, string] + ): int {.gcsafe, raises: [].} = + var pos = 0 + if pos + 1 > cap: + return -1 + let isOk = byte(if res.isOk: 1 else: 0) + buf[pos] = isOk + pos += 1 + if res.isOk: + when not (`payloadType` is void): + let val = res.value + if not mtMarshalValue(buf, cap, val, pos): + return -1 + else: + let errMsg = res.error + if not mtMarshalValue(buf, cap, errMsg, pos): + return -1 + return pos + + proc `unmarshalRespIdent`( + buf: ptr UncheckedArray[byte], + len: int, + dst: var Result[`payloadType`, string], + ): bool {.gcsafe, raises: [].} = + var pos = 0 + if pos + 1 > len: + return false + let isOk = buf[pos] + pos += 1 + if isOk == 1'u8: + when (`payloadType` is void): + dst.ok() + else: + var val: `payloadType` + if not mtUnmarshalValue(buf, len, val, pos): + return false + dst = ok(Result[`payloadType`, string], val) + else: + var errMsg: string + if not mtUnmarshalValue(buf, len, errMsg, pos): + return false + dst = err(Result[`payloadType`, string], errMsg) + return true + + proc `marshalRespSizeIdent`( + res: Result[`payloadType`, string] + ): int {.gcsafe, raises: [].} = + ## Exact marshaled byte length of a response — mirrors `marshalRespIdent` + ## (1 isOk byte + value-or-error). Used to size a heap-spill buffer. + result = 1 + if res.isOk: + when not (`payloadType` is void): + result += mtMarshalSizeValue(res.value) + else: + result += mtMarshalSizeValue(res.error) + + ) + + # ── Global state ──────────────────────────────────────────────────── + result.add( + quote do: + var `globalBucketsIdent`: ptr UncheckedArray[`bucketName`] + var `globalBucketCountIdent`: int + var `globalBucketCapIdent`: int + var `globalLockIdent`: Lock + var `globalInitIdent`: Atomic[int] + ) + + # ── Timeout knob (per broker type) ────────────────────────────────── + result.add( + quote do: + var `timeoutVarIdent`*: Duration = chronos.seconds(5) + ## Default timeout for cross-thread requests. + + proc setRequestTimeout*(_: typedesc[`typeIdent`], timeout: Duration) = + `timeoutVarIdent` = timeout + + proc requestTimeout*(_: typedesc[`typeIdent`]): Duration = + `timeoutVarIdent` + + ) + + # ── Init + grow ────────────────────────────────────────────────────── + result.add( + quote do: + proc `initProcIdent`() = + if `globalInitIdent`.load(moRelaxed) == 2: + return + var expected = 0 + if `globalInitIdent`.compareExchange(expected, 1, moAcquire, moRelaxed): + initLock(`globalLockIdent`) + `globalBucketCapIdent` = 4 + `globalBucketsIdent` = cast[ptr UncheckedArray[`bucketName`]](createShared( + `bucketName`, `globalBucketCapIdent` + )) + `globalBucketCountIdent` = 0 + `globalInitIdent`.store(2, moRelease) + else: + while `globalInitIdent`.load(moAcquire) != 2: + discard + + proc `growProcIdent`() = + let newCap = `globalBucketCapIdent` * 2 + let newBuf = + cast[ptr UncheckedArray[`bucketName`]](createShared(`bucketName`, newCap)) + for i in 0 ..< `globalBucketCountIdent`: + newBuf[i] = `globalBucketsIdent`[i] + `globalBucketsIdent` = newBuf + `globalBucketCapIdent` = newCap + + proc `shardHintIdent`(): uint32 {.inline.} = + cast[uint32](cast[uint](currentMtThreadId()) shr 4) + + ) + + # ── Threadvar provider storage ────────────────────────────────────── + var tvNoArgCtxIdent, tvNoArgHandlerIdent: NimNode + if not zeroArgSig.isNil(): + tvNoArgCtxIdent = ident("g" & typeDisplayName & "TvNoArgCtxs") + tvNoArgHandlerIdent = ident("g" & typeDisplayName & "TvNoArgHandlers") + result.add( + quote do: + var `tvNoArgCtxIdent` {.threadvar.}: seq[BrokerContext] + var `tvNoArgHandlerIdent` {.threadvar.}: seq[`zeroArgProviderName`] + ) + + var tvWithArgCtxIdent, tvWithArgHandlerIdent: NimNode + if not argSig.isNil(): + tvWithArgCtxIdent = ident("g" & typeDisplayName & "TvWithArgCtxs") + tvWithArgHandlerIdent = ident("g" & typeDisplayName & "TvWithArgHandlers") + result.add( + quote do: + var `tvWithArgCtxIdent` {.threadvar.}: seq[BrokerContext] + var `tvWithArgHandlerIdent` {.threadvar.}: seq[`argProviderName`] + ) + + # ── sendReply helper (marshals Result into response slot bytes) ────── + # Protocol: + # 1. CAS Empty→Writing via pool.beginWrite. If it fails the + # requester abandoned; release the slot without writing. + # 2. Marshal `resp` into slotPayloadPtr(idx). + # 3. commitWrite (stores size + flips state to Ready, release-ordered). + # 4. Fire requester's signal. + result.add( + quote do: + proc `sendReplyIdent`( + pool: ptr ResponseSlotPool, + slotIdx: uint32, + requesterSignal: ThreadSignalPtr, + resp: Result[`payloadType`, string], + ) {.gcsafe, raises: [].} = + if pool.isNil or slotIdx == EmptyIdx: + return + if not pool[].beginWrite(slotIdx): + # Requester already abandoned — provider owns the release. + pool[].release(slotIdx, `shardHintIdent`()) + return + let payloadPtr = pool[].slotPayloadPtr(slotIdx) + let written = + try: + `marshalRespIdent`(payloadPtr, int(pool[].slotPayloadCap), resp) + except Exception: + -1 + if written >= 0: + pool[].commitWrite(slotIdx, uint32(written)) + else: + # Response exceeded the inline slot — auto-spill onto the heap so the + # full response is delivered instead of replaced by an err. Falls back + # to an err only if the spill itself cannot be sized/allocated. + let needed = + try: + `marshalRespSizeIdent`(resp) + except Exception: + -1 + var spilled = false + if needed >= 0 and needed <= `maxDynPayloadLit`: + let spillBuf = allocShared0(needed) + if not spillBuf.isNil: + let w2 = + try: + `marshalRespIdent`( + cast[ptr UncheckedArray[byte]](spillBuf), needed, resp + ) + except Exception: + -1 + if w2 < 0: + deallocShared(spillBuf) + else: + pool[].commitWriteOverflow(slotIdx, spillBuf, uint32(w2)) + spilled = true + if not spilled: + # Could not spill (over ceiling / OOM / marshal error) — commit a + # compact err so the requester gets a clean failure, not garbage. + let fallback = err( + Result[`payloadType`, string], + "RequestBroker(" & `typeNameLit` & "): response too large to deliver", + ) + let writtenFb = + try: + `marshalRespIdent`(payloadPtr, int(pool[].slotPayloadCap), fallback) + except Exception: + -1 + if writtenFb < 0: + pool[].commitWrite(slotIdx, 0'u32) + else: + pool[].commitWrite(slotIdx, uint32(writtenFb)) + if not requesterSignal.isNil: + fireBrokerSignal(requesterSignal) + + ) + + # ── handleMsg async (provider-side dispatch of a single ReqMsg) ────── + let msgIdent = ident("msg") + let loopCtxIdent = ident("loopCtx") + let poolIdent = ident("pool") + + var handleBody = newStmtList() + + if not zeroArgSig.isNil(): + let handlerIdent0 = ident("handler0") + handleBody.add( + quote do: + if `msgIdent`.requestKind == 0: + var `handlerIdent0`: `zeroArgProviderName` + for i in 0 ..< `tvNoArgCtxIdent`.len: + if `tvNoArgCtxIdent`[i] == `loopCtxIdent`: + `handlerIdent0` = `tvNoArgHandlerIdent`[i] + break + if `handlerIdent0`.isNil(): + `sendReplyIdent`( + `poolIdent`, + `msgIdent`.responseSlotIdx, + `msgIdent`.requesterSignal, + err( + Result[`payloadType`, string], + "RequestBroker(" & `typeNameLit` & "): no zero-arg provider registered", + ), + ) + else: + let catchedRes = catch: + await `handlerIdent0`() + if catchedRes.isErr(): + `sendReplyIdent`( + `poolIdent`, + `msgIdent`.responseSlotIdx, + `msgIdent`.requesterSignal, + err( + Result[`payloadType`, string], + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & + catchedRes.error.msg, + ), + ) + else: + let providerRes = catchedRes.get() + when not (`payloadType` is void): + if providerRes.isOk(): + let resultValue = providerRes.get() + when compiles(resultValue.isNil()) and + not (typeof(resultValue) is string): + if resultValue.isNil(): + `sendReplyIdent`( + `poolIdent`, + `msgIdent`.responseSlotIdx, + `msgIdent`.requesterSignal, + err( + Result[`payloadType`, string], + "RequestBroker(" & `typeNameLit` & + "): provider returned nil result", + ), + ) + return + `sendReplyIdent`( + `poolIdent`, `msgIdent`.responseSlotIdx, `msgIdent`.requesterSignal, + providerRes, + ) + ) + + if not argSig.isNil(): + let argNameIdents = collectParamNames(argParams) + let handlerIdent1 = ident("handler1") + var providerCall = newCall(handlerIdent1) + for argName in argNameIdents: + providerCall.add(newDotExpr(msgIdent, argName)) + + handleBody.add( + quote do: + if `msgIdent`.requestKind == 1: + var `handlerIdent1`: `argProviderName` + for i in 0 ..< `tvWithArgCtxIdent`.len: + if `tvWithArgCtxIdent`[i] == `loopCtxIdent`: + `handlerIdent1` = `tvWithArgHandlerIdent`[i] + break + if `handlerIdent1`.isNil(): + `sendReplyIdent`( + `poolIdent`, + `msgIdent`.responseSlotIdx, + `msgIdent`.requesterSignal, + err( + Result[`payloadType`, string], + "RequestBroker(" & `typeNameLit` & + "): no provider registered for input signature", + ), + ) + else: + let catchedRes = catch: + await `providerCall` + if catchedRes.isErr(): + `sendReplyIdent`( + `poolIdent`, + `msgIdent`.responseSlotIdx, + `msgIdent`.requesterSignal, + err( + Result[`payloadType`, string], + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & + catchedRes.error.msg, + ), + ) + else: + let providerRes = catchedRes.get() + when not (`payloadType` is void): + if providerRes.isOk(): + let resultValue = providerRes.get() + when compiles(resultValue.isNil()) and + not (typeof(resultValue) is string): + if resultValue.isNil(): + `sendReplyIdent`( + `poolIdent`, + `msgIdent`.responseSlotIdx, + `msgIdent`.requesterSignal, + err( + Result[`payloadType`, string], + "RequestBroker(" & `typeNameLit` & + "): provider returned nil result", + ), + ) + return + `sendReplyIdent`( + `poolIdent`, `msgIdent`.responseSlotIdx, `msgIdent`.requesterSignal, + providerRes, + ) + ) + + result.add( + quote do: + proc `handleMsgIdent`( + `msgIdent`: `requestMsgName`, + `loopCtxIdent`: BrokerContext, + `poolIdent`: ptr ResponseSlotPool, + ) {.async: (raises: []).} = + `handleBody` + + ) + + # ── Poll fn maker ──────────────────────────────────────────────────── + # Dequeues cell idx from ring, unmarshals ReqMsg, dispatches. When the + # ring is closed and empty, registers its (ring, slab, pool) triple for + # synchronous deferred free at thread exit (see drainPendingRingFrees). + result.add( + quote do: + proc `pollFnMakerIdent`( + ring: ptr VyukovMpscRing[uint32], + slab: ptr PayloadSlab, + pool: ptr ResponseSlotPool, + loopCtx: BrokerContext, + ): ThreadDispatchPollFn = + let capturedRing = ring + let capturedSlab = slab + let capturedPool = pool + let capturedCtx = loopCtx + return proc(): int {.gcsafe, raises: [].} = + {.cast(gcsafe).}: + var cellIdx: uint32 + if not capturedRing.tryDequeue(cellIdx): + if capturedRing.isClosed(): + # Hand off to the thread-local pending-free registry; the + # processing-thread proc drains it synchronously after + # drainAsyncOps. Doing the free asynchronously here ran the + # refc allocator during shutdown teardown and SEGV'd on + # Linux + macOS ASAN (PR #13, deferredFreeReqRing path). + enqueuePendingRingFree(capturedRing, capturedSlab, capturedPool) + return 2 + return 0 + # Got a cell — unmarshal ReqMsg, dispatch. dataPtr/dataLen resolve + # the heap-spill buffer when the request spilled, else inline. + var msg: `requestMsgName` + let payloadPtr = capturedSlab[].dataPtr(cellIdx) + let payloadLen = capturedSlab[].dataLen(cellIdx) + let ok = + try: + `unmarshalIdent`(payloadPtr, payloadLen, msg) + except Exception: + false + if ok: + asyncSpawn `handleMsgIdent`(msg, capturedCtx, capturedPool) + else: + error "Failed to unmarshal request payload", requestType = `typeNameLit` + # Release the cell back to the slab — the unmarshaled msg + # holds its own copy on this thread's GC heap. + capturedSlab[].release(cellIdx, `shardHintIdent`()) + return 1 + + ) + + # ── setProvider impl helper (reused by 4 public overloads) ─────────── + # Allocates ring + slab + pool on the calling thread, registers the + # bucket, and starts the poller. Returns Result[void, string]. + let setupBucketIdent = ident("setupBucket" & typeDisplayName) + result.add( + quote do: + proc `setupBucketIdent`(brokerCtx: BrokerContext): Result[void, string] = + let myThreadId = currentMtThreadId() + let myThreadGen = currentMtThreadGen() + var ring: ptr VyukovMpscRing[uint32] + var slab: ptr PayloadSlab + var pool: ptr ResponseSlotPool + withLock(`globalLockIdent`): + for i in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[i].brokerCtx == brokerCtx: + if `globalBucketsIdent`[i].threadId == myThreadId and + `globalBucketsIdent`[i].threadGen == myThreadGen: + return ok() # already set on this thread + return err( + "RequestBroker(" & `typeNameLit` & + "): provider already set from another thread" + ) + if `globalBucketCountIdent` >= `globalBucketCapIdent`: + `growProcIdent`() + ring = newVyukovMpscRing[uint32](`queueDepthLit`) + slab = cast[ptr PayloadSlab](createShared(PayloadSlab, 1)) + initPayloadSlab( + slab[], + capacity = uint32(`slabCapacityLit`), + payloadBytes = uint32(`payloadBytesLit`), + nShards = `freeListShardsLit`, + ) + pool = cast[ptr ResponseSlotPool](createShared(ResponseSlotPool, 1)) + initResponseSlotPool( + pool[], + capacity = uint32(`responseSlotsLit`), + maxPayloadBytes = uint32(`responseBytesLit`), + nShards = `freeListShardsLit`, + ) + let providerSig = getOrInitBrokerSignal() + let idx = `globalBucketCountIdent` + `globalBucketsIdent`[idx] = `bucketName`( + brokerCtx: brokerCtx, + ring: ring, + slab: slab, + responseSlotPool: pool, + providerSignal: providerSig, + threadId: myThreadId, + threadGen: myThreadGen, + ) + `globalBucketCountIdent` += 1 + registerBrokerPoller(`pollFnMakerIdent`(ring, slab, pool, brokerCtx)) + ensureBrokerDispatchStarted() + ok() + + ) + + # ── setProvider (zero-arg) ────────────────────────────────────────── + if not zeroArgSig.isNil(): + result.add( + quote do: + proc setProvider*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + handler: `zeroArgProviderName`, + ): Result[void, string] = + `initProcIdent`() + let myThreadGen = currentMtThreadGen() + for i in 0 ..< `tvNoArgCtxIdent`.len: + if `tvNoArgCtxIdent`[i] == brokerCtx: + var isStale = true + withLock(`globalLockIdent`): + for j in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[j].brokerCtx == brokerCtx and + `globalBucketsIdent`[j].threadId == currentMtThreadId() and + `globalBucketsIdent`[j].threadGen == myThreadGen: + isStale = false + break + if isStale: + `tvNoArgCtxIdent`.del(i) + `tvNoArgHandlerIdent`.del(i) + break + else: + return err( + "RequestBroker(" & `typeNameLit` & + "): provider already set for broker context" + ) + `tvNoArgCtxIdent`.add(brokerCtx) + `tvNoArgHandlerIdent`.add(handler) + let r = `setupBucketIdent`(brokerCtx) + if r.isErr(): + `tvNoArgCtxIdent`.setLen(`tvNoArgCtxIdent`.len - 1) + `tvNoArgHandlerIdent`.setLen(`tvNoArgHandlerIdent`.len - 1) + return r + ok() + + proc setProvider*( + _: typedesc[`typeIdent`], handler: `zeroArgProviderName` + ): Result[void, string] = + setProvider(`typeIdent`, DefaultBrokerContext, handler) + + ) + + # ── setProvider (with-args) ───────────────────────────────────────── + if not argSig.isNil(): + result.add( + quote do: + proc setProvider*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + handler: `argProviderName`, + ): Result[void, string] = + `initProcIdent`() + let myThreadGen = currentMtThreadGen() + for i in 0 ..< `tvWithArgCtxIdent`.len: + if `tvWithArgCtxIdent`[i] == brokerCtx: + var isStale = true + withLock(`globalLockIdent`): + for j in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[j].brokerCtx == brokerCtx and + `globalBucketsIdent`[j].threadId == currentMtThreadId() and + `globalBucketsIdent`[j].threadGen == myThreadGen: + isStale = false + break + if isStale: + `tvWithArgCtxIdent`.del(i) + `tvWithArgHandlerIdent`.del(i) + break + else: + return err( + "RequestBroker(" & `typeNameLit` & + "): provider already set for broker context" + ) + `tvWithArgCtxIdent`.add(brokerCtx) + `tvWithArgHandlerIdent`.add(handler) + let r = `setupBucketIdent`(brokerCtx) + if r.isErr(): + `tvWithArgCtxIdent`.setLen(`tvWithArgCtxIdent`.len - 1) + `tvWithArgHandlerIdent`.setLen(`tvWithArgHandlerIdent`.len - 1) + return r + ok() + + proc setProvider*( + _: typedesc[`typeIdent`], handler: `argProviderName` + ): Result[void, string] = + setProvider(`typeIdent`, DefaultBrokerContext, handler) + + ) + + # ── request helper: send and await one ReqMsg cross-thread ────────── + # Returns the response Result or an err on timeout / queue full. + let sendAndAwaitIdent = ident("sendAndAwait" & typeDisplayName) + result.add( + quote do: + proc `sendAndAwaitIdent`( + ring: ptr VyukovMpscRing[uint32], + slab: ptr PayloadSlab, + pool: ptr ResponseSlotPool, + providerSignal: ThreadSignalPtr, + msg: sink `requestMsgName`, + ): Future[Result[`payloadType`, string]] {.async: (raises: []).} = + ensureBrokerDispatchStarted() + let mySignal = getOrInitBrokerSignal() + # Reserve the response slot. + let slotIdx = pool[].claim(`shardHintIdent`()) + if slotIdx == EmptyIdx: + return + err("RequestBroker(" & `typeNameLit` & "): response slot pool exhausted") + # Reserve a slab cell, marshal ReqMsg into it. + let cellIdx = slab[].claim(`shardHintIdent`()) + if cellIdx == EmptyIdx: + pool[].release(slotIdx, `shardHintIdent`()) + return err("RequestBroker(" & `typeNameLit` & "): request slab exhausted") + let cellPtr = slab[].cellPtr(cellIdx) + let payloadPtr = slab[].cellPayloadPtr(cellIdx) + var msgCopy = msg + msgCopy.responseSlotIdx = slotIdx + msgCopy.requesterSignal = mySignal + let written = + try: + `marshalIdent`(payloadPtr, int(slab[].cellPayloadCap), msgCopy) + except Exception: + -1 + if written >= 0: + cellPtr.payloadSize = uint32(written) + else: + # Auto-spill the request onto the heap instead of failing. + let needed = + try: + `marshalSizeIdent`(msgCopy) + except Exception: + -1 + if needed < 0 or needed > `maxDynPayloadLit`: + slab[].release(cellIdx, `shardHintIdent`()) + pool[].release(slotIdx, `shardHintIdent`()) + return err( + "RequestBroker(" & `typeNameLit` & + "): request payload exceeds maxDynamicPayloadBytes" + ) + let spillBuf = allocShared0(needed) + if spillBuf.isNil: + slab[].release(cellIdx, `shardHintIdent`()) + pool[].release(slotIdx, `shardHintIdent`()) + return + err("RequestBroker(" & `typeNameLit` & "): request spill alloc failed") + let w2 = + try: + `marshalIdent`(cast[ptr UncheckedArray[byte]](spillBuf), needed, msgCopy) + except Exception: + -1 + if w2 < 0: + deallocShared(spillBuf) + slab[].release(cellIdx, `shardHintIdent`()) + pool[].release(slotIdx, `shardHintIdent`()) + return err("RequestBroker(" & `typeNameLit` & "): request marshal failed") + slab[].setOverflow(cellIdx, spillBuf, uint32(w2)) + cellPtr.refcount.store(1, moRelease) + if not ring.tryEnqueue(cellIdx): + slab[].release(cellIdx, `shardHintIdent`()) + pool[].release(slotIdx, `shardHintIdent`()) + return err("RequestBroker(" & `typeNameLit` & "): provider queue full") + fireBrokerSignal(providerSignal) + # Register a one-shot response poller for this slot. + let responseFut = + newFuture[Result[`payloadType`, string]]("request." & `typeNameLit`) + let capturedPool = pool + let capturedSlotIdx = slotIdx + let capturedResponseFut = responseFut + registerBrokerPoller( + proc(): int {.gcsafe, raises: [].} = + {.cast(gcsafe).}: + if not capturedPool[].readyState(capturedSlotIdx): + return 0 + # Unmarshal Result from slot bytes on THIS (requester) thread, + # so any string/seq inside lives on this thread's GC heap. + # This is the §2.2 fix: no cross-thread `=copy` of the typed + # Result value. + var decoded: Result[`payloadType`, string] + let payloadPtr = capturedPool[].respDataPtr(capturedSlotIdx) + let payloadSize = capturedPool[].respDataLen(capturedSlotIdx) + let ok = + try: + `unmarshalRespIdent`(payloadPtr, payloadSize, decoded) + except Exception: + false + if not ok: + decoded = err( + Result[`payloadType`, string], + "RequestBroker(" & `typeNameLit` & "): response unmarshal failed", + ) + if not capturedResponseFut.finished: + capturedResponseFut.complete(decoded) + capturedPool[].release(capturedSlotIdx, `shardHintIdent`()) + return 2 + ) + let completedRes = catch: + await withTimeout(responseFut, `timeoutVarIdent`) + if completedRes.isErr(): + responseFut.cancelSoon() + discard capturedPool[].abandon(capturedSlotIdx) + return err( + "RequestBroker(" & `typeNameLit` & "): recv failed: " & + completedRes.error.msg + ) + if not completedRes.get(): + responseFut.cancelSoon() + discard capturedPool[].abandon(capturedSlotIdx) + return err( + "RequestBroker(" & `typeNameLit` & "): cross-thread request timed out after " & + $`timeoutVarIdent` + ) + let recvRes = catch: + responseFut.read() + if recvRes.isErr(): + return err( + "RequestBroker(" & `typeNameLit` & "): recv failed: " & recvRes.error.msg + ) + recvRes.get() + + ) + + # ── blockingRequest helper: same as above but synchronous ──────────── + let blockingSendAndAwaitIdent = ident("blockingSendAndAwait" & typeDisplayName) + result.add( + quote do: + proc `blockingSendAndAwaitIdent`( + ring: ptr VyukovMpscRing[uint32], + slab: ptr PayloadSlab, + pool: ptr ResponseSlotPool, + providerSignal: ThreadSignalPtr, + msg: sink `requestMsgName`, + ): Result[`payloadType`, string] {.gcsafe, raises: [].} = + let slotIdx = pool[].claim(`shardHintIdent`()) + if slotIdx == EmptyIdx: + return + err("RequestBroker(" & `typeNameLit` & "): response slot pool exhausted") + let cellIdx = slab[].claim(`shardHintIdent`()) + if cellIdx == EmptyIdx: + pool[].release(slotIdx, `shardHintIdent`()) + return err("RequestBroker(" & `typeNameLit` & "): request slab exhausted") + let cellPtr = slab[].cellPtr(cellIdx) + let payloadPtr = slab[].cellPayloadPtr(cellIdx) + var msgCopy = msg + msgCopy.responseSlotIdx = slotIdx + msgCopy.requesterSignal = nil # no async loop on this thread + let written = + try: + `marshalIdent`(payloadPtr, int(slab[].cellPayloadCap), msgCopy) + except Exception: + -1 + if written >= 0: + cellPtr.payloadSize = uint32(written) + else: + # Auto-spill the request onto the heap instead of failing. + let needed = + try: + `marshalSizeIdent`(msgCopy) + except Exception: + -1 + if needed < 0 or needed > `maxDynPayloadLit`: + slab[].release(cellIdx, `shardHintIdent`()) + pool[].release(slotIdx, `shardHintIdent`()) + return err( + "RequestBroker(" & `typeNameLit` & + "): request payload exceeds maxDynamicPayloadBytes" + ) + let spillBuf = allocShared0(needed) + if spillBuf.isNil: + slab[].release(cellIdx, `shardHintIdent`()) + pool[].release(slotIdx, `shardHintIdent`()) + return + err("RequestBroker(" & `typeNameLit` & "): request spill alloc failed") + let w2 = + try: + `marshalIdent`(cast[ptr UncheckedArray[byte]](spillBuf), needed, msgCopy) + except Exception: + -1 + if w2 < 0: + deallocShared(spillBuf) + slab[].release(cellIdx, `shardHintIdent`()) + pool[].release(slotIdx, `shardHintIdent`()) + return err("RequestBroker(" & `typeNameLit` & "): request marshal failed") + slab[].setOverflow(cellIdx, spillBuf, uint32(w2)) + cellPtr.refcount.store(1, moRelease) + if not ring.tryEnqueue(cellIdx): + slab[].release(cellIdx, `shardHintIdent`()) + pool[].release(slotIdx, `shardHintIdent`()) + return err("RequestBroker(" & `typeNameLit` & "): provider queue full") + fireBrokerSignal(providerSignal) + # Busy-poll the response slot until ready or timeout. + let deadline = Moment.now() + `timeoutVarIdent` + while Moment.now() < deadline: + if pool[].readyState(slotIdx): + var decoded: Result[`payloadType`, string] + let payloadPtr = pool[].respDataPtr(slotIdx) + let payloadSize = pool[].respDataLen(slotIdx) + let ok = + try: + `unmarshalRespIdent`(payloadPtr, payloadSize, decoded) + except Exception: + false + pool[].release(slotIdx, `shardHintIdent`()) + if ok: + return decoded + return + err("RequestBroker(" & `typeNameLit` & "): response unmarshal failed") + sleep(1) + # Timeout: abandon the slot so a late provider write returns + # the slot to the pool instead of leaving it stranded. + discard pool[].abandon(slotIdx) + return err( + "RequestBroker(" & `typeNameLit` & "): cross-thread request timed out after " & + $`timeoutVarIdent` + ) + + ) + + # ── request (zero-arg) ────────────────────────────────────────────── + if not zeroArgSig.isNil(): + result.add( + quote do: + proc request*( + _: typedesc[`typeIdent`], brokerCtx: BrokerContext + ): Future[Result[`payloadType`, string]] {.async: (raises: []).} = + `initProcIdent`() + var ring: ptr VyukovMpscRing[uint32] + var slab: ptr PayloadSlab + var pool: ptr ResponseSlotPool + var providerSignal: ThreadSignalPtr + var sameThread = false + let myThreadGen = currentMtThreadGen() + withLock(`globalLockIdent`): + for i in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[i].brokerCtx == brokerCtx: + if `globalBucketsIdent`[i].threadId == currentMtThreadId() and + `globalBucketsIdent`[i].threadGen == myThreadGen: + sameThread = true + else: + ring = `globalBucketsIdent`[i].ring + slab = `globalBucketsIdent`[i].slab + pool = `globalBucketsIdent`[i].responseSlotPool + providerSignal = `globalBucketsIdent`[i].providerSignal + break + if sameThread: + var provider: `zeroArgProviderName` + for i in 0 ..< `tvNoArgCtxIdent`.len: + if `tvNoArgCtxIdent`[i] == brokerCtx: + provider = `tvNoArgHandlerIdent`[i] + break + if provider.isNil(): + return err( + "RequestBroker(" & `typeNameLit` & "): no zero-arg provider registered" + ) + let catchedRes = catch: + await provider() + if catchedRes.isErr(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & + catchedRes.error.msg + ) + return catchedRes.get() + if ring.isNil: + return err( + "RequestBroker(" & `typeNameLit` & + "): no zero-arg provider registered for broker context " & $brokerCtx + ) + var msg = `requestMsgName`(requestKind: 0) + return await `sendAndAwaitIdent`(ring, slab, pool, providerSignal, msg) + + proc request*( + _: typedesc[`typeIdent`] + ): Future[Result[`payloadType`, string]] {.async: (raises: []).} = + return await request(`typeIdent`, DefaultBrokerContext) + + ) + else: + result.add( + quote do: + proc request*( + _: typedesc[`typeIdent`] + ): Future[Result[`payloadType`, string]] {.async: (raises: []).} = + return + err("RequestBroker(" & `typeNameLit` & "): no zero-arg provider registered") + + ) + + # ── blockingRequest (zero-arg) ────────────────────────────────────── + if not zeroArgSig.isNil(): + result.add( + quote do: + proc blockingRequest*( + _: typedesc[`typeIdent`], brokerCtx: BrokerContext + ): Result[`payloadType`, string] {.gcsafe, raises: [].} = + `initProcIdent`() + var ring: ptr VyukovMpscRing[uint32] + var slab: ptr PayloadSlab + var pool: ptr ResponseSlotPool + var providerSignal: ThreadSignalPtr + var sameThread = false + let myThreadGen = currentMtThreadGen() + withLock(`globalLockIdent`): + for i in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[i].brokerCtx == brokerCtx: + if `globalBucketsIdent`[i].threadId == currentMtThreadId() and + `globalBucketsIdent`[i].threadGen == myThreadGen: + sameThread = true + else: + ring = `globalBucketsIdent`[i].ring + slab = `globalBucketsIdent`[i].slab + pool = `globalBucketsIdent`[i].responseSlotPool + providerSignal = `globalBucketsIdent`[i].providerSignal + break + if sameThread: + var provider: `zeroArgProviderName` + for i in 0 ..< `tvNoArgCtxIdent`.len: + if `tvNoArgCtxIdent`[i] == brokerCtx: + provider = `tvNoArgHandlerIdent`[i] + break + if provider.isNil(): + return err( + "RequestBroker(" & `typeNameLit` & "): no zero-arg provider registered" + ) + let catchedRes = catch: + blockingAwait(provider()) + if catchedRes.isErr(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & + catchedRes.error.msg + ) + return catchedRes.get() + if ring.isNil: + return err( + "RequestBroker(" & `typeNameLit` & + "): no zero-arg provider registered for broker context " & $brokerCtx + ) + var msg = `requestMsgName`(requestKind: 0) + `blockingSendAndAwaitIdent`(ring, slab, pool, providerSignal, msg) + + proc blockingRequest*( + _: typedesc[`typeIdent`] + ): Result[`payloadType`, string] {.gcsafe, raises: [].} = + blockingRequest(`typeIdent`, DefaultBrokerContext) + + ) + else: + result.add( + quote do: + proc blockingRequest*( + _: typedesc[`typeIdent`] + ): Result[`payloadType`, string] {.gcsafe, raises: [].} = + return + err("RequestBroker(" & `typeNameLit` & "): no zero-arg provider registered") + + ) + + # ── request (with-args) ───────────────────────────────────────────── + if not argSig.isNil(): + let requestParamDefs = cloneParams(argParams) + let argNameIdents = collectParamNames(requestParamDefs) + + # Build the keyed (ctx-explicit) request proc. + let reqPragmas = quote: + {.async: (raises: []).} + let typedescParam = + newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)) + + var keyedFormalParams = newTree(nnkFormalParams) + keyedFormalParams.add(copyNimTree(returnType)) + keyedFormalParams.add( + newTree(nnkIdentDefs, ident("_"), typedescParam, newEmptyNode()) + ) + keyedFormalParams.add( + newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode()) + ) + for paramDef in requestParamDefs: + keyedFormalParams.add(paramDef) + + let providerSym = genSym(nskVar, "provider") + var providerCall = newCall(providerSym) + for argName in argNameIdents: + providerCall.add(argName) + + var msgCtor = newTree(nnkObjConstr, requestMsgName) + msgCtor.add(newTree(nnkExprColonExpr, ident("requestKind"), newLit(1))) + for argName in argNameIdents: + msgCtor.add(newTree(nnkExprColonExpr, argName, argName)) + + let keyedBody = quote: + `initProcIdent`() + var ring: ptr VyukovMpscRing[uint32] + var slab: ptr PayloadSlab + var pool: ptr ResponseSlotPool + var providerSignal: ThreadSignalPtr + var sameThread = false + let myThreadGen = currentMtThreadGen() + withLock(`globalLockIdent`): + for i in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[i].brokerCtx == brokerCtx: + if `globalBucketsIdent`[i].threadId == currentMtThreadId() and + `globalBucketsIdent`[i].threadGen == myThreadGen: + sameThread = true + else: + ring = `globalBucketsIdent`[i].ring + slab = `globalBucketsIdent`[i].slab + pool = `globalBucketsIdent`[i].responseSlotPool + providerSignal = `globalBucketsIdent`[i].providerSignal + break + if sameThread: + var `providerSym`: `argProviderName` + for i in 0 ..< `tvWithArgCtxIdent`.len: + if `tvWithArgCtxIdent`[i] == brokerCtx: + `providerSym` = `tvWithArgHandlerIdent`[i] + break + if `providerSym`.isNil(): + return err( + "RequestBroker(" & `typeNameLit` & + "): no provider registered for input signature" + ) + let catchedRes = catch: + await `providerCall` + if catchedRes.isErr(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & + catchedRes.error.msg + ) + return catchedRes.get() + if ring.isNil: + return err( + "RequestBroker(" & `typeNameLit` & + "): no provider registered for broker context " & $brokerCtx + ) + var msg = `msgCtor` + return await `sendAndAwaitIdent`(ring, slab, pool, providerSignal, msg) + + result.add( + newTree( + nnkProcDef, + postfix(ident("request"), "*"), + newEmptyNode(), + newEmptyNode(), + keyedFormalParams, + reqPragmas, + newEmptyNode(), + keyedBody, + ) + ) + + # Non-keyed forwarder. + var nonKeyedFormalParams = newTree(nnkFormalParams) + nonKeyedFormalParams.add(copyNimTree(returnType)) + nonKeyedFormalParams.add( + newTree(nnkIdentDefs, ident("_"), typedescParam, newEmptyNode()) + ) + for paramDef in cloneParams(argParams): + nonKeyedFormalParams.add(paramDef) + + var forwardCall = newCall(ident("request")) + forwardCall.add(copyNimTree(typeIdent)) + forwardCall.add(ident("DefaultBrokerContext")) + for argName in argNameIdents: + forwardCall.add(argName) + let forwardBody = quote: + return await `forwardCall` + + result.add( + newTree( + nnkProcDef, + postfix(ident("request"), "*"), + newEmptyNode(), + newEmptyNode(), + nonKeyedFormalParams, + reqPragmas, + newEmptyNode(), + forwardBody, + ) + ) + + # ── blockingRequest (with-args) ───────────────────────────────────── + if not argSig.isNil(): + let brParamDefs = cloneParams(argParams) + let brArgNameIdents = collectParamNames(brParamDefs) + let brPragmas = quote: + {.gcsafe, raises: [].} + let typedescParam = + newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)) + + var brKeyedFormalParams = newTree(nnkFormalParams) + brKeyedFormalParams.add( + newTree(nnkBracketExpr, ident("Result"), copyNimTree(typeIdent), ident("string")) + ) + brKeyedFormalParams.add( + newTree(nnkIdentDefs, ident("_"), typedescParam, newEmptyNode()) + ) + brKeyedFormalParams.add( + newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode()) + ) + for paramDef in brParamDefs: + brKeyedFormalParams.add(paramDef) + + let brProviderSym = genSym(nskVar, "provider") + var brProviderCall = newCall(brProviderSym) + for argName in brArgNameIdents: + brProviderCall.add(argName) + + var brMsgCtor = newTree(nnkObjConstr, requestMsgName) + brMsgCtor.add(newTree(nnkExprColonExpr, ident("requestKind"), newLit(1))) + for argName in brArgNameIdents: + brMsgCtor.add(newTree(nnkExprColonExpr, argName, argName)) + + let brKeyedBody = quote: + `initProcIdent`() + var ring: ptr VyukovMpscRing[uint32] + var slab: ptr PayloadSlab + var pool: ptr ResponseSlotPool + var providerSignal: ThreadSignalPtr + var sameThread = false + let myThreadGen = currentMtThreadGen() + withLock(`globalLockIdent`): + for i in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[i].brokerCtx == brokerCtx: + if `globalBucketsIdent`[i].threadId == currentMtThreadId() and + `globalBucketsIdent`[i].threadGen == myThreadGen: + sameThread = true + else: + ring = `globalBucketsIdent`[i].ring + slab = `globalBucketsIdent`[i].slab + pool = `globalBucketsIdent`[i].responseSlotPool + providerSignal = `globalBucketsIdent`[i].providerSignal + break + if sameThread: + var `brProviderSym`: `argProviderName` + for i in 0 ..< `tvWithArgCtxIdent`.len: + if `tvWithArgCtxIdent`[i] == brokerCtx: + `brProviderSym` = `tvWithArgHandlerIdent`[i] + break + if `brProviderSym`.isNil(): + return err( + "RequestBroker(" & `typeNameLit` & + "): no provider registered for input signature" + ) + let catchedRes = catch: + blockingAwait(`brProviderCall`) + if catchedRes.isErr(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & + catchedRes.error.msg + ) + return catchedRes.get() + if ring.isNil: + return err( + "RequestBroker(" & `typeNameLit` & + "): no provider registered for broker context " & $brokerCtx + ) + var msg = `brMsgCtor` + `blockingSendAndAwaitIdent`(ring, slab, pool, providerSignal, msg) + + result.add( + newTree( + nnkProcDef, + postfix(ident("blockingRequest"), "*"), + newEmptyNode(), + newEmptyNode(), + brKeyedFormalParams, + brPragmas, + newEmptyNode(), + brKeyedBody, + ) + ) + + # Non-keyed forwarder. + var brNonKeyedFormalParams = newTree(nnkFormalParams) + brNonKeyedFormalParams.add( + newTree(nnkBracketExpr, ident("Result"), copyNimTree(typeIdent), ident("string")) + ) + brNonKeyedFormalParams.add( + newTree(nnkIdentDefs, ident("_"), typedescParam, newEmptyNode()) + ) + for paramDef in cloneParams(argParams): + brNonKeyedFormalParams.add(paramDef) + + var brForwardCall = newCall(ident("blockingRequest")) + brForwardCall.add(copyNimTree(typeIdent)) + brForwardCall.add(ident("DefaultBrokerContext")) + for argName in brArgNameIdents: + brForwardCall.add(argName) + let brForwardBody = quote: + `brForwardCall` + + result.add( + newTree( + nnkProcDef, + postfix(ident("blockingRequest"), "*"), + newEmptyNode(), + newEmptyNode(), + brNonKeyedFormalParams, + brPragmas, + newEmptyNode(), + brForwardBody, + ) + ) + + # ── clearProvider ─────────────────────────────────────────────────── + let brokerCtxParam = ident("brokerCtx") + var tvCleanup = newStmtList() + if not zeroArgSig.isNil(): + tvCleanup.add( + quote do: + for i in countdown(`tvNoArgCtxIdent`.len - 1, 0): + if `tvNoArgCtxIdent`[i] == `brokerCtxParam`: + `tvNoArgCtxIdent`.del(i) + `tvNoArgHandlerIdent`.del(i) + break + ) + if not argSig.isNil(): + tvCleanup.add( + quote do: + for i in countdown(`tvWithArgCtxIdent`.len - 1, 0): + if `tvWithArgCtxIdent`[i] == `brokerCtxParam`: + `tvWithArgCtxIdent`.del(i) + `tvWithArgHandlerIdent`.del(i) + break + ) + + let clearBody = quote: + `initProcIdent`() + var ring: ptr VyukovMpscRing[uint32] + var providerSignal: ThreadSignalPtr + var isProviderThread = false + let myThreadGen = currentMtThreadGen() + withLock(`globalLockIdent`): + var foundIdx = -1 + for i in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[i].brokerCtx == `brokerCtxParam`: + ring = `globalBucketsIdent`[i].ring + providerSignal = `globalBucketsIdent`[i].providerSignal + isProviderThread = ( + `globalBucketsIdent`[i].threadId == currentMtThreadId() and + `globalBucketsIdent`[i].threadGen == myThreadGen + ) + foundIdx = i + break + if foundIdx >= 0: + for i in foundIdx ..< `globalBucketCountIdent` - 1: + `globalBucketsIdent`[i] = `globalBucketsIdent`[i + 1] + `globalBucketCountIdent` -= 1 + if isProviderThread: + `tvCleanup` + if not ring.isNil: + ring.close() + fireBrokerSignal(providerSignal) + + var formalParamsClear = newTree(nnkFormalParams) + formalParamsClear.add(newEmptyNode()) + formalParamsClear.add( + newTree( + nnkIdentDefs, + ident("_"), + newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)), + newEmptyNode(), + ) + ) + formalParamsClear.add( + newTree(nnkIdentDefs, brokerCtxParam, ident("BrokerContext"), newEmptyNode()) + ) + result.add( + newTree( + nnkProcDef, + postfix(ident("clearProvider"), "*"), + newEmptyNode(), + newEmptyNode(), + formalParamsClear, + newEmptyNode(), + newEmptyNode(), + clearBody, + ) + ) + + result.add( + quote do: + proc clearProvider*(_: typedesc[`typeIdent`]) = + clearProvider(`typeIdent`, DefaultBrokerContext) + + ) + + # ── isProvided ───────────────────────────────────────────────────── + let isProvidedCtxParam = ident("brokerCtx") + let isProvidedBody = quote: + `initProcIdent`() + withLock(`globalLockIdent`): + for i in 0 ..< `globalBucketCountIdent`: + if `globalBucketsIdent`[i].brokerCtx == `isProvidedCtxParam`: + return true + return false + + var formalParamsIsProvided = newTree(nnkFormalParams) + formalParamsIsProvided.add(ident("bool")) + formalParamsIsProvided.add( + newTree( + nnkIdentDefs, + ident("_"), + newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)), + newEmptyNode(), + ) + ) + formalParamsIsProvided.add( + newTree(nnkIdentDefs, isProvidedCtxParam, ident("BrokerContext"), newEmptyNode()) + ) + + result.add( + newTree( + nnkProcDef, + postfix(ident("isProvided"), "*"), + newEmptyNode(), + newEmptyNode(), + formalParamsIsProvided, + newEmptyNode(), + newEmptyNode(), + isProvidedBody, + ) + ) + + result.add( + quote do: + proc isProvided*(_: typedesc[`typeIdent`]): bool = + isProvided(`typeIdent`, DefaultBrokerContext) + + ) + + when defined(brokerDebug): + writeBrokerDebug("RequestBrokerMt", typeDisplayName, result) + when defined(brokerDebugStdout): + echo result.repr + + return result + +{.pop.} diff --git a/wasm-deps/brokers/brokers/multi_request_broker.nim b/wasm-deps/brokers/brokers/multi_request_broker.nim new file mode 100644 index 000000000..d204fe7e1 --- /dev/null +++ b/wasm-deps/brokers/brokers/multi_request_broker.nim @@ -0,0 +1,746 @@ +## MultiRequestBroker +## -------------------- +## MultiRequestBroker represents a proactive decoupling pattern, that +## allows defining request-response style interactions between modules without +## need for direct dependencies in between. +## Worth considering using it for use cases where you need to collect data from multiple providers. +## +## Generates a standalone, type-safe request broker for the declared type. +## The macro exports the value type itself plus a broker companion that manages +## providers via thread-local storage. +## +## Unlike `RequestBroker`, every call to `request` fan-outs to every registered +## provider and returns all collected responses. +## The request succeeds only if all providers succeed, otherwise it fails. +## +## Type definitions: +## - Inline `object` / `ref object` definitions are supported. +## - Native types, aliases, and externally-defined types are also supported. +## In that case, MultiRequestBroker will automatically wrap the declared RHS +## type in `distinct` unless you already used `distinct`. +## This keeps request types unique even when multiple brokers share the same +## underlying base type. +## +## Default vs. context aware use: +## Every generated broker is a thread-local global instance. +## Sometimes you want multiple independent provider sets for the same request +## type within the same thread (e.g. multiple components). For that, you can use +## context-aware MultiRequestBroker. +## +## Context awareness is supported through the `BrokerContext` argument for +## `setProvider`, `request`, `removeProvider`, and `clearProviders`. +## Provider stores are kept separate per broker context. +## +## Default broker context is defined as `DefaultBrokerContext`. If you don't +## need context awareness, you can keep using the interfaces without the context +## argument, which operate on `DefaultBrokerContext`. +## +## Usage: +## +## Declare collectable request data type inside a `MultiRequestBroker` macro, add any number of fields: +## ```nim +## MultiRequestBroker: +## type TypeName = object +## field1*: Type1 +## field2*: Type2 +## +## ## Define the request and provider signature, that is enforced at compile time. +## proc signature*(): Future[Result[TypeName, string]] {.async: (raises: []).} +## +## ## Also possible to define signature with arbitrary input arguments. +## proc signature*(arg1: ArgType, arg2: AnotherArgType): Future[Result[TypeName, string]] {.async: (raises: []).} +## +## ``` +## +## You can register a request processor (provider) anywhere without the need to +## know who will request. +## Register provider functions with `TypeName.setProvider(...)`. +## Providers are async procs or lambdas that return `Future[Result[TypeName, string]]`. +## `setProvider` returns a handle (or an error) that can later be used to remove +## the provider. + +## Requests can be made from anywhere with no direct dependency on the provider(s) +## by calling `TypeName.request()` (with arguments respecting the declared signature). +## This will asynchronously call all registered providers and return the collected +## responses as `Future[Result[seq[TypeName], string]]`. +## +## Whenever you don't want to process requests anymore (or your object instance that provides the request goes out of scope), +## you can remove it from the broker with `TypeName.removeProvider(handle)`. +## Alternatively, you can remove all registered providers through `TypeName.clearProviders()`. +## +## Example: +## ```nim +## MultiRequestBroker: +## type Greeting = object +## text*: string +## +## ## Define the request and provider signature, that is enforced at compile time. +## proc signature*(): Future[Result[Greeting, string]] {.async: (raises: []).} +## +## ## Also possible to define signature with arbitrary input arguments. +## proc signature*(lang: string): Future[Result[Greeting, string]] {.async: (raises: []).} +## +## ... +## let handle = Greeting.setProvider( +## proc(): Future[Result[Greeting, string]] {.async: (raises: []).} = +## ok(Greeting(text: "hello")) +## ) +## +## let anotherHandle = Greeting.setProvider( +## proc(): Future[Result[Greeting, string]] {.async: (raises: []).} = +## ok(Greeting(text: "szia")) +## ) +## +## let responses = (await Greeting.request()).valueOr(@[Greeting(text: "default")]) +## +## echo responses.len +## Greeting.clearProviders() +## ``` +## If no `signature` proc is declared, a zero-argument form is generated +## automatically, so the caller only needs to provide the type definition. + +import std/[macros, strutils, tables, sugar] +import chronos +import results +import ./internal/helper/broker_utils +import ./broker_context +import ./internal/broker_debug + +export results, chronos, broker_context + +proc isReturnTypeValid(returnType, typeIdent: NimNode): bool = + ## Accept Future[Result[TypeIdent, string]] as the contract. + if returnType.kind != nnkBracketExpr or returnType.len != 2: + return false + if returnType[0].kind != nnkIdent or not returnType[0].eqIdent("Future"): + return false + let inner = returnType[1] + if inner.kind != nnkBracketExpr or inner.len != 3: + return false + if inner[0].kind != nnkIdent or not inner[0].eqIdent("Result"): + return false + if inner[1].kind != nnkIdent or not inner[1].eqIdent($typeIdent): + return false + inner[2].kind == nnkIdent and inner[2].eqIdent("string") + +proc makeProcType(returnType: NimNode, params: seq[NimNode]): NimNode = + var formal = newTree(nnkFormalParams) + formal.add(returnType) + for param in params: + formal.add(param) + + let pragmas = quote: + {.async.} + + newTree(nnkProcTy, formal, pragmas) + +macro MultiRequestBroker*(body: untyped): untyped = + when defined(brokerDebug): + echo body.treeRepr + let parsed = parseSingleTypeDef(body, "MultiRequestBroker") + let typeIdent = parsed.typeIdent + let objectDef = parsed.objectDef + let isRefObject = parsed.isRefObject + + when defined(brokerDebug): + echo "MultiRequestBroker generating type: ", $typeIdent + + let exportedTypeIdent = postfix(copyNimTree(typeIdent), "*") + let sanitized = sanitizeIdentName(typeIdent) + let typeNameLit = newLit($typeIdent) + let isRefObjectLit = newLit(isRefObject) + let uint64Ident = ident("uint64") + let providerKindIdent = ident(sanitized & "ProviderKind") + let providerHandleIdent = ident(sanitized & "ProviderHandle") + let exportedProviderHandleIdent = postfix(copyNimTree(providerHandleIdent), "*") + let bucketTypeIdent = ident(sanitized & "CtxBucket") + let findBucketIdxIdent = ident(sanitized & "FindBucketIdx") + let getOrCreateBucketIdxIdent = ident(sanitized & "GetOrCreateBucketIdx") + let zeroKindIdent = ident("pk" & sanitized & "NoArgs") + let argKindIdent = ident("pk" & sanitized & "WithArgs") + var zeroArgSig: NimNode = nil + var zeroArgProviderName: NimNode = nil + var zeroArgFieldName: NimNode = nil + var argSig: NimNode = nil + var argParams: seq[NimNode] = @[] + var argProviderName: NimNode = nil + var argFieldName: NimNode = nil + + for stmt in body: + case stmt.kind + of nnkProcDef: + let procName = stmt[0] + let procNameIdent = + case procName.kind + of nnkIdent: + procName + of nnkPostfix: + procName[1] + else: + procName + let procNameStr = $procNameIdent + if not procNameStr.startsWith("signature"): + error("Signature proc names must start with `signature`", procName) + let params = stmt.params + if params.len == 0: + error("Signature must declare a return type", stmt) + let returnType = params[0] + if not isReturnTypeValid(returnType, typeIdent): + error( + "Signature must return Future[Result[`" & $typeIdent & "`, string]]", stmt + ) + let paramCount = params.len - 1 + if paramCount == 0: + if zeroArgSig != nil: + error("Only one zero-argument signature is allowed", stmt) + zeroArgSig = stmt + zeroArgProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderNoArgs") + zeroArgFieldName = ident("providerNoArgs") + elif paramCount >= 1: + if argSig != nil: + error("Only one argument-based signature is allowed", stmt) + argSig = stmt + argParams = @[] + for idx in 1 ..< params.len: + let paramDef = params[idx] + if paramDef.kind != nnkIdentDefs: + error( + "Signature parameter must be a standard identifier declaration", paramDef + ) + let paramTypeNode = paramDef[paramDef.len - 2] + if paramTypeNode.kind == nnkEmpty: + error("Signature parameter must declare a type", paramDef) + var hasName = false + for i in 0 ..< paramDef.len - 2: + if paramDef[i].kind != nnkEmpty: + hasName = true + if not hasName: + error("Signature parameter must declare a name", paramDef) + argParams.add(copyNimTree(paramDef)) + argProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderWithArgs") + argFieldName = ident("providerWithArgs") + of nnkTypeSection, nnkEmpty: + discard + else: + error("Unsupported statement inside MultiRequestBroker definition", stmt) + + if zeroArgSig.isNil() and argSig.isNil(): + zeroArgSig = newEmptyNode() + zeroArgProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderNoArgs") + zeroArgFieldName = ident("providerNoArgs") + + var typeSection = newTree(nnkTypeSection) + typeSection.add(newTree(nnkTypeDef, exportedTypeIdent, newEmptyNode(), objectDef)) + + var kindEnum = newTree(nnkEnumTy, newEmptyNode()) + if not zeroArgSig.isNil(): + kindEnum.add(zeroKindIdent) + if not argSig.isNil(): + kindEnum.add(argKindIdent) + typeSection.add(newTree(nnkTypeDef, providerKindIdent, newEmptyNode(), kindEnum)) + + var handleRecList = newTree(nnkRecList) + handleRecList.add(newTree(nnkIdentDefs, ident("id"), uint64Ident, newEmptyNode())) + handleRecList.add( + newTree(nnkIdentDefs, ident("kind"), providerKindIdent, newEmptyNode()) + ) + typeSection.add( + newTree( + nnkTypeDef, + exportedProviderHandleIdent, + newEmptyNode(), + newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), handleRecList), + ) + ) + + let returnType = quote: + Future[Result[`typeIdent`, string]] + + if not zeroArgSig.isNil(): + let procType = makeProcType(returnType, @[]) + typeSection.add(newTree(nnkTypeDef, zeroArgProviderName, newEmptyNode(), procType)) + if not argSig.isNil(): + let procType = makeProcType(returnType, cloneParams(argParams)) + typeSection.add(newTree(nnkTypeDef, argProviderName, newEmptyNode(), procType)) + + var bucketRecList = newTree(nnkRecList) + bucketRecList.add( + newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode()) + ) + if not zeroArgSig.isNil(): + bucketRecList.add( + newTree( + nnkIdentDefs, + zeroArgFieldName, + newTree(nnkBracketExpr, ident("seq"), zeroArgProviderName), + newEmptyNode(), + ) + ) + if not argSig.isNil(): + bucketRecList.add( + newTree( + nnkIdentDefs, + argFieldName, + newTree(nnkBracketExpr, ident("seq"), argProviderName), + newEmptyNode(), + ) + ) + typeSection.add( + newTree( + nnkTypeDef, + bucketTypeIdent, + newEmptyNode(), + newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), bucketRecList), + ) + ) + + var brokerRecList = newTree(nnkRecList) + brokerRecList.add( + newTree( + nnkIdentDefs, + ident("buckets"), + newTree(nnkBracketExpr, ident("seq"), bucketTypeIdent), + newEmptyNode(), + ) + ) + let brokerTypeIdent = ident(sanitizeIdentName(typeIdent) & "Broker") + typeSection.add( + newTree( + nnkTypeDef, + brokerTypeIdent, + newEmptyNode(), + newTree( + nnkRefTy, newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), brokerRecList) + ), + ) + ) + result = newStmtList() + result.add(typeSection) + + let globalVarIdent = ident("g" & sanitizeIdentName(typeIdent) & "Broker") + let accessProcIdent = ident("access" & sanitizeIdentName(typeIdent) & "Broker") + result.add( + quote do: + var `globalVarIdent` {.threadvar.}: `brokerTypeIdent` + + proc `findBucketIdxIdent`( + broker: `brokerTypeIdent`, brokerCtx: BrokerContext + ): int = + if brokerCtx == DefaultBrokerContext: + return 0 + for i in 1 ..< broker.buckets.len: + if broker.buckets[i].brokerCtx == brokerCtx: + return i + return -1 + + proc `getOrCreateBucketIdxIdent`( + broker: `brokerTypeIdent`, brokerCtx: BrokerContext + ): int = + let idx = `findBucketIdxIdent`(broker, brokerCtx) + if idx >= 0: + return idx + broker.buckets.add(`bucketTypeIdent`(brokerCtx: brokerCtx)) + return broker.buckets.high + + proc `accessProcIdent`(): `brokerTypeIdent` = + if `globalVarIdent`.isNil(): + new(`globalVarIdent`) + `globalVarIdent`.buckets = + @[`bucketTypeIdent`(brokerCtx: DefaultBrokerContext)] + return `globalVarIdent` + + ) + + var clearBody = newStmtList() + if not zeroArgSig.isNil(): + result.add( + quote do: + proc setProvider*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + handler: `zeroArgProviderName`, + ): Result[`providerHandleIdent`, string] = + if handler.isNil(): + return err("Provider handler must be provided") + let broker = `accessProcIdent`() + let bucketIdx = `getOrCreateBucketIdxIdent`(broker, brokerCtx) + for i, existing in broker.buckets[bucketIdx].`zeroArgFieldName`: + if not existing.isNil() and existing == handler: + return ok(`providerHandleIdent`(id: uint64(i + 1), kind: `zeroKindIdent`)) + broker.buckets[bucketIdx].`zeroArgFieldName`.add(handler) + return ok( + `providerHandleIdent`( + id: uint64(broker.buckets[bucketIdx].`zeroArgFieldName`.len), + kind: `zeroKindIdent`, + ) + ) + + proc setProvider*( + _: typedesc[`typeIdent`], handler: `zeroArgProviderName` + ): Result[`providerHandleIdent`, string] = + return setProvider(`typeIdent`, DefaultBrokerContext, handler) + + ) + result.add( + quote do: + proc request*( + _: typedesc[`typeIdent`], brokerCtx: BrokerContext + ): Future[Result[seq[`typeIdent`], string]] {.async: (raises: []), gcsafe.} = + var aggregated: seq[`typeIdent`] = @[] + let broker = `accessProcIdent`() + let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx) + if bucketIdx < 0: + return ok(aggregated) + let providers = broker.buckets[bucketIdx].`zeroArgFieldName` + if providers.len == 0: + return ok(aggregated) + # var providersFut: seq[Future[Result[`typeIdent`, string]]] = collect: + var providersFut = collect(newSeq): + for provider in providers: + if provider.isNil(): + continue + provider() + + let catchable = catch: + await allFinished(providersFut) + + catchable.isOkOr: + return err("Some provider(s) failed:" & error.msg) + + for fut in catchable.get(): + if fut.failed(): + return err("Some provider(s) failed:" & fut.error.msg) + elif fut.finished(): + let providerResult = fut.value() + if providerResult.isOk: + let providerValue = providerResult.get() + when `isRefObjectLit`: + if providerValue.isNil(): + return err( + "MultiRequestBroker(" & `typeNameLit` & + "): provider returned nil result" + ) + aggregated.add(providerValue) + else: + return err("Some provider(s) failed:" & providerResult.error) + + return ok(aggregated) + + proc request*( + _: typedesc[`typeIdent`] + ): Future[Result[seq[`typeIdent`], string]] = + return request(`typeIdent`, DefaultBrokerContext) + + ) + if not argSig.isNil(): + result.add( + quote do: + proc setProvider*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + handler: `argProviderName`, + ): Result[`providerHandleIdent`, string] = + if handler.isNil(): + return err("Provider handler must be provided") + let broker = `accessProcIdent`() + let bucketIdx = `getOrCreateBucketIdxIdent`(broker, brokerCtx) + for i, existing in broker.buckets[bucketIdx].`argFieldName`: + if not existing.isNil() and existing == handler: + return ok(`providerHandleIdent`(id: uint64(i + 1), kind: `argKindIdent`)) + broker.buckets[bucketIdx].`argFieldName`.add(handler) + return ok( + `providerHandleIdent`( + id: uint64(broker.buckets[bucketIdx].`argFieldName`.len), + kind: `argKindIdent`, + ) + ) + + proc setProvider*( + _: typedesc[`typeIdent`], handler: `argProviderName` + ): Result[`providerHandleIdent`, string] = + return setProvider(`typeIdent`, DefaultBrokerContext, handler) + + ) + let requestParamDefs = cloneParams(argParams) + let argNameIdents = collectParamNames(requestParamDefs) + let providerSym = genSym(nskLet, "providerVal") + var providerCall = newCall(providerSym) + for argName in argNameIdents: + providerCall.add(argName) + var formalParams = newTree(nnkFormalParams) + formalParams.add( + quote do: + Future[Result[seq[`typeIdent`], string]] + ) + formalParams.add( + newTree( + nnkIdentDefs, + ident("_"), + newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)), + newEmptyNode(), + ) + ) + formalParams.add( + newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode()) + ) + for paramDef in requestParamDefs: + formalParams.add(paramDef) + let requestPragmas = quote: + {.async: (raises: []), gcsafe.} + let requestBody = quote: + var aggregated: seq[`typeIdent`] = @[] + let broker = `accessProcIdent`() + let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx) + if bucketIdx < 0: + return ok(aggregated) + let providers = broker.buckets[bucketIdx].`argFieldName` + if providers.len == 0: + return ok(aggregated) + var providersFut = collect(newSeq): + for provider in providers: + if provider.isNil(): + continue + let `providerSym` = provider + `providerCall` + let catchable = catch: + await allFinished(providersFut) + catchable.isOkOr: + return err("Some provider(s) failed:" & error.msg) + for fut in catchable.get(): + if fut.failed(): + return err("Some provider(s) failed:" & fut.error.msg) + elif fut.finished(): + let providerResult = fut.value() + if providerResult.isOk: + let providerValue = providerResult.get() + when `isRefObjectLit`: + if providerValue.isNil(): + return err( + "MultiRequestBroker(" & `typeNameLit` & + "): provider returned nil result" + ) + aggregated.add(providerValue) + else: + return err("Some provider(s) failed:" & providerResult.error) + return ok(aggregated) + + result.add( + newTree( + nnkProcDef, + postfix(ident("request"), "*"), + newEmptyNode(), + newEmptyNode(), + formalParams, + requestPragmas, + newEmptyNode(), + requestBody, + ) + ) + + # Backward-compatible default-context overload (no brokerCtx parameter). + var formalParamsDefault = newTree(nnkFormalParams) + formalParamsDefault.add( + quote do: + Future[Result[seq[`typeIdent`], string]] + ) + formalParamsDefault.add( + newTree( + nnkIdentDefs, + ident("_"), + newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)), + newEmptyNode(), + ) + ) + for paramDef in requestParamDefs: + formalParamsDefault.add(copyNimTree(paramDef)) + + var wrapperCall = newCall(ident("request")) + wrapperCall.add(copyNimTree(typeIdent)) + wrapperCall.add(ident("DefaultBrokerContext")) + for argName in argNameIdents: + wrapperCall.add(copyNimTree(argName)) + + result.add( + newTree( + nnkProcDef, + postfix(ident("request"), "*"), + newEmptyNode(), + newEmptyNode(), + formalParamsDefault, + newEmptyNode(), + newEmptyNode(), + newStmtList(newTree(nnkReturnStmt, wrapperCall)), + ) + ) + let removeHandleCtxSym = genSym(nskParam, "handle") + let removeHandleDefaultSym = genSym(nskParam, "handle") + + when true: + # Generate clearProviders / removeProvider with macro-time knowledge about which + # provider lists exist (zero-arg and/or arg providers). + if not zeroArgSig.isNil() and not argSig.isNil(): + result.add( + quote do: + proc clearProviders*(_: typedesc[`typeIdent`], brokerCtx: BrokerContext) = + let broker = `accessProcIdent`() + if broker.isNil(): + return + let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx) + if bucketIdx < 0: + return + broker.buckets[bucketIdx].`zeroArgFieldName`.setLen(0) + broker.buckets[bucketIdx].`argFieldName`.setLen(0) + if brokerCtx != DefaultBrokerContext: + broker.buckets.delete(bucketIdx) + + proc clearProviders*(_: typedesc[`typeIdent`]) = + clearProviders(`typeIdent`, DefaultBrokerContext) + + proc removeProvider*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + `removeHandleCtxSym`: `providerHandleIdent`, + ) = + if `removeHandleCtxSym`.id == 0'u64: + return + let broker = `accessProcIdent`() + if broker.isNil(): + return + let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx) + if bucketIdx < 0: + return + + if `removeHandleCtxSym`.kind == `zeroKindIdent`: + let idx = int(`removeHandleCtxSym`.id) - 1 + if idx >= 0 and idx < broker.buckets[bucketIdx].`zeroArgFieldName`.len: + broker.buckets[bucketIdx].`zeroArgFieldName`[idx] = nil + elif `removeHandleCtxSym`.kind == `argKindIdent`: + let idx = int(`removeHandleCtxSym`.id) - 1 + if idx >= 0 and idx < broker.buckets[bucketIdx].`argFieldName`.len: + broker.buckets[bucketIdx].`argFieldName`[idx] = nil + + if brokerCtx != DefaultBrokerContext: + var hasAny = false + for p in broker.buckets[bucketIdx].`zeroArgFieldName`: + if not p.isNil(): + hasAny = true + break + if not hasAny: + for p in broker.buckets[bucketIdx].`argFieldName`: + if not p.isNil(): + hasAny = true + break + if not hasAny: + broker.buckets.delete(bucketIdx) + + proc removeProvider*( + _: typedesc[`typeIdent`], `removeHandleDefaultSym`: `providerHandleIdent` + ) = + removeProvider(`typeIdent`, DefaultBrokerContext, `removeHandleDefaultSym`) + + ) + elif not zeroArgSig.isNil(): + result.add( + quote do: + proc clearProviders*(_: typedesc[`typeIdent`], brokerCtx: BrokerContext) = + let broker = `accessProcIdent`() + if broker.isNil(): + return + let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx) + if bucketIdx < 0: + return + broker.buckets[bucketIdx].`zeroArgFieldName`.setLen(0) + if brokerCtx != DefaultBrokerContext: + broker.buckets.delete(bucketIdx) + + proc clearProviders*(_: typedesc[`typeIdent`]) = + clearProviders(`typeIdent`, DefaultBrokerContext) + + proc removeProvider*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + `removeHandleCtxSym`: `providerHandleIdent`, + ) = + if `removeHandleCtxSym`.id == 0'u64: + return + let broker = `accessProcIdent`() + if broker.isNil(): + return + let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx) + if bucketIdx < 0: + return + if `removeHandleCtxSym`.kind != `zeroKindIdent`: + return + let idx = int(`removeHandleCtxSym`.id) - 1 + if idx >= 0 and idx < broker.buckets[bucketIdx].`zeroArgFieldName`.len: + broker.buckets[bucketIdx].`zeroArgFieldName`[idx] = nil + if brokerCtx != DefaultBrokerContext: + var hasAny = false + for p in broker.buckets[bucketIdx].`zeroArgFieldName`: + if not p.isNil(): + hasAny = true + break + if not hasAny: + broker.buckets.delete(bucketIdx) + + proc removeProvider*( + _: typedesc[`typeIdent`], `removeHandleDefaultSym`: `providerHandleIdent` + ) = + removeProvider(`typeIdent`, DefaultBrokerContext, `removeHandleDefaultSym`) + + ) + else: + result.add( + quote do: + proc clearProviders*(_: typedesc[`typeIdent`], brokerCtx: BrokerContext) = + let broker = `accessProcIdent`() + if broker.isNil(): + return + let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx) + if bucketIdx < 0: + return + broker.buckets[bucketIdx].`argFieldName`.setLen(0) + if brokerCtx != DefaultBrokerContext: + broker.buckets.delete(bucketIdx) + + proc clearProviders*(_: typedesc[`typeIdent`]) = + clearProviders(`typeIdent`, DefaultBrokerContext) + + proc removeProvider*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + `removeHandleCtxSym`: `providerHandleIdent`, + ) = + if `removeHandleCtxSym`.id == 0'u64: + return + let broker = `accessProcIdent`() + if broker.isNil(): + return + let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx) + if bucketIdx < 0: + return + if `removeHandleCtxSym`.kind != `argKindIdent`: + return + let idx = int(`removeHandleCtxSym`.id) - 1 + if idx >= 0 and idx < broker.buckets[bucketIdx].`argFieldName`.len: + broker.buckets[bucketIdx].`argFieldName`[idx] = nil + if brokerCtx != DefaultBrokerContext: + var hasAny = false + for p in broker.buckets[bucketIdx].`argFieldName`: + if not p.isNil(): + hasAny = true + break + if not hasAny: + broker.buckets.delete(bucketIdx) + + proc removeProvider*( + _: typedesc[`typeIdent`], `removeHandleDefaultSym`: `providerHandleIdent` + ) = + removeProvider(`typeIdent`, DefaultBrokerContext, `removeHandleDefaultSym`) + + ) + + when defined(brokerDebug): + writeBrokerDebug("MultiRequestBroker", sanitized, result) + when defined(brokerDebugStdout): + echo result.repr diff --git a/wasm-deps/brokers/brokers/request_broker.nim b/wasm-deps/brokers/brokers/request_broker.nim new file mode 100644 index 000000000..d28df3905 --- /dev/null +++ b/wasm-deps/brokers/brokers/request_broker.nim @@ -0,0 +1,1033 @@ +## RequestBroker +## -------------------- +## RequestBroker represents a proactive decoupling pattern, that +## allows defining request-response style interactions between modules without +## need for direct dependencies in between. +## Worth considering using it in a single provider, many requester scenario. +## +## Provides a declarative way to define an immutable value type together with a +## thread-local broker that can register an asynchronous or synchronous provider, +## dispatch typed requests and clear provider. +## +## For consideration use `sync` mode RequestBroker when you need to provide simple value(s) +## where there is no long-running async operation involved. +## Typically it act as a accessor for the local state of generic setting. +## +## `async` mode is better to be used when you request date that may involve some long IO operation +## or action. +## +## Default vs. context aware use: +## Every generated broker is a thread-local global instance. This means each RequestBroker enables decoupled +## data exchange threadwise. Sometimes we use brokers inside a context - like inside a component that has many modules or subsystems. +## In case you would instantiate multiple such components in a single thread, and each component must has its own provider for the same RequestBroker type, +## in order to avoid provider collision, you can use context aware RequestBroker. +## Context awareness is supported through the `BrokerContext` argument for `setProvider`, `request`, `clearProvider` interfaces. +## Suce use requires generating a new unique `BrokerContext` value per component instance, and spread it to all modules using the brokers. +## Example, store the `BrokerContext` as a field inside the top level component instance, and spread around at initialization of the subcomponents.. +## +## Default broker context is defined as `DefaultBrokerContext` constant. But if you don't need context awareness, you can use the +## interfaces without context argument. +## +## Usage: +## Declare your desired request type inside a `RequestBroker` macro, add any number of fields. +## Define the provider signature, that is enforced at compile time. +## +## ```nim +## RequestBroker: +## type TypeName = object +## field1*: FieldType +## field2*: AnotherFieldType +## +## proc signature*(): Future[Result[TypeName, string]] +## ## Also possible to define signature with arbitrary input arguments. +## proc signature*(arg1: ArgType, arg2: AnotherArgType): Future[Result[TypeName, string]] +## +## ``` +## +## Sync mode (no `async` / `Future`) can be generated with: +## +## ```nim +## RequestBroker(sync): +## type TypeName = object +## field1*: FieldType +## +## proc signature*(): Result[TypeName, string] +## proc signature*(arg1: ArgType): Result[TypeName, string] +## ``` +## +## Note: When the request type is declared as a native type / alias / externally-defined +## type (i.e. not an inline `object` / `ref object` definition), RequestBroker +## will wrap it in `distinct` automatically unless you already used `distinct`. +## This avoids overload ambiguity when multiple brokers share the same +## underlying base type (Nim overload resolution does not consider return type). +## +## This means that for non-object request types you typically: +## - construct values with an explicit cast/constructor, e.g. `MyType("x")` +## - unwrap with a cast when needed, e.g. `string(myVal)` or `BaseType(myVal)` +## +## Example (native response type): +## ```nim +## RequestBroker(sync): +## type MyCount = int # exported as: `distinct int` +## +## MyCount.setProvider(proc(): Result[MyCount, string] = ok(MyCount(42))) +## let res = MyCount.request() +## if res.isOk(): +## let raw = int(res.get()) +## ``` +## +## Example (externally-defined type): +## ```nim +## type External = object +## label*: string +## +## RequestBroker: +## type MyExternal = External # exported as: `distinct External` +## +## MyExternal.setProvider( +## proc(): Future[Result[MyExternal, string]] {.async.} = +## ok(MyExternal(External(label: "hi"))) +## ) +## let res = await MyExternal.request() +## if res.isOk(): +## let base = External(res.get()) +## echo base.label +## ``` +## The 'TypeName' object defines the requestable data (but also can be seen as request for action with return value). +## The 'signature' proc defines the provider(s) signature, that is enforced at compile time. +## One signature can be with no arguments, another with any number of arguments - where the input arguments are +## not related to the request type - but alternative inputs for the request to be processed. +## +## After this, you can register a provider anywhere in your code with +## `TypeName.setProvider(...)`, which returns error if already having a provider. +## Providers are async procs/lambdas in default mode and sync procs in sync mode. +## +## Providers are stored as a broker-context keyed list: +## - the default provider is always stored at index 0 (reserved broker context: 0) +## - additional providers can be registered under arbitrary non-zero broker contexts +## +## The original `setProvider(handler)` / `request(...)` APIs continue to operate +## on the default provider (broker context 0) for backward compatibility. +## +## Requests can be made from anywhere with no direct dependency on the provider by +## calling `TypeName.request()` - with arguments respecting the signature(s). +## In async mode, this returns a Future[Result[TypeName, string]]. In sync mode, it returns Result[TypeName, string]. +## +## Whenever you no want to process requests (or your object instance that provides the request goes out of scope), +## you can remove it from the broker with `TypeName.clearProvider()`. +## +## +## Example: +## ```nim +## RequestBroker: +## type Greeting = object +## text*: string +## +## ## Define the request and provider signature, that is enforced at compile time. +## proc signature*(): Future[Result[Greeting, string]] {.async.} +## +## ## Also possible to define signature with arbitrary input arguments. +## proc signature*(lang: string): Future[Result[Greeting, string]] {.async.} +## +## ... +## Greeting.setProvider( +## proc(): Future[Result[Greeting, string]] {.async.} = +## ok(Greeting(text: "hello")) +## ) +## let res = await Greeting.request() +## +## +## ... +## # using native type as response for a synchronous request. +## RequestBroker(sync): +## type NeedThatInfo = string +## +##... +## NeedThatInfo.setProvider( +## proc(): Result[NeedThatInfo, string] = +## ok("this is the info you wanted") +## ) +## let res = NeedThatInfo.request().valueOr: +## echo "not ok due to: " & error +## NeedThatInfo(":-(") +## +## echo string(res) +## ``` +## If no `signature` proc is declared, a zero-argument form is generated +## automatically, so the caller only needs to provide the type definition. + +import std/[macros, strutils] +from std/sequtils import keepItIf +import chronos +import results +import ./internal/helper/broker_utils, ./broker_context +import ./internal/broker_debug + +when compileOption("threads"): + import ./internal/mt_config, ./internal/mt_request_broker + export mt_config, mt_request_broker + +when compileOption("threads") and defined(BrokerFfiApi): + # CBOR is the only FFI codegen strategy. The historical + # `-d:BrokerFfiApiNative` / `-d:BrokerFfiApiCBOR` flags were retired + # in favour of a single `-d:BrokerFfiApi`. + import ./internal/api_request_broker_cbor + export api_request_broker_cbor + +export results, chronos, keepItIf, broker_context + +proc errorFuture[T](message: string): Future[Result[T, string]] {.inline.} = + ## Build a future that is already completed with an error result. + let fut = newFuture[Result[T, string]]("request_broker.errorFuture") + fut.complete(err(Result[T, string], message)) + fut + +type RequestBrokerMode = enum + rbAsync + rbSync + rbMultiThread + rbApi + +proc isAsyncReturnTypeValid(returnType, typeIdent: NimNode): bool = + ## Accept Future[Result[TypeIdent, string]] as the contract. + if returnType.kind != nnkBracketExpr or returnType.len != 2: + return false + if returnType[0].kind != nnkIdent or not returnType[0].eqIdent("Future"): + return false + let inner = returnType[1] + if inner.kind != nnkBracketExpr or inner.len != 3: + return false + if inner[0].kind != nnkIdent or not inner[0].eqIdent("Result"): + return false + if inner[1].kind != nnkIdent or not inner[1].eqIdent($typeIdent): + return false + inner[2].kind == nnkIdent and inner[2].eqIdent("string") + +proc isSyncReturnTypeValid(returnType, typeIdent: NimNode): bool = + ## Accept Result[TypeIdent, string] as the contract. + if returnType.kind != nnkBracketExpr or returnType.len != 3: + return false + if returnType[0].kind != nnkIdent or not returnType[0].eqIdent("Result"): + return false + if returnType[1].kind != nnkIdent or not returnType[1].eqIdent($typeIdent): + return false + returnType[2].kind == nnkIdent and returnType[2].eqIdent("string") + +proc isReturnTypeValid(returnType, typeIdent: NimNode, mode: RequestBrokerMode): bool = + case mode + of rbAsync, rbMultiThread, rbApi: + isAsyncReturnTypeValid(returnType, typeIdent) + of rbSync: + isSyncReturnTypeValid(returnType, typeIdent) + +proc makeProcType( + returnType: NimNode, params: seq[NimNode], mode: RequestBrokerMode +): NimNode = + var formal = newTree(nnkFormalParams) + formal.add(returnType) + for param in params: + formal.add(param) + case mode + of rbAsync, rbMultiThread, rbApi: + let pragmas = newTree(nnkPragma, ident("async")) + newTree(nnkProcTy, formal, pragmas) + of rbSync: + let raisesPragma = newTree( + nnkExprColonExpr, ident("raises"), newTree(nnkBracket, ident("CatchableError")) + ) + let pragmas = newTree(nnkPragma, raisesPragma, ident("gcsafe")) + newTree(nnkProcTy, formal, pragmas) + +proc parseMode(modeNode: NimNode): RequestBrokerMode = + ## Parses the mode selector for the 2-argument macro overload. + ## Supported spellings: `sync` / `async` / `mt` / `API` (case-insensitive). + let raw = ($modeNode).strip().toLowerAscii() + case raw + of "sync": + rbSync + of "async": + rbAsync + of "mt": + rbMultiThread + of "api": + rbApi + else: + error("RequestBroker mode must be `sync`, `async`, `mt` or `API`", modeNode) + +proc generateRequestBroker(body: NimNode, mode: RequestBrokerMode): NimNode = + when defined(brokerDebug): + echo body.treeRepr + echo "RequestBroker mode: ", $mode + # Classify: legacy uses `proc signature*` procs (Ok type == broker type); + # the new proc-sugar uses lowercase verb procs paired to a Capitalized payload + # type by name, with the payload decoupled from the dispatch tag (option B). + # A body with no procs at all is legacy (zero-arg default). + var hasSignatureProc = false + var hasOtherProc = false + for stmt in body: + if stmt.kind == nnkProcDef: + let nm = stmt[0] + let nmId = (if nm.kind == nnkPostfix: nm[1] else: nm) + if ($nmId).startsWith("signature"): + hasSignatureProc = true + else: + hasOtherProc = true + let isSugar = hasOtherProc and not hasSignatureProc + + var typeIdent: NimNode = nil + var objectDef: NimNode = nil + var payloadType: NimNode = nil + var zeroArgSig: NimNode = nil + var zeroArgProviderName: NimNode = nil + var argSig: NimNode = nil + var argParams: seq[NimNode] = @[] + var argProviderName: NimNode = nil + + if not isSugar: + let parsed = parseSingleTypeDef(body, "RequestBroker", allowRefToNonObject = true) + typeIdent = parsed.typeIdent + objectDef = parsed.objectDef + payloadType = copyNimTree(typeIdent) # legacy: dispatch tag == payload + + for stmt in body: + case stmt.kind + of nnkProcDef: + let procName = stmt[0] + let procNameIdent = + case procName.kind + of nnkIdent: + procName + of nnkPostfix: + procName[1] + else: + procName + let procNameStr = $procNameIdent + if not procNameStr.startsWith("signature"): + error("Signature proc names must start with `signature`", procName) + let params = stmt.params + if params.len == 0: + error("Signature must declare a return type", stmt) + let returnType = params[0] + if not isReturnTypeValid(returnType, typeIdent, mode): + case mode + of rbAsync, rbMultiThread, rbApi: + error( + "Signature must return Future[Result[`" & $typeIdent & "`, string]]", stmt + ) + of rbSync: + error("Signature must return Result[`" & $typeIdent & "`, string]", stmt) + let paramCount = params.len - 1 + if paramCount == 0: + if zeroArgSig != nil: + error("Only one zero-argument signature is allowed", stmt) + zeroArgSig = stmt + zeroArgProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderNoArgs") + elif paramCount >= 1: + if argSig != nil: + error("Only one argument-based signature is allowed", stmt) + argSig = stmt + argParams = @[] + for idx in 1 ..< params.len: + let paramDef = params[idx] + if paramDef.kind != nnkIdentDefs: + error( + "Signature parameter must be a standard identifier declaration", + paramDef, + ) + let paramTypeNode = paramDef[paramDef.len - 2] + if paramTypeNode.kind == nnkEmpty: + error("Signature parameter must declare a type", paramDef) + var hasName = false + for i in 0 ..< paramDef.len - 2: + if paramDef[i].kind != nnkEmpty: + hasName = true + if not hasName: + error("Signature parameter must declare a name", paramDef) + argParams.add(copyNimTree(paramDef)) + argProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderWithArgs") + of nnkTypeSection, nnkEmpty: + discard + else: + error("Unsupported statement inside RequestBroker definition", stmt) + + if zeroArgSig.isNil() and argSig.isNil(): + zeroArgSig = newEmptyNode() + zeroArgProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderNoArgs") + else: + # ---- New proc-sugar form (option B / decoupled payload) ---- + let sg = parseRequestSugar(body, "RequestBroker", async = (mode != rbSync)) + typeIdent = sg.typeIdent + objectDef = sg.objectDef + payloadType = sg.payloadType + if not sg.zeroArgProc.isNil: + zeroArgSig = sg.zeroArgProc + zeroArgProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderNoArgs") + if not sg.argProc.isNil: + argSig = sg.argProc + argParams = sg.argParams + argProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderWithArgs") + + when defined(brokerDebug): + echo "RequestBroker generating type: ", $typeIdent + + let exportedTypeIdent = postfix(copyNimTree(typeIdent), "*") + let typeDisplayName = sanitizeIdentName(typeIdent) + let typeNameLit = newLit(typeDisplayName) + + var typeSection = newTree(nnkTypeSection) + typeSection.add(newTree(nnkTypeDef, exportedTypeIdent, newEmptyNode(), objectDef)) + + let returnType = + case mode + of rbAsync, rbMultiThread, rbApi: + quote: + Future[Result[`payloadType`, string]] + of rbSync: + quote: + Result[`payloadType`, string] + + if not zeroArgSig.isNil(): + let procType = makeProcType(returnType, @[], mode) + typeSection.add(newTree(nnkTypeDef, zeroArgProviderName, newEmptyNode(), procType)) + if not argSig.isNil(): + let procType = makeProcType(returnType, cloneParams(argParams), mode) + typeSection.add(newTree(nnkTypeDef, argProviderName, newEmptyNode(), procType)) + + var brokerRecList = newTree(nnkRecList) + if not zeroArgSig.isNil(): + let zeroArgProvidersFieldName = ident("providersNoArgs") + let zeroArgProvidersTupleTy = newTree( + nnkTupleTy, + newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode()), + newTree(nnkIdentDefs, ident("handler"), zeroArgProviderName, newEmptyNode()), + ) + let zeroArgProvidersSeqTy = + newTree(nnkBracketExpr, ident("seq"), zeroArgProvidersTupleTy) + brokerRecList.add( + newTree( + nnkIdentDefs, zeroArgProvidersFieldName, zeroArgProvidersSeqTy, newEmptyNode() + ) + ) + if not argSig.isNil(): + let argProvidersFieldName = ident("providersWithArgs") + let argProvidersTupleTy = newTree( + nnkTupleTy, + newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode()), + newTree(nnkIdentDefs, ident("handler"), argProviderName, newEmptyNode()), + ) + let argProvidersSeqTy = newTree(nnkBracketExpr, ident("seq"), argProvidersTupleTy) + brokerRecList.add( + newTree(nnkIdentDefs, argProvidersFieldName, argProvidersSeqTy, newEmptyNode()) + ) + let brokerTypeIdent = ident(sanitizeIdentName(typeIdent) & "Broker") + let brokerTypeDef = newTree( + nnkTypeDef, + brokerTypeIdent, + newEmptyNode(), + newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), brokerRecList), + ) + typeSection.add(brokerTypeDef) + result = newStmtList() + result.add(typeSection) + + let globalVarIdent = ident("g" & sanitizeIdentName(typeIdent) & "Broker") + let accessProcIdent = ident("access" & sanitizeIdentName(typeIdent) & "Broker") + + var brokerNewBody = newStmtList() + if not zeroArgSig.isNil(): + brokerNewBody.add( + quote do: + result.providersNoArgs = + @[(brokerCtx: DefaultBrokerContext, handler: default(`zeroArgProviderName`))] + ) + if not argSig.isNil(): + brokerNewBody.add( + quote do: + result.providersWithArgs = + @[(brokerCtx: DefaultBrokerContext, handler: default(`argProviderName`))] + ) + + var brokerInitChecks = newStmtList() + if not zeroArgSig.isNil(): + brokerInitChecks.add( + quote do: + if `globalVarIdent`.providersNoArgs.len == 0: + `globalVarIdent` = `brokerTypeIdent`.new() + ) + if not argSig.isNil(): + brokerInitChecks.add( + quote do: + if `globalVarIdent`.providersWithArgs.len == 0: + `globalVarIdent` = `brokerTypeIdent`.new() + ) + + result.add( + quote do: + var `globalVarIdent` {.threadvar.}: `brokerTypeIdent` + + proc new(_: type `brokerTypeIdent`): `brokerTypeIdent` = + result = `brokerTypeIdent`() + `brokerNewBody` + + proc `accessProcIdent`(): var `brokerTypeIdent` = + `brokerInitChecks` + `globalVarIdent` + + ) + + var clearBodyKeyed = newStmtList() + let brokerCtxParamIdent = ident("brokerCtx") + if not zeroArgSig.isNil(): + let zeroArgProvidersFieldName = ident("providersNoArgs") + result.add( + quote do: + proc setProvider*( + _: typedesc[`typeIdent`], handler: `zeroArgProviderName` + ): Result[void, string] = + if not `accessProcIdent`().`zeroArgProvidersFieldName`[0].handler.isNil(): + return err("Zero-arg provider already set") + `accessProcIdent`().`zeroArgProvidersFieldName`[0].handler = handler + return ok() + + ) + + result.add( + quote do: + proc setProvider*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + handler: `zeroArgProviderName`, + ): Result[void, string] = + if brokerCtx == DefaultBrokerContext: + return setProvider(`typeIdent`, handler) + + for entry in `accessProcIdent`().`zeroArgProvidersFieldName`: + if entry.brokerCtx == brokerCtx: + return err( + "RequestBroker(" & `typeNameLit` & + "): provider already set for broker context " & $brokerCtx + ) + + `accessProcIdent`().`zeroArgProvidersFieldName`.add( + (brokerCtx: brokerCtx, handler: handler) + ) + return ok() + + ) + clearBodyKeyed.add( + quote do: + if `brokerCtxParamIdent` == DefaultBrokerContext: + `accessProcIdent`().`zeroArgProvidersFieldName`[0].handler = + default(`zeroArgProviderName`) + else: + `accessProcIdent`().`zeroArgProvidersFieldName`.keepItIf( + it.brokerCtx != `brokerCtxParamIdent` + ) + ) + case mode + of rbAsync, rbMultiThread, rbApi: + result.add( + quote do: + proc request*( + _: typedesc[`typeIdent`] + ): Future[Result[`payloadType`, string]] {.async: (raises: []).} = + return await request(`typeIdent`, DefaultBrokerContext) + + ) + + result.add( + quote do: + proc request*( + _: typedesc[`typeIdent`], brokerCtx: BrokerContext + ): Future[Result[`payloadType`, string]] {.async: (raises: []).} = + var provider: `zeroArgProviderName` + if brokerCtx == DefaultBrokerContext: + provider = `accessProcIdent`().`zeroArgProvidersFieldName`[0].handler + else: + for entry in `accessProcIdent`().`zeroArgProvidersFieldName`: + if entry.brokerCtx == brokerCtx: + provider = entry.handler + break + + if provider.isNil(): + if brokerCtx == DefaultBrokerContext: + return err( + "RequestBroker(" & `typeNameLit` & "): no zero-arg provider registered" + ) + return err( + "RequestBroker(" & `typeNameLit` & + "): no provider registered for broker context " & $brokerCtx + ) + + let catchedRes = catch: + await provider() + + if catchedRes.isErr(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & + catchedRes.error.msg + ) + + let providerRes = catchedRes.get() + if providerRes.isOk(): + when compiles(providerRes.get().isNil()) and + not (typeof(providerRes.get()) is string): + if providerRes.get().isNil(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider returned nil result" + ) + return providerRes + + ) + of rbSync: + # Keyed variant first — the forwarder below calls it, and `sync` bodies + # are sem-checked eagerly (no forward references for overloaded routines). + result.add( + quote do: + proc request*( + _: typedesc[`typeIdent`], brokerCtx: BrokerContext + ): Result[`payloadType`, string] {.gcsafe, raises: [].} = + var provider: `zeroArgProviderName` + if brokerCtx == DefaultBrokerContext: + provider = `accessProcIdent`().`zeroArgProvidersFieldName`[0].handler + else: + for entry in `accessProcIdent`().`zeroArgProvidersFieldName`: + if entry.brokerCtx == brokerCtx: + provider = entry.handler + break + + if provider.isNil(): + if brokerCtx == DefaultBrokerContext: + return err( + "RequestBroker(" & `typeNameLit` & "): no zero-arg provider registered" + ) + return err( + "RequestBroker(" & `typeNameLit` & + "): no provider registered for broker context " & $brokerCtx + ) + + var providerRes: Result[`payloadType`, string] + try: + providerRes = provider() + except CatchableError as e: + return err( + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & + e.msg + ) + + if providerRes.isOk(): + when compiles(providerRes.get().isNil()) and + not (typeof(providerRes.get()) is string): + if providerRes.get().isNil(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider returned nil result" + ) + return providerRes + + ) + + result.add( + quote do: + proc request*( + _: typedesc[`typeIdent`] + ): Result[`payloadType`, string] {.gcsafe, raises: [].} = + return request(`typeIdent`, DefaultBrokerContext) + + ) + if not argSig.isNil(): + let argProvidersFieldName = ident("providersWithArgs") + result.add( + quote do: + proc setProvider*( + _: typedesc[`typeIdent`], handler: `argProviderName` + ): Result[void, string] = + if not `accessProcIdent`().`argProvidersFieldName`[0].handler.isNil(): + return err("Provider already set") + `accessProcIdent`().`argProvidersFieldName`[0].handler = handler + return ok() + + ) + + result.add( + quote do: + proc setProvider*( + _: typedesc[`typeIdent`], + brokerCtx: BrokerContext, + handler: `argProviderName`, + ): Result[void, string] = + if brokerCtx == DefaultBrokerContext: + return setProvider(`typeIdent`, handler) + + for entry in `accessProcIdent`().`argProvidersFieldName`: + if entry.brokerCtx == brokerCtx: + return err( + "RequestBroker(" & `typeNameLit` & + "): provider already set for broker context " & $brokerCtx + ) + + `accessProcIdent`().`argProvidersFieldName`.add( + (brokerCtx: brokerCtx, handler: handler) + ) + return ok() + + ) + clearBodyKeyed.add( + quote do: + if `brokerCtxParamIdent` == DefaultBrokerContext: + `accessProcIdent`().`argProvidersFieldName`[0].handler = + default(`argProviderName`) + else: + `accessProcIdent`().`argProvidersFieldName`.keepItIf( + it.brokerCtx != `brokerCtxParamIdent` + ) + ) + let requestParamDefs = cloneParams(argParams) + let argNameIdents = collectParamNames(requestParamDefs) + var formalParams = newTree(nnkFormalParams) + formalParams.add(copyNimTree(returnType)) + formalParams.add( + newTree( + nnkIdentDefs, + ident("_"), + newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)), + newEmptyNode(), + ) + ) + for paramDef in requestParamDefs: + formalParams.add(paramDef) + + let requestPragmas = + case mode + of rbAsync, rbMultiThread, rbApi: + quote: + {.async: (raises: []).} + of rbSync: + quote: + {.gcsafe, raises: [].} + + var forwardCall = newCall(ident("request")) + forwardCall.add(copyNimTree(typeIdent)) + forwardCall.add(ident("DefaultBrokerContext")) + for argName in argNameIdents: + forwardCall.add(argName) + + var requestBody = newStmtList() + case mode + of rbAsync, rbMultiThread, rbApi: + requestBody.add( + quote do: + return await `forwardCall` + ) + of rbSync: + requestBody.add( + quote do: + return `forwardCall` + ) + + # Built now, but added to `result` only *after* the keyed variant below: + # the forwarder's body calls the keyed `request`, and in `sync` mode the + # body is sem-checked eagerly, so the keyed overload must already be + # declared (Nim has no forward references for overloaded routines). Async + # tolerates either order because the `async` macro reprocesses the body + # after the surrounding scope is fully populated. + let nonKeyedRequestProc = newTree( + nnkProcDef, + postfix(ident("request"), "*"), + newEmptyNode(), + newEmptyNode(), + formalParams, + requestPragmas, + newEmptyNode(), + requestBody, + ) + + # Keyed request variant for the argument-based signature. + let requestParamDefsKeyed = cloneParams(argParams) + let argNameIdentsKeyed = collectParamNames(requestParamDefsKeyed) + let providerSymKeyed = genSym(nskVar, "provider") + var formalParamsKeyed = newTree(nnkFormalParams) + formalParamsKeyed.add(copyNimTree(returnType)) + formalParamsKeyed.add( + newTree( + nnkIdentDefs, + ident("_"), + newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)), + newEmptyNode(), + ) + ) + formalParamsKeyed.add( + newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode()) + ) + for paramDef in requestParamDefsKeyed: + formalParamsKeyed.add(paramDef) + + let requestPragmasKeyed = requestPragmas + var providerCallKeyed = newCall(providerSymKeyed) + for argName in argNameIdentsKeyed: + providerCallKeyed.add(argName) + + var requestBodyKeyed = newStmtList() + requestBodyKeyed.add( + quote do: + var `providerSymKeyed`: `argProviderName` + if brokerCtx == DefaultBrokerContext: + `providerSymKeyed` = `accessProcIdent`().`argProvidersFieldName`[0].handler + else: + for entry in `accessProcIdent`().`argProvidersFieldName`: + if entry.brokerCtx == brokerCtx: + `providerSymKeyed` = entry.handler + break + ) + requestBodyKeyed.add( + quote do: + if `providerSymKeyed`.isNil(): + if brokerCtx == DefaultBrokerContext: + return err( + "RequestBroker(" & `typeNameLit` & + "): no provider registered for input signature" + ) + return err( + "RequestBroker(" & `typeNameLit` & + "): no provider registered for broker context " & $brokerCtx + ) + ) + + case mode + of rbAsync, rbMultiThread, rbApi: + requestBodyKeyed.add( + quote do: + let catchedRes = catch: + await `providerCallKeyed` + if catchedRes.isErr(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & + catchedRes.error.msg + ) + + let providerRes = catchedRes.get() + if providerRes.isOk(): + when compiles(providerRes.get().isNil()) and + not (typeof(providerRes.get()) is string): + if providerRes.get().isNil(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider returned nil result" + ) + return providerRes + ) + of rbSync: + requestBodyKeyed.add( + quote do: + var providerRes: Result[`payloadType`, string] + try: + providerRes = `providerCallKeyed` + except CatchableError as e: + return err( + "RequestBroker(" & `typeNameLit` & "): provider threw exception: " & e.msg + ) + + if providerRes.isOk(): + when compiles(providerRes.get().isNil()) and + not (typeof(providerRes.get()) is string): + if providerRes.get().isNil(): + return err( + "RequestBroker(" & `typeNameLit` & "): provider returned nil result" + ) + return providerRes + ) + + result.add( + newTree( + nnkProcDef, + postfix(ident("request"), "*"), + newEmptyNode(), + newEmptyNode(), + formalParamsKeyed, + requestPragmasKeyed, + newEmptyNode(), + requestBodyKeyed, + ) + ) + + # Now the keyed overload is in scope for the forwarder's body. + result.add(nonKeyedRequestProc) + + block: + var formalParamsClearKeyed = newTree(nnkFormalParams) + formalParamsClearKeyed.add(newEmptyNode()) + formalParamsClearKeyed.add( + newTree( + nnkIdentDefs, + ident("_"), + newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)), + newEmptyNode(), + ) + ) + formalParamsClearKeyed.add( + newTree(nnkIdentDefs, brokerCtxParamIdent, ident("BrokerContext"), newEmptyNode()) + ) + + result.add( + newTree( + nnkProcDef, + postfix(ident("clearProvider"), "*"), + newEmptyNode(), + newEmptyNode(), + formalParamsClearKeyed, + newEmptyNode(), + newEmptyNode(), + clearBodyKeyed, + ) + ) + + result.add( + quote do: + proc clearProvider*(_: typedesc[`typeIdent`]) = + clearProvider(`typeIdent`, DefaultBrokerContext) + + ) + + # ── isProvided ───────────────────────────────────────────────────── + # Returns true when at least one provider slot (zero-arg or with-arg) + # is filled for the given broker context. + block: + var isProvidedBody = newStmtList() + if not zeroArgSig.isNil(): + let zeroArgProvidersFieldName = ident("providersNoArgs") + isProvidedBody.add( + quote do: + if `brokerCtxParamIdent` == DefaultBrokerContext: + if not `accessProcIdent`().`zeroArgProvidersFieldName`[0].handler.isNil(): + return true + else: + for entry in `accessProcIdent`().`zeroArgProvidersFieldName`: + if entry.brokerCtx == `brokerCtxParamIdent` and not entry.handler.isNil(): + return true + ) + if not argSig.isNil(): + let argProvidersFieldName = ident("providersWithArgs") + isProvidedBody.add( + quote do: + if `brokerCtxParamIdent` == DefaultBrokerContext: + if not `accessProcIdent`().`argProvidersFieldName`[0].handler.isNil(): + return true + else: + for entry in `accessProcIdent`().`argProvidersFieldName`: + if entry.brokerCtx == `brokerCtxParamIdent` and not entry.handler.isNil(): + return true + ) + isProvidedBody.add( + quote do: + return false + ) + + var formalParamsIsProvided = newTree(nnkFormalParams) + formalParamsIsProvided.add(ident("bool")) + formalParamsIsProvided.add( + newTree( + nnkIdentDefs, + ident("_"), + newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)), + newEmptyNode(), + ) + ) + formalParamsIsProvided.add( + newTree(nnkIdentDefs, brokerCtxParamIdent, ident("BrokerContext"), newEmptyNode()) + ) + + result.add( + newTree( + nnkProcDef, + postfix(ident("isProvided"), "*"), + newEmptyNode(), + newEmptyNode(), + formalParamsIsProvided, + newEmptyNode(), + newEmptyNode(), + isProvidedBody, + ) + ) + + result.add( + quote do: + proc isProvided*(_: typedesc[`typeIdent`]): bool = + isProvided(`typeIdent`, DefaultBrokerContext) + + ) + + when defined(brokerDebug): + writeBrokerDebug("RequestBroker", typeDisplayName, result, header = "mode=" & $mode) + when defined(brokerDebugStdout): + echo result.repr + + return result + +macro RequestBroker*(args: varargs[untyped]): untyped = + ## Default (async) mode, or explicit mode selector with optional kwargs. + ## + ## Examples: + ## RequestBroker: + ## type Foo = object + ## proc signature*(): Result[Foo, string] + ## + ## RequestBroker(sync): + ## type Foo = object + ## proc signature*(): Result[Foo, string] + ## + ## RequestBroker(mt): + ## type Foo = object + ## proc signature*(arg: string): Future[Result[Foo, string]] {.async.} + ## + ## RequestBroker(mt, queueDepth = 1024, responseSlots = 64, + ## maxResponseBytes = 4096): + ## type Foo = object + ## proc signature*(arg: string): Future[Result[Foo, string]] {.async.} + if args.len == 0: + macros.error("RequestBroker requires a body block") + if args.len == 1: + return generateRequestBroker(args[0], rbAsync) + let mode = args[0] + let body = args[^1] + if body.kind notin {nnkStmtList, nnkTypeDef, nnkTypeSection}: + error( + "RequestBroker(" & mode.repr & + ") body must be a `:` block of type definitions (got " & $body.kind & ")", + body, + ) + var kwargs: seq[NimNode] + for i in 1 ..< args.len - 1: + kwargs.add(args[i]) + let m = parseMode(mode) + let split = (kwargs: kwargs, body: body) + case m + of rbMultiThread: + when not compileOption("threads"): + macros.error("RequestBroker(mt) requires --threads:on. " & + "Compile with `--threads:on` to use multi-thread RequestBroker.") + else: + let cfg = parseMtReqKwargs(split.kwargs) + generateMtRequestBroker(body, cfg) + of rbApi: + when not compileOption("threads"): + macros.error("RequestBroker(API) requires --threads:on. " & + "Compile with `--threads:on` to use API RequestBroker.") + else: + when defined(BrokerFfiApi): + # Validate kwargs at the outer macro for clear error origin, + # then hand them to the deferred codegen which re-parses into + # MtReqCfg. RequestBroker(API) runs on the same MT lane + # underneath, so it accepts the same capacity knobs as + # RequestBroker(mt) (queueDepth / slabCapacity / + # maxPayloadBytes / responseSlots / maxResponseBytes / + # freeListShards / preset). + discard parseMtReqKwargs(split.kwargs) + generateApiCborRequestBroker(body, split.kwargs) + else: + let cfg = parseMtReqKwargs(split.kwargs) + generateMtRequestBroker(body, cfg) + of rbAsync, rbSync: + if split.kwargs.len > 0: + error( + "RequestBroker(" & mode.repr & ") does not accept kwargs (kwargs are mt-only)", + split.kwargs[0], + ) + generateRequestBroker(body, m) diff --git a/wasm-deps/brokers/nimblemeta.json b/wasm-deps/brokers/nimblemeta.json new file mode 100644 index 000000000..600d7f2f0 --- /dev/null +++ b/wasm-deps/brokers/nimblemeta.json @@ -0,0 +1,51 @@ +{ + "version": 1, + "metaData": { + "url": "https://github.com/NagyZoltanPeter/nim-brokers.git", + "downloadMethod": "git", + "vcsRevision": "a7316a35f1b62e3497ae8ee0fc1aace74df0beb2", + "files": [ + "/brokers/internal/mt_broker_common.nim", + "/brokers.nim", + "/brokers/event_broker.nim", + "/brokers/internal/mt_queue.nim", + "/brokers/internal/api_cbor_descriptor.nim", + "/brokers/internal/api_codegen_cbor_hpp.nim", + "/brokers/api_library.nim", + "/brokers/internal/api_codegen_cmake.nim", + "/brokers/internal/api_cbor_tuple.nim", + "/brokers/internal/mt_codec.nim", + "/brokers/broker_context.nim", + "/brokers/internal/api_type_resolver.nim", + "/brokers/multi_request_broker.nim", + "/brokers/internal/api_request_broker_cbor.nim", + "/brokers/internal/api_outdir.nim", + "/brokers/internal/api_codegen_cbor_h.nim", + "/brokers/broker_interface.nim", + "/brokers/internal/api_codegen_cbor_go.nim", + "/brokers/internal/api_event_broker_cbor.nim", + "/brokers/broker_implement.nim", + "/brokers/internal/api_schema.nim", + "/brokers/internal/api_codegen_cbor_py.nim", + "/brokers/internal/api_codegen_cbor_cddl.nim", + "/brokers/internal/helper/broker_utils.nim", + "/brokers/internal/mt_config.nim", + "/brokers/internal/api_common.nim", + "/brokers/internal/mt_event_broker.nim", + "/brokers/internal/api_cbor_codec.nim", + "/brokers/request_broker.nim", + "/brokers/internal/api_cbor_subs_registry.nim", + "/brokers/internal/api_cbor_event_courier.nim", + "/brokers/internal/broker_debug.nim", + "/brokers.nimble", + "/brokers/internal/api_codegen_cbor_rust.nim", + "/brokers/internal/api_cbor_courier.nim", + "/brokers/internal/mt_request_broker.nim" + ], + "binaries": [], + "specialVersions": [ + "3.1.1", + "#v3.1.1" + ] + } +} \ No newline at end of file diff --git a/wasm-deps/edge_builders.nim b/wasm-deps/edge_builders.nim new file mode 100644 index 000000000..ed9471c9e --- /dev/null +++ b/wasm-deps/edge_builders.nim @@ -0,0 +1,547 @@ +# SPDX-License-Identifier: Apache-2.0 OR MIT +# Copyright (c) Status Research & Development GmbH + +## This module contains a Switch Building helper. +runnableExamples: + let switch = SwitchBuilder.new().withRng(rng).withAddresses(multiaddress) + # etc + .build() + +{.push raises: [].} + +# NOTE: wasm/edge override of libp2p/builders. Identical to upstream EXCEPT the +# QUIC transport is removed — quictransport pulls lsquic + boringssl x86 asm that +# cannot build for wasm32, and an edge browser node never uses QUIC. Imports are +# rewritten to absolute `libp2p/...` form so this single-file override resolves +# the rest of the package (Nim keys modules by absolute path, so type identity is +# preserved). Placed first on the path via --path so it shadows upstream builders. +import options, tables, chronos, chronicles, sequtils +import + libp2p/switch, + libp2p/peerid, + libp2p/peerinfo, + libp2p/stream/connection, + libp2p/multiaddress, + libp2p/crypto/crypto, + # wstransport dropped: it imports autotls/service -> certificate_ffi -> lsquic. + # The edge node uses WsBrowserTransport; builders' withWsTransport is removed. + libp2p/transports/[transport, tcptransport, memorytransport], + libp2p/muxers/[muxer, mplex/mplex, yamux/yamux], + libp2p/protocols/[identify, secure/secure, secure/noise, rendezvous, kademlia], + libp2p/protocols/connectivity/[ + autonat/server, + autonatv2/server, + autonatv2/service, + autonatv2/client, + relay/relay, + relay/client, + relay/rtransport, + ], + libp2p/connmanager, + libp2p/upgrademngrs/muxedupgrade, + libp2p/observedaddrmanager, + libp2p/nameresolving/nameresolver, + libp2p/errors, + libp2p/utility +import libp2p/services/wildcardresolverservice + +# autotls is trimmed in the wasm/edge override: its `certificate_ffi` pulls +# lsquic/boringssl, which won't build for wasm, and a browser edge node never +# provisions TLS certs (the browser's WebSocket handles wss). The field stays as +# a never-`Some` stub so the SwitchBuilder shape is otherwise unchanged. +type AutotlsService* = ref object + +# TLSPrivateKey/TLSCertificate/TLSFlags dropped from exports — they came from the +# removed wstransport. ServerFlags stays (from tcptransport). +export switch, peerid, peerinfo, connection, multiaddress, crypto, errors, ServerFlags + +const MemoryAutoAddress* = memorytransport.MemoryAutoAddress + +type + TransportProvider* {.deprecated: "Use TransportBuilder instead".} = + proc(upgr: Upgrade, privateKey: PrivateKey): Transport {.gcsafe, raises: [].} + + TransportBuilder* {.public.} = + proc(config: TransportConfig): Transport {.gcsafe, raises: [].} + + TransportConfig* = ref object + upgr*: Upgrade + privateKey*: PrivateKey + autotls*: Opt[AutotlsService] + + SecureProtocol* {.pure.} = enum + Noise + + KadInfo = object + config*: KadDHTConfig + bootstrapNodes*: seq[(PeerId, seq[MultiAddress])] + + SwitchBuilder* = ref object + privKey: Opt[PrivateKey] + addresses: seq[MultiAddress] + secureManagers: seq[SecureProtocol] + muxers: seq[MuxerProvider] + transports: seq[TransportBuilder] + rng: ref HmacDrbgContext + maxConnections: int + maxIn: int + sendSignedPeerRecord: bool + maxOut: int + maxConnsPerPeer: int + protoVersion: string + agentVersion: string + nameResolver: NameResolver + peerStoreCapacity: Opt[int] + autonat: bool + autonatV2ServerConfig: Opt[AutonatV2Config] + autonatV2Client: AutonatV2Client + autonatV2ServiceConfig: AutonatV2ServiceConfig + autotls: Opt[AutotlsService] + circuitRelay: Opt[Relay] + rdv: Opt[RendezVous] + kad: Opt[KadInfo] + services: seq[Service] + observedAddrManager: ObservedAddrManager + enableWildcardResolver: bool + +proc new*(T: type[SwitchBuilder]): T {.public.} = + ## Creates a SwitchBuilder + + let address = + MultiAddress.init("/ip4/127.0.0.1/tcp/0").expect("Should initialize to default") + + SwitchBuilder( + privKey: Opt.none(PrivateKey), + addresses: @[address], + secureManagers: @[], + maxConnections: MaxConnections, + maxIn: -1, + maxOut: -1, + maxConnsPerPeer: MaxConnectionsPerPeer, + protoVersion: ProtoVersion, + agentVersion: AgentVersion, + autotls: Opt.none(AutotlsService), + circuitRelay: Opt.none(Relay), + rdv: Opt.none(RendezVous), + kad: Opt.none(KadInfo), + enableWildcardResolver: true, + ) + +proc withPrivateKey*( + b: SwitchBuilder, privateKey: PrivateKey +): SwitchBuilder {.public.} = + ## Set the private key of the switch. Will be used to + ## generate a PeerId + + b.privKey = Opt.some(privateKey) + b + +proc withAddresses*( + b: SwitchBuilder, addresses: seq[MultiAddress], enableWildcardResolver: bool = true +): SwitchBuilder {.public.} = + ## | Set the listening addresses of the switch + ## | Calling it multiple time will override the value + b.addresses = addresses + b.enableWildcardResolver = enableWildcardResolver + b + +proc withAddress*( + b: SwitchBuilder, address: MultiAddress, enableWildcardResolver: bool = true +): SwitchBuilder {.public.} = + ## | Set the listening address of the switch + ## | Calling it multiple time will override the value + b.withAddresses(@[address], enableWildcardResolver) + +proc withSignedPeerRecord*(b: SwitchBuilder, sendIt = true): SwitchBuilder {.public.} = + b.sendSignedPeerRecord = sendIt + b + +proc withMplex*( + b: SwitchBuilder, inTimeout = 5.minutes, outTimeout = 5.minutes, maxChannCount = 200 +): SwitchBuilder {.public.} = + ## | Uses `Mplex `_ as a multiplexer + ## | `Timeout` is the duration after which a inactive connection will be closed + proc newMuxer(conn: Connection): Muxer = + Mplex.new(conn, inTimeout, outTimeout, maxChannCount) + + assert b.muxers.countIt(it.codec == MplexCodec) == 0, "Mplex build multiple times" + b.muxers.add(MuxerProvider.new(newMuxer, MplexCodec)) + b + +proc withYamux*( + b: SwitchBuilder, + maxChannCount: int = MaxChannelCount, + windowSize: int = YamuxDefaultWindowSize, + inTimeout: Duration = 5.minutes, + outTimeout: Duration = 5.minutes, +): SwitchBuilder = + proc newMuxer(conn: Connection): Muxer = + Yamux.new( + conn, + maxChannCount = maxChannCount, + windowSize = windowSize, + inTimeout = inTimeout, + outTimeout = outTimeout, + ) + + assert b.muxers.countIt(it.codec == YamuxCodec) == 0, "Yamux build multiple times" + b.muxers.add(MuxerProvider.new(newMuxer, YamuxCodec)) + b + +proc withNoise*(b: SwitchBuilder): SwitchBuilder {.public.} = + b.secureManagers.add(SecureProtocol.Noise) + b + +proc withTransport*( + b: SwitchBuilder, prov: TransportBuilder +): SwitchBuilder {.public.} = + ## Use a custom transport + runnableExamples: + let switch = SwitchBuilder + .new() + .withTransport( + proc(config: TransportConfig): Transport = + TcpTransport.new(flags, config.upgr) + ) + .build() + b.transports.add(prov) + b + +proc withTransport*( + b: SwitchBuilder, prov: TransportProvider +): SwitchBuilder {.deprecated: "Use TransportBuilder instead".} = + ## Use a custom transport + runnableExamples: + let switch = SwitchBuilder + .new() + .withTransport( + proc(upgr: Upgrade, privateKey: PrivateKey): Transport = + TcpTransport.new(flags, upgr) + ) + .build() + let tBuilder: TransportBuilder = proc(config: TransportConfig): Transport = + prov(config.upgr, config.privateKey) + b.withTransport(tBuilder) + +proc withTcpTransport*( + b: SwitchBuilder, flags: set[ServerFlags] = {} +): SwitchBuilder {.public.} = + b.withTransport( + proc(config: TransportConfig): Transport = + TcpTransport.new(flags, config.upgr) + ) + +# withWsTransport removed in the wasm/edge override: it threads `config.autotls` +# into the real WsTransport, and the edge node uses the browser-WebSocket +# transport instead. (withQuicTransport removed too — no QUIC in the browser.) + +proc withMemoryTransport*(b: SwitchBuilder): SwitchBuilder {.public.} = + b.withTransport( + proc(config: TransportConfig): Transport = + MemoryTransport.new(config.upgr) + ) + +proc withRng*(b: SwitchBuilder, rng: ref HmacDrbgContext): SwitchBuilder {.public.} = + b.rng = rng + b + +proc withMaxConnections*( + b: SwitchBuilder, maxConnections: int +): SwitchBuilder {.public.} = + ## Maximum concurrent connections of the switch. You should either use this, or + ## `withMaxIn <#withMaxIn,SwitchBuilder,int>`_ & `withMaxOut<#withMaxOut,SwitchBuilder,int>`_ + b.maxConnections = maxConnections + b + +proc withMaxIn*(b: SwitchBuilder, maxIn: int): SwitchBuilder {.public.} = + ## Maximum concurrent incoming connections. Should be used with `withMaxOut<#withMaxOut,SwitchBuilder,int>`_ + b.maxIn = maxIn + b + +proc withMaxOut*(b: SwitchBuilder, maxOut: int): SwitchBuilder {.public.} = + ## Maximum concurrent outgoing connections. Should be used with `withMaxIn<#withMaxIn,SwitchBuilder,int>`_ + b.maxOut = maxOut + b + +proc withMaxConnsPerPeer*( + b: SwitchBuilder, maxConnsPerPeer: int +): SwitchBuilder {.public.} = + b.maxConnsPerPeer = maxConnsPerPeer + b + +proc withPeerStore*(b: SwitchBuilder, capacity: int): SwitchBuilder {.public.} = + b.peerStoreCapacity = Opt.some(capacity) + b + +proc withProtoVersion*( + b: SwitchBuilder, protoVersion: string +): SwitchBuilder {.public.} = + b.protoVersion = protoVersion + b + +proc withAgentVersion*( + b: SwitchBuilder, agentVersion: string +): SwitchBuilder {.public.} = + b.agentVersion = agentVersion + b + +proc withNameResolver*( + b: SwitchBuilder, nameResolver: NameResolver +): SwitchBuilder {.public.} = + b.nameResolver = nameResolver + b + +proc withAutonat*(b: SwitchBuilder): SwitchBuilder = + b.autonat = true + b + +proc withAutonatV2Server*( + b: SwitchBuilder, config: AutonatV2Config = AutonatV2Config.new() +): SwitchBuilder = + b.autonatV2ServerConfig = Opt.some(config) + b + +proc withAutonatV2*( + b: SwitchBuilder, serviceConfig = AutonatV2ServiceConfig.new() +): SwitchBuilder = + b.autonatV2Client = AutonatV2Client.new(b.rng) + b.autonatV2ServiceConfig = serviceConfig + b + +when defined(libp2p_autotls_support): + proc withAutotls*( + b: SwitchBuilder, config: AutotlsConfig = AutotlsConfig.new() + ): SwitchBuilder {.public.} = + b.autotls = Opt.some(AutotlsService.new(config = config)) + b + +proc withCircuitRelay*(b: SwitchBuilder, r: Relay = Relay.new()): SwitchBuilder = + b.circuitRelay = Opt.some(r) + b + +proc withRendezVous*(b: SwitchBuilder, rdv: RendezVous): SwitchBuilder = + var lrdv = rdv + if rdv.isNil(): + lrdv = RendezVous.new() + + b.rdv = Opt.some(lrdv) + b + +proc withKademlia*( + b: SwitchBuilder, + bootstrapNodes: seq[(PeerId, seq[MultiAddress])] = @[], + config: KadDHTConfig = KadDHTConfig.new(), +): SwitchBuilder = + b.kad = Opt.some(KadInfo(config: config, bootstrapNodes: bootstrapNodes)) + b + +proc withServices*(b: SwitchBuilder, services: seq[Service]): SwitchBuilder = + b.services = services + b + +proc withObservedAddrManager*( + b: SwitchBuilder, observedAddrManager: ObservedAddrManager +): SwitchBuilder = + b.observedAddrManager = observedAddrManager + b + +proc build*(b: SwitchBuilder): Switch {.raises: [LPError], public.} = + if b.rng == nil: # newRng could fail + raise newException(Defect, "Cannot initialize RNG") + + let pkRes = PrivateKey.random(b.rng[]) + let seckey = b.privKey.get(otherwise = pkRes.expect("Expected default Private Key")) + + if b.secureManagers.len == 0: + debug "no secure managers defined. Adding noise by default" + b.secureManagers.add(SecureProtocol.Noise) + + var secureManagerInstances: seq[Secure] + if SecureProtocol.Noise in b.secureManagers: + secureManagerInstances.add(Noise.new(b.rng, seckey).Secure) + + let peerInfo = PeerInfo.new( + seckey, b.addresses, protoVersion = b.protoVersion, agentVersion = b.agentVersion + ) + + let identify = + if b.observedAddrManager != nil: + Identify.new(peerInfo, b.sendSignedPeerRecord, b.observedAddrManager) + else: + Identify.new(peerInfo, b.sendSignedPeerRecord) + + let + connManager = + ConnManager.new(b.maxConnsPerPeer, b.maxConnections, b.maxIn, b.maxOut) + ms = MultistreamSelect.new() + muxedUpgrade = MuxedUpgrade.new(b.muxers, secureManagerInstances, ms) + + # autotls service is never created in the edge override (field is always none). + + let transports = block: + var transports: seq[Transport] + for tProvider in b.transports: + transports.add( + tProvider( + TransportConfig(upgr: muxedUpgrade, privateKey: seckey, autotls: b.autotls) + ) + ) + transports + + if b.secureManagers.len == 0: + b.secureManagers &= SecureProtocol.Noise + + if isNil(b.rng): + b.rng = newRng() + + let peerStore = block: + b.peerStoreCapacity.withValue(capacity): + PeerStore.new(identify, capacity) + else: + PeerStore.new(identify) + + if b.enableWildcardResolver: + b.services.add(WildcardAddressResolverService.new()) + + if not isNil(b.autonatV2Client): + b.services.add( + AutonatV2Service.new( + b.rng, client = b.autonatV2Client, config = b.autonatV2ServiceConfig + ) + ) + + let switch = newSwitch( + peerInfo = peerInfo, + transports = transports, + secureManagers = secureManagerInstances, + connManager = connManager, + ms = ms, + nameResolver = b.nameResolver, + peerStore = peerStore, + services = b.services, + ) + + switch.mount(identify) + + if not isNil(b.autonatV2Client): + b.autonatV2Client.setup(switch) + switch.mount(b.autonatV2Client) + + b.autonatV2ServerConfig.withValue(config): + switch.mount(AutonatV2.new(switch, config = config)) + + if b.autonat: + switch.mount(Autonat.new(switch)) + + b.circuitRelay.withValue(relay): + if relay of RelayClient: + switch.addTransport(RelayTransport.new(RelayClient(relay), muxedUpgrade)) + relay.setup(switch) + switch.mount(relay) + + b.rdv.withValue(rdvService): + rdvService.setup(switch) + switch.mount(rdvService) + + b.kad.withValue(kadInfo): + let kad = KadDHT.new( + switch, bootstrapNodes = kadInfo.bootstrapNodes, config = kadInfo.config + ) + switch.mount(kad) + + return switch + +type TransportType* {.pure.} = enum + TCP + Memory + +proc newStandardSwitchBuilder*( + privKey = Opt.none(PrivateKey), + addrs: MultiAddress | seq[MultiAddress] = newSeq[MultiAddress](), + transport: TransportType = TransportType.TCP, + transportFlags: set[ServerFlags] = {}, + rng = newRng(), + secureManagers: openArray[SecureProtocol] = [SecureProtocol.Noise], + inTimeout: Duration = 5.minutes, + outTimeout: Duration = 5.minutes, + maxConnections = MaxConnections, + maxIn = -1, + maxOut = -1, + maxConnsPerPeer = MaxConnectionsPerPeer, + nameResolver = Opt.none(NameResolver), + sendSignedPeerRecord = false, + peerStoreCapacity = 1000, +): SwitchBuilder {.raises: [LPError], public.} = + ## Helper for common switch configurations. + var b = SwitchBuilder + .new() + .withRng(rng) + .withSignedPeerRecord(sendSignedPeerRecord) + .withMaxConnections(maxConnections) + .withMaxIn(maxIn) + .withMaxOut(maxOut) + .withMaxConnsPerPeer(maxConnsPerPeer) + .withPeerStore(capacity = peerStoreCapacity) + .withNoise() + + privKey.withValue(pkey): + b = b.withPrivateKey(pkey) + + nameResolver.withValue(nr): + b = b.withNameResolver(nr) + + var addrs = + when addrs is MultiAddress: + @[addrs] + else: + addrs + + case transport + of TransportType.TCP: + if addrs.len == 0: + addrs = @[MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet()] + b = b.withTcpTransport(transportFlags).withAddresses(addrs).withMplex( + inTimeout, outTimeout + ) + of TransportType.Memory: + if addrs.len == 0: + addrs = @[MultiAddress.init(MemoryAutoAddress).tryGet()] + b = b.withMemoryTransport().withAddresses(addrs).withMplex(inTimeout, outTimeout) + + b + +proc newStandardSwitch*( + privKey = Opt.none(PrivateKey), + addrs: MultiAddress | seq[MultiAddress] = newSeq[MultiAddress](), + transport: TransportType = TransportType.TCP, + transportFlags: set[ServerFlags] = {}, + rng = newRng(), + secureManagers: openArray[SecureProtocol] = [SecureProtocol.Noise], + inTimeout: Duration = 5.minutes, + outTimeout: Duration = 5.minutes, + maxConnections = MaxConnections, + maxIn = -1, + maxOut = -1, + maxConnsPerPeer = MaxConnectionsPerPeer, + nameResolver = Opt.none(NameResolver), + sendSignedPeerRecord = false, + peerStoreCapacity = 1000, +): Switch {.raises: [LPError], public.} = + newStandardSwitchBuilder( + privKey = privKey, + addrs = addrs, + transport = transport, + transportFlags = transportFlags, + rng = rng, + secureManagers = secureManagers, + inTimeout = inTimeout, + outTimeout = outTimeout, + maxConnections = maxConnections, + maxIn = maxIn, + maxOut = maxOut, + maxConnsPerPeer = maxConnsPerPeer, + nameResolver = nameResolver, + sendSignedPeerRecord = sendSignedPeerRecord, + peerStoreCapacity = peerStoreCapacity, + ) + .build() diff --git a/wasm-deps/ffi/ffi.nim b/wasm-deps/ffi/ffi.nim new file mode 100644 index 000000000..0ef64acd5 --- /dev/null +++ b/wasm-deps/ffi/ffi.nim @@ -0,0 +1,10 @@ +import std/[atomics, tables] +import chronos, chronicles +import + ffi/internal/[ffi_library, ffi_macro], + ffi/[alloc, ffi_types, ffi_context, ffi_thread_request] + +export atomics, tables +export chronos, chronicles +export + atomics, alloc, ffi_library, ffi_macro, ffi_types, ffi_context, ffi_thread_request diff --git a/wasm-deps/ffi/ffi.nimble b/wasm-deps/ffi/ffi.nimble new file mode 100644 index 000000000..dc39f4ca6 --- /dev/null +++ b/wasm-deps/ffi/ffi.nimble @@ -0,0 +1,22 @@ +# ffi.nimble + +version = "0.1.3" +author = "Institute of Free Technology" +description = "FFI framework with custom header generation" +license = "MIT or Apache License 2.0" + +packageName = "ffi" + +requires "nim >= 2.2.4" +requires "chronos" +requires "chronicles" +requires "taskpools" + +# Source files to include +# srcDir = "src" +# installFiles = @["src/ffi.nim", "mylib.h"] + +# # 💡 Custom build step before installation +# before install: +# echo "Generating custom C header..." +# exec "nim r tools/gen_header.nim" diff --git a/wasm-deps/ffi/ffi/alloc.nim b/wasm-deps/ffi/ffi/alloc.nim new file mode 100644 index 000000000..1a6f118b5 --- /dev/null +++ b/wasm-deps/ffi/ffi/alloc.nim @@ -0,0 +1,42 @@ +## Can be shared safely between threads +type SharedSeq*[T] = tuple[data: ptr UncheckedArray[T], len: int] + +proc alloc*(str: cstring): cstring = + # Byte allocation from the given address. + # There should be the corresponding manual deallocation with deallocShared ! + if str.isNil(): + var ret = cast[cstring](allocShared(1)) # Allocate memory for the null terminator + ret[0] = '\0' # Set the null terminator + return ret + + let ret = cast[cstring](allocShared(len(str) + 1)) + copyMem(ret, str, len(str) + 1) + return ret + +proc alloc*(str: string): cstring = + ## Byte allocation from the given address. + ## There should be the corresponding manual deallocation with deallocShared ! + var ret = cast[cstring](allocShared(str.len + 1)) + let s = cast[seq[char]](str) + for i in 0 ..< str.len: + ret[i] = s[i] + ret[str.len] = '\0' + return ret + +proc allocSharedSeq*[T](s: seq[T]): SharedSeq[T] = + let data = allocShared(sizeof(T) * s.len) + if s.len != 0: + copyMem(data, unsafeAddr s[0], s.len) + return (cast[ptr UncheckedArray[T]](data), s.len) + +proc deallocSharedSeq*[T](s: var SharedSeq[T]) = + deallocShared(s.data) + s.len = 0 + +proc toSeq*[T](s: SharedSeq[T]): seq[T] = + ## Creates a seq[T] from a SharedSeq[T]. No explicit dealloc is required + ## as req[T] is a GC managed type. + var ret = newSeq[T]() + for i in 0 ..< s.len: + ret.add(s.data[i]) + return ret diff --git a/wasm-deps/ffi/ffi/ffi_config.nim b/wasm-deps/ffi/ffi/ffi_config.nim new file mode 100644 index 000000000..0b0012217 --- /dev/null +++ b/wasm-deps/ffi/ffi/ffi_config.nim @@ -0,0 +1,11 @@ +## Compile-time selection of the execution transport. +## +## Default (threaded): each FFIContext spawns an FFI worker thread + a watchdog +## thread and hands requests over a chronos ThreadSignalPtr + SPSC channel. +## Those rely on OS threads + eventfd-style signalling, absent in a baseline +## WebAssembly sandbox. +## +## `singleThreaded` collapses the worker onto the calling thread: a request runs +## inline to completion. Auto-selected for Emscripten/WASM; forceable anywhere +## with `-d:ffiSingleThreaded`. +const singleThreaded* = defined(ffiSingleThreaded) or defined(emscripten) diff --git a/wasm-deps/ffi/ffi/ffi_context.nim b/wasm-deps/ffi/ffi/ffi_context.nim new file mode 100644 index 000000000..e3d276c94 --- /dev/null +++ b/wasm-deps/ffi/ffi/ffi_context.nim @@ -0,0 +1,302 @@ +{.pragma: exported, exportc, cdecl, raises: [].} +{.pragma: callback, cdecl, raises: [], gcsafe.} +{.passc: "-fPIC".} + +import std/[options, atomics, os, net, locks, json, tables] +import chronicles, chronos, results +import ./ffi_config +when not singleThreaded: + # ThreadSignalPtr requires threads enabled; the SPSC channel only carries + # requests across the worker-thread boundary. Neither exists inline. + import chronos/threadsync, taskpools/channels_spsc_single +import ./ffi_types, ./ffi_thread_request, ./internal/ffi_macro, ./logging + +type FFIContext*[T] = object + myLib*: ptr T + # main library object (e.g., Waku, LibP2P, SDS, the one to be exposed as a library) + when not singleThreaded: + ffiThread: Thread[(ptr FFIContext[T])] + # represents the main FFI thread in charge of attending API consumer actions + watchdogThread: Thread[(ptr FFIContext[T])] + # monitors the FFI thread and notifies the FFI API consumer if it hangs + reqChannel: ChannelSPSCSingle[ptr FFIThreadRequest] + reqSignal: ThreadSignalPtr # to notify the FFI Thread that a new request is sent + reqReceivedSignal: ThreadSignalPtr + # to signal main thread, interfacing with the FFI thread, that FFI thread received the request + else: + myLibStorage: T + # Threaded mode roots the library object on the FFI worker thread's stack + # (`ffiReqHandler`). With no worker thread we keep that backing store in + # the context instead, GC-rooted via the holder in createFFIContext. + lock: Lock + userData*: pointer + eventCallback*: pointer + eventUserdata*: pointer + running: Atomic[bool] # To control when the threads are running + registeredRequests: ptr Table[cstring, FFIRequestProc] + # Pointer to with the registered requests at compile time + +const git_version* {.strdefine.} = "n/a" + +template callEventCallback*(ctx: ptr FFIContext, eventName: string, body: untyped) = + if isNil(ctx[].eventCallback): + chronicles.error eventName & " - eventCallback is nil" + return + + foreignThreadGc: + try: + let event = body + cast[FFICallBack](ctx[].eventCallback)( + RET_OK, unsafeAddr event[0], cast[csize_t](len(event)), ctx[].eventUserData + ) + except Exception, CatchableError: + let msg = + "Exception " & eventName & " when calling 'eventCallBack': " & + getCurrentExceptionMsg() + cast[FFICallBack](ctx[].eventCallback)( + RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), ctx[].eventUserData + ) + +when not singleThreaded: + proc sendRequestToFFIThread*( + ctx: ptr FFIContext, ffiRequest: ptr FFIThreadRequest, timeout = InfiniteDuration + ): Result[void, string] = + ctx.lock.acquire() + # This lock is only necessary while we use a SP Channel and while the signalling + # between threads assumes that there aren't concurrent requests. + # Rearchitecting the signaling + migrating to a MP Channel will allow us to receive + # requests concurrently and spare us the need of locks + defer: + ctx.lock.release() + + ## Sending the request + let sentOk = ctx.reqChannel.trySend(ffiRequest) + if not sentOk: + return err("Couldn't send a request to the ffi thread") + + let fireSyncRes = ctx.reqSignal.fireSync() + if fireSyncRes.isErr(): + return err("failed fireSync: " & $fireSyncRes.error) + + if fireSyncRes.get() == false: + return err("Couldn't fireSync in time") + + ## wait until the FFI working thread properly received the request + let res = ctx.reqReceivedSignal.waitSync(timeout) + if res.isErr(): + return err("Couldn't receive reqReceivedSignal signal") + + ## Notice that in case of "ok", the deallocShared(req) is performed by the FFI Thread in the + ## process proc. + return ok() + +type Foo = object +registerReqFFI(WatchdogReq, foo: ptr Foo): + proc(): Future[Result[string, string]] {.async.} = + return ok("FFI thread is not blocked") + +type JsonNotRespondingEvent = object + eventType: string + +proc init(T: type JsonNotRespondingEvent): T = + return JsonNotRespondingEvent(eventType: "not_responding") + +proc `$`(event: JsonNotRespondingEvent): string = + $(%*event) + +proc onNotResponding*(ctx: ptr FFIContext) = + callEventCallback(ctx, "onNotResponding"): + $JsonNotRespondingEvent.init() + +when not singleThreaded: + proc watchdogThreadBody(ctx: ptr FFIContext) {.thread.} = + ## Watchdog thread that monitors the FFI thread and notifies the library user if it hangs. + ## This thread never blocks. + + let watchdogRun = proc(ctx: ptr FFIContext) {.async.} = + const WatchdogStartDelay = 10.seconds + const WatchdogTimeinterval = 1.seconds + const WatchdogTimeout = 20.seconds + + # Give time for the node to be created and up before sending watchdog requests + await sleepAsync(WatchdogStartDelay) + while true: + await sleepAsync(WatchdogTimeinterval) + + if ctx.running.load == false: + debug "Watchdog thread exiting because FFIContext is not running" + break + + let callback = proc( + callerRet: cint, msg: ptr cchar, len: csize_t, userData: pointer + ) {.cdecl, gcsafe, raises: [].} = + discard ## Don't do anything. Just respecting the callback signature. + const nilUserData = nil + + trace "Sending watchdog request to FFI thread" + + sendRequestToFFIThread(ctx, WatchdogReq.ffiNewReq(callback, nilUserData), WatchdogTimeout).isOkOr: + error "Failed to send watchdog request to FFI thread", error = $error + onNotResponding(ctx) + + waitFor watchdogRun(ctx) + +proc processRequest[T]( + request: ptr FFIThreadRequest, ctx: ptr FFIContext[T] +) {.async.} = + ## Invoked within the FFI thread to process a request coming from the FFI API consumer thread. + + let reqId = $request[].reqId + ## The reqId determines which proc will handle the request. + ## The registeredRequests represents a table defined at compile time. + ## Then, registeredRequests == Table[reqId, proc-handling-the-request-asynchronously] + + let retFut = + if not ctx[].registeredRequests[].contains(reqId): + ## That shouldn't happen because only registered requests should be sent to the FFI thread. + nilProcess(request[].reqId) + else: + ctx[].registeredRequests[][reqId](request[].reqContent, ctx) + handleRes(await retFut, request) + +when not singleThreaded: + proc ffiThreadBody[T](ctx: ptr FFIContext[T]) {.thread.} = + ## FFI thread body that attends library user API requests + + logging.setupLog(logging.LogLevel.DEBUG, logging.LogFormat.TEXT) + + let ffiRun = proc(ctx: ptr FFIContext[T]) {.async.} = + var ffiReqHandler: T + ## Holds the main library object, i.e., in charge of handling the ffi requests. + ## e.g., Waku, LibP2P, SDS, etc. + + while true: + await ctx.reqSignal.wait() + + if ctx.running.load == false: + break + + ## Wait for a request from the ffi consumer thread + var request: ptr FFIThreadRequest + let recvOk = ctx.reqChannel.tryRecv(request) + if not recvOk: + chronicles.error "ffi thread could not receive a request" + continue + + ctx.myLib = addr ffiReqHandler + + ## Handle the request + asyncSpawn processRequest(request, ctx) + + let fireRes = ctx.reqReceivedSignal.fireSync() + if fireRes.isErr(): + error "could not fireSync back to requester thread", error = fireRes.error + + waitFor ffiRun(ctx) + +when singleThreaded: + type SingleThreadedHolder[T] = ref object of RootObj + ## GC-traced cell so the library object stored in `ctx.myLibStorage` (a `ref` + ## for e.g. Waku) stays scanned. Kept alive in `gSingleThreadedRoots`. + ## `of RootObj` so holders can be stored uniformly as `RootRef`. + ctx: FFIContext[T] + + var gSingleThreadedRoots {.threadvar.}: seq[RootRef] + + proc sendRequestToFFIThread*( + ctx: ptr FFIContext, ffiRequest: ptr FFIThreadRequest, timeout = InfiniteDuration + ): Result[void, string] = + ## Single-threaded transport. `processRequest` fires the callback and frees + ## the request via `handleRes`. + when defined(emscripten): + # Browser: handlers await the network (WebSocket). Blocking with `waitFor` + # would starve the JS event loop and deadlock. Fire-and-forget instead; the + # host drives chronos via `ffi_poll()` and the callback fires on completion. + asyncSpawn processRequest(ffiRequest, ctx) + poll() # kick the handler up to its first await + else: + try: + waitFor processRequest(ffiRequest, ctx) + except CatchableError as e: + return err("processRequest failed: " & e.msg) + return ok() + + proc ffiPoll*() {.exportc: "ffi_poll", cdecl.} = + ## Advance chronos one step. The browser host calls this from its event loop + ## (setTimeout / requestAnimationFrame) so async handlers progress without + ## blocking the JS thread; callbacks fire as work completes. + poll() + + proc createFFIContext*[T](): Result[ptr FFIContext[T], string] = + ## No worker/watchdog threads. The context lives inside a GC-rooted holder so + ## `myLibStorage` (the library `ref`) is scanned; `myLib` points at it. + let holder = SingleThreadedHolder[T]() + gSingleThreadedRoots.add(holder) + let ctx = addr holder.ctx + ctx.lock.initLock() + ctx.registeredRequests = addr ffi_types.registeredRequests + ctx.running.store(true) + ctx.myLib = addr ctx.myLibStorage + return ok(ctx) + + proc destroyFFIContext*[T](ctx: ptr FFIContext[T]): Result[void, string] = + ctx.running.store(false) + ctx.lock.deinitLock() + # Drop the GC root so the holder (and its library object) can be collected. + for i in 0 ..< gSingleThreadedRoots.len: + let h = cast[SingleThreadedHolder[T]](gSingleThreadedRoots[i]) + if cast[pointer](addr h.ctx) == cast[pointer](ctx): + gSingleThreadedRoots.del(i) + break + return ok() +else: + proc createFFIContext*[T](): Result[ptr FFIContext[T], string] = + ## This proc is called from the main thread and it creates + ## the FFI working thread. + var ctx = createShared(FFIContext[T], 1) + ctx.reqSignal = ThreadSignalPtr.new().valueOr: + return err("couldn't create reqSignal ThreadSignalPtr") + ctx.reqReceivedSignal = ThreadSignalPtr.new().valueOr: + return err("couldn't create reqReceivedSignal ThreadSignalPtr") + ctx.lock.initLock() + ctx.registeredRequests = addr ffi_types.registeredRequests + + ctx.running.store(true) + + try: + createThread(ctx.ffiThread, ffiThreadBody[T], ctx) + except ValueError, ResourceExhaustedError: + freeShared(ctx) + return err("failed to create the FFI thread: " & getCurrentExceptionMsg()) + + try: + createThread(ctx.watchdogThread, watchdogThreadBody, ctx) + except ValueError, ResourceExhaustedError: + freeShared(ctx) + return err("failed to create the watchdog thread: " & getCurrentExceptionMsg()) + + return ok(ctx) + + proc destroyFFIContext*[T](ctx: ptr FFIContext[T]): Result[void, string] = + ctx.running.store(false) + + let signaledOnTime = ctx.reqSignal.fireSync().valueOr: + return err("error in destroyFFIContext: " & $error) + if not signaledOnTime: + return err("failed to signal reqSignal on time in destroyFFIContext") + + joinThread(ctx.ffiThread) + joinThread(ctx.watchdogThread) + ctx.lock.deinitLock() + ?ctx.reqSignal.close() + ?ctx.reqReceivedSignal.close() + freeShared(ctx) + + return ok() + +template checkParams*(ctx: ptr FFIContext, callback: FFICallBack, userData: pointer) = + if not isNil(ctx): + ctx[].userData = userData + + if isNil(callback): + return RET_MISSING_CALLBACK diff --git a/wasm-deps/ffi/ffi/ffi_thread_request.nim b/wasm-deps/ffi/ffi/ffi_thread_request.nim new file mode 100644 index 000000000..93c8b0cf1 --- /dev/null +++ b/wasm-deps/ffi/ffi/ffi_thread_request.nim @@ -0,0 +1,64 @@ +## This file contains the base message request type that will be handled. +## The requests are created by the main thread and processed by +## the FFI Thread. + +import std/[json, macros], results, tables +import chronos +import ./ffi_config +when not singleThreaded: + import chronos/threadsync # ThreadSignalPtr requires threads enabled +import ./ffi_types, ./internal/ffi_macro, ./alloc + +type FFIThreadRequest* = object + callback: FFICallBack + userData: pointer + reqId*: cstring + reqContent*: pointer + +proc init*( + T: typedesc[FFIThreadRequest], + callback: FFICallBack, + userData: pointer, + reqId: cstring, + reqContent: pointer, +): ptr type T = + var ret = createShared(FFIThreadRequest) + ret[].callback = callback + ret[].userData = userData + ret[].reqId = reqId.alloc() + ret[].reqContent = reqContent + return ret + +proc deleteRequest(request: ptr FFIThreadRequest) = + deallocShared(request[].reqId) + deallocShared(request) + +proc handleRes*[T: string | void]( + res: Result[T, string], request: ptr FFIThreadRequest +) = + ## Handles the Result responses, which can either be Result[string, string] or + ## Result[void, string]. + + defer: + deleteRequest(request) + + if res.isErr(): + foreignThreadGc: + let msg = "ffi error: handleRes fireSyncRes error: " & $res.error + request[].callback( + RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), request[].userData + ) + return + + foreignThreadGc: + var msg: cstring = "" + when T is string: + msg = res.get().cstring() + request[].callback( + RET_OK, unsafeAddr msg[0], cast[csize_t](len(msg)), request[].userData + ) + return + +proc nilProcess*(reqId: cstring): Future[Result[string, string]] {.async.} = + return err("This request type is not implemented: " & $reqId) + diff --git a/wasm-deps/ffi/ffi/ffi_types.nim b/wasm-deps/ffi/ffi/ffi_types.nim new file mode 100644 index 000000000..76ead50ea --- /dev/null +++ b/wasm-deps/ffi/ffi/ffi_types.nim @@ -0,0 +1,39 @@ +import std/tables +import chronos + +################################################################################ +### Exported types + +type FFICallBack* = proc( + callerRet: cint, msg: ptr cchar, len: csize_t, userData: pointer +) {.cdecl, gcsafe, raises: [].} + +const RET_OK*: cint = 0 +const RET_ERR*: cint = 1 +const RET_MISSING_CALLBACK*: cint = 2 + +### End of exported types +################################################################################ + +################################################################################ +### FFI utils + +type FFIRequestProc* = + proc(request: pointer, reqHandler: pointer): Future[Result[string, string]] {.async.} + +template foreignThreadGc*(body: untyped) = + when declared(setupForeignThreadGc): + setupForeignThreadGc() + + body + + when declared(tearDownForeignThreadGc): + tearDownForeignThreadGc() + +## Registered requests table populated at compile time and never updated at run time. +## The key represents the request type name as cstring, e.g., "CreateNodeRequest". +## The value is a proc that handles the request asynchronously. +var registeredRequests*: Table[cstring, FFIRequestProc] + +### End of FFI utils +################################################################################ diff --git a/wasm-deps/ffi/ffi/internal/ffi_library.nim b/wasm-deps/ffi/ffi/internal/ffi_library.nim new file mode 100644 index 000000000..a9387f769 --- /dev/null +++ b/wasm-deps/ffi/ffi/internal/ffi_library.nim @@ -0,0 +1,84 @@ +import std/[macros, atomics], strformat, chronicles, chronos + +macro declareLibrary*(libraryName: static[string]): untyped = + var res = newStmtList() + + ## Generate {.pragma: exported, exportc, cdecl, raises: [].} + res.add nnkPragma.newTree( + nnkExprColonExpr.newTree(ident"pragma", ident"exported"), + ident"exportc", + ident"cdecl", + nnkExprColonExpr.newTree(ident"raises", nnkBracket.newTree()), + ) + + ## Generate {.pragma: callback, cdecl, raises: [], gcsafe.} + res.add nnkPragma.newTree( + nnkExprColonExpr.newTree(ident"pragma", ident"callback"), + ident"cdecl", + nnkExprColonExpr.newTree(ident"raises", nnkBracket.newTree()), + ident"gcsafe", + ) + + ## Generate {.passc: "-fPIC".} + res.add nnkPragma.newTree(nnkExprColonExpr.newTree(ident"passc", newLit("-fPIC"))) + + when defined(linux) and not defined(emscripten): + # NB: under emscripten (--os:linux) a `-Wl,-soname` makes emcc build a wasm + # SIDE module, which breaks EXPORTED_FUNCTIONS/malloc. The wasm/edge build is + # a MAIN module, so skip the soname there. + ## Generates {.passl: "-Wl,-soname,libwaku.so".} (considering libraryName=="waku", for example) + let soName = fmt"-Wl,-soname,lib{libraryName}.so" + res.add( + newNimNode(nnkPragma).add( + nnkExprColonExpr.newTree(ident"passl", newStrLitNode(soName)) + ) + ) + + ## proc lib{libraryName}NimMain() {.importc.} + let libNimMainName = ident(fmt"lib{libraryName}NimMain") + let importcPragma = nnkPragma.newTree(ident"importc") + let procDef = newProc( + name = libNimMainName, + params = @[ident"void"], + pragmas = importcPragma, + body = newEmptyNode(), + ) + res.add(procDef) + + # Create: var initialized: Atomic[bool] + let atomicType = nnkBracketExpr.newTree(ident("Atomic"), ident("bool")) + let varStmt = nnkVarSection.newTree( + nnkIdentDefs.newTree(ident("initialized"), atomicType, newEmptyNode()) + ) + res.add(varStmt) + + ## Android chronicles redirection + let chroniclesBlock = quote: + when defined(android) and compiles(defaultChroniclesStream.outputs[0].writer): + defaultChroniclesStream.outputs[0].writer = proc( + logLevel: LogLevel, msg: LogOutputStr + ) {.raises: [].} = + echo logLevel, msg + result.add(chroniclesBlock) + + let procName = ident("initializeLibrary") + let nimMainName = ident("lib" & libraryName & "NimMain") + + let initializeLibraryProc = quote: + proc `procName`*() {.exported.} = + if not initialized.exchange(true): + ## Every Nim library needs to call `NimMain` once exactly, + ## to initialize the Nim runtime. + ## Being `` the value given in the optional + ## compilation flag --nimMainPrefix:yourprefix + `nimMainName`() + when declared(setupForeignThreadGc): + setupForeignThreadGc() + when declared(nimGC_setStackBottom): + var locals {.volatile, noinit.}: pointer + locals = addr(locals) + nimGC_setStackBottom(locals) + + res.add(initializeLibraryProc) + + return res diff --git a/wasm-deps/ffi/ffi/internal/ffi_macro.nim b/wasm-deps/ffi/ffi/internal/ffi_macro.nim new file mode 100644 index 000000000..95e6377d5 --- /dev/null +++ b/wasm-deps/ffi/ffi/internal/ffi_macro.nim @@ -0,0 +1,549 @@ +import std/[macros, tables] +import chronos +import ../ffi_types + +proc extractFieldsFromLambda(body: NimNode): seq[NimNode] = + ## Extracts the fields (params) from the given lambda body, when using the registerReqFFI macro. + ## e.g., for: + ## registerReqFFI(CreateNodeRequest, ctx: ptr FFIContext[Waku]): + ## proc( + ## configJson: cstring, appCallbacks: AppCallbacks + ## ): Future[Result[string, string]] {.async.} = + ## ... + ## The extracted fields will be: + ## - configJson: cstring + ## - appCallbacks: AppCallbacks + ## + + var procNode = body + if procNode.kind == nnkStmtList and procNode.len == 1: + procNode = procNode[0] + if procNode.kind != nnkLambda and procNode.kind != nnkProcDef: + error "registerReqFFI expects a lambda proc, found: " & $procNode.kind + + let params = procNode[3] # parameters list + result = @[] + for p in params[1 .. ^1]: # skip return type + result.add newIdentDefs(p[0], p[1]) + + when defined(ffiDumpMacros): + echo result.repr + +proc buildRequestType(reqTypeName: NimNode, body: NimNode): NimNode = + ## Builds: + ## type * = object + ## : + ## ... + ## e.g.: + ## type CreateNodeRequest* = object + ## configJson: cstring + ## appCallbacks: AppCallbacks + ## + + var procNode = body + if procNode.kind == nnkStmtList and procNode.len == 1: + procNode = procNode[0] + if procNode.kind != nnkLambda and procNode.kind != nnkProcDef: + error "registerReqFFI expects a lambda proc, found: " & $procNode.kind + + let params = procNode[3] # formal params of the lambda + var fields: seq[NimNode] = @[] + for p in params[1 .. ^1]: # skip return type at index 0 + let name = p[0] + let typ = p[1] + # Field must be nnkIdentDefs(name, type, defaultExpr) + fields.add newTree(nnkIdentDefs, name, typ, newEmptyNode()) + + # Wrap fields in a rec list + let recList = newTree(nnkRecList, fields) + + # object type node: object [of?] [] [pragma?] recList + let objTy = newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), recList) + + # Export the type (CreateNodeRequest*) + let typeName = + if reqTypeName.kind == nnkPostfix: + reqTypeName + else: + postfix(reqTypeName, "*") + + result = + newNimNode(nnkTypeSection).add(newTree(nnkTypeDef, typeName, newEmptyNode(), objTy)) + + when defined(ffiDumpMacros): + echo result.repr + +proc buildFfiNewReqProc(reqTypeName, body: NimNode): NimNode = + ## Builds the ffiNewProc in charge of creating the FFIThreadRequest in shared memory. + ## Then, a pointer to this request will be sent to the FFI thread for processing. + ## e.g.: + ## proc ffiNewReq*(T: typedesc[CreateNodeRequest]; callback: FFICallBack; + ## userData: pointer; configJson: cstring; + ## appCallbacks: AppCallbacks): ptr FFIThreadRequest = + ## var reqObj = createShared(T) + ## reqObj[].configJson = configJson.alloc() + ## reqObj[].appCallbacks = appCallbacks + ## let typeStr`gensym2866 = $T + ## var ret`gensym2866 = FFIThreadRequest.init(callback, userData, + ## typeStr`gensym2866.cstring, reqObj) + ## return ret`gensym2866 + ## + ## This should be invoked by the ffi consumer thread (generally, main thread.) + ## Notice that the shared memory allocated by the main thread is freed by the FFI thread + ## after processing the request. + + var formalParams = newSeq[NimNode]() + + var procNode: NimNode + if body.kind == nnkStmtList and body.len == 1: + procNode = body[0] # unwrap single statement + else: + procNode = body + + if procNode.kind != nnkLambda and procNode.kind != nnkProcDef: + error "registerReqFFI expects a lambda definition. Found: " & $procNode.kind + + # T: typedesc[CreateNodeRequest] + let typedescParam = newIdentDefs( + ident("T"), # param name + nnkBracketExpr.newTree(ident("typedesc"), reqTypeName), # typedesc[T] + ) + formalParams.add(typedescParam) + + # Other fixed FFI params + formalParams.add(newIdentDefs(ident("callback"), ident("FFICallBack"))) + formalParams.add(newIdentDefs(ident("userData"), ident("pointer"))) + + # Add original lambda params + let procParams = procNode[3] + for p in procParams[1 .. ^1]: + formalParams.add(p) + + # Build `ptr FFIThreadRequest` + let retType = newNimNode(nnkPtrTy) + retType.add(ident("FFIThreadRequest")) + + formalParams = @[retType] & formalParams + + # Build body + let reqObjIdent = ident("reqObj") + var newBody = newStmtList() + newBody.add( + quote do: + var `reqObjIdent` = createShared(T) + ) + + for p in procParams[1 .. ^1]: + let fieldNameIdent = ident($p[0]) + let fieldTypeNode = p[1] + + # Extract type name as string + var typeStr: string + if fieldTypeNode.kind == nnkIdent: + typeStr = $fieldTypeNode + elif fieldTypeNode.kind == nnkBracketExpr: + typeStr = $fieldTypeNode[0] # e.g., `ptr` in `ptr[Waku]` + else: + typeStr = "" # fallback + + # Apply .alloc() only to cstrings + if typeStr == "cstring": + newBody.add( + quote do: + `reqObjIdent`[].`fieldNameIdent` = `fieldNameIdent`.alloc() + ) + else: + newBody.add( + quote do: + `reqObjIdent`[].`fieldNameIdent` = `fieldNameIdent` + ) + + # FFIThreadRequest.init using fnv1aHash32 + newBody.add( + quote do: + let typeStr = $T + var ret = + FFIThreadRequest.init(callback, userData, typeStr.cstring, `reqObjIdent`) + return ret + ) + + # Build the proc node + result = newProc( + name = postfix(ident("ffiNewReq"), "*"), + params = formalParams, + body = newBody, + pragmas = newEmptyNode(), + ) + + when defined(ffiDumpMacros): + echo result.repr + +proc buildFfiDeleteReqProc(reqTypeName: NimNode, fields: seq[NimNode]): NimNode = + ## Generates: + ## proc ffiDeleteReq(self: ptr ) = + ## deallocShared(self[].) + ## deallocShared(self) + + # Build the body + var body = newStmtList() + for f in fields: + if $f[1] == "cstring": # only dealloc cstring fields + body.add newCall( + ident("deallocShared"), + newDotExpr(newTree(nnkDerefExpr, ident("self")), ident($f[0])), + ) + + # Always free the whole object at the end + body.add newCall(ident("deallocShared"), ident("self")) + + # Build the parameter: (self: ptr ) + let selfParam = newIdentDefs(ident("self"), newTree(nnkPtrTy, reqTypeName)) + + # Build the proc definition + result = newProc( + name = postfix(ident("ffiDeleteReq"), "*"), + params = @[newEmptyNode()] & @[selfParam], # ✅ properly wrapped in a sequence + body = body, + ) + + when defined(ffiDumpMacros): + echo result.repr + +proc buildProcessFFIRequestProc(reqTypeName, reqHandler, body: NimNode): NimNode = + ## Builds, f.e.: + ## proc processFFIRequest(T: typedesc[CreateNodeRequest]; + ## configJson: cstring; + ## appCallbacks: AppCallbacks; + ## ctx: ptr FFIContext[Waku]) ... + + if reqHandler.kind != nnkExprColonExpr: + error( + "Second argument must be a typed parameter, e.g., waku: ptr Waku. Found: " & + $reqHandler.kind + ) + + let rhs = reqHandler[1] + if rhs.kind != nnkPtrTy: + error("Second argument must be a pointer type, e.g., waku: ptr Waku") + + var procNode = body + if procNode.kind == nnkStmtList and procNode.len == 1: + procNode = procNode[0] + if procNode.kind != nnkLambda and procNode.kind != nnkProcDef: + error "registerReqFFI expects a lambda definition. Found: " & $procNode.kind + + let typedescParam = + newIdentDefs(ident("T"), nnkBracketExpr.newTree(ident("typedesc"), reqTypeName)) + + # Build formal params: (returnType, request: pointer, waku: ptr Waku) + let procParams = procNode[3] + var formalParams: seq[NimNode] = @[] + formalParams.add(procParams[0]) # return type + formalParams.add(typedescParam) + formalParams.add(newIdentDefs(ident("request"), ident("pointer"))) + formalParams.add(newIdentDefs(reqHandler[0], rhs)) # e.g. waku: ptr Waku + + # Inject cast/unpack/defer into the body + let bodyNode = + if procNode.body.kind == nnkStmtList: + procNode.body + else: + newStmtList(procNode.body) + + let newBody = newStmtList() + let reqIdent = ident("req") + + newBody.add quote do: + let `reqIdent`: ptr `reqTypeName` = cast[ptr `reqTypeName`](request) + defer: + ffiDeleteReq(`reqIdent`) + + # automatically unpack fields into locals + for p in procParams[1 ..^ 1]: + let fieldName = p[0] # Ident + + newBody.add quote do: + let `fieldName` = `reqIdent`[].`fieldName` + + # Append user's lambda body + newBody.add(bodyNode) + + result = newProc( + name = postfix(ident("processFFIRequest"), "*"), + params = formalParams, + body = newBody, + procType = nnkProcDef, + pragmas = + if procNode.len >= 5: + procNode[4] + else: + newEmptyNode(), + ) + + when defined(ffiDumpMacros): + echo result.repr + +proc addNewRequestToRegistry(reqTypeName, reqHandler: NimNode): NimNode = + ## Adds a new request to the registeredRequests table. + ## The key is a representation of the request, e.g. "CreateNodeReq". + ## The value is a proc definition in charge of handling the request from FFI thread. + + # Build: request[].reqContent + let reqContent = + newDotExpr(newTree(nnkDerefExpr, ident("request")), ident("reqContent")) + + # Build Future[Result[string, string]] return type + let returnType = nnkBracketExpr.newTree( + ident("Future"), + nnkBracketExpr.newTree(ident("Result"), ident("string"), ident("string")), + ) + + # Extract the type from reqHandler (generic: ptr Waku, ptr Foo, ptr Bar, etc.) + let rhsType = + if reqHandler.kind == nnkExprColonExpr: + reqHandler[1] # Use the explicit type + else: + error "Second argument must be a typed parameter, e.g. waku: ptr Waku" + + # Build: cast[ptr Waku](reqHandler) or cast[ptr Foo](reqHandler) dynamically + let castedHandler = newTree( + nnkCast, + rhsType, # The type, e.g. ptr Waku + ident("reqHandler"), # The expression to cast + ) + + let callExpr = newCall( + newDotExpr(reqTypeName, ident("processFFIRequest")), ident("request"), castedHandler + ) + + var newBody = newStmtList() + newBody.add( + quote do: + return await `callExpr` + ) + + # Build: + # proc(request: pointer, reqHandler: pointer): + # Future[Result[string, string]] {.async.} = + # CreateNodeRequest.processFFIRequest(request, reqHandler) + let asyncProc = newProc( + name = newEmptyNode(), # anonymous proc + params = + @[ + returnType, + newIdentDefs(ident("request"), ident("pointer")), + newIdentDefs(ident("reqHandler"), ident("pointer")), + ], + body = newBody, + pragmas = nnkPragma.newTree(ident("async")), + ) + + let reqTypeNameStr = $reqTypeName + + let key = newLit($reqTypeName) + # Generate: registeredRequests["CreateNodeRequest"] = + result = + newAssignment(newTree(nnkBracketExpr, ident("registeredRequests"), key), asyncProc) + + when defined(ffiDumpMacros): + echo result.repr + +macro registerReqFFI*(reqTypeName, reqHandler, body: untyped): untyped = + ## Registers a request that will be handled by the FFI/working thread. + ## The request should be sent from the ffi consumer thread. + ## + ## e.g.: + ## In this example, we register a CreateNodeRequest that will be handled by a proc that contains + ## the provided lambda body and parameters, by the FFI/working thread. + ## + ## The lambda passed to this macro must: + ## - only have no-GC'ed types. + ## - Return Future[Result[string, string]] and be annotated with {.async.} + ## And notice that the returned values will be sent back to the ffi consumer thread. + ## + ## registerReqFFI(CreateNodeRequest, ctx: ptr FFIContext[Waku]): + ## proc( + ## configJson: cstring, appCallbacks: AppCallbacks + ## ): Future[Result[string, string]] {.async.} = + ## ctx.myLib[] = (await createWaku(configJson, cast[AppCallbacks](appCallbacks))).valueOr: + ## return err($error) + ## return ok("") + ## + ## On the other hand, the created FFI request should be dispatched from the ffi consumer thread + ## (generally, the main thread) following something like: + ## + ## ffi.sendRequestToFFIThread( + ## ctx, CreateNodeRequest.ffiNewReq(callback, userData, configJson, appCallbacks) + ## ).isOkOr: + ## ... + ## ... + ## + + # Extract lambda params to generate fields + let fields = extractFieldsFromLambda(body) + + let typeDef = buildRequestType(reqTypeName, body) + let ffiNewReqProc = buildFfiNewReqProc(reqTypeName, body) + let processProc = buildProcessFFIRequestProc(reqTypeName, reqHandler, body) + let addNewReqToReg = addNewRequestToRegistry(reqTypeName, reqHandler) + let deleteProc = buildFfiDeleteReqProc(reqTypeName, fields) + result = newStmtList(typeDef, ffiNewReqProc, deleteProc, processProc, addNewReqToReg) + + when defined(ffiDumpMacros): + echo result.repr + +macro processReq*( + reqType, ctx, callback, userData: untyped, args: varargs[untyped] +): untyped = + ## Expands T.processReq(ctx, callback, userData, a, b, ...) + ## e.g.: + ## waku_dial_peerReq.processReq(ctx, callback, userData, peerMultiAddr, protocol, timeoutMs) + ## + + var callArgs = @[reqType, callback, userData] + for a in args: + callArgs.add a + + let newReqCall = newCall(ident("ffiNewReq"), callArgs) + + let sendCall = newCall( + newDotExpr(ident("ffi_context"), ident("sendRequestToFFIThread")), ctx, newReqCall + ) + + result = quote: + block: + let res = `sendCall` + if res.isErr(): + let msg = "error in sendRequestToFFIThread: " & res.error + `callback`(RET_ERR, unsafeAddr msg[0], cast[csize_t](msg.len), `userData`) + return RET_ERR + return RET_OK + + when defined(ffiDumpMacros): + echo result.repr + +macro ffi*(prc: untyped): untyped = + ## Defines an FFI-exported proc that registers a request handler to be executed + ## asynchronously in the FFI thread. + ## + ## {.ffi.} implicitly implies: ...Return[Future[Result[string, string]] {.async.} + ## + ## When using {.ffi.}, the first three parameters must be: + ## - ctx: ptr FFIContext[T] <-- T is the type that handles the FFI requests + ## - callback: FFICallBack + ## - userData: pointer + ## Then, additional parameters may be defined as needed, after these first three, always + ## considering that only no-GC'ed (or C-like) types are allowed. + ## + ## e.g.: + ## proc waku_version( + ## ctx: ptr FFIContext[Waku], callback: FFICallBack, userData: pointer + ## ) {.ffi.} = + ## return ok(WakuNodeVersionString) + ## + ## e.g2.: + ## proc waku_start( + ## ctx: ptr FFIContext[Waku], callback: FFICallBack, userData: pointer + ## ) {.ffi.} = + ## (await startWaku(ctx[].myLib)).isOkOr: + ## error "START_NODE failed", error = error + ## return err("failed to start: " & $error) + ## return ok("") + ## + ## e.g3.: + ## proc waku_peer_exchange_request( + ## ctx: ptr FFIContext[Waku], + ## callback: FFICallBack, + ## userData: pointer, + ## numPeers: uint64, + ## ) {.ffi.} = + ## let numValidPeers = (await performPeerExchangeRequestTo(numPeers, ctx.myLib[])).valueOr: + ## error "waku_peer_exchange_request failed", error = error + ## return err("failed peer exchange: " & $error) + ## return ok($numValidPeers) + ## + ## In these examples, notice that ctx.myLib is of type "ptr Waku", being Waku main library type. + ## + + let procName = prc[0] + let formalParams = prc[3] + let bodyNode = prc[^1] + + if formalParams.len < 2: + error("`.ffi.` procs require at least 1 parameter") + + let firstParam = formalParams[1] + let paramIdent = firstParam[0] + let paramType = firstParam[1] + + let reqName = ident($procName & "Req") + let returnType = ident("cint") + + # Build parameter list (skip return type) + var newParams = newSeq[NimNode]() + newParams.add(returnType) + for i in 1 ..< formalParams.len: + newParams.add(newIdentDefs(formalParams[i][0], formalParams[i][1])) + + # Build Future[Result[string, string]] return type + let futReturnType = quote: + Future[Result[string, string]] + + var userParams = newSeq[NimNode]() + userParams.add(futReturnType) + if formalParams.len > 3: + for i in 4 ..< formalParams.len: + userParams.add(newIdentDefs(formalParams[i][0], formalParams[i][1])) + + # Build argument list for processReq + var argsList = newSeq[NimNode]() + for i in 1 ..< formalParams.len: + argsList.add(formalParams[i][0]) + + # 1. Build the dot expression. e.g.: waku_is_onlineReq.processReq + let dotExpr = newTree(nnkDotExpr, reqName, ident"processReq") + + # 2. Build the call node with dotExpr as callee + let callNode = newTree(nnkCall, dotExpr) + for arg in argsList: + callNode.add(arg) + + # Proc body + let ffiBody = newStmtList( + quote do: + initializeLibrary() + if not isNil(ctx): + ctx[].userData = userData + if isNil(callback): + return RET_MISSING_CALLBACK + ) + + ffiBody.add(callNode) + + # Under emscripten, `dynlib` makes Nim emit `emcc -shared` (a wasm SIDE module), + # which breaks EXPORTED_FUNCTIONS/malloc. The wasm/edge build is a MAIN module, + # so export with plain `exportc` there. + let exportPragmas = + when defined(emscripten): + newTree(nnkPragma, ident "exportc", ident "cdecl") + else: + newTree(nnkPragma, ident "dynlib", ident "exportc", ident "cdecl") + let ffiProc = + newProc(name = procName, params = newParams, body = ffiBody, pragmas = exportPragmas) + + var anonymousProcNode = newProc( + name = newEmptyNode(), # anonymous proc + params = userParams, + body = newStmtList(bodyNode), + pragmas = newTree(nnkPragma, ident"async"), + ) + + # registerReqFFI wrapper + let registerReq = quote: + registerReqFFI(`reqName`, `paramIdent`: `paramType`): + `anonymousProcNode` + + result = newStmtList(registerReq, ffiProc) + + when defined(ffiDumpMacros): + echo result.repr diff --git a/wasm-deps/ffi/ffi/logging.nim b/wasm-deps/ffi/ffi/logging.nim new file mode 100644 index 000000000..b82ec117a --- /dev/null +++ b/wasm-deps/ffi/ffi/logging.nim @@ -0,0 +1,106 @@ +## This code has been copied and addapted from `status-im/nimbu-eth2` project. +## Link: https://github.com/status-im/nimbus-eth2/blob/c585b0a5b1ae4d55af38ad7f4715ad455e791552/beacon_chain/nimbus_binary_common.nim +## This is also copied in logos-messaging-nim repository (2025-12-10) +import + std/[typetraits, os, strutils, syncio], + chronicles, + chronicles/log_output, + chronicles/topics_registry + +export chronicles.LogLevel + +{.push raises: [].} + +type LogFormat* = enum + TEXT + JSON + +## Utils + +proc stripAnsi(v: string): string = + ## Copied from: https://github.com/status-im/nimbus-eth2/blob/stable/beacon_chain/nimbus_binary_common.nim#L41 + ## Silly chronicles, colors is a compile-time property + var + res = newStringOfCap(v.len) + i: int + + while i < v.len: + let c = v[i] + if c == '\x1b': + var + x = i + 1 + found = false + + while x < v.len: # look for [..m + let c2 = v[x] + if x == i + 1: + if c2 != '[': + break + else: + if c2 in {'0' .. '9'} + {';'}: + discard # keep looking + elif c2 == 'm': + i = x + 1 + found = true + break + else: + break + inc x + + if found: # skip adding c + continue + res.add c + inc i + + res + +proc writeAndFlush(f: syncio.File, s: LogOutputStr) = + try: + f.write(s) + f.flushFile() + except CatchableError: + logLoggingFailure(cstring(s), getCurrentException()) + +## Setup + +proc setupLogLevel(level: LogLevel) = + # TODO: Support per topic level configuratio + topics_registry.setLogLevel(level) + +proc setupLogFormat(format: LogFormat, color = true) = + proc noOutputWriter(logLevel: LogLevel, msg: LogOutputStr) = + discard + + proc stdoutOutputWriter(logLevel: LogLevel, msg: LogOutputStr) = + writeAndFlush(syncio.stdout, msg) + + proc stdoutNoColorOutputWriter(logLevel: LogLevel, msg: LogOutputStr) = + writeAndFlush(syncio.stdout, stripAnsi(msg)) + + when defaultChroniclesStream.outputs.type.arity == 2: + case format + of LogFormat.Text: + defaultChroniclesStream.outputs[0].writer = + if color: stdoutOutputWriter else: stdoutNoColorOutputWriter + defaultChroniclesStream.outputs[1].writer = noOutputWriter + of LogFormat.Json: + defaultChroniclesStream.outputs[0].writer = noOutputWriter + defaultChroniclesStream.outputs[1].writer = stdoutOutputWriter + else: + {. + warning: + "the present module should be compiled with '-d:chronicles_default_output_device=dynamic' " & + "and '-d:chronicles_sinks=\"textlines,json\"' options" + .} + +proc setupLog*(level: LogLevel, format: LogFormat) = + ## Logging setup + # Adhere to NO_COLOR initiative: https://no-color.org/ + let color = + try: + not parseBool(os.getEnv("NO_COLOR", "false")) + except CatchableError: + true + + setupLogLevel(level) + setupLogFormat(format, color) diff --git a/wasm-deps/ffi/nimblemeta.json b/wasm-deps/ffi/nimblemeta.json new file mode 100644 index 000000000..7dfc5b79e --- /dev/null +++ b/wasm-deps/ffi/nimblemeta.json @@ -0,0 +1,23 @@ +{ + "version": 1, + "metaData": { + "url": "https://github.com/logos-messaging/nim-ffi", + "downloadMethod": "git", + "vcsRevision": "06111de155253b34e47ed2aaed1d61d08d62cc1b", + "files": [ + "/ffi.nim", + "/ffi/ffi_types.nim", + "/ffi.nimble", + "/ffi/ffi_thread_request.nim", + "/ffi/alloc.nim", + "/ffi/logging.nim", + "/ffi/internal/ffi_library.nim", + "/ffi/internal/ffi_macro.nim", + "/ffi/ffi_context.nim" + ], + "binaries": [], + "specialVersions": [ + "0.1.3" + ] + } +} \ No newline at end of file diff --git a/wasm-deps/shim-include/sys/queue.h b/wasm-deps/shim-include/sys/queue.h new file mode 100644 index 000000000..2470343d0 --- /dev/null +++ b/wasm-deps/shim-include/sys/queue.h @@ -0,0 +1,909 @@ +/* + * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/*- + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)queue.h 8.5 (Berkeley) 8/20/94 + */ + +#ifndef _SYS_QUEUE_H_ +#define _SYS_QUEUE_H_ + +#ifndef __improbable +#define __improbable(x) (x) /* noop in userspace */ +#endif /* __improbable */ + +/* + * This file defines five types of data structures: singly-linked lists, + * singly-linked tail queues, lists, tail queues, and circular queues. + * + * A singly-linked list is headed by a single forward pointer. The elements + * are singly linked for minimum space and pointer manipulation overhead at + * the expense of O(n) removal for arbitrary elements. New elements can be + * added to the list after an existing element or at the head of the list. + * Elements being removed from the head of the list should use the explicit + * macro for this purpose for optimum efficiency. A singly-linked list may + * only be traversed in the forward direction. Singly-linked lists are ideal + * for applications with large datasets and few or no removals or for + * implementing a LIFO queue. + * + * A singly-linked tail queue is headed by a pair of pointers, one to the + * head of the list and the other to the tail of the list. The elements are + * singly linked for minimum space and pointer manipulation overhead at the + * expense of O(n) removal for arbitrary elements. New elements can be added + * to the list after an existing element, at the head of the list, or at the + * end of the list. Elements being removed from the head of the tail queue + * should use the explicit macro for this purpose for optimum efficiency. + * A singly-linked tail queue may only be traversed in the forward direction. + * Singly-linked tail queues are ideal for applications with large datasets + * and few or no removals or for implementing a FIFO queue. + * + * A list is headed by a single forward pointer (or an array of forward + * pointers for a hash table header). The elements are doubly linked + * so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before + * or after an existing element or at the head of the list. A list + * may only be traversed in the forward direction. + * + * A tail queue is headed by a pair of pointers, one to the head of the + * list and the other to the tail of the list. The elements are doubly + * linked so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before or + * after an existing element, at the head of the list, or at the end of + * the list. A tail queue may be traversed in either direction. + * + * A circle queue is headed by a pair of pointers, one to the head of the + * list and the other to the tail of the list. The elements are doubly + * linked so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before or after + * an existing element, at the head of the list, or at the end of the list. + * A circle queue may be traversed in either direction, but has a more + * complex end of list detection. + * Note that circle queues are deprecated, because, as the removal log + * in FreeBSD states, "CIRCLEQs are a disgrace to everything Knuth taught + * us in Volume 1 Chapter 2. [...] Use TAILQ instead, it provides the same + * functionality." Code using them will continue to compile, but they + * are no longer documented on the man page. + * + * For details on the use of these macros, see the queue(3) manual page. + * + * + * SLIST LIST STAILQ TAILQ CIRCLEQ + * _HEAD + + + + + + * _HEAD_INITIALIZER + + + + - + * _ENTRY + + + + + + * _INIT + + + + + + * _EMPTY + + + + + + * _FIRST + + + + + + * _NEXT + + + + + + * _PREV - - - + + + * _LAST - - + + + + * _FOREACH + + + + + + * _FOREACH_SAFE + + + + - + * _FOREACH_REVERSE - - - + - + * _FOREACH_REVERSE_SAFE - - - + - + * _INSERT_HEAD + + + + + + * _INSERT_BEFORE - + - + + + * _INSERT_AFTER + + + + + + * _INSERT_TAIL - - + + + + * _CONCAT - - + + - + * _REMOVE_AFTER + - + - - + * _REMOVE_HEAD + - + - - + * _REMOVE_HEAD_UNTIL - - + - - + * _REMOVE + + + + + + * _SWAP - + + + - + * + */ +#ifdef QUEUE_MACRO_DEBUG +/* Store the last 2 places the queue element or head was altered */ +struct qm_trace { + char * lastfile; + int lastline; + char * prevfile; + int prevline; +}; + +#define TRACEBUF struct qm_trace trace; +#define TRASHIT(x) do {(x) = (void *)-1;} while (0) + +#define QMD_TRACE_HEAD(head) do { \ + (head)->trace.prevline = (head)->trace.lastline; \ + (head)->trace.prevfile = (head)->trace.lastfile; \ + (head)->trace.lastline = __LINE__; \ + (head)->trace.lastfile = __FILE__; \ +} while (0) + +#define QMD_TRACE_ELEM(elem) do { \ + (elem)->trace.prevline = (elem)->trace.lastline; \ + (elem)->trace.prevfile = (elem)->trace.lastfile; \ + (elem)->trace.lastline = __LINE__; \ + (elem)->trace.lastfile = __FILE__; \ +} while (0) + +#else +#define QMD_TRACE_ELEM(elem) +#define QMD_TRACE_HEAD(head) +#define TRACEBUF +#define TRASHIT(x) +#endif /* QUEUE_MACRO_DEBUG */ + +/* + * Horrible macros to enable use of code that was meant to be C-specific + * (and which push struct onto type) in C++; without these, C++ code + * that uses these macros in the context of a class will blow up + * due to "struct" being preprended to "type" by the macros, causing + * inconsistent use of tags. + * + * This approach is necessary because these are macros; we have to use + * these on a per-macro basis (because the queues are implemented as + * macros, disabling this warning in the scope of the header file is + * insufficient), whuch means we can't use #pragma, and have to use + * _Pragma. We only need to use these for the queue macros that + * prepend "struct" to "type" and will cause C++ to blow up. + */ +#if defined(__clang__) && defined(__cplusplus) +#define __MISMATCH_TAGS_PUSH \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wmismatched-tags\"") +#define __MISMATCH_TAGS_POP \ + _Pragma("clang diagnostic pop") +#else +#define __MISMATCH_TAGS_PUSH +#define __MISMATCH_TAGS_POP +#endif + +/*! + * Ensures that these macros can safely be used in structs when compiling with + * clang. The macros do not allow for nullability attributes to be specified due + * to how they are expanded. For example: + * + * SLIST_HEAD(, foo _Nullable) bar; + * + * expands to + * + * struct { + * struct foo _Nullable *slh_first; + * } + * + * which is not valid because the nullability specifier has to apply to the + * pointer. So just ignore nullability completeness in all the places where this + * is an issue. + */ +#if defined(__clang__) +#define __NULLABILITY_COMPLETENESS_PUSH \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wnullability-completeness\"") +#define __NULLABILITY_COMPLETENESS_POP \ + _Pragma("clang diagnostic pop") +#else +#define __NULLABILITY_COMPLETENESS_PUSH +#define __NULLABILITY_COMPLETENESS_POP +#endif + +/* + * Singly-linked List declarations. + */ +#define SLIST_HEAD(name, type) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +struct name { \ + struct type *slh_first; /* first element */ \ +} \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +#define SLIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define SLIST_ENTRY(type) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +struct { \ + struct type *sle_next; /* next element */ \ +} \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +/* + * Singly-linked List functions. + */ +#define SLIST_EMPTY(head) ((head)->slh_first == NULL) + +#define SLIST_FIRST(head) ((head)->slh_first) + +#define SLIST_FOREACH(var, head, field) \ + for ((var) = SLIST_FIRST((head)); \ + (var); \ + (var) = SLIST_NEXT((var), field)) + +#define SLIST_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = SLIST_FIRST((head)); \ + (var) && ((tvar) = SLIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define SLIST_FOREACH_PREVPTR(var, varp, head, field) \ + for ((varp) = &SLIST_FIRST((head)); \ + ((var) = *(varp)) != NULL; \ + (varp) = &SLIST_NEXT((var), field)) + +#define SLIST_INIT(head) do { \ + SLIST_FIRST((head)) = NULL; \ +} while (0) + +#define SLIST_INSERT_AFTER(slistelm, elm, field) do { \ + SLIST_NEXT((elm), field) = SLIST_NEXT((slistelm), field); \ + SLIST_NEXT((slistelm), field) = (elm); \ +} while (0) + +#define SLIST_INSERT_HEAD(head, elm, field) do { \ + SLIST_NEXT((elm), field) = SLIST_FIRST((head)); \ + SLIST_FIRST((head)) = (elm); \ +} while (0) + +#define SLIST_NEXT(elm, field) ((elm)->field.sle_next) + +#define SLIST_REMOVE(head, elm, type, field) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +do { \ + if (SLIST_FIRST((head)) == (elm)) { \ + SLIST_REMOVE_HEAD((head), field); \ + } \ + else { \ + struct type *curelm = SLIST_FIRST((head)); \ + while (SLIST_NEXT(curelm, field) != (elm)) \ + curelm = SLIST_NEXT(curelm, field); \ + SLIST_REMOVE_AFTER(curelm, field); \ + } \ + TRASHIT((elm)->field.sle_next); \ +} while (0) \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +#define SLIST_REMOVE_AFTER(elm, field) do { \ + SLIST_NEXT(elm, field) = \ + SLIST_NEXT(SLIST_NEXT(elm, field), field); \ +} while (0) + +#define SLIST_REMOVE_HEAD(head, field) do { \ + SLIST_FIRST((head)) = SLIST_NEXT(SLIST_FIRST((head)), field); \ +} while (0) + +/* + * Singly-linked Tail queue declarations. + */ +#define STAILQ_HEAD(name, type) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +struct name { \ + struct type *stqh_first;/* first element */ \ + struct type **stqh_last;/* addr of last next element */ \ +} \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +#define STAILQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).stqh_first } + +#define STAILQ_ENTRY(type) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +struct { \ + struct type *stqe_next; /* next element */ \ +} \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +/* + * Singly-linked Tail queue functions. + */ +#define STAILQ_CONCAT(head1, head2) do { \ + if (!STAILQ_EMPTY((head2))) { \ + *(head1)->stqh_last = (head2)->stqh_first; \ + (head1)->stqh_last = (head2)->stqh_last; \ + STAILQ_INIT((head2)); \ + } \ +} while (0) + +#define STAILQ_EMPTY(head) ((head)->stqh_first == NULL) + +#define STAILQ_FIRST(head) ((head)->stqh_first) + +#define STAILQ_FOREACH(var, head, field) \ + for((var) = STAILQ_FIRST((head)); \ + (var); \ + (var) = STAILQ_NEXT((var), field)) + + +#define STAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = STAILQ_FIRST((head)); \ + (var) && ((tvar) = STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define STAILQ_INIT(head) do { \ + STAILQ_FIRST((head)) = NULL; \ + (head)->stqh_last = &STAILQ_FIRST((head)); \ +} while (0) + +#define STAILQ_INSERT_AFTER(head, tqelm, elm, field) do { \ + if ((STAILQ_NEXT((elm), field) = STAILQ_NEXT((tqelm), field)) == NULL)\ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ + STAILQ_NEXT((tqelm), field) = (elm); \ +} while (0) + +#define STAILQ_INSERT_HEAD(head, elm, field) do { \ + if ((STAILQ_NEXT((elm), field) = STAILQ_FIRST((head))) == NULL) \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ + STAILQ_FIRST((head)) = (elm); \ +} while (0) + +#define STAILQ_INSERT_TAIL(head, elm, field) do { \ + STAILQ_NEXT((elm), field) = NULL; \ + *(head)->stqh_last = (elm); \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ +} while (0) + +#define STAILQ_LAST(head, type, field) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ + (STAILQ_EMPTY((head)) ? \ + NULL : \ + ((struct type *)(void *) \ + ((char *)((head)->stqh_last) - __offsetof(struct type, field))))\ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +#define STAILQ_NEXT(elm, field) ((elm)->field.stqe_next) + +#define STAILQ_REMOVE(head, elm, type, field) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +do { \ + if (STAILQ_FIRST((head)) == (elm)) { \ + STAILQ_REMOVE_HEAD((head), field); \ + } \ + else { \ + struct type *curelm = STAILQ_FIRST((head)); \ + while (STAILQ_NEXT(curelm, field) != (elm)) \ + curelm = STAILQ_NEXT(curelm, field); \ + STAILQ_REMOVE_AFTER(head, curelm, field); \ + } \ + TRASHIT((elm)->field.stqe_next); \ +} while (0) \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +#define STAILQ_REMOVE_HEAD(head, field) do { \ + if ((STAILQ_FIRST((head)) = \ + STAILQ_NEXT(STAILQ_FIRST((head)), field)) == NULL) \ + (head)->stqh_last = &STAILQ_FIRST((head)); \ +} while (0) + +#define STAILQ_REMOVE_HEAD_UNTIL(head, elm, field) do { \ + if ((STAILQ_FIRST((head)) = STAILQ_NEXT((elm), field)) == NULL) \ + (head)->stqh_last = &STAILQ_FIRST((head)); \ +} while (0) + +#define STAILQ_REMOVE_AFTER(head, elm, field) do { \ + if ((STAILQ_NEXT(elm, field) = \ + STAILQ_NEXT(STAILQ_NEXT(elm, field), field)) == NULL) \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ +} while (0) + +#define STAILQ_SWAP(head1, head2, type) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +do { \ + struct type *swap_first = STAILQ_FIRST(head1); \ + struct type **swap_last = (head1)->stqh_last; \ + STAILQ_FIRST(head1) = STAILQ_FIRST(head2); \ + (head1)->stqh_last = (head2)->stqh_last; \ + STAILQ_FIRST(head2) = swap_first; \ + (head2)->stqh_last = swap_last; \ + if (STAILQ_EMPTY(head1)) \ + (head1)->stqh_last = &STAILQ_FIRST(head1); \ + if (STAILQ_EMPTY(head2)) \ + (head2)->stqh_last = &STAILQ_FIRST(head2); \ +} while (0) \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + + +/* + * List declarations. + */ +#define LIST_HEAD(name, type) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +struct name { \ + struct type *lh_first; /* first element */ \ +} \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +#define LIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define LIST_ENTRY(type) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +struct { \ + struct type *le_next; /* next element */ \ + struct type **le_prev; /* address of previous next element */ \ +} \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +/* + * List functions. + */ + +#define LIST_CHECK_HEAD(head, field) +#define LIST_CHECK_NEXT(elm, field) +#define LIST_CHECK_PREV(elm, field) + +#define LIST_EMPTY(head) ((head)->lh_first == NULL) + +#define LIST_FIRST(head) ((head)->lh_first) + +#define LIST_FOREACH(var, head, field) \ + for ((var) = LIST_FIRST((head)); \ + (var); \ + (var) = LIST_NEXT((var), field)) + +#define LIST_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = LIST_FIRST((head)); \ + (var) && ((tvar) = LIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define LIST_INIT(head) do { \ + LIST_FIRST((head)) = NULL; \ +} while (0) + +#define LIST_INSERT_AFTER(listelm, elm, field) do { \ + LIST_CHECK_NEXT(listelm, field); \ + if ((LIST_NEXT((elm), field) = LIST_NEXT((listelm), field)) != NULL)\ + LIST_NEXT((listelm), field)->field.le_prev = \ + &LIST_NEXT((elm), field); \ + LIST_NEXT((listelm), field) = (elm); \ + (elm)->field.le_prev = &LIST_NEXT((listelm), field); \ +} while (0) + +#define LIST_INSERT_BEFORE(listelm, elm, field) do { \ + LIST_CHECK_PREV(listelm, field); \ + (elm)->field.le_prev = (listelm)->field.le_prev; \ + LIST_NEXT((elm), field) = (listelm); \ + *(listelm)->field.le_prev = (elm); \ + (listelm)->field.le_prev = &LIST_NEXT((elm), field); \ +} while (0) + +#define LIST_INSERT_HEAD(head, elm, field) do { \ + LIST_CHECK_HEAD((head), field); \ + if ((LIST_NEXT((elm), field) = LIST_FIRST((head))) != NULL) \ + LIST_FIRST((head))->field.le_prev = &LIST_NEXT((elm), field);\ + LIST_FIRST((head)) = (elm); \ + (elm)->field.le_prev = &LIST_FIRST((head)); \ +} while (0) + +#define LIST_NEXT(elm, field) ((elm)->field.le_next) + +#define LIST_REMOVE(elm, field) do { \ + LIST_CHECK_NEXT(elm, field); \ + LIST_CHECK_PREV(elm, field); \ + if (LIST_NEXT((elm), field) != NULL) \ + LIST_NEXT((elm), field)->field.le_prev = \ + (elm)->field.le_prev; \ + *(elm)->field.le_prev = LIST_NEXT((elm), field); \ + TRASHIT((elm)->field.le_next); \ + TRASHIT((elm)->field.le_prev); \ +} while (0) + +#define LIST_SWAP(head1, head2, type, field) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +do { \ + struct type *swap_tmp = LIST_FIRST((head1)); \ + LIST_FIRST((head1)) = LIST_FIRST((head2)); \ + LIST_FIRST((head2)) = swap_tmp; \ + if ((swap_tmp = LIST_FIRST((head1))) != NULL) \ + swap_tmp->field.le_prev = &LIST_FIRST((head1)); \ + if ((swap_tmp = LIST_FIRST((head2))) != NULL) \ + swap_tmp->field.le_prev = &LIST_FIRST((head2)); \ +} while (0) \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +/* + * Tail queue declarations. + */ +#define TAILQ_HEAD(name, type) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +struct name { \ + struct type *tqh_first; /* first element */ \ + struct type **tqh_last; /* addr of last next element */ \ + TRACEBUF \ +} \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +#define TAILQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).tqh_first } + +#define TAILQ_ENTRY(type) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +struct { \ + struct type *tqe_next; /* next element */ \ + struct type **tqe_prev; /* address of previous next element */ \ + TRACEBUF \ +} \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +/* + * Tail queue functions. + */ +#define TAILQ_CHECK_HEAD(head, field) +#define TAILQ_CHECK_NEXT(elm, field) +#define TAILQ_CHECK_PREV(elm, field) + +#define TAILQ_CONCAT(head1, head2, field) do { \ + if (!TAILQ_EMPTY(head2)) { \ + *(head1)->tqh_last = (head2)->tqh_first; \ + (head2)->tqh_first->field.tqe_prev = (head1)->tqh_last; \ + (head1)->tqh_last = (head2)->tqh_last; \ + TAILQ_INIT((head2)); \ + QMD_TRACE_HEAD(head1); \ + QMD_TRACE_HEAD(head2); \ + } \ +} while (0) + +#define TAILQ_EMPTY(head) ((head)->tqh_first == NULL) + +#define TAILQ_FIRST(head) ((head)->tqh_first) + +#define TAILQ_FOREACH(var, head, field) \ + for ((var) = TAILQ_FIRST((head)); \ + (var); \ + (var) = TAILQ_NEXT((var), field)) + +#define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = TAILQ_FIRST((head)); \ + (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define TAILQ_FOREACH_REVERSE(var, head, headname, field) \ + for ((var) = TAILQ_LAST((head), headname); \ + (var); \ + (var) = TAILQ_PREV((var), headname, field)) + +#define TAILQ_FOREACH_REVERSE_SAFE(var, head, headname, field, tvar) \ + for ((var) = TAILQ_LAST((head), headname); \ + (var) && ((tvar) = TAILQ_PREV((var), headname, field), 1); \ + (var) = (tvar)) + + +#define TAILQ_INIT(head) do { \ + TAILQ_FIRST((head)) = NULL; \ + (head)->tqh_last = &TAILQ_FIRST((head)); \ + QMD_TRACE_HEAD(head); \ +} while (0) + + +#define TAILQ_INSERT_AFTER(head, listelm, elm, field) do { \ + TAILQ_CHECK_NEXT(listelm, field); \ + if ((TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field)) != NULL)\ + TAILQ_NEXT((elm), field)->field.tqe_prev = \ + &TAILQ_NEXT((elm), field); \ + else { \ + (head)->tqh_last = &TAILQ_NEXT((elm), field); \ + QMD_TRACE_HEAD(head); \ + } \ + TAILQ_NEXT((listelm), field) = (elm); \ + (elm)->field.tqe_prev = &TAILQ_NEXT((listelm), field); \ + QMD_TRACE_ELEM(&(elm)->field); \ + QMD_TRACE_ELEM(&listelm->field); \ +} while (0) + +#define TAILQ_INSERT_BEFORE(listelm, elm, field) do { \ + TAILQ_CHECK_PREV(listelm, field); \ + (elm)->field.tqe_prev = (listelm)->field.tqe_prev; \ + TAILQ_NEXT((elm), field) = (listelm); \ + *(listelm)->field.tqe_prev = (elm); \ + (listelm)->field.tqe_prev = &TAILQ_NEXT((elm), field); \ + QMD_TRACE_ELEM(&(elm)->field); \ + QMD_TRACE_ELEM(&listelm->field); \ +} while (0) + +#define TAILQ_INSERT_HEAD(head, elm, field) do { \ + TAILQ_CHECK_HEAD(head, field); \ + if ((TAILQ_NEXT((elm), field) = TAILQ_FIRST((head))) != NULL) \ + TAILQ_FIRST((head))->field.tqe_prev = \ + &TAILQ_NEXT((elm), field); \ + else \ + (head)->tqh_last = &TAILQ_NEXT((elm), field); \ + TAILQ_FIRST((head)) = (elm); \ + (elm)->field.tqe_prev = &TAILQ_FIRST((head)); \ + QMD_TRACE_HEAD(head); \ + QMD_TRACE_ELEM(&(elm)->field); \ +} while (0) + +#define TAILQ_INSERT_TAIL(head, elm, field) do { \ + TAILQ_NEXT((elm), field) = NULL; \ + (elm)->field.tqe_prev = (head)->tqh_last; \ + *(head)->tqh_last = (elm); \ + (head)->tqh_last = &TAILQ_NEXT((elm), field); \ + QMD_TRACE_HEAD(head); \ + QMD_TRACE_ELEM(&(elm)->field); \ +} while (0) + +#define TAILQ_LAST(head, headname) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ + (*(((struct headname *)((head)->tqh_last))->tqh_last)) \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +#define TAILQ_NEXT(elm, field) ((elm)->field.tqe_next) + +#define TAILQ_PREV(elm, headname, field) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ + (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last)) \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +#define TAILQ_REMOVE(head, elm, field) do { \ + TAILQ_CHECK_NEXT(elm, field); \ + TAILQ_CHECK_PREV(elm, field); \ + if ((TAILQ_NEXT((elm), field)) != NULL) \ + TAILQ_NEXT((elm), field)->field.tqe_prev = \ + (elm)->field.tqe_prev; \ + else { \ + (head)->tqh_last = (elm)->field.tqe_prev; \ + QMD_TRACE_HEAD(head); \ + } \ + *(elm)->field.tqe_prev = TAILQ_NEXT((elm), field); \ + TRASHIT((elm)->field.tqe_next); \ + TRASHIT((elm)->field.tqe_prev); \ + QMD_TRACE_ELEM(&(elm)->field); \ +} while (0) + +/* + * Why did they switch to spaces for this one macro? + */ +#define TAILQ_SWAP(head1, head2, type, field) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +do { \ + struct type *swap_first = (head1)->tqh_first; \ + struct type **swap_last = (head1)->tqh_last; \ + (head1)->tqh_first = (head2)->tqh_first; \ + (head1)->tqh_last = (head2)->tqh_last; \ + (head2)->tqh_first = swap_first; \ + (head2)->tqh_last = swap_last; \ + if ((swap_first = (head1)->tqh_first) != NULL) \ + swap_first->field.tqe_prev = &(head1)->tqh_first; \ + else \ + (head1)->tqh_last = &(head1)->tqh_first; \ + if ((swap_first = (head2)->tqh_first) != NULL) \ + swap_first->field.tqe_prev = &(head2)->tqh_first; \ + else \ + (head2)->tqh_last = &(head2)->tqh_first; \ +} while (0) \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +/* + * Circular queue definitions. + */ +#define CIRCLEQ_HEAD(name, type) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +struct name { \ + struct type *cqh_first; /* first element */ \ + struct type *cqh_last; /* last element */ \ +} \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +#define CIRCLEQ_ENTRY(type) \ +__MISMATCH_TAGS_PUSH \ +__NULLABILITY_COMPLETENESS_PUSH \ +struct { \ + struct type *cqe_next; /* next element */ \ + struct type *cqe_prev; /* previous element */ \ +} \ +__NULLABILITY_COMPLETENESS_POP \ +__MISMATCH_TAGS_POP + +/* + * Circular queue functions. + */ +#define CIRCLEQ_CHECK_HEAD(head, field) +#define CIRCLEQ_CHECK_NEXT(head, elm, field) +#define CIRCLEQ_CHECK_PREV(head, elm, field) + +#define CIRCLEQ_EMPTY(head) ((head)->cqh_first == (void *)(head)) + +#define CIRCLEQ_FIRST(head) ((head)->cqh_first) + +#define CIRCLEQ_FOREACH(var, head, field) \ + for((var) = (head)->cqh_first; \ + (var) != (void *)(head); \ + (var) = (var)->field.cqe_next) + +#define CIRCLEQ_INIT(head) do { \ + (head)->cqh_first = (void *)(head); \ + (head)->cqh_last = (void *)(head); \ +} while (0) + +#define CIRCLEQ_INSERT_AFTER(head, listelm, elm, field) do { \ + CIRCLEQ_CHECK_NEXT(head, listelm, field); \ + (elm)->field.cqe_next = (listelm)->field.cqe_next; \ + (elm)->field.cqe_prev = (listelm); \ + if ((listelm)->field.cqe_next == (void *)(head)) \ + (head)->cqh_last = (elm); \ + else \ + (listelm)->field.cqe_next->field.cqe_prev = (elm); \ + (listelm)->field.cqe_next = (elm); \ +} while (0) + +#define CIRCLEQ_INSERT_BEFORE(head, listelm, elm, field) do { \ + CIRCLEQ_CHECK_PREV(head, listelm, field); \ + (elm)->field.cqe_next = (listelm); \ + (elm)->field.cqe_prev = (listelm)->field.cqe_prev; \ + if ((listelm)->field.cqe_prev == (void *)(head)) \ + (head)->cqh_first = (elm); \ + else \ + (listelm)->field.cqe_prev->field.cqe_next = (elm); \ + (listelm)->field.cqe_prev = (elm); \ +} while (0) + +#define CIRCLEQ_INSERT_HEAD(head, elm, field) do { \ + CIRCLEQ_CHECK_HEAD(head, field); \ + (elm)->field.cqe_next = (head)->cqh_first; \ + (elm)->field.cqe_prev = (void *)(head); \ + if ((head)->cqh_last == (void *)(head)) \ + (head)->cqh_last = (elm); \ + else \ + (head)->cqh_first->field.cqe_prev = (elm); \ + (head)->cqh_first = (elm); \ +} while (0) + +#define CIRCLEQ_INSERT_TAIL(head, elm, field) do { \ + (elm)->field.cqe_next = (void *)(head); \ + (elm)->field.cqe_prev = (head)->cqh_last; \ + if ((head)->cqh_first == (void *)(head)) \ + (head)->cqh_first = (elm); \ + else \ + (head)->cqh_last->field.cqe_next = (elm); \ + (head)->cqh_last = (elm); \ +} while (0) + +#define CIRCLEQ_LAST(head) ((head)->cqh_last) + +#define CIRCLEQ_NEXT(elm, field) ((elm)->field.cqe_next) + +#define CIRCLEQ_PREV(elm, field) ((elm)->field.cqe_prev) + +#define CIRCLEQ_REMOVE(head, elm, field) do { \ + CIRCLEQ_CHECK_NEXT(head, elm, field); \ + CIRCLEQ_CHECK_PREV(head, elm, field); \ + if ((elm)->field.cqe_next == (void *)(head)) \ + (head)->cqh_last = (elm)->field.cqe_prev; \ + else \ + (elm)->field.cqe_next->field.cqe_prev = \ + (elm)->field.cqe_prev; \ + if ((elm)->field.cqe_prev == (void *)(head)) \ + (head)->cqh_first = (elm)->field.cqe_next; \ + else \ + (elm)->field.cqe_prev->field.cqe_next = \ + (elm)->field.cqe_next; \ +} while (0) + +#ifdef _KERNEL + +#if NOTFB31 + +/* + * XXX insque() and remque() are an old way of handling certain queues. + * They bogusly assumes that all queue heads look alike. + */ + +struct quehead { + struct quehead *qh_link; + struct quehead *qh_rlink; +}; + +#ifdef __GNUC__ +#define chkquenext(a) +#define chkqueprev(a) + +static __inline void +insque(void *a, void *b) +{ + struct quehead *element = (struct quehead *)a, + *head = (struct quehead *)b; + chkquenext(head); + + element->qh_link = head->qh_link; + element->qh_rlink = head; + head->qh_link = element; + element->qh_link->qh_rlink = element; +} + +static __inline void +remque(void *a) +{ + struct quehead *element = (struct quehead *)a; + chkquenext(element); + chkqueprev(element); + + element->qh_link->qh_rlink = element->qh_rlink; + element->qh_rlink->qh_link = element->qh_link; + element->qh_rlink = 0; +} + +#else /* !__GNUC__ */ + +void insque(void *a, void *b); +void remque(void *a); + +#endif /* __GNUC__ */ + +#endif /* NOTFB31 */ +#endif /* _KERNEL */ + +#endif /* !_SYS_QUEUE_H_ */