feat(host): {.ffiHost.} metadata + Go wrapper (increment 5)

5a: record {.ffiHost.} procs in a compile-time registry (FFIHostMeta /
ffiHostRegistry), populated by the macro, so generators can see host fns.

5b: the Go generator emits an idiomatic wrapper over the host C ABI:
- a single //export cgo trampoline backs every host fn; a cgo.Handle in
  userData selects the Go closure;
- the closure runs on a fresh GOROUTINE so the FFI thread is never blocked
  (the non-blocking contract), then answers via <lib>_host_complete by token;
- a per-host `Set<Name>(func(string) (string, error))` method registers it.

Validated end to end with `go run` (examples/host_demo): Go UseToken -> Nim
{.ffi.} handler -> await fetchToken {.ffiHost.} -> Go trampoline -> goroutine
runs the closure -> host_complete -> future resolves on the loop thread ->
"token[TOK-session]" back in Go. Timer's Go output is unchanged (no host fns);
its regenerated .h just gains the always-exported host ABI decls.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

examples/host_demo/go_bindings/.gitignore

@@ -0,0 +1,3 @@
+*.dylib
+*.so
+example/example

examples/host_demo/go_bindings/Makefile

@@ -0,0 +1,35 @@
+# Build the Nim dylib next to the generated Go package and run the host-callback
+# example.
+#
+#   make run     # build libhost_demo + run the example
+#   make clean
+#
+# The generated package's cgo directives use ${SRCDIR}, so the library only has
+# to sit in this directory (-L/-rpath point here). It is compiled from the repo
+# root so the vendored Nimble dependencies resolve.
+
+REPO_ROOT := $(abspath ../../..)
+NIM_SRC   := $(REPO_ROOT)/examples/host_demo/host_demo.nim
+
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Darwin)
+  LIBNAME := libhost_demo.dylib
+else
+  LIBNAME := libhost_demo.so
+endif
+
+NIMFLAGS := --mm:orc -d:chronicles_log_level=WARN --app:lib --noMain \
+            --nimMainPrefix:libhost_demo
+
+.PHONY: all run clean
+
+all: $(LIBNAME)
+
+$(LIBNAME):
+	cd $(REPO_ROOT) && nim c $(NIMFLAGS) -o:$(CURDIR)/$(LIBNAME) $(NIM_SRC)
+
+run: $(LIBNAME)
+	cd example && go run .
+
+clean:
+	rm -f $(LIBNAME) example/example

examples/host_demo/go_bindings/example/go.mod

@@ -0,0 +1,7 @@
+module example
+
+go 1.21
+
+require host_demo v0.0.0
+
+replace host_demo => ../

examples/host_demo/go_bindings/example/main.go

@@ -0,0 +1,37 @@
+// Go example for a {.ffiHost.} host callback.
+//
+// `fetchToken` is implemented HERE (the Go app) and registered with
+// SetFetchToken. When we call UseToken, the Nim library calls back into this Go
+// closure for a token — the closure runs on a goroutine the generated wrapper
+// spawns (never blocking the FFI thread) and answers via host_complete.
+package main
+
+import (
+	"fmt"
+	"log"
+
+	hd "host_demo"
+)
+
+func main() {
+	node, err := hd.NewHost_demo()
+	if err != nil {
+		log.Fatalf("create: %v", err)
+	}
+	defer node.Destroy()
+
+	// The host's implementation of the {.ffiHost.} fetchToken.
+	node.SetFetchToken(func(key string) (string, error) {
+		return "TOK-" + key, nil
+	})
+
+	res, err := node.UseToken("session")
+	if err != nil {
+		log.Fatalf("useToken: %v", err)
+	}
+	fmt.Printf("result: %s\n", res)
+	if res != "token[TOK-session]" {
+		log.Fatalf("unexpected result: %q", res)
+	}
+	fmt.Println("OK")
+}

examples/host_demo/go_bindings/go.mod

@@ -0,0 +1,3 @@
+module host_demo
+
+go 1.21

examples/host_demo/go_bindings/host_demo.go

@@ -0,0 +1,173 @@
+// Code generated by nim-ffi Go codegen. DO NOT EDIT.
+package host_demo
+
+/*
+#cgo CFLAGS: -I${SRCDIR}
+#cgo LDFLAGS: -L${SRCDIR} -lhost_demo -Wl,-rpath,${SRCDIR}
+#include "host_demo.h"
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+
+extern void host_demoGoEvent(int ret, char* msg, size_t len, void* userData);
+extern void host_demoHostTrampoline(uint64_t token, char* req, size_t reqLen, void* userData);
+static int host_demoRegisterHost(void* ctx, const char* name, void* ud) {
+  return host_demo_register_host_fn(ctx, name, (FFIHostFn)host_demoHostTrampoline, ud);
+}
+
+typedef struct {
+  int ret; char* msg; size_t len; int done;
+  pthread_mutex_t mu; pthread_cond_t cv;
+} Host_demoResp;
+
+static Host_demoResp* host_demoRespNew() {
+  Host_demoResp* r = (Host_demoResp*)calloc(1, sizeof(Host_demoResp));
+  pthread_mutex_init(&r->mu, NULL); pthread_cond_init(&r->cv, NULL);
+  return r;
+}
+static void host_demoRespFree(Host_demoResp* r) {
+  if (!r) return;
+  if (r->msg) free(r->msg);
+  pthread_mutex_destroy(&r->mu); pthread_cond_destroy(&r->cv); free(r);
+}
+static int host_demoRespRet(Host_demoResp* r) { return r->ret; }
+static char* host_demoRespMsg(Host_demoResp* r) { return r->msg; }
+static size_t host_demoRespLen(Host_demoResp* r) { return r->len; }
+
+static void host_demoRespCb(int ret, const char* msg, size_t len, void* ud) {
+  Host_demoResp* r = (Host_demoResp*)ud;
+  pthread_mutex_lock(&r->mu);
+  r->ret = ret;
+  // Native ABI: (msg, len) is the raw result (RET_OK) or error (RET_ERR).
+  // Copy it so it survives past the callback.
+  char* e = (char*)malloc(len + 1); if (e) { memcpy(e, msg, len); e[len] = 0; }
+  r->msg = e; r->len = len;
+  r->done = 1; pthread_cond_signal(&r->cv); pthread_mutex_unlock(&r->mu);
+}
+static void host_demoRespWait(Host_demoResp* r) {
+  pthread_mutex_lock(&r->mu);
+  while (!r->done) pthread_cond_wait(&r->cv, &r->mu);
+  pthread_mutex_unlock(&r->mu);
+}
+
+static void* host_demoCall_demo_create(Host_demoResp* r) {
+  void* ctx = demo_create(host_demoRespCb, r);
+  host_demoRespWait(r);
+  return ctx;
+}
+static int host_demoCall_use_token(void* ctx, const char* key, Host_demoResp* r) {
+  int rc = use_token(ctx, host_demoRespCb, r, key);
+  if (rc == RET_OK) host_demoRespWait(r);
+  return rc;
+}
+static int host_demoCall_demo_destroy(void* ctx) { return demo_destroy(ctx); }
+static uint64_t host_demoRegisterEvents(void* ctx) { return host_demo_add_event_listener(ctx, "", (FFICallBack)host_demoGoEvent, ctx); }
+*/
+import "C"
+
+import (
+	"errors"
+	"runtime/cgo"
+	"sync"
+	"unsafe"
+)
+
+type resultSlot struct {
+	val  any
+	err  error
+	done chan struct{}
+}
+
+type Host_demoNode struct {
+	ctx unsafe.Pointer
+}
+
+// goStr extracts and frees the captured response string.
+func respStr(r *C.Host_demoResp) string {
+	return C.GoStringN(C.host_demoRespMsg(r), C.int(C.host_demoRespLen(r)))
+}
+
+var (
+	eventMu      sync.Mutex
+	eventHandler func(string)
+)
+
+// SetEventHandler installs the catch-all handler for library-initiated
+// events (delivered as raw JSON strings).
+func (n *Host_demoNode) SetEventHandler(h func(string)) {
+	eventMu.Lock()
+	eventHandler = h
+	eventMu.Unlock()
+	C.host_demoRegisterEvents(n.ctx)
+}
+
+//export host_demoGoEvent
+func host_demoGoEvent(ret C.int, msg *C.char, length C.size_t, userData unsafe.Pointer) {
+	eventMu.Lock()
+	h := eventHandler
+	eventMu.Unlock()
+	if h != nil && ret == C.RET_OK {
+		h(C.GoStringN(msg, C.int(length)))
+	}
+}
+
+type hostEntry struct {
+	ctx unsafe.Pointer
+	fn  func(string) (string, error)
+}
+
+//export host_demoHostTrampoline
+func host_demoHostTrampoline(token C.uint64_t, req *C.char, reqLen C.size_t, userData unsafe.Pointer) {
+	e := cgo.Handle(uintptr(userData)).Value().(hostEntry)
+	reqStr := C.GoStringN(req, C.int(reqLen))
+	go func() {
+		res, err := e.fn(reqStr)
+		if err != nil {
+			msg := err.Error()
+			cmsg := C.CString(msg)
+			C.host_demo_host_complete(e.ctx, token, C.int(C.RET_ERR), cmsg, C.size_t(len(msg)))
+			C.free(unsafe.Pointer(cmsg))
+		} else {
+			cmsg := C.CString(res)
+			C.host_demo_host_complete(e.ctx, token, C.int(C.RET_OK), cmsg, C.size_t(len(res)))
+			C.free(unsafe.Pointer(cmsg))
+		}
+	}()
+}
+
+// SetFetchToken registers the host implementation of the 'fetch_token' {.ffiHost.} call.
+func (n *Host_demoNode) SetFetchToken(fn func(string) (string, error)) {
+	handle := cgo.NewHandle(hostEntry{ctx: n.ctx, fn: fn})
+	cname := C.CString("fetch_token")
+	C.host_demoRegisterHost(n.ctx, cname, unsafe.Pointer(handle))
+	C.free(unsafe.Pointer(cname))
+}
+
+func NewHost_demo() (*Host_demoNode, error) {
+	r := C.host_demoRespNew()
+	defer C.host_demoRespFree(r)
+	ctx := C.host_demoCall_demo_create(r)
+	if C.host_demoRespRet(r) != C.RET_OK {
+		return nil, errors.New(respStr(r))
+	}
+	return &Host_demoNode{ctx: ctx}, nil
+}
+
+func (n *Host_demoNode) UseToken(key string) (string, error) {
+	c_key := C.CString(key)
+	defer C.free(unsafe.Pointer(c_key))
+	r := C.host_demoRespNew()
+	defer C.host_demoRespFree(r)
+	C.host_demoCall_use_token(n.ctx, c_key, r)
+	if C.host_demoRespRet(r) != C.RET_OK {
+		return "", errors.New(respStr(r))
+	}
+	return respStr(r), nil
+}
+
+func (n *Host_demoNode) Destroy() error {
+	if C.host_demoCall_demo_destroy(n.ctx) != C.RET_OK {
+		return errors.New("host_demo destroy failed")
+	}
+	return nil
+}

examples/host_demo/go_bindings/host_demo.h

@@ -0,0 +1,58 @@
+// Generated by nim-ffi C codegen. Do not edit by hand.
+//
+// Native (zero-serialization) C ABI. Each call delivers its result to the
+// callback. On RET_OK:
+//   - string-returning procs: (msg, len) is the raw string bytes (not
+//     NUL-terminated; use len).
+//   - struct-returning procs: msg is a pointer to the returned C struct — cast
+//     it to `const <Type>*` (len is sizeof). It is valid ONLY for the duration
+//     of the callback; copy out anything you need before returning. The library
+//     deep-frees it right after the callback (you free nothing).
+// On RET_ERR, (msg, len) is the raw error text. A `<name>_cbor` variant of each
+// proc also exists for generic/cross-language callers that prefer CBOR.
+#ifndef NIM_FFI_GEN_HOST_DEMO_H
+#define NIM_FFI_GEN_HOST_DEMO_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef NIM_FFI_RET_CODES
+#define NIM_FFI_RET_CODES
+#define RET_OK 0
+#define RET_ERR 1
+#define RET_MISSING_CALLBACK 2
+#endif
+
+#ifndef NIM_FFI_CALLBACK_T
+#define NIM_FFI_CALLBACK_T
+typedef void (*FFICallBack)(int callerRet, const char *msg, size_t len, void *userData);
+#endif
+
+
+
+void *demo_create(FFICallBack callback, void *userData);
+
+int demo_destroy(void *ctx);
+
+int use_token(void *ctx, FFICallBack callback, void *userData, const char* key);
+
+uint64_t host_demo_add_event_listener(void *ctx, const char *eventName, FFICallBack callback, void *userData);
+int host_demo_remove_event_listener(void *ctx, uint64_t listenerId);
+
+// --- host callbacks ({.ffiHost.}) — host-implemented functions --------
+#ifndef NIM_FFI_HOST_FN_T
+#define NIM_FFI_HOST_FN_T
+typedef void (*FFIHostFn)(uint64_t token, const char *req, size_t reqLen, void *userData);
+#endif
+int host_demo_register_host_fn(void *ctx, const char *name, FFIHostFn fn, void *userData);
+int host_demo_host_complete(void *ctx, uint64_t token, int ret, const char *msg, size_t len);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif /* NIM_FFI_GEN_HOST_DEMO_H */

examples/host_demo/host_demo.nim

@@ -0,0 +1,31 @@
+## Minimal example exercising a {.ffiHost.} host callback end-to-end from Go.
+##
+## `fetchToken` is implemented by the *host* (the Go app); `useToken` is a normal
+## {.ffi.} method the host calls, which in turn asks the host for a token via
+## `fetchToken` and awaits it. This proves the inverted call direction across the
+## real FFI boundary with the generated Go wrapper.
+
+import ffi, chronos, results
+
+type Demo = object
+
+declareLibrary("host_demo", Demo)
+
+# Ctor first: the {.ffiCtor.} macro declares the per-lib FFI pool that the
+# {.ffi.} method below references.
+proc demoCreate(): Future[Result[Demo, string]] {.ffiCtor.} =
+  return ok(Demo())
+
+proc demoDestroy(d: Demo) {.ffiDtor.} =
+  discard
+
+# Host-implemented: the Go app registers this with SetFetchToken.
+proc fetchToken(key: string): Future[Result[string, string]] {.ffiHost.}
+
+# A {.ffi.} method the host calls; it asks the host for a token and wraps it.
+proc useToken(d: Demo, key: string): Future[Result[string, string]] {.ffi.} =
+  let tok = (await fetchToken(key)).valueOr:
+    return err("host error: " & error)
+  return ok("token[" & tok & "]")
+
+genBindings()

examples/timer/go_bindings/my_timer.h

@@ -114,6 +114,14 @@ int my_timer_destroy(void *ctx);
 uint64_t my_timer_add_event_listener(void *ctx, const char *eventName, FFICallBack callback, void *userData);
 int my_timer_remove_event_listener(void *ctx, uint64_t listenerId);
 
+// --- host callbacks ({.ffiHost.}) — host-implemented functions --------
+#ifndef NIM_FFI_HOST_FN_T
+#define NIM_FFI_HOST_FN_T
+typedef void (*FFIHostFn)(uint64_t token, const char *req, size_t reqLen, void *userData);
+#endif
+int my_timer_register_host_fn(void *ctx, const char *name, FFIHostFn fn, void *userData);
+int my_timer_host_complete(void *ctx, uint64_t token, int ret, const char *msg, size_t len);
+
 #ifdef __cplusplus
 } // extern "C"
 #endif

ffi/codegen/go.nim

@@ -334,6 +334,7 @@ proc generateGoFile*(
     types: seq[FFITypeMeta],
     libName: string,
     events: seq[FFIEventMeta] = @[],
+    hosts: seq[FFIHostMeta] = @[],
 ): string =
   let nodeType = capitalizeFirstLetter(libName) & "Node"
   let respT = capitalizeFirstLetter(libName) & "Resp"
@@ -370,6 +371,25 @@ proc generateGoFile*(
   L.add(
     "extern void " & libName & "GoEvent(int ret, char* msg, size_t len, void* userData);"
   )
+  # Host callbacks ({.ffiHost.}): a single exported Go trampoline backs every
+  # registered host fn; the static helper hands its address to register_host_fn
+  # (cgo drops const, so the forward decl uses char*).
+  if hosts.len > 0:
+    L.add(
+      "extern void " & libName &
+        "HostTrampoline(uint64_t token, char* req, size_t reqLen, void* userData);"
+    )
+    L.add(
+      "static int " & libName &
+        "RegisterHost(void* ctx, const char* name, void* ud) {"
+    )
+    # cgo exports the trampoline with `char*` (it drops const); cast to FFIHostFn
+    # so the function-pointer types match.
+    L.add(
+      "  return " & libName & "_register_host_fn(ctx, name, (FFIHostFn)" & libName &
+        "HostTrampoline, ud);"
+    )
+    L.add("}")
   # One exported Go result callback per struct-returning proc (it reads the typed
   # return POD in-callback). Forward-declared here so cgo's `char*` shape matches.
   for p in procs:
@@ -559,6 +579,63 @@ proc generateGoFile*(
   L.add("}")
   L.add("")
 
+  # ---- host callbacks ({.ffiHost.}) ----------------------------------------
+  # One exported trampoline serves all host fns; the cgo.Handle in userData
+  # selects which Go closure. The closure runs on a fresh goroutine so the FFI
+  # thread is never blocked (the non-blocking contract), then answers by token.
+  if hosts.len > 0:
+    L.add("type hostEntry struct {")
+    L.add("\tctx unsafe.Pointer")
+    L.add("\tfn  func(string) (string, error)")
+    L.add("}")
+    L.add("")
+    L.add("//export " & libName & "HostTrampoline")
+    L.add(
+      "func " & libName &
+        "HostTrampoline(token C.uint64_t, req *C.char, reqLen C.size_t, userData unsafe.Pointer) {"
+    )
+    L.add("\te := cgo.Handle(uintptr(userData)).Value().(hostEntry)")
+    L.add("\treqStr := C.GoStringN(req, C.int(reqLen))")
+    L.add("\tgo func() {")
+    L.add("\t\tres, err := e.fn(reqStr)")
+    L.add("\t\tif err != nil {")
+    L.add("\t\t\tmsg := err.Error()")
+    L.add("\t\t\tcmsg := C.CString(msg)")
+    L.add(
+      "\t\t\tC." & libName &
+        "_host_complete(e.ctx, token, C.int(C.RET_ERR), cmsg, C.size_t(len(msg)))"
+    )
+    L.add("\t\t\tC.free(unsafe.Pointer(cmsg))")
+    L.add("\t\t} else {")
+    L.add("\t\t\tcmsg := C.CString(res)")
+    L.add(
+      "\t\t\tC." & libName &
+        "_host_complete(e.ctx, token, C.int(C.RET_OK), cmsg, C.size_t(len(res)))"
+    )
+    L.add("\t\t\tC.free(unsafe.Pointer(cmsg))")
+    L.add("\t\t}")
+    L.add("\t}()")
+    L.add("}")
+    L.add("")
+    for h in hosts:
+      let setName = "Set" & capitalizeFirstLetter(h.nimProcName)
+      L.add(
+        "// " & setName & " registers the host implementation of the '" & h.wireName &
+          "' {.ffiHost.} call."
+      )
+      L.add(
+        "func (n *" & nodeType & ") " & setName &
+          "(fn func(string) (string, error)) {"
+      )
+      L.add("\thandle := cgo.NewHandle(hostEntry{ctx: n.ctx, fn: fn})")
+      L.add("\tcname := C.CString(\"" & h.wireName & "\")")
+      L.add(
+        "\tC." & libName & "RegisterHost(n.ctx, cname, unsafe.Pointer(handle))"
+      )
+      L.add("\tC.free(unsafe.Pointer(cname))")
+      L.add("}")
+      L.add("")
+
   # ---- constructor ---------------------------------------------------------
   if haveCtor:
     let (goParams, conv, callArgs) = goParamConv(ctor.extraParams, types)
@@ -690,9 +767,11 @@ proc generateGoBindings*(
     outputDir: string,
     nimSrcRelPath: string,
     events: seq[FFIEventMeta] = @[],
+    hosts: seq[FFIHostMeta] = @[],
 ) =
   writeFile(
-    outputDir / (libName & ".go"), generateGoFile(procs, types, libName, events)
+    outputDir / (libName & ".go"),
+    generateGoFile(procs, types, libName, events, hosts),
   )
   # cgo `#include "<lib>.h"` resolves against this package directory, so emit the
   # native C header here too — the Go package is then self-contained (just stage

ffi/codegen/meta.nim

@@ -40,10 +40,24 @@ type
     libName*: string
     payloadTypeName*: string
 
+  FFIHostMeta* = object
+    ## Host-provided function declared with `{.ffiHost.}` — the host implements
+    ## it and a `{.ffi.}` handler awaits it. `wireName` is the snake_case name
+    ## the host registers under. First slice: one `string` arg, `string` return;
+    ## `argName`/`argTypeName`/`returnTypeName` carry the shape so generators can
+    ## emit a typed wrapper.
+    wireName*: string
+    nimProcName*: string
+    libName*: string
+    argName*: string
+    argTypeName*: string
+    returnTypeName*: string
+
 # Compile-time registries populated by the macros
 var ffiProcRegistry* {.compileTime.}: seq[FFIProcMeta]
 var ffiTypeRegistry* {.compileTime.}: seq[FFITypeMeta]
 var ffiEventRegistry* {.compileTime.}: seq[FFIEventMeta]
+var ffiHostRegistry* {.compileTime.}: seq[FFIHostMeta]
 var currentLibName* {.compileTime.}: string
 
 # Target language for binding generation; override with -d:targetLang=cpp

ffi/internal/ffi_macro.nim

@@ -1982,7 +1982,21 @@ macro ffiHost*(prc: untyped): untyped =
     block:
       let raw = $procName
       if raw.endsWith("*"): raw[0 ..^ 2] else: raw
-  let wireNameLit = newStrLitNode(camelToSnakeCase(procNameStr))
+  let wireName = camelToSnakeCase(procNameStr)
+  let wireNameLit = newStrLitNode(wireName)
+
+  # Record metadata so the per-language generators can emit an idiomatic wrapper
+  # (register a closure + a trampoline that answers via <lib>_host_complete).
+  ffiHostRegistry.add(
+    FFIHostMeta(
+      wireName: wireName,
+      nimProcName: procNameStr,
+      libName: currentLibName,
+      argName: $argName,
+      argTypeName: "string",
+      returnTypeName: "string",
+    )
+  )
 
   # The generated async body: resolve the thread-local host context, look up the
   # registered fn, allocate a pending token, invoke the host with the raw request
@@ -2088,7 +2102,7 @@ macro genBindings*(
     of "go":
       generateGoBindings(
         ffiProcRegistry, ffiTypeRegistry, libName, outputDir, nimSrcRelPath,
-        ffiEventRegistry,
+        ffiEventRegistry, ffiHostRegistry,
       )
     else:
       error(