diff --git a/CHANGELOG.md b/CHANGELOG.md index 497f2e2..f6e3654 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,9 @@ All notable changes to this project are documented in this file. composite fields follow. The `c` proc-dispatch path and the foreign (C++ / Rust) generators are still pending, so `c` remains rejected on proc/ctor/dtor/event annotations for now. +- `tests/bench/bench_codec.nim` (+ `nimble bench_codec`): a single-process + microbenchmark comparing the `cbor` and `c` codecs across payload shapes, + isolating codec cost from the (identical) thread/callback round-trip. - Queue-overflow handling: when the bounded event queue is full, the library sets a sticky "stuck" flag, logs an error, fires `not_responding` from the event thread, and rejects subsequent diff --git a/ffi.nimble b/ffi.nimble index dc168db..6be83b5 100644 --- a/ffi.nimble +++ b/ffi.nimble @@ -85,6 +85,11 @@ task test_serial, "Run CBOR codec unit tests": exec "nim c -r " & nimFlagsOrc & " tests/unit/test_serial.nim" exec "nim c -r " & nimFlagsRefc & " tests/unit/test_serial.nim" +task bench_codec, "Microbenchmark: cbor vs c (cwire) wire-format codecs": + # Built with -d:danger so the numbers reflect optimized codegen, not the + # debug build. Not part of `test` — timing is a measurement, not a gate. + exec "nim c -r " & nimFlagsOrc & " -d:danger tests/bench/bench_codec.nim" + task test_cpp_e2e, "Build and run the C++ end-to-end tests for the timer example": # Regenerate the C++ bindings so the suite always runs against fresh codegen. runOrQuit "nimble genbindings_cpp" diff --git a/tests/bench/README.md b/tests/bench/README.md new file mode 100644 index 0000000..63ffede --- /dev/null +++ b/tests/bench/README.md @@ -0,0 +1,47 @@ +# FFI wire-format codec benchmark + +`bench_codec.nim` is a single-process Nim microbenchmark comparing the two FFI +wire-format codecs head-to-head on identical payloads: + +- **cbor** — `cborEncode` / `cborDecode`, self-describing bytes over + `seq[byte]`. The codec the `cbor` ABI uses on every boundary crossing. +- **c (cwire)** — `cwirePack` / `cwireUnpack` / `cwireFree`, flat C-struct + shared-memory packing. The codec the `c` ABI uses, emitted for every + `{.ffi: "abi = c".}` type as its `_CWire` companion. + +Both paths run in the same process on the same values, so the numbers isolate +**codec cost only** — no thread hop, no callback dispatch, no chronos work. The +full FFI round-trip (thread channel + callback) is identical for both ABIs, so +the codec is where the ABI difference actually lives. + +## Running + +```sh +nimble bench_codec +# or directly, with the size sweep extended to 1 MiB: +nim c -r --mm:orc -d:danger tests/bench/bench_codec.nim --include-1mib +``` + +Build with `-d:danger` (the nimble task does) so the figures reflect optimized +codegen rather than a debug build. + +## Payload shapes covered + +| Type | Shape | +|------------------|----------------------------------------------------| +| `EchoRequest` | 1 string + 1 int (small struct) | +| `EchoResponse` | 2 strings | +| `ComplexRequest` | `seq[EchoRequest]`, `seq[string]`, `Option[...]` | +| `BytesPayload` | `seq[byte]`, swept 100 B → 150 KiB (`--include-1mib` adds 1 MiB) | + +## Interpreting + +Small structs are dominated by CBOR's per-field tag/length framing, so cwire +wins by a large factor. As payloads grow into big `seq[byte]` blobs, both codecs +become `memcpy`-bound and the ratio converges toward ~1×. The byte-blob sweep +also reports throughput (MiB/s) for each codec. + +> Note: this benchmark exercises the `c` ABI **codec** (the cwire companions on +> the Nim side). Wiring the `c` ABI through the full proc-dispatch path and the +> foreign (C++/Rust) generators is tracked separately; only `cbor` currently +> generates working end-to-end bindings. diff --git a/tests/bench/bench_codec.nim b/tests/bench/bench_codec.nim new file mode 100644 index 0000000..10879e4 --- /dev/null +++ b/tests/bench/bench_codec.nim @@ -0,0 +1,169 @@ +## Microbenchmark comparing the `cbor` and `c` (cwire) codecs on identical +## payloads in one process, isolating codec cost from the thread/callback hop. + +import std/[monotimes, options, os, strformat, strutils, times] +import ../../ffi + +# Payload types mirror the timer example so e2e and bench stay comparable. + +type EchoRequest {.ffi: "abi = c".} = object + message: string + delayMs: int + +type EchoResponse {.ffi: "abi = c".} = object + echoed: string + timerName: string + +type ComplexRequest {.ffi: "abi = c".} = object + messages: seq[EchoRequest] + tags: seq[string] + note: Option[string] + retries: Option[int] + +type BytesPayload {.ffi: "abi = c".} = object + payload: seq[byte] + +# Flush the cwire companions for the types above. +genBindings() + +const Iterations = 200_000 +const HeaderWidth = 66 + +proc reportTiming(label: string, perOp: float, iters: int) = + let padded = label & repeat(' ', max(0, 46 - label.len())) + echo " " & padded & " " & formatFloat(perOp, ffDecimal, 2) & " ns/op (" & $iters & + " iters)" + +template timeItN(labelArg: string, iters: int, body: untyped): float = + ## Returns nanoseconds per iteration averaged over `iters`. + let start = getMonoTime() + for i in 0 ..< iters: + body + let elapsed = (getMonoTime() - start).inNanoseconds.float + let perOp = elapsed / iters.float + reportTiming(labelArg, perOp, iters) + perOp + +template timeIt(labelArg: string, body: untyped): float = + timeItN(labelArg, Iterations, body) + +proc mibPerSec(sizeBytes: int, perOpNs: float): float = + (sizeBytes.float / perOpNs) * 1_000_000_000.0 / (1024.0 * 1024.0) + +proc benchEchoRequest() = + echo "── EchoRequest (small: 1 string + 1 int) ─────────────────────────" + let req = EchoRequest(message: "hello world", delayMs: 100) + + let cborRtNs = timeIt "cbor encode + decode": + let bytes = cborEncode(req) + let back = cborDecode(bytes, EchoRequest).valueOr: + doAssert false, error + default(EchoRequest) + doAssert back.delayMs == 100 + + let cwireRtNs = timeIt "cwire pack + unpack + free": + var wire: EchoRequest_CWire + cwirePack(wire, req) + let back = cwireUnpack(wire) + doAssert back.delayMs == 100 + cwireFree(wire) + + echo &" ratio (cbor RT / cwire RT) = {cborRtNs / cwireRtNs:.2f}x" + echo "" + +proc benchEchoResponse() = + echo "── EchoResponse (small: 2 strings) ───────────────────────────────" + let resp = EchoResponse(echoed: "hello world", timerName: "bench-timer") + + let cborRtNs = timeIt "cbor encode + decode": + let bytes = cborEncode(resp) + let back = cborDecode(bytes, EchoResponse).valueOr: + doAssert false, error + default(EchoResponse) + doAssert back.timerName == "bench-timer" + + let cwireRtNs = timeIt "cwire pack + unpack + free": + var wire: EchoResponse_CWire + cwirePack(wire, resp) + let back = cwireUnpack(wire) + doAssert back.timerName == "bench-timer" + cwireFree(wire) + + echo &" ratio (cbor RT / cwire RT) = {cborRtNs / cwireRtNs:.2f}x" + echo "" + +proc benchComplexRequest() = + echo "── ComplexRequest (seq[EchoRequest] x4, seq[string] x3, options) ─" + let req = ComplexRequest( + messages: @[ + EchoRequest(message: "alpha", delayMs: 1), + EchoRequest(message: "beta", delayMs: 2), + EchoRequest(message: "gamma", delayMs: 3), + EchoRequest(message: "delta", delayMs: 4), + ], + tags: @["fast", "async", "bench"], + note: some("a slightly longer note that survives the round-trip"), + retries: some(7), + ) + + let cborRtNs = timeIt "cbor encode + decode": + let bytes = cborEncode(req) + let back = cborDecode(bytes, ComplexRequest).valueOr: + doAssert false, error + default(ComplexRequest) + doAssert back.messages.len() == 4 + + let cwireRtNs = timeIt "cwire pack + unpack + free": + var wire: ComplexRequest_CWire + cwirePack(wire, req) + let back = cwireUnpack(wire) + doAssert back.messages.len() == 4 + cwireFree(wire) + + echo &" ratio (cbor RT / cwire RT) = {cborRtNs / cwireRtNs:.2f}x" + echo "" + +proc benchBytesAtSize(sizeBytes: int, iters: int) = + # Iteration count scales down with size: a 1 MiB op costs ~100s of µs. + let label = "── BytesPayload (seq[byte], " & $sizeBytes & " bytes) " + echo label & repeat('-', max(0, HeaderWidth - label.len())) + var blob = newSeq[byte](sizeBytes) + for i in 0 ..< sizeBytes: + blob[i] = byte(i and 0xff) + let req = BytesPayload(payload: blob) + + let cborRtNs = timeItN("cbor encode + decode", iters): + let bytes = cborEncode(req) + let back = cborDecode(bytes, BytesPayload).valueOr: + doAssert false, error + default(BytesPayload) + doAssert back.payload.len() == sizeBytes + + let cwireRtNs = timeItN("cwire pack + unpack + free", iters): + var wire: BytesPayload_CWire + cwirePack(wire, req) + let back = cwireUnpack(wire) + doAssert back.payload.len() == sizeBytes + cwireFree(wire) + + echo &" ratio (cbor RT / cwire RT) = {cborRtNs / cwireRtNs:.2f}x" & + &" | throughput cbor={mibPerSec(sizeBytes, cborRtNs):.1f} MiB/s cwire={mibPerSec(sizeBytes, cwireRtNs):.1f} MiB/s" + echo "" + +echo &"nim-ffi codec microbench (cbor vs c) {now()}" +echo "──────────────────────────────────────────────────────────────────" +echo "" +benchEchoRequest() +benchEchoResponse() +benchComplexRequest() + +# Payload-size sweep across real-consumer wire caps (1 MiB is opt-in). +echo "═══ Payload-size sweep (BytesPayload.payload = seq[byte]) ═════════" +echo "" +benchBytesAtSize(100, 200_000) +benchBytesAtSize(1024, 50_000) +benchBytesAtSize(10 * 1024, 5_000) +benchBytesAtSize(64 * 1024, 500) +benchBytesAtSize(150 * 1024, 200) +if paramCount() >= 1 and paramStr(1) == "--include-1mib": + benchBytesAtSize(1024 * 1024, 50)