From a50df01e4baf25d86e3312a3eba44417bcea8710 Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:49:59 +0200 Subject: [PATCH] feat(perf): C++ e2e perf test harness (tests/perf) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measures the full C++ -> Nim -> C++ foreign round trip across setups: scalar family (2 x int64 + float64), payload family (seq[byte], size sweep incl. large payloads), and event delivery — through sync, async (bounded in-flight window) and event lanes, over a thread-count sweep. All handlers compute the same O(1) parity predicate, so setups differ purely in transport cost. Every reply is verified; a bad reply or lost event exits non-zero. `nimble perf_cpp_e2e` regenerates the cpp bindings, builds libperfbench with -d:danger over the NIM_FFI_MM matrix (orc + refc) — deliberately bypassing the debug-orc nim_ffi_lib.cmake template build — and runs the Release driver. NIM_FFI_PERF_* env knobs control threads, volume, iterations and payload sizes; each table row also emits a csv line for diff-friendly capture. Co-Authored-By: Claude Fable 5 --- .gitignore | 6 + ffi.nimble | 38 +++ tests/perf/README.md | 60 ++++ tests/perf/benchlib/perfbench.nim | 94 ++++++ tests/perf/cpp/CMakeLists.txt | 95 ++++++ tests/perf/cpp/perf_driver.cpp | 461 ++++++++++++++++++++++++++++++ 6 files changed, 754 insertions(+) create mode 100644 tests/perf/README.md create mode 100644 tests/perf/benchlib/perfbench.nim create mode 100644 tests/perf/cpp/CMakeLists.txt create mode 100644 tests/perf/cpp/perf_driver.cpp diff --git a/.gitignore b/.gitignore index b8dbe58..0cc0f12 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,12 @@ tests/test_* # E2E test build artifacts (e.g. CMake build dirs under tests/e2e/cpp/build/) tests/e2e/**/build/ +# Perf harness artifacts (bench lib + driver, cmake build dir, generated bindings +# — regenerated by `nimble perf_cpp_e2e`) +tests/perf/build/ +tests/perf/cpp/build/ +tests/perf/benchlib/cpp_bindings/ + # Generated binding crates (regenerated by `nimble genbindings_*`) examples/**/rust_bindings/target/ diff --git a/ffi.nimble b/ffi.nimble index 489343c..5824eab 100644 --- a/ffi.nimble +++ b/ffi.nimble @@ -18,6 +18,7 @@ const nimFlagsRefc = "--mm:refc -d:chronicles_log_level=WARN" const timerSrc = "examples/timer/timer.nim" const echoSrc = "examples/echo/echo.nim" +const perfbenchSrc = "tests/perf/benchlib/perfbench.nim" import std/[algorithm, os, strutils] @@ -153,6 +154,43 @@ task bench_ffi_submit, for flags in mmModes(): exec "nim c -r " & flags & " -d:danger" & extra & " tests/bench/bench_ffi_submit.nim" +proc buildPerfbenchLib(flags: string) = + ## Builds libperfbench for the perf harness. The cpp_bindings cmake template + ## builds Nim libs as debug orc — useless for perf numbers — so the harness + ## builds the dylib itself: -d:danger to match the tests/bench methodology, + ## `flags` supplying the mm. + mkDir "tests/perf/build" + var cmd = + "nim c " & flags & " -d:danger --app:lib --noMain --nimMainPrefix:libperfbench" + when defined(windows): + # mingw gcc emits no import library unless told to; MSVC consumers need it. + cmd.add " --passL:-Wl,--out-implib,tests/perf/build/perfbench.lib" + cmd.add " -o:tests/perf/build/perfbench.dll" + elif defined(macosx): + cmd.add " -o:tests/perf/build/libperfbench.dylib" + else: + cmd.add " -o:tests/perf/build/libperfbench.so" + cmd.add " " & perfbenchSrc + runOrQuit cmd + +task genbindings_cpp_perfbench, "Generate C++ bindings for the perf bench library": + exec genBindingsCmd(nimFlagsOrc, perfbenchSrc, "cpp") + +task perf_cpp_e2e, + "Build and run the C++ e2e perf harness (NIM_FFI_MM matrix, -d:danger; NIM_FFI_PERF_* knobs)": + # Not part of `test` — timing is a measurement, not a gate. The driver still + # exits non-zero on any correctness failure (bad reply, lost event). + runOrQuit "nimble genbindings_cpp_perfbench" + for flags in mmModes(): + echo "\n=== perf_cpp_e2e: " & flags & " (danger) ===" + buildPerfbenchLib(flags) + runOrQuit "cmake -S tests/perf/cpp -B tests/perf/cpp/build -DCMAKE_BUILD_TYPE=Release" + runOrQuit "cmake --build tests/perf/cpp/build --config Release" + when defined(windows): + runOrQuit "tests/perf/build/perf_driver.exe" + else: + runOrQuit "tests/perf/build/perf_driver" + task test_cpp_e2e, "Build and run the C++ end-to-end tests for the timer example": # Regenerate the C++ bindings so the suite always runs against fresh codegen. runOrQuit "nimble genbindings_cpp" diff --git a/tests/perf/README.md b/tests/perf/README.md new file mode 100644 index 0000000..1e4cd4f --- /dev/null +++ b/tests/perf/README.md @@ -0,0 +1,60 @@ +# C++ e2e perf harness + +Measures the full foreign round trip — C++ driver → generated `perfbench.hpp` +wrapper (CBOR encode) → `libperfbench` C ABI → FFI thread → Nim handler → back +to C++ — across payload shapes, call lanes and thread counts. The nim-ffi +analog of nim-brokers' `test/ffibench` (`bench_e2e_driver.cpp` / +`perf_driver.cpp`), using the same O(1) parity predicate so the tables compare +directly. + +Timing is a measurement, not a gate: the harness is not part of `nimble test`. +It still exits non-zero on any correctness failure — every reply is verified +against the driver-side predicate, and every fired event must be delivered. + +## Running + +```sh +nimble perf_cpp_e2e +``` + +Builds `tests/perf/benchlib/perfbench.nim` as a shared library with +`-d:danger` (matching the `tests/bench` methodology) for each mm in the +`NIM_FFI_MM` matrix (empty = orc + refc), regenerates the C++ bindings, builds +the driver Release via CMake, and runs it. The generated +`nim_ffi_lib.cmake` template is deliberately **not** used to build the lib — +it produces a debug orc build, which would make the numbers meaningless. + +## What is measured + +Every handler computes the same O(1) parity predicate +`((a + b + int64(x)) and 1) == 0`, so setups differ purely in **transport** +cost (encode / copy / decode / thread crossing), never in handler work: + +| Setup | Wire shape | Lane | +| --- | --- | --- | +| scalar | 2 × `int64` + 1 × `float64` in, `bool` out | sync + async | +| payload N B | `seq[byte]` of N in, `bool` out; predicate reads (first, last, len) only | sync + async | +| event | sync trigger fires one `on_perf_ping` event of N B | delivery-latency | + +- **sync** — blocking round trips; per-call latency sampled (p50/p99). +- **async** — `*Async` future turnaround with a bounded in-flight window per + thread (`NIM_FFI_PERF_ASYNC_WINDOW`); throughput only. +- **event** — the listener computes delivery latency from a driver-side + `steady_clock` stamp passed through the Nim provider verbatim, so the delta + stays inside one clock domain. + +Each table row is one thread count: median msg/s over `NIM_FFI_PERF_ITERS` +runs, latency percentiles pooled across runs, plus a `csv,perf_ffi,...` line +per row for diff-friendly capture. + +## Env knobs + +| Knob | Default | Meaning | +| --- | --- | --- | +| `NIM_FFI_MM` | both | `orc` / `refc` — mm matrix for the Nim lib build | +| `NIM_FFI_PERF_THREADS` | `1,2,4,8` | driver thread counts swept | +| `NIM_FFI_PERF_PER_THREAD` | `2000` | round trips per thread per run | +| `NIM_FFI_PERF_ITERS` | `3` | runs per row, median reported | +| `NIM_FFI_PERF_PAYLOAD_SIZES` | `64,512,4096,65536` | payload family sizes (bytes) | +| `NIM_FFI_PERF_EVENT_PAYLOAD` | `512` | event payload size (bytes) | +| `NIM_FFI_PERF_ASYNC_WINDOW` | `64` | in-flight futures per thread, async lane | diff --git a/tests/perf/benchlib/perfbench.nim b/tests/perf/benchlib/perfbench.nim new file mode 100644 index 0000000..ea5624f --- /dev/null +++ b/tests/perf/benchlib/perfbench.nim @@ -0,0 +1,94 @@ +## Perf bench library — the Nim side of the C++ e2e perf harness +## (tests/perf/cpp/perf_driver.cpp). +## +## Every handler computes the same O(1) parity predicate, so the payload and +## scalar families differ purely in TRANSPORT cost (CBOR encode -> FFI thread +## -> decode), never in handler work. The C++ driver re-computes the predicate +## and verifies every reply, making each timed call a correctness check too. + +import ffi, chronos + +type Perfbench = object + name: string + +declareLibrary("perfbench", Perfbench) + +type PerfbenchConfig {.ffi.} = object + name: string + +type PayloadCheckRequest {.ffi.} = object + data: seq[byte] + +type PayloadCheckResponse {.ffi.} = object + ok: bool + +type ScalarCheckRequest {.ffi.} = object + a: int64 + b: int64 + x: float64 + +type ScalarCheckResponse {.ffi.} = object + ok: bool + +type TriggerPingRequest {.ffi.} = object + count: int64 + payloadBytes: int64 + stampNs: int64 # driver-side steady_clock stamp, passed through verbatim + +type TriggerPingResponse {.ffi.} = object + emitted: int64 + +type PerfPingEvent {.ffi.} = object + seqNo: int64 + stampNs: int64 + data: seq[byte] + +proc onPerfPing*(evt: PerfPingEvent) {.ffiEvent: "on_perf_ping".} + +# The one shared predicate — identical formula in the C++ driver +# (`parityPred`), so results are exactly predictable there. +func parityPred(a, b: int64, x: float64): bool = + ((a + b + int64(x)) and 1) == 0 + +proc perfbenchCreate*( + config: PerfbenchConfig +): Future[Result[Perfbench, string]] {.ffiCtor.} = + ## Creates a bench context. No sleeps: handlers must add zero think time. + return ok(Perfbench(name: config.name)) + +proc perfbenchPayloadCheck*( + p: Perfbench, req: PayloadCheckRequest +): Future[Result[PayloadCheckResponse, string]] {.ffi.} = + ## O(1) predicate over (first byte, last byte, length) — the payload bytes + ## are never walked, so the cost measured is transport, not compute. + if req.data.len == 0: + return ok(PayloadCheckResponse(ok: parityPred(0, 0, 0.0))) + return ok( + PayloadCheckResponse( + ok: parityPred(int64(req.data[0]), int64(req.data[^1]), float64(req.data.len)) + ) + ) + +proc perfbenchScalarCheck*( + p: Perfbench, req: ScalarCheckRequest +): Future[Result[ScalarCheckResponse, string]] {.ffi.} = + ## Scalar family: 2 x int64 + 1 x float64 in, bool out, same predicate. + return ok(ScalarCheckResponse(ok: parityPred(req.a, req.b, req.x))) + +proc perfbenchTriggerPing*( + p: Perfbench, req: TriggerPingRequest +): Future[Result[TriggerPingResponse, string]] {.ffi.} = + ## Fires `count` on_perf_ping events of `payloadBytes` each, passing the + ## driver's clock stamp through so the listener can compute delivery latency + ## inside a single clock domain. + for i in 0 ..< req.count: + onPerfPing( + PerfPingEvent(seqNo: i, stampNs: req.stampNs, data: newSeq[byte](req.payloadBytes)) + ) + return ok(TriggerPingResponse(emitted: req.count)) + +proc perfbench_destroy*(p: Perfbench) {.ffiDtor.} = + ## Releases the bench context. + discard + +genBindings() diff --git a/tests/perf/cpp/CMakeLists.txt b/tests/perf/cpp/CMakeLists.txt new file mode 100644 index 0000000..56a99c4 --- /dev/null +++ b/tests/perf/cpp/CMakeLists.txt @@ -0,0 +1,95 @@ +cmake_minimum_required(VERSION 3.16) +project(nim_ffi_cpp_perf CXX C) + +# C++ -> Nim -> C++ e2e perf harness. Unlike tests/e2e/cpp this does NOT +# add_subdirectory the generated cpp_bindings: their nim_ffi_lib.cmake template +# builds the Nim dylib as a debug orc build, which would make the numbers +# meaningless. The `perf_cpp_e2e` nimble task builds libperfbench itself +# (-d:danger, NIM_FFI_MM matrix) into tests/perf/build/ and this project +# imports that prebuilt library. + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# MSVC defaults __cplusplus to 199711L regardless of the active /std:c++XX +# level — the generated header's C++20 guard would then misfire. +if(MSVC) + add_compile_options(/Zc:__cplusplus) +endif() + +# ── Locate the repository root (contains ffi.nimble) ───────────────────────── +set(_search_dir "${CMAKE_CURRENT_SOURCE_DIR}") +set(REPO_ROOT "") +foreach(_i RANGE 10) + if(EXISTS "${_search_dir}/ffi.nimble") + set(REPO_ROOT "${_search_dir}") + break() + endif() + get_filename_component(_search_dir "${_search_dir}" DIRECTORY) +endforeach() +if("${REPO_ROOT}" STREQUAL "") + message(FATAL_ERROR "Cannot find repo root (no ffi.nimble in any ancestor)") +endif() + +# ── Prebuilt libperfbench (built by the nimble task, not by cmake) ─────────── +if(NOT PERF_LIB_DIR) + set(PERF_LIB_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../build") +endif() +get_filename_component(PERF_LIB_DIR "${PERF_LIB_DIR}" ABSOLUTE) + +set(_bindings_dir "${CMAKE_CURRENT_SOURCE_DIR}/../benchlib/cpp_bindings") +if(NOT EXISTS "${_bindings_dir}/perfbench.hpp") + message(FATAL_ERROR + "perfbench.hpp not found in ${_bindings_dir}.\n" + "Run `nimble perf_cpp_e2e` (or `nimble genbindings_cpp_perfbench`) first.") +endif() + +set(_lib_file + "${PERF_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}perfbench${CMAKE_SHARED_LIBRARY_SUFFIX}") +if(NOT EXISTS "${_lib_file}") + message(FATAL_ERROR + "${_lib_file} not found.\n" + "Run `nimble perf_cpp_e2e` to build the bench library first.") +endif() + +add_library(perfbench SHARED IMPORTED) +set_target_properties(perfbench PROPERTIES + IMPORTED_LOCATION "${_lib_file}" + INTERFACE_INCLUDE_DIRECTORIES "${_bindings_dir}") +if(WIN32) + set_target_properties(perfbench PROPERTIES + IMPORTED_IMPLIB "${PERF_LIB_DIR}/perfbench.lib") +endif() + +# ── TinyCBOR (vendored; used by the generated CBOR codec) ──────────────────── +set(TINYCBOR_SRC_DIR "${REPO_ROOT}/ffi/codegen/templates/cpp/vendor") +add_library(tinycbor STATIC + "${TINYCBOR_SRC_DIR}/tinycbor/cborencoder.c" + "${TINYCBOR_SRC_DIR}/tinycbor/cborencoder_close_container_checked.c" + "${TINYCBOR_SRC_DIR}/tinycbor/cborparser.c" + "${TINYCBOR_SRC_DIR}/tinycbor/cborparser_dup_string.c" + "${TINYCBOR_SRC_DIR}/tinycbor/cborerrorstrings.c") +target_include_directories(tinycbor PUBLIC + "${TINYCBOR_SRC_DIR}" + "${TINYCBOR_SRC_DIR}/tinycbor") +set_property(TARGET tinycbor PROPERTY C_STANDARD 99) +set_property(TARGET tinycbor PROPERTY POSITION_INDEPENDENT_CODE ON) + +# ── The driver ─────────────────────────────────────────────────────────────── +add_executable(perf_driver perf_driver.cpp) +target_link_libraries(perf_driver PRIVATE perfbench tinycbor) +set_target_properties(perf_driver PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${PERF_LIB_DIR}") + +# Nim-built dylibs use `@rpath/lib*.dylib|so` install_names on macOS and +# Linux; the driver sits in the same dir as the lib, so @loader_path/$ORIGIN +# resolves it. Windows has no rpath — the DLL already lives next to the exe. +if(APPLE) + set_target_properties(perf_driver PROPERTIES + BUILD_RPATH "@loader_path" + INSTALL_RPATH "@loader_path") +elseif(UNIX) + set_target_properties(perf_driver PROPERTIES + BUILD_RPATH "$ORIGIN" + INSTALL_RPATH "$ORIGIN") +endif() diff --git a/tests/perf/cpp/perf_driver.cpp b/tests/perf/cpp/perf_driver.cpp new file mode 100644 index 0000000..2479a47 --- /dev/null +++ b/tests/perf/cpp/perf_driver.cpp @@ -0,0 +1,461 @@ +// perf_driver — C++ -> Nim -> C++ e2e perf harness for nim-ffi. +// +// Drives the ACTUAL foreign path: this C++ process -> generated +// perfbench.hpp wrapper (CBOR encode) -> libperfbench C ABI -> FFI thread +// -> Nim handler, and back. +// +// Three lanes: +// sync C++ -> Nim -> C++ blocking round trip; per-call latency sampled +// async C++ -> Nim -> C++ `*Async` future turnaround, bounded in-flight +// window per thread; throughput only +// event C++ -> Nim -> C++ a sync trigger fires one on_perf_ping event +// per call; delivery latency is measured from a +// driver-side steady_clock stamp passed through +// the Nim provider verbatim +// +// Two payload families share ONE trivial O(1) parity predicate as the handler +// compute, so setups differ purely in TRANSPORT cost, never in handler work: +// scalar — 2 x int64 + 1 x float64 in, bool out +// payload — N-byte byte payload (N swept), predicate over (first byte, +// last byte, length); the bytes are never walked by the handler +// +// Correctness is enforced on every reply: a wrong or failed result aborts the +// run rather than being mistimed. +// +// Env knobs: NIM_FFI_PERF_THREADS ("1,2,4,8"), NIM_FFI_PERF_PER_THREAD (2000), +// NIM_FFI_PERF_ITERS (3, median reported), NIM_FFI_PERF_PAYLOAD_SIZES +// ("64,512,4096,65536"), NIM_FFI_PERF_EVENT_PAYLOAD (512), +// NIM_FFI_PERF_ASYNC_WINDOW (64 in-flight futures per thread). + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "perfbench.hpp" + +using sclock = std::chrono::steady_clock; + +static int envInt(const char* name, int def) { + const char* v = std::getenv(name); + if (v == nullptr || *v == '\0') return def; + return std::atoi(v); +} + +static std::vector envIntList(const char* name, const char* def) { + const char* v = std::getenv(name); + std::string s = (v != nullptr && *v != '\0') ? v : def; + std::vector out; + size_t pos = 0; + while (pos < s.size()) { + size_t comma = s.find(',', pos); + if (comma == std::string::npos) comma = s.size(); + out.push_back(std::atoi(s.substr(pos, comma - pos).c_str())); + pos = comma + 1; + } + return out; +} + +[[noreturn]] static void fatal(const char* what, const std::string& detail) { + std::fprintf(stderr, "FATAL: %s: %s\n", what, detail.c_str()); + std::exit(2); +} + +// --------------------------------------------------------------------------- +// Formatting — human-readable ns and table helpers. +// --------------------------------------------------------------------------- + +static void fmtNs(char* buf, size_t cap, int64_t ns) { + if (ns >= 1'000'000'000) { + std::snprintf(buf, cap, "%.3f s", static_cast(ns) / 1e9); + } else if (ns >= 1'000'000) { + std::snprintf(buf, cap, "%.3f ms", static_cast(ns) / 1e6); + } else if (ns >= 1'000) { + std::snprintf(buf, cap, "%.1f µs", static_cast(ns) / 1e3); + } else { + std::snprintf(buf, cap, "%" PRId64 " ns", ns); + } +} + +static double median(std::vector xs) { + std::sort(xs.begin(), xs.end()); + const size_t n = xs.size(); + return (n % 2 == 1) ? xs[n / 2] : (xs[n / 2 - 1] + xs[n / 2]) / 2.0; +} + +static int64_t percentile(const std::vector& sorted, int pct) { + if (sorted.empty()) return 0; + const size_t i = (sorted.size() * static_cast(pct)) / 100; + return sorted[std::min(i, sorted.size() - 1)]; +} + +// --------------------------------------------------------------------------- +// The one shared predicate — identical formula in the Nim provider +// (`parityPred` in perfbench.nim), so every reply is exactly predictable here. +// --------------------------------------------------------------------------- + +static bool parityPred(int64_t a, int64_t b, double x) { + return ((a + b + static_cast(x)) & 1) == 0; +} + +static std::vector makePayload(int size) { + std::vector p(static_cast(size)); + for (size_t i = 0; i < p.size(); ++i) p[i] = static_cast(i & 0xFF); + return p; +} + +static bool payloadPredOf(const std::vector& p) { + if (p.empty()) return parityPred(0, 0, 0.0); + return parityPred(static_cast(p.front()), static_cast(p.back()), + static_cast(p.size())); +} + +// Scalar family inputs, derived per message index i. +static int64_t scalarA(int64_t i) { return i; } +static int64_t scalarB(int64_t i) { return 3 * i + 1; } +static double scalarX(int64_t i) { return 0.5 * static_cast(i); } +static bool scalarPred(int64_t i) { + return parityPred(scalarA(i), scalarB(i), scalarX(i)); +} + +static std::unique_ptr makeCtx() { + auto r = PerfbenchCtx::create(PerfbenchConfig{"perf"}); + if (r.isErr()) fatal("PerfbenchCtx::create", r.error()); + return r.take(); +} + +struct IterOut { + double msgPerSec = 0.0; + std::vector latNs; // empty for lanes without per-call samples +}; + +// --------------------------------------------------------------------------- +// sync lane — blocking round trips, per-call latency sampled. +// `call(ctx, i)` performs one verified round trip. +// --------------------------------------------------------------------------- + +template +static IterOut runSync(int threads, int perThread, CallFn&& call) { + auto ctx = makeCtx(); + std::atomic start{false}; + std::vector> perThreadLat(static_cast(threads)); + + std::vector workers; + workers.reserve(static_cast(threads)); + for (int t = 0; t < threads; ++t) { + workers.emplace_back([&, t] { + auto& lat = perThreadLat[static_cast(t)]; + lat.reserve(static_cast(perThread)); + while (!start.load(std::memory_order_acquire)) {} + for (int64_t i = 0; i < perThread; ++i) { + const auto c0 = sclock::now(); + call(*ctx, i); + const auto c1 = sclock::now(); + lat.push_back( + std::chrono::duration_cast(c1 - c0) + .count()); + } + }); + } + + const auto t0 = sclock::now(); + start.store(true, std::memory_order_release); + for (auto& w : workers) w.join(); // blocking calls: join == all done + const auto t1 = sclock::now(); + + IterOut out; + for (auto& v : perThreadLat) + out.latNs.insert(out.latNs.end(), v.begin(), v.end()); + const double sec = std::chrono::duration(t1 - t0).count(); + out.msgPerSec = + static_cast(static_cast(threads) * + static_cast(perThread)) / + sec; + return out; +} + +// --------------------------------------------------------------------------- +// async lane — `*Async` future turnaround with a bounded in-flight window. +// `issue(ctx, i)` returns the future; `expect(i)` the predicted predicate. +// --------------------------------------------------------------------------- + +template +static IterOut runAsync(int threads, int perThread, int window, IssueFn&& issue, + ExpectFn&& expect) { + auto ctx = makeCtx(); + std::atomic start{false}; + std::atomic bad{0}; + + std::vector workers; + workers.reserve(static_cast(threads)); + for (int t = 0; t < threads; ++t) { + workers.emplace_back([&] { + using Fut = decltype(issue(*ctx, int64_t{0})); + std::deque> inflight; + auto drainOne = [&] { + auto [idx, fut] = std::move(inflight.front()); + inflight.pop_front(); + auto r = fut.get(); + if (r.isErr() || r.value().ok != expect(idx)) + bad.fetch_add(1, std::memory_order_relaxed); + }; + while (!start.load(std::memory_order_acquire)) {} + for (int64_t i = 0; i < perThread; ++i) { + inflight.emplace_back(i, issue(*ctx, i)); + if (inflight.size() >= static_cast(window)) drainOne(); + } + while (!inflight.empty()) drainOne(); + }); + } + + const auto t0 = sclock::now(); + start.store(true, std::memory_order_release); + for (auto& w : workers) w.join(); + const auto t1 = sclock::now(); + + if (bad.load() != 0) + fatal("async correctness", + std::to_string(bad.load()) + " failed or mispredicted replies"); + + IterOut out; + const double sec = std::chrono::duration(t1 - t0).count(); + out.msgPerSec = + static_cast(static_cast(threads) * + static_cast(perThread)) / + sec; + return out; +} + +// --------------------------------------------------------------------------- +// event lane — each verified sync trigger fires one on_perf_ping event; the +// listener measures delivery latency from the driver-side stamp. The clock is +// std::chrono::steady_clock on both ends (the stamp is passed through Nim +// verbatim), so the delta is a meaningful per-event delivery latency. +// --------------------------------------------------------------------------- + +static IterOut runEvent(int threads, int perThread, int payloadBytes) { + auto ctx = makeCtx(); + const uint64_t total = + static_cast(threads) * static_cast(perThread); + + std::vector latencies; + latencies.reserve(total); + std::mutex latMu; + std::atomic delivered{0}; + + const auto handle = ctx->addOnPerfPingListener([&](const PerfPingEvent& evt) { + const int64_t nowNs = std::chrono::duration_cast( + sclock::now().time_since_epoch()) + .count(); + { + std::lock_guard g(latMu); + latencies.push_back(nowNs - evt.stampNs); + } + delivered.fetch_add(1, std::memory_order_relaxed); + }); + if (handle.id == 0) fatal("addOnPerfPingListener", "returned zero id"); + + std::atomic start{false}; + std::vector workers; + workers.reserve(static_cast(threads)); + for (int t = 0; t < threads; ++t) { + workers.emplace_back([&] { + while (!start.load(std::memory_order_acquire)) {} + for (int64_t i = 0; i < perThread; ++i) { + const int64_t stampNs = + std::chrono::duration_cast( + sclock::now().time_since_epoch()) + .count(); + auto r = ctx->trigger_ping( + TriggerPingRequest{1, payloadBytes, stampNs}); + if (r.isErr()) fatal("triggerPing", r.error()); + if (r.value().emitted != 1) + fatal("triggerPing", "emitted != 1"); + } + }); + } + + const auto t0 = sclock::now(); + start.store(true, std::memory_order_release); + for (auto& w : workers) w.join(); + + // Drain: the run is over only when every event reached the listener. + const auto deadline = sclock::now() + std::chrono::seconds(60); + while (delivered.load(std::memory_order_acquire) < total) { + if (sclock::now() > deadline) + fatal("event drain", "timeout waiting for event delivery"); + std::this_thread::sleep_for(std::chrono::microseconds(200)); + } + const auto t1 = sclock::now(); + + ctx->removeEventListener(handle); + if (delivered.load() != total) + fatal("event correctness", "delivered != emitted"); + + IterOut out; + { + std::lock_guard g(latMu); + out.latNs = std::move(latencies); + } + const double sec = std::chrono::duration(t1 - t0).count(); + out.msgPerSec = static_cast(total) / sec; + return out; +} + +// --------------------------------------------------------------------------- +// Sweep driver — one compact table per (lane, family) with a row per thread +// count: median msg/s over iters, latency percentiles pooled across iters. +// --------------------------------------------------------------------------- + +struct Knobs { + std::vector threadCounts; + int perThread = 0; + int iters = 0; +}; + +static void runScenario(const std::string& name, const std::string& csvTag, + const Knobs& k, int bytesPerMsg, + const std::function& fn) { + std::printf("── %s — %d msgs/thread (median of %d) ──────\n", name.c_str(), + k.perThread, k.iters); + std::printf(" %-9s%-11s%-13s%-11s%-12s%-12s%s\n", "threads", "msgs", "msg/s", + "MB/s", "p50", "p99", "vs 1-thread"); + double base = 0.0; + for (int threads : k.threadCounts) { + std::vector rates; + std::vector lat; + for (int i = 0; i < k.iters; ++i) { + IterOut out = fn(threads); + rates.push_back(out.msgPerSec); + lat.insert(lat.end(), out.latNs.begin(), out.latNs.end()); + } + const double med = median(rates); + if (base == 0.0) base = med; + + std::sort(lat.begin(), lat.end()); + const int64_t p50 = percentile(lat, 50); + const int64_t p99 = percentile(lat, 99); + char mbs[32], p50Buf[32], p99Buf[32]; + if (bytesPerMsg > 0) + std::snprintf(mbs, sizeof(mbs), "%.2f", + med * static_cast(bytesPerMsg) / 1e6); + else + std::snprintf(mbs, sizeof(mbs), "-"); + if (lat.empty()) { + std::snprintf(p50Buf, sizeof(p50Buf), "-"); + std::snprintf(p99Buf, sizeof(p99Buf), "-"); + } else { + fmtNs(p50Buf, sizeof(p50Buf), p50); + fmtNs(p99Buf, sizeof(p99Buf), p99); + } + std::printf(" %-9d%-11llu%-13.0f%-11s%-12s%-12s%s\n", threads, + static_cast( + static_cast(threads) * + static_cast(k.perThread)), + med, mbs, p50Buf, p99Buf, + (std::to_string(med / base).substr(0, 4) + "x").c_str()); + // CSV row: tag,threads,msg/s,p50_ns,p99_ns — diff-friendly capture. + std::printf("csv,perf_ffi,%s,%d,%.0f,%" PRId64 ",%" PRId64 "\n", + csvTag.c_str(), threads, med, p50, p99); + std::fflush(stdout); + } + std::printf("\n"); +} + +int main() { + Knobs k; + k.threadCounts = envIntList("NIM_FFI_PERF_THREADS", "1,2,4,8"); + k.perThread = envInt("NIM_FFI_PERF_PER_THREAD", 2000); + k.iters = envInt("NIM_FFI_PERF_ITERS", 3); + const std::vector payloadSizes = + envIntList("NIM_FFI_PERF_PAYLOAD_SIZES", "64,512,4096,65536"); + const int eventPayload = envInt("NIM_FFI_PERF_EVENT_PAYLOAD", 512); + const int window = envInt("NIM_FFI_PERF_ASYNC_WINDOW", 64); + if (k.perThread < 1 || k.iters < 1 || window < 1 || k.threadCounts.empty() || + payloadSizes.empty()) { + std::fprintf(stderr, "invalid NIM_FFI_PERF_* configuration\n"); + return 2; + } + + std::printf("# perfbench FFI e2e — shared O(1) parity predicate; setups " + "differ only in transport (scalars vs N-byte payload)\n\n"); + + // ── scalar family ──────────────────────────────────────────────────────── + runScenario("sync C++ -> Nim -> C++ (blocking round trip) [scalar]", + "sync,scalar,0", k, 0, [&](int threads) { + return runSync(threads, k.perThread, [](PerfbenchCtx& ctx, + int64_t i) { + auto r = ctx.scalar_check( + ScalarCheckRequest{scalarA(i), scalarB(i), scalarX(i)}); + if (r.isErr()) fatal("scalarCheck", r.error()); + if (r.value().ok != scalarPred(i)) + fatal("sync scalar correctness", "predicate mismatch"); + }); + }); + + runScenario("async C++ -> Nim -> C++ (future turnaround) [scalar]", + "async,scalar,0", k, 0, [&](int threads) { + return runAsync( + threads, k.perThread, window, + [](PerfbenchCtx& ctx, int64_t i) { + return ctx.scalar_checkAsync(ScalarCheckRequest{ + scalarA(i), scalarB(i), scalarX(i)}); + }, + [](int64_t i) { return scalarPred(i); }); + }); + + // ── payload family, size sweep ─────────────────────────────────────────── + for (int size : payloadSizes) { + const auto payload = makePayload(size); + const bool expected = payloadPredOf(payload); + const std::string tag = "[" + std::to_string(size) + "B payload]"; + + runScenario("sync C++ -> Nim -> C++ (blocking round trip) " + tag, + "sync,payload," + std::to_string(size), k, size, + [&](int threads) { + return runSync(threads, k.perThread, + [&](PerfbenchCtx& ctx, int64_t) { + auto r = ctx.payload_check( + PayloadCheckRequest{payload}); + if (r.isErr()) + fatal("payloadCheck", r.error()); + if (r.value().ok != expected) + fatal("sync payload correctness", + "predicate mismatch"); + }); + }); + + runScenario("async C++ -> Nim -> C++ (future turnaround) " + tag, + "async,payload," + std::to_string(size), k, size, + [&](int threads) { + return runAsync( + threads, k.perThread, window, + [&](PerfbenchCtx& ctx, int64_t) { + return ctx.payload_checkAsync( + PayloadCheckRequest{payload}); + }, + [&](int64_t) { return expected; }); + }); + } + + // ── event lane ─────────────────────────────────────────────────────────── + runScenario("event C++ -> Nim -> C++ (on_perf_ping delivery) [" + + std::to_string(eventPayload) + "B payload]", + "event,payload," + std::to_string(eventPayload), k, eventPayload, + [&](int threads) { + return runEvent(threads, k.perThread, eventPayload); + }); + + std::printf(" correctness: all counts and predicates matched expectations.\n"); + return 0; +}