diff --git a/tools/benchmarks/Equi-X/.github/workflows/ci.yml b/tools/benchmarks/Equi-X/.github/workflows/ci.yml new file mode 100644 index 0000000..8321e66 --- /dev/null +++ b/tools/benchmarks/Equi-X/.github/workflows/ci.yml @@ -0,0 +1,66 @@ +name: ci + +on: + push: + pull_request: + +jobs: + build-and-smoke: + # ubuntu-24.04 (x86-64) and macos-14 (Apple Silicon / aarch64) validate both + # platforms, including the Apple Silicon JIT guard in the C runner. + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-14] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Build runners + run: ./scripts/setup.sh + + - name: Install harness + run: pip install -e ./harness + + - name: Run reference C unit tests + run: | + cmake --build build/runners/c --target equix-tests + ./build/runners/c/equix_ext/equix-tests + + - name: Harness pytest + run: | + pip install pytest + pytest -q harness/tests + + - name: Smoke benchmark + run: python -m equix_bench run --config configs/smoke.toml --out results/ + + - name: Cross-implementation correctness gate + run: python -m equix_bench run --config configs/smoke.toml --out results/ --crosscheck-only + + - name: Multi-CPU combine (two labelled runs, faceted + cross-CPU figures) + run: | + python -m equix_bench run --config configs/smoke.toml --out runA/ --device-label ci-cpu-a + python -m equix_bench run --config configs/smoke.toml --out runB/ --device-label ci-cpu-b + python -m equix_bench combine --inputs runA runB --out combined/ + test -f combined/plots/xdev_throughput.png + test -f combined/plots/solve_time_by_runtime.png + + - name: Upload results (report + plots) + if: always() + uses: actions/upload-artifact@v4 + with: + name: smoke-results-${{ matrix.os }} + path: | + results/ + combined/ diff --git a/tools/benchmarks/Equi-X/.gitignore b/tools/benchmarks/Equi-X/.gitignore new file mode 100644 index 0000000..3616ded --- /dev/null +++ b/tools/benchmarks/Equi-X/.gitignore @@ -0,0 +1,26 @@ +# Build outputs +/build/ +runners/rust/target/ +**/*.o +**/*.a + +# Benchmark outputs +/results/ +/combined/ +/run[A-Z]/ +/runA/ +/runB/ + +# Auto-generated compiler-flag variant manifests (reference machine-specific build paths) +/adapters/generated/ + +# Python +__pycache__/ +*.pyc +.venv/ +*.egg-info/ +.pytest_cache/ + +# Editor/OS +.DS_Store +*.swp diff --git a/tools/benchmarks/Equi-X/.gitmodules b/tools/benchmarks/Equi-X/.gitmodules new file mode 100644 index 0000000..ff1d707 --- /dev/null +++ b/tools/benchmarks/Equi-X/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vendored/equix"] + path = vendored/equix + url = https://github.com/tevador/equix diff --git a/tools/benchmarks/Equi-X/LICENSE b/tools/benchmarks/Equi-X/LICENSE new file mode 100644 index 0000000..696e4d5 --- /dev/null +++ b/tools/benchmarks/Equi-X/LICENSE @@ -0,0 +1,36 @@ +MIT License + +Copyright (c) 2026 Equi-X benchmark framework contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------- + +This license applies to the framework code authored in this repository +(the C/Rust runners and the Python harness). + +THIRD-PARTY COMPONENTS (not covered by the above license): + + * tevador/equix and tevador/hashx — vendored as a git submodule under + vendored/equix — are licensed LGPL-3.0-only. + * The Rust crates `equix` and `hashx` (dependencies of runners/rust) are + licensed LGPL-3.0-only. + +These components are used as dependencies/submodules and retain their original +licenses; see their respective LICENSE files. diff --git a/tools/benchmarks/Equi-X/Makefile b/tools/benchmarks/Equi-X/Makefile new file mode 100644 index 0000000..b5f051c --- /dev/null +++ b/tools/benchmarks/Equi-X/Makefile @@ -0,0 +1,83 @@ +# Equi-X benchmark — make wrapper around the scripts/ + harness entry points. +# +# make setup bootstrap deps, build both runners, autotune C flags +# make check report which dependencies are present (installs nothing) +# make test run the harness unit tests (skips cleanly if pytest missing) +# make benchmark main C-vs-Rust benchmark, quick profile (depends: setup test) +# make benchmark-full deep sweep incl. concurrency + mining (depends: setup test) +# make compiler-flags build the gcc/clang -O matrix and compare (depends: setup) +# make mining standalone mining sweep, idle-gated (depends: setup) +# make control difficulty-controller simulation demo +# make combine merge per-device runs under ROOT into one faceted report +# make everything full pipeline via scripts/run_all.sh --full +# make clean remove C build trees + generated variant manifests +# make distclean clean + Rust target/ + results/ +# +# Variables: OUT=results make benchmark OUT=/tmp/out +# ROOT=results-by-device make combine (tree of per-device runs) +# EQUIX_NO_AUTOTUNE=1 make setup (skip flag autotuning) + +SHELL := /usr/bin/env bash +OUT ?= results +ROOT ?= $(OUT) +# Prefer the project venv scripts/setup.sh creates (harness + pytest installed, +# PEP-668-proof); fall back to system python3 when it is absent. +PY := $(if $(wildcard .venv/bin/python),.venv/bin/python,python3) +export PYTHONPATH := $(CURDIR)/harness$(if $(PYTHONPATH),:$(PYTHONPATH),) + +.PHONY: all help setup check test benchmark benchmark-full compiler-flags \ + mining control combine everything clean distclean + +all: benchmark + +help: + @sed -n '2,17p' Makefile | sed 's/^# \{0,1\}//' + +setup: + ./scripts/setup.sh + +check: + ./scripts/setup.sh --check + +# Tests are a soft dependency: run when pytest is available, skip loudly (but +# without failing the chain) when it is not — mirroring scripts/run_all.sh. +test: setup + @if $(PY) -c 'import pytest' >/dev/null 2>&1; then \ + $(PY) -m pytest -q harness/tests; \ + else \ + echo "WARNING: pytest not importable; skipping unit tests." >&2; \ + echo " fix: re-run ./scripts/setup.sh (creates .venv with pytest)," >&2; \ + echo " or: python3 -m pip install pytest (or --break-system-packages)" >&2; \ + fi + +benchmark: setup test + $(PY) -m equix_bench run --config configs/smoke.toml --out $(OUT)/main + +benchmark-full: setup test + $(PY) -m equix_bench run --config configs/full.toml --out $(OUT)/main + +compiler-flags: setup + ./scripts/build_variants.sh + $(PY) -m equix_bench run --config configs/compiler_flags.toml --out $(OUT)/compiler_flags + +mining: setup + ./scripts/run_when_idle.sh $(PY) -m equix_bench run --config configs/mining.toml --out $(OUT)/mining + +control: + $(PY) -m equix_bench.difficulty_control --out $(OUT)/control + +# Merge every run found under ROOT (each device's results, in any layout) into +# one faceted, per-device report. ROOT defaults to OUT; OUT names the report dir. +combine: + $(PY) -m equix_bench combine --root $(ROOT) --out $(OUT)/combined + @echo "combined report -> $(OUT)/combined/report.md" + +everything: + ./scripts/run_all.sh --full --out $(OUT) + +clean: + rm -rf build/runners/c build/autotune build/variants adapters/generated + @echo "cleaned C build trees + generated manifests (Rust target/ and $(OUT)/ kept; use distclean)" + +distclean: clean + rm -rf runners/rust/target $(OUT) combined build diff --git a/tools/benchmarks/Equi-X/PARAMETERS.md b/tools/benchmarks/Equi-X/PARAMETERS.md new file mode 100644 index 0000000..dd645d7 --- /dev/null +++ b/tools/benchmarks/Equi-X/PARAMETERS.md @@ -0,0 +1,226 @@ +# Equi-X Benchmark Parameters + +This document describes **every parameter** the benchmarking framework exposes, +what it controls, and its implications for **execution time, memory, and cost**. +Parameters are set in a TOML config (see `configs/`) and travel to each runner as +the JSON job-spec (see `adapters/README.md`). + +Equi-X itself is a fixed puzzle: **Equihash(n=60, k=3)** over the **HashX** +pseudo-random hash function. The algorithm constants (n, k, solution size = 8× +16-bit indices, hash size) are **not** tunable — they define the puzzle. What the +framework varies is *how* the puzzle is executed and measured, plus the Tor-style +*effort* layer stacked on top. + +--- + +## 1. `operation` — what is being measured + +| value | what it does | dominant cost | +|-------|--------------|---------------| +| `solve` | Generate the HashX program from the challenge, then run the Equihash solver to find all solutions. | **Milliseconds.** The headline PoW cost. ~1.7 solutions/challenge on average. | +| `verify` | Check that a given solution is valid for a challenge (index ordering + partial/final XOR sums). | **Microseconds.** ~1000× cheaper than solving — this asymmetry is the whole point of a client puzzle. | +| `effort` | Repeatedly solve over an incrementing nonce until a solution meets a target *effort* (difficulty). | **Scales with `target_effort`** (see §5). Models the real cost of producing a PoW at a difficulty. | +| `hashx_compile` | Isolate HashX **program generation + compilation** (`hashx_make` / `EquiXBuilder::build`) from execution, using the HashX API directly. | **Microseconds.** The only clean way to measure JIT/compile cost (see §3). | + +**Implication:** `solve` and `verify` are the two faces of an asymmetric PoW; +report them together to show the work/verify ratio. `effort` is the attacker/ +client cost model. `hashx_compile` explains *why* compiled mode is faster. + +--- + +## 2. `runtime` — HashX execution backend + +HashX generates a unique straight-line program per challenge and can either +**interpret** it or **JIT-compile** it to native code. + +| value | meaning | implication | +|-------|---------|-------------| +| `interpret` | Force the pure interpreter; never compile. | Portable, no executable memory. **~9× slower solve** in practice (measured ~68 ms vs ~7.6 ms). | +| `try-compile` | Compile if supported, else fall back to the interpreter. **Default.** | Best speed where a JIT exists (x86-64/aarch64); safe elsewhere. `runtime_effective` reports which path ran. | +| `must-compile` | Require the JIT; **fail** if unsupported. | Use to guarantee you are measuring compiled performance; errors out on unsupported targets instead of silently interpreting. | + +- C mapping: `interpret` → `equix_alloc(SOLVE)`; compiled → `equix_alloc(SOLVE | COMPILE)`. Unsupported JIT returns the `EQUIX_NOTSUPP` sentinel. +- Rust mapping: `RuntimeOption::InterpretOnly` / `TryCompile` / `CompileOnly`. + +**Cost implication:** compiling adds a one-off ~50–80 µs per program (see +`hashx_compile`), amortized across the millions of HashX evaluations in a solve — +so compiled mode wins decisively for `solve`, is marginal for a single `verify`. + +--- + +## 3. Compile-time isolation (why `hashx_compile` exists) + +The HashX program is seeded by the **challenge**, so in the C library the program +is (re)generated *inside* `equix_solve` — libequix's public API cannot separate +"compile" from "solve". Rust *can* (build vs solve are distinct calls), but to +keep C and Rust comparable, **both** runners implement a dedicated +`hashx_compile` operation that times `hashx_make` (program-gen + JIT) separately +from one `hashx_exec`. Treat the `compile_ns` field as meaningful **only** for the +`hashx_compile` operation; it is `0` for `solve`/`verify`. + +--- + +## 4. Challenge parameters + +| parameter | applies to | meaning | +|-----------|-----------|---------| +| `challenges` (`challenge_hex`) | solve, verify, hashx_compile | Hex-encoded challenge bytes. The challenge is the HashX seed — **each distinct challenge is a different one-way function**. | +| `bases` (`challenge_base_hex`) | effort, hashx_compile | Fixed prefix; the runner appends a nonce to form each attempt's challenge. | +| `nonce_bytes` | effort | Width of the little-endian nonce counter appended to the base (`challenge = base ‖ LE(nonce)`). Must be ≤ 8. | +| `nonce_start` | effort, hashx_compile | Starting nonce value (reproducibility / sharding the search space). | +| `solution_hex` | verify | 16-byte packed solution (8× uint16 LE). The harness auto-fills this from a `solve` of the same challenge. | + +**Edge case — invalid programs:** roughly **1 in 2^k** challenges produce a HashX +program that fails validation (by design). The runner treats this as a valid +*measured outcome* (`solutions: 0`, verify → `CHALLENGE`), not an error; the +effort search simply advances the nonce. Some perfectly valid challenges also +have **0 Equihash solutions** — e.g. the all-zero challenge — so pick challenges +with known solutions for solve/verify cells. + +--- + +## 5. Effort / difficulty parameters (Tor proposal 327) + +The effort layer sits **above** Equi-X. For a solved `(challenge, solution)`: + +``` +hash32 = first 32 bits (big-endian) of BLAKE2b-256(challenge ‖ solution_bytes) +achieved = floor((2^32 - 1) / hash32) # "how hard was this solution" +valid at effort E ⇔ hash32 · E ≤ 2^32 - 1 ⇔ achieved ≥ E +``` + +| parameter | meaning | implication | +|-----------|---------|-------------| +| `targets` (`target_effort`) | Difficulty to reach: stop when a solution's `achieved ≥ target`. | Cost grows **~linearly** with target — a 10× harder target costs ~10× more work. Each solution meets effort `E` with probability `1/E`; a solve yields ~1.7 solutions, so expected solves ≈ `E / 1.7`. | +| `max_attempts` | Safety cap on the nonce search per repetition. | Bounds worst-case runtime; if hit before the target, `achieved` reports the best found. Set comfortably above the target. | + +The preimage layout and byte order are **identical in C and Rust** — the +cross-check asserts both produce the same `achieved` effort for a fixed input, so +a mismatch (a broken port) fails the build rather than silently skewing results. +(Verified against Python's standard `hashlib.blake2b(digest_size=32)`.) + +**Notes:** +- This models Tor-327's effort *concept* (a difficulty proxy for benchmarking); the + preimage is `challenge ‖ solution_bytes` with standard BLAKE2b-256, a + simplification of Tor's production wire layout (which folds in seed/nonce/ + personalization fields), so values are not byte-compatible with a live Tor PoW. +- The search is **deterministic** given `(base, nonce_start)`: every repetition + runs the same nonce sequence, so `repetitions` measures timing variance of the + same search, not a difficulty distribution. Vary `nonce_start`/`bases` to sample + different searches. + +--- + +## 6. Measurement parameters + +| parameter | meaning | implication | +|-----------|---------|-------------| +| `repetitions` | Number of **timed** iterations per cell. | More reps → tighter median/p95, longer runs. The report uses median + p95 + stddev because there is no `perf`/`taskset` here, so noise is real. | +| `warmup` | Untimed iterations run **before** timing. | Excludes cold caches, first-touch paging, and initial JIT warmth from the measurement. Warmups are never counted in `runs[]`. | +| `seed` | Optional RNG seed for reproducible challenge generation (reserved for generators). | Reproducibility. | +| `impls` | Which implementations to run (must match adapter manifest names). | Determines what appears on every comparison plot — needs ≥2 for the C-vs-Rust figures. | + +--- + +## 7. Metrics reported (and their units) + +| metric | source | notes | +|--------|--------|-------| +| `wall_ns` | `clock_gettime(CLOCK_MONOTONIC)` (C) / `Instant` (Rust) | Per-rep solve/verify/effort time. | +| `compile_ns` | `hashx_make` / `EquiXBuilder::build` timing | Meaningful only for `hashx_compile`. | +| `solves_per_sec`, `hashes_per_sec` | derived from median solve time | Throughput; hash-rate = solves/sec × the per-solve HashX count (2^16, the equix 16-bit index space; both impls use the same constant so comparisons are exact). | +| `peak_rss_kb` | `getrusage.ru_maxrss` (C) / `/proc/self/status VmHWM` (Rust) | Always **kilobytes** (Linux reports KB; macOS reports bytes and the runner converts). One process per cell keeps this attributable. | +| `attempts`, `achieved_effort` | effort search | Attacker/client cost at a difficulty. | +| `verify_result` | `equix_verify` result enum | `OK` / `CHALLENGE` / `ORDER` / `PARTIAL_SUM` / `FINAL_SUM`. | +| `protection_factor` | DoS analysis (§9) | attacker time/token ÷ defender verify time — the core DoS asymmetry. | +| `verify_per_sec`, `attacker_tokens_per_sec` | DoS analysis (§9) | defender screening capacity vs attacker output, per core. | + +--- + +## 8. Device / CPU tracking & multi-CPU figures + +Every run records the **device** it executed on — the runner self-reports +`env.cpu` (model), `env.arch`, and `env.device` (`cpu`/`gpu`), which the harness +turns into a device record `{type, name, arch, label}` carried on every result +(and in `results.csv` / `run_meta.json`). + +| parameter | meaning | implication | +|-----------|---------|-------------| +| `--device-label` (a.k.a. `--cpu-label`) | Human label for the executing device. | Defaults to a slug of the **CPU model + OS/kernel version** (e.g. `intel-xeon-2-80ghz-6-18-5`); override to disambiguate machines that still collide (e.g. `--device-label ryzen-9950x`). | + +**Reflecting the CPU on plots:** with a single device, the CPU is shown in each +plot's title and the report header. To compare **multiple CPUs**, run on each +machine and merge the outputs: + +```bash +python -m equix_bench run --config configs/full.toml --out runA/ --device-label host-a +python -m equix_bench run --config configs/full.toml --out runB/ --device-label host-b +python -m equix_bench combine --inputs runA runB --out combined/ +``` + +`combine` re-aggregates the saved per-run data (no re-benchmarking) and renders: +- **faceted plots** — one subplot per CPU, C-vs-Rust compared within each; and +- **`xdev_*` cross-CPU charts** — x=CPU, series=implementation — for headline + metrics (solve throughput, solve time, peak RSS, verify time). + +### GPU + +**Equi-X is not benchmarked on GPU, and no GPU implementation is bundled.** HashX +(the hash Equi-X is built on) is deliberately designed to resist GPU/ASIC +acceleration — it depends on branch prediction and out-of-order execution that +favor general-purpose CPUs — so a GPU solver would be far slower and none exists in +practice. The framework is nonetheless **GPU-ready**: a runner that reports +`device: "gpu"` plugs in through the adapter protocol and appears on all figures as +another device, with no harness change. + +## 9. DoS-protection effectiveness + +Equi-X is a client puzzle for DoS defense: a requester must **solve** (expensive) +before a service acts, while the service only **verifies** (cheap). Any run that +includes both the `effort` and `verify` operations gets a DoS-protection section +(and `dos_protection.png`) computed from **measured** numbers on the running system. + +| quantity | definition | +|----------|------------| +| `attacker_s(E)` | measured median time to craft one accepted token at effort `E` (the `effort` op), using the fastest impl | +| `defender_s` | measured fastest median `verify` time on that device | +| **`protection_factor(E)`** | `attacker_s(E) / defender_s` — how many verifies the defender does in the time the attacker needs for one accepted request | +| `verify_per_sec` | `1 / defender_s` — defender screening capacity per core | +| `attacker_tokens_per_sec` | `1 / attacker_s(E)` — attacker output per core | +| **verdict** | *effective* if some tested effort reaches the threshold; the report states the **minimum effort** `E*` from which protection holds on this system | + +The threshold defaults to **10 000×** (`dosprotect.DEFAULT_THRESHOLD`). Run it with: + +```bash +python -m equix_bench run --config configs/dos_protection.toml --out results/ +``` + +Because it uses measured attacker cost, the answer is specific to the CPU it runs +on — the same effort gives a different protection factor on a fast vs slow machine. + +## 10. Compiler-flag variants (performance vs build flags) + +The same C implementation can be built under different compiler/optimization flags +and compared as separate impls. `scripts/build_variants.sh` builds a matrix +(`gcc -O0/-O2/-O3`, `-march=native`, `-flto`, `clang -O3`, …), writing one +`equix-c-` manifest per variant to `adapters/generated/` (loaded alongside the +built-in adapters). The flags apply to the whole `libequix`+`hashx`+runner build, so +they affect the Equihash solver and the HashX interpreter (the JIT executes the same +generated machine code regardless). + +```bash +./scripts/build_variants.sh +python -m equix_bench run --config configs/compiler_flags.toml --out results/ +``` + +Every comparison plot then compares the flag variants; all variants produce +identical solutions, so the interop cross-check still holds. + +## 11. Not benchmarked by default (and why) + +- **HugePages** (`EQUIX_CTX_HUGEPAGES`): off by default; it changes RSS accounting + and requires host configuration, which would distort memory comparisons. +- **Threads / multi-core solving**: the framework measures single-thread cost per + cell for clean per-implementation comparison; parallel scaling is orthogonal. +- **HW performance counters** (cycles, cache misses): `perf` is unavailable in the + reference environment, so cost is reported as wall-time + RSS. diff --git a/tools/benchmarks/Equi-X/README.md b/tools/benchmarks/Equi-X/README.md new file mode 100644 index 0000000..d79ee2a --- /dev/null +++ b/tools/benchmarks/Equi-X/README.md @@ -0,0 +1,296 @@ +# Equi-X + +A benchmarking framework for the **Equi-X** proof-of-work algorithm — tevador's +`Equihash(n=60, k=3)` over the **HashX** pseudo-random hash function (the +client-puzzle used by Tor onion-service PoW, proposal 327). + +It benchmarks **multiple implementations** side by side — the reference **C** +(`tevador/equix` + `hashx`) and the **Rust** (Tor `arti` `equix` / `hashx` crates) +— across **all parameters** (runtime backend, operation, challenge, difficulty) +and measures **execution time and cost** (solve/verify throughput, peak memory, +JIT-compile overhead, and difficulty/effort cost). It is extensible to any other +implementation via a small language-agnostic plugin protocol. + +## What it measures + +- **Solve & verify time + throughput** — the asymmetric core of the PoW. +- **Peak RSS memory** — per implementation and runtime. +- **JIT compile overhead** — interpreter vs compiled HashX, isolated. +- **Difficulty / effort sweep** — Tor prop-327 effort: expected attempts & time to + reach a target difficulty. +- **DoS-protection effectiveness** — the attacker-solve vs defender-verify asymmetry + on *this* system, with a verdict (effective from what effort). This figure is + *per-core* (derived as 1/latency from a single serial op). +- **Sustained throughput under concurrency** *(--full)* — the complementary + *measured* answer: N worker processes run at once (N = 1, 2, 4, … up to the core + count) to capture real memory-bandwidth contention, reporting the machine's true + aggregate solves/s and verifies/s and the saturation knee. Additive — it never + overwrites the per-core estimate above. +- **Mining rate vs difficulty** *(--full, or standalone via `configs/mining.toml`)* — + the measured whole-machine token mint rate at each effort target (pooled over + many independent nonce ranges, one streaming search per core), the basis for + "control the mint rate by setting difficulty". Outputs `mining.csv` + a report + section; best measured under idle (`scripts/run_when_idle.sh ` gates on CPU idle). +- **Compiler-flag comparison** — the same C impl built under different + `-O` levels / `-march` / `-flto` / gcc-vs-clang, compared side by side. + +Two companion documents build on the measurements: `docs/findings.md` (the full +findings report: how Equi-X works, message-exchange schemas, DoS and mining +usage with the measured numbers) and `docs/difficulty-control.md` (closed-loop +difficulty controllers — mint-rate + load — with a simulator calibrated on the +measured curve: `python -m equix_bench.difficulty_control`). + +Every generated plot compares the implementations (C vs Rust) on the same axes, +and a correctness **cross-check** proves the implementations agree (solutions from +one verify under the other; effort values match byte-for-byte). + +## Architecture + +```mermaid +flowchart TB + cfg["configs/*.toml
parameter matrix"] + + subgraph harness["Python harness — equix_bench"] + direction TB + config["config.py
expand matrix → cells"] + registry["registry.py
adapter manifests"] + runner["runner.py
spawn 1 process / cell"] + stats["stats.py
aggregate median/p95"] + crosscheck["crosscheck.py
interop gate"] + dos["dosprotect.py
DoS asymmetry"] + report["report.py
plots + report.md"] + end + + subgraph runners["Runners — JSON job/result over stdio"] + direction LR + c["runners/c
libequix + hashx"] + rust["runners/rust
equix + hashx crates"] + plugin["your adapter
(any language)"] + end + + outputs["report.md · results.csv · raw/*.json
plots/*.png (C-vs-Rust, faceted per CPU, DoS)"] + + cfg --> config + registry --> runner + config --> runner + runner -- job JSON --> c + runner -- job JSON --> rust + runner -- job JSON --> plugin + c -- result JSON --> stats + rust -- result JSON --> stats + plugin -- result JSON --> stats + stats --> report + crosscheck --> report + dos --> report + report --> outputs + + manifests["adapters/*.manifest.toml"] -.-> registry +``` + +- **Runners** wrap one implementation and speak a JSON-over-stdio protocol + (`adapters/README.md`). One process per parameter cell keeps memory attributable. +- **Harness** (`harness/equix_bench`) expands the TOML matrix, runs every cell, + aggregates statistics, cross-checks implementations, evaluates DoS-protection, + and renders `results/report.md`, `results/results.csv`, and comparison plots. +- **Extensible**: add an implementation by dropping in a runner that speaks the + protocol plus a manifest — including compiler-flag variants and future GPU runners. + +## Run everything (one command) + +```bash +./scripts/run_all.sh # bootstrap deps + build + test + benchmark + compiler variants +./scripts/run_all.sh --full # deeper sweep (full config + effort sweep; takes longer) +``` + +Or via make — `make benchmark` depends on `setup` and `test`, so one command gets a +correct-by-construction run: + +```bash +make benchmark # setup -> tests -> quick main benchmark +make benchmark-full # setup -> tests -> full sweep (concurrency + mining) +make help # all targets: check/test/compiler-flags/mining/control/clean/distclean +``` + +Copied or moved the repo (e.g. rsync'd to another machine)? `setup.sh` (and +`make setup`) detects the stale CMake caches that a copy carries and cleans them +automatically; `make clean` / `make distclean` are also available. + +`run_all.sh` does the whole pipeline: installs/builds dependencies +(`setup.sh`), runs the unit tests, runs the **main C-vs-Rust benchmark** (all +operations + the **DoS-protection verdict**, with the correctness gate), then +builds the **compiler-flag variants** and compares them. Outputs: + +- `results/main/report.md` — C vs Rust: time, throughput, RSS, compile, effort, DoS + (with `--full` also the concurrency and mining sections + `concurrency.csv`, `mining.csv`) +- `results/compiler_flags/report.md` — compiler-flag comparison + +Flags: `--out DIR`, `--no-variants`, `--no-setup`, `--no-tests` (see `--help`). + +## Quick start (step by step) + +```bash +# 1. Install any missing deps (cmake/compiler/cargo/python) + fetch + build. +# setup.sh auto-installs via the system package manager / rustup when possible. +# Use `--check` to only report what's missing; EQUIX_NO_AUTO_INSTALL=1 to disable. +./scripts/setup.sh # installs the harness + pytest into a project .venv + +# 2. Run the smoke benchmark (seconds). Use the venv interpreter setup.sh made: +.venv/bin/python -m equix_bench run --config configs/smoke.toml --out results/ +# (or `make benchmark` / `scripts/run_all.sh`, which auto-prefer .venv) + +# 3. Full sweep (minutes) +python -m equix_bench run --config configs/full.toml --out results/ + +# One-shot end-to-end check +./scripts/verify.sh +``` + +Outputs land in `results/`: `report.md`, `results.csv`, `raw/results.json`, and +`plots/*.png` (each comparing C vs Rust). + +## DoS-protection evaluation + +Equi-X is a DoS defense: requesters *solve* (expensive), the service *verifies* +(cheap). Any run that includes `effort` + `verify` gets a **DoS-protection** +section in the report — measured attacker-cost vs defender-cost on this system, +the asymmetry factor, verify throughput, and a verdict ("effective from effort ≥ E*"). + +```bash +python -m equix_bench run --config configs/dos_protection.toml --out results/ +# report.md -> "DoS-protection effectiveness (this system)" + dos_protection.png +``` + +## Comparing compiler flags + +Build the C runner (and libequix/hashx) under several compiler/optimization flag +sets, then benchmark them as separate impls — every plot compares the variants: + +```bash +./scripts/build_variants.sh # gcc -O0/-O2/-O3/-march=native/-flto, clang -O3, ... +python -m equix_bench run --config configs/compiler_flags.toml --out results/ +``` + +Each variant becomes `equix-c-` via an auto-generated manifest in +`adapters/generated/`; unbuilt variants (missing compiler) are skipped. + +The **main** `equix-c` runner is not one fixed guess: `setup.sh` runs +`scripts/autotune_c_flags.sh`, which builds the fast-tier flag candidates +(`-O2`, `-O3`, `-O3 -march=native`, `-O3 -flto`) with the default compiler, +benchmarks the JIT solve path, and installs the **fastest** binary as the main +runner (recording the winning flags in `build/runners/c/equix_runner.flags` and +`build/provenance.json`). Solve is JIT-dominated so the margin is usually small; +when plain `-O3` is within 1% of the best it is preferred (portable, no native +lock-in). Skip the tuning with `EQUIX_NO_AUTOTUNE=1` (falls back to `-O3 -DNDEBUG`). + +## Comparing multiple CPUs (and GPUs) + +Every run records the **CPU it executed on** (model, arch) and shows it on each +plot. To compare across machines, run on each and merge — no re-benchmarking: + +```bash +python -m equix_bench run --config configs/full.toml --out runA/ --device-label host-a +# ...on another machine... +python -m equix_bench run --config configs/full.toml --out runB/ --device-label host-b +python -m equix_bench combine --inputs runA runB --out combined/ +``` + +`combine` produces **faceted plots** (one panel per CPU, C-vs-Rust within each) plus +**`xdev_*` cross-CPU charts** (x=CPU, series=implementation) for headline metrics. +The **concurrency** and **mining** sections/figures are carried across every device +too (faceted per CPU), not just the solve/verify/effort plots. + +### Automatic, when results from many devices are collected in one tree + +Copy or `rsync` each device's output under a single directory, then combine the +whole tree in one command — no need to list every run by hand: + +```bash +scripts/combine_all.sh results-by-device --out combined/ # or: make combine ROOT=results-by-device +``` + +Discovery is **layout-agnostic**: any directory holding a `raw/results.json` is +treated as a run, and each run's **device identity comes from inside its records** +(CPU model + OS), so folder names are free-form and two machines are never +conflated. Re-runs of the same device are de-duplicated automatically (newest +wins). The layout can be anything, e.g.: + +``` +results-by-device/ + laptop-x1/main/{raw/results.json, concurrency.csv, mining.csv, ...} + server-epyc/main/{raw/results.json, ...} + rpi5/results/main/{raw/results.json, ...} +``` + +**GPU?** Equi-X/HashX is CPU-oriented by design (deliberately GPU/ASIC-hostile), so +no GPU implementation is benchmarked or bundled. The framework is GPU-ready though: +a runner reporting `device: "gpu"` plugs in via the adapter protocol and appears on +every figure as another device automatically. See `PARAMETERS.md` §8. + +## Requirements + +- C: `cmake` ≥ 3.10, `gcc`/`clang`. Rust: `cargo`/`rustc`. Python ≥ 3.11 (`matplotlib`). +- x86-64 or aarch64 for HashX JIT; other targets fall back to the interpreter. +- `setup.sh` installs the harness (matplotlib/numpy) and `pytest` into a project + `.venv`. This sidesteps the Linux "externally-managed-environment" / "pytest not + importable" failure (PEP-668): a virtualenv has its own writable site-packages, so + `pip` just works — no `pyenv` or `--break-system-packages` needed. On Debian/Ubuntu + it provisions `python3-venv` if missing, and falls back to system `pip` if a venv + can't be created. + +## Platform support (Linux, macOS, ARM64, Raspberry Pi 5) + +Runners and harness are portable C / Rust / Python. CI builds and runs the smoke +benchmark on **Linux (x86-64)** and **macOS (Apple Silicon)**; both are supported, +along with ARM64 Linux (Raspberry Pi 5). + +### macOS (Intel & Apple Silicon) + +- **Interpreter runtime: fully supported** on both Intel and Apple Silicon. +- **JIT (compiled runtime):** works on Linux and Intel macOS. On **Apple Silicon** + the bundled HashX C JIT uses `mmap`+`mprotect` without `MAP_JIT`, which the + kernel rejects — so the **C runner detects this and uses the interpreter** + (`try-compile` → interpreter, `must-compile` → clean error), never crashing. The + Rust impl JITs via `dynasmrt` (which handles Apple Silicon), so `runtime_effective` + honestly reports what each impl actually ran. +- Platform specifics are handled: peak RSS via `getrusage` (macOS reports bytes, not + Linux's KB — converted), CPU name via `sysctl` (no `/proc`), OS via `uname`. + +### ARM64 / Raspberry Pi 5 + +The framework is portable C / Rust / Python with no x86-specific code, so it runs +on **ARM64 including the Raspberry Pi 5**: + +- **64-bit OS (recommended for Pi 5):** full support, **including the JIT** — HashX + ships an aarch64 compiler backend (`compiler_a64.c`; the Rust side uses + `dynasmrt`), so `interpret`, `try-compile`, and `must-compile` all work. +- **32-bit OS (armv7/armhf):** runs **interpreter-only** — there is no 32-bit-ARM + JIT, so `try-compile` transparently falls back to the interpreter and + `must-compile` fails by design. Use a 64-bit OS on the Pi 5 to benchmark the JIT. +- CPU identification handles ARM `/proc/cpuinfo` (no `model name` field): the device + label uses the board `Model` (e.g. `raspberry-pi-5-model-b-rev-1-0-`). +- **Comparing x86 vs ARM** is a first-class use case — run on each and `combine` for + faceted x86-vs-Pi figures (the arch is recorded per device). +- **Caveat:** the Pi 5 can thermal-throttle under sustained solving; use active + cooling and watch the reported stddev/p95 for stability. + +Build on the Pi exactly as elsewhere: `./scripts/setup.sh` (it creates the `.venv` +with the harness installed; use `.venv/bin/python -m equix_bench ...` to run). + +## Parameters + +See **[PARAMETERS.md](PARAMETERS.md)** for a full description of every parameter +(operation, runtime, challenge/nonce, effort/difficulty, repetitions/warmup) and +its implications for time, memory, and cost. + +## Adding an implementation + +Write a runner that speaks the protocol in `adapters/README.md`, drop a +`.manifest.toml` next to the examples, and add its name to a config's +`run.impls`. No harness code changes required. + +## Licensing + +The framework code (runners glue, harness) is MIT (`LICENSE`). The vendored +`tevador/equix` + `hashx` (git submodule) and the Rust `equix`/`hashx` crates are +**LGPL-3.0-only**; they are used as dependencies/submodules and are not +relicensed here. diff --git a/tools/benchmarks/Equi-X/adapters/README.md b/tools/benchmarks/Equi-X/adapters/README.md new file mode 100644 index 0000000..7df98be --- /dev/null +++ b/tools/benchmarks/Equi-X/adapters/README.md @@ -0,0 +1,127 @@ +# Adapter / Plugin Protocol + +The benchmark harness is **implementation-agnostic**. Any Equi-X implementation +can be benchmarked by wrapping it in a **runner**: an executable that speaks a +tiny JSON-over-stdio protocol. The reference C and Rust runners are just two +adapters; add your own (in any language) by satisfying this contract and dropping +in a manifest. + +## Contract + +1. The runner reads **one** job-spec JSON object from **stdin**. +2. It performs the requested operation, measuring in-process. +3. It writes **one** result JSON object to **stdout** (the last line of stdout, so + you may print progress/diagnostics before it — everything else goes to stderr). +4. Exit code `0` on success, non-zero on error (still print a result JSON with + `"ok": false` and an `"error"` message when possible). + +One job = one process. The harness spawns a fresh process per parameter cell so +`peak_rss_kb` is attributable. + +## Job-spec (stdin) + +```jsonc +{ + "schema_version": 1, + "operation": "solve", // "solve" | "verify" | "effort" | "hashx_compile" + "runtime": "try-compile", // "interpret" | "try-compile" | "must-compile" + "repetitions": 20, // timed iterations + "warmup": 3, // untimed iterations before timing + + // solve / verify / hashx_compile: + "challenge_hex": "deadbeef", // hex challenge bytes (HashX seed) + "solution_hex": "6f27...0dc9", // verify only: 16-byte packed solution (8x u16 LE) + "challenge_seed_hex": "abcd", // solve/verify only, OPTIONAL, ALTERNATIVE to + // challenge_hex: each rep uses a fresh challenge + // from a SHA-256 chain over this seed + // (challenge_0 = SHA256(seed), challenge_{i+1} = + // SHA256(challenge_i)) so measurements span many + // challenges. Challenge generation, and (for + // verify) the setup solve that yields a token, + // MUST be excluded from every timed region. + // No solution_hex needed for verify: the runner + // self-solves each derived challenge. + + // effort / hashx_compile (nonce search): + "challenge_base_hex": "abcd", // challenge = base || little_endian(nonce, nonce_bytes) + "nonce_bytes": 8, + "nonce_start": 0, + "target_effort": 1000, // effort: stop when achieved >= target + "max_attempts": 5000000 // effort: safety cap +} +``` + +Only the fields relevant to the operation are present. Unknown fields must be +ignored. + +## Result (stdout) + +```jsonc +{ + "schema_version": 1, + "ok": true, + "impl": { "name": "equix-c", "version": "1.0.0", "commit": "b7bb7d9", + "runtime_effective": "compiled" }, + "operation": "solve", + "runtime_requested": "try-compile", + "runtime_effective": "compiled", // may differ (try-compile fallback) + "env": { "os": "linux", "compiler": "gcc-13.3.0", + "cpu": "Intel(R) Xeon(R) ... @ 2.10GHz", // device model string + "arch": "x86_64", // x86_64 | aarch64 | ... + "device": "cpu", // "cpu" | "gpu" + "os_version": "6.18.5" }, // kernel release; folded into the auto device label + "runs": [ // one entry per TIMED rep (warmups excluded) + { "index": 0, "wall_ns": 7582269, "solutions": 4, "compile_ns": 0, + "attempts": 0, "achieved_effort": 0, "verify_result": null } + ], + "solutions_hex": ["6f27...", "..."], // solve: final-rep solutions; effort: the winning solution; else null + "winning_nonce_hex": "0300000000000000", // effort only, OPTIONAL: wire bytes (LE, nonce_bytes long) + // of the winning token's nonce — lets the harness + // measure message sizes vs difficulty + "peak_rss_kb": 4548, // whole-process high-water; Linux KB + "error": null +} +``` + +### Field semantics per operation + +| operation | must populate | notes | +|-----------|---------------|-------| +| `solve` | `runs[].wall_ns`, `runs[].solutions`, `solutions_hex` | `solutions_hex` (final rep) enables the interop cross-check. | +| `verify` | `runs[].wall_ns`, `runs[].verify_result`, `runs[].solutions` (1/0) | `verify_result` ∈ `OK`/`CHALLENGE`/`ORDER`/`PARTIAL_SUM`/`FINAL_SUM` (or impl-specific string). | +| `effort` | `runs[].wall_ns`, `runs[].attempts`, `runs[].achieved_effort` | Effort formula: BLAKE2b-256(`challenge‖solution_bytes`), `achieved = (2^32-1)/hash32` (hash32 = first 4 bytes big-endian). **Must match byte-for-byte** — the cross-check enforces it. Report the winning token via `solutions_hex` + `winning_nonce_hex` when the target was reached (enables message-size measurement). | +| `hashx_compile` | `runs[].compile_ns`, `runs[].wall_ns` | `compile_ns` = program-gen + compile; `wall_ns` = one execution. | + +`solution_bytes` = the 8 solution indices as little-endian `uint16` (16 bytes). + +### Device (CPU/GPU) reporting + +Each runner reports the hardware it ran on in `env.cpu` / `env.arch` / `env.device`. +The harness stamps this onto every result so figures reflect the executing device +and results from different machines can be merged (`combine`). A CPU runner sets +`device: "cpu"`; **a GPU implementation would set `device: "gpu"`** and its device +name, and it then appears on the comparison figures automatically — no harness +change. Equi-X/HashX is CPU-oriented by design, so no GPU runner ships today, but +the protocol is ready for one. + +## Manifest + +Register an adapter by adding `.manifest.toml` to a manifest directory +(default `adapters/examples/`, override with `--manifests`): + +```toml +name = "my-equix" +exec = "path/to/runner" # or ["python3", "my_runner.py"]; paths are relative to repo root +protocol_version = 1 +capabilities = ["solve", "verify", "effort", "hashx_compile"] +runtimes = ["interpret", "try-compile", "must-compile"] + +[env] # optional environment for the runner process +MY_VAR = "value" +``` + +The harness skips (with a warning) any cell whose operation/runtime is not in the +adapter's declared `capabilities`/`runtimes`, and any adapter whose `exec` is not +found — so a partially built tree degrades gracefully instead of crashing. + +See `manifest.schema.json` for a machine-readable schema. diff --git a/tools/benchmarks/Equi-X/adapters/examples/c.manifest.toml b/tools/benchmarks/Equi-X/adapters/examples/c.manifest.toml new file mode 100644 index 0000000..a90c974 --- /dev/null +++ b/tools/benchmarks/Equi-X/adapters/examples/c.manifest.toml @@ -0,0 +1,10 @@ +# Adapter manifest for the reference C implementation (tevador/equix + hashx). +name = "equix-c" +exec = "build/runners/c/equix_runner" +protocol_version = 1 +capabilities = ["solve", "verify", "effort", "hashx_compile"] +runtimes = ["interpret", "try-compile", "must-compile"] + +# Optional provenance passed to the runner as environment variables. +[env] +EQUIX_C_VERSION = "1.0.0" diff --git a/tools/benchmarks/Equi-X/adapters/examples/rust.manifest.toml b/tools/benchmarks/Equi-X/adapters/examples/rust.manifest.toml new file mode 100644 index 0000000..f43e538 --- /dev/null +++ b/tools/benchmarks/Equi-X/adapters/examples/rust.manifest.toml @@ -0,0 +1,9 @@ +# Adapter manifest for the Rust implementation (Tor arti equix + hashx crates). +name = "equix-rust" +exec = "runners/rust/target/release/equix_runner" +protocol_version = 1 +capabilities = ["solve", "verify", "effort", "hashx_compile"] +runtimes = ["interpret", "try-compile", "must-compile"] + +[env] +EQUIX_RUST_VERSION = "0.7.0" diff --git a/tools/benchmarks/Equi-X/adapters/manifest.schema.json b/tools/benchmarks/Equi-X/adapters/manifest.schema.json new file mode 100644 index 0000000..1c301d3 --- /dev/null +++ b/tools/benchmarks/Equi-X/adapters/manifest.schema.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/madxor/equi-x/adapters/manifest.schema.json", + "title": "Equi-X benchmark adapter manifest", + "type": "object", + "required": ["name", "exec"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Unique implementation name, referenced by configs' run.impls." + }, + "exec": { + "description": "Runner argv. A string is a single-element argv. Paths are resolved relative to the repo root.", + "oneOf": [ + { "type": "string" }, + { "type": "array", "items": { "type": "string" }, "minItems": 1 } + ] + }, + "protocol_version": { + "type": "integer", + "default": 1, + "description": "Wire protocol version the runner speaks." + }, + "capabilities": { + "type": "array", + "items": { "enum": ["solve", "verify", "effort", "hashx_compile"] }, + "description": "Operations this runner supports; unsupported cells are skipped." + }, + "runtimes": { + "type": "array", + "items": { "enum": ["interpret", "try-compile", "must-compile"] }, + "description": "HashX runtimes this runner supports." + }, + "env": { + "type": "object", + "additionalProperties": { "type": "string" }, + "description": "Extra environment variables set for the runner process." + } + } +} diff --git a/tools/benchmarks/Equi-X/configs/compiler_flags.toml b/tools/benchmarks/Equi-X/configs/compiler_flags.toml new file mode 100644 index 0000000..6acd3a6 --- /dev/null +++ b/tools/benchmarks/Equi-X/configs/compiler_flags.toml @@ -0,0 +1,35 @@ +# Compare C-runner performance across compiler/optimization flags. +# First build the variants: ./scripts/build_variants.sh +# Impls that were not built (e.g. a missing compiler) are skipped with a warning. +[run] +# Flag differences are small and JIT-dominated, so use a generous rep count to +# keep the per-variant median stable enough to rank them meaningfully. +warmup = 5 +repetitions = 30 +impls = [ + "equix-c-gcc-o0", + "equix-c-gcc-o2", + "equix-c-gcc-o3", + "equix-c-gcc-o3-native", + "equix-c-gcc-o3-lto", + "equix-c-clang-o3", + "equix-c-clang-o3-native", +] + +# Solve dominates PoW cost; compare interpreter and JIT across flag sets. +[[jobs]] +operation = "solve" +runtimes = ["interpret", "try-compile"] +challenges = ["deadbeef", "0102030405060708"] + +# Program generation + JIT compile cost can also shift with flags. +[[jobs]] +operation = "hashx_compile" +runtimes = ["must-compile"] +challenges = ["deadbeef"] + +# All variants compute identical solutions -> the interop check still holds. +[crosscheck] +enabled = true +challenges = ["deadbeef"] +pairs = [["equix-c-gcc-o3", "equix-c-clang-o3"]] diff --git a/tools/benchmarks/Equi-X/configs/dos_protection.toml b/tools/benchmarks/Equi-X/configs/dos_protection.toml new file mode 100644 index 0000000..45dc6f7 --- /dev/null +++ b/tools/benchmarks/Equi-X/configs/dos_protection.toml @@ -0,0 +1,28 @@ +# DoS-protection evaluation: measures the attacker-solve vs defender-verify +# asymmetry across a range of efforts to answer "is Equi-X an effective DoS +# defense on THIS system, and at what effort?". A few minutes to run. +[run] +warmup = 2 +repetitions = 8 +impls = ["equix-c", "equix-rust"] + +# Defender cost: fast verify (both interpreter and JIT so you can compare). +[[jobs]] +operation = "verify" +runtimes = ["interpret", "try-compile"] +challenges = ["deadbeef"] + +# Attacker cost: measured time to craft one accepted token at each effort. +[[jobs]] +operation = "effort" +runtimes = ["try-compile"] +bases = ["abcd"] +targets = [200, 1000, 4000] +nonce_bytes = 8 +max_attempts = 50000000 +repetitions = 1 + +[crosscheck] +enabled = true +challenges = ["deadbeef"] +pairs = [["equix-c", "equix-rust"], ["equix-rust", "equix-c"]] diff --git a/tools/benchmarks/Equi-X/configs/full.toml b/tools/benchmarks/Equi-X/configs/full.toml new file mode 100644 index 0000000..0898904 --- /dev/null +++ b/tools/benchmarks/Equi-X/configs/full.toml @@ -0,0 +1,77 @@ +# Full sweep: all runtimes x all operations x multiple challenges + an effort +# sweep, with enough repetitions for stable medians. Minutes-scale. +[run] +warmup = 8 +repetitions = 100 +impls = ["equix-c", "equix-rust"] + +# --- solve: interpreter vs JIT. vary_challenge = each listed value is a SEED; +# every rep solves a fresh challenge from a SHA-256 chain, so the medians and +# p95 reflect the spread ACROSS challenges, not one fixed instance. Challenge +# generation is excluded from the timing. --- +[[jobs]] +operation = "solve" +runtimes = ["interpret", "try-compile", "must-compile"] +challenges = ["deadbeef", "0000000000000002", "cafe", "0102030405060708"] +vary_challenge = true + +# --- verify: cheap; in seed mode the runner self-solves each derived challenge +# (setup solve excluded from timing), so no harness-resolved solution. --- +[[jobs]] +operation = "verify" +runtimes = ["interpret", "try-compile", "must-compile"] +challenges = ["deadbeef", "0000000000000002", "cafe"] +vary_challenge = true + +# --- compile isolation: interpreter (program-gen only) vs JIT compile --- +[[jobs]] +operation = "hashx_compile" +runtimes = ["interpret", "must-compile"] +challenges = ["deadbeef", "0102030405060708"] + +# --- effort/difficulty sweep: cost grows ~linearly with target effort --- +[[jobs]] +operation = "effort" +runtimes = ["try-compile"] +bases = ["abcd"] +targets = [100, 1000, 10000] +nonce_bytes = 8 +nonce_start = 0 +max_attempts = 20000000 +repetitions = 10 + +[crosscheck] +enabled = true +challenges = ["deadbeef", "cafe", "0000000000000002"] +pairs = [["equix-c", "equix-rust"], ["equix-rust", "equix-c"]] + +# --- concurrency / saturation: MEASURED sustained solve & verify capacity --- +# Runs N worker processes at once (N = 1,2,4,... up to the core count) and +# measures aggregate throughput, so the report can state the machine's real +# parallel capacity and the saturation knee. Additive to the per-core DoS +# estimate — it never overwrites it. max_workers = 0 means os.cpu_count(). +[concurrency] +enabled = true +operations = ["solve", "verify"] +challenge = "deadbeef" +reps = 50 +warmup = 5 +max_workers = 0 +levels = [] + +# --- mining rate vs difficulty: MEASURED whole-machine token mint rate --- +# Basis for "control the mint rate by setting difficulty". Uses the fastest +# solver (open-network case). Each 1-core point pools independent searches from +# distinct nonce ranges; the machine point streams tokens_per_worker searches +# per core concurrently and pools tokens over busy time (failed searches +# charged to the denominator). +[mining] +enabled = true +impls = ["equix-rust"] +challenge_base = "abcd" +efforts = [100, 300, 1000, 3000] +samples = 50 +workers = 0 +tokens_per_worker = 5 +nonce_bytes = 8 +max_attempts = 1500000 diff --git a/tools/benchmarks/Equi-X/configs/mining.toml b/tools/benchmarks/Equi-X/configs/mining.toml new file mode 100644 index 0000000..afba610 --- /dev/null +++ b/tools/benchmarks/Equi-X/configs/mining.toml @@ -0,0 +1,24 @@ +# Mining-rate benchmark: measure the whole-machine token mint rate vs difficulty, +# using the fastest solver (the open-network case). Isolated so it can be run +# alone under idle conditions: +# python -m equix_bench run --config configs/mining.toml --out results/mining +[run] +impls = ["equix-rust"] + +[crosscheck] +enabled = false + +[mining] +enabled = true +impls = ["equix-rust"] # everyone in an open network uses the fastest solver +challenge_base = "abcd" +# Expected attempts per token grow ~linearly with effort (measured ≈0.3-0.5·E, +# with several solutions per solve attempt each drawing an effort value), so a +# token at high E is genuinely expensive to mint AND to average. We measure a +# 100→3000 curve (30× span, enough to show the ~1/E trend) and extrapolate. +efforts = [100, 300, 1000, 3000] +samples = 10 # independent 1-core mints per difficulty (distinct nonces) +workers = 0 # 0 = all cores, for the whole-machine rate +tokens_per_worker = 5 # tokens each concurrent worker streams (averages nonce variance) +nonce_bytes = 8 +max_attempts = 1500000 diff --git a/tools/benchmarks/Equi-X/configs/smoke.toml b/tools/benchmarks/Equi-X/configs/smoke.toml new file mode 100644 index 0000000..13b43f9 --- /dev/null +++ b/tools/benchmarks/Equi-X/configs/smoke.toml @@ -0,0 +1,35 @@ +# Smoke config: seconds-scale sanity run used by CI and for quick local checks. +[run] +warmup = 1 +repetitions = 3 +impls = ["equix-c", "equix-rust"] + +[[jobs]] +operation = "solve" +runtimes = ["interpret", "try-compile"] +challenges = ["deadbeef"] +vary_challenge = true + +[[jobs]] +operation = "verify" +runtimes = ["interpret"] +challenges = ["deadbeef"] +vary_challenge = true + +[[jobs]] +operation = "hashx_compile" +runtimes = ["interpret", "must-compile"] +challenges = ["deadbeef"] + +[[jobs]] +operation = "effort" +runtimes = ["try-compile"] +bases = ["abcd"] +targets = [50, 200] +nonce_bytes = 8 +max_attempts = 200000 + +[crosscheck] +enabled = true +challenges = ["deadbeef", "cafe"] +pairs = [["equix-c", "equix-rust"], ["equix-rust", "equix-c"]] diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/__init__.py b/tools/benchmarks/Equi-X/harness/equix_bench/__init__.py new file mode 100644 index 0000000..421308f --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/__init__.py @@ -0,0 +1,8 @@ +"""Equi-X PoW benchmarking harness. + +Orchestrates language-agnostic runners over a parameter matrix, aggregates +timing/cost statistics, cross-checks implementations, and renders comparison +reports and plots. +""" + +__version__ = "0.1.0" diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/__main__.py b/tools/benchmarks/Equi-X/harness/equix_bench/__main__.py new file mode 100644 index 0000000..bfdcd0c --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/__main__.py @@ -0,0 +1,4 @@ +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/cli.py b/tools/benchmarks/Equi-X/harness/equix_bench/cli.py new file mode 100644 index 0000000..6caabed --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/cli.py @@ -0,0 +1,398 @@ +"""Command-line entrypoint: `python -m equix_bench run --config ... --out ...`.""" +from __future__ import annotations + +import argparse +import json +import os +import signal +import sys +from datetime import datetime, timezone +from pathlib import Path + +from . import concurrency as concmod +from . import config as cfgmod +from . import mining as miningmod +from . import report as reportmod +from . import stats as statsmod +from .crosscheck import run_crosscheck +from .device import device_from_env +from .protocol import JobSpec, Result +from .registry import load_manifests +from .runner import RunnerError, run + + +def _repo_root(override: str | None) -> Path: + if override: + return Path(override).resolve() + return Path(__file__).resolve().parents[2] + + +def _cpu_model() -> str: + from .device import _host_cpu # cross-platform (Linux /proc, macOS sysctl) + + return _host_cpu() + + +def _resolve_verify_solutions(cells, adapters, repo_root): + """Fill solution_hex for verify cells by solving each challenge once with the + first capable implementation. Returns (usable_cells, warnings).""" + warnings = [] + cache: dict[str, str | None] = {} + solver = None + for name, a in adapters.items(): + if not a.capabilities or "solve" in a.capabilities: + if a.available(repo_root): + solver = (name, a) + break + out = [] + for c in cells: + # Seed-mode verify cells self-solve each derived challenge in the runner, + # so they need no pre-resolved solution. + if c.job.operation != "verify" or c.job.challenge_seed_hex is not None: + out.append(c) + continue + chal = c.job.challenge_hex + if chal not in cache: + if solver is None: + cache[chal] = None + else: + r = run(solver[1], JobSpec(operation="solve", runtime="try-compile", + repetitions=1, warmup=0, challenge_hex=chal), + repo_root) + sols = r.solutions_hex or [] + cache[chal] = sols[0] if sols else None + sol = cache[chal] + if sol is None: + warnings.append(f"verify skipped for challenge {chal}: no solution found") + continue + c.job.solution_hex = sol + out.append(c) + return out, warnings + + +def cmd_run(args) -> int: + repo_root = _repo_root(args.root) + if args.manifests: + manifest_dirs = [Path(args.manifests)] + else: + # built-in adapters + generated compiler-flag variants (if any) + manifest_dirs = [repo_root / "adapters" / "examples", repo_root / "adapters" / "generated"] + adapters = load_manifests(manifest_dirs) + if not adapters: + print(f"error: no adapter manifests found in {manifest_dirs}", file=sys.stderr) + return 2 + + # Keep only adapters whose runner is actually built/available. + available = {n: a for n, a in adapters.items() if a.available(repo_root)} + for n in adapters: + if n not in available: + print(f"warning: adapter '{n}' runner not found; skipping", file=sys.stderr) + + config = cfgmod.load_config(Path(args.config)) + out_dir = Path(args.out) + + # ---- cross-check only ---- + if args.crosscheck_only: + challenges = config.crosscheck.get("challenges", ["deadbeef", "cafe"]) + pairs = [tuple(p) for p in config.crosscheck.get("pairs", [])] or None + from .crosscheck import _pairs + pair_list = _pairs(config.crosscheck.get("pairs", []), list(available.keys())) + checks, ok = run_crosscheck(available, repo_root, challenges, pair_list) + for c in checks: + print(f"[{'PASS' if c.passed else 'FAIL'}] {c.kind}: {c.detail}") + print(f"\nCross-check overall: {'PASS' if ok else 'FAIL'}") + return 0 if ok else 1 + + # ---- full run ---- + cells, warns = cfgmod.expand(config, available) + for w in warns: + print(f"warning: {w}", file=sys.stderr) + cells, vwarns = _resolve_verify_solutions(cells, available, repo_root) + for w in vwarns: + print(f"warning: {w}", file=sys.stderr) + + print(f"Running {len(cells)} cells across {len(available)} implementations...") + all_stats = [] + raw = [] + for i, c in enumerate(cells, 1): + adapter = available[c.impl] + try: + result = run(adapter, c.job, repo_root, timeout=args.timeout) + except RunnerError as e: + print(f" [{i}/{len(cells)}] {c.impl} {c.group} FAILED: {e}", file=sys.stderr) + continue + # Device identity: derived from what the runner reported (accurate even + # for a remote or GPU runner), with the CLI label override applied. + device = device_from_env(result.env, override_label=args.device_label) + # Enrich the raw record so a run is self-contained for later `combine`. + result.raw["_label"] = c.label + result.raw["_device"] = device + result.raw["_impl"] = c.impl + result.raw["_group"] = c.group + raw.append(result.raw) + st = statsmod.summarize(c.impl, c.group, c.job.runtime, c.label, result, device) + all_stats.append(st) + tag = f"{c.impl}/{c.group}/{c.job.runtime} {c.label}" + if st.ok: + print(f" [{i}/{len(cells)}] {tag}: median {st.median_ns/1e6:.3f} ms") + else: + print(f" [{i}/{len(cells)}] {tag}: ERROR {st.error}", file=sys.stderr) + + # cross-check + checks = [] + if config.crosscheck.get("enabled", True) and len(available) >= 2: + from .crosscheck import _pairs + pair_list = _pairs(config.crosscheck.get("pairs", []), list(available.keys())) + challenges = config.crosscheck.get("challenges", ["deadbeef"]) + checks, _ = run_crosscheck(available, repo_root, challenges, pair_list) + + # concurrency / saturation benchmark (opt-in via a [concurrency] config block). + # Measures sustained parallel solve/verify capacity; additive to the per-core + # DoS estimate, which it never modifies. + concurrency = None + conc_cfg = config.raw.get("concurrency", {}) + if conc_cfg.get("enabled", False): + print("Running concurrency / saturation ladder...") + resolver = lambda env: device_from_env(env, override_label=args.device_label).get("label", "host") + # Default to the impls this run selected (not every built variant); the + # [concurrency] block can still name its own `impls` to override. + conc_adapters = {n: available[n] for n in config.impls if n in available} or available + concurrency = concmod.run_concurrency(conc_cfg, conc_adapters, repo_root, resolver, args.timeout) + for r in concurrency: + if r.error: + print(f" concurrency {r.impl}/{r.operation}: {r.error}", file=sys.stderr) + else: + print(f" concurrency {r.impl}/{r.operation}: peak " + f"{r.peak_ops_per_sec:,.0f} ops/s at {r.knee_workers} workers") + + # mining-rate benchmark (opt-in via a [mining] config block): measures + # whole-machine token production vs difficulty, the basis for rate control. + mining = None + mining_cfg = config.raw.get("mining", {}) + if mining_cfg.get("enabled", False): + print("Running mining-rate / difficulty ladder...") + resolver = lambda env: device_from_env(env, override_label=args.device_label).get("label", "host") + mine_adapters = {n: available[n] for n in config.impls if n in available} or available + mining = miningmod.run_mining(mining_cfg, mine_adapters, repo_root, resolver, args.timeout) + for r in mining: + for p in r.points: + print(f" mining {r.impl} E={p.effort}: {p.tokens_per_sec_1core:,.2f} tok/s/core, " + f"{p.tokens_per_sec_machine:,.2f} tok/s machine ({p.ok_workers} workers)") + + devices = sorted({s.device_label for s in all_stats}) + meta = { + "timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "config": args.config, + "cpu": _cpu_model(), + "nproc": os.cpu_count() or "?", + "devices": devices, + } + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "run_meta.json").write_text(json.dumps({ + **meta, + "device_records": {s.device_label: {"type": s.device_type, "name": s.device_name, + "arch": s.device_arch} for s in all_stats}, + }, indent=2)) + if concurrency: + concmod.write_csv(concurrency, out_dir / "concurrency.csv") + if mining: + miningmod.write_csv(mining, out_dir / "mining.csv") + reportmod.generate(all_stats, checks, raw, out_dir, meta, concurrency=concurrency, mining=mining) + print(f"\nReport written to {out_dir/'report.md'} (plots in {out_dir/'plots'})") + + if checks and not all(c.passed for c in checks): + print("Cross-check FAILED", file=sys.stderr) + return 1 + return 0 + + +def _load_cells_from_raw(raw_list: list[dict]) -> list[statsmod.CellStats]: + """Rebuild CellStats from enriched raw records (each carries _label/_device).""" + out = [] + for d in raw_list: + try: + result = Result.from_dict(d) + except ValueError: + continue + label = d.get("_label", {}) + device = d.get("_device", {}) + impl = d.get("_impl", result.impl_name) + group = d.get("_group", result.operation) + out.append(statsmod.summarize(impl, group, result.runtime_requested, label, result, device)) + return out + + +def _discover_runs(root: Path) -> list[Path]: + """Every run-output directory under `root`, identified by its `raw/results.json`. + Layout-agnostic: works whether devices are laid out as `//main`, + `//results/main`, or arbitrary rsync'd trees — a run is anything + with a raw record file, and device identity comes from the records, not paths.""" + runs = {p.parent.parent for p in root.rglob("raw/results.json")} + return sorted(runs) + + +def _dedup_key(r: dict) -> tuple: + """Identity of one measured cell for de-duplication across discovered runs: + (device, impl, operation, runtime, label). Includes runtime because two + runtimes of the same op/challenge share _group and _label and would otherwise + collide (dropping one).""" + dev = (r.get("_device") or {}).get("label", "") + try: + res = Result.from_dict(r) + op, rt = res.operation, res.runtime_requested + except (ValueError, KeyError, TypeError): + op, rt = r.get("operation", ""), r.get("runtime_requested", "") + return (dev, r.get("_impl"), op, rt, json.dumps(r.get("_label", {}), sort_keys=True)) + + +def _collect_runs(inputs: list[Path]): + """Load and de-duplicate raw records + concurrency/mining results across runs. + When the same cell (or device's concurrency/mining ladder) appears in more + than one run, the record from the newest run (by run_meta timestamp) wins, so + re-runs replace rather than double-count. Returns (raw, conc, mining, seen_dirs).""" + from . import concurrency as concmod + from . import mining as miningmod + + raw_by_key: dict[tuple, tuple[str, dict]] = {} # key -> (ts, record) + conc_by_key: dict[tuple, tuple[str, Any]] = {} + mine_by_key: dict[tuple, tuple[str, Any]] = {} + seen_dirs: list[Path] = [] + for d in inputs: + raw_path = d / "raw" / "results.json" + if not raw_path.exists(): + print(f"warning: skipping '{d}' (no raw/results.json)", file=sys.stderr) + continue + seen_dirs.append(d) + meta_path = d / "run_meta.json" + ts = "" + if meta_path.exists(): + try: + ts = json.loads(meta_path.read_text()).get("timestamp", "") + except (ValueError, OSError): + ts = "" + for r in json.loads(raw_path.read_text()): + k = _dedup_key(r) + if k not in raw_by_key or ts >= raw_by_key[k][0]: + raw_by_key[k] = (ts, r) + cpath = d / "concurrency.csv" + if cpath.exists(): + for cr in concmod.read_csv(cpath): + k = (cr.device, cr.impl, cr.operation) + if k not in conc_by_key or ts >= conc_by_key[k][0]: + conc_by_key[k] = (ts, cr) + mpath = d / "mining.csv" + if mpath.exists(): + for mr in miningmod.read_csv(mpath): + k = (mr.device, mr.impl, mr.challenge_base) + if k not in mine_by_key or ts >= mine_by_key[k][0]: + mine_by_key[k] = (ts, mr) + raw = [rec for _ts, rec in raw_by_key.values()] + conc = [cr for _ts, cr in conc_by_key.values()] + mining = [mr for _ts, mr in mine_by_key.values()] + return raw, conc, mining, seen_dirs + + +def cmd_combine(args) -> int: + """Merge multiple prior runs into a single faceted (per-device) report — with + the concurrency and mining sections/figures carried across all runs. Inputs + are either listed explicitly (--inputs) or discovered under a tree (--root).""" + inputs: list[Path] = [Path(p) for p in (args.inputs or [])] + if args.root: + discovered = _discover_runs(Path(args.root)) + if not discovered: + print(f"error: no run directories (with raw/results.json) found under " + f"'{args.root}'", file=sys.stderr) + return 2 + inputs.extend(discovered) + if not inputs: + print("error: provide run dirs via --inputs DIR... or a tree via --root DIR", + file=sys.stderr) + return 2 + # De-dup identical paths (e.g. --root and --inputs overlapping) preserving order. + seen: set[str] = set() + inputs = [p for p in inputs if not (str(p) in seen or seen.add(str(p)))] + + all_raw, conc, mining, seen_dirs = _collect_runs(inputs) + stats = _load_cells_from_raw(all_raw) + if not stats: + print("error: no usable records found in inputs", file=sys.stderr) + return 2 + + devices_seen = sorted({s.device_label for s in stats}) + # Manifest: show exactly which dirs contributed which devices, so a missed + # tree can't silently masquerade as full coverage. + print(f"Discovered {len(seen_dirs)} run(s) across {len(devices_seen)} device(s):") + for d in seen_dirs: + try: + recs = json.loads((d / "raw" / "results.json").read_text()) + devs = sorted({(r.get("_device") or {}).get("label", "?") for r in recs}) + except (ValueError, OSError): + devs = ["?"] + extra = [] + if (d / "concurrency.csv").exists(): + extra.append("concurrency") + if (d / "mining.csv").exists(): + extra.append("mining") + tail = f" (+{', '.join(extra)})" if extra else "" + print(f" - {d} -> {', '.join(devs)}{tail}") + + meta = { + "timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "config": f"combine of {len(seen_dirs)} runs", + "cpu": ", ".join(devices_seen), + "nproc": "?", + "devices": devices_seen, + } + out_dir = Path(args.out) + reportmod.generate(stats, [], all_raw, out_dir, meta, + concurrency=conc or None, mining=mining or None) + print(f"\nCombined report for devices {devices_seen} -> {out_dir/'report.md'}") + return 0 + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(prog="equix_bench", description="Equi-X PoW benchmarking framework") + sub = p.add_subparsers(dest="cmd", required=True) + + r = sub.add_parser("run", help="run a benchmark config") + r.add_argument("--config", required=True, help="path to a TOML config") + r.add_argument("--out", default="results", help="output directory") + r.add_argument("--root", default=None, help="repo root (default: inferred)") + r.add_argument("--manifests", default=None, help="adapter manifest directory") + r.add_argument("--timeout", type=float, default=900.0, help="per-cell timeout (s)") + r.add_argument("--crosscheck-only", action="store_true", help="only run the interop cross-check") + r.add_argument("--device-label", "--cpu-label", dest="device_label", default=None, + help="human label for the executing device/CPU (default: auto from CPU model)") + r.set_defaults(func=cmd_run) + + c = sub.add_parser("combine", help="merge multiple runs into per-device comparison figures") + c.add_argument("--inputs", nargs="+", default=None, help="run output directories to merge") + c.add_argument("--root", default=None, + help="auto-discover every run (dir with raw/results.json) under this tree") + c.add_argument("--out", default="combined", help="output directory") + c.set_defaults(func=cmd_combine) + + args = p.parse_args(argv) + + # Clean Ctrl+C: kill any live runner subprocesses (worker threads in the + # concurrency/mining pools never receive KeyboardInterrupt themselves, so the + # handler — which always runs in the main thread — reaps them promptly so a + # blocked pool.shutdown can't hang), then raise KeyboardInterrupt so the run + # unwinds normally. We catch it below to exit 130 without a traceback. + from .runner import terminate_all_children + + def _on_sigint(signum, frame): + terminate_all_children() + raise KeyboardInterrupt + + signal.signal(signal.SIGINT, _on_sigint) + try: + return args.func(args) + except KeyboardInterrupt: + terminate_all_children() + print("\nInterrupted (Ctrl+C) — stopped; runner subprocesses killed.", file=sys.stderr) + return 130 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/concurrency.py b/tools/benchmarks/Equi-X/harness/equix_bench/concurrency.py new file mode 100644 index 0000000..1734271 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/concurrency.py @@ -0,0 +1,349 @@ +"""Concurrency / saturation benchmark: the machine's *measured* sustained +solve and verify capacity under parallel load. + +`dosprotect.py` reports *per-core* figures derived as 1/latency from a single +serial operation -- it never runs anything concurrently, so multiplying by the +core count over-estimates (Equi-X solving is memory-hard, so N parallel solvers +contend for cache/memory bandwidth and scale sub-linearly). + +This module answers the complementary, measured question: run N worker +processes at once, for N stepping up a ladder to the core count, and measure the +aggregate throughput at each level. That yields: + + * the real sustained solves/sec and verifies/sec the machine handles, + * the "knee" -- the worker count at peak throughput, beyond which adding + workers stops helping (bandwidth saturated), + * a scaling efficiency vs. ideal linear scaling. + +It is additive: it does not touch or replace the per-core DoS estimate. +""" +from __future__ import annotations + +import math +import statistics +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional + +from .protocol import JobSpec, Result +from .registry import Adapter +from .runner import RunnerError, run + + +@dataclass +class LevelStat: + workers: int # concurrency level attempted + ok_workers: int # workers that returned a usable result + per_worker_median_s: float # median across workers of each worker's median op time + aggregate_ops_per_sec: float # sum over workers of 1/(that worker's median op time) + per_worker_ops_per_sec: float # aggregate / ok_workers + scaling_efficiency: float # aggregate(N) / (N * single-worker baseline) + total_peak_rss_kb: int # summed peak RSS across the concurrent workers + + +@dataclass +class ConcResult: + device: str + impl: str + operation: str + nproc: int + reps: int + challenge: str + baseline_ops_per_sec: float # single-worker (level 1) throughput + peak_ops_per_sec: float # best aggregate across levels + knee_workers: int # worker count at peak aggregate throughput + levels: list[LevelStat] = field(default_factory=list) + error: Optional[str] = None + + +def _ladder(max_workers: int, explicit: Optional[list[int]] = None) -> list[int]: + """Worker-count ladder: 1, 2, 4, 8, ... capped at max_workers, with + max_workers itself always included (so an 6- or 10-core box gets its top). + Explicit lists are clamped to [1, max_workers]; level 1 is always included + because it anchors the per-worker baseline every derived figure needs.""" + if explicit: + levels = sorted({n for n in explicit if 1 <= n <= max_workers}) + if not levels: + raise ValueError( + f"concurrency levels {explicit} all outside [1, {max_workers}]" + ) + return sorted({1, *levels}) + levels = [] + n = 1 + while n < max_workers: + levels.append(n) + n *= 2 + levels.append(max_workers) + return sorted(set(levels)) + + +def _worker_median_s(res: Result) -> Optional[float]: + """One worker's median per-op time in seconds (runner-internal timing, which + already excludes process startup and warmup). None if the worker had no runs.""" + if not res.ok or not res.runs: + return None + walls = [float(r.wall_ns) for r in res.runs if r.wall_ns > 0] + if not walls: + return None + return statistics.median(walls) / 1e9 + + +def _measure_level( + adapter: Adapter, + make_spec: Callable[[], JobSpec], + n: int, + repo_root: Path, + timeout: float, +) -> tuple[list[float], int]: + """Launch n identical workers concurrently; return (per-worker median seconds + for the workers that succeeded, summed peak RSS KB across all workers). + + Real subprocesses run in parallel: the thread pool only blocks on their I/O, + so contention is measured on the actual runner binary, not in Python.""" + def one(_i: int) -> Optional[Result]: + try: + return run(adapter, make_spec(), repo_root, timeout=timeout) + except RunnerError: + return None + + with ThreadPoolExecutor(max_workers=n) as pool: + results = list(pool.map(one, range(n))) + + medians: list[float] = [] + rss = 0 + for res in results: + if res is None: + continue + rss += max(0, res.peak_rss_kb) + m = _worker_median_s(res) + if m and m > 0: + medians.append(m) + return medians, rss + + +def _solution_for(adapter: Adapter, challenge: str, repo_root: Path, timeout: float) -> Optional[str]: + """Solve `challenge` once to obtain a solution to feed the verify workers.""" + try: + r = run(adapter, JobSpec(operation="solve", runtime="try-compile", + repetitions=1, warmup=0, challenge_hex=challenge), + repo_root, timeout=timeout) + except RunnerError: + return None + sols = r.solutions_hex or [] + return sols[0] if sols else None + + +def _first_env(adapter: Adapter, challenge: str, repo_root: Path, timeout: float) -> dict[str, Any]: + """Cheap single run purely to learn the executing device (env) for labeling. + Uses a 1-rep solve: always valid (a verify probe would need a solution and + fail, silently mislabeling the device).""" + try: + r = run(adapter, JobSpec(operation="solve", runtime="try-compile", repetitions=1, + warmup=0, challenge_hex=challenge), + repo_root, timeout=timeout) + return r.env + except RunnerError: + return {} + + +# Each worker's runner-internal measured window must dwarf the multi-ms +# subprocess start skew, or the workers' windows barely overlap and the +# "measured under concurrency" number degenerates to serial-rate x N. +MIN_WINDOW_S = 0.5 + + +def measure( + adapter: Adapter, + operation: str, + challenge: str, + solution_hex: Optional[str], + max_workers: int, + reps: int, + warmup: int, + repo_root: Path, + device_label: str, + timeout: float, + levels: Optional[list[int]] = None, + min_window_s: float = MIN_WINDOW_S, +) -> ConcResult: + """Run the saturation ladder for one (impl, operation) and summarize it.""" + def make_spec(n_reps: int) -> JobSpec: + return JobSpec( + operation=operation, + runtime="try-compile", + repetitions=n_reps, + warmup=warmup, + challenge_hex=challenge, + solution_hex=solution_hex if operation == "verify" else None, + ) + + result = ConcResult( + device=device_label, impl=adapter.name, operation=operation, + nproc=max_workers, reps=reps, challenge=challenge, + baseline_ops_per_sec=0.0, peak_ops_per_sec=0.0, knee_workers=0, + ) + + # Calibrate: one uncontended run tells us the per-op time, from which we + # size reps so every worker's measured window is at least min_window_s + # (fast ops like verify at ~17us need tens of thousands of reps to overlap + # meaningfully; slow ops like solve already exceed the window with a few). + cal_medians, _ = _measure_level(adapter, lambda: make_spec(reps), 1, repo_root, timeout) + eff_reps = reps + if cal_medians and min_window_s > 0: + eff_reps = max(reps, math.ceil(min_window_s / cal_medians[0])) + result.reps = eff_reps + + baseline = 0.0 + for n in _ladder(max_workers, levels): + medians, rss = _measure_level(adapter, lambda: make_spec(eff_reps), n, repo_root, timeout) + if not medians: + result.levels.append(LevelStat(n, 0, 0.0, 0.0, 0.0, 0.0, rss)) + continue + aggregate = sum(1.0 / m for m in medians) + if baseline == 0.0: + # Per-WORKER throughput anchors ideal scaling; falling back to a + # level n>1 must divide by n or every derived figure is n-fold off. + baseline = aggregate / n + ideal = baseline * n + result.levels.append(LevelStat( + workers=n, + ok_workers=len(medians), + per_worker_median_s=statistics.median(medians), + aggregate_ops_per_sec=aggregate, + per_worker_ops_per_sec=aggregate / len(medians), + scaling_efficiency=(aggregate / ideal) if ideal > 0 else 0.0, + total_peak_rss_kb=rss, + )) + + result.baseline_ops_per_sec = baseline + usable = [lv for lv in result.levels if lv.aggregate_ops_per_sec > 0] + if usable: + peak = max(usable, key=lambda lv: lv.aggregate_ops_per_sec) + result.peak_ops_per_sec = peak.aggregate_ops_per_sec + result.knee_workers = peak.workers + else: + result.error = "no worker produced a usable measurement" + return result + + +def run_concurrency( + cfg: dict[str, Any], + adapters: dict[str, Adapter], + repo_root: Path, + device_resolver: Callable[[dict[str, Any]], str], + timeout: float, +) -> list[ConcResult]: + """Drive the concurrency benchmark from a `[concurrency]` config block. + + cfg keys (all optional): operations (["solve","verify"]), impls ([] = all + available), challenge ("deadbeef"), reps (40), warmup (5), max_workers + (0 = os.cpu_count()), levels ([] = auto power-of-two ladder).""" + import os + + operations = list(cfg.get("operations", ["solve", "verify"])) + challenge = str(cfg.get("challenge", "deadbeef")) + reps = int(cfg.get("reps", 40)) + warmup = int(cfg.get("warmup", 5)) + max_workers = int(cfg.get("max_workers", 0)) or (os.cpu_count() or 4) + levels = [int(x) for x in cfg.get("levels", [])] or None + want_impls = list(cfg.get("impls", [])) or list(adapters.keys()) + + impls = [(n, adapters[n]) for n in want_impls if n in adapters] + out: list[ConcResult] = [] + if not impls: + return out + + # Learn the device label once from a cheap probe on the first impl. + probe_env = _first_env(impls[0][1], challenge, repo_root, timeout) + device_label = device_resolver(probe_env) + + # A verify needs a valid solution; solve once (impl-independent) and reuse. + solution_hex = None + if "verify" in operations: + for _, a in impls: + solution_hex = _solution_for(a, challenge, repo_root, timeout) + if solution_hex: + break + + for op in operations: + if op == "verify" and not solution_hex: + continue # nothing to verify against; skip rather than error the run + for name, adapter in impls: + out.append(measure( + adapter, op, challenge, solution_hex, max_workers, reps, warmup, + repo_root, device_label, timeout, levels, + )) + return out + + +def write_csv(results: list[ConcResult], path: Path) -> None: + import csv + + with open(path, "w", newline="") as f: + w = csv.writer(f) + w.writerow([ + "device", "impl", "operation", "workers", "ok_workers", + "per_worker_median_s", "aggregate_ops_per_sec", + "per_worker_ops_per_sec", "scaling_efficiency", "total_peak_rss_kb", + ]) + for r in results: + for lv in r.levels: + w.writerow([ + r.device, r.impl, r.operation, lv.workers, lv.ok_workers, + f"{lv.per_worker_median_s:.9f}", f"{lv.aggregate_ops_per_sec:.3f}", + f"{lv.per_worker_ops_per_sec:.3f}", f"{lv.scaling_efficiency:.4f}", + lv.total_peak_rss_kb, + ]) + + +def read_csv(path: Path) -> list[ConcResult]: + """Reconstruct ConcResults from a concurrency.csv (the inverse of write_csv), + so `combine` can fold in each device's measured saturation ladder. + + The CSV is per-level; the per-result summary fields (baseline/peak/knee/nproc) + are re-derived from the levels — baseline is the single-worker aggregate, + nproc the top level, and peak/knee the best aggregate — matching how + `measure()` computed them originally.""" + import csv + + def _i(row, k): + v = row.get(k, "") + return int(v) if v not in ("", None) else 0 + + def _f(row, k): + v = row.get(k, "") + return float(v) if v not in ("", None) else 0.0 + + by_key: dict[tuple[str, str, str], ConcResult] = {} + with open(path, newline="") as f: + for row in csv.DictReader(f): + key = (row["device"], row["impl"], row["operation"]) + res = by_key.get(key) + if res is None: + res = ConcResult( + device=row["device"], impl=row["impl"], operation=row["operation"], + nproc=0, reps=0, challenge="", baseline_ops_per_sec=0.0, + peak_ops_per_sec=0.0, knee_workers=0, + ) + by_key[key] = res + res.levels.append(LevelStat( + workers=_i(row, "workers"), + ok_workers=_i(row, "ok_workers"), + per_worker_median_s=_f(row, "per_worker_median_s"), + aggregate_ops_per_sec=_f(row, "aggregate_ops_per_sec"), + per_worker_ops_per_sec=_f(row, "per_worker_ops_per_sec"), + scaling_efficiency=_f(row, "scaling_efficiency"), + total_peak_rss_kb=_i(row, "total_peak_rss_kb"), + )) + for res in by_key.values(): + res.levels.sort(key=lambda lv: lv.workers) + res.nproc = max((lv.workers for lv in res.levels), default=0) + base = next((lv for lv in res.levels if lv.workers == 1), None) + res.baseline_ops_per_sec = base.aggregate_ops_per_sec if base else 0.0 + usable = [lv for lv in res.levels if lv.aggregate_ops_per_sec > 0] + if usable: + peak = max(usable, key=lambda lv: lv.aggregate_ops_per_sec) + res.peak_ops_per_sec = peak.aggregate_ops_per_sec + res.knee_workers = peak.workers + return list(by_key.values()) diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/config.py b/tools/benchmarks/Equi-X/harness/equix_bench/config.py new file mode 100644 index 0000000..9fbcb83 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/config.py @@ -0,0 +1,155 @@ +"""Config loading + parameter-matrix expansion into concrete runner cells. + +A "cell" is one fully-specified runner invocation: (impl, JobSpec) plus grouping +metadata used for aggregation and plotting. Expansion is the cartesian product +of impls x operations x runtimes x challenges/targets, filtered by each adapter's +declared capabilities/runtimes. +""" +from __future__ import annotations + +import tomllib +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +from .protocol import JobSpec +from .registry import Adapter + + +@dataclass +class Cell: + impl: str + group: str # operation, used as the primary plot grouping + label: dict[str, Any] # human-facing params (challenge, target, ...) + job: JobSpec + + +@dataclass +class Config: + warmup: int + repetitions: int + impls: list[str] + jobs: list[dict[str, Any]] + crosscheck: dict[str, Any] = field(default_factory=dict) + raw: dict[str, Any] = field(default_factory=dict) + + +def load_config(path: Path) -> Config: + with open(path, "rb") as f: + d = tomllib.load(f) + run = d.get("run", {}) + return Config( + warmup=int(run.get("warmup", 3)), + repetitions=int(run.get("repetitions", 10)), + impls=list(run.get("impls", [])), + jobs=list(d.get("jobs", [])), + crosscheck=dict(d.get("crosscheck", {})), + raw=d, + ) + + +def _supports(adapter: Adapter, operation: str, runtime: str) -> bool: + if adapter.capabilities and operation not in adapter.capabilities: + return False + if adapter.runtimes and runtime not in adapter.runtimes: + return False + return True + + +def expand(config: Config, adapters: dict[str, Adapter]) -> tuple[list[Cell], list[str]]: + """Return (cells, warnings). Cells whose impl/op/runtime is unsupported are + skipped with a warning rather than crashing the run.""" + cells: list[Cell] = [] + warnings: list[str] = [] + + for job in config.jobs: + op = job["operation"] + runtimes = job.get("runtimes", ["try-compile"]) + reps = int(job.get("repetitions", config.repetitions)) + warmup = int(job.get("warmup", config.warmup)) + + for impl in config.impls: + adapter = adapters.get(impl) + if adapter is None: + warnings.append(f"impl '{impl}' has no manifest; skipped") + continue + for runtime in runtimes: + if not _supports(adapter, op, runtime): + warnings.append( + f"{impl} does not support {op}/{runtime}; skipped" + ) + continue + cells.extend( + _expand_job(impl, op, runtime, reps, warmup, job) + ) + return cells, warnings + + +def _expand_job( + impl: str, + op: str, + runtime: str, + reps: int, + warmup: int, + job: dict[str, Any], +) -> list[Cell]: + out: list[Cell] = [] + + if op in ("solve", "verify", "hashx_compile"): + challenges = job.get("challenges", ["deadbeef"]) + # vary_challenge: treat each listed value as a SEED, deriving a fresh + # challenge per rep (SHA-256 chain) so the measurement spans many + # challenges instead of assuming one fixed instance. + vary = bool(job.get("vary_challenge", False)) and op in ("solve", "verify") + for chal in challenges: + spec = JobSpec( + operation=op, + runtime=runtime, + repetitions=reps, + warmup=warmup, + ) + if vary: + spec.challenge_seed_hex = chal + else: + spec.challenge_hex = chal + if op == "hashx_compile": + # hashx_compile varies the seed per rep via a nonce counter. + spec.challenge_hex = None + spec.challenge_base_hex = chal + spec.nonce_start = int(job.get("nonce_start", 0)) + # verify needs a solution_hex (filled by resolve_verify_solutions) — + # UNLESS in seed mode, where the runner self-solves each challenge. + out.append( + Cell(impl=impl, group=op, + label={"challenge": chal, "varied": True} if vary else {"challenge": chal}, + job=spec) + ) + + elif op == "effort": + bases = job.get("bases", ["abcd"]) + targets = job.get("targets", [1000]) + for base in bases: + for target in targets: + spec = JobSpec( + operation="effort", + runtime=runtime, + repetitions=reps, + warmup=warmup, + challenge_base_hex=base, + nonce_bytes=int(job.get("nonce_bytes", 8)), + nonce_start=int(job.get("nonce_start", 0)), + target_effort=int(target), + max_attempts=int(job.get("max_attempts", 5_000_000)), + ) + out.append( + Cell( + impl=impl, + group="effort", + label={"base": base, "target_effort": int(target)}, + job=spec, + ) + ) + else: + raise ValueError(f"unknown operation in config: {op}") + + return out diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/crosscheck.py b/tools/benchmarks/Equi-X/harness/equix_bench/crosscheck.py new file mode 100644 index 0000000..58dd2e4 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/crosscheck.py @@ -0,0 +1,116 @@ +"""Cross-implementation correctness gate. + +Two independent checks that any conforming implementations must pass: + +1. Interop: solve a challenge with impl A, then verify EACH returned solution + with impl B (both directions). Every solution must verify OK. This proves the + implementations compute the same HashX/Equihash puzzle. + +2. Effort agreement: run the effort op on A and B with identical parameters. Since + solving is deterministic, both must report the same attempts and achieved + effort -- this guards that the BLAKE2b effort preimage is byte-identical + across languages. +""" +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .protocol import JobSpec +from .registry import Adapter +from .runner import run + + +@dataclass +class Check: + kind: str + detail: str + passed: bool + + +def _pairs(config_pairs: list[list[str]], impls: list[str]) -> list[tuple[str, str]]: + if config_pairs: + return [(p[0], p[1]) for p in config_pairs] + # default: all ordered pairs + out = [] + for a in impls: + for b in impls: + if a != b: + out.append((a, b)) + return out + + +def run_crosscheck( + adapters: dict[str, Adapter], + repo_root: Path, + challenges: list[str], + pairs: list[tuple[str, str]], + effort_base: str = "abcd", + effort_target: int = 200, + effort_max_attempts: int = 200_000, +) -> tuple[list[Check], bool]: + checks: list[Check] = [] + + # 1. Interop: solutions from A verify under B. + for a_name, b_name in pairs: + a, b = adapters.get(a_name), adapters.get(b_name) + if not a or not b: + checks.append(Check("interop", f"{a_name}->{b_name}: missing adapter", False)) + continue + for chal in challenges: + solve = run( + a, + JobSpec(operation="solve", runtime="try-compile", repetitions=1, warmup=0, challenge_hex=chal), + repo_root, + ) + sols = solve.solutions_hex or [] + if not sols: + checks.append(Check("interop", f"{a_name} found 0 solutions for {chal} (nothing to verify)", True)) + continue + all_ok = True + for sol in sols: + v = run( + b, + JobSpec(operation="verify", runtime="interpret", repetitions=1, warmup=0, challenge_hex=chal, solution_hex=sol), + repo_root, + ) + vr = v.runs[-1].verify_result if v.runs else None + if vr != "OK": + all_ok = False + checks.append(Check("interop", f"{a_name} solution {sol} for {chal} did NOT verify under {b_name} (got {vr})", False)) + if all_ok: + checks.append(Check("interop", f"{a_name}->{b_name}: all {len(sols)} solutions for {chal} verify OK", True)) + + # 2. Effort agreement across all impls. + impls = list(adapters.keys()) + effort_results = {} + for name in impls: + a = adapters[name] + if a.capabilities and "effort" not in a.capabilities: + continue + r = run( + a, + JobSpec( + operation="effort", runtime="try-compile", repetitions=1, warmup=0, + challenge_base_hex=effort_base, nonce_bytes=8, nonce_start=0, + target_effort=effort_target, max_attempts=effort_max_attempts, + ), + repo_root, + ) + if r.runs: + effort_results[name] = (r.runs[-1].attempts, r.runs[-1].achieved_effort) + if len(effort_results) >= 2: + vals = set(effort_results.values()) + passed = len(vals) == 1 + checks.append( + Check( + "effort-agreement", + f"effort (attempts, achieved) across impls: {effort_results} " + + ("-- AGREE" if passed else "-- DISAGREE"), + passed, + ) + ) + + overall = all(c.passed for c in checks) and len(checks) > 0 + return checks, overall diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/device.py b/tools/benchmarks/Equi-X/harness/equix_bench/device.py new file mode 100644 index 0000000..a6435a3 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/device.py @@ -0,0 +1,81 @@ +"""Device (CPU/GPU) identity. + +Each runner self-reports its hardware in the result `env` (cpu, arch, device). +The harness derives a device record from that, so identity is correct even for a +remote runner or a future GPU adapter (which would report device="gpu"). A CLI +`--device-label` overrides only the human label, e.g. to disambiguate two machines +that report the same CPU model string. +""" +from __future__ import annotations + +import platform +import re +from typing import Any, Optional + + +def slug(s: str) -> str: + out = re.sub(r"[^a-zA-Z0-9]+", "-", s.strip().lower()).strip("-") + return out or "unknown" + + +def device_from_env(env: dict[str, Any], override_label: Optional[str] = None) -> dict[str, str]: + """Build a device record from a runner's reported env, falling back to the + host when the runner did not report hardware (older/minimal adapters). + + The auto label combines the device model with the OS version so runs on the + same CPU under different OS/kernel versions get distinct labels.""" + name = env.get("cpu") or _host_cpu() + arch = env.get("arch") or platform.machine() or "unknown" + dtype = env.get("device") or "cpu" + os_name = env.get("os") or platform.system().lower() or "unknown" + os_version = env.get("os_version") or platform.release() or "unknown" + return { + "type": dtype, + "name": name, + "arch": arch, + "os": os_name, + "os_version": os_version, + "label": override_label or slug(f"{name}-{os_version}"), + } + + +# Field priority works across arches: "model name" (x86), "Model" (Raspberry Pi +# board), "Hardware" (older ARM), "cpu model" (others). +_CPU_FIELDS = ("model name", "Model", "Hardware", "cpu model") + + +def parse_cpu_model(cpuinfo_text: str) -> str: + found: dict[str, str] = {} + for line in cpuinfo_text.splitlines(): + if ":" not in line: + continue + k, _, v = line.partition(":") + k, v = k.strip(), v.strip() + if k in _CPU_FIELDS and k not in found and v: + found[k] = v + for f in _CPU_FIELDS: + if f in found: + return found[f] + return "unknown" + + +def _host_cpu() -> str: + try: + model = parse_cpu_model(open("/proc/cpuinfo").read()) + if model != "unknown": + return model + except OSError: + pass + if platform.system() == "Darwin": # macOS has no /proc + try: + import subprocess + + out = subprocess.run( + ["sysctl", "-n", "machdep.cpu.brand_string"], + capture_output=True, text=True, timeout=2, + ) + if out.returncode == 0 and out.stdout.strip(): + return out.stdout.strip() + except Exception: # noqa: BLE001 + pass + return platform.processor() or "unknown" diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/difficulty_control.py b/tools/benchmarks/Equi-X/harness/equix_bench/difficulty_control.py new file mode 100644 index 0000000..f0b395f --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/difficulty_control.py @@ -0,0 +1,340 @@ +"""Reference difficulty (effort `E`) controllers + a simulator. + +Two closed-loop controllers that adjust the Equi-X effort parameter to demand, +calibrated on the MEASURED mint-rate curve (docs/findings.md §7a): + + * MintRateController (Design A) — hold a network's token MINT RATE at a target, + the way PoW difficulty retargeting works, specialized to the measured 1/E law. + * LoadController (Design B) — hold a single node's admission PRESSURE (offered + valid-token rate ÷ service capacity) at a target, so an attack is throttled + while honest clients pay the least difficulty that keeps the node healthy. + +Both rest on one measured fact: token rate ∝ 1/E on fixed hardware, and verify +cost is constant and ~free. So the control law is a cheap MULTIPLICATIVE step and +needs no absolute capacity model — it self-corrects from what it observes. + +Run the demo (writes plots + a summary): + python -m equix_bench.difficulty_control --out results/control +""" +from __future__ import annotations + +import math +import random +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional + +# Measured mint rate per reference machine (Apple M4 Pro, 14 cores, Rust-JIT), +# from results/main/mining.csv: tokens/second at each effort target. POOLED +# estimator (total tokens over total busy seconds across all workers) — a sum +# of per-worker 1/t ratios would carry a +6..25% upward bias at these CVs. +MEASURED_MINT: list[tuple[int, float]] = [(100, 69.1), (300, 23.5), (1000, 5.49), (3000, 1.75), (10000, 0.67)] + + +def mint_rate_per_machine(E: float, points: list[tuple[int, float]] = MEASURED_MINT) -> float: + """Tokens/s one reference machine mints at effort `E`. + + Log-log linear interpolation between the measured points; outside the + measured range it extrapolates as pure 1/E (slope -1 in log-log), the design + asymptote. Monotonically decreasing in E.""" + pts = sorted(points) + if E <= pts[0][0]: + e0, r0 = pts[0] + return r0 * (e0 / E) # 1/E extrapolation below range + if E >= pts[-1][0]: + e1, r1 = pts[-1] + return r1 * (e1 / E) # 1/E extrapolation above range + for (e0, r0), (e1, r1) in zip(pts, pts[1:]): + if e0 <= E <= e1: + f = (math.log(E) - math.log(e0)) / (math.log(e1) - math.log(e0)) + return math.exp(math.log(r0) + f * (math.log(r1) - math.log(r0))) + return pts[-1][1] # unreachable + + +def _clip(x: float, lo: float, hi: float) -> float: + return max(lo, min(hi, x)) + + +def equilibrium_E(machines: float, target_rate: float, + points: list[tuple[int, float]] = MEASURED_MINT, + lo: float = 1.0, hi: float = 1e9) -> float: + """The E at which `machines` reference machines mint `target_rate` tokens/s + (the natural controller seed). Solved by bisection on the measured curve — + no hard-coded constants to drift out of sync when MEASURED_MINT is updated.""" + for _ in range(200): + mid = math.sqrt(lo * hi) # geometric: the curve lives in log-log space + if machines * mint_rate_per_machine(mid, points) > target_rate: + lo = mid + else: + hi = mid + if hi / lo < 1.0001: + break + return math.sqrt(lo * hi) + + +@dataclass +class MintRateController: + """Design A: E ← E · clamp(R_obs / R*). Because rate ∝ 1/E, minting twice too + fast means E must double. Robust to capacity drift (only uses observed rate).""" + target_rate: float # R* (tokens/s the network should mint) + E: float = 1000.0 # current difficulty + e_min: float = 100.0 + e_max: float = 1e9 + max_factor: float = 4.0 # per-epoch clamp (anti-oscillation, Bitcoin-style) + ewma: float = 0.3 # smoothing of the observed rate (0..1); 1 = no smoothing + _r: Optional[float] = None # smoothed observed rate + + def update(self, observed_rate: float) -> float: + self._r = observed_rate if self._r is None else self.ewma * observed_rate + (1 - self.ewma) * self._r + factor = _clip((self._r / self.target_rate) if self.target_rate > 0 else 1.0, + 1.0 / self.max_factor, self.max_factor) + self.E = _clip(self.E * factor, self.e_min, self.e_max) + return self.E + + +@dataclass +class LoadController: + """Design B: E ← E · clamp(exp(k · (p - p_set))), where p = offered valid-token + rate ÷ service capacity. Raise E under pressure (throttle), let it decay toward + e_min when idle (cheap for honest clients). Deadband avoids flapping — note the + intended consequence: while pressure sits within the deadband of p_set, E HOLDS + (including post-attack, if load keeps utilization pinned at target); it only + decays once pressure drops below p_set - deadband.""" + p_set: float = 0.8 # target pressure (headroom below saturation) + k: float = 1.5 # gain + E: float = 300.0 + e_min: float = 300.0 + e_max: float = 1e9 + max_factor: float = 4.0 + deadband: float = 0.03 + + def update(self, pressure: float) -> float: + e = pressure - self.p_set + if abs(e) < self.deadband: + return self.E + factor = _clip(math.exp(self.k * e), 1.0 / self.max_factor, self.max_factor) + self.E = _clip(self.E * factor, self.e_min, self.e_max) + return self.E + + +# --------------------------------------------------------------------- simulators + + +@dataclass +class Trace: + t: list[float] = field(default_factory=list) + E: list[float] = field(default_factory=list) + signal: list[float] = field(default_factory=list) # observed rate (A) or pressure (B) + target: list[float] = field(default_factory=list) + extra: dict = field(default_factory=dict) + + +def simulate_mining(capacity: Callable[[int], float], target_rate: float, steps: int, + ctrl: Optional[MintRateController] = None, + rate_model: Callable[[float], float] = mint_rate_per_machine, + noise: float = 0.0, seed: int = 0) -> Trace: + """Network of `capacity(t)` machines; controller holds mint rate at target. + `noise` (fractional stddev) models real measurement jitter on the observed + rate; `seed` makes it reproducible.""" + ctrl = ctrl or MintRateController(target_rate=target_rate) + rng = random.Random(seed) + tr = Trace(target=[], extra={"capacity": [], "true_rate": []}) + for t in range(steps): + C = capacity(t) + R = C * rate_model(ctrl.E) # true tokens/s at current E + R_obs = R * (1 + rng.gauss(0, noise)) if noise else R + tr.t.append(t); tr.E.append(ctrl.E); tr.signal.append(R_obs) + tr.target.append(target_rate); tr.extra["capacity"].append(C) + tr.extra["true_rate"].append(R) # noise-free, for honest error stats + ctrl.update(R_obs) # set E for the next epoch + return tr + + +def simulate_dos(legit: Callable[[int], float], attackers: Callable[[int], float], + service_capacity: float, steps: int, ctrl: Optional[LoadController] = None, + honest_cores: float = 1.0, + rate_model: Callable[[float], float] = mint_rate_per_machine, + adaptive_attackers: Optional[Callable[[int, float], float]] = None) -> Trace: + """One protected node: honest req/s `legit(t)` plus attacker machines each + minting valid tokens at the current E. Controller holds admission pressure at + p_set. Records honest latency = time for one honest `honest_cores`-core client + to mint a token at the current E (the cost of defense to legitimate users). + + `adaptive_attackers(t, E)`, when given, replaces `attackers(t)`: the attacker + OBSERVES the current difficulty and can pause when solving is too expensive + and resume when E decays — the rational strategy against a decaying controller.""" + ctrl = ctrl or LoadController() + tr = Trace(target=[], extra={"attacker_rate": [], "presented": [], "honest_latency": [], "util": []}) + machine_cores = 14.0 # the reference machine the curve was measured on + for t in range(steps): + A = adaptive_attackers(t, ctrl.E) if adaptive_attackers else attackers(t) + m = rate_model(ctrl.E) # tokens/s per 14-core machine at current E + attacker_rate = A * m # valid tokens/s the attacker can present + presented = legit(t) + attacker_rate + pressure = presented / service_capacity + honest_latency = 1.0 / (m * honest_cores / machine_cores) # 1 honest client's token time + tr.t.append(t); tr.E.append(ctrl.E); tr.signal.append(pressure); tr.target.append(ctrl.p_set) + tr.extra["attacker_rate"].append(attacker_rate) + tr.extra["presented"].append(presented) + tr.extra["honest_latency"].append(honest_latency) + tr.extra["util"].append(min(1.0, pressure)) + ctrl.update(pressure) + return tr + + +# ------------------------------------------------------------------------- demo + + +def _plot_mining(tr: Trace, path: Path, title: str = "Design A — mint-rate controller") -> None: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + fig, ax = plt.subplots(3, 1, figsize=(8, 7), sharex=True) + ax[0].plot(tr.t, tr.extra["capacity"], color="#8172B3") + ax[0].set_ylabel("miners\n(machines)"); ax[0].set_title(title) + ax[1].plot(tr.t, tr.signal, color="#4C72B0", label="observed mint rate") + ax[1].plot(tr.t, tr.target, "--", color="#C44E52", label="target R*") + ax[1].set_ylabel("tokens / s"); ax[1].legend(fontsize=8) + ax[2].plot(tr.t, tr.E, color="#55A868"); ax[2].set_yscale("log") + ax[2].set_ylabel("effort E"); ax[2].set_xlabel("epoch") + for a in ax: a.grid(True, alpha=0.3) + fig.tight_layout(); fig.savefig(path, dpi=110); plt.close(fig) + + +def _plot_dos(tr: Trace, S: float, path: Path, + title: str = "Design B — single-node load controller") -> None: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + fig, ax = plt.subplots(4, 1, figsize=(8, 9), sharex=True) + ax[0].plot(tr.t, tr.extra["presented"], color="#DD8452", label="offered valid-token req/s") + ax[0].plot(tr.t, tr.extra["attacker_rate"], ":", color="#C44E52", label="attacker share") + ax[0].axhline(S, ls="--", color="#333", label=f"service capacity S={S:g}") + ax[0].set_ylabel("requests / s"); ax[0].set_title(title); ax[0].legend(fontsize=8) + ax[1].plot(tr.t, tr.extra["util"], color="#4C72B0", label="utilization") + ax[1].plot(tr.t, tr.target, "--", color="#C44E52", label="p_set") + ax[1].set_ylabel("utilization"); ax[1].set_ylim(0, 1.05); ax[1].legend(fontsize=8) + ax[2].plot(tr.t, tr.E, color="#55A868"); ax[2].set_yscale("log"); ax[2].set_ylabel("effort E") + ax[3].plot(tr.t, tr.extra["honest_latency"], color="#937860") + ax[3].set_ylabel("honest solve\ntime (s)"); ax[3].set_xlabel("tick") + for a in ax: a.grid(True, alpha=0.3) + fig.tight_layout(); fig.savefig(path, dpi=110); plt.close(fig) + + +def main(argv=None) -> int: + import argparse + p = argparse.ArgumentParser(description="Equi-X difficulty (E) controller demo") + p.add_argument("--out", default="results/control", help="output directory for plots") + p.add_argument("--steps", type=int, default=80) + args = p.parse_args(argv) + out = Path(args.out); out.mkdir(parents=True, exist_ok=True) + + # Design A: miners ramp 2→12, then a hashpower spike to 20 at 60%. Hold R* = 2 tok/s. + def capacity(t: int) -> float: + if t < 48: + return 2 + 10 * (t / 48) + return 20.0 + mining = simulate_mining(capacity, target_rate=2.0, steps=args.steps) + _plot_mining(mining, out / "control_mining.png") + + # Design B: honest 8 req/s throughout; a 6-machine flood during [24, 56). S = 40 req/s. + def legit(t: int) -> float: + return 8.0 + def attackers(t: int) -> float: + return 6.0 if 24 <= t < 56 else 0.0 + S = 40.0 + dos = simulate_dos(legit, attackers, service_capacity=S, steps=args.steps) + _plot_dos(dos, S, out / "control_dos.png") + + # Production run: how you'd actually deploy Design A. Gentle gains (max_factor + # 2, heavy EWMA), E SEEDED near equilibrium from the capacity estimate (no cold + # start), organic capacity growth, and ±8% measurement noise on the observed + # rate. Result: smooth tracking with no overshoot, stable under noise. + steps_p = max(args.steps, 120) + C0, R_target = 4.0, 2.0 + E_seed = equilibrium_E(C0, R_target) # seed at equilibrium: C0 machines mint R_target + def capacity_prod(t: int) -> float: + # organic growth 4 → ~13 with a slow wobble (diurnal-like), no hard steps + return 4.0 + 9.0 * (t / steps_p) + 0.8 * math.sin(t / 9.0) + prod_ctrl = MintRateController(target_rate=R_target, E=E_seed, max_factor=2.0, ewma=0.15) + prod = simulate_mining(capacity_prod, target_rate=R_target, steps=steps_p, + ctrl=prod_ctrl, noise=0.08, seed=1) + _plot_mining(prod, out / "control_production.png", + title="Production run — mint-rate controller (seeded E, gentle gains, ±8% noise)") + + # Summary numbers. + def _final(tr, key_signal): + return tr.E[-1], key_signal + # Run 4 — ADAPTIVE attacker vs Design B: 6 machines that only attack while + # E is below their give-up point (solving cheap enough to bother), pausing + # when the controller escalates and resuming as E decays. The rational + # strategy — does the loop oscillate, and what does the attacker still get? + E_GIVEUP = 800.0 + def adaptive(t: int, E: float) -> float: + if t < 16 or t >= 104: + return 0.0 + return 6.0 if E < E_GIVEUP else 0.0 + adaptive_tr = simulate_dos(legit, lambda t: 0.0, service_capacity=S, + steps=120, adaptive_attackers=adaptive) + _plot_dos(adaptive_tr, S, out / "control_adaptive.png", + title="Design B vs an ADAPTIVE attacker (attacks only while E < give-up)") + atk_window = adaptive_tr.extra["attacker_rate"][16:104] + duty = sum(1 for a in atk_window if a > 0) / len(atk_window) + sat = sum(1 for u in adaptive_tr.extra["util"][16:104] if u >= 0.999) + + # Run 5 — miner CHURN for Design A production tuning: miners join/leave as a + # seeded random walk instead of a smooth ramp; noise on the observed rate. + rng_churn = random.Random(7) + miners = 8.0 + churn_path = [] + for _ in range(140): + if rng_churn.random() < 0.15: miners += 1 + if rng_churn.random() < 0.15 and miners > 2: miners -= 1 + churn_path.append(miners) + churn_ctrl = MintRateController(target_rate=R_target, E=equilibrium_E(8.0, R_target), + max_factor=2.0, ewma=0.15) + churn = simulate_mining(lambda t: churn_path[t], target_rate=R_target, steps=140, + ctrl=churn_ctrl, noise=0.08, seed=2) + _plot_mining(churn, out / "control_churn.png", + title="Production tuning under miner churn (join/leave random walk, ±8% noise)") + churn_tail = churn.extra["true_rate"][40:] + churn_err = 100.0 * (sum(churn_tail) / len(churn_tail) - R_target) / R_target + + # Production-run steady-state error over the settled tail (last third). + # Report BOTH: the noisy observed mean (what an operator sees) and the + # noise-free true-rate mean (the controller's systematic tracking lag — + # averaging only the noisy signal would understate it). + n_tail = len(prod.signal) * 2 // 3 + obs_tail = prod.signal[n_tail:] + true_tail = prod.extra["true_rate"][n_tail:] + prod_err_obs = 100.0 * (sum(obs_tail) / len(obs_tail) - R_target) / R_target + prod_err_true = 100.0 * (sum(true_tail) / len(true_tail) - R_target) / R_target + lines = ["# Difficulty-control simulation\n", + f"- Design A (mining): final E={mining.E[-1]:,.0f}, " + f"mint rate {mining.signal[-1]:.2f} tok/s vs target {mining.target[-1]:.2f} " + f"(with {mining.extra['capacity'][-1]:.0f} miners).", + f"- Design B (DoS): peak E under attack ≈ {max(dos.E):,.0f}; " + f"steady honest solve time {dos.extra['honest_latency'][0]:.2f}s at rest → " + f"{max(dos.extra['honest_latency']):.2f}s under attack; " + f"utilization held near p_set={dos.target[-1]:.2f}.", + f"- Production run (seeded E, gentle gains, ±8% noise): no cold-start " + f"overshoot; settled tail within {prod_err_obs:+.1f}% of target (observed) / " + f"{prod_err_true:+.1f}% (noise-free — the systematic lag against the capacity " + f"ramp), final E={prod.E[-1]:,.0f} at {prod.extra['capacity'][-1]:.0f} miners.", + f"- Adaptive attacker (give-up E={E_GIVEUP:,.0f}): duty-cycled to " + f"{duty*100:.0f}% attack-on time, {sat} saturated tick(s) in the whole attack " + f"window; E oscillates {min(adaptive_tr.E[20:104]):,.0f}–{max(adaptive_tr.E[20:104]):,.0f} " + f"(sawtooth around the give-up point — see the doc for the mitigation).", + f"- Miner churn (random walk {min(churn_path):.0f}–{max(churn_path):.0f} miners): " + f"settled noise-free rate within {churn_err:+.1f}% of target.", + "\nPlots: `control_mining.png`, `control_dos.png`, `control_production.png`, " + "`control_adaptive.png`, `control_churn.png`.\n"] + (out / "summary.md").write_text("\n".join(lines)) + print("\n".join(lines)) + print(f"\nWrote {out}/control_mining.png, {out}/control_dos.png, " + f"{out}/control_production.png, {out}/summary.md") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/dosprotect.py b/tools/benchmarks/Equi-X/harness/equix_bench/dosprotect.py new file mode 100644 index 0000000..a722315 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/dosprotect.py @@ -0,0 +1,92 @@ +"""DoS-protection effectiveness evaluation. + +Equi-X is a client puzzle: a requester must *solve* the puzzle (expensive) before +a service will act, while the service only *verifies* (cheap). The protection is +effective on a given system when the attacker's cost to produce one accepted +request vastly exceeds the defender's cost to check it. + +This module answers that question from MEASURED data on the running system: + + attacker_s(E) = measured median time to produce one token at effort E + (the `effort` op: solve over nonces until achieved >= E) + defender_s = measured median time for one `verify` + protection_factor(E) = attacker_s(E) / defender_s + -> how many requests the defender can screen in the time an attacker needs + to craft one accepted request. Large = strong DoS protection. + +Also derived: + verify_throughput = 1 / defender_s (verifications/sec per core) + attacker_token_rate = 1 / attacker_s(E) (accepted tokens/sec per core) +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +from .stats import CellStats + +# Protection is judged "effective" when an attacker spends at least this many +# times the defender's per-verify cost to produce one accepted request. +DEFAULT_THRESHOLD = 10_000.0 + + +def min_verify_seconds(stats: list[CellStats], device: str) -> Optional[float]: + """Defender's best (fastest) verify time on `device`, in seconds.""" + vs = [ + s.median_ns / 1e9 + for s in stats + if s.operation == "verify" and s.ok and s.device_label == device and s.median_ns > 0 + ] + return min(vs) if vs else None + + +@dataclass +class ProtectionRow: + device: str + effort: int + attacker_impl: str + attacker_s: float # fastest measured time to craft a token at this effort + defender_verify_s: float # fastest measured verify time + protection_factor: float # attacker_s / defender_verify_s + verify_per_sec: float # defender screening capacity (per core) + attacker_tokens_per_sec: float # attacker output (per core) + + +def assess(stats: list[CellStats], threshold: float = DEFAULT_THRESHOLD): + """Return (rows, effective, threshold). Uses the attacker's fastest impl and + the defender's fastest verify per device -- the realistic optimized case.""" + rows: list[ProtectionRow] = [] + for dev in sorted({s.device_label for s in stats if s.ok}): + vs = min_verify_seconds(stats, dev) + if vs is None or vs <= 0: + continue + by_effort: dict[int, list[tuple[float, str]]] = {} + for s in stats: + if s.operation == "effort" and s.ok and s.device_label == dev and s.median_ns > 0: + t = int(s.label.get("target_effort", 0)) + by_effort.setdefault(t, []).append((s.median_ns / 1e9, s.impl)) + for t in sorted(by_effort): + attacker_s, attacker_impl = min(by_effort[t], key=lambda x: x[0]) + rows.append(ProtectionRow( + device=dev, effort=t, attacker_impl=attacker_impl, + attacker_s=attacker_s, defender_verify_s=vs, + protection_factor=attacker_s / vs, + verify_per_sec=1.0 / vs, + attacker_tokens_per_sec=1.0 / attacker_s, + )) + # "Effective on this system" means SOME reachable effort clears the bar; the + # smallest such effort tells the operator how hard to set the puzzle here. + effective = any(r.protection_factor >= threshold for r in rows) + return rows, effective, threshold + + +def min_effective_effort(rows, threshold: float = DEFAULT_THRESHOLD): + """Smallest measured effort whose protection factor clears the threshold, + grouped per device. Returns {device: effort or None}.""" + out: dict[str, Optional[int]] = {} + for r in rows: + if r.device not in out: + out[r.device] = None + if r.protection_factor >= threshold and (out[r.device] is None or r.effort < out[r.device]): + out[r.device] = r.effort + return out diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/mining.py b/tools/benchmarks/Equi-X/harness/equix_bench/mining.py new file mode 100644 index 0000000..89e774c --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/mining.py @@ -0,0 +1,338 @@ +"""Mining-rate benchmark: how fast can tokens (effort-qualified solutions) be +minted at a given difficulty, on one core and on the whole machine. + +This is the measured basis for the "control the mint rate by setting difficulty" +use case. It differs from the plain effort sweep in two ways that make it a real +statistical measurement rather than a single anecdote: + + * The runner's effort search is DETERMINISTIC from `nonce_start` (every rep + restarts at the same nonce), so repetitions alone re-time one identical + search. Here each sample uses a DISTINCT `nonce_start`, spaced beyond + `max_attempts` so the search ranges never overlap -- independent draws from + the geometric token-finding process, which we average. + * It measures the whole-machine mint rate directly: `workers` (= core count) + independent searches run at once, and we sum their token rates. + +Outputs, per difficulty E (all POOLED estimators — total tokens over total busy +seconds, failed searches charged to the denominator; a mean of per-sample 1/t +ratios would carry a +6..25% Jensen-style upward bias at the CVs we measure): + tokens_per_sec_1core successes / total busy seconds, sequential [1 core] + tokens_per_sec_machine workers * minted / total busy seconds [N cores] + attempts_mean attempts per token, pooled over every search +""" +from __future__ import annotations + +import statistics +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional + +from .protocol import JobSpec, Result +from .registry import Adapter +from .runner import RunnerError, run + +# Nonce range reserved per sample/worker; must exceed any single token's attempts +# so independent searches never touch the same nonces. +STRIDE = 4_000_000 + + +@dataclass +class MiningPoint: + effort: int + samples: int # independent 1-core samples that succeeded + token_s_mean: float # mean seconds to mint one token [1 core] + token_s_median: float + token_s_stddev: float + attempts_mean: float # POOLED over 1-core samples AND worker mints + achieved_mean: float + tokens_per_sec_1core: float # successes / total busy seconds [1 core] + workers: int + ok_workers: int + tokens_per_sec_machine: float # pooled: workers * minted / total busy s + scaling_efficiency: float # machine / (workers * 1core rate) + failed_searches: int = 0 # searches that hit max_attempts w/o a token + # Message sizes, MEASURED from real minted tokens' wire bytes. The design + # predicts both are constant in E (a solution is always 8 x u16 = 16 bytes; + # the nonce is a protocol constant): min==max demonstrates it. + solution_bytes_min: int = 0 + solution_bytes_max: int = 0 + nonce_bytes_wire: int = 0 + + +@dataclass +class MiningResult: + device: str + impl: str + challenge_base: str + nproc: int + points: list[MiningPoint] = field(default_factory=list) + error: Optional[str] = None + + +def _effort_spec(base: str, effort: int, nonce_start: int, nonce_bytes: int, + reps: int, warmup: int, max_attempts: int) -> JobSpec: + return JobSpec( + operation="effort", + runtime="try-compile", # fastest available path per impl + repetitions=reps, + warmup=warmup, + challenge_base_hex=base, + nonce_bytes=nonce_bytes, + nonce_start=nonce_start, + target_effort=effort, + max_attempts=max_attempts, + ) + + +def _one_token(adapter: Adapter, base: str, effort: int, nonce_start: int, + nonce_bytes: int, max_attempts: int, repo_root: Path, + timeout: float) -> Optional[tuple[bool, float, int, int, tuple[int, int]]]: + """Search one fresh nonce range for a token. Returns (reached_target, + seconds, attempts, achieved, (solution_bytes, nonce_bytes_wire)) — elapsed + time is reported even when the target was NOT reached, so callers can charge + failed searches to the denominator instead of silently conditioning the rate + on success. Sizes are (0, 0) when no token was minted. + None only when the runner itself errored (no timing available).""" + try: + r = run(adapter, _effort_spec(base, effort, nonce_start, nonce_bytes, 1, 0, max_attempts), + repo_root, timeout=timeout) + except RunnerError: + return None + if not r.ok or not r.runs: + return None + run0 = r.runs[0] + if run0.wall_ns <= 0: + return None + # Wire sizes of the actual minted token (hex chars / 2 = bytes). + sol_b = len(r.solutions_hex[0]) // 2 if r.solutions_hex else 0 + nonce_b = len(r.winning_nonce_hex) // 2 if r.winning_nonce_hex else 0 + return (run0.achieved_effort >= effort, run0.wall_ns / 1e9, + run0.attempts, run0.achieved_effort, (sol_b, nonce_b)) + + +def _worker_batch(adapter: Adapter, base: str, effort: int, nonce_start: int, + count: int, nonce_bytes: int, max_attempts: int, + repo_root: Path, timeout: float) -> tuple[int, float, int, int, list[tuple[int, int]]]: + """One concurrent worker STREAMING `count` searches over advancing nonce + ranges (like a real miner). Returns (tokens_minted, total_busy_seconds, + attempts_total, failed_searches, token_sizes). Failed searches contribute + their full solve time to the denominator — a real miner pays for them too.""" + minted, total, attempts, failed = 0, 0.0, 0, 0 + sizes: list[tuple[int, int]] = [] + for k in range(count): + got = _one_token(adapter, base, effort, nonce_start + k * STRIDE, nonce_bytes, + max_attempts, repo_root, timeout) + if got is None: + continue + ok, secs, atts, _ach, size = got + total += secs + attempts += atts + if ok: + minted += 1 + sizes.append(size) + else: + failed += 1 + return (minted, total, attempts, failed, sizes) + + +def measure_point(adapter: Adapter, base: str, effort: int, samples: int, + workers: int, tokens_per_worker: int, nonce_bytes: int, + max_attempts: int, repo_root: Path, timeout: float) -> MiningPoint: + if max_attempts > STRIDE: + raise ValueError( + f"max_attempts ({max_attempts}) must not exceed STRIDE ({STRIDE}); " + "independent nonce ranges would overlap" + ) + + # --- 1-core: `samples` independent, sequential searches (distinct nonces) --- + secs: list[float] = [] # successful token times + attempts_all: list[int] = [] # attempts, pooled over ALL mints (see below) + achieved: list[int] = [] + all_sizes: list[tuple[int, int]] = [] # (solution_bytes, nonce_bytes) per token + fail_s, fails = 0.0, 0 + for s in range(samples): + got = _one_token(adapter, base, effort, s * STRIDE, nonce_bytes, + max_attempts, repo_root, timeout) + if got is None: + continue + ok, t, atts, ach, size = got + if ok: + secs.append(t); attempts_all.append(atts); achieved.append(ach) + all_sizes.append(size) + else: + fail_s += t; fails += 1 + + if not secs: + return MiningPoint(effort, 0, 0, 0, 0, 0, 0, 0, workers, 0, 0, 0, fails) + + # Pooled rate: total tokens over total busy time (failed searches included + # in the denominator — a miner pays for them too). Stable for the + # heavy-tailed token-time distribution, unlike a mean of 1/t ratios. + token_s_mean = statistics.fmean(secs) + onecore_rate = len(secs) / (sum(secs) + fail_s) + + # --- whole machine: `workers` concurrent streams on disjoint nonce ranges --- + def one(w: int) -> tuple[int, float, int, int]: + start = (samples + w * tokens_per_worker) * STRIDE + return _worker_batch(adapter, base, effort, start, tokens_per_worker, + nonce_bytes, max_attempts, repo_root, timeout) + + with ThreadPoolExecutor(max_workers=workers) as pool: + batches = list(pool.map(one, range(workers))) + + # Pooled machine estimator: workers x (all tokens / all busy seconds). + # Summing per-worker m/t ratios (few heavy-tailed samples each) carries a + # +6..25% Jensen-style upward bias at the CVs we measure; pooling first + # reduces the residual bias to ~+1% at 70 mints. + minted = sum(b[0] for b in batches) + busy = sum(b[1] for b in batches) + ok_workers = sum(1 for b in batches if b[0] > 0) + w_fails = sum(b[3] for b in batches) + machine_rate = (workers * minted / busy) if busy > 0 else 0.0 + # Attempts per TOKEN, pooled across every search (1-core + workers, failed + # searches' attempts charged to the numerator): the real cost of a token, + # at ~8x the sample size of the 1-core batch alone. + total_attempts = sum(attempts_all) + sum(b[2] for b in batches) + total_tokens = len(secs) + minted + attempts_mean = total_attempts / total_tokens if total_tokens else 0.0 + # Token wire sizes across EVERY minted token (1-core + workers). + for b in batches: + all_sizes.extend(b[4]) + sol_sizes = [s for (s, _n) in all_sizes if s > 0] + nonce_sizes = [n for (_s, n) in all_sizes if n > 0] + + ideal = onecore_rate * workers + return MiningPoint( + effort=effort, + samples=len(secs), + token_s_mean=token_s_mean, + token_s_median=statistics.median(secs), + token_s_stddev=statistics.pstdev(secs) if len(secs) > 1 else 0.0, + attempts_mean=attempts_mean, + achieved_mean=statistics.fmean(achieved), + tokens_per_sec_1core=onecore_rate, + workers=workers, + ok_workers=ok_workers, + tokens_per_sec_machine=machine_rate, + scaling_efficiency=(machine_rate / ideal) if ideal > 0 else 0.0, + failed_searches=fails + w_fails, + solution_bytes_min=min(sol_sizes) if sol_sizes else 0, + solution_bytes_max=max(sol_sizes) if sol_sizes else 0, + nonce_bytes_wire=max(nonce_sizes) if nonce_sizes else 0, + ) + + +def run_mining(cfg: dict[str, Any], adapters: dict[str, Adapter], repo_root: Path, + device_resolver: Callable[[dict[str, Any]], str], + timeout: float) -> list[MiningResult]: + """Drive the mining benchmark from a `[mining]` config block. + + cfg keys (optional): impls ([] = all given), challenge_base ("abcd"), + efforts ([100,300,1000,3000,10000]), samples (12, 1-core mints per effort), + workers (0 = cpu count), tokens_per_worker (6, streamed per concurrent + worker), nonce_bytes (8), max_attempts (1_500_000).""" + import os + + base = str(cfg.get("challenge_base", "abcd")) + efforts = [int(e) for e in cfg.get("efforts", [100, 300, 1000, 3000, 10000])] + samples = int(cfg.get("samples", 12)) + workers = int(cfg.get("workers", 0)) or (os.cpu_count() or 4) + tokens_per_worker = int(cfg.get("tokens_per_worker", cfg.get("reps", 6))) + nonce_bytes = int(cfg.get("nonce_bytes", 8)) + max_attempts = int(cfg.get("max_attempts", 1_500_000)) + want = list(cfg.get("impls", [])) or list(adapters.keys()) + + impls = [(n, adapters[n]) for n in want if n in adapters] + out: list[MiningResult] = [] + for name, adapter in impls: + # Learn device from a cheap probe: target_effort=1 with a single attempt + # (a probe at efforts[0] would run a full search — minutes on a slow + # interpreted impl at high E, just to read the env). + try: + probe = run(adapter, _effort_spec(base, 1, 0, nonce_bytes, 1, 0, 1), + repo_root, timeout=timeout) + device = device_resolver(probe.env) + except RunnerError: + device = device_resolver({}) + res = MiningResult(device=device, impl=name, challenge_base=base, nproc=workers) + for e in efforts: + res.points.append(measure_point(adapter, base, e, samples, workers, + tokens_per_worker, nonce_bytes, max_attempts, + repo_root, timeout)) + out.append(res) + return out + + +def write_csv(results: list[MiningResult], path: Path) -> None: + import csv + + with open(path, "w", newline="") as f: + w = csv.writer(f) + w.writerow([ + "device", "impl", "challenge_base", "effort", "samples", + "attempts_mean", "achieved_mean", "token_s_mean", "token_s_median", + "token_s_stddev", "tokens_per_sec_1core", "workers", "ok_workers", + "tokens_per_sec_machine", "scaling_efficiency", "failed_searches", + "solution_bytes_min", "solution_bytes_max", "nonce_bytes_wire", + ]) + for r in results: + for p in r.points: + w.writerow([ + r.device, r.impl, r.challenge_base, p.effort, p.samples, + f"{p.attempts_mean:.2f}", f"{p.achieved_mean:.1f}", + f"{p.token_s_mean:.6f}", f"{p.token_s_median:.6f}", + f"{p.token_s_stddev:.6f}", f"{p.tokens_per_sec_1core:.4f}", + p.workers, p.ok_workers, f"{p.tokens_per_sec_machine:.4f}", + f"{p.scaling_efficiency:.4f}", p.failed_searches, + p.solution_bytes_min, p.solution_bytes_max, p.nonce_bytes_wire, + ]) + + +def read_csv(path: Path) -> list[MiningResult]: + """Reconstruct MiningResults from a mining.csv (the inverse of write_csv), so + `combine` can fold in each device's measured mint-rate ladder. Every field is + round-tripped from the CSV; nproc is recovered from the per-point worker count.""" + import csv + + # Columns added over time (failed_searches, message-size fields) may be absent + # in older CSVs; default them so a mixed-vintage `combine` still round-trips. + def _i(row, k): + v = row.get(k, "") + return int(v) if v not in ("", None) else 0 + + def _f(row, k): + v = row.get(k, "") + return float(v) if v not in ("", None) else 0.0 + + by_key: dict[tuple[str, str, str], MiningResult] = {} + with open(path, newline="") as f: + for row in csv.DictReader(f): + key = (row["device"], row["impl"], row["challenge_base"]) + res = by_key.get(key) + if res is None: + res = MiningResult(device=row["device"], impl=row["impl"], + challenge_base=row["challenge_base"], nproc=0) + by_key[key] = res + res.points.append(MiningPoint( + effort=_i(row, "effort"), + samples=_i(row, "samples"), + token_s_mean=_f(row, "token_s_mean"), + token_s_median=_f(row, "token_s_median"), + token_s_stddev=_f(row, "token_s_stddev"), + attempts_mean=_f(row, "attempts_mean"), + achieved_mean=_f(row, "achieved_mean"), + tokens_per_sec_1core=_f(row, "tokens_per_sec_1core"), + workers=_i(row, "workers"), + ok_workers=_i(row, "ok_workers"), + tokens_per_sec_machine=_f(row, "tokens_per_sec_machine"), + scaling_efficiency=_f(row, "scaling_efficiency"), + failed_searches=_i(row, "failed_searches"), + solution_bytes_min=_i(row, "solution_bytes_min"), + solution_bytes_max=_i(row, "solution_bytes_max"), + nonce_bytes_wire=_i(row, "nonce_bytes_wire"), + )) + for res in by_key.values(): + res.points.sort(key=lambda p: p.effort) + res.nproc = max((p.workers for p in res.points), default=0) + return list(by_key.values()) diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/protocol.py b/tools/benchmarks/Equi-X/harness/equix_bench/protocol.py new file mode 100644 index 0000000..1eb140a --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/protocol.py @@ -0,0 +1,103 @@ +"""Wire protocol: job-spec (harness -> runner) and result (runner -> harness). + +The schema is documented in adapters/README.md. Runners are language-agnostic; +this module is the single source of truth for how the Python harness speaks it. +""" +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass, field +from typing import Any, Optional + +SCHEMA_VERSION = 1 + + +@dataclass +class JobSpec: + """One runner invocation. Only the fields relevant to the operation are set.""" + + operation: str # solve | verify | effort | hashx_compile + runtime: str # interpret | try-compile | must-compile + repetitions: int = 10 + warmup: int = 3 + challenge_hex: Optional[str] = None + # When set, each rep derives a fresh challenge by SHA-256-chaining this seed + # (solve/verify only); challenge generation is excluded from timed regions. + challenge_seed_hex: Optional[str] = None + challenge_base_hex: Optional[str] = None + solution_hex: Optional[str] = None + nonce_bytes: Optional[int] = None + nonce_start: Optional[int] = None + target_effort: Optional[int] = None + max_attempts: Optional[int] = None + seed: Optional[int] = None + + def to_json(self) -> str: + d: dict[str, Any] = {"schema_version": SCHEMA_VERSION} + d.update({k: v for k, v in asdict(self).items() if v is not None}) + return json.dumps(d) + + +@dataclass +class Run: + index: int + wall_ns: int + solutions: int + compile_ns: int + attempts: int + achieved_effort: int + verify_result: Optional[str] + + +@dataclass +class Result: + ok: bool + impl_name: str + impl_version: str + impl_commit: str + operation: str + runtime_requested: str + runtime_effective: Optional[str] + env: dict[str, Any] + runs: list[Run] + solutions_hex: Optional[list[str]] + peak_rss_kb: int + error: Optional[str] + raw: dict[str, Any] = field(default_factory=dict) + # effort op only: wire bytes (hex) of the winning token's nonce. + winning_nonce_hex: Optional[str] = None + + @staticmethod + def from_dict(d: dict[str, Any]) -> "Result": + sv = d.get("schema_version") + if sv != SCHEMA_VERSION: + raise ValueError(f"unsupported schema_version {sv!r} (expected {SCHEMA_VERSION})") + impl = d.get("impl", {}) or {} + runs = [ + Run( + index=r.get("index", i), + wall_ns=int(r.get("wall_ns", 0)), + solutions=int(r.get("solutions", 0)), + compile_ns=int(r.get("compile_ns", 0)), + attempts=int(r.get("attempts", 0)), + achieved_effort=int(r.get("achieved_effort", 0)), + verify_result=r.get("verify_result"), + ) + for i, r in enumerate(d.get("runs", []) or []) + ] + return Result( + ok=bool(d.get("ok", False)), + impl_name=impl.get("name", "?"), + impl_version=impl.get("version", "?"), + impl_commit=impl.get("commit", "?"), + operation=d.get("operation", "?"), + runtime_requested=d.get("runtime_requested", "?"), + runtime_effective=d.get("runtime_effective"), + env=d.get("env", {}) or {}, + runs=runs, + solutions_hex=d.get("solutions_hex"), + peak_rss_kb=int(d.get("peak_rss_kb", 0)), + error=d.get("error"), + raw=d, + winning_nonce_hex=d.get("winning_nonce_hex"), + ) diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/registry.py b/tools/benchmarks/Equi-X/harness/equix_bench/registry.py new file mode 100644 index 0000000..6c89a25 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/registry.py @@ -0,0 +1,72 @@ +"""Adapter registry: discover implementations from TOML manifests. + +Each implementation ("adapter") ships a manifest declaring how to invoke its +runner and what it supports. New implementations plug in by adding a manifest -- +no harness code changes. See adapters/README.md. +""" +from __future__ import annotations + +import os +import tomllib +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class Adapter: + name: str + exec: list[str] # argv; paths resolved against repo root + protocol_version: int + capabilities: list[str] + runtimes: list[str] + env: dict[str, str] + + def resolve(self, repo_root: Path) -> list[str]: + """Resolve the executable path (first argv element) against repo root.""" + argv = list(self.exec) + p = Path(argv[0]) + if not p.is_absolute(): + p = (repo_root / p).resolve() + argv[0] = str(p) + return argv + + def available(self, repo_root: Path) -> bool: + argv = self.resolve(repo_root) + first = argv[0] + # A bare interpreter name (e.g. "python3") is looked up on PATH. + if "/" not in self.exec[0]: + return True + return os.path.exists(first) and os.access(first, os.X_OK) + + +def load_manifest(path: Path) -> Adapter: + with open(path, "rb") as f: + d = tomllib.load(f) + exec_field = d["exec"] + if isinstance(exec_field, str): + exec_field = [exec_field] + return Adapter( + name=d["name"], + exec=list(exec_field), + protocol_version=int(d.get("protocol_version", 1)), + capabilities=list(d.get("capabilities", [])), + runtimes=list(d.get("runtimes", [])), + env=dict(d.get("env", {})), + ) + + +def load_manifests(manifest_dirs) -> dict[str, Adapter]: + """Load adapter manifests from one directory or several (later dirs win on + name collision). This lets generated compiler-flag variants in a second dir + coexist with the built-in adapters.""" + if isinstance(manifest_dirs, (str, Path)): + manifest_dirs = [manifest_dirs] + adapters: dict[str, Adapter] = {} + for d in manifest_dirs: + d = Path(d) + if not d.is_dir(): + continue + for p in sorted(d.glob("*.manifest.toml")): + a = load_manifest(p) + adapters[a.name] = a + return adapters diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/report.py b/tools/benchmarks/Equi-X/harness/equix_bench/report.py new file mode 100644 index 0000000..efcf262 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/report.py @@ -0,0 +1,767 @@ +"""Reporting: CSV, raw JSON, comparison plots, and a markdown report. + +Design rules (per requirements): + * EVERY plot compares all implementations on the same axes. `_require_multi_impl` + fails loudly if a figure would show fewer than two implementations. + * Plots reflect the executing device (CPU). With ONE device, the device is shown + in the title. With MULTIPLE devices (e.g. after `combine`), each plot becomes a + small-multiples grid (one facet per device, C-vs-Rust within each), plus + dedicated cross-device charts (x=device, series=impl) for headline metrics. +""" +from __future__ import annotations + +import csv +import json +from collections import defaultdict +from pathlib import Path +from typing import Any, Callable, Iterable, Optional + +import matplotlib + +matplotlib.use("Agg") # headless +import matplotlib.pyplot as plt # noqa: E402 +import numpy as np # noqa: E402 + +from .crosscheck import Check +from .dosprotect import assess as dos_assess +from .dosprotect import min_verify_seconds +from .stats import CellStats + +_PALETTE = ["#4C72B0", "#DD8452", "#55A868", "#C44E52", "#8172B3", "#937860"] + + +def _impl_color(impl: str, all_impls: list[str]) -> str: + idx = all_impls.index(impl) if impl in all_impls else len(all_impls) + return _PALETTE[idx % len(_PALETTE)] + + +def _require_multi_impl(impls_present: Iterable[str], plot: str) -> list[str]: + impls = sorted(set(impls_present)) + if len(impls) < 2: + raise RuntimeError( + f"plot '{plot}' requires >=2 implementations to compare, " + f"but only found: {impls}. Build/enable both C and Rust runners." + ) + return impls + + +# --------------------------------------------------------------------- outputs + + +def write_csv(stats: list[CellStats], path: Path) -> None: + cols = [ + "impl", "device_label", "device_type", "device_name", "device_arch", + "operation", "runtime_requested", "runtime_effective", "label", + "reps", "ok", "min_ns", "median_ns", "mean_ns", "stddev_ns", "p95_ns", + "solutions_mean", "compile_median_ns", "attempts_mean", + "achieved_effort_mean", "solves_per_sec", "hashes_per_sec", + "peak_rss_kb", "verify_result", "error", + ] + with open(path, "w", newline="") as f: + w = csv.writer(f) + w.writerow(cols) + for s in stats: + w.writerow([ + s.impl, s.device_label, s.device_type, s.device_name, s.device_arch, + s.operation, s.runtime_requested, s.runtime_effective, + json.dumps(s.label), s.reps, s.ok, f"{s.min_ns:.1f}", + f"{s.median_ns:.1f}", f"{s.mean_ns:.1f}", f"{s.stddev_ns:.1f}", + f"{s.p95_ns:.1f}", f"{s.solutions_mean:.3f}", + f"{s.compile_median_ns:.1f}", f"{s.attempts_mean:.2f}", + f"{s.achieved_effort_mean:.1f}", f"{s.solves_per_sec:.2f}", + f"{s.hashes_per_sec:.0f}", s.peak_rss_kb, s.verify_result, s.error, + ]) + + +def write_raw(raw: list[dict[str, Any]], path: Path) -> None: + with open(path, "w") as f: + json.dump(raw, f, indent=2) + + +# ----------------------------------------------------------------- plot helpers + + +def _grouped_bar(ax, categories, series, all_impls, errors=None) -> None: + names = sorted(series.keys()) + n = len(names) + width = 0.8 / max(n, 1) + x = np.arange(len(categories)) + for i, name in enumerate(names): + off = (i - (n - 1) / 2) * width + err = errors.get(name) if errors else None + ax.bar(x + off, series[name], width, label=name, yerr=err, capsize=3, + color=_impl_color(name, all_impls), edgecolor="black", linewidth=0.4) + ax.set_xticks(x) + ax.set_xticklabels(categories, rotation=0) + ax.legend(title="implementation", fontsize=8) + ax.grid(axis="y", linestyle=":", alpha=0.5) + + +def _agg(stats, cat_key, val): + """Aggregate (mean) `val` grouped by (category, impl) over a pre-filtered set.""" + buckets: dict[tuple[str, str], list[float]] = defaultdict(list) + impls_present: set[str] = set() + cats: list[str] = [] + for s in stats: + c = cat_key(s) + buckets[(s.impl, c)].append(val(s)) + impls_present.add(s.impl) + if c not in cats: + cats.append(c) + cats = sorted(cats) + impls = sorted(impls_present) + series = { + impl: [float(np.mean(buckets[(impl, c)])) if buckets.get((impl, c)) else 0.0 + for c in cats] + for impl in impls + } + return cats, series, impls + + +def _save(fig, path: Path) -> str: + # Only run tight_layout when no layout engine is active (faceted figures use + # constrained layout, which is incompatible with tight_layout and can produce + # NaN axes geometry if both run). + try: + if fig.get_layout_engine() is None: + fig.tight_layout() + except Exception: # noqa: BLE001 - layout is best-effort, never fatal + pass + fig.savefig(path, dpi=110) + plt.close(fig) + return path.name + + +# ------------------------------------------------------------------ panels (per device) + + +def _panel_time(ax, stats, all_impls): + cats, series, _ = _agg(stats, lambda s: s.runtime_requested, lambda s: s.median_ns) + _, errs, _ = _agg(stats, lambda s: s.runtime_requested, + lambda s: max(0.0, s.p95_ns - s.median_ns)) + _grouped_bar(ax, cats, series, all_impls, errs) + ax.set_xlabel("HashX runtime") + + +def _panel_throughput(ax, stats, all_impls): + cats, series, _ = _agg(stats, lambda s: s.runtime_requested, lambda s: s.solves_per_sec) + _grouped_bar(ax, cats, series, all_impls) + ax.set_xlabel("HashX runtime") + + +def _panel_rss(ax, stats, all_impls): + cats, series, _ = _agg(stats, lambda s: s.runtime_requested, lambda s: float(s.peak_rss_kb)) + _grouped_bar(ax, cats, series, all_impls) + ax.set_xlabel("HashX runtime") + + +def _panel_compile(ax, stats, all_impls): + cats, series, _ = _agg(stats, lambda s: s.runtime_requested, lambda s: s.compile_median_ns) + _grouped_bar(ax, cats, series, all_impls) + ax.set_xlabel("HashX runtime") + + +def _panel_speedup(ax, stats, all_impls): + med: dict[tuple[str, str], list[float]] = defaultdict(list) + for s in stats: + med[(s.impl, s.runtime_requested)].append(s.median_ns) + impls = sorted({s.impl for s in stats}) + speedups = [] + for impl in impls: + interp = med.get((impl, "interpret")) + comp = med.get((impl, "try-compile")) or med.get((impl, "must-compile")) + speedups.append(float(np.mean(interp)) / float(np.mean(comp)) if interp and comp else 0.0) + x = np.arange(len(impls)) + ax.bar(x, speedups, 0.5, color=[_impl_color(i, all_impls) for i in impls], + edgecolor="black", linewidth=0.4) + for xi, v in zip(x, speedups): + ax.text(xi, v, f"{v:.1f}x", ha="center", va="bottom", fontsize=8) + ax.set_xticks(x) + ax.set_xticklabels(impls) + ax.set_xlabel("implementation") + + +def _panel_distribution(ax, stats, all_impls): + runtimes: list[str] = [] + data: dict[tuple[str, str], list[float]] = defaultdict(list) + impls = sorted({s.impl for s in stats}) + for s in stats: + if s.walls: + if s.runtime_requested not in runtimes: + runtimes.append(s.runtime_requested) + data[(s.impl, s.runtime_requested)].extend([w / 1e6 for w in s.walls]) + runtimes = sorted(runtimes) + positions, box_data, colors = [], [], [] + width = 0.8 / max(len(impls), 1) + for ri, rt in enumerate(runtimes): + for ii, impl in enumerate(impls): + vals = data.get((impl, rt)) + if not vals: + continue + positions.append(ri + (ii - (len(impls) - 1) / 2) * width) + box_data.append(vals) + colors.append(_impl_color(impl, all_impls)) + if box_data: + bp = ax.boxplot(box_data, positions=positions, widths=width * 0.9, + patch_artist=True, showfliers=False) + for patch, c in zip(bp["boxes"], colors): + patch.set_facecolor(c) + patch.set_alpha(0.8) + ax.set_xticks(range(len(runtimes))) + ax.set_xticklabels(runtimes) + ax.set_xlabel("HashX runtime") + from matplotlib.patches import Patch + ax.legend(handles=[Patch(facecolor=_impl_color(i, all_impls), label=i) for i in impls], + title="implementation", fontsize=8) + ax.grid(axis="y", linestyle=":", alpha=0.5) + + +def _panel_effort(ax, stats, all_impls, which): + pts: dict[str, dict[int, tuple[float, float]]] = defaultdict(dict) + for s in stats: + t = int(s.label.get("target_effort", 0)) + pts[s.impl][t] = (s.attempts_mean, s.median_ns / 1e9) + for impl in sorted(pts): + xs = sorted(pts[impl].keys()) + ys = [pts[impl][x][0 if which == "attempts" else 1] for x in xs] + ax.plot(xs, ys, "o-", label=impl, color=_impl_color(impl, all_impls)) + ax.set_xscale("log") + if which == "attempts": + ax.set_yscale("log") + ax.set_ylabel("mean attempts") + else: + ax.set_ylabel("median time (s)") + ax.set_xlabel("target effort") + ax.legend(fontsize=8) + ax.grid(True, which="both", linestyle=":", alpha=0.5) + + +# ------------------------------------------------------------------ facet driver + + +def _facet(stats, panel_fn, title, ylabel, path, all_impls, keep) -> str: + subset = [s for s in stats if keep(s) and s.ok] + devices = sorted({s.device_label for s in subset}) + _require_multi_impl({s.impl for s in subset}, path.name) + + if len(devices) <= 1: + fig, ax = plt.subplots(figsize=(8, 4.5)) + panel_fn(ax, subset, all_impls) + dev = devices[0] if devices else "host" + ax.set_title(f"{title}\n[{dev}]") + ax.set_ylabel(ylabel) + return _save(fig, path) + + n = len(devices) + fig, axes = plt.subplots(1, n, figsize=(min(7 * n, 22), 4.8), sharey=True, + squeeze=False, layout="constrained") + for ax, dev in zip(axes[0], devices): + panel_fn(ax, [s for s in subset if s.device_label == dev], all_impls) + ax.set_title(dev) + axes[0][0].set_ylabel(ylabel) + fig.suptitle(title, fontweight="bold") + return _save(fig, path) + + +def _plot_dos(stats, path, all_impls) -> Optional[str]: + """DoS-protection asymmetry: attacker cost to craft a token / defender verify + cost, vs effort. One line per implementation (attacker), faceted per device.""" + panels = [] + for dev in sorted({s.device_label for s in stats if s.ok and s.operation == "effort"}): + vs = min_verify_seconds(stats, dev) + if not vs: + continue + per_impl: dict[str, dict[int, float]] = defaultdict(dict) + for s in stats: + if s.operation == "effort" and s.ok and s.device_label == dev and s.median_ns > 0: + e = int(s.label.get("target_effort", 0)) + per_impl[s.impl][e] = (s.median_ns / 1e9) / vs + if per_impl: + panels.append((dev, per_impl)) + if not panels: + return None + _require_multi_impl({im for _, pi in panels for im in pi}, path.name) + + from .dosprotect import DEFAULT_THRESHOLD + n = len(panels) + fig, axes = plt.subplots(1, n, figsize=(min(7 * n, 22), 4.8), squeeze=False, + sharey=True, layout="constrained" if n > 1 else None) + for ax, (dev, per_impl) in zip(axes[0], panels): + for impl in sorted(per_impl): + xs = sorted(per_impl[impl]) + ys = [per_impl[impl][x] for x in xs] + ax.plot(xs, ys, "o-", label=impl, color=_impl_color(impl, all_impls)) + ax.axhline(DEFAULT_THRESHOLD, ls="--", color="red", alpha=0.7, + label=f"effective ≥ {DEFAULT_THRESHOLD:.0f}×") + ax.set_xscale("log") + ax.set_yscale("log") + ax.set_xlabel("target effort (attacker difficulty)") + ax.set_title(dev) + ax.grid(True, which="both", linestyle=":", alpha=0.5) + ax.legend(fontsize=8) + axes[0][0].set_ylabel("protection factor (attacker time / verify time)") + fig.suptitle("DoS-protection asymmetry: cost to attack vs cost to verify", fontweight="bold") + return _save(fig, path) + + +def _cross_device_bar(stats, keep, val, title, ylabel, path, all_impls) -> Optional[str]: + subset = [s for s in stats if keep(s) and s.ok] + devices = sorted({s.device_label for s in subset}) + if len(devices) < 2: + return None # only meaningful across multiple devices + cats, series, impls = _agg(subset, lambda s: s.device_label, val) + _require_multi_impl(impls, path.name) + fig, ax = plt.subplots(figsize=(max(6, 2 + 2 * len(devices)), 4.5)) + _grouped_bar(ax, cats, series, all_impls) + ax.set_xlabel("device / CPU") + ax.set_ylabel(ylabel) + ax.set_title(title) + return _save(fig, path) + + +def _devices_of(results) -> list[str]: + """Devices present in a concurrency/mining result set, in stable order. + Blank device labels (e.g. a single-host run before `combine`) collapse to one + unnamed facet so the plot still renders.""" + return sorted({(getattr(r, "device", "") or "") for r in results}) + + +def _facet_axes(devices, figsize_per, **kw): + """One subplot column per device (constrained layout when faceted, so the + shared suptitle never overlaps the panels).""" + n = max(len(devices), 1) + w_per, h = figsize_per + fig, axes = plt.subplots(1, n, figsize=(min(w_per * n, 22), h), squeeze=False, + layout="constrained" if n > 1 else None, **kw) + return fig, axes[0] + + +def _plot_concurrency(results, operation: str, path: Path, all_impls) -> Optional[str]: + """Aggregate throughput vs worker count for one operation, one line per impl, + with the ideal-linear-scaling reference. Shows where the machine saturates. + With >1 device (after `combine`), each device gets its own panel.""" + rows = [r for r in results if r.operation == operation and r.levels] + if not rows: + return None + devices = _devices_of(rows) + multi = len(devices) > 1 + fig, axes = _facet_axes(devices, (7, 4.5), sharey=True) + for ax, dev in zip(axes, devices): + for r in sorted([x for x in rows if (getattr(x, "device", "") or "") == dev], + key=lambda r: r.impl): + pts = [(lv.workers, lv.aggregate_ops_per_sec) for lv in r.levels + if lv.aggregate_ops_per_sec > 0] + if not pts: + continue + xs, ys = zip(*pts) + color = _impl_color(r.impl, all_impls) + ax.plot(xs, ys, marker="o", color=color, label=f"{r.impl} (measured)") + # ideal linear scaling from this impl's single-worker baseline + if r.baseline_ops_per_sec > 0: + ax.plot(xs, [r.baseline_ops_per_sec * x for x in xs], linestyle=":", + color=color, alpha=0.5, label=f"{r.impl} (ideal linear)") + ax.set_xlabel("concurrent workers") + ax.set_title(dev if multi else f"[{dev}]" if dev else "") + ax.grid(True, alpha=0.3) + ax.legend(fontsize=8) + axes[0].set_ylabel(f"aggregate {operation}s / second") + fig.suptitle(f"Sustained {operation} throughput under concurrency (measured)", + fontweight="bold") + return _save(fig, path) + + +def _plot_mining(results, path: Path, all_impls) -> Optional[str]: + """Token mint rate vs difficulty (effort), log-log. Per impl: 1-core and + whole-machine lines. A straight line here means rate ∝ 1/effort. With >1 + device (after `combine`), each device gets its own panel.""" + rows = [r for r in results if r.points] + if not rows: + return None + devices = _devices_of(rows) + multi = len(devices) > 1 + fig, axes = _facet_axes(devices, (7, 4.5), sharey=True) + for ax, dev in zip(axes, devices): + for r in sorted([x for x in rows if (getattr(x, "device", "") or "") == dev], + key=lambda r: r.impl): + pts = [(p.effort, p.tokens_per_sec_1core, p.tokens_per_sec_machine) + for p in r.points if p.samples > 0 and p.tokens_per_sec_1core > 0] + if not pts: + continue + xs = [p[0] for p in pts] + color = _impl_color(r.impl, all_impls) + ax.plot(xs, [p[1] for p in pts], marker="o", color=color, + label=f"{r.impl} — 1 core") + ax.plot(xs, [p[2] for p in pts], marker="s", linestyle="--", color=color, + label=f"{r.impl} — {r.nproc} cores") + ax.set_xscale("log"); ax.set_yscale("log") + ax.set_xlabel("difficulty (effort target)") + ax.set_title(dev if multi else "") + ax.grid(True, which="both", alpha=0.3) + ax.legend(fontsize=8) + axes[0].set_ylabel("tokens minted / second") + fig.suptitle("Mining rate vs difficulty (measured)", fontweight="bold") + return _save(fig, path) + + +# ---------------------------------------------------------------- report driver + + +def generate(stats, checks, raw, out_dir: Path, meta, concurrency=None, mining=None) -> None: + out_dir.mkdir(parents=True, exist_ok=True) + plots_dir = out_dir / "plots" + plots_dir.mkdir(exist_ok=True) + (out_dir / "raw").mkdir(exist_ok=True) + + write_csv(stats, out_dir / "results.csv") + write_raw(raw, out_dir / "raw" / "results.json") + + impl_meta: dict[str, dict[str, Any]] = {} + for r in raw: + im = r.get("impl", {}) or {} + name = im.get("name") + if name and name not in impl_meta: + impl_meta[name] = {"version": im.get("version"), "commit": im.get("commit"), + "compiler": (r.get("env", {}) or {}).get("compiler")} + meta = {**meta, "impls": impl_meta} + + all_impls = sorted({s.impl for s in stats}) + devices = sorted({s.device_label for s in stats if s.ok}) + plots: list[str] = [] + + def try_plot(fn, *a, **k): + try: + r = fn(*a, **k) + if isinstance(r, list): + plots.extend([x for x in r if x]) + elif r: + plots.append(r) + except RuntimeError as e: + plots.append(f"__error__:{e}") + + has = lambda op: any(s.operation == op for s in stats) + + if has("solve"): + try_plot(_facet, stats, _panel_time, "Solve time by runtime: C vs Rust", + "median solve time (ns)", plots_dir / "solve_time_by_runtime.png", + all_impls, lambda s: s.operation == "solve") + try_plot(_facet, stats, _panel_throughput, "Solve throughput: C vs Rust", + "solves / second", plots_dir / "throughput.png", all_impls, + lambda s: s.operation == "solve") + try_plot(_facet, stats, _panel_speedup, "JIT speedup over interpreter: C vs Rust", + "interpret / compiled (x faster)", plots_dir / "jit_speedup.png", + all_impls, lambda s: s.operation == "solve") + try_plot(_facet, stats, _panel_rss, "Peak memory during solve: C vs Rust", + "peak RSS (KB)", plots_dir / "peak_rss.png", all_impls, + lambda s: s.operation == "solve") + try_plot(_facet, stats, _panel_distribution, "Solve-time distribution: C vs Rust", + "solve time (ms)", plots_dir / "solve_distribution.png", all_impls, + lambda s: s.operation == "solve") + if has("verify"): + try_plot(_facet, stats, _panel_time, "Verify time by runtime: C vs Rust", + "median verify time (ns)", plots_dir / "verify_time.png", all_impls, + lambda s: s.operation == "verify") + if has("hashx_compile"): + try_plot(_facet, stats, _panel_compile, "HashX compile overhead: C vs Rust", + "median program-gen + compile (ns)", plots_dir / "compile_overhead.png", + all_impls, lambda s: s.operation == "hashx_compile") + if has("effort"): + try_plot(_facet, stats, lambda ax, s, im: _panel_effort(ax, s, im, "attempts"), + "Effort cost (attempts): C vs Rust", "mean attempts", + plots_dir / "effort_attempts.png", all_impls, lambda s: s.operation == "effort") + try_plot(_facet, stats, lambda ax, s, im: _panel_effort(ax, s, im, "time"), + "Effort cost (time): C vs Rust", "median time (s)", + plots_dir / "effort_time.png", all_impls, lambda s: s.operation == "effort") + + # Cross-device (CPU vs CPU) headline charts -- emitted only when >1 device. + if len(devices) >= 2: + try_plot(_cross_device_bar, stats, + lambda s: s.operation == "solve" and s.runtime_requested in ("try-compile", "must-compile"), + lambda s: s.solves_per_sec, "Solve throughput across CPUs: C vs Rust", + "solves / second", plots_dir / "xdev_throughput.png", all_impls) + try_plot(_cross_device_bar, stats, + lambda s: s.operation == "solve" and s.runtime_requested in ("try-compile", "must-compile"), + lambda s: s.median_ns, "Solve time across CPUs: C vs Rust", + "median solve time (ns)", plots_dir / "xdev_solve_time.png", all_impls) + try_plot(_cross_device_bar, stats, lambda s: s.operation == "solve", + lambda s: float(s.peak_rss_kb), "Peak RSS across CPUs: C vs Rust", + "peak RSS (KB)", plots_dir / "xdev_peak_rss.png", all_impls) + try_plot(_cross_device_bar, stats, lambda s: s.operation == "verify", + lambda s: s.median_ns, "Verify time across CPUs: C vs Rust", + "median verify time (ns)", plots_dir / "xdev_verify_time.png", all_impls) + + # DoS-protection effectiveness (needs both effort and verify measurements). + dos = None + if any(s.operation == "effort" for s in stats) and any(s.operation == "verify" for s in stats): + try_plot(_plot_dos, stats, plots_dir / "dos_protection.png", all_impls) + dos = dos_assess(stats) + + # Concurrency / saturation curves (one figure per operation; C vs Rust lines). + if concurrency: + for op in sorted({r.operation for r in concurrency}): + try_plot(_plot_concurrency, concurrency, op, + plots_dir / f"concurrency_{op}.png", all_impls) + + # Mining rate vs difficulty (tokens/s vs effort, 1-core and whole-machine). + if mining: + try_plot(_plot_mining, mining, plots_dir / "mining_rate.png", all_impls) + + _write_markdown(stats, checks, plots, out_dir, meta, devices, dos, concurrency, mining) + + +def _fmt_ns(ns: float) -> str: + if ns >= 1e9: + return f"{ns/1e9:.3f} s" + if ns >= 1e6: + return f"{ns/1e6:.3f} ms" + if ns >= 1e3: + return f"{ns/1e3:.3f} µs" + return f"{ns:.0f} ns" + + +def _concurrency_section(lines: list, concurrency, dos=None) -> None: + """Render the measured concurrency/saturation results as a self-contained + section. Deliberately additive: it reports the machine's real sustained + capacity next to the per-core estimate, without altering the DoS numbers.""" + ok = [r for r in concurrency if r.levels and r.peak_ops_per_sec > 0] + lines.append("## Sustained throughput under concurrency (measured)\n") + lines.append( + "The DoS section above reports **per-core** capacity as 1/latency from a " + "single serial op. This section instead **measures** aggregate throughput " + "with *N* worker processes running at once (N stepping up to the core " + "count), so it captures real memory-bandwidth contention. It is additive — " + "the per-core figures above are unchanged.\n" + ) + if not ok: + lines.append("_No usable concurrency measurements._\n") + return + + devices = sorted({(r.device or "") for r in ok}) + multi = len(devices) > 1 + dev_hdr = "device | " if multi else "" + dev_sep = "---|" if multi else "" + nproc = max(r.nproc for r in ok) + lines.append( + f"Measured on up to **{nproc}** concurrent workers" + + (f" across **{len(devices)}** devices" if multi else "") + ". " + "*Peak* is the best aggregate ops/s observed; *knee* is the worker count " + "where it peaks (adding workers past it stops helping). *Naïve N×* is the " + "per-core figure multiplied by the core count — what a linear extrapolation " + "would (over)predict; the *efficiency* column is measured peak ÷ naïve N×.\n" + ) + lines.append(f"| {dev_hdr}impl | operation | 1 worker (per-core) | knee | measured peak | naïve N× | scaling efficiency |") + lines.append(f"|{dev_sep}---|---|---|---|---|---|---|") + for r in sorted(ok, key=lambda r: ((r.device or ""), r.operation, r.impl)): + naive = r.baseline_ops_per_sec * r.nproc + eff = (r.peak_ops_per_sec / naive) if naive > 0 else 0.0 + dev_cell = f"{r.device} | " if multi else "" + lines.append( + f"| {dev_cell}{r.impl} | {r.operation} | {r.baseline_ops_per_sec:,.0f} ops/s | " + f"{r.knee_workers} workers | **{r.peak_ops_per_sec:,.0f} ops/s** | " + f"{naive:,.0f} ops/s | {eff*100:.0f}% |" + ) + lines.append("") + + # Per-level detail, grouped by operation then (device, impl). + for op in sorted({r.operation for r in ok}): + lines.append(f"### {op}: throughput vs. concurrency\n") + lines.append(f"| {dev_hdr}impl | workers | ok | aggregate ops/s | per-worker ops/s | scaling eff. | peak RSS |") + lines.append(f"|{dev_sep}---|---|---|---|---|---|---|") + for r in sorted([x for x in ok if x.operation == op], key=lambda r: ((r.device or ""), r.impl)): + dev_cell = f"{r.device} | " if multi else "" + for lv in r.levels: + rss = f"{lv.total_peak_rss_kb/1024:.0f} MB" if lv.total_peak_rss_kb else "n/a" + lines.append( + f"| {dev_cell}{r.impl} | {lv.workers} | {lv.ok_workers} | " + f"{lv.aggregate_ops_per_sec:,.0f} | {lv.per_worker_ops_per_sec:,.0f} | " + f"{lv.scaling_efficiency*100:.0f}% | {rss} |" + ) + lines.append("") + + +def _mining_section(lines: list, mining) -> None: + """Measured mint rate vs difficulty: 1-core and whole-machine tokens/s.""" + ok = [r for r in mining if any(p.samples > 0 for p in r.points)] + lines.append("## Mining rate vs difficulty (measured)\n") + lines.append( + "How many effort-qualified tokens can be minted per second at a given " + "difficulty. The **whole-machine** rate is the reliable figure: it averages " + "one streaming search per core over independent nonce ranges. Per-core is that " + "rate divided by the core count (token-find time is heavy-tailed, so the " + "separately-sampled single-core mean is noisier and can even exceed the machine " + "rate ÷ cores at low sample counts — prefer the derived per-core). Mint rate " + "falls ~1/effort, so difficulty sets the rate directly.\n" + ) + if not ok: + lines.append("_No usable mining measurements._\n") + return + for r in ok: + nproc = r.nproc + lines.append(f"**`{r.impl}`** on `{r.device}` (base `{r.challenge_base}`), " + f"whole-machine = {nproc} cores:\n") + lines.append("| difficulty (effort) | mean attempts/token | tokens/s [%d cores] | " + "tokens/s [1 core, ÷%d] |" % (nproc, nproc)) + lines.append("|---|---|---|---|") + for p in sorted(r.points, key=lambda p: p.effort): + if p.samples == 0: + lines.append(f"| {p.effort} | _no sample reached target_ | — | — |") + continue + per_core = p.tokens_per_sec_machine / nproc if nproc else 0.0 + lines.append( + f"| {p.effort} | {p.attempts_mean:,.0f} | " + f"**{p.tokens_per_sec_machine:,.2f}** | {per_core:,.3f} |" + ) + lines.append("") + # Message sizes vs difficulty, measured from real minted tokens. The + # interesting result is constancy: cost scales with E, bytes do not. + sized = [p for p in r.points if p.solution_bytes_max > 0] + if sized: + lines.append("**Message sizes (measured from every minted token):** ") + rows_sz = [] + for p in sorted(sized, key=lambda p: p.effort): + span = (f"{p.solution_bytes_min}" if p.solution_bytes_min == p.solution_bytes_max + else f"{p.solution_bytes_min}-{p.solution_bytes_max}") + rows_sz.append(f"E={p.effort}: solution {span} B + nonce {p.nonce_bytes_wire} B") + const = all(p.solution_bytes_min == sized[0].solution_bytes_max == p.solution_bytes_max + for p in sized) + lines.append("; ".join(rows_sz) + ".") + if const: + lines.append( + f"Token size is **constant in difficulty**: every token at every " + f"measured E is exactly {sized[0].solution_bytes_max} B solution + " + f"{sized[0].nonce_bytes_wire} B nonce — raising E raises solve cost, " + f"never message size.\n" + ) + else: + lines.append("Token size VARIED across efforts — investigate.\n") + # Headline: 1/effort check across the measured span, on the machine rate. + # (sorted by effort so lo/hi are the true endpoints regardless of config order) + good = sorted([p for p in r.points if p.samples > 0 and p.tokens_per_sec_machine > 0], + key=lambda p: p.effort) + if len(good) >= 2: + lo, hi = good[0], good[-1] + fold_e = hi.effort / lo.effort if lo.effort else 0 + fold_r = (lo.tokens_per_sec_machine / hi.tokens_per_sec_machine) if hi.tokens_per_sec_machine else 0 + lines.append( + f"> Over a {fold_e:.0f}× rise in difficulty ({lo.effort}→{hi.effort}), " + f"the machine mint rate fell {fold_r:.0f}× — ~1/effort, " + f"so halving the target roughly doubles the mint rate.\n" + ) + + +def _write_markdown(stats, checks, plots, out_dir: Path, meta, devices, dos=None, concurrency=None, mining=None) -> None: + lines: list[str] = [] + lines.append("# Equi-X Benchmark Report\n") + lines.append(f"- Generated: {meta.get('timestamp', 'n/a')}") + lines.append(f"- Config: `{meta.get('config', 'n/a')}`") + lines.append(f"- Devices (CPUs): {', '.join(devices) if devices else 'n/a'}") + for dl in devices: + ex = next((s for s in stats if s.device_label == dl), None) + if ex: + lines.append(f" - `{dl}`: {ex.device_name} ({ex.device_arch}, {ex.device_type})") + for impl, info in (meta.get("impls") or {}).items(): + lines.append( + f"- `{impl}`: version {info.get('version')}, commit {info.get('commit')}, " + f"built with {info.get('compiler')}" + ) + lines.append("") + + lines.append("## Correctness cross-check (interop gate)\n") + if checks: + overall = all(c.passed for c in checks) + lines.append(f"**Overall: {'PASS ✅' if overall else 'FAIL ❌'}**\n") + lines.append("| kind | detail | result |") + lines.append("|------|--------|--------|") + for c in checks: + lines.append(f"| {c.kind} | {c.detail} | {'PASS' if c.passed else 'FAIL'} |") + else: + lines.append("_No cross-checks run (e.g. combined report)._") + lines.append("") + + # DoS-protection effectiveness + if dos is not None: + rows, effective, threshold = dos + lines.append("## DoS-protection effectiveness (this system)\n") + lines.append( + "Equi-X defends by making requesters *solve* (expensive) while the service " + "only *verifies* (cheap). Protection factor = measured attacker time to craft " + "one accepted token at a given effort ÷ measured verify time. " + f"Judged **effective** when ≥ {threshold:.0f}× on every measured point.\n" + ) + if rows: + from .dosprotect import min_effective_effort + verdict = "EFFECTIVE ✅" if effective else "WEAK ⚠️" + mee = min_effective_effort(rows, threshold) + eff_where = ", ".join( + f"`{d}`: effort ≥ {e}" if e is not None else f"`{d}`: not reached in tested range" + for d, e in mee.items() + ) + best = max(rows, key=lambda r: r.effort) + lines.append( + f"**Verdict: {verdict}** (effective from — {eff_where}).\n\n" + f"At effort {best.effort}, an attacker needs ~{best.attacker_s:.3g}s " + f"(impl `{best.attacker_impl}`) to craft one accepted request, while the " + f"defender verifies in ~{best.defender_verify_s*1e6:.2f}µs " + f"(**{best.protection_factor:,.0f}×** asymmetry; one core screens " + f"~{best.verify_per_sec:,.0f} requests/s vs the attacker's " + f"~{best.attacker_tokens_per_sec:,.2f} tokens/s).\n" + ) + lines.append("| device | effort | attacker time/token | attacker impl | verify time | protection factor | verify/s | attacker tokens/s |") + lines.append("|---|---|---|---|---|---|---|---|") + for r in rows: + lines.append( + f"| {r.device} | {r.effort} | {_fmt_ns(r.attacker_s*1e9)} | {r.attacker_impl} | " + f"{_fmt_ns(r.defender_verify_s*1e9)} | {r.protection_factor:,.0f}× | " + f"{r.verify_per_sec:,.0f} | {r.attacker_tokens_per_sec:,.3f} |" + ) + else: + lines.append("_Insufficient effort/verify measurements to assess._") + lines.append("") + + # Measured concurrency / saturation -- complements (never overwrites) the + # per-core DoS estimate above. + if concurrency: + _concurrency_section(lines, concurrency, dos) + + # Measured mining rate vs difficulty. + if mining: + _mining_section(lines, mining) + + note = ("every plot compares C vs Rust; with multiple CPUs each plot is faceted " + "per CPU and `xdev_*` charts compare CPUs directly") + lines.append(f"## Comparison plots ({note})\n") + for p in plots: + if p.startswith("__error__:"): + lines.append(f"> ⚠️ {p[len('__error__:'):]}\n") + else: + lines.append(f"### {p.replace('_', ' ').replace('.png','').title()}\n") + lines.append(f"![{p}](plots/{p})\n") + + def table(op, cols, rowfn): + rows = [s for s in stats if s.operation == op] + if not rows: + return + lines.append(f"## {op} results\n") + lines.append("| " + " | ".join(cols) + " |") + lines.append("|" + "|".join(["---"] * len(cols)) + "|") + for s in sorted(rows, key=lambda s: (s.device_label, json.dumps(s.label), s.impl, s.runtime_requested)): + lines.append("| " + " | ".join(rowfn(s)) + " |") + lines.append("") + + table("solve", ["device", "challenge", "impl", "runtime", "median", "p95", "solves/s", "hashes/s", "sols/solve", "RSS(KB)"], + lambda s: [s.device_label, str(s.label.get("challenge")), s.impl, str(s.runtime_effective), + _fmt_ns(s.median_ns), _fmt_ns(s.p95_ns), f"{s.solves_per_sec:.1f}", + f"{s.hashes_per_sec:,.0f}", f"{s.solutions_mean:.2f}", str(s.peak_rss_kb)]) + table("verify", ["device", "challenge", "impl", "runtime", "median", "p95", "result", "RSS(KB)"], + lambda s: [s.device_label, str(s.label.get("challenge")), s.impl, str(s.runtime_effective), + _fmt_ns(s.median_ns), _fmt_ns(s.p95_ns), str(s.verify_result), str(s.peak_rss_kb)]) + table("hashx_compile", ["device", "challenge", "impl", "runtime", "median compile", "median exec", "RSS(KB)"], + lambda s: [s.device_label, str(s.label.get("challenge")), s.impl, str(s.runtime_effective), + _fmt_ns(s.compile_median_ns), _fmt_ns(s.median_ns), str(s.peak_rss_kb)]) + table("effort", ["device", "base", "target", "impl", "runtime", "mean attempts", "median time", "mean achieved"], + lambda s: [s.device_label, str(s.label.get("base")), str(s.label.get("target_effort")), s.impl, + str(s.runtime_effective), f"{s.attempts_mean:.1f}", _fmt_ns(s.median_ns), + f"{s.achieved_effort_mean:.0f}"]) + + lines.append("---") + lines.append("_See `results.csv` for the full flat dataset and `raw/results.json` for per-rep data._") + (out_dir / "report.md").write_text("\n".join(lines)) diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/runner.py b/tools/benchmarks/Equi-X/harness/equix_bench/runner.py new file mode 100644 index 0000000..ba00326 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/runner.py @@ -0,0 +1,98 @@ +"""Execute a runner cell as a subprocess (one process per cell). + +One process per cell keeps peak_rss_kb attributable and isolates each +(impl, operation, runtime, params) measurement from the others. +""" +from __future__ import annotations + +import os +import subprocess +import threading +from pathlib import Path + +from .protocol import JobSpec, Result +from .registry import Adapter + + +class RunnerError(RuntimeError): + pass + + +# Live runner subprocesses, so a Ctrl+C can kill them promptly instead of leaving +# them running. Worker threads (concurrency/mining) never receive KeyboardInterrupt +# themselves, so the main-thread SIGINT handler (installed in cli.main) reaps them +# via terminate_all_children(); the per-call finally cleans up the common case. +_active: set[subprocess.Popen] = set() +_active_lock = threading.Lock() + + +def terminate_all_children() -> None: + """Kill every runner subprocess still running. Safe to call from a signal + handler (main thread) and idempotent.""" + with _active_lock: + procs = list(_active) + for p in procs: + try: + p.kill() + except Exception: # noqa: BLE001 - best-effort teardown, never raise + pass + + +def run( + adapter: Adapter, + spec: JobSpec, + repo_root: Path, + timeout: float = 900.0, +) -> Result: + argv = adapter.resolve(repo_root) + env = dict(os.environ) + env.update(adapter.env) + try: + proc = subprocess.Popen( + argv, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + except FileNotFoundError as e: + raise RunnerError(f"{adapter.name} executable not found: {argv[0]}") from e + with _active_lock: + _active.add(proc) + try: + stdout, stderr = proc.communicate(input=spec.to_json(), timeout=timeout) + except subprocess.TimeoutExpired as e: + proc.kill() + proc.communicate() # reap the killed child so it can't linger as a zombie + raise RunnerError(f"{adapter.name} timed out after {timeout}s") from e + except BaseException: + # KeyboardInterrupt (or anything else): never leave the child running. + proc.kill() + try: + proc.communicate(timeout=5) + except Exception: # noqa: BLE001 + pass + raise + finally: + with _active_lock: + _active.discard(proc) + + out = (stdout or "").strip() + if not out: + raise RunnerError( + f"{adapter.name} produced no output (exit {proc.returncode}); " + f"stderr: {(stderr or '').strip()[:500]}" + ) + # A runner may print progress lines to stdout before the JSON; take the last line. + line = out.splitlines()[-1] + try: + import json + + result = Result.from_dict(json.loads(line)) + except Exception as e: # noqa: BLE001 + raise RunnerError( + f"{adapter.name} emitted invalid result JSON: {e}; " + f"raw: {line[:500]}" + ) from e + return result diff --git a/tools/benchmarks/Equi-X/harness/equix_bench/stats.py b/tools/benchmarks/Equi-X/harness/equix_bench/stats.py new file mode 100644 index 0000000..3985530 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/equix_bench/stats.py @@ -0,0 +1,130 @@ +"""Aggregate a runner Result's per-rep timings into summary statistics.""" +from __future__ import annotations + +import math +import statistics +from dataclasses import dataclass, field +from typing import Any, Optional + +from .protocol import Result + +# Equi-X solve: stage 0 evaluates the HashX function once for every index in the +# 16-bit index space (equix solver_heap.h: INDEX_SPACE = 1 << 16), which dominates +# the per-solve hashing. Used to derive an effective hash-rate from throughput. +HASHX_PER_SOLVE = 1 << 16 + + +def _percentile(values: list[float], pct: float) -> float: + if not values: + return 0.0 + s = sorted(values) + # Nearest-rank method: rank = ceil(P/100 * N). + k = max(1, min(len(s), math.ceil(pct / 100.0 * len(s)))) + return s[k - 1] + + +@dataclass +class CellStats: + impl: str + operation: str + runtime_requested: str + runtime_effective: Optional[str] + label: dict[str, Any] + reps: int + ok: bool + # wall-time stats (ns) + min_ns: float + median_ns: float + mean_ns: float + stddev_ns: float + p95_ns: float + # derived / auxiliary + solutions_mean: float + compile_median_ns: float + attempts_mean: float + achieved_effort_mean: float + solves_per_sec: float + hashes_per_sec: float + peak_rss_kb: int + verify_result: Optional[str] + walls: list[float] = field(default_factory=list) # per-rep wall_ns + error: Optional[str] = None + extra: dict[str, Any] = field(default_factory=dict) + # executing hardware (see device.py) + device_label: str = "host" + device_type: str = "cpu" + device_name: str = "unknown" + device_arch: str = "unknown" + + +def summarize( + impl: str, + operation: str, + runtime_requested: str, + label: dict[str, Any], + result: Result, + device: Optional[dict[str, Any]] = None, +) -> CellStats: + device = device or {} + dkw = dict( + device_label=device.get("label", "host"), + device_type=device.get("type", "cpu"), + device_name=device.get("name", result.env.get("cpu", "unknown")), + device_arch=device.get("arch", result.env.get("arch", "unknown")), + ) + if not result.ok or not result.runs: + return CellStats( + impl=impl, + operation=operation, + runtime_requested=runtime_requested, + runtime_effective=result.runtime_effective, + label=label, + reps=0, + ok=False, + min_ns=0, median_ns=0, mean_ns=0, stddev_ns=0, p95_ns=0, + solutions_mean=0, compile_median_ns=0, attempts_mean=0, + achieved_effort_mean=0, solves_per_sec=0, hashes_per_sec=0, + peak_rss_kb=result.peak_rss_kb, + verify_result=None, + error=result.error or "no runs", + **dkw, + ) + + walls = [float(r.wall_ns) for r in result.runs] + compiles = [float(r.compile_ns) for r in result.runs] + sols = [float(r.solutions) for r in result.runs] + attempts = [float(r.attempts) for r in result.runs] + efforts = [float(r.achieved_effort) for r in result.runs] + + median_ns = statistics.median(walls) + mean_ns = statistics.fmean(walls) + stddev_ns = statistics.pstdev(walls) if len(walls) > 1 else 0.0 + + # throughput: solves/sec from median solve time; hash-rate from solves/sec. + solves_per_sec = 1e9 / median_ns if operation == "solve" and median_ns > 0 else 0.0 + hashes_per_sec = solves_per_sec * HASHX_PER_SOLVE + + return CellStats( + impl=impl, + operation=operation, + runtime_requested=runtime_requested, + runtime_effective=result.runtime_effective, + label=label, + reps=len(result.runs), + ok=True, + min_ns=min(walls), + median_ns=median_ns, + mean_ns=mean_ns, + stddev_ns=stddev_ns, + p95_ns=_percentile(walls, 95), + solutions_mean=statistics.fmean(sols), + compile_median_ns=statistics.median(compiles), + attempts_mean=statistics.fmean(attempts), + achieved_effort_mean=statistics.fmean(efforts), + solves_per_sec=solves_per_sec, + hashes_per_sec=hashes_per_sec, + peak_rss_kb=result.peak_rss_kb, + verify_result=result.runs[-1].verify_result, + walls=walls, + **dkw, + ) diff --git a/tools/benchmarks/Equi-X/harness/pyproject.toml b/tools/benchmarks/Equi-X/harness/pyproject.toml new file mode 100644 index 0000000..7b206cf --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "equix-bench" +version = "0.1.0" +description = "Benchmarking framework for the Equi-X proof-of-work algorithm (C and Rust implementations)" +requires-python = ">=3.11" +dependencies = [ + "matplotlib>=3.5", + "numpy>=1.21", +] + +[project.scripts] +equix-bench = "equix_bench.cli:main" + +[tool.setuptools] +packages = ["equix_bench"] diff --git a/tools/benchmarks/Equi-X/harness/tests/test_combine.py b/tools/benchmarks/Equi-X/harness/tests/test_combine.py new file mode 100644 index 0000000..1b02964 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_combine.py @@ -0,0 +1,143 @@ +"""`combine` merges runs from many devices into ONE faceted report — including +the concurrency and mining sections/figures, which the per-run CSVs carry and +which must survive the CSV round-trip. Auto-discovery walks a tree and identifies +runs by their raw record file, taking device identity from the records (not paths) +so an arbitrary collected layout still combines correctly.""" +import json +from pathlib import Path + +from equix_bench import report as reportmod +from equix_bench.cli import _discover_runs, cmd_combine +from equix_bench.concurrency import ConcResult, LevelStat +from equix_bench.concurrency import read_csv as conc_read +from equix_bench.concurrency import write_csv as conc_write +from equix_bench.mining import MiningPoint, MiningResult +from equix_bench.mining import read_csv as mining_read +from equix_bench.mining import write_csv as mining_write +from equix_bench.protocol import Result, Run +from equix_bench.stats import summarize + + +# --------------------------------------------------------------- CSV round-trips + + +def _conc(device, impl): + lv = [LevelStat(1, 1, 0.01, 100.0, 100.0, 1.0, 1000), + LevelStat(2, 2, 0.011, 182.0, 91.0, 0.91, 2000)] + return ConcResult(device=device, impl=impl, operation="solve", nproc=2, reps=40, + challenge="deadbeef", baseline_ops_per_sec=100.0, + peak_ops_per_sec=182.0, knee_workers=2, levels=lv) + + +def _mining(device, impl): + pts = [MiningPoint(100, 10, 0.165, 0.11, 0.14, 34.0, 350.0, 6.06, 2, 2, 12.2, 1.0, 0, 16, 16, 8), + MiningPoint(1000, 10, 3.6, 2.5, 3.4, 500.0, 3000.0, 0.28, 2, 2, 0.55, 0.98, 1, 16, 16, 8)] + return MiningResult(device=device, impl=impl, challenge_base="abcd", nproc=2, points=pts) + + +def test_concurrency_csv_roundtrip(tmp_path): + orig = [_conc("cpuA", "equix-c"), _conc("cpuA", "equix-rust")] + conc_write(orig, tmp_path / "c.csv") + back = conc_read(tmp_path / "c.csv") + assert {r.impl for r in back} == {"equix-c", "equix-rust"} + r = next(x for x in back if x.impl == "equix-c") + # Summary fields are re-derived from the levels, matching measure(). + assert r.nproc == 2 and r.knee_workers == 2 + assert r.baseline_ops_per_sec == 100.0 and r.peak_ops_per_sec == 182.0 + assert [lv.workers for lv in r.levels] == [1, 2] + + +def test_mining_csv_roundtrip(tmp_path): + mining_write([_mining("cpuA", "equix-rust")], tmp_path / "m.csv") + back = mining_read(tmp_path / "m.csv") + assert len(back) == 1 + r = back[0] + assert r.nproc == 2 and r.challenge_base == "abcd" + assert [p.effort for p in r.points] == [100, 1000] # sorted on read + assert r.points[0].solution_bytes_max == 16 + + +# ------------------------------------------------------------ discovery + combine + + +def _raw_record(device_label, impl, op, runtime, median_ns): + """A minimal enriched raw record in the on-wire schema cmd_run persists to + raw/results.json (schema_version + nested impl block + _* enrichment).""" + return { + "schema_version": 1, "ok": True, + "impl": {"name": impl, "version": "1", "commit": "c"}, + "operation": op, "runtime_requested": runtime, "runtime_effective": "compiled", + "env": {"cpu": device_label, "arch": "arm"}, + "runs": [{"index": 0, "wall_ns": median_ns, "solutions": 1, "compile_ns": 0, + "attempts": 0, "achieved_effort": 0, "verify_result": "OK"}], + "solutions_hex": None, "peak_rss_kb": 1000, "error": None, + "_label": {"challenge": "deadbeef"}, "_impl": impl, "_group": op, + "_device": {"label": device_label, "type": "cpu", "name": device_label, "arch": "arm"}, + } + + +def _write_run(dirpath: Path, device_label, ts, with_extras=False): + dirpath.mkdir(parents=True, exist_ok=True) + (dirpath / "raw").mkdir(exist_ok=True) + recs = [_raw_record(device_label, "equix-c", "solve", "try-compile", 39_000_000), + _raw_record(device_label, "equix-rust", "solve", "try-compile", 4_500_000), + # two runtimes of the same op must NOT collide during de-dup + _raw_record(device_label, "equix-c", "solve", "interpret", 42_000_000), + _raw_record(device_label, "equix-rust", "solve", "interpret", 12_000_000)] + (dirpath / "raw" / "results.json").write_text(json.dumps(recs)) + (dirpath / "run_meta.json").write_text(json.dumps({"timestamp": ts, "devices": [device_label]})) + if with_extras: + conc_write([_conc(device_label, "equix-c"), _conc(device_label, "equix-rust")], + dirpath / "concurrency.csv") + mining_write([_mining(device_label, "equix-rust")], dirpath / "mining.csv") + + +def test_discover_runs_is_layout_agnostic(tmp_path): + # Two different nesting depths under one tree; discovery finds both. + _write_run(tmp_path / "deviceA" / "main", "cpu-a", "2026-07-27T10:00:00+00:00") + _write_run(tmp_path / "host-b" / "results" / "main", "cpu-b", "2026-07-27T11:00:00+00:00") + found = _discover_runs(tmp_path) + assert len(found) == 2 + assert {p.name for p in found} == {"main"} + + +class _Args: + def __init__(self, **kw): + self.inputs = None + self.root = None + self.out = None + self.__dict__.update(kw) + + +def test_combine_root_produces_faceted_multidevice_report(tmp_path): + _write_run(tmp_path / "A" / "main", "cpu-a", "2026-07-27T10:00:00+00:00", with_extras=True) + _write_run(tmp_path / "B" / "main", "cpu-b", "2026-07-27T11:00:00+00:00", with_extras=True) + out = tmp_path / "combined" + rc = cmd_combine(_Args(root=str(tmp_path), out=str(out))) + assert rc == 0 + md = (out / "report.md").read_text() + # Both devices named as CPUs in the header. + assert "cpu-a" in md and "cpu-b" in md + # Concurrency + mining sections survived the combine (they were dropped before). + assert "Sustained throughput under concurrency" in md + assert "Mining rate vs difficulty" in md + # Multi-device concurrency table gains a device column. + assert "| device | impl | operation |" in md + # Cross-device (xdev_*) headline figures are emitted with >1 device. + assert any((out / "plots").glob("xdev_*.png")) + # The faceted concurrency/mining figures exist. + assert (out / "plots" / "concurrency_solve.png").exists() + assert (out / "plots" / "mining_rate.png").exists() + + +def test_combine_dedups_reruns_keeping_newest(tmp_path): + # Same device measured twice (a re-run): the newer run must replace, not + # double the cell count. + _write_run(tmp_path / "old" / "main", "cpu-a", "2026-07-27T09:00:00+00:00") + _write_run(tmp_path / "new" / "main", "cpu-a", "2026-07-27T15:00:00+00:00") + out = tmp_path / "combined" + rc = cmd_combine(_Args(root=str(tmp_path), out=str(out))) + assert rc == 0 + rows = (out / "results.csv").read_text().strip().splitlines()[1:] + # 2 impls x 2 runtimes x 1 device = 4 cells, NOT 8 (dedup collapsed the re-run). + assert len(rows) == 4 diff --git a/tools/benchmarks/Equi-X/harness/tests/test_concurrency.py b/tools/benchmarks/Equi-X/harness/tests/test_concurrency.py new file mode 100644 index 0000000..cbc3e7b --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_concurrency.py @@ -0,0 +1,93 @@ +"""The concurrency benchmark must MEASURE aggregate parallel throughput and the +saturation knee -- distinct from dosprotect.py's per-core 1/latency estimate.""" +from pathlib import Path + +import equix_bench.concurrency as conc +from equix_bench.concurrency import _ladder, _worker_median_s, measure +from equix_bench.protocol import Result, Run +from equix_bench.registry import Adapter + + +def _result(wall_ns, ok=True, rss=1000): + runs = [ + Run(index=i, wall_ns=w, solutions=1, compile_ns=0, attempts=0, + achieved_effort=0, verify_result="OK") + for i, w in enumerate(wall_ns) + ] + return Result( + ok=ok, impl_name="equix-c", impl_version="1", impl_commit="c", + operation="solve", runtime_requested="try-compile", runtime_effective="compiled", + env={}, runs=runs, solutions_hex=["ab"], peak_rss_kb=rss, error=None, + ) + + +def _adapter(): + return Adapter(name="equix-c", exec=["/bin/true"], protocol_version=1, + capabilities=[], runtimes=[], env={}) + + +def test_ladder_powers_of_two_plus_top(): + assert _ladder(1) == [1] + assert _ladder(4) == [1, 2, 4] + assert _ladder(8) == [1, 2, 4, 8] + assert _ladder(6) == [1, 2, 4, 6] # non-power-of-two core count still included + assert _ladder(10, [1, 3, 10]) == [1, 3, 10] # explicit levels honored + assert _ladder(10, [3, 10]) == [1, 3, 10] # level 1 forced in (baseline anchor) + import pytest + with pytest.raises(ValueError): + _ladder(14, [16, 32]) # all out of range -> loud error, not empty + + +def test_worker_median_seconds(): + assert _worker_median_s(_result([10, 20, 30])) == 20 / 1e9 + assert _worker_median_s(_result([], ok=True)) is None + assert _worker_median_s(_result([10], ok=False)) is None + + +def test_measure_aggregates_throughput_and_scaling(monkeypatch): + # No contention in the fake: each worker sustains 100 ops/s (10 ms/op), so + # aggregate must scale linearly and peak at the max worker count. + monkeypatch.setattr(conc, "run", + lambda a, s, r, timeout=900.0: _result([10_000_000] * 3)) + res = measure(_adapter(), "solve", "deadbeef", None, max_workers=4, reps=3, + warmup=1, repo_root=Path("."), device_label="cpuX", timeout=10, + levels=[1, 2, 4]) + + assert res.baseline_ops_per_sec == 100.0 + agg = {lv.workers: lv.aggregate_ops_per_sec for lv in res.levels} + assert agg == {1: 100.0, 2: 200.0, 4: 400.0} + assert res.peak_ops_per_sec == 400.0 and res.knee_workers == 4 + # perfect linear scaling -> efficiency 1.0 at every level + assert all(abs(lv.scaling_efficiency - 1.0) < 1e-9 for lv in res.levels) + # per-level RSS is summed across the concurrent workers + assert {lv.workers: lv.total_peak_rss_kb for lv in res.levels} == {1: 1000, 2: 2000, 4: 4000} + + +def test_baseline_falls_back_per_worker_when_level1_fails(monkeypatch): + # Level 1 fails, level 2 succeeds at 100 ops/s per worker: the baseline must + # anchor to PER-WORKER throughput (100), not the 2-worker aggregate (200) -- + # otherwise every efficiency/naive-Nx figure is off by the level's width. + calls = {"n": 0} + def fake_run(a, s, r, timeout=900.0): + calls["n"] += 1 + if calls["n"] <= 2: # calibration + the level-1 worker fail + return _result([], ok=False) + return _result([10_000_000] * 3) + monkeypatch.setattr(conc, "run", fake_run) + res = measure(_adapter(), "solve", "deadbeef", None, max_workers=2, reps=3, + warmup=1, repo_root=Path("."), device_label="cpuX", timeout=10, + levels=[1, 2], min_window_s=0) + assert res.baseline_ops_per_sec == 100.0 # per-worker, not aggregate + lv2 = [lv for lv in res.levels if lv.workers == 2][0] + assert abs(lv2.scaling_efficiency - 1.0) < 1e-9 # 200/(100*2), not 200/(200*2) + + +def test_measure_handles_failed_workers(monkeypatch): + monkeypatch.setattr(conc, "run", + lambda a, s, r, timeout=900.0: _result([], ok=False)) + res = measure(_adapter(), "solve", "deadbeef", None, max_workers=2, reps=3, + warmup=1, repo_root=Path("."), device_label="cpuX", timeout=10, + levels=[1, 2]) + assert res.peak_ops_per_sec == 0.0 + assert res.error == "no worker produced a usable measurement" + assert all(lv.aggregate_ops_per_sec == 0.0 for lv in res.levels) diff --git a/tools/benchmarks/Equi-X/harness/tests/test_device.py b/tools/benchmarks/Equi-X/harness/tests/test_device.py new file mode 100644 index 0000000..8406d50 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_device.py @@ -0,0 +1,104 @@ +from pathlib import Path + +from equix_bench import report as reportmod +from equix_bench.cli import _load_cells_from_raw +from equix_bench.device import device_from_env, parse_cpu_model, slug +from equix_bench.protocol import Result +from equix_bench.stats import summarize + + +def test_slug(): + assert slug("Intel(R) Xeon(R) @ 2.10GHz") == "intel-r-xeon-r-2-10ghz" + assert slug("") == "unknown" + + +X86_CPUINFO = """processor\t: 0 +vendor_id\t: GenuineIntel +model name\t: Intel(R) Xeon(R) Processor @ 2.10GHz +cpu MHz\t\t: 2100.000 +""" + +# Raspberry Pi 5 (aarch64) has NO "model name"; the board is under "Model". +PI5_CPUINFO = """processor\t: 0 +BogoMIPS\t: 108.00 +Features\t: fp asimd evtstrm aes pmull sha1 sha2 crc32 +CPU implementer\t: 0x41 +CPU part\t: 0xd0b +processor\t: 1 +CPU part\t: 0xd0b +Revision\t: d04170 +Model\t\t: Raspberry Pi 5 Model B Rev 1.0 +""" + + +def test_parse_cpu_model_x86_and_arm(): + assert parse_cpu_model(X86_CPUINFO) == "Intel(R) Xeon(R) Processor @ 2.10GHz" + # On ARM/Pi we fall back to the board "Model" line instead of "unknown". + assert parse_cpu_model(PI5_CPUINFO) == "Raspberry Pi 5 Model B Rev 1.0" + assert parse_cpu_model("no fields here\n") == "unknown" + + +def test_pi_auto_label_is_meaningful(): + env = {"cpu": "Raspberry Pi 5 Model B Rev 1.0", "arch": "aarch64", + "device": "cpu", "os_version": "6.6.31-rpi"} + d = device_from_env(env) + assert d["arch"] == "aarch64" + assert d["label"] == "raspberry-pi-5-model-b-rev-1-0-6-6-31-rpi" + + +def test_device_from_env_precedence(): + env = {"cpu": "My CPU", "arch": "x86_64", "device": "cpu", "os_version": "6.1.0"} + d = device_from_env(env) + assert d["type"] == "cpu" and d["name"] == "My CPU" and d["arch"] == "x86_64" + # OS version is folded into the auto label + assert d["os_version"] == "6.1.0" + assert d["label"] == "my-cpu-6-1-0" + # explicit label override wins + assert device_from_env(env, override_label="box1")["label"] == "box1" + # a GPU runner is honored + g = device_from_env({"cpu": "Some GPU", "arch": "sm_90", "device": "gpu", "os_version": "1"}) + assert g["type"] == "gpu" + + +def _raw(impl, device_label, wall): + return { + "schema_version": 1, "ok": True, + "impl": {"name": impl, "version": "1", "commit": "c", "runtime_effective": "compiled"}, + "operation": "solve", "runtime_requested": "try-compile", + "runtime_effective": "compiled", + "env": {"cpu": device_label, "arch": "x86_64", "device": "cpu"}, + "runs": [{"index": 0, "wall_ns": wall, "solutions": 4, "compile_ns": 0, + "attempts": 0, "achieved_effort": 0, "verify_result": None}], + "solutions_hex": ["00" * 16], "peak_rss_kb": 100, "error": None, + "_label": {"challenge": "deadbeef"}, + "_device": {"type": "cpu", "name": device_label, "arch": "x86_64", "label": device_label}, + "_impl": impl, "_group": "solve", + } + + +def test_summarize_stamps_device(): + r = Result.from_dict(_raw("equix-c", "cpuA", 10)) + st = summarize("equix-c", "solve", "try-compile", {"challenge": "deadbeef"}, r, + {"type": "cpu", "name": "cpuA", "arch": "x86_64", "label": "cpuA"}) + assert st.device_label == "cpuA" and st.device_arch == "x86_64" + + +def test_combine_loader_reconstructs_cells(): + raws = [_raw("equix-c", "cpuA", 10), _raw("equix-rust", "cpuA", 12), + _raw("equix-c", "cpuB", 20), _raw("equix-rust", "cpuB", 22)] + cells = _load_cells_from_raw(raws) + assert len(cells) == 4 + assert {c.device_label for c in cells} == {"cpuA", "cpuB"} + assert {c.impl for c in cells} == {"equix-c", "equix-rust"} + + +def test_generate_emits_cross_device_plots(tmp_path: Path): + raws = [_raw("equix-c", "cpuA", 10), _raw("equix-rust", "cpuA", 12), + _raw("equix-c", "cpuB", 20), _raw("equix-rust", "cpuB", 22)] + stats = _load_cells_from_raw(raws) + reportmod.generate(stats, [], raws, tmp_path, + {"timestamp": "t", "config": "test", "devices": ["cpuA", "cpuB"]}) + # faceted solve plot + cross-device chart both present + assert (tmp_path / "plots" / "solve_time_by_runtime.png").exists() + assert (tmp_path / "plots" / "xdev_throughput.png").exists() + assert (tmp_path / "report.md").exists() diff --git a/tools/benchmarks/Equi-X/harness/tests/test_difficulty_control.py b/tools/benchmarks/Equi-X/harness/tests/test_difficulty_control.py new file mode 100644 index 0000000..f11b37c --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_difficulty_control.py @@ -0,0 +1,96 @@ +"""The difficulty controllers must converge: drive E so the observed signal +(mint rate for Design A, pressure for Design B) reaches its target, using only +the measured 1/E mint-rate curve.""" +import math + +from equix_bench.difficulty_control import ( + LoadController, MEASURED_MINT, MintRateController, equilibrium_E, + mint_rate_per_machine, simulate_dos, simulate_mining, +) + + +def test_mint_rate_matches_measured_and_is_monotonic(): + for e, r in MEASURED_MINT: + assert abs(mint_rate_per_machine(e) - r) < 1e-6 + rates = [mint_rate_per_machine(e) for e in (50, 100, 300, 1000, 3000, 10000, 100000)] + assert all(a > b for a, b in zip(rates, rates[1:])) # strictly decreasing + # 1/E extrapolation only OUTSIDE the measured range: doubling E halves the rate. + top_e, top_r = max(MEASURED_MINT) + assert abs(mint_rate_per_machine(2 * top_e) - top_r / 2) < 1e-6 + assert abs(mint_rate_per_machine(50) - MEASURED_MINT[0][1] * MEASURED_MINT[0][0] / 50) < 1e-6 + + +def test_mint_rate_controller_converges_to_target(): + # 10 machines, target 3 tok/s. Controller should settle E so 10·M(E) ≈ 3. + tr = simulate_mining(lambda t: 10.0, target_rate=3.0, steps=60) + assert abs(tr.signal[-1] - 3.0) / 3.0 < 0.05 # within 5% + assert 10 * mint_rate_per_machine(tr.E[-1]) == tr.signal[-1] + + +def test_mint_rate_controller_tracks_capacity_step(): + # Capacity doubles at t=30; controller must roughly double E to hold the rate. + tr = simulate_mining(lambda t: 5.0 if t < 30 else 10.0, target_rate=2.0, steps=80) + assert abs(tr.signal[-1] - 2.0) / 2.0 < 0.05 + E_before = tr.E[29] + assert tr.E[-1] > 1.6 * E_before # ~2× more difficulty + + +def test_equilibrium_E_inverts_the_curve(): + # At the solved E, `machines` x M(E) must equal the target rate. + for machines, target in [(6.0, 2.0), (1.0, 0.5), (20.0, 2.0)]: + E = equilibrium_E(machines, target) + assert abs(machines * mint_rate_per_machine(E) - target) / target < 0.001 + + +def test_production_tuning_is_stable_under_noise_and_reproducible(): + # Gentle gains + seeded E + measurement noise: settled rate stays near target, + # and the same seed gives the same trace. + def cap(t): + return 6.0 + mk = lambda: MintRateController(target_rate=2.0, E=equilibrium_E(6.0, 2.0), max_factor=2.0, ewma=0.15) + a = simulate_mining(cap, 2.0, steps=120, ctrl=mk(), noise=0.08, seed=1) + b = simulate_mining(cap, 2.0, steps=120, ctrl=mk(), noise=0.08, seed=1) + assert a.E == b.E # reproducible for a fixed seed + tail = a.signal[80:] + mean = sum(tail) / len(tail) + assert abs(mean - 2.0) / 2.0 < 0.05 # settled within 5% of target + # Seeded near equilibrium -> no big cold-start excursion in E. + assert max(a.E) / min(a.E) < 3.0 + + +def test_load_controller_direction_and_deadband(): + c = LoadController(p_set=0.8, E=1000.0, e_min=100.0) + assert c.update(1.5) > 1000.0 # overloaded -> raise E + c2 = LoadController(p_set=0.8, E=1000.0, e_min=100.0) + assert c2.update(0.2) < 1000.0 # idle -> lower E + c3 = LoadController(p_set=0.8, E=1000.0, deadband=0.05) + assert c3.update(0.81) == 1000.0 # within deadband -> hold + + +def test_adaptive_attacker_bounds_mean_load_but_sawtooths(): + # An attacker that only attacks while E is below a give-up point must not be + # able to hold the node saturated continuously (mean load bounded), yet the + # decay controller visibly oscillates (E spans a wide range). + def adaptive(t, E): + return 6.0 if (10 <= t < 80 and E < 800.0) else 0.0 + tr = simulate_dos(lambda t: 8.0, lambda t: 0.0, service_capacity=40.0, + steps=100, adaptive_attackers=adaptive) + window = slice(10, 80) + util = tr.extra["util"][window] + assert sum(util) / len(util) < 0.9 # mean load bounded, not pinned at 1 + assert max(tr.E[window]) / min(tr.E[window]) > 3 # E sawtooths, not settled + # Attacker gets a nonzero share (it isn't shut out) but not a constant one. + on = [a for a in tr.extra["attacker_rate"][window] if a > 0] + assert 0 < len(on) < 70 + + +def test_load_controller_throttles_attack(): + # Flood during [20,50); node capacity 40 req/s, honest 8 req/s. + tr = simulate_dos(lambda t: 8.0, lambda t: 6.0 if 20 <= t < 50 else 0.0, + service_capacity=40.0, steps=90) + # After the controller reacts, sustained utilization must fall back to ~p_set. + late_attack_util = tr.extra["util"][45] + assert late_attack_util < 0.95 # not saturated anymore + # E rises during the attack and decays again afterwards. + assert max(tr.E[20:50]) > 3 * tr.E[19] + assert tr.E[-1] < max(tr.E[20:50]) diff --git a/tools/benchmarks/Equi-X/harness/tests/test_dosprotect.py b/tools/benchmarks/Equi-X/harness/tests/test_dosprotect.py new file mode 100644 index 0000000..bdb10ec --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_dosprotect.py @@ -0,0 +1,54 @@ +"""The DoS-protection evaluation must correctly quantify the attacker/defender +asymmetry that makes Equi-X an effective client puzzle.""" +from equix_bench.dosprotect import DEFAULT_THRESHOLD, assess, min_verify_seconds +from equix_bench.stats import CellStats + + +def _cell(op, impl, median_ns, label=None, device="cpuX"): + return CellStats( + impl=impl, operation=op, runtime_requested="try-compile", + runtime_effective="compiled", label=label or {}, + reps=3, ok=True, min_ns=median_ns, median_ns=median_ns, mean_ns=median_ns, + stddev_ns=0, p95_ns=median_ns, solutions_mean=1, compile_median_ns=0, + attempts_mean=0, achieved_effort_mean=0, solves_per_sec=0, hashes_per_sec=0, + peak_rss_kb=1000, verify_result="OK", device_label=device, + ) + + +def test_min_verify_seconds(): + stats = [ + _cell("verify", "equix-c", 40_000), # 40 µs + _cell("verify", "equix-rust", 44_000), # 44 µs + ] + assert abs(min_verify_seconds(stats, "cpuX") - 40e-6) < 1e-12 + assert min_verify_seconds(stats, "other") is None + + +def test_effective_protection_on_realistic_numbers(): + # verify ~40 µs; crafting a token at effort 1000 ~ 4.4 s -> huge asymmetry. + stats = [ + _cell("verify", "equix-c", 40_000), + _cell("effort", "equix-c", 4_400_000_000, {"target_effort": 1000}), + _cell("effort", "equix-rust", 4_600_000_000, {"target_effort": 1000}), + ] + rows, effective, threshold = assess(stats) + assert threshold == DEFAULT_THRESHOLD + assert len(rows) == 1 + r = rows[0] + # attacker uses the FASTER impl (equix-c here, 4.4s) + assert r.attacker_impl == "equix-c" + assert abs(r.protection_factor - (4.4 / 40e-6)) < 1.0 # ~110,000x + assert effective is True + assert r.verify_per_sec > 20_000 # defender screens >20k/s + assert r.attacker_tokens_per_sec < 1 # attacker <1 token/s + + +def test_weak_protection_flagged(): + # Pathological: verify as expensive as crafting -> not effective. + stats = [ + _cell("verify", "equix-c", 1_000_000_000), # 1 s verify + _cell("effort", "equix-c", 2_000_000_000, {"target_effort": 10}), # 2 s craft + ] + rows, effective, _ = assess(stats) + assert rows and effective is False + assert rows[0].protection_factor < 10 diff --git a/tools/benchmarks/Equi-X/harness/tests/test_mining.py b/tools/benchmarks/Equi-X/harness/tests/test_mining.py new file mode 100644 index 0000000..7cd1037 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_mining.py @@ -0,0 +1,75 @@ +"""The mining benchmark must turn effort searches into a measured mint rate: +per-core and whole-machine tokens/s, with failed searches excluded.""" +from pathlib import Path + +import equix_bench.mining as mining +from equix_bench.mining import measure_point +from equix_bench.protocol import Result, Run +from equix_bench.registry import Adapter + + +def _result(wall_ns, attempts, achieved, ok=True, target=1000): + runs = [Run(index=0, wall_ns=wall_ns, solutions=1 if achieved else 0, compile_ns=0, + attempts=attempts, achieved_effort=achieved, verify_result=None)] + # Winning-token wire bytes are emitted only when the target was reached: + # a 16-byte solution and an 8-byte nonce, as the real runners produce. + minted = achieved >= target + return Result(ok=ok, impl_name="equix-rust", impl_version="1", impl_commit="c", + operation="effort", runtime_requested="try-compile", runtime_effective="compiled", + env={}, runs=runs, solutions_hex=["ab" * 16] if minted else None, + peak_rss_kb=4000, error=None, + winning_nonce_hex="00" * 8 if minted else None) + + +def _adapter(): + return Adapter(name="equix-rust", exec=["/bin/true"], protocol_version=1, + capabilities=["effort"], runtimes=[], env={}) + + +def test_measure_point_rates_and_scaling(monkeypatch): + # Every search: 0.5 s, 100 attempts, reaches the target -> 2 tokens/s/core. + monkeypatch.setattr(mining, "run", + lambda a, s, r, timeout=900.0: _result(500_000_000, 100, 1000)) + p = measure_point(_adapter(), "abcd", effort=1000, samples=5, workers=4, + tokens_per_worker=3, nonce_bytes=8, max_attempts=1_000_000, + repo_root=Path("."), timeout=10) + assert p.samples == 5 + assert abs(p.tokens_per_sec_1core - 2.0) < 1e-9 # 5 tokens / 2.5 s + assert abs(p.token_s_mean - 0.5) < 1e-9 + assert p.attempts_mean == 100 + # 4 workers, each streams 3 tokens (3/1.5s = 2/s), no contention -> 8 tokens/s. + assert p.ok_workers == 4 + assert abs(p.tokens_per_sec_machine - 8.0) < 1e-9 + assert abs(p.scaling_efficiency - 1.0) < 1e-9 + # Message sizes measured from every minted token: constant 16 B + 8 B. + assert (p.solution_bytes_min, p.solution_bytes_max, p.nonce_bytes_wire) == (16, 16, 8) + + +def test_measure_point_excludes_failed_searches(monkeypatch): + # achieved (500) never reaches target (1000) -> every sample rejected. + monkeypatch.setattr(mining, "run", + lambda a, s, r, timeout=900.0: _result(500_000_000, 100, 500)) + p = measure_point(_adapter(), "abcd", effort=1000, samples=4, workers=2, + tokens_per_worker=2, nonce_bytes=8, max_attempts=1_000_000, + repo_root=Path("."), timeout=10) + assert p.samples == 0 + assert p.tokens_per_sec_1core == 0 + assert p.tokens_per_sec_machine == 0 + + +def test_distinct_nonce_ranges_do_not_overlap(monkeypatch): + # Record the nonce_start each invocation used; ranges must be STRIDE-spaced + # and disjoint between the 1-core samples and the concurrent worker streams. + seen = [] + def rec(a, spec, r, timeout=900.0): + seen.append(spec.nonce_start) + return _result(100_000_000, 10, 1000) + monkeypatch.setattr(mining, "run", rec) + measure_point(_adapter(), "abcd", effort=1000, samples=3, workers=2, + tokens_per_worker=2, nonce_bytes=8, max_attempts=1_000_000, + repo_root=Path("."), timeout=10) + S = mining.STRIDE + # 3 sequential 1-core samples at 0,S,2S; then worker0 streams 3S,4S, worker1 5S,6S. + assert seen[:3] == [0, S, 2 * S] + assert sorted(seen[3:]) == [3 * S, 4 * S, 5 * S, 6 * S] + assert len(set(seen)) == len(seen) # all disjoint diff --git a/tools/benchmarks/Equi-X/harness/tests/test_mock_runner.py b/tools/benchmarks/Equi-X/harness/tests/test_mock_runner.py new file mode 100644 index 0000000..f2b95ba --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_mock_runner.py @@ -0,0 +1,48 @@ +"""The harness must speak the protocol to ANY conforming runner, and a job with +`repetitions=N` must yield exactly N timed entries (warmups excluded).""" +import sys + +from equix_bench.protocol import JobSpec +from equix_bench.registry import Adapter +from equix_bench.runner import run + +MOCK = '''#!/usr/bin/env python3 +import sys, json +job = json.loads(sys.stdin.read()) +reps = job.get("repetitions", 1) +# A conforming runner emits exactly `repetitions` timed runs (no warmups). +runs = [{"index": i, "wall_ns": 100 + i, "solutions": 1, "compile_ns": 0, + "attempts": 0, "achieved_effort": 0, "verify_result": None} for i in range(reps)] +print(json.dumps({ + "schema_version": 1, "ok": True, + "impl": {"name": "mock", "version": "0", "commit": "0", "runtime_effective": "interpreted"}, + "operation": job["operation"], "runtime_requested": job.get("runtime", "?"), + "runtime_effective": "interpreted", "env": {}, "runs": runs, + "solutions_hex": ["00" * 16], "peak_rss_kb": 42, "error": None})) +''' + + +def test_run_parses_conforming_runner(tmp_path): + script = tmp_path / "mock.py" + script.write_text(MOCK) + adapter = Adapter( + name="mock", exec=[sys.executable, str(script)], protocol_version=1, + capabilities=["solve"], runtimes=["interpret"], env={}, + ) + r = run(adapter, JobSpec(operation="solve", runtime="interpret", + repetitions=5, warmup=2, challenge_hex="ab"), tmp_path) + assert r.ok + assert len(r.runs) == 5 # exactly repetitions; warmups excluded + assert r.solutions_hex == ["00" * 16] + + +def test_run_tolerates_progress_lines(tmp_path): + script = tmp_path / "chatty.py" + script.write_text(MOCK.replace( + 'print(json.dumps({', 'print("progress: warming up", flush=True)\nprint(json.dumps({' + )) + adapter = Adapter(name="chatty", exec=[sys.executable, str(script)], + protocol_version=1, capabilities=["solve"], runtimes=["interpret"], env={}) + r = run(adapter, JobSpec(operation="solve", runtime="interpret", + repetitions=2, warmup=0, challenge_hex="ab"), tmp_path) + assert r.ok and len(r.runs) == 2 # last stdout line is the JSON result diff --git a/tools/benchmarks/Equi-X/harness/tests/test_protocol.py b/tools/benchmarks/Equi-X/harness/tests/test_protocol.py new file mode 100644 index 0000000..ebf0e18 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_protocol.py @@ -0,0 +1,42 @@ +import json + +import pytest + +from equix_bench.protocol import SCHEMA_VERSION, JobSpec, Result + + +def test_jobspec_omits_none_fields(): + d = json.loads(JobSpec(operation="solve", runtime="interpret", challenge_hex="ab").to_json()) + assert d["schema_version"] == SCHEMA_VERSION + assert d["operation"] == "solve" + assert d["challenge_hex"] == "ab" + assert "solution_hex" not in d # None fields are dropped from the wire form + + +def test_result_rejects_wrong_schema(): + with pytest.raises(ValueError): + Result.from_dict({"schema_version": 999}) + + +def test_result_parse_roundtrip(): + d = { + "schema_version": 1, + "ok": True, + "impl": {"name": "x", "version": "1", "commit": "c", "runtime_effective": "compiled"}, + "operation": "solve", + "runtime_requested": "try-compile", + "runtime_effective": "compiled", + "env": {"os": "linux"}, + "runs": [ + {"index": 0, "wall_ns": 10, "solutions": 4, "compile_ns": 0, + "attempts": 0, "achieved_effort": 0, "verify_result": None}, + {"index": 1, "wall_ns": 20, "solutions": 4, "compile_ns": 0, + "attempts": 0, "achieved_effort": 0, "verify_result": None}, + ], + "solutions_hex": ["00" * 16], + "peak_rss_kb": 100, + "error": None, + } + r = Result.from_dict(d) + assert r.ok and r.impl_name == "x" + assert len(r.runs) == 2 and r.runs[1].wall_ns == 20 diff --git a/tools/benchmarks/Equi-X/harness/tests/test_report_sections.py b/tools/benchmarks/Equi-X/harness/tests/test_report_sections.py new file mode 100644 index 0000000..510f024 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_report_sections.py @@ -0,0 +1,65 @@ +"""End-to-end smoke test for report generation with concurrency + mining data: +a field/format error in these sections must fail here, not at the end of an +hours-long --full run.""" +import json +from pathlib import Path + +from equix_bench import report as reportmod +from equix_bench.concurrency import ConcResult, LevelStat +from equix_bench.concurrency import write_csv as conc_csv +from equix_bench.mining import MiningPoint, MiningResult +from equix_bench.mining import write_csv as mining_csv +from equix_bench.protocol import Result, Run +from equix_bench.stats import summarize + + +def _stats_cell(impl, op, median_ns): + runs = [Run(index=0, wall_ns=median_ns, solutions=1, compile_ns=0, attempts=0, + achieved_effort=0, verify_result="OK")] + res = Result(ok=True, impl_name=impl, impl_version="1", impl_commit="c", + operation=op, runtime_requested="try-compile", runtime_effective="compiled", + env={}, runs=runs, solutions_hex=None, peak_rss_kb=1000, error=None) + return summarize(impl, op, "try-compile", {"challenge": "deadbeef"}, res, + {"label": "cpuX", "type": "cpu", "name": "X", "arch": "arm"}) + + +def _conc(impl, op): + lv = [LevelStat(1, 1, 0.01, 100.0, 100.0, 1.0, 1000), + LevelStat(2, 2, 0.011, 182.0, 91.0, 0.91, 2000)] + return ConcResult(device="cpuX", impl=impl, operation=op, nproc=2, reps=40, + challenge="deadbeef", baseline_ops_per_sec=100.0, + peak_ops_per_sec=182.0, knee_workers=2, levels=lv) + + +def _mining(impl): + # Deliberately non-ascending efforts: the section must sort, not garble. + pts = [MiningPoint(1000, 10, 3.6, 2.5, 3.4, 500.0, 3000.0, 0.28, 2, 2, 0.55, 0.98, 1, 16, 16, 8), + MiningPoint(100, 10, 0.165, 0.11, 0.14, 34.0, 350.0, 6.06, 2, 2, 12.2, 1.0, 0, 16, 16, 8)] + return MiningResult(device="cpuX", impl=impl, challenge_base="abcd", nproc=2, points=pts) + + +def test_generate_report_with_concurrency_and_mining(tmp_path): + stats = [_stats_cell("equix-c", "solve", 39_000_000), + _stats_cell("equix-rust", "solve", 4_500_000), + _stats_cell("equix-c", "verify", 16_000), + _stats_cell("equix-rust", "verify", 15_000)] + conc = [_conc("equix-c", "solve"), _conc("equix-rust", "solve")] + mining = [_mining("equix-rust")] + meta = {"timestamp": "t", "config": "test", "cpu": "X", "nproc": 2, "devices": ["cpuX"]} + + reportmod.generate(stats, [], [], tmp_path, meta, concurrency=conc, mining=mining) + md = (tmp_path / "report.md").read_text() + assert "Sustained throughput under concurrency" in md + assert "Mining rate vs difficulty" in md + # Mining table rows must come out effort-ascending despite input order. + assert md.index("| 100 |") < md.index("| 1000 |") + # The 1/effort headline must use the true endpoints (100 -> 1000, a 10x rise). + assert "10× rise" in md or "10x rise" in md + # Message-size constancy must be reported when sizes are uniform. + assert "constant in difficulty" in md + + conc_csv(conc, tmp_path / "concurrency.csv") + mining_csv(mining, tmp_path / "mining.csv") + head = (tmp_path / "mining.csv").read_text().splitlines() + assert head[0].endswith("nonce_bytes_wire") + assert len(head) == 3 # header + 2 points diff --git a/tools/benchmarks/Equi-X/harness/tests/test_seed_challenge.py b/tools/benchmarks/Equi-X/harness/tests/test_seed_challenge.py new file mode 100644 index 0000000..8e9be9c --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_seed_challenge.py @@ -0,0 +1,56 @@ +"""vary_challenge turns each configured challenge into a SEED: the runner derives +a fresh challenge per rep by SHA-256-chaining it, so measurements span many +challenges. These tests pin the harness wiring (config expansion, protocol +serialization, verify-resolution skip).""" +import json + +from equix_bench.cli import _resolve_verify_solutions +from equix_bench.config import Config, expand +from equix_bench.protocol import JobSpec +from equix_bench.registry import Adapter + + +def _adapter(name, caps): + return Adapter(name=name, exec=["/bin/true"], protocol_version=1, + capabilities=caps, runtimes=["interpret", "try-compile"], env={}) + + +def test_expand_sets_seed_not_challenge_when_varied(): + cfg = Config(warmup=1, repetitions=5, impls=["x"], + jobs=[{"operation": "solve", "runtimes": ["interpret"], + "challenges": ["deadbeef", "cafe"], "vary_challenge": True}]) + cells, _ = expand(cfg, {"x": _adapter("x", ["solve"])}) + assert len(cells) == 2 + for c in cells: + assert c.job.challenge_seed_hex in ("deadbeef", "cafe") + assert c.job.challenge_hex is None # seed replaces the fixed challenge + assert c.label.get("varied") is True + + +def test_expand_fixed_challenge_when_not_varied(): + cfg = Config(warmup=1, repetitions=5, impls=["x"], + jobs=[{"operation": "solve", "runtimes": ["interpret"], + "challenges": ["deadbeef"]}]) + cells, _ = expand(cfg, {"x": _adapter("x", ["solve"])}) + assert cells[0].job.challenge_hex == "deadbeef" + assert cells[0].job.challenge_seed_hex is None + + +def test_jobspec_serializes_seed_and_omits_none_challenge(): + d = json.loads(JobSpec(operation="solve", runtime="interpret", + challenge_seed_hex="abcd").to_json()) + assert d["challenge_seed_hex"] == "abcd" + assert "challenge_hex" not in d # None fields dropped from the wire + + +def test_verify_resolution_skips_seed_mode_cells(): + # A seed-mode verify cell must pass through untouched (runner self-solves) — + # never dropped for "no solution found", never sent to a solver. + cfg = Config(warmup=1, repetitions=3, impls=["x"], + jobs=[{"operation": "verify", "runtimes": ["interpret"], + "challenges": ["deadbeef"], "vary_challenge": True}]) + cells, _ = expand(cfg, {"x": _adapter("x", ["verify"])}) + # adapters/repo_root are irrelevant here: seed cells short-circuit before any run. + out, warnings = _resolve_verify_solutions(cells, {}, repo_root=None) + assert len(out) == 1 and out[0].job.challenge_seed_hex == "deadbeef" + assert warnings == [] diff --git a/tools/benchmarks/Equi-X/harness/tests/test_seed_runner_integration.py b/tools/benchmarks/Equi-X/harness/tests/test_seed_runner_integration.py new file mode 100644 index 0000000..4fd1b2a --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_seed_runner_integration.py @@ -0,0 +1,49 @@ +"""Integration checks on the real runners' seed mode (skipped if not built). +The load-bearing property: because both impls derive challenges with STANDARD +SHA-256, the same seed yields the same challenge stream, so a seed-varied +measurement still compares the two implementations on identical inputs.""" +import hashlib +import json +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +C = ROOT / "build/runners/c/equix_runner" +RUST = ROOT / "runners/rust/target/release/equix_runner" + + +def _run(binary, spec): + p = subprocess.run([str(binary)], input=json.dumps(spec), + capture_output=True, text=True, timeout=120) + return json.loads(p.stdout.strip().splitlines()[-1]) + + +@pytest.mark.skipif(not (C.exists() and RUST.exists()), reason="runners not built") +def test_seed_mode_derives_identical_challenges_across_impls(): + # challenge_0 = SHA256(seed): solving it in seed mode must give, in BOTH + # impls, exactly the solutions of that derived challenge (fixed mode). + seed = "abcd" + derived = hashlib.sha256(bytes.fromhex(seed)).hexdigest() + outs = {} + for name, b in (("c", C), ("rust", RUST)): + seeded = _run(b, {"schema_version": 1, "operation": "solve", "runtime": "interpret", + "repetitions": 1, "warmup": 0, "challenge_seed_hex": seed}) + fixed = _run(b, {"schema_version": 1, "operation": "solve", "runtime": "interpret", + "repetitions": 1, "warmup": 0, "challenge_hex": derived}) + # seed mode's first challenge IS sha256(seed): same solutions as fixed. + assert sorted(seeded["solutions_hex"]) == sorted(fixed["solutions_hex"]) + outs[name] = sorted(seeded["solutions_hex"]) + # And both implementations agree on that derived challenge. + assert outs["c"] == outs["rust"] + + +@pytest.mark.skipif(not RUST.exists(), reason="rust runner not built") +def test_seed_mode_verify_selfsolves_valid_tokens(): + # Seed-mode verify needs no solution_hex and every timed sample is a real, + # accepted token (the runner self-solves each derived challenge). + d = _run(RUST, {"schema_version": 1, "operation": "verify", "runtime": "try-compile", + "repetitions": 8, "warmup": 2, "challenge_seed_hex": "deadbeef"}) + assert d["ok"] and len(d["runs"]) == 8 + assert all(r["verify_result"] == "OK" for r in d["runs"]) diff --git a/tools/benchmarks/Equi-X/harness/tests/test_stats.py b/tools/benchmarks/Equi-X/harness/tests/test_stats.py new file mode 100644 index 0000000..d72c808 --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_stats.py @@ -0,0 +1,43 @@ +from equix_bench.protocol import Result +from equix_bench.stats import _percentile, summarize + + +def _make(walls): + runs = [ + {"index": i, "wall_ns": w, "solutions": 4, "compile_ns": 0, + "attempts": 0, "achieved_effort": 0, "verify_result": None} + for i, w in enumerate(walls) + ] + return Result.from_dict({ + "schema_version": 1, "ok": True, + "impl": {"name": "x", "version": "1", "commit": "c", "runtime_effective": "compiled"}, + "operation": "solve", "runtime_requested": "try-compile", + "runtime_effective": "compiled", "env": {}, "runs": runs, + "solutions_hex": None, "peak_rss_kb": 100, "error": None, + }) + + +def test_percentile_nearest_rank(): + assert _percentile([1, 2, 3, 4, 5], 95) == 5 + assert _percentile([1, 2, 3, 4, 5], 50) == 3 + assert _percentile([], 95) == 0.0 + + +def test_summarize_basic(): + s = summarize("x", "solve", "try-compile", {}, _make([10, 20, 30])) + assert s.ok and s.reps == 3 + assert s.min_ns == 10 and s.median_ns == 20 + assert s.solves_per_sec > 0 # 1e9 / 20 ns + assert s.walls == [10, 20, 30] + + +def test_summarize_handles_failure(): + r = Result.from_dict({ + "schema_version": 1, "ok": False, + "impl": {"name": "x", "version": "1", "commit": "c", "runtime_effective": None}, + "operation": "solve", "runtime_requested": "must-compile", + "runtime_effective": None, "env": {}, "runs": [], + "solutions_hex": None, "peak_rss_kb": 0, "error": "boom", + }) + s = summarize("x", "solve", "must-compile", {}, r) + assert not s.ok and s.error == "boom" diff --git a/tools/benchmarks/Equi-X/harness/tests/test_variants.py b/tools/benchmarks/Equi-X/harness/tests/test_variants.py new file mode 100644 index 0000000..556eecc --- /dev/null +++ b/tools/benchmarks/Equi-X/harness/tests/test_variants.py @@ -0,0 +1,59 @@ +"""Compiler-flag variants ride on the multi-implementation machinery: several C +builds register as distinct impls (from a second manifest dir) and expand into +comparable cells.""" +from equix_bench.config import Config, expand +from equix_bench.registry import Adapter, load_manifests + +_MANIFEST = ( + 'name = "{name}"\n' + 'exec = "build/runners/c/equix_runner"\n' + 'protocol_version = 1\n' + 'capabilities = ["solve", "hashx_compile"]\n' + 'runtimes = ["try-compile", "interpret"]\n' +) + + +def test_load_manifests_merges_multiple_dirs(tmp_path): + d1, d2 = tmp_path / "examples", tmp_path / "generated" + d1.mkdir() + d2.mkdir() + (d1 / "base.manifest.toml").write_text(_MANIFEST.format(name="equix-c")) + (d2 / "o2.manifest.toml").write_text(_MANIFEST.format(name="equix-c-gcc-o2")) + (d2 / "o3.manifest.toml").write_text(_MANIFEST.format(name="equix-c-gcc-o3")) + adapters = load_manifests([d1, d2]) + assert set(adapters) == {"equix-c", "equix-c-gcc-o2", "equix-c-gcc-o3"} + # a non-existent dir is simply skipped + assert set(load_manifests([d1, tmp_path / "nope"])) == {"equix-c"} + + +def test_expand_sweeps_variant_impls(): + names = ["equix-c-gcc-o0", "equix-c-gcc-o3", "equix-c-clang-o3"] + adapters = { + n: Adapter(name=n, exec=["/bin/true"], protocol_version=1, + capabilities=["solve"], runtimes=["try-compile"], env={}) + for n in names + } + cfg = Config( + warmup=1, repetitions=2, impls=names, + jobs=[{"operation": "solve", "runtimes": ["try-compile"], + "challenges": ["deadbeef", "cafe"]}], + ) + cells, warnings = expand(cfg, adapters) + # 3 variants x 2 challenges x 1 runtime + assert len(cells) == 6 + assert {c.impl for c in cells} == set(names) + assert not warnings + + +def test_expand_warns_on_unsupported_runtime(): + adapters = { + "equix-c-gcc-o3": Adapter(name="equix-c-gcc-o3", exec=["/bin/true"], + protocol_version=1, capabilities=["solve"], + runtimes=["try-compile"], env={}), + } + cfg = Config(warmup=1, repetitions=1, impls=["equix-c-gcc-o3"], + jobs=[{"operation": "solve", "runtimes": ["must-compile"], + "challenges": ["deadbeef"]}]) + cells, warnings = expand(cfg, adapters) + assert cells == [] + assert any("must-compile" in w for w in warnings) diff --git a/tools/benchmarks/Equi-X/runners/c/CMakeLists.txt b/tools/benchmarks/Equi-X/runners/c/CMakeLists.txt new file mode 100644 index 0000000..6ee075c --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/c/CMakeLists.txt @@ -0,0 +1,57 @@ +# Equi-X C benchmark runner. +# +# Builds the vendored tevador/equix + hashx static libraries (via add_subdirectory) +# and links them into `equix_runner`. blake2.c symbols come from hashx_static. +cmake_minimum_required(VERSION 3.10) +project(equix_runner C) + +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) + message(STATUS "Setting default build type: ${CMAKE_BUILD_TYPE}") +endif() + +# Location of the vendored equix source tree (contains hashx/ submodule). +if(NOT DEFINED EQUIX_DIR) + set(EQUIX_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../vendored/equix") +endif() +get_filename_component(EQUIX_DIR "${EQUIX_DIR}" ABSOLUTE) + +if(NOT EXISTS "${EQUIX_DIR}/include/equix.h") + message(FATAL_ERROR "equix source not found at ${EQUIX_DIR}. Run scripts/setup.sh first.") +endif() + +# The vendored equix/hashx use cmake_minimum_required(VERSION 2.8.8). CMake >= 4 +# removes compatibility with < 3.5 (hard error) and deprecation-warns for < 3.10. +# Pin the policy floor to 3.10 so those submodules configure cleanly with neither +# the error nor the "Compatibility with CMake < 3.10 will be removed" warning. +if(NOT DEFINED CMAKE_POLICY_VERSION_MINIMUM) + set(CMAKE_POLICY_VERSION_MINIMUM 3.10 CACHE STRING "" FORCE) +endif() + +add_subdirectory("${EQUIX_DIR}" "${CMAKE_BINARY_DIR}/equix_ext") + +add_executable(equix_runner + equix_runner.c + effort.c + json_min.c + sha256.c) + +target_compile_definitions(equix_runner PRIVATE HASHX_SIZE=8 EQUIX_STATIC HASHX_STATIC) + +target_include_directories(equix_runner PRIVATE + "${EQUIX_DIR}/include" + "${EQUIX_DIR}/hashx/include" + "${EQUIX_DIR}/hashx/src") # for the internal blake2.h + +target_link_libraries(equix_runner PRIVATE equix_static hashx_static) + +# Optional build-time provenance (set by scripts/setup.sh). +if(DEFINED EQUIX_C_COMMIT) + target_compile_definitions(equix_runner PRIVATE EQUIX_C_COMMIT="${EQUIX_C_COMMIT}") +endif() +if(DEFINED EQUIX_C_VERSION) + target_compile_definitions(equix_runner PRIVATE EQUIX_C_VERSION="${EQUIX_C_VERSION}") +endif() diff --git a/tools/benchmarks/Equi-X/runners/c/effort.c b/tools/benchmarks/Equi-X/runners/c/effort.c new file mode 100644 index 0000000..b1a2d69 --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/c/effort.c @@ -0,0 +1,46 @@ +#include "effort.h" + +#include + +/* blake2.h is an internal hashx header (not installed); referenced by include + * path into the vendored submodule. We use the full (12-round) standard + * BLAKE2b via init_param/update/final -- NOT the reduced hashx_blake2b_4r. */ +#include + +void effort_solution_bytes(const equix_solution *sol, uint8_t out[16]) { + for (int i = 0; i < EQUIX_NUM_IDX; i++) { + out[2 * i] = (uint8_t)(sol->idx[i] & 0xff); + out[2 * i + 1] = (uint8_t)((sol->idx[i] >> 8) & 0xff); + } +} + +/* Standard unkeyed BLAKE2b-256 (digest 32, fanout 1, depth 1, all else zero). */ +static void blake2b256(const uint8_t *in1, size_t len1, const uint8_t *in2, + size_t len2, uint8_t out[32]) { + blake2b_param p; + memset(&p, 0, sizeof p); + p.digest_length = 32; + p.fanout = 1; + p.depth = 1; + + blake2b_state s; + hashx_blake2b_init_param(&s, &p); + hashx_blake2b_update(&s, in1, len1); + hashx_blake2b_update(&s, in2, len2); + hashx_blake2b_final(&s, out, 32); +} + +uint32_t effort_of(const uint8_t *challenge, size_t challenge_len, + const equix_solution *sol) { + uint8_t sol_bytes[16]; + effort_solution_bytes(sol, sol_bytes); + + uint8_t h[32]; + blake2b256(challenge, challenge_len, sol_bytes, sizeof sol_bytes, h); + + uint32_t hash32 = ((uint32_t)h[0] << 24) | ((uint32_t)h[1] << 16) | + ((uint32_t)h[2] << 8) | (uint32_t)h[3]; + if (hash32 == 0) + return 0xFFFFFFFFu; + return 0xFFFFFFFFu / hash32; +} diff --git a/tools/benchmarks/Equi-X/runners/c/effort.h b/tools/benchmarks/Equi-X/runners/c/effort.h new file mode 100644 index 0000000..0947840 --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/c/effort.h @@ -0,0 +1,30 @@ +/* Tor proposal-327 style effort computation for Equi-X solutions. + * + * The effort layer sits ABOVE Equi-X: given a solved (challenge, solution), + * we hash them with standard BLAKE2b-256 and read the first 32 bits big-endian + * as `hash32`. A solution is "valid at effort E" iff hash32 * E <= 2^32-1, so + * the achieved effort of a solution is floor((2^32-1) / hash32). + * + * The preimage layout and byte order defined here MUST be identical to the Rust + * runner (runners/rust/src/effort.rs) -- the Python cross-check asserts both + * implementations produce the same effort for a fixed (challenge, solution). + * + * Preimage = challenge_bytes || solution_bytes + * solution_bytes = 8 x uint16 little-endian (equix_solution.idx[]) + */ +#ifndef EQUIX_RUNNER_EFFORT_H +#define EQUIX_RUNNER_EFFORT_H + +#include +#include + +#include + +/* Serialize an Equi-X solution to its canonical 16-byte little-endian form. */ +void effort_solution_bytes(const equix_solution *sol, uint8_t out[16]); + +/* Achieved effort of `sol` for `challenge`. Returns UINT32_MAX when hash32==0. */ +uint32_t effort_of(const uint8_t *challenge, size_t challenge_len, + const equix_solution *sol); + +#endif /* EQUIX_RUNNER_EFFORT_H */ diff --git a/tools/benchmarks/Equi-X/runners/c/equix_runner.c b/tools/benchmarks/Equi-X/runners/c/equix_runner.c new file mode 100644 index 0000000..ac8c0da --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/c/equix_runner.c @@ -0,0 +1,711 @@ +/* Equi-X C benchmark runner. + * + * Reads one job-spec JSON object on stdin, runs the requested operation against + * the reference C implementation (tevador/equix + hashx), and writes one result + * JSON object on stdout. All diagnostics go to stderr. See adapters/README.md + * for the protocol. + */ +#include +#include +#include +#include +#include +#if defined(__APPLE__) +#include +#define OS_STR "macos" +#else +#define OS_STR "linux" +#endif + +/* The bundled HashX JITs via mmap(RW)+mprotect(RX) with no MAP_JIT, which the + * Apple Silicon kernel (hard W^X) rejects and would crash on execution. So we + * treat the compiler as unsupported there and fall back to the interpreter. */ +#if defined(__APPLE__) && defined(__aarch64__) +#define JIT_SUPPORTED 0 +#else +#define JIT_SUPPORTED 1 +#endif + +#include +#include + +#include "effort.h" +#include "json_min.h" +#include "sha256.h" +#include "timing.h" + +#ifndef EQUIX_C_COMMIT +#define EQUIX_C_COMMIT "unknown" +#endif +#ifndef EQUIX_C_VERSION +#define EQUIX_C_VERSION "1.0.0" +#endif + +#if defined(__clang__) +#define COMPILER_STR "clang-" __clang_version__ +#elif defined(__GNUC__) +#define COMPILER_STR "gcc-" __VERSION__ +#else +#define COMPILER_STR "unknown" +#endif + +#if defined(__x86_64__) || defined(_M_X64) +#define ARCH_STR "x86_64" +#elif defined(__aarch64__) +#define ARCH_STR "aarch64" +#elif defined(__i386__) +#define ARCH_STR "x86" +#elif defined(__arm__) +#define ARCH_STR "arm" +#else +#define ARCH_STR "unknown" +#endif + +#define MAX_CHALLENGE 256 + +/* CPU model string from /proc/cpuinfo, JSON-escaped, cached. Tries fields in + * priority order so it works across architectures: "model name" (x86), + * "Model" (Raspberry Pi board), "Hardware" (older ARM), "cpu model" (others). + * Returns "unknown" when none are present. */ +static const char *cpu_model(void) { + static char model[256]; + if (model[0]) + return model; + strcpy(model, "unknown"); +#if defined(__APPLE__) + /* macOS has no /proc; the CPU brand comes from sysctl (works on both Intel + * and Apple Silicon, e.g. "Apple M2"). */ + size_t sz = sizeof model; + if (sysctlbyname("machdep.cpu.brand_string", model, &sz, NULL, 0) != 0 || !model[0]) + strcpy(model, "unknown"); + return model; +#else + FILE *f = fopen("/proc/cpuinfo", "r"); + if (!f) + return model; + /* Large enough that ARM's trailing "Model"/"Hardware" lines are captured + * even on many-core machines. */ + static char buf[131072]; + size_t n = fread(buf, 1, sizeof buf - 1, f); + buf[n] = '\0'; + fclose(f); + + static const char *fields[] = {"model name", "Model", "Hardware", "cpu model"}; + for (size_t fi = 0; fi < sizeof fields / sizeof fields[0]; fi++) { + size_t fl = strlen(fields[fi]); + for (const char *p = buf; p; p = strchr(p, '\n') ? strchr(p, '\n') + 1 : NULL) { + if (strncmp(p, fields[fi], fl) != 0) + continue; + /* field must be followed by whitespace/':' (avoid partial matches) */ + char after = p[fl]; + if (after != ' ' && after != '\t' && after != ':') + continue; + const char *c = strchr(p, ':'); + const char *eol = strchr(p, '\n'); + if (!c || (eol && c > eol)) + continue; + c++; + while (*c == ' ' || *c == '\t') + c++; + size_t k = 0; + for (size_t i = 0; c[i] && c[i] != '\n' && c[i] != '\r' && k + 2 < sizeof model; i++) { + if (c[i] == '"' || c[i] == '\\') + model[k++] = '\\'; + model[k++] = c[i]; + } + while (k > 0 && model[k - 1] == ' ') + k--; + model[k] = '\0'; + if (k > 0) + return model; + } + } + return model; +#endif /* !__APPLE__ */ +} + +/* OS kernel release (uname -r), e.g. "6.18.5" (Linux) or "23.5.0" (macOS/Darwin). + * Cached; "unknown" on failure. uname() is portable across Linux and macOS. */ +static const char *os_version(void) { + static char v[128]; + if (v[0]) + return v; + strcpy(v, "unknown"); + struct utsname u; + if (uname(&u) == 0) + snprintf(v, sizeof v, "%s", u.release); + return v; +} + +/* ------------------------------------------------------------------ helpers */ + +static char *read_all_stdin(void) { + size_t cap = 4096, len = 0; + char *buf = malloc(cap); + if (!buf) + return NULL; + size_t n; + while ((n = fread(buf + len, 1, cap - len - 1, stdin)) > 0) { + len += n; + if (len + 1 >= cap) { + cap *= 2; + char *nb = realloc(buf, cap); + if (!nb) { + free(buf); + return NULL; + } + buf = nb; + } + } + buf[len] = '\0'; + return buf; +} + +/* Decode a hex string into `out` (capacity outcap). Returns byte length, or -1. */ +static int hex_decode(const char *hex, uint8_t *out, size_t outcap) { + size_t hl = strlen(hex); + if (hl % 2 != 0) + return -1; + size_t bl = hl / 2; + if (bl > outcap) + return -1; + for (size_t i = 0; i < bl; i++) { + char c0 = hex[2 * i], c1 = hex[2 * i + 1]; + int hi = (c0 >= '0' && c0 <= '9') ? c0 - '0' + : (c0 >= 'a' && c0 <= 'f') ? c0 - 'a' + 10 + : (c0 >= 'A' && c0 <= 'F') ? c0 - 'A' + 10 + : -1; + int lo = (c1 >= '0' && c1 <= '9') ? c1 - '0' + : (c1 >= 'a' && c1 <= 'f') ? c1 - 'a' + 10 + : (c1 >= 'A' && c1 <= 'F') ? c1 - 'A' + 10 + : -1; + if (hi < 0 || lo < 0) + return -1; + out[i] = (uint8_t)((hi << 4) | lo); + } + return (int)bl; +} + +/* One measured repetition. */ +typedef struct { + uint64_t wall_ns; + int solutions; + uint64_t compile_ns; + uint64_t attempts; + uint32_t achieved_effort; + const char *verify_result; /* NULL unless a verify op */ +} run_t; + +static const char *verify_result_str(equix_result r) { + switch (r) { + case EQUIX_OK: return "OK"; + case EQUIX_CHALLENGE: return "CHALLENGE"; + case EQUIX_ORDER: return "ORDER"; + case EQUIX_PARTIAL_SUM: return "PARTIAL_SUM"; + case EQUIX_FINAL_SUM: return "FINAL_SUM"; + default: return "UNKNOWN"; + } +} + +/* Emit a failure result JSON and exit. */ +static void fail(const char *op, const char *runtime_req, const char *msg) { + printf("{\"schema_version\":1,\"ok\":false," + "\"impl\":{\"name\":\"equix-c\",\"version\":\"%s\",\"commit\":\"%s\"," + "\"runtime_effective\":null}," + "\"operation\":\"%s\",\"runtime_requested\":\"%s\"," + "\"runtime_effective\":null," + "\"env\":{\"os\":\"%s\",\"compiler\":\"%s\",\"cpu\":\"%s\"," + "\"arch\":\"%s\",\"device\":\"cpu\",\"os_version\":\"%s\"}," + "\"runs\":[],\"peak_rss_kb\":%ld,\"error\":\"%s\"}\n", + EQUIX_C_VERSION, EQUIX_C_COMMIT, op ? op : "", runtime_req ? runtime_req : "", + OS_STR, COMPILER_STR, cpu_model(), ARCH_STR, os_version(), peak_rss_kb(), msg); + exit(1); +} + +/* solutions_hex_json: pre-formatted JSON array (e.g. ["aabb..",".."]) or NULL. */ +static void emit_ex(const char *op, const char *runtime_req, + const char *runtime_eff, const run_t *runs, size_t nruns, + const char *solutions_hex_json, + const char *winning_nonce_hex); + +static void emit(const char *op, const char *runtime_req, + const char *runtime_eff, const run_t *runs, size_t nruns, + const char *solutions_hex_json) { + emit_ex(op, runtime_req, runtime_eff, runs, nruns, solutions_hex_json, NULL); +} + +/* Like emit, plus an optional winning_nonce_hex field (effort op: the wire + * bytes of the winning token's nonce, so the harness can measure sizes). */ +static void emit_ex(const char *op, const char *runtime_req, + const char *runtime_eff, const run_t *runs, size_t nruns, + const char *solutions_hex_json, + const char *winning_nonce_hex) { + printf("{\"schema_version\":1,\"ok\":true," + "\"impl\":{\"name\":\"equix-c\",\"version\":\"%s\",\"commit\":\"%s\"," + "\"runtime_effective\":\"%s\"}," + "\"operation\":\"%s\",\"runtime_requested\":\"%s\"," + "\"runtime_effective\":\"%s\"," + "\"env\":{\"os\":\"%s\",\"compiler\":\"%s\",\"cpu\":\"%s\"," + "\"arch\":\"%s\",\"device\":\"cpu\",\"os_version\":\"%s\"}," + "\"runs\":[", + EQUIX_C_VERSION, EQUIX_C_COMMIT, runtime_eff, op, runtime_req, + runtime_eff, OS_STR, COMPILER_STR, cpu_model(), ARCH_STR, os_version()); + for (size_t i = 0; i < nruns; i++) { + const run_t *r = &runs[i]; + printf("%s{\"index\":%zu,\"wall_ns\":%llu,\"solutions\":%d," + "\"compile_ns\":%llu,\"attempts\":%llu,\"achieved_effort\":%u," + "\"verify_result\":", + i ? "," : "", i, (unsigned long long)r->wall_ns, r->solutions, + (unsigned long long)r->compile_ns, + (unsigned long long)r->attempts, r->achieved_effort); + if (r->verify_result) + printf("\"%s\"}", r->verify_result); + else + printf("null}"); + } + printf("],\"solutions_hex\":%s,", + solutions_hex_json ? solutions_hex_json : "null"); + if (winning_nonce_hex) + printf("\"winning_nonce_hex\":\"%s\",", winning_nonce_hex); + printf("\"peak_rss_kb\":%ld,\"error\":null}\n", peak_rss_kb()); +} + +/* Format a solution as 32 lowercase hex chars (16 bytes little-endian). */ +static void solution_to_hex(const equix_solution *sol, char out[33]) { + static const char hx[] = "0123456789abcdef"; + uint8_t sb[16]; + effort_solution_bytes(sol, sb); + for (int i = 0; i < 16; i++) { + out[2 * i] = hx[sb[i] >> 4]; + out[2 * i + 1] = hx[sb[i] & 0xf]; + } + out[32] = '\0'; +} + +/* ------------------------------------------------------- equix ctx creation */ + +/* Allocate an equix context honoring the requested runtime, reporting the + * effective runtime. base_flag is EQUIX_CTX_SOLVE or EQUIX_CTX_VERIFY. + * Returns NULL and sets *err on hard failure. */ +static equix_ctx *alloc_ctx(int base_flag, const char *runtime, + const char **eff, const char **err) { + equix_ctx *ctx; + if (strcmp(runtime, "interpret") == 0) { + ctx = equix_alloc((equix_ctx_flags)base_flag); + *eff = "interpreted"; + } else if (strcmp(runtime, "must-compile") == 0) { + if (!JIT_SUPPORTED) { + *err = "must-compile requested but JIT compiler not supported on this platform"; + return NULL; + } + ctx = equix_alloc((equix_ctx_flags)(base_flag | EQUIX_CTX_COMPILE)); + if (ctx == EQUIX_NOTSUPP) { + *err = "must-compile requested but JIT compiler not supported"; + return NULL; + } + *eff = "compiled"; + } else { /* try-compile (default) */ + ctx = JIT_SUPPORTED + ? equix_alloc((equix_ctx_flags)(base_flag | EQUIX_CTX_COMPILE)) + : EQUIX_NOTSUPP; + if (ctx == EQUIX_NOTSUPP) { + ctx = equix_alloc((equix_ctx_flags)base_flag); + *eff = "interpreted (fallback)"; + } else { + *eff = "compiled"; + } + } + if (ctx == NULL || ctx == EQUIX_NOTSUPP) { + *err = "equix_alloc failed"; + return NULL; + } + return ctx; +} + +/* --------------------------------------------------------------- operations */ + +/* Read either a fixed challenge (challenge_hex) or a seed (challenge_seed_hex). + * In seed mode each rep hashes the current challenge to get the next (a SHA-256 + * chain), so measurements span many challenges; that derivation is done OUTSIDE + * every timed region. Returns 1 for seed mode, 0 for fixed, and fills the first + * challenge into `chal`/`clen`. Fails the op if neither field is present. */ +static int read_challenge_or_seed(const char *json, const char *op, + const char *runtime, uint8_t chal[MAX_CHALLENGE], + int *clen) { + char hex[2 * MAX_CHALLENGE + 1] = {0}; + if (jm_get_str(json, "challenge_seed_hex", hex, sizeof hex)) { + uint8_t seed[MAX_CHALLENGE]; + int slen = hex_decode(hex, seed, sizeof seed); + if (slen < 0) + fail(op, runtime, "invalid challenge_seed_hex"); + sha256(seed, (size_t)slen, chal); /* challenge for iteration 0 */ + *clen = 32; + return 1; + } + if (!jm_get_str(json, "challenge_hex", hex, sizeof hex)) + fail(op, runtime, "requires challenge_hex or challenge_seed_hex"); + *clen = hex_decode(hex, chal, MAX_CHALLENGE); + if (*clen < 0) + fail(op, runtime, "invalid challenge_hex"); + return 0; +} + +static void op_solve(const char *json, const char *runtime, uint64_t reps, + uint64_t warmup) { + uint8_t chal[MAX_CHALLENGE]; + int clen; + int seeded = read_challenge_or_seed(json, "solve", runtime, chal, &clen); + + const char *eff = "?", *err = NULL; + equix_ctx *ctx = alloc_ctx(EQUIX_CTX_SOLVE, runtime, &eff, &err); + if (!ctx) + fail("solve", runtime, err); + + equix_solution sols[EQUIX_MAX_SOLS]; + for (uint64_t w = 0; w < warmup; w++) { + (void)equix_solve(ctx, chal, clen, sols); + if (seeded) + sha256(chal, 32, chal); /* advance the chain (untimed) */ + } + + run_t *runs = calloc(reps ? reps : 1, sizeof(run_t)); + int last_n = 0; + for (uint64_t i = 0; i < reps; i++) { + uint64_t t0 = now_ns(); + int n = equix_solve(ctx, chal, clen, sols); + uint64_t t1 = now_ns(); + if (seeded) + sha256(chal, 32, chal); /* derive next challenge AFTER stopping the timer */ + runs[i].wall_ns = t1 - t0; + runs[i].solutions = n; + last_n = n; + } + /* Publish the solutions found in the final rep for cross-implementation + * verification. Each solution = 32 hex chars; array fits comfortably. */ + char shex[EQUIX_MAX_SOLS * 40 + 4]; + size_t off = 0; + off += (size_t)snprintf(shex + off, sizeof shex - off, "["); + for (int s = 0; s < last_n; s++) { + char h[33]; + solution_to_hex(&sols[s], h); + off += (size_t)snprintf(shex + off, sizeof shex - off, "%s\"%s\"", + s ? "," : "", h); + } + snprintf(shex + off, sizeof shex - off, "]"); + emit("solve", runtime, eff, runs, reps, shex); + free(runs); + equix_free(ctx); +} + +/* Seed mode, two-phase so the timed region contains ONLY equix_verify: + * phase 1 (untimed): walk the SHA-256 chain, self-solving each challenge to + * collect (challenge, solution) pairs — the setup solve, which touches the + * ~1.8 MB solver table, is kept out of timing so it cannot pollute the cache + * the tiny verify reads from; + * phase 2 (timed): verify the collected pairs back-to-back. + * Uses a SOLVE context (it can verify too). Solution-less challenges are skipped. */ +static void op_verify_seeded(const char *json, const char *runtime, uint64_t reps, + uint64_t warmup) { + uint8_t chal[MAX_CHALLENGE]; + int clen; + (void)read_challenge_or_seed(json, "verify", runtime, chal, &clen); + + const char *eff = "?", *err = NULL; + equix_ctx *ctx = alloc_ctx(EQUIX_CTX_SOLVE, runtime, &eff, &err); + if (!ctx) + fail("verify", runtime, err); + + uint64_t want = warmup + reps; + uint8_t *chals = malloc(want * 32); + equix_solution *toks = malloc(want * sizeof(equix_solution)); + equix_solution sols[EQUIX_MAX_SOLS]; + + /* Phase 1 — collect `want` valid (challenge, solution) pairs, untimed. + * Cap draws so solution-less challenges cannot loop forever. */ + uint64_t got = 0, guard = 0, guard_max = want * 8 + 128; + while (got < want && guard++ < guard_max) { + if (equix_solve(ctx, chal, clen, sols) > 0) { + memcpy(chals + got * 32, chal, 32); + toks[got] = sols[0]; + got++; + } + sha256(chal, 32, chal); /* advance the chain (untimed) */ + } + + uint64_t warm = warmup < got ? warmup : got; + uint64_t timed = got - warm; + for (uint64_t w = 0; w < warm; w++) + (void)equix_verify(ctx, chals + w * 32, 32, &toks[w]); + + run_t *runs = calloc(timed ? timed : 1, sizeof(run_t)); + for (uint64_t i = 0; i < timed; i++) { + uint64_t idx = warm + i; + uint64_t t0 = now_ns(); + equix_result r = equix_verify(ctx, chals + idx * 32, 32, &toks[idx]); + uint64_t t1 = now_ns(); + runs[i].wall_ns = t1 - t0; + runs[i].solutions = (r == EQUIX_OK) ? 1 : 0; + runs[i].verify_result = verify_result_str(r); + } + emit("verify", runtime, eff, runs, timed, NULL); + free(runs); + free(chals); + free(toks); + equix_free(ctx); +} + +static void op_verify(const char *json, const char *runtime, uint64_t reps, + uint64_t warmup) { + char seed_probe[4] = {0}; + if (jm_get_str(json, "challenge_seed_hex", seed_probe, sizeof seed_probe)) { + op_verify_seeded(json, runtime, reps, warmup); + return; + } + + char chal_hex[2 * MAX_CHALLENGE + 1] = {0}; + char sol_hex[64] = {0}; + if (!jm_get_str(json, "challenge_hex", chal_hex, sizeof chal_hex)) + fail("verify", runtime, "verify requires challenge_hex"); + if (!jm_get_str(json, "solution_hex", sol_hex, sizeof sol_hex)) + fail("verify", runtime, "verify requires solution_hex"); + uint8_t chal[MAX_CHALLENGE]; + int clen = hex_decode(chal_hex, chal, sizeof chal); + if (clen < 0) + fail("verify", runtime, "invalid challenge_hex"); + uint8_t sb[16]; + if (hex_decode(sol_hex, sb, sizeof sb) != 16) + fail("verify", runtime, "solution_hex must be 16 bytes"); + equix_solution sol; + for (int i = 0; i < EQUIX_NUM_IDX; i++) + sol.idx[i] = (equix_idx)(sb[2 * i] | ((uint16_t)sb[2 * i + 1] << 8)); + + const char *eff = "?", *err = NULL; + equix_ctx *ctx = alloc_ctx(EQUIX_CTX_VERIFY, runtime, &eff, &err); + if (!ctx) + fail("verify", runtime, err); + + for (uint64_t w = 0; w < warmup; w++) + (void)equix_verify(ctx, chal, clen, &sol); + + run_t *runs = calloc(reps ? reps : 1, sizeof(run_t)); + for (uint64_t i = 0; i < reps; i++) { + uint64_t t0 = now_ns(); + equix_result r = equix_verify(ctx, chal, clen, &sol); + uint64_t t1 = now_ns(); + runs[i].wall_ns = t1 - t0; + runs[i].solutions = (r == EQUIX_OK) ? 1 : 0; + runs[i].verify_result = verify_result_str(r); + } + emit("verify", runtime, eff, runs, reps, NULL); + free(runs); + equix_free(ctx); +} + +/* Build challenge = base || little-endian(nonce, nonce_bytes) into buf. */ +static int build_nonce_challenge(const uint8_t *base, int base_len, + uint64_t nonce, int nonce_bytes, uint8_t *buf) { + memcpy(buf, base, base_len); + for (int i = 0; i < nonce_bytes; i++) + buf[base_len + i] = (uint8_t)((nonce >> (8 * i)) & 0xff); + return base_len + nonce_bytes; +} + +static void op_effort(const char *json, const char *runtime, uint64_t reps, + uint64_t warmup) { + char base_hex[2 * MAX_CHALLENGE + 1] = {0}; + if (!jm_get_str(json, "challenge_base_hex", base_hex, sizeof base_hex)) + fail("effort", runtime, "effort requires challenge_base_hex"); + uint8_t base[MAX_CHALLENGE]; + int base_len = hex_decode(base_hex, base, sizeof base); + if (base_len < 0) + fail("effort", runtime, "invalid challenge_base_hex"); + + uint64_t nonce_bytes = 8, nonce_start = 0, target = 1000, max_attempts = 5000000; + jm_get_u64(json, "nonce_bytes", &nonce_bytes); + jm_get_u64(json, "nonce_start", &nonce_start); + jm_get_u64(json, "target_effort", &target); + jm_get_u64(json, "max_attempts", &max_attempts); + if (nonce_bytes > 8 || (size_t)base_len + nonce_bytes > MAX_CHALLENGE) + fail("effort", runtime, "nonce_bytes out of range"); + + const char *eff = "?", *err = NULL; + equix_ctx *ctx = alloc_ctx(EQUIX_CTX_SOLVE, runtime, &eff, &err); + if (!ctx) + fail("effort", runtime, err); + + equix_solution sols[EQUIX_MAX_SOLS]; + uint8_t chal[MAX_CHALLENGE]; + + /* One search = one repetition; warmups run a full search but are discarded. */ + for (uint64_t w = 0; w < warmup; w++) { + uint64_t nonce = nonce_start; + for (uint64_t a = 0; a < max_attempts; a++, nonce++) { + int clen = build_nonce_challenge(base, base_len, nonce, nonce_bytes, chal); + int n = equix_solve(ctx, chal, clen, sols); + int done = 0; + for (int s = 0; s < n; s++) + if (effort_of(chal, clen, &sols[s]) >= target) { done = 1; break; } + if (done) break; + } + } + + run_t *runs = calloc(reps ? reps : 1, sizeof(run_t)); + /* The winning token's wire bytes (nonce LE + 16-byte solution): reported so + * the harness can measure message sizes vs difficulty. */ + equix_solution win_sol; + uint8_t win_nonce[8]; + int have_token = 0; + for (uint64_t i = 0; i < reps; i++) { + uint64_t nonce = nonce_start; + uint64_t attempts = 0; + uint32_t best = 0; + uint64_t t0 = now_ns(); + for (uint64_t a = 0; a < max_attempts; a++, nonce++) { + int clen = build_nonce_challenge(base, base_len, nonce, nonce_bytes, chal); + int n = equix_solve(ctx, chal, clen, sols); + attempts++; + int done = 0; + for (int s = 0; s < n; s++) { + uint32_t e = effort_of(chal, clen, &sols[s]); + if (e > best) best = e; + if (e >= target) { + if (!done) { + win_sol = sols[s]; + memcpy(win_nonce, chal + base_len, nonce_bytes); + have_token = 1; + } + done = 1; + } + } + if (done) break; + } + uint64_t t1 = now_ns(); + runs[i].wall_ns = t1 - t0; + runs[i].attempts = attempts; + runs[i].achieved_effort = best; + runs[i].solutions = (best >= target) ? 1 : 0; + } + if (have_token) { + static const char hx[] = "0123456789abcdef"; + char shex[33], sjson[40], nhex[17]; + solution_to_hex(&win_sol, shex); + snprintf(sjson, sizeof sjson, "[\"%s\"]", shex); + for (uint64_t b = 0; b < nonce_bytes; b++) { + nhex[2 * b] = hx[win_nonce[b] >> 4]; + nhex[2 * b + 1] = hx[win_nonce[b] & 0xf]; + } + nhex[2 * nonce_bytes] = '\0'; + emit_ex("effort", runtime, eff, runs, reps, sjson, nhex); + } else { + emit("effort", runtime, eff, runs, reps, NULL); + } + free(runs); + equix_free(ctx); +} + +/* Isolate program generation+compile (hashx_make) from execution (hashx_exec) + * using the hashx API directly -- libequix's public API cannot separate them. + * Each rep uses a distinct seed (base || LE(nonce_start+i)) so we sample the + * compile-time distribution across different generated programs. */ +static void op_hashx_compile(const char *json, const char *runtime, + uint64_t reps, uint64_t warmup) { + char base_hex[2 * MAX_CHALLENGE + 1] = {0}; + if (!jm_get_str(json, "challenge_base_hex", base_hex, sizeof base_hex) && + !jm_get_str(json, "challenge_hex", base_hex, sizeof base_hex)) + fail("hashx_compile", runtime, "hashx_compile requires a challenge"); + uint8_t base[MAX_CHALLENGE]; + int base_len = hex_decode(base_hex, base, sizeof base); + if (base_len < 0) + fail("hashx_compile", runtime, "invalid challenge"); + uint64_t nonce_start = 0; + jm_get_u64(json, "nonce_start", &nonce_start); + + hashx_type type; + const char *eff; + if (strcmp(runtime, "interpret") == 0) { + type = HASHX_INTERPRETED; + eff = "interpreted"; + } else if (!JIT_SUPPORTED) { + /* Avoid the Apple Silicon JIT crash: force interpreter / clean error. */ + if (strcmp(runtime, "must-compile") == 0) + fail("hashx_compile", runtime, + "must-compile requested but JIT not supported on this platform"); + type = HASHX_INTERPRETED; + eff = "interpreted (fallback)"; + } else { + type = HASHX_COMPILED; + eff = "compiled"; + } + hashx_ctx *hx = hashx_alloc(type); + if (hx == HASHX_NOTSUPP) { + if (strcmp(runtime, "must-compile") == 0) + fail("hashx_compile", runtime, "compiler not supported"); + hx = hashx_alloc(HASHX_INTERPRETED); + eff = "interpreted (fallback)"; + } + if (hx == NULL) + fail("hashx_compile", runtime, "hashx_alloc failed"); + + uint8_t seed[MAX_CHALLENGE + 8]; + uint8_t hout[HASHX_SIZE]; + + for (uint64_t w = 0; w < warmup; w++) { + int sl = build_nonce_challenge(base, base_len, nonce_start, 8, seed); + if (hashx_make(hx, seed, sl)) + hashx_exec(hx, 0, hout); + } + + run_t *runs = calloc(reps ? reps : 1, sizeof(run_t)); + for (uint64_t i = 0; i < reps; i++) { + int sl = build_nonce_challenge(base, base_len, nonce_start + i, 8, seed); + uint64_t t0 = now_ns(); + int made = hashx_make(hx, seed, sl); + uint64_t t1 = now_ns(); + runs[i].compile_ns = t1 - t0; + if (made) { + /* Time a single hashx_exec as the per-hash execution cost. */ + uint64_t e0 = now_ns(); + hashx_exec(hx, 0, hout); + uint64_t e1 = now_ns(); + runs[i].wall_ns = e1 - e0; + runs[i].solutions = 1; /* program generated successfully */ + } else { + runs[i].wall_ns = 0; + runs[i].solutions = 0; /* rare invalid seed */ + } + } + emit("hashx_compile", runtime, eff, runs, reps, NULL); + free(runs); + hashx_free(hx); +} + +int main(void) { + char *json = read_all_stdin(); + if (!json) + fail(NULL, NULL, "failed to read stdin"); + + char op[32] = "solve"; + char runtime[32] = "try-compile"; + jm_get_str(json, "operation", op, sizeof op); + jm_get_str(json, "runtime", runtime, sizeof runtime); + + uint64_t reps = 10, warmup = 3; + jm_get_u64(json, "repetitions", &reps); + jm_get_u64(json, "warmup", &warmup); + if (reps == 0) + reps = 1; + + if (strcmp(op, "solve") == 0) + op_solve(json, runtime, reps, warmup); + else if (strcmp(op, "verify") == 0) + op_verify(json, runtime, reps, warmup); + else if (strcmp(op, "effort") == 0) + op_effort(json, runtime, reps, warmup); + else if (strcmp(op, "hashx_compile") == 0) + op_hashx_compile(json, runtime, reps, warmup); + else + fail(op, runtime, "unknown operation"); + + free(json); + return 0; +} diff --git a/tools/benchmarks/Equi-X/runners/c/json_min.c b/tools/benchmarks/Equi-X/runners/c/json_min.c new file mode 100644 index 0000000..1eb699a --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/c/json_min.c @@ -0,0 +1,88 @@ +#include "json_min.h" + +#include +#include +#include +#include + +/* Return a pointer to the first non-space char of the value for `key`, or NULL + * if the key is absent / malformed. */ +static const char *find_val(const char *json, const char *key) { + char pat[128]; + int n = snprintf(pat, sizeof pat, "\"%s\"", key); + if (n <= 0 || (size_t)n >= sizeof pat) + return NULL; + const char *p = strstr(json, pat); + if (!p) + return NULL; + p += (size_t)n; + while (*p && *p != ':') + p++; + if (*p != ':') + return NULL; + p++; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') + p++; + return p; +} + +int jm_get_str(const char *json, const char *key, char *out, size_t outsz) { + const char *p = find_val(json, key); + if (!p || *p != '"') + return 0; /* absent or null/non-string */ + p++; + size_t i = 0; + while (*p && *p != '"') { + char c = *p++; + if (c == '\\' && *p) { + char e = *p++; + switch (e) { + case 'n': c = '\n'; break; + case 't': c = '\t'; break; + case 'r': c = '\r'; break; + case '"': c = '"'; break; + case '\\': c = '\\'; break; + case '/': c = '/'; break; + default: c = e; break; + } + } + if (i + 1 < outsz) + out[i++] = c; + } + if (i < outsz) + out[i] = '\0'; + else if (outsz) + out[outsz - 1] = '\0'; + return 1; +} + +int jm_get_u64(const char *json, const char *key, uint64_t *out) { + const char *p = find_val(json, key); + if (!p || (!isdigit((unsigned char)*p) && *p != '+')) + return 0; + *out = strtoull(p, NULL, 10); + return 1; +} + +int jm_get_i64(const char *json, const char *key, int64_t *out) { + const char *p = find_val(json, key); + if (!p || (!isdigit((unsigned char)*p) && *p != '-' && *p != '+')) + return 0; + *out = strtoll(p, NULL, 10); + return 1; +} + +int jm_get_bool(const char *json, const char *key, int *out) { + const char *p = find_val(json, key); + if (!p) + return 0; + if (strncmp(p, "true", 4) == 0) { + *out = 1; + return 1; + } + if (strncmp(p, "false", 5) == 0) { + *out = 0; + return 1; + } + return 0; +} diff --git a/tools/benchmarks/Equi-X/runners/c/json_min.h b/tools/benchmarks/Equi-X/runners/c/json_min.h new file mode 100644 index 0000000..6b94497 --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/c/json_min.h @@ -0,0 +1,23 @@ +/* Minimal JSON reader for the Equi-X runner protocol. + * + * The job-spec is a FLAT object of scalar fields (no nested objects/arrays on + * the input side), so a tiny key->value scanner is sufficient and avoids any + * external dependency. Result JSON is emitted by hand in equix_runner.c. + * + * Each getter searches for the exact quoted key ("key") which makes prefix + * collisions impossible (e.g. searching "nonce" never matches "nonce_bytes", + * because the char after the opening key is '_' not '"'). + */ +#ifndef EQUIX_RUNNER_JSON_MIN_H +#define EQUIX_RUNNER_JSON_MIN_H + +#include +#include + +/* All return 1 if the key was found (and, for typed getters, parsed), else 0. */ +int jm_get_str(const char *json, const char *key, char *out, size_t outsz); +int jm_get_u64(const char *json, const char *key, uint64_t *out); +int jm_get_i64(const char *json, const char *key, int64_t *out); +int jm_get_bool(const char *json, const char *key, int *out); + +#endif /* EQUIX_RUNNER_JSON_MIN_H */ diff --git a/tools/benchmarks/Equi-X/runners/c/sha256.c b/tools/benchmarks/Equi-X/runners/c/sha256.c new file mode 100644 index 0000000..e6ca89e --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/c/sha256.c @@ -0,0 +1,75 @@ +/* Minimal standalone SHA-256 (FIPS 180-4). Verified against Python hashlib in + * the test suite. Off the hot path (challenge generation only). */ +#include "sha256.h" + +#include + +static uint32_t rotr(uint32_t x, uint32_t n) { return (x >> n) | (x << (32 - n)); } + +static const uint32_t K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, + 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, + 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, + 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +static void sha256_block(uint32_t state[8], const uint8_t block[64]) { + uint32_t w[64]; + for (int i = 0; i < 16; i++) + w[i] = ((uint32_t)block[i * 4] << 24) | ((uint32_t)block[i * 4 + 1] << 16) | + ((uint32_t)block[i * 4 + 2] << 8) | (uint32_t)block[i * 4 + 3]; + for (int i = 16; i < 64; i++) { + uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = state[0], b = state[1], c = state[2], d = state[3]; + uint32_t e = state[4], f = state[5], g = state[6], h = state[7]; + for (int i = 0; i < 64; i++) { + uint32_t S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + uint32_t ch = (e & f) ^ (~e & g); + uint32_t t1 = h + S1 + ch + K[i] + w[i]; + uint32_t S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + uint32_t t2 = S0 + maj; + h = g; g = f; f = e; e = d + t1; + d = c; c = b; b = a; a = t1 + t2; + } + state[0] += a; state[1] += b; state[2] += c; state[3] += d; + state[4] += e; state[5] += f; state[6] += g; state[7] += h; +} + +void sha256(const uint8_t *data, size_t len, uint8_t out[32]) { + uint32_t state[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + size_t full = len / 64; + for (size_t i = 0; i < full; i++) + sha256_block(state, data + i * 64); + + /* Final block(s): remaining bytes, 0x80, zero pad, 64-bit big-endian length. */ + uint8_t buf[128]; + size_t rem = len - full * 64; + memset(buf, 0, sizeof buf); + memcpy(buf, data + full * 64, rem); + buf[rem] = 0x80; + size_t padlen = (rem < 56) ? 64 : 128; + uint64_t bitlen = (uint64_t)len * 8; + for (int i = 0; i < 8; i++) + buf[padlen - 1 - i] = (uint8_t)(bitlen >> (8 * i)); + sha256_block(state, buf); + if (padlen == 128) + sha256_block(state, buf + 64); + + for (int i = 0; i < 8; i++) { + out[i * 4] = (uint8_t)(state[i] >> 24); + out[i * 4 + 1] = (uint8_t)(state[i] >> 16); + out[i * 4 + 2] = (uint8_t)(state[i] >> 8); + out[i * 4 + 3] = (uint8_t)(state[i]); + } +} diff --git a/tools/benchmarks/Equi-X/runners/c/sha256.h b/tools/benchmarks/Equi-X/runners/c/sha256.h new file mode 100644 index 0000000..2aab938 --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/c/sha256.h @@ -0,0 +1,13 @@ +/* Minimal standalone SHA-256 (FIPS 180-4) for deriving per-rep challenges from + * a seed. Not on any hot path — used only to generate challenges between timed + * measurements, so plain portable C is fine. */ +#ifndef EQUIX_RUNNER_SHA256_H +#define EQUIX_RUNNER_SHA256_H + +#include +#include + +/* One-shot: hash `len` bytes of `data` into the 32-byte `out`. */ +void sha256(const uint8_t *data, size_t len, uint8_t out[32]); + +#endif /* EQUIX_RUNNER_SHA256_H */ diff --git a/tools/benchmarks/Equi-X/runners/c/timing.h b/tools/benchmarks/Equi-X/runners/c/timing.h new file mode 100644 index 0000000..e7ed13c --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/c/timing.h @@ -0,0 +1,35 @@ +/* Monotonic timing + peak-RSS helpers for the Equi-X C runner. */ +#ifndef EQUIX_RUNNER_TIMING_H +#define EQUIX_RUNNER_TIMING_H + +#include +#include +#include + +/* Monotonic wall-clock nanoseconds (immune to wall-clock adjustments). + * macOS: clock_gettime(CLOCK_MONOTONIC) only has microsecond granularity, which + * quantizes ~16 us operations (verify) by +/-3-6%; CLOCK_UPTIME_RAW via + * clock_gettime_nsec_np gives true nanosecond resolution. */ +static inline uint64_t now_ns(void) { +#if defined(__APPLE__) + return clock_gettime_nsec_np(CLOCK_UPTIME_RAW); +#else + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; +#endif +} + +/* Whole-process peak resident set size, in KILOBYTES. + * ru_maxrss units differ by OS: Linux reports kilobytes, macOS/BSD report bytes. */ +static inline long peak_rss_kb(void) { + struct rusage ru; + getrusage(RUSAGE_SELF, &ru); +#if defined(__APPLE__) + return ru.ru_maxrss / 1024; /* macOS: bytes -> KB */ +#else + return ru.ru_maxrss; /* Linux: already KB */ +#endif +} + +#endif /* EQUIX_RUNNER_TIMING_H */ diff --git a/tools/benchmarks/Equi-X/runners/rust/Cargo.lock b/tools/benchmarks/Equi-X/runners/rust/Cargo.lock new file mode 100644 index 0000000..89ce5f1 --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/rust/Cargo.lock @@ -0,0 +1,391 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "dynasm" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd358e74d2f8d71a11b7a2c51c67ad5df3de06003b0596875e47bc09126c894e" +dependencies = [ + "bitflags", + "byteorder", + "lazy_static", + "proc-macro-error3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dynasmrt" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6efe03f6bd356ae85eeea7ee14c2bf54e664a949d089d238364b5a99afbc4116" +dependencies = [ + "byteorder", + "dynasm", + "fnv", + "memmap2", +] + +[[package]] +name = "equix" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "207dd01b4071086d80c21d3b39f029b5ebb658f9910c0348c465ca1dd5346d90" +dependencies = [ + "arrayvec", + "hashx", + "num-traits", + "thiserror", + "visibility", +] + +[[package]] +name = "equix-runner" +version = "0.1.0" +dependencies = [ + "blake2", + "equix", + "hashx", + "libc", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "fixed-capacity-vec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b31a14f5ee08ed1a40e1252b35af18bed062e3f39b69aab34decde36bc43e40" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "hashx" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc106a3b059d84aad44624a2b3b10e173584249e413596a9b6ed4a4a2542e41" +dependencies = [ + "arrayvec", + "blake2", + "dynasmrt", + "fixed-capacity-vec", + "hex", + "rand_core", + "thiserror", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "proc-macro-error-attr3" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be5bfc63c4dc85083c9daaf7112d0261701d4058677c3bff7f2afc44e30ef3e1" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error3" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0d42490f6b7b143eef32b9e3522e42bf25dadc02c69ed72236f80adb949b5c" +dependencies = [ + "proc-macro-error-attr3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/benchmarks/Equi-X/runners/rust/Cargo.toml b/tools/benchmarks/Equi-X/runners/rust/Cargo.toml new file mode 100644 index 0000000..e111528 --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/rust/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "equix-runner" +version = "0.1.0" +edition = "2021" +description = "Equi-X benchmark runner (Rust / arti equix crate) speaking the JSON-over-stdio protocol" +license = "MIT" + +[[bin]] +name = "equix_runner" +path = "src/main.rs" + +# Exact version pins + committed Cargo.lock + `--locked` freeze the whole tree. +[dependencies] +equix = "=0.7.0" +hashx = "=0.9.0" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +blake2 = "0.10" +sha2 = "0.10" + +# macOS has no /proc; getrusage (via libc) provides peak RSS there. +[target.'cfg(target_os = "macos")'.dependencies] +libc = "0.2" + +[profile.release] +opt-level = 3 diff --git a/tools/benchmarks/Equi-X/runners/rust/src/effort.rs b/tools/benchmarks/Equi-X/runners/rust/src/effort.rs new file mode 100644 index 0000000..80bdc72 --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/rust/src/effort.rs @@ -0,0 +1,25 @@ +//! Tor proposal-327 style effort computation. +//! +//! MUST be byte-identical to the C runner (runners/c/effort.c): standard +//! BLAKE2b-256 over `challenge || solution_bytes`, first 32 bits big-endian as +//! `hash32`, achieved effort = floor((2^32-1) / hash32). The Python cross-check +//! asserts both runners agree on a fixed (challenge, solution). + +use blake2::digest::consts::U32; +use blake2::{Blake2b, Digest}; + +type Blake2b256 = Blake2b; + +/// Achieved effort of `solution_bytes` (16-byte packed form) for `challenge`. +pub fn effort_of(challenge: &[u8], solution_bytes: &[u8]) -> u32 { + let mut h = Blake2b256::new(); + h.update(challenge); + h.update(solution_bytes); + let out = h.finalize(); + let hash32 = u32::from_be_bytes([out[0], out[1], out[2], out[3]]); + if hash32 == 0 { + u32::MAX + } else { + u32::MAX / hash32 + } +} diff --git a/tools/benchmarks/Equi-X/runners/rust/src/job.rs b/tools/benchmarks/Equi-X/runners/rust/src/job.rs new file mode 100644 index 0000000..221726d --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/rust/src/job.rs @@ -0,0 +1,83 @@ +//! Protocol structs: the job-spec read from stdin and the result written to +//! stdout. Mirrors runners/c/equix_runner.c and adapters/README.md. + +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize)] +pub struct Job { + #[serde(default)] + pub operation: Option, + #[serde(default)] + pub runtime: Option, + #[serde(default)] + pub challenge_hex: Option, + /// When set, each rep derives a fresh challenge by SHA-256-chaining this + /// seed (challenge generation is excluded from every timed region). + #[serde(default)] + pub challenge_seed_hex: Option, + #[serde(default)] + pub challenge_base_hex: Option, + #[serde(default)] + pub solution_hex: Option, + #[serde(default)] + pub nonce_bytes: Option, + #[serde(default)] + pub nonce_start: Option, + #[serde(default)] + pub target_effort: Option, + #[serde(default)] + pub max_attempts: Option, + #[serde(default)] + pub repetitions: Option, + #[serde(default)] + pub warmup: Option, +} + +#[derive(Serialize)] +pub struct RunOut { + pub index: usize, + pub wall_ns: u64, + pub solutions: i64, + pub compile_ns: u64, + pub attempts: u64, + pub achieved_effort: u32, + pub verify_result: Option, +} + +#[derive(Serialize)] +pub struct ImplInfo { + pub name: String, + pub version: String, + pub commit: String, + pub runtime_effective: Option, +} + +#[derive(Serialize)] +pub struct EnvInfo { + pub os: String, + pub compiler: String, + pub cpu: String, + pub arch: String, + pub device: String, + pub os_version: String, +} + +#[derive(Serialize)] +pub struct Output { + pub schema_version: u32, + pub ok: bool, + #[serde(rename = "impl")] + pub impl_info: ImplInfo, + pub operation: String, + pub runtime_requested: String, + pub runtime_effective: Option, + pub env: EnvInfo, + pub runs: Vec, + pub solutions_hex: Option>, + /// effort op only: the wire bytes of the winning token's nonce (LE, + /// exactly `nonce_bytes` long) — lets the harness measure message sizes. + #[serde(skip_serializing_if = "Option::is_none")] + pub winning_nonce_hex: Option, + pub peak_rss_kb: i64, + pub error: Option, +} diff --git a/tools/benchmarks/Equi-X/runners/rust/src/main.rs b/tools/benchmarks/Equi-X/runners/rust/src/main.rs new file mode 100644 index 0000000..d87b9db --- /dev/null +++ b/tools/benchmarks/Equi-X/runners/rust/src/main.rs @@ -0,0 +1,590 @@ +//! Equi-X Rust benchmark runner (arti `equix` + `hashx` crates). +//! +//! Reads one job-spec JSON on stdin, runs the requested operation, writes one +//! result JSON on stdout. Mirrors the C runner's protocol exactly so the Python +//! harness can compare them cell-for-cell. + +mod effort; +mod job; + +use std::io::Read; +use std::time::Instant; + +use equix::{EquiXBuilder, RuntimeOption, SolverMemory}; + +use job::{EnvInfo, ImplInfo, Job, Output, RunOut}; + +const MAX_CHALLENGE: usize = 256; + +#[cfg(target_os = "linux")] +fn peak_rss_kb() -> i64 { + std::fs::read_to_string("/proc/self/status") + .ok() + .and_then(|s| { + s.lines() + .find(|l| l.starts_with("VmHWM:")) + .and_then(|l| l.split_whitespace().nth(1)) + .and_then(|v| v.parse().ok()) + }) + .unwrap_or(-1) +} + +#[cfg(target_os = "macos")] +fn peak_rss_kb() -> i64 { + // macOS has no /proc; getrusage.ru_maxrss is in BYTES here (Linux uses KB). + unsafe { + let mut ru: libc::rusage = std::mem::zeroed(); + if libc::getrusage(libc::RUSAGE_SELF, &mut ru) == 0 { + (ru.ru_maxrss as i64) / 1024 + } else { + -1 + } + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn peak_rss_kb() -> i64 { + -1 +} + +fn hex_decode(s: &str) -> Option> { + if s.len() % 2 != 0 { + return None; + } + (0..s.len() / 2) + .map(|i| u8::from_str_radix(&s[2 * i..2 * i + 2], 16).ok()) + .collect() +} + +fn hex_encode(b: &[u8]) -> String { + let mut o = String::with_capacity(b.len() * 2); + for x in b { + o.push_str(&format!("{:02x}", x)); + } + o +} + +fn runtime_option(s: &str) -> RuntimeOption { + match s { + "interpret" => RuntimeOption::InterpretOnly, + "must-compile" => RuntimeOption::CompileOnly, + _ => RuntimeOption::TryCompile, + } +} + +fn eff_str(runtime_dbg: &str, requested: &str) -> String { + let compiled = runtime_dbg.to_lowercase().contains("compil"); + if compiled { + "compiled".to_string() + } else if requested == "try-compile" { + "interpreted (fallback)".to_string() + } else { + "interpreted".to_string() + } +} + +fn impl_info(runtime_effective: Option) -> ImplInfo { + ImplInfo { + name: "equix-rust".to_string(), + version: std::env::var("EQUIX_RUST_VERSION").unwrap_or_else(|_| "0.7.0".to_string()), + commit: std::env::var("EQUIX_RUST_COMMIT").unwrap_or_else(|_| "crate-0.7.0".to_string()), + runtime_effective, + } +} + +#[cfg(target_os = "linux")] +fn cpu_model() -> String { + // Priority order works across arches: "model name" (x86), "Model" (Raspberry + // Pi board), "Hardware" (older ARM), "cpu model" (others). + let text = std::fs::read_to_string("/proc/cpuinfo").unwrap_or_default(); + for field in ["model name", "Model", "Hardware", "cpu model"] { + for line in text.lines() { + if let Some((k, v)) = line.split_once(':') { + if k.trim() == field && !v.trim().is_empty() { + return v.trim().to_string(); + } + } + } + } + "unknown".to_string() +} + +#[cfg(not(target_os = "linux"))] +fn cpu_model() -> String { + // macOS (and other non-Linux): no /proc. `sysctl` reports the CPU brand on + // both Intel and Apple Silicon (e.g. "Apple M2"). + std::process::Command::new("sysctl") + .args(["-n", "machdep.cpu.brand_string"]) + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "unknown".to_string()) +} + +#[cfg(target_os = "linux")] +fn os_version() -> String { + std::fs::read_to_string("/proc/sys/kernel/osrelease") + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "unknown".to_string()) +} + +#[cfg(not(target_os = "linux"))] +fn os_version() -> String { + // macOS/BSD: kernel release via `uname -r` (e.g. Darwin "23.5.0"). + std::process::Command::new("uname") + .arg("-r") + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "unknown".to_string()) +} + +fn env_info() -> EnvInfo { + EnvInfo { + os: std::env::consts::OS.to_string(), // "linux", "macos", ... + compiler: std::env::var("EQUIX_RUST_RUSTC").unwrap_or_else(|_| "rustc".to_string()), + cpu: cpu_model(), + arch: std::env::consts::ARCH.to_string(), + device: "cpu".to_string(), + os_version: os_version(), + } +} + +fn fail(op: &str, req: &str, msg: &str) -> ! { + let out = Output { + schema_version: 1, + ok: false, + impl_info: impl_info(None), + operation: op.to_string(), + runtime_requested: req.to_string(), + runtime_effective: None, + env: env_info(), + runs: vec![], + solutions_hex: None, + winning_nonce_hex: None, + peak_rss_kb: peak_rss_kb(), + error: Some(msg.to_string()), + }; + println!("{}", serde_json::to_string(&out).unwrap()); + std::process::exit(1); +} + +fn emit( + op: &str, + req: &str, + eff: &str, + runs: Vec, + solutions_hex: Option>, +) { + emit_with_nonce(op, req, eff, runs, solutions_hex, None) +} + +fn emit_with_nonce( + op: &str, + req: &str, + eff: &str, + runs: Vec, + solutions_hex: Option>, + winning_nonce_hex: Option, +) { + let out = Output { + schema_version: 1, + ok: true, + impl_info: impl_info(Some(eff.to_string())), + operation: op.to_string(), + runtime_requested: req.to_string(), + runtime_effective: Some(eff.to_string()), + env: env_info(), + runs, + solutions_hex, + winning_nonce_hex, + peak_rss_kb: peak_rss_kb(), + error: None, + }; + println!("{}", serde_json::to_string(&out).unwrap()); +} + +/// Probe-build to discover the effective runtime (and surface hard failures). +fn effective_runtime( + builder: &EquiXBuilder, + challenge: &[u8], + req: &str, +) -> Result { + match builder.build(challenge) { + Ok(eq) => Ok(eff_str(&format!("{:?}", eq.runtime()), req)), + Err(e) => Err(format!("{:?}", e)), + } +} + +/// SHA-256 of `data`. Used only to derive per-rep challenges from a seed +/// (between timed measurements) — never on a timed path. +fn sha256(data: &[u8]) -> [u8; 32] { + use sha2::{Digest, Sha256}; + let mut h = Sha256::new(); + h.update(data); + h.finalize().into() +} + +fn build_nonce_challenge(base: &[u8], nonce: u64, nonce_bytes: usize) -> Vec { + let mut c = Vec::with_capacity(base.len() + nonce_bytes); + c.extend_from_slice(base); + for i in 0..nonce_bytes { + c.push(((nonce >> (8 * i)) & 0xff) as u8); + } + c +} + +/// First challenge for a solve/verify op: either the fixed `challenge_hex`, or +/// (seed mode) SHA-256(seed). Returns (challenge_bytes, seeded?). +fn first_challenge(job: &Job, op: &str, req: &str) -> (Vec, bool) { + if let Some(seed_hex) = job.challenge_seed_hex.as_ref() { + let seed = hex_decode(seed_hex) + .unwrap_or_else(|| fail(op, req, "invalid challenge_seed_hex")); + (sha256(&seed).to_vec(), true) + } else { + let chex = job + .challenge_hex + .as_ref() + .unwrap_or_else(|| fail(op, req, "requires challenge_hex or challenge_seed_hex")); + (hex_decode(chex).unwrap_or_else(|| fail(op, req, "invalid challenge_hex")), false) + } +} + +fn op_solve(job: &Job, req: &str, reps: u64, warmup: u64) { + let (mut challenge, seeded) = first_challenge(job, "solve", req); + + let mut builder = EquiXBuilder::new(); + builder.runtime(runtime_option(req)); + let eff = match effective_runtime(&builder, &challenge, req) { + Ok(e) => e, + Err(e) => fail("solve", req, &e), + }; + + let mut mem = SolverMemory::new(); + for _ in 0..warmup { + if let Ok(eq) = builder.build(&challenge) { + let _ = eq.solve_with_memory(&mut mem); + } + if seeded { + challenge = sha256(&challenge).to_vec(); // advance the chain (untimed) + } + } + + let mut runs = Vec::with_capacity(reps as usize); + let mut last_sols: Vec = vec![]; + for i in 0..reps { + let t0 = Instant::now(); + let arr = match builder.build(&challenge) { + Ok(eq) => eq.solve_with_memory(&mut mem), + Err(_) => Default::default(), + }; + let ns = t0.elapsed().as_nanos() as u64; + if seeded { + challenge = sha256(&challenge).to_vec(); // next challenge AFTER stopping the timer + } + if i + 1 == reps { + last_sols = arr.iter().map(|s| hex_encode(&s.to_bytes())).collect(); + } + runs.push(RunOut { + index: i as usize, + wall_ns: ns, + solutions: arr.len() as i64, + compile_ns: 0, + attempts: 0, + achieved_effort: 0, + verify_result: None, + }); + } + emit("solve", req, &eff, runs, Some(last_sols)); +} + +/// Seed mode, two-phase so the timed region contains ONLY verify_bytes: +/// phase 1 (untimed) walks the SHA-256 chain, self-solving each challenge to +/// collect (challenge, solution) pairs — keeping the ~1.8 MB solver pass out of +/// timing so it cannot pollute the cache the tiny verify reads from; phase 2 +/// (timed) verifies the collected pairs back-to-back. Solution-less skipped. +fn op_verify_seeded(job: &Job, req: &str, reps: u64, warmup: u64) { + let (mut challenge, _) = first_challenge(job, "verify", req); + let mut builder = EquiXBuilder::new(); + builder.runtime(runtime_option(req)); + let eff = match effective_runtime(&builder, &challenge, req) { + Ok(e) => e, + Err(e) => fail("verify", req, &e), + }; + + let mut mem = SolverMemory::new(); + let want = warmup + reps; + let guard_max = want * 8 + 128; + // Phase 1: collect valid (challenge, solution) pairs, untimed. + let mut pairs: Vec<(Vec, [u8; 16])> = Vec::with_capacity(want as usize); + let mut guard = 0u64; + while (pairs.len() as u64) < want && guard < guard_max { + let sol = match builder.build(&challenge) { + Ok(eq) => eq.solve_with_memory(&mut mem).iter().next().map(|s| s.to_bytes()), + Err(_) => None, + }; + if let Some(sb) = sol { + pairs.push((challenge.clone(), sb)); + } + challenge = sha256(&challenge).to_vec(); // advance (untimed) + guard += 1; + } + + let warm = (warmup as usize).min(pairs.len()); + for (c, sb) in &pairs[..warm] { + let _ = builder.verify_bytes(c, sb); + } + + let mut runs = Vec::with_capacity(pairs.len() - warm); + for (c, sb) in &pairs[warm..] { + let t0 = Instant::now(); + let r = builder.verify_bytes(c, sb); + let ns = t0.elapsed().as_nanos() as u64; + let (vr, ok) = match &r { + Ok(_) => ("OK".to_string(), 1), + Err(e) => (format!("{:?}", e), 0), + }; + runs.push(RunOut { + index: runs.len(), + wall_ns: ns, + solutions: ok, + compile_ns: 0, + attempts: 0, + achieved_effort: 0, + verify_result: Some(vr), + }); + } + emit("verify", req, &eff, runs, None); +} + +fn op_verify(job: &Job, req: &str, reps: u64, warmup: u64) { + if job.challenge_seed_hex.is_some() { + return op_verify_seeded(job, req, reps, warmup); + } + let chex = job + .challenge_hex + .as_ref() + .unwrap_or_else(|| fail("verify", req, "verify requires challenge_hex")); + let shex = job + .solution_hex + .as_ref() + .unwrap_or_else(|| fail("verify", req, "verify requires solution_hex")); + let challenge = + hex_decode(chex).unwrap_or_else(|| fail("verify", req, "invalid challenge_hex")); + let sb = hex_decode(shex).unwrap_or_else(|| fail("verify", req, "invalid solution_hex")); + if sb.len() != 16 { + fail("verify", req, "solution_hex must be 16 bytes"); + } + let mut sol_bytes = [0u8; 16]; + sol_bytes.copy_from_slice(&sb); + + let mut builder = EquiXBuilder::new(); + builder.runtime(runtime_option(req)); + let eff = match effective_runtime(&builder, &challenge, req) { + Ok(e) => e, + Err(e) => fail("verify", req, &e), + }; + + for _ in 0..warmup { + let _ = builder.verify_bytes(&challenge, &sol_bytes); + } + + let mut runs = Vec::with_capacity(reps as usize); + for i in 0..reps { + let t0 = Instant::now(); + let r = builder.verify_bytes(&challenge, &sol_bytes); + let ns = t0.elapsed().as_nanos() as u64; + let (vr, ok) = match &r { + Ok(_) => ("OK".to_string(), 1), + Err(e) => (format!("{:?}", e), 0), + }; + runs.push(RunOut { + index: i as usize, + wall_ns: ns, + solutions: ok, + compile_ns: 0, + attempts: 0, + achieved_effort: 0, + verify_result: Some(vr), + }); + } + emit("verify", req, &eff, runs, None); +} + +fn op_effort(job: &Job, req: &str, reps: u64, warmup: u64) { + let bhex = job + .challenge_base_hex + .as_ref() + .unwrap_or_else(|| fail("effort", req, "effort requires challenge_base_hex")); + let base = hex_decode(bhex).unwrap_or_else(|| fail("effort", req, "invalid challenge_base_hex")); + let nonce_bytes = job.nonce_bytes.unwrap_or(8) as usize; + let nonce_start = job.nonce_start.unwrap_or(0); + let target = job.target_effort.unwrap_or(1000) as u32; + let max_attempts = job.max_attempts.unwrap_or(5_000_000); + if nonce_bytes > 8 || base.len() + nonce_bytes > MAX_CHALLENGE { + fail("effort", req, "nonce_bytes out of range"); + } + + let mut builder = EquiXBuilder::new(); + builder.runtime(runtime_option(req)); + let eff = match effective_runtime(&builder, &base, req) { + Ok(e) => e, + Err(e) => fail("effort", req, &e), + }; + + let mut mem = SolverMemory::new(); + // (attempts, best effort, winning token bytes: (nonce wire bytes, solution)) + let search = |mem: &mut SolverMemory| -> (u64, u32, Option<(Vec, Vec)>) { + let mut nonce = nonce_start; + let mut attempts = 0u64; + let mut best = 0u32; + let mut token: Option<(Vec, Vec)> = None; + for _ in 0..max_attempts { + let chal = build_nonce_challenge(&base, nonce, nonce_bytes); + let arr = match builder.build(&chal) { + Ok(eq) => eq.solve_with_memory(mem), + Err(_) => Default::default(), + }; + attempts += 1; + let mut done = false; + for s in arr.iter() { + let e = effort::effort_of(&chal, &s.to_bytes()); + if e > best { + best = e; + } + if e >= target && !done { + // The token as it would go on the wire: the nonce's + // nonce_bytes-long LE encoding + the 16-byte solution. + token = Some((chal[base.len()..].to_vec(), s.to_bytes().to_vec())); + done = true; + } + } + if done { + break; + } + nonce = nonce.wrapping_add(1); + } + (attempts, best, token) + }; + + for _ in 0..warmup { + let _ = search(&mut mem); + } + + let mut runs = Vec::with_capacity(reps as usize); + let mut winning: Option<(Vec, Vec)> = None; + for i in 0..reps { + let t0 = Instant::now(); + let (attempts, best, token) = search(&mut mem); + let ns = t0.elapsed().as_nanos() as u64; + if token.is_some() { + winning = token; + } + runs.push(RunOut { + index: i as usize, + wall_ns: ns, + solutions: (best >= target) as i64, + compile_ns: 0, + attempts, + achieved_effort: best, + verify_result: None, + }); + } + let (sols_hex, nonce_hex) = match winning { + Some((nonce_bytes_wire, sol)) => ( + Some(vec![hex_encode(&sol)]), + Some(hex_encode(&nonce_bytes_wire)), + ), + None => (None, None), + }; + emit_with_nonce("effort", req, &eff, runs, sols_hex, nonce_hex); +} + +fn op_hashx_compile(job: &Job, req: &str, reps: u64, warmup: u64) { + use hashx::{HashXBuilder, RuntimeOption as HxRt}; + let bhex = job + .challenge_base_hex + .as_ref() + .or(job.challenge_hex.as_ref()) + .unwrap_or_else(|| fail("hashx_compile", req, "hashx_compile requires a challenge")); + let base = hex_decode(bhex).unwrap_or_else(|| fail("hashx_compile", req, "invalid challenge")); + let nonce_start = job.nonce_start.unwrap_or(0); + + let hxrt = match req { + "interpret" => HxRt::InterpretOnly, + "must-compile" => HxRt::CompileOnly, + _ => HxRt::TryCompile, + }; + let mut hb = HashXBuilder::new(); + hb.runtime(hxrt); + + // Probe for effective runtime / hard failure. + let probe_seed = build_nonce_challenge(&base, nonce_start, 8); + let eff = match hb.build(&probe_seed) { + Ok(h) => eff_str(&format!("{:?}", h.runtime()), req), + Err(e) => fail("hashx_compile", req, &format!("{:?}", e)), + }; + + for _ in 0..warmup { + let seed = build_nonce_challenge(&base, nonce_start, 8); + if let Ok(h) = hb.build(&seed) { + let _ = h.hash_to_bytes(0); + } + } + + let mut runs = Vec::with_capacity(reps as usize); + for i in 0..reps { + let seed = build_nonce_challenge(&base, nonce_start + i, 8); + let t0 = Instant::now(); + let built = hb.build(&seed); + let compile_ns = t0.elapsed().as_nanos() as u64; + let (wall_ns, sols) = match built { + Ok(h) => { + let e0 = Instant::now(); + let _ = h.hash_to_bytes(0); + (e0.elapsed().as_nanos() as u64, 1) + } + Err(_) => (0, 0), + }; + runs.push(RunOut { + index: i as usize, + wall_ns, + solutions: sols, + compile_ns, + attempts: 0, + achieved_effort: 0, + verify_result: None, + }); + } + emit("hashx_compile", req, &eff, runs, None); +} + +fn main() { + let mut input = String::new(); + if std::io::stdin().read_to_string(&mut input).is_err() { + fail("", "", "failed to read stdin"); + } + let job: Job = match serde_json::from_str(&input) { + Ok(j) => j, + Err(e) => fail("", "", &format!("invalid job JSON: {}", e)), + }; + + let op = job.operation.clone().unwrap_or_else(|| "solve".to_string()); + let req = job.runtime.clone().unwrap_or_else(|| "try-compile".to_string()); + let reps = job.repetitions.unwrap_or(10).max(1); + let warmup = job.warmup.unwrap_or(3); + + match op.as_str() { + "solve" => op_solve(&job, &req, reps, warmup), + "verify" => op_verify(&job, &req, reps, warmup), + "effort" => op_effort(&job, &req, reps, warmup), + "hashx_compile" => op_hashx_compile(&job, &req, reps, warmup), + _ => fail(&op, &req, "unknown operation"), + } +} diff --git a/tools/benchmarks/Equi-X/scripts/autotune_c_flags.sh b/tools/benchmarks/Equi-X/scripts/autotune_c_flags.sh new file mode 100755 index 0000000..bd1a794 --- /dev/null +++ b/tools/benchmarks/Equi-X/scripts/autotune_c_flags.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Auto-select the FASTEST optimization flags for the MAIN C runner on THIS +# machine, then install the winner as build/runners/c/equix_runner (the path the +# `equix-c` adapter uses). So the main benchmark always runs the fastest build +# this host can produce, rather than a hard-coded guess. +# +# It builds a few candidate flag sets with the default C compiler, benchmarks the +# JIT solve path (which dominates PoW cost) with enough reps for a stable median, +# ranks them, and copies the fastest binary over the main runner. Each rep solves +# a FRESH challenge derived from the seed (a SHA-256 chain), so tuning spans the +# same varied-challenge distribution the main solve benchmark measures — not one +# fixed challenge. The seed is deterministic, so every candidate sees the IDENTICAL +# challenge stream, keeping the flag comparison a fair apples-to-apples A/B. +# Idempotent: candidate build dirs are reused, so re-runs are cheap. +# +# Tunables (env): +# CC compiler to tune (default: cc) +# EQUIX_AUTOTUNE_REPS timed solves per candidate (default: 1000) +# EQUIX_AUTOTUNE_WARMUP warmup solves per candidate (default: 8) +# EQUIX_AUTOTUNE_SEED challenge SEED hex (default: deadbeef); each rep +# solves the next SHA-256-derived challenge in the chain +# EQUIX_AUTOTUNE_EPSILON if plain -O3 is within this fraction of the best, +# prefer -O3 (portable, no native/LTO lock-in on a +# sub-noise win). Default: 0.01 (1%). Set 0 for strict. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +# Ctrl+C mid-tune: exit cleanly (130) instead of a bare SIGINT kill. Nothing to +# roll back — candidate build dirs are reused and the main runner is only ever +# replaced by the atomic mv at the very end, so an interrupt leaves it untouched. +trap 'echo; echo "autotune: interrupted (Ctrl+C); main runner left unchanged." >&2; exit 130' INT + +CC="${CC:-cc}" +REPS="${EQUIX_AUTOTUNE_REPS:-1000}" +WARMUP="${EQUIX_AUTOTUNE_WARMUP:-8}" +# Accept the legacy EQUIX_AUTOTUNE_CHALLENGE name as a fallback for the seed. +SEED="${EQUIX_AUTOTUNE_SEED:-${EQUIX_AUTOTUNE_CHALLENGE:-deadbeef}}" +EPS="${EQUIX_AUTOTUNE_EPSILON:-0.01}" +NPROC="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)" +EQUIX_COMMIT="$(git -C vendored/equix rev-parse --short HEAD 2>/dev/null || echo unknown)" + +command -v "$CC" >/dev/null 2>&1 || { echo "autotune: compiler '$CC' not found"; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo "autotune: python3 required"; exit 1; } + +# Candidate flag sets on the default compiler. -O0/-O1 are excluded on purpose +# (the flag sweep shows ~2x regression); these are the fast-tier contenders. +CANDIDATES=( + "o2|-O2 -DNDEBUG" + "o3|-O3 -DNDEBUG" + "o3-native|-O3 -march=native -DNDEBUG" + "o3-lto|-O3 -flto -DNDEBUG" +) + +# Median solve wall-time (ns) for a runner binary, or non-zero exit if unusable. +# NB: the Python script is passed via -c so the runner's piped JSON reaches its +# stdin (a heredoc would be consumed as the script instead). +measure() { + printf '{"schema_version":1,"operation":"solve","runtime":"try-compile","repetitions":%d,"warmup":%d,"challenge_seed_hex":"%s"}' \ + "$REPS" "$WARMUP" "$SEED" | "$1" 2>/dev/null | python3 -c ' +import sys, json, statistics +try: + d = json.loads(sys.stdin.read().splitlines()[-1]) + assert d.get("ok") and d.get("runs") + runs = d["runs"] + walls = [r["wall_ns"] for r in runs if r.get("wall_ns", 0) > 0] + # Sanity: every rep must be validly timed, and the build must really solve + # SOMETHING across the stream. Under the varied-challenge seed, individual + # derived challenges legitimately yield 0 solutions (~17% of them), so the + # guard is total-solutions>0, not the old per-rep all(>0) which this breaks. + assert len(walls) == len(runs) + assert sum(r.get("solutions", 0) for r in runs) > 0 + print(int(statistics.median(walls))) +except BaseException: # incl. KeyboardInterrupt on Ctrl+C: exit quietly, no stray traceback + sys.exit(1) +' +} + +# Parallel indexed arrays (macOS ships bash 3.2, which has no associative arrays). +echo "autotune: compiler=$CC reps=$REPS warmup=$WARMUP seed=$SEED (challenge varied per rep)" +# NB: candidates are built with the plain version "1.0.0" — the winner becomes +# the MAIN runner, and a tuning-suffixed version would leak machine-specific +# provenance into published results. The chosen flags are recorded separately +# in build/runners/c/equix_runner.flags and build/provenance.json. +NAMES=(); OKFLAGS=(); MEDIANS=() +for entry in "${CANDIDATES[@]}"; do + IFS='|' read -r name flags <<<"$entry" + bdir="build/autotune/$name" + log="build/autotune/$name.log" + mkdir -p build/autotune + # A copied/moved repo carries a CMake cache with old absolute paths; clean it. + if [ -f "$bdir/CMakeCache.txt" ]; then + recorded="$(sed -n 's/^CMAKE_CACHEFILE_DIR:INTERNAL=//p' "$bdir/CMakeCache.txt" | head -1)" + if [ -n "$recorded" ] && [ "$recorded" != "$(cd "$bdir" && pwd)" ]; then + rm -rf "$bdir" + fi + fi + if ! cmake -S runners/c -B "$bdir" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER="$CC" \ + -DCMAKE_C_FLAGS_RELEASE="$flags" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.10 \ + -DEQUIX_C_COMMIT="$EQUIX_COMMIT" \ + -DEQUIX_C_VERSION="1.0.0" >/dev/null 2>"$log" \ + || ! cmake --build "$bdir" -j"$NPROC" --target equix_runner >>"$log" 2>&1; then + echo " skip $name ($flags): build failed (see $log)" + continue + fi + if ! median="$(measure "$bdir/equix_runner")"; then + echo " skip $name ($flags): benchmark failed / no solutions" + continue + fi + NAMES+=("$name"); OKFLAGS+=("$flags"); MEDIANS+=("$median") + printf ' %-10s %-26s median %s ms\n' "$name" "$flags" \ + "$(python3 -c "print(f'{$median/1e6:.3f}')")" +done + +[ "${#NAMES[@]}" -gt 0 ] || { echo "autotune: no candidate usable; keeping existing runner"; exit 1; } + +# Pick the minimum-median candidate; find plain -O3's index for the tie-break. +best_i=0; o3_i=-1 +for i in "${!NAMES[@]}"; do + [ "${MEDIANS[$i]}" -lt "${MEDIANS[$best_i]}" ] && best_i=$i + [ "${NAMES[$i]}" = "o3" ] && o3_i=$i +done + +# Tie-break: if plain -O3 is within EPS of the winner, prefer it -- avoids +# locking the main runner into a native/LTO build over a sub-noise difference. +if [ "$o3_i" -ge 0 ] && [ "${NAMES[$best_i]}" != "o3" ] \ + && awk "BEGIN{exit !(${MEDIANS[$o3_i]} <= ${MEDIANS[$best_i]}*(1+$EPS))}"; then + echo " note: -O3 within ${EPS} of best (${NAMES[$best_i]}); preferring -O3 for portability" + best_i=$o3_i +fi + +best_name="${NAMES[$best_i]}"; best_flags="${OKFLAGS[$best_i]}"; best_median="${MEDIANS[$best_i]}" +mkdir -p build/runners/c +# Install atomically: copy to a temp path, then rename over the live runner, so a +# Ctrl+C during the copy can never leave a half-written (corrupt) main binary. +cp -f "build/autotune/$best_name/equix_runner" build/runners/c/equix_runner.tmp +mv -f build/runners/c/equix_runner.tmp build/runners/c/equix_runner +printf '%s' "$best_flags" > build/runners/c/equix_runner.flags +echo "==> fastest flags on this machine: '$best_flags' ($best_name), median $(python3 -c "print(f'{$best_median/1e6:.3f}')") ms" +echo " installed as build/runners/c/equix_runner (used by the 'equix-c' main run)" diff --git a/tools/benchmarks/Equi-X/scripts/build_variants.sh b/tools/benchmarks/Equi-X/scripts/build_variants.sh new file mode 100755 index 0000000..7fd39e1 --- /dev/null +++ b/tools/benchmarks/Equi-X/scripts/build_variants.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Build the C runner (and its vendored libequix/hashx) under several compiler / +# optimization-flag combinations, registering each as its own benchmark impl so +# the harness can compare performance across compiler flags. +# +# Each variant -> build/variants//equix_runner and a manifest at +# adapters/generated/equix-c-.manifest.toml (impl name "equix-c-"). +# Then: python -m equix_bench run --config configs/compiler_flags.toml --out results/ +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" +NPROC="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)" +have() { command -v "$1" >/dev/null 2>&1; } + +git submodule update --init --recursive >/dev/null 2>&1 || true +EQUIX_COMMIT="$(git -C vendored/equix rev-parse --short HEAD 2>/dev/null || echo unknown)" +OUT="adapters/generated" +mkdir -p "$OUT" build/variants + +# name | compiler | CFLAGS (-DNDEBUG kept so all variants match Release semantics) +VARIANTS=( + "gcc-o0|gcc|-O0 -DNDEBUG" + "gcc-o2|gcc|-O2 -DNDEBUG" + "gcc-o3|gcc|-O3 -DNDEBUG" + "gcc-o3-native|gcc|-O3 -march=native -DNDEBUG" + "gcc-o3-lto|gcc|-O3 -flto -DNDEBUG" + "clang-o3|clang|-O3 -DNDEBUG" + "clang-o3-native|clang|-O3 -march=native -DNDEBUG" +) + +built=() +for entry in "${VARIANTS[@]}"; do + IFS='|' read -r name cc flags <<<"$entry" + if ! have "$cc"; then echo "skip $name ($cc not found)"; continue; fi + bdir="build/variants/$name" + # A copied/moved repo carries a CMake cache with old absolute paths; clean it. + if [ -f "$bdir/CMakeCache.txt" ]; then + recorded="$(sed -n 's/^CMAKE_CACHEFILE_DIR:INTERNAL=//p' "$bdir/CMakeCache.txt" | head -1)" + if [ -n "$recorded" ] && [ "$recorded" != "$(cd "$bdir" && pwd)" ]; then + rm -rf "$bdir" + fi + fi + echo "==> building $name ($cc $flags)" + if ! cmake -S runners/c -B "$bdir" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.10 \ + -DCMAKE_C_COMPILER="$cc" \ + -DCMAKE_C_FLAGS_RELEASE="$flags" \ + -DEQUIX_C_COMMIT="$EQUIX_COMMIT" \ + -DEQUIX_C_VERSION="1.0.0-$name" >/dev/null 2>"build/variants/$name.log"; then + echo " configure FAILED for $name (see build/variants/$name.log)"; continue + fi + if ! cmake --build "$bdir" -j"$NPROC" --target equix_runner >>"build/variants/$name.log" 2>&1; then + echo " build FAILED for $name (see build/variants/$name.log)"; continue + fi + cat > "$OUT/equix-c-$name.manifest.toml" </combined or results/combined) +set -uo pipefail + +ROOT_ARG="" +OUT="" +while [ $# -gt 0 ]; do + case "$1" in + --out) [ $# -ge 2 ] || { echo "error: --out needs a directory" >&2; exit 2; }; OUT="$2"; shift ;; + --out=*) OUT="${1#*=}" ;; + -h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + -*) echo "unknown option: $1" >&2; exit 2 ;; + *) ROOT_ARG="$1" ;; + esac + shift +done + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO" + +# Default ROOT: a dedicated per-device collection dir if present, else results/. +ROOT="$ROOT_ARG" +if [ -z "$ROOT" ]; then + if [ -d results-by-device ]; then ROOT="results-by-device"; else ROOT="results"; fi +fi +[ -d "$ROOT" ] || { echo "error: ROOT '$ROOT' is not a directory" >&2; exit 2; } +[ -n "$OUT" ] || OUT="$ROOT/combined" + +# Prefer the venv from setup.sh (harness installed, PEP-668-proof). +PY="python3" +[ -x "$REPO/.venv/bin/python" ] && PY="$REPO/.venv/bin/python" +export PYTHONPATH="$REPO/harness${PYTHONPATH:+:$PYTHONPATH}" + +echo "==> Combining all runs under '$ROOT' -> '$OUT'" +$PY -m equix_bench combine --root "$ROOT" --out "$OUT" || { + echo "error: combine failed" >&2; exit 1; +} +echo " report: $OUT/report.md" +echo " figures: $OUT/plots/*.png data: $OUT/results.csv" diff --git a/tools/benchmarks/Equi-X/scripts/run_all.sh b/tools/benchmarks/Equi-X/scripts/run_all.sh new file mode 100755 index 0000000..8f5e162 --- /dev/null +++ b/tools/benchmarks/Equi-X/scripts/run_all.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# Run the ENTIRE Equi-X benchmark pipeline end to end: +# 1. bootstrap dependencies + build both runners (scripts/setup.sh) +# 2. harness unit tests +# 3. main benchmark: C vs Rust across all operations, incl. the DoS-protection +# verdict, with the cross-implementation correctness gate; the --full +# profile additionally measures sustained concurrency (saturation ladder) +# and the mining rate vs difficulty +# 4. compiler-flag variants: build a gcc/clang/-O matrix and compare them +# +# Usage: +# ./scripts/run_all.sh # quick profile (smoke config; a few minutes) +# ./scripts/run_all.sh --full # deep sweep (full config + effort sweep; longer) +# ./scripts/run_all.sh --out DIR # output base dir (default: results/) +# ./scripts/run_all.sh --no-variants # skip the compiler-flag matrix +# ./scripts/run_all.sh --no-setup # assume deps already installed/built +# ./scripts/run_all.sh --no-tests # skip the unit tests +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" +export PYTHONPATH="$ROOT/harness${PYTHONPATH:+:$PYTHONPATH}" + +PROFILE=quick +OUT=results +DO_SETUP=1 +DO_VARIANTS=1 +DO_TESTS=1 + +while [ $# -gt 0 ]; do + case "$1" in + --full) PROFILE=full ;; + --quick) PROFILE=quick ;; + --out) [ $# -ge 2 ] || { echo "error: --out requires a directory argument" >&2; exit 2; } + OUT="$2"; shift ;; + --out=*) OUT="${1#*=}" ;; + --no-setup) DO_SETUP=0 ;; + --no-variants) DO_VARIANTS=0 ;; + --no-tests) DO_TESTS=0 ;; + -h|--help) sed -n '2,/^set -uo/p' "$0" | sed '$d'; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac + shift +done + +MAIN_CONFIG="configs/smoke.toml" +[ "$PROFILE" = full ] && MAIN_CONFIG="configs/full.toml" +# Prefer the project venv scripts/setup.sh creates: it has the harness + pytest +# installed and sidesteps PEP-668 "externally-managed" pip failures on Linux. +PY="python3" +[ -x "$ROOT/.venv/bin/python" ] && PY="$ROOT/.venv/bin/python" +BENCH="$PY -m equix_bench" +fail() { echo "ERROR: $*" >&2; exit 1; } + +# Ctrl+C: report the interruption plainly and exit 130, rather than letting the +# child's non-zero exit trip a misleading "ERROR: ... failed". The harness kills +# its own runner subprocesses on SIGINT; this trap just makes the pipeline's own +# exit clean. 128+SIGINT(2) = 130, the conventional interrupted-by-Ctrl+C code. +on_interrupt() { echo; echo "Interrupted (Ctrl+C) — stopping the pipeline." >&2; exit 130; } +trap on_interrupt INT + +echo "======================================================================" +echo " Equi-X full pipeline profile=$PROFILE out=$OUT/" +echo "======================================================================" + +if [ "$DO_SETUP" = 1 ]; then + echo; echo "### [1/4] Bootstrap dependencies + build runners" + ./scripts/setup.sh || fail "setup.sh failed" +fi + +if [ "$DO_TESTS" = 1 ]; then + echo; echo "### [2/4] Harness unit tests" + # Ensure pytest is importable. Externally-managed interpreters (PEP 668, e.g. + # Homebrew/Debian Python) reject a plain `pip install`, so fall back through + # --user and finally --break-system-packages before giving up. + # With the venv from setup.sh, pytest is already present. Otherwise fall back + # through --user and --break-system-packages before giving up. + if ! $PY -c 'import pytest' >/dev/null 2>&1; then + $PY -m pip install -q pytest >/dev/null 2>&1 \ + || $PY -m pip install -q --user pytest >/dev/null 2>&1 \ + || $PY -m pip install -q --break-system-packages pytest >/dev/null 2>&1 \ + || true + fi + if $PY -c 'import pytest' >/dev/null 2>&1; then + $PY -m pytest -q harness/tests || fail "unit tests failed" + else + echo " WARNING: could not install pytest; skipping unit tests." >&2 + echo " Fix: re-run ./scripts/setup.sh (creates .venv with pytest), or:" >&2 + echo " python3 -m pip install --break-system-packages pytest" >&2 + fi +fi + +echo; echo "### [3/4] Main benchmark: C vs Rust (all ops + DoS-protection; --full adds concurrency + mining) [$MAIN_CONFIG]" +# cmd_run runs the interop cross-check internally and exits non-zero if it fails, +# so this step is also the correctness gate. +$BENCH run --config "$MAIN_CONFIG" --out "$OUT/main" || fail "main benchmark / cross-check failed" + +if [ "$DO_VARIANTS" = 1 ]; then + echo; echo "### [4/4] Compiler-flag variants (gcc/clang x -O levels)" + ./scripts/build_variants.sh || echo " (some variants failed to build; continuing with those that did)" + $BENCH run --config configs/compiler_flags.toml --out "$OUT/compiler_flags" \ + || echo " (compiler-flags run returned non-zero; see $OUT/compiler_flags/)" +fi + +echo; echo "======================================================================" +echo " Done. Reports:" +echo " $OUT/main/report.md C vs Rust: time, throughput, RSS, compile, effort, DoS" +[ "$DO_VARIANTS" = 1 ] && echo " $OUT/compiler_flags/report.md compiler-flag comparison" +echo " plots: $OUT/*/plots/*.png data: $OUT/*/results.csv" +# The --full profile also measures sustained parallel solve/verify capacity. +[ -f "$OUT/main/concurrency.csv" ] && \ + echo " $OUT/main/concurrency.csv measured concurrency/saturation (solves/s, verify/s, knee)" +[ -f "$OUT/main/mining.csv" ] && \ + echo " $OUT/main/mining.csv measured mining rate vs difficulty (tokens/s, 1-core + machine)" +# Surface the DoS verdict inline. +if [ -f "$OUT/main/report.md" ]; then + grep -h 'Verdict:' "$OUT/main/report.md" | sed 's/\*\*//g; s/^/ DoS → /' +fi +echo "======================================================================" diff --git a/tools/benchmarks/Equi-X/scripts/run_when_idle.sh b/tools/benchmarks/Equi-X/scripts/run_when_idle.sh new file mode 100755 index 0000000..fef8b4f --- /dev/null +++ b/tools/benchmarks/Equi-X/scripts/run_when_idle.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Run a command only once the machine is idle — so other workloads can't skew a +# benchmark. This is how the published mining numbers were measured (see +# docs/findings.md §7a): the run starts only after CPU idle stays above a +# threshold for several consecutive polls. +# +# ./scripts/run_when_idle.sh python3 -m equix_bench run --config configs/mining.toml --out results/mining +# +# Env overrides: +# IDLE_THRESH=85 required % CPU idle IDLE_NEED=3 consecutive polls +# IDLE_POLL=25 seconds between polls IDLE_MAXWAIT=21600 give-up (s) +set -uo pipefail + +[ $# -ge 1 ] || { echo "usage: $0 [args...]" >&2; exit 2; } + +THRESH="${IDLE_THRESH:-85}" +NEED="${IDLE_NEED:-3}" +POLL="${IDLE_POLL:-25}" +MAXWAIT="${IDLE_MAXWAIT:-21600}" + +cpu_idle() { + case "$(uname -s)" in + Darwin) top -l 2 -s 1 -n 0 2>/dev/null | grep 'CPU usage' | tail -1 \ + | sed -E 's/.* ([0-9.]+)% idle.*/\1/' ;; + Linux) vmstat 1 2 2>/dev/null | tail -1 | awk '{print $15}' ;; + *) echo 100 ;; # unknown platform: don't block + esac +} + +ok=0; elapsed=0 +echo "waiting for idle (>= ${THRESH}% CPU idle x ${NEED} consecutive polls)..." +while :; do + idle="$(cpu_idle)"; idle="${idle:-0}" + if awk "BEGIN{exit !(${idle}+0 >= ${THRESH})}"; then ok=$((ok+1)); else ok=0; fi + echo " idle=${idle}% streak=${ok}/${NEED} (waited ${elapsed}s)" + [ "$ok" -ge "$NEED" ] && break + sleep "$POLL"; elapsed=$((elapsed+POLL)) + [ "$elapsed" -ge "$MAXWAIT" ] && { echo "gave up waiting for idle after ${elapsed}s" >&2; exit 3; } +done + +echo "system idle; running: $*" +exec "$@" diff --git a/tools/benchmarks/Equi-X/scripts/setup.sh b/tools/benchmarks/Equi-X/scripts/setup.sh new file mode 100755 index 0000000..c1a4c7c --- /dev/null +++ b/tools/benchmarks/Equi-X/scripts/setup.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +# Bootstrap the Equi-X benchmark: install/compile any missing dependencies, fetch +# the vendored reference implementation, and build both runners. Idempotent. +# +# ./scripts/setup.sh # install missing deps (best effort) + build +# ./scripts/setup.sh --check # only report what's present/missing, install nothing +# EQUIX_NO_AUTO_INSTALL=1 ./scripts/setup.sh # never auto-install; fail with guidance +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +CHECK_ONLY=0 +[ "${1:-}" = "--check" ] && CHECK_ONLY=1 +: "${EQUIX_NO_AUTO_INSTALL:=0}" + +NPROC="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)" +have() { command -v "$1" >/dev/null 2>&1; } + +# clean_stale_cmake_dir : CMake caches record ABSOLUTE paths, so a +# repo that was copied/moved (e.g. rsync'd to another machine) fails to +# configure with "CMakeCache.txt directory ... is different". Detect a cache +# created at a different path and remove the build dir so cmake starts fresh. +clean_stale_cmake_dir() { + local dir="$1" cache="$1/CMakeCache.txt" recorded + [ -f "$cache" ] || return 0 + recorded="$(sed -n 's/^CMAKE_CACHEFILE_DIR:INTERNAL=//p' "$cache" | head -1)" + if [ -n "$recorded" ] && [ "$recorded" != "$(cd "$dir" && pwd)" ]; then + echo " (stale CMake cache from '$recorded'; cleaning $dir)" + rm -rf "$dir" + fi +} + +# Minimum Rust toolchain the runner's dependency tree needs to compile. +RUST_MIN="1.91.0" + +# rust_version_ok -> 0 if have >= min (compares major.minor.patch). +# Pre-release suffixes ("1.91.0-nightly", "1.91.1-beta.2") are stripped first; +# without that, a non-numeric patch component errors the [ -ge ] comparison. +rust_version_ok() { + local have="${1%%-*}" min="${2%%-*}" IFS=. + # shellcheck disable=SC2086 + set -- $have; local h_maj=${1:-0} h_min=${2:-0} h_pat=${3:-0} + # shellcheck disable=SC2086 + set -- $min; local m_maj=${1:-0} m_min=${2:-0} m_pat=${3:-0} + [ "$h_maj" -ne "$m_maj" ] && { [ "$h_maj" -gt "$m_maj" ]; return; } + [ "$h_min" -ne "$m_min" ] && { [ "$h_min" -gt "$m_min" ]; return; } + [ "$h_pat" -ge "$m_pat" ] +} + +# --- dependency provisioning ------------------------------------------------- + +SUDO="" +if [ "$(id -u)" -ne 0 ] && have sudo; then SUDO="sudo"; fi + +PM="" +for pm in apt-get dnf yum pacman zypper brew; do + if have "$pm"; then PM="$pm"; break; fi +done + +pm_install() { # pm_install + case "$PM" in + apt-get) $SUDO apt-get update -qq && $SUDO apt-get install -y "$@" ;; + dnf|yum) $SUDO "$PM" install -y "$@" ;; + pacman) $SUDO pacman -Sy --noconfirm "$@" ;; + zypper) $SUDO zypper install -y "$@" ;; + brew) brew install "$@" ;; + *) return 1 ;; + esac +} + +# pkg name for the current package manager: ensure +ensure() { + local cmd="$1" apt="$2" dnf="$3" pac="$4" brew="$5" pkg="" + if have "$cmd"; then echo " ok: $cmd"; return 0; fi + echo " MISSING: $cmd" + [ "$CHECK_ONLY" = 1 ] && { MISSING=1; return 0; } + if [ "$EQUIX_NO_AUTO_INSTALL" = 1 ] || [ -z "$PM" ]; then + echo " -> please install '$cmd' (no package manager auto-install available)" + MISSING=1; return 0 + fi + case "$PM" in + apt-get) pkg="$apt" ;; dnf|yum) pkg="$dnf" ;; pacman) pkg="$pac" ;; + zypper) pkg="$dnf" ;; brew) pkg="$brew" ;; + esac + # An empty package name means "not installable via this PM" (e.g. cc/pip3 on + # macOS come from the Xcode CLT / python): don't run a bare `$PM install`. + if [ -z "$pkg" ]; then + case "$cmd" in + cc) echo " -> please install the Xcode Command Line Tools: xcode-select --install" ;; + *) echo " -> please install '$cmd' manually (not packaged for $PM here)" ;; + esac + MISSING=1; return 0 + fi + echo " -> installing $pkg via $PM" + if pm_install $pkg; then echo " installed: $cmd"; else echo " FAILED to install $cmd"; MISSING=1; fi +} + +MISSING=0 +echo "==> [1/6] Checking dependencies (package manager: ${PM:-none})" +ensure git git git git git +ensure cmake cmake cmake cmake cmake +ensure cc build-essential 'gcc gcc-c++ make' base-devel "" # clang ships with Xcode CLT on macOS +ensure python3 python3 python3 python python +ensure pip3 python3-pip python3-pip python-pip "" + +# Rust toolchain via rustup when missing, and kept at >= RUST_MIN. +# (A distro/rustup rustc that is too old — e.g. 1.87 vs the 1.91 the deps need — +# fails the build just like a missing one, so treat both the same way.) +# NB: must return 0 when the file is absent — a bare `[ -f ] && .` one-liner +# returns 1 there, and under `set -e` a plain call would silently kill setup. +maybe_source_cargo_env() { + # shellcheck disable=SC1091 + if [ -f "$HOME/.cargo/env" ]; then . "$HOME/.cargo/env"; fi +} + +if have cargo; then + RUSTC_VER="$(rustc --version 2>/dev/null | awk '{print $2}')" + if rust_version_ok "${RUSTC_VER:-0.0.0}" "$RUST_MIN"; then + echo " ok: cargo (rustc ${RUSTC_VER})" + elif [ "$CHECK_ONLY" = 1 ]; then + echo " OUTDATED: rustc ${RUSTC_VER:-unknown} < ${RUST_MIN}"; MISSING=1 + elif [ "$EQUIX_NO_AUTO_INSTALL" = 1 ]; then + echo " -> rustc ${RUSTC_VER:-unknown} < ${RUST_MIN}; please update Rust (https://rustup.rs)"; MISSING=1 + elif have rustup; then + echo " -> rustc ${RUSTC_VER:-unknown} < ${RUST_MIN}; updating via 'rustup update stable'" + if rustup update stable && rustup default stable; then + maybe_source_cargo_env + RUSTC_VER="$(rustc --version 2>/dev/null | awk '{print $2}')" + rust_version_ok "${RUSTC_VER:-0.0.0}" "$RUST_MIN" \ + || { echo " FAILED: rustc still ${RUSTC_VER:-unknown} < ${RUST_MIN} after update"; MISSING=1; } + else + echo " FAILED to update Rust via rustup"; MISSING=1 + fi + else + # cargo present but not managed by rustup (e.g. a distro package): install + # rustup so we can get a current toolchain, then re-check. + echo " -> rustc ${RUSTC_VER:-unknown} < ${RUST_MIN} and no rustup; installing rustup" + if have curl && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable; then + maybe_source_cargo_env + RUSTC_VER="$(rustc --version 2>/dev/null | awk '{print $2}')" + rust_version_ok "${RUSTC_VER:-0.0.0}" "$RUST_MIN" \ + || { echo " FAILED: rustc still ${RUSTC_VER:-unknown} < ${RUST_MIN}; ensure ~/.cargo/bin precedes the system rustc on PATH"; MISSING=1; } + else + echo " FAILED to install rustup (need curl + network)"; MISSING=1 + fi + fi +elif [ "$CHECK_ONLY" = 1 ]; then + echo " MISSING: cargo"; MISSING=1 +elif [ "$EQUIX_NO_AUTO_INSTALL" = 1 ]; then + echo " -> please install Rust (https://rustup.rs)"; MISSING=1 +else + echo " -> installing Rust via rustup" + if have curl && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable; then + maybe_source_cargo_env + else + echo " FAILED to install cargo (need curl + network)"; MISSING=1 + fi +fi + +if [ "$CHECK_ONLY" = 1 ]; then + [ "$MISSING" = 0 ] && echo "==> All dependencies present." || echo "==> Some dependencies missing (see above)." + exit "$MISSING" +fi +[ "$MISSING" = 0 ] || { echo "ERROR: missing dependencies above; install them and re-run."; exit 1; } + +# --- build ------------------------------------------------------------------- + +echo "==> [2/6] Initializing vendored submodules (equix + hashx)" +git submodule update --init --recursive + +EQUIX_COMMIT="$(git -C vendored/equix rev-parse --short HEAD)" +HASHX_COMMIT="$(git -C vendored/equix/hashx rev-parse --short HEAD 2>/dev/null || echo unknown)" +echo " equix @ ${EQUIX_COMMIT}, hashx @ ${HASHX_COMMIT}" + +echo "==> [3/6] Building the C runner (compiles libequix + libhashx)" +# Baseline build with explicit, known-fast flags -- `-O3 -DNDEBUG` (no -O0/-O1, +# no -march=native). This guarantees a working runner and is the fallback if +# autotune is disabled or fails. +MAIN_C_FLAGS="-O3 -DNDEBUG" +# A copied/moved repo carries CMake caches with the old absolute paths — clean +# them or cmake refuses to configure. +clean_stale_cmake_dir build/runners/c +for d in build/autotune/* build/variants/*; do + if [ -d "$d" ]; then clean_stale_cmake_dir "$d"; fi +done +# If a previous autotune installed a winner built with DIFFERENT flags, the +# incremental build below would be a no-op (the copied binary is newer than the +# sources) and rewriting the .flags file would misdescribe the binary. Force a +# clean rebuild in that case so binary and provenance always agree. +CLEAN_ARG="" +if [ -f build/runners/c/equix_runner.flags ] \ + && [ "$(cat build/runners/c/equix_runner.flags)" != "${MAIN_C_FLAGS}" ]; then + CLEAN_ARG="--clean-first" +fi +cmake -S runners/c -B build/runners/c \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS_RELEASE="${MAIN_C_FLAGS}" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.10 \ + -DEQUIX_C_COMMIT="${EQUIX_COMMIT}" \ + -DEQUIX_C_VERSION="1.0.0" >/dev/null +cmake --build build/runners/c -j"$NPROC" --target equix_runner ${CLEAN_ARG} +printf '%s' "${MAIN_C_FLAGS}" > build/runners/c/equix_runner.flags +echo " -> build/runners/c/equix_runner (baseline flags: ${MAIN_C_FLAGS})" + +# Then pick the fastest optimization flags on THIS machine and install the winner +# as the main runner, so the benchmark runs the fastest build the host can make +# (solve is JIT-dominated, so the win is usually small, but it is now measured, +# not assumed). Set EQUIX_NO_AUTOTUNE=1 to skip (keeps the baseline above). +if [ "${EQUIX_NO_AUTOTUNE:-0}" = 1 ]; then + echo " (autotune disabled via EQUIX_NO_AUTOTUNE; using ${MAIN_C_FLAGS})" +else + echo "==> [3b/6] Auto-selecting the fastest C flags for the main runner" + ./scripts/autotune_c_flags.sh || echo " (autotune failed; keeping the ${MAIN_C_FLAGS} baseline)" +fi + +echo "==> [4/6] Building the Rust runner (pinned via Cargo.lock)" +cargo build --locked --release --manifest-path runners/rust/Cargo.toml +echo " -> runners/rust/target/release/equix_runner" + +echo "==> [5/6] Installing the Python harness into a project venv (.venv)" +# A project virtualenv is the robust fix for the Linux failure "pytest not +# importable / externally-managed-environment": PEP-668 interpreters (Debian, +# Homebrew) reject a plain `pip install` into the system site-packages, but a +# venv has its own writable site-packages, so pip (matplotlib/numpy/pytest) just +# works and stays isolated from the system Python. It is stdlib-only — no pyenv +# or extra tooling required. run_all.sh and the Makefile auto-prefer .venv/bin. +VENV_DIR=".venv" +if [ ! -x "$VENV_DIR/bin/python" ]; then + if ! python3 -m venv "$VENV_DIR" >/dev/null 2>&1; then + # Debian/Ubuntu split venv into the python3-venv package; provision and retry. + if [ "$PM" = apt-get ] && [ "$EQUIX_NO_AUTO_INSTALL" != 1 ]; then + echo " (python3 -m venv unavailable; installing python3-venv)" + pm_install python3-venv >/dev/null 2>&1 || true + python3 -m venv "$VENV_DIR" >/dev/null 2>&1 || true + fi + fi +fi +if [ -x "$VENV_DIR/bin/python" ]; then + VENV_PY="$VENV_DIR/bin/python" + "$VENV_PY" -m pip install -q --upgrade pip >/dev/null 2>&1 || true + if "$VENV_PY" -m pip install -q -e ./harness pytest; then + echo " -> equix_bench + pytest installed into $VENV_DIR" + echo " (run via: $VENV_DIR/bin/python -m equix_bench ...; run_all.sh / make use it automatically)" + else + echo " (pip install into $VENV_DIR failed; see errors above)" + fi +else + # No venv possible (e.g. locked-down host): fall back through system pip modes. + echo " (could not create $VENV_DIR; falling back to system pip)" + if python3 -m pip install -e ./harness >/dev/null 2>&1 \ + || python3 -m pip install --user -e ./harness >/dev/null 2>&1 \ + || python3 -m pip install --break-system-packages -e ./harness >/dev/null 2>&1; then + echo " -> equix_bench installed (system interpreter)" + else + echo " (could not pip install automatically; run: python3 -m pip install -e ./harness)" + fi +fi + +echo "==> [6/6] Writing provenance" +mkdir -p build +cat > build/provenance.json </dev/null || echo n/a) | head -1)", + "cc_flags": "$(cat build/runners/c/equix_runner.flags 2>/dev/null || echo n/a)", + "rustc": "$(rustc --version 2>/dev/null | awk '{print $2}' || echo n/a)" +} +EOF +cat build/provenance.json + +echo "==> Done. Try:" +PY_HINT="python3"; [ -x "$VENV_DIR/bin/python" ] && PY_HINT="$VENV_DIR/bin/python" +echo " $PY_HINT -m equix_bench run --config configs/smoke.toml --out results/" diff --git a/tools/benchmarks/Equi-X/scripts/verify.sh b/tools/benchmarks/Equi-X/scripts/verify.sh new file mode 100755 index 0000000..69ed0df --- /dev/null +++ b/tools/benchmarks/Equi-X/scripts/verify.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# End-to-end verification: build, probe each runner directly, run the smoke +# config, and assert the cross-implementation correctness gate passes. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +CRUN="build/runners/c/equix_runner" +RRUN="runners/rust/target/release/equix_runner" + +echo "==> Ensuring runners are built" +[ -x "$CRUN" ] && [ -x "$RRUN" ] || ./scripts/setup.sh + +probe() { + local runner="$1" name="$2" + local job='{"schema_version":1,"operation":"solve","runtime":"try-compile","challenge_hex":"deadbeef","repetitions":1,"warmup":0}' + local out; out="$(echo "$job" | "$runner" | tail -1)" + echo "$out" | grep -q '"ok":true' || { echo "FAIL: $name did not return ok:true"; echo "$out"; exit 1; } + echo "$out" | grep -q '"solutions":4' || { echo "FAIL: $name did not find 4 solutions for deadbeef"; echo "$out"; exit 1; } + echo " OK: $name solve/deadbeef -> 4 solutions" +} + +echo "==> Probing runners directly" +probe "$CRUN" "equix-c" +probe "$RRUN" "equix-rust" + +echo "==> Running smoke config" +PYTHONPATH=harness python3 -m equix_bench run --config configs/smoke.toml --out results --root . + +echo "==> Cross-check gate" +PYTHONPATH=harness python3 -m equix_bench run --config configs/smoke.toml --out results --root . --crosscheck-only + +echo "==> Outputs" +ls -1 results/plots/*.png +test -f results/report.md && echo " report.md OK" +test -f results/results.csv && echo " results.csv OK" +echo "==> VERIFY PASSED"