mirror of
https://github.com/logos-blockchain/research.git
synced 2026-08-07 11:43:20 +00:00
Importing Equi-X benchmarking suite
This commit is contained in:
parent
400559f146
commit
ae54acd83d
66
tools/benchmarks/Equi-X/.github/workflows/ci.yml
vendored
Normal file
66
tools/benchmarks/Equi-X/.github/workflows/ci.yml
vendored
Normal file
@ -0,0 +1,66 @@
|
|||||||
|
name: ci
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-and-smoke:
|
||||||
|
# ubuntu-24.04 (x86-64) and macos-14 (Apple Silicon / aarch64) validate both
|
||||||
|
# platforms, including the Apple Silicon JIT guard in the C runner.
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
os: [ubuntu-24.04, macos-14]
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
submodules: recursive
|
||||||
|
|
||||||
|
- name: Install Rust toolchain
|
||||||
|
uses: dtolnay/rust-toolchain@stable
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.11"
|
||||||
|
|
||||||
|
- name: Build runners
|
||||||
|
run: ./scripts/setup.sh
|
||||||
|
|
||||||
|
- name: Install harness
|
||||||
|
run: pip install -e ./harness
|
||||||
|
|
||||||
|
- name: Run reference C unit tests
|
||||||
|
run: |
|
||||||
|
cmake --build build/runners/c --target equix-tests
|
||||||
|
./build/runners/c/equix_ext/equix-tests
|
||||||
|
|
||||||
|
- name: Harness pytest
|
||||||
|
run: |
|
||||||
|
pip install pytest
|
||||||
|
pytest -q harness/tests
|
||||||
|
|
||||||
|
- name: Smoke benchmark
|
||||||
|
run: python -m equix_bench run --config configs/smoke.toml --out results/
|
||||||
|
|
||||||
|
- name: Cross-implementation correctness gate
|
||||||
|
run: python -m equix_bench run --config configs/smoke.toml --out results/ --crosscheck-only
|
||||||
|
|
||||||
|
- name: Multi-CPU combine (two labelled runs, faceted + cross-CPU figures)
|
||||||
|
run: |
|
||||||
|
python -m equix_bench run --config configs/smoke.toml --out runA/ --device-label ci-cpu-a
|
||||||
|
python -m equix_bench run --config configs/smoke.toml --out runB/ --device-label ci-cpu-b
|
||||||
|
python -m equix_bench combine --inputs runA runB --out combined/
|
||||||
|
test -f combined/plots/xdev_throughput.png
|
||||||
|
test -f combined/plots/solve_time_by_runtime.png
|
||||||
|
|
||||||
|
- name: Upload results (report + plots)
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: smoke-results-${{ matrix.os }}
|
||||||
|
path: |
|
||||||
|
results/
|
||||||
|
combined/
|
||||||
26
tools/benchmarks/Equi-X/.gitignore
vendored
Normal file
26
tools/benchmarks/Equi-X/.gitignore
vendored
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
# Build outputs
|
||||||
|
/build/
|
||||||
|
runners/rust/target/
|
||||||
|
**/*.o
|
||||||
|
**/*.a
|
||||||
|
|
||||||
|
# Benchmark outputs
|
||||||
|
/results/
|
||||||
|
/combined/
|
||||||
|
/run[A-Z]/
|
||||||
|
/runA/
|
||||||
|
/runB/
|
||||||
|
|
||||||
|
# Auto-generated compiler-flag variant manifests (reference machine-specific build paths)
|
||||||
|
/adapters/generated/
|
||||||
|
|
||||||
|
# Python
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
.venv/
|
||||||
|
*.egg-info/
|
||||||
|
.pytest_cache/
|
||||||
|
|
||||||
|
# Editor/OS
|
||||||
|
.DS_Store
|
||||||
|
*.swp
|
||||||
3
tools/benchmarks/Equi-X/.gitmodules
vendored
Normal file
3
tools/benchmarks/Equi-X/.gitmodules
vendored
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
[submodule "vendored/equix"]
|
||||||
|
path = vendored/equix
|
||||||
|
url = https://github.com/tevador/equix
|
||||||
36
tools/benchmarks/Equi-X/LICENSE
Normal file
36
tools/benchmarks/Equi-X/LICENSE
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2026 Equi-X benchmark framework contributors
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
|
||||||
|
-------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
This license applies to the framework code authored in this repository
|
||||||
|
(the C/Rust runners and the Python harness).
|
||||||
|
|
||||||
|
THIRD-PARTY COMPONENTS (not covered by the above license):
|
||||||
|
|
||||||
|
* tevador/equix and tevador/hashx — vendored as a git submodule under
|
||||||
|
vendored/equix — are licensed LGPL-3.0-only.
|
||||||
|
* The Rust crates `equix` and `hashx` (dependencies of runners/rust) are
|
||||||
|
licensed LGPL-3.0-only.
|
||||||
|
|
||||||
|
These components are used as dependencies/submodules and retain their original
|
||||||
|
licenses; see their respective LICENSE files.
|
||||||
83
tools/benchmarks/Equi-X/Makefile
Normal file
83
tools/benchmarks/Equi-X/Makefile
Normal file
@ -0,0 +1,83 @@
|
|||||||
|
# Equi-X benchmark — make wrapper around the scripts/ + harness entry points.
|
||||||
|
#
|
||||||
|
# make setup bootstrap deps, build both runners, autotune C flags
|
||||||
|
# make check report which dependencies are present (installs nothing)
|
||||||
|
# make test run the harness unit tests (skips cleanly if pytest missing)
|
||||||
|
# make benchmark main C-vs-Rust benchmark, quick profile (depends: setup test)
|
||||||
|
# make benchmark-full deep sweep incl. concurrency + mining (depends: setup test)
|
||||||
|
# make compiler-flags build the gcc/clang -O matrix and compare (depends: setup)
|
||||||
|
# make mining standalone mining sweep, idle-gated (depends: setup)
|
||||||
|
# make control difficulty-controller simulation demo
|
||||||
|
# make combine merge per-device runs under ROOT into one faceted report
|
||||||
|
# make everything full pipeline via scripts/run_all.sh --full
|
||||||
|
# make clean remove C build trees + generated variant manifests
|
||||||
|
# make distclean clean + Rust target/ + results/
|
||||||
|
#
|
||||||
|
# Variables: OUT=results make benchmark OUT=/tmp/out
|
||||||
|
# ROOT=results-by-device make combine (tree of per-device runs)
|
||||||
|
# EQUIX_NO_AUTOTUNE=1 make setup (skip flag autotuning)
|
||||||
|
|
||||||
|
SHELL := /usr/bin/env bash
|
||||||
|
OUT ?= results
|
||||||
|
ROOT ?= $(OUT)
|
||||||
|
# Prefer the project venv scripts/setup.sh creates (harness + pytest installed,
|
||||||
|
# PEP-668-proof); fall back to system python3 when it is absent.
|
||||||
|
PY := $(if $(wildcard .venv/bin/python),.venv/bin/python,python3)
|
||||||
|
export PYTHONPATH := $(CURDIR)/harness$(if $(PYTHONPATH),:$(PYTHONPATH),)
|
||||||
|
|
||||||
|
.PHONY: all help setup check test benchmark benchmark-full compiler-flags \
|
||||||
|
mining control combine everything clean distclean
|
||||||
|
|
||||||
|
all: benchmark
|
||||||
|
|
||||||
|
help:
|
||||||
|
@sed -n '2,17p' Makefile | sed 's/^# \{0,1\}//'
|
||||||
|
|
||||||
|
setup:
|
||||||
|
./scripts/setup.sh
|
||||||
|
|
||||||
|
check:
|
||||||
|
./scripts/setup.sh --check
|
||||||
|
|
||||||
|
# Tests are a soft dependency: run when pytest is available, skip loudly (but
|
||||||
|
# without failing the chain) when it is not — mirroring scripts/run_all.sh.
|
||||||
|
test: setup
|
||||||
|
@if $(PY) -c 'import pytest' >/dev/null 2>&1; then \
|
||||||
|
$(PY) -m pytest -q harness/tests; \
|
||||||
|
else \
|
||||||
|
echo "WARNING: pytest not importable; skipping unit tests." >&2; \
|
||||||
|
echo " fix: re-run ./scripts/setup.sh (creates .venv with pytest)," >&2; \
|
||||||
|
echo " or: python3 -m pip install pytest (or --break-system-packages)" >&2; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
benchmark: setup test
|
||||||
|
$(PY) -m equix_bench run --config configs/smoke.toml --out $(OUT)/main
|
||||||
|
|
||||||
|
benchmark-full: setup test
|
||||||
|
$(PY) -m equix_bench run --config configs/full.toml --out $(OUT)/main
|
||||||
|
|
||||||
|
compiler-flags: setup
|
||||||
|
./scripts/build_variants.sh
|
||||||
|
$(PY) -m equix_bench run --config configs/compiler_flags.toml --out $(OUT)/compiler_flags
|
||||||
|
|
||||||
|
mining: setup
|
||||||
|
./scripts/run_when_idle.sh $(PY) -m equix_bench run --config configs/mining.toml --out $(OUT)/mining
|
||||||
|
|
||||||
|
control:
|
||||||
|
$(PY) -m equix_bench.difficulty_control --out $(OUT)/control
|
||||||
|
|
||||||
|
# Merge every run found under ROOT (each device's results, in any layout) into
|
||||||
|
# one faceted, per-device report. ROOT defaults to OUT; OUT names the report dir.
|
||||||
|
combine:
|
||||||
|
$(PY) -m equix_bench combine --root $(ROOT) --out $(OUT)/combined
|
||||||
|
@echo "combined report -> $(OUT)/combined/report.md"
|
||||||
|
|
||||||
|
everything:
|
||||||
|
./scripts/run_all.sh --full --out $(OUT)
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -rf build/runners/c build/autotune build/variants adapters/generated
|
||||||
|
@echo "cleaned C build trees + generated manifests (Rust target/ and $(OUT)/ kept; use distclean)"
|
||||||
|
|
||||||
|
distclean: clean
|
||||||
|
rm -rf runners/rust/target $(OUT) combined build
|
||||||
226
tools/benchmarks/Equi-X/PARAMETERS.md
Normal file
226
tools/benchmarks/Equi-X/PARAMETERS.md
Normal file
@ -0,0 +1,226 @@
|
|||||||
|
# Equi-X Benchmark Parameters
|
||||||
|
|
||||||
|
This document describes **every parameter** the benchmarking framework exposes,
|
||||||
|
what it controls, and its implications for **execution time, memory, and cost**.
|
||||||
|
Parameters are set in a TOML config (see `configs/`) and travel to each runner as
|
||||||
|
the JSON job-spec (see `adapters/README.md`).
|
||||||
|
|
||||||
|
Equi-X itself is a fixed puzzle: **Equihash(n=60, k=3)** over the **HashX**
|
||||||
|
pseudo-random hash function. The algorithm constants (n, k, solution size = 8×
|
||||||
|
16-bit indices, hash size) are **not** tunable — they define the puzzle. What the
|
||||||
|
framework varies is *how* the puzzle is executed and measured, plus the Tor-style
|
||||||
|
*effort* layer stacked on top.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. `operation` — what is being measured
|
||||||
|
|
||||||
|
| value | what it does | dominant cost |
|
||||||
|
|-------|--------------|---------------|
|
||||||
|
| `solve` | Generate the HashX program from the challenge, then run the Equihash solver to find all solutions. | **Milliseconds.** The headline PoW cost. ~1.7 solutions/challenge on average. |
|
||||||
|
| `verify` | Check that a given solution is valid for a challenge (index ordering + partial/final XOR sums). | **Microseconds.** ~1000× cheaper than solving — this asymmetry is the whole point of a client puzzle. |
|
||||||
|
| `effort` | Repeatedly solve over an incrementing nonce until a solution meets a target *effort* (difficulty). | **Scales with `target_effort`** (see §5). Models the real cost of producing a PoW at a difficulty. |
|
||||||
|
| `hashx_compile` | Isolate HashX **program generation + compilation** (`hashx_make` / `EquiXBuilder::build`) from execution, using the HashX API directly. | **Microseconds.** The only clean way to measure JIT/compile cost (see §3). |
|
||||||
|
|
||||||
|
**Implication:** `solve` and `verify` are the two faces of an asymmetric PoW;
|
||||||
|
report them together to show the work/verify ratio. `effort` is the attacker/
|
||||||
|
client cost model. `hashx_compile` explains *why* compiled mode is faster.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. `runtime` — HashX execution backend
|
||||||
|
|
||||||
|
HashX generates a unique straight-line program per challenge and can either
|
||||||
|
**interpret** it or **JIT-compile** it to native code.
|
||||||
|
|
||||||
|
| value | meaning | implication |
|
||||||
|
|-------|---------|-------------|
|
||||||
|
| `interpret` | Force the pure interpreter; never compile. | Portable, no executable memory. **~9× slower solve** in practice (measured ~68 ms vs ~7.6 ms). |
|
||||||
|
| `try-compile` | Compile if supported, else fall back to the interpreter. **Default.** | Best speed where a JIT exists (x86-64/aarch64); safe elsewhere. `runtime_effective` reports which path ran. |
|
||||||
|
| `must-compile` | Require the JIT; **fail** if unsupported. | Use to guarantee you are measuring compiled performance; errors out on unsupported targets instead of silently interpreting. |
|
||||||
|
|
||||||
|
- C mapping: `interpret` → `equix_alloc(SOLVE)`; compiled → `equix_alloc(SOLVE | COMPILE)`. Unsupported JIT returns the `EQUIX_NOTSUPP` sentinel.
|
||||||
|
- Rust mapping: `RuntimeOption::InterpretOnly` / `TryCompile` / `CompileOnly`.
|
||||||
|
|
||||||
|
**Cost implication:** compiling adds a one-off ~50–80 µs per program (see
|
||||||
|
`hashx_compile`), amortized across the millions of HashX evaluations in a solve —
|
||||||
|
so compiled mode wins decisively for `solve`, is marginal for a single `verify`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Compile-time isolation (why `hashx_compile` exists)
|
||||||
|
|
||||||
|
The HashX program is seeded by the **challenge**, so in the C library the program
|
||||||
|
is (re)generated *inside* `equix_solve` — libequix's public API cannot separate
|
||||||
|
"compile" from "solve". Rust *can* (build vs solve are distinct calls), but to
|
||||||
|
keep C and Rust comparable, **both** runners implement a dedicated
|
||||||
|
`hashx_compile` operation that times `hashx_make` (program-gen + JIT) separately
|
||||||
|
from one `hashx_exec`. Treat the `compile_ns` field as meaningful **only** for the
|
||||||
|
`hashx_compile` operation; it is `0` for `solve`/`verify`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Challenge parameters
|
||||||
|
|
||||||
|
| parameter | applies to | meaning |
|
||||||
|
|-----------|-----------|---------|
|
||||||
|
| `challenges` (`challenge_hex`) | solve, verify, hashx_compile | Hex-encoded challenge bytes. The challenge is the HashX seed — **each distinct challenge is a different one-way function**. |
|
||||||
|
| `bases` (`challenge_base_hex`) | effort, hashx_compile | Fixed prefix; the runner appends a nonce to form each attempt's challenge. |
|
||||||
|
| `nonce_bytes` | effort | Width of the little-endian nonce counter appended to the base (`challenge = base ‖ LE(nonce)`). Must be ≤ 8. |
|
||||||
|
| `nonce_start` | effort, hashx_compile | Starting nonce value (reproducibility / sharding the search space). |
|
||||||
|
| `solution_hex` | verify | 16-byte packed solution (8× uint16 LE). The harness auto-fills this from a `solve` of the same challenge. |
|
||||||
|
|
||||||
|
**Edge case — invalid programs:** roughly **1 in 2^k** challenges produce a HashX
|
||||||
|
program that fails validation (by design). The runner treats this as a valid
|
||||||
|
*measured outcome* (`solutions: 0`, verify → `CHALLENGE`), not an error; the
|
||||||
|
effort search simply advances the nonce. Some perfectly valid challenges also
|
||||||
|
have **0 Equihash solutions** — e.g. the all-zero challenge — so pick challenges
|
||||||
|
with known solutions for solve/verify cells.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Effort / difficulty parameters (Tor proposal 327)
|
||||||
|
|
||||||
|
The effort layer sits **above** Equi-X. For a solved `(challenge, solution)`:
|
||||||
|
|
||||||
|
```
|
||||||
|
hash32 = first 32 bits (big-endian) of BLAKE2b-256(challenge ‖ solution_bytes)
|
||||||
|
achieved = floor((2^32 - 1) / hash32) # "how hard was this solution"
|
||||||
|
valid at effort E ⇔ hash32 · E ≤ 2^32 - 1 ⇔ achieved ≥ E
|
||||||
|
```
|
||||||
|
|
||||||
|
| parameter | meaning | implication |
|
||||||
|
|-----------|---------|-------------|
|
||||||
|
| `targets` (`target_effort`) | Difficulty to reach: stop when a solution's `achieved ≥ target`. | Cost grows **~linearly** with target — a 10× harder target costs ~10× more work. Each solution meets effort `E` with probability `1/E`; a solve yields ~1.7 solutions, so expected solves ≈ `E / 1.7`. |
|
||||||
|
| `max_attempts` | Safety cap on the nonce search per repetition. | Bounds worst-case runtime; if hit before the target, `achieved` reports the best found. Set comfortably above the target. |
|
||||||
|
|
||||||
|
The preimage layout and byte order are **identical in C and Rust** — the
|
||||||
|
cross-check asserts both produce the same `achieved` effort for a fixed input, so
|
||||||
|
a mismatch (a broken port) fails the build rather than silently skewing results.
|
||||||
|
(Verified against Python's standard `hashlib.blake2b(digest_size=32)`.)
|
||||||
|
|
||||||
|
**Notes:**
|
||||||
|
- This models Tor-327's effort *concept* (a difficulty proxy for benchmarking); the
|
||||||
|
preimage is `challenge ‖ solution_bytes` with standard BLAKE2b-256, a
|
||||||
|
simplification of Tor's production wire layout (which folds in seed/nonce/
|
||||||
|
personalization fields), so values are not byte-compatible with a live Tor PoW.
|
||||||
|
- The search is **deterministic** given `(base, nonce_start)`: every repetition
|
||||||
|
runs the same nonce sequence, so `repetitions` measures timing variance of the
|
||||||
|
same search, not a difficulty distribution. Vary `nonce_start`/`bases` to sample
|
||||||
|
different searches.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Measurement parameters
|
||||||
|
|
||||||
|
| parameter | meaning | implication |
|
||||||
|
|-----------|---------|-------------|
|
||||||
|
| `repetitions` | Number of **timed** iterations per cell. | More reps → tighter median/p95, longer runs. The report uses median + p95 + stddev because there is no `perf`/`taskset` here, so noise is real. |
|
||||||
|
| `warmup` | Untimed iterations run **before** timing. | Excludes cold caches, first-touch paging, and initial JIT warmth from the measurement. Warmups are never counted in `runs[]`. |
|
||||||
|
| `seed` | Optional RNG seed for reproducible challenge generation (reserved for generators). | Reproducibility. |
|
||||||
|
| `impls` | Which implementations to run (must match adapter manifest names). | Determines what appears on every comparison plot — needs ≥2 for the C-vs-Rust figures. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Metrics reported (and their units)
|
||||||
|
|
||||||
|
| metric | source | notes |
|
||||||
|
|--------|--------|-------|
|
||||||
|
| `wall_ns` | `clock_gettime(CLOCK_MONOTONIC)` (C) / `Instant` (Rust) | Per-rep solve/verify/effort time. |
|
||||||
|
| `compile_ns` | `hashx_make` / `EquiXBuilder::build` timing | Meaningful only for `hashx_compile`. |
|
||||||
|
| `solves_per_sec`, `hashes_per_sec` | derived from median solve time | Throughput; hash-rate = solves/sec × the per-solve HashX count (2^16, the equix 16-bit index space; both impls use the same constant so comparisons are exact). |
|
||||||
|
| `peak_rss_kb` | `getrusage.ru_maxrss` (C) / `/proc/self/status VmHWM` (Rust) | Always **kilobytes** (Linux reports KB; macOS reports bytes and the runner converts). One process per cell keeps this attributable. |
|
||||||
|
| `attempts`, `achieved_effort` | effort search | Attacker/client cost at a difficulty. |
|
||||||
|
| `verify_result` | `equix_verify` result enum | `OK` / `CHALLENGE` / `ORDER` / `PARTIAL_SUM` / `FINAL_SUM`. |
|
||||||
|
| `protection_factor` | DoS analysis (§9) | attacker time/token ÷ defender verify time — the core DoS asymmetry. |
|
||||||
|
| `verify_per_sec`, `attacker_tokens_per_sec` | DoS analysis (§9) | defender screening capacity vs attacker output, per core. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Device / CPU tracking & multi-CPU figures
|
||||||
|
|
||||||
|
Every run records the **device** it executed on — the runner self-reports
|
||||||
|
`env.cpu` (model), `env.arch`, and `env.device` (`cpu`/`gpu`), which the harness
|
||||||
|
turns into a device record `{type, name, arch, label}` carried on every result
|
||||||
|
(and in `results.csv` / `run_meta.json`).
|
||||||
|
|
||||||
|
| parameter | meaning | implication |
|
||||||
|
|-----------|---------|-------------|
|
||||||
|
| `--device-label` (a.k.a. `--cpu-label`) | Human label for the executing device. | Defaults to a slug of the **CPU model + OS/kernel version** (e.g. `intel-xeon-2-80ghz-6-18-5`); override to disambiguate machines that still collide (e.g. `--device-label ryzen-9950x`). |
|
||||||
|
|
||||||
|
**Reflecting the CPU on plots:** with a single device, the CPU is shown in each
|
||||||
|
plot's title and the report header. To compare **multiple CPUs**, run on each
|
||||||
|
machine and merge the outputs:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m equix_bench run --config configs/full.toml --out runA/ --device-label host-a
|
||||||
|
python -m equix_bench run --config configs/full.toml --out runB/ --device-label host-b
|
||||||
|
python -m equix_bench combine --inputs runA runB --out combined/
|
||||||
|
```
|
||||||
|
|
||||||
|
`combine` re-aggregates the saved per-run data (no re-benchmarking) and renders:
|
||||||
|
- **faceted plots** — one subplot per CPU, C-vs-Rust compared within each; and
|
||||||
|
- **`xdev_*` cross-CPU charts** — x=CPU, series=implementation — for headline
|
||||||
|
metrics (solve throughput, solve time, peak RSS, verify time).
|
||||||
|
|
||||||
|
### GPU
|
||||||
|
|
||||||
|
**Equi-X is not benchmarked on GPU, and no GPU implementation is bundled.** HashX
|
||||||
|
(the hash Equi-X is built on) is deliberately designed to resist GPU/ASIC
|
||||||
|
acceleration — it depends on branch prediction and out-of-order execution that
|
||||||
|
favor general-purpose CPUs — so a GPU solver would be far slower and none exists in
|
||||||
|
practice. The framework is nonetheless **GPU-ready**: a runner that reports
|
||||||
|
`device: "gpu"` plugs in through the adapter protocol and appears on all figures as
|
||||||
|
another device, with no harness change.
|
||||||
|
|
||||||
|
## 9. DoS-protection effectiveness
|
||||||
|
|
||||||
|
Equi-X is a client puzzle for DoS defense: a requester must **solve** (expensive)
|
||||||
|
before a service acts, while the service only **verifies** (cheap). Any run that
|
||||||
|
includes both the `effort` and `verify` operations gets a DoS-protection section
|
||||||
|
(and `dos_protection.png`) computed from **measured** numbers on the running system.
|
||||||
|
|
||||||
|
| quantity | definition |
|
||||||
|
|----------|------------|
|
||||||
|
| `attacker_s(E)` | measured median time to craft one accepted token at effort `E` (the `effort` op), using the fastest impl |
|
||||||
|
| `defender_s` | measured fastest median `verify` time on that device |
|
||||||
|
| **`protection_factor(E)`** | `attacker_s(E) / defender_s` — how many verifies the defender does in the time the attacker needs for one accepted request |
|
||||||
|
| `verify_per_sec` | `1 / defender_s` — defender screening capacity per core |
|
||||||
|
| `attacker_tokens_per_sec` | `1 / attacker_s(E)` — attacker output per core |
|
||||||
|
| **verdict** | *effective* if some tested effort reaches the threshold; the report states the **minimum effort** `E*` from which protection holds on this system |
|
||||||
|
|
||||||
|
The threshold defaults to **10 000×** (`dosprotect.DEFAULT_THRESHOLD`). Run it with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m equix_bench run --config configs/dos_protection.toml --out results/
|
||||||
|
```
|
||||||
|
|
||||||
|
Because it uses measured attacker cost, the answer is specific to the CPU it runs
|
||||||
|
on — the same effort gives a different protection factor on a fast vs slow machine.
|
||||||
|
|
||||||
|
## 10. Compiler-flag variants (performance vs build flags)
|
||||||
|
|
||||||
|
The same C implementation can be built under different compiler/optimization flags
|
||||||
|
and compared as separate impls. `scripts/build_variants.sh` builds a matrix
|
||||||
|
(`gcc -O0/-O2/-O3`, `-march=native`, `-flto`, `clang -O3`, …), writing one
|
||||||
|
`equix-c-<name>` manifest per variant to `adapters/generated/` (loaded alongside the
|
||||||
|
built-in adapters). The flags apply to the whole `libequix`+`hashx`+runner build, so
|
||||||
|
they affect the Equihash solver and the HashX interpreter (the JIT executes the same
|
||||||
|
generated machine code regardless).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/build_variants.sh
|
||||||
|
python -m equix_bench run --config configs/compiler_flags.toml --out results/
|
||||||
|
```
|
||||||
|
|
||||||
|
Every comparison plot then compares the flag variants; all variants produce
|
||||||
|
identical solutions, so the interop cross-check still holds.
|
||||||
|
|
||||||
|
## 11. Not benchmarked by default (and why)
|
||||||
|
|
||||||
|
- **HugePages** (`EQUIX_CTX_HUGEPAGES`): off by default; it changes RSS accounting
|
||||||
|
and requires host configuration, which would distort memory comparisons.
|
||||||
|
- **Threads / multi-core solving**: the framework measures single-thread cost per
|
||||||
|
cell for clean per-implementation comparison; parallel scaling is orthogonal.
|
||||||
|
- **HW performance counters** (cycles, cache misses): `perf` is unavailable in the
|
||||||
|
reference environment, so cost is reported as wall-time + RSS.
|
||||||
296
tools/benchmarks/Equi-X/README.md
Normal file
296
tools/benchmarks/Equi-X/README.md
Normal file
@ -0,0 +1,296 @@
|
|||||||
|
# Equi-X
|
||||||
|
|
||||||
|
A benchmarking framework for the **Equi-X** proof-of-work algorithm — tevador's
|
||||||
|
`Equihash(n=60, k=3)` over the **HashX** pseudo-random hash function (the
|
||||||
|
client-puzzle used by Tor onion-service PoW, proposal 327).
|
||||||
|
|
||||||
|
It benchmarks **multiple implementations** side by side — the reference **C**
|
||||||
|
(`tevador/equix` + `hashx`) and the **Rust** (Tor `arti` `equix` / `hashx` crates)
|
||||||
|
— across **all parameters** (runtime backend, operation, challenge, difficulty)
|
||||||
|
and measures **execution time and cost** (solve/verify throughput, peak memory,
|
||||||
|
JIT-compile overhead, and difficulty/effort cost). It is extensible to any other
|
||||||
|
implementation via a small language-agnostic plugin protocol.
|
||||||
|
|
||||||
|
## What it measures
|
||||||
|
|
||||||
|
- **Solve & verify time + throughput** — the asymmetric core of the PoW.
|
||||||
|
- **Peak RSS memory** — per implementation and runtime.
|
||||||
|
- **JIT compile overhead** — interpreter vs compiled HashX, isolated.
|
||||||
|
- **Difficulty / effort sweep** — Tor prop-327 effort: expected attempts & time to
|
||||||
|
reach a target difficulty.
|
||||||
|
- **DoS-protection effectiveness** — the attacker-solve vs defender-verify asymmetry
|
||||||
|
on *this* system, with a verdict (effective from what effort). This figure is
|
||||||
|
*per-core* (derived as 1/latency from a single serial op).
|
||||||
|
- **Sustained throughput under concurrency** *(--full)* — the complementary
|
||||||
|
*measured* answer: N worker processes run at once (N = 1, 2, 4, … up to the core
|
||||||
|
count) to capture real memory-bandwidth contention, reporting the machine's true
|
||||||
|
aggregate solves/s and verifies/s and the saturation knee. Additive — it never
|
||||||
|
overwrites the per-core estimate above.
|
||||||
|
- **Mining rate vs difficulty** *(--full, or standalone via `configs/mining.toml`)* —
|
||||||
|
the measured whole-machine token mint rate at each effort target (pooled over
|
||||||
|
many independent nonce ranges, one streaming search per core), the basis for
|
||||||
|
"control the mint rate by setting difficulty". Outputs `mining.csv` + a report
|
||||||
|
section; best measured under idle (`scripts/run_when_idle.sh <cmd>` gates on CPU idle).
|
||||||
|
- **Compiler-flag comparison** — the same C impl built under different
|
||||||
|
`-O` levels / `-march` / `-flto` / gcc-vs-clang, compared side by side.
|
||||||
|
|
||||||
|
Two companion documents build on the measurements: `docs/findings.md` (the full
|
||||||
|
findings report: how Equi-X works, message-exchange schemas, DoS and mining
|
||||||
|
usage with the measured numbers) and `docs/difficulty-control.md` (closed-loop
|
||||||
|
difficulty controllers — mint-rate + load — with a simulator calibrated on the
|
||||||
|
measured curve: `python -m equix_bench.difficulty_control`).
|
||||||
|
|
||||||
|
Every generated plot compares the implementations (C vs Rust) on the same axes,
|
||||||
|
and a correctness **cross-check** proves the implementations agree (solutions from
|
||||||
|
one verify under the other; effort values match byte-for-byte).
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart TB
|
||||||
|
cfg["configs/*.toml<br/>parameter matrix"]
|
||||||
|
|
||||||
|
subgraph harness["Python harness — equix_bench"]
|
||||||
|
direction TB
|
||||||
|
config["config.py<br/>expand matrix → cells"]
|
||||||
|
registry["registry.py<br/>adapter manifests"]
|
||||||
|
runner["runner.py<br/>spawn 1 process / cell"]
|
||||||
|
stats["stats.py<br/>aggregate median/p95"]
|
||||||
|
crosscheck["crosscheck.py<br/>interop gate"]
|
||||||
|
dos["dosprotect.py<br/>DoS asymmetry"]
|
||||||
|
report["report.py<br/>plots + report.md"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph runners["Runners — JSON job/result over stdio"]
|
||||||
|
direction LR
|
||||||
|
c["runners/c<br/>libequix + hashx"]
|
||||||
|
rust["runners/rust<br/>equix + hashx crates"]
|
||||||
|
plugin["your adapter<br/>(any language)"]
|
||||||
|
end
|
||||||
|
|
||||||
|
outputs["report.md · results.csv · raw/*.json<br/>plots/*.png (C-vs-Rust, faceted per CPU, DoS)"]
|
||||||
|
|
||||||
|
cfg --> config
|
||||||
|
registry --> runner
|
||||||
|
config --> runner
|
||||||
|
runner -- job JSON --> c
|
||||||
|
runner -- job JSON --> rust
|
||||||
|
runner -- job JSON --> plugin
|
||||||
|
c -- result JSON --> stats
|
||||||
|
rust -- result JSON --> stats
|
||||||
|
plugin -- result JSON --> stats
|
||||||
|
stats --> report
|
||||||
|
crosscheck --> report
|
||||||
|
dos --> report
|
||||||
|
report --> outputs
|
||||||
|
|
||||||
|
manifests["adapters/*.manifest.toml"] -.-> registry
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Runners** wrap one implementation and speak a JSON-over-stdio protocol
|
||||||
|
(`adapters/README.md`). One process per parameter cell keeps memory attributable.
|
||||||
|
- **Harness** (`harness/equix_bench`) expands the TOML matrix, runs every cell,
|
||||||
|
aggregates statistics, cross-checks implementations, evaluates DoS-protection,
|
||||||
|
and renders `results/report.md`, `results/results.csv`, and comparison plots.
|
||||||
|
- **Extensible**: add an implementation by dropping in a runner that speaks the
|
||||||
|
protocol plus a manifest — including compiler-flag variants and future GPU runners.
|
||||||
|
|
||||||
|
## Run everything (one command)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/run_all.sh # bootstrap deps + build + test + benchmark + compiler variants
|
||||||
|
./scripts/run_all.sh --full # deeper sweep (full config + effort sweep; takes longer)
|
||||||
|
```
|
||||||
|
|
||||||
|
Or via make — `make benchmark` depends on `setup` and `test`, so one command gets a
|
||||||
|
correct-by-construction run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make benchmark # setup -> tests -> quick main benchmark
|
||||||
|
make benchmark-full # setup -> tests -> full sweep (concurrency + mining)
|
||||||
|
make help # all targets: check/test/compiler-flags/mining/control/clean/distclean
|
||||||
|
```
|
||||||
|
|
||||||
|
Copied or moved the repo (e.g. rsync'd to another machine)? `setup.sh` (and
|
||||||
|
`make setup`) detects the stale CMake caches that a copy carries and cleans them
|
||||||
|
automatically; `make clean` / `make distclean` are also available.
|
||||||
|
|
||||||
|
`run_all.sh` does the whole pipeline: installs/builds dependencies
|
||||||
|
(`setup.sh`), runs the unit tests, runs the **main C-vs-Rust benchmark** (all
|
||||||
|
operations + the **DoS-protection verdict**, with the correctness gate), then
|
||||||
|
builds the **compiler-flag variants** and compares them. Outputs:
|
||||||
|
|
||||||
|
- `results/main/report.md` — C vs Rust: time, throughput, RSS, compile, effort, DoS
|
||||||
|
(with `--full` also the concurrency and mining sections + `concurrency.csv`, `mining.csv`)
|
||||||
|
- `results/compiler_flags/report.md` — compiler-flag comparison
|
||||||
|
|
||||||
|
Flags: `--out DIR`, `--no-variants`, `--no-setup`, `--no-tests` (see `--help`).
|
||||||
|
|
||||||
|
## Quick start (step by step)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Install any missing deps (cmake/compiler/cargo/python) + fetch + build.
|
||||||
|
# setup.sh auto-installs via the system package manager / rustup when possible.
|
||||||
|
# Use `--check` to only report what's missing; EQUIX_NO_AUTO_INSTALL=1 to disable.
|
||||||
|
./scripts/setup.sh # installs the harness + pytest into a project .venv
|
||||||
|
|
||||||
|
# 2. Run the smoke benchmark (seconds). Use the venv interpreter setup.sh made:
|
||||||
|
.venv/bin/python -m equix_bench run --config configs/smoke.toml --out results/
|
||||||
|
# (or `make benchmark` / `scripts/run_all.sh`, which auto-prefer .venv)
|
||||||
|
|
||||||
|
# 3. Full sweep (minutes)
|
||||||
|
python -m equix_bench run --config configs/full.toml --out results/
|
||||||
|
|
||||||
|
# One-shot end-to-end check
|
||||||
|
./scripts/verify.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Outputs land in `results/`: `report.md`, `results.csv`, `raw/results.json`, and
|
||||||
|
`plots/*.png` (each comparing C vs Rust).
|
||||||
|
|
||||||
|
## DoS-protection evaluation
|
||||||
|
|
||||||
|
Equi-X is a DoS defense: requesters *solve* (expensive), the service *verifies*
|
||||||
|
(cheap). Any run that includes `effort` + `verify` gets a **DoS-protection**
|
||||||
|
section in the report — measured attacker-cost vs defender-cost on this system,
|
||||||
|
the asymmetry factor, verify throughput, and a verdict ("effective from effort ≥ E*").
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m equix_bench run --config configs/dos_protection.toml --out results/
|
||||||
|
# report.md -> "DoS-protection effectiveness (this system)" + dos_protection.png
|
||||||
|
```
|
||||||
|
|
||||||
|
## Comparing compiler flags
|
||||||
|
|
||||||
|
Build the C runner (and libequix/hashx) under several compiler/optimization flag
|
||||||
|
sets, then benchmark them as separate impls — every plot compares the variants:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/build_variants.sh # gcc -O0/-O2/-O3/-march=native/-flto, clang -O3, ...
|
||||||
|
python -m equix_bench run --config configs/compiler_flags.toml --out results/
|
||||||
|
```
|
||||||
|
|
||||||
|
Each variant becomes `equix-c-<name>` via an auto-generated manifest in
|
||||||
|
`adapters/generated/`; unbuilt variants (missing compiler) are skipped.
|
||||||
|
|
||||||
|
The **main** `equix-c` runner is not one fixed guess: `setup.sh` runs
|
||||||
|
`scripts/autotune_c_flags.sh`, which builds the fast-tier flag candidates
|
||||||
|
(`-O2`, `-O3`, `-O3 -march=native`, `-O3 -flto`) with the default compiler,
|
||||||
|
benchmarks the JIT solve path, and installs the **fastest** binary as the main
|
||||||
|
runner (recording the winning flags in `build/runners/c/equix_runner.flags` and
|
||||||
|
`build/provenance.json`). Solve is JIT-dominated so the margin is usually small;
|
||||||
|
when plain `-O3` is within 1% of the best it is preferred (portable, no native
|
||||||
|
lock-in). Skip the tuning with `EQUIX_NO_AUTOTUNE=1` (falls back to `-O3 -DNDEBUG`).
|
||||||
|
|
||||||
|
## Comparing multiple CPUs (and GPUs)
|
||||||
|
|
||||||
|
Every run records the **CPU it executed on** (model, arch) and shows it on each
|
||||||
|
plot. To compare across machines, run on each and merge — no re-benchmarking:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m equix_bench run --config configs/full.toml --out runA/ --device-label host-a
|
||||||
|
# ...on another machine...
|
||||||
|
python -m equix_bench run --config configs/full.toml --out runB/ --device-label host-b
|
||||||
|
python -m equix_bench combine --inputs runA runB --out combined/
|
||||||
|
```
|
||||||
|
|
||||||
|
`combine` produces **faceted plots** (one panel per CPU, C-vs-Rust within each) plus
|
||||||
|
**`xdev_*` cross-CPU charts** (x=CPU, series=implementation) for headline metrics.
|
||||||
|
The **concurrency** and **mining** sections/figures are carried across every device
|
||||||
|
too (faceted per CPU), not just the solve/verify/effort plots.
|
||||||
|
|
||||||
|
### Automatic, when results from many devices are collected in one tree
|
||||||
|
|
||||||
|
Copy or `rsync` each device's output under a single directory, then combine the
|
||||||
|
whole tree in one command — no need to list every run by hand:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/combine_all.sh results-by-device --out combined/ # or: make combine ROOT=results-by-device
|
||||||
|
```
|
||||||
|
|
||||||
|
Discovery is **layout-agnostic**: any directory holding a `raw/results.json` is
|
||||||
|
treated as a run, and each run's **device identity comes from inside its records**
|
||||||
|
(CPU model + OS), so folder names are free-form and two machines are never
|
||||||
|
conflated. Re-runs of the same device are de-duplicated automatically (newest
|
||||||
|
wins). The layout can be anything, e.g.:
|
||||||
|
|
||||||
|
```
|
||||||
|
results-by-device/
|
||||||
|
laptop-x1/main/{raw/results.json, concurrency.csv, mining.csv, ...}
|
||||||
|
server-epyc/main/{raw/results.json, ...}
|
||||||
|
rpi5/results/main/{raw/results.json, ...}
|
||||||
|
```
|
||||||
|
|
||||||
|
**GPU?** Equi-X/HashX is CPU-oriented by design (deliberately GPU/ASIC-hostile), so
|
||||||
|
no GPU implementation is benchmarked or bundled. The framework is GPU-ready though:
|
||||||
|
a runner reporting `device: "gpu"` plugs in via the adapter protocol and appears on
|
||||||
|
every figure as another device automatically. See `PARAMETERS.md` §8.
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
- C: `cmake` ≥ 3.10, `gcc`/`clang`. Rust: `cargo`/`rustc`. Python ≥ 3.11 (`matplotlib`).
|
||||||
|
- x86-64 or aarch64 for HashX JIT; other targets fall back to the interpreter.
|
||||||
|
- `setup.sh` installs the harness (matplotlib/numpy) and `pytest` into a project
|
||||||
|
`.venv`. This sidesteps the Linux "externally-managed-environment" / "pytest not
|
||||||
|
importable" failure (PEP-668): a virtualenv has its own writable site-packages, so
|
||||||
|
`pip` just works — no `pyenv` or `--break-system-packages` needed. On Debian/Ubuntu
|
||||||
|
it provisions `python3-venv` if missing, and falls back to system `pip` if a venv
|
||||||
|
can't be created.
|
||||||
|
|
||||||
|
## Platform support (Linux, macOS, ARM64, Raspberry Pi 5)
|
||||||
|
|
||||||
|
Runners and harness are portable C / Rust / Python. CI builds and runs the smoke
|
||||||
|
benchmark on **Linux (x86-64)** and **macOS (Apple Silicon)**; both are supported,
|
||||||
|
along with ARM64 Linux (Raspberry Pi 5).
|
||||||
|
|
||||||
|
### macOS (Intel & Apple Silicon)
|
||||||
|
|
||||||
|
- **Interpreter runtime: fully supported** on both Intel and Apple Silicon.
|
||||||
|
- **JIT (compiled runtime):** works on Linux and Intel macOS. On **Apple Silicon**
|
||||||
|
the bundled HashX C JIT uses `mmap`+`mprotect` without `MAP_JIT`, which the
|
||||||
|
kernel rejects — so the **C runner detects this and uses the interpreter**
|
||||||
|
(`try-compile` → interpreter, `must-compile` → clean error), never crashing. The
|
||||||
|
Rust impl JITs via `dynasmrt` (which handles Apple Silicon), so `runtime_effective`
|
||||||
|
honestly reports what each impl actually ran.
|
||||||
|
- Platform specifics are handled: peak RSS via `getrusage` (macOS reports bytes, not
|
||||||
|
Linux's KB — converted), CPU name via `sysctl` (no `/proc`), OS via `uname`.
|
||||||
|
|
||||||
|
### ARM64 / Raspberry Pi 5
|
||||||
|
|
||||||
|
The framework is portable C / Rust / Python with no x86-specific code, so it runs
|
||||||
|
on **ARM64 including the Raspberry Pi 5**:
|
||||||
|
|
||||||
|
- **64-bit OS (recommended for Pi 5):** full support, **including the JIT** — HashX
|
||||||
|
ships an aarch64 compiler backend (`compiler_a64.c`; the Rust side uses
|
||||||
|
`dynasmrt`), so `interpret`, `try-compile`, and `must-compile` all work.
|
||||||
|
- **32-bit OS (armv7/armhf):** runs **interpreter-only** — there is no 32-bit-ARM
|
||||||
|
JIT, so `try-compile` transparently falls back to the interpreter and
|
||||||
|
`must-compile` fails by design. Use a 64-bit OS on the Pi 5 to benchmark the JIT.
|
||||||
|
- CPU identification handles ARM `/proc/cpuinfo` (no `model name` field): the device
|
||||||
|
label uses the board `Model` (e.g. `raspberry-pi-5-model-b-rev-1-0-<kernel>`).
|
||||||
|
- **Comparing x86 vs ARM** is a first-class use case — run on each and `combine` for
|
||||||
|
faceted x86-vs-Pi figures (the arch is recorded per device).
|
||||||
|
- **Caveat:** the Pi 5 can thermal-throttle under sustained solving; use active
|
||||||
|
cooling and watch the reported stddev/p95 for stability.
|
||||||
|
|
||||||
|
Build on the Pi exactly as elsewhere: `./scripts/setup.sh` (it creates the `.venv`
|
||||||
|
with the harness installed; use `.venv/bin/python -m equix_bench ...` to run).
|
||||||
|
|
||||||
|
## Parameters
|
||||||
|
|
||||||
|
See **[PARAMETERS.md](PARAMETERS.md)** for a full description of every parameter
|
||||||
|
(operation, runtime, challenge/nonce, effort/difficulty, repetitions/warmup) and
|
||||||
|
its implications for time, memory, and cost.
|
||||||
|
|
||||||
|
## Adding an implementation
|
||||||
|
|
||||||
|
Write a runner that speaks the protocol in `adapters/README.md`, drop a
|
||||||
|
`<name>.manifest.toml` next to the examples, and add its name to a config's
|
||||||
|
`run.impls`. No harness code changes required.
|
||||||
|
|
||||||
|
## Licensing
|
||||||
|
|
||||||
|
The framework code (runners glue, harness) is MIT (`LICENSE`). The vendored
|
||||||
|
`tevador/equix` + `hashx` (git submodule) and the Rust `equix`/`hashx` crates are
|
||||||
|
**LGPL-3.0-only**; they are used as dependencies/submodules and are not
|
||||||
|
relicensed here.
|
||||||
127
tools/benchmarks/Equi-X/adapters/README.md
Normal file
127
tools/benchmarks/Equi-X/adapters/README.md
Normal file
@ -0,0 +1,127 @@
|
|||||||
|
# Adapter / Plugin Protocol
|
||||||
|
|
||||||
|
The benchmark harness is **implementation-agnostic**. Any Equi-X implementation
|
||||||
|
can be benchmarked by wrapping it in a **runner**: an executable that speaks a
|
||||||
|
tiny JSON-over-stdio protocol. The reference C and Rust runners are just two
|
||||||
|
adapters; add your own (in any language) by satisfying this contract and dropping
|
||||||
|
in a manifest.
|
||||||
|
|
||||||
|
## Contract
|
||||||
|
|
||||||
|
1. The runner reads **one** job-spec JSON object from **stdin**.
|
||||||
|
2. It performs the requested operation, measuring in-process.
|
||||||
|
3. It writes **one** result JSON object to **stdout** (the last line of stdout, so
|
||||||
|
you may print progress/diagnostics before it — everything else goes to stderr).
|
||||||
|
4. Exit code `0` on success, non-zero on error (still print a result JSON with
|
||||||
|
`"ok": false` and an `"error"` message when possible).
|
||||||
|
|
||||||
|
One job = one process. The harness spawns a fresh process per parameter cell so
|
||||||
|
`peak_rss_kb` is attributable.
|
||||||
|
|
||||||
|
## Job-spec (stdin)
|
||||||
|
|
||||||
|
```jsonc
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"operation": "solve", // "solve" | "verify" | "effort" | "hashx_compile"
|
||||||
|
"runtime": "try-compile", // "interpret" | "try-compile" | "must-compile"
|
||||||
|
"repetitions": 20, // timed iterations
|
||||||
|
"warmup": 3, // untimed iterations before timing
|
||||||
|
|
||||||
|
// solve / verify / hashx_compile:
|
||||||
|
"challenge_hex": "deadbeef", // hex challenge bytes (HashX seed)
|
||||||
|
"solution_hex": "6f27...0dc9", // verify only: 16-byte packed solution (8x u16 LE)
|
||||||
|
"challenge_seed_hex": "abcd", // solve/verify only, OPTIONAL, ALTERNATIVE to
|
||||||
|
// challenge_hex: each rep uses a fresh challenge
|
||||||
|
// from a SHA-256 chain over this seed
|
||||||
|
// (challenge_0 = SHA256(seed), challenge_{i+1} =
|
||||||
|
// SHA256(challenge_i)) so measurements span many
|
||||||
|
// challenges. Challenge generation, and (for
|
||||||
|
// verify) the setup solve that yields a token,
|
||||||
|
// MUST be excluded from every timed region.
|
||||||
|
// No solution_hex needed for verify: the runner
|
||||||
|
// self-solves each derived challenge.
|
||||||
|
|
||||||
|
// effort / hashx_compile (nonce search):
|
||||||
|
"challenge_base_hex": "abcd", // challenge = base || little_endian(nonce, nonce_bytes)
|
||||||
|
"nonce_bytes": 8,
|
||||||
|
"nonce_start": 0,
|
||||||
|
"target_effort": 1000, // effort: stop when achieved >= target
|
||||||
|
"max_attempts": 5000000 // effort: safety cap
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Only the fields relevant to the operation are present. Unknown fields must be
|
||||||
|
ignored.
|
||||||
|
|
||||||
|
## Result (stdout)
|
||||||
|
|
||||||
|
```jsonc
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"ok": true,
|
||||||
|
"impl": { "name": "equix-c", "version": "1.0.0", "commit": "b7bb7d9",
|
||||||
|
"runtime_effective": "compiled" },
|
||||||
|
"operation": "solve",
|
||||||
|
"runtime_requested": "try-compile",
|
||||||
|
"runtime_effective": "compiled", // may differ (try-compile fallback)
|
||||||
|
"env": { "os": "linux", "compiler": "gcc-13.3.0",
|
||||||
|
"cpu": "Intel(R) Xeon(R) ... @ 2.10GHz", // device model string
|
||||||
|
"arch": "x86_64", // x86_64 | aarch64 | ...
|
||||||
|
"device": "cpu", // "cpu" | "gpu"
|
||||||
|
"os_version": "6.18.5" }, // kernel release; folded into the auto device label
|
||||||
|
"runs": [ // one entry per TIMED rep (warmups excluded)
|
||||||
|
{ "index": 0, "wall_ns": 7582269, "solutions": 4, "compile_ns": 0,
|
||||||
|
"attempts": 0, "achieved_effort": 0, "verify_result": null }
|
||||||
|
],
|
||||||
|
"solutions_hex": ["6f27...", "..."], // solve: final-rep solutions; effort: the winning solution; else null
|
||||||
|
"winning_nonce_hex": "0300000000000000", // effort only, OPTIONAL: wire bytes (LE, nonce_bytes long)
|
||||||
|
// of the winning token's nonce — lets the harness
|
||||||
|
// measure message sizes vs difficulty
|
||||||
|
"peak_rss_kb": 4548, // whole-process high-water; Linux KB
|
||||||
|
"error": null
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Field semantics per operation
|
||||||
|
|
||||||
|
| operation | must populate | notes |
|
||||||
|
|-----------|---------------|-------|
|
||||||
|
| `solve` | `runs[].wall_ns`, `runs[].solutions`, `solutions_hex` | `solutions_hex` (final rep) enables the interop cross-check. |
|
||||||
|
| `verify` | `runs[].wall_ns`, `runs[].verify_result`, `runs[].solutions` (1/0) | `verify_result` ∈ `OK`/`CHALLENGE`/`ORDER`/`PARTIAL_SUM`/`FINAL_SUM` (or impl-specific string). |
|
||||||
|
| `effort` | `runs[].wall_ns`, `runs[].attempts`, `runs[].achieved_effort` | Effort formula: BLAKE2b-256(`challenge‖solution_bytes`), `achieved = (2^32-1)/hash32` (hash32 = first 4 bytes big-endian). **Must match byte-for-byte** — the cross-check enforces it. Report the winning token via `solutions_hex` + `winning_nonce_hex` when the target was reached (enables message-size measurement). |
|
||||||
|
| `hashx_compile` | `runs[].compile_ns`, `runs[].wall_ns` | `compile_ns` = program-gen + compile; `wall_ns` = one execution. |
|
||||||
|
|
||||||
|
`solution_bytes` = the 8 solution indices as little-endian `uint16` (16 bytes).
|
||||||
|
|
||||||
|
### Device (CPU/GPU) reporting
|
||||||
|
|
||||||
|
Each runner reports the hardware it ran on in `env.cpu` / `env.arch` / `env.device`.
|
||||||
|
The harness stamps this onto every result so figures reflect the executing device
|
||||||
|
and results from different machines can be merged (`combine`). A CPU runner sets
|
||||||
|
`device: "cpu"`; **a GPU implementation would set `device: "gpu"`** and its device
|
||||||
|
name, and it then appears on the comparison figures automatically — no harness
|
||||||
|
change. Equi-X/HashX is CPU-oriented by design, so no GPU runner ships today, but
|
||||||
|
the protocol is ready for one.
|
||||||
|
|
||||||
|
## Manifest
|
||||||
|
|
||||||
|
Register an adapter by adding `<name>.manifest.toml` to a manifest directory
|
||||||
|
(default `adapters/examples/`, override with `--manifests`):
|
||||||
|
|
||||||
|
```toml
|
||||||
|
name = "my-equix"
|
||||||
|
exec = "path/to/runner" # or ["python3", "my_runner.py"]; paths are relative to repo root
|
||||||
|
protocol_version = 1
|
||||||
|
capabilities = ["solve", "verify", "effort", "hashx_compile"]
|
||||||
|
runtimes = ["interpret", "try-compile", "must-compile"]
|
||||||
|
|
||||||
|
[env] # optional environment for the runner process
|
||||||
|
MY_VAR = "value"
|
||||||
|
```
|
||||||
|
|
||||||
|
The harness skips (with a warning) any cell whose operation/runtime is not in the
|
||||||
|
adapter's declared `capabilities`/`runtimes`, and any adapter whose `exec` is not
|
||||||
|
found — so a partially built tree degrades gracefully instead of crashing.
|
||||||
|
|
||||||
|
See `manifest.schema.json` for a machine-readable schema.
|
||||||
10
tools/benchmarks/Equi-X/adapters/examples/c.manifest.toml
Normal file
10
tools/benchmarks/Equi-X/adapters/examples/c.manifest.toml
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
# Adapter manifest for the reference C implementation (tevador/equix + hashx).
|
||||||
|
name = "equix-c"
|
||||||
|
exec = "build/runners/c/equix_runner"
|
||||||
|
protocol_version = 1
|
||||||
|
capabilities = ["solve", "verify", "effort", "hashx_compile"]
|
||||||
|
runtimes = ["interpret", "try-compile", "must-compile"]
|
||||||
|
|
||||||
|
# Optional provenance passed to the runner as environment variables.
|
||||||
|
[env]
|
||||||
|
EQUIX_C_VERSION = "1.0.0"
|
||||||
@ -0,0 +1,9 @@
|
|||||||
|
# Adapter manifest for the Rust implementation (Tor arti equix + hashx crates).
|
||||||
|
name = "equix-rust"
|
||||||
|
exec = "runners/rust/target/release/equix_runner"
|
||||||
|
protocol_version = 1
|
||||||
|
capabilities = ["solve", "verify", "effort", "hashx_compile"]
|
||||||
|
runtimes = ["interpret", "try-compile", "must-compile"]
|
||||||
|
|
||||||
|
[env]
|
||||||
|
EQUIX_RUST_VERSION = "0.7.0"
|
||||||
41
tools/benchmarks/Equi-X/adapters/manifest.schema.json
Normal file
41
tools/benchmarks/Equi-X/adapters/manifest.schema.json
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://github.com/madxor/equi-x/adapters/manifest.schema.json",
|
||||||
|
"title": "Equi-X benchmark adapter manifest",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["name", "exec"],
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"name": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Unique implementation name, referenced by configs' run.impls."
|
||||||
|
},
|
||||||
|
"exec": {
|
||||||
|
"description": "Runner argv. A string is a single-element argv. Paths are resolved relative to the repo root.",
|
||||||
|
"oneOf": [
|
||||||
|
{ "type": "string" },
|
||||||
|
{ "type": "array", "items": { "type": "string" }, "minItems": 1 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"protocol_version": {
|
||||||
|
"type": "integer",
|
||||||
|
"default": 1,
|
||||||
|
"description": "Wire protocol version the runner speaks."
|
||||||
|
},
|
||||||
|
"capabilities": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "enum": ["solve", "verify", "effort", "hashx_compile"] },
|
||||||
|
"description": "Operations this runner supports; unsupported cells are skipped."
|
||||||
|
},
|
||||||
|
"runtimes": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "enum": ["interpret", "try-compile", "must-compile"] },
|
||||||
|
"description": "HashX runtimes this runner supports."
|
||||||
|
},
|
||||||
|
"env": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": { "type": "string" },
|
||||||
|
"description": "Extra environment variables set for the runner process."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
35
tools/benchmarks/Equi-X/configs/compiler_flags.toml
Normal file
35
tools/benchmarks/Equi-X/configs/compiler_flags.toml
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
# Compare C-runner performance across compiler/optimization flags.
|
||||||
|
# First build the variants: ./scripts/build_variants.sh
|
||||||
|
# Impls that were not built (e.g. a missing compiler) are skipped with a warning.
|
||||||
|
[run]
|
||||||
|
# Flag differences are small and JIT-dominated, so use a generous rep count to
|
||||||
|
# keep the per-variant median stable enough to rank them meaningfully.
|
||||||
|
warmup = 5
|
||||||
|
repetitions = 30
|
||||||
|
impls = [
|
||||||
|
"equix-c-gcc-o0",
|
||||||
|
"equix-c-gcc-o2",
|
||||||
|
"equix-c-gcc-o3",
|
||||||
|
"equix-c-gcc-o3-native",
|
||||||
|
"equix-c-gcc-o3-lto",
|
||||||
|
"equix-c-clang-o3",
|
||||||
|
"equix-c-clang-o3-native",
|
||||||
|
]
|
||||||
|
|
||||||
|
# Solve dominates PoW cost; compare interpreter and JIT across flag sets.
|
||||||
|
[[jobs]]
|
||||||
|
operation = "solve"
|
||||||
|
runtimes = ["interpret", "try-compile"]
|
||||||
|
challenges = ["deadbeef", "0102030405060708"]
|
||||||
|
|
||||||
|
# Program generation + JIT compile cost can also shift with flags.
|
||||||
|
[[jobs]]
|
||||||
|
operation = "hashx_compile"
|
||||||
|
runtimes = ["must-compile"]
|
||||||
|
challenges = ["deadbeef"]
|
||||||
|
|
||||||
|
# All variants compute identical solutions -> the interop check still holds.
|
||||||
|
[crosscheck]
|
||||||
|
enabled = true
|
||||||
|
challenges = ["deadbeef"]
|
||||||
|
pairs = [["equix-c-gcc-o3", "equix-c-clang-o3"]]
|
||||||
28
tools/benchmarks/Equi-X/configs/dos_protection.toml
Normal file
28
tools/benchmarks/Equi-X/configs/dos_protection.toml
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
# DoS-protection evaluation: measures the attacker-solve vs defender-verify
|
||||||
|
# asymmetry across a range of efforts to answer "is Equi-X an effective DoS
|
||||||
|
# defense on THIS system, and at what effort?". A few minutes to run.
|
||||||
|
[run]
|
||||||
|
warmup = 2
|
||||||
|
repetitions = 8
|
||||||
|
impls = ["equix-c", "equix-rust"]
|
||||||
|
|
||||||
|
# Defender cost: fast verify (both interpreter and JIT so you can compare).
|
||||||
|
[[jobs]]
|
||||||
|
operation = "verify"
|
||||||
|
runtimes = ["interpret", "try-compile"]
|
||||||
|
challenges = ["deadbeef"]
|
||||||
|
|
||||||
|
# Attacker cost: measured time to craft one accepted token at each effort.
|
||||||
|
[[jobs]]
|
||||||
|
operation = "effort"
|
||||||
|
runtimes = ["try-compile"]
|
||||||
|
bases = ["abcd"]
|
||||||
|
targets = [200, 1000, 4000]
|
||||||
|
nonce_bytes = 8
|
||||||
|
max_attempts = 50000000
|
||||||
|
repetitions = 1
|
||||||
|
|
||||||
|
[crosscheck]
|
||||||
|
enabled = true
|
||||||
|
challenges = ["deadbeef"]
|
||||||
|
pairs = [["equix-c", "equix-rust"], ["equix-rust", "equix-c"]]
|
||||||
77
tools/benchmarks/Equi-X/configs/full.toml
Normal file
77
tools/benchmarks/Equi-X/configs/full.toml
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
# Full sweep: all runtimes x all operations x multiple challenges + an effort
|
||||||
|
# sweep, with enough repetitions for stable medians. Minutes-scale.
|
||||||
|
[run]
|
||||||
|
warmup = 8
|
||||||
|
repetitions = 100
|
||||||
|
impls = ["equix-c", "equix-rust"]
|
||||||
|
|
||||||
|
# --- solve: interpreter vs JIT. vary_challenge = each listed value is a SEED;
|
||||||
|
# every rep solves a fresh challenge from a SHA-256 chain, so the medians and
|
||||||
|
# p95 reflect the spread ACROSS challenges, not one fixed instance. Challenge
|
||||||
|
# generation is excluded from the timing. ---
|
||||||
|
[[jobs]]
|
||||||
|
operation = "solve"
|
||||||
|
runtimes = ["interpret", "try-compile", "must-compile"]
|
||||||
|
challenges = ["deadbeef", "0000000000000002", "cafe", "0102030405060708"]
|
||||||
|
vary_challenge = true
|
||||||
|
|
||||||
|
# --- verify: cheap; in seed mode the runner self-solves each derived challenge
|
||||||
|
# (setup solve excluded from timing), so no harness-resolved solution. ---
|
||||||
|
[[jobs]]
|
||||||
|
operation = "verify"
|
||||||
|
runtimes = ["interpret", "try-compile", "must-compile"]
|
||||||
|
challenges = ["deadbeef", "0000000000000002", "cafe"]
|
||||||
|
vary_challenge = true
|
||||||
|
|
||||||
|
# --- compile isolation: interpreter (program-gen only) vs JIT compile ---
|
||||||
|
[[jobs]]
|
||||||
|
operation = "hashx_compile"
|
||||||
|
runtimes = ["interpret", "must-compile"]
|
||||||
|
challenges = ["deadbeef", "0102030405060708"]
|
||||||
|
|
||||||
|
# --- effort/difficulty sweep: cost grows ~linearly with target effort ---
|
||||||
|
[[jobs]]
|
||||||
|
operation = "effort"
|
||||||
|
runtimes = ["try-compile"]
|
||||||
|
bases = ["abcd"]
|
||||||
|
targets = [100, 1000, 10000]
|
||||||
|
nonce_bytes = 8
|
||||||
|
nonce_start = 0
|
||||||
|
max_attempts = 20000000
|
||||||
|
repetitions = 10
|
||||||
|
|
||||||
|
[crosscheck]
|
||||||
|
enabled = true
|
||||||
|
challenges = ["deadbeef", "cafe", "0000000000000002"]
|
||||||
|
pairs = [["equix-c", "equix-rust"], ["equix-rust", "equix-c"]]
|
||||||
|
|
||||||
|
# --- concurrency / saturation: MEASURED sustained solve & verify capacity ---
|
||||||
|
# Runs N worker processes at once (N = 1,2,4,... up to the core count) and
|
||||||
|
# measures aggregate throughput, so the report can state the machine's real
|
||||||
|
# parallel capacity and the saturation knee. Additive to the per-core DoS
|
||||||
|
# estimate — it never overwrites it. max_workers = 0 means os.cpu_count().
|
||||||
|
[concurrency]
|
||||||
|
enabled = true
|
||||||
|
operations = ["solve", "verify"]
|
||||||
|
challenge = "deadbeef"
|
||||||
|
reps = 50
|
||||||
|
warmup = 5
|
||||||
|
max_workers = 0
|
||||||
|
levels = []
|
||||||
|
|
||||||
|
# --- mining rate vs difficulty: MEASURED whole-machine token mint rate ---
|
||||||
|
# Basis for "control the mint rate by setting difficulty". Uses the fastest
|
||||||
|
# solver (open-network case). Each 1-core point pools independent searches from
|
||||||
|
# distinct nonce ranges; the machine point streams tokens_per_worker searches
|
||||||
|
# per core concurrently and pools tokens over busy time (failed searches
|
||||||
|
# charged to the denominator).
|
||||||
|
[mining]
|
||||||
|
enabled = true
|
||||||
|
impls = ["equix-rust"]
|
||||||
|
challenge_base = "abcd"
|
||||||
|
efforts = [100, 300, 1000, 3000]
|
||||||
|
samples = 50
|
||||||
|
workers = 0
|
||||||
|
tokens_per_worker = 5
|
||||||
|
nonce_bytes = 8
|
||||||
|
max_attempts = 1500000
|
||||||
24
tools/benchmarks/Equi-X/configs/mining.toml
Normal file
24
tools/benchmarks/Equi-X/configs/mining.toml
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
# Mining-rate benchmark: measure the whole-machine token mint rate vs difficulty,
|
||||||
|
# using the fastest solver (the open-network case). Isolated so it can be run
|
||||||
|
# alone under idle conditions:
|
||||||
|
# python -m equix_bench run --config configs/mining.toml --out results/mining
|
||||||
|
[run]
|
||||||
|
impls = ["equix-rust"]
|
||||||
|
|
||||||
|
[crosscheck]
|
||||||
|
enabled = false
|
||||||
|
|
||||||
|
[mining]
|
||||||
|
enabled = true
|
||||||
|
impls = ["equix-rust"] # everyone in an open network uses the fastest solver
|
||||||
|
challenge_base = "abcd"
|
||||||
|
# Expected attempts per token grow ~linearly with effort (measured ≈0.3-0.5·E,
|
||||||
|
# with several solutions per solve attempt each drawing an effort value), so a
|
||||||
|
# token at high E is genuinely expensive to mint AND to average. We measure a
|
||||||
|
# 100→3000 curve (30× span, enough to show the ~1/E trend) and extrapolate.
|
||||||
|
efforts = [100, 300, 1000, 3000]
|
||||||
|
samples = 10 # independent 1-core mints per difficulty (distinct nonces)
|
||||||
|
workers = 0 # 0 = all cores, for the whole-machine rate
|
||||||
|
tokens_per_worker = 5 # tokens each concurrent worker streams (averages nonce variance)
|
||||||
|
nonce_bytes = 8
|
||||||
|
max_attempts = 1500000
|
||||||
35
tools/benchmarks/Equi-X/configs/smoke.toml
Normal file
35
tools/benchmarks/Equi-X/configs/smoke.toml
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
# Smoke config: seconds-scale sanity run used by CI and for quick local checks.
|
||||||
|
[run]
|
||||||
|
warmup = 1
|
||||||
|
repetitions = 3
|
||||||
|
impls = ["equix-c", "equix-rust"]
|
||||||
|
|
||||||
|
[[jobs]]
|
||||||
|
operation = "solve"
|
||||||
|
runtimes = ["interpret", "try-compile"]
|
||||||
|
challenges = ["deadbeef"]
|
||||||
|
vary_challenge = true
|
||||||
|
|
||||||
|
[[jobs]]
|
||||||
|
operation = "verify"
|
||||||
|
runtimes = ["interpret"]
|
||||||
|
challenges = ["deadbeef"]
|
||||||
|
vary_challenge = true
|
||||||
|
|
||||||
|
[[jobs]]
|
||||||
|
operation = "hashx_compile"
|
||||||
|
runtimes = ["interpret", "must-compile"]
|
||||||
|
challenges = ["deadbeef"]
|
||||||
|
|
||||||
|
[[jobs]]
|
||||||
|
operation = "effort"
|
||||||
|
runtimes = ["try-compile"]
|
||||||
|
bases = ["abcd"]
|
||||||
|
targets = [50, 200]
|
||||||
|
nonce_bytes = 8
|
||||||
|
max_attempts = 200000
|
||||||
|
|
||||||
|
[crosscheck]
|
||||||
|
enabled = true
|
||||||
|
challenges = ["deadbeef", "cafe"]
|
||||||
|
pairs = [["equix-c", "equix-rust"], ["equix-rust", "equix-c"]]
|
||||||
8
tools/benchmarks/Equi-X/harness/equix_bench/__init__.py
Normal file
8
tools/benchmarks/Equi-X/harness/equix_bench/__init__.py
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
"""Equi-X PoW benchmarking harness.
|
||||||
|
|
||||||
|
Orchestrates language-agnostic runners over a parameter matrix, aggregates
|
||||||
|
timing/cost statistics, cross-checks implementations, and renders comparison
|
||||||
|
reports and plots.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__version__ = "0.1.0"
|
||||||
4
tools/benchmarks/Equi-X/harness/equix_bench/__main__.py
Normal file
4
tools/benchmarks/Equi-X/harness/equix_bench/__main__.py
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
from .cli import main
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
398
tools/benchmarks/Equi-X/harness/equix_bench/cli.py
Normal file
398
tools/benchmarks/Equi-X/harness/equix_bench/cli.py
Normal file
@ -0,0 +1,398 @@
|
|||||||
|
"""Command-line entrypoint: `python -m equix_bench run --config ... --out ...`."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from . import concurrency as concmod
|
||||||
|
from . import config as cfgmod
|
||||||
|
from . import mining as miningmod
|
||||||
|
from . import report as reportmod
|
||||||
|
from . import stats as statsmod
|
||||||
|
from .crosscheck import run_crosscheck
|
||||||
|
from .device import device_from_env
|
||||||
|
from .protocol import JobSpec, Result
|
||||||
|
from .registry import load_manifests
|
||||||
|
from .runner import RunnerError, run
|
||||||
|
|
||||||
|
|
||||||
|
def _repo_root(override: str | None) -> Path:
|
||||||
|
if override:
|
||||||
|
return Path(override).resolve()
|
||||||
|
return Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
def _cpu_model() -> str:
|
||||||
|
from .device import _host_cpu # cross-platform (Linux /proc, macOS sysctl)
|
||||||
|
|
||||||
|
return _host_cpu()
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_verify_solutions(cells, adapters, repo_root):
|
||||||
|
"""Fill solution_hex for verify cells by solving each challenge once with the
|
||||||
|
first capable implementation. Returns (usable_cells, warnings)."""
|
||||||
|
warnings = []
|
||||||
|
cache: dict[str, str | None] = {}
|
||||||
|
solver = None
|
||||||
|
for name, a in adapters.items():
|
||||||
|
if not a.capabilities or "solve" in a.capabilities:
|
||||||
|
if a.available(repo_root):
|
||||||
|
solver = (name, a)
|
||||||
|
break
|
||||||
|
out = []
|
||||||
|
for c in cells:
|
||||||
|
# Seed-mode verify cells self-solve each derived challenge in the runner,
|
||||||
|
# so they need no pre-resolved solution.
|
||||||
|
if c.job.operation != "verify" or c.job.challenge_seed_hex is not None:
|
||||||
|
out.append(c)
|
||||||
|
continue
|
||||||
|
chal = c.job.challenge_hex
|
||||||
|
if chal not in cache:
|
||||||
|
if solver is None:
|
||||||
|
cache[chal] = None
|
||||||
|
else:
|
||||||
|
r = run(solver[1], JobSpec(operation="solve", runtime="try-compile",
|
||||||
|
repetitions=1, warmup=0, challenge_hex=chal),
|
||||||
|
repo_root)
|
||||||
|
sols = r.solutions_hex or []
|
||||||
|
cache[chal] = sols[0] if sols else None
|
||||||
|
sol = cache[chal]
|
||||||
|
if sol is None:
|
||||||
|
warnings.append(f"verify skipped for challenge {chal}: no solution found")
|
||||||
|
continue
|
||||||
|
c.job.solution_hex = sol
|
||||||
|
out.append(c)
|
||||||
|
return out, warnings
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_run(args) -> int:
|
||||||
|
repo_root = _repo_root(args.root)
|
||||||
|
if args.manifests:
|
||||||
|
manifest_dirs = [Path(args.manifests)]
|
||||||
|
else:
|
||||||
|
# built-in adapters + generated compiler-flag variants (if any)
|
||||||
|
manifest_dirs = [repo_root / "adapters" / "examples", repo_root / "adapters" / "generated"]
|
||||||
|
adapters = load_manifests(manifest_dirs)
|
||||||
|
if not adapters:
|
||||||
|
print(f"error: no adapter manifests found in {manifest_dirs}", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
# Keep only adapters whose runner is actually built/available.
|
||||||
|
available = {n: a for n, a in adapters.items() if a.available(repo_root)}
|
||||||
|
for n in adapters:
|
||||||
|
if n not in available:
|
||||||
|
print(f"warning: adapter '{n}' runner not found; skipping", file=sys.stderr)
|
||||||
|
|
||||||
|
config = cfgmod.load_config(Path(args.config))
|
||||||
|
out_dir = Path(args.out)
|
||||||
|
|
||||||
|
# ---- cross-check only ----
|
||||||
|
if args.crosscheck_only:
|
||||||
|
challenges = config.crosscheck.get("challenges", ["deadbeef", "cafe"])
|
||||||
|
pairs = [tuple(p) for p in config.crosscheck.get("pairs", [])] or None
|
||||||
|
from .crosscheck import _pairs
|
||||||
|
pair_list = _pairs(config.crosscheck.get("pairs", []), list(available.keys()))
|
||||||
|
checks, ok = run_crosscheck(available, repo_root, challenges, pair_list)
|
||||||
|
for c in checks:
|
||||||
|
print(f"[{'PASS' if c.passed else 'FAIL'}] {c.kind}: {c.detail}")
|
||||||
|
print(f"\nCross-check overall: {'PASS' if ok else 'FAIL'}")
|
||||||
|
return 0 if ok else 1
|
||||||
|
|
||||||
|
# ---- full run ----
|
||||||
|
cells, warns = cfgmod.expand(config, available)
|
||||||
|
for w in warns:
|
||||||
|
print(f"warning: {w}", file=sys.stderr)
|
||||||
|
cells, vwarns = _resolve_verify_solutions(cells, available, repo_root)
|
||||||
|
for w in vwarns:
|
||||||
|
print(f"warning: {w}", file=sys.stderr)
|
||||||
|
|
||||||
|
print(f"Running {len(cells)} cells across {len(available)} implementations...")
|
||||||
|
all_stats = []
|
||||||
|
raw = []
|
||||||
|
for i, c in enumerate(cells, 1):
|
||||||
|
adapter = available[c.impl]
|
||||||
|
try:
|
||||||
|
result = run(adapter, c.job, repo_root, timeout=args.timeout)
|
||||||
|
except RunnerError as e:
|
||||||
|
print(f" [{i}/{len(cells)}] {c.impl} {c.group} FAILED: {e}", file=sys.stderr)
|
||||||
|
continue
|
||||||
|
# Device identity: derived from what the runner reported (accurate even
|
||||||
|
# for a remote or GPU runner), with the CLI label override applied.
|
||||||
|
device = device_from_env(result.env, override_label=args.device_label)
|
||||||
|
# Enrich the raw record so a run is self-contained for later `combine`.
|
||||||
|
result.raw["_label"] = c.label
|
||||||
|
result.raw["_device"] = device
|
||||||
|
result.raw["_impl"] = c.impl
|
||||||
|
result.raw["_group"] = c.group
|
||||||
|
raw.append(result.raw)
|
||||||
|
st = statsmod.summarize(c.impl, c.group, c.job.runtime, c.label, result, device)
|
||||||
|
all_stats.append(st)
|
||||||
|
tag = f"{c.impl}/{c.group}/{c.job.runtime} {c.label}"
|
||||||
|
if st.ok:
|
||||||
|
print(f" [{i}/{len(cells)}] {tag}: median {st.median_ns/1e6:.3f} ms")
|
||||||
|
else:
|
||||||
|
print(f" [{i}/{len(cells)}] {tag}: ERROR {st.error}", file=sys.stderr)
|
||||||
|
|
||||||
|
# cross-check
|
||||||
|
checks = []
|
||||||
|
if config.crosscheck.get("enabled", True) and len(available) >= 2:
|
||||||
|
from .crosscheck import _pairs
|
||||||
|
pair_list = _pairs(config.crosscheck.get("pairs", []), list(available.keys()))
|
||||||
|
challenges = config.crosscheck.get("challenges", ["deadbeef"])
|
||||||
|
checks, _ = run_crosscheck(available, repo_root, challenges, pair_list)
|
||||||
|
|
||||||
|
# concurrency / saturation benchmark (opt-in via a [concurrency] config block).
|
||||||
|
# Measures sustained parallel solve/verify capacity; additive to the per-core
|
||||||
|
# DoS estimate, which it never modifies.
|
||||||
|
concurrency = None
|
||||||
|
conc_cfg = config.raw.get("concurrency", {})
|
||||||
|
if conc_cfg.get("enabled", False):
|
||||||
|
print("Running concurrency / saturation ladder...")
|
||||||
|
resolver = lambda env: device_from_env(env, override_label=args.device_label).get("label", "host")
|
||||||
|
# Default to the impls this run selected (not every built variant); the
|
||||||
|
# [concurrency] block can still name its own `impls` to override.
|
||||||
|
conc_adapters = {n: available[n] for n in config.impls if n in available} or available
|
||||||
|
concurrency = concmod.run_concurrency(conc_cfg, conc_adapters, repo_root, resolver, args.timeout)
|
||||||
|
for r in concurrency:
|
||||||
|
if r.error:
|
||||||
|
print(f" concurrency {r.impl}/{r.operation}: {r.error}", file=sys.stderr)
|
||||||
|
else:
|
||||||
|
print(f" concurrency {r.impl}/{r.operation}: peak "
|
||||||
|
f"{r.peak_ops_per_sec:,.0f} ops/s at {r.knee_workers} workers")
|
||||||
|
|
||||||
|
# mining-rate benchmark (opt-in via a [mining] config block): measures
|
||||||
|
# whole-machine token production vs difficulty, the basis for rate control.
|
||||||
|
mining = None
|
||||||
|
mining_cfg = config.raw.get("mining", {})
|
||||||
|
if mining_cfg.get("enabled", False):
|
||||||
|
print("Running mining-rate / difficulty ladder...")
|
||||||
|
resolver = lambda env: device_from_env(env, override_label=args.device_label).get("label", "host")
|
||||||
|
mine_adapters = {n: available[n] for n in config.impls if n in available} or available
|
||||||
|
mining = miningmod.run_mining(mining_cfg, mine_adapters, repo_root, resolver, args.timeout)
|
||||||
|
for r in mining:
|
||||||
|
for p in r.points:
|
||||||
|
print(f" mining {r.impl} E={p.effort}: {p.tokens_per_sec_1core:,.2f} tok/s/core, "
|
||||||
|
f"{p.tokens_per_sec_machine:,.2f} tok/s machine ({p.ok_workers} workers)")
|
||||||
|
|
||||||
|
devices = sorted({s.device_label for s in all_stats})
|
||||||
|
meta = {
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||||
|
"config": args.config,
|
||||||
|
"cpu": _cpu_model(),
|
||||||
|
"nproc": os.cpu_count() or "?",
|
||||||
|
"devices": devices,
|
||||||
|
}
|
||||||
|
out_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
(out_dir / "run_meta.json").write_text(json.dumps({
|
||||||
|
**meta,
|
||||||
|
"device_records": {s.device_label: {"type": s.device_type, "name": s.device_name,
|
||||||
|
"arch": s.device_arch} for s in all_stats},
|
||||||
|
}, indent=2))
|
||||||
|
if concurrency:
|
||||||
|
concmod.write_csv(concurrency, out_dir / "concurrency.csv")
|
||||||
|
if mining:
|
||||||
|
miningmod.write_csv(mining, out_dir / "mining.csv")
|
||||||
|
reportmod.generate(all_stats, checks, raw, out_dir, meta, concurrency=concurrency, mining=mining)
|
||||||
|
print(f"\nReport written to {out_dir/'report.md'} (plots in {out_dir/'plots'})")
|
||||||
|
|
||||||
|
if checks and not all(c.passed for c in checks):
|
||||||
|
print("Cross-check FAILED", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _load_cells_from_raw(raw_list: list[dict]) -> list[statsmod.CellStats]:
|
||||||
|
"""Rebuild CellStats from enriched raw records (each carries _label/_device)."""
|
||||||
|
out = []
|
||||||
|
for d in raw_list:
|
||||||
|
try:
|
||||||
|
result = Result.from_dict(d)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
label = d.get("_label", {})
|
||||||
|
device = d.get("_device", {})
|
||||||
|
impl = d.get("_impl", result.impl_name)
|
||||||
|
group = d.get("_group", result.operation)
|
||||||
|
out.append(statsmod.summarize(impl, group, result.runtime_requested, label, result, device))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _discover_runs(root: Path) -> list[Path]:
|
||||||
|
"""Every run-output directory under `root`, identified by its `raw/results.json`.
|
||||||
|
Layout-agnostic: works whether devices are laid out as `<root>/<device>/main`,
|
||||||
|
`<root>/<host>/results/main`, or arbitrary rsync'd trees — a run is anything
|
||||||
|
with a raw record file, and device identity comes from the records, not paths."""
|
||||||
|
runs = {p.parent.parent for p in root.rglob("raw/results.json")}
|
||||||
|
return sorted(runs)
|
||||||
|
|
||||||
|
|
||||||
|
def _dedup_key(r: dict) -> tuple:
|
||||||
|
"""Identity of one measured cell for de-duplication across discovered runs:
|
||||||
|
(device, impl, operation, runtime, label). Includes runtime because two
|
||||||
|
runtimes of the same op/challenge share _group and _label and would otherwise
|
||||||
|
collide (dropping one)."""
|
||||||
|
dev = (r.get("_device") or {}).get("label", "")
|
||||||
|
try:
|
||||||
|
res = Result.from_dict(r)
|
||||||
|
op, rt = res.operation, res.runtime_requested
|
||||||
|
except (ValueError, KeyError, TypeError):
|
||||||
|
op, rt = r.get("operation", ""), r.get("runtime_requested", "")
|
||||||
|
return (dev, r.get("_impl"), op, rt, json.dumps(r.get("_label", {}), sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_runs(inputs: list[Path]):
|
||||||
|
"""Load and de-duplicate raw records + concurrency/mining results across runs.
|
||||||
|
When the same cell (or device's concurrency/mining ladder) appears in more
|
||||||
|
than one run, the record from the newest run (by run_meta timestamp) wins, so
|
||||||
|
re-runs replace rather than double-count. Returns (raw, conc, mining, seen_dirs)."""
|
||||||
|
from . import concurrency as concmod
|
||||||
|
from . import mining as miningmod
|
||||||
|
|
||||||
|
raw_by_key: dict[tuple, tuple[str, dict]] = {} # key -> (ts, record)
|
||||||
|
conc_by_key: dict[tuple, tuple[str, Any]] = {}
|
||||||
|
mine_by_key: dict[tuple, tuple[str, Any]] = {}
|
||||||
|
seen_dirs: list[Path] = []
|
||||||
|
for d in inputs:
|
||||||
|
raw_path = d / "raw" / "results.json"
|
||||||
|
if not raw_path.exists():
|
||||||
|
print(f"warning: skipping '{d}' (no raw/results.json)", file=sys.stderr)
|
||||||
|
continue
|
||||||
|
seen_dirs.append(d)
|
||||||
|
meta_path = d / "run_meta.json"
|
||||||
|
ts = ""
|
||||||
|
if meta_path.exists():
|
||||||
|
try:
|
||||||
|
ts = json.loads(meta_path.read_text()).get("timestamp", "")
|
||||||
|
except (ValueError, OSError):
|
||||||
|
ts = ""
|
||||||
|
for r in json.loads(raw_path.read_text()):
|
||||||
|
k = _dedup_key(r)
|
||||||
|
if k not in raw_by_key or ts >= raw_by_key[k][0]:
|
||||||
|
raw_by_key[k] = (ts, r)
|
||||||
|
cpath = d / "concurrency.csv"
|
||||||
|
if cpath.exists():
|
||||||
|
for cr in concmod.read_csv(cpath):
|
||||||
|
k = (cr.device, cr.impl, cr.operation)
|
||||||
|
if k not in conc_by_key or ts >= conc_by_key[k][0]:
|
||||||
|
conc_by_key[k] = (ts, cr)
|
||||||
|
mpath = d / "mining.csv"
|
||||||
|
if mpath.exists():
|
||||||
|
for mr in miningmod.read_csv(mpath):
|
||||||
|
k = (mr.device, mr.impl, mr.challenge_base)
|
||||||
|
if k not in mine_by_key or ts >= mine_by_key[k][0]:
|
||||||
|
mine_by_key[k] = (ts, mr)
|
||||||
|
raw = [rec for _ts, rec in raw_by_key.values()]
|
||||||
|
conc = [cr for _ts, cr in conc_by_key.values()]
|
||||||
|
mining = [mr for _ts, mr in mine_by_key.values()]
|
||||||
|
return raw, conc, mining, seen_dirs
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_combine(args) -> int:
|
||||||
|
"""Merge multiple prior runs into a single faceted (per-device) report — with
|
||||||
|
the concurrency and mining sections/figures carried across all runs. Inputs
|
||||||
|
are either listed explicitly (--inputs) or discovered under a tree (--root)."""
|
||||||
|
inputs: list[Path] = [Path(p) for p in (args.inputs or [])]
|
||||||
|
if args.root:
|
||||||
|
discovered = _discover_runs(Path(args.root))
|
||||||
|
if not discovered:
|
||||||
|
print(f"error: no run directories (with raw/results.json) found under "
|
||||||
|
f"'{args.root}'", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
inputs.extend(discovered)
|
||||||
|
if not inputs:
|
||||||
|
print("error: provide run dirs via --inputs DIR... or a tree via --root DIR",
|
||||||
|
file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
# De-dup identical paths (e.g. --root and --inputs overlapping) preserving order.
|
||||||
|
seen: set[str] = set()
|
||||||
|
inputs = [p for p in inputs if not (str(p) in seen or seen.add(str(p)))]
|
||||||
|
|
||||||
|
all_raw, conc, mining, seen_dirs = _collect_runs(inputs)
|
||||||
|
stats = _load_cells_from_raw(all_raw)
|
||||||
|
if not stats:
|
||||||
|
print("error: no usable records found in inputs", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
devices_seen = sorted({s.device_label for s in stats})
|
||||||
|
# Manifest: show exactly which dirs contributed which devices, so a missed
|
||||||
|
# tree can't silently masquerade as full coverage.
|
||||||
|
print(f"Discovered {len(seen_dirs)} run(s) across {len(devices_seen)} device(s):")
|
||||||
|
for d in seen_dirs:
|
||||||
|
try:
|
||||||
|
recs = json.loads((d / "raw" / "results.json").read_text())
|
||||||
|
devs = sorted({(r.get("_device") or {}).get("label", "?") for r in recs})
|
||||||
|
except (ValueError, OSError):
|
||||||
|
devs = ["?"]
|
||||||
|
extra = []
|
||||||
|
if (d / "concurrency.csv").exists():
|
||||||
|
extra.append("concurrency")
|
||||||
|
if (d / "mining.csv").exists():
|
||||||
|
extra.append("mining")
|
||||||
|
tail = f" (+{', '.join(extra)})" if extra else ""
|
||||||
|
print(f" - {d} -> {', '.join(devs)}{tail}")
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||||
|
"config": f"combine of {len(seen_dirs)} runs",
|
||||||
|
"cpu": ", ".join(devices_seen),
|
||||||
|
"nproc": "?",
|
||||||
|
"devices": devices_seen,
|
||||||
|
}
|
||||||
|
out_dir = Path(args.out)
|
||||||
|
reportmod.generate(stats, [], all_raw, out_dir, meta,
|
||||||
|
concurrency=conc or None, mining=mining or None)
|
||||||
|
print(f"\nCombined report for devices {devices_seen} -> {out_dir/'report.md'}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None) -> int:
|
||||||
|
p = argparse.ArgumentParser(prog="equix_bench", description="Equi-X PoW benchmarking framework")
|
||||||
|
sub = p.add_subparsers(dest="cmd", required=True)
|
||||||
|
|
||||||
|
r = sub.add_parser("run", help="run a benchmark config")
|
||||||
|
r.add_argument("--config", required=True, help="path to a TOML config")
|
||||||
|
r.add_argument("--out", default="results", help="output directory")
|
||||||
|
r.add_argument("--root", default=None, help="repo root (default: inferred)")
|
||||||
|
r.add_argument("--manifests", default=None, help="adapter manifest directory")
|
||||||
|
r.add_argument("--timeout", type=float, default=900.0, help="per-cell timeout (s)")
|
||||||
|
r.add_argument("--crosscheck-only", action="store_true", help="only run the interop cross-check")
|
||||||
|
r.add_argument("--device-label", "--cpu-label", dest="device_label", default=None,
|
||||||
|
help="human label for the executing device/CPU (default: auto from CPU model)")
|
||||||
|
r.set_defaults(func=cmd_run)
|
||||||
|
|
||||||
|
c = sub.add_parser("combine", help="merge multiple runs into per-device comparison figures")
|
||||||
|
c.add_argument("--inputs", nargs="+", default=None, help="run output directories to merge")
|
||||||
|
c.add_argument("--root", default=None,
|
||||||
|
help="auto-discover every run (dir with raw/results.json) under this tree")
|
||||||
|
c.add_argument("--out", default="combined", help="output directory")
|
||||||
|
c.set_defaults(func=cmd_combine)
|
||||||
|
|
||||||
|
args = p.parse_args(argv)
|
||||||
|
|
||||||
|
# Clean Ctrl+C: kill any live runner subprocesses (worker threads in the
|
||||||
|
# concurrency/mining pools never receive KeyboardInterrupt themselves, so the
|
||||||
|
# handler — which always runs in the main thread — reaps them promptly so a
|
||||||
|
# blocked pool.shutdown can't hang), then raise KeyboardInterrupt so the run
|
||||||
|
# unwinds normally. We catch it below to exit 130 without a traceback.
|
||||||
|
from .runner import terminate_all_children
|
||||||
|
|
||||||
|
def _on_sigint(signum, frame):
|
||||||
|
terminate_all_children()
|
||||||
|
raise KeyboardInterrupt
|
||||||
|
|
||||||
|
signal.signal(signal.SIGINT, _on_sigint)
|
||||||
|
try:
|
||||||
|
return args.func(args)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
terminate_all_children()
|
||||||
|
print("\nInterrupted (Ctrl+C) — stopped; runner subprocesses killed.", file=sys.stderr)
|
||||||
|
return 130
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
349
tools/benchmarks/Equi-X/harness/equix_bench/concurrency.py
Normal file
349
tools/benchmarks/Equi-X/harness/equix_bench/concurrency.py
Normal file
@ -0,0 +1,349 @@
|
|||||||
|
"""Concurrency / saturation benchmark: the machine's *measured* sustained
|
||||||
|
solve and verify capacity under parallel load.
|
||||||
|
|
||||||
|
`dosprotect.py` reports *per-core* figures derived as 1/latency from a single
|
||||||
|
serial operation -- it never runs anything concurrently, so multiplying by the
|
||||||
|
core count over-estimates (Equi-X solving is memory-hard, so N parallel solvers
|
||||||
|
contend for cache/memory bandwidth and scale sub-linearly).
|
||||||
|
|
||||||
|
This module answers the complementary, measured question: run N worker
|
||||||
|
processes at once, for N stepping up a ladder to the core count, and measure the
|
||||||
|
aggregate throughput at each level. That yields:
|
||||||
|
|
||||||
|
* the real sustained solves/sec and verifies/sec the machine handles,
|
||||||
|
* the "knee" -- the worker count at peak throughput, beyond which adding
|
||||||
|
workers stops helping (bandwidth saturated),
|
||||||
|
* a scaling efficiency vs. ideal linear scaling.
|
||||||
|
|
||||||
|
It is additive: it does not touch or replace the per-core DoS estimate.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
import statistics
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Callable, Optional
|
||||||
|
|
||||||
|
from .protocol import JobSpec, Result
|
||||||
|
from .registry import Adapter
|
||||||
|
from .runner import RunnerError, run
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class LevelStat:
|
||||||
|
workers: int # concurrency level attempted
|
||||||
|
ok_workers: int # workers that returned a usable result
|
||||||
|
per_worker_median_s: float # median across workers of each worker's median op time
|
||||||
|
aggregate_ops_per_sec: float # sum over workers of 1/(that worker's median op time)
|
||||||
|
per_worker_ops_per_sec: float # aggregate / ok_workers
|
||||||
|
scaling_efficiency: float # aggregate(N) / (N * single-worker baseline)
|
||||||
|
total_peak_rss_kb: int # summed peak RSS across the concurrent workers
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ConcResult:
|
||||||
|
device: str
|
||||||
|
impl: str
|
||||||
|
operation: str
|
||||||
|
nproc: int
|
||||||
|
reps: int
|
||||||
|
challenge: str
|
||||||
|
baseline_ops_per_sec: float # single-worker (level 1) throughput
|
||||||
|
peak_ops_per_sec: float # best aggregate across levels
|
||||||
|
knee_workers: int # worker count at peak aggregate throughput
|
||||||
|
levels: list[LevelStat] = field(default_factory=list)
|
||||||
|
error: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
def _ladder(max_workers: int, explicit: Optional[list[int]] = None) -> list[int]:
|
||||||
|
"""Worker-count ladder: 1, 2, 4, 8, ... capped at max_workers, with
|
||||||
|
max_workers itself always included (so an 6- or 10-core box gets its top).
|
||||||
|
Explicit lists are clamped to [1, max_workers]; level 1 is always included
|
||||||
|
because it anchors the per-worker baseline every derived figure needs."""
|
||||||
|
if explicit:
|
||||||
|
levels = sorted({n for n in explicit if 1 <= n <= max_workers})
|
||||||
|
if not levels:
|
||||||
|
raise ValueError(
|
||||||
|
f"concurrency levels {explicit} all outside [1, {max_workers}]"
|
||||||
|
)
|
||||||
|
return sorted({1, *levels})
|
||||||
|
levels = []
|
||||||
|
n = 1
|
||||||
|
while n < max_workers:
|
||||||
|
levels.append(n)
|
||||||
|
n *= 2
|
||||||
|
levels.append(max_workers)
|
||||||
|
return sorted(set(levels))
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_median_s(res: Result) -> Optional[float]:
|
||||||
|
"""One worker's median per-op time in seconds (runner-internal timing, which
|
||||||
|
already excludes process startup and warmup). None if the worker had no runs."""
|
||||||
|
if not res.ok or not res.runs:
|
||||||
|
return None
|
||||||
|
walls = [float(r.wall_ns) for r in res.runs if r.wall_ns > 0]
|
||||||
|
if not walls:
|
||||||
|
return None
|
||||||
|
return statistics.median(walls) / 1e9
|
||||||
|
|
||||||
|
|
||||||
|
def _measure_level(
|
||||||
|
adapter: Adapter,
|
||||||
|
make_spec: Callable[[], JobSpec],
|
||||||
|
n: int,
|
||||||
|
repo_root: Path,
|
||||||
|
timeout: float,
|
||||||
|
) -> tuple[list[float], int]:
|
||||||
|
"""Launch n identical workers concurrently; return (per-worker median seconds
|
||||||
|
for the workers that succeeded, summed peak RSS KB across all workers).
|
||||||
|
|
||||||
|
Real subprocesses run in parallel: the thread pool only blocks on their I/O,
|
||||||
|
so contention is measured on the actual runner binary, not in Python."""
|
||||||
|
def one(_i: int) -> Optional[Result]:
|
||||||
|
try:
|
||||||
|
return run(adapter, make_spec(), repo_root, timeout=timeout)
|
||||||
|
except RunnerError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=n) as pool:
|
||||||
|
results = list(pool.map(one, range(n)))
|
||||||
|
|
||||||
|
medians: list[float] = []
|
||||||
|
rss = 0
|
||||||
|
for res in results:
|
||||||
|
if res is None:
|
||||||
|
continue
|
||||||
|
rss += max(0, res.peak_rss_kb)
|
||||||
|
m = _worker_median_s(res)
|
||||||
|
if m and m > 0:
|
||||||
|
medians.append(m)
|
||||||
|
return medians, rss
|
||||||
|
|
||||||
|
|
||||||
|
def _solution_for(adapter: Adapter, challenge: str, repo_root: Path, timeout: float) -> Optional[str]:
|
||||||
|
"""Solve `challenge` once to obtain a solution to feed the verify workers."""
|
||||||
|
try:
|
||||||
|
r = run(adapter, JobSpec(operation="solve", runtime="try-compile",
|
||||||
|
repetitions=1, warmup=0, challenge_hex=challenge),
|
||||||
|
repo_root, timeout=timeout)
|
||||||
|
except RunnerError:
|
||||||
|
return None
|
||||||
|
sols = r.solutions_hex or []
|
||||||
|
return sols[0] if sols else None
|
||||||
|
|
||||||
|
|
||||||
|
def _first_env(adapter: Adapter, challenge: str, repo_root: Path, timeout: float) -> dict[str, Any]:
|
||||||
|
"""Cheap single run purely to learn the executing device (env) for labeling.
|
||||||
|
Uses a 1-rep solve: always valid (a verify probe would need a solution and
|
||||||
|
fail, silently mislabeling the device)."""
|
||||||
|
try:
|
||||||
|
r = run(adapter, JobSpec(operation="solve", runtime="try-compile", repetitions=1,
|
||||||
|
warmup=0, challenge_hex=challenge),
|
||||||
|
repo_root, timeout=timeout)
|
||||||
|
return r.env
|
||||||
|
except RunnerError:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
# Each worker's runner-internal measured window must dwarf the multi-ms
|
||||||
|
# subprocess start skew, or the workers' windows barely overlap and the
|
||||||
|
# "measured under concurrency" number degenerates to serial-rate x N.
|
||||||
|
MIN_WINDOW_S = 0.5
|
||||||
|
|
||||||
|
|
||||||
|
def measure(
|
||||||
|
adapter: Adapter,
|
||||||
|
operation: str,
|
||||||
|
challenge: str,
|
||||||
|
solution_hex: Optional[str],
|
||||||
|
max_workers: int,
|
||||||
|
reps: int,
|
||||||
|
warmup: int,
|
||||||
|
repo_root: Path,
|
||||||
|
device_label: str,
|
||||||
|
timeout: float,
|
||||||
|
levels: Optional[list[int]] = None,
|
||||||
|
min_window_s: float = MIN_WINDOW_S,
|
||||||
|
) -> ConcResult:
|
||||||
|
"""Run the saturation ladder for one (impl, operation) and summarize it."""
|
||||||
|
def make_spec(n_reps: int) -> JobSpec:
|
||||||
|
return JobSpec(
|
||||||
|
operation=operation,
|
||||||
|
runtime="try-compile",
|
||||||
|
repetitions=n_reps,
|
||||||
|
warmup=warmup,
|
||||||
|
challenge_hex=challenge,
|
||||||
|
solution_hex=solution_hex if operation == "verify" else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = ConcResult(
|
||||||
|
device=device_label, impl=adapter.name, operation=operation,
|
||||||
|
nproc=max_workers, reps=reps, challenge=challenge,
|
||||||
|
baseline_ops_per_sec=0.0, peak_ops_per_sec=0.0, knee_workers=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Calibrate: one uncontended run tells us the per-op time, from which we
|
||||||
|
# size reps so every worker's measured window is at least min_window_s
|
||||||
|
# (fast ops like verify at ~17us need tens of thousands of reps to overlap
|
||||||
|
# meaningfully; slow ops like solve already exceed the window with a few).
|
||||||
|
cal_medians, _ = _measure_level(adapter, lambda: make_spec(reps), 1, repo_root, timeout)
|
||||||
|
eff_reps = reps
|
||||||
|
if cal_medians and min_window_s > 0:
|
||||||
|
eff_reps = max(reps, math.ceil(min_window_s / cal_medians[0]))
|
||||||
|
result.reps = eff_reps
|
||||||
|
|
||||||
|
baseline = 0.0
|
||||||
|
for n in _ladder(max_workers, levels):
|
||||||
|
medians, rss = _measure_level(adapter, lambda: make_spec(eff_reps), n, repo_root, timeout)
|
||||||
|
if not medians:
|
||||||
|
result.levels.append(LevelStat(n, 0, 0.0, 0.0, 0.0, 0.0, rss))
|
||||||
|
continue
|
||||||
|
aggregate = sum(1.0 / m for m in medians)
|
||||||
|
if baseline == 0.0:
|
||||||
|
# Per-WORKER throughput anchors ideal scaling; falling back to a
|
||||||
|
# level n>1 must divide by n or every derived figure is n-fold off.
|
||||||
|
baseline = aggregate / n
|
||||||
|
ideal = baseline * n
|
||||||
|
result.levels.append(LevelStat(
|
||||||
|
workers=n,
|
||||||
|
ok_workers=len(medians),
|
||||||
|
per_worker_median_s=statistics.median(medians),
|
||||||
|
aggregate_ops_per_sec=aggregate,
|
||||||
|
per_worker_ops_per_sec=aggregate / len(medians),
|
||||||
|
scaling_efficiency=(aggregate / ideal) if ideal > 0 else 0.0,
|
||||||
|
total_peak_rss_kb=rss,
|
||||||
|
))
|
||||||
|
|
||||||
|
result.baseline_ops_per_sec = baseline
|
||||||
|
usable = [lv for lv in result.levels if lv.aggregate_ops_per_sec > 0]
|
||||||
|
if usable:
|
||||||
|
peak = max(usable, key=lambda lv: lv.aggregate_ops_per_sec)
|
||||||
|
result.peak_ops_per_sec = peak.aggregate_ops_per_sec
|
||||||
|
result.knee_workers = peak.workers
|
||||||
|
else:
|
||||||
|
result.error = "no worker produced a usable measurement"
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def run_concurrency(
|
||||||
|
cfg: dict[str, Any],
|
||||||
|
adapters: dict[str, Adapter],
|
||||||
|
repo_root: Path,
|
||||||
|
device_resolver: Callable[[dict[str, Any]], str],
|
||||||
|
timeout: float,
|
||||||
|
) -> list[ConcResult]:
|
||||||
|
"""Drive the concurrency benchmark from a `[concurrency]` config block.
|
||||||
|
|
||||||
|
cfg keys (all optional): operations (["solve","verify"]), impls ([] = all
|
||||||
|
available), challenge ("deadbeef"), reps (40), warmup (5), max_workers
|
||||||
|
(0 = os.cpu_count()), levels ([] = auto power-of-two ladder)."""
|
||||||
|
import os
|
||||||
|
|
||||||
|
operations = list(cfg.get("operations", ["solve", "verify"]))
|
||||||
|
challenge = str(cfg.get("challenge", "deadbeef"))
|
||||||
|
reps = int(cfg.get("reps", 40))
|
||||||
|
warmup = int(cfg.get("warmup", 5))
|
||||||
|
max_workers = int(cfg.get("max_workers", 0)) or (os.cpu_count() or 4)
|
||||||
|
levels = [int(x) for x in cfg.get("levels", [])] or None
|
||||||
|
want_impls = list(cfg.get("impls", [])) or list(adapters.keys())
|
||||||
|
|
||||||
|
impls = [(n, adapters[n]) for n in want_impls if n in adapters]
|
||||||
|
out: list[ConcResult] = []
|
||||||
|
if not impls:
|
||||||
|
return out
|
||||||
|
|
||||||
|
# Learn the device label once from a cheap probe on the first impl.
|
||||||
|
probe_env = _first_env(impls[0][1], challenge, repo_root, timeout)
|
||||||
|
device_label = device_resolver(probe_env)
|
||||||
|
|
||||||
|
# A verify needs a valid solution; solve once (impl-independent) and reuse.
|
||||||
|
solution_hex = None
|
||||||
|
if "verify" in operations:
|
||||||
|
for _, a in impls:
|
||||||
|
solution_hex = _solution_for(a, challenge, repo_root, timeout)
|
||||||
|
if solution_hex:
|
||||||
|
break
|
||||||
|
|
||||||
|
for op in operations:
|
||||||
|
if op == "verify" and not solution_hex:
|
||||||
|
continue # nothing to verify against; skip rather than error the run
|
||||||
|
for name, adapter in impls:
|
||||||
|
out.append(measure(
|
||||||
|
adapter, op, challenge, solution_hex, max_workers, reps, warmup,
|
||||||
|
repo_root, device_label, timeout, levels,
|
||||||
|
))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def write_csv(results: list[ConcResult], path: Path) -> None:
|
||||||
|
import csv
|
||||||
|
|
||||||
|
with open(path, "w", newline="") as f:
|
||||||
|
w = csv.writer(f)
|
||||||
|
w.writerow([
|
||||||
|
"device", "impl", "operation", "workers", "ok_workers",
|
||||||
|
"per_worker_median_s", "aggregate_ops_per_sec",
|
||||||
|
"per_worker_ops_per_sec", "scaling_efficiency", "total_peak_rss_kb",
|
||||||
|
])
|
||||||
|
for r in results:
|
||||||
|
for lv in r.levels:
|
||||||
|
w.writerow([
|
||||||
|
r.device, r.impl, r.operation, lv.workers, lv.ok_workers,
|
||||||
|
f"{lv.per_worker_median_s:.9f}", f"{lv.aggregate_ops_per_sec:.3f}",
|
||||||
|
f"{lv.per_worker_ops_per_sec:.3f}", f"{lv.scaling_efficiency:.4f}",
|
||||||
|
lv.total_peak_rss_kb,
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def read_csv(path: Path) -> list[ConcResult]:
|
||||||
|
"""Reconstruct ConcResults from a concurrency.csv (the inverse of write_csv),
|
||||||
|
so `combine` can fold in each device's measured saturation ladder.
|
||||||
|
|
||||||
|
The CSV is per-level; the per-result summary fields (baseline/peak/knee/nproc)
|
||||||
|
are re-derived from the levels — baseline is the single-worker aggregate,
|
||||||
|
nproc the top level, and peak/knee the best aggregate — matching how
|
||||||
|
`measure()` computed them originally."""
|
||||||
|
import csv
|
||||||
|
|
||||||
|
def _i(row, k):
|
||||||
|
v = row.get(k, "")
|
||||||
|
return int(v) if v not in ("", None) else 0
|
||||||
|
|
||||||
|
def _f(row, k):
|
||||||
|
v = row.get(k, "")
|
||||||
|
return float(v) if v not in ("", None) else 0.0
|
||||||
|
|
||||||
|
by_key: dict[tuple[str, str, str], ConcResult] = {}
|
||||||
|
with open(path, newline="") as f:
|
||||||
|
for row in csv.DictReader(f):
|
||||||
|
key = (row["device"], row["impl"], row["operation"])
|
||||||
|
res = by_key.get(key)
|
||||||
|
if res is None:
|
||||||
|
res = ConcResult(
|
||||||
|
device=row["device"], impl=row["impl"], operation=row["operation"],
|
||||||
|
nproc=0, reps=0, challenge="", baseline_ops_per_sec=0.0,
|
||||||
|
peak_ops_per_sec=0.0, knee_workers=0,
|
||||||
|
)
|
||||||
|
by_key[key] = res
|
||||||
|
res.levels.append(LevelStat(
|
||||||
|
workers=_i(row, "workers"),
|
||||||
|
ok_workers=_i(row, "ok_workers"),
|
||||||
|
per_worker_median_s=_f(row, "per_worker_median_s"),
|
||||||
|
aggregate_ops_per_sec=_f(row, "aggregate_ops_per_sec"),
|
||||||
|
per_worker_ops_per_sec=_f(row, "per_worker_ops_per_sec"),
|
||||||
|
scaling_efficiency=_f(row, "scaling_efficiency"),
|
||||||
|
total_peak_rss_kb=_i(row, "total_peak_rss_kb"),
|
||||||
|
))
|
||||||
|
for res in by_key.values():
|
||||||
|
res.levels.sort(key=lambda lv: lv.workers)
|
||||||
|
res.nproc = max((lv.workers for lv in res.levels), default=0)
|
||||||
|
base = next((lv for lv in res.levels if lv.workers == 1), None)
|
||||||
|
res.baseline_ops_per_sec = base.aggregate_ops_per_sec if base else 0.0
|
||||||
|
usable = [lv for lv in res.levels if lv.aggregate_ops_per_sec > 0]
|
||||||
|
if usable:
|
||||||
|
peak = max(usable, key=lambda lv: lv.aggregate_ops_per_sec)
|
||||||
|
res.peak_ops_per_sec = peak.aggregate_ops_per_sec
|
||||||
|
res.knee_workers = peak.workers
|
||||||
|
return list(by_key.values())
|
||||||
155
tools/benchmarks/Equi-X/harness/equix_bench/config.py
Normal file
155
tools/benchmarks/Equi-X/harness/equix_bench/config.py
Normal file
@ -0,0 +1,155 @@
|
|||||||
|
"""Config loading + parameter-matrix expansion into concrete runner cells.
|
||||||
|
|
||||||
|
A "cell" is one fully-specified runner invocation: (impl, JobSpec) plus grouping
|
||||||
|
metadata used for aggregation and plotting. Expansion is the cartesian product
|
||||||
|
of impls x operations x runtimes x challenges/targets, filtered by each adapter's
|
||||||
|
declared capabilities/runtimes.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tomllib
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
from .protocol import JobSpec
|
||||||
|
from .registry import Adapter
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Cell:
|
||||||
|
impl: str
|
||||||
|
group: str # operation, used as the primary plot grouping
|
||||||
|
label: dict[str, Any] # human-facing params (challenge, target, ...)
|
||||||
|
job: JobSpec
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Config:
|
||||||
|
warmup: int
|
||||||
|
repetitions: int
|
||||||
|
impls: list[str]
|
||||||
|
jobs: list[dict[str, Any]]
|
||||||
|
crosscheck: dict[str, Any] = field(default_factory=dict)
|
||||||
|
raw: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(path: Path) -> Config:
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
d = tomllib.load(f)
|
||||||
|
run = d.get("run", {})
|
||||||
|
return Config(
|
||||||
|
warmup=int(run.get("warmup", 3)),
|
||||||
|
repetitions=int(run.get("repetitions", 10)),
|
||||||
|
impls=list(run.get("impls", [])),
|
||||||
|
jobs=list(d.get("jobs", [])),
|
||||||
|
crosscheck=dict(d.get("crosscheck", {})),
|
||||||
|
raw=d,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _supports(adapter: Adapter, operation: str, runtime: str) -> bool:
|
||||||
|
if adapter.capabilities and operation not in adapter.capabilities:
|
||||||
|
return False
|
||||||
|
if adapter.runtimes and runtime not in adapter.runtimes:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def expand(config: Config, adapters: dict[str, Adapter]) -> tuple[list[Cell], list[str]]:
|
||||||
|
"""Return (cells, warnings). Cells whose impl/op/runtime is unsupported are
|
||||||
|
skipped with a warning rather than crashing the run."""
|
||||||
|
cells: list[Cell] = []
|
||||||
|
warnings: list[str] = []
|
||||||
|
|
||||||
|
for job in config.jobs:
|
||||||
|
op = job["operation"]
|
||||||
|
runtimes = job.get("runtimes", ["try-compile"])
|
||||||
|
reps = int(job.get("repetitions", config.repetitions))
|
||||||
|
warmup = int(job.get("warmup", config.warmup))
|
||||||
|
|
||||||
|
for impl in config.impls:
|
||||||
|
adapter = adapters.get(impl)
|
||||||
|
if adapter is None:
|
||||||
|
warnings.append(f"impl '{impl}' has no manifest; skipped")
|
||||||
|
continue
|
||||||
|
for runtime in runtimes:
|
||||||
|
if not _supports(adapter, op, runtime):
|
||||||
|
warnings.append(
|
||||||
|
f"{impl} does not support {op}/{runtime}; skipped"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
cells.extend(
|
||||||
|
_expand_job(impl, op, runtime, reps, warmup, job)
|
||||||
|
)
|
||||||
|
return cells, warnings
|
||||||
|
|
||||||
|
|
||||||
|
def _expand_job(
|
||||||
|
impl: str,
|
||||||
|
op: str,
|
||||||
|
runtime: str,
|
||||||
|
reps: int,
|
||||||
|
warmup: int,
|
||||||
|
job: dict[str, Any],
|
||||||
|
) -> list[Cell]:
|
||||||
|
out: list[Cell] = []
|
||||||
|
|
||||||
|
if op in ("solve", "verify", "hashx_compile"):
|
||||||
|
challenges = job.get("challenges", ["deadbeef"])
|
||||||
|
# vary_challenge: treat each listed value as a SEED, deriving a fresh
|
||||||
|
# challenge per rep (SHA-256 chain) so the measurement spans many
|
||||||
|
# challenges instead of assuming one fixed instance.
|
||||||
|
vary = bool(job.get("vary_challenge", False)) and op in ("solve", "verify")
|
||||||
|
for chal in challenges:
|
||||||
|
spec = JobSpec(
|
||||||
|
operation=op,
|
||||||
|
runtime=runtime,
|
||||||
|
repetitions=reps,
|
||||||
|
warmup=warmup,
|
||||||
|
)
|
||||||
|
if vary:
|
||||||
|
spec.challenge_seed_hex = chal
|
||||||
|
else:
|
||||||
|
spec.challenge_hex = chal
|
||||||
|
if op == "hashx_compile":
|
||||||
|
# hashx_compile varies the seed per rep via a nonce counter.
|
||||||
|
spec.challenge_hex = None
|
||||||
|
spec.challenge_base_hex = chal
|
||||||
|
spec.nonce_start = int(job.get("nonce_start", 0))
|
||||||
|
# verify needs a solution_hex (filled by resolve_verify_solutions) —
|
||||||
|
# UNLESS in seed mode, where the runner self-solves each challenge.
|
||||||
|
out.append(
|
||||||
|
Cell(impl=impl, group=op,
|
||||||
|
label={"challenge": chal, "varied": True} if vary else {"challenge": chal},
|
||||||
|
job=spec)
|
||||||
|
)
|
||||||
|
|
||||||
|
elif op == "effort":
|
||||||
|
bases = job.get("bases", ["abcd"])
|
||||||
|
targets = job.get("targets", [1000])
|
||||||
|
for base in bases:
|
||||||
|
for target in targets:
|
||||||
|
spec = JobSpec(
|
||||||
|
operation="effort",
|
||||||
|
runtime=runtime,
|
||||||
|
repetitions=reps,
|
||||||
|
warmup=warmup,
|
||||||
|
challenge_base_hex=base,
|
||||||
|
nonce_bytes=int(job.get("nonce_bytes", 8)),
|
||||||
|
nonce_start=int(job.get("nonce_start", 0)),
|
||||||
|
target_effort=int(target),
|
||||||
|
max_attempts=int(job.get("max_attempts", 5_000_000)),
|
||||||
|
)
|
||||||
|
out.append(
|
||||||
|
Cell(
|
||||||
|
impl=impl,
|
||||||
|
group="effort",
|
||||||
|
label={"base": base, "target_effort": int(target)},
|
||||||
|
job=spec,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"unknown operation in config: {op}")
|
||||||
|
|
||||||
|
return out
|
||||||
116
tools/benchmarks/Equi-X/harness/equix_bench/crosscheck.py
Normal file
116
tools/benchmarks/Equi-X/harness/equix_bench/crosscheck.py
Normal file
@ -0,0 +1,116 @@
|
|||||||
|
"""Cross-implementation correctness gate.
|
||||||
|
|
||||||
|
Two independent checks that any conforming implementations must pass:
|
||||||
|
|
||||||
|
1. Interop: solve a challenge with impl A, then verify EACH returned solution
|
||||||
|
with impl B (both directions). Every solution must verify OK. This proves the
|
||||||
|
implementations compute the same HashX/Equihash puzzle.
|
||||||
|
|
||||||
|
2. Effort agreement: run the effort op on A and B with identical parameters. Since
|
||||||
|
solving is deterministic, both must report the same attempts and achieved
|
||||||
|
effort -- this guards that the BLAKE2b effort preimage is byte-identical
|
||||||
|
across languages.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from .protocol import JobSpec
|
||||||
|
from .registry import Adapter
|
||||||
|
from .runner import run
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Check:
|
||||||
|
kind: str
|
||||||
|
detail: str
|
||||||
|
passed: bool
|
||||||
|
|
||||||
|
|
||||||
|
def _pairs(config_pairs: list[list[str]], impls: list[str]) -> list[tuple[str, str]]:
|
||||||
|
if config_pairs:
|
||||||
|
return [(p[0], p[1]) for p in config_pairs]
|
||||||
|
# default: all ordered pairs
|
||||||
|
out = []
|
||||||
|
for a in impls:
|
||||||
|
for b in impls:
|
||||||
|
if a != b:
|
||||||
|
out.append((a, b))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def run_crosscheck(
|
||||||
|
adapters: dict[str, Adapter],
|
||||||
|
repo_root: Path,
|
||||||
|
challenges: list[str],
|
||||||
|
pairs: list[tuple[str, str]],
|
||||||
|
effort_base: str = "abcd",
|
||||||
|
effort_target: int = 200,
|
||||||
|
effort_max_attempts: int = 200_000,
|
||||||
|
) -> tuple[list[Check], bool]:
|
||||||
|
checks: list[Check] = []
|
||||||
|
|
||||||
|
# 1. Interop: solutions from A verify under B.
|
||||||
|
for a_name, b_name in pairs:
|
||||||
|
a, b = adapters.get(a_name), adapters.get(b_name)
|
||||||
|
if not a or not b:
|
||||||
|
checks.append(Check("interop", f"{a_name}->{b_name}: missing adapter", False))
|
||||||
|
continue
|
||||||
|
for chal in challenges:
|
||||||
|
solve = run(
|
||||||
|
a,
|
||||||
|
JobSpec(operation="solve", runtime="try-compile", repetitions=1, warmup=0, challenge_hex=chal),
|
||||||
|
repo_root,
|
||||||
|
)
|
||||||
|
sols = solve.solutions_hex or []
|
||||||
|
if not sols:
|
||||||
|
checks.append(Check("interop", f"{a_name} found 0 solutions for {chal} (nothing to verify)", True))
|
||||||
|
continue
|
||||||
|
all_ok = True
|
||||||
|
for sol in sols:
|
||||||
|
v = run(
|
||||||
|
b,
|
||||||
|
JobSpec(operation="verify", runtime="interpret", repetitions=1, warmup=0, challenge_hex=chal, solution_hex=sol),
|
||||||
|
repo_root,
|
||||||
|
)
|
||||||
|
vr = v.runs[-1].verify_result if v.runs else None
|
||||||
|
if vr != "OK":
|
||||||
|
all_ok = False
|
||||||
|
checks.append(Check("interop", f"{a_name} solution {sol} for {chal} did NOT verify under {b_name} (got {vr})", False))
|
||||||
|
if all_ok:
|
||||||
|
checks.append(Check("interop", f"{a_name}->{b_name}: all {len(sols)} solutions for {chal} verify OK", True))
|
||||||
|
|
||||||
|
# 2. Effort agreement across all impls.
|
||||||
|
impls = list(adapters.keys())
|
||||||
|
effort_results = {}
|
||||||
|
for name in impls:
|
||||||
|
a = adapters[name]
|
||||||
|
if a.capabilities and "effort" not in a.capabilities:
|
||||||
|
continue
|
||||||
|
r = run(
|
||||||
|
a,
|
||||||
|
JobSpec(
|
||||||
|
operation="effort", runtime="try-compile", repetitions=1, warmup=0,
|
||||||
|
challenge_base_hex=effort_base, nonce_bytes=8, nonce_start=0,
|
||||||
|
target_effort=effort_target, max_attempts=effort_max_attempts,
|
||||||
|
),
|
||||||
|
repo_root,
|
||||||
|
)
|
||||||
|
if r.runs:
|
||||||
|
effort_results[name] = (r.runs[-1].attempts, r.runs[-1].achieved_effort)
|
||||||
|
if len(effort_results) >= 2:
|
||||||
|
vals = set(effort_results.values())
|
||||||
|
passed = len(vals) == 1
|
||||||
|
checks.append(
|
||||||
|
Check(
|
||||||
|
"effort-agreement",
|
||||||
|
f"effort (attempts, achieved) across impls: {effort_results} "
|
||||||
|
+ ("-- AGREE" if passed else "-- DISAGREE"),
|
||||||
|
passed,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
overall = all(c.passed for c in checks) and len(checks) > 0
|
||||||
|
return checks, overall
|
||||||
81
tools/benchmarks/Equi-X/harness/equix_bench/device.py
Normal file
81
tools/benchmarks/Equi-X/harness/equix_bench/device.py
Normal file
@ -0,0 +1,81 @@
|
|||||||
|
"""Device (CPU/GPU) identity.
|
||||||
|
|
||||||
|
Each runner self-reports its hardware in the result `env` (cpu, arch, device).
|
||||||
|
The harness derives a device record from that, so identity is correct even for a
|
||||||
|
remote runner or a future GPU adapter (which would report device="gpu"). A CLI
|
||||||
|
`--device-label` overrides only the human label, e.g. to disambiguate two machines
|
||||||
|
that report the same CPU model string.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import platform
|
||||||
|
import re
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
|
||||||
|
def slug(s: str) -> str:
|
||||||
|
out = re.sub(r"[^a-zA-Z0-9]+", "-", s.strip().lower()).strip("-")
|
||||||
|
return out or "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def device_from_env(env: dict[str, Any], override_label: Optional[str] = None) -> dict[str, str]:
|
||||||
|
"""Build a device record from a runner's reported env, falling back to the
|
||||||
|
host when the runner did not report hardware (older/minimal adapters).
|
||||||
|
|
||||||
|
The auto label combines the device model with the OS version so runs on the
|
||||||
|
same CPU under different OS/kernel versions get distinct labels."""
|
||||||
|
name = env.get("cpu") or _host_cpu()
|
||||||
|
arch = env.get("arch") or platform.machine() or "unknown"
|
||||||
|
dtype = env.get("device") or "cpu"
|
||||||
|
os_name = env.get("os") or platform.system().lower() or "unknown"
|
||||||
|
os_version = env.get("os_version") or platform.release() or "unknown"
|
||||||
|
return {
|
||||||
|
"type": dtype,
|
||||||
|
"name": name,
|
||||||
|
"arch": arch,
|
||||||
|
"os": os_name,
|
||||||
|
"os_version": os_version,
|
||||||
|
"label": override_label or slug(f"{name}-{os_version}"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Field priority works across arches: "model name" (x86), "Model" (Raspberry Pi
|
||||||
|
# board), "Hardware" (older ARM), "cpu model" (others).
|
||||||
|
_CPU_FIELDS = ("model name", "Model", "Hardware", "cpu model")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_cpu_model(cpuinfo_text: str) -> str:
|
||||||
|
found: dict[str, str] = {}
|
||||||
|
for line in cpuinfo_text.splitlines():
|
||||||
|
if ":" not in line:
|
||||||
|
continue
|
||||||
|
k, _, v = line.partition(":")
|
||||||
|
k, v = k.strip(), v.strip()
|
||||||
|
if k in _CPU_FIELDS and k not in found and v:
|
||||||
|
found[k] = v
|
||||||
|
for f in _CPU_FIELDS:
|
||||||
|
if f in found:
|
||||||
|
return found[f]
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def _host_cpu() -> str:
|
||||||
|
try:
|
||||||
|
model = parse_cpu_model(open("/proc/cpuinfo").read())
|
||||||
|
if model != "unknown":
|
||||||
|
return model
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
if platform.system() == "Darwin": # macOS has no /proc
|
||||||
|
try:
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
out = subprocess.run(
|
||||||
|
["sysctl", "-n", "machdep.cpu.brand_string"],
|
||||||
|
capture_output=True, text=True, timeout=2,
|
||||||
|
)
|
||||||
|
if out.returncode == 0 and out.stdout.strip():
|
||||||
|
return out.stdout.strip()
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
pass
|
||||||
|
return platform.processor() or "unknown"
|
||||||
@ -0,0 +1,340 @@
|
|||||||
|
"""Reference difficulty (effort `E`) controllers + a simulator.
|
||||||
|
|
||||||
|
Two closed-loop controllers that adjust the Equi-X effort parameter to demand,
|
||||||
|
calibrated on the MEASURED mint-rate curve (docs/findings.md §7a):
|
||||||
|
|
||||||
|
* MintRateController (Design A) — hold a network's token MINT RATE at a target,
|
||||||
|
the way PoW difficulty retargeting works, specialized to the measured 1/E law.
|
||||||
|
* LoadController (Design B) — hold a single node's admission PRESSURE (offered
|
||||||
|
valid-token rate ÷ service capacity) at a target, so an attack is throttled
|
||||||
|
while honest clients pay the least difficulty that keeps the node healthy.
|
||||||
|
|
||||||
|
Both rest on one measured fact: token rate ∝ 1/E on fixed hardware, and verify
|
||||||
|
cost is constant and ~free. So the control law is a cheap MULTIPLICATIVE step and
|
||||||
|
needs no absolute capacity model — it self-corrects from what it observes.
|
||||||
|
|
||||||
|
Run the demo (writes plots + a summary):
|
||||||
|
python -m equix_bench.difficulty_control --out results/control
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
import random
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Callable, Optional
|
||||||
|
|
||||||
|
# Measured mint rate per reference machine (Apple M4 Pro, 14 cores, Rust-JIT),
|
||||||
|
# from results/main/mining.csv: tokens/second at each effort target. POOLED
|
||||||
|
# estimator (total tokens over total busy seconds across all workers) — a sum
|
||||||
|
# of per-worker 1/t ratios would carry a +6..25% upward bias at these CVs.
|
||||||
|
MEASURED_MINT: list[tuple[int, float]] = [(100, 69.1), (300, 23.5), (1000, 5.49), (3000, 1.75), (10000, 0.67)]
|
||||||
|
|
||||||
|
|
||||||
|
def mint_rate_per_machine(E: float, points: list[tuple[int, float]] = MEASURED_MINT) -> float:
|
||||||
|
"""Tokens/s one reference machine mints at effort `E`.
|
||||||
|
|
||||||
|
Log-log linear interpolation between the measured points; outside the
|
||||||
|
measured range it extrapolates as pure 1/E (slope -1 in log-log), the design
|
||||||
|
asymptote. Monotonically decreasing in E."""
|
||||||
|
pts = sorted(points)
|
||||||
|
if E <= pts[0][0]:
|
||||||
|
e0, r0 = pts[0]
|
||||||
|
return r0 * (e0 / E) # 1/E extrapolation below range
|
||||||
|
if E >= pts[-1][0]:
|
||||||
|
e1, r1 = pts[-1]
|
||||||
|
return r1 * (e1 / E) # 1/E extrapolation above range
|
||||||
|
for (e0, r0), (e1, r1) in zip(pts, pts[1:]):
|
||||||
|
if e0 <= E <= e1:
|
||||||
|
f = (math.log(E) - math.log(e0)) / (math.log(e1) - math.log(e0))
|
||||||
|
return math.exp(math.log(r0) + f * (math.log(r1) - math.log(r0)))
|
||||||
|
return pts[-1][1] # unreachable
|
||||||
|
|
||||||
|
|
||||||
|
def _clip(x: float, lo: float, hi: float) -> float:
|
||||||
|
return max(lo, min(hi, x))
|
||||||
|
|
||||||
|
|
||||||
|
def equilibrium_E(machines: float, target_rate: float,
|
||||||
|
points: list[tuple[int, float]] = MEASURED_MINT,
|
||||||
|
lo: float = 1.0, hi: float = 1e9) -> float:
|
||||||
|
"""The E at which `machines` reference machines mint `target_rate` tokens/s
|
||||||
|
(the natural controller seed). Solved by bisection on the measured curve —
|
||||||
|
no hard-coded constants to drift out of sync when MEASURED_MINT is updated."""
|
||||||
|
for _ in range(200):
|
||||||
|
mid = math.sqrt(lo * hi) # geometric: the curve lives in log-log space
|
||||||
|
if machines * mint_rate_per_machine(mid, points) > target_rate:
|
||||||
|
lo = mid
|
||||||
|
else:
|
||||||
|
hi = mid
|
||||||
|
if hi / lo < 1.0001:
|
||||||
|
break
|
||||||
|
return math.sqrt(lo * hi)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MintRateController:
|
||||||
|
"""Design A: E ← E · clamp(R_obs / R*). Because rate ∝ 1/E, minting twice too
|
||||||
|
fast means E must double. Robust to capacity drift (only uses observed rate)."""
|
||||||
|
target_rate: float # R* (tokens/s the network should mint)
|
||||||
|
E: float = 1000.0 # current difficulty
|
||||||
|
e_min: float = 100.0
|
||||||
|
e_max: float = 1e9
|
||||||
|
max_factor: float = 4.0 # per-epoch clamp (anti-oscillation, Bitcoin-style)
|
||||||
|
ewma: float = 0.3 # smoothing of the observed rate (0..1); 1 = no smoothing
|
||||||
|
_r: Optional[float] = None # smoothed observed rate
|
||||||
|
|
||||||
|
def update(self, observed_rate: float) -> float:
|
||||||
|
self._r = observed_rate if self._r is None else self.ewma * observed_rate + (1 - self.ewma) * self._r
|
||||||
|
factor = _clip((self._r / self.target_rate) if self.target_rate > 0 else 1.0,
|
||||||
|
1.0 / self.max_factor, self.max_factor)
|
||||||
|
self.E = _clip(self.E * factor, self.e_min, self.e_max)
|
||||||
|
return self.E
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class LoadController:
|
||||||
|
"""Design B: E ← E · clamp(exp(k · (p - p_set))), where p = offered valid-token
|
||||||
|
rate ÷ service capacity. Raise E under pressure (throttle), let it decay toward
|
||||||
|
e_min when idle (cheap for honest clients). Deadband avoids flapping — note the
|
||||||
|
intended consequence: while pressure sits within the deadband of p_set, E HOLDS
|
||||||
|
(including post-attack, if load keeps utilization pinned at target); it only
|
||||||
|
decays once pressure drops below p_set - deadband."""
|
||||||
|
p_set: float = 0.8 # target pressure (headroom below saturation)
|
||||||
|
k: float = 1.5 # gain
|
||||||
|
E: float = 300.0
|
||||||
|
e_min: float = 300.0
|
||||||
|
e_max: float = 1e9
|
||||||
|
max_factor: float = 4.0
|
||||||
|
deadband: float = 0.03
|
||||||
|
|
||||||
|
def update(self, pressure: float) -> float:
|
||||||
|
e = pressure - self.p_set
|
||||||
|
if abs(e) < self.deadband:
|
||||||
|
return self.E
|
||||||
|
factor = _clip(math.exp(self.k * e), 1.0 / self.max_factor, self.max_factor)
|
||||||
|
self.E = _clip(self.E * factor, self.e_min, self.e_max)
|
||||||
|
return self.E
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------- simulators
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Trace:
|
||||||
|
t: list[float] = field(default_factory=list)
|
||||||
|
E: list[float] = field(default_factory=list)
|
||||||
|
signal: list[float] = field(default_factory=list) # observed rate (A) or pressure (B)
|
||||||
|
target: list[float] = field(default_factory=list)
|
||||||
|
extra: dict = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
def simulate_mining(capacity: Callable[[int], float], target_rate: float, steps: int,
|
||||||
|
ctrl: Optional[MintRateController] = None,
|
||||||
|
rate_model: Callable[[float], float] = mint_rate_per_machine,
|
||||||
|
noise: float = 0.0, seed: int = 0) -> Trace:
|
||||||
|
"""Network of `capacity(t)` machines; controller holds mint rate at target.
|
||||||
|
`noise` (fractional stddev) models real measurement jitter on the observed
|
||||||
|
rate; `seed` makes it reproducible."""
|
||||||
|
ctrl = ctrl or MintRateController(target_rate=target_rate)
|
||||||
|
rng = random.Random(seed)
|
||||||
|
tr = Trace(target=[], extra={"capacity": [], "true_rate": []})
|
||||||
|
for t in range(steps):
|
||||||
|
C = capacity(t)
|
||||||
|
R = C * rate_model(ctrl.E) # true tokens/s at current E
|
||||||
|
R_obs = R * (1 + rng.gauss(0, noise)) if noise else R
|
||||||
|
tr.t.append(t); tr.E.append(ctrl.E); tr.signal.append(R_obs)
|
||||||
|
tr.target.append(target_rate); tr.extra["capacity"].append(C)
|
||||||
|
tr.extra["true_rate"].append(R) # noise-free, for honest error stats
|
||||||
|
ctrl.update(R_obs) # set E for the next epoch
|
||||||
|
return tr
|
||||||
|
|
||||||
|
|
||||||
|
def simulate_dos(legit: Callable[[int], float], attackers: Callable[[int], float],
|
||||||
|
service_capacity: float, steps: int, ctrl: Optional[LoadController] = None,
|
||||||
|
honest_cores: float = 1.0,
|
||||||
|
rate_model: Callable[[float], float] = mint_rate_per_machine,
|
||||||
|
adaptive_attackers: Optional[Callable[[int, float], float]] = None) -> Trace:
|
||||||
|
"""One protected node: honest req/s `legit(t)` plus attacker machines each
|
||||||
|
minting valid tokens at the current E. Controller holds admission pressure at
|
||||||
|
p_set. Records honest latency = time for one honest `honest_cores`-core client
|
||||||
|
to mint a token at the current E (the cost of defense to legitimate users).
|
||||||
|
|
||||||
|
`adaptive_attackers(t, E)`, when given, replaces `attackers(t)`: the attacker
|
||||||
|
OBSERVES the current difficulty and can pause when solving is too expensive
|
||||||
|
and resume when E decays — the rational strategy against a decaying controller."""
|
||||||
|
ctrl = ctrl or LoadController()
|
||||||
|
tr = Trace(target=[], extra={"attacker_rate": [], "presented": [], "honest_latency": [], "util": []})
|
||||||
|
machine_cores = 14.0 # the reference machine the curve was measured on
|
||||||
|
for t in range(steps):
|
||||||
|
A = adaptive_attackers(t, ctrl.E) if adaptive_attackers else attackers(t)
|
||||||
|
m = rate_model(ctrl.E) # tokens/s per 14-core machine at current E
|
||||||
|
attacker_rate = A * m # valid tokens/s the attacker can present
|
||||||
|
presented = legit(t) + attacker_rate
|
||||||
|
pressure = presented / service_capacity
|
||||||
|
honest_latency = 1.0 / (m * honest_cores / machine_cores) # 1 honest client's token time
|
||||||
|
tr.t.append(t); tr.E.append(ctrl.E); tr.signal.append(pressure); tr.target.append(ctrl.p_set)
|
||||||
|
tr.extra["attacker_rate"].append(attacker_rate)
|
||||||
|
tr.extra["presented"].append(presented)
|
||||||
|
tr.extra["honest_latency"].append(honest_latency)
|
||||||
|
tr.extra["util"].append(min(1.0, pressure))
|
||||||
|
ctrl.update(pressure)
|
||||||
|
return tr
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------- demo
|
||||||
|
|
||||||
|
|
||||||
|
def _plot_mining(tr: Trace, path: Path, title: str = "Design A — mint-rate controller") -> None:
|
||||||
|
import matplotlib
|
||||||
|
matplotlib.use("Agg")
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
fig, ax = plt.subplots(3, 1, figsize=(8, 7), sharex=True)
|
||||||
|
ax[0].plot(tr.t, tr.extra["capacity"], color="#8172B3")
|
||||||
|
ax[0].set_ylabel("miners\n(machines)"); ax[0].set_title(title)
|
||||||
|
ax[1].plot(tr.t, tr.signal, color="#4C72B0", label="observed mint rate")
|
||||||
|
ax[1].plot(tr.t, tr.target, "--", color="#C44E52", label="target R*")
|
||||||
|
ax[1].set_ylabel("tokens / s"); ax[1].legend(fontsize=8)
|
||||||
|
ax[2].plot(tr.t, tr.E, color="#55A868"); ax[2].set_yscale("log")
|
||||||
|
ax[2].set_ylabel("effort E"); ax[2].set_xlabel("epoch")
|
||||||
|
for a in ax: a.grid(True, alpha=0.3)
|
||||||
|
fig.tight_layout(); fig.savefig(path, dpi=110); plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
def _plot_dos(tr: Trace, S: float, path: Path,
|
||||||
|
title: str = "Design B — single-node load controller") -> None:
|
||||||
|
import matplotlib
|
||||||
|
matplotlib.use("Agg")
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
fig, ax = plt.subplots(4, 1, figsize=(8, 9), sharex=True)
|
||||||
|
ax[0].plot(tr.t, tr.extra["presented"], color="#DD8452", label="offered valid-token req/s")
|
||||||
|
ax[0].plot(tr.t, tr.extra["attacker_rate"], ":", color="#C44E52", label="attacker share")
|
||||||
|
ax[0].axhline(S, ls="--", color="#333", label=f"service capacity S={S:g}")
|
||||||
|
ax[0].set_ylabel("requests / s"); ax[0].set_title(title); ax[0].legend(fontsize=8)
|
||||||
|
ax[1].plot(tr.t, tr.extra["util"], color="#4C72B0", label="utilization")
|
||||||
|
ax[1].plot(tr.t, tr.target, "--", color="#C44E52", label="p_set")
|
||||||
|
ax[1].set_ylabel("utilization"); ax[1].set_ylim(0, 1.05); ax[1].legend(fontsize=8)
|
||||||
|
ax[2].plot(tr.t, tr.E, color="#55A868"); ax[2].set_yscale("log"); ax[2].set_ylabel("effort E")
|
||||||
|
ax[3].plot(tr.t, tr.extra["honest_latency"], color="#937860")
|
||||||
|
ax[3].set_ylabel("honest solve\ntime (s)"); ax[3].set_xlabel("tick")
|
||||||
|
for a in ax: a.grid(True, alpha=0.3)
|
||||||
|
fig.tight_layout(); fig.savefig(path, dpi=110); plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None) -> int:
|
||||||
|
import argparse
|
||||||
|
p = argparse.ArgumentParser(description="Equi-X difficulty (E) controller demo")
|
||||||
|
p.add_argument("--out", default="results/control", help="output directory for plots")
|
||||||
|
p.add_argument("--steps", type=int, default=80)
|
||||||
|
args = p.parse_args(argv)
|
||||||
|
out = Path(args.out); out.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Design A: miners ramp 2→12, then a hashpower spike to 20 at 60%. Hold R* = 2 tok/s.
|
||||||
|
def capacity(t: int) -> float:
|
||||||
|
if t < 48:
|
||||||
|
return 2 + 10 * (t / 48)
|
||||||
|
return 20.0
|
||||||
|
mining = simulate_mining(capacity, target_rate=2.0, steps=args.steps)
|
||||||
|
_plot_mining(mining, out / "control_mining.png")
|
||||||
|
|
||||||
|
# Design B: honest 8 req/s throughout; a 6-machine flood during [24, 56). S = 40 req/s.
|
||||||
|
def legit(t: int) -> float:
|
||||||
|
return 8.0
|
||||||
|
def attackers(t: int) -> float:
|
||||||
|
return 6.0 if 24 <= t < 56 else 0.0
|
||||||
|
S = 40.0
|
||||||
|
dos = simulate_dos(legit, attackers, service_capacity=S, steps=args.steps)
|
||||||
|
_plot_dos(dos, S, out / "control_dos.png")
|
||||||
|
|
||||||
|
# Production run: how you'd actually deploy Design A. Gentle gains (max_factor
|
||||||
|
# 2, heavy EWMA), E SEEDED near equilibrium from the capacity estimate (no cold
|
||||||
|
# start), organic capacity growth, and ±8% measurement noise on the observed
|
||||||
|
# rate. Result: smooth tracking with no overshoot, stable under noise.
|
||||||
|
steps_p = max(args.steps, 120)
|
||||||
|
C0, R_target = 4.0, 2.0
|
||||||
|
E_seed = equilibrium_E(C0, R_target) # seed at equilibrium: C0 machines mint R_target
|
||||||
|
def capacity_prod(t: int) -> float:
|
||||||
|
# organic growth 4 → ~13 with a slow wobble (diurnal-like), no hard steps
|
||||||
|
return 4.0 + 9.0 * (t / steps_p) + 0.8 * math.sin(t / 9.0)
|
||||||
|
prod_ctrl = MintRateController(target_rate=R_target, E=E_seed, max_factor=2.0, ewma=0.15)
|
||||||
|
prod = simulate_mining(capacity_prod, target_rate=R_target, steps=steps_p,
|
||||||
|
ctrl=prod_ctrl, noise=0.08, seed=1)
|
||||||
|
_plot_mining(prod, out / "control_production.png",
|
||||||
|
title="Production run — mint-rate controller (seeded E, gentle gains, ±8% noise)")
|
||||||
|
|
||||||
|
# Summary numbers.
|
||||||
|
def _final(tr, key_signal):
|
||||||
|
return tr.E[-1], key_signal
|
||||||
|
# Run 4 — ADAPTIVE attacker vs Design B: 6 machines that only attack while
|
||||||
|
# E is below their give-up point (solving cheap enough to bother), pausing
|
||||||
|
# when the controller escalates and resuming as E decays. The rational
|
||||||
|
# strategy — does the loop oscillate, and what does the attacker still get?
|
||||||
|
E_GIVEUP = 800.0
|
||||||
|
def adaptive(t: int, E: float) -> float:
|
||||||
|
if t < 16 or t >= 104:
|
||||||
|
return 0.0
|
||||||
|
return 6.0 if E < E_GIVEUP else 0.0
|
||||||
|
adaptive_tr = simulate_dos(legit, lambda t: 0.0, service_capacity=S,
|
||||||
|
steps=120, adaptive_attackers=adaptive)
|
||||||
|
_plot_dos(adaptive_tr, S, out / "control_adaptive.png",
|
||||||
|
title="Design B vs an ADAPTIVE attacker (attacks only while E < give-up)")
|
||||||
|
atk_window = adaptive_tr.extra["attacker_rate"][16:104]
|
||||||
|
duty = sum(1 for a in atk_window if a > 0) / len(atk_window)
|
||||||
|
sat = sum(1 for u in adaptive_tr.extra["util"][16:104] if u >= 0.999)
|
||||||
|
|
||||||
|
# Run 5 — miner CHURN for Design A production tuning: miners join/leave as a
|
||||||
|
# seeded random walk instead of a smooth ramp; noise on the observed rate.
|
||||||
|
rng_churn = random.Random(7)
|
||||||
|
miners = 8.0
|
||||||
|
churn_path = []
|
||||||
|
for _ in range(140):
|
||||||
|
if rng_churn.random() < 0.15: miners += 1
|
||||||
|
if rng_churn.random() < 0.15 and miners > 2: miners -= 1
|
||||||
|
churn_path.append(miners)
|
||||||
|
churn_ctrl = MintRateController(target_rate=R_target, E=equilibrium_E(8.0, R_target),
|
||||||
|
max_factor=2.0, ewma=0.15)
|
||||||
|
churn = simulate_mining(lambda t: churn_path[t], target_rate=R_target, steps=140,
|
||||||
|
ctrl=churn_ctrl, noise=0.08, seed=2)
|
||||||
|
_plot_mining(churn, out / "control_churn.png",
|
||||||
|
title="Production tuning under miner churn (join/leave random walk, ±8% noise)")
|
||||||
|
churn_tail = churn.extra["true_rate"][40:]
|
||||||
|
churn_err = 100.0 * (sum(churn_tail) / len(churn_tail) - R_target) / R_target
|
||||||
|
|
||||||
|
# Production-run steady-state error over the settled tail (last third).
|
||||||
|
# Report BOTH: the noisy observed mean (what an operator sees) and the
|
||||||
|
# noise-free true-rate mean (the controller's systematic tracking lag —
|
||||||
|
# averaging only the noisy signal would understate it).
|
||||||
|
n_tail = len(prod.signal) * 2 // 3
|
||||||
|
obs_tail = prod.signal[n_tail:]
|
||||||
|
true_tail = prod.extra["true_rate"][n_tail:]
|
||||||
|
prod_err_obs = 100.0 * (sum(obs_tail) / len(obs_tail) - R_target) / R_target
|
||||||
|
prod_err_true = 100.0 * (sum(true_tail) / len(true_tail) - R_target) / R_target
|
||||||
|
lines = ["# Difficulty-control simulation\n",
|
||||||
|
f"- Design A (mining): final E={mining.E[-1]:,.0f}, "
|
||||||
|
f"mint rate {mining.signal[-1]:.2f} tok/s vs target {mining.target[-1]:.2f} "
|
||||||
|
f"(with {mining.extra['capacity'][-1]:.0f} miners).",
|
||||||
|
f"- Design B (DoS): peak E under attack ≈ {max(dos.E):,.0f}; "
|
||||||
|
f"steady honest solve time {dos.extra['honest_latency'][0]:.2f}s at rest → "
|
||||||
|
f"{max(dos.extra['honest_latency']):.2f}s under attack; "
|
||||||
|
f"utilization held near p_set={dos.target[-1]:.2f}.",
|
||||||
|
f"- Production run (seeded E, gentle gains, ±8% noise): no cold-start "
|
||||||
|
f"overshoot; settled tail within {prod_err_obs:+.1f}% of target (observed) / "
|
||||||
|
f"{prod_err_true:+.1f}% (noise-free — the systematic lag against the capacity "
|
||||||
|
f"ramp), final E={prod.E[-1]:,.0f} at {prod.extra['capacity'][-1]:.0f} miners.",
|
||||||
|
f"- Adaptive attacker (give-up E={E_GIVEUP:,.0f}): duty-cycled to "
|
||||||
|
f"{duty*100:.0f}% attack-on time, {sat} saturated tick(s) in the whole attack "
|
||||||
|
f"window; E oscillates {min(adaptive_tr.E[20:104]):,.0f}–{max(adaptive_tr.E[20:104]):,.0f} "
|
||||||
|
f"(sawtooth around the give-up point — see the doc for the mitigation).",
|
||||||
|
f"- Miner churn (random walk {min(churn_path):.0f}–{max(churn_path):.0f} miners): "
|
||||||
|
f"settled noise-free rate within {churn_err:+.1f}% of target.",
|
||||||
|
"\nPlots: `control_mining.png`, `control_dos.png`, `control_production.png`, "
|
||||||
|
"`control_adaptive.png`, `control_churn.png`.\n"]
|
||||||
|
(out / "summary.md").write_text("\n".join(lines))
|
||||||
|
print("\n".join(lines))
|
||||||
|
print(f"\nWrote {out}/control_mining.png, {out}/control_dos.png, "
|
||||||
|
f"{out}/control_production.png, {out}/summary.md")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
92
tools/benchmarks/Equi-X/harness/equix_bench/dosprotect.py
Normal file
92
tools/benchmarks/Equi-X/harness/equix_bench/dosprotect.py
Normal file
@ -0,0 +1,92 @@
|
|||||||
|
"""DoS-protection effectiveness evaluation.
|
||||||
|
|
||||||
|
Equi-X is a client puzzle: a requester must *solve* the puzzle (expensive) before
|
||||||
|
a service will act, while the service only *verifies* (cheap). The protection is
|
||||||
|
effective on a given system when the attacker's cost to produce one accepted
|
||||||
|
request vastly exceeds the defender's cost to check it.
|
||||||
|
|
||||||
|
This module answers that question from MEASURED data on the running system:
|
||||||
|
|
||||||
|
attacker_s(E) = measured median time to produce one token at effort E
|
||||||
|
(the `effort` op: solve over nonces until achieved >= E)
|
||||||
|
defender_s = measured median time for one `verify`
|
||||||
|
protection_factor(E) = attacker_s(E) / defender_s
|
||||||
|
-> how many requests the defender can screen in the time an attacker needs
|
||||||
|
to craft one accepted request. Large = strong DoS protection.
|
||||||
|
|
||||||
|
Also derived:
|
||||||
|
verify_throughput = 1 / defender_s (verifications/sec per core)
|
||||||
|
attacker_token_rate = 1 / attacker_s(E) (accepted tokens/sec per core)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from .stats import CellStats
|
||||||
|
|
||||||
|
# Protection is judged "effective" when an attacker spends at least this many
|
||||||
|
# times the defender's per-verify cost to produce one accepted request.
|
||||||
|
DEFAULT_THRESHOLD = 10_000.0
|
||||||
|
|
||||||
|
|
||||||
|
def min_verify_seconds(stats: list[CellStats], device: str) -> Optional[float]:
|
||||||
|
"""Defender's best (fastest) verify time on `device`, in seconds."""
|
||||||
|
vs = [
|
||||||
|
s.median_ns / 1e9
|
||||||
|
for s in stats
|
||||||
|
if s.operation == "verify" and s.ok and s.device_label == device and s.median_ns > 0
|
||||||
|
]
|
||||||
|
return min(vs) if vs else None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ProtectionRow:
|
||||||
|
device: str
|
||||||
|
effort: int
|
||||||
|
attacker_impl: str
|
||||||
|
attacker_s: float # fastest measured time to craft a token at this effort
|
||||||
|
defender_verify_s: float # fastest measured verify time
|
||||||
|
protection_factor: float # attacker_s / defender_verify_s
|
||||||
|
verify_per_sec: float # defender screening capacity (per core)
|
||||||
|
attacker_tokens_per_sec: float # attacker output (per core)
|
||||||
|
|
||||||
|
|
||||||
|
def assess(stats: list[CellStats], threshold: float = DEFAULT_THRESHOLD):
|
||||||
|
"""Return (rows, effective, threshold). Uses the attacker's fastest impl and
|
||||||
|
the defender's fastest verify per device -- the realistic optimized case."""
|
||||||
|
rows: list[ProtectionRow] = []
|
||||||
|
for dev in sorted({s.device_label for s in stats if s.ok}):
|
||||||
|
vs = min_verify_seconds(stats, dev)
|
||||||
|
if vs is None or vs <= 0:
|
||||||
|
continue
|
||||||
|
by_effort: dict[int, list[tuple[float, str]]] = {}
|
||||||
|
for s in stats:
|
||||||
|
if s.operation == "effort" and s.ok and s.device_label == dev and s.median_ns > 0:
|
||||||
|
t = int(s.label.get("target_effort", 0))
|
||||||
|
by_effort.setdefault(t, []).append((s.median_ns / 1e9, s.impl))
|
||||||
|
for t in sorted(by_effort):
|
||||||
|
attacker_s, attacker_impl = min(by_effort[t], key=lambda x: x[0])
|
||||||
|
rows.append(ProtectionRow(
|
||||||
|
device=dev, effort=t, attacker_impl=attacker_impl,
|
||||||
|
attacker_s=attacker_s, defender_verify_s=vs,
|
||||||
|
protection_factor=attacker_s / vs,
|
||||||
|
verify_per_sec=1.0 / vs,
|
||||||
|
attacker_tokens_per_sec=1.0 / attacker_s,
|
||||||
|
))
|
||||||
|
# "Effective on this system" means SOME reachable effort clears the bar; the
|
||||||
|
# smallest such effort tells the operator how hard to set the puzzle here.
|
||||||
|
effective = any(r.protection_factor >= threshold for r in rows)
|
||||||
|
return rows, effective, threshold
|
||||||
|
|
||||||
|
|
||||||
|
def min_effective_effort(rows, threshold: float = DEFAULT_THRESHOLD):
|
||||||
|
"""Smallest measured effort whose protection factor clears the threshold,
|
||||||
|
grouped per device. Returns {device: effort or None}."""
|
||||||
|
out: dict[str, Optional[int]] = {}
|
||||||
|
for r in rows:
|
||||||
|
if r.device not in out:
|
||||||
|
out[r.device] = None
|
||||||
|
if r.protection_factor >= threshold and (out[r.device] is None or r.effort < out[r.device]):
|
||||||
|
out[r.device] = r.effort
|
||||||
|
return out
|
||||||
338
tools/benchmarks/Equi-X/harness/equix_bench/mining.py
Normal file
338
tools/benchmarks/Equi-X/harness/equix_bench/mining.py
Normal file
@ -0,0 +1,338 @@
|
|||||||
|
"""Mining-rate benchmark: how fast can tokens (effort-qualified solutions) be
|
||||||
|
minted at a given difficulty, on one core and on the whole machine.
|
||||||
|
|
||||||
|
This is the measured basis for the "control the mint rate by setting difficulty"
|
||||||
|
use case. It differs from the plain effort sweep in two ways that make it a real
|
||||||
|
statistical measurement rather than a single anecdote:
|
||||||
|
|
||||||
|
* The runner's effort search is DETERMINISTIC from `nonce_start` (every rep
|
||||||
|
restarts at the same nonce), so repetitions alone re-time one identical
|
||||||
|
search. Here each sample uses a DISTINCT `nonce_start`, spaced beyond
|
||||||
|
`max_attempts` so the search ranges never overlap -- independent draws from
|
||||||
|
the geometric token-finding process, which we average.
|
||||||
|
* It measures the whole-machine mint rate directly: `workers` (= core count)
|
||||||
|
independent searches run at once, and we sum their token rates.
|
||||||
|
|
||||||
|
Outputs, per difficulty E (all POOLED estimators — total tokens over total busy
|
||||||
|
seconds, failed searches charged to the denominator; a mean of per-sample 1/t
|
||||||
|
ratios would carry a +6..25% Jensen-style upward bias at the CVs we measure):
|
||||||
|
tokens_per_sec_1core successes / total busy seconds, sequential [1 core]
|
||||||
|
tokens_per_sec_machine workers * minted / total busy seconds [N cores]
|
||||||
|
attempts_mean attempts per token, pooled over every search
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import statistics
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Callable, Optional
|
||||||
|
|
||||||
|
from .protocol import JobSpec, Result
|
||||||
|
from .registry import Adapter
|
||||||
|
from .runner import RunnerError, run
|
||||||
|
|
||||||
|
# Nonce range reserved per sample/worker; must exceed any single token's attempts
|
||||||
|
# so independent searches never touch the same nonces.
|
||||||
|
STRIDE = 4_000_000
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MiningPoint:
|
||||||
|
effort: int
|
||||||
|
samples: int # independent 1-core samples that succeeded
|
||||||
|
token_s_mean: float # mean seconds to mint one token [1 core]
|
||||||
|
token_s_median: float
|
||||||
|
token_s_stddev: float
|
||||||
|
attempts_mean: float # POOLED over 1-core samples AND worker mints
|
||||||
|
achieved_mean: float
|
||||||
|
tokens_per_sec_1core: float # successes / total busy seconds [1 core]
|
||||||
|
workers: int
|
||||||
|
ok_workers: int
|
||||||
|
tokens_per_sec_machine: float # pooled: workers * minted / total busy s
|
||||||
|
scaling_efficiency: float # machine / (workers * 1core rate)
|
||||||
|
failed_searches: int = 0 # searches that hit max_attempts w/o a token
|
||||||
|
# Message sizes, MEASURED from real minted tokens' wire bytes. The design
|
||||||
|
# predicts both are constant in E (a solution is always 8 x u16 = 16 bytes;
|
||||||
|
# the nonce is a protocol constant): min==max demonstrates it.
|
||||||
|
solution_bytes_min: int = 0
|
||||||
|
solution_bytes_max: int = 0
|
||||||
|
nonce_bytes_wire: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MiningResult:
|
||||||
|
device: str
|
||||||
|
impl: str
|
||||||
|
challenge_base: str
|
||||||
|
nproc: int
|
||||||
|
points: list[MiningPoint] = field(default_factory=list)
|
||||||
|
error: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
def _effort_spec(base: str, effort: int, nonce_start: int, nonce_bytes: int,
|
||||||
|
reps: int, warmup: int, max_attempts: int) -> JobSpec:
|
||||||
|
return JobSpec(
|
||||||
|
operation="effort",
|
||||||
|
runtime="try-compile", # fastest available path per impl
|
||||||
|
repetitions=reps,
|
||||||
|
warmup=warmup,
|
||||||
|
challenge_base_hex=base,
|
||||||
|
nonce_bytes=nonce_bytes,
|
||||||
|
nonce_start=nonce_start,
|
||||||
|
target_effort=effort,
|
||||||
|
max_attempts=max_attempts,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _one_token(adapter: Adapter, base: str, effort: int, nonce_start: int,
|
||||||
|
nonce_bytes: int, max_attempts: int, repo_root: Path,
|
||||||
|
timeout: float) -> Optional[tuple[bool, float, int, int, tuple[int, int]]]:
|
||||||
|
"""Search one fresh nonce range for a token. Returns (reached_target,
|
||||||
|
seconds, attempts, achieved, (solution_bytes, nonce_bytes_wire)) — elapsed
|
||||||
|
time is reported even when the target was NOT reached, so callers can charge
|
||||||
|
failed searches to the denominator instead of silently conditioning the rate
|
||||||
|
on success. Sizes are (0, 0) when no token was minted.
|
||||||
|
None only when the runner itself errored (no timing available)."""
|
||||||
|
try:
|
||||||
|
r = run(adapter, _effort_spec(base, effort, nonce_start, nonce_bytes, 1, 0, max_attempts),
|
||||||
|
repo_root, timeout=timeout)
|
||||||
|
except RunnerError:
|
||||||
|
return None
|
||||||
|
if not r.ok or not r.runs:
|
||||||
|
return None
|
||||||
|
run0 = r.runs[0]
|
||||||
|
if run0.wall_ns <= 0:
|
||||||
|
return None
|
||||||
|
# Wire sizes of the actual minted token (hex chars / 2 = bytes).
|
||||||
|
sol_b = len(r.solutions_hex[0]) // 2 if r.solutions_hex else 0
|
||||||
|
nonce_b = len(r.winning_nonce_hex) // 2 if r.winning_nonce_hex else 0
|
||||||
|
return (run0.achieved_effort >= effort, run0.wall_ns / 1e9,
|
||||||
|
run0.attempts, run0.achieved_effort, (sol_b, nonce_b))
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_batch(adapter: Adapter, base: str, effort: int, nonce_start: int,
|
||||||
|
count: int, nonce_bytes: int, max_attempts: int,
|
||||||
|
repo_root: Path, timeout: float) -> tuple[int, float, int, int, list[tuple[int, int]]]:
|
||||||
|
"""One concurrent worker STREAMING `count` searches over advancing nonce
|
||||||
|
ranges (like a real miner). Returns (tokens_minted, total_busy_seconds,
|
||||||
|
attempts_total, failed_searches, token_sizes). Failed searches contribute
|
||||||
|
their full solve time to the denominator — a real miner pays for them too."""
|
||||||
|
minted, total, attempts, failed = 0, 0.0, 0, 0
|
||||||
|
sizes: list[tuple[int, int]] = []
|
||||||
|
for k in range(count):
|
||||||
|
got = _one_token(adapter, base, effort, nonce_start + k * STRIDE, nonce_bytes,
|
||||||
|
max_attempts, repo_root, timeout)
|
||||||
|
if got is None:
|
||||||
|
continue
|
||||||
|
ok, secs, atts, _ach, size = got
|
||||||
|
total += secs
|
||||||
|
attempts += atts
|
||||||
|
if ok:
|
||||||
|
minted += 1
|
||||||
|
sizes.append(size)
|
||||||
|
else:
|
||||||
|
failed += 1
|
||||||
|
return (minted, total, attempts, failed, sizes)
|
||||||
|
|
||||||
|
|
||||||
|
def measure_point(adapter: Adapter, base: str, effort: int, samples: int,
|
||||||
|
workers: int, tokens_per_worker: int, nonce_bytes: int,
|
||||||
|
max_attempts: int, repo_root: Path, timeout: float) -> MiningPoint:
|
||||||
|
if max_attempts > STRIDE:
|
||||||
|
raise ValueError(
|
||||||
|
f"max_attempts ({max_attempts}) must not exceed STRIDE ({STRIDE}); "
|
||||||
|
"independent nonce ranges would overlap"
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- 1-core: `samples` independent, sequential searches (distinct nonces) ---
|
||||||
|
secs: list[float] = [] # successful token times
|
||||||
|
attempts_all: list[int] = [] # attempts, pooled over ALL mints (see below)
|
||||||
|
achieved: list[int] = []
|
||||||
|
all_sizes: list[tuple[int, int]] = [] # (solution_bytes, nonce_bytes) per token
|
||||||
|
fail_s, fails = 0.0, 0
|
||||||
|
for s in range(samples):
|
||||||
|
got = _one_token(adapter, base, effort, s * STRIDE, nonce_bytes,
|
||||||
|
max_attempts, repo_root, timeout)
|
||||||
|
if got is None:
|
||||||
|
continue
|
||||||
|
ok, t, atts, ach, size = got
|
||||||
|
if ok:
|
||||||
|
secs.append(t); attempts_all.append(atts); achieved.append(ach)
|
||||||
|
all_sizes.append(size)
|
||||||
|
else:
|
||||||
|
fail_s += t; fails += 1
|
||||||
|
|
||||||
|
if not secs:
|
||||||
|
return MiningPoint(effort, 0, 0, 0, 0, 0, 0, 0, workers, 0, 0, 0, fails)
|
||||||
|
|
||||||
|
# Pooled rate: total tokens over total busy time (failed searches included
|
||||||
|
# in the denominator — a miner pays for them too). Stable for the
|
||||||
|
# heavy-tailed token-time distribution, unlike a mean of 1/t ratios.
|
||||||
|
token_s_mean = statistics.fmean(secs)
|
||||||
|
onecore_rate = len(secs) / (sum(secs) + fail_s)
|
||||||
|
|
||||||
|
# --- whole machine: `workers` concurrent streams on disjoint nonce ranges ---
|
||||||
|
def one(w: int) -> tuple[int, float, int, int]:
|
||||||
|
start = (samples + w * tokens_per_worker) * STRIDE
|
||||||
|
return _worker_batch(adapter, base, effort, start, tokens_per_worker,
|
||||||
|
nonce_bytes, max_attempts, repo_root, timeout)
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||||
|
batches = list(pool.map(one, range(workers)))
|
||||||
|
|
||||||
|
# Pooled machine estimator: workers x (all tokens / all busy seconds).
|
||||||
|
# Summing per-worker m/t ratios (few heavy-tailed samples each) carries a
|
||||||
|
# +6..25% Jensen-style upward bias at the CVs we measure; pooling first
|
||||||
|
# reduces the residual bias to ~+1% at 70 mints.
|
||||||
|
minted = sum(b[0] for b in batches)
|
||||||
|
busy = sum(b[1] for b in batches)
|
||||||
|
ok_workers = sum(1 for b in batches if b[0] > 0)
|
||||||
|
w_fails = sum(b[3] for b in batches)
|
||||||
|
machine_rate = (workers * minted / busy) if busy > 0 else 0.0
|
||||||
|
# Attempts per TOKEN, pooled across every search (1-core + workers, failed
|
||||||
|
# searches' attempts charged to the numerator): the real cost of a token,
|
||||||
|
# at ~8x the sample size of the 1-core batch alone.
|
||||||
|
total_attempts = sum(attempts_all) + sum(b[2] for b in batches)
|
||||||
|
total_tokens = len(secs) + minted
|
||||||
|
attempts_mean = total_attempts / total_tokens if total_tokens else 0.0
|
||||||
|
# Token wire sizes across EVERY minted token (1-core + workers).
|
||||||
|
for b in batches:
|
||||||
|
all_sizes.extend(b[4])
|
||||||
|
sol_sizes = [s for (s, _n) in all_sizes if s > 0]
|
||||||
|
nonce_sizes = [n for (_s, n) in all_sizes if n > 0]
|
||||||
|
|
||||||
|
ideal = onecore_rate * workers
|
||||||
|
return MiningPoint(
|
||||||
|
effort=effort,
|
||||||
|
samples=len(secs),
|
||||||
|
token_s_mean=token_s_mean,
|
||||||
|
token_s_median=statistics.median(secs),
|
||||||
|
token_s_stddev=statistics.pstdev(secs) if len(secs) > 1 else 0.0,
|
||||||
|
attempts_mean=attempts_mean,
|
||||||
|
achieved_mean=statistics.fmean(achieved),
|
||||||
|
tokens_per_sec_1core=onecore_rate,
|
||||||
|
workers=workers,
|
||||||
|
ok_workers=ok_workers,
|
||||||
|
tokens_per_sec_machine=machine_rate,
|
||||||
|
scaling_efficiency=(machine_rate / ideal) if ideal > 0 else 0.0,
|
||||||
|
failed_searches=fails + w_fails,
|
||||||
|
solution_bytes_min=min(sol_sizes) if sol_sizes else 0,
|
||||||
|
solution_bytes_max=max(sol_sizes) if sol_sizes else 0,
|
||||||
|
nonce_bytes_wire=max(nonce_sizes) if nonce_sizes else 0,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def run_mining(cfg: dict[str, Any], adapters: dict[str, Adapter], repo_root: Path,
|
||||||
|
device_resolver: Callable[[dict[str, Any]], str],
|
||||||
|
timeout: float) -> list[MiningResult]:
|
||||||
|
"""Drive the mining benchmark from a `[mining]` config block.
|
||||||
|
|
||||||
|
cfg keys (optional): impls ([] = all given), challenge_base ("abcd"),
|
||||||
|
efforts ([100,300,1000,3000,10000]), samples (12, 1-core mints per effort),
|
||||||
|
workers (0 = cpu count), tokens_per_worker (6, streamed per concurrent
|
||||||
|
worker), nonce_bytes (8), max_attempts (1_500_000)."""
|
||||||
|
import os
|
||||||
|
|
||||||
|
base = str(cfg.get("challenge_base", "abcd"))
|
||||||
|
efforts = [int(e) for e in cfg.get("efforts", [100, 300, 1000, 3000, 10000])]
|
||||||
|
samples = int(cfg.get("samples", 12))
|
||||||
|
workers = int(cfg.get("workers", 0)) or (os.cpu_count() or 4)
|
||||||
|
tokens_per_worker = int(cfg.get("tokens_per_worker", cfg.get("reps", 6)))
|
||||||
|
nonce_bytes = int(cfg.get("nonce_bytes", 8))
|
||||||
|
max_attempts = int(cfg.get("max_attempts", 1_500_000))
|
||||||
|
want = list(cfg.get("impls", [])) or list(adapters.keys())
|
||||||
|
|
||||||
|
impls = [(n, adapters[n]) for n in want if n in adapters]
|
||||||
|
out: list[MiningResult] = []
|
||||||
|
for name, adapter in impls:
|
||||||
|
# Learn device from a cheap probe: target_effort=1 with a single attempt
|
||||||
|
# (a probe at efforts[0] would run a full search — minutes on a slow
|
||||||
|
# interpreted impl at high E, just to read the env).
|
||||||
|
try:
|
||||||
|
probe = run(adapter, _effort_spec(base, 1, 0, nonce_bytes, 1, 0, 1),
|
||||||
|
repo_root, timeout=timeout)
|
||||||
|
device = device_resolver(probe.env)
|
||||||
|
except RunnerError:
|
||||||
|
device = device_resolver({})
|
||||||
|
res = MiningResult(device=device, impl=name, challenge_base=base, nproc=workers)
|
||||||
|
for e in efforts:
|
||||||
|
res.points.append(measure_point(adapter, base, e, samples, workers,
|
||||||
|
tokens_per_worker, nonce_bytes, max_attempts,
|
||||||
|
repo_root, timeout))
|
||||||
|
out.append(res)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def write_csv(results: list[MiningResult], path: Path) -> None:
|
||||||
|
import csv
|
||||||
|
|
||||||
|
with open(path, "w", newline="") as f:
|
||||||
|
w = csv.writer(f)
|
||||||
|
w.writerow([
|
||||||
|
"device", "impl", "challenge_base", "effort", "samples",
|
||||||
|
"attempts_mean", "achieved_mean", "token_s_mean", "token_s_median",
|
||||||
|
"token_s_stddev", "tokens_per_sec_1core", "workers", "ok_workers",
|
||||||
|
"tokens_per_sec_machine", "scaling_efficiency", "failed_searches",
|
||||||
|
"solution_bytes_min", "solution_bytes_max", "nonce_bytes_wire",
|
||||||
|
])
|
||||||
|
for r in results:
|
||||||
|
for p in r.points:
|
||||||
|
w.writerow([
|
||||||
|
r.device, r.impl, r.challenge_base, p.effort, p.samples,
|
||||||
|
f"{p.attempts_mean:.2f}", f"{p.achieved_mean:.1f}",
|
||||||
|
f"{p.token_s_mean:.6f}", f"{p.token_s_median:.6f}",
|
||||||
|
f"{p.token_s_stddev:.6f}", f"{p.tokens_per_sec_1core:.4f}",
|
||||||
|
p.workers, p.ok_workers, f"{p.tokens_per_sec_machine:.4f}",
|
||||||
|
f"{p.scaling_efficiency:.4f}", p.failed_searches,
|
||||||
|
p.solution_bytes_min, p.solution_bytes_max, p.nonce_bytes_wire,
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def read_csv(path: Path) -> list[MiningResult]:
|
||||||
|
"""Reconstruct MiningResults from a mining.csv (the inverse of write_csv), so
|
||||||
|
`combine` can fold in each device's measured mint-rate ladder. Every field is
|
||||||
|
round-tripped from the CSV; nproc is recovered from the per-point worker count."""
|
||||||
|
import csv
|
||||||
|
|
||||||
|
# Columns added over time (failed_searches, message-size fields) may be absent
|
||||||
|
# in older CSVs; default them so a mixed-vintage `combine` still round-trips.
|
||||||
|
def _i(row, k):
|
||||||
|
v = row.get(k, "")
|
||||||
|
return int(v) if v not in ("", None) else 0
|
||||||
|
|
||||||
|
def _f(row, k):
|
||||||
|
v = row.get(k, "")
|
||||||
|
return float(v) if v not in ("", None) else 0.0
|
||||||
|
|
||||||
|
by_key: dict[tuple[str, str, str], MiningResult] = {}
|
||||||
|
with open(path, newline="") as f:
|
||||||
|
for row in csv.DictReader(f):
|
||||||
|
key = (row["device"], row["impl"], row["challenge_base"])
|
||||||
|
res = by_key.get(key)
|
||||||
|
if res is None:
|
||||||
|
res = MiningResult(device=row["device"], impl=row["impl"],
|
||||||
|
challenge_base=row["challenge_base"], nproc=0)
|
||||||
|
by_key[key] = res
|
||||||
|
res.points.append(MiningPoint(
|
||||||
|
effort=_i(row, "effort"),
|
||||||
|
samples=_i(row, "samples"),
|
||||||
|
token_s_mean=_f(row, "token_s_mean"),
|
||||||
|
token_s_median=_f(row, "token_s_median"),
|
||||||
|
token_s_stddev=_f(row, "token_s_stddev"),
|
||||||
|
attempts_mean=_f(row, "attempts_mean"),
|
||||||
|
achieved_mean=_f(row, "achieved_mean"),
|
||||||
|
tokens_per_sec_1core=_f(row, "tokens_per_sec_1core"),
|
||||||
|
workers=_i(row, "workers"),
|
||||||
|
ok_workers=_i(row, "ok_workers"),
|
||||||
|
tokens_per_sec_machine=_f(row, "tokens_per_sec_machine"),
|
||||||
|
scaling_efficiency=_f(row, "scaling_efficiency"),
|
||||||
|
failed_searches=_i(row, "failed_searches"),
|
||||||
|
solution_bytes_min=_i(row, "solution_bytes_min"),
|
||||||
|
solution_bytes_max=_i(row, "solution_bytes_max"),
|
||||||
|
nonce_bytes_wire=_i(row, "nonce_bytes_wire"),
|
||||||
|
))
|
||||||
|
for res in by_key.values():
|
||||||
|
res.points.sort(key=lambda p: p.effort)
|
||||||
|
res.nproc = max((p.workers for p in res.points), default=0)
|
||||||
|
return list(by_key.values())
|
||||||
103
tools/benchmarks/Equi-X/harness/equix_bench/protocol.py
Normal file
103
tools/benchmarks/Equi-X/harness/equix_bench/protocol.py
Normal file
@ -0,0 +1,103 @@
|
|||||||
|
"""Wire protocol: job-spec (harness -> runner) and result (runner -> harness).
|
||||||
|
|
||||||
|
The schema is documented in adapters/README.md. Runners are language-agnostic;
|
||||||
|
this module is the single source of truth for how the Python harness speaks it.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from dataclasses import asdict, dataclass, field
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
SCHEMA_VERSION = 1
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class JobSpec:
|
||||||
|
"""One runner invocation. Only the fields relevant to the operation are set."""
|
||||||
|
|
||||||
|
operation: str # solve | verify | effort | hashx_compile
|
||||||
|
runtime: str # interpret | try-compile | must-compile
|
||||||
|
repetitions: int = 10
|
||||||
|
warmup: int = 3
|
||||||
|
challenge_hex: Optional[str] = None
|
||||||
|
# When set, each rep derives a fresh challenge by SHA-256-chaining this seed
|
||||||
|
# (solve/verify only); challenge generation is excluded from timed regions.
|
||||||
|
challenge_seed_hex: Optional[str] = None
|
||||||
|
challenge_base_hex: Optional[str] = None
|
||||||
|
solution_hex: Optional[str] = None
|
||||||
|
nonce_bytes: Optional[int] = None
|
||||||
|
nonce_start: Optional[int] = None
|
||||||
|
target_effort: Optional[int] = None
|
||||||
|
max_attempts: Optional[int] = None
|
||||||
|
seed: Optional[int] = None
|
||||||
|
|
||||||
|
def to_json(self) -> str:
|
||||||
|
d: dict[str, Any] = {"schema_version": SCHEMA_VERSION}
|
||||||
|
d.update({k: v for k, v in asdict(self).items() if v is not None})
|
||||||
|
return json.dumps(d)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Run:
|
||||||
|
index: int
|
||||||
|
wall_ns: int
|
||||||
|
solutions: int
|
||||||
|
compile_ns: int
|
||||||
|
attempts: int
|
||||||
|
achieved_effort: int
|
||||||
|
verify_result: Optional[str]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Result:
|
||||||
|
ok: bool
|
||||||
|
impl_name: str
|
||||||
|
impl_version: str
|
||||||
|
impl_commit: str
|
||||||
|
operation: str
|
||||||
|
runtime_requested: str
|
||||||
|
runtime_effective: Optional[str]
|
||||||
|
env: dict[str, Any]
|
||||||
|
runs: list[Run]
|
||||||
|
solutions_hex: Optional[list[str]]
|
||||||
|
peak_rss_kb: int
|
||||||
|
error: Optional[str]
|
||||||
|
raw: dict[str, Any] = field(default_factory=dict)
|
||||||
|
# effort op only: wire bytes (hex) of the winning token's nonce.
|
||||||
|
winning_nonce_hex: Optional[str] = None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def from_dict(d: dict[str, Any]) -> "Result":
|
||||||
|
sv = d.get("schema_version")
|
||||||
|
if sv != SCHEMA_VERSION:
|
||||||
|
raise ValueError(f"unsupported schema_version {sv!r} (expected {SCHEMA_VERSION})")
|
||||||
|
impl = d.get("impl", {}) or {}
|
||||||
|
runs = [
|
||||||
|
Run(
|
||||||
|
index=r.get("index", i),
|
||||||
|
wall_ns=int(r.get("wall_ns", 0)),
|
||||||
|
solutions=int(r.get("solutions", 0)),
|
||||||
|
compile_ns=int(r.get("compile_ns", 0)),
|
||||||
|
attempts=int(r.get("attempts", 0)),
|
||||||
|
achieved_effort=int(r.get("achieved_effort", 0)),
|
||||||
|
verify_result=r.get("verify_result"),
|
||||||
|
)
|
||||||
|
for i, r in enumerate(d.get("runs", []) or [])
|
||||||
|
]
|
||||||
|
return Result(
|
||||||
|
ok=bool(d.get("ok", False)),
|
||||||
|
impl_name=impl.get("name", "?"),
|
||||||
|
impl_version=impl.get("version", "?"),
|
||||||
|
impl_commit=impl.get("commit", "?"),
|
||||||
|
operation=d.get("operation", "?"),
|
||||||
|
runtime_requested=d.get("runtime_requested", "?"),
|
||||||
|
runtime_effective=d.get("runtime_effective"),
|
||||||
|
env=d.get("env", {}) or {},
|
||||||
|
runs=runs,
|
||||||
|
solutions_hex=d.get("solutions_hex"),
|
||||||
|
peak_rss_kb=int(d.get("peak_rss_kb", 0)),
|
||||||
|
error=d.get("error"),
|
||||||
|
raw=d,
|
||||||
|
winning_nonce_hex=d.get("winning_nonce_hex"),
|
||||||
|
)
|
||||||
72
tools/benchmarks/Equi-X/harness/equix_bench/registry.py
Normal file
72
tools/benchmarks/Equi-X/harness/equix_bench/registry.py
Normal file
@ -0,0 +1,72 @@
|
|||||||
|
"""Adapter registry: discover implementations from TOML manifests.
|
||||||
|
|
||||||
|
Each implementation ("adapter") ships a manifest declaring how to invoke its
|
||||||
|
runner and what it supports. New implementations plug in by adding a manifest --
|
||||||
|
no harness code changes. See adapters/README.md.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import tomllib
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Adapter:
|
||||||
|
name: str
|
||||||
|
exec: list[str] # argv; paths resolved against repo root
|
||||||
|
protocol_version: int
|
||||||
|
capabilities: list[str]
|
||||||
|
runtimes: list[str]
|
||||||
|
env: dict[str, str]
|
||||||
|
|
||||||
|
def resolve(self, repo_root: Path) -> list[str]:
|
||||||
|
"""Resolve the executable path (first argv element) against repo root."""
|
||||||
|
argv = list(self.exec)
|
||||||
|
p = Path(argv[0])
|
||||||
|
if not p.is_absolute():
|
||||||
|
p = (repo_root / p).resolve()
|
||||||
|
argv[0] = str(p)
|
||||||
|
return argv
|
||||||
|
|
||||||
|
def available(self, repo_root: Path) -> bool:
|
||||||
|
argv = self.resolve(repo_root)
|
||||||
|
first = argv[0]
|
||||||
|
# A bare interpreter name (e.g. "python3") is looked up on PATH.
|
||||||
|
if "/" not in self.exec[0]:
|
||||||
|
return True
|
||||||
|
return os.path.exists(first) and os.access(first, os.X_OK)
|
||||||
|
|
||||||
|
|
||||||
|
def load_manifest(path: Path) -> Adapter:
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
d = tomllib.load(f)
|
||||||
|
exec_field = d["exec"]
|
||||||
|
if isinstance(exec_field, str):
|
||||||
|
exec_field = [exec_field]
|
||||||
|
return Adapter(
|
||||||
|
name=d["name"],
|
||||||
|
exec=list(exec_field),
|
||||||
|
protocol_version=int(d.get("protocol_version", 1)),
|
||||||
|
capabilities=list(d.get("capabilities", [])),
|
||||||
|
runtimes=list(d.get("runtimes", [])),
|
||||||
|
env=dict(d.get("env", {})),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def load_manifests(manifest_dirs) -> dict[str, Adapter]:
|
||||||
|
"""Load adapter manifests from one directory or several (later dirs win on
|
||||||
|
name collision). This lets generated compiler-flag variants in a second dir
|
||||||
|
coexist with the built-in adapters."""
|
||||||
|
if isinstance(manifest_dirs, (str, Path)):
|
||||||
|
manifest_dirs = [manifest_dirs]
|
||||||
|
adapters: dict[str, Adapter] = {}
|
||||||
|
for d in manifest_dirs:
|
||||||
|
d = Path(d)
|
||||||
|
if not d.is_dir():
|
||||||
|
continue
|
||||||
|
for p in sorted(d.glob("*.manifest.toml")):
|
||||||
|
a = load_manifest(p)
|
||||||
|
adapters[a.name] = a
|
||||||
|
return adapters
|
||||||
767
tools/benchmarks/Equi-X/harness/equix_bench/report.py
Normal file
767
tools/benchmarks/Equi-X/harness/equix_bench/report.py
Normal file
@ -0,0 +1,767 @@
|
|||||||
|
"""Reporting: CSV, raw JSON, comparison plots, and a markdown report.
|
||||||
|
|
||||||
|
Design rules (per requirements):
|
||||||
|
* EVERY plot compares all implementations on the same axes. `_require_multi_impl`
|
||||||
|
fails loudly if a figure would show fewer than two implementations.
|
||||||
|
* Plots reflect the executing device (CPU). With ONE device, the device is shown
|
||||||
|
in the title. With MULTIPLE devices (e.g. after `combine`), each plot becomes a
|
||||||
|
small-multiples grid (one facet per device, C-vs-Rust within each), plus
|
||||||
|
dedicated cross-device charts (x=device, series=impl) for headline metrics.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
|
from collections import defaultdict
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Callable, Iterable, Optional
|
||||||
|
|
||||||
|
import matplotlib
|
||||||
|
|
||||||
|
matplotlib.use("Agg") # headless
|
||||||
|
import matplotlib.pyplot as plt # noqa: E402
|
||||||
|
import numpy as np # noqa: E402
|
||||||
|
|
||||||
|
from .crosscheck import Check
|
||||||
|
from .dosprotect import assess as dos_assess
|
||||||
|
from .dosprotect import min_verify_seconds
|
||||||
|
from .stats import CellStats
|
||||||
|
|
||||||
|
_PALETTE = ["#4C72B0", "#DD8452", "#55A868", "#C44E52", "#8172B3", "#937860"]
|
||||||
|
|
||||||
|
|
||||||
|
def _impl_color(impl: str, all_impls: list[str]) -> str:
|
||||||
|
idx = all_impls.index(impl) if impl in all_impls else len(all_impls)
|
||||||
|
return _PALETTE[idx % len(_PALETTE)]
|
||||||
|
|
||||||
|
|
||||||
|
def _require_multi_impl(impls_present: Iterable[str], plot: str) -> list[str]:
|
||||||
|
impls = sorted(set(impls_present))
|
||||||
|
if len(impls) < 2:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"plot '{plot}' requires >=2 implementations to compare, "
|
||||||
|
f"but only found: {impls}. Build/enable both C and Rust runners."
|
||||||
|
)
|
||||||
|
return impls
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------- outputs
|
||||||
|
|
||||||
|
|
||||||
|
def write_csv(stats: list[CellStats], path: Path) -> None:
|
||||||
|
cols = [
|
||||||
|
"impl", "device_label", "device_type", "device_name", "device_arch",
|
||||||
|
"operation", "runtime_requested", "runtime_effective", "label",
|
||||||
|
"reps", "ok", "min_ns", "median_ns", "mean_ns", "stddev_ns", "p95_ns",
|
||||||
|
"solutions_mean", "compile_median_ns", "attempts_mean",
|
||||||
|
"achieved_effort_mean", "solves_per_sec", "hashes_per_sec",
|
||||||
|
"peak_rss_kb", "verify_result", "error",
|
||||||
|
]
|
||||||
|
with open(path, "w", newline="") as f:
|
||||||
|
w = csv.writer(f)
|
||||||
|
w.writerow(cols)
|
||||||
|
for s in stats:
|
||||||
|
w.writerow([
|
||||||
|
s.impl, s.device_label, s.device_type, s.device_name, s.device_arch,
|
||||||
|
s.operation, s.runtime_requested, s.runtime_effective,
|
||||||
|
json.dumps(s.label), s.reps, s.ok, f"{s.min_ns:.1f}",
|
||||||
|
f"{s.median_ns:.1f}", f"{s.mean_ns:.1f}", f"{s.stddev_ns:.1f}",
|
||||||
|
f"{s.p95_ns:.1f}", f"{s.solutions_mean:.3f}",
|
||||||
|
f"{s.compile_median_ns:.1f}", f"{s.attempts_mean:.2f}",
|
||||||
|
f"{s.achieved_effort_mean:.1f}", f"{s.solves_per_sec:.2f}",
|
||||||
|
f"{s.hashes_per_sec:.0f}", s.peak_rss_kb, s.verify_result, s.error,
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def write_raw(raw: list[dict[str, Any]], path: Path) -> None:
|
||||||
|
with open(path, "w") as f:
|
||||||
|
json.dump(raw, f, indent=2)
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------- plot helpers
|
||||||
|
|
||||||
|
|
||||||
|
def _grouped_bar(ax, categories, series, all_impls, errors=None) -> None:
|
||||||
|
names = sorted(series.keys())
|
||||||
|
n = len(names)
|
||||||
|
width = 0.8 / max(n, 1)
|
||||||
|
x = np.arange(len(categories))
|
||||||
|
for i, name in enumerate(names):
|
||||||
|
off = (i - (n - 1) / 2) * width
|
||||||
|
err = errors.get(name) if errors else None
|
||||||
|
ax.bar(x + off, series[name], width, label=name, yerr=err, capsize=3,
|
||||||
|
color=_impl_color(name, all_impls), edgecolor="black", linewidth=0.4)
|
||||||
|
ax.set_xticks(x)
|
||||||
|
ax.set_xticklabels(categories, rotation=0)
|
||||||
|
ax.legend(title="implementation", fontsize=8)
|
||||||
|
ax.grid(axis="y", linestyle=":", alpha=0.5)
|
||||||
|
|
||||||
|
|
||||||
|
def _agg(stats, cat_key, val):
|
||||||
|
"""Aggregate (mean) `val` grouped by (category, impl) over a pre-filtered set."""
|
||||||
|
buckets: dict[tuple[str, str], list[float]] = defaultdict(list)
|
||||||
|
impls_present: set[str] = set()
|
||||||
|
cats: list[str] = []
|
||||||
|
for s in stats:
|
||||||
|
c = cat_key(s)
|
||||||
|
buckets[(s.impl, c)].append(val(s))
|
||||||
|
impls_present.add(s.impl)
|
||||||
|
if c not in cats:
|
||||||
|
cats.append(c)
|
||||||
|
cats = sorted(cats)
|
||||||
|
impls = sorted(impls_present)
|
||||||
|
series = {
|
||||||
|
impl: [float(np.mean(buckets[(impl, c)])) if buckets.get((impl, c)) else 0.0
|
||||||
|
for c in cats]
|
||||||
|
for impl in impls
|
||||||
|
}
|
||||||
|
return cats, series, impls
|
||||||
|
|
||||||
|
|
||||||
|
def _save(fig, path: Path) -> str:
|
||||||
|
# Only run tight_layout when no layout engine is active (faceted figures use
|
||||||
|
# constrained layout, which is incompatible with tight_layout and can produce
|
||||||
|
# NaN axes geometry if both run).
|
||||||
|
try:
|
||||||
|
if fig.get_layout_engine() is None:
|
||||||
|
fig.tight_layout()
|
||||||
|
except Exception: # noqa: BLE001 - layout is best-effort, never fatal
|
||||||
|
pass
|
||||||
|
fig.savefig(path, dpi=110)
|
||||||
|
plt.close(fig)
|
||||||
|
return path.name
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ panels (per device)
|
||||||
|
|
||||||
|
|
||||||
|
def _panel_time(ax, stats, all_impls):
|
||||||
|
cats, series, _ = _agg(stats, lambda s: s.runtime_requested, lambda s: s.median_ns)
|
||||||
|
_, errs, _ = _agg(stats, lambda s: s.runtime_requested,
|
||||||
|
lambda s: max(0.0, s.p95_ns - s.median_ns))
|
||||||
|
_grouped_bar(ax, cats, series, all_impls, errs)
|
||||||
|
ax.set_xlabel("HashX runtime")
|
||||||
|
|
||||||
|
|
||||||
|
def _panel_throughput(ax, stats, all_impls):
|
||||||
|
cats, series, _ = _agg(stats, lambda s: s.runtime_requested, lambda s: s.solves_per_sec)
|
||||||
|
_grouped_bar(ax, cats, series, all_impls)
|
||||||
|
ax.set_xlabel("HashX runtime")
|
||||||
|
|
||||||
|
|
||||||
|
def _panel_rss(ax, stats, all_impls):
|
||||||
|
cats, series, _ = _agg(stats, lambda s: s.runtime_requested, lambda s: float(s.peak_rss_kb))
|
||||||
|
_grouped_bar(ax, cats, series, all_impls)
|
||||||
|
ax.set_xlabel("HashX runtime")
|
||||||
|
|
||||||
|
|
||||||
|
def _panel_compile(ax, stats, all_impls):
|
||||||
|
cats, series, _ = _agg(stats, lambda s: s.runtime_requested, lambda s: s.compile_median_ns)
|
||||||
|
_grouped_bar(ax, cats, series, all_impls)
|
||||||
|
ax.set_xlabel("HashX runtime")
|
||||||
|
|
||||||
|
|
||||||
|
def _panel_speedup(ax, stats, all_impls):
|
||||||
|
med: dict[tuple[str, str], list[float]] = defaultdict(list)
|
||||||
|
for s in stats:
|
||||||
|
med[(s.impl, s.runtime_requested)].append(s.median_ns)
|
||||||
|
impls = sorted({s.impl for s in stats})
|
||||||
|
speedups = []
|
||||||
|
for impl in impls:
|
||||||
|
interp = med.get((impl, "interpret"))
|
||||||
|
comp = med.get((impl, "try-compile")) or med.get((impl, "must-compile"))
|
||||||
|
speedups.append(float(np.mean(interp)) / float(np.mean(comp)) if interp and comp else 0.0)
|
||||||
|
x = np.arange(len(impls))
|
||||||
|
ax.bar(x, speedups, 0.5, color=[_impl_color(i, all_impls) for i in impls],
|
||||||
|
edgecolor="black", linewidth=0.4)
|
||||||
|
for xi, v in zip(x, speedups):
|
||||||
|
ax.text(xi, v, f"{v:.1f}x", ha="center", va="bottom", fontsize=8)
|
||||||
|
ax.set_xticks(x)
|
||||||
|
ax.set_xticklabels(impls)
|
||||||
|
ax.set_xlabel("implementation")
|
||||||
|
|
||||||
|
|
||||||
|
def _panel_distribution(ax, stats, all_impls):
|
||||||
|
runtimes: list[str] = []
|
||||||
|
data: dict[tuple[str, str], list[float]] = defaultdict(list)
|
||||||
|
impls = sorted({s.impl for s in stats})
|
||||||
|
for s in stats:
|
||||||
|
if s.walls:
|
||||||
|
if s.runtime_requested not in runtimes:
|
||||||
|
runtimes.append(s.runtime_requested)
|
||||||
|
data[(s.impl, s.runtime_requested)].extend([w / 1e6 for w in s.walls])
|
||||||
|
runtimes = sorted(runtimes)
|
||||||
|
positions, box_data, colors = [], [], []
|
||||||
|
width = 0.8 / max(len(impls), 1)
|
||||||
|
for ri, rt in enumerate(runtimes):
|
||||||
|
for ii, impl in enumerate(impls):
|
||||||
|
vals = data.get((impl, rt))
|
||||||
|
if not vals:
|
||||||
|
continue
|
||||||
|
positions.append(ri + (ii - (len(impls) - 1) / 2) * width)
|
||||||
|
box_data.append(vals)
|
||||||
|
colors.append(_impl_color(impl, all_impls))
|
||||||
|
if box_data:
|
||||||
|
bp = ax.boxplot(box_data, positions=positions, widths=width * 0.9,
|
||||||
|
patch_artist=True, showfliers=False)
|
||||||
|
for patch, c in zip(bp["boxes"], colors):
|
||||||
|
patch.set_facecolor(c)
|
||||||
|
patch.set_alpha(0.8)
|
||||||
|
ax.set_xticks(range(len(runtimes)))
|
||||||
|
ax.set_xticklabels(runtimes)
|
||||||
|
ax.set_xlabel("HashX runtime")
|
||||||
|
from matplotlib.patches import Patch
|
||||||
|
ax.legend(handles=[Patch(facecolor=_impl_color(i, all_impls), label=i) for i in impls],
|
||||||
|
title="implementation", fontsize=8)
|
||||||
|
ax.grid(axis="y", linestyle=":", alpha=0.5)
|
||||||
|
|
||||||
|
|
||||||
|
def _panel_effort(ax, stats, all_impls, which):
|
||||||
|
pts: dict[str, dict[int, tuple[float, float]]] = defaultdict(dict)
|
||||||
|
for s in stats:
|
||||||
|
t = int(s.label.get("target_effort", 0))
|
||||||
|
pts[s.impl][t] = (s.attempts_mean, s.median_ns / 1e9)
|
||||||
|
for impl in sorted(pts):
|
||||||
|
xs = sorted(pts[impl].keys())
|
||||||
|
ys = [pts[impl][x][0 if which == "attempts" else 1] for x in xs]
|
||||||
|
ax.plot(xs, ys, "o-", label=impl, color=_impl_color(impl, all_impls))
|
||||||
|
ax.set_xscale("log")
|
||||||
|
if which == "attempts":
|
||||||
|
ax.set_yscale("log")
|
||||||
|
ax.set_ylabel("mean attempts")
|
||||||
|
else:
|
||||||
|
ax.set_ylabel("median time (s)")
|
||||||
|
ax.set_xlabel("target effort")
|
||||||
|
ax.legend(fontsize=8)
|
||||||
|
ax.grid(True, which="both", linestyle=":", alpha=0.5)
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ facet driver
|
||||||
|
|
||||||
|
|
||||||
|
def _facet(stats, panel_fn, title, ylabel, path, all_impls, keep) -> str:
|
||||||
|
subset = [s for s in stats if keep(s) and s.ok]
|
||||||
|
devices = sorted({s.device_label for s in subset})
|
||||||
|
_require_multi_impl({s.impl for s in subset}, path.name)
|
||||||
|
|
||||||
|
if len(devices) <= 1:
|
||||||
|
fig, ax = plt.subplots(figsize=(8, 4.5))
|
||||||
|
panel_fn(ax, subset, all_impls)
|
||||||
|
dev = devices[0] if devices else "host"
|
||||||
|
ax.set_title(f"{title}\n[{dev}]")
|
||||||
|
ax.set_ylabel(ylabel)
|
||||||
|
return _save(fig, path)
|
||||||
|
|
||||||
|
n = len(devices)
|
||||||
|
fig, axes = plt.subplots(1, n, figsize=(min(7 * n, 22), 4.8), sharey=True,
|
||||||
|
squeeze=False, layout="constrained")
|
||||||
|
for ax, dev in zip(axes[0], devices):
|
||||||
|
panel_fn(ax, [s for s in subset if s.device_label == dev], all_impls)
|
||||||
|
ax.set_title(dev)
|
||||||
|
axes[0][0].set_ylabel(ylabel)
|
||||||
|
fig.suptitle(title, fontweight="bold")
|
||||||
|
return _save(fig, path)
|
||||||
|
|
||||||
|
|
||||||
|
def _plot_dos(stats, path, all_impls) -> Optional[str]:
|
||||||
|
"""DoS-protection asymmetry: attacker cost to craft a token / defender verify
|
||||||
|
cost, vs effort. One line per implementation (attacker), faceted per device."""
|
||||||
|
panels = []
|
||||||
|
for dev in sorted({s.device_label for s in stats if s.ok and s.operation == "effort"}):
|
||||||
|
vs = min_verify_seconds(stats, dev)
|
||||||
|
if not vs:
|
||||||
|
continue
|
||||||
|
per_impl: dict[str, dict[int, float]] = defaultdict(dict)
|
||||||
|
for s in stats:
|
||||||
|
if s.operation == "effort" and s.ok and s.device_label == dev and s.median_ns > 0:
|
||||||
|
e = int(s.label.get("target_effort", 0))
|
||||||
|
per_impl[s.impl][e] = (s.median_ns / 1e9) / vs
|
||||||
|
if per_impl:
|
||||||
|
panels.append((dev, per_impl))
|
||||||
|
if not panels:
|
||||||
|
return None
|
||||||
|
_require_multi_impl({im for _, pi in panels for im in pi}, path.name)
|
||||||
|
|
||||||
|
from .dosprotect import DEFAULT_THRESHOLD
|
||||||
|
n = len(panels)
|
||||||
|
fig, axes = plt.subplots(1, n, figsize=(min(7 * n, 22), 4.8), squeeze=False,
|
||||||
|
sharey=True, layout="constrained" if n > 1 else None)
|
||||||
|
for ax, (dev, per_impl) in zip(axes[0], panels):
|
||||||
|
for impl in sorted(per_impl):
|
||||||
|
xs = sorted(per_impl[impl])
|
||||||
|
ys = [per_impl[impl][x] for x in xs]
|
||||||
|
ax.plot(xs, ys, "o-", label=impl, color=_impl_color(impl, all_impls))
|
||||||
|
ax.axhline(DEFAULT_THRESHOLD, ls="--", color="red", alpha=0.7,
|
||||||
|
label=f"effective ≥ {DEFAULT_THRESHOLD:.0f}×")
|
||||||
|
ax.set_xscale("log")
|
||||||
|
ax.set_yscale("log")
|
||||||
|
ax.set_xlabel("target effort (attacker difficulty)")
|
||||||
|
ax.set_title(dev)
|
||||||
|
ax.grid(True, which="both", linestyle=":", alpha=0.5)
|
||||||
|
ax.legend(fontsize=8)
|
||||||
|
axes[0][0].set_ylabel("protection factor (attacker time / verify time)")
|
||||||
|
fig.suptitle("DoS-protection asymmetry: cost to attack vs cost to verify", fontweight="bold")
|
||||||
|
return _save(fig, path)
|
||||||
|
|
||||||
|
|
||||||
|
def _cross_device_bar(stats, keep, val, title, ylabel, path, all_impls) -> Optional[str]:
|
||||||
|
subset = [s for s in stats if keep(s) and s.ok]
|
||||||
|
devices = sorted({s.device_label for s in subset})
|
||||||
|
if len(devices) < 2:
|
||||||
|
return None # only meaningful across multiple devices
|
||||||
|
cats, series, impls = _agg(subset, lambda s: s.device_label, val)
|
||||||
|
_require_multi_impl(impls, path.name)
|
||||||
|
fig, ax = plt.subplots(figsize=(max(6, 2 + 2 * len(devices)), 4.5))
|
||||||
|
_grouped_bar(ax, cats, series, all_impls)
|
||||||
|
ax.set_xlabel("device / CPU")
|
||||||
|
ax.set_ylabel(ylabel)
|
||||||
|
ax.set_title(title)
|
||||||
|
return _save(fig, path)
|
||||||
|
|
||||||
|
|
||||||
|
def _devices_of(results) -> list[str]:
|
||||||
|
"""Devices present in a concurrency/mining result set, in stable order.
|
||||||
|
Blank device labels (e.g. a single-host run before `combine`) collapse to one
|
||||||
|
unnamed facet so the plot still renders."""
|
||||||
|
return sorted({(getattr(r, "device", "") or "") for r in results})
|
||||||
|
|
||||||
|
|
||||||
|
def _facet_axes(devices, figsize_per, **kw):
|
||||||
|
"""One subplot column per device (constrained layout when faceted, so the
|
||||||
|
shared suptitle never overlaps the panels)."""
|
||||||
|
n = max(len(devices), 1)
|
||||||
|
w_per, h = figsize_per
|
||||||
|
fig, axes = plt.subplots(1, n, figsize=(min(w_per * n, 22), h), squeeze=False,
|
||||||
|
layout="constrained" if n > 1 else None, **kw)
|
||||||
|
return fig, axes[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _plot_concurrency(results, operation: str, path: Path, all_impls) -> Optional[str]:
|
||||||
|
"""Aggregate throughput vs worker count for one operation, one line per impl,
|
||||||
|
with the ideal-linear-scaling reference. Shows where the machine saturates.
|
||||||
|
With >1 device (after `combine`), each device gets its own panel."""
|
||||||
|
rows = [r for r in results if r.operation == operation and r.levels]
|
||||||
|
if not rows:
|
||||||
|
return None
|
||||||
|
devices = _devices_of(rows)
|
||||||
|
multi = len(devices) > 1
|
||||||
|
fig, axes = _facet_axes(devices, (7, 4.5), sharey=True)
|
||||||
|
for ax, dev in zip(axes, devices):
|
||||||
|
for r in sorted([x for x in rows if (getattr(x, "device", "") or "") == dev],
|
||||||
|
key=lambda r: r.impl):
|
||||||
|
pts = [(lv.workers, lv.aggregate_ops_per_sec) for lv in r.levels
|
||||||
|
if lv.aggregate_ops_per_sec > 0]
|
||||||
|
if not pts:
|
||||||
|
continue
|
||||||
|
xs, ys = zip(*pts)
|
||||||
|
color = _impl_color(r.impl, all_impls)
|
||||||
|
ax.plot(xs, ys, marker="o", color=color, label=f"{r.impl} (measured)")
|
||||||
|
# ideal linear scaling from this impl's single-worker baseline
|
||||||
|
if r.baseline_ops_per_sec > 0:
|
||||||
|
ax.plot(xs, [r.baseline_ops_per_sec * x for x in xs], linestyle=":",
|
||||||
|
color=color, alpha=0.5, label=f"{r.impl} (ideal linear)")
|
||||||
|
ax.set_xlabel("concurrent workers")
|
||||||
|
ax.set_title(dev if multi else f"[{dev}]" if dev else "")
|
||||||
|
ax.grid(True, alpha=0.3)
|
||||||
|
ax.legend(fontsize=8)
|
||||||
|
axes[0].set_ylabel(f"aggregate {operation}s / second")
|
||||||
|
fig.suptitle(f"Sustained {operation} throughput under concurrency (measured)",
|
||||||
|
fontweight="bold")
|
||||||
|
return _save(fig, path)
|
||||||
|
|
||||||
|
|
||||||
|
def _plot_mining(results, path: Path, all_impls) -> Optional[str]:
|
||||||
|
"""Token mint rate vs difficulty (effort), log-log. Per impl: 1-core and
|
||||||
|
whole-machine lines. A straight line here means rate ∝ 1/effort. With >1
|
||||||
|
device (after `combine`), each device gets its own panel."""
|
||||||
|
rows = [r for r in results if r.points]
|
||||||
|
if not rows:
|
||||||
|
return None
|
||||||
|
devices = _devices_of(rows)
|
||||||
|
multi = len(devices) > 1
|
||||||
|
fig, axes = _facet_axes(devices, (7, 4.5), sharey=True)
|
||||||
|
for ax, dev in zip(axes, devices):
|
||||||
|
for r in sorted([x for x in rows if (getattr(x, "device", "") or "") == dev],
|
||||||
|
key=lambda r: r.impl):
|
||||||
|
pts = [(p.effort, p.tokens_per_sec_1core, p.tokens_per_sec_machine)
|
||||||
|
for p in r.points if p.samples > 0 and p.tokens_per_sec_1core > 0]
|
||||||
|
if not pts:
|
||||||
|
continue
|
||||||
|
xs = [p[0] for p in pts]
|
||||||
|
color = _impl_color(r.impl, all_impls)
|
||||||
|
ax.plot(xs, [p[1] for p in pts], marker="o", color=color,
|
||||||
|
label=f"{r.impl} — 1 core")
|
||||||
|
ax.plot(xs, [p[2] for p in pts], marker="s", linestyle="--", color=color,
|
||||||
|
label=f"{r.impl} — {r.nproc} cores")
|
||||||
|
ax.set_xscale("log"); ax.set_yscale("log")
|
||||||
|
ax.set_xlabel("difficulty (effort target)")
|
||||||
|
ax.set_title(dev if multi else "")
|
||||||
|
ax.grid(True, which="both", alpha=0.3)
|
||||||
|
ax.legend(fontsize=8)
|
||||||
|
axes[0].set_ylabel("tokens minted / second")
|
||||||
|
fig.suptitle("Mining rate vs difficulty (measured)", fontweight="bold")
|
||||||
|
return _save(fig, path)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- report driver
|
||||||
|
|
||||||
|
|
||||||
|
def generate(stats, checks, raw, out_dir: Path, meta, concurrency=None, mining=None) -> None:
|
||||||
|
out_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
plots_dir = out_dir / "plots"
|
||||||
|
plots_dir.mkdir(exist_ok=True)
|
||||||
|
(out_dir / "raw").mkdir(exist_ok=True)
|
||||||
|
|
||||||
|
write_csv(stats, out_dir / "results.csv")
|
||||||
|
write_raw(raw, out_dir / "raw" / "results.json")
|
||||||
|
|
||||||
|
impl_meta: dict[str, dict[str, Any]] = {}
|
||||||
|
for r in raw:
|
||||||
|
im = r.get("impl", {}) or {}
|
||||||
|
name = im.get("name")
|
||||||
|
if name and name not in impl_meta:
|
||||||
|
impl_meta[name] = {"version": im.get("version"), "commit": im.get("commit"),
|
||||||
|
"compiler": (r.get("env", {}) or {}).get("compiler")}
|
||||||
|
meta = {**meta, "impls": impl_meta}
|
||||||
|
|
||||||
|
all_impls = sorted({s.impl for s in stats})
|
||||||
|
devices = sorted({s.device_label for s in stats if s.ok})
|
||||||
|
plots: list[str] = []
|
||||||
|
|
||||||
|
def try_plot(fn, *a, **k):
|
||||||
|
try:
|
||||||
|
r = fn(*a, **k)
|
||||||
|
if isinstance(r, list):
|
||||||
|
plots.extend([x for x in r if x])
|
||||||
|
elif r:
|
||||||
|
plots.append(r)
|
||||||
|
except RuntimeError as e:
|
||||||
|
plots.append(f"__error__:{e}")
|
||||||
|
|
||||||
|
has = lambda op: any(s.operation == op for s in stats)
|
||||||
|
|
||||||
|
if has("solve"):
|
||||||
|
try_plot(_facet, stats, _panel_time, "Solve time by runtime: C vs Rust",
|
||||||
|
"median solve time (ns)", plots_dir / "solve_time_by_runtime.png",
|
||||||
|
all_impls, lambda s: s.operation == "solve")
|
||||||
|
try_plot(_facet, stats, _panel_throughput, "Solve throughput: C vs Rust",
|
||||||
|
"solves / second", plots_dir / "throughput.png", all_impls,
|
||||||
|
lambda s: s.operation == "solve")
|
||||||
|
try_plot(_facet, stats, _panel_speedup, "JIT speedup over interpreter: C vs Rust",
|
||||||
|
"interpret / compiled (x faster)", plots_dir / "jit_speedup.png",
|
||||||
|
all_impls, lambda s: s.operation == "solve")
|
||||||
|
try_plot(_facet, stats, _panel_rss, "Peak memory during solve: C vs Rust",
|
||||||
|
"peak RSS (KB)", plots_dir / "peak_rss.png", all_impls,
|
||||||
|
lambda s: s.operation == "solve")
|
||||||
|
try_plot(_facet, stats, _panel_distribution, "Solve-time distribution: C vs Rust",
|
||||||
|
"solve time (ms)", plots_dir / "solve_distribution.png", all_impls,
|
||||||
|
lambda s: s.operation == "solve")
|
||||||
|
if has("verify"):
|
||||||
|
try_plot(_facet, stats, _panel_time, "Verify time by runtime: C vs Rust",
|
||||||
|
"median verify time (ns)", plots_dir / "verify_time.png", all_impls,
|
||||||
|
lambda s: s.operation == "verify")
|
||||||
|
if has("hashx_compile"):
|
||||||
|
try_plot(_facet, stats, _panel_compile, "HashX compile overhead: C vs Rust",
|
||||||
|
"median program-gen + compile (ns)", plots_dir / "compile_overhead.png",
|
||||||
|
all_impls, lambda s: s.operation == "hashx_compile")
|
||||||
|
if has("effort"):
|
||||||
|
try_plot(_facet, stats, lambda ax, s, im: _panel_effort(ax, s, im, "attempts"),
|
||||||
|
"Effort cost (attempts): C vs Rust", "mean attempts",
|
||||||
|
plots_dir / "effort_attempts.png", all_impls, lambda s: s.operation == "effort")
|
||||||
|
try_plot(_facet, stats, lambda ax, s, im: _panel_effort(ax, s, im, "time"),
|
||||||
|
"Effort cost (time): C vs Rust", "median time (s)",
|
||||||
|
plots_dir / "effort_time.png", all_impls, lambda s: s.operation == "effort")
|
||||||
|
|
||||||
|
# Cross-device (CPU vs CPU) headline charts -- emitted only when >1 device.
|
||||||
|
if len(devices) >= 2:
|
||||||
|
try_plot(_cross_device_bar, stats,
|
||||||
|
lambda s: s.operation == "solve" and s.runtime_requested in ("try-compile", "must-compile"),
|
||||||
|
lambda s: s.solves_per_sec, "Solve throughput across CPUs: C vs Rust",
|
||||||
|
"solves / second", plots_dir / "xdev_throughput.png", all_impls)
|
||||||
|
try_plot(_cross_device_bar, stats,
|
||||||
|
lambda s: s.operation == "solve" and s.runtime_requested in ("try-compile", "must-compile"),
|
||||||
|
lambda s: s.median_ns, "Solve time across CPUs: C vs Rust",
|
||||||
|
"median solve time (ns)", plots_dir / "xdev_solve_time.png", all_impls)
|
||||||
|
try_plot(_cross_device_bar, stats, lambda s: s.operation == "solve",
|
||||||
|
lambda s: float(s.peak_rss_kb), "Peak RSS across CPUs: C vs Rust",
|
||||||
|
"peak RSS (KB)", plots_dir / "xdev_peak_rss.png", all_impls)
|
||||||
|
try_plot(_cross_device_bar, stats, lambda s: s.operation == "verify",
|
||||||
|
lambda s: s.median_ns, "Verify time across CPUs: C vs Rust",
|
||||||
|
"median verify time (ns)", plots_dir / "xdev_verify_time.png", all_impls)
|
||||||
|
|
||||||
|
# DoS-protection effectiveness (needs both effort and verify measurements).
|
||||||
|
dos = None
|
||||||
|
if any(s.operation == "effort" for s in stats) and any(s.operation == "verify" for s in stats):
|
||||||
|
try_plot(_plot_dos, stats, plots_dir / "dos_protection.png", all_impls)
|
||||||
|
dos = dos_assess(stats)
|
||||||
|
|
||||||
|
# Concurrency / saturation curves (one figure per operation; C vs Rust lines).
|
||||||
|
if concurrency:
|
||||||
|
for op in sorted({r.operation for r in concurrency}):
|
||||||
|
try_plot(_plot_concurrency, concurrency, op,
|
||||||
|
plots_dir / f"concurrency_{op}.png", all_impls)
|
||||||
|
|
||||||
|
# Mining rate vs difficulty (tokens/s vs effort, 1-core and whole-machine).
|
||||||
|
if mining:
|
||||||
|
try_plot(_plot_mining, mining, plots_dir / "mining_rate.png", all_impls)
|
||||||
|
|
||||||
|
_write_markdown(stats, checks, plots, out_dir, meta, devices, dos, concurrency, mining)
|
||||||
|
|
||||||
|
|
||||||
|
def _fmt_ns(ns: float) -> str:
|
||||||
|
if ns >= 1e9:
|
||||||
|
return f"{ns/1e9:.3f} s"
|
||||||
|
if ns >= 1e6:
|
||||||
|
return f"{ns/1e6:.3f} ms"
|
||||||
|
if ns >= 1e3:
|
||||||
|
return f"{ns/1e3:.3f} µs"
|
||||||
|
return f"{ns:.0f} ns"
|
||||||
|
|
||||||
|
|
||||||
|
def _concurrency_section(lines: list, concurrency, dos=None) -> None:
|
||||||
|
"""Render the measured concurrency/saturation results as a self-contained
|
||||||
|
section. Deliberately additive: it reports the machine's real sustained
|
||||||
|
capacity next to the per-core estimate, without altering the DoS numbers."""
|
||||||
|
ok = [r for r in concurrency if r.levels and r.peak_ops_per_sec > 0]
|
||||||
|
lines.append("## Sustained throughput under concurrency (measured)\n")
|
||||||
|
lines.append(
|
||||||
|
"The DoS section above reports **per-core** capacity as 1/latency from a "
|
||||||
|
"single serial op. This section instead **measures** aggregate throughput "
|
||||||
|
"with *N* worker processes running at once (N stepping up to the core "
|
||||||
|
"count), so it captures real memory-bandwidth contention. It is additive — "
|
||||||
|
"the per-core figures above are unchanged.\n"
|
||||||
|
)
|
||||||
|
if not ok:
|
||||||
|
lines.append("_No usable concurrency measurements._\n")
|
||||||
|
return
|
||||||
|
|
||||||
|
devices = sorted({(r.device or "") for r in ok})
|
||||||
|
multi = len(devices) > 1
|
||||||
|
dev_hdr = "device | " if multi else ""
|
||||||
|
dev_sep = "---|" if multi else ""
|
||||||
|
nproc = max(r.nproc for r in ok)
|
||||||
|
lines.append(
|
||||||
|
f"Measured on up to **{nproc}** concurrent workers"
|
||||||
|
+ (f" across **{len(devices)}** devices" if multi else "") + ". "
|
||||||
|
"*Peak* is the best aggregate ops/s observed; *knee* is the worker count "
|
||||||
|
"where it peaks (adding workers past it stops helping). *Naïve N×* is the "
|
||||||
|
"per-core figure multiplied by the core count — what a linear extrapolation "
|
||||||
|
"would (over)predict; the *efficiency* column is measured peak ÷ naïve N×.\n"
|
||||||
|
)
|
||||||
|
lines.append(f"| {dev_hdr}impl | operation | 1 worker (per-core) | knee | measured peak | naïve N× | scaling efficiency |")
|
||||||
|
lines.append(f"|{dev_sep}---|---|---|---|---|---|---|")
|
||||||
|
for r in sorted(ok, key=lambda r: ((r.device or ""), r.operation, r.impl)):
|
||||||
|
naive = r.baseline_ops_per_sec * r.nproc
|
||||||
|
eff = (r.peak_ops_per_sec / naive) if naive > 0 else 0.0
|
||||||
|
dev_cell = f"{r.device} | " if multi else ""
|
||||||
|
lines.append(
|
||||||
|
f"| {dev_cell}{r.impl} | {r.operation} | {r.baseline_ops_per_sec:,.0f} ops/s | "
|
||||||
|
f"{r.knee_workers} workers | **{r.peak_ops_per_sec:,.0f} ops/s** | "
|
||||||
|
f"{naive:,.0f} ops/s | {eff*100:.0f}% |"
|
||||||
|
)
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
# Per-level detail, grouped by operation then (device, impl).
|
||||||
|
for op in sorted({r.operation for r in ok}):
|
||||||
|
lines.append(f"### {op}: throughput vs. concurrency\n")
|
||||||
|
lines.append(f"| {dev_hdr}impl | workers | ok | aggregate ops/s | per-worker ops/s | scaling eff. | peak RSS |")
|
||||||
|
lines.append(f"|{dev_sep}---|---|---|---|---|---|---|")
|
||||||
|
for r in sorted([x for x in ok if x.operation == op], key=lambda r: ((r.device or ""), r.impl)):
|
||||||
|
dev_cell = f"{r.device} | " if multi else ""
|
||||||
|
for lv in r.levels:
|
||||||
|
rss = f"{lv.total_peak_rss_kb/1024:.0f} MB" if lv.total_peak_rss_kb else "n/a"
|
||||||
|
lines.append(
|
||||||
|
f"| {dev_cell}{r.impl} | {lv.workers} | {lv.ok_workers} | "
|
||||||
|
f"{lv.aggregate_ops_per_sec:,.0f} | {lv.per_worker_ops_per_sec:,.0f} | "
|
||||||
|
f"{lv.scaling_efficiency*100:.0f}% | {rss} |"
|
||||||
|
)
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
|
||||||
|
def _mining_section(lines: list, mining) -> None:
|
||||||
|
"""Measured mint rate vs difficulty: 1-core and whole-machine tokens/s."""
|
||||||
|
ok = [r for r in mining if any(p.samples > 0 for p in r.points)]
|
||||||
|
lines.append("## Mining rate vs difficulty (measured)\n")
|
||||||
|
lines.append(
|
||||||
|
"How many effort-qualified tokens can be minted per second at a given "
|
||||||
|
"difficulty. The **whole-machine** rate is the reliable figure: it averages "
|
||||||
|
"one streaming search per core over independent nonce ranges. Per-core is that "
|
||||||
|
"rate divided by the core count (token-find time is heavy-tailed, so the "
|
||||||
|
"separately-sampled single-core mean is noisier and can even exceed the machine "
|
||||||
|
"rate ÷ cores at low sample counts — prefer the derived per-core). Mint rate "
|
||||||
|
"falls ~1/effort, so difficulty sets the rate directly.\n"
|
||||||
|
)
|
||||||
|
if not ok:
|
||||||
|
lines.append("_No usable mining measurements._\n")
|
||||||
|
return
|
||||||
|
for r in ok:
|
||||||
|
nproc = r.nproc
|
||||||
|
lines.append(f"**`{r.impl}`** on `{r.device}` (base `{r.challenge_base}`), "
|
||||||
|
f"whole-machine = {nproc} cores:\n")
|
||||||
|
lines.append("| difficulty (effort) | mean attempts/token | tokens/s [%d cores] | "
|
||||||
|
"tokens/s [1 core, ÷%d] |" % (nproc, nproc))
|
||||||
|
lines.append("|---|---|---|---|")
|
||||||
|
for p in sorted(r.points, key=lambda p: p.effort):
|
||||||
|
if p.samples == 0:
|
||||||
|
lines.append(f"| {p.effort} | _no sample reached target_ | — | — |")
|
||||||
|
continue
|
||||||
|
per_core = p.tokens_per_sec_machine / nproc if nproc else 0.0
|
||||||
|
lines.append(
|
||||||
|
f"| {p.effort} | {p.attempts_mean:,.0f} | "
|
||||||
|
f"**{p.tokens_per_sec_machine:,.2f}** | {per_core:,.3f} |"
|
||||||
|
)
|
||||||
|
lines.append("")
|
||||||
|
# Message sizes vs difficulty, measured from real minted tokens. The
|
||||||
|
# interesting result is constancy: cost scales with E, bytes do not.
|
||||||
|
sized = [p for p in r.points if p.solution_bytes_max > 0]
|
||||||
|
if sized:
|
||||||
|
lines.append("**Message sizes (measured from every minted token):** ")
|
||||||
|
rows_sz = []
|
||||||
|
for p in sorted(sized, key=lambda p: p.effort):
|
||||||
|
span = (f"{p.solution_bytes_min}" if p.solution_bytes_min == p.solution_bytes_max
|
||||||
|
else f"{p.solution_bytes_min}-{p.solution_bytes_max}")
|
||||||
|
rows_sz.append(f"E={p.effort}: solution {span} B + nonce {p.nonce_bytes_wire} B")
|
||||||
|
const = all(p.solution_bytes_min == sized[0].solution_bytes_max == p.solution_bytes_max
|
||||||
|
for p in sized)
|
||||||
|
lines.append("; ".join(rows_sz) + ".")
|
||||||
|
if const:
|
||||||
|
lines.append(
|
||||||
|
f"Token size is **constant in difficulty**: every token at every "
|
||||||
|
f"measured E is exactly {sized[0].solution_bytes_max} B solution + "
|
||||||
|
f"{sized[0].nonce_bytes_wire} B nonce — raising E raises solve cost, "
|
||||||
|
f"never message size.\n"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
lines.append("Token size VARIED across efforts — investigate.\n")
|
||||||
|
# Headline: 1/effort check across the measured span, on the machine rate.
|
||||||
|
# (sorted by effort so lo/hi are the true endpoints regardless of config order)
|
||||||
|
good = sorted([p for p in r.points if p.samples > 0 and p.tokens_per_sec_machine > 0],
|
||||||
|
key=lambda p: p.effort)
|
||||||
|
if len(good) >= 2:
|
||||||
|
lo, hi = good[0], good[-1]
|
||||||
|
fold_e = hi.effort / lo.effort if lo.effort else 0
|
||||||
|
fold_r = (lo.tokens_per_sec_machine / hi.tokens_per_sec_machine) if hi.tokens_per_sec_machine else 0
|
||||||
|
lines.append(
|
||||||
|
f"> Over a {fold_e:.0f}× rise in difficulty ({lo.effort}→{hi.effort}), "
|
||||||
|
f"the machine mint rate fell {fold_r:.0f}× — ~1/effort, "
|
||||||
|
f"so halving the target roughly doubles the mint rate.\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_markdown(stats, checks, plots, out_dir: Path, meta, devices, dos=None, concurrency=None, mining=None) -> None:
|
||||||
|
lines: list[str] = []
|
||||||
|
lines.append("# Equi-X Benchmark Report\n")
|
||||||
|
lines.append(f"- Generated: {meta.get('timestamp', 'n/a')}")
|
||||||
|
lines.append(f"- Config: `{meta.get('config', 'n/a')}`")
|
||||||
|
lines.append(f"- Devices (CPUs): {', '.join(devices) if devices else 'n/a'}")
|
||||||
|
for dl in devices:
|
||||||
|
ex = next((s for s in stats if s.device_label == dl), None)
|
||||||
|
if ex:
|
||||||
|
lines.append(f" - `{dl}`: {ex.device_name} ({ex.device_arch}, {ex.device_type})")
|
||||||
|
for impl, info in (meta.get("impls") or {}).items():
|
||||||
|
lines.append(
|
||||||
|
f"- `{impl}`: version {info.get('version')}, commit {info.get('commit')}, "
|
||||||
|
f"built with {info.get('compiler')}"
|
||||||
|
)
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
lines.append("## Correctness cross-check (interop gate)\n")
|
||||||
|
if checks:
|
||||||
|
overall = all(c.passed for c in checks)
|
||||||
|
lines.append(f"**Overall: {'PASS ✅' if overall else 'FAIL ❌'}**\n")
|
||||||
|
lines.append("| kind | detail | result |")
|
||||||
|
lines.append("|------|--------|--------|")
|
||||||
|
for c in checks:
|
||||||
|
lines.append(f"| {c.kind} | {c.detail} | {'PASS' if c.passed else 'FAIL'} |")
|
||||||
|
else:
|
||||||
|
lines.append("_No cross-checks run (e.g. combined report)._")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
# DoS-protection effectiveness
|
||||||
|
if dos is not None:
|
||||||
|
rows, effective, threshold = dos
|
||||||
|
lines.append("## DoS-protection effectiveness (this system)\n")
|
||||||
|
lines.append(
|
||||||
|
"Equi-X defends by making requesters *solve* (expensive) while the service "
|
||||||
|
"only *verifies* (cheap). Protection factor = measured attacker time to craft "
|
||||||
|
"one accepted token at a given effort ÷ measured verify time. "
|
||||||
|
f"Judged **effective** when ≥ {threshold:.0f}× on every measured point.\n"
|
||||||
|
)
|
||||||
|
if rows:
|
||||||
|
from .dosprotect import min_effective_effort
|
||||||
|
verdict = "EFFECTIVE ✅" if effective else "WEAK ⚠️"
|
||||||
|
mee = min_effective_effort(rows, threshold)
|
||||||
|
eff_where = ", ".join(
|
||||||
|
f"`{d}`: effort ≥ {e}" if e is not None else f"`{d}`: not reached in tested range"
|
||||||
|
for d, e in mee.items()
|
||||||
|
)
|
||||||
|
best = max(rows, key=lambda r: r.effort)
|
||||||
|
lines.append(
|
||||||
|
f"**Verdict: {verdict}** (effective from — {eff_where}).\n\n"
|
||||||
|
f"At effort {best.effort}, an attacker needs ~{best.attacker_s:.3g}s "
|
||||||
|
f"(impl `{best.attacker_impl}`) to craft one accepted request, while the "
|
||||||
|
f"defender verifies in ~{best.defender_verify_s*1e6:.2f}µs "
|
||||||
|
f"(**{best.protection_factor:,.0f}×** asymmetry; one core screens "
|
||||||
|
f"~{best.verify_per_sec:,.0f} requests/s vs the attacker's "
|
||||||
|
f"~{best.attacker_tokens_per_sec:,.2f} tokens/s).\n"
|
||||||
|
)
|
||||||
|
lines.append("| device | effort | attacker time/token | attacker impl | verify time | protection factor | verify/s | attacker tokens/s |")
|
||||||
|
lines.append("|---|---|---|---|---|---|---|---|")
|
||||||
|
for r in rows:
|
||||||
|
lines.append(
|
||||||
|
f"| {r.device} | {r.effort} | {_fmt_ns(r.attacker_s*1e9)} | {r.attacker_impl} | "
|
||||||
|
f"{_fmt_ns(r.defender_verify_s*1e9)} | {r.protection_factor:,.0f}× | "
|
||||||
|
f"{r.verify_per_sec:,.0f} | {r.attacker_tokens_per_sec:,.3f} |"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
lines.append("_Insufficient effort/verify measurements to assess._")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
# Measured concurrency / saturation -- complements (never overwrites) the
|
||||||
|
# per-core DoS estimate above.
|
||||||
|
if concurrency:
|
||||||
|
_concurrency_section(lines, concurrency, dos)
|
||||||
|
|
||||||
|
# Measured mining rate vs difficulty.
|
||||||
|
if mining:
|
||||||
|
_mining_section(lines, mining)
|
||||||
|
|
||||||
|
note = ("every plot compares C vs Rust; with multiple CPUs each plot is faceted "
|
||||||
|
"per CPU and `xdev_*` charts compare CPUs directly")
|
||||||
|
lines.append(f"## Comparison plots ({note})\n")
|
||||||
|
for p in plots:
|
||||||
|
if p.startswith("__error__:"):
|
||||||
|
lines.append(f"> ⚠️ {p[len('__error__:'):]}\n")
|
||||||
|
else:
|
||||||
|
lines.append(f"### {p.replace('_', ' ').replace('.png','').title()}\n")
|
||||||
|
lines.append(f"\n")
|
||||||
|
|
||||||
|
def table(op, cols, rowfn):
|
||||||
|
rows = [s for s in stats if s.operation == op]
|
||||||
|
if not rows:
|
||||||
|
return
|
||||||
|
lines.append(f"## {op} results\n")
|
||||||
|
lines.append("| " + " | ".join(cols) + " |")
|
||||||
|
lines.append("|" + "|".join(["---"] * len(cols)) + "|")
|
||||||
|
for s in sorted(rows, key=lambda s: (s.device_label, json.dumps(s.label), s.impl, s.runtime_requested)):
|
||||||
|
lines.append("| " + " | ".join(rowfn(s)) + " |")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
table("solve", ["device", "challenge", "impl", "runtime", "median", "p95", "solves/s", "hashes/s", "sols/solve", "RSS(KB)"],
|
||||||
|
lambda s: [s.device_label, str(s.label.get("challenge")), s.impl, str(s.runtime_effective),
|
||||||
|
_fmt_ns(s.median_ns), _fmt_ns(s.p95_ns), f"{s.solves_per_sec:.1f}",
|
||||||
|
f"{s.hashes_per_sec:,.0f}", f"{s.solutions_mean:.2f}", str(s.peak_rss_kb)])
|
||||||
|
table("verify", ["device", "challenge", "impl", "runtime", "median", "p95", "result", "RSS(KB)"],
|
||||||
|
lambda s: [s.device_label, str(s.label.get("challenge")), s.impl, str(s.runtime_effective),
|
||||||
|
_fmt_ns(s.median_ns), _fmt_ns(s.p95_ns), str(s.verify_result), str(s.peak_rss_kb)])
|
||||||
|
table("hashx_compile", ["device", "challenge", "impl", "runtime", "median compile", "median exec", "RSS(KB)"],
|
||||||
|
lambda s: [s.device_label, str(s.label.get("challenge")), s.impl, str(s.runtime_effective),
|
||||||
|
_fmt_ns(s.compile_median_ns), _fmt_ns(s.median_ns), str(s.peak_rss_kb)])
|
||||||
|
table("effort", ["device", "base", "target", "impl", "runtime", "mean attempts", "median time", "mean achieved"],
|
||||||
|
lambda s: [s.device_label, str(s.label.get("base")), str(s.label.get("target_effort")), s.impl,
|
||||||
|
str(s.runtime_effective), f"{s.attempts_mean:.1f}", _fmt_ns(s.median_ns),
|
||||||
|
f"{s.achieved_effort_mean:.0f}"])
|
||||||
|
|
||||||
|
lines.append("---")
|
||||||
|
lines.append("_See `results.csv` for the full flat dataset and `raw/results.json` for per-rep data._")
|
||||||
|
(out_dir / "report.md").write_text("\n".join(lines))
|
||||||
98
tools/benchmarks/Equi-X/harness/equix_bench/runner.py
Normal file
98
tools/benchmarks/Equi-X/harness/equix_bench/runner.py
Normal file
@ -0,0 +1,98 @@
|
|||||||
|
"""Execute a runner cell as a subprocess (one process per cell).
|
||||||
|
|
||||||
|
One process per cell keeps peak_rss_kb attributable and isolates each
|
||||||
|
(impl, operation, runtime, params) measurement from the others.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .protocol import JobSpec, Result
|
||||||
|
from .registry import Adapter
|
||||||
|
|
||||||
|
|
||||||
|
class RunnerError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# Live runner subprocesses, so a Ctrl+C can kill them promptly instead of leaving
|
||||||
|
# them running. Worker threads (concurrency/mining) never receive KeyboardInterrupt
|
||||||
|
# themselves, so the main-thread SIGINT handler (installed in cli.main) reaps them
|
||||||
|
# via terminate_all_children(); the per-call finally cleans up the common case.
|
||||||
|
_active: set[subprocess.Popen] = set()
|
||||||
|
_active_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def terminate_all_children() -> None:
|
||||||
|
"""Kill every runner subprocess still running. Safe to call from a signal
|
||||||
|
handler (main thread) and idempotent."""
|
||||||
|
with _active_lock:
|
||||||
|
procs = list(_active)
|
||||||
|
for p in procs:
|
||||||
|
try:
|
||||||
|
p.kill()
|
||||||
|
except Exception: # noqa: BLE001 - best-effort teardown, never raise
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def run(
|
||||||
|
adapter: Adapter,
|
||||||
|
spec: JobSpec,
|
||||||
|
repo_root: Path,
|
||||||
|
timeout: float = 900.0,
|
||||||
|
) -> Result:
|
||||||
|
argv = adapter.resolve(repo_root)
|
||||||
|
env = dict(os.environ)
|
||||||
|
env.update(adapter.env)
|
||||||
|
try:
|
||||||
|
proc = subprocess.Popen(
|
||||||
|
argv,
|
||||||
|
stdin=subprocess.PIPE,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
except FileNotFoundError as e:
|
||||||
|
raise RunnerError(f"{adapter.name} executable not found: {argv[0]}") from e
|
||||||
|
with _active_lock:
|
||||||
|
_active.add(proc)
|
||||||
|
try:
|
||||||
|
stdout, stderr = proc.communicate(input=spec.to_json(), timeout=timeout)
|
||||||
|
except subprocess.TimeoutExpired as e:
|
||||||
|
proc.kill()
|
||||||
|
proc.communicate() # reap the killed child so it can't linger as a zombie
|
||||||
|
raise RunnerError(f"{adapter.name} timed out after {timeout}s") from e
|
||||||
|
except BaseException:
|
||||||
|
# KeyboardInterrupt (or anything else): never leave the child running.
|
||||||
|
proc.kill()
|
||||||
|
try:
|
||||||
|
proc.communicate(timeout=5)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
with _active_lock:
|
||||||
|
_active.discard(proc)
|
||||||
|
|
||||||
|
out = (stdout or "").strip()
|
||||||
|
if not out:
|
||||||
|
raise RunnerError(
|
||||||
|
f"{adapter.name} produced no output (exit {proc.returncode}); "
|
||||||
|
f"stderr: {(stderr or '').strip()[:500]}"
|
||||||
|
)
|
||||||
|
# A runner may print progress lines to stdout before the JSON; take the last line.
|
||||||
|
line = out.splitlines()[-1]
|
||||||
|
try:
|
||||||
|
import json
|
||||||
|
|
||||||
|
result = Result.from_dict(json.loads(line))
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
raise RunnerError(
|
||||||
|
f"{adapter.name} emitted invalid result JSON: {e}; "
|
||||||
|
f"raw: {line[:500]}"
|
||||||
|
) from e
|
||||||
|
return result
|
||||||
130
tools/benchmarks/Equi-X/harness/equix_bench/stats.py
Normal file
130
tools/benchmarks/Equi-X/harness/equix_bench/stats.py
Normal file
@ -0,0 +1,130 @@
|
|||||||
|
"""Aggregate a runner Result's per-rep timings into summary statistics."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
import statistics
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
from .protocol import Result
|
||||||
|
|
||||||
|
# Equi-X solve: stage 0 evaluates the HashX function once for every index in the
|
||||||
|
# 16-bit index space (equix solver_heap.h: INDEX_SPACE = 1 << 16), which dominates
|
||||||
|
# the per-solve hashing. Used to derive an effective hash-rate from throughput.
|
||||||
|
HASHX_PER_SOLVE = 1 << 16
|
||||||
|
|
||||||
|
|
||||||
|
def _percentile(values: list[float], pct: float) -> float:
|
||||||
|
if not values:
|
||||||
|
return 0.0
|
||||||
|
s = sorted(values)
|
||||||
|
# Nearest-rank method: rank = ceil(P/100 * N).
|
||||||
|
k = max(1, min(len(s), math.ceil(pct / 100.0 * len(s))))
|
||||||
|
return s[k - 1]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class CellStats:
|
||||||
|
impl: str
|
||||||
|
operation: str
|
||||||
|
runtime_requested: str
|
||||||
|
runtime_effective: Optional[str]
|
||||||
|
label: dict[str, Any]
|
||||||
|
reps: int
|
||||||
|
ok: bool
|
||||||
|
# wall-time stats (ns)
|
||||||
|
min_ns: float
|
||||||
|
median_ns: float
|
||||||
|
mean_ns: float
|
||||||
|
stddev_ns: float
|
||||||
|
p95_ns: float
|
||||||
|
# derived / auxiliary
|
||||||
|
solutions_mean: float
|
||||||
|
compile_median_ns: float
|
||||||
|
attempts_mean: float
|
||||||
|
achieved_effort_mean: float
|
||||||
|
solves_per_sec: float
|
||||||
|
hashes_per_sec: float
|
||||||
|
peak_rss_kb: int
|
||||||
|
verify_result: Optional[str]
|
||||||
|
walls: list[float] = field(default_factory=list) # per-rep wall_ns
|
||||||
|
error: Optional[str] = None
|
||||||
|
extra: dict[str, Any] = field(default_factory=dict)
|
||||||
|
# executing hardware (see device.py)
|
||||||
|
device_label: str = "host"
|
||||||
|
device_type: str = "cpu"
|
||||||
|
device_name: str = "unknown"
|
||||||
|
device_arch: str = "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def summarize(
|
||||||
|
impl: str,
|
||||||
|
operation: str,
|
||||||
|
runtime_requested: str,
|
||||||
|
label: dict[str, Any],
|
||||||
|
result: Result,
|
||||||
|
device: Optional[dict[str, Any]] = None,
|
||||||
|
) -> CellStats:
|
||||||
|
device = device or {}
|
||||||
|
dkw = dict(
|
||||||
|
device_label=device.get("label", "host"),
|
||||||
|
device_type=device.get("type", "cpu"),
|
||||||
|
device_name=device.get("name", result.env.get("cpu", "unknown")),
|
||||||
|
device_arch=device.get("arch", result.env.get("arch", "unknown")),
|
||||||
|
)
|
||||||
|
if not result.ok or not result.runs:
|
||||||
|
return CellStats(
|
||||||
|
impl=impl,
|
||||||
|
operation=operation,
|
||||||
|
runtime_requested=runtime_requested,
|
||||||
|
runtime_effective=result.runtime_effective,
|
||||||
|
label=label,
|
||||||
|
reps=0,
|
||||||
|
ok=False,
|
||||||
|
min_ns=0, median_ns=0, mean_ns=0, stddev_ns=0, p95_ns=0,
|
||||||
|
solutions_mean=0, compile_median_ns=0, attempts_mean=0,
|
||||||
|
achieved_effort_mean=0, solves_per_sec=0, hashes_per_sec=0,
|
||||||
|
peak_rss_kb=result.peak_rss_kb,
|
||||||
|
verify_result=None,
|
||||||
|
error=result.error or "no runs",
|
||||||
|
**dkw,
|
||||||
|
)
|
||||||
|
|
||||||
|
walls = [float(r.wall_ns) for r in result.runs]
|
||||||
|
compiles = [float(r.compile_ns) for r in result.runs]
|
||||||
|
sols = [float(r.solutions) for r in result.runs]
|
||||||
|
attempts = [float(r.attempts) for r in result.runs]
|
||||||
|
efforts = [float(r.achieved_effort) for r in result.runs]
|
||||||
|
|
||||||
|
median_ns = statistics.median(walls)
|
||||||
|
mean_ns = statistics.fmean(walls)
|
||||||
|
stddev_ns = statistics.pstdev(walls) if len(walls) > 1 else 0.0
|
||||||
|
|
||||||
|
# throughput: solves/sec from median solve time; hash-rate from solves/sec.
|
||||||
|
solves_per_sec = 1e9 / median_ns if operation == "solve" and median_ns > 0 else 0.0
|
||||||
|
hashes_per_sec = solves_per_sec * HASHX_PER_SOLVE
|
||||||
|
|
||||||
|
return CellStats(
|
||||||
|
impl=impl,
|
||||||
|
operation=operation,
|
||||||
|
runtime_requested=runtime_requested,
|
||||||
|
runtime_effective=result.runtime_effective,
|
||||||
|
label=label,
|
||||||
|
reps=len(result.runs),
|
||||||
|
ok=True,
|
||||||
|
min_ns=min(walls),
|
||||||
|
median_ns=median_ns,
|
||||||
|
mean_ns=mean_ns,
|
||||||
|
stddev_ns=stddev_ns,
|
||||||
|
p95_ns=_percentile(walls, 95),
|
||||||
|
solutions_mean=statistics.fmean(sols),
|
||||||
|
compile_median_ns=statistics.median(compiles),
|
||||||
|
attempts_mean=statistics.fmean(attempts),
|
||||||
|
achieved_effort_mean=statistics.fmean(efforts),
|
||||||
|
solves_per_sec=solves_per_sec,
|
||||||
|
hashes_per_sec=hashes_per_sec,
|
||||||
|
peak_rss_kb=result.peak_rss_kb,
|
||||||
|
verify_result=result.runs[-1].verify_result,
|
||||||
|
walls=walls,
|
||||||
|
**dkw,
|
||||||
|
)
|
||||||
19
tools/benchmarks/Equi-X/harness/pyproject.toml
Normal file
19
tools/benchmarks/Equi-X/harness/pyproject.toml
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=61"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "equix-bench"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "Benchmarking framework for the Equi-X proof-of-work algorithm (C and Rust implementations)"
|
||||||
|
requires-python = ">=3.11"
|
||||||
|
dependencies = [
|
||||||
|
"matplotlib>=3.5",
|
||||||
|
"numpy>=1.21",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.scripts]
|
||||||
|
equix-bench = "equix_bench.cli:main"
|
||||||
|
|
||||||
|
[tool.setuptools]
|
||||||
|
packages = ["equix_bench"]
|
||||||
143
tools/benchmarks/Equi-X/harness/tests/test_combine.py
Normal file
143
tools/benchmarks/Equi-X/harness/tests/test_combine.py
Normal file
@ -0,0 +1,143 @@
|
|||||||
|
"""`combine` merges runs from many devices into ONE faceted report — including
|
||||||
|
the concurrency and mining sections/figures, which the per-run CSVs carry and
|
||||||
|
which must survive the CSV round-trip. Auto-discovery walks a tree and identifies
|
||||||
|
runs by their raw record file, taking device identity from the records (not paths)
|
||||||
|
so an arbitrary collected layout still combines correctly."""
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from equix_bench import report as reportmod
|
||||||
|
from equix_bench.cli import _discover_runs, cmd_combine
|
||||||
|
from equix_bench.concurrency import ConcResult, LevelStat
|
||||||
|
from equix_bench.concurrency import read_csv as conc_read
|
||||||
|
from equix_bench.concurrency import write_csv as conc_write
|
||||||
|
from equix_bench.mining import MiningPoint, MiningResult
|
||||||
|
from equix_bench.mining import read_csv as mining_read
|
||||||
|
from equix_bench.mining import write_csv as mining_write
|
||||||
|
from equix_bench.protocol import Result, Run
|
||||||
|
from equix_bench.stats import summarize
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- CSV round-trips
|
||||||
|
|
||||||
|
|
||||||
|
def _conc(device, impl):
|
||||||
|
lv = [LevelStat(1, 1, 0.01, 100.0, 100.0, 1.0, 1000),
|
||||||
|
LevelStat(2, 2, 0.011, 182.0, 91.0, 0.91, 2000)]
|
||||||
|
return ConcResult(device=device, impl=impl, operation="solve", nproc=2, reps=40,
|
||||||
|
challenge="deadbeef", baseline_ops_per_sec=100.0,
|
||||||
|
peak_ops_per_sec=182.0, knee_workers=2, levels=lv)
|
||||||
|
|
||||||
|
|
||||||
|
def _mining(device, impl):
|
||||||
|
pts = [MiningPoint(100, 10, 0.165, 0.11, 0.14, 34.0, 350.0, 6.06, 2, 2, 12.2, 1.0, 0, 16, 16, 8),
|
||||||
|
MiningPoint(1000, 10, 3.6, 2.5, 3.4, 500.0, 3000.0, 0.28, 2, 2, 0.55, 0.98, 1, 16, 16, 8)]
|
||||||
|
return MiningResult(device=device, impl=impl, challenge_base="abcd", nproc=2, points=pts)
|
||||||
|
|
||||||
|
|
||||||
|
def test_concurrency_csv_roundtrip(tmp_path):
|
||||||
|
orig = [_conc("cpuA", "equix-c"), _conc("cpuA", "equix-rust")]
|
||||||
|
conc_write(orig, tmp_path / "c.csv")
|
||||||
|
back = conc_read(tmp_path / "c.csv")
|
||||||
|
assert {r.impl for r in back} == {"equix-c", "equix-rust"}
|
||||||
|
r = next(x for x in back if x.impl == "equix-c")
|
||||||
|
# Summary fields are re-derived from the levels, matching measure().
|
||||||
|
assert r.nproc == 2 and r.knee_workers == 2
|
||||||
|
assert r.baseline_ops_per_sec == 100.0 and r.peak_ops_per_sec == 182.0
|
||||||
|
assert [lv.workers for lv in r.levels] == [1, 2]
|
||||||
|
|
||||||
|
|
||||||
|
def test_mining_csv_roundtrip(tmp_path):
|
||||||
|
mining_write([_mining("cpuA", "equix-rust")], tmp_path / "m.csv")
|
||||||
|
back = mining_read(tmp_path / "m.csv")
|
||||||
|
assert len(back) == 1
|
||||||
|
r = back[0]
|
||||||
|
assert r.nproc == 2 and r.challenge_base == "abcd"
|
||||||
|
assert [p.effort for p in r.points] == [100, 1000] # sorted on read
|
||||||
|
assert r.points[0].solution_bytes_max == 16
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ discovery + combine
|
||||||
|
|
||||||
|
|
||||||
|
def _raw_record(device_label, impl, op, runtime, median_ns):
|
||||||
|
"""A minimal enriched raw record in the on-wire schema cmd_run persists to
|
||||||
|
raw/results.json (schema_version + nested impl block + _* enrichment)."""
|
||||||
|
return {
|
||||||
|
"schema_version": 1, "ok": True,
|
||||||
|
"impl": {"name": impl, "version": "1", "commit": "c"},
|
||||||
|
"operation": op, "runtime_requested": runtime, "runtime_effective": "compiled",
|
||||||
|
"env": {"cpu": device_label, "arch": "arm"},
|
||||||
|
"runs": [{"index": 0, "wall_ns": median_ns, "solutions": 1, "compile_ns": 0,
|
||||||
|
"attempts": 0, "achieved_effort": 0, "verify_result": "OK"}],
|
||||||
|
"solutions_hex": None, "peak_rss_kb": 1000, "error": None,
|
||||||
|
"_label": {"challenge": "deadbeef"}, "_impl": impl, "_group": op,
|
||||||
|
"_device": {"label": device_label, "type": "cpu", "name": device_label, "arch": "arm"},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _write_run(dirpath: Path, device_label, ts, with_extras=False):
|
||||||
|
dirpath.mkdir(parents=True, exist_ok=True)
|
||||||
|
(dirpath / "raw").mkdir(exist_ok=True)
|
||||||
|
recs = [_raw_record(device_label, "equix-c", "solve", "try-compile", 39_000_000),
|
||||||
|
_raw_record(device_label, "equix-rust", "solve", "try-compile", 4_500_000),
|
||||||
|
# two runtimes of the same op must NOT collide during de-dup
|
||||||
|
_raw_record(device_label, "equix-c", "solve", "interpret", 42_000_000),
|
||||||
|
_raw_record(device_label, "equix-rust", "solve", "interpret", 12_000_000)]
|
||||||
|
(dirpath / "raw" / "results.json").write_text(json.dumps(recs))
|
||||||
|
(dirpath / "run_meta.json").write_text(json.dumps({"timestamp": ts, "devices": [device_label]}))
|
||||||
|
if with_extras:
|
||||||
|
conc_write([_conc(device_label, "equix-c"), _conc(device_label, "equix-rust")],
|
||||||
|
dirpath / "concurrency.csv")
|
||||||
|
mining_write([_mining(device_label, "equix-rust")], dirpath / "mining.csv")
|
||||||
|
|
||||||
|
|
||||||
|
def test_discover_runs_is_layout_agnostic(tmp_path):
|
||||||
|
# Two different nesting depths under one tree; discovery finds both.
|
||||||
|
_write_run(tmp_path / "deviceA" / "main", "cpu-a", "2026-07-27T10:00:00+00:00")
|
||||||
|
_write_run(tmp_path / "host-b" / "results" / "main", "cpu-b", "2026-07-27T11:00:00+00:00")
|
||||||
|
found = _discover_runs(tmp_path)
|
||||||
|
assert len(found) == 2
|
||||||
|
assert {p.name for p in found} == {"main"}
|
||||||
|
|
||||||
|
|
||||||
|
class _Args:
|
||||||
|
def __init__(self, **kw):
|
||||||
|
self.inputs = None
|
||||||
|
self.root = None
|
||||||
|
self.out = None
|
||||||
|
self.__dict__.update(kw)
|
||||||
|
|
||||||
|
|
||||||
|
def test_combine_root_produces_faceted_multidevice_report(tmp_path):
|
||||||
|
_write_run(tmp_path / "A" / "main", "cpu-a", "2026-07-27T10:00:00+00:00", with_extras=True)
|
||||||
|
_write_run(tmp_path / "B" / "main", "cpu-b", "2026-07-27T11:00:00+00:00", with_extras=True)
|
||||||
|
out = tmp_path / "combined"
|
||||||
|
rc = cmd_combine(_Args(root=str(tmp_path), out=str(out)))
|
||||||
|
assert rc == 0
|
||||||
|
md = (out / "report.md").read_text()
|
||||||
|
# Both devices named as CPUs in the header.
|
||||||
|
assert "cpu-a" in md and "cpu-b" in md
|
||||||
|
# Concurrency + mining sections survived the combine (they were dropped before).
|
||||||
|
assert "Sustained throughput under concurrency" in md
|
||||||
|
assert "Mining rate vs difficulty" in md
|
||||||
|
# Multi-device concurrency table gains a device column.
|
||||||
|
assert "| device | impl | operation |" in md
|
||||||
|
# Cross-device (xdev_*) headline figures are emitted with >1 device.
|
||||||
|
assert any((out / "plots").glob("xdev_*.png"))
|
||||||
|
# The faceted concurrency/mining figures exist.
|
||||||
|
assert (out / "plots" / "concurrency_solve.png").exists()
|
||||||
|
assert (out / "plots" / "mining_rate.png").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_combine_dedups_reruns_keeping_newest(tmp_path):
|
||||||
|
# Same device measured twice (a re-run): the newer run must replace, not
|
||||||
|
# double the cell count.
|
||||||
|
_write_run(tmp_path / "old" / "main", "cpu-a", "2026-07-27T09:00:00+00:00")
|
||||||
|
_write_run(tmp_path / "new" / "main", "cpu-a", "2026-07-27T15:00:00+00:00")
|
||||||
|
out = tmp_path / "combined"
|
||||||
|
rc = cmd_combine(_Args(root=str(tmp_path), out=str(out)))
|
||||||
|
assert rc == 0
|
||||||
|
rows = (out / "results.csv").read_text().strip().splitlines()[1:]
|
||||||
|
# 2 impls x 2 runtimes x 1 device = 4 cells, NOT 8 (dedup collapsed the re-run).
|
||||||
|
assert len(rows) == 4
|
||||||
93
tools/benchmarks/Equi-X/harness/tests/test_concurrency.py
Normal file
93
tools/benchmarks/Equi-X/harness/tests/test_concurrency.py
Normal file
@ -0,0 +1,93 @@
|
|||||||
|
"""The concurrency benchmark must MEASURE aggregate parallel throughput and the
|
||||||
|
saturation knee -- distinct from dosprotect.py's per-core 1/latency estimate."""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import equix_bench.concurrency as conc
|
||||||
|
from equix_bench.concurrency import _ladder, _worker_median_s, measure
|
||||||
|
from equix_bench.protocol import Result, Run
|
||||||
|
from equix_bench.registry import Adapter
|
||||||
|
|
||||||
|
|
||||||
|
def _result(wall_ns, ok=True, rss=1000):
|
||||||
|
runs = [
|
||||||
|
Run(index=i, wall_ns=w, solutions=1, compile_ns=0, attempts=0,
|
||||||
|
achieved_effort=0, verify_result="OK")
|
||||||
|
for i, w in enumerate(wall_ns)
|
||||||
|
]
|
||||||
|
return Result(
|
||||||
|
ok=ok, impl_name="equix-c", impl_version="1", impl_commit="c",
|
||||||
|
operation="solve", runtime_requested="try-compile", runtime_effective="compiled",
|
||||||
|
env={}, runs=runs, solutions_hex=["ab"], peak_rss_kb=rss, error=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _adapter():
|
||||||
|
return Adapter(name="equix-c", exec=["/bin/true"], protocol_version=1,
|
||||||
|
capabilities=[], runtimes=[], env={})
|
||||||
|
|
||||||
|
|
||||||
|
def test_ladder_powers_of_two_plus_top():
|
||||||
|
assert _ladder(1) == [1]
|
||||||
|
assert _ladder(4) == [1, 2, 4]
|
||||||
|
assert _ladder(8) == [1, 2, 4, 8]
|
||||||
|
assert _ladder(6) == [1, 2, 4, 6] # non-power-of-two core count still included
|
||||||
|
assert _ladder(10, [1, 3, 10]) == [1, 3, 10] # explicit levels honored
|
||||||
|
assert _ladder(10, [3, 10]) == [1, 3, 10] # level 1 forced in (baseline anchor)
|
||||||
|
import pytest
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
_ladder(14, [16, 32]) # all out of range -> loud error, not empty
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_median_seconds():
|
||||||
|
assert _worker_median_s(_result([10, 20, 30])) == 20 / 1e9
|
||||||
|
assert _worker_median_s(_result([], ok=True)) is None
|
||||||
|
assert _worker_median_s(_result([10], ok=False)) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_measure_aggregates_throughput_and_scaling(monkeypatch):
|
||||||
|
# No contention in the fake: each worker sustains 100 ops/s (10 ms/op), so
|
||||||
|
# aggregate must scale linearly and peak at the max worker count.
|
||||||
|
monkeypatch.setattr(conc, "run",
|
||||||
|
lambda a, s, r, timeout=900.0: _result([10_000_000] * 3))
|
||||||
|
res = measure(_adapter(), "solve", "deadbeef", None, max_workers=4, reps=3,
|
||||||
|
warmup=1, repo_root=Path("."), device_label="cpuX", timeout=10,
|
||||||
|
levels=[1, 2, 4])
|
||||||
|
|
||||||
|
assert res.baseline_ops_per_sec == 100.0
|
||||||
|
agg = {lv.workers: lv.aggregate_ops_per_sec for lv in res.levels}
|
||||||
|
assert agg == {1: 100.0, 2: 200.0, 4: 400.0}
|
||||||
|
assert res.peak_ops_per_sec == 400.0 and res.knee_workers == 4
|
||||||
|
# perfect linear scaling -> efficiency 1.0 at every level
|
||||||
|
assert all(abs(lv.scaling_efficiency - 1.0) < 1e-9 for lv in res.levels)
|
||||||
|
# per-level RSS is summed across the concurrent workers
|
||||||
|
assert {lv.workers: lv.total_peak_rss_kb for lv in res.levels} == {1: 1000, 2: 2000, 4: 4000}
|
||||||
|
|
||||||
|
|
||||||
|
def test_baseline_falls_back_per_worker_when_level1_fails(monkeypatch):
|
||||||
|
# Level 1 fails, level 2 succeeds at 100 ops/s per worker: the baseline must
|
||||||
|
# anchor to PER-WORKER throughput (100), not the 2-worker aggregate (200) --
|
||||||
|
# otherwise every efficiency/naive-Nx figure is off by the level's width.
|
||||||
|
calls = {"n": 0}
|
||||||
|
def fake_run(a, s, r, timeout=900.0):
|
||||||
|
calls["n"] += 1
|
||||||
|
if calls["n"] <= 2: # calibration + the level-1 worker fail
|
||||||
|
return _result([], ok=False)
|
||||||
|
return _result([10_000_000] * 3)
|
||||||
|
monkeypatch.setattr(conc, "run", fake_run)
|
||||||
|
res = measure(_adapter(), "solve", "deadbeef", None, max_workers=2, reps=3,
|
||||||
|
warmup=1, repo_root=Path("."), device_label="cpuX", timeout=10,
|
||||||
|
levels=[1, 2], min_window_s=0)
|
||||||
|
assert res.baseline_ops_per_sec == 100.0 # per-worker, not aggregate
|
||||||
|
lv2 = [lv for lv in res.levels if lv.workers == 2][0]
|
||||||
|
assert abs(lv2.scaling_efficiency - 1.0) < 1e-9 # 200/(100*2), not 200/(200*2)
|
||||||
|
|
||||||
|
|
||||||
|
def test_measure_handles_failed_workers(monkeypatch):
|
||||||
|
monkeypatch.setattr(conc, "run",
|
||||||
|
lambda a, s, r, timeout=900.0: _result([], ok=False))
|
||||||
|
res = measure(_adapter(), "solve", "deadbeef", None, max_workers=2, reps=3,
|
||||||
|
warmup=1, repo_root=Path("."), device_label="cpuX", timeout=10,
|
||||||
|
levels=[1, 2])
|
||||||
|
assert res.peak_ops_per_sec == 0.0
|
||||||
|
assert res.error == "no worker produced a usable measurement"
|
||||||
|
assert all(lv.aggregate_ops_per_sec == 0.0 for lv in res.levels)
|
||||||
104
tools/benchmarks/Equi-X/harness/tests/test_device.py
Normal file
104
tools/benchmarks/Equi-X/harness/tests/test_device.py
Normal file
@ -0,0 +1,104 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from equix_bench import report as reportmod
|
||||||
|
from equix_bench.cli import _load_cells_from_raw
|
||||||
|
from equix_bench.device import device_from_env, parse_cpu_model, slug
|
||||||
|
from equix_bench.protocol import Result
|
||||||
|
from equix_bench.stats import summarize
|
||||||
|
|
||||||
|
|
||||||
|
def test_slug():
|
||||||
|
assert slug("Intel(R) Xeon(R) @ 2.10GHz") == "intel-r-xeon-r-2-10ghz"
|
||||||
|
assert slug("") == "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
X86_CPUINFO = """processor\t: 0
|
||||||
|
vendor_id\t: GenuineIntel
|
||||||
|
model name\t: Intel(R) Xeon(R) Processor @ 2.10GHz
|
||||||
|
cpu MHz\t\t: 2100.000
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Raspberry Pi 5 (aarch64) has NO "model name"; the board is under "Model".
|
||||||
|
PI5_CPUINFO = """processor\t: 0
|
||||||
|
BogoMIPS\t: 108.00
|
||||||
|
Features\t: fp asimd evtstrm aes pmull sha1 sha2 crc32
|
||||||
|
CPU implementer\t: 0x41
|
||||||
|
CPU part\t: 0xd0b
|
||||||
|
processor\t: 1
|
||||||
|
CPU part\t: 0xd0b
|
||||||
|
Revision\t: d04170
|
||||||
|
Model\t\t: Raspberry Pi 5 Model B Rev 1.0
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_cpu_model_x86_and_arm():
|
||||||
|
assert parse_cpu_model(X86_CPUINFO) == "Intel(R) Xeon(R) Processor @ 2.10GHz"
|
||||||
|
# On ARM/Pi we fall back to the board "Model" line instead of "unknown".
|
||||||
|
assert parse_cpu_model(PI5_CPUINFO) == "Raspberry Pi 5 Model B Rev 1.0"
|
||||||
|
assert parse_cpu_model("no fields here\n") == "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def test_pi_auto_label_is_meaningful():
|
||||||
|
env = {"cpu": "Raspberry Pi 5 Model B Rev 1.0", "arch": "aarch64",
|
||||||
|
"device": "cpu", "os_version": "6.6.31-rpi"}
|
||||||
|
d = device_from_env(env)
|
||||||
|
assert d["arch"] == "aarch64"
|
||||||
|
assert d["label"] == "raspberry-pi-5-model-b-rev-1-0-6-6-31-rpi"
|
||||||
|
|
||||||
|
|
||||||
|
def test_device_from_env_precedence():
|
||||||
|
env = {"cpu": "My CPU", "arch": "x86_64", "device": "cpu", "os_version": "6.1.0"}
|
||||||
|
d = device_from_env(env)
|
||||||
|
assert d["type"] == "cpu" and d["name"] == "My CPU" and d["arch"] == "x86_64"
|
||||||
|
# OS version is folded into the auto label
|
||||||
|
assert d["os_version"] == "6.1.0"
|
||||||
|
assert d["label"] == "my-cpu-6-1-0"
|
||||||
|
# explicit label override wins
|
||||||
|
assert device_from_env(env, override_label="box1")["label"] == "box1"
|
||||||
|
# a GPU runner is honored
|
||||||
|
g = device_from_env({"cpu": "Some GPU", "arch": "sm_90", "device": "gpu", "os_version": "1"})
|
||||||
|
assert g["type"] == "gpu"
|
||||||
|
|
||||||
|
|
||||||
|
def _raw(impl, device_label, wall):
|
||||||
|
return {
|
||||||
|
"schema_version": 1, "ok": True,
|
||||||
|
"impl": {"name": impl, "version": "1", "commit": "c", "runtime_effective": "compiled"},
|
||||||
|
"operation": "solve", "runtime_requested": "try-compile",
|
||||||
|
"runtime_effective": "compiled",
|
||||||
|
"env": {"cpu": device_label, "arch": "x86_64", "device": "cpu"},
|
||||||
|
"runs": [{"index": 0, "wall_ns": wall, "solutions": 4, "compile_ns": 0,
|
||||||
|
"attempts": 0, "achieved_effort": 0, "verify_result": None}],
|
||||||
|
"solutions_hex": ["00" * 16], "peak_rss_kb": 100, "error": None,
|
||||||
|
"_label": {"challenge": "deadbeef"},
|
||||||
|
"_device": {"type": "cpu", "name": device_label, "arch": "x86_64", "label": device_label},
|
||||||
|
"_impl": impl, "_group": "solve",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_summarize_stamps_device():
|
||||||
|
r = Result.from_dict(_raw("equix-c", "cpuA", 10))
|
||||||
|
st = summarize("equix-c", "solve", "try-compile", {"challenge": "deadbeef"}, r,
|
||||||
|
{"type": "cpu", "name": "cpuA", "arch": "x86_64", "label": "cpuA"})
|
||||||
|
assert st.device_label == "cpuA" and st.device_arch == "x86_64"
|
||||||
|
|
||||||
|
|
||||||
|
def test_combine_loader_reconstructs_cells():
|
||||||
|
raws = [_raw("equix-c", "cpuA", 10), _raw("equix-rust", "cpuA", 12),
|
||||||
|
_raw("equix-c", "cpuB", 20), _raw("equix-rust", "cpuB", 22)]
|
||||||
|
cells = _load_cells_from_raw(raws)
|
||||||
|
assert len(cells) == 4
|
||||||
|
assert {c.device_label for c in cells} == {"cpuA", "cpuB"}
|
||||||
|
assert {c.impl for c in cells} == {"equix-c", "equix-rust"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_emits_cross_device_plots(tmp_path: Path):
|
||||||
|
raws = [_raw("equix-c", "cpuA", 10), _raw("equix-rust", "cpuA", 12),
|
||||||
|
_raw("equix-c", "cpuB", 20), _raw("equix-rust", "cpuB", 22)]
|
||||||
|
stats = _load_cells_from_raw(raws)
|
||||||
|
reportmod.generate(stats, [], raws, tmp_path,
|
||||||
|
{"timestamp": "t", "config": "test", "devices": ["cpuA", "cpuB"]})
|
||||||
|
# faceted solve plot + cross-device chart both present
|
||||||
|
assert (tmp_path / "plots" / "solve_time_by_runtime.png").exists()
|
||||||
|
assert (tmp_path / "plots" / "xdev_throughput.png").exists()
|
||||||
|
assert (tmp_path / "report.md").exists()
|
||||||
@ -0,0 +1,96 @@
|
|||||||
|
"""The difficulty controllers must converge: drive E so the observed signal
|
||||||
|
(mint rate for Design A, pressure for Design B) reaches its target, using only
|
||||||
|
the measured 1/E mint-rate curve."""
|
||||||
|
import math
|
||||||
|
|
||||||
|
from equix_bench.difficulty_control import (
|
||||||
|
LoadController, MEASURED_MINT, MintRateController, equilibrium_E,
|
||||||
|
mint_rate_per_machine, simulate_dos, simulate_mining,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_mint_rate_matches_measured_and_is_monotonic():
|
||||||
|
for e, r in MEASURED_MINT:
|
||||||
|
assert abs(mint_rate_per_machine(e) - r) < 1e-6
|
||||||
|
rates = [mint_rate_per_machine(e) for e in (50, 100, 300, 1000, 3000, 10000, 100000)]
|
||||||
|
assert all(a > b for a, b in zip(rates, rates[1:])) # strictly decreasing
|
||||||
|
# 1/E extrapolation only OUTSIDE the measured range: doubling E halves the rate.
|
||||||
|
top_e, top_r = max(MEASURED_MINT)
|
||||||
|
assert abs(mint_rate_per_machine(2 * top_e) - top_r / 2) < 1e-6
|
||||||
|
assert abs(mint_rate_per_machine(50) - MEASURED_MINT[0][1] * MEASURED_MINT[0][0] / 50) < 1e-6
|
||||||
|
|
||||||
|
|
||||||
|
def test_mint_rate_controller_converges_to_target():
|
||||||
|
# 10 machines, target 3 tok/s. Controller should settle E so 10·M(E) ≈ 3.
|
||||||
|
tr = simulate_mining(lambda t: 10.0, target_rate=3.0, steps=60)
|
||||||
|
assert abs(tr.signal[-1] - 3.0) / 3.0 < 0.05 # within 5%
|
||||||
|
assert 10 * mint_rate_per_machine(tr.E[-1]) == tr.signal[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_mint_rate_controller_tracks_capacity_step():
|
||||||
|
# Capacity doubles at t=30; controller must roughly double E to hold the rate.
|
||||||
|
tr = simulate_mining(lambda t: 5.0 if t < 30 else 10.0, target_rate=2.0, steps=80)
|
||||||
|
assert abs(tr.signal[-1] - 2.0) / 2.0 < 0.05
|
||||||
|
E_before = tr.E[29]
|
||||||
|
assert tr.E[-1] > 1.6 * E_before # ~2× more difficulty
|
||||||
|
|
||||||
|
|
||||||
|
def test_equilibrium_E_inverts_the_curve():
|
||||||
|
# At the solved E, `machines` x M(E) must equal the target rate.
|
||||||
|
for machines, target in [(6.0, 2.0), (1.0, 0.5), (20.0, 2.0)]:
|
||||||
|
E = equilibrium_E(machines, target)
|
||||||
|
assert abs(machines * mint_rate_per_machine(E) - target) / target < 0.001
|
||||||
|
|
||||||
|
|
||||||
|
def test_production_tuning_is_stable_under_noise_and_reproducible():
|
||||||
|
# Gentle gains + seeded E + measurement noise: settled rate stays near target,
|
||||||
|
# and the same seed gives the same trace.
|
||||||
|
def cap(t):
|
||||||
|
return 6.0
|
||||||
|
mk = lambda: MintRateController(target_rate=2.0, E=equilibrium_E(6.0, 2.0), max_factor=2.0, ewma=0.15)
|
||||||
|
a = simulate_mining(cap, 2.0, steps=120, ctrl=mk(), noise=0.08, seed=1)
|
||||||
|
b = simulate_mining(cap, 2.0, steps=120, ctrl=mk(), noise=0.08, seed=1)
|
||||||
|
assert a.E == b.E # reproducible for a fixed seed
|
||||||
|
tail = a.signal[80:]
|
||||||
|
mean = sum(tail) / len(tail)
|
||||||
|
assert abs(mean - 2.0) / 2.0 < 0.05 # settled within 5% of target
|
||||||
|
# Seeded near equilibrium -> no big cold-start excursion in E.
|
||||||
|
assert max(a.E) / min(a.E) < 3.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_controller_direction_and_deadband():
|
||||||
|
c = LoadController(p_set=0.8, E=1000.0, e_min=100.0)
|
||||||
|
assert c.update(1.5) > 1000.0 # overloaded -> raise E
|
||||||
|
c2 = LoadController(p_set=0.8, E=1000.0, e_min=100.0)
|
||||||
|
assert c2.update(0.2) < 1000.0 # idle -> lower E
|
||||||
|
c3 = LoadController(p_set=0.8, E=1000.0, deadband=0.05)
|
||||||
|
assert c3.update(0.81) == 1000.0 # within deadband -> hold
|
||||||
|
|
||||||
|
|
||||||
|
def test_adaptive_attacker_bounds_mean_load_but_sawtooths():
|
||||||
|
# An attacker that only attacks while E is below a give-up point must not be
|
||||||
|
# able to hold the node saturated continuously (mean load bounded), yet the
|
||||||
|
# decay controller visibly oscillates (E spans a wide range).
|
||||||
|
def adaptive(t, E):
|
||||||
|
return 6.0 if (10 <= t < 80 and E < 800.0) else 0.0
|
||||||
|
tr = simulate_dos(lambda t: 8.0, lambda t: 0.0, service_capacity=40.0,
|
||||||
|
steps=100, adaptive_attackers=adaptive)
|
||||||
|
window = slice(10, 80)
|
||||||
|
util = tr.extra["util"][window]
|
||||||
|
assert sum(util) / len(util) < 0.9 # mean load bounded, not pinned at 1
|
||||||
|
assert max(tr.E[window]) / min(tr.E[window]) > 3 # E sawtooths, not settled
|
||||||
|
# Attacker gets a nonzero share (it isn't shut out) but not a constant one.
|
||||||
|
on = [a for a in tr.extra["attacker_rate"][window] if a > 0]
|
||||||
|
assert 0 < len(on) < 70
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_controller_throttles_attack():
|
||||||
|
# Flood during [20,50); node capacity 40 req/s, honest 8 req/s.
|
||||||
|
tr = simulate_dos(lambda t: 8.0, lambda t: 6.0 if 20 <= t < 50 else 0.0,
|
||||||
|
service_capacity=40.0, steps=90)
|
||||||
|
# After the controller reacts, sustained utilization must fall back to ~p_set.
|
||||||
|
late_attack_util = tr.extra["util"][45]
|
||||||
|
assert late_attack_util < 0.95 # not saturated anymore
|
||||||
|
# E rises during the attack and decays again afterwards.
|
||||||
|
assert max(tr.E[20:50]) > 3 * tr.E[19]
|
||||||
|
assert tr.E[-1] < max(tr.E[20:50])
|
||||||
54
tools/benchmarks/Equi-X/harness/tests/test_dosprotect.py
Normal file
54
tools/benchmarks/Equi-X/harness/tests/test_dosprotect.py
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
"""The DoS-protection evaluation must correctly quantify the attacker/defender
|
||||||
|
asymmetry that makes Equi-X an effective client puzzle."""
|
||||||
|
from equix_bench.dosprotect import DEFAULT_THRESHOLD, assess, min_verify_seconds
|
||||||
|
from equix_bench.stats import CellStats
|
||||||
|
|
||||||
|
|
||||||
|
def _cell(op, impl, median_ns, label=None, device="cpuX"):
|
||||||
|
return CellStats(
|
||||||
|
impl=impl, operation=op, runtime_requested="try-compile",
|
||||||
|
runtime_effective="compiled", label=label or {},
|
||||||
|
reps=3, ok=True, min_ns=median_ns, median_ns=median_ns, mean_ns=median_ns,
|
||||||
|
stddev_ns=0, p95_ns=median_ns, solutions_mean=1, compile_median_ns=0,
|
||||||
|
attempts_mean=0, achieved_effort_mean=0, solves_per_sec=0, hashes_per_sec=0,
|
||||||
|
peak_rss_kb=1000, verify_result="OK", device_label=device,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_min_verify_seconds():
|
||||||
|
stats = [
|
||||||
|
_cell("verify", "equix-c", 40_000), # 40 µs
|
||||||
|
_cell("verify", "equix-rust", 44_000), # 44 µs
|
||||||
|
]
|
||||||
|
assert abs(min_verify_seconds(stats, "cpuX") - 40e-6) < 1e-12
|
||||||
|
assert min_verify_seconds(stats, "other") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_effective_protection_on_realistic_numbers():
|
||||||
|
# verify ~40 µs; crafting a token at effort 1000 ~ 4.4 s -> huge asymmetry.
|
||||||
|
stats = [
|
||||||
|
_cell("verify", "equix-c", 40_000),
|
||||||
|
_cell("effort", "equix-c", 4_400_000_000, {"target_effort": 1000}),
|
||||||
|
_cell("effort", "equix-rust", 4_600_000_000, {"target_effort": 1000}),
|
||||||
|
]
|
||||||
|
rows, effective, threshold = assess(stats)
|
||||||
|
assert threshold == DEFAULT_THRESHOLD
|
||||||
|
assert len(rows) == 1
|
||||||
|
r = rows[0]
|
||||||
|
# attacker uses the FASTER impl (equix-c here, 4.4s)
|
||||||
|
assert r.attacker_impl == "equix-c"
|
||||||
|
assert abs(r.protection_factor - (4.4 / 40e-6)) < 1.0 # ~110,000x
|
||||||
|
assert effective is True
|
||||||
|
assert r.verify_per_sec > 20_000 # defender screens >20k/s
|
||||||
|
assert r.attacker_tokens_per_sec < 1 # attacker <1 token/s
|
||||||
|
|
||||||
|
|
||||||
|
def test_weak_protection_flagged():
|
||||||
|
# Pathological: verify as expensive as crafting -> not effective.
|
||||||
|
stats = [
|
||||||
|
_cell("verify", "equix-c", 1_000_000_000), # 1 s verify
|
||||||
|
_cell("effort", "equix-c", 2_000_000_000, {"target_effort": 10}), # 2 s craft
|
||||||
|
]
|
||||||
|
rows, effective, _ = assess(stats)
|
||||||
|
assert rows and effective is False
|
||||||
|
assert rows[0].protection_factor < 10
|
||||||
75
tools/benchmarks/Equi-X/harness/tests/test_mining.py
Normal file
75
tools/benchmarks/Equi-X/harness/tests/test_mining.py
Normal file
@ -0,0 +1,75 @@
|
|||||||
|
"""The mining benchmark must turn effort searches into a measured mint rate:
|
||||||
|
per-core and whole-machine tokens/s, with failed searches excluded."""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import equix_bench.mining as mining
|
||||||
|
from equix_bench.mining import measure_point
|
||||||
|
from equix_bench.protocol import Result, Run
|
||||||
|
from equix_bench.registry import Adapter
|
||||||
|
|
||||||
|
|
||||||
|
def _result(wall_ns, attempts, achieved, ok=True, target=1000):
|
||||||
|
runs = [Run(index=0, wall_ns=wall_ns, solutions=1 if achieved else 0, compile_ns=0,
|
||||||
|
attempts=attempts, achieved_effort=achieved, verify_result=None)]
|
||||||
|
# Winning-token wire bytes are emitted only when the target was reached:
|
||||||
|
# a 16-byte solution and an 8-byte nonce, as the real runners produce.
|
||||||
|
minted = achieved >= target
|
||||||
|
return Result(ok=ok, impl_name="equix-rust", impl_version="1", impl_commit="c",
|
||||||
|
operation="effort", runtime_requested="try-compile", runtime_effective="compiled",
|
||||||
|
env={}, runs=runs, solutions_hex=["ab" * 16] if minted else None,
|
||||||
|
peak_rss_kb=4000, error=None,
|
||||||
|
winning_nonce_hex="00" * 8 if minted else None)
|
||||||
|
|
||||||
|
|
||||||
|
def _adapter():
|
||||||
|
return Adapter(name="equix-rust", exec=["/bin/true"], protocol_version=1,
|
||||||
|
capabilities=["effort"], runtimes=[], env={})
|
||||||
|
|
||||||
|
|
||||||
|
def test_measure_point_rates_and_scaling(monkeypatch):
|
||||||
|
# Every search: 0.5 s, 100 attempts, reaches the target -> 2 tokens/s/core.
|
||||||
|
monkeypatch.setattr(mining, "run",
|
||||||
|
lambda a, s, r, timeout=900.0: _result(500_000_000, 100, 1000))
|
||||||
|
p = measure_point(_adapter(), "abcd", effort=1000, samples=5, workers=4,
|
||||||
|
tokens_per_worker=3, nonce_bytes=8, max_attempts=1_000_000,
|
||||||
|
repo_root=Path("."), timeout=10)
|
||||||
|
assert p.samples == 5
|
||||||
|
assert abs(p.tokens_per_sec_1core - 2.0) < 1e-9 # 5 tokens / 2.5 s
|
||||||
|
assert abs(p.token_s_mean - 0.5) < 1e-9
|
||||||
|
assert p.attempts_mean == 100
|
||||||
|
# 4 workers, each streams 3 tokens (3/1.5s = 2/s), no contention -> 8 tokens/s.
|
||||||
|
assert p.ok_workers == 4
|
||||||
|
assert abs(p.tokens_per_sec_machine - 8.0) < 1e-9
|
||||||
|
assert abs(p.scaling_efficiency - 1.0) < 1e-9
|
||||||
|
# Message sizes measured from every minted token: constant 16 B + 8 B.
|
||||||
|
assert (p.solution_bytes_min, p.solution_bytes_max, p.nonce_bytes_wire) == (16, 16, 8)
|
||||||
|
|
||||||
|
|
||||||
|
def test_measure_point_excludes_failed_searches(monkeypatch):
|
||||||
|
# achieved (500) never reaches target (1000) -> every sample rejected.
|
||||||
|
monkeypatch.setattr(mining, "run",
|
||||||
|
lambda a, s, r, timeout=900.0: _result(500_000_000, 100, 500))
|
||||||
|
p = measure_point(_adapter(), "abcd", effort=1000, samples=4, workers=2,
|
||||||
|
tokens_per_worker=2, nonce_bytes=8, max_attempts=1_000_000,
|
||||||
|
repo_root=Path("."), timeout=10)
|
||||||
|
assert p.samples == 0
|
||||||
|
assert p.tokens_per_sec_1core == 0
|
||||||
|
assert p.tokens_per_sec_machine == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_distinct_nonce_ranges_do_not_overlap(monkeypatch):
|
||||||
|
# Record the nonce_start each invocation used; ranges must be STRIDE-spaced
|
||||||
|
# and disjoint between the 1-core samples and the concurrent worker streams.
|
||||||
|
seen = []
|
||||||
|
def rec(a, spec, r, timeout=900.0):
|
||||||
|
seen.append(spec.nonce_start)
|
||||||
|
return _result(100_000_000, 10, 1000)
|
||||||
|
monkeypatch.setattr(mining, "run", rec)
|
||||||
|
measure_point(_adapter(), "abcd", effort=1000, samples=3, workers=2,
|
||||||
|
tokens_per_worker=2, nonce_bytes=8, max_attempts=1_000_000,
|
||||||
|
repo_root=Path("."), timeout=10)
|
||||||
|
S = mining.STRIDE
|
||||||
|
# 3 sequential 1-core samples at 0,S,2S; then worker0 streams 3S,4S, worker1 5S,6S.
|
||||||
|
assert seen[:3] == [0, S, 2 * S]
|
||||||
|
assert sorted(seen[3:]) == [3 * S, 4 * S, 5 * S, 6 * S]
|
||||||
|
assert len(set(seen)) == len(seen) # all disjoint
|
||||||
48
tools/benchmarks/Equi-X/harness/tests/test_mock_runner.py
Normal file
48
tools/benchmarks/Equi-X/harness/tests/test_mock_runner.py
Normal file
@ -0,0 +1,48 @@
|
|||||||
|
"""The harness must speak the protocol to ANY conforming runner, and a job with
|
||||||
|
`repetitions=N` must yield exactly N timed entries (warmups excluded)."""
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from equix_bench.protocol import JobSpec
|
||||||
|
from equix_bench.registry import Adapter
|
||||||
|
from equix_bench.runner import run
|
||||||
|
|
||||||
|
MOCK = '''#!/usr/bin/env python3
|
||||||
|
import sys, json
|
||||||
|
job = json.loads(sys.stdin.read())
|
||||||
|
reps = job.get("repetitions", 1)
|
||||||
|
# A conforming runner emits exactly `repetitions` timed runs (no warmups).
|
||||||
|
runs = [{"index": i, "wall_ns": 100 + i, "solutions": 1, "compile_ns": 0,
|
||||||
|
"attempts": 0, "achieved_effort": 0, "verify_result": None} for i in range(reps)]
|
||||||
|
print(json.dumps({
|
||||||
|
"schema_version": 1, "ok": True,
|
||||||
|
"impl": {"name": "mock", "version": "0", "commit": "0", "runtime_effective": "interpreted"},
|
||||||
|
"operation": job["operation"], "runtime_requested": job.get("runtime", "?"),
|
||||||
|
"runtime_effective": "interpreted", "env": {}, "runs": runs,
|
||||||
|
"solutions_hex": ["00" * 16], "peak_rss_kb": 42, "error": None}))
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_parses_conforming_runner(tmp_path):
|
||||||
|
script = tmp_path / "mock.py"
|
||||||
|
script.write_text(MOCK)
|
||||||
|
adapter = Adapter(
|
||||||
|
name="mock", exec=[sys.executable, str(script)], protocol_version=1,
|
||||||
|
capabilities=["solve"], runtimes=["interpret"], env={},
|
||||||
|
)
|
||||||
|
r = run(adapter, JobSpec(operation="solve", runtime="interpret",
|
||||||
|
repetitions=5, warmup=2, challenge_hex="ab"), tmp_path)
|
||||||
|
assert r.ok
|
||||||
|
assert len(r.runs) == 5 # exactly repetitions; warmups excluded
|
||||||
|
assert r.solutions_hex == ["00" * 16]
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_tolerates_progress_lines(tmp_path):
|
||||||
|
script = tmp_path / "chatty.py"
|
||||||
|
script.write_text(MOCK.replace(
|
||||||
|
'print(json.dumps({', 'print("progress: warming up", flush=True)\nprint(json.dumps({'
|
||||||
|
))
|
||||||
|
adapter = Adapter(name="chatty", exec=[sys.executable, str(script)],
|
||||||
|
protocol_version=1, capabilities=["solve"], runtimes=["interpret"], env={})
|
||||||
|
r = run(adapter, JobSpec(operation="solve", runtime="interpret",
|
||||||
|
repetitions=2, warmup=0, challenge_hex="ab"), tmp_path)
|
||||||
|
assert r.ok and len(r.runs) == 2 # last stdout line is the JSON result
|
||||||
42
tools/benchmarks/Equi-X/harness/tests/test_protocol.py
Normal file
42
tools/benchmarks/Equi-X/harness/tests/test_protocol.py
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from equix_bench.protocol import SCHEMA_VERSION, JobSpec, Result
|
||||||
|
|
||||||
|
|
||||||
|
def test_jobspec_omits_none_fields():
|
||||||
|
d = json.loads(JobSpec(operation="solve", runtime="interpret", challenge_hex="ab").to_json())
|
||||||
|
assert d["schema_version"] == SCHEMA_VERSION
|
||||||
|
assert d["operation"] == "solve"
|
||||||
|
assert d["challenge_hex"] == "ab"
|
||||||
|
assert "solution_hex" not in d # None fields are dropped from the wire form
|
||||||
|
|
||||||
|
|
||||||
|
def test_result_rejects_wrong_schema():
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
Result.from_dict({"schema_version": 999})
|
||||||
|
|
||||||
|
|
||||||
|
def test_result_parse_roundtrip():
|
||||||
|
d = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"ok": True,
|
||||||
|
"impl": {"name": "x", "version": "1", "commit": "c", "runtime_effective": "compiled"},
|
||||||
|
"operation": "solve",
|
||||||
|
"runtime_requested": "try-compile",
|
||||||
|
"runtime_effective": "compiled",
|
||||||
|
"env": {"os": "linux"},
|
||||||
|
"runs": [
|
||||||
|
{"index": 0, "wall_ns": 10, "solutions": 4, "compile_ns": 0,
|
||||||
|
"attempts": 0, "achieved_effort": 0, "verify_result": None},
|
||||||
|
{"index": 1, "wall_ns": 20, "solutions": 4, "compile_ns": 0,
|
||||||
|
"attempts": 0, "achieved_effort": 0, "verify_result": None},
|
||||||
|
],
|
||||||
|
"solutions_hex": ["00" * 16],
|
||||||
|
"peak_rss_kb": 100,
|
||||||
|
"error": None,
|
||||||
|
}
|
||||||
|
r = Result.from_dict(d)
|
||||||
|
assert r.ok and r.impl_name == "x"
|
||||||
|
assert len(r.runs) == 2 and r.runs[1].wall_ns == 20
|
||||||
@ -0,0 +1,65 @@
|
|||||||
|
"""End-to-end smoke test for report generation with concurrency + mining data:
|
||||||
|
a field/format error in these sections must fail here, not at the end of an
|
||||||
|
hours-long --full run."""
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from equix_bench import report as reportmod
|
||||||
|
from equix_bench.concurrency import ConcResult, LevelStat
|
||||||
|
from equix_bench.concurrency import write_csv as conc_csv
|
||||||
|
from equix_bench.mining import MiningPoint, MiningResult
|
||||||
|
from equix_bench.mining import write_csv as mining_csv
|
||||||
|
from equix_bench.protocol import Result, Run
|
||||||
|
from equix_bench.stats import summarize
|
||||||
|
|
||||||
|
|
||||||
|
def _stats_cell(impl, op, median_ns):
|
||||||
|
runs = [Run(index=0, wall_ns=median_ns, solutions=1, compile_ns=0, attempts=0,
|
||||||
|
achieved_effort=0, verify_result="OK")]
|
||||||
|
res = Result(ok=True, impl_name=impl, impl_version="1", impl_commit="c",
|
||||||
|
operation=op, runtime_requested="try-compile", runtime_effective="compiled",
|
||||||
|
env={}, runs=runs, solutions_hex=None, peak_rss_kb=1000, error=None)
|
||||||
|
return summarize(impl, op, "try-compile", {"challenge": "deadbeef"}, res,
|
||||||
|
{"label": "cpuX", "type": "cpu", "name": "X", "arch": "arm"})
|
||||||
|
|
||||||
|
|
||||||
|
def _conc(impl, op):
|
||||||
|
lv = [LevelStat(1, 1, 0.01, 100.0, 100.0, 1.0, 1000),
|
||||||
|
LevelStat(2, 2, 0.011, 182.0, 91.0, 0.91, 2000)]
|
||||||
|
return ConcResult(device="cpuX", impl=impl, operation=op, nproc=2, reps=40,
|
||||||
|
challenge="deadbeef", baseline_ops_per_sec=100.0,
|
||||||
|
peak_ops_per_sec=182.0, knee_workers=2, levels=lv)
|
||||||
|
|
||||||
|
|
||||||
|
def _mining(impl):
|
||||||
|
# Deliberately non-ascending efforts: the section must sort, not garble.
|
||||||
|
pts = [MiningPoint(1000, 10, 3.6, 2.5, 3.4, 500.0, 3000.0, 0.28, 2, 2, 0.55, 0.98, 1, 16, 16, 8),
|
||||||
|
MiningPoint(100, 10, 0.165, 0.11, 0.14, 34.0, 350.0, 6.06, 2, 2, 12.2, 1.0, 0, 16, 16, 8)]
|
||||||
|
return MiningResult(device="cpuX", impl=impl, challenge_base="abcd", nproc=2, points=pts)
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_report_with_concurrency_and_mining(tmp_path):
|
||||||
|
stats = [_stats_cell("equix-c", "solve", 39_000_000),
|
||||||
|
_stats_cell("equix-rust", "solve", 4_500_000),
|
||||||
|
_stats_cell("equix-c", "verify", 16_000),
|
||||||
|
_stats_cell("equix-rust", "verify", 15_000)]
|
||||||
|
conc = [_conc("equix-c", "solve"), _conc("equix-rust", "solve")]
|
||||||
|
mining = [_mining("equix-rust")]
|
||||||
|
meta = {"timestamp": "t", "config": "test", "cpu": "X", "nproc": 2, "devices": ["cpuX"]}
|
||||||
|
|
||||||
|
reportmod.generate(stats, [], [], tmp_path, meta, concurrency=conc, mining=mining)
|
||||||
|
md = (tmp_path / "report.md").read_text()
|
||||||
|
assert "Sustained throughput under concurrency" in md
|
||||||
|
assert "Mining rate vs difficulty" in md
|
||||||
|
# Mining table rows must come out effort-ascending despite input order.
|
||||||
|
assert md.index("| 100 |") < md.index("| 1000 |")
|
||||||
|
# The 1/effort headline must use the true endpoints (100 -> 1000, a 10x rise).
|
||||||
|
assert "10× rise" in md or "10x rise" in md
|
||||||
|
# Message-size constancy must be reported when sizes are uniform.
|
||||||
|
assert "constant in difficulty" in md
|
||||||
|
|
||||||
|
conc_csv(conc, tmp_path / "concurrency.csv")
|
||||||
|
mining_csv(mining, tmp_path / "mining.csv")
|
||||||
|
head = (tmp_path / "mining.csv").read_text().splitlines()
|
||||||
|
assert head[0].endswith("nonce_bytes_wire")
|
||||||
|
assert len(head) == 3 # header + 2 points
|
||||||
56
tools/benchmarks/Equi-X/harness/tests/test_seed_challenge.py
Normal file
56
tools/benchmarks/Equi-X/harness/tests/test_seed_challenge.py
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
"""vary_challenge turns each configured challenge into a SEED: the runner derives
|
||||||
|
a fresh challenge per rep by SHA-256-chaining it, so measurements span many
|
||||||
|
challenges. These tests pin the harness wiring (config expansion, protocol
|
||||||
|
serialization, verify-resolution skip)."""
|
||||||
|
import json
|
||||||
|
|
||||||
|
from equix_bench.cli import _resolve_verify_solutions
|
||||||
|
from equix_bench.config import Config, expand
|
||||||
|
from equix_bench.protocol import JobSpec
|
||||||
|
from equix_bench.registry import Adapter
|
||||||
|
|
||||||
|
|
||||||
|
def _adapter(name, caps):
|
||||||
|
return Adapter(name=name, exec=["/bin/true"], protocol_version=1,
|
||||||
|
capabilities=caps, runtimes=["interpret", "try-compile"], env={})
|
||||||
|
|
||||||
|
|
||||||
|
def test_expand_sets_seed_not_challenge_when_varied():
|
||||||
|
cfg = Config(warmup=1, repetitions=5, impls=["x"],
|
||||||
|
jobs=[{"operation": "solve", "runtimes": ["interpret"],
|
||||||
|
"challenges": ["deadbeef", "cafe"], "vary_challenge": True}])
|
||||||
|
cells, _ = expand(cfg, {"x": _adapter("x", ["solve"])})
|
||||||
|
assert len(cells) == 2
|
||||||
|
for c in cells:
|
||||||
|
assert c.job.challenge_seed_hex in ("deadbeef", "cafe")
|
||||||
|
assert c.job.challenge_hex is None # seed replaces the fixed challenge
|
||||||
|
assert c.label.get("varied") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_expand_fixed_challenge_when_not_varied():
|
||||||
|
cfg = Config(warmup=1, repetitions=5, impls=["x"],
|
||||||
|
jobs=[{"operation": "solve", "runtimes": ["interpret"],
|
||||||
|
"challenges": ["deadbeef"]}])
|
||||||
|
cells, _ = expand(cfg, {"x": _adapter("x", ["solve"])})
|
||||||
|
assert cells[0].job.challenge_hex == "deadbeef"
|
||||||
|
assert cells[0].job.challenge_seed_hex is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_jobspec_serializes_seed_and_omits_none_challenge():
|
||||||
|
d = json.loads(JobSpec(operation="solve", runtime="interpret",
|
||||||
|
challenge_seed_hex="abcd").to_json())
|
||||||
|
assert d["challenge_seed_hex"] == "abcd"
|
||||||
|
assert "challenge_hex" not in d # None fields dropped from the wire
|
||||||
|
|
||||||
|
|
||||||
|
def test_verify_resolution_skips_seed_mode_cells():
|
||||||
|
# A seed-mode verify cell must pass through untouched (runner self-solves) —
|
||||||
|
# never dropped for "no solution found", never sent to a solver.
|
||||||
|
cfg = Config(warmup=1, repetitions=3, impls=["x"],
|
||||||
|
jobs=[{"operation": "verify", "runtimes": ["interpret"],
|
||||||
|
"challenges": ["deadbeef"], "vary_challenge": True}])
|
||||||
|
cells, _ = expand(cfg, {"x": _adapter("x", ["verify"])})
|
||||||
|
# adapters/repo_root are irrelevant here: seed cells short-circuit before any run.
|
||||||
|
out, warnings = _resolve_verify_solutions(cells, {}, repo_root=None)
|
||||||
|
assert len(out) == 1 and out[0].job.challenge_seed_hex == "deadbeef"
|
||||||
|
assert warnings == []
|
||||||
@ -0,0 +1,49 @@
|
|||||||
|
"""Integration checks on the real runners' seed mode (skipped if not built).
|
||||||
|
The load-bearing property: because both impls derive challenges with STANDARD
|
||||||
|
SHA-256, the same seed yields the same challenge stream, so a seed-varied
|
||||||
|
measurement still compares the two implementations on identical inputs."""
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
C = ROOT / "build/runners/c/equix_runner"
|
||||||
|
RUST = ROOT / "runners/rust/target/release/equix_runner"
|
||||||
|
|
||||||
|
|
||||||
|
def _run(binary, spec):
|
||||||
|
p = subprocess.run([str(binary)], input=json.dumps(spec),
|
||||||
|
capture_output=True, text=True, timeout=120)
|
||||||
|
return json.loads(p.stdout.strip().splitlines()[-1])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not (C.exists() and RUST.exists()), reason="runners not built")
|
||||||
|
def test_seed_mode_derives_identical_challenges_across_impls():
|
||||||
|
# challenge_0 = SHA256(seed): solving it in seed mode must give, in BOTH
|
||||||
|
# impls, exactly the solutions of that derived challenge (fixed mode).
|
||||||
|
seed = "abcd"
|
||||||
|
derived = hashlib.sha256(bytes.fromhex(seed)).hexdigest()
|
||||||
|
outs = {}
|
||||||
|
for name, b in (("c", C), ("rust", RUST)):
|
||||||
|
seeded = _run(b, {"schema_version": 1, "operation": "solve", "runtime": "interpret",
|
||||||
|
"repetitions": 1, "warmup": 0, "challenge_seed_hex": seed})
|
||||||
|
fixed = _run(b, {"schema_version": 1, "operation": "solve", "runtime": "interpret",
|
||||||
|
"repetitions": 1, "warmup": 0, "challenge_hex": derived})
|
||||||
|
# seed mode's first challenge IS sha256(seed): same solutions as fixed.
|
||||||
|
assert sorted(seeded["solutions_hex"]) == sorted(fixed["solutions_hex"])
|
||||||
|
outs[name] = sorted(seeded["solutions_hex"])
|
||||||
|
# And both implementations agree on that derived challenge.
|
||||||
|
assert outs["c"] == outs["rust"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not RUST.exists(), reason="rust runner not built")
|
||||||
|
def test_seed_mode_verify_selfsolves_valid_tokens():
|
||||||
|
# Seed-mode verify needs no solution_hex and every timed sample is a real,
|
||||||
|
# accepted token (the runner self-solves each derived challenge).
|
||||||
|
d = _run(RUST, {"schema_version": 1, "operation": "verify", "runtime": "try-compile",
|
||||||
|
"repetitions": 8, "warmup": 2, "challenge_seed_hex": "deadbeef"})
|
||||||
|
assert d["ok"] and len(d["runs"]) == 8
|
||||||
|
assert all(r["verify_result"] == "OK" for r in d["runs"])
|
||||||
43
tools/benchmarks/Equi-X/harness/tests/test_stats.py
Normal file
43
tools/benchmarks/Equi-X/harness/tests/test_stats.py
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
from equix_bench.protocol import Result
|
||||||
|
from equix_bench.stats import _percentile, summarize
|
||||||
|
|
||||||
|
|
||||||
|
def _make(walls):
|
||||||
|
runs = [
|
||||||
|
{"index": i, "wall_ns": w, "solutions": 4, "compile_ns": 0,
|
||||||
|
"attempts": 0, "achieved_effort": 0, "verify_result": None}
|
||||||
|
for i, w in enumerate(walls)
|
||||||
|
]
|
||||||
|
return Result.from_dict({
|
||||||
|
"schema_version": 1, "ok": True,
|
||||||
|
"impl": {"name": "x", "version": "1", "commit": "c", "runtime_effective": "compiled"},
|
||||||
|
"operation": "solve", "runtime_requested": "try-compile",
|
||||||
|
"runtime_effective": "compiled", "env": {}, "runs": runs,
|
||||||
|
"solutions_hex": None, "peak_rss_kb": 100, "error": None,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def test_percentile_nearest_rank():
|
||||||
|
assert _percentile([1, 2, 3, 4, 5], 95) == 5
|
||||||
|
assert _percentile([1, 2, 3, 4, 5], 50) == 3
|
||||||
|
assert _percentile([], 95) == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_summarize_basic():
|
||||||
|
s = summarize("x", "solve", "try-compile", {}, _make([10, 20, 30]))
|
||||||
|
assert s.ok and s.reps == 3
|
||||||
|
assert s.min_ns == 10 and s.median_ns == 20
|
||||||
|
assert s.solves_per_sec > 0 # 1e9 / 20 ns
|
||||||
|
assert s.walls == [10, 20, 30]
|
||||||
|
|
||||||
|
|
||||||
|
def test_summarize_handles_failure():
|
||||||
|
r = Result.from_dict({
|
||||||
|
"schema_version": 1, "ok": False,
|
||||||
|
"impl": {"name": "x", "version": "1", "commit": "c", "runtime_effective": None},
|
||||||
|
"operation": "solve", "runtime_requested": "must-compile",
|
||||||
|
"runtime_effective": None, "env": {}, "runs": [],
|
||||||
|
"solutions_hex": None, "peak_rss_kb": 0, "error": "boom",
|
||||||
|
})
|
||||||
|
s = summarize("x", "solve", "must-compile", {}, r)
|
||||||
|
assert not s.ok and s.error == "boom"
|
||||||
59
tools/benchmarks/Equi-X/harness/tests/test_variants.py
Normal file
59
tools/benchmarks/Equi-X/harness/tests/test_variants.py
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
"""Compiler-flag variants ride on the multi-implementation machinery: several C
|
||||||
|
builds register as distinct impls (from a second manifest dir) and expand into
|
||||||
|
comparable cells."""
|
||||||
|
from equix_bench.config import Config, expand
|
||||||
|
from equix_bench.registry import Adapter, load_manifests
|
||||||
|
|
||||||
|
_MANIFEST = (
|
||||||
|
'name = "{name}"\n'
|
||||||
|
'exec = "build/runners/c/equix_runner"\n'
|
||||||
|
'protocol_version = 1\n'
|
||||||
|
'capabilities = ["solve", "hashx_compile"]\n'
|
||||||
|
'runtimes = ["try-compile", "interpret"]\n'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_manifests_merges_multiple_dirs(tmp_path):
|
||||||
|
d1, d2 = tmp_path / "examples", tmp_path / "generated"
|
||||||
|
d1.mkdir()
|
||||||
|
d2.mkdir()
|
||||||
|
(d1 / "base.manifest.toml").write_text(_MANIFEST.format(name="equix-c"))
|
||||||
|
(d2 / "o2.manifest.toml").write_text(_MANIFEST.format(name="equix-c-gcc-o2"))
|
||||||
|
(d2 / "o3.manifest.toml").write_text(_MANIFEST.format(name="equix-c-gcc-o3"))
|
||||||
|
adapters = load_manifests([d1, d2])
|
||||||
|
assert set(adapters) == {"equix-c", "equix-c-gcc-o2", "equix-c-gcc-o3"}
|
||||||
|
# a non-existent dir is simply skipped
|
||||||
|
assert set(load_manifests([d1, tmp_path / "nope"])) == {"equix-c"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_expand_sweeps_variant_impls():
|
||||||
|
names = ["equix-c-gcc-o0", "equix-c-gcc-o3", "equix-c-clang-o3"]
|
||||||
|
adapters = {
|
||||||
|
n: Adapter(name=n, exec=["/bin/true"], protocol_version=1,
|
||||||
|
capabilities=["solve"], runtimes=["try-compile"], env={})
|
||||||
|
for n in names
|
||||||
|
}
|
||||||
|
cfg = Config(
|
||||||
|
warmup=1, repetitions=2, impls=names,
|
||||||
|
jobs=[{"operation": "solve", "runtimes": ["try-compile"],
|
||||||
|
"challenges": ["deadbeef", "cafe"]}],
|
||||||
|
)
|
||||||
|
cells, warnings = expand(cfg, adapters)
|
||||||
|
# 3 variants x 2 challenges x 1 runtime
|
||||||
|
assert len(cells) == 6
|
||||||
|
assert {c.impl for c in cells} == set(names)
|
||||||
|
assert not warnings
|
||||||
|
|
||||||
|
|
||||||
|
def test_expand_warns_on_unsupported_runtime():
|
||||||
|
adapters = {
|
||||||
|
"equix-c-gcc-o3": Adapter(name="equix-c-gcc-o3", exec=["/bin/true"],
|
||||||
|
protocol_version=1, capabilities=["solve"],
|
||||||
|
runtimes=["try-compile"], env={}),
|
||||||
|
}
|
||||||
|
cfg = Config(warmup=1, repetitions=1, impls=["equix-c-gcc-o3"],
|
||||||
|
jobs=[{"operation": "solve", "runtimes": ["must-compile"],
|
||||||
|
"challenges": ["deadbeef"]}])
|
||||||
|
cells, warnings = expand(cfg, adapters)
|
||||||
|
assert cells == []
|
||||||
|
assert any("must-compile" in w for w in warnings)
|
||||||
57
tools/benchmarks/Equi-X/runners/c/CMakeLists.txt
Normal file
57
tools/benchmarks/Equi-X/runners/c/CMakeLists.txt
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
# Equi-X C benchmark runner.
|
||||||
|
#
|
||||||
|
# Builds the vendored tevador/equix + hashx static libraries (via add_subdirectory)
|
||||||
|
# and links them into `equix_runner`. blake2.c symbols come from hashx_static.
|
||||||
|
cmake_minimum_required(VERSION 3.10)
|
||||||
|
project(equix_runner C)
|
||||||
|
|
||||||
|
set(CMAKE_C_STANDARD 11)
|
||||||
|
set(CMAKE_C_STANDARD_REQUIRED ON)
|
||||||
|
|
||||||
|
if(NOT CMAKE_BUILD_TYPE)
|
||||||
|
set(CMAKE_BUILD_TYPE Release)
|
||||||
|
message(STATUS "Setting default build type: ${CMAKE_BUILD_TYPE}")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
# Location of the vendored equix source tree (contains hashx/ submodule).
|
||||||
|
if(NOT DEFINED EQUIX_DIR)
|
||||||
|
set(EQUIX_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../vendored/equix")
|
||||||
|
endif()
|
||||||
|
get_filename_component(EQUIX_DIR "${EQUIX_DIR}" ABSOLUTE)
|
||||||
|
|
||||||
|
if(NOT EXISTS "${EQUIX_DIR}/include/equix.h")
|
||||||
|
message(FATAL_ERROR "equix source not found at ${EQUIX_DIR}. Run scripts/setup.sh first.")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
# The vendored equix/hashx use cmake_minimum_required(VERSION 2.8.8). CMake >= 4
|
||||||
|
# removes compatibility with < 3.5 (hard error) and deprecation-warns for < 3.10.
|
||||||
|
# Pin the policy floor to 3.10 so those submodules configure cleanly with neither
|
||||||
|
# the error nor the "Compatibility with CMake < 3.10 will be removed" warning.
|
||||||
|
if(NOT DEFINED CMAKE_POLICY_VERSION_MINIMUM)
|
||||||
|
set(CMAKE_POLICY_VERSION_MINIMUM 3.10 CACHE STRING "" FORCE)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
add_subdirectory("${EQUIX_DIR}" "${CMAKE_BINARY_DIR}/equix_ext")
|
||||||
|
|
||||||
|
add_executable(equix_runner
|
||||||
|
equix_runner.c
|
||||||
|
effort.c
|
||||||
|
json_min.c
|
||||||
|
sha256.c)
|
||||||
|
|
||||||
|
target_compile_definitions(equix_runner PRIVATE HASHX_SIZE=8 EQUIX_STATIC HASHX_STATIC)
|
||||||
|
|
||||||
|
target_include_directories(equix_runner PRIVATE
|
||||||
|
"${EQUIX_DIR}/include"
|
||||||
|
"${EQUIX_DIR}/hashx/include"
|
||||||
|
"${EQUIX_DIR}/hashx/src") # for the internal blake2.h
|
||||||
|
|
||||||
|
target_link_libraries(equix_runner PRIVATE equix_static hashx_static)
|
||||||
|
|
||||||
|
# Optional build-time provenance (set by scripts/setup.sh).
|
||||||
|
if(DEFINED EQUIX_C_COMMIT)
|
||||||
|
target_compile_definitions(equix_runner PRIVATE EQUIX_C_COMMIT="${EQUIX_C_COMMIT}")
|
||||||
|
endif()
|
||||||
|
if(DEFINED EQUIX_C_VERSION)
|
||||||
|
target_compile_definitions(equix_runner PRIVATE EQUIX_C_VERSION="${EQUIX_C_VERSION}")
|
||||||
|
endif()
|
||||||
46
tools/benchmarks/Equi-X/runners/c/effort.c
Normal file
46
tools/benchmarks/Equi-X/runners/c/effort.c
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
#include "effort.h"
|
||||||
|
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
/* blake2.h is an internal hashx header (not installed); referenced by include
|
||||||
|
* path into the vendored submodule. We use the full (12-round) standard
|
||||||
|
* BLAKE2b via init_param/update/final -- NOT the reduced hashx_blake2b_4r. */
|
||||||
|
#include <blake2.h>
|
||||||
|
|
||||||
|
void effort_solution_bytes(const equix_solution *sol, uint8_t out[16]) {
|
||||||
|
for (int i = 0; i < EQUIX_NUM_IDX; i++) {
|
||||||
|
out[2 * i] = (uint8_t)(sol->idx[i] & 0xff);
|
||||||
|
out[2 * i + 1] = (uint8_t)((sol->idx[i] >> 8) & 0xff);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Standard unkeyed BLAKE2b-256 (digest 32, fanout 1, depth 1, all else zero). */
|
||||||
|
static void blake2b256(const uint8_t *in1, size_t len1, const uint8_t *in2,
|
||||||
|
size_t len2, uint8_t out[32]) {
|
||||||
|
blake2b_param p;
|
||||||
|
memset(&p, 0, sizeof p);
|
||||||
|
p.digest_length = 32;
|
||||||
|
p.fanout = 1;
|
||||||
|
p.depth = 1;
|
||||||
|
|
||||||
|
blake2b_state s;
|
||||||
|
hashx_blake2b_init_param(&s, &p);
|
||||||
|
hashx_blake2b_update(&s, in1, len1);
|
||||||
|
hashx_blake2b_update(&s, in2, len2);
|
||||||
|
hashx_blake2b_final(&s, out, 32);
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t effort_of(const uint8_t *challenge, size_t challenge_len,
|
||||||
|
const equix_solution *sol) {
|
||||||
|
uint8_t sol_bytes[16];
|
||||||
|
effort_solution_bytes(sol, sol_bytes);
|
||||||
|
|
||||||
|
uint8_t h[32];
|
||||||
|
blake2b256(challenge, challenge_len, sol_bytes, sizeof sol_bytes, h);
|
||||||
|
|
||||||
|
uint32_t hash32 = ((uint32_t)h[0] << 24) | ((uint32_t)h[1] << 16) |
|
||||||
|
((uint32_t)h[2] << 8) | (uint32_t)h[3];
|
||||||
|
if (hash32 == 0)
|
||||||
|
return 0xFFFFFFFFu;
|
||||||
|
return 0xFFFFFFFFu / hash32;
|
||||||
|
}
|
||||||
30
tools/benchmarks/Equi-X/runners/c/effort.h
Normal file
30
tools/benchmarks/Equi-X/runners/c/effort.h
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
/* Tor proposal-327 style effort computation for Equi-X solutions.
|
||||||
|
*
|
||||||
|
* The effort layer sits ABOVE Equi-X: given a solved (challenge, solution),
|
||||||
|
* we hash them with standard BLAKE2b-256 and read the first 32 bits big-endian
|
||||||
|
* as `hash32`. A solution is "valid at effort E" iff hash32 * E <= 2^32-1, so
|
||||||
|
* the achieved effort of a solution is floor((2^32-1) / hash32).
|
||||||
|
*
|
||||||
|
* The preimage layout and byte order defined here MUST be identical to the Rust
|
||||||
|
* runner (runners/rust/src/effort.rs) -- the Python cross-check asserts both
|
||||||
|
* implementations produce the same effort for a fixed (challenge, solution).
|
||||||
|
*
|
||||||
|
* Preimage = challenge_bytes || solution_bytes
|
||||||
|
* solution_bytes = 8 x uint16 little-endian (equix_solution.idx[])
|
||||||
|
*/
|
||||||
|
#ifndef EQUIX_RUNNER_EFFORT_H
|
||||||
|
#define EQUIX_RUNNER_EFFORT_H
|
||||||
|
|
||||||
|
#include <stddef.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
#include <equix.h>
|
||||||
|
|
||||||
|
/* Serialize an Equi-X solution to its canonical 16-byte little-endian form. */
|
||||||
|
void effort_solution_bytes(const equix_solution *sol, uint8_t out[16]);
|
||||||
|
|
||||||
|
/* Achieved effort of `sol` for `challenge`. Returns UINT32_MAX when hash32==0. */
|
||||||
|
uint32_t effort_of(const uint8_t *challenge, size_t challenge_len,
|
||||||
|
const equix_solution *sol);
|
||||||
|
|
||||||
|
#endif /* EQUIX_RUNNER_EFFORT_H */
|
||||||
711
tools/benchmarks/Equi-X/runners/c/equix_runner.c
Normal file
711
tools/benchmarks/Equi-X/runners/c/equix_runner.c
Normal file
@ -0,0 +1,711 @@
|
|||||||
|
/* Equi-X C benchmark runner.
|
||||||
|
*
|
||||||
|
* Reads one job-spec JSON object on stdin, runs the requested operation against
|
||||||
|
* the reference C implementation (tevador/equix + hashx), and writes one result
|
||||||
|
* JSON object on stdout. All diagnostics go to stderr. See adapters/README.md
|
||||||
|
* for the protocol.
|
||||||
|
*/
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/utsname.h>
|
||||||
|
#if defined(__APPLE__)
|
||||||
|
#include <sys/sysctl.h>
|
||||||
|
#define OS_STR "macos"
|
||||||
|
#else
|
||||||
|
#define OS_STR "linux"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* The bundled HashX JITs via mmap(RW)+mprotect(RX) with no MAP_JIT, which the
|
||||||
|
* Apple Silicon kernel (hard W^X) rejects and would crash on execution. So we
|
||||||
|
* treat the compiler as unsupported there and fall back to the interpreter. */
|
||||||
|
#if defined(__APPLE__) && defined(__aarch64__)
|
||||||
|
#define JIT_SUPPORTED 0
|
||||||
|
#else
|
||||||
|
#define JIT_SUPPORTED 1
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#include <equix.h>
|
||||||
|
#include <hashx.h>
|
||||||
|
|
||||||
|
#include "effort.h"
|
||||||
|
#include "json_min.h"
|
||||||
|
#include "sha256.h"
|
||||||
|
#include "timing.h"
|
||||||
|
|
||||||
|
#ifndef EQUIX_C_COMMIT
|
||||||
|
#define EQUIX_C_COMMIT "unknown"
|
||||||
|
#endif
|
||||||
|
#ifndef EQUIX_C_VERSION
|
||||||
|
#define EQUIX_C_VERSION "1.0.0"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if defined(__clang__)
|
||||||
|
#define COMPILER_STR "clang-" __clang_version__
|
||||||
|
#elif defined(__GNUC__)
|
||||||
|
#define COMPILER_STR "gcc-" __VERSION__
|
||||||
|
#else
|
||||||
|
#define COMPILER_STR "unknown"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if defined(__x86_64__) || defined(_M_X64)
|
||||||
|
#define ARCH_STR "x86_64"
|
||||||
|
#elif defined(__aarch64__)
|
||||||
|
#define ARCH_STR "aarch64"
|
||||||
|
#elif defined(__i386__)
|
||||||
|
#define ARCH_STR "x86"
|
||||||
|
#elif defined(__arm__)
|
||||||
|
#define ARCH_STR "arm"
|
||||||
|
#else
|
||||||
|
#define ARCH_STR "unknown"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#define MAX_CHALLENGE 256
|
||||||
|
|
||||||
|
/* CPU model string from /proc/cpuinfo, JSON-escaped, cached. Tries fields in
|
||||||
|
* priority order so it works across architectures: "model name" (x86),
|
||||||
|
* "Model" (Raspberry Pi board), "Hardware" (older ARM), "cpu model" (others).
|
||||||
|
* Returns "unknown" when none are present. */
|
||||||
|
static const char *cpu_model(void) {
|
||||||
|
static char model[256];
|
||||||
|
if (model[0])
|
||||||
|
return model;
|
||||||
|
strcpy(model, "unknown");
|
||||||
|
#if defined(__APPLE__)
|
||||||
|
/* macOS has no /proc; the CPU brand comes from sysctl (works on both Intel
|
||||||
|
* and Apple Silicon, e.g. "Apple M2"). */
|
||||||
|
size_t sz = sizeof model;
|
||||||
|
if (sysctlbyname("machdep.cpu.brand_string", model, &sz, NULL, 0) != 0 || !model[0])
|
||||||
|
strcpy(model, "unknown");
|
||||||
|
return model;
|
||||||
|
#else
|
||||||
|
FILE *f = fopen("/proc/cpuinfo", "r");
|
||||||
|
if (!f)
|
||||||
|
return model;
|
||||||
|
/* Large enough that ARM's trailing "Model"/"Hardware" lines are captured
|
||||||
|
* even on many-core machines. */
|
||||||
|
static char buf[131072];
|
||||||
|
size_t n = fread(buf, 1, sizeof buf - 1, f);
|
||||||
|
buf[n] = '\0';
|
||||||
|
fclose(f);
|
||||||
|
|
||||||
|
static const char *fields[] = {"model name", "Model", "Hardware", "cpu model"};
|
||||||
|
for (size_t fi = 0; fi < sizeof fields / sizeof fields[0]; fi++) {
|
||||||
|
size_t fl = strlen(fields[fi]);
|
||||||
|
for (const char *p = buf; p; p = strchr(p, '\n') ? strchr(p, '\n') + 1 : NULL) {
|
||||||
|
if (strncmp(p, fields[fi], fl) != 0)
|
||||||
|
continue;
|
||||||
|
/* field must be followed by whitespace/':' (avoid partial matches) */
|
||||||
|
char after = p[fl];
|
||||||
|
if (after != ' ' && after != '\t' && after != ':')
|
||||||
|
continue;
|
||||||
|
const char *c = strchr(p, ':');
|
||||||
|
const char *eol = strchr(p, '\n');
|
||||||
|
if (!c || (eol && c > eol))
|
||||||
|
continue;
|
||||||
|
c++;
|
||||||
|
while (*c == ' ' || *c == '\t')
|
||||||
|
c++;
|
||||||
|
size_t k = 0;
|
||||||
|
for (size_t i = 0; c[i] && c[i] != '\n' && c[i] != '\r' && k + 2 < sizeof model; i++) {
|
||||||
|
if (c[i] == '"' || c[i] == '\\')
|
||||||
|
model[k++] = '\\';
|
||||||
|
model[k++] = c[i];
|
||||||
|
}
|
||||||
|
while (k > 0 && model[k - 1] == ' ')
|
||||||
|
k--;
|
||||||
|
model[k] = '\0';
|
||||||
|
if (k > 0)
|
||||||
|
return model;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return model;
|
||||||
|
#endif /* !__APPLE__ */
|
||||||
|
}
|
||||||
|
|
||||||
|
/* OS kernel release (uname -r), e.g. "6.18.5" (Linux) or "23.5.0" (macOS/Darwin).
|
||||||
|
* Cached; "unknown" on failure. uname() is portable across Linux and macOS. */
|
||||||
|
static const char *os_version(void) {
|
||||||
|
static char v[128];
|
||||||
|
if (v[0])
|
||||||
|
return v;
|
||||||
|
strcpy(v, "unknown");
|
||||||
|
struct utsname u;
|
||||||
|
if (uname(&u) == 0)
|
||||||
|
snprintf(v, sizeof v, "%s", u.release);
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ------------------------------------------------------------------ helpers */
|
||||||
|
|
||||||
|
static char *read_all_stdin(void) {
|
||||||
|
size_t cap = 4096, len = 0;
|
||||||
|
char *buf = malloc(cap);
|
||||||
|
if (!buf)
|
||||||
|
return NULL;
|
||||||
|
size_t n;
|
||||||
|
while ((n = fread(buf + len, 1, cap - len - 1, stdin)) > 0) {
|
||||||
|
len += n;
|
||||||
|
if (len + 1 >= cap) {
|
||||||
|
cap *= 2;
|
||||||
|
char *nb = realloc(buf, cap);
|
||||||
|
if (!nb) {
|
||||||
|
free(buf);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
buf = nb;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
buf[len] = '\0';
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Decode a hex string into `out` (capacity outcap). Returns byte length, or -1. */
|
||||||
|
static int hex_decode(const char *hex, uint8_t *out, size_t outcap) {
|
||||||
|
size_t hl = strlen(hex);
|
||||||
|
if (hl % 2 != 0)
|
||||||
|
return -1;
|
||||||
|
size_t bl = hl / 2;
|
||||||
|
if (bl > outcap)
|
||||||
|
return -1;
|
||||||
|
for (size_t i = 0; i < bl; i++) {
|
||||||
|
char c0 = hex[2 * i], c1 = hex[2 * i + 1];
|
||||||
|
int hi = (c0 >= '0' && c0 <= '9') ? c0 - '0'
|
||||||
|
: (c0 >= 'a' && c0 <= 'f') ? c0 - 'a' + 10
|
||||||
|
: (c0 >= 'A' && c0 <= 'F') ? c0 - 'A' + 10
|
||||||
|
: -1;
|
||||||
|
int lo = (c1 >= '0' && c1 <= '9') ? c1 - '0'
|
||||||
|
: (c1 >= 'a' && c1 <= 'f') ? c1 - 'a' + 10
|
||||||
|
: (c1 >= 'A' && c1 <= 'F') ? c1 - 'A' + 10
|
||||||
|
: -1;
|
||||||
|
if (hi < 0 || lo < 0)
|
||||||
|
return -1;
|
||||||
|
out[i] = (uint8_t)((hi << 4) | lo);
|
||||||
|
}
|
||||||
|
return (int)bl;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* One measured repetition. */
|
||||||
|
typedef struct {
|
||||||
|
uint64_t wall_ns;
|
||||||
|
int solutions;
|
||||||
|
uint64_t compile_ns;
|
||||||
|
uint64_t attempts;
|
||||||
|
uint32_t achieved_effort;
|
||||||
|
const char *verify_result; /* NULL unless a verify op */
|
||||||
|
} run_t;
|
||||||
|
|
||||||
|
static const char *verify_result_str(equix_result r) {
|
||||||
|
switch (r) {
|
||||||
|
case EQUIX_OK: return "OK";
|
||||||
|
case EQUIX_CHALLENGE: return "CHALLENGE";
|
||||||
|
case EQUIX_ORDER: return "ORDER";
|
||||||
|
case EQUIX_PARTIAL_SUM: return "PARTIAL_SUM";
|
||||||
|
case EQUIX_FINAL_SUM: return "FINAL_SUM";
|
||||||
|
default: return "UNKNOWN";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Emit a failure result JSON and exit. */
|
||||||
|
static void fail(const char *op, const char *runtime_req, const char *msg) {
|
||||||
|
printf("{\"schema_version\":1,\"ok\":false,"
|
||||||
|
"\"impl\":{\"name\":\"equix-c\",\"version\":\"%s\",\"commit\":\"%s\","
|
||||||
|
"\"runtime_effective\":null},"
|
||||||
|
"\"operation\":\"%s\",\"runtime_requested\":\"%s\","
|
||||||
|
"\"runtime_effective\":null,"
|
||||||
|
"\"env\":{\"os\":\"%s\",\"compiler\":\"%s\",\"cpu\":\"%s\","
|
||||||
|
"\"arch\":\"%s\",\"device\":\"cpu\",\"os_version\":\"%s\"},"
|
||||||
|
"\"runs\":[],\"peak_rss_kb\":%ld,\"error\":\"%s\"}\n",
|
||||||
|
EQUIX_C_VERSION, EQUIX_C_COMMIT, op ? op : "", runtime_req ? runtime_req : "",
|
||||||
|
OS_STR, COMPILER_STR, cpu_model(), ARCH_STR, os_version(), peak_rss_kb(), msg);
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* solutions_hex_json: pre-formatted JSON array (e.g. ["aabb..",".."]) or NULL. */
|
||||||
|
static void emit_ex(const char *op, const char *runtime_req,
|
||||||
|
const char *runtime_eff, const run_t *runs, size_t nruns,
|
||||||
|
const char *solutions_hex_json,
|
||||||
|
const char *winning_nonce_hex);
|
||||||
|
|
||||||
|
static void emit(const char *op, const char *runtime_req,
|
||||||
|
const char *runtime_eff, const run_t *runs, size_t nruns,
|
||||||
|
const char *solutions_hex_json) {
|
||||||
|
emit_ex(op, runtime_req, runtime_eff, runs, nruns, solutions_hex_json, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Like emit, plus an optional winning_nonce_hex field (effort op: the wire
|
||||||
|
* bytes of the winning token's nonce, so the harness can measure sizes). */
|
||||||
|
static void emit_ex(const char *op, const char *runtime_req,
|
||||||
|
const char *runtime_eff, const run_t *runs, size_t nruns,
|
||||||
|
const char *solutions_hex_json,
|
||||||
|
const char *winning_nonce_hex) {
|
||||||
|
printf("{\"schema_version\":1,\"ok\":true,"
|
||||||
|
"\"impl\":{\"name\":\"equix-c\",\"version\":\"%s\",\"commit\":\"%s\","
|
||||||
|
"\"runtime_effective\":\"%s\"},"
|
||||||
|
"\"operation\":\"%s\",\"runtime_requested\":\"%s\","
|
||||||
|
"\"runtime_effective\":\"%s\","
|
||||||
|
"\"env\":{\"os\":\"%s\",\"compiler\":\"%s\",\"cpu\":\"%s\","
|
||||||
|
"\"arch\":\"%s\",\"device\":\"cpu\",\"os_version\":\"%s\"},"
|
||||||
|
"\"runs\":[",
|
||||||
|
EQUIX_C_VERSION, EQUIX_C_COMMIT, runtime_eff, op, runtime_req,
|
||||||
|
runtime_eff, OS_STR, COMPILER_STR, cpu_model(), ARCH_STR, os_version());
|
||||||
|
for (size_t i = 0; i < nruns; i++) {
|
||||||
|
const run_t *r = &runs[i];
|
||||||
|
printf("%s{\"index\":%zu,\"wall_ns\":%llu,\"solutions\":%d,"
|
||||||
|
"\"compile_ns\":%llu,\"attempts\":%llu,\"achieved_effort\":%u,"
|
||||||
|
"\"verify_result\":",
|
||||||
|
i ? "," : "", i, (unsigned long long)r->wall_ns, r->solutions,
|
||||||
|
(unsigned long long)r->compile_ns,
|
||||||
|
(unsigned long long)r->attempts, r->achieved_effort);
|
||||||
|
if (r->verify_result)
|
||||||
|
printf("\"%s\"}", r->verify_result);
|
||||||
|
else
|
||||||
|
printf("null}");
|
||||||
|
}
|
||||||
|
printf("],\"solutions_hex\":%s,",
|
||||||
|
solutions_hex_json ? solutions_hex_json : "null");
|
||||||
|
if (winning_nonce_hex)
|
||||||
|
printf("\"winning_nonce_hex\":\"%s\",", winning_nonce_hex);
|
||||||
|
printf("\"peak_rss_kb\":%ld,\"error\":null}\n", peak_rss_kb());
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Format a solution as 32 lowercase hex chars (16 bytes little-endian). */
|
||||||
|
static void solution_to_hex(const equix_solution *sol, char out[33]) {
|
||||||
|
static const char hx[] = "0123456789abcdef";
|
||||||
|
uint8_t sb[16];
|
||||||
|
effort_solution_bytes(sol, sb);
|
||||||
|
for (int i = 0; i < 16; i++) {
|
||||||
|
out[2 * i] = hx[sb[i] >> 4];
|
||||||
|
out[2 * i + 1] = hx[sb[i] & 0xf];
|
||||||
|
}
|
||||||
|
out[32] = '\0';
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ------------------------------------------------------- equix ctx creation */
|
||||||
|
|
||||||
|
/* Allocate an equix context honoring the requested runtime, reporting the
|
||||||
|
* effective runtime. base_flag is EQUIX_CTX_SOLVE or EQUIX_CTX_VERIFY.
|
||||||
|
* Returns NULL and sets *err on hard failure. */
|
||||||
|
static equix_ctx *alloc_ctx(int base_flag, const char *runtime,
|
||||||
|
const char **eff, const char **err) {
|
||||||
|
equix_ctx *ctx;
|
||||||
|
if (strcmp(runtime, "interpret") == 0) {
|
||||||
|
ctx = equix_alloc((equix_ctx_flags)base_flag);
|
||||||
|
*eff = "interpreted";
|
||||||
|
} else if (strcmp(runtime, "must-compile") == 0) {
|
||||||
|
if (!JIT_SUPPORTED) {
|
||||||
|
*err = "must-compile requested but JIT compiler not supported on this platform";
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
ctx = equix_alloc((equix_ctx_flags)(base_flag | EQUIX_CTX_COMPILE));
|
||||||
|
if (ctx == EQUIX_NOTSUPP) {
|
||||||
|
*err = "must-compile requested but JIT compiler not supported";
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
*eff = "compiled";
|
||||||
|
} else { /* try-compile (default) */
|
||||||
|
ctx = JIT_SUPPORTED
|
||||||
|
? equix_alloc((equix_ctx_flags)(base_flag | EQUIX_CTX_COMPILE))
|
||||||
|
: EQUIX_NOTSUPP;
|
||||||
|
if (ctx == EQUIX_NOTSUPP) {
|
||||||
|
ctx = equix_alloc((equix_ctx_flags)base_flag);
|
||||||
|
*eff = "interpreted (fallback)";
|
||||||
|
} else {
|
||||||
|
*eff = "compiled";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (ctx == NULL || ctx == EQUIX_NOTSUPP) {
|
||||||
|
*err = "equix_alloc failed";
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return ctx;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --------------------------------------------------------------- operations */
|
||||||
|
|
||||||
|
/* Read either a fixed challenge (challenge_hex) or a seed (challenge_seed_hex).
|
||||||
|
* In seed mode each rep hashes the current challenge to get the next (a SHA-256
|
||||||
|
* chain), so measurements span many challenges; that derivation is done OUTSIDE
|
||||||
|
* every timed region. Returns 1 for seed mode, 0 for fixed, and fills the first
|
||||||
|
* challenge into `chal`/`clen`. Fails the op if neither field is present. */
|
||||||
|
static int read_challenge_or_seed(const char *json, const char *op,
|
||||||
|
const char *runtime, uint8_t chal[MAX_CHALLENGE],
|
||||||
|
int *clen) {
|
||||||
|
char hex[2 * MAX_CHALLENGE + 1] = {0};
|
||||||
|
if (jm_get_str(json, "challenge_seed_hex", hex, sizeof hex)) {
|
||||||
|
uint8_t seed[MAX_CHALLENGE];
|
||||||
|
int slen = hex_decode(hex, seed, sizeof seed);
|
||||||
|
if (slen < 0)
|
||||||
|
fail(op, runtime, "invalid challenge_seed_hex");
|
||||||
|
sha256(seed, (size_t)slen, chal); /* challenge for iteration 0 */
|
||||||
|
*clen = 32;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
if (!jm_get_str(json, "challenge_hex", hex, sizeof hex))
|
||||||
|
fail(op, runtime, "requires challenge_hex or challenge_seed_hex");
|
||||||
|
*clen = hex_decode(hex, chal, MAX_CHALLENGE);
|
||||||
|
if (*clen < 0)
|
||||||
|
fail(op, runtime, "invalid challenge_hex");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void op_solve(const char *json, const char *runtime, uint64_t reps,
|
||||||
|
uint64_t warmup) {
|
||||||
|
uint8_t chal[MAX_CHALLENGE];
|
||||||
|
int clen;
|
||||||
|
int seeded = read_challenge_or_seed(json, "solve", runtime, chal, &clen);
|
||||||
|
|
||||||
|
const char *eff = "?", *err = NULL;
|
||||||
|
equix_ctx *ctx = alloc_ctx(EQUIX_CTX_SOLVE, runtime, &eff, &err);
|
||||||
|
if (!ctx)
|
||||||
|
fail("solve", runtime, err);
|
||||||
|
|
||||||
|
equix_solution sols[EQUIX_MAX_SOLS];
|
||||||
|
for (uint64_t w = 0; w < warmup; w++) {
|
||||||
|
(void)equix_solve(ctx, chal, clen, sols);
|
||||||
|
if (seeded)
|
||||||
|
sha256(chal, 32, chal); /* advance the chain (untimed) */
|
||||||
|
}
|
||||||
|
|
||||||
|
run_t *runs = calloc(reps ? reps : 1, sizeof(run_t));
|
||||||
|
int last_n = 0;
|
||||||
|
for (uint64_t i = 0; i < reps; i++) {
|
||||||
|
uint64_t t0 = now_ns();
|
||||||
|
int n = equix_solve(ctx, chal, clen, sols);
|
||||||
|
uint64_t t1 = now_ns();
|
||||||
|
if (seeded)
|
||||||
|
sha256(chal, 32, chal); /* derive next challenge AFTER stopping the timer */
|
||||||
|
runs[i].wall_ns = t1 - t0;
|
||||||
|
runs[i].solutions = n;
|
||||||
|
last_n = n;
|
||||||
|
}
|
||||||
|
/* Publish the solutions found in the final rep for cross-implementation
|
||||||
|
* verification. Each solution = 32 hex chars; array fits comfortably. */
|
||||||
|
char shex[EQUIX_MAX_SOLS * 40 + 4];
|
||||||
|
size_t off = 0;
|
||||||
|
off += (size_t)snprintf(shex + off, sizeof shex - off, "[");
|
||||||
|
for (int s = 0; s < last_n; s++) {
|
||||||
|
char h[33];
|
||||||
|
solution_to_hex(&sols[s], h);
|
||||||
|
off += (size_t)snprintf(shex + off, sizeof shex - off, "%s\"%s\"",
|
||||||
|
s ? "," : "", h);
|
||||||
|
}
|
||||||
|
snprintf(shex + off, sizeof shex - off, "]");
|
||||||
|
emit("solve", runtime, eff, runs, reps, shex);
|
||||||
|
free(runs);
|
||||||
|
equix_free(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Seed mode, two-phase so the timed region contains ONLY equix_verify:
|
||||||
|
* phase 1 (untimed): walk the SHA-256 chain, self-solving each challenge to
|
||||||
|
* collect (challenge, solution) pairs — the setup solve, which touches the
|
||||||
|
* ~1.8 MB solver table, is kept out of timing so it cannot pollute the cache
|
||||||
|
* the tiny verify reads from;
|
||||||
|
* phase 2 (timed): verify the collected pairs back-to-back.
|
||||||
|
* Uses a SOLVE context (it can verify too). Solution-less challenges are skipped. */
|
||||||
|
static void op_verify_seeded(const char *json, const char *runtime, uint64_t reps,
|
||||||
|
uint64_t warmup) {
|
||||||
|
uint8_t chal[MAX_CHALLENGE];
|
||||||
|
int clen;
|
||||||
|
(void)read_challenge_or_seed(json, "verify", runtime, chal, &clen);
|
||||||
|
|
||||||
|
const char *eff = "?", *err = NULL;
|
||||||
|
equix_ctx *ctx = alloc_ctx(EQUIX_CTX_SOLVE, runtime, &eff, &err);
|
||||||
|
if (!ctx)
|
||||||
|
fail("verify", runtime, err);
|
||||||
|
|
||||||
|
uint64_t want = warmup + reps;
|
||||||
|
uint8_t *chals = malloc(want * 32);
|
||||||
|
equix_solution *toks = malloc(want * sizeof(equix_solution));
|
||||||
|
equix_solution sols[EQUIX_MAX_SOLS];
|
||||||
|
|
||||||
|
/* Phase 1 — collect `want` valid (challenge, solution) pairs, untimed.
|
||||||
|
* Cap draws so solution-less challenges cannot loop forever. */
|
||||||
|
uint64_t got = 0, guard = 0, guard_max = want * 8 + 128;
|
||||||
|
while (got < want && guard++ < guard_max) {
|
||||||
|
if (equix_solve(ctx, chal, clen, sols) > 0) {
|
||||||
|
memcpy(chals + got * 32, chal, 32);
|
||||||
|
toks[got] = sols[0];
|
||||||
|
got++;
|
||||||
|
}
|
||||||
|
sha256(chal, 32, chal); /* advance the chain (untimed) */
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t warm = warmup < got ? warmup : got;
|
||||||
|
uint64_t timed = got - warm;
|
||||||
|
for (uint64_t w = 0; w < warm; w++)
|
||||||
|
(void)equix_verify(ctx, chals + w * 32, 32, &toks[w]);
|
||||||
|
|
||||||
|
run_t *runs = calloc(timed ? timed : 1, sizeof(run_t));
|
||||||
|
for (uint64_t i = 0; i < timed; i++) {
|
||||||
|
uint64_t idx = warm + i;
|
||||||
|
uint64_t t0 = now_ns();
|
||||||
|
equix_result r = equix_verify(ctx, chals + idx * 32, 32, &toks[idx]);
|
||||||
|
uint64_t t1 = now_ns();
|
||||||
|
runs[i].wall_ns = t1 - t0;
|
||||||
|
runs[i].solutions = (r == EQUIX_OK) ? 1 : 0;
|
||||||
|
runs[i].verify_result = verify_result_str(r);
|
||||||
|
}
|
||||||
|
emit("verify", runtime, eff, runs, timed, NULL);
|
||||||
|
free(runs);
|
||||||
|
free(chals);
|
||||||
|
free(toks);
|
||||||
|
equix_free(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void op_verify(const char *json, const char *runtime, uint64_t reps,
|
||||||
|
uint64_t warmup) {
|
||||||
|
char seed_probe[4] = {0};
|
||||||
|
if (jm_get_str(json, "challenge_seed_hex", seed_probe, sizeof seed_probe)) {
|
||||||
|
op_verify_seeded(json, runtime, reps, warmup);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
char chal_hex[2 * MAX_CHALLENGE + 1] = {0};
|
||||||
|
char sol_hex[64] = {0};
|
||||||
|
if (!jm_get_str(json, "challenge_hex", chal_hex, sizeof chal_hex))
|
||||||
|
fail("verify", runtime, "verify requires challenge_hex");
|
||||||
|
if (!jm_get_str(json, "solution_hex", sol_hex, sizeof sol_hex))
|
||||||
|
fail("verify", runtime, "verify requires solution_hex");
|
||||||
|
uint8_t chal[MAX_CHALLENGE];
|
||||||
|
int clen = hex_decode(chal_hex, chal, sizeof chal);
|
||||||
|
if (clen < 0)
|
||||||
|
fail("verify", runtime, "invalid challenge_hex");
|
||||||
|
uint8_t sb[16];
|
||||||
|
if (hex_decode(sol_hex, sb, sizeof sb) != 16)
|
||||||
|
fail("verify", runtime, "solution_hex must be 16 bytes");
|
||||||
|
equix_solution sol;
|
||||||
|
for (int i = 0; i < EQUIX_NUM_IDX; i++)
|
||||||
|
sol.idx[i] = (equix_idx)(sb[2 * i] | ((uint16_t)sb[2 * i + 1] << 8));
|
||||||
|
|
||||||
|
const char *eff = "?", *err = NULL;
|
||||||
|
equix_ctx *ctx = alloc_ctx(EQUIX_CTX_VERIFY, runtime, &eff, &err);
|
||||||
|
if (!ctx)
|
||||||
|
fail("verify", runtime, err);
|
||||||
|
|
||||||
|
for (uint64_t w = 0; w < warmup; w++)
|
||||||
|
(void)equix_verify(ctx, chal, clen, &sol);
|
||||||
|
|
||||||
|
run_t *runs = calloc(reps ? reps : 1, sizeof(run_t));
|
||||||
|
for (uint64_t i = 0; i < reps; i++) {
|
||||||
|
uint64_t t0 = now_ns();
|
||||||
|
equix_result r = equix_verify(ctx, chal, clen, &sol);
|
||||||
|
uint64_t t1 = now_ns();
|
||||||
|
runs[i].wall_ns = t1 - t0;
|
||||||
|
runs[i].solutions = (r == EQUIX_OK) ? 1 : 0;
|
||||||
|
runs[i].verify_result = verify_result_str(r);
|
||||||
|
}
|
||||||
|
emit("verify", runtime, eff, runs, reps, NULL);
|
||||||
|
free(runs);
|
||||||
|
equix_free(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Build challenge = base || little-endian(nonce, nonce_bytes) into buf. */
|
||||||
|
static int build_nonce_challenge(const uint8_t *base, int base_len,
|
||||||
|
uint64_t nonce, int nonce_bytes, uint8_t *buf) {
|
||||||
|
memcpy(buf, base, base_len);
|
||||||
|
for (int i = 0; i < nonce_bytes; i++)
|
||||||
|
buf[base_len + i] = (uint8_t)((nonce >> (8 * i)) & 0xff);
|
||||||
|
return base_len + nonce_bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void op_effort(const char *json, const char *runtime, uint64_t reps,
|
||||||
|
uint64_t warmup) {
|
||||||
|
char base_hex[2 * MAX_CHALLENGE + 1] = {0};
|
||||||
|
if (!jm_get_str(json, "challenge_base_hex", base_hex, sizeof base_hex))
|
||||||
|
fail("effort", runtime, "effort requires challenge_base_hex");
|
||||||
|
uint8_t base[MAX_CHALLENGE];
|
||||||
|
int base_len = hex_decode(base_hex, base, sizeof base);
|
||||||
|
if (base_len < 0)
|
||||||
|
fail("effort", runtime, "invalid challenge_base_hex");
|
||||||
|
|
||||||
|
uint64_t nonce_bytes = 8, nonce_start = 0, target = 1000, max_attempts = 5000000;
|
||||||
|
jm_get_u64(json, "nonce_bytes", &nonce_bytes);
|
||||||
|
jm_get_u64(json, "nonce_start", &nonce_start);
|
||||||
|
jm_get_u64(json, "target_effort", &target);
|
||||||
|
jm_get_u64(json, "max_attempts", &max_attempts);
|
||||||
|
if (nonce_bytes > 8 || (size_t)base_len + nonce_bytes > MAX_CHALLENGE)
|
||||||
|
fail("effort", runtime, "nonce_bytes out of range");
|
||||||
|
|
||||||
|
const char *eff = "?", *err = NULL;
|
||||||
|
equix_ctx *ctx = alloc_ctx(EQUIX_CTX_SOLVE, runtime, &eff, &err);
|
||||||
|
if (!ctx)
|
||||||
|
fail("effort", runtime, err);
|
||||||
|
|
||||||
|
equix_solution sols[EQUIX_MAX_SOLS];
|
||||||
|
uint8_t chal[MAX_CHALLENGE];
|
||||||
|
|
||||||
|
/* One search = one repetition; warmups run a full search but are discarded. */
|
||||||
|
for (uint64_t w = 0; w < warmup; w++) {
|
||||||
|
uint64_t nonce = nonce_start;
|
||||||
|
for (uint64_t a = 0; a < max_attempts; a++, nonce++) {
|
||||||
|
int clen = build_nonce_challenge(base, base_len, nonce, nonce_bytes, chal);
|
||||||
|
int n = equix_solve(ctx, chal, clen, sols);
|
||||||
|
int done = 0;
|
||||||
|
for (int s = 0; s < n; s++)
|
||||||
|
if (effort_of(chal, clen, &sols[s]) >= target) { done = 1; break; }
|
||||||
|
if (done) break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
run_t *runs = calloc(reps ? reps : 1, sizeof(run_t));
|
||||||
|
/* The winning token's wire bytes (nonce LE + 16-byte solution): reported so
|
||||||
|
* the harness can measure message sizes vs difficulty. */
|
||||||
|
equix_solution win_sol;
|
||||||
|
uint8_t win_nonce[8];
|
||||||
|
int have_token = 0;
|
||||||
|
for (uint64_t i = 0; i < reps; i++) {
|
||||||
|
uint64_t nonce = nonce_start;
|
||||||
|
uint64_t attempts = 0;
|
||||||
|
uint32_t best = 0;
|
||||||
|
uint64_t t0 = now_ns();
|
||||||
|
for (uint64_t a = 0; a < max_attempts; a++, nonce++) {
|
||||||
|
int clen = build_nonce_challenge(base, base_len, nonce, nonce_bytes, chal);
|
||||||
|
int n = equix_solve(ctx, chal, clen, sols);
|
||||||
|
attempts++;
|
||||||
|
int done = 0;
|
||||||
|
for (int s = 0; s < n; s++) {
|
||||||
|
uint32_t e = effort_of(chal, clen, &sols[s]);
|
||||||
|
if (e > best) best = e;
|
||||||
|
if (e >= target) {
|
||||||
|
if (!done) {
|
||||||
|
win_sol = sols[s];
|
||||||
|
memcpy(win_nonce, chal + base_len, nonce_bytes);
|
||||||
|
have_token = 1;
|
||||||
|
}
|
||||||
|
done = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (done) break;
|
||||||
|
}
|
||||||
|
uint64_t t1 = now_ns();
|
||||||
|
runs[i].wall_ns = t1 - t0;
|
||||||
|
runs[i].attempts = attempts;
|
||||||
|
runs[i].achieved_effort = best;
|
||||||
|
runs[i].solutions = (best >= target) ? 1 : 0;
|
||||||
|
}
|
||||||
|
if (have_token) {
|
||||||
|
static const char hx[] = "0123456789abcdef";
|
||||||
|
char shex[33], sjson[40], nhex[17];
|
||||||
|
solution_to_hex(&win_sol, shex);
|
||||||
|
snprintf(sjson, sizeof sjson, "[\"%s\"]", shex);
|
||||||
|
for (uint64_t b = 0; b < nonce_bytes; b++) {
|
||||||
|
nhex[2 * b] = hx[win_nonce[b] >> 4];
|
||||||
|
nhex[2 * b + 1] = hx[win_nonce[b] & 0xf];
|
||||||
|
}
|
||||||
|
nhex[2 * nonce_bytes] = '\0';
|
||||||
|
emit_ex("effort", runtime, eff, runs, reps, sjson, nhex);
|
||||||
|
} else {
|
||||||
|
emit("effort", runtime, eff, runs, reps, NULL);
|
||||||
|
}
|
||||||
|
free(runs);
|
||||||
|
equix_free(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Isolate program generation+compile (hashx_make) from execution (hashx_exec)
|
||||||
|
* using the hashx API directly -- libequix's public API cannot separate them.
|
||||||
|
* Each rep uses a distinct seed (base || LE(nonce_start+i)) so we sample the
|
||||||
|
* compile-time distribution across different generated programs. */
|
||||||
|
static void op_hashx_compile(const char *json, const char *runtime,
|
||||||
|
uint64_t reps, uint64_t warmup) {
|
||||||
|
char base_hex[2 * MAX_CHALLENGE + 1] = {0};
|
||||||
|
if (!jm_get_str(json, "challenge_base_hex", base_hex, sizeof base_hex) &&
|
||||||
|
!jm_get_str(json, "challenge_hex", base_hex, sizeof base_hex))
|
||||||
|
fail("hashx_compile", runtime, "hashx_compile requires a challenge");
|
||||||
|
uint8_t base[MAX_CHALLENGE];
|
||||||
|
int base_len = hex_decode(base_hex, base, sizeof base);
|
||||||
|
if (base_len < 0)
|
||||||
|
fail("hashx_compile", runtime, "invalid challenge");
|
||||||
|
uint64_t nonce_start = 0;
|
||||||
|
jm_get_u64(json, "nonce_start", &nonce_start);
|
||||||
|
|
||||||
|
hashx_type type;
|
||||||
|
const char *eff;
|
||||||
|
if (strcmp(runtime, "interpret") == 0) {
|
||||||
|
type = HASHX_INTERPRETED;
|
||||||
|
eff = "interpreted";
|
||||||
|
} else if (!JIT_SUPPORTED) {
|
||||||
|
/* Avoid the Apple Silicon JIT crash: force interpreter / clean error. */
|
||||||
|
if (strcmp(runtime, "must-compile") == 0)
|
||||||
|
fail("hashx_compile", runtime,
|
||||||
|
"must-compile requested but JIT not supported on this platform");
|
||||||
|
type = HASHX_INTERPRETED;
|
||||||
|
eff = "interpreted (fallback)";
|
||||||
|
} else {
|
||||||
|
type = HASHX_COMPILED;
|
||||||
|
eff = "compiled";
|
||||||
|
}
|
||||||
|
hashx_ctx *hx = hashx_alloc(type);
|
||||||
|
if (hx == HASHX_NOTSUPP) {
|
||||||
|
if (strcmp(runtime, "must-compile") == 0)
|
||||||
|
fail("hashx_compile", runtime, "compiler not supported");
|
||||||
|
hx = hashx_alloc(HASHX_INTERPRETED);
|
||||||
|
eff = "interpreted (fallback)";
|
||||||
|
}
|
||||||
|
if (hx == NULL)
|
||||||
|
fail("hashx_compile", runtime, "hashx_alloc failed");
|
||||||
|
|
||||||
|
uint8_t seed[MAX_CHALLENGE + 8];
|
||||||
|
uint8_t hout[HASHX_SIZE];
|
||||||
|
|
||||||
|
for (uint64_t w = 0; w < warmup; w++) {
|
||||||
|
int sl = build_nonce_challenge(base, base_len, nonce_start, 8, seed);
|
||||||
|
if (hashx_make(hx, seed, sl))
|
||||||
|
hashx_exec(hx, 0, hout);
|
||||||
|
}
|
||||||
|
|
||||||
|
run_t *runs = calloc(reps ? reps : 1, sizeof(run_t));
|
||||||
|
for (uint64_t i = 0; i < reps; i++) {
|
||||||
|
int sl = build_nonce_challenge(base, base_len, nonce_start + i, 8, seed);
|
||||||
|
uint64_t t0 = now_ns();
|
||||||
|
int made = hashx_make(hx, seed, sl);
|
||||||
|
uint64_t t1 = now_ns();
|
||||||
|
runs[i].compile_ns = t1 - t0;
|
||||||
|
if (made) {
|
||||||
|
/* Time a single hashx_exec as the per-hash execution cost. */
|
||||||
|
uint64_t e0 = now_ns();
|
||||||
|
hashx_exec(hx, 0, hout);
|
||||||
|
uint64_t e1 = now_ns();
|
||||||
|
runs[i].wall_ns = e1 - e0;
|
||||||
|
runs[i].solutions = 1; /* program generated successfully */
|
||||||
|
} else {
|
||||||
|
runs[i].wall_ns = 0;
|
||||||
|
runs[i].solutions = 0; /* rare invalid seed */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
emit("hashx_compile", runtime, eff, runs, reps, NULL);
|
||||||
|
free(runs);
|
||||||
|
hashx_free(hx);
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(void) {
|
||||||
|
char *json = read_all_stdin();
|
||||||
|
if (!json)
|
||||||
|
fail(NULL, NULL, "failed to read stdin");
|
||||||
|
|
||||||
|
char op[32] = "solve";
|
||||||
|
char runtime[32] = "try-compile";
|
||||||
|
jm_get_str(json, "operation", op, sizeof op);
|
||||||
|
jm_get_str(json, "runtime", runtime, sizeof runtime);
|
||||||
|
|
||||||
|
uint64_t reps = 10, warmup = 3;
|
||||||
|
jm_get_u64(json, "repetitions", &reps);
|
||||||
|
jm_get_u64(json, "warmup", &warmup);
|
||||||
|
if (reps == 0)
|
||||||
|
reps = 1;
|
||||||
|
|
||||||
|
if (strcmp(op, "solve") == 0)
|
||||||
|
op_solve(json, runtime, reps, warmup);
|
||||||
|
else if (strcmp(op, "verify") == 0)
|
||||||
|
op_verify(json, runtime, reps, warmup);
|
||||||
|
else if (strcmp(op, "effort") == 0)
|
||||||
|
op_effort(json, runtime, reps, warmup);
|
||||||
|
else if (strcmp(op, "hashx_compile") == 0)
|
||||||
|
op_hashx_compile(json, runtime, reps, warmup);
|
||||||
|
else
|
||||||
|
fail(op, runtime, "unknown operation");
|
||||||
|
|
||||||
|
free(json);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
88
tools/benchmarks/Equi-X/runners/c/json_min.c
Normal file
88
tools/benchmarks/Equi-X/runners/c/json_min.c
Normal file
@ -0,0 +1,88 @@
|
|||||||
|
#include "json_min.h"
|
||||||
|
|
||||||
|
#include <ctype.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
/* Return a pointer to the first non-space char of the value for `key`, or NULL
|
||||||
|
* if the key is absent / malformed. */
|
||||||
|
static const char *find_val(const char *json, const char *key) {
|
||||||
|
char pat[128];
|
||||||
|
int n = snprintf(pat, sizeof pat, "\"%s\"", key);
|
||||||
|
if (n <= 0 || (size_t)n >= sizeof pat)
|
||||||
|
return NULL;
|
||||||
|
const char *p = strstr(json, pat);
|
||||||
|
if (!p)
|
||||||
|
return NULL;
|
||||||
|
p += (size_t)n;
|
||||||
|
while (*p && *p != ':')
|
||||||
|
p++;
|
||||||
|
if (*p != ':')
|
||||||
|
return NULL;
|
||||||
|
p++;
|
||||||
|
while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r')
|
||||||
|
p++;
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
int jm_get_str(const char *json, const char *key, char *out, size_t outsz) {
|
||||||
|
const char *p = find_val(json, key);
|
||||||
|
if (!p || *p != '"')
|
||||||
|
return 0; /* absent or null/non-string */
|
||||||
|
p++;
|
||||||
|
size_t i = 0;
|
||||||
|
while (*p && *p != '"') {
|
||||||
|
char c = *p++;
|
||||||
|
if (c == '\\' && *p) {
|
||||||
|
char e = *p++;
|
||||||
|
switch (e) {
|
||||||
|
case 'n': c = '\n'; break;
|
||||||
|
case 't': c = '\t'; break;
|
||||||
|
case 'r': c = '\r'; break;
|
||||||
|
case '"': c = '"'; break;
|
||||||
|
case '\\': c = '\\'; break;
|
||||||
|
case '/': c = '/'; break;
|
||||||
|
default: c = e; break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (i + 1 < outsz)
|
||||||
|
out[i++] = c;
|
||||||
|
}
|
||||||
|
if (i < outsz)
|
||||||
|
out[i] = '\0';
|
||||||
|
else if (outsz)
|
||||||
|
out[outsz - 1] = '\0';
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
int jm_get_u64(const char *json, const char *key, uint64_t *out) {
|
||||||
|
const char *p = find_val(json, key);
|
||||||
|
if (!p || (!isdigit((unsigned char)*p) && *p != '+'))
|
||||||
|
return 0;
|
||||||
|
*out = strtoull(p, NULL, 10);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
int jm_get_i64(const char *json, const char *key, int64_t *out) {
|
||||||
|
const char *p = find_val(json, key);
|
||||||
|
if (!p || (!isdigit((unsigned char)*p) && *p != '-' && *p != '+'))
|
||||||
|
return 0;
|
||||||
|
*out = strtoll(p, NULL, 10);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
int jm_get_bool(const char *json, const char *key, int *out) {
|
||||||
|
const char *p = find_val(json, key);
|
||||||
|
if (!p)
|
||||||
|
return 0;
|
||||||
|
if (strncmp(p, "true", 4) == 0) {
|
||||||
|
*out = 1;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
if (strncmp(p, "false", 5) == 0) {
|
||||||
|
*out = 0;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
23
tools/benchmarks/Equi-X/runners/c/json_min.h
Normal file
23
tools/benchmarks/Equi-X/runners/c/json_min.h
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
/* Minimal JSON reader for the Equi-X runner protocol.
|
||||||
|
*
|
||||||
|
* The job-spec is a FLAT object of scalar fields (no nested objects/arrays on
|
||||||
|
* the input side), so a tiny key->value scanner is sufficient and avoids any
|
||||||
|
* external dependency. Result JSON is emitted by hand in equix_runner.c.
|
||||||
|
*
|
||||||
|
* Each getter searches for the exact quoted key ("key") which makes prefix
|
||||||
|
* collisions impossible (e.g. searching "nonce" never matches "nonce_bytes",
|
||||||
|
* because the char after the opening key is '_' not '"').
|
||||||
|
*/
|
||||||
|
#ifndef EQUIX_RUNNER_JSON_MIN_H
|
||||||
|
#define EQUIX_RUNNER_JSON_MIN_H
|
||||||
|
|
||||||
|
#include <stddef.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
/* All return 1 if the key was found (and, for typed getters, parsed), else 0. */
|
||||||
|
int jm_get_str(const char *json, const char *key, char *out, size_t outsz);
|
||||||
|
int jm_get_u64(const char *json, const char *key, uint64_t *out);
|
||||||
|
int jm_get_i64(const char *json, const char *key, int64_t *out);
|
||||||
|
int jm_get_bool(const char *json, const char *key, int *out);
|
||||||
|
|
||||||
|
#endif /* EQUIX_RUNNER_JSON_MIN_H */
|
||||||
75
tools/benchmarks/Equi-X/runners/c/sha256.c
Normal file
75
tools/benchmarks/Equi-X/runners/c/sha256.c
Normal file
@ -0,0 +1,75 @@
|
|||||||
|
/* Minimal standalone SHA-256 (FIPS 180-4). Verified against Python hashlib in
|
||||||
|
* the test suite. Off the hot path (challenge generation only). */
|
||||||
|
#include "sha256.h"
|
||||||
|
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
static uint32_t rotr(uint32_t x, uint32_t n) { return (x >> n) | (x << (32 - n)); }
|
||||||
|
|
||||||
|
static const uint32_t K[64] = {
|
||||||
|
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1,
|
||||||
|
0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
|
||||||
|
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
|
||||||
|
0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
||||||
|
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147,
|
||||||
|
0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
|
||||||
|
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
|
||||||
|
0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
||||||
|
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
|
||||||
|
0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
|
||||||
|
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};
|
||||||
|
|
||||||
|
static void sha256_block(uint32_t state[8], const uint8_t block[64]) {
|
||||||
|
uint32_t w[64];
|
||||||
|
for (int i = 0; i < 16; i++)
|
||||||
|
w[i] = ((uint32_t)block[i * 4] << 24) | ((uint32_t)block[i * 4 + 1] << 16) |
|
||||||
|
((uint32_t)block[i * 4 + 2] << 8) | (uint32_t)block[i * 4 + 3];
|
||||||
|
for (int i = 16; i < 64; i++) {
|
||||||
|
uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3);
|
||||||
|
uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10);
|
||||||
|
w[i] = w[i - 16] + s0 + w[i - 7] + s1;
|
||||||
|
}
|
||||||
|
uint32_t a = state[0], b = state[1], c = state[2], d = state[3];
|
||||||
|
uint32_t e = state[4], f = state[5], g = state[6], h = state[7];
|
||||||
|
for (int i = 0; i < 64; i++) {
|
||||||
|
uint32_t S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25);
|
||||||
|
uint32_t ch = (e & f) ^ (~e & g);
|
||||||
|
uint32_t t1 = h + S1 + ch + K[i] + w[i];
|
||||||
|
uint32_t S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22);
|
||||||
|
uint32_t maj = (a & b) ^ (a & c) ^ (b & c);
|
||||||
|
uint32_t t2 = S0 + maj;
|
||||||
|
h = g; g = f; f = e; e = d + t1;
|
||||||
|
d = c; c = b; b = a; a = t1 + t2;
|
||||||
|
}
|
||||||
|
state[0] += a; state[1] += b; state[2] += c; state[3] += d;
|
||||||
|
state[4] += e; state[5] += f; state[6] += g; state[7] += h;
|
||||||
|
}
|
||||||
|
|
||||||
|
void sha256(const uint8_t *data, size_t len, uint8_t out[32]) {
|
||||||
|
uint32_t state[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
|
||||||
|
0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19};
|
||||||
|
size_t full = len / 64;
|
||||||
|
for (size_t i = 0; i < full; i++)
|
||||||
|
sha256_block(state, data + i * 64);
|
||||||
|
|
||||||
|
/* Final block(s): remaining bytes, 0x80, zero pad, 64-bit big-endian length. */
|
||||||
|
uint8_t buf[128];
|
||||||
|
size_t rem = len - full * 64;
|
||||||
|
memset(buf, 0, sizeof buf);
|
||||||
|
memcpy(buf, data + full * 64, rem);
|
||||||
|
buf[rem] = 0x80;
|
||||||
|
size_t padlen = (rem < 56) ? 64 : 128;
|
||||||
|
uint64_t bitlen = (uint64_t)len * 8;
|
||||||
|
for (int i = 0; i < 8; i++)
|
||||||
|
buf[padlen - 1 - i] = (uint8_t)(bitlen >> (8 * i));
|
||||||
|
sha256_block(state, buf);
|
||||||
|
if (padlen == 128)
|
||||||
|
sha256_block(state, buf + 64);
|
||||||
|
|
||||||
|
for (int i = 0; i < 8; i++) {
|
||||||
|
out[i * 4] = (uint8_t)(state[i] >> 24);
|
||||||
|
out[i * 4 + 1] = (uint8_t)(state[i] >> 16);
|
||||||
|
out[i * 4 + 2] = (uint8_t)(state[i] >> 8);
|
||||||
|
out[i * 4 + 3] = (uint8_t)(state[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
13
tools/benchmarks/Equi-X/runners/c/sha256.h
Normal file
13
tools/benchmarks/Equi-X/runners/c/sha256.h
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
/* Minimal standalone SHA-256 (FIPS 180-4) for deriving per-rep challenges from
|
||||||
|
* a seed. Not on any hot path — used only to generate challenges between timed
|
||||||
|
* measurements, so plain portable C is fine. */
|
||||||
|
#ifndef EQUIX_RUNNER_SHA256_H
|
||||||
|
#define EQUIX_RUNNER_SHA256_H
|
||||||
|
|
||||||
|
#include <stddef.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
/* One-shot: hash `len` bytes of `data` into the 32-byte `out`. */
|
||||||
|
void sha256(const uint8_t *data, size_t len, uint8_t out[32]);
|
||||||
|
|
||||||
|
#endif /* EQUIX_RUNNER_SHA256_H */
|
||||||
35
tools/benchmarks/Equi-X/runners/c/timing.h
Normal file
35
tools/benchmarks/Equi-X/runners/c/timing.h
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
/* Monotonic timing + peak-RSS helpers for the Equi-X C runner. */
|
||||||
|
#ifndef EQUIX_RUNNER_TIMING_H
|
||||||
|
#define EQUIX_RUNNER_TIMING_H
|
||||||
|
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <time.h>
|
||||||
|
#include <sys/resource.h>
|
||||||
|
|
||||||
|
/* Monotonic wall-clock nanoseconds (immune to wall-clock adjustments).
|
||||||
|
* macOS: clock_gettime(CLOCK_MONOTONIC) only has microsecond granularity, which
|
||||||
|
* quantizes ~16 us operations (verify) by +/-3-6%; CLOCK_UPTIME_RAW via
|
||||||
|
* clock_gettime_nsec_np gives true nanosecond resolution. */
|
||||||
|
static inline uint64_t now_ns(void) {
|
||||||
|
#if defined(__APPLE__)
|
||||||
|
return clock_gettime_nsec_np(CLOCK_UPTIME_RAW);
|
||||||
|
#else
|
||||||
|
struct timespec ts;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||||
|
return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Whole-process peak resident set size, in KILOBYTES.
|
||||||
|
* ru_maxrss units differ by OS: Linux reports kilobytes, macOS/BSD report bytes. */
|
||||||
|
static inline long peak_rss_kb(void) {
|
||||||
|
struct rusage ru;
|
||||||
|
getrusage(RUSAGE_SELF, &ru);
|
||||||
|
#if defined(__APPLE__)
|
||||||
|
return ru.ru_maxrss / 1024; /* macOS: bytes -> KB */
|
||||||
|
#else
|
||||||
|
return ru.ru_maxrss; /* Linux: already KB */
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif /* EQUIX_RUNNER_TIMING_H */
|
||||||
391
tools/benchmarks/Equi-X/runners/rust/Cargo.lock
generated
Normal file
391
tools/benchmarks/Equi-X/runners/rust/Cargo.lock
generated
Normal file
@ -0,0 +1,391 @@
|
|||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "arrayvec"
|
||||||
|
version = "0.7.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "autocfg"
|
||||||
|
version = "1.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bitflags"
|
||||||
|
version = "2.13.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "blake2"
|
||||||
|
version = "0.10.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
|
||||||
|
dependencies = [
|
||||||
|
"digest",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "block-buffer"
|
||||||
|
version = "0.10.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||||
|
dependencies = [
|
||||||
|
"generic-array",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "byteorder"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cfg-if"
|
||||||
|
version = "1.0.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cpufeatures"
|
||||||
|
version = "0.2.17"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crypto-common"
|
||||||
|
version = "0.1.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||||
|
dependencies = [
|
||||||
|
"generic-array",
|
||||||
|
"typenum",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "digest"
|
||||||
|
version = "0.10.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||||
|
dependencies = [
|
||||||
|
"block-buffer",
|
||||||
|
"crypto-common",
|
||||||
|
"subtle",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dynasm"
|
||||||
|
version = "5.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fd358e74d2f8d71a11b7a2c51c67ad5df3de06003b0596875e47bc09126c894e"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"byteorder",
|
||||||
|
"lazy_static",
|
||||||
|
"proc-macro-error3",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dynasmrt"
|
||||||
|
version = "5.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6efe03f6bd356ae85eeea7ee14c2bf54e664a949d089d238364b5a99afbc4116"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
"dynasm",
|
||||||
|
"fnv",
|
||||||
|
"memmap2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "equix"
|
||||||
|
version = "0.7.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "207dd01b4071086d80c21d3b39f029b5ebb658f9910c0348c465ca1dd5346d90"
|
||||||
|
dependencies = [
|
||||||
|
"arrayvec",
|
||||||
|
"hashx",
|
||||||
|
"num-traits",
|
||||||
|
"thiserror",
|
||||||
|
"visibility",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "equix-runner"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"blake2",
|
||||||
|
"equix",
|
||||||
|
"hashx",
|
||||||
|
"libc",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"sha2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fixed-capacity-vec"
|
||||||
|
version = "1.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6b31a14f5ee08ed1a40e1252b35af18bed062e3f39b69aab34decde36bc43e40"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fnv"
|
||||||
|
version = "1.0.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "generic-array"
|
||||||
|
version = "0.14.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||||
|
dependencies = [
|
||||||
|
"typenum",
|
||||||
|
"version_check",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hashx"
|
||||||
|
version = "0.9.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2bc106a3b059d84aad44624a2b3b10e173584249e413596a9b6ed4a4a2542e41"
|
||||||
|
dependencies = [
|
||||||
|
"arrayvec",
|
||||||
|
"blake2",
|
||||||
|
"dynasmrt",
|
||||||
|
"fixed-capacity-vec",
|
||||||
|
"hex",
|
||||||
|
"rand_core",
|
||||||
|
"thiserror",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hex"
|
||||||
|
version = "0.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "itoa"
|
||||||
|
version = "1.0.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lazy_static"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libc"
|
||||||
|
version = "0.2.186"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memchr"
|
||||||
|
version = "2.8.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memmap2"
|
||||||
|
version = "0.9.11"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-traits"
|
||||||
|
version = "0.2.19"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro-error-attr3"
|
||||||
|
version = "3.0.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "be5bfc63c4dc85083c9daaf7112d0261701d4058677c3bff7f2afc44e30ef3e1"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro-error3"
|
||||||
|
version = "3.0.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "dd0d42490f6b7b143eef32b9e3522e42bf25dadc02c69ed72236f80adb949b5c"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro-error-attr3",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro2"
|
||||||
|
version = "1.0.106"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quote"
|
||||||
|
version = "1.0.46"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand_core"
|
||||||
|
version = "0.10.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde"
|
||||||
|
version = "1.0.228"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||||
|
dependencies = [
|
||||||
|
"serde_core",
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_core"
|
||||||
|
version = "1.0.228"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||||
|
dependencies = [
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_derive"
|
||||||
|
version = "1.0.228"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_json"
|
||||||
|
version = "1.0.150"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
||||||
|
dependencies = [
|
||||||
|
"itoa",
|
||||||
|
"memchr",
|
||||||
|
"serde",
|
||||||
|
"serde_core",
|
||||||
|
"zmij",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sha2"
|
||||||
|
version = "0.10.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"cpufeatures",
|
||||||
|
"digest",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "subtle"
|
||||||
|
version = "2.6.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn"
|
||||||
|
version = "2.0.119"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "thiserror"
|
||||||
|
version = "2.0.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
|
||||||
|
dependencies = [
|
||||||
|
"thiserror-impl",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "thiserror-impl"
|
||||||
|
version = "2.0.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "typenum"
|
||||||
|
version = "1.20.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-ident"
|
||||||
|
version = "1.0.24"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "version_check"
|
||||||
|
version = "0.9.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "visibility"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zmij"
|
||||||
|
version = "1.0.23"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||||
26
tools/benchmarks/Equi-X/runners/rust/Cargo.toml
Normal file
26
tools/benchmarks/Equi-X/runners/rust/Cargo.toml
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
[package]
|
||||||
|
name = "equix-runner"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2021"
|
||||||
|
description = "Equi-X benchmark runner (Rust / arti equix crate) speaking the JSON-over-stdio protocol"
|
||||||
|
license = "MIT"
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "equix_runner"
|
||||||
|
path = "src/main.rs"
|
||||||
|
|
||||||
|
# Exact version pins + committed Cargo.lock + `--locked` freeze the whole tree.
|
||||||
|
[dependencies]
|
||||||
|
equix = "=0.7.0"
|
||||||
|
hashx = "=0.9.0"
|
||||||
|
serde = { version = "1", features = ["derive"] }
|
||||||
|
serde_json = "1"
|
||||||
|
blake2 = "0.10"
|
||||||
|
sha2 = "0.10"
|
||||||
|
|
||||||
|
# macOS has no /proc; getrusage (via libc) provides peak RSS there.
|
||||||
|
[target.'cfg(target_os = "macos")'.dependencies]
|
||||||
|
libc = "0.2"
|
||||||
|
|
||||||
|
[profile.release]
|
||||||
|
opt-level = 3
|
||||||
25
tools/benchmarks/Equi-X/runners/rust/src/effort.rs
Normal file
25
tools/benchmarks/Equi-X/runners/rust/src/effort.rs
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
//! Tor proposal-327 style effort computation.
|
||||||
|
//!
|
||||||
|
//! MUST be byte-identical to the C runner (runners/c/effort.c): standard
|
||||||
|
//! BLAKE2b-256 over `challenge || solution_bytes`, first 32 bits big-endian as
|
||||||
|
//! `hash32`, achieved effort = floor((2^32-1) / hash32). The Python cross-check
|
||||||
|
//! asserts both runners agree on a fixed (challenge, solution).
|
||||||
|
|
||||||
|
use blake2::digest::consts::U32;
|
||||||
|
use blake2::{Blake2b, Digest};
|
||||||
|
|
||||||
|
type Blake2b256 = Blake2b<U32>;
|
||||||
|
|
||||||
|
/// Achieved effort of `solution_bytes` (16-byte packed form) for `challenge`.
|
||||||
|
pub fn effort_of(challenge: &[u8], solution_bytes: &[u8]) -> u32 {
|
||||||
|
let mut h = Blake2b256::new();
|
||||||
|
h.update(challenge);
|
||||||
|
h.update(solution_bytes);
|
||||||
|
let out = h.finalize();
|
||||||
|
let hash32 = u32::from_be_bytes([out[0], out[1], out[2], out[3]]);
|
||||||
|
if hash32 == 0 {
|
||||||
|
u32::MAX
|
||||||
|
} else {
|
||||||
|
u32::MAX / hash32
|
||||||
|
}
|
||||||
|
}
|
||||||
83
tools/benchmarks/Equi-X/runners/rust/src/job.rs
Normal file
83
tools/benchmarks/Equi-X/runners/rust/src/job.rs
Normal file
@ -0,0 +1,83 @@
|
|||||||
|
//! Protocol structs: the job-spec read from stdin and the result written to
|
||||||
|
//! stdout. Mirrors runners/c/equix_runner.c and adapters/README.md.
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct Job {
|
||||||
|
#[serde(default)]
|
||||||
|
pub operation: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub runtime: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub challenge_hex: Option<String>,
|
||||||
|
/// When set, each rep derives a fresh challenge by SHA-256-chaining this
|
||||||
|
/// seed (challenge generation is excluded from every timed region).
|
||||||
|
#[serde(default)]
|
||||||
|
pub challenge_seed_hex: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub challenge_base_hex: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub solution_hex: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub nonce_bytes: Option<u64>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub nonce_start: Option<u64>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub target_effort: Option<u64>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub max_attempts: Option<u64>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub repetitions: Option<u64>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub warmup: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct RunOut {
|
||||||
|
pub index: usize,
|
||||||
|
pub wall_ns: u64,
|
||||||
|
pub solutions: i64,
|
||||||
|
pub compile_ns: u64,
|
||||||
|
pub attempts: u64,
|
||||||
|
pub achieved_effort: u32,
|
||||||
|
pub verify_result: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct ImplInfo {
|
||||||
|
pub name: String,
|
||||||
|
pub version: String,
|
||||||
|
pub commit: String,
|
||||||
|
pub runtime_effective: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct EnvInfo {
|
||||||
|
pub os: String,
|
||||||
|
pub compiler: String,
|
||||||
|
pub cpu: String,
|
||||||
|
pub arch: String,
|
||||||
|
pub device: String,
|
||||||
|
pub os_version: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct Output {
|
||||||
|
pub schema_version: u32,
|
||||||
|
pub ok: bool,
|
||||||
|
#[serde(rename = "impl")]
|
||||||
|
pub impl_info: ImplInfo,
|
||||||
|
pub operation: String,
|
||||||
|
pub runtime_requested: String,
|
||||||
|
pub runtime_effective: Option<String>,
|
||||||
|
pub env: EnvInfo,
|
||||||
|
pub runs: Vec<RunOut>,
|
||||||
|
pub solutions_hex: Option<Vec<String>>,
|
||||||
|
/// effort op only: the wire bytes of the winning token's nonce (LE,
|
||||||
|
/// exactly `nonce_bytes` long) — lets the harness measure message sizes.
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub winning_nonce_hex: Option<String>,
|
||||||
|
pub peak_rss_kb: i64,
|
||||||
|
pub error: Option<String>,
|
||||||
|
}
|
||||||
590
tools/benchmarks/Equi-X/runners/rust/src/main.rs
Normal file
590
tools/benchmarks/Equi-X/runners/rust/src/main.rs
Normal file
@ -0,0 +1,590 @@
|
|||||||
|
//! Equi-X Rust benchmark runner (arti `equix` + `hashx` crates).
|
||||||
|
//!
|
||||||
|
//! Reads one job-spec JSON on stdin, runs the requested operation, writes one
|
||||||
|
//! result JSON on stdout. Mirrors the C runner's protocol exactly so the Python
|
||||||
|
//! harness can compare them cell-for-cell.
|
||||||
|
|
||||||
|
mod effort;
|
||||||
|
mod job;
|
||||||
|
|
||||||
|
use std::io::Read;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use equix::{EquiXBuilder, RuntimeOption, SolverMemory};
|
||||||
|
|
||||||
|
use job::{EnvInfo, ImplInfo, Job, Output, RunOut};
|
||||||
|
|
||||||
|
const MAX_CHALLENGE: usize = 256;
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
fn peak_rss_kb() -> i64 {
|
||||||
|
std::fs::read_to_string("/proc/self/status")
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| {
|
||||||
|
s.lines()
|
||||||
|
.find(|l| l.starts_with("VmHWM:"))
|
||||||
|
.and_then(|l| l.split_whitespace().nth(1))
|
||||||
|
.and_then(|v| v.parse().ok())
|
||||||
|
})
|
||||||
|
.unwrap_or(-1)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
fn peak_rss_kb() -> i64 {
|
||||||
|
// macOS has no /proc; getrusage.ru_maxrss is in BYTES here (Linux uses KB).
|
||||||
|
unsafe {
|
||||||
|
let mut ru: libc::rusage = std::mem::zeroed();
|
||||||
|
if libc::getrusage(libc::RUSAGE_SELF, &mut ru) == 0 {
|
||||||
|
(ru.ru_maxrss as i64) / 1024
|
||||||
|
} else {
|
||||||
|
-1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||||
|
fn peak_rss_kb() -> i64 {
|
||||||
|
-1
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hex_decode(s: &str) -> Option<Vec<u8>> {
|
||||||
|
if s.len() % 2 != 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
(0..s.len() / 2)
|
||||||
|
.map(|i| u8::from_str_radix(&s[2 * i..2 * i + 2], 16).ok())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hex_encode(b: &[u8]) -> String {
|
||||||
|
let mut o = String::with_capacity(b.len() * 2);
|
||||||
|
for x in b {
|
||||||
|
o.push_str(&format!("{:02x}", x));
|
||||||
|
}
|
||||||
|
o
|
||||||
|
}
|
||||||
|
|
||||||
|
fn runtime_option(s: &str) -> RuntimeOption {
|
||||||
|
match s {
|
||||||
|
"interpret" => RuntimeOption::InterpretOnly,
|
||||||
|
"must-compile" => RuntimeOption::CompileOnly,
|
||||||
|
_ => RuntimeOption::TryCompile,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn eff_str(runtime_dbg: &str, requested: &str) -> String {
|
||||||
|
let compiled = runtime_dbg.to_lowercase().contains("compil");
|
||||||
|
if compiled {
|
||||||
|
"compiled".to_string()
|
||||||
|
} else if requested == "try-compile" {
|
||||||
|
"interpreted (fallback)".to_string()
|
||||||
|
} else {
|
||||||
|
"interpreted".to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn impl_info(runtime_effective: Option<String>) -> ImplInfo {
|
||||||
|
ImplInfo {
|
||||||
|
name: "equix-rust".to_string(),
|
||||||
|
version: std::env::var("EQUIX_RUST_VERSION").unwrap_or_else(|_| "0.7.0".to_string()),
|
||||||
|
commit: std::env::var("EQUIX_RUST_COMMIT").unwrap_or_else(|_| "crate-0.7.0".to_string()),
|
||||||
|
runtime_effective,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
fn cpu_model() -> String {
|
||||||
|
// Priority order works across arches: "model name" (x86), "Model" (Raspberry
|
||||||
|
// Pi board), "Hardware" (older ARM), "cpu model" (others).
|
||||||
|
let text = std::fs::read_to_string("/proc/cpuinfo").unwrap_or_default();
|
||||||
|
for field in ["model name", "Model", "Hardware", "cpu model"] {
|
||||||
|
for line in text.lines() {
|
||||||
|
if let Some((k, v)) = line.split_once(':') {
|
||||||
|
if k.trim() == field && !v.trim().is_empty() {
|
||||||
|
return v.trim().to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"unknown".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(target_os = "linux"))]
|
||||||
|
fn cpu_model() -> String {
|
||||||
|
// macOS (and other non-Linux): no /proc. `sysctl` reports the CPU brand on
|
||||||
|
// both Intel and Apple Silicon (e.g. "Apple M2").
|
||||||
|
std::process::Command::new("sysctl")
|
||||||
|
.args(["-n", "machdep.cpu.brand_string"])
|
||||||
|
.output()
|
||||||
|
.ok()
|
||||||
|
.and_then(|o| String::from_utf8(o.stdout).ok())
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.unwrap_or_else(|| "unknown".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
fn os_version() -> String {
|
||||||
|
std::fs::read_to_string("/proc/sys/kernel/osrelease")
|
||||||
|
.ok()
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.unwrap_or_else(|| "unknown".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(target_os = "linux"))]
|
||||||
|
fn os_version() -> String {
|
||||||
|
// macOS/BSD: kernel release via `uname -r` (e.g. Darwin "23.5.0").
|
||||||
|
std::process::Command::new("uname")
|
||||||
|
.arg("-r")
|
||||||
|
.output()
|
||||||
|
.ok()
|
||||||
|
.and_then(|o| String::from_utf8(o.stdout).ok())
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.unwrap_or_else(|| "unknown".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn env_info() -> EnvInfo {
|
||||||
|
EnvInfo {
|
||||||
|
os: std::env::consts::OS.to_string(), // "linux", "macos", ...
|
||||||
|
compiler: std::env::var("EQUIX_RUST_RUSTC").unwrap_or_else(|_| "rustc".to_string()),
|
||||||
|
cpu: cpu_model(),
|
||||||
|
arch: std::env::consts::ARCH.to_string(),
|
||||||
|
device: "cpu".to_string(),
|
||||||
|
os_version: os_version(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fail(op: &str, req: &str, msg: &str) -> ! {
|
||||||
|
let out = Output {
|
||||||
|
schema_version: 1,
|
||||||
|
ok: false,
|
||||||
|
impl_info: impl_info(None),
|
||||||
|
operation: op.to_string(),
|
||||||
|
runtime_requested: req.to_string(),
|
||||||
|
runtime_effective: None,
|
||||||
|
env: env_info(),
|
||||||
|
runs: vec![],
|
||||||
|
solutions_hex: None,
|
||||||
|
winning_nonce_hex: None,
|
||||||
|
peak_rss_kb: peak_rss_kb(),
|
||||||
|
error: Some(msg.to_string()),
|
||||||
|
};
|
||||||
|
println!("{}", serde_json::to_string(&out).unwrap());
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn emit(
|
||||||
|
op: &str,
|
||||||
|
req: &str,
|
||||||
|
eff: &str,
|
||||||
|
runs: Vec<RunOut>,
|
||||||
|
solutions_hex: Option<Vec<String>>,
|
||||||
|
) {
|
||||||
|
emit_with_nonce(op, req, eff, runs, solutions_hex, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn emit_with_nonce(
|
||||||
|
op: &str,
|
||||||
|
req: &str,
|
||||||
|
eff: &str,
|
||||||
|
runs: Vec<RunOut>,
|
||||||
|
solutions_hex: Option<Vec<String>>,
|
||||||
|
winning_nonce_hex: Option<String>,
|
||||||
|
) {
|
||||||
|
let out = Output {
|
||||||
|
schema_version: 1,
|
||||||
|
ok: true,
|
||||||
|
impl_info: impl_info(Some(eff.to_string())),
|
||||||
|
operation: op.to_string(),
|
||||||
|
runtime_requested: req.to_string(),
|
||||||
|
runtime_effective: Some(eff.to_string()),
|
||||||
|
env: env_info(),
|
||||||
|
runs,
|
||||||
|
solutions_hex,
|
||||||
|
winning_nonce_hex,
|
||||||
|
peak_rss_kb: peak_rss_kb(),
|
||||||
|
error: None,
|
||||||
|
};
|
||||||
|
println!("{}", serde_json::to_string(&out).unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Probe-build to discover the effective runtime (and surface hard failures).
|
||||||
|
fn effective_runtime(
|
||||||
|
builder: &EquiXBuilder,
|
||||||
|
challenge: &[u8],
|
||||||
|
req: &str,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
match builder.build(challenge) {
|
||||||
|
Ok(eq) => Ok(eff_str(&format!("{:?}", eq.runtime()), req)),
|
||||||
|
Err(e) => Err(format!("{:?}", e)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// SHA-256 of `data`. Used only to derive per-rep challenges from a seed
|
||||||
|
/// (between timed measurements) — never on a timed path.
|
||||||
|
fn sha256(data: &[u8]) -> [u8; 32] {
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
let mut h = Sha256::new();
|
||||||
|
h.update(data);
|
||||||
|
h.finalize().into()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_nonce_challenge(base: &[u8], nonce: u64, nonce_bytes: usize) -> Vec<u8> {
|
||||||
|
let mut c = Vec::with_capacity(base.len() + nonce_bytes);
|
||||||
|
c.extend_from_slice(base);
|
||||||
|
for i in 0..nonce_bytes {
|
||||||
|
c.push(((nonce >> (8 * i)) & 0xff) as u8);
|
||||||
|
}
|
||||||
|
c
|
||||||
|
}
|
||||||
|
|
||||||
|
/// First challenge for a solve/verify op: either the fixed `challenge_hex`, or
|
||||||
|
/// (seed mode) SHA-256(seed). Returns (challenge_bytes, seeded?).
|
||||||
|
fn first_challenge(job: &Job, op: &str, req: &str) -> (Vec<u8>, bool) {
|
||||||
|
if let Some(seed_hex) = job.challenge_seed_hex.as_ref() {
|
||||||
|
let seed = hex_decode(seed_hex)
|
||||||
|
.unwrap_or_else(|| fail(op, req, "invalid challenge_seed_hex"));
|
||||||
|
(sha256(&seed).to_vec(), true)
|
||||||
|
} else {
|
||||||
|
let chex = job
|
||||||
|
.challenge_hex
|
||||||
|
.as_ref()
|
||||||
|
.unwrap_or_else(|| fail(op, req, "requires challenge_hex or challenge_seed_hex"));
|
||||||
|
(hex_decode(chex).unwrap_or_else(|| fail(op, req, "invalid challenge_hex")), false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn op_solve(job: &Job, req: &str, reps: u64, warmup: u64) {
|
||||||
|
let (mut challenge, seeded) = first_challenge(job, "solve", req);
|
||||||
|
|
||||||
|
let mut builder = EquiXBuilder::new();
|
||||||
|
builder.runtime(runtime_option(req));
|
||||||
|
let eff = match effective_runtime(&builder, &challenge, req) {
|
||||||
|
Ok(e) => e,
|
||||||
|
Err(e) => fail("solve", req, &e),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut mem = SolverMemory::new();
|
||||||
|
for _ in 0..warmup {
|
||||||
|
if let Ok(eq) = builder.build(&challenge) {
|
||||||
|
let _ = eq.solve_with_memory(&mut mem);
|
||||||
|
}
|
||||||
|
if seeded {
|
||||||
|
challenge = sha256(&challenge).to_vec(); // advance the chain (untimed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut runs = Vec::with_capacity(reps as usize);
|
||||||
|
let mut last_sols: Vec<String> = vec![];
|
||||||
|
for i in 0..reps {
|
||||||
|
let t0 = Instant::now();
|
||||||
|
let arr = match builder.build(&challenge) {
|
||||||
|
Ok(eq) => eq.solve_with_memory(&mut mem),
|
||||||
|
Err(_) => Default::default(),
|
||||||
|
};
|
||||||
|
let ns = t0.elapsed().as_nanos() as u64;
|
||||||
|
if seeded {
|
||||||
|
challenge = sha256(&challenge).to_vec(); // next challenge AFTER stopping the timer
|
||||||
|
}
|
||||||
|
if i + 1 == reps {
|
||||||
|
last_sols = arr.iter().map(|s| hex_encode(&s.to_bytes())).collect();
|
||||||
|
}
|
||||||
|
runs.push(RunOut {
|
||||||
|
index: i as usize,
|
||||||
|
wall_ns: ns,
|
||||||
|
solutions: arr.len() as i64,
|
||||||
|
compile_ns: 0,
|
||||||
|
attempts: 0,
|
||||||
|
achieved_effort: 0,
|
||||||
|
verify_result: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
emit("solve", req, &eff, runs, Some(last_sols));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Seed mode, two-phase so the timed region contains ONLY verify_bytes:
|
||||||
|
/// phase 1 (untimed) walks the SHA-256 chain, self-solving each challenge to
|
||||||
|
/// collect (challenge, solution) pairs — keeping the ~1.8 MB solver pass out of
|
||||||
|
/// timing so it cannot pollute the cache the tiny verify reads from; phase 2
|
||||||
|
/// (timed) verifies the collected pairs back-to-back. Solution-less skipped.
|
||||||
|
fn op_verify_seeded(job: &Job, req: &str, reps: u64, warmup: u64) {
|
||||||
|
let (mut challenge, _) = first_challenge(job, "verify", req);
|
||||||
|
let mut builder = EquiXBuilder::new();
|
||||||
|
builder.runtime(runtime_option(req));
|
||||||
|
let eff = match effective_runtime(&builder, &challenge, req) {
|
||||||
|
Ok(e) => e,
|
||||||
|
Err(e) => fail("verify", req, &e),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut mem = SolverMemory::new();
|
||||||
|
let want = warmup + reps;
|
||||||
|
let guard_max = want * 8 + 128;
|
||||||
|
// Phase 1: collect valid (challenge, solution) pairs, untimed.
|
||||||
|
let mut pairs: Vec<(Vec<u8>, [u8; 16])> = Vec::with_capacity(want as usize);
|
||||||
|
let mut guard = 0u64;
|
||||||
|
while (pairs.len() as u64) < want && guard < guard_max {
|
||||||
|
let sol = match builder.build(&challenge) {
|
||||||
|
Ok(eq) => eq.solve_with_memory(&mut mem).iter().next().map(|s| s.to_bytes()),
|
||||||
|
Err(_) => None,
|
||||||
|
};
|
||||||
|
if let Some(sb) = sol {
|
||||||
|
pairs.push((challenge.clone(), sb));
|
||||||
|
}
|
||||||
|
challenge = sha256(&challenge).to_vec(); // advance (untimed)
|
||||||
|
guard += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
let warm = (warmup as usize).min(pairs.len());
|
||||||
|
for (c, sb) in &pairs[..warm] {
|
||||||
|
let _ = builder.verify_bytes(c, sb);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut runs = Vec::with_capacity(pairs.len() - warm);
|
||||||
|
for (c, sb) in &pairs[warm..] {
|
||||||
|
let t0 = Instant::now();
|
||||||
|
let r = builder.verify_bytes(c, sb);
|
||||||
|
let ns = t0.elapsed().as_nanos() as u64;
|
||||||
|
let (vr, ok) = match &r {
|
||||||
|
Ok(_) => ("OK".to_string(), 1),
|
||||||
|
Err(e) => (format!("{:?}", e), 0),
|
||||||
|
};
|
||||||
|
runs.push(RunOut {
|
||||||
|
index: runs.len(),
|
||||||
|
wall_ns: ns,
|
||||||
|
solutions: ok,
|
||||||
|
compile_ns: 0,
|
||||||
|
attempts: 0,
|
||||||
|
achieved_effort: 0,
|
||||||
|
verify_result: Some(vr),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
emit("verify", req, &eff, runs, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn op_verify(job: &Job, req: &str, reps: u64, warmup: u64) {
|
||||||
|
if job.challenge_seed_hex.is_some() {
|
||||||
|
return op_verify_seeded(job, req, reps, warmup);
|
||||||
|
}
|
||||||
|
let chex = job
|
||||||
|
.challenge_hex
|
||||||
|
.as_ref()
|
||||||
|
.unwrap_or_else(|| fail("verify", req, "verify requires challenge_hex"));
|
||||||
|
let shex = job
|
||||||
|
.solution_hex
|
||||||
|
.as_ref()
|
||||||
|
.unwrap_or_else(|| fail("verify", req, "verify requires solution_hex"));
|
||||||
|
let challenge =
|
||||||
|
hex_decode(chex).unwrap_or_else(|| fail("verify", req, "invalid challenge_hex"));
|
||||||
|
let sb = hex_decode(shex).unwrap_or_else(|| fail("verify", req, "invalid solution_hex"));
|
||||||
|
if sb.len() != 16 {
|
||||||
|
fail("verify", req, "solution_hex must be 16 bytes");
|
||||||
|
}
|
||||||
|
let mut sol_bytes = [0u8; 16];
|
||||||
|
sol_bytes.copy_from_slice(&sb);
|
||||||
|
|
||||||
|
let mut builder = EquiXBuilder::new();
|
||||||
|
builder.runtime(runtime_option(req));
|
||||||
|
let eff = match effective_runtime(&builder, &challenge, req) {
|
||||||
|
Ok(e) => e,
|
||||||
|
Err(e) => fail("verify", req, &e),
|
||||||
|
};
|
||||||
|
|
||||||
|
for _ in 0..warmup {
|
||||||
|
let _ = builder.verify_bytes(&challenge, &sol_bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut runs = Vec::with_capacity(reps as usize);
|
||||||
|
for i in 0..reps {
|
||||||
|
let t0 = Instant::now();
|
||||||
|
let r = builder.verify_bytes(&challenge, &sol_bytes);
|
||||||
|
let ns = t0.elapsed().as_nanos() as u64;
|
||||||
|
let (vr, ok) = match &r {
|
||||||
|
Ok(_) => ("OK".to_string(), 1),
|
||||||
|
Err(e) => (format!("{:?}", e), 0),
|
||||||
|
};
|
||||||
|
runs.push(RunOut {
|
||||||
|
index: i as usize,
|
||||||
|
wall_ns: ns,
|
||||||
|
solutions: ok,
|
||||||
|
compile_ns: 0,
|
||||||
|
attempts: 0,
|
||||||
|
achieved_effort: 0,
|
||||||
|
verify_result: Some(vr),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
emit("verify", req, &eff, runs, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn op_effort(job: &Job, req: &str, reps: u64, warmup: u64) {
|
||||||
|
let bhex = job
|
||||||
|
.challenge_base_hex
|
||||||
|
.as_ref()
|
||||||
|
.unwrap_or_else(|| fail("effort", req, "effort requires challenge_base_hex"));
|
||||||
|
let base = hex_decode(bhex).unwrap_or_else(|| fail("effort", req, "invalid challenge_base_hex"));
|
||||||
|
let nonce_bytes = job.nonce_bytes.unwrap_or(8) as usize;
|
||||||
|
let nonce_start = job.nonce_start.unwrap_or(0);
|
||||||
|
let target = job.target_effort.unwrap_or(1000) as u32;
|
||||||
|
let max_attempts = job.max_attempts.unwrap_or(5_000_000);
|
||||||
|
if nonce_bytes > 8 || base.len() + nonce_bytes > MAX_CHALLENGE {
|
||||||
|
fail("effort", req, "nonce_bytes out of range");
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut builder = EquiXBuilder::new();
|
||||||
|
builder.runtime(runtime_option(req));
|
||||||
|
let eff = match effective_runtime(&builder, &base, req) {
|
||||||
|
Ok(e) => e,
|
||||||
|
Err(e) => fail("effort", req, &e),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut mem = SolverMemory::new();
|
||||||
|
// (attempts, best effort, winning token bytes: (nonce wire bytes, solution))
|
||||||
|
let search = |mem: &mut SolverMemory| -> (u64, u32, Option<(Vec<u8>, Vec<u8>)>) {
|
||||||
|
let mut nonce = nonce_start;
|
||||||
|
let mut attempts = 0u64;
|
||||||
|
let mut best = 0u32;
|
||||||
|
let mut token: Option<(Vec<u8>, Vec<u8>)> = None;
|
||||||
|
for _ in 0..max_attempts {
|
||||||
|
let chal = build_nonce_challenge(&base, nonce, nonce_bytes);
|
||||||
|
let arr = match builder.build(&chal) {
|
||||||
|
Ok(eq) => eq.solve_with_memory(mem),
|
||||||
|
Err(_) => Default::default(),
|
||||||
|
};
|
||||||
|
attempts += 1;
|
||||||
|
let mut done = false;
|
||||||
|
for s in arr.iter() {
|
||||||
|
let e = effort::effort_of(&chal, &s.to_bytes());
|
||||||
|
if e > best {
|
||||||
|
best = e;
|
||||||
|
}
|
||||||
|
if e >= target && !done {
|
||||||
|
// The token as it would go on the wire: the nonce's
|
||||||
|
// nonce_bytes-long LE encoding + the 16-byte solution.
|
||||||
|
token = Some((chal[base.len()..].to_vec(), s.to_bytes().to_vec()));
|
||||||
|
done = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if done {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
nonce = nonce.wrapping_add(1);
|
||||||
|
}
|
||||||
|
(attempts, best, token)
|
||||||
|
};
|
||||||
|
|
||||||
|
for _ in 0..warmup {
|
||||||
|
let _ = search(&mut mem);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut runs = Vec::with_capacity(reps as usize);
|
||||||
|
let mut winning: Option<(Vec<u8>, Vec<u8>)> = None;
|
||||||
|
for i in 0..reps {
|
||||||
|
let t0 = Instant::now();
|
||||||
|
let (attempts, best, token) = search(&mut mem);
|
||||||
|
let ns = t0.elapsed().as_nanos() as u64;
|
||||||
|
if token.is_some() {
|
||||||
|
winning = token;
|
||||||
|
}
|
||||||
|
runs.push(RunOut {
|
||||||
|
index: i as usize,
|
||||||
|
wall_ns: ns,
|
||||||
|
solutions: (best >= target) as i64,
|
||||||
|
compile_ns: 0,
|
||||||
|
attempts,
|
||||||
|
achieved_effort: best,
|
||||||
|
verify_result: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let (sols_hex, nonce_hex) = match winning {
|
||||||
|
Some((nonce_bytes_wire, sol)) => (
|
||||||
|
Some(vec![hex_encode(&sol)]),
|
||||||
|
Some(hex_encode(&nonce_bytes_wire)),
|
||||||
|
),
|
||||||
|
None => (None, None),
|
||||||
|
};
|
||||||
|
emit_with_nonce("effort", req, &eff, runs, sols_hex, nonce_hex);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn op_hashx_compile(job: &Job, req: &str, reps: u64, warmup: u64) {
|
||||||
|
use hashx::{HashXBuilder, RuntimeOption as HxRt};
|
||||||
|
let bhex = job
|
||||||
|
.challenge_base_hex
|
||||||
|
.as_ref()
|
||||||
|
.or(job.challenge_hex.as_ref())
|
||||||
|
.unwrap_or_else(|| fail("hashx_compile", req, "hashx_compile requires a challenge"));
|
||||||
|
let base = hex_decode(bhex).unwrap_or_else(|| fail("hashx_compile", req, "invalid challenge"));
|
||||||
|
let nonce_start = job.nonce_start.unwrap_or(0);
|
||||||
|
|
||||||
|
let hxrt = match req {
|
||||||
|
"interpret" => HxRt::InterpretOnly,
|
||||||
|
"must-compile" => HxRt::CompileOnly,
|
||||||
|
_ => HxRt::TryCompile,
|
||||||
|
};
|
||||||
|
let mut hb = HashXBuilder::new();
|
||||||
|
hb.runtime(hxrt);
|
||||||
|
|
||||||
|
// Probe for effective runtime / hard failure.
|
||||||
|
let probe_seed = build_nonce_challenge(&base, nonce_start, 8);
|
||||||
|
let eff = match hb.build(&probe_seed) {
|
||||||
|
Ok(h) => eff_str(&format!("{:?}", h.runtime()), req),
|
||||||
|
Err(e) => fail("hashx_compile", req, &format!("{:?}", e)),
|
||||||
|
};
|
||||||
|
|
||||||
|
for _ in 0..warmup {
|
||||||
|
let seed = build_nonce_challenge(&base, nonce_start, 8);
|
||||||
|
if let Ok(h) = hb.build(&seed) {
|
||||||
|
let _ = h.hash_to_bytes(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut runs = Vec::with_capacity(reps as usize);
|
||||||
|
for i in 0..reps {
|
||||||
|
let seed = build_nonce_challenge(&base, nonce_start + i, 8);
|
||||||
|
let t0 = Instant::now();
|
||||||
|
let built = hb.build(&seed);
|
||||||
|
let compile_ns = t0.elapsed().as_nanos() as u64;
|
||||||
|
let (wall_ns, sols) = match built {
|
||||||
|
Ok(h) => {
|
||||||
|
let e0 = Instant::now();
|
||||||
|
let _ = h.hash_to_bytes(0);
|
||||||
|
(e0.elapsed().as_nanos() as u64, 1)
|
||||||
|
}
|
||||||
|
Err(_) => (0, 0),
|
||||||
|
};
|
||||||
|
runs.push(RunOut {
|
||||||
|
index: i as usize,
|
||||||
|
wall_ns,
|
||||||
|
solutions: sols,
|
||||||
|
compile_ns,
|
||||||
|
attempts: 0,
|
||||||
|
achieved_effort: 0,
|
||||||
|
verify_result: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
emit("hashx_compile", req, &eff, runs, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
let mut input = String::new();
|
||||||
|
if std::io::stdin().read_to_string(&mut input).is_err() {
|
||||||
|
fail("", "", "failed to read stdin");
|
||||||
|
}
|
||||||
|
let job: Job = match serde_json::from_str(&input) {
|
||||||
|
Ok(j) => j,
|
||||||
|
Err(e) => fail("", "", &format!("invalid job JSON: {}", e)),
|
||||||
|
};
|
||||||
|
|
||||||
|
let op = job.operation.clone().unwrap_or_else(|| "solve".to_string());
|
||||||
|
let req = job.runtime.clone().unwrap_or_else(|| "try-compile".to_string());
|
||||||
|
let reps = job.repetitions.unwrap_or(10).max(1);
|
||||||
|
let warmup = job.warmup.unwrap_or(3);
|
||||||
|
|
||||||
|
match op.as_str() {
|
||||||
|
"solve" => op_solve(&job, &req, reps, warmup),
|
||||||
|
"verify" => op_verify(&job, &req, reps, warmup),
|
||||||
|
"effort" => op_effort(&job, &req, reps, warmup),
|
||||||
|
"hashx_compile" => op_hashx_compile(&job, &req, reps, warmup),
|
||||||
|
_ => fail(&op, &req, "unknown operation"),
|
||||||
|
}
|
||||||
|
}
|
||||||
144
tools/benchmarks/Equi-X/scripts/autotune_c_flags.sh
Executable file
144
tools/benchmarks/Equi-X/scripts/autotune_c_flags.sh
Executable file
@ -0,0 +1,144 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Auto-select the FASTEST optimization flags for the MAIN C runner on THIS
|
||||||
|
# machine, then install the winner as build/runners/c/equix_runner (the path the
|
||||||
|
# `equix-c` adapter uses). So the main benchmark always runs the fastest build
|
||||||
|
# this host can produce, rather than a hard-coded guess.
|
||||||
|
#
|
||||||
|
# It builds a few candidate flag sets with the default C compiler, benchmarks the
|
||||||
|
# JIT solve path (which dominates PoW cost) with enough reps for a stable median,
|
||||||
|
# ranks them, and copies the fastest binary over the main runner. Each rep solves
|
||||||
|
# a FRESH challenge derived from the seed (a SHA-256 chain), so tuning spans the
|
||||||
|
# same varied-challenge distribution the main solve benchmark measures — not one
|
||||||
|
# fixed challenge. The seed is deterministic, so every candidate sees the IDENTICAL
|
||||||
|
# challenge stream, keeping the flag comparison a fair apples-to-apples A/B.
|
||||||
|
# Idempotent: candidate build dirs are reused, so re-runs are cheap.
|
||||||
|
#
|
||||||
|
# Tunables (env):
|
||||||
|
# CC compiler to tune (default: cc)
|
||||||
|
# EQUIX_AUTOTUNE_REPS timed solves per candidate (default: 1000)
|
||||||
|
# EQUIX_AUTOTUNE_WARMUP warmup solves per candidate (default: 8)
|
||||||
|
# EQUIX_AUTOTUNE_SEED challenge SEED hex (default: deadbeef); each rep
|
||||||
|
# solves the next SHA-256-derived challenge in the chain
|
||||||
|
# EQUIX_AUTOTUNE_EPSILON if plain -O3 is within this fraction of the best,
|
||||||
|
# prefer -O3 (portable, no native/LTO lock-in on a
|
||||||
|
# sub-noise win). Default: 0.01 (1%). Set 0 for strict.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
# Ctrl+C mid-tune: exit cleanly (130) instead of a bare SIGINT kill. Nothing to
|
||||||
|
# roll back — candidate build dirs are reused and the main runner is only ever
|
||||||
|
# replaced by the atomic mv at the very end, so an interrupt leaves it untouched.
|
||||||
|
trap 'echo; echo "autotune: interrupted (Ctrl+C); main runner left unchanged." >&2; exit 130' INT
|
||||||
|
|
||||||
|
CC="${CC:-cc}"
|
||||||
|
REPS="${EQUIX_AUTOTUNE_REPS:-1000}"
|
||||||
|
WARMUP="${EQUIX_AUTOTUNE_WARMUP:-8}"
|
||||||
|
# Accept the legacy EQUIX_AUTOTUNE_CHALLENGE name as a fallback for the seed.
|
||||||
|
SEED="${EQUIX_AUTOTUNE_SEED:-${EQUIX_AUTOTUNE_CHALLENGE:-deadbeef}}"
|
||||||
|
EPS="${EQUIX_AUTOTUNE_EPSILON:-0.01}"
|
||||||
|
NPROC="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)"
|
||||||
|
EQUIX_COMMIT="$(git -C vendored/equix rev-parse --short HEAD 2>/dev/null || echo unknown)"
|
||||||
|
|
||||||
|
command -v "$CC" >/dev/null 2>&1 || { echo "autotune: compiler '$CC' not found"; exit 1; }
|
||||||
|
command -v python3 >/dev/null 2>&1 || { echo "autotune: python3 required"; exit 1; }
|
||||||
|
|
||||||
|
# Candidate flag sets on the default compiler. -O0/-O1 are excluded on purpose
|
||||||
|
# (the flag sweep shows ~2x regression); these are the fast-tier contenders.
|
||||||
|
CANDIDATES=(
|
||||||
|
"o2|-O2 -DNDEBUG"
|
||||||
|
"o3|-O3 -DNDEBUG"
|
||||||
|
"o3-native|-O3 -march=native -DNDEBUG"
|
||||||
|
"o3-lto|-O3 -flto -DNDEBUG"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Median solve wall-time (ns) for a runner binary, or non-zero exit if unusable.
|
||||||
|
# NB: the Python script is passed via -c so the runner's piped JSON reaches its
|
||||||
|
# stdin (a heredoc would be consumed as the script instead).
|
||||||
|
measure() {
|
||||||
|
printf '{"schema_version":1,"operation":"solve","runtime":"try-compile","repetitions":%d,"warmup":%d,"challenge_seed_hex":"%s"}' \
|
||||||
|
"$REPS" "$WARMUP" "$SEED" | "$1" 2>/dev/null | python3 -c '
|
||||||
|
import sys, json, statistics
|
||||||
|
try:
|
||||||
|
d = json.loads(sys.stdin.read().splitlines()[-1])
|
||||||
|
assert d.get("ok") and d.get("runs")
|
||||||
|
runs = d["runs"]
|
||||||
|
walls = [r["wall_ns"] for r in runs if r.get("wall_ns", 0) > 0]
|
||||||
|
# Sanity: every rep must be validly timed, and the build must really solve
|
||||||
|
# SOMETHING across the stream. Under the varied-challenge seed, individual
|
||||||
|
# derived challenges legitimately yield 0 solutions (~17% of them), so the
|
||||||
|
# guard is total-solutions>0, not the old per-rep all(>0) which this breaks.
|
||||||
|
assert len(walls) == len(runs)
|
||||||
|
assert sum(r.get("solutions", 0) for r in runs) > 0
|
||||||
|
print(int(statistics.median(walls)))
|
||||||
|
except BaseException: # incl. KeyboardInterrupt on Ctrl+C: exit quietly, no stray traceback
|
||||||
|
sys.exit(1)
|
||||||
|
'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Parallel indexed arrays (macOS ships bash 3.2, which has no associative arrays).
|
||||||
|
echo "autotune: compiler=$CC reps=$REPS warmup=$WARMUP seed=$SEED (challenge varied per rep)"
|
||||||
|
# NB: candidates are built with the plain version "1.0.0" — the winner becomes
|
||||||
|
# the MAIN runner, and a tuning-suffixed version would leak machine-specific
|
||||||
|
# provenance into published results. The chosen flags are recorded separately
|
||||||
|
# in build/runners/c/equix_runner.flags and build/provenance.json.
|
||||||
|
NAMES=(); OKFLAGS=(); MEDIANS=()
|
||||||
|
for entry in "${CANDIDATES[@]}"; do
|
||||||
|
IFS='|' read -r name flags <<<"$entry"
|
||||||
|
bdir="build/autotune/$name"
|
||||||
|
log="build/autotune/$name.log"
|
||||||
|
mkdir -p build/autotune
|
||||||
|
# A copied/moved repo carries a CMake cache with old absolute paths; clean it.
|
||||||
|
if [ -f "$bdir/CMakeCache.txt" ]; then
|
||||||
|
recorded="$(sed -n 's/^CMAKE_CACHEFILE_DIR:INTERNAL=//p' "$bdir/CMakeCache.txt" | head -1)"
|
||||||
|
if [ -n "$recorded" ] && [ "$recorded" != "$(cd "$bdir" && pwd)" ]; then
|
||||||
|
rm -rf "$bdir"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if ! cmake -S runners/c -B "$bdir" \
|
||||||
|
-DCMAKE_BUILD_TYPE=Release \
|
||||||
|
-DCMAKE_C_COMPILER="$CC" \
|
||||||
|
-DCMAKE_C_FLAGS_RELEASE="$flags" \
|
||||||
|
-DCMAKE_POLICY_VERSION_MINIMUM=3.10 \
|
||||||
|
-DEQUIX_C_COMMIT="$EQUIX_COMMIT" \
|
||||||
|
-DEQUIX_C_VERSION="1.0.0" >/dev/null 2>"$log" \
|
||||||
|
|| ! cmake --build "$bdir" -j"$NPROC" --target equix_runner >>"$log" 2>&1; then
|
||||||
|
echo " skip $name ($flags): build failed (see $log)"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if ! median="$(measure "$bdir/equix_runner")"; then
|
||||||
|
echo " skip $name ($flags): benchmark failed / no solutions"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
NAMES+=("$name"); OKFLAGS+=("$flags"); MEDIANS+=("$median")
|
||||||
|
printf ' %-10s %-26s median %s ms\n' "$name" "$flags" \
|
||||||
|
"$(python3 -c "print(f'{$median/1e6:.3f}')")"
|
||||||
|
done
|
||||||
|
|
||||||
|
[ "${#NAMES[@]}" -gt 0 ] || { echo "autotune: no candidate usable; keeping existing runner"; exit 1; }
|
||||||
|
|
||||||
|
# Pick the minimum-median candidate; find plain -O3's index for the tie-break.
|
||||||
|
best_i=0; o3_i=-1
|
||||||
|
for i in "${!NAMES[@]}"; do
|
||||||
|
[ "${MEDIANS[$i]}" -lt "${MEDIANS[$best_i]}" ] && best_i=$i
|
||||||
|
[ "${NAMES[$i]}" = "o3" ] && o3_i=$i
|
||||||
|
done
|
||||||
|
|
||||||
|
# Tie-break: if plain -O3 is within EPS of the winner, prefer it -- avoids
|
||||||
|
# locking the main runner into a native/LTO build over a sub-noise difference.
|
||||||
|
if [ "$o3_i" -ge 0 ] && [ "${NAMES[$best_i]}" != "o3" ] \
|
||||||
|
&& awk "BEGIN{exit !(${MEDIANS[$o3_i]} <= ${MEDIANS[$best_i]}*(1+$EPS))}"; then
|
||||||
|
echo " note: -O3 within ${EPS} of best (${NAMES[$best_i]}); preferring -O3 for portability"
|
||||||
|
best_i=$o3_i
|
||||||
|
fi
|
||||||
|
|
||||||
|
best_name="${NAMES[$best_i]}"; best_flags="${OKFLAGS[$best_i]}"; best_median="${MEDIANS[$best_i]}"
|
||||||
|
mkdir -p build/runners/c
|
||||||
|
# Install atomically: copy to a temp path, then rename over the live runner, so a
|
||||||
|
# Ctrl+C during the copy can never leave a half-written (corrupt) main binary.
|
||||||
|
cp -f "build/autotune/$best_name/equix_runner" build/runners/c/equix_runner.tmp
|
||||||
|
mv -f build/runners/c/equix_runner.tmp build/runners/c/equix_runner
|
||||||
|
printf '%s' "$best_flags" > build/runners/c/equix_runner.flags
|
||||||
|
echo "==> fastest flags on this machine: '$best_flags' ($best_name), median $(python3 -c "print(f'{$best_median/1e6:.3f}')") ms"
|
||||||
|
echo " installed as build/runners/c/equix_runner (used by the 'equix-c' main run)"
|
||||||
75
tools/benchmarks/Equi-X/scripts/build_variants.sh
Executable file
75
tools/benchmarks/Equi-X/scripts/build_variants.sh
Executable file
@ -0,0 +1,75 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Build the C runner (and its vendored libequix/hashx) under several compiler /
|
||||||
|
# optimization-flag combinations, registering each as its own benchmark impl so
|
||||||
|
# the harness can compare performance across compiler flags.
|
||||||
|
#
|
||||||
|
# Each variant -> build/variants/<name>/equix_runner and a manifest at
|
||||||
|
# adapters/generated/equix-c-<name>.manifest.toml (impl name "equix-c-<name>").
|
||||||
|
# Then: python -m equix_bench run --config configs/compiler_flags.toml --out results/
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$ROOT"
|
||||||
|
NPROC="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)"
|
||||||
|
have() { command -v "$1" >/dev/null 2>&1; }
|
||||||
|
|
||||||
|
git submodule update --init --recursive >/dev/null 2>&1 || true
|
||||||
|
EQUIX_COMMIT="$(git -C vendored/equix rev-parse --short HEAD 2>/dev/null || echo unknown)"
|
||||||
|
OUT="adapters/generated"
|
||||||
|
mkdir -p "$OUT" build/variants
|
||||||
|
|
||||||
|
# name | compiler | CFLAGS (-DNDEBUG kept so all variants match Release semantics)
|
||||||
|
VARIANTS=(
|
||||||
|
"gcc-o0|gcc|-O0 -DNDEBUG"
|
||||||
|
"gcc-o2|gcc|-O2 -DNDEBUG"
|
||||||
|
"gcc-o3|gcc|-O3 -DNDEBUG"
|
||||||
|
"gcc-o3-native|gcc|-O3 -march=native -DNDEBUG"
|
||||||
|
"gcc-o3-lto|gcc|-O3 -flto -DNDEBUG"
|
||||||
|
"clang-o3|clang|-O3 -DNDEBUG"
|
||||||
|
"clang-o3-native|clang|-O3 -march=native -DNDEBUG"
|
||||||
|
)
|
||||||
|
|
||||||
|
built=()
|
||||||
|
for entry in "${VARIANTS[@]}"; do
|
||||||
|
IFS='|' read -r name cc flags <<<"$entry"
|
||||||
|
if ! have "$cc"; then echo "skip $name ($cc not found)"; continue; fi
|
||||||
|
bdir="build/variants/$name"
|
||||||
|
# A copied/moved repo carries a CMake cache with old absolute paths; clean it.
|
||||||
|
if [ -f "$bdir/CMakeCache.txt" ]; then
|
||||||
|
recorded="$(sed -n 's/^CMAKE_CACHEFILE_DIR:INTERNAL=//p' "$bdir/CMakeCache.txt" | head -1)"
|
||||||
|
if [ -n "$recorded" ] && [ "$recorded" != "$(cd "$bdir" && pwd)" ]; then
|
||||||
|
rm -rf "$bdir"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
echo "==> building $name ($cc $flags)"
|
||||||
|
if ! cmake -S runners/c -B "$bdir" \
|
||||||
|
-DCMAKE_BUILD_TYPE=Release \
|
||||||
|
-DCMAKE_POLICY_VERSION_MINIMUM=3.10 \
|
||||||
|
-DCMAKE_C_COMPILER="$cc" \
|
||||||
|
-DCMAKE_C_FLAGS_RELEASE="$flags" \
|
||||||
|
-DEQUIX_C_COMMIT="$EQUIX_COMMIT" \
|
||||||
|
-DEQUIX_C_VERSION="1.0.0-$name" >/dev/null 2>"build/variants/$name.log"; then
|
||||||
|
echo " configure FAILED for $name (see build/variants/$name.log)"; continue
|
||||||
|
fi
|
||||||
|
if ! cmake --build "$bdir" -j"$NPROC" --target equix_runner >>"build/variants/$name.log" 2>&1; then
|
||||||
|
echo " build FAILED for $name (see build/variants/$name.log)"; continue
|
||||||
|
fi
|
||||||
|
cat > "$OUT/equix-c-$name.manifest.toml" <<EOF
|
||||||
|
# Auto-generated by scripts/build_variants.sh -- do not edit.
|
||||||
|
name = "equix-c-$name"
|
||||||
|
exec = "build/variants/$name/equix_runner"
|
||||||
|
protocol_version = 1
|
||||||
|
capabilities = ["solve", "verify", "effort", "hashx_compile"]
|
||||||
|
runtimes = ["interpret", "try-compile", "must-compile"]
|
||||||
|
|
||||||
|
[env]
|
||||||
|
EQUIX_C_VERSION = "1.0.0-$name"
|
||||||
|
EQUIX_C_FLAGS = "$cc $flags"
|
||||||
|
EOF
|
||||||
|
built+=("equix-c-$name")
|
||||||
|
done
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "Built variants: ${built[*]:-none}"
|
||||||
|
echo "Manifests in $OUT/. Run:"
|
||||||
|
echo " python -m equix_bench run --config configs/compiler_flags.toml --out results/"
|
||||||
58
tools/benchmarks/Equi-X/scripts/combine_all.sh
Executable file
58
tools/benchmarks/Equi-X/scripts/combine_all.sh
Executable file
@ -0,0 +1,58 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Combine benchmark results collected from MULTIPLE devices into ONE report with
|
||||||
|
# figures drawn from all of them. Point it at a tree that holds every device's
|
||||||
|
# run outputs (each device's `results/` copied/rsync'd under it, in any layout):
|
||||||
|
#
|
||||||
|
# results-by-device/
|
||||||
|
# laptop-x1/main/{raw/results.json, concurrency.csv, mining.csv, ...}
|
||||||
|
# server-epyc/main/{raw/results.json, ...}
|
||||||
|
# rpi5/results/main/{raw/results.json, ...}
|
||||||
|
#
|
||||||
|
# Discovery is layout-agnostic: any directory containing raw/results.json is a
|
||||||
|
# run, and each run's DEVICE identity comes from inside its records (CPU model +
|
||||||
|
# OS), so folder names are free-form and two devices never get conflated. Re-runs
|
||||||
|
# of the same device are de-duplicated (newest wins). Every core plot is faceted
|
||||||
|
# per device; cross-device `xdev_*` charts compare CPUs directly; the concurrency
|
||||||
|
# and mining sections/figures are carried across all devices too.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ./scripts/combine_all.sh [ROOT] [--out DIR]
|
||||||
|
# ROOT tree to scan for runs (default: results-by-device, else results)
|
||||||
|
# --out DIR combined report directory (default: <ROOT>/combined or results/combined)
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
ROOT_ARG=""
|
||||||
|
OUT=""
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--out) [ $# -ge 2 ] || { echo "error: --out needs a directory" >&2; exit 2; }; OUT="$2"; shift ;;
|
||||||
|
--out=*) OUT="${1#*=}" ;;
|
||||||
|
-h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||||
|
-*) echo "unknown option: $1" >&2; exit 2 ;;
|
||||||
|
*) ROOT_ARG="$1" ;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$REPO"
|
||||||
|
|
||||||
|
# Default ROOT: a dedicated per-device collection dir if present, else results/.
|
||||||
|
ROOT="$ROOT_ARG"
|
||||||
|
if [ -z "$ROOT" ]; then
|
||||||
|
if [ -d results-by-device ]; then ROOT="results-by-device"; else ROOT="results"; fi
|
||||||
|
fi
|
||||||
|
[ -d "$ROOT" ] || { echo "error: ROOT '$ROOT' is not a directory" >&2; exit 2; }
|
||||||
|
[ -n "$OUT" ] || OUT="$ROOT/combined"
|
||||||
|
|
||||||
|
# Prefer the venv from setup.sh (harness installed, PEP-668-proof).
|
||||||
|
PY="python3"
|
||||||
|
[ -x "$REPO/.venv/bin/python" ] && PY="$REPO/.venv/bin/python"
|
||||||
|
export PYTHONPATH="$REPO/harness${PYTHONPATH:+:$PYTHONPATH}"
|
||||||
|
|
||||||
|
echo "==> Combining all runs under '$ROOT' -> '$OUT'"
|
||||||
|
$PY -m equix_bench combine --root "$ROOT" --out "$OUT" || {
|
||||||
|
echo "error: combine failed" >&2; exit 1;
|
||||||
|
}
|
||||||
|
echo " report: $OUT/report.md"
|
||||||
|
echo " figures: $OUT/plots/*.png data: $OUT/results.csv"
|
||||||
119
tools/benchmarks/Equi-X/scripts/run_all.sh
Executable file
119
tools/benchmarks/Equi-X/scripts/run_all.sh
Executable file
@ -0,0 +1,119 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Run the ENTIRE Equi-X benchmark pipeline end to end:
|
||||||
|
# 1. bootstrap dependencies + build both runners (scripts/setup.sh)
|
||||||
|
# 2. harness unit tests
|
||||||
|
# 3. main benchmark: C vs Rust across all operations, incl. the DoS-protection
|
||||||
|
# verdict, with the cross-implementation correctness gate; the --full
|
||||||
|
# profile additionally measures sustained concurrency (saturation ladder)
|
||||||
|
# and the mining rate vs difficulty
|
||||||
|
# 4. compiler-flag variants: build a gcc/clang/-O matrix and compare them
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ./scripts/run_all.sh # quick profile (smoke config; a few minutes)
|
||||||
|
# ./scripts/run_all.sh --full # deep sweep (full config + effort sweep; longer)
|
||||||
|
# ./scripts/run_all.sh --out DIR # output base dir (default: results/)
|
||||||
|
# ./scripts/run_all.sh --no-variants # skip the compiler-flag matrix
|
||||||
|
# ./scripts/run_all.sh --no-setup # assume deps already installed/built
|
||||||
|
# ./scripts/run_all.sh --no-tests # skip the unit tests
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$ROOT"
|
||||||
|
export PYTHONPATH="$ROOT/harness${PYTHONPATH:+:$PYTHONPATH}"
|
||||||
|
|
||||||
|
PROFILE=quick
|
||||||
|
OUT=results
|
||||||
|
DO_SETUP=1
|
||||||
|
DO_VARIANTS=1
|
||||||
|
DO_TESTS=1
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--full) PROFILE=full ;;
|
||||||
|
--quick) PROFILE=quick ;;
|
||||||
|
--out) [ $# -ge 2 ] || { echo "error: --out requires a directory argument" >&2; exit 2; }
|
||||||
|
OUT="$2"; shift ;;
|
||||||
|
--out=*) OUT="${1#*=}" ;;
|
||||||
|
--no-setup) DO_SETUP=0 ;;
|
||||||
|
--no-variants) DO_VARIANTS=0 ;;
|
||||||
|
--no-tests) DO_TESTS=0 ;;
|
||||||
|
-h|--help) sed -n '2,/^set -uo/p' "$0" | sed '$d'; exit 0 ;;
|
||||||
|
*) echo "unknown argument: $1" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
MAIN_CONFIG="configs/smoke.toml"
|
||||||
|
[ "$PROFILE" = full ] && MAIN_CONFIG="configs/full.toml"
|
||||||
|
# Prefer the project venv scripts/setup.sh creates: it has the harness + pytest
|
||||||
|
# installed and sidesteps PEP-668 "externally-managed" pip failures on Linux.
|
||||||
|
PY="python3"
|
||||||
|
[ -x "$ROOT/.venv/bin/python" ] && PY="$ROOT/.venv/bin/python"
|
||||||
|
BENCH="$PY -m equix_bench"
|
||||||
|
fail() { echo "ERROR: $*" >&2; exit 1; }
|
||||||
|
|
||||||
|
# Ctrl+C: report the interruption plainly and exit 130, rather than letting the
|
||||||
|
# child's non-zero exit trip a misleading "ERROR: ... failed". The harness kills
|
||||||
|
# its own runner subprocesses on SIGINT; this trap just makes the pipeline's own
|
||||||
|
# exit clean. 128+SIGINT(2) = 130, the conventional interrupted-by-Ctrl+C code.
|
||||||
|
on_interrupt() { echo; echo "Interrupted (Ctrl+C) — stopping the pipeline." >&2; exit 130; }
|
||||||
|
trap on_interrupt INT
|
||||||
|
|
||||||
|
echo "======================================================================"
|
||||||
|
echo " Equi-X full pipeline profile=$PROFILE out=$OUT/"
|
||||||
|
echo "======================================================================"
|
||||||
|
|
||||||
|
if [ "$DO_SETUP" = 1 ]; then
|
||||||
|
echo; echo "### [1/4] Bootstrap dependencies + build runners"
|
||||||
|
./scripts/setup.sh || fail "setup.sh failed"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$DO_TESTS" = 1 ]; then
|
||||||
|
echo; echo "### [2/4] Harness unit tests"
|
||||||
|
# Ensure pytest is importable. Externally-managed interpreters (PEP 668, e.g.
|
||||||
|
# Homebrew/Debian Python) reject a plain `pip install`, so fall back through
|
||||||
|
# --user and finally --break-system-packages before giving up.
|
||||||
|
# With the venv from setup.sh, pytest is already present. Otherwise fall back
|
||||||
|
# through --user and --break-system-packages before giving up.
|
||||||
|
if ! $PY -c 'import pytest' >/dev/null 2>&1; then
|
||||||
|
$PY -m pip install -q pytest >/dev/null 2>&1 \
|
||||||
|
|| $PY -m pip install -q --user pytest >/dev/null 2>&1 \
|
||||||
|
|| $PY -m pip install -q --break-system-packages pytest >/dev/null 2>&1 \
|
||||||
|
|| true
|
||||||
|
fi
|
||||||
|
if $PY -c 'import pytest' >/dev/null 2>&1; then
|
||||||
|
$PY -m pytest -q harness/tests || fail "unit tests failed"
|
||||||
|
else
|
||||||
|
echo " WARNING: could not install pytest; skipping unit tests." >&2
|
||||||
|
echo " Fix: re-run ./scripts/setup.sh (creates .venv with pytest), or:" >&2
|
||||||
|
echo " python3 -m pip install --break-system-packages pytest" >&2
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo; echo "### [3/4] Main benchmark: C vs Rust (all ops + DoS-protection; --full adds concurrency + mining) [$MAIN_CONFIG]"
|
||||||
|
# cmd_run runs the interop cross-check internally and exits non-zero if it fails,
|
||||||
|
# so this step is also the correctness gate.
|
||||||
|
$BENCH run --config "$MAIN_CONFIG" --out "$OUT/main" || fail "main benchmark / cross-check failed"
|
||||||
|
|
||||||
|
if [ "$DO_VARIANTS" = 1 ]; then
|
||||||
|
echo; echo "### [4/4] Compiler-flag variants (gcc/clang x -O levels)"
|
||||||
|
./scripts/build_variants.sh || echo " (some variants failed to build; continuing with those that did)"
|
||||||
|
$BENCH run --config configs/compiler_flags.toml --out "$OUT/compiler_flags" \
|
||||||
|
|| echo " (compiler-flags run returned non-zero; see $OUT/compiler_flags/)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo; echo "======================================================================"
|
||||||
|
echo " Done. Reports:"
|
||||||
|
echo " $OUT/main/report.md C vs Rust: time, throughput, RSS, compile, effort, DoS"
|
||||||
|
[ "$DO_VARIANTS" = 1 ] && echo " $OUT/compiler_flags/report.md compiler-flag comparison"
|
||||||
|
echo " plots: $OUT/*/plots/*.png data: $OUT/*/results.csv"
|
||||||
|
# The --full profile also measures sustained parallel solve/verify capacity.
|
||||||
|
[ -f "$OUT/main/concurrency.csv" ] && \
|
||||||
|
echo " $OUT/main/concurrency.csv measured concurrency/saturation (solves/s, verify/s, knee)"
|
||||||
|
[ -f "$OUT/main/mining.csv" ] && \
|
||||||
|
echo " $OUT/main/mining.csv measured mining rate vs difficulty (tokens/s, 1-core + machine)"
|
||||||
|
# Surface the DoS verdict inline.
|
||||||
|
if [ -f "$OUT/main/report.md" ]; then
|
||||||
|
grep -h 'Verdict:' "$OUT/main/report.md" | sed 's/\*\*//g; s/^/ DoS → /'
|
||||||
|
fi
|
||||||
|
echo "======================================================================"
|
||||||
42
tools/benchmarks/Equi-X/scripts/run_when_idle.sh
Executable file
42
tools/benchmarks/Equi-X/scripts/run_when_idle.sh
Executable file
@ -0,0 +1,42 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Run a command only once the machine is idle — so other workloads can't skew a
|
||||||
|
# benchmark. This is how the published mining numbers were measured (see
|
||||||
|
# docs/findings.md §7a): the run starts only after CPU idle stays above a
|
||||||
|
# threshold for several consecutive polls.
|
||||||
|
#
|
||||||
|
# ./scripts/run_when_idle.sh python3 -m equix_bench run --config configs/mining.toml --out results/mining
|
||||||
|
#
|
||||||
|
# Env overrides:
|
||||||
|
# IDLE_THRESH=85 required % CPU idle IDLE_NEED=3 consecutive polls
|
||||||
|
# IDLE_POLL=25 seconds between polls IDLE_MAXWAIT=21600 give-up (s)
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
[ $# -ge 1 ] || { echo "usage: $0 <command> [args...]" >&2; exit 2; }
|
||||||
|
|
||||||
|
THRESH="${IDLE_THRESH:-85}"
|
||||||
|
NEED="${IDLE_NEED:-3}"
|
||||||
|
POLL="${IDLE_POLL:-25}"
|
||||||
|
MAXWAIT="${IDLE_MAXWAIT:-21600}"
|
||||||
|
|
||||||
|
cpu_idle() {
|
||||||
|
case "$(uname -s)" in
|
||||||
|
Darwin) top -l 2 -s 1 -n 0 2>/dev/null | grep 'CPU usage' | tail -1 \
|
||||||
|
| sed -E 's/.* ([0-9.]+)% idle.*/\1/' ;;
|
||||||
|
Linux) vmstat 1 2 2>/dev/null | tail -1 | awk '{print $15}' ;;
|
||||||
|
*) echo 100 ;; # unknown platform: don't block
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
ok=0; elapsed=0
|
||||||
|
echo "waiting for idle (>= ${THRESH}% CPU idle x ${NEED} consecutive polls)..."
|
||||||
|
while :; do
|
||||||
|
idle="$(cpu_idle)"; idle="${idle:-0}"
|
||||||
|
if awk "BEGIN{exit !(${idle}+0 >= ${THRESH})}"; then ok=$((ok+1)); else ok=0; fi
|
||||||
|
echo " idle=${idle}% streak=${ok}/${NEED} (waited ${elapsed}s)"
|
||||||
|
[ "$ok" -ge "$NEED" ] && break
|
||||||
|
sleep "$POLL"; elapsed=$((elapsed+POLL))
|
||||||
|
[ "$elapsed" -ge "$MAXWAIT" ] && { echo "gave up waiting for idle after ${elapsed}s" >&2; exit 3; }
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "system idle; running: $*"
|
||||||
|
exec "$@"
|
||||||
277
tools/benchmarks/Equi-X/scripts/setup.sh
Executable file
277
tools/benchmarks/Equi-X/scripts/setup.sh
Executable file
@ -0,0 +1,277 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Bootstrap the Equi-X benchmark: install/compile any missing dependencies, fetch
|
||||||
|
# the vendored reference implementation, and build both runners. Idempotent.
|
||||||
|
#
|
||||||
|
# ./scripts/setup.sh # install missing deps (best effort) + build
|
||||||
|
# ./scripts/setup.sh --check # only report what's present/missing, install nothing
|
||||||
|
# EQUIX_NO_AUTO_INSTALL=1 ./scripts/setup.sh # never auto-install; fail with guidance
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
CHECK_ONLY=0
|
||||||
|
[ "${1:-}" = "--check" ] && CHECK_ONLY=1
|
||||||
|
: "${EQUIX_NO_AUTO_INSTALL:=0}"
|
||||||
|
|
||||||
|
NPROC="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)"
|
||||||
|
have() { command -v "$1" >/dev/null 2>&1; }
|
||||||
|
|
||||||
|
# clean_stale_cmake_dir <build-dir>: CMake caches record ABSOLUTE paths, so a
|
||||||
|
# repo that was copied/moved (e.g. rsync'd to another machine) fails to
|
||||||
|
# configure with "CMakeCache.txt directory ... is different". Detect a cache
|
||||||
|
# created at a different path and remove the build dir so cmake starts fresh.
|
||||||
|
clean_stale_cmake_dir() {
|
||||||
|
local dir="$1" cache="$1/CMakeCache.txt" recorded
|
||||||
|
[ -f "$cache" ] || return 0
|
||||||
|
recorded="$(sed -n 's/^CMAKE_CACHEFILE_DIR:INTERNAL=//p' "$cache" | head -1)"
|
||||||
|
if [ -n "$recorded" ] && [ "$recorded" != "$(cd "$dir" && pwd)" ]; then
|
||||||
|
echo " (stale CMake cache from '$recorded'; cleaning $dir)"
|
||||||
|
rm -rf "$dir"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Minimum Rust toolchain the runner's dependency tree needs to compile.
|
||||||
|
RUST_MIN="1.91.0"
|
||||||
|
|
||||||
|
# rust_version_ok <have> <min> -> 0 if have >= min (compares major.minor.patch).
|
||||||
|
# Pre-release suffixes ("1.91.0-nightly", "1.91.1-beta.2") are stripped first;
|
||||||
|
# without that, a non-numeric patch component errors the [ -ge ] comparison.
|
||||||
|
rust_version_ok() {
|
||||||
|
local have="${1%%-*}" min="${2%%-*}" IFS=.
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
set -- $have; local h_maj=${1:-0} h_min=${2:-0} h_pat=${3:-0}
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
set -- $min; local m_maj=${1:-0} m_min=${2:-0} m_pat=${3:-0}
|
||||||
|
[ "$h_maj" -ne "$m_maj" ] && { [ "$h_maj" -gt "$m_maj" ]; return; }
|
||||||
|
[ "$h_min" -ne "$m_min" ] && { [ "$h_min" -gt "$m_min" ]; return; }
|
||||||
|
[ "$h_pat" -ge "$m_pat" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- dependency provisioning -------------------------------------------------
|
||||||
|
|
||||||
|
SUDO=""
|
||||||
|
if [ "$(id -u)" -ne 0 ] && have sudo; then SUDO="sudo"; fi
|
||||||
|
|
||||||
|
PM=""
|
||||||
|
for pm in apt-get dnf yum pacman zypper brew; do
|
||||||
|
if have "$pm"; then PM="$pm"; break; fi
|
||||||
|
done
|
||||||
|
|
||||||
|
pm_install() { # pm_install <pkg...>
|
||||||
|
case "$PM" in
|
||||||
|
apt-get) $SUDO apt-get update -qq && $SUDO apt-get install -y "$@" ;;
|
||||||
|
dnf|yum) $SUDO "$PM" install -y "$@" ;;
|
||||||
|
pacman) $SUDO pacman -Sy --noconfirm "$@" ;;
|
||||||
|
zypper) $SUDO zypper install -y "$@" ;;
|
||||||
|
brew) brew install "$@" ;;
|
||||||
|
*) return 1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# pkg name for the current package manager: ensure <cmd> <apt> <dnf> <pacman> <brew>
|
||||||
|
ensure() {
|
||||||
|
local cmd="$1" apt="$2" dnf="$3" pac="$4" brew="$5" pkg=""
|
||||||
|
if have "$cmd"; then echo " ok: $cmd"; return 0; fi
|
||||||
|
echo " MISSING: $cmd"
|
||||||
|
[ "$CHECK_ONLY" = 1 ] && { MISSING=1; return 0; }
|
||||||
|
if [ "$EQUIX_NO_AUTO_INSTALL" = 1 ] || [ -z "$PM" ]; then
|
||||||
|
echo " -> please install '$cmd' (no package manager auto-install available)"
|
||||||
|
MISSING=1; return 0
|
||||||
|
fi
|
||||||
|
case "$PM" in
|
||||||
|
apt-get) pkg="$apt" ;; dnf|yum) pkg="$dnf" ;; pacman) pkg="$pac" ;;
|
||||||
|
zypper) pkg="$dnf" ;; brew) pkg="$brew" ;;
|
||||||
|
esac
|
||||||
|
# An empty package name means "not installable via this PM" (e.g. cc/pip3 on
|
||||||
|
# macOS come from the Xcode CLT / python): don't run a bare `$PM install`.
|
||||||
|
if [ -z "$pkg" ]; then
|
||||||
|
case "$cmd" in
|
||||||
|
cc) echo " -> please install the Xcode Command Line Tools: xcode-select --install" ;;
|
||||||
|
*) echo " -> please install '$cmd' manually (not packaged for $PM here)" ;;
|
||||||
|
esac
|
||||||
|
MISSING=1; return 0
|
||||||
|
fi
|
||||||
|
echo " -> installing $pkg via $PM"
|
||||||
|
if pm_install $pkg; then echo " installed: $cmd"; else echo " FAILED to install $cmd"; MISSING=1; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
MISSING=0
|
||||||
|
echo "==> [1/6] Checking dependencies (package manager: ${PM:-none})"
|
||||||
|
ensure git git git git git
|
||||||
|
ensure cmake cmake cmake cmake cmake
|
||||||
|
ensure cc build-essential 'gcc gcc-c++ make' base-devel "" # clang ships with Xcode CLT on macOS
|
||||||
|
ensure python3 python3 python3 python python
|
||||||
|
ensure pip3 python3-pip python3-pip python-pip ""
|
||||||
|
|
||||||
|
# Rust toolchain via rustup when missing, and kept at >= RUST_MIN.
|
||||||
|
# (A distro/rustup rustc that is too old — e.g. 1.87 vs the 1.91 the deps need —
|
||||||
|
# fails the build just like a missing one, so treat both the same way.)
|
||||||
|
# NB: must return 0 when the file is absent — a bare `[ -f ] && .` one-liner
|
||||||
|
# returns 1 there, and under `set -e` a plain call would silently kill setup.
|
||||||
|
maybe_source_cargo_env() {
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
if [ -f "$HOME/.cargo/env" ]; then . "$HOME/.cargo/env"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
if have cargo; then
|
||||||
|
RUSTC_VER="$(rustc --version 2>/dev/null | awk '{print $2}')"
|
||||||
|
if rust_version_ok "${RUSTC_VER:-0.0.0}" "$RUST_MIN"; then
|
||||||
|
echo " ok: cargo (rustc ${RUSTC_VER})"
|
||||||
|
elif [ "$CHECK_ONLY" = 1 ]; then
|
||||||
|
echo " OUTDATED: rustc ${RUSTC_VER:-unknown} < ${RUST_MIN}"; MISSING=1
|
||||||
|
elif [ "$EQUIX_NO_AUTO_INSTALL" = 1 ]; then
|
||||||
|
echo " -> rustc ${RUSTC_VER:-unknown} < ${RUST_MIN}; please update Rust (https://rustup.rs)"; MISSING=1
|
||||||
|
elif have rustup; then
|
||||||
|
echo " -> rustc ${RUSTC_VER:-unknown} < ${RUST_MIN}; updating via 'rustup update stable'"
|
||||||
|
if rustup update stable && rustup default stable; then
|
||||||
|
maybe_source_cargo_env
|
||||||
|
RUSTC_VER="$(rustc --version 2>/dev/null | awk '{print $2}')"
|
||||||
|
rust_version_ok "${RUSTC_VER:-0.0.0}" "$RUST_MIN" \
|
||||||
|
|| { echo " FAILED: rustc still ${RUSTC_VER:-unknown} < ${RUST_MIN} after update"; MISSING=1; }
|
||||||
|
else
|
||||||
|
echo " FAILED to update Rust via rustup"; MISSING=1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# cargo present but not managed by rustup (e.g. a distro package): install
|
||||||
|
# rustup so we can get a current toolchain, then re-check.
|
||||||
|
echo " -> rustc ${RUSTC_VER:-unknown} < ${RUST_MIN} and no rustup; installing rustup"
|
||||||
|
if have curl && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable; then
|
||||||
|
maybe_source_cargo_env
|
||||||
|
RUSTC_VER="$(rustc --version 2>/dev/null | awk '{print $2}')"
|
||||||
|
rust_version_ok "${RUSTC_VER:-0.0.0}" "$RUST_MIN" \
|
||||||
|
|| { echo " FAILED: rustc still ${RUSTC_VER:-unknown} < ${RUST_MIN}; ensure ~/.cargo/bin precedes the system rustc on PATH"; MISSING=1; }
|
||||||
|
else
|
||||||
|
echo " FAILED to install rustup (need curl + network)"; MISSING=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
elif [ "$CHECK_ONLY" = 1 ]; then
|
||||||
|
echo " MISSING: cargo"; MISSING=1
|
||||||
|
elif [ "$EQUIX_NO_AUTO_INSTALL" = 1 ]; then
|
||||||
|
echo " -> please install Rust (https://rustup.rs)"; MISSING=1
|
||||||
|
else
|
||||||
|
echo " -> installing Rust via rustup"
|
||||||
|
if have curl && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable; then
|
||||||
|
maybe_source_cargo_env
|
||||||
|
else
|
||||||
|
echo " FAILED to install cargo (need curl + network)"; MISSING=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$CHECK_ONLY" = 1 ]; then
|
||||||
|
[ "$MISSING" = 0 ] && echo "==> All dependencies present." || echo "==> Some dependencies missing (see above)."
|
||||||
|
exit "$MISSING"
|
||||||
|
fi
|
||||||
|
[ "$MISSING" = 0 ] || { echo "ERROR: missing dependencies above; install them and re-run."; exit 1; }
|
||||||
|
|
||||||
|
# --- build -------------------------------------------------------------------
|
||||||
|
|
||||||
|
echo "==> [2/6] Initializing vendored submodules (equix + hashx)"
|
||||||
|
git submodule update --init --recursive
|
||||||
|
|
||||||
|
EQUIX_COMMIT="$(git -C vendored/equix rev-parse --short HEAD)"
|
||||||
|
HASHX_COMMIT="$(git -C vendored/equix/hashx rev-parse --short HEAD 2>/dev/null || echo unknown)"
|
||||||
|
echo " equix @ ${EQUIX_COMMIT}, hashx @ ${HASHX_COMMIT}"
|
||||||
|
|
||||||
|
echo "==> [3/6] Building the C runner (compiles libequix + libhashx)"
|
||||||
|
# Baseline build with explicit, known-fast flags -- `-O3 -DNDEBUG` (no -O0/-O1,
|
||||||
|
# no -march=native). This guarantees a working runner and is the fallback if
|
||||||
|
# autotune is disabled or fails.
|
||||||
|
MAIN_C_FLAGS="-O3 -DNDEBUG"
|
||||||
|
# A copied/moved repo carries CMake caches with the old absolute paths — clean
|
||||||
|
# them or cmake refuses to configure.
|
||||||
|
clean_stale_cmake_dir build/runners/c
|
||||||
|
for d in build/autotune/* build/variants/*; do
|
||||||
|
if [ -d "$d" ]; then clean_stale_cmake_dir "$d"; fi
|
||||||
|
done
|
||||||
|
# If a previous autotune installed a winner built with DIFFERENT flags, the
|
||||||
|
# incremental build below would be a no-op (the copied binary is newer than the
|
||||||
|
# sources) and rewriting the .flags file would misdescribe the binary. Force a
|
||||||
|
# clean rebuild in that case so binary and provenance always agree.
|
||||||
|
CLEAN_ARG=""
|
||||||
|
if [ -f build/runners/c/equix_runner.flags ] \
|
||||||
|
&& [ "$(cat build/runners/c/equix_runner.flags)" != "${MAIN_C_FLAGS}" ]; then
|
||||||
|
CLEAN_ARG="--clean-first"
|
||||||
|
fi
|
||||||
|
cmake -S runners/c -B build/runners/c \
|
||||||
|
-DCMAKE_BUILD_TYPE=Release \
|
||||||
|
-DCMAKE_C_FLAGS_RELEASE="${MAIN_C_FLAGS}" \
|
||||||
|
-DCMAKE_POLICY_VERSION_MINIMUM=3.10 \
|
||||||
|
-DEQUIX_C_COMMIT="${EQUIX_COMMIT}" \
|
||||||
|
-DEQUIX_C_VERSION="1.0.0" >/dev/null
|
||||||
|
cmake --build build/runners/c -j"$NPROC" --target equix_runner ${CLEAN_ARG}
|
||||||
|
printf '%s' "${MAIN_C_FLAGS}" > build/runners/c/equix_runner.flags
|
||||||
|
echo " -> build/runners/c/equix_runner (baseline flags: ${MAIN_C_FLAGS})"
|
||||||
|
|
||||||
|
# Then pick the fastest optimization flags on THIS machine and install the winner
|
||||||
|
# as the main runner, so the benchmark runs the fastest build the host can make
|
||||||
|
# (solve is JIT-dominated, so the win is usually small, but it is now measured,
|
||||||
|
# not assumed). Set EQUIX_NO_AUTOTUNE=1 to skip (keeps the baseline above).
|
||||||
|
if [ "${EQUIX_NO_AUTOTUNE:-0}" = 1 ]; then
|
||||||
|
echo " (autotune disabled via EQUIX_NO_AUTOTUNE; using ${MAIN_C_FLAGS})"
|
||||||
|
else
|
||||||
|
echo "==> [3b/6] Auto-selecting the fastest C flags for the main runner"
|
||||||
|
./scripts/autotune_c_flags.sh || echo " (autotune failed; keeping the ${MAIN_C_FLAGS} baseline)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> [4/6] Building the Rust runner (pinned via Cargo.lock)"
|
||||||
|
cargo build --locked --release --manifest-path runners/rust/Cargo.toml
|
||||||
|
echo " -> runners/rust/target/release/equix_runner"
|
||||||
|
|
||||||
|
echo "==> [5/6] Installing the Python harness into a project venv (.venv)"
|
||||||
|
# A project virtualenv is the robust fix for the Linux failure "pytest not
|
||||||
|
# importable / externally-managed-environment": PEP-668 interpreters (Debian,
|
||||||
|
# Homebrew) reject a plain `pip install` into the system site-packages, but a
|
||||||
|
# venv has its own writable site-packages, so pip (matplotlib/numpy/pytest) just
|
||||||
|
# works and stays isolated from the system Python. It is stdlib-only — no pyenv
|
||||||
|
# or extra tooling required. run_all.sh and the Makefile auto-prefer .venv/bin.
|
||||||
|
VENV_DIR=".venv"
|
||||||
|
if [ ! -x "$VENV_DIR/bin/python" ]; then
|
||||||
|
if ! python3 -m venv "$VENV_DIR" >/dev/null 2>&1; then
|
||||||
|
# Debian/Ubuntu split venv into the python3-venv package; provision and retry.
|
||||||
|
if [ "$PM" = apt-get ] && [ "$EQUIX_NO_AUTO_INSTALL" != 1 ]; then
|
||||||
|
echo " (python3 -m venv unavailable; installing python3-venv)"
|
||||||
|
pm_install python3-venv >/dev/null 2>&1 || true
|
||||||
|
python3 -m venv "$VENV_DIR" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [ -x "$VENV_DIR/bin/python" ]; then
|
||||||
|
VENV_PY="$VENV_DIR/bin/python"
|
||||||
|
"$VENV_PY" -m pip install -q --upgrade pip >/dev/null 2>&1 || true
|
||||||
|
if "$VENV_PY" -m pip install -q -e ./harness pytest; then
|
||||||
|
echo " -> equix_bench + pytest installed into $VENV_DIR"
|
||||||
|
echo " (run via: $VENV_DIR/bin/python -m equix_bench ...; run_all.sh / make use it automatically)"
|
||||||
|
else
|
||||||
|
echo " (pip install into $VENV_DIR failed; see errors above)"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# No venv possible (e.g. locked-down host): fall back through system pip modes.
|
||||||
|
echo " (could not create $VENV_DIR; falling back to system pip)"
|
||||||
|
if python3 -m pip install -e ./harness >/dev/null 2>&1 \
|
||||||
|
|| python3 -m pip install --user -e ./harness >/dev/null 2>&1 \
|
||||||
|
|| python3 -m pip install --break-system-packages -e ./harness >/dev/null 2>&1; then
|
||||||
|
echo " -> equix_bench installed (system interpreter)"
|
||||||
|
else
|
||||||
|
echo " (could not pip install automatically; run: python3 -m pip install -e ./harness)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> [6/6] Writing provenance"
|
||||||
|
mkdir -p build
|
||||||
|
cat > build/provenance.json <<EOF
|
||||||
|
{
|
||||||
|
"equix_commit": "${EQUIX_COMMIT}",
|
||||||
|
"hashx_commit": "${HASHX_COMMIT}",
|
||||||
|
"rust_equix": "0.7.0",
|
||||||
|
"rust_hashx": "0.9.0",
|
||||||
|
"cc": "$( (cc --version 2>/dev/null || echo n/a) | head -1)",
|
||||||
|
"cc_flags": "$(cat build/runners/c/equix_runner.flags 2>/dev/null || echo n/a)",
|
||||||
|
"rustc": "$(rustc --version 2>/dev/null | awk '{print $2}' || echo n/a)"
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
cat build/provenance.json
|
||||||
|
|
||||||
|
echo "==> Done. Try:"
|
||||||
|
PY_HINT="python3"; [ -x "$VENV_DIR/bin/python" ] && PY_HINT="$VENV_DIR/bin/python"
|
||||||
|
echo " $PY_HINT -m equix_bench run --config configs/smoke.toml --out results/"
|
||||||
38
tools/benchmarks/Equi-X/scripts/verify.sh
Executable file
38
tools/benchmarks/Equi-X/scripts/verify.sh
Executable file
@ -0,0 +1,38 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# End-to-end verification: build, probe each runner directly, run the smoke
|
||||||
|
# config, and assert the cross-implementation correctness gate passes.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
CRUN="build/runners/c/equix_runner"
|
||||||
|
RRUN="runners/rust/target/release/equix_runner"
|
||||||
|
|
||||||
|
echo "==> Ensuring runners are built"
|
||||||
|
[ -x "$CRUN" ] && [ -x "$RRUN" ] || ./scripts/setup.sh
|
||||||
|
|
||||||
|
probe() {
|
||||||
|
local runner="$1" name="$2"
|
||||||
|
local job='{"schema_version":1,"operation":"solve","runtime":"try-compile","challenge_hex":"deadbeef","repetitions":1,"warmup":0}'
|
||||||
|
local out; out="$(echo "$job" | "$runner" | tail -1)"
|
||||||
|
echo "$out" | grep -q '"ok":true' || { echo "FAIL: $name did not return ok:true"; echo "$out"; exit 1; }
|
||||||
|
echo "$out" | grep -q '"solutions":4' || { echo "FAIL: $name did not find 4 solutions for deadbeef"; echo "$out"; exit 1; }
|
||||||
|
echo " OK: $name solve/deadbeef -> 4 solutions"
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "==> Probing runners directly"
|
||||||
|
probe "$CRUN" "equix-c"
|
||||||
|
probe "$RRUN" "equix-rust"
|
||||||
|
|
||||||
|
echo "==> Running smoke config"
|
||||||
|
PYTHONPATH=harness python3 -m equix_bench run --config configs/smoke.toml --out results --root .
|
||||||
|
|
||||||
|
echo "==> Cross-check gate"
|
||||||
|
PYTHONPATH=harness python3 -m equix_bench run --config configs/smoke.toml --out results --root . --crosscheck-only
|
||||||
|
|
||||||
|
echo "==> Outputs"
|
||||||
|
ls -1 results/plots/*.png
|
||||||
|
test -f results/report.md && echo " report.md OK"
|
||||||
|
test -f results/results.csv && echo " results.csv OK"
|
||||||
|
echo "==> VERIFY PASSED"
|
||||||
Loading…
x
Reference in New Issue
Block a user