logos-blockchain/research
1
0
Fork 0
mirror of https://github.com/logos-blockchain/research.git synced 2026-08-07 03:33:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Importing tsi-sim v3

Browse Source
This commit is contained in:
Marcin Pawlowski 2026-07-30 18:57:10 +02:00
parent e86bb0cb6c
commit 24da2fc8b3
No known key found for this signature in database
98 changed files with 8942 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
tools/simulators/tsi/tsi-sim-pernode/.gitignore vendored Normal file
View File

@ -0,0 +1,24 @@
# Generated artifacts
runs/
results/
figures/
report-figures/
!results/.gitkeep
!figures/.gitkeep
# Python
__pycache__/
*.py[cod]
*.egg-info/
.venv/
venv/
.pytest_cache/
.mypy_cache/
.ruff_cache/
*.parquet
*.csv
# Rendered reports — the report of record (markdown + figures) lives in reports/tsi/;
# anything built or generated in the simulation folder is a local artifact, never committed.
*.pdf
*.html

52
tools/simulators/tsi/tsi-sim-pernode/Makefile Normal file
View File

@ -0,0 +1,52 @@
# Sweep targets are auto-discovered from configs/*.yaml (see the pattern rule below), so any
# new config becomes runnable as `make <stem>` with no Makefile edit.
SWEEP_CONFIGS := $(patsubst configs/%.yaml,%,$(wildcard configs/*.yaml))
.PHONY: install $(SWEEP_CONFIGS) figures verify test test-slow test-all lint clean
VENV ?= .venv
PY := $(VENV)/bin/python
PIP := $(VENV)/bin/pip
# Pin BLAS/OpenMP to a single thread so joblib's N worker processes don't oversubscribe
# the cores (numpy in each worker would otherwise each spawn a full BLAS thread pool).
export OMP_NUM_THREADS := 1
export OPENBLAS_NUM_THREADS := 1
export MKL_NUM_THREADS := 1
export NUMEXPR_NUM_THREADS := 1
$(VENV):
python3 -m venv $(VENV)
$(PIP) install --upgrade pip
install: $(VENV)
$(PIP) install -e ".[dev,accel]" # accel = numba (per-node measurement speedup)
# Run any sweep config by its file stem, e.g. `make smoke`, `make default`, `make fullscale`
# (or any new configs/<name>.yaml). Each writes results + figures into a fresh dated folder under
# runs/ (runs/<YYYY-MM-DD_HHMMSS>_<label>/) so runs never overwrite. Pass extra flags via
# SWEEP_ARGS, e.g. make fullscale SWEEP_ARGS="--batch-size 1 --mem-frac 0.6"
$(SWEEP_CONFIGS): install
$(PY) scripts/run_sweep.py --config configs/$@.yaml --label $@ $(SWEEP_ARGS)
# Re-render figures from a run's parquet into a fresh dated figures/ folder:
# make figures RESULTS=runs/<dir>/results.parquet
figures: install
$(PY) scripts/make_figures.py --results $(RESULTS)
# Analytic sanity checks (simulator vs closed-form theory; replicates run across cores).
verify: install
$(PY) scripts/verify.py
test: install # fast subset (addopts already excludes slow)
$(PY) -m pytest
test-slow: install
$(PY) -m pytest -m slow
test-all: install
$(PY) -m pytest -m ''
lint: install
$(VENV)/bin/ruff check src scripts tests
clean:
rm -rf runs/* results/*.parquet figures/* .pytest_cache .ruff_cache .mypy_cache
find . -name __pycache__ -type d -prune -exec rm -rf {} +

150
tools/simulators/tsi/tsi-sim-pernode/README.md Normal file
View File

@ -0,0 +1,150 @@
# tsi-sim-pernode — Cryptarchia TSI **per-node** network simulator (Phase 2)
> The reduced-model simulators (`../tsi-sim/`, `../tsi-sim-mc/`) collapse the network to one
> global canonical chain and one scalar `D_est` per epoch. **This package removes that
> collapse:** every one of the `N` nodes runs TSI individually with its **own** `D_est`, from
> its **own** partial view of the block tree under explicit message propagation over a peering
> graph. Its job is to *test* the reduced model's assumption that all honest nodes agree.
## What it models
- **Per-node lottery:** node `i` wins a slot with `φ_f(w_i / D_est_i)` — `D_est` is a length-`N`
**vector**, each node self-updating from its own view (the reduced model's key reuse: the
sparse sampler already takes a per-node probability vector).
- **Topology** (`topology`): three propagation models over the network.
- `full_mesh`: every node one hop away with uniform latency `L` — reproduces the reduced
model exactly (validation baseline).
- `regular`: a random **d-regular** peering graph (configurable `degree`) with per-link
latency (`link_latency_dist ∈ {fixed, uniform, exp, geo}`, all with mean
`link_latency_mean`). A block reaches a node after the shortest **weighted** path from its
producer (gossip flooding). Models **direct block gossip**.
- `blend`: the **same** d-regular graph, but a block is first routed through the **Blend
mixnet** before it is public — the producer picks `blend_hops` distinct relay nodes
uniformly at random, the block hops `producer → r₁ → … → r_hops` over the graph, each relay
waiting a `Uniform(0, blend_delay_max)` **mixing delay** before forwarding, and the last
relay's forward is the final network-wide gossip that makes the block visible. Relays are
blind forwarders (they learn the block only from that final gossip). The dominant latency is
the per-hop mixing, not the graph transport — this is the multi-slot regime where forks and
the stake underestimate appear and uncle references matter. Because the mixing delays are
`Uniform`-bounded, the windowed fork choice stays **exact** (horizon
`(blend_hops+1)·max_path_latency + blend_hops·blend_delay_max`).
- **Real-world latency (units).** Latency is in **slots** and a slot is **1 s**, so measured
internet latencies (tens–hundreds of ms) are *fractions* of a slot; arrivals are therefore
kept **sub-slot (float)**, not rounded to whole slots. `link_latency_dist=geo` draws each
link from a geographic band mixture (`~15 ms` metro → `~200 ms` antipodal, EU↔EU ≪ EU↔AU),
rescaled so `link_latency_mean` stays the mean-latency knob. So `regular` runs the realistic
sub-slot direct-gossip regime (`~0.05–0.2` slot), where forks are rare, and `blend` runs the
multi-slot Blend-mixnet regime, where per-hop mixing delays dominate.
- **Per-node views:** one global block tree plus an `(N × n_blocks)` **arrival matrix** `A`;
each node builds on / measures density over the blocks that have arrived at it. Uncle refs
are **baked at production** from the producer's view (faithful — immutable once adopted).
- **Metrics:** per-node `D_est` spread (`range`, `IQR`), canonical-chain **agreement**
(window prefix vs current tip), mean accuracy, and — with `init_dest=heterogeneous` —
transient re-convergence.
## Headline result
**Per-node `D_est` disagreement collapses to zero.** Because TSI reads density from a window
buried far past `k`-finality, and all nodes seed the recursion from a common hardcoded
genesis `D`, every node computes the **same** measured density `m` → **identical** `D_est`
(`range ≈ 0`, `agreement_window = 1`) — *even under a sparse graph with high latency and heavy
tip-level forking* (`agreement_tip` can drop well below 1). This **validates the reduced
model**. Topology/latency instead shift the shared *mean* accuracy (via fork rate → `q`),
which uncle references recover just as in the reduced model. (Injected heterogeneous
disagreement, which the real protocol never creates, is *preserved* by the common
multiplicative update — a cautionary note, not protocol behaviour.)
## Quick start
```bash
make install # venv + editable install
make test # unit + fast per-node checks
make verify # per-node validation (parity, spread→0, agreement, topology effect)
# Run any configs/<name>.yaml by its stem (auto-discovered); each writes a dated runs/ folder:
make smoke # tiny end-to-end grid + figures (configs/smoke.yaml)
make default # scaled-k divergence/topology sweep + figures (configs/default.yaml)
make fullscale # full-scale (true k) confirmation (configs/fullscale.yaml)
make figures RESULTS=runs/<dir>/results.parquet # re-render figures from a run
# Extra sweep flags: make fullscale SWEEP_ARGS="--batch-size 1 --mem-frac 0.6"
```
## Scale & performance
- **Representation:** one global block tree + `(N × n_blocks)` `float64` arrival matrix `A`
(sub-slot arrivals); topology `path_latency[N,N]` (per-node Dijkstra, once per trajectory).
- **`n_blocks` is NOT `~10·k` in general — it tracks block production.** `n_blocks` is the number
of lottery wins in an epoch, `≈ E·Σᵢφ(wᵢ/D_est)`. At equilibrium that is `~10·k` (≈ 22k at
k=2160), but when `D_est` is far below the true stake — the **collapsed-estimate regime**, e.g. a
small `genesis_d_factor` — `Σ(stake)/D_est = 1/genesis_d_factor` is large and block production
explodes proportionally. At `genesis_d_factor=0.01`, genesis epoch-0 produces **~2.0M blocks**
(100× equilibrium) → `A ≈ 15 GB` for a *single* worker; `D_est` self-corrects to equilibrium
within ~2 epochs, so only the earliest epoch(s) are heavy. **Raising `genesis_d_factor` toward
0.1–0.5 collapses this cost** (0.1 → ~0.22M blocks → ~1.6 GB; 0.5 → ~44k → ~0.4 GB) and does not
change the equilibrium result, which is measured after burn-in.
- **Sliding-window prune (`prune_arrival`, default on):** the arrival matrix never needs per-node
columns for blocks past the horizon — under deterministic latency a block with `slot ≤ t − H` has
reached *every* node, so its column is finalized and dropped. We keep columns only for blocks
inside `max(horizon, uncle_window)` slots in a base-offset buffer, turning the `O(N·n_blocks)`
matrix into `O(N · keep-span-blocks)`. This is what makes the collapsed regime affordable: at
N=1000/k=2160/`gdf=0.01` the buffer is ~tens of MB instead of the ~15 GB full matrix (fork choice,
the parent clamp, uncle selection, and per-node tips all reconstruct exactly from it). It is
**bit-identical** to the full matrix at `jitter_mean == 0` (proven by `test_prune_matches_full_matrix`
across topologies/uncles/`gdf`); with jitter it falls back to the full matrix (whose safety clamp
keeps the tree valid). Set `prune_arrival: false` to force the full matrix (the parity oracle).
The measurement pass also argmaxes in node-row bands so it adds only a small temporary. Divergence
sweeps run at scaled **k=256** (`configs/default.yaml`); full-scale k=2160 is validated to **N ≤ 2000**.
- **Worker sizing (auto, RAM-safe):** both `A` (`~N·n_blocks`, incl. the block explosion above)
and `path_latency` (`~N²`) grow, so the sweep runner sizes the loky pool to fit a RAM budget
(`--mem-frac`, default 0.7 of physical RAM) instead of blindly using every core. The per-worker
estimate realises the seeded stake to compute the **genesis-epoch** block count
(`expected_peak_blocks`), so it reflects a low-`genesis_d_factor` explosion rather than assuming
`~10·k`. A **calibration probe** measures a real worker's peak RSS (one genesis epoch of the
heaviest config in a spawned process) whenever the estimate is heavy or `N > 2000`
(`--calibrate {auto,always,never}`, default `auto`; the probe bounds itself to physical RAM so it
fails loud rather than freezing).
- **Fail-loud memory guard (`memguard.py`):** every worker checks size *before* allocating both
big arrays — the `(N × n_blocks)` `A` (in `build_tree_pernode`) and the `(N × N)` `path_latency`
(in `build_path_latency`, built first) — and raises `ArrivalMatrixTooLarge` if it would exceed
the budget `TSI_ARRIVAL_BYTES_BUDGET`. The sweep sets that to each worker's RAM share; **unset or
`0` is not "unlimited"** — it resolves to `DEFAULT_BUDGET_FRAC` (0.9) of physical RAM, so a bare
`run_trajectory`, `tsi-verify`, the probe, or a `--mem-frac 0` run all keep an absolute
per-process ceiling. So a mis-estimated block explosion (or a huge `N`) fails with a clear
message instead of freezing the machine.
- **Cost:** dominated by the per-node fork choice (batched per slot) and the arrival-matrix
fill; the sparse lottery is negligible. Across-config joblib **loky** parallelism reused.
- **Measurement optimisation (`measure.py`):** the per-node canonical/density/agreement pass
was ~95% of an epoch. It is now **deduped by tip** (nodes sharing a tip share every derived
quantity — high agreement collapses `N` to a handful of computations) and the per-tip chain
walk runs as a cached **numba** kernel (pure-Python fallback if numba is absent). Exact —
bit-identical to the naive loop (`test_measure`). Measured **~9× end-to-end** (heavy config
11.3 s → 1.2 s) and ~14× on measurement-bound configs. numba comes via the `accel` extra
(`pip install -e ".[dev,accel]"`, done by `make install`).
- **Windowed fork choice (`windowed_fork_choice`, default on):** bounds the block-tree build's
per-slot candidate scan to a horizon of the max path latency plus the fully-propagated best
tip, turning `O(n_blocks^2)` fork choice into `O(n_blocks*H)`. **Exact** when link latency is
deterministic (`jitter_mean == 0`) — bit-identical to a full scan (parity test). With
`jitter_mean > 0` it becomes a tiny approximation and **warns**; a safety clamp still keeps
the tree valid, and `windowed_fork_choice=False` forces a guaranteed-exact full scan.
- **Reproducibility:** every draw spawns off `SeedSequence(hash(config))` — child 0 stake,
1 graph, 2 init, 3+e epoch `e`. `graph_seed`/`degree`/`link_latency_*` are part of the
config identity.
- **numpy-version caveat:** the `accel` extra (numba) requires `numpy<2.5`, so installing it
pins numpy to 2.4.x. numpy's `Generator.choice(replace=False)` is **not** stream-stable
across the 2.4↔2.5 boundary, and the sparse lottery uses it heavily only in the degenerate
*collapsed-estimate* regime (`D_est → 0` ⇒ win-prob → 1 ⇒ `count ≈ n_slots`). So a run on
numpy 2.5 and a run on numpy 2.4 give **identical results for all normal configs** but can
diverge chaotically in that one extreme regime (e.g. `degree=4, link_latency=8`, where the
estimate has already collapsed to ~0.13 — off the safe chart). The differences are tiny
(max |Δ mean_ratio| ≈ 3e-3) and change no conclusion; pin numpy if bit-reproducibility
across environments is required.
## Layout
```
src/tsi_sim/ constants config rng stake lottery topology blocktree(+build_tree_pernode)
uncles(+select_uncles_at_production) tsi(+update_D_vec) epoch engine metrics
theory verify plotting/{style, figures_pernode, make_figures}
configs/ smoke.yaml default.yaml fullscale.yaml
tests/ test_{pernode,config,rng,lottery,blocktree,uncles,tsi_counting,stake,
theory,latency,theory_convergence}.py
```

25
tools/simulators/tsi/tsi-sim-pernode/configs/blend-hops-delay.yaml Normal file
View File

@ -0,0 +1,25 @@
# Blend mixnet: find the (hops, delay) regime where ONE uncle stops being enough.
# More relay hops and larger per-hop mixing delay push block visibility later -> more concurrent
# proposals -> more orphaned honest blocks per canonical block. A block can reference at most U
# uncles, so once the number of orphans per canonical block exceeds ~1, U=1 can no longer recover
# the full block density and the stake estimate stays low. This sweep brackets that transition.
# Latency is in SLOTS (1 slot = 1 s). Sweep axes (cartesian product x replicates).
n_nodes: [1000, 2000] # network sizes to compare
stake_dist: [pareto] # heavy-tailed (realistic) stake distribution
topology: [blend] # Blend mixnet only (this study is about the cascade)
degree: [6] # peering degree of the underlying d-regular graph
link_latency_mean: [0.5] # per-link graph latency (secondary to mixing delay)
link_latency_dist: [geo] # real-world geographic band mixture
blend_hops: [3, 4, 5, 6] # number of random relay hops in the mix cascade
blend_delay_max: [4.0, 8.0, 16.0, 32.0] # max per-relay mixing delay (slots); delay ~ U(0, this)
max_uncles: [0, 1, 2, 4] # U: 0 baseline, 1 = the question, 2/4 = how many needed
uncle_strategy: [oldest] # uncle selection: oldest-first fill
init_dest: [common] # per-node initial D_est from agreement
replicates: 10 # independent RNG replicates per grid cell
base: # per-run settings shared by every cell (not swept)
k: 2160 # true security parameter
epochs: 20 # equilibrium is reached within ~2 epochs; burn 50%
f: 0.03333333333333333 # slot activation coefficient (1/30)
genesis_d_factor: 0.5 # start near true stake (cheap epoch 0; equilibrium is
# what the U-recovery is measured on, gdf-independent)
early_stop: true

26
tools/simulators/tsi/tsi-sim-pernode/configs/block-rate.yaml Normal file
View File

@ -0,0 +1,26 @@
# How do the findings change with the block production rate f? All prior studies fixed f=1/30
# (one block every 30 s, since 1 slot = 1 s). f sets the block interval 1/f, which the uncle-window
# study showed is what drives the required W — and a denser block rate means a given delay spans
# more blocks, so more forking. Here we compare block intervals of 10, 15, 20, 30 s (f = 1/10,
# 1/15, 1/20, 1/30) across delay, uncle cap U, and window W. Blend mixnet, geographic transport.
n_nodes: [1000] # single N (the f-dependence is the focus)
stake_dist: [pareto] # heavy-tailed (realistic) stake distribution
topology: [blend] # Blend mixnet (multi-slot delay)
degree: [6] # peering degree of the d-regular graph
link_latency_mean: [0.5] # natural geographic transport (sub-slot)
link_latency_dist: [geo] # real-world geographic band mixture
blend_hops: [3] # fixed hop count
blend_delay_max: [4.0, 8.0, 16.0, 32.0] # per-hop mixing delay (slots = seconds)
uncle_window: [30, 100, 300] # W: a few windows to see the W-vs-f interaction
max_uncles: [0, 1, 2] # U: baseline, one uncle, two uncles
uncle_strategy: [oldest] # uncle selection: oldest-first fill
init_dest: [common] # per-node initial D_est from agreement
# f = 1 / (block interval in seconds): 1/10, 1/15, 1/20, 1/30. Everything else (epoch/window
# geometry) derives from (k, f), so a denser f simply shortens the epoch and packs blocks closer.
f: [0.1, 0.06666666666666667, 0.05, 0.03333333333333333]
replicates: 10 # independent RNG replicates per grid cell
base: # per-run settings shared by every cell
k: 2160 # true security parameter
epochs: 20 # equilibrium reached within ~2 epochs; burn 50%
genesis_d_factor: 0.5 # start near true stake (cheap epoch 0)
early_stop: true

28
tools/simulators/tsi/tsi-sim-pernode/configs/default.yaml Normal file
View File

@ -0,0 +1,28 @@
# Per-node divergence + topology sweep (scaled k). Headline: per-node D_est spread stays ~0
# (window agreement ~1) while topology/latency shift the shared mean accuracy, which uncles
# recover. Two propagation models are swept, each covering a real-world regime:
# * regular : direct block gossip over a random d-regular graph. Realistic links are sub-slot
# (~0.05-0.2 slot = 50-200 ms), so forks are rare and TSI barely underestimates.
# * blend : the same graph, but each block is relayed through `blend_hops` random nodes
# (a mix cascade, per-hop delay ~ U(0, blend_delay_max)) before a final gossip.
# This is the multi-slot Blend-mixnet regime where forks and the stake
# underestimate appear and uncle references matter.
# Latency is in SLOTS (1 slot = 1 s). Sweep axes (cartesian product x replicates).
n_nodes: [400] # number of nodes / stake holders
stake_dist: [uniform] # stake distribution (uniform = equal stake)
topology: [regular, blend] # direct gossip vs Blend mixnet (both on the d-regular graph)
degree: [4, 8, 16] # peering degree (higher = better connected)
link_latency_mean: [0.05, 0.1, 0.2] # mean one-way per-link latency (slots); regular's main axis
link_latency_dist: [geo] # real-world geographic band mixture (see constants)
blend_hops: [3] # blend: number of random relay hops in the mix cascade
blend_delay_max: [0.5, 1.0, 2.0, 4.0] # blend: max per-relay mixing delay (slots); blend's main axis
max_uncles: [0, 1, 2, 4] # U: max uncle references per block (0 = baseline)
uncle_strategy: [oldest] # uncle selection: oldest-first fill
init_dest: [common, heterogeneous] # initial per-node D_est: agreement vs disagreement
replicates: 5 # independent RNG replicates per grid cell
base: # per-run settings shared by every cell (not swept)
k: 256 # security parameter (scaled; T = 6*floor(256/f) slots)
epochs: 35 # epochs simulated per trajectory
f: 0.03333333333333333 # slot activation coefficient (default 1/30); configurable
genesis_d_factor: 0.5 # genesis D_est = factor x true total stake
init_spread: 0.5 # heterogeneous-start relative spread of initial D_est

20
tools/simulators/tsi/tsi-sim-pernode/configs/expdist.yaml Normal file
View File

@ -0,0 +1,20 @@
# S1: link-latency SHAPE sensitivity — exponential vs geo at the same mean (report §2).
# The geo baseline at identical cells lives in the nscaling-a run; this config runs the exp arm.
n_nodes: [1000, 4000]
stake_dist: [pareto]
topology: [blend]
degree: [6]
link_latency_mean: [0.5]
link_latency_dist: [exp]
blend_hops: [3]
blend_delay_max: [4.0, 8.0]
max_uncles: [0, 1, 2]
uncle_strategy: [oldest]
init_dest: [common]
replicates: 4
base:
k: 256
epochs: 40
f: 0.03333333333333333
genesis_d_factor: 0.5
early_stop: true

20
tools/simulators/tsi/tsi-sim-pernode/configs/fullscale-small.yaml Normal file
View File

@ -0,0 +1,20 @@
# N = 1000/2000 tier of the full-scale confirmation (same grid as fullscale.yaml, which covers
# 5000/10000) — committed so all four N tiers reproduce without editing configs.
n_nodes: [1000, 2000]
stake_dist: [pareto]
topology: [regular, blend]
degree: [4, 6]
link_latency_mean: [0.1, 0.2, 0.5, 1.0]
link_latency_dist: [geo]
blend_hops: [3]
blend_delay_max: [1.0, 2.0, 3.0]
max_uncles: [0, 1, 2, 3]
uncle_strategy: [oldest]
init_dest: [common]
replicates: 10
base:
k: 2160
epochs: 100
f: 0.03333333333333333
genesis_d_factor: 0.1
early_stop: true

22
tools/simulators/tsi/tsi-sim-pernode/configs/fullscale.yaml Normal file
View File

@ -0,0 +1,22 @@
# Full-scale confirmation at TRUE k=2160, HEAVY tier N=5000/10000 (the N=1000/2000 tier is
# fullscale-small.yaml). Lean grid: the corrected mechanism shifts the U>=1 plateau to 1.0 and
# leaves consensus/thresholds unchanged, so this confirms the level + N-scaling, not the full
# product. degree 4/6 kept for the fig18/fig19 (Appendix C) comparisons.
n_nodes: [5000, 10000]
stake_dist: [pareto]
topology: [regular, blend]
degree: [4, 6]
link_latency_mean: [0.5]
link_latency_dist: [geo]
blend_hops: [3]
blend_delay_max: [1.0, 2.0, 3.0]
max_uncles: [0, 1, 2]
uncle_strategy: [oldest]
init_dest: [common]
replicates: 6
base:
k: 2160
epochs: 100
f: 0.03333333333333333
genesis_d_factor: 0.1
early_stop: true

23
tools/simulators/tsi/tsi-sim-pernode/configs/nscaling-a.yaml Normal file
View File

@ -0,0 +1,23 @@
# N-scaling of the U=1 boundary, case (a): blend, geo delays, NO jitter. Measures how the
# one-uncle sufficiency erodes as the network grows (gossip diameter ~ log N raises D_vis and
# the load rho = f*D_vis). Scaled k=256 (robustness-study convention); ladder to N=16000
# (the 32k tier is configs/nscaling32-a.yaml; N >= 10^5 is covered by the exact topology
# probe scripts/topology_probe.py + the validated load law — report §3.8).
n_nodes: [1000, 2000, 4000, 8000, 16000]
stake_dist: [pareto]
topology: [blend]
degree: [4, 6, 8]
link_latency_mean: [0.5]
link_latency_dist: [geo]
blend_hops: [3]
blend_delay_max: [4.0, 8.0]
max_uncles: [0, 1, 2]
uncle_strategy: [oldest]
init_dest: [common]
replicates: 4
base:
k: 256
epochs: 40
f: 0.03333333333333333
genesis_d_factor: 0.5
early_stop: true

26
tools/simulators/tsi/tsi-sim-pernode/configs/nscaling-b.yaml Normal file
View File

@ -0,0 +1,26 @@
# N-scaling of the U=1 boundary, case (b): geo delays + Poisson long-tail jitter (10% of deliveries straggle by Poisson(3) slots). Measures how the
# one-uncle sufficiency erodes as the network grows (gossip diameter ~ log N raises D_vis and
# the load rho = f*D_vis). Scaled k=256 (robustness-study convention); ladder to N=16000
# (the 32k tier is configs/nscaling32-b.yaml; N >= 10^5 is covered by the exact topology
# probe scripts/topology_probe.py + the validated load law — report §3.8).
n_nodes: [1000, 2000, 4000, 8000, 16000]
stake_dist: [pareto]
topology: [blend]
degree: [4, 6, 8]
link_latency_mean: [0.5]
link_latency_dist: [geo]
blend_hops: [3]
blend_delay_max: [4.0, 8.0]
max_uncles: [0, 1, 2]
uncle_strategy: [oldest]
init_dest: [common]
replicates: 4
base:
k: 256
epochs: 40
f: 0.03333333333333333
genesis_d_factor: 0.5
jitter_mean: 3.0
jitter_dist: poisson
jitter_frac: 0.1
early_stop: true

20
tools/simulators/tsi/tsi-sim-pernode/configs/nscaling32-a.yaml Normal file
View File

@ -0,0 +1,20 @@
# 32k tier of the N-scaling ladder, case (a) — boundary delay only (memory-bound: the N x N
# path matrix is ~8 GB/worker, so this tier runs with few workers).
n_nodes: [32000]
stake_dist: [pareto]
topology: [blend]
degree: [4, 6, 8]
link_latency_mean: [0.5]
link_latency_dist: [geo]
blend_hops: [3]
blend_delay_max: [8.0]
max_uncles: [1, 2]
uncle_strategy: [oldest]
init_dest: [common]
replicates: 3
base:
k: 256
epochs: 40
f: 0.03333333333333333
genesis_d_factor: 0.5
early_stop: true

23
tools/simulators/tsi/tsi-sim-pernode/configs/nscaling32-b.yaml Normal file
View File

@ -0,0 +1,23 @@
# 32k tier of the N-scaling ladder, case (b): + Poisson long-tail jitter — boundary delay only (memory-bound: the N x N
# path matrix is ~8 GB/worker, so this tier runs with few workers).
n_nodes: [32000]
stake_dist: [pareto]
topology: [blend]
degree: [4, 6, 8]
link_latency_mean: [0.5]
link_latency_dist: [geo]
blend_hops: [3]
blend_delay_max: [8.0]
max_uncles: [1, 2]
uncle_strategy: [oldest]
init_dest: [common]
replicates: 3
base:
k: 256
epochs: 40
f: 0.03333333333333333
genesis_d_factor: 0.5
jitter_mean: 3.0
jitter_dist: poisson
jitter_frac: 0.1
early_stop: true

21
tools/simulators/tsi/tsi-sim-pernode/configs/pareto133.yaml Normal file
View File

@ -0,0 +1,21 @@
# S2: stake-concentration sensitivity — Pareto tail 1.33 vs the default 1.16 (report §2).
# The 1.16 baseline at identical cells lives in the nscaling-a run; this config runs the 1.33 arm.
n_nodes: [1000, 4000]
stake_dist: [pareto]
topology: [blend]
degree: [6]
link_latency_mean: [0.5]
link_latency_dist: [geo]
blend_hops: [3]
blend_delay_max: [4.0, 8.0]
max_uncles: [0, 1, 2]
uncle_strategy: [oldest]
init_dest: [common]
replicates: 4
base:
k: 256
epochs: 40
f: 0.03333333333333333
genesis_d_factor: 0.5
pareto_shape: 1.33
early_stop: true

25
tools/simulators/tsi/tsi-sim-pernode/configs/reorg-depth.yaml Normal file
View File

@ -0,0 +1,25 @@
# Fork rate and reorg depth vs uncle cap, under a private-chain "deepest-reorg" adversary.
# adversary_frac {0, 0.1, 0.2, 0.3} x U {0,1,2,3}, Blend, N = 1000, k = 256. The adversary
# (adversary_strategy: private_chain) builds a hidden chain on its own tip and releases it to
# maximise the override depth. Metrics: fork_rate (orphaned/total) and max_reorg_depth per epoch.
# NOTE: run only AFTER the corrected-mechanism engine changes land (needs the new strategy +
# fork-depth measurement). Placeholder committed with the design; wired by scripts/reorg_depth.py.
n_nodes: [1000]
stake_dist: [pareto]
topology: [blend]
degree: [6]
link_latency_mean: [0.5]
link_latency_dist: [geo]
blend_hops: [3]
blend_delay_max: [8.0]
max_uncles: [0, 1, 2, 3]
uncle_strategy: [oldest]
init_dest: [common]
replicates: 8
base:
k: 256
epochs: 30
f: 0.03333333333333333
genesis_d_factor: 0.5
adversary_strategy: private_chain
early_stop: false # reorg-depth statistics need the full epoch sample

27
tools/simulators/tsi/tsi-sim-pernode/configs/rho-boundary.yaml Normal file
View File

@ -0,0 +1,27 @@
# Rho-boundary deficit sweep (Phase 2 of the "below the block rate" reframing).
# Goal: densely resolve the UNDER-COUNT DEFICIT 1 - D̂/D as a function of the load ρ, and confirm
# the equilibrium is bounded by 1 (no cell above 1 beyond sampling noise) with tight SEM.
# hops is fixed at 3 so ρ = f·D_vis ∝ blend_delay_max (a clean ρ axis); δ is sampled densely around
# the ρ≈1 boundary. High replicate count beats down the per-cell SEM so the ≤1 bound and the
# U=⌈ρ⌉ recovery boundary are resolved cleanly. k=256 (the robustness/scaling convention, §3.7):
# the ρ-deficit mechanics are k-invariant (§3.1), and k=256's short epochs make a dense, 20-replicate
# sweep tractable where k=2160 would not.
# Latency is in SLOTS (1 slot = 1 s). Sweep axes (cartesian product x replicates).
n_nodes: [1000] # fix N to isolate ρ (N enters only through ℓ_mean, §3.7)
stake_dist: [pareto] # heavy-tailed (realistic) stake distribution
topology: [blend] # Blend mixnet only (the regime where U/W matter)
degree: [6] # peering degree of the underlying d-regular graph
link_latency_mean: [0.5] # per-link graph latency (secondary to mixing delay)
link_latency_dist: [geo] # real-world geographic band mixture
blend_hops: [3] # fixed → ρ ∝ blend_delay_max
blend_delay_max: [8.0, 12.0, 15.0, 16.0, 17.0, 18.0, 20.0, 24.0, 28.0, 36.0] # ρ ≈ 0.56 … 2.0, dense near 1
max_uncles: [0, 1, 2, 3] # U: 0 (raw deficit), 1/2/3 (recovery vs ⌈ρ⌉)
uncle_strategy: [oldest] # uncle selection: oldest-first fill
init_dest: [common] # per-node initial D_est from agreement
replicates: 20 # high replicate count → tight SEM on the deficit
base: # per-run settings shared by every cell (not swept)
k: 256 # scaled security parameter (ρ mechanics are k-invariant)
epochs: 20 # equilibrium within ~2 epochs; burn 50 %
f: 0.03333333333333333 # slot activation coefficient (1/30)
genesis_d_factor: 0.5 # start near true stake; equilibrium is gdf-independent
early_stop: true

21
tools/simulators/tsi/tsi-sim-pernode/configs/smoke.yaml Normal file
View File

@ -0,0 +1,21 @@
# Tiny end-to-end per-node smoke grid (fast; dev + CI-style check).
# Sweep axes (cartesian product x replicates); every value is a list.
n_nodes: [200] # number of nodes / stake holders in the network
stake_dist: [uniform, pareto] # stake distribution across nodes (equal vs heavy-tailed)
topology: [full_mesh, regular, blend] # full mesh (uniform L), d-regular graph, or Blend mixnet
degree: [4, 8] # peering degree of the regular / blend graph (ignored by full_mesh)
link_latency_mean: [0.1, 2.0] # mean one-way per-link latency in SLOTS (1 slot = 1 s):
# 0.1 = realistic direct gossip (~100 ms); 2.0 = slow transport
link_latency_dist: [geo] # per-link latency law: geo = real-world geographic band mixture
blend_hops: [3] # blend: number of random relay hops in the mix cascade
blend_delay_max: [1.0, 6.0] # blend: max per-relay mixing delay (slots); delay ~ U(0, this)
latency: [2] # full_mesh uniform link latency L (slots); ignored otherwise
max_uncles: [0, 2] # U: max uncle references per block (0 = baseline, no uncles)
uncle_strategy: [oldest] # uncle selection: oldest-first (deterministic fill) or random
init_dest: [common] # per-node initial D_est: common (from agreement) or heterogeneous
replicates: 3 # independent RNG replicates per grid cell
base: # per-run settings shared by every cell (not swept)
k: 16 # security parameter (scaled down; true value 2160)
epochs: 15 # number of epochs simulated per trajectory
f: 0.03333333333333333 # slot activation coefficient (default 1/30); configurable
genesis_d_factor: 0.5 # genesis D_est = factor x true total stake

26
tools/simulators/tsi/tsi-sim-pernode/configs/uncle-window.yaml Normal file
View File

@ -0,0 +1,26 @@
# When does the uncle window W fail? At a FIXED single uncle (U=1), sweep the reference window W
# against the block-visibility delay. An uncle can only reference an orphan whose slot is within W
# of the referencing block; larger delay spreads the orphans over a wider slot range, so a small W
# can no longer reach them and the stake estimate stays low. This finds the (W, delay) relation.
# Transport is the natural geographic latency (link_latency_dist: geo), as in the prior runs; the
# delay that stresses W is the Blend mixnet per-hop mixing (blend_delay_max).
# Latency is in SLOTS (1 slot = 1 s). Sweep axes (cartesian product x replicates).
n_nodes: [1000, 2000] # network sizes to compare
stake_dist: [pareto] # heavy-tailed (realistic) stake distribution
topology: [blend] # Blend mixnet (multi-slot delay stresses W)
degree: [6] # peering degree of the d-regular graph
link_latency_mean: [0.5] # natural geographic transport (sub-slot)
link_latency_dist: [geo] # real-world geographic band mixture
blend_hops: [3] # fixed hop count; delay is the swept knob
blend_delay_max: [2.0, 4.0, 8.0, 16.0, 32.0] # max per-relay mixing delay (slots) = the delay
uncle_window: [10, 15, 20, 25, 30, 35, 50, 100, 200, 300] # W: reference window (slots) to sweep
max_uncles: [1] # FIXED at one uncle (the question is about W)
uncle_strategy: [oldest] # uncle selection: oldest-first fill
init_dest: [common] # per-node initial D_est from agreement
replicates: 10 # independent RNG replicates per grid cell
base: # per-run settings shared by every cell
k: 2160 # true security parameter
epochs: 20 # equilibrium reached within ~2 epochs; burn 50%
f: 0.03333333333333333 # slot activation coefficient (1/30)
genesis_d_factor: 0.5 # start near true stake (cheap epoch 0)
early_stop: true

22
tools/simulators/tsi/tsi-sim-pernode/configs/window-scale.yaml Normal file
View File

@ -0,0 +1,22 @@
# S3: uncle-window sufficiency at scale (N = 1 000 vs 10 000) and the W-as-buffer question:
# does a wider window absorb block-production fluctuations near the load boundary (rho ~ 1)
# and soften the near-integer U undershoot? Report §3.4 / §8.
n_nodes: [1000, 10000]
stake_dist: [pareto]
topology: [blend]
degree: [6]
link_latency_mean: [0.5]
link_latency_dist: [geo]
blend_hops: [3]
blend_delay_max: [8.0, 16.0, 32.0]
max_uncles: [1, 2]
uncle_window: [50, 100, 200, 300, 450, 600]
uncle_strategy: [oldest]
init_dest: [common]
replicates: 3
base:
k: 256
epochs: 40
f: 0.03333333333333333
genesis_d_factor: 0.5
early_stop: true

23
tools/simulators/tsi/tsi-sim-pernode/configs/window-uncles.yaml Normal file
View File

@ -0,0 +1,23 @@
# Joint (W, U) safe region: co-sweep the uncle window W and the uncle cap U at the transition
# delays found earlier (U=1 is W-limited at delay ~8 and U-limited at delay >=16). Shows where
# "widen the window" stops helping and "add an uncle" takes over. Blend mixnet, geographic
# transport (geo), hops=3. Latency is in SLOTS (1 slot = 1 s).
n_nodes: [1000, 2000] # network sizes to compare
stake_dist: [pareto] # heavy-tailed (realistic) stake distribution
topology: [blend] # Blend mixnet (multi-slot delay)
degree: [6] # peering degree of the d-regular graph
link_latency_mean: [0.5] # natural geographic transport (sub-slot)
link_latency_dist: [geo] # real-world geographic band mixture
blend_hops: [3] # fixed hop count
blend_delay_max: [8.0, 16.0, 32.0] # delays spanning the U=1 W-limited -> U-limited transition
uncle_window: [10, 20, 30, 50, 100, 200, 300] # W: reference window (slots)
max_uncles: [1, 2, 3, 4] # U: uncle cap (the co-swept lever)
uncle_strategy: [oldest] # uncle selection: oldest-first fill
init_dest: [common] # per-node initial D_est from agreement
replicates: 10 # independent RNG replicates per grid cell
base: # per-run settings shared by every cell
k: 2160 # true security parameter
epochs: 20 # equilibrium reached within ~2 epochs; burn 50%
f: 0.03333333333333333 # slot activation coefficient (1/30)
genesis_d_factor: 0.5 # start near true stake (cheap epoch 0)
early_stop: true

47
tools/simulators/tsi/tsi-sim-pernode/pyproject.toml Normal file
View File

@ -0,0 +1,47 @@
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project]
name = "tsi-sim-pernode"
version = "0.1.0"
description = "Per-node network simulation of Cryptarchia Total Stake Inference (Phase 2)"
readme = "README.md"
requires-python = ">=3.11"
license = { text = "MIT" }
authors = [{ name = "Logos" }]
dependencies = [
"numpy>=1.26",
"pandas>=2.1",
"pyarrow>=14",
"matplotlib>=3.8",
"scipy>=1.11",
"pyyaml>=6.0",
"tqdm>=4.66",
"joblib>=1.3",
]
[project.optional-dependencies]
dev = ["pytest>=8.0", "pytest-xdist>=3.5", "ruff>=0.5", "mypy>=1.8"]
accel = ["numba>=0.60"] # optional; only used if a single-config tree build is a bottleneck
[project.scripts]
tsi-sweep = "tsi_sim.sweep:main"
tsi-verify = "tsi_sim.verify:main"
tsi-figures = "tsi_sim.plotting.make_figures:main"
[tool.hatch.build.targets.wheel]
packages = ["src/tsi_sim"]
[tool.ruff]
line-length = 100
target-version = "py311"
[tool.ruff.lint]
select = ["E", "F", "I", "UP", "B", "NPY"]
ignore = ["E741"] # allow single-char names like L (latency), q, m — domain symbols
[tool.pytest.ini_options]
testpaths = ["tests"]
markers = ["slow: long-running (full-scale k) checks"]
addopts = "-q -m 'not slow'" # fast subset by default; `make test-all` runs everything

1
tools/simulators/tsi/tsi-sim-pernode/requirements-dev.txt Normal file
View File

@ -0,0 +1 @@
-e .[dev]

1
tools/simulators/tsi/tsi-sim-pernode/requirements.txt Normal file
View File

@ -0,0 +1 @@
-e .

91
tools/simulators/tsi/tsi-sim-pernode/scripts/adversary_figs.py Normal file
View File

@ -0,0 +1,91 @@
"""Render fig8 (uncle-suppression grinding, §6.3) and fig9 (withhold vs suppress, §6.4).
Sources (committed):
runs/adversary_grid/suppress.parquet -> fig8_adversary.png
runs/adversary_grid/withhold.parquet -> fig9_withhold.png
Both plot D_hat/D vs beta_adv, mean over replicates with a min/max band. Uses the shared
tsi_sim.plotting.style theme. Note post-fix: the honest equilibrium is now 1.0 (not 1.017),
so suppression's ratios sit near 1.0 at low load rather than above it.
"""
from __future__ import annotations
from pathlib import Path
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from tsi_sim.plotting import style
HERE = Path(__file__).resolve().parent.parent
GRID = HERE / "runs" / "adversary_grid"
FIGS = HERE / "report-figures"
# blending budget (s) -> operating load rho (per §6.3)
LOAD = {8.0: 0.56, 16.0: 0.96, 24.0: 1.36}
def _agg(df: pd.DataFrame, keys: list[str]) -> pd.DataFrame:
return df.groupby(keys).mean_ratio.agg(["mean", "min", "max"]).reset_index()
def fig8_suppress() -> Path:
df = pd.read_parquet(GRID / "suppress.parquet")
g = _agg(df, ["delay", "beta_adv"])
fig, ax = plt.subplots()
for i, delay in enumerate(sorted(g.delay.unique())):
s = g[g.delay == delay].sort_values("beta_adv")
c = style.color_for(i)
x = s.beta_adv.to_numpy()
ax.fill_between(x, s["min"], s["max"], color=c, alpha=0.15, linewidth=0)
ax.plot(x, s["mean"], color=c, marker="o", ms=4,
label=rf"$\rho$ = {LOAD[delay]:.2f}")
ax.axhline(1.0, color="0.4", ls="--", lw=1.0, zorder=0)
ax.set_xlabel(r"adversary stake fraction $\beta_{\mathrm{adv}}$")
ax.set_ylabel(r"$\hat{D}/D$ (uncle suppression)")
ax.set_title("Uncle-suppression grinding scales with load")
ax.set_xticks([0.1, 0.3, 0.5])
ax.legend(title="load", loc="lower left")
out = FIGS / "fig8_adversary"
return style.save(fig, out, provenance="runs/adversary_grid/suppress.parquet")[0]
def fig9_withhold() -> Path:
df = pd.read_parquet(GRID / "withhold.parquet")
g = _agg(df, ["strategy", "topo", "beta_adv"])
fig, ax = plt.subplots()
order = [("withhold", "regular"), ("withhold", "blend"),
("suppress", "regular"), ("suppress", "blend")]
styles = {"withhold": "-", "suppress": "--"}
colors = {"regular": style.color_for(1), "blend": style.color_for(0)}
for strat, topo in order:
s = g[(g.strategy == strat) & (g.topo == topo)].sort_values("beta_adv")
c = colors[topo]
x = s.beta_adv.to_numpy()
ax.fill_between(x, s["min"], s["max"], color=c, alpha=0.10, linewidth=0)
ax.plot(x, s["mean"], color=c, ls=styles[strat], marker="o", ms=4,
label=f"{strat}, {topo}")
# active-stake reference line (1 - beta_adv)
xr = np.linspace(0.1, 0.5, 50)
ax.plot(xr, 1 - xr, color="0.4", ls=":", lw=1.2,
label=r"$1-\beta_{\mathrm{adv}}$ (active stake)")
ax.set_xlabel(r"adversary stake fraction $\beta_{\mathrm{adv}}$")
ax.set_ylabel(r"$\hat{D}/D$")
ax.set_title("Withholding tracks active stake; suppression barely moves it")
ax.set_xticks([0.1, 0.2, 0.3, 0.4, 0.5])
ax.legend(loc="lower left", ncol=1)
out = FIGS / "fig9_withhold"
return style.save(fig, out, provenance="runs/adversary_grid/withhold.parquet")[0]
def main() -> None:
style.apply_style()
p8 = fig8_suppress()
p9 = fig9_withhold()
print("wrote", p8)
print("wrote", p9)
if __name__ == "__main__":
main()

74
tools/simulators/tsi/tsi-sim-pernode/scripts/adversary_grid.py Normal file
View File

@ -0,0 +1,74 @@
#!/usr/bin/env python
"""Adversary grids (§6.3 uncle suppression, §6.4 withhold vs suppress) — committed generators.
Suppression x load (§6.3): beta_adv {0.1, 0.3, 0.5} x blending budget {8, 16, 24} s
(loads rho ~ 0.56 / 0.96 / 1.36), U = 2, N = 1000 -> runs/adversary_grid/suppress.parquet.
Withhold vs suppress (§6.4): beta_adv {0.1..0.5} x {regular, blend} x strategy, N = 400,
exact oracle -> runs/adversary_grid/withhold.parquet.
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import pandas as pd
from joblib import Parallel, delayed
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
EPOCHS = 20
REPS = 16 # withhold at N=400 is noisy; average enough replicates for a clean curve
def _suppress_cell(beta_adv: float, delay: float, rep: int) -> dict:
cfg = SimConfig(n_nodes=1000, stake_dist="pareto", topology="blend", degree=6,
link_latency_mean=0.5, link_latency_dist="geo", blend_hops=3,
blend_delay_max=delay, max_uncles=2, uncle_window=300, k=256,
epochs=EPOCHS, genesis_d_factor=0.5,
adversary_frac=beta_adv, adversary_strategy="suppress", replicate=rep)
df = pd.DataFrame(run_trajectory(cfg))
t = df[df.epoch >= EPOCHS // 2]
return dict(beta_adv=beta_adv, delay=delay, rep=rep,
mean_ratio=float(t.mean_ratio.mean()))
def _withhold_cell(beta_adv: float, topo: str, strategy: str, rep: int) -> dict:
cfg = SimConfig(n_nodes=400, stake_dist="pareto", topology=topo, degree=6,
link_latency_mean=0.5, link_latency_dist="geo", blend_hops=3,
blend_delay_max=4.0, max_uncles=2, uncle_window=300, k=256,
epochs=EPOCHS, genesis_d_factor=0.5,
adversary_frac=beta_adv, adversary_strategy=strategy,
windowed_fork_choice=False, prune_arrival=False, replicate=rep)
df = pd.DataFrame(run_trajectory(cfg))
t = df[df.epoch >= EPOCHS // 2]
return dict(beta_adv=beta_adv, topo=topo, strategy=strategy, rep=rep,
mean_ratio=float(t.mean_ratio.mean()))
def main() -> None:
out = Path(__file__).resolve().parents[1] / "runs" / "adversary_grid"
out.mkdir(parents=True, exist_ok=True)
sup_jobs = [(b, d, r) for b in (0.1, 0.3, 0.5) for d in (8.0, 16.0, 24.0)
for r in range(REPS)]
sup = pd.DataFrame(Parallel(n_jobs=6, backend="loky", inner_max_num_threads=1)(
delayed(_suppress_cell)(b, d, r) for b, d, r in sup_jobs))
sup.to_parquet(out / "suppress.parquet", index=False)
print("suppression (D_hat/D by beta_adv x delay):")
print(sup.groupby(["delay", "beta_adv"]).mean_ratio.mean().unstack().round(3).to_string())
wh_jobs = [(b, t, s, r) for b in (0.1, 0.2, 0.3, 0.4, 0.5) for t in ("regular", "blend")
for s in ("withhold", "suppress") for r in range(REPS)]
wh = pd.DataFrame(Parallel(n_jobs=6, backend="loky", inner_max_num_threads=1)(
delayed(_withhold_cell)(b, t, s, r) for b, t, s, r in wh_jobs))
wh.to_parquet(out / "withhold.parquet", index=False)
print("withhold vs suppress (D_hat/D):")
print(wh.groupby(["strategy", "topo", "beta_adv"]).mean_ratio.mean()
.unstack().round(3).to_string())
print(f"wrote {out}/suppress.parquet + withhold.parquet")
if __name__ == "__main__":
main()

173
tools/simulators/tsi/tsi-sim-pernode/scripts/appendix_fluct.py Normal file
View File

@ -0,0 +1,173 @@
"""Appendix B: the U=0 estimate fluctuates around 1 — sampling noise, not bias (figB1, figB2).
Data:
(1) clean zero-delay series (full_mesh, L=0, U=0, uniform stakes) at k in {256, 1024, 2160}
-> runs/fluctuation_u0.parquet (this script, --run)
(2) the committed full-scale N=1000 run (regular sub-slot links and blend, U=0, k=2160)
-> per-epoch tails read directly.
Figures:
figB1 — high-precision per-epoch trace of (D_hat/D - 1) in per-mil at k=2160: the clean
zero-delay series and the realistic 0.1-slot direct-gossip series, with the
+-sigma_th = sqrt((1-f)/(f T)) band.
figB2 — left: per-epoch deviation distributions vs k with the 1/sqrt(T) law; right: the
delay progression (0.1 -> 1.0-slot links, blend): mean drops below 1 and
P(D_hat/D > 1) -> 0 as orphan loss takes over.
Run: python scripts/appendix_fluct.py --run (simulate series (1), ~30-60 min)
python scripts/appendix_fluct.py (render figures + print stats)
"""
from __future__ import annotations
import sys
from pathlib import Path
import numpy as np
import pandas as pd
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from joblib import Parallel, delayed # noqa: E402
from tsi_sim.config import SimConfig # noqa: E402
from tsi_sim.engine import run_trajectory # noqa: E402
from tsi_sim.plotting import style # noqa: E402
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
FIGS = HERE / "report-figures"
F = 1.0 / 30.0
KS = (256, 1024, 2160)
REPS = 4
EPOCHS = 120
def sigma_theory(k: int) -> float:
t_win = 6 * int(k / F)
return float(np.sqrt((1 - F) / (F * t_win)))
def _one(k: int, rep: int) -> pd.DataFrame:
cfg = SimConfig(n_nodes=400, stake_dist="uniform", topology="full_mesh", latency=0,
max_uncles=0, uncle_window=300, k=k, epochs=EPOCHS,
genesis_d_factor=1.0, replicate=rep)
df = pd.DataFrame(run_trajectory(cfg))
df["k_run"] = k
return df[["k_run", "replicate", "epoch", "mean_ratio", "range_ratio"]]
def run() -> None:
jobs = [(k, r) for k in KS for r in range(REPS)]
parts = Parallel(n_jobs=3, prefer="processes")(delayed(_one)(k, r) for k, r in jobs)
out = pd.concat(parts, ignore_index=True)
out.to_parquet(RUNS / "fluctuation_u0.parquet")
print(f"wrote {len(out)} rows -> runs/fluctuation_u0.parquet")
def figs() -> None:
import matplotlib.pyplot as plt
style.apply_style()
clean = pd.read_parquet(RUNS / "fluctuation_u0.parquet")
full = pd.read_parquet(sorted(RUNS.glob("2026-07-23_*_fullscale-small/results.parquet"))[-1])
u0 = full[full.max_uncles == 0]
# ---- figB1: high-precision traces at k=2160 ----
fig, ax = plt.subplots(figsize=(8.6, 4.0))
s = clean[(clean.k_run == 2160) & (clean.replicate == 0) & (clean.epoch >= 4)]
ax.plot(s.epoch, (s.mean_ratio - 1) * 1e3, "-o", ms=3,
color=style.OKABE_ITO[0], label="zero delay (full mesh), U = 0")
r = (u0[(u0.topology == "regular") & (u0.link_latency_mean == 0.1) & (u0.degree == 6)
& (u0.replicate == 0) & (u0.epoch >= 4)])
ax.plot(r.epoch, (r.mean_ratio - 1) * 1e3, "-s", ms=3,
color=style.OKABE_ITO[1], label="direct gossip, 0.1-slot links, U = 0")
sg = sigma_theory(2160) * 1e3
ax.axhspan(-sg, sg, color="0.9", zorder=0)
ax.axhline(0.0, color="0.5", lw=0.8)
ax.text(119, -sg * 1.45, r"$\pm\sigma_{th} = \sqrt{(1-f)/(fT)}$", fontsize=8,
color="0.4", ha="right")
ax.set_xlabel("epoch")
ax.set_ylabel(r"$(\hat D / D - 1) \times 10^{3}$ (per-mil)")
ax.set_title("U = 0, k = 2160: per-epoch sampling noise around the ≤1 equilibrium")
ax.legend(fontsize=8)
style.save(fig, FIGS / "figB1_fluctuation_trace", provenance="scripts/appendix_fluct.py")
plt.close(fig)
# ---- figB2: sigma vs k (left), delay progression (middle), sigma vs delay/U (right) ----
fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(13.6, 4.0))
for i, k in enumerate(KS):
s = clean[(clean.k_run == k) & (clean.epoch >= 8)]
dev = (s.mean_ratio - 1) * 1e3
ax1.hist(dev, bins=31, density=True, histtype="step", lw=1.4,
color=style.OKABE_ITO[i],
label=f"k={k}: sd {dev.std()/1e3:.4f} (th {sigma_theory(k):.4f})")
ax1.axvline(0, color="0.5", lw=0.8)
ax1.set_xlabel(r"$(\hat D / D - 1) \times 10^{3}$")
ax1.set_ylabel("density")
ax1.set_title(r"noise shrinks as $1/\sqrt{T}$ (window size)")
ax1.legend(fontsize=7)
rows = []
for lat in (0.1, 0.2, 0.5, 1.0):
s = u0[(u0.topology == "regular") & (u0.link_latency_mean == lat)
& (u0.epoch >= 15)].mean_ratio
rows.append(dict(case=f"gossip {lat}", mean=s.mean(), p_gt1=(s > 1).mean(),
lo=s.quantile(0.05), hi=s.quantile(0.95)))
b = u0[(u0.topology == "blend") & (u0.epoch >= 15)].mean_ratio
rows.append(dict(case="Blend", mean=b.mean(), p_gt1=(b > 1).mean(),
lo=b.quantile(0.05), hi=b.quantile(0.95)))
dd = pd.DataFrame(rows)
x = np.arange(len(dd))
ax2.errorbar(x, dd["mean"], yerr=[dd["mean"] - dd.lo, dd.hi - dd["mean"]],
fmt="o", ms=5, capsize=3, color=style.OKABE_ITO[0])
for xi, (_, row) in zip(x, dd.iterrows(), strict=True):
ax2.annotate(f"P(>1)={row.p_gt1:.0%}", (xi, row.hi), textcoords="offset points",
xytext=(0, 6), ha="center", fontsize=7, color="0.35")
ax2.axhline(1.0, color="0.5", lw=0.8, ls=":")
ax2.set_xticks(x, dd.case, rotation=20, ha="right", fontsize=8)
ax2.set_ylabel(r"$\hat D / D$ (U = 0, k = 2160)")
ax2.set_title("orphan loss pulls the mean below 1;\nexcursions above 1 vanish with delay")
# right: per-epoch sigma (within a trajectory) vs case, U=0 vs U=1
def per_epoch_sigma(s: pd.DataFrame) -> float:
return float(s.groupby(["degree", "replicate"]).mean_ratio.std().mean())
cases: list[tuple[str, pd.DataFrame]] = []
for lat in (0.1, 0.2, 0.5, 1.0):
cases.append((f"gossip {lat}",
full[(full.topology == "regular") & (full.link_latency_mean == lat)
& (full.epoch >= 15)]))
for dl in (1.0, 2.0, 3.0):
cases.append((f"blend δ={dl:g}",
full[(full.topology == "blend") & (full.blend_delay_max == dl)
& (full.epoch >= 15)]))
x3 = np.arange(len(cases))
for u, marker, lbl in ((0, "o", "U = 0"), (1, "s", "U = 1")):
sig = [per_epoch_sigma(s[s.max_uncles == u]) for _, s in cases]
ax3.plot(x3, sig, marker, ms=6, ls="-", lw=1.0,
color=style.OKABE_ITO[0 if u else 1], label=lbl)
ax3.axhline(sigma_theory(2160), color="0.5", lw=0.9, ls="--")
ax3.text(0.05, sigma_theory(2160) * 1.15, r"sampling floor $\sigma_{th}$",
fontsize=7, color="0.4")
ax3.set_yscale("log")
ax3.set_xticks(x3, [c for c, _ in cases], rotation=20, ha="right", fontsize=8)
ax3.set_ylabel(r"per-epoch $\sigma$ of $\hat D / D$")
ax3.set_title("Blend delay amplifies U = 0 noise ~17×;\none uncle restores the floor")
ax3.legend(fontsize=8)
style.save(fig, FIGS / "figB2_fluctuation_stats", provenance="scripts/appendix_fluct.py")
plt.close(fig)
# ---- stats for the appendix text ----
print("=== clean zero-delay series ===")
for k in KS:
s = clean[(clean.k_run == k) & (clean.epoch >= 8)].mean_ratio
print(f"k={k}: mean={s.mean():.5f} sd={s.std():.5f} (th {sigma_theory(k):.5f}) "
f"P(>1)={(s > 1).mean():.2f} min={s.min():.4f} max={s.max():.4f}")
if __name__ == "__main__":
if "--run" in sys.argv:
run()
else:
figs()

100
tools/simulators/tsi/tsi-sim-pernode/scripts/bootstrap_dynamics.py Normal file
View File

@ -0,0 +1,100 @@
#!/usr/bin/env python
"""Full-scale bootstrap study: block production self-stabilises from any genesis guess (fig1).
Runs at the TRUE security parameter k = 2160, under the Blend transport, at N = 1 000 and
N = 5 000, WITH and WITHOUT uncle references (U = 2 vs U = 0) — so the cold-start behaviour of
the deployed configuration is measured, not extrapolated, and the role of uncles during
bootstrap is visible. genesis_d_factor = initial D_est / true stake (0.01x .. 2x).
Writes runs/bootstrap_fullscale/results.parquet and renders fig1_bootstrap (block-production
rate and D_est/D per epoch; solid = U 2, dashed = U 0; one colour per genesis guess).
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import numpy as np
import pandas as pd
from joblib import Parallel, delayed
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
from tsi_sim.plotting import style
F = 1.0 / 30.0
EPOCHS = 12
# gdf 0.01 floods epoch 0 with ~100x blocks (memory-heavy); run it only at N = 1000.
GRID = [(1000, gdf, rep) for gdf in (0.01, 0.1, 0.5, 1.0, 2.0) for rep in range(3)] + \
[(5000, gdf, rep) for gdf in (0.1, 1.0, 2.0) for rep in range(2)]
def _one(n: int, gdf: float, u: int, rep: int) -> list[dict]:
cfg = SimConfig(n_nodes=n, k=2160, stake_dist="pareto", genesis_d_factor=gdf,
topology="blend", degree=6, blend_hops=3, blend_delay_max=8.0,
link_latency_dist="geo", link_latency_mean=0.5,
max_uncles=u, uncle_window=300, epochs=EPOCHS, replicate=rep)
rows = run_trajectory(cfg)
for r in rows:
r["gdf"] = gdf
r["u"] = u
return rows
def fig1(df: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
d = df[df.n_nodes == 1000]
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8.2, 6.2), sharex=True)
gdfs = sorted(d.gdf.unique())
for i, gdf in enumerate(gdfs):
for u, ls in ((2, "-"), (0, "--")):
s = (d[(d.gdf == gdf) & (d.u == u)]
.groupby("epoch").agg(rate=("n_blocks", "mean"), ratio=("mean_ratio", "mean")))
rate = s.rate / (10 * int(2160 / F)) # blocks per slot
ax1.plot(s.index, rate, ls, color=style.OKABE_ITO[i], lw=1.4, ms=3,
marker="o" if u == 2 else None,
label=f"{gdf:g}×" if u == 2 else None)
ax2.plot(s.index, s.ratio, ls, color=style.OKABE_ITO[i], lw=1.4, ms=3,
marker="o" if u == 2 else None)
ax1.axhline(F, color="0.5", lw=0.9, ls=":")
ax1.text(EPOCHS - 0.4, F * 1.25, "target f", fontsize=8, color="0.4", ha="right")
ax1.set_yscale("log")
ax1.set_ylabel("block production (blocks / slot)")
ax1.set_title("Bootstrap at full scale (k = 2160, Blend, N = 1000): "
"solid = U 2, dashed = U 0")
ax1.legend(fontsize=8, title="genesis D̂ / D", ncols=5)
ax2.axhline(1.0, color="0.5", lw=0.9, ls=":")
ax2.set_yscale("log")
ax2.set_xlabel("epoch")
ax2.set_ylabel(r"$\hat D / D$")
style.save(fig, Path(__file__).resolve().parents[1] / "report-figures" / "fig1_bootstrap",
provenance="scripts/bootstrap_dynamics.py (k=2160)")
plt.close(fig)
def main() -> None:
out = Path(__file__).resolve().parents[1] / "runs" / "bootstrap_fullscale"
out.mkdir(parents=True, exist_ok=True)
jobs = [(n, g, u, r) for (n, g, r) in GRID for u in (0, 2)]
results = Parallel(n_jobs=3, backend="loky", inner_max_num_threads=1)(
delayed(_one)(n, g, u, r) for n, g, u, r in jobs)
df = pd.DataFrame([row for traj in results for row in traj])
df.to_parquet(out / "results.parquet", index=False)
fig1(df)
# settle epochs: first epoch with block rate within 10% of f, per (n, gdf, u)
el = 10 * int(2160 / F)
df["rate"] = df.n_blocks / el
st = (df.assign(ok=lambda x: (x.rate - F).abs() <= 0.1 * F)
.groupby(["n_nodes", "gdf", "u", "replicate"])
.apply(lambda g: int(g[g.ok].epoch.min()) if g.ok.any() else np.nan,
include_groups=False))
print("settle epoch (first epoch within 10% of f):")
print(st.groupby(["n_nodes", "gdf", "u"]).mean().round(2).to_string())
print(f"wrote {out/'results.parquet'} ({len(df)} rows) and fig1_bootstrap")
if __name__ == "__main__":
main()

91
tools/simulators/tsi/tsi-sim-pernode/scripts/build_html.py Normal file
View File

@ -0,0 +1,91 @@
"""Render the TSI report markdown set (index + 4 parts) to standalone, print-friendly HTML.
Committed replacement for the ad-hoc HTML build. Markdown is the source of truth; the HTML is a
build artifact (not committed). Code blocks are syntax-highlighted (codehilite + Pygments),
and cross-document `.md` links are rewritten to `.html` so the rendered set navigates internally.
Run: python scripts/build_html.py --all # index + 4 parts
python scripts/build_html.py <file.md> ... # specific docs
"""
from __future__ import annotations
import argparse
import re
from pathlib import Path
import markdown
from pygments.formatters import HtmlFormatter
HERE = Path(__file__).resolve().parent.parent
DOCS = [
"REPORT-tsi-parameter-selection.md",
"tsi-report-1-overview-and-recommendations.md",
"tsi-report-2-accuracy-and-design.md",
"tsi-report-3-robustness-and-incentives.md",
"tsi-report-4-reproducibility-and-appendices.md",
]
CSS_BASE = r"""
@page { size: A4; margin: 18mm 16mm 20mm 16mm; }
html { -webkit-print-color-adjust: exact; }
body { font-family: -apple-system, "Helvetica Neue", "Arial Unicode MS", sans-serif;
font-size: 9.5pt; line-height: 1.45; color: #1a1a1a; max-width: 100%; margin: 0; }
h1 { font-size: 17pt; line-height: 1.25; border-bottom: 2px solid #333; padding-bottom: 6px; }
h2 { font-size: 13.5pt; margin-top: 22px; border-bottom: 1px solid #999; padding-bottom: 3px;
page-break-after: avoid; }
h3 { font-size: 11pt; margin-top: 16px; page-break-after: avoid; }
p, li { text-align: justify; }
code { font-family: "SF Mono", Menlo, monospace; font-size: 8.5pt;
background: #f4f4f4; padding: 0 2px; border-radius: 2px; }
pre { background: #f4f4f4; padding: 8px 10px; border-radius: 4px; overflow-x: hidden;
white-space: pre-wrap; page-break-inside: avoid; }
pre code { background: none; font-size: 8pt; }
table { border-collapse: collapse; width: 100%; font-size: 8pt; margin: 10px 0; }
th, td { border: 1px solid #bbb; padding: 3px 5px; text-align: left; vertical-align: top; }
th { background: #ececec; }
tr { page-break-inside: avoid; }
img { max-width: 100%; height: auto; }
figure { margin: 12px 0; text-align: center; page-break-inside: avoid; }
figcaption { font-size: 8pt; color: #444; text-align: justify; margin-top: 4px; padding: 0 8mm; }
blockquote { border-left: 3px solid #999; margin-left: 0; padding-left: 12px; color: #333; }
hr { border: none; border-top: 1px solid #ccc; margin: 18px 0; }
em { color: inherit; }
"""
def render(md_path: Path) -> Path:
body = markdown.markdown(
md_path.read_text(),
extensions=["tables", "fenced_code", "sane_lists", "codehilite", "md_in_html"],
extension_configs={"codehilite": {"guess_lang": False}},
)
# rewrite intra-set links so the rendered HTML navigates to .html, not .md
body = re.sub(r'(href="[^"]*?)\.md(#|")', r"\1.html\2", body)
pyg = HtmlFormatter(style="default").get_style_defs(".codehilite")
style = (CSS_BASE + "\n.codehilite{background:#f4f4f4;border-radius:4px;}\n"
".codehilite pre{background:none;margin:0;}\n" + pyg)
html = (f"<!DOCTYPE html><html><head><meta charset='utf-8'>"
f"<title>{md_path.stem}</title><style>{style}</style></head><body>\n"
f"{body}\n</body></html>")
out = md_path.with_suffix(".html")
out.write_text(html)
return out
def main() -> None:
ap = argparse.ArgumentParser(description="Render the TSI report markdown set to HTML")
ap.add_argument("docs", nargs="*", help="specific .md files (default: the whole set)")
ap.add_argument("--all", action="store_true", help="render the index + 4 parts")
args = ap.parse_args()
targets = DOCS if (args.all or not args.docs) else args.docs
for d in targets:
p = HERE / d
if not p.exists():
print(f"skip (missing): {d}")
continue
print(f"wrote {render(p).name}")
if __name__ == "__main__":
main()

53
tools/simulators/tsi/tsi-sim-pernode/scripts/capstone.py Normal file
View File

@ -0,0 +1,53 @@
#!/usr/bin/env python
"""Capstone: the recommended configuration end-to-end at true k=2160 (report §8).
One config — f=1/30, W=300, U=2, β=1, degree 6, Blend 3 hops × 8 s, Pareto stake — run honest
and under a 30 % uncle-suppression adversary, confirming accuracy, consensus, fork rate, reorg
depth, and the emergent reference rate p_ref ALL hold together. Writes runs/capstone.parquet.
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import pandas as pd
from joblib import Parallel, delayed
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
REC = dict(n_nodes=1000, stake_dist="pareto", topology="blend", degree=6,
link_latency_mean=0.5, link_latency_dist="geo", blend_hops=3, blend_delay_max=8.0,
max_uncles=2, uncle_window=300, uncle_strategy="oldest", k=2160, epochs=40,
genesis_d_factor=0.5, early_stop=True)
def _one(adv: float, rep: int) -> list[dict]:
cfg = SimConfig(**REC, adversary_frac=adv, adversary_strategy="suppress", replicate=rep)
rows = run_trajectory(cfg)
for r in rows:
r["adv"] = adv
return rows
def main() -> None:
out = Path(__file__).resolve().parents[1] / "runs"
jobs = [(a, r) for a in (0.0, 0.3) for r in range(8)]
res = Parallel(n_jobs=4, backend="loky", inner_max_num_threads=1)(
delayed(_one)(a, r) for a, r in jobs)
df = pd.DataFrame([row for traj in res for row in traj])
df.to_parquet(out / "capstone.parquet", index=False)
print("=== Capstone: recommended config, all metrics together (equilibrium tail) ===")
for adv, g in df.groupby("adv"):
t = g[g.epoch >= g.epoch.max() // 2]
print(f"adversary {adv:.0%}: D̂/D {t.mean_ratio.mean():.4f} "
f"range_ratio {t.range_ratio.max():.4f} agreement {t.agreement_window.min():.4f} "
f"fork_rate {t.fork_rate.mean():.3f} max_reorg_depth {t.max_reorg_depth.max()} "
f"p_ref {t.p_ref.mean():.3f}")
print(f"wrote {out/'capstone.parquet'} ({len(df)} rows)")
if __name__ == "__main__":
main()

78
tools/simulators/tsi/tsi-sim-pernode/scripts/churn.py Normal file
View File

@ -0,0 +1,78 @@
#!/usr/bin/env python
"""Organic honest stake churn: does TSI track active stake within-epoch? (report §6.x, fig29).
The active honest stake oscillates (sine, weekly cycle), ramps, or steps down; TSI should
track it with a one-epoch lag (β=1, §6.5 EMA law). We measure D̂/D_active (should stay ~1)
and D̂/D_total (follows the active fraction), plus the fork rate the transient induces.
Blend, U=2, degree 6, k=256. Writes runs/churn.parquet and fig29_churn.
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import pandas as pd
from joblib import Parallel, delayed
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
from tsi_sim.plotting import style
BASE = dict(n_nodes=1000, stake_dist="pareto", topology="blend", degree=6,
link_latency_mean=0.5, link_latency_dist="geo", blend_hops=3, blend_delay_max=8.0,
max_uncles=2, uncle_window=300, k=256, epochs=40, genesis_d_factor=0.5)
GRID = [(m, p, r) for m in ("sine", "ramp", "step") for p in (2, 4, 8) for r in range(4)]
def _one(mode: str, period: int, rep: int) -> list[dict]:
cfg = SimConfig(**BASE, churn_amp=0.3, churn_period=period, churn_mode=mode, replicate=rep)
rows = run_trajectory(cfg)
for r in rows:
r["mode"], r["period"] = mode, period
return rows
def fig29(df: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
fig, axes = plt.subplots(1, 3, figsize=(13.2, 4.0), sharey=True)
for ax, mode in zip(axes, ("sine", "ramp", "step"), strict=True):
s = df[(df["mode"] == mode) & (df.period == 4)]
g = s.groupby("epoch").agg(active=("active_stake_frac", "mean"),
tot=("mean_ratio", "mean"))
g["corr"] = g.tot / g.active
ax.plot(g.index, g.active, "--", color="0.6", lw=1.4, label="active stake / total")
ax.plot(g.index, g.tot, "-o", ms=3, color=style.OKABE_ITO[0], label="D̂ / D_total")
ax.plot(g.index, g["corr"], "-s", ms=3, color=style.OKABE_ITO[1], label="D̂ / D_active")
ax.axhline(1.0, color="0.8", lw=0.7, ls=":")
ax.set_xlabel("epoch")
ax.set_title(f"{mode} churn (30 %, period 4)")
axes[0].set_ylabel("stake fraction / accuracy")
axes[0].legend(fontsize=8, loc="lower left")
fig.suptitle("TSI tracks active stake under organic churn (β=1, one-epoch lag); "
"corrected accuracy stays ~1", y=1.03)
style.save(fig, Path(__file__).resolve().parents[1] / "report-figures" / "fig29_churn",
provenance="scripts/churn.py")
plt.close(fig)
def main() -> None:
out = Path(__file__).resolve().parents[1] / "runs"
res = Parallel(n_jobs=6, backend="loky", inner_max_num_threads=1)(
delayed(_one)(m, p, r) for m, p, r in GRID)
df = pd.DataFrame([row for traj in res for row in traj])
df.to_parquet(out / "churn.parquet", index=False)
fig29(df)
print("=== churn: D̂/D_active (tracking accuracy) and worst lag, period 4 ===")
for (mode,), g in df[df.period == 4].groupby(["mode"]):
t = g[g.epoch >= 8]
corr = (t.mean_ratio / t.active_stake_frac)
print(f"{mode}: D̂/D_active {corr.mean():.3f} (min {corr.min():.3f}), "
f"range_ratio {t.range_ratio.max():.4f}, fork_rate {t.fork_rate.mean():.3f}")
print(f"wrote {out/'churn.parquet'} and fig29_churn")
if __name__ == "__main__":
main()

89
tools/simulators/tsi/tsi-sim-pernode/scripts/clock_skew.py Normal file
View File

@ -0,0 +1,89 @@
#!/usr/bin/env python
"""Per-node clock skew: does a whole-timeline slot-clock offset break consensus? (report §6.1).
Unlike per-arrival jitter (which leaves range_ratio EXACTLY 0, §6.1), a constant per-node clock
offset shifts each node's measurement window by δ_i slots, so nodes count different blocks in the
boundary slots and their occupied-slot counts differ. We build one honest finalized tree, then for
each node evaluate its occupied-slot density over its OWN shifted window [δ_i, T+δ_i), and report
the resulting inter-node spread vs skew. Bound: |Δm|/m ≲ 2·skew/T, so the effect is O(skew/T) —
tiny at the production window (T ≈ 4.3e5 slots) but, unlike jitter, not exactly zero.
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import numpy as np
from tsi_sim import lottery
from tsi_sim.blocktree import build_tree_pernode
from tsi_sim.config import SimConfig
from tsi_sim.topology import build_path_latency
SKEWS = (0, 1, 2, 5, 10, 20)
def build_honest_tree(cfg: SimConfig, seed: int):
rng = np.random.default_rng(seed)
stake = np.random.default_rng(seed + 1).random(cfg.n_nodes) + 0.1
d_est = np.full(cfg.n_nodes, float(stake.sum()))
pl = build_path_latency(cfg, np.random.default_rng(seed + 2))
p = lottery.win_probs(stake, d_est, cfg.f)
ws, wn = lottery.sample_wins(p, cfg.epoch_len, np.random.default_rng(seed + 3))
active, groups = lottery.group_by_slot(ws, wn)
tree, _A = build_tree_pernode(active, groups, pl, cfg, rng)
return tree
def occupied_slots(tree, T: int, lo: int) -> int:
"""Occupied canonical+uncle slots in the window [lo, lo+T) (one count per slot)."""
nb = tree.n_blocks
ids = np.arange(nb)
h = tree.height.copy()
best = int(np.lexsort((-ids, -tree.slot, h))[-1])
canon = []
b = best
while b > 0:
canon.append(b)
b = int(tree.parent[b])
occ = set()
for b in canon:
s = int(tree.slot[b])
if lo <= s < lo + T:
occ.add(s)
for b in canon:
for u in tree.uncles[b]:
su = int(tree.slot[u])
if lo <= su < lo + T and su not in occ:
occ.add(su)
return len(occ)
def main() -> None:
cfg = SimConfig(n_nodes=400, stake_dist="pareto", topology="blend", degree=6,
link_latency_mean=0.5, link_latency_dist="geo", blend_hops=3,
blend_delay_max=8.0, max_uncles=2, uncle_window=300, k=256, epochs=1)
T = cfg.period_T
print(f"window T = {T} slots (k={cfg.k}); production k=2160 -> T≈{6 * int(2160 / cfg.f):.0e}")
tree = build_honest_tree(cfg, seed=20260724)
m0 = occupied_slots(tree, T, 0)
print(f"baseline occupied slots m0 = {m0}")
print("skew (slots) | inter-node range(m)/m0 | bound 2·skew/T")
for skew in SKEWS:
if skew == 0:
print(f"{skew:>4} | 0.000000 | 0")
continue
rng = np.random.default_rng(7)
offs = rng.integers(-skew, skew + 1, size=cfg.n_nodes)
ms = np.array([occupied_slots(tree, T, int(o)) for o in offs])
rng_ratio = (ms.max() - ms.min()) / m0
print(f"{skew:>4} | {rng_ratio:.6f} | {2 * skew / T:.6f}")
print("\nInterpretation: the spread is O(skew/T) — bounded and vanishing at the production "
"window, but (unlike per-arrival jitter) not exactly 0, so bounded clock skew is a "
"small, quantifiable consensus cost, not a break.")
if __name__ == "__main__":
main()

293
tools/simulators/tsi/tsi-sim-pernode/scripts/dynamic_withhold.py Normal file
View File

@ -0,0 +1,293 @@
"""Dynamic (withhold-then-rejoin) grinding analysis — REPORT §6.5.
Static withholding (§6.4) deflates D_est to the active-stake line (1-beta_adv) but is
self-punishing (the coalition forfeits every withheld block). The dynamic variant abstains to
depress D_est, then re-activates to mine at the depressed difficulty. Its feasibility is set by
how fast D_est moves, i.e. the estimator gain `beta` (config.beta), since the update
D_{t+1} = (1-beta)*D_t + beta * S_t * D* (exact, leading order)
is an EMA of the active-stake signal S_t*D* with memory ~1/beta epochs (S_t = active fraction,
D* = honest-equilibrium estimate). Four studies:
1. SAWTOOTH + EMA LAW — D_est(t) trajectory vs beta, overlaid with the EMA prediction (fig10)
2. PROFITABILITY — realized reward / stake share vs withhold duty and beta_adv (fig11)
3. GRIEFING FRONTIER — estimator distortion achieved vs reward forfeited (fig11, right)
4. TIPPING near rho~=1 — does a withhold PULSE recover, or tip into the §6.2 collapsed
branch and stay deflated (self-sustaining)? (fig12)
Run: python scripts/dynamic_withhold.py (writes runs/dynamic_withhold_*.parquet + fig10-12)
"""
from __future__ import annotations
from pathlib import Path
import numpy as np
import pandas as pd
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
from tsi_sim.plotting import style
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
FIGS = HERE / "report-figures"
RUNS.mkdir(exist_ok=True)
FIGS.mkdir(exist_ok=True)
# equal stakes => coalition_frac == adversary_frac EXACTLY (clean parametric control of beta_adv)
BASE = dict(n_nodes=1000, stake_dist="uniform", topology="regular", degree=8,
link_latency_mean=0.3, link_latency_dist="geo", max_uncles=2, uncle_window=300,
genesis_d_factor=0.5, k=64, adversary_strategy="withhold")
def traj(reps: int, **cfg) -> pd.DataFrame:
"""Run `reps` replicates of one config, tagged with replicate id."""
out = []
for r in range(reps):
df = pd.DataFrame(run_trajectory(SimConfig(replicate=r, **cfg)))
out.append(df)
return pd.concat(out, ignore_index=True)
def honest_equilibrium(beta: float, reps: int = 4, **over) -> float:
"""Honest-run tail-mean D_est/D_true (the level D*/D_true the EMA relaxes toward)."""
kw = {**BASE, **over}
kw.pop("adversary_strategy", None)
df = traj(reps, epochs=20, beta=beta, adversary_frac=0.0, **kw)
return float(df[df.epoch >= 12].mean_ratio.mean())
# ---------------------------------------------------------------------------------------------
# STUDY 1 — sawtooth trajectory + EMA-law overlay
# ---------------------------------------------------------------------------------------------
def study1_sawtooth() -> pd.DataFrame:
betas = [0.25, 0.5, 1.0]
period, wh = 6, 3 # 50% duty, wide on/off blocks so the sawtooth shape is visible
epochs, reps, badv = 42, 6, 0.3
rows = []
for beta in betas:
df = traj(reps, epochs=epochs, beta=beta, adversary_frac=badv,
adversary_period=period, adversary_withhold_epochs=wh, **BASE)
g = df.groupby("epoch")
d_hat = g.mean_ratio.mean().to_numpy()
active = g.active_stake_frac.mean().to_numpy() # 1 or (1-badv) per epoch
rstar = honest_equilibrium(beta)
# EMA prediction: D_hat[t] = (1-beta)*D_hat[t-1] + beta*S_t*D*, seeded from genesis. The
# end-of-epoch-t estimate uses THIS epoch's active fraction active[t] (not active[t-1]).
pred = np.empty(epochs)
prev = BASE["genesis_d_factor"]
for t in range(epochs):
pred[t] = (1 - beta) * prev + beta * active[t] * rstar
prev = pred[t]
for t in range(epochs):
rows.append(dict(beta=beta, epoch=t, d_hat=d_hat[t], active=active[t],
ema_pred=pred[t], rstar=rstar))
out = pd.DataFrame(rows)
out.to_parquet(RUNS / "dynamic_withhold_sawtooth.parquet")
rms = float(np.sqrt(((out.d_hat - out.ema_pred) ** 2)[out.epoch >= 3].mean()))
print(f"[S1] sawtooth: EMA-law RMS(D_hat - pred), epoch>=3 = {rms:.4f}")
for beta in betas:
s = out[(out.beta == beta) & (out.epoch >= 12)]
lo, hi = s.d_hat.min(), s.d_hat.max()
print(f" beta={beta}: swing D_hat in [{lo:.3f}, {hi:.3f}] amplitude={hi-lo:.3f}")
return out
# ---------------------------------------------------------------------------------------------
# STUDY 2 + 3 — profitability and griefing frontier
# ---------------------------------------------------------------------------------------------
# duty psi -> (period, withhold_epochs); 0.0 anchor is "never withhold" (always participate)
SCHEDULES = [
("never", 0.0, None),
("p10/1", 0.10, (10, 1)),
("p4/1", 0.25, (4, 1)),
("p3/1", 1 / 3, (3, 1)),
("p2/1", 0.50, (2, 1)),
("p3/2", 2 / 3, (3, 2)),
("p4/3", 0.75, (4, 3)),
("static", 1.0, (1, 1)),
]
def _reward_over_stake(df: pd.DataFrame, badv: float, burn: int) -> float:
t = df[df.epoch >= burn]
total = float((t.adv_blocks + t.honest_blocks).sum())
return float(t.adv_blocks.sum() / (badv * total)) if total else float("nan")
def study2_profitability() -> pd.DataFrame:
epochs, reps, burn = 26, 8, 12
rows = []
# (a) duty x beta at beta_adv = 0.3
for beta in (0.5, 1.0):
for label, psi, sched in SCHEDULES:
if sched is None: # never withhold == honest participation
df = traj(reps, epochs=epochs, beta=beta, adversary_frac=0.3,
adversary_strategy="suppress",
**{k: v for k, v in BASE.items() if k != "adversary_strategy"})
else:
p, w = sched
df = traj(reps, epochs=epochs, beta=beta, adversary_frac=0.3,
adversary_period=p, adversary_withhold_epochs=w, **BASE)
ros = _reward_over_stake(df, 0.3, burn)
distortion = float(1.0 - df[df.epoch >= burn].mean_ratio.mean()) # mean deflation
rows.append(dict(kind="duty", beta=beta, badv=0.3, label=label, duty=psi,
reward_over_stake=ros, distortion=distortion))
print(f"[S2] beta={beta} badv=0.30 {label:7s} duty={psi:.2f} "
f"reward/stake={ros:.3f} distortion={distortion:.3f}")
# (b) beta_adv sweep at the alternate schedule (period2/wh1), beta=1
for badv in (0.1, 0.2, 0.3, 0.4):
df = traj(reps, epochs=epochs, beta=1.0, adversary_frac=badv,
adversary_period=2, adversary_withhold_epochs=1, **BASE)
ros = _reward_over_stake(df, badv, burn)
rows.append(dict(kind="badv", beta=1.0, badv=badv, label="p2/1", duty=0.5,
reward_over_stake=ros, distortion=float(1 - df[df.epoch >= burn]
.mean_ratio.mean())))
print(f"[S2b] beta=1.0 badv={badv:.2f} alternate reward/stake={ros:.3f}")
out = pd.DataFrame(rows)
out.to_parquet(RUNS / "dynamic_withhold_profit.parquet")
return out
# ---------------------------------------------------------------------------------------------
# STUDY 4 — tipping: does a withhold PULSE recover or trap the estimator?
# ---------------------------------------------------------------------------------------------
def study4_tipping() -> pd.DataFrame:
# Push the operating load rho = f * D_vis up via block rate f AND mixnet delay, then apply a
# short withhold pulse and watch whether D_est recovers to the honest level or stays collapsed.
epochs, reps, pulse, badv = 34, 5, 4, 0.4
base = dict(n_nodes=400, stake_dist="uniform", topology="blend", degree=8,
link_latency_mean=0.3, link_latency_dist="geo", blend_hops=4,
max_uncles=2, uncle_window=300, genesis_d_factor=0.6, k=32,
adversary_strategy="withhold")
# operating points from mild to aggressive load
points = [
("f=1/30 d=6", dict(f=1 / 30, blend_delay_max=6.0)),
("f=1/15 d=12", dict(f=1 / 15, blend_delay_max=12.0)),
("f=1/10 d=18", dict(f=1 / 10, blend_delay_max=18.0)),
("f=1/10 d=30", dict(f=1 / 10, blend_delay_max=30.0)),
]
rows = []
for label, over in points:
# honest reference (no pulse)
hon = traj(reps, epochs=epochs, beta=1.0, adversary_frac=0.0,
**{**base, **over, "adversary_strategy": "suppress"})
hon_g = hon.groupby("epoch").mean_ratio.mean()
# pulse: withhold first `pulse` epochs, honest forever after
pul = traj(reps, epochs=epochs, beta=1.0, adversary_frac=badv,
adversary_period=epochs, adversary_withhold_epochs=pulse, **{**base, **over})
pul_g = pul.groupby("epoch").mean_ratio.mean()
hon_eq = float(hon_g[hon_g.index >= epochs - 8].mean())
post = float(pul_g[pul_g.index >= epochs - 8].mean()) # long after the pulse
recovered = post > 0.9 * hon_eq
print(f"[S4] {label:14s} honest_eq={hon_eq:.3f} post-pulse={post:.3f} "
f"{'RECOVERS' if recovered else 'TRAPPED (collapsed)'}")
for t in range(epochs):
rows.append(dict(point=label, epoch=t, honest=float(hon_g.get(t, np.nan)),
pulse=float(pul_g.get(t, np.nan)), hon_eq=hon_eq,
recovered=recovered, pulse_len=pulse))
out = pd.DataFrame(rows)
out.to_parquet(RUNS / "dynamic_withhold_tipping.parquet")
return out
# ---------------------------------------------------------------------------------------------
# FIGURES
# ---------------------------------------------------------------------------------------------
def fig10(saw: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
fig, axes = plt.subplots(1, 3, figsize=(11.5, 3.4), sharey=True)
for ax, beta in zip(axes, sorted(saw.beta.unique()), strict=True):
s = saw[saw.beta == beta].sort_values("epoch")
rstar = s.rstar.iloc[0]
ax.axhline(rstar, color="0.6", lw=0.8, ls=":", label="honest $D^*$")
ax.axhline((1 - 0.3) * rstar, color="0.6", lw=0.8, ls="--",
label=r"active $(1-\beta_{adv})D^*$")
ax.plot(s.epoch, s.d_hat, "-o", ms=3, color=style.OKABE_ITO[0], label=r"$\hat D$ (sim)")
ax.plot(s.epoch, s.ema_pred, "-", lw=1.4, color=style.OKABE_ITO[1],
label="EMA law")
ax.set_title(rf"$\beta={beta}$ ($\tau={-1/np.log(1-beta):.1f}$ ep)" if beta < 1
else rf"$\beta={beta}$ (1-epoch)")
ax.set_xlabel("epoch")
ax.set_xlim(6, s.epoch.max())
axes[0].set_ylabel(r"$\hat D / D_{\rm true}$")
axes[0].legend(fontsize=7, loc="lower right")
fig.suptitle(r"Dynamic withholding drives an EMA sawtooth; estimator gain $\beta$ sets its "
"speed & depth", y=1.02)
style.save(fig, FIGS / "fig10_sawtooth", provenance="scripts/dynamic_withhold.py::study1")
plt.close(fig)
def fig11(prof: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
fig, axes = plt.subplots(1, 2, figsize=(9.2, 3.6))
# left: reward/stake vs duty, per beta (+ beta_adv sweep inset points)
ax = axes[0]
duty = prof[prof.kind == "duty"]
for i, beta in enumerate(sorted(duty.beta.unique())):
s = duty[duty.beta == beta].sort_values("duty")
ax.plot(s.duty, s.reward_over_stake, "-o", ms=4, color=style.OKABE_ITO[i],
label=rf"$\beta={beta}$")
ax.axhline(1.0, color="0.5", lw=0.9, ls="--", label="break-even")
ax.set_xlabel(r"withhold duty $\psi$ (fraction of epochs)")
ax.set_ylabel(r"realized reward / stake share")
ax.set_title(r"Dynamic withholding is unprofitable ($\beta_{adv}=0.3$)")
ax.legend(fontsize=8)
# right: griefing frontier — distortion achieved vs reward forfeited
ax = axes[1]
for i, beta in enumerate(sorted(duty.beta.unique())):
s = duty[duty.beta == beta].sort_values("duty")
ax.plot(1 - s.reward_over_stake, s.distortion, "-o", ms=4, color=style.OKABE_ITO[i],
label=rf"$\beta={beta}$")
lim = max(0.01, float((1 - duty.reward_over_stake).max()))
ax.plot([0, lim], [0, lim], color="0.6", lw=0.8, ls=":", label="1:1 (linear cost)")
ax.set_xlabel("reward forfeited (1 - reward/stake)")
ax.set_ylabel(r"mean estimator distortion $1-\overline{\hat D/D}$")
ax.set_title("Griefing is bounded & linearly costly")
ax.legend(fontsize=8)
style.save(fig, FIGS / "fig11_profitability", provenance="scripts/dynamic_withhold.py::study2")
plt.close(fig)
def fig12(tip: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
fig, ax = plt.subplots(figsize=(7.0, 4.2))
points = list(dict.fromkeys(tip.point))
for i, pt in enumerate(points):
s = tip[tip.point == pt].sort_values("epoch")
rec = bool(s.recovered.iloc[0])
c = style.OKABE_ITO[i]
ax.plot(s.epoch, s.pulse, "-o", ms=3, color=c,
label=f"{pt} {'recovers' if rec else 'TRAPPED'}")
ax.plot(s.epoch, s.honest, "-", lw=0.8, color=c, alpha=0.35)
plen = int(tip.pulse_len.iloc[0])
ax.axvspan(0, plen - 1, color="0.85", label=f"withhold pulse ({plen} ep)")
ax.set_xlabel("epoch")
ax.set_ylabel(r"$\hat D / D_{\rm true}$")
ax.set_title("Withhold pulse vs operating load: transient recovery or collapse-branch trap")
ax.legend(fontsize=7, loc="lower right")
style.save(fig, FIGS / "fig12_tipping", provenance="scripts/dynamic_withhold.py::study4")
plt.close(fig)
def main() -> None:
print("=== STUDY 1: sawtooth + EMA law ===")
saw = study1_sawtooth()
print("\n=== STUDY 2/3: profitability + griefing ===")
prof = study2_profitability()
print("\n=== STUDY 4: tipping near rho~=1 ===")
tip = study4_tipping()
print("\n=== FIGURES ===")
fig10(saw)
fig11(prof)
fig12(tip)
print("wrote fig10_sawtooth, fig11_profitability, fig12_tipping")
if __name__ == "__main__":
main()

60
tools/simulators/tsi/tsi-sim-pernode/scripts/hops_delay_grid.py Normal file
View File

@ -0,0 +1,60 @@
"""fig3 — uncle recovery under the Blend cascade: hops × per-hop delay × U (4-panel grid).
Committed generator for report fig3. Previously fig3 was produced ad hoc (via make_figures on the
blend-hops-delay run, then hand-copied into report-figures/) and had NO reproducible source in the
repo; this script closes that gap. Per uncle cap U it plots mean D̂/D vs the per-hop budget δ_max,
one curve per hop count, at N=1000 from the canonical blend-hops-delay sweep (per-trajectory 50%
burn-in via figures_pernode.equilibrium). Accuracy is bounded by 1 (slot-counting cannot over-count
occupied slots), so the y-axis is capped at the exact-recovery bound — no above-1 headroom.
Run: python scripts/hops_delay_grid.py
"""
from __future__ import annotations
import sys
from pathlib import Path
import pandas as pd
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from tsi_sim.plotting import style # noqa: E402
from tsi_sim.plotting.figures_pernode import equilibrium # noqa: E402
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
FIGS = HERE / "report-figures"
def main() -> None:
import matplotlib.pyplot as plt
src = sorted(RUNS.glob("*_blend-hops-delay/results.parquet"))[-1]
eq = equilibrium(pd.read_parquet(src))
eq = eq[(eq.n_nodes == 1000) & (eq.topology == "blend") & (eq.stake_dist == "pareto")]
us = sorted(eq.max_uncles.unique())
hops = sorted(eq.blend_hops.unique())
style.apply_style()
fig, axes = plt.subplots(1, len(us), figsize=(3.2 * len(us), 3.6), sharey=True)
for ax, U in zip(axes, us, strict=True):
s = eq[eq.max_uncles == U]
for i, h in enumerate(hops):
g = s[s.blend_hops == h].groupby("blend_delay_max").mean_ratio.agg(["mean", "sem"])
ax.errorbar(g.index, g["mean"], yerr=g["sem"], fmt="-o", ms=4, capsize=2,
color=style.OKABE_ITO[i], label=f"{int(h)} hops")
ax.axhline(1.0, color="0.4", lw=1.0, ls="--", zorder=0)
ax.axhline(0.98, color="0.75", lw=0.8, ls=":", zorder=0)
ax.set_title(f"U = {int(U)}")
ax.set_xlabel(r"per-hop budget $\delta_{max}$ (s)")
axes[0].set_ylabel(r"mean $\hat D / D$")
axes[0].set_ylim(top=1.01) # bounded by 1: cap at the exact-recovery bound
axes[0].legend(fontsize=8, loc="lower left")
fig.suptitle(r"Uncle recovery under Blend cascade: hops × per-hop delay × U "
r"(pareto, N=1000, f=1/30)", y=1.02)
style.save(fig, FIGS / "fig3_hops_delay", provenance="scripts/hops_delay_grid.py")
plt.close(fig)
print("wrote fig3_hops_delay")
if __name__ == "__main__":
main()

61
tools/simulators/tsi/tsi-sim-pernode/scripts/jitter_grid.py Normal file
View File

@ -0,0 +1,61 @@
#!/usr/bin/env python
"""Jitter/consensus grid (§6.1): exact oracle, per-(block,node) Exp jitter up to 3 slots.
jitter_mean {0, 0.1, 0.3, 1.0, 3.0} x {regular, blend} x N {1000, 2000}, 10 replicates, U = 2,
windowed_fork_choice/prune_arrival OFF (guaranteed-exact full-matrix mode). Writes one
tail-aggregated row per (topo, N, jitter, rep) to runs/jitter_grid/results.parquet.
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import pandas as pd
from joblib import Parallel, delayed
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
JITTERS = [0.0, 0.1, 0.3, 1.0, 3.0]
TOPOS = ["regular", "blend"]
NS = [1000, 2000]
REPS = 10
EPOCHS = 20
def _one(topo: str, n: int, jm: float, rep: int) -> dict:
cfg = SimConfig(n_nodes=n, stake_dist="pareto", topology=topo, degree=6,
link_latency_mean=0.5, link_latency_dist="geo",
blend_hops=3, blend_delay_max=4.0,
max_uncles=2, uncle_window=300, k=256, epochs=EPOCHS,
genesis_d_factor=0.5, jitter_mean=jm,
windowed_fork_choice=False, prune_arrival=False, replicate=rep)
df = pd.DataFrame(run_trajectory(cfg))
t = df[df.epoch >= EPOCHS // 2]
return dict(topo=topo, N=n, jitter=jm, rep=rep,
range_ratio=float(t.range_ratio.max()),
agreement_window=float(t.agreement_window.min()),
agreement_tip=float(t.agreement_tip.mean()),
mean_ratio=float(t.mean_ratio.mean()))
def main() -> None:
out = Path(__file__).resolve().parents[1] / "runs" / "jitter_grid"
out.mkdir(parents=True, exist_ok=True)
jobs = [(t, n, j, r) for t in TOPOS for n in NS for j in JITTERS for r in range(REPS)]
rows = Parallel(n_jobs=4, backend="loky", inner_max_num_threads=1)(
delayed(_one)(t, n, j, r) for t, n, j, r in jobs)
df = pd.DataFrame(rows)
df.to_parquet(out / "results.parquet", index=False)
print(df.groupby("jitter").agg(range_max=("range_ratio", "max"),
agr_min=("agreement_window", "min"),
acc_lo=("mean_ratio", "min"),
acc_hi=("mean_ratio", "max"),
tip_worst=("agreement_tip", "min")).round(4).to_string())
print(f"wrote {out/'results.parquet'} ({len(df)} rows)")
if __name__ == "__main__":
main()

14
tools/simulators/tsi/tsi-sim-pernode/scripts/make_figures.py Normal file
View File

@ -0,0 +1,14 @@
#!/usr/bin/env python
"""Thin shim so `python scripts/make_figures.py` works without installing."""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from tsi_sim.plotting.make_figures import main # noqa: E402
if __name__ == "__main__":
main()

161
tools/simulators/tsi/tsi-sim-pernode/scripts/nscaling_analysis.py Normal file
View File

@ -0,0 +1,161 @@
"""N-scaling of the one-uncle boundary: figures fig23/fig24 + §3.8 numbers.
Combines the direct ladder (nscaling-a/b + 32k tiers, N = 1k..32k) with the exact topology
probe (l_mean(N, degree) to N = 10^6) and the load law rho = f*D_vis:
fig23 — U=1 accuracy vs N per degree, case (a) vs (b), at the 8-s blending budget.
fig24 — the ladder collapsed onto rho (validating that N enters only via l_mean), with the
probe's rho(N) curves extrapolating each degree to 10^6 and the U=1 boundary marked.
Run: python scripts/nscaling_analysis.py
"""
from __future__ import annotations
import sys
from pathlib import Path
import numpy as np
import pandas as pd
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from tsi_sim.plotting import style # noqa: E402
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
FIGS = HERE / "report-figures"
F = 1.0 / 30.0
HOPS = 3
def load_ladder() -> pd.DataFrame:
parts = []
for stem in ("nscaling-a", "nscaling-b", "nscaling32-a", "nscaling32-b"):
cands = sorted(RUNS.glob(f"*_{stem}/results.parquet"))
if not cands:
raise SystemExit(f"missing run for {stem}")
d = pd.read_parquet(cands[-1])
d["case"] = "b" if stem.endswith("-b") else "a"
parts.append(d)
df = pd.concat(parts, ignore_index=True)
# Per-trajectory 50% burn-in (matches the report's line-55 convention). A global
# epoch.max()//2 threshold discards the early-stopping large-N tiers entirely, which
# dropped deg6/deg8's N=32000 points and left deg4's on a single replicate.
_keys = ["case", "n_nodes", "degree", "blend_delay_max", "max_uncles", "replicate"]
tail = df[df.epoch >= df.groupby(_keys).epoch.transform("max") // 2]
eq = tail.groupby(["case", "n_nodes", "degree", "blend_delay_max", "max_uncles",
"replicate"], as_index=False).mean_ratio.mean()
return eq
def lmean_fits(probe: pd.DataFrame) -> dict[int, tuple[float, float]]:
"""Per-degree log-fit l_mean ~ a*ln(N) + b from the exact probe."""
lm = probe.groupby(["n", "degree"], as_index=False).l_mean.mean()
fits = {}
for deg in sorted(lm.degree.unique()):
s = lm[lm.degree == deg]
a, b = np.polyfit(np.log(s.n), s.l_mean, 1)
fits[int(deg)] = (float(a), float(b))
return fits
def rho_of(eq: pd.DataFrame, probe: pd.DataFrame) -> pd.DataFrame:
# fitted l_mean covers every ladder N (the probe grid is log-spaced, not the ladder's)
fits = lmean_fits(probe)
m = eq.copy()
m["l_mean"] = [fits[int(d)][0] * np.log(n) + fits[int(d)][1]
for d, n in zip(m.degree, m.n_nodes, strict=True)]
m["d_vis"] = HOPS * m.blend_delay_max / 2.0 + (HOPS + 1) * m.l_mean
m["rho"] = F * m.d_vis
return m
def fig23(eq: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
fig, axes = plt.subplots(1, 2, figsize=(9.6, 4.2), sharey=True)
for ax, case, title in ((axes[0], "a", "case (a): geo delays"),
(axes[1], "b", "case (b): + 10% Poisson(3) stragglers")):
s = eq[(eq.case == case) & (eq.blend_delay_max == 8.0) & (eq.max_uncles == 1)]
for i, deg in enumerate((4, 6, 8)):
g = s[s.degree == deg].groupby("n_nodes").mean_ratio.agg(["mean", "sem"])
ax.errorbar(g.index, g["mean"], yerr=g["sem"], fmt="-o", ms=4, capsize=2,
color=style.OKABE_ITO[i], label=f"degree {deg}")
u2 = eq[(eq.case == case) & (eq.blend_delay_max == 8.0) & (eq.max_uncles == 2)]
g2 = u2.groupby("n_nodes").mean_ratio.mean()
ax.plot(g2.index, g2.values, ":", color="0.5", lw=1.2, label="U = 2 (all degrees)")
ax.axhline(0.98, color="0.75", lw=0.8, ls="--")
ax.text(1100, 0.982, "0.98 recovery bar", fontsize=7, color="0.5")
ax.set_xscale("log")
ax.set_xlabel("network size N")
ax.set_title(title)
ax.legend(fontsize=8, loc="lower left")
axes[0].set_ylim(top=1.01) # bounded by 1: cap at the exact-recovery bound (above-1 is noise)
axes[0].set_ylabel(r"stake estimate accuracy $\hat D / D$")
fig.suptitle(r"U = 1 erodes with network size (blend, $\delta_{max}$ = 8 s, f = 1/30)",
y=1.02)
style.save(fig, FIGS / "fig23_nscaling_u1", provenance="scripts/nscaling_analysis.py")
plt.close(fig)
def fig24(m: pd.DataFrame, probe: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(9.6, 4.2))
# left: ladder accuracy vs rho, all (N, degree, delay) cells collapse onto one curve
for case, marker, lbl in (("a", "o", "case (a)"), ("b", "s", "case (b)")):
s = m[(m.case == case) & (m.max_uncles == 1)]
g = s.groupby(["n_nodes", "degree", "blend_delay_max"]).agg(
rho=("rho", "mean"), acc=("mean_ratio", "mean")).reset_index()
ax1.scatter(g.rho, g.acc, s=18, marker=marker, alpha=0.75, label=lbl)
ax1.axvline(1.0, color="0.6", lw=0.8, ls=":")
ax1.text(0.98, 0.05, r"$\rho=1$", rotation=90, fontsize=7, color="0.4",
va="bottom", ha="right", transform=ax1.get_xaxis_transform())
ax1.set_xlabel(r"load $\rho = f\,D_{vis}(N, d, \delta)$")
ax1.set_ylabel(r"$\hat D / D$ at U = 1")
ax1.set_ylim(top=1.01) # bounded by 1: cap at the exact-recovery bound
ax1.set_title("ladder collapses onto the load law")
ax1.legend(fontsize=8)
# right: probe rho(N) per degree to 1M, delta=8
lm = probe.groupby(["n", "degree"], as_index=False).l_mean.mean()
for i, deg in enumerate((4, 6, 8)):
s = lm[lm.degree == deg].sort_values("n")
rho = F * (HOPS * 8.0 / 2.0 + (HOPS + 1) * s.l_mean)
ax2.plot(s.n, rho, "-o", ms=4, color=style.OKABE_ITO[i], label=f"degree {deg}")
ax2.axhline(1.0, color="0.6", lw=0.8, ls=":")
ax2.axhline(0.96, color="tab:red", lw=0.8, ls="--")
ax2.text(1.0e6, 0.955, "measured U=1 failure (ρ=0.96, §3.6)", fontsize=7, color="tab:red",
ha="right", va="top")
ax2.set_xscale("log")
ax2.set_xlabel("network size N")
ax2.set_ylabel(r"load $\rho$ at $\delta_{max}$ = 8 s")
ax2.set_title(r"exact probe: $\rho(N)$ to $10^6$ nodes")
ax2.legend(fontsize=8, loc="upper left")
style.save(fig, FIGS / "fig24_nscaling_probe", provenance="scripts/nscaling_analysis.py")
plt.close(fig)
def main() -> None:
probe = pd.read_parquet(RUNS / "topology_probe.parquet")
eq = load_ladder()
m = rho_of(eq, probe)
fig23(eq)
fig24(m, probe)
# §3.8 numbers
print("=== U=1 accuracy vs N (delta=8) ===")
t = eq[(eq.blend_delay_max == 8.0) & (eq.max_uncles == 1)]
print(t.groupby(["case", "degree", "n_nodes"]).mean_ratio.mean().round(3).to_string())
print("\n=== probe rho(delta=8) at 1M ===")
lm = probe.groupby(["n", "degree"]).l_mean.mean().reset_index()
one = lm[lm.n == 1_000_000]
for _, r in one.iterrows():
rho = F * (HOPS * 4.0 + 4 * r.l_mean)
print(f"degree {int(r.degree)}: l_mean={r.l_mean:.2f} rho={rho:.3f}")
for d, a, b in ((4, 0.326, -0.23), (6, 0.192, -0.06), (8, 0.143, 0.00)):
nstar = np.exp(((0.96 / F - HOPS * 4.0) / 4 - b) / a)
print(f"degree {d}: U=1 failure crossing at N* ≈ {nstar:.2e}")
if __name__ == "__main__":
main()

57
tools/simulators/tsi/tsi-sim-pernode/scripts/regenerate_extra_figs.py Normal file
View File

@ -0,0 +1,57 @@
"""Render the per-node simulator figure TYPES that the report was missing (fig17-fig22, plus fig5).
Each comes from an existing ``figures_pernode`` function on committed sweep data, so the report
includes and discusses every figure the simulator generates (§3). The bootstrap type
(``block_production_stabilization``) is fig1, rendered by scripts/bootstrap_dynamics.py. To run:
python scripts/regenerate_extra_figs.py
"""
from __future__ import annotations
from pathlib import Path
import pandas as pd
from tsi_sim.plotting import figures_pernode as F
from tsi_sim.plotting import style
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
FIGS = HERE / "report-figures"
# Large-scale source: the N=10000 fullscale run (also holds N-scaling context via §3.2 table).
FULL = sorted(RUNS.glob("2026-07-2*_fullscale/results.parquet"))[-1]
HET = sorted(RUNS.glob("2026-07-2*_default/results.parquet"))[-1]
WIN = sorted(RUNS.glob("2026-07-2*_uncle-window/results.parquet"))[-1]
WU = sorted(RUNS.glob("2026-07-2*_window-uncles/results.parquet"))[-1]
def main() -> None:
full = pd.read_parquet(FULL)
het = pd.read_parquet(HET)
win = pd.read_parquet(WIN)
wu = pd.read_parquet(WU)
jobs = [
("fig2_uncle_recovery", F.accuracy_vs_u(full, "pareto", "blend")),
("fig4_window", F.accuracy_vs_uncle_window(win, "pareto", "blend")),
("fig17_divergence", F.divergence_vs_epoch(full, "pareto", "blend")),
("fig18_tip_agreement", F.tip_agreement_vs_latency(full, "pareto", "blend")),
("fig19_accuracy_vs_latency", F.accuracy_vs_link_latency(full, "pareto", "regular")),
("fig20_heatmap_accuracy", F.heatmap_accuracy(full, "pareto", 6, "blend")),
("fig21_heterogeneous_recovery", F.heterogeneous_recovery(het, "uniform", "regular")),
("fig22_heatmap_window_delay", F.heatmap_window_delay(win, "pareto", "blend")),
# fig5 (W x U safe region at one delay): the (window x uncles) sweep, blend, delay=16 s
# (the delay the report's §3.5 "W=100, U=4 -> 0.96" callout reads off).
("fig5_window_uncles", F.heatmap_window_uncles(wu, "pareto", 16.0, "blend")),
]
for stem, fig in jobs:
if fig is None:
print(f"SKIP {stem} (no data)")
continue
style.save(fig, FIGS / stem, provenance="scripts/regenerate_extra_figs.py")
print(f"wrote {stem}")
if __name__ == "__main__":
main()

193
tools/simulators/tsi/tsi-sim-pernode/scripts/reorg_depth.py Normal file
View File

@ -0,0 +1,193 @@
"""Fork depth and private-chain reorg depth vs parameters and adversary stake (fig27, fig28).
Honest fork depth (adversary 0 %) is measured by the engine (runs/fork_rate_vs_delay.parquet:
fork_rate and max_reorg_depth vs Blend delay and uncle cap). The adversarial deepest-reorg tail
comes from src/tsi_sim/reorg.py, coupled to the measured honest orphan rate via alpha_eff.
fig27 — P(reorg depth >= d) vs d, per adversary stake {0, 10, 20, 30 %}, at the recommended
operating point; closed-form tail with Monte-Carlo validation markers.
fig28 — reorg depth vs Blend delay: the honest max depth (engine, U=0 vs U=2) and the
adversarial 99.9-percentile depth per stake — both fall steeply as delay drops
(fewer forks) and as uncles keep the block rate at f.
Run: python scripts/reorg_depth.py
"""
from __future__ import annotations
import sys
from pathlib import Path
import numpy as np
import pandas as pd
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from tsi_sim.plotting import style # noqa: E402
from tsi_sim.reorg import alpha_effective, reorg_depth_tail # noqa: E402
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
FIGS = HERE / "report-figures"
F = 1.0 / 30.0
STAKES = [0.0, 0.1, 0.2, 0.3]
COL = {0.0: "0.5", 0.1: style.OKABE_ITO[0], 0.2: style.OKABE_ITO[1], 0.3: style.OKABE_ITO[2]}
def depth_for_prob(alpha_eff: float, p: float = 1e-3) -> float:
"""Smallest depth d with P(reorg >= d) < p (a practical worst-case reorg to defend against)."""
if alpha_eff <= 0.0:
return 0.0
if alpha_eff >= 0.5:
return float("inf")
r = alpha_eff / (1.0 - alpha_eff)
return float(np.ceil(np.log(p) / np.log(r)))
def fig27(o_ref: float) -> None:
import matplotlib.pyplot as plt
style.apply_style()
rng = np.random.default_rng(20260723)
fig, ax = plt.subplots(figsize=(7.6, 4.4))
ds = np.arange(1, 13)
for alpha in STAKES:
ae = alpha_effective(alpha, o_ref)
if alpha == 0.0:
ax.plot(ds, [0] * len(ds), "-", color=COL[alpha], lw=1.4,
label="0 % (no private chain)")
continue
tail = [reorg_depth_tail(ae, int(d)) for d in ds]
ax.plot(ds, tail, "-", color=COL[alpha], lw=1.6,
label=f"{alpha:.0%} (α_eff={ae:.2f})")
# Monte-Carlo of the stationary catch-up tail: the fraction of time a reflected
# random walk (adversary lead over the public chain, down-drift since ae<1/2) sits
# at least d ahead converges to the closed form r^d — the quantity the lines plot.
walk = rng.random(4_000_000) < ae
lead = 0
occ = np.zeros(len(ds) + 1, dtype=np.int64)
for up in walk:
lead = lead + 1 if up else max(lead - 1, 0)
if lead:
occ[1:min(lead, len(ds)) + 1] += 1
pts = [occ[int(d)] / walk.size for d in ds[:6]]
ax.plot(ds[:6], pts, "s", ms=4, color=COL[alpha], alpha=0.5)
ax.set_yscale("log")
ax.set_ylim(1e-6, 1.5)
ax.set_xlabel("reorg depth d (confirmations reversed)")
ax.set_ylabel(r"$P(\mathrm{reorg\ depth} \geq d)$")
ax.set_title(f"Deepest-reorg tail vs adversary stake (Blend, operating point o≈{o_ref:.2f})\n"
"lines: closed form; squares: Monte-Carlo")
ax.legend(fontsize=8, title="adversary stake")
style.save(fig, FIGS / "fig27_reorg_tail", provenance="scripts/reorg_depth.py")
plt.close(fig)
def fig28(fr: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
fig, ax = plt.subplots(figsize=(7.8, 4.4))
# honest measured max reorg depth (engine), U=0 vs U=2
for U, ls, lbl in ((0, ":", "honest, U=0 (overproduces)"), (2, "-", "honest, U=2")):
s = fr[fr.U == U].sort_values("delta")
ax.plot(s.delta, s.max_depth, ls, color="0.4", lw=1.4, marker="o", ms=4, label=lbl)
# adversarial 99.9-pct depth vs delay, per stake, using U=2 honest orphan rate.
# inf (alpha_eff >= 1/2: unbounded) is drawn as an off-top marker with a "∞" callout.
base = fr[fr.U == 2].sort_values("delta")
ax.set_ylim(0, 40)
for alpha in (0.1, 0.2, 0.3):
raw = [depth_for_prob(alpha_effective(alpha, o)) for o in base.fork_rate]
depths = [min(d, 39) for d in raw]
ax.plot(base.delta, depths, "-", color=COL[alpha], lw=1.6, marker="s", ms=4,
label=f"adversary {alpha:.0%} (99.9-pct)")
for x, r in zip(base.delta, raw, strict=True):
if np.isinf(r):
ax.annotate("∞ (unbounded)", (x, 39), color=COL[alpha], fontsize=7,
ha="center", va="top")
ax.axvline(16.8, color="0.8", lw=0.8, ls="--")
ax.text(15.8, 37, "ρ≈1 (δ≈17 s)", fontsize=7, color="0.5", ha="right")
ax.set_xlabel("Blend per-hop blending budget δ (s) [more delay → more forks]")
ax.set_ylabel("reorg depth (blocks)")
ax.set_title("Reorg depth grows with delay and adversary stake — "
"shrunk by uncles (block rate at f) and by ρ<1")
ax.legend(fontsize=8, ncols=2)
style.save(fig, FIGS / "fig28_reorg_depth_vs_delay", provenance="scripts/reorg_depth.py")
plt.close(fig)
def measure_fork_rate() -> pd.DataFrame:
"""Honest fork rate + max reorg depth vs Blend delay and uncle cap (engine, adversary 0 %)."""
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
rows = []
for delta in (2.0, 4.0, 8.0, 16.0, 32.0):
for u in (0, 2):
frs, mds = [], []
for rep in range(3):
df = pd.DataFrame(run_trajectory(SimConfig(
n_nodes=1000, stake_dist="pareto", topology="blend", degree=6,
link_latency_mean=0.5, link_latency_dist="geo", blend_hops=3,
blend_delay_max=delta, max_uncles=u, uncle_window=300, k=256,
epochs=16, genesis_d_factor=0.5, early_stop=True, replicate=rep)))
t = df[df.epoch >= 6]
frs.append(t.fork_rate.mean())
mds.append(t.max_reorg_depth.max())
rows.append(dict(delta=delta, U=u, fork_rate=float(np.mean(frs)),
max_depth=int(np.max(mds))))
out = pd.DataFrame(rows)
out.to_parquet(RUNS / "fork_rate_vs_delay.parquet")
return out
def measure_fork_rate_scale() -> pd.DataFrame:
"""Honest fork rate vs N and peering degree at the recommended budget (U=2, δ=8 s).
Widens the reorg study (§6.10): since a bigger/sparser network raises the honest fork rate,
and the adversary's effective share depends on it, reorg depth should be shown vs N/degree.
"""
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
rows = []
for n in (1000, 4000, 16000):
for deg in (4, 6, 8):
frs = []
for rep in range(3):
df = pd.DataFrame(run_trajectory(SimConfig(
n_nodes=n, stake_dist="pareto", topology="blend", degree=deg,
link_latency_mean=0.5, link_latency_dist="geo", blend_hops=3,
blend_delay_max=8.0, max_uncles=2, uncle_window=300, k=256,
epochs=16, genesis_d_factor=0.5, early_stop=True, replicate=rep)))
frs.append(df[df.epoch >= 6].fork_rate.mean())
o = float(np.mean(frs))
row = dict(n=n, degree=deg, fork_rate=o)
for alpha in (0.1, 0.2, 0.3):
row[f"d999_{int(alpha * 100)}"] = depth_for_prob(alpha_effective(alpha, o))
rows.append(row)
out = pd.DataFrame(rows)
out.to_parquet(RUNS / "fork_rate_vs_scale.parquet")
print(out.round(3).to_string(index=False))
return out
def main() -> None:
if "--measure" in sys.argv:
print(measure_fork_rate().to_string(index=False))
return
if "--measure-scale" in sys.argv:
measure_fork_rate_scale()
return
fr = pd.read_parquet(RUNS / "fork_rate_vs_delay.parquet")
o_ref = float(fr[(fr.U == 2) & (fr.delta == 8.0)].fork_rate.iloc[0])
fig27(o_ref)
fig28(fr)
print("=== reorg-depth summary (U=2 operating points) ===")
for _, row in fr[fr.U == 2].sort_values("delta").iterrows():
line = f"δ={row.delta:4.0f}s honest o={row.fork_rate:.2f} max_depth={row.max_depth}"
for alpha in (0.1, 0.2, 0.3):
dd = depth_for_prob(alpha_effective(alpha, row.fork_rate))
line += f" | {alpha:.0%}: d99.9={dd:.0f}"
print(line)
print("wrote fig27_reorg_tail, fig28_reorg_depth_vs_delay")
if __name__ == "__main__":
main()

119
tools/simulators/tsi/tsi-sim-pernode/scripts/reward_mandate.py Normal file
View File

@ -0,0 +1,119 @@
"""Soft (reward-weighted) uncle inclusion — REPORT §6.7 / §6.8 (fig15).
Inclusion is a SOFT rule (omission forfeits the nephew reward; it is NOT a block-validity rule) —
because no node can prove which forks a producer could see within the window, so a *validity* rule
cannot be encoded fork-safely (§6.8). Under a soft rule the reference rate ``p_ref`` is EMERGENT: an
honest orphan was published, so any honest canonical block that sees it within ``W`` references it
for the nephew reward — the attacker only suppresses on *its own* canonical blocks. So ``p_ref`` is
high in practice (honest referencers), driven toward 1 by a larger ``W`` and toward 0 only by deep
reorgs whose orphans age out of the window before an honest block references them.
fig15 sweeps that emergent ``p_ref`` for the SM1 selfish attacker (gamma=0, self-uncle on):
LEFT — reward share vs ``p_ref``: the ``p_ref → 0`` end (attacker suppresses all / tiny ``W``) is
the *backfire* (share above block-only); the honest-referencer / large-``W`` end
(``p_ref → 1``) reaches ~stake. The crossover below block-only is near ``p_ref ≈ 0.3``.
RIGHT — honest-orphan reward recovery vs ``p_ref`` (the fairness metric).
So the soft rule delivers the fairness + selfish-mitigation of a hard mandate *without* the fork
risk, to the extent ``W``/visibility keep ``p_ref`` high; the residual is exactly the "can't
guarantee a node sees every fork in the window" gap. Also prints the bribery bound (§6.9).
Run: python scripts/reward_mandate.py (writes runs/reward_mandate.parquet + fig15)
"""
from __future__ import annotations
from pathlib import Path
import numpy as np
import pandas as pd
from tsi_sim.plotting import style
from tsi_sim.selfish import RewardParams, honest_reward_recovery, race_from_alpha, reward_shares
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
FIGS = HERE / "report-figures"
RUNS.mkdir(exist_ok=True)
FIGS.mkdir(exist_ok=True)
N_EVENTS = 4_000_000
ALPHAS = [0.35, 0.40, 0.46] # near / above the selfish threshold, where uncle rewards matter
P_REFS = [0.0, 0.2, 0.3, 0.5, 0.7, 0.85, 1.0]
W_U, W_N = 0.875, 0.03125 # Ethereum-like: w_u + w_n = 0.906 < 1 (farming-safe)
def sweep() -> pd.DataFrame:
rng = np.random.default_rng(11)
rows = []
for alpha in ALPHAS:
r = race_from_alpha(alpha, N_EVENTS, 0.0, rng)
for p in P_REFS:
# soft rule: honest referencers take the nephew (adv_nephew=0), attacker self-uncles
rp = RewardParams(w_uncle=W_U, w_nephew=W_N, p_ref=p, p_ref_adv=1.0, adv_nephew=0.0)
rec = honest_reward_recovery(r, RewardParams(w_uncle=W_U, p_ref=p))
rows.append(dict(alpha=alpha, p_ref=p, block=r.revenue_share,
reward_share=reward_shares(r, rp).adv_reward_share, recovery=rec))
out = pd.DataFrame(rows)
out.to_parquet(RUNS / "reward_mandate.parquet")
return out
def report(df: pd.DataFrame) -> None:
print(f"Soft rule, Ethereum-like w_u={W_U}, w_n={W_N} (sum {W_U+W_N:.3f} < 1, farming-safe):")
for alpha in ALPHAS:
s = df[df.alpha == alpha].sort_values("p_ref")
block = s.block.iloc[0]
print(f"alpha={alpha} block_share={block:.3f} stake={alpha}: reward_share by p_ref")
for _, row in s.iterrows():
tag = " BACKFIRE" if row.reward_share > block + 1e-3 else ""
print(f" p_ref={row.p_ref:.2f}: share={row.reward_share:.3f} "
f"rec={row.recovery:.2f}{tag}")
print("\nbribery to suppress a reference (§6.9): a soft rule costs the briber only w_nephew =",
W_N, "(cheap) but never forks; a hard rule would cost a full block but cannot be encoded "
"fork-safely.")
def fig15(df: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
fig, axes = plt.subplots(1, 2, figsize=(9.6, 3.8))
ax = axes[0]
for i, alpha in enumerate(ALPHAS):
s = df[df.alpha == alpha].sort_values("p_ref")
c = style.OKABE_ITO[i]
ax.plot(s.p_ref, s.reward_share, "-o", ms=4, color=c, label=rf"$\alpha={alpha}$")
ax.axhline(s.block.iloc[0], color=c, lw=0.8, ls=":", alpha=0.7) # block-only reference
ax.axvspan(0, 0.3, color="0.9", label="attacker-suppressed / small W")
ax.set_xlabel(r"emergent reference rate $p_{\rm ref}$ (grows with $W$)")
ax.set_ylabel("attacker reward share")
ax.set_title("Soft rule: high $p_{\\rm ref}$ (honest refs) → ~stake")
ax.legend(fontsize=7, loc="upper right")
ax = axes[1]
for i, alpha in enumerate(ALPHAS):
s = df[df.alpha == alpha].sort_values("p_ref")
ax.plot(s.p_ref, s.recovery, "-o", ms=4, color=style.OKABE_ITO[i],
label=rf"$\alpha={alpha}$")
ax.axhline(1.0, color="0.5", lw=0.8, ls="--")
ax.set_xlabel(r"emergent reference rate $p_{\rm ref}$")
ax.set_ylabel("honest reward recovery")
ax.set_title(r"Honest-orphan compensation ($w_u=0.875$)")
ax.set_ylim(0, 1.05)
ax.legend(fontsize=7, loc="lower right")
style.save(fig, FIGS / "fig15_mandate", provenance="scripts/reward_mandate.py")
plt.close(fig)
def main() -> None:
print("=== soft (reward-weighted) uncle inclusion: reward share vs emergent p_ref ===")
df = sweep()
report(df)
fig15(df)
print("wrote fig15_mandate")
if __name__ == "__main__":
main()

90
tools/simulators/tsi/tsi-sim-pernode/scripts/rho_boundary_analysis.py Normal file
View File

@ -0,0 +1,90 @@
"""Deficit-vs-load figure (fig26) from the rho-boundary sweep (configs/rho-boundary.yaml).
The "region below the block rate": the estimator equilibrium is bounded by 1 (it cannot over-count
occupied slots), so the signal of interest is the UNDER-COUNT DEFICIT 1 - D̂/D >= 0 as a function of
the load rho = f*D_vis, per uncle cap U. hops is fixed at 3 in the sweep so rho ∝ blend_delay_max.
Left panel: deficit 1 - D̂/D vs rho, per U (log-y), with the U=⌈ρ⌉ boundary visible.
Right panel: the same as accuracy D̂/D vs rho, y-axis capped at the 1.0 bound — no above-1 headroom;
residual above-1 shows only as ±σ error bars (sampling noise around ≤1).
Run: python scripts/rho_boundary_analysis.py
"""
from __future__ import annotations
import sys
from pathlib import Path
import numpy as np
import pandas as pd
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from tsi_sim.plotting import style # noqa: E402
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
FIGS = HERE / "report-figures"
F = 1.0 / 30.0
HOPS = 3
LMEAN = 1.2 # degree-6, N=1000 geo graph (matches §4)
def load() -> pd.DataFrame:
src = sorted(RUNS.glob("*_rho-boundary/results.parquet"))[-1]
df = pd.read_parquet(src)
keys = ["blend_delay_max", "max_uncles", "replicate"]
df["emax"] = df.groupby(keys).epoch.transform("max")
tail = df[df.epoch >= df.emax // 2]
g = (tail.groupby(["blend_delay_max", "max_uncles"])
.mean_ratio.agg(["mean", "sem"]).reset_index())
g["rho"] = F * (HOPS * g.blend_delay_max / 2.0 + (HOPS + 1) * LMEAN)
g["deficit"] = 1.0 - g["mean"]
return g
def fig26(g: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(9.6, 4.2))
for i, U in enumerate((0, 1, 2, 3)):
s = g[g.max_uncles == U].sort_values("rho")
c = style.OKABE_ITO[i]
# left: deficit (floored at a small positive value for the log axis)
d = np.clip(s.deficit.values, 3e-4, None)
ax1.plot(s.rho, d, "-o", ms=4, color=c, label=f"U = {U}")
# right: accuracy, capped at 1.0
ax2.errorbar(s.rho, s["mean"], yerr=s["sem"], fmt="-o", ms=4, capsize=2,
color=c, label=f"U = {U}")
ax1.set_yscale("log")
ax1.set_xlabel(r"load $\rho = f\,D_{vis}$")
ax1.set_ylabel(r"under-count deficit $1 - \hat D/D$")
ax1.set_title(r"deficit grows once $\rho$ exceeds the uncle cap")
ax1.axvline(1.0, color="0.6", lw=0.8, ls=":")
ax1.legend(fontsize=8, title="uncle cap")
ax2.axhline(1.0, color="0.4", lw=1.0, ls="--")
ax2.text(g.rho.min(), 1.001, r"$\hat D/D = 1$ bound (cannot over-count)",
fontsize=7, color="0.4", va="bottom")
ax2.set_ylim(0.0, 1.02) # cap at the bound: no above-1 headroom
ax2.set_xlabel(r"load $\rho = f\,D_{vis}$")
ax2.set_ylabel(r"accuracy $\hat D/D$ (bounded by 1)")
ax2.set_title("equilibrium sits at or below 1 at every load")
ax2.legend(fontsize=8, loc="lower left", title="uncle cap")
fig.suptitle(r"The region below the block rate: under-count deficit vs load "
r"(blend, N=1000, f=1/30, hops=3)", y=1.02)
style.save(fig, FIGS / "fig26_deficit_vs_rho", provenance="scripts/rho_boundary_analysis.py")
plt.close(fig)
def main() -> None:
g = load()
fig26(g)
above = g[g["mean"] > 1 + 2 * g["sem"]]
print(f"bounded-by-1 check: {len(above)}/{len(g)} cells above 1 by >2 SEM; "
f"max D̂/D = {g['mean'].max():.4f}")
print("wrote fig26_deficit_vs_rho")
if __name__ == "__main__":
main()

43
tools/simulators/tsi/tsi-sim-pernode/scripts/run_all_reruns.sh Executable file
View File

@ -0,0 +1,43 @@
#!/usr/bin/env bash
# Resilient corrected-mechanism rerun: each step runs independently; a failure is logged and
# the batch continues. Summary at the end. Light studies first, heavy fullscale last.
cd "$(dirname "$0")/.." || exit 1
source .venv/bin/activate 2>/dev/null
LOG=runs/rerun_status.log
: > "$LOG"
step() {
local name="$1"; shift
echo "=== [$(date +%H:%M:%S)] START $name ===" | tee -a "$LOG"
if "$@" >>"$LOG" 2>&1; then
echo "=== OK $name ===" | tee -a "$LOG"
else
echo "=== FAIL $name (exit $?) ===" | tee -a "$LOG"
fi
}
# light config sweeps
step nscaling-a make nscaling-a SWEEP_ARGS="--n-jobs 6"
step nscaling-b make nscaling-b SWEEP_ARGS="--n-jobs 6"
step nscaling32-a make nscaling32-a SWEEP_ARGS="--n-jobs 2"
step nscaling32-b make nscaling32-b SWEEP_ARGS="--n-jobs 2"
step uncle-window make uncle-window SWEEP_ARGS="--n-jobs 6"
step window-uncles make window-uncles SWEEP_ARGS="--n-jobs 6"
step block-rate make block-rate SWEEP_ARGS="--n-jobs 6"
step blend-hops-delay make blend-hops-delay SWEEP_ARGS="--n-jobs 6"
step window-scale make window-scale SWEEP_ARGS="--n-jobs 6"
step expdist make expdist SWEEP_ARGS="--n-jobs 8"
step pareto133 make pareto133 SWEEP_ARGS="--n-jobs 8"
step default make default SWEEP_ARGS="--n-jobs 6"
# scripts
step stake_vs_delay python scripts/stake_vs_delay.py
step bootstrap python scripts/bootstrap_dynamics.py
step fluctuation python scripts/appendix_fluct.py --run
step jitter_grid python scripts/jitter_grid.py
step adversary_grid python scripts/adversary_grid.py
step dynamic_withhold python scripts/dynamic_withhold.py
step selfish_mining python scripts/selfish_mining.py
step selfish_rewards python scripts/selfish_rewards.py
step reward_mandate python scripts/reward_mandate.py
# heavy last so nothing waits on it
step fullscale make fullscale SWEEP_ARGS="--n-jobs 3 --mem-frac 0.55"
echo "=== [$(date +%H:%M:%S)] BATCH DONE ===" | tee -a "$LOG"
grep -E "OK|FAIL" "$LOG" | tee -a "$LOG"

14
tools/simulators/tsi/tsi-sim-pernode/scripts/run_sweep.py Normal file
View File

@ -0,0 +1,14 @@
#!/usr/bin/env python
"""Run a TSI parameter sweep from a YAML config -> results parquet."""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from tsi_sim.sweep import main # noqa: E402
if __name__ == "__main__":
main()

136
tools/simulators/tsi/tsi-sim-pernode/scripts/selfish_mining.py Normal file
View File

@ -0,0 +1,136 @@
"""Selfish / private-chain withholding vs TSI — REPORT §6.6, and the per-block issuance question.
Two panels (fig13):
LEFT — revenue share adv/(adv+hon) vs stake alpha, for gamma in {0, 0.5, 1}, with the Eyal-Sirer
closed form overlaid and the profitability thresholds marked. Above threshold the share
exceeds the diagonal (share = stake), so private-chain withholding IS profitable — the
opposite of §6.5's abstention withholding.
RIGHT — TSI coupling: the counted canonical density deflates D_hat to D*·(density fraction);
uncle references recover orphaned honest blocks back into the count, lifting D_hat toward
D* (uncle_recovery u in {0, 0.5, 1}). The mechanism that fixes the honest under-count
(§3.2) also blunts the selfish attacker's estimator deflation.
Issuance / absolute-reward note (the §6.5 GAP-2 question): TSI targets *counted* density = f, so the
canonical block rate is held at ~f regardless of the attack — the canonical "pie" does not inflate
when D_hat deflates (the extra lottery wins are orphans that earn no canonical reward). Hence for a
per-block reward schedule the adversary's ABSOLUTE reward per unit stake equals revenue_share/alpha,
identical to the share metric: §6.5's "unprofitable" abstention result is robust to per-block
issuance, and §6.6's selfish premium (revenue_share/alpha > 1 above threshold) is the real profit.
Run: python scripts/selfish_mining.py (writes runs/selfish_*.parquet + fig13)
"""
from __future__ import annotations
from pathlib import Path
import numpy as np
import pandas as pd
from tsi_sim.plotting import style
from tsi_sim.selfish import (
race_from_alpha,
selfish_revenue_closed_form,
selfish_threshold,
tsi_dhat_ratio,
)
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
FIGS = HERE / "report-figures"
RUNS.mkdir(exist_ok=True)
FIGS.mkdir(exist_ok=True)
N_EVENTS = 6_000_000 # per (alpha, gamma) cell; MC noise ~ 1e-3 on the share
GAMMAS = [0.0, 0.5, 1.0]
ALPHAS = [0.05, 0.10, 0.15, 0.20, 0.25, 1 / 3, 0.40, 0.45, 0.49]
def sweep() -> pd.DataFrame:
rng = np.random.default_rng(20240719)
rows = []
for gamma in GAMMAS:
for alpha in ALPHAS:
r = race_from_alpha(alpha, N_EVENTS, gamma, rng)
rows.append(dict(
alpha=alpha, gamma=gamma,
share=r.revenue_share,
closed_form=selfish_revenue_closed_form(alpha, gamma),
reward_per_stake=r.revenue_share / alpha, # absolute per-block NPV ratio
density_fraction=r.density_fraction, # D_hat/D* at u=0
dhat_u0=tsi_dhat_ratio(r, 0.0),
dhat_u50=tsi_dhat_ratio(r, 0.5),
dhat_u100=tsi_dhat_ratio(r, 1.0),
orphan_hon_frac=r.orphan_hon / r.events,
))
out = pd.DataFrame(rows)
out.to_parquet(RUNS / "selfish_sweep.parquet")
return out
def report(df: pd.DataFrame) -> None:
print(f"{'gamma':>5} {'thresh':>7} " + " ".join(f"a={a:.2f}" for a in [0.2, 1 / 3, 0.4]))
for gamma in GAMMAS:
g = df[df.gamma == gamma]
cells = []
for a in (0.2, 1 / 3, 0.4):
row = g[np.isclose(g.alpha, a)].iloc[0]
cells.append(f"{row.share:.3f}({row.reward_per_stake:.2f}x)")
print(f"{gamma:5.1f} {selfish_threshold(gamma):7.3f} " + " ".join(cells))
print("(share(reward/stake x); >1x = profitable). D_hat/D* deflation at alpha=0.4, gamma=0:")
r = df[(df.gamma == 0.0) & np.isclose(df.alpha, 0.4)].iloc[0]
print(f" u=0: {r.dhat_u0:.3f} u=0.5: {r.dhat_u50:.3f} u=1: {r.dhat_u100:.3f} "
f"(orphaned honest {r.orphan_hon_frac*100:.1f}% of blocks)")
def fig13(df: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
fig, axes = plt.subplots(1, 2, figsize=(9.6, 3.8))
# LEFT: revenue share vs alpha, per gamma, with closed form + diagonal + thresholds
ax = axes[0]
aa = np.array(ALPHAS)
ax.plot(aa, aa, color="0.5", lw=0.9, ls="--", label="honest (share = stake)")
for i, gamma in enumerate(GAMMAS):
g = df[df.gamma == gamma].sort_values("alpha")
c = style.OKABE_ITO[i]
ax.plot(g.alpha, g.share, "o", ms=4, color=c)
fine = np.linspace(0.02, 0.49, 200)
ax.plot(fine, [selfish_revenue_closed_form(a, gamma) for a in fine], "-", lw=1.3,
color=c, label=rf"$\gamma={gamma}$ (Eyal–Sirer)")
thr = selfish_threshold(gamma)
if 0 < thr < 0.5:
ax.axvline(thr, color=c, lw=0.7, ls=":")
ax.set_xlabel(r"adversary stake $\alpha$")
ax.set_ylabel("revenue share (canonical blocks)")
ax.set_title("Private-chain withholding is profitable above threshold")
ax.legend(fontsize=7, loc="upper left")
# RIGHT: TSI D_hat deflation vs alpha and uncle recovery (gamma=0, worst-case connectivity)
ax = axes[1]
g0 = df[df.gamma == 0.0].sort_values("alpha")
for u, col, lab in [("dhat_u0", style.OKABE_ITO[1], r"no uncles ($\eta$=0)"),
("dhat_u50", style.OKABE_ITO[4], r"$\eta$=0.5"),
("dhat_u100", style.OKABE_ITO[2], r"honest-orphan recovery ($\eta$=1)")]:
ax.plot(g0.alpha, g0[u], "-o", ms=4, color=col, label=lab)
ax.axhline(1.0, color="0.5", lw=0.9, ls="--", label=r"honest $D^*$")
ax.set_xlabel(r"adversary stake $\alpha$")
ax.set_ylabel(r"$\hat D / D^*$ (estimator deflation)")
ax.set_title("Selfish orphaning deflates $\\hat D$; uncles recover it")
ax.legend(fontsize=7, loc="lower left")
style.save(fig, FIGS / "fig13_selfish", provenance="scripts/selfish_mining.py")
plt.close(fig)
def main() -> None:
print("=== selfish-mining sweep (validated vs Eyal-Sirer) ===")
df = sweep()
report(df)
fig13(df)
print("wrote fig13_selfish")
if __name__ == "__main__":
main()

144
tools/simulators/tsi/tsi-sim-pernode/scripts/selfish_rewards.py Normal file
View File

@ -0,0 +1,144 @@
"""Optimal selfish mining + uncle-reward incentive design — REPORT §6.6 / §6.7 (fig14).
Two questions:
A. How much does the *optimal* (Sapirshtein MDP) selfish strategy beat SM1, and where is the
profitability threshold? (fig14, left)
B. Do block/uncle REWARDS defuse the attack? Paying an uncle reward to orphaned honest blocks
compensates them, so the selfish attacker's *reward* share falls below its block share and the
profitability threshold moves up. (fig14, right)
Adversarial framing (see report §6.7): uncle rewards (i) compensate honestly-orphaned producers,
(ii) disincentivise hiding (a withheld block never propagates -> can never be an uncle -> forfeits
both block and uncle reward), and (iii) shrink the selfish premium. The reward scheme's own attack
surface — "uncle farming" (deliberately orphaning your own blocks to collect uncle rewards) — is
bounded because uncles must be real VRF winners and an uncle pays w_uncle < 1 < a canonical block.
Run: python scripts/selfish_rewards.py (writes runs/selfish_rewards.parquet + fig14)
"""
from __future__ import annotations
from pathlib import Path
import numpy as np
import pandas as pd
from tsi_sim.plotting import style
from tsi_sim.selfish import (
RewardParams,
honest_reward_recovery,
race_from_alpha,
reward_shares,
selfish_revenue_closed_form,
)
from tsi_sim.selfish_mdp import optimal_selfish_revenue
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
FIGS = HERE / "report-figures"
RUNS.mkdir(exist_ok=True)
FIGS.mkdir(exist_ok=True)
N_EVENTS = 4_000_000
ALPHAS = [0.10, 0.15, 0.20, 0.25, 0.30, 1 / 3, 0.36, 0.40, 0.43, 0.46]
W_UNCLES = [0.0, 0.25, 0.5, 1.0]
def sweep_optimal() -> pd.DataFrame:
"""Optimal (MDP) vs SM1 vs honest revenue, plus reward-share under each uncle reward (g=0)."""
rng = np.random.default_rng(7)
rows = []
for alpha in ALPHAS:
for gamma in (0.0, 0.5):
opt = optimal_selfish_revenue(alpha, gamma, cap=40, iters=3000)
rows.append(dict(kind="revenue", alpha=alpha, gamma=gamma,
sm1=selfish_revenue_closed_form(alpha, gamma), optimal=opt))
# reward-share (gamma=0 SM1 race) under each uncle reward
r = race_from_alpha(alpha, N_EVENTS, 0.0, rng)
for wu in W_UNCLES:
rp = RewardParams(w_uncle=wu, p_ref=1.0)
rows.append(dict(kind="reward", alpha=alpha, w_uncle=wu,
block_share=r.revenue_share,
reward_share=reward_shares(r, rp).adv_reward_share,
honest_recovery=honest_reward_recovery(r, rp)))
out = pd.DataFrame(rows)
out.to_parquet(RUNS / "selfish_rewards.parquet")
return out
def _threshold(alphas, shares):
"""First alpha where share > alpha (profitability boundary), by linear interp; None if never."""
a = np.array(alphas)
d = np.array(shares) - a
for i in range(1, len(a)):
if d[i - 1] <= 0 < d[i]:
t = a[i - 1] + (a[i] - a[i - 1]) * (-d[i - 1]) / (d[i] - d[i - 1])
return float(t)
return None
def report(df: pd.DataFrame) -> None:
rev = df[df.kind == "revenue"]
print("optimal (MDP) vs SM1 revenue, gamma=0 / 0.5:")
for alpha in (1 / 3, 0.4, 0.46):
for g in (0.0, 0.5):
row = rev[(np.isclose(rev.alpha, alpha)) & (rev.gamma == g)].iloc[0]
print(f" a={alpha:.3f} g={g}: optimal={row.optimal:.3f} SM1={row.sm1:.3f} "
f"(gap {row.optimal-row.sm1:+.3f})")
rw = df[df.kind == "reward"]
print("\nuncle reward -> selfish profitability threshold (gamma=0, SM1):")
for wu in W_UNCLES:
s = rw[rw.w_uncle == wu].sort_values("alpha")
thr = _threshold(s.alpha.tolist(), s.reward_share.tolist())
rec = s[np.isclose(s.alpha, 0.40)].honest_recovery.iloc[0]
print(f" w_uncle={wu}: threshold alpha* = {thr if thr is None else round(thr,3)} "
f"(honest reward recovery @a=0.4: {rec:.3f})")
def fig14(df: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
fig, axes = plt.subplots(1, 2, figsize=(9.6, 3.8))
aa = np.array(ALPHAS)
# LEFT: optimal vs SM1 vs honest revenue
ax = axes[0]
rev = df[df.kind == "revenue"]
ax.plot(aa, aa, color="0.5", lw=0.9, ls="--", label="honest (= stake)")
for i, g in enumerate((0.0, 0.5)):
s = rev[rev.gamma == g].sort_values("alpha")
c = style.OKABE_ITO[i]
ax.plot(s.alpha, s.optimal, "-o", ms=4, color=c, label=rf"optimal, $\gamma={g}$")
ax.plot(s.alpha, s.sm1, ":", lw=1.4, color=c, label=rf"SM1, $\gamma={g}$")
ax.set_xlabel(r"adversary stake $\alpha$")
ax.set_ylabel("revenue share")
ax.set_title("Optimal selfish (MDP) vs SM1")
ax.legend(fontsize=7, loc="upper left")
# RIGHT: reward-share vs alpha under uncle rewards (gamma=0)
ax = axes[1]
rw = df[df.kind == "reward"]
ax.plot(aa, aa, color="0.5", lw=0.9, ls="--", label="break-even (= stake)")
for i, wu in enumerate(W_UNCLES):
s = rw[rw.w_uncle == wu].sort_values("alpha")
ax.plot(s.alpha, s.reward_share, "-o", ms=4, color=style.OKABE_ITO[i],
label=rf"$w_u={wu}$")
ax.set_xlabel(r"adversary stake $\alpha$")
ax.set_ylabel("attacker reward share")
ax.set_title("Uncle rewards shrink the selfish premium")
ax.legend(fontsize=7, loc="upper left")
style.save(fig, FIGS / "fig14_optimal_rewards", provenance="scripts/selfish_rewards.py")
plt.close(fig)
def main() -> None:
print("=== optimal-selfish + uncle-reward sweep ===")
df = sweep_optimal()
report(df)
fig14(df)
print("wrote fig14_optimal_rewards")
if __name__ == "__main__":
main()

214
tools/simulators/tsi/tsi-sim-pernode/scripts/split_report.py Normal file
View File

@ -0,0 +1,214 @@
"""One-time migration: split REPORT-tsi-parameter-selection.md into a thematic 4-part set + index.
The single report grew dense and heavily cross-referenced; this slices it into four cohesive parts
(kept in tsi-sim-pernode/ so all report-figures/ links stay valid) plus a short index that reuses the
canonical filename as the entry point. Section NUMBERS (§1-§9, A-C) are preserved as stable identifiers
across files; every §ref is rewritten into a clickable link to a portable `<a id="s6-5">` anchor,
same-file or cross-file as appropriate. Figure embeds and §9's config/script/run paths are untouched.
Run: python scripts/split_report.py (reads REPORT-...md, writes the 4 parts + overwrites the index)
"""
from __future__ import annotations
# ruff: noqa: E501 (one-time migration; index/nav strings are intentionally long prose)
import re
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent.parent
SRC = HERE / "REPORT-tsi-parameter-selection.md"
INDEX = "REPORT-tsi-parameter-selection.md"
P1 = "tsi-report-1-overview-and-recommendations.md"
P2 = "tsi-report-2-accuracy-and-design.md"
P3 = "tsi-report-3-robustness-and-incentives.md"
P4 = "tsi-report-4-reproducibility-and-appendices.md"
# top-level section id -> part filename, and the section order within each part
PART_SECTIONS = {
P1: ["1", "7", "8"],
P2: ["2", "3", "4", "5"],
P3: ["6"],
P4: ["9", "A", "B", "C"],
}
PART_TITLE = {
P1: "Part 1 — Overview and recommendations",
P2: "Part 2 — Accuracy and design",
P3: "Part 3 — Robustness and incentives",
P4: "Part 4 — Reproducibility and appendices",
}
SEC_TO_FILE = {s: f for f, secs in PART_SECTIONS.items() for s in secs}
def top_id(line: str) -> str | None:
m = re.match(r"^##\s+Appendix\s+([A-C])\b", line)
if m:
return m.group(1)
m = re.match(r"^##\s+(\d+)\.", line)
return m.group(1) if m else None
def header_anchor(line: str) -> str | None:
"""Anchor id for a section/subsection header line, e.g. §6.5 -> s6-5, App B.2 -> sB-2."""
m = re.match(r"^##\s+Appendix\s+([A-C])\b", line)
if m:
return "s" + m.group(1)
m = re.match(r"^##\s+(\d+)\.", line)
if m:
return "s" + m.group(1)
m = re.match(r"^###\s+([0-9A-C]+)\.(\d+)", line)
if m:
return f"s{m.group(1)}-{m.group(2)}"
return None
def make_ref_rewriter(current_file: str):
"""Rewrite §N/§N.M and 'Appendix X' refs into links to their anchor (same- or cross-file)."""
def link(top: str, sub: str | None, label: str) -> str:
anchor = "s" + top + (f"-{sub}" if sub else "")
tgt = SEC_TO_FILE.get(top)
if tgt is None:
return label # unknown target: leave as text
dest = f"#{anchor}" if tgt == current_file else f"{tgt}#{anchor}"
return f"[{label}]({dest})"
def sec_sub(m: re.Match) -> str:
top, sub = m.group(1), m.group(2)
return link(top, sub, m.group(0))
def appendix(m: re.Match) -> str:
return link(m.group(1), None, m.group(0))
sec_re = re.compile(r"§\s?(\d+)(?:\.(\d+))?")
app_re = re.compile(r"\bAppendix\s+([A-C])\b")
def rewrite(text: str) -> str:
return app_re.sub(appendix, sec_re.sub(sec_sub, text))
return rewrite
def render_lines(lines: list[str], current_file: str) -> list[str]:
"""Inject anchors before headers and rewrite §refs, skipping fenced code blocks."""
rewrite = make_ref_rewriter(current_file)
out: list[str] = []
in_fence = False
for ln in lines:
if ln.lstrip().startswith("```"):
in_fence = not in_fence
out.append(ln)
continue
if in_fence:
out.append(ln) # never touch code (refs there stay plain text)
continue
aid = header_anchor(ln)
if aid is not None:
out.append(f'<a id="{aid}"></a>')
out.append(ln) # header title kept verbatim (no links inside headers)
continue
out.append(rewrite(ln))
return out
def part_header(fname: str, units_note: str) -> list[str]:
nav = (f"*[Part 1 — Overview & recommendations]({P1}) · [Part 2 — Accuracy & design]({P2}) · "
f"[Part 3 — Robustness & incentives]({P3}) · [Part 4 — Reproducibility & appendices]({P4}) · "
f"[Index]({INDEX})*")
where = ("*Sections live across the set: §1/§7/§8 in Part 1, §2–§5 in Part 2, §6 in Part 3, "
"§9 and Appendices A–C in Part 4.*")
return [
f"# Total-Stake-Inference parameter selection — {PART_TITLE[fname].split('— ')[1]}",
"",
units_note,
"",
nav,
"",
where,
"",
"---",
"",
]
def build_index(units_note: str) -> str:
lines = [
"# Total-Stake-Inference parameter selection",
"",
units_note,
"",
"This analysis selects and justifies the TSI parameters for Cryptarchia, from a per-node "
"network simulation (`tsi-sim-pernode`). It is split into four parts:",
"",
f"1. **[Overview and recommendations]({P1})** — the executive summary, the per-knob parameter "
"reference (§7), and the safest selection with residual risks and the recommendation-vs-spec "
"deltas (§8).",
f"2. **[Accuracy and design]({P2})** — the model and counting rule (§2), the seven findings and "
"their evidence (§3), and the design equations / selection algorithm (§4–§5).",
f"3. **[Robustness and incentives]({P3})** — jitter, grinding, withholding, selfish mining, the "
"reward design, fork/reorg depth, and organic churn (§6).",
f"4. **[Reproducibility and appendices]({P4})** — how to re-run every study (§9), the residual "
"f-rounding offset (App A), the per-epoch noise floor (App B), and consensus detail (App C).",
"",
f"**Headline recommendation** (Cryptarchia baseline f = 1/30): security `k = 2160`, uncle "
f"window `W = 300` slots, uncle cap `U = ⌈ρ⌉ + 1` (2 at the Blend target), learning rate "
f"`β = 1`, peering degree ≥ 6 at scale, soft uncle rewards with `w_u + w_n < 1`, and operate "
f"at load `ρ = f·D_vis < 1`. The full recommended-configuration table and rationale are in "
f"**[Part 1 →]({P1})**.",
"",
]
return "\n".join(lines) + "\n"
def main() -> None:
text = SRC.read_text()
if "## 6. Robustness" not in text:
sys.exit("Source has already been split (no '## 6. Robustness' found in "
f"{SRC.name}, which is now the index). This one-time migration is complete; "
"re-run against the pre-split backup only.")
raw = text.split("\n")
# frontmatter (title + units note) is everything before the first "## " header
first_h = next(i for i, l in enumerate(raw) if l.startswith("## "))
units_note = raw[2] # the italic "*Per-node network simulation ... 1 slot = 1 s.*" line
# slice into top-level sections
sections: dict[str, list[str]] = {}
cur: str | None = None
for l in raw[first_h:]:
tid = top_id(l)
if tid is not None:
cur = tid
sections[cur] = []
if cur is not None:
sections[cur].append(l)
# update the reading-order note (in §1) to describe the 4-part structure
ro_old_prefix = "The rest of the report, in reading order:"
ro_new = ("This report is split into four parts (see the [index](" + INDEX + ")): "
"**Part 1** — the recommended configuration, the per-knob parameter reference (§7) and "
"the safest selection with residual risks and spec deltas (§8); **Part 2** — the model "
"and counting rule (§2), the evidence behind each finding (§3), and the design equations "
"and selection algorithm (§4–§5); **Part 3** — robustness against noise, attacks and the "
"incentive design (§6); **Part 4** — reproducibility (§9) and the appendices (the residual "
"~1 % f-rounding offset, the ±0.9 % per-epoch noise floor, and consensus detail).")
sections["1"] = [
ro_new if l.startswith(ro_old_prefix) else l for l in sections["1"]
]
# assemble each part
for fname, sec_ids in PART_SECTIONS.items():
body: list[str] = list(part_header(fname, units_note))
for sid in sec_ids:
body.extend(sections[sid])
body.append("") # spacer between sections
rendered = render_lines(body, fname)
(HERE / fname).write_text("\n".join(rendered).rstrip() + "\n")
print(f"wrote {fname} ({len(rendered)} lines)")
# index last (overwrites the source-name file)
(HERE / INDEX).write_text(build_index(units_note))
print(f"wrote {INDEX} (index)")
if __name__ == "__main__":
main()

101
tools/simulators/tsi/tsi-sim-pernode/scripts/stake_vs_delay.py Normal file
View File

@ -0,0 +1,101 @@
"""Relative stake estimate vs network delay (fig16).
Shows the report's central relationship: the recovered *relative stake* ``D̂/D`` as a function of the
mean block-visibility delay ``D_vis`` (seconds), one curve per uncle cap ``U``. Accuracy holds near
the ceiling ``c(f)`` while the load ``ρ = f·D_vis`` stays below ``⌈U⌉``, then collapses — so larger
delay needs more uncles. Blend transport, f = 1/30 (30 s blocks); delay swept via the
per-hop blending budget ``blend_delay_max``.
Run: python scripts/stake_vs_delay.py (writes runs/stake_vs_delay.parquet + fig16)
"""
from __future__ import annotations
from pathlib import Path
import numpy as np
import pandas as pd
from tsi_sim import topology
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
from tsi_sim.plotting import style
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
FIGS = HERE / "report-figures"
RUNS.mkdir(exist_ok=True)
FIGS.mkdir(exist_ok=True)
F = 1.0 / 30.0
HOPS = 3
DELAYS = [1.0, 4.0, 8.0, 12.0, 16.0, 20.0, 26.0, 32.0, 40.0] # per-hop blending budget (s)
UNCLES = [0, 1, 2, 3]
BASE = dict(n_nodes=800, k=48, epochs=16, stake_dist="uniform", topology="blend", degree=8,
link_latency_mean=0.3, link_latency_dist="geo", blend_hops=HOPS, uncle_window=300,
genesis_d_factor=0.5, f=F)
def d_vis(delay: float) -> float:
"""Mean visibility delay D_vis = hops·δ/2 + (hops+1)·ℓ_mean, ℓ_mean = mean shortest-path."""
cfg = SimConfig(**BASE, blend_delay_max=delay, max_uncles=1)
pl = topology.build_path_latency(cfg, np.random.default_rng(np.random.SeedSequence(0)))
off = pl[~np.eye(pl.shape[0], dtype=bool)]
l_mean = float(off[np.isfinite(off)].mean())
return HOPS * delay / 2.0 + (HOPS + 1) * l_mean
def sweep() -> pd.DataFrame:
rows = []
for delay in DELAYS:
dv = d_vis(delay)
for U in UNCLES:
reps = []
for r in range(5):
df = pd.DataFrame(run_trajectory(SimConfig(
replicate=r, blend_delay_max=delay, max_uncles=U, **BASE)))
reps.append(df[df.epoch >= 8].mean_ratio.mean())
ratio = float(np.mean(reps))
sem = float(np.std(reps) / np.sqrt(len(reps)))
rows.append(dict(delay=delay, d_vis=dv, rho=F * dv, U=U, ratio=ratio, sem=sem))
print(f"delay={delay:4.0f}s D_vis={dv:5.1f} rho={F*dv:4.2f} U={U}: D̂/D={ratio:.3f}")
out = pd.DataFrame(rows)
out.to_parquet(RUNS / "stake_vs_delay.parquet")
return out
def fig16(df: pd.DataFrame) -> None:
import matplotlib.pyplot as plt
style.apply_style()
fig, ax = plt.subplots(figsize=(7.2, 4.4))
ax.axhline(1.0, color="0.5", lw=0.9, ls="--", label="exact recovery (1.0)")
for i, U in enumerate(UNCLES):
s = df[df.U == U].sort_values("d_vis")
ax.errorbar(s.d_vis, s.ratio, yerr=s["sem"], fmt="-o", ms=4, capsize=2,
color=style.OKABE_ITO[i], label=f"U = {U}")
# rho = 1, 2, 3 boundaries: D_vis = k/f <-> rho = f*D_vis = k
for k in (1, 2, 3):
dv = k / F
if dv <= df.d_vis.max() * 1.02:
ax.axvline(dv, color="0.7", lw=0.7, ls=":")
ax.text(dv, 0.32, f"ρ={k}", rotation=90, va="bottom", ha="right", fontsize=7,
color="0.4")
ax.set_xlabel(r"mean block-visibility delay $D_{\rm vis}$ (s) [load $\rho = f\,D_{\rm vis}$]")
ax.set_ylabel(r"relative stake estimate $\hat D / D$")
ax.set_title(r"Recovered relative stake vs delay (blend, $f=1/30$): "
r"$U$ must grow with $\rho=\lceil f D_{\rm vis}\rceil$")
ax.set_ylim(0.3, 1.02) # bounded by 1: cap at the exact-recovery bound, no above-1 headroom
ax.legend(fontsize=8, loc="lower left")
style.save(fig, FIGS / "fig16_stake_vs_delay", provenance="scripts/stake_vs_delay.py")
plt.close(fig)
def main() -> None:
print("=== relative stake estimate vs delay ===")
df = sweep()
fig16(df)
print("wrote fig16_stake_vs_delay")
if __name__ == "__main__":
main()

95
tools/simulators/tsi/tsi-sim-pernode/scripts/topology_probe.py Normal file
View File

@ -0,0 +1,95 @@
"""Exact large-N topology probe: mean gossip path latency l_mean(N, degree) up to N = 10^6.
The only N-dependent term in the load law rho = f*D_vis (report §3.3/§4, eq 1) is the mean
shortest-path transport latency l_mean over the peering graph, which grows ~ log_(d-1) N.
Direct per-node simulation is memory-bound at N ~ 3*10^4 (the N x N matrix), but l_mean is
measurable EXACTLY at any N with sampled-source Dijkstra on the same graph generator the
simulator uses (circulant base + 10x Maslov-Sneppen swaps, geo per-link latencies).
Writes runs/topology_probe.parquet: one row per (n, degree, replicate) with l_mean, quantiles,
and the derived D_vis / rho for the study's mixing budgets. Used by report §3.8.
Run: python scripts/topology_probe.py (~2-3 h, parallel over cells)
"""
from __future__ import annotations
import sys
import time
from pathlib import Path
import numpy as np
import pandas as pd
from joblib import Parallel, delayed
from scipy.sparse import csr_matrix
from scipy.sparse.csgraph import dijkstra
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from tsi_sim.config import SimConfig # noqa: E402
from tsi_sim.topology import ( # noqa: E402
_circulant_edges,
_double_edge_swaps,
_sample_link_latencies,
)
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
F = 1.0 / 30.0
HOPS = 3
DELAYS = (4.0, 8.0) # blend mixing budgets studied in the N-scaling ladder
LINK_MEAN = 0.5 # per-link geo mean (slots), matching the ladder configs
N_GRID = (1_000, 4_000, 16_000, 64_000, 250_000, 1_000_000)
DEGREES = (4, 6, 8)
N_SOURCES = 64 # sampled Dijkstra sources per graph
REPS = {n: (3 if n <= 64_000 else 1) for n in N_GRID}
def probe_cell(n: int, degree: int, rep: int) -> dict:
rng = np.random.default_rng(np.random.SeedSequence([n, degree, rep, 20260721]))
cfg = SimConfig(n_nodes=min(n, 10_000), degree=degree, link_latency_mean=LINK_MEAN,
link_latency_dist="geo") # only used for latency sampling params
t0 = time.time()
edges = _circulant_edges(n, degree)
edges = _double_edge_swaps(edges, n_swaps=10 * len(edges), rng=rng)
e = np.array(sorted(edges), dtype=np.int64)
w = _sample_link_latencies(e.shape[0], cfg, rng)
adj = csr_matrix(
(np.concatenate([w, w]),
(np.concatenate([e[:, 0], e[:, 1]]), np.concatenate([e[:, 1], e[:, 0]]))),
shape=(n, n))
sources = rng.choice(n, size=min(N_SOURCES, n), replace=False)
dist = dijkstra(adj, directed=False, indices=sources)
mask = np.isfinite(dist) & (dist > 0)
d = dist[mask]
row = dict(
n=n, degree=degree, replicate=rep, n_sources=len(sources), n_edges=e.shape[0],
l_mean=float(d.mean()), l_p50=float(np.percentile(d, 50)),
l_p90=float(np.percentile(d, 90)), l_p99=float(np.percentile(d, 99)),
l_max=float(d.max()), build_s=float(time.time() - t0),
)
for delay in DELAYS:
dvis = HOPS * delay / 2.0 + (HOPS + 1) * row["l_mean"]
row[f"d_vis_{delay:g}"] = dvis
row[f"rho_{delay:g}"] = F * dvis
print(f"n={n:>9,} deg={degree} rep={rep}: l_mean={row['l_mean']:.2f} "
f"p99={row['l_p99']:.2f} rho(8)={row['rho_8']:.3f} [{row['build_s']:.0f}s]",
flush=True)
return row
def main() -> None:
cells = [(n, d, r) for n in N_GRID for d in DEGREES for r in range(REPS[n])]
# large-N cells first so the slowest work starts immediately
cells.sort(key=lambda c: -c[0])
rows = Parallel(n_jobs=4, prefer="processes")(
delayed(probe_cell)(n, d, r) for n, d, r in cells)
df = pd.DataFrame(rows)
out = RUNS / "topology_probe.parquet"
df.to_parquet(out)
print(f"wrote {len(df)} rows -> {out}")
if __name__ == "__main__":
main()

14
tools/simulators/tsi/tsi-sim-pernode/scripts/verify.py Normal file
View File

@ -0,0 +1,14 @@
#!/usr/bin/env python
"""Thin shim so `python scripts/verify.py` works without installing; see tsi_sim.verify."""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from tsi_sim.verify import main # noqa: E402
if __name__ == "__main__":
raise SystemExit(main())

73
tools/simulators/tsi/tsi-sim-pernode/scripts/window_scale_analysis.py Normal file
View File

@ -0,0 +1,73 @@
"""Window sufficiency at scale + the W-as-buffer question (fig25, report §3.4).
From the window-scale sweep (N = 1 000 vs 10 000, W = 50..600, delta in {8, 16, 32} s, U in
{1, 2}, k = 256): the window floor's position is N-invariant, a wider window buys back the
near-boundary (rho ~ 1) undershoot at U = 1, and no window fixes sustained overload (rho > U).
Run: python scripts/window_scale_analysis.py
"""
from __future__ import annotations
import sys
from pathlib import Path
import pandas as pd
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from tsi_sim.plotting import style # noqa: E402
HERE = Path(__file__).resolve().parent.parent
RUNS = HERE / "runs"
FIGS = HERE / "report-figures"
BAR = 0.98
def main() -> None:
src = sorted(RUNS.glob("*_window-scale/results.parquet"))[-1]
df = pd.read_parquet(src)
# Early stop terminates each config once it converges (~epoch 16), well short of the
# nominal ``epochs`` (40). A fixed ``epoch >= 20`` tail would drop almost every config;
# take the second half of each config's *actually-run* epochs instead (matches the
# burn_frac=0.5 tail used elsewhere).
keys = ["n_nodes", "blend_delay_max", "uncle_window", "max_uncles", "replicate"]
tail_from = df.groupby(keys).epoch.transform("max") * 0.5
t = df[df.epoch >= tail_from]
eq = (t.groupby(["n_nodes", "blend_delay_max", "uncle_window", "max_uncles"],
as_index=False).mean_ratio.mean())
import matplotlib.pyplot as plt
style.apply_style()
fig, axes = plt.subplots(1, 3, figsize=(13.2, 4.0), sharey=True)
for ax, delay in zip(axes, (8.0, 16.0, 32.0), strict=True):
for n, u in ((1000, 1), (10000, 1), (1000, 2), (10000, 2)):
s = eq[(eq.blend_delay_max == delay) & (eq.n_nodes == n)
& (eq.max_uncles == u)].sort_values("uncle_window")
ax.plot(s.uncle_window, s.mean_ratio,
"-o" if u == 1 else "--s", ms=4, lw=1.3,
color=style.OKABE_ITO[0 if n == 1000 else 1],
label=f"N={n:,}, U={u}" if delay == 8.0 else None)
ax.axhline(BAR, color="0.7", lw=0.8, ls="--")
ax.set_xlabel("uncle window W (slots)")
ax.set_title(f"blending budget δ = {delay:g} s")
axes[0].set_ylabel(r"$\hat D / D$")
axes[0].text(60, BAR + 0.006, "0.98 recovery bar", fontsize=7, color="0.5")
axes[0].legend(fontsize=8, loc="lower right")
fig.suptitle("Window sufficiency at scale: a wider W buys back the ρ ≈ 1 boundary (middle) "
"but cannot fix sustained overload (right)", y=1.03)
style.save(fig, FIGS / "fig25_window_scale",
provenance=f"scripts/window_scale_analysis.py ({src.parent.name})")
plt.close(fig)
print("wrote fig25_window_scale")
for delay in (8.0, 16.0, 32.0):
s = eq[(eq.blend_delay_max == delay) & (eq.max_uncles == 1)]
piv = s.pivot(index="n_nodes", columns="uncle_window", values="mean_ratio")
print(f"\nU=1 δ={delay:g}:")
print(piv.round(3).to_string())
if __name__ == "__main__":
main()

3
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/__init__.py Normal file
View File

@ -0,0 +1,3 @@
"""Cryptarchia Total Stake Inference simulator (uncle references)."""
__version__ = "0.1.0"

480
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/blocktree.py Normal file
View File

@ -0,0 +1,480 @@
"""Block tree, latency-driven forks, and honest longest-chain fork choice.
Blocks are stored in parallel arrays (id == index). A virtual genesis is block 0 at
slot -1, height 0. Every real block is produced at an active slot by one winning node
and points at the best tip *visible to that node at production time*, which is what makes
network latency (and same-slot multi-winners) produce forks.
Fork choice is honest longest-chain with a first-seen tie-break (prefer higher height,
then earlier slot, then lower id) — no adversary is modelled, so the spec's density /
deep-fork rules never engage.
"""
from __future__ import annotations
import heapq
from dataclasses import dataclass
import numpy as np
from .latency import LatencyModel
from .memguard import ArrivalMatrixTooLarge, check_alloc
__all__ = ["ArrivalMatrixTooLarge", "BlockTree", "build_tree", "build_tree_pernode",
"tips_for_all_nodes"]
GENESIS = 0
@dataclass
class BlockTree:
slot: np.ndarray # int64, slot of each block (genesis = -1)
parent: np.ndarray # int64, parent id (genesis = -1)
height: np.ndarray # int64, chain height (genesis = 0)
leader: np.ndarray # int64, producing node id (genesis = -1)
uncles: list[tuple[int, ...]] # referenced uncle ids per block (filled later)
@property
def n_blocks(self) -> int:
return self.slot.shape[0]
def ancestors(self, block_id: int) -> list[int]:
"""Ancestor chain of ``block_id`` from itself down to (excluding) genesis."""
out: list[int] = []
b = block_id
while b > GENESIS:
out.append(b)
b = int(self.parent[b])
return out
def canonical_chain(self) -> list[int]:
"""Honest longest-chain: ancestors of the best tip over the whole tree.
Returns real block ids (genesis excluded), tip-first.
"""
tip = self._best_over_all()
return self.ancestors(tip)
def _rank(self, bid: int) -> tuple[int, int, int]:
# Preference order for "better tip": higher height, earlier slot, lower id.
return (int(self.height[bid]), -int(self.slot[bid]), -bid)
def _best_over_all(self) -> int:
best = GENESIS
best_rank = self._rank(GENESIS)
for bid in range(1, self.n_blocks):
r = self._rank(bid)
if r > best_rank:
best_rank, best = r, bid
return best
def build_tree(
active_slots: np.ndarray,
winners_per_slot: list[np.ndarray],
latency: LatencyModel,
rng: np.random.Generator,
) -> BlockTree:
"""Construct the block tree from grouped lottery winners under a latency model."""
# Preallocate with genesis in slot 0.
slot = [-1]
parent = [-1]
height = [0]
leader = [-1]
# global_best = best publicly-visible tip so far, as (height, slot, id).
def better(a: tuple[int, int, int], b: tuple[int, int, int]) -> tuple[int, int, int]:
# higher height, then earlier slot, then lower id
ah, as_, ai = a
bh, bs, bi = b
if ah != bh:
return a if ah > bh else b
if as_ != bs:
return a if as_ < bs else b
return a if ai < bi else b
global_best = (0, -1, GENESIS)
own_best: dict[int, tuple[int, int, int]] = {}
# min-heap of (visible_at, block_id) awaiting public visibility
pending: list[tuple[int, int]] = []
next_id = 1
for si in range(active_slots.shape[0]):
t = int(active_slots[si])
# advance visibility frontier to slot t
while pending and pending[0][0] <= t:
_, bid = heapq.heappop(pending)
cand = (height[bid], slot[bid], bid)
global_best = better(global_best, cand)
for v in winners_per_slot[si].tolist():
gb = global_best
ob = own_best.get(v, (0, -1, GENESIS))
chosen = better(gb, ob)
p_id = chosen[2]
h = chosen[0] + 1
bid = next_id
next_id += 1
slot.append(t)
parent.append(p_id)
height.append(h)
leader.append(v)
own_best[v] = (h, t, bid)
va = latency.visible_at(t, rng)
heapq.heappush(pending, (va, bid))
return BlockTree(
slot=np.asarray(slot, np.int64),
parent=np.asarray(parent, np.int64),
height=np.asarray(height, np.int64),
leader=np.asarray(leader, np.int64),
uncles=[() for _ in range(next_id)],
)
# --- Per-node engine -------------------------------------------------------
def _rank_keys(height: np.ndarray, slot: np.ndarray, ids: np.ndarray,
epoch_len: int) -> np.ndarray:
"""Composite int64 sort key so argmax reproduces the (height, −slot, −id) tie-break."""
n = ids.shape[0]
c2 = np.int64(n + 1)
c1 = np.int64(epoch_len + 2) * c2
return height.astype(np.int64) * c1 - slot.astype(np.int64) * c2 - ids.astype(np.int64)
@dataclass
class SlidingArrival:
"""Pruned arrival store: per-node columns only for blocks still inside the keep-span.
Blocks with ``slot <= t - horizon`` are finalized — under deterministic latency every node
has received them — so their per-node columns are dropped. ``buf[:, b - base]`` holds the
arrival column for any live block ``b`` (``b >= base``); a block id ``< base`` is finalized and
treated as "arrived at every node". This is what turns the ``O(N * n_blocks)`` arrival matrix
into ``O(N * keep-span-blocks)``; ``tips_for_all_nodes`` reconstructs exact per-node tips from
it. Equivalent to the full matrix when ``jitter_mean == 0``.
"""
buf: np.ndarray # (N, buf_width) base-offset column buffer of recent arrivals
base: int # absolute block id stored at buf[:, 0]
horizon: float # slot <= t - horizon => arrived at every node
n: int # N (node count)
nb: int # number of blocks
def _max_span_blocks(active_slots: np.ndarray, counts: np.ndarray, span: float) -> int:
"""Max number of blocks whose slot lies in any ``span``-wide slot window (for buffer sizing)."""
if active_slots.size == 0:
return 1
# inclusive window slot >= t - span (matches the sliding buffer's kept set / uncle window)
cum = np.concatenate([[0], np.cumsum(counts)])
best, left = 0, 0
for r in range(active_slots.shape[0]):
while active_slots[left] < active_slots[r] - span:
left += 1
best = max(best, int(cum[r + 1] - cum[left]))
return best
def build_tree_pernode(
active_slots: np.ndarray,
winners_per_slot: list[np.ndarray],
path_latency: np.ndarray,
config,
rng: np.random.Generator,
adversary_mask: np.ndarray | None = None,
):
"""Build the global block tree AND the per-node arrival matrix.
``adversary_mask[v] == True`` marks a node that suppresses uncle references in its own blocks
(references none), to deflate the TSI density count (grinding). ``None`` = fully honest.
Each winner builds on the best tip *in its own arrival-filtered view*; uncle refs are
baked at production from the producer's view. Returns ``(BlockTree, A)`` where
``A[i, b]`` is the slot block ``b`` becomes usable at node ``i``.
Fork choice — full scan vs windowed horizon
-------------------------------------------
A winner ``v`` at slot ``t`` builds on the highest-key block it has received
(``A[v, b] <= t``). Naively this scans all ``nb`` blocks so far → ``O(n_blocks^2)`` per
epoch. With ``config.windowed_fork_choice`` (default) we scan only a horizon and add one
representative of everything older:
* ``H = max path latency`` over the graph (for ``blend``, ``H`` also adds the whole mix
cascade: ``(blend_hops+1)*max_path_latency + blend_hops*blend_delay_max``, a hard bound
since the per-relay mixing delays are ``Uniform``-bounded). Any block with
``slot <= t - H`` has, under *deterministic* latency, reached **every** node
(``slot + propagation <= t``), so the best of them — the "fully-propagated tip" ``gb`` —
is a valid candidate for *all* nodes and is tracked incrementally. Only blocks with
``slot > t - H`` need a per-node arrival check. Result: ``O(n_blocks * H * f)``, and
**exact** when latency is deterministic (including blend's bounded mixing delays).
CAVEAT: exactness assumes actual arrival never exceeds ``slot + H``. That holds only when
``jitter_mean == 0``. With ``jitter_mean > 0`` the stochastic jitter can delay a block past
the horizon, so ``gb`` may be offered to a node that has not actually received it, or a
node's true best old tip may sit just outside the window — a (usually tiny) approximation.
We warn in that case; a guaranteed-exact result is available via
``windowed_fork_choice=False`` (full scan). A safety clamp below still guarantees no node
ever builds on a block it has not received, so the tree stays valid regardless.
"""
import warnings
from .topology import arrival_column
from .uncles import select_uncles_at_production
n = config.n_nodes
n_blocks = 1 + sum(int(g.shape[0]) for g in winners_per_slot)
E = config.epoch_len
slot = np.empty(n_blocks, np.int64)
parent = np.empty(n_blocks, np.int64)
height = np.empty(n_blocks, np.int64)
leader = np.empty(n_blocks, np.int64)
uncles: list[tuple[int, ...]] = [() for _ in range(n_blocks)]
slot[0], parent[0], height[0], leader[0] = -1, -1, 0, -1
c2 = np.int64(n_blocks + 1)
c1 = np.int64(E + 2) * c2
key = np.empty(n_blocks, np.int64)
key[0] = np.int64(0) * c1 - np.int64(-1) * c2 - np.int64(0)
NEG = np.iinfo(np.int64).min
windowed = bool(config.windowed_fork_choice)
if not windowed:
horizon = float(E) # full scan (gb unused)
elif config.topology == "blend":
# blend arrival = cascade of (hops+1) transport legs + hops Uniform(0, delay_max) mix
# delays; all bounded, so this is a HARD upper bound on (arrival - slot) -> still exact.
max_pl = float(path_latency.max())
dmax = float(config.blend_delay_max)
horizon = (config.blend_hops + 1) * max_pl + config.blend_hops * dmax
else:
horizon = float(path_latency.max()) # H; disconnected -> full scan
if windowed and config.jitter_mean > 0.0:
warnings.warn(
"windowed_fork_choice / prune_arrival are only approximate when jitter_mean > 0: "
"stochastic arrival jitter can push a block past the deterministic horizon, so a "
"node's true best older tip may be missed. Set windowed_fork_choice=False for a "
"guaranteed-exact full scan.",
RuntimeWarning, stacklevel=2,
)
# Sliding-window prune needs the deterministic horizon, so it only applies with windowed fork
# choice AND jitter_mean == 0. With jitter the full matrix's safety clamp is required. A
# withholding adversary produces blocks that NEVER arrive (arrival > E), violating the prune's
# "finalized => arrived-everywhere" assumption, so it too forces the full matrix.
withholding = (adversary_mask is not None and config.adversary_frac > 0.0
and config.adversary_strategy == "withhold")
if config.prune_arrival and windowed and config.jitter_mean == 0.0 and not withholding:
return _build_pruned(active_slots, winners_per_slot, path_latency, config, rng,
slot, parent, height, leader, uncles, key, c1, c2,
float(horizon), n_blocks, E, n, adversary_mask)
# --- full (N x n_blocks) matrix path: the exact parity oracle -----------------
# Guard BEFORE the big allocation: A is (N x n_blocks) float64. A collapsed D_est (small
# genesis_d_factor) inflates lottery wins, so n_blocks can explode far past the ~10*k
# equilibrium and make A tens of GB. Fail loud rather than freeze the machine.
check_alloc(
n * n_blocks * 8, f"arrival matrix A (N={n} x n_blocks={n_blocks} x 8B)",
f"n_blocks={n_blocks} is ~{n_blocks / max(10 * config.k, 1):.0f}x the ~{10 * config.k} "
f"equilibrium, driven by genesis_d_factor={config.genesis_d_factor} "
f"(sum(stake)/D_est_genesis={1.0 / config.genesis_d_factor:.0f}). Raise "
f"genesis_d_factor, lower n_nodes/k, prune_arrival, or raise --mem-frac.")
# arrival times are sub-slot (float): latency is in slots and a slot is 1 s, so realistic
# inter-node latencies are fractions of a slot (see topology.build_path_latency).
A = np.full((n, n_blocks), float(E), np.float64) # sentinel = epoch_len ("never" arrives)
A[:, 0] = 0.0 # genesis known to all from slot 0
withheld = np.zeros(n_blocks, dtype=bool) # adversary "withhold": block never arrives anywhere
gb_key = key[0] # running best fully-propagated tip (slot <= t - H)
gb_id = 0
fp_idx = 1 # frontier pointer over fully-propagated blocks
nb = 1
for si in range(active_slots.shape[0]):
t = int(active_slots[si])
winners = winners_per_slot[si]
# --- fork choice: window [lo, nb) + fully-propagated best gb ---
if windowed:
thr = t - horizon
while fp_idx < nb and int(slot[fp_idx]) <= thr: # advance propagated frontier
if not withheld[fp_idx] and key[fp_idx] > gb_key: # withheld blocks reach no node
gb_key, gb_id = int(key[fp_idx]), fp_idx
fp_idx += 1
lo = int(np.searchsorted(slot[:nb], thr, side="right")) # first slot > t - H
else:
lo = 0 # full scan (gb unused)
if lo < nb:
sub = A[winners, lo:nb] <= t # (w, nb-lo)
masked = np.where(sub, key[lo:nb], NEG)
win_key = masked.max(axis=1)
parents = masked.argmax(axis=1) + lo
else:
win_key = np.full(winners.shape[0], NEG, np.int64)
parents = np.zeros(winners.shape[0], np.int64)
if windowed:
gb_ok = A[winners, gb_id] <= t # gb actually received? (jitter)
use_gb = gb_ok & (gb_key > win_key)
parents = np.where(use_gb, gb_id, parents)
# safety: never build on a block a node has not received (jitter edge) -> genesis
bad = A[winners, parents] > t
if bad.any():
parents = np.where(bad, 0, parents)
for wi in range(winners.shape[0]):
v = int(winners[wi])
p_id = int(parents[wi])
h = int(height[p_id]) + 1
b = nb
slot[b], parent[b], height[b], leader[b] = t, p_id, h, v
key[b] = np.int64(h) * c1 - np.int64(t) * c2 - np.int64(b)
adv = adversary_mask is not None and adversary_mask[v]
hide = adv and config.adversary_strategy == "withhold"
if adv: # suppress refs (both adversary modes)
uncles[b] = ()
else:
uncles[b] = select_uncles_at_production(
slot, parent, uncles, A[v], b, p_id, t, config, rng
)
col = arrival_column(path_latency, v, t, config, rng) # (rng drawn either way)
if hide:
A[:, b] = float(E) + 1.0 # withheld: never arrives -> orphan
withheld[b] = True
else:
np.maximum(col, A[:, p_id], out=col)
A[:, b] = col
A[v, b] = max(float(t), float(A[v, p_id])) # producer sees own block at its slot
nb += 1
tree = BlockTree(slot=slot, parent=parent, height=height, leader=leader, uncles=uncles)
return tree, A
def _build_pruned(active_slots, winners_per_slot, path_latency, config, rng,
slot, parent, height, leader, uncles, key, c1, c2, horizon, n_blocks, E, n,
adversary_mask=None):
"""Windowed build with a sliding-window arrival buffer (see ``SlidingArrival``).
Identical tree/uncles to the full-matrix path when ``jitter_mean == 0`` (the guaranteed regime
for ``windowed_fork_choice``): a block ``slot <= t - horizon`` is received by everyone, so its
per-node column is never needed again — fork choice only scans the horizon window, the parent
clamp on a finalized parent is a no-op (its arrival ``<= t <= col``), and uncle candidates
older than the horizon are trivially received. We therefore keep columns only for blocks inside
``max(horizon, uncle_window)`` slots, in a base-offset buffer, and finalize (drop) the rest.
"""
from .topology import arrival_column
from .uncles import select_uncles_at_production
NEG = np.iinfo(np.int64).min
keepspan = max(float(horizon), float(config.uncle_window)) # columns kept within this span
counts = np.array([int(g.shape[0]) for g in winners_per_slot], dtype=np.int64)
cap = _max_span_blocks(active_slots, counts, keepspan) # max live blocks at once
max_slot = int(counts.max()) if counts.size else 0
buf_width = 2 * (cap + max_slot) + 8 # headroom => rare compaction
check_alloc(
n * buf_width * 8, f"pruned arrival buffer (N={n} x {buf_width} cols x 8B)",
f"sliding-window prune keeps ~{cap} of {n_blocks} block-columns "
f"(keepspan={keepspan:g} slots); raise --mem-frac if genuinely too large.")
buf = np.full((n, buf_width), float(E), np.float64) # sentinel = E ("never arrives")
buf[:, 0] = 0.0 # genesis (id 0) known to all
base = 0 # absolute id at buf[:, 0]
gb_key, gb_id, fp_idx = int(key[0]), 0, 1
nb = 1
for si in range(active_slots.shape[0]):
t = int(active_slots[si])
winners = winners_per_slot[si]
thr = t - horizon
while fp_idx < nb and int(slot[fp_idx]) <= thr: # advance fully-propagated frontier
if int(key[fp_idx]) > gb_key:
gb_key, gb_id = int(key[fp_idx]), fp_idx
fp_idx += 1
lo = int(np.searchsorted(slot[:nb], thr, side="right")) # first block with slot > t - H
if lo < nb:
sub = buf[winners, lo - base:nb - base] <= t # window blocks are all live
masked = np.where(sub, key[lo:nb], NEG)
win_key = masked.max(axis=1)
parents = masked.argmax(axis=1) + lo
else:
win_key = np.full(winners.shape[0], NEG, np.int64)
parents = np.zeros(winners.shape[0], np.int64)
# gb is fully-propagated (slot <= t - H) => received by all under jitter=0 (gb_ok=True), and
# the finally-chosen parent is always received, so no bad-clamp is needed (parity: the full
# path's gb_ok/bad are likewise no-ops at jitter=0).
parents = np.where(gb_key > win_key, gb_id, parents)
for wi in range(winners.shape[0]):
v = int(winners[wi])
p_id = int(parents[wi])
h = int(height[p_id]) + 1
b = nb
slot[b], parent[b], height[b], leader[b] = t, p_id, h, v
key[b] = np.int64(h) * c1 - np.int64(t) * c2 - np.int64(b)
if b - base >= buf_width: # compact: drop finalized columns
# keep slot >= t - keepspan (side="left"): the uncle window's lower bound is also
# inclusive (slot >= t-W), so base must not advance past a block it may still read.
live_lo = int(np.searchsorted(slot[:nb], t - keepspan, side="left"))
if live_lo > base:
keep = nb - live_lo
if keep > 0:
buf[:, :keep] = buf[:, live_lo - base:nb - base].copy()
base = live_lo
if adversary_mask is not None and adversary_mask[v]:
uncles[b] = () # adversary suppresses uncle refs
else:
uncles[b] = select_uncles_at_production(
slot, parent, uncles, buf[v], b, p_id, t, config, rng, arr_base=base)
col = arrival_column(path_latency, v, t, config, rng)
if p_id >= base: # live parent -> clamp; else no-op
np.maximum(col, buf[:, p_id - base], out=col)
buf[:, b - base] = col
pv = float(buf[v, p_id - base]) if p_id >= base else float(t) # finalized parent <= t
buf[v, b - base] = max(float(t), pv)
nb += 1
tree = BlockTree(slot=slot, parent=parent, height=height, leader=leader, uncles=uncles)
return tree, SlidingArrival(buf=buf, base=base, horizon=float(horizon), n=n, nb=nb)
def _tips_pruned(tree: BlockTree, arr: SlidingArrival, cutoff: int) -> np.ndarray:
"""Per-node tips from the sliding buffer: best fully-propagated block (global) vs each node's
best recent (still-in-window) arrival. Exact equivalent of the full-matrix argmax at jitter=0.
"""
nb = tree.n_blocks
key = _rank_keys(tree.height, tree.slot, np.arange(nb), cutoff + 2)
NEG = np.iinfo(np.int64).min
recent = tree.slot > (cutoff - arr.horizon) # slot > E - H: per-node arrival varies
recent[0] = False # genesis is finalized (arrived at all)
# best over finalized/"arrived-everywhere" blocks (slot <= E - H): a candidate for every node
fin_ids = np.nonzero(~recent)[0]
gb_final = int(fin_ids[np.argmax(key[fin_ids])])
recent_ids = np.nonzero(recent)[0]
if recent_ids.size == 0:
return np.full(arr.n, gb_final, np.int64)
arrived = arr.buf[:, recent_ids - arr.base] <= cutoff # (N, R) recent blocks in buffer
masked = np.where(arrived, key[recent_ids][None, :], NEG)
best_recent = recent_ids[masked.argmax(axis=1)]
use_recent = masked.max(axis=1) > int(key[gb_final])
return np.where(use_recent, best_recent, gb_final)
def tips_for_all_nodes(tree: BlockTree, arrival, cutoff: int,
row_chunk: int = 64) -> np.ndarray:
"""Per-node best tip = argmax (height, −slot, −id) over blocks arrived by ``cutoff``.
``arrival`` is either the full ``(N, n_blocks)`` matrix or a pruned ``SlidingArrival``; both
yield the same tips at ``jitter_mean == 0``. For the full matrix, each node's argmax is
independent, so we process it in ``row_chunk`` node-row bands — capping the transient
``np.where`` mask at ``(row_chunk, nb)`` instead of a second full ``(N, nb)`` int64 array
(bitwise-identical to the unchunked argmax).
"""
if isinstance(arrival, SlidingArrival):
return _tips_pruned(tree, arrival, cutoff)
nb = tree.n_blocks
n = arrival.shape[0]
ids = np.arange(nb)
key = _rank_keys(tree.height, tree.slot, ids, cutoff + 2)
NEG = np.iinfo(np.int64).min
tips = np.empty(n, np.int64)
for lo in range(0, n, row_chunk):
hi = min(lo + row_chunk, n)
masked = np.where(arrival[lo:hi] <= cutoff, key[None, :], NEG) # (row_chunk, nb)
tips[lo:hi] = masked.argmax(axis=1)
return tips # (N,)

62
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/concurrency.py Normal file
View File

@ -0,0 +1,62 @@
"""Concurrent block-proposal analysis.
Every lottery win is a block *proposal*. Two proposals produced within ``L`` slots of each
other cannot see one another (a block becomes visible only after the network latency ``L``),
so they are mutually concurrent — competing forks. Bucketing the timeline into
non-overlapping windows of ``L`` slots and counting proposals per bucket gives a direct
view of how many proposals are concurrent, and the busiest bucket is the peak number of
concurrent proposals. This is the quantity that bounds how many uncles can appear, so it
informs the ``MAX_UNCLES`` choice.
"""
from __future__ import annotations
from dataclasses import replace
import numpy as np
from .config import SimConfig
from .lottery import sample_wins, win_probs
from .rng import seedseq_for
from .stake import make_stake
def window_counts(winner_slots: np.ndarray, epoch_len: int, bucket: int) -> np.ndarray:
"""Proposals per non-overlapping ``bucket``-slot window over ``[0, epoch_len)``."""
if epoch_len <= 0:
return np.empty(0, np.int64)
bucket = max(int(bucket), 1)
n_windows = (epoch_len + bucket - 1) // bucket
if winner_slots.size == 0:
return np.zeros(n_windows, np.int64)
return np.bincount(winner_slots // bucket, minlength=n_windows)
def proposal_slots(config: SimConfig, replicate: int = 0) -> np.ndarray:
"""Simulate one epoch of block proposals at the *true* lottery difficulty (D=D_true).
Returns the sorted slots at which proposals (all lottery winners, including forks) occur.
Independent of the TSI trajectory — the proposal process depends only on stake, ``f``,
and ``epoch_len`` — so this is a cheap, self-contained re-simulation for the plots.
"""
cfg = replace(config, replicate=replicate)
root = seedseq_for(cfg)
children = root.spawn(2)
stake = make_stake(cfg, np.random.default_rng(children[0]))
p_win = win_probs(stake, float(stake.sum()), cfg.f)
winner_slots, _ = sample_wins(p_win, cfg.epoch_len, np.random.default_rng(children[1]))
return winner_slots
def concurrency_stats(config: SimConfig, replicate: int = 0) -> dict:
"""Per-bucket proposal-count stats for one simulated epoch (bucket = ``max(L, 1)``)."""
ws = proposal_slots(config, replicate)
bucket = max(config.latency, 1)
counts = window_counts(ws, config.epoch_len, bucket)
return {
"bucket": bucket,
"counts": counts,
"max": int(counts.max()) if counts.size else 0,
"mean": float(counts.mean()) if counts.size else 0.0,
"p99": float(np.percentile(counts, 99)) if counts.size else 0.0,
}

371
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/config.py Normal file
View File

@ -0,0 +1,371 @@
"""Configuration dataclasses for single runs and parameter sweeps."""
from __future__ import annotations
import itertools
from dataclasses import dataclass, field, replace
from typing import Any, Literal
from . import constants
StakeDist = Literal["uniform", "pareto"]
UncleStrategy = Literal["oldest", "random"]
Topology = Literal["full_mesh", "regular", "blend"]
LinkLatencyDist = Literal["fixed", "uniform", "exp", "geo"]
JitterDist = Literal["exp", "poisson"]
ChurnMode = Literal["sine", "ramp", "step"]
InitDest = Literal["common", "heterogeneous"]
# How the adversary_frac coalition attacks the TSI density count:
# "suppress" — produces normally but references NO uncles (starves the recovered density; weak);
# "withhold" — never gossips its blocks (they are orphaned, its won slots become gaps in the
# canonical chain), so the counted density drops ~adversary_frac and TSI deflates D_est toward
# the reduced ACTIVE stake. Stronger, but the withheld blocks earn nothing (griefing/grinding).
AdversaryStrategy = Literal["suppress", "withhold"]
@dataclass(frozen=True)
class SimConfig:
"""A single fully-specified simulation run (one grid cell, one replicate)."""
# --- network / stake ---
n_nodes: int = 1000
stake_dist: StakeDist = "uniform"
pareto_shape: float = 1.16 # Pareto (Lomax) tail index; ~80/20 by default
uniform_random: bool = False # if True, draw i.i.d. uniform stakes; else equal
total_stake: float = 1.0e9 # FIXED across distributions for comparability
# --- network latency (slots) ---
latency: int = 0 # L: full-mesh uniform link latency (block seen at t+L)
latency_stochastic: bool = False # if True, L is the mean of a stochastic model
# --- network topology (per-node model) ---
# "full_mesh": every node one hop away, uniform latency = `latency` (reproduces the
# reduced model). "regular": random d-regular peering graph with per-link latency;
# a block reaches a node after the shortest WEIGHTED path from its producer.
# "blend": same d-regular graph, but a block is first relayed through `blend_hops` random
# nodes (a mix cascade, each adding a Uniform(0, blend_delay_max) mixing delay) before a
# final network-wide gossip makes it visible — models routing over the Blend mixnet.
topology: Topology = "full_mesh"
degree: int = 8 # peering degree (regular / blend graph)
# One entropy contributor to the per-trajectory RNG (via key()), NOT an independent topology
# knob: the graph is seeded from the config's full-key spawn hierarchy (engine.run_trajectory),
# so it is fixed per trajectory but is re-rolled by ANY key() field (stake_dist, f,
# adversary_frac, replicate, ...). Consequently two configs that differ only in a non-topology
# field draw different graphs; adversary-vs-honest comparisons are therefore unpaired in the
# graph sample (a variance source averaged out over replicates, not a bias — the main deflation
# effects are topology-independent, §6.4). Making it a paired/independent knob would require
# seeding the graph from topology-only entropy and re-running every sweep.
graph_seed: int = 0
# Blend mixnet cascade (topology == "blend"): the producer picks `blend_hops` distinct
# relay nodes uniformly at random; the block hops producer -> r1 -> ... -> r_hops over the
# graph, each relay waiting Uniform(0, blend_delay_max) slots before forwarding; the last
# relay's forward is the final network-wide gossip. Ignored by full_mesh / regular.
blend_hops: int = 3 # number of random relay hops in the mix cascade
adversary_strategy: AdversaryStrategy = "suppress" # how adversary_frac attacks (see above)
blend_delay_max: float = 3.0 # max per-relay mixing delay (slots); delay ~ U(0, this)
# Mean one-way per-link latency in SLOTS (1 slot = 1 s). Realistic direct-gossip links are
# sub-slot (~0.04-0.15 slot = 40-150 ms); whole-slot values (1, 2, ...) model routing over
# the Blend mixnet, where each hop costs seconds. Arrivals are kept sub-slot (float).
link_latency_mean: float = 1.0
# Per-link latency distribution (all have mean = link_latency_mean): "fixed" (all equal),
# "uniform" (0..2*mean), "exp" (long tail), "geo" (real-world geographic band mixture:
# short intra-region links, long inter-continental ones — see constants.GEO_LATENCY_*).
link_latency_dist: LinkLatencyDist = "fixed"
jitter_mean: float = 0.0 # extra per-(block,node) jitter (slots); 0 = none.
# Jitter model (active when jitter_mean > 0):
# "exp" — EVERY delivery gets +Exp(jitter_mean); the §6.1 robustness model.
# "poisson" — a random fraction `jitter_frac` of deliveries gets +Poisson(jitter_mean)
# whole slots; the rest arrive on time. A LONG-TAIL model: most deliveries are
# unaffected, a few straggle by multiple slots (case (b) of the N-scaling study).
jitter_dist: JitterDist = "exp"
jitter_frac: float = 1.0 # fraction of deliveries hit (poisson model; exp uses all)
# --- uncle references ---
uncle_window: int = constants.W_DEFAULT # W
max_uncles: int = 0 # U (0 = baseline, no uncles)
uncle_strategy: UncleStrategy = "oldest"
# Coin-flip inclusion prob for the "random" strategy. Only 0.5 reproduces the spec's
# unbiased coin (cryptarchia-v1-protocol.md); other values are a deliberate, non-spec
# sensitivity knob, not protocol behaviour.
uncle_random_p: float = 0.5
# --- adversary (grinding via D_est deflation) ---
# Fraction of TOTAL STAKE controlled by an adversary that suppresses uncle references in its
# own blocks (references no uncles), starving the TSI density count so honest nodes under-count
# blocks and infer a LOW D_est -> everyone's win probability phi(f, w/D_est) rises, which is the
# grinding payoff. 0.0 = fully honest (the studied baseline). The coalition is a RANDOM node set
# whose stake sums to adversary_frac (see engine._adversary_mask); block production is
# stake-proportional, so the deflation depends only on that summed share, not on whether the
# coalition is one whale or many small nodes. Withholding is a separate, stronger lever.
adversary_frac: float = 0.0
# Dynamic (withhold-then-rejoin) schedule for the withholding lever (§6.5). The coalition is
# FIXED (identity from adversary_frac); this only gates whether it withholds in a given epoch.
# adversary_period == 0 -> STATIC: the coalition attacks (withholds) every epoch (the §6.4
# model; backward-compatible default).
# adversary_period > 0 -> PERIODIC: withhold for the first `adversary_withhold_epochs` of
# every `adversary_period`-epoch cycle, then behave honestly (produce + gossip) for the
# rest — an abstain-then-rejoin grinder. A single downward pulse (does D_est recover, or
# tip into the §6.2 collapsed branch?) is period == epochs, withhold_epochs == pulse length.
# Only affects adversary_strategy == "withhold"; suppression stays static.
adversary_period: int = 0
adversary_withhold_epochs: int = 0
# --- consensus / TSI ---
f: float = constants.F # slot activation coefficient (configurable; sweepable)
beta: float = constants.BETA_DEFAULT
k: int = 64 # scaled by default; full scale = 2160
genesis_d_factor: float = 0.5 # genesis D = factor * true total stake
epochs: int = 40
# If True, mirror the spec's integer fixed-point f-truncation (f_p = int(f*1000)/1000),
# which the on-chain estimator uses; this reproduces its ~1% systematic overestimate.
# Default False keeps the analysis-faithful exact-f behaviour.
fixed_point: bool = False
# If True, count uncle references per BLOCK ID (the pre-fix behaviour, which double-counts
# same-slot co-winners and inflates the equilibrium by c(f)). The correct default counts
# per SLOT (one count per slot, matching the pre-uncle design invariant). Kept as a flag
# for reproducing historical runs only; no study uses it.
legacy_block_count: bool = False
# Early stop: when the per-epoch estimate has converged (trailing epochs statistically
# flat), run ES_MEASURE more epochs as the equilibrium sample and stop. Truncation-only:
# per-epoch RNG streams are pre-spawned, so the epochs that DO run are bit-identical to a
# full run's prefix (hence excluded from key()). Auto-disabled for periodic-adversary
# schedules (sawtooths must run their full budget).
early_stop: bool = False
# Organic (non-adversarial) participation churn: each epoch a `churn_amp` fraction of honest
# stake goes inactive following a schedule, so the ACTIVE stake oscillates/ramps and TSI must
# track it. churn_amp = peak inactive fraction; churn_period = epochs per cycle; churn_mode:
# "sine" — active fraction = 1 - churn_amp*(1-cos(2π·epoch/period))/2 (smooth weekly cycle)
# "ramp" — active fraction declines linearly to 1-churn_amp over churn_period, then holds
# "step" — one-time drop to (1-churn_amp) at churn_period (mass leave)
churn_amp: float = 0.0
churn_period: int = 4
churn_mode: ChurnMode = "sine"
# Per-node constant slot-clock offset (~Uniform(-clock_skew_max, +clock_skew_max) slots),
# applied to each node's measurement-window bounds — tests whether a whole-timeline clock shift
# (unlike per-arrival jitter) can split slot-occupancy at the window edges and break consensus.
clock_skew_max: int = 0
# Each node updates its OWN D_est from its OWN view — the point of this simulator, and the ONLY
# mode implemented here (always True). The global-consensus-D_est baseline (per_node_dest=False)
# is not built in this package; it lives in the sibling reduced model (tsi-sim). Retained as a
# key() seed contributor for compatibility; do not set False (no code path reads it).
per_node_dest: bool = True
# "common": all nodes start at genesis_d_factor*D_true (studies convergence FROM
# agreement). "heterogeneous": per-node initial D_est drawn with relative spread
# `init_spread` around genesis (studies transient re-convergence from disagreement).
init_dest: InitDest = "common"
init_spread: float = 0.0 # relative spread of heterogeneous initial D_est
# --- performance ---
# >1 parallelises the per-slot lottery across slot-chunks (opt-in; must be pinned and
# recorded because it changes the RNG stream — see lottery.sample_wins_chunked).
lottery_chunks: int = 1
# Windowed fork choice bounds the per-slot candidate scan to a horizon of the max path
# latency (plus the fully-propagated best tip), turning O(n_blocks^2) into O(n_blocks*H).
# EXACT when link latency is deterministic (jitter_mean == 0). With jitter_mean > 0 it is
# a (usually tiny) approximation and emits a warning — see blocktree.build_tree_pernode.
# Set False for a guaranteed-exact full scan.
windowed_fork_choice: bool = True
# Sliding-window pruning of the (N x n_blocks) arrival matrix: keep per-node arrival columns
# only for blocks still inside the keep-span max(horizon, uncle_window); blocks past that are
# finalized (arrived at every node under the deterministic horizon), so their columns are
# dropped. Turns O(N * n_blocks) memory into O(N * keep-span-blocks) — the fix for the
# collapsed-D_est block explosion. EXACT vs the full matrix when jitter_mean == 0 (needs the
# horizon, so it only applies when windowed_fork_choice is on); set False to store the whole
# matrix (the parity oracle, and required for a guaranteed-exact jitter>0 run).
prune_arrival: bool = True
# --- bookkeeping ---
replicate: int = 0
root_seed: int = 12345
def __post_init__(self) -> None:
# frozen dataclass: validation only (no attribute assignment)
if self.stake_dist not in ("uniform", "pareto"):
raise ValueError(f"stake_dist must be uniform|pareto, got {self.stake_dist!r}")
if self.uncle_strategy not in ("oldest", "random"):
raise ValueError(f"uncle_strategy must be oldest|random, got {self.uncle_strategy!r}")
if self.topology not in ("full_mesh", "regular", "blend"):
raise ValueError(f"topology must be full_mesh|regular|blend, got {self.topology!r}")
if self.link_latency_dist not in ("fixed", "uniform", "exp", "geo"):
raise ValueError(f"link_latency_dist must be fixed|uniform|exp|geo, got "
f"{self.link_latency_dist!r}")
if self.jitter_dist not in ("exp", "poisson"):
raise ValueError(f"jitter_dist must be exp|poisson, got {self.jitter_dist!r}")
if not 0.0 <= self.jitter_frac <= 1.0:
raise ValueError(f"jitter_frac must be in [0, 1], got {self.jitter_frac}")
if self.init_dest not in ("common", "heterogeneous"):
raise ValueError(f"init_dest must be common|heterogeneous, got {self.init_dest!r}")
if self.churn_mode not in ("sine", "ramp", "step"):
raise ValueError(f"churn_mode must be sine|ramp|step, got {self.churn_mode!r}")
if not 0.0 <= self.churn_amp < 1.0:
raise ValueError(f"churn_amp must be in [0, 1), got {self.churn_amp}")
if self.churn_period < 1:
raise ValueError(f"churn_period must be >= 1, got {self.churn_period}")
if self.clock_skew_max < 0:
raise ValueError(f"clock_skew_max must be >= 0, got {self.clock_skew_max}")
if self.adversary_strategy not in ("suppress", "withhold"):
raise ValueError(f"adversary_strategy must be suppress|withhold, got "
f"{self.adversary_strategy!r}")
checks = {
"n_nodes": self.n_nodes >= 1,
"k": self.k >= 1,
"epochs": self.epochs >= 1,
"latency": self.latency >= 0,
"max_uncles": self.max_uncles >= 0,
"uncle_window": self.uncle_window >= 1,
"lottery_chunks": self.lottery_chunks >= 1,
"uncle_random_p": 0.0 <= self.uncle_random_p <= 1.0,
"f": 0.0 < self.f < 1.0,
"beta": self.beta > 0.0,
"genesis_d_factor": self.genesis_d_factor > 0.0,
"pareto_shape": self.pareto_shape > 0.0,
"total_stake": self.total_stake > 0.0,
"degree": self.degree >= 1,
"link_latency_mean": self.link_latency_mean >= 0.0,
"jitter_mean": self.jitter_mean >= 0.0,
"init_spread": self.init_spread >= 0.0,
"blend_hops": self.blend_hops >= 1,
"blend_delay_max": self.blend_delay_max >= 0.0,
"adversary_frac": 0.0 <= self.adversary_frac < 1.0,
"adversary_period": self.adversary_period >= 0,
"adversary_withhold_epochs": self.adversary_withhold_epochs >= 0,
}
bad = [name for name, ok in checks.items() if not ok]
if bad:
raise ValueError(f"invalid SimConfig field(s): {bad}")
if self.adversary_period > 0 and self.adversary_withhold_epochs > self.adversary_period:
raise ValueError(
f"adversary_withhold_epochs ({self.adversary_withhold_epochs}) must be "
f"<= adversary_period ({self.adversary_period})")
if self.topology in ("regular", "blend"):
# a d-regular graph on n nodes needs degree < n and n*degree even
if self.degree >= self.n_nodes:
raise ValueError(f"degree ({self.degree}) must be < n_nodes ({self.n_nodes})")
if (self.n_nodes * self.degree) % 2 != 0:
raise ValueError("regular graph requires n_nodes*degree to be even")
if self.topology == "blend":
# need `blend_hops` DISTINCT relay nodes drawn from the non-producer pool
if self.blend_hops > self.n_nodes - 1:
raise ValueError(
f"blend_hops ({self.blend_hops}) must be <= n_nodes-1 ({self.n_nodes - 1})")
def adversary_withholds(self, epoch: int) -> bool:
"""Whether the (fixed) coalition withholds this epoch under its schedule.
Static (``adversary_period == 0``) attacks every epoch; periodic attacks the first
``adversary_withhold_epochs`` epochs of each ``adversary_period``-epoch cycle. Meaningful
only for ``adversary_strategy == "withhold"`` with ``adversary_frac > 0``.
"""
if self.adversary_period <= 0:
return True
return (epoch % self.adversary_period) < self.adversary_withhold_epochs
# derived geometry -------------------------------------------------------
@property
def epoch_len(self) -> int:
return constants.epoch_len(self.k, self.f)
@property
def period_T(self) -> int:
return constants.period_T(self.k, self.f)
def key(self) -> tuple:
"""Hashable identity used to seed the RNG deterministically.
Must include EVERY field that affects the run (guarded by test_rng), otherwise two
distinct configs would share an RNG stream.
"""
return (
self.n_nodes, self.stake_dist, self.pareto_shape, self.uniform_random,
self.total_stake, self.latency, self.latency_stochastic, self.uncle_window,
self.max_uncles, self.uncle_strategy, self.uncle_random_p, self.f, self.beta,
self.k, self.genesis_d_factor, self.epochs, self.fixed_point,
self.legacy_block_count, self.churn_amp, self.churn_period, self.churn_mode,
self.clock_skew_max, self.per_node_dest,
self.lottery_chunks, self.topology, self.degree, self.graph_seed,
self.link_latency_mean, self.link_latency_dist, self.jitter_mean,
self.jitter_dist, self.jitter_frac,
self.blend_hops, self.blend_delay_max, self.adversary_frac, self.adversary_strategy,
self.adversary_period, self.adversary_withhold_epochs,
self.init_dest, self.init_spread, self.replicate,
)
# NOTE: windowed_fork_choice and prune_arrival are deliberately excluded — they are pure
# compute/memory optimisations that consume no RNG and (at jitter_mean == 0) change no
# result, so pruned and full-matrix runs must share a seed (see test_pernode parity).
# Axes that can be swept; every SimConfig field is legal here.
_SWEEP_AXES = (
"n_nodes", "stake_dist", "latency", "max_uncles", "uncle_strategy", "uncle_window",
"topology", "degree", "link_latency_mean", "link_latency_dist",
"blend_hops", "blend_delay_max", "init_dest", "f",
)
@dataclass
class SweepConfig:
"""A cartesian grid of runs plus replicates, all sharing ``base`` settings."""
n_nodes: list[int] = field(default_factory=lambda: [1000])
stake_dist: list[StakeDist] = field(default_factory=lambda: ["uniform"])
latency: list[int] = field(default_factory=lambda: [0])
max_uncles: list[int] = field(default_factory=lambda: [0, 1, 2, 4])
uncle_strategy: list[UncleStrategy] = field(default_factory=lambda: ["oldest"])
uncle_window: list[int] = field(default_factory=lambda: [constants.W_DEFAULT])
topology: list[Topology] = field(default_factory=lambda: ["regular"])
degree: list[int] = field(default_factory=lambda: [8])
link_latency_mean: list[float] = field(default_factory=lambda: [1.0])
link_latency_dist: list[LinkLatencyDist] = field(default_factory=lambda: ["fixed"])
blend_hops: list[int] = field(default_factory=lambda: [3])
blend_delay_max: list[float] = field(default_factory=lambda: [3.0])
init_dest: list[InitDest] = field(default_factory=lambda: ["common"])
f: list[float] = field(default_factory=lambda: [constants.F])
replicates: int = 8
base: dict[str, Any] = field(default_factory=dict)
def expand(self) -> list[SimConfig]:
"""Materialise every ``SimConfig`` in the grid × replicates."""
base = SimConfig(**self.base)
cells: list[SimConfig] = []
axis_values = [getattr(self, ax) for ax in _SWEEP_AXES]
for combo in itertools.product(*axis_values):
overrides = dict(zip(_SWEEP_AXES, combo, strict=True))
# U=0 references no uncles, so it is independent of uncle_strategy AND uncle_window;
# keep only the first of each to avoid duplicate (identical) work.
if overrides["max_uncles"] == 0 and (
overrides["uncle_strategy"] != self.uncle_strategy[0]
or overrides["uncle_window"] != self.uncle_window[0]
):
continue
# full mesh ignores degree / link-latency model; keep only the first to avoid dupes.
if overrides["topology"] == "full_mesh" and (
overrides["degree"] != self.degree[0]
or overrides["link_latency_mean"] != self.link_latency_mean[0]
or overrides["link_latency_dist"] != self.link_latency_dist[0]
):
continue
# only blend uses the mix-cascade knobs; collapse them elsewhere to avoid dupes.
if overrides["topology"] != "blend" and (
overrides["blend_hops"] != self.blend_hops[0]
or overrides["blend_delay_max"] != self.blend_delay_max[0]
):
continue
# `latency` is the full_mesh uniform-L knob; regular/blend ignore it — collapse it
# for them so sweeping latency doesn't emit duplicate (seed-shifted) graph cells.
if overrides["topology"] != "full_mesh" and overrides["latency"] != self.latency[0]:
continue
for rep in range(self.replicates):
cells.append(replace(base, **overrides, replicate=rep))
return cells
@classmethod
def from_dict(cls, d: dict[str, Any]) -> SweepConfig:
d = dict(d)
base = d.pop("base", {})
known = {*_SWEEP_AXES, "replicates"}
unknown = set(d) - known
if unknown:
raise ValueError(
f"unknown sweep keys: {sorted(unknown)} (valid: {sorted(known)}; "
"per-run settings belong under 'base:')"
)
return cls(base=base, **d)

63
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/constants.py Normal file
View File

@ -0,0 +1,63 @@
"""Protocol constants and epoch/window geometry.
All slot geometry derives from the pair ``(k, f)`` so a scaled-down ``k`` (used for
parameter sweeps) automatically shrinks the epoch and measurement window. See
``cryptarchia-v1-protocol.md`` and ``cryptarchia-total-stake-inference.md``.
"""
from __future__ import annotations
# --- True protocol values (full scale) -------------------------------------
K_TRUE = 2160 # security parameter (blocks)
F = 1.0 / 30.0 # slot activation coefficient (default; configurable per run)
W_DEFAULT = 300 # uncle reference window w_u (slots)
BETA_DEFAULT = 1.0 # TSI learning rate
SLOT_SECONDS = 1 # slot length (seconds) — so 1 slot == 1 s
# --- Real-world inter-node network latency (per gossip link) ---------------
# A slot is SLOT_SECONDS = 1 s, so measured internet latencies (tens–hundreds of ms) are
# FRACTIONS of a slot. The values below are one-way, application-level latencies between two
# directly-peered nodes, bucketed by the geographic relationship of the peers — in a globally
# distributed node set a random peer is usually on another continent. (≈ RTT/2 from public
# latency measurements plus a little gossip processing/serialization overhead.) A block
# gossip-floods over the peering graph, so its end-to-end delay to a far node is the sum of
# a few such per-link latencies along the fastest path (Dijkstra) — see topology.py.
GEO_LATENCY_BANDS_SLOTS = (
0.015, # metro / same country (~15 ms one-way)
0.040, # same continent, e.g. EU↔EU (~40 ms)
0.090, # transatlantic, e.g. EU↔US-East (~90 ms)
0.200, # antipodal, e.g. EU↔AU / EU↔JP (~200 ms)
)
# Share of random peer links falling in each band for a globally distributed node set
# (NA/EU/Asia-weighted). Most peer pairs are cross-continent, hence the long-latency mass.
GEO_LATENCY_WEIGHTS = (0.15, 0.35, 0.35, 0.15)
# Mean one-way latency of a random global peer link under the mixture above (~0.078 slot,
# i.e. ~78 ms). Used to rescale the "geo" link-latency distribution to a requested mean.
GEO_LATENCY_MEAN_SLOTS = sum(
b * w for b, w in zip(GEO_LATENCY_BANDS_SLOTS, GEO_LATENCY_WEIGHTS, strict=True)
)
def floor_k_over_f(k: int, f: float = F) -> int:
"""``floor(k / f)`` — the base quantum of the epoch schedule."""
return int(k / f)
def epoch_len(k: int, f: float = F) -> int:
"""Epoch length in slots: ``10 * floor(k/f)``."""
return 10 * floor_k_over_f(k, f)
def period_T(k: int, f: float = F) -> int:
"""TSI measurement window length ``T`` in slots: ``6 * floor(k/f)``.
This is the first ``6*floor(k/f)`` slots of the (previous) epoch over which the
block density is measured.
"""
return 6 * floor_k_over_f(k, f)
def expected_blocks_in_window(k: int, f: float = F) -> float:
"""Expected honest-chain block count in the measurement window at equilibrium."""
return period_T(k, f) * f

167
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/engine.py Normal file
View File

@ -0,0 +1,167 @@
"""Multi-epoch per-node trajectory driver for a single config."""
from __future__ import annotations
from typing import Any
import numpy as np
from . import topology
from .config import SimConfig
from .epoch import simulate_epoch
from .metrics import divergence_row
from .rng import seedseq_for
from .stake import make_stake
# Early-stop (config.early_stop): detector + measurement budget. The detector uses a short
# 2-epoch delta window (convergence at beta=1 is abrupt, ~2-5 epochs); ES_MIN_EPOCH keeps it
# out of the genesis transient, and ES_MEASURE post-detection epochs form the equilibrium
# sample, so a slightly eager detection still averages over converging epochs. Thresholds are
# in units of the per-epoch sampling noise sigma_th = sqrt((1-f)/(f*T)); regimes noisier than
# that (e.g. Blend U=0 fork-race noise) never trigger and simply run their full budget.
ES_MIN_EPOCH = 6 # first epoch at which the detector may fire
ES_MEASURE = 10 # measurement epochs run after detection
def _adversary_mask(config: SimConfig, stake: np.ndarray) -> np.ndarray | None:
"""Nodes controlled by the uncle-suppressing adversary — a random coalition whose stake sums to
``adversary_frac`` of the total, giving smooth control of the adversary's block share. (For
uncle suppression the deflation depends only on that block share, not on whether the coalition
is one whale or many small nodes, so concentration is not modelled here.) ``None`` if honest.
Seeded from a standalone ``SeedSequence([root_seed, replicate, 0xADEADBEEF])`` (independent of
the main spawn hierarchy), and drawn only after the ``adversary_frac <= 0`` early return, so an
``adversary_frac == 0`` run is bit-identical to the honest baseline.
"""
if config.adversary_frac <= 0.0:
return None
adv_seed = np.random.SeedSequence([config.root_seed, config.replicate, 0xADEADBEEF])
order = np.random.default_rng(adv_seed).permutation(config.n_nodes)
target = config.adversary_frac * float(stake.sum())
cum = np.cumsum(stake[order])
take = int(np.searchsorted(cum, target, side="left")) + 1 # smallest coalition >= target
mask = np.zeros(config.n_nodes, dtype=bool)
mask[order[:take]] = True
return mask
def _initial_d_est(config: SimConfig, d_true: float, rng: np.random.Generator) -> np.ndarray:
"""Per-node initial estimate: common genesis, or heterogeneous around it."""
base = config.genesis_d_factor * d_true
n = config.n_nodes
if config.init_dest == "common" or config.init_spread <= 0.0:
return np.full(n, base, dtype=float)
# heterogeneous: uniform in base*(1 ± init_spread), clamped positive
lo = max(base * (1.0 - config.init_spread), 1.0)
hi = base * (1.0 + config.init_spread)
return rng.uniform(lo, hi, size=n)
def _churn_active_fraction(config: SimConfig, epoch: int) -> float:
"""Active honest-stake fraction this epoch, per the churn schedule (1.0 if no churn)."""
if config.churn_amp <= 0.0:
return 1.0
a, per = config.churn_amp, config.churn_period
if config.churn_mode == "sine":
return 1.0 - a * (1.0 - np.cos(2.0 * np.pi * epoch / per)) / 2.0
if config.churn_mode == "ramp":
return 1.0 - a * min(epoch / per, 1.0)
# step: drop to 1-a at epoch `per`, hold
return 1.0 - a if epoch >= per else 1.0
def _churn_inactive_mask(stake: np.ndarray, active_frac: float,
rng: np.random.Generator) -> np.ndarray:
"""A random node subset whose stake sums to ~(1-active_frac) of the total, marked inactive.
Greedy nearest-fill in random order: while the running inactive stake is short of the
target, a node is deactivated only if it lands the total closer to the target than stopping
short would (the gap is at least half the node's stake). A whale that would overshoot is
skipped and the fill continues with smaller nodes. Under a heavy-tailed (Pareto) stake
distribution a plain cumulative-prefix cut lets a single whale straddling the cutoff
overshoot the amplitude badly (a 30 % label realising up to ~53 %); nearest-fill keeps the
realised amplitude on-label.
"""
n = stake.shape[0]
if active_frac >= 1.0:
return np.zeros(n, dtype=bool)
target = (1.0 - active_frac) * float(stake.sum())
mask = np.zeros(n, dtype=bool)
acc = 0.0
for i in rng.permutation(n):
if acc >= target:
break
s = float(stake[i])
if (target - acc) >= 0.5 * s: # including node i lands closer than stopping short
mask[i] = True
acc += s
return mask
def run_trajectory(config: SimConfig) -> list[dict[str, Any]]:
"""Run ``config.epochs`` per-node epochs, one divergence-summary row per epoch.
Each of the ``N`` nodes carries its OWN ``d_est`` and self-updates from its own view.
The topology (``path_latency``) is built once (invariant across epochs). RNG is a spawn
hierarchy off the config's root SeedSequence: child 0 = stake, 1 = graph, 2 = init,
3+e = epoch e — so results are deterministic and order-independent.
"""
root = seedseq_for(config)
children = root.spawn(config.epochs + 3)
stake = make_stake(config, np.random.default_rng(children[0]))
d_true = float(stake.sum())
path_latency = topology.build_path_latency(config, np.random.default_rng(children[1]))
d_est = _initial_d_est(config, d_true, np.random.default_rng(children[2]))
adv_mask = _adversary_mask(config, stake)
# exact stake fraction of the (integer-rounded) coalition, for the active-stake bookkeeping
coalition_frac = float(stake[adv_mask].sum() / d_true) if adv_mask is not None else 0.0
withholding = adv_mask is not None and config.adversary_strategy == "withhold"
# churn RNG is standalone (drawn only when churn_amp>0) so churn=0 stays bit-identical
churn_rng = (np.random.default_rng(np.random.SeedSequence([config.root_seed,
config.replicate, 0xC4084])) if config.churn_amp > 0.0 else None)
def _sigma_th() -> float:
t_win = config.period_T
return float(np.sqrt((1.0 - config.f) / (config.f * t_win)))
def _converged(series: list[float]) -> bool:
"""2-epoch delta window: last step within sigma_th, 2-step drift within 1.5x."""
if len(series) < 3:
return False
sig = _sigma_th()
return (abs(series[-1] - series[-2]) <= sig
and abs(series[-1] - series[-3]) <= 1.5 * sig)
# sawtooth schedules must run their full budget; the detector would misread a rejoin ramp
allow_early = config.early_stop and config.adversary_period == 0
rows: list[dict[str, Any]] = []
stop_after: int | None = None
for epoch in range(config.epochs):
# The coalition is fixed; the schedule only gates whether it withholds THIS epoch. On a
# rejoin epoch it behaves fully honestly (behaviour mask None == honest baseline). A
# suppressing coalition (or the static default) attacks every epoch.
attacks = config.adversary_withholds(epoch) if withholding else adv_mask is not None
behaviour_mask = adv_mask if attacks else None
active_stake_frac = 1.0 - coalition_frac if (withholding and attacks) else 1.0
# organic honest churn: deactivate a scheduled stake fraction this epoch
inactive_mask = None
if churn_rng is not None:
active_frac = _churn_active_fraction(config, epoch)
inactive_mask = _churn_inactive_mask(stake, active_frac, churn_rng)
active_stake_frac = float(stake[~inactive_mask].sum() / d_true) * active_stake_frac
er = simulate_epoch(config, stake, d_est, path_latency, children[epoch + 3],
adversary_mask=behaviour_mask, coalition_mask=adv_mask,
inactive_mask=inactive_mask)
row = divergence_row(config, epoch, d_est, er, d_true)
row["adversary_withholding"] = bool(withholding and attacks)
row["active_stake_frac"] = active_stake_frac
rows.append(row)
d_est = er.d_next
if allow_early and stop_after is None and epoch >= ES_MIN_EPOCH:
if _converged([r["mean_ratio"] for r in rows]):
stop_after = epoch + ES_MEASURE
if stop_after is not None and epoch >= stop_after:
break
return rows

121
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/epoch.py Normal file
View File

@ -0,0 +1,121 @@
"""Single per-node epoch: per-node lottery -> global tree + arrival matrix -> per-node
canonical chain, density, and self-update of each node's own D_est."""
from __future__ import annotations
from dataclasses import dataclass
import numpy as np
from . import fork, lottery, tsi
from .blocktree import build_tree_pernode
from .config import SimConfig
from .measure import measure
@dataclass
class EpochResult:
d_next: np.ndarray # (N,) each node's updated D_est
m: np.ndarray # (N,) per-node measured slot count (canonical + recovered)
q: np.ndarray # (N,) per-node honest active-slot fraction
q_eff: np.ndarray # (N,) per-node uncle-recovered fraction
n_blocks: int # real blocks produced
n_active_window: int # global active slots in window
agreement_window: float # fraction of nodes sharing the modal window prefix
agreement_tip: float # fraction of nodes sharing the modal current tip
mean_orphan_rate: float # mean over nodes of (blocks not on my chain)/blocks
adv_blocks: int # coalition blocks on the canonical chain, in window (reward)
honest_blocks: int # non-coalition blocks on the canonical chain, in window
fork_rate: float # orphaned / total blocks in window
max_reorg_depth: int # deepest maximal orphan branch (blocks a reorg would discard)
mean_reorg_depth: float # mean maximal-orphan-branch depth
p_ref: float # emergent reference rate: in-window orphans referenced as uncles
def _canonical_producer_split(
tree, A, coalition_mask: np.ndarray | None, T: int, cutoff: int
) -> tuple[int, int]:
"""Split the finalized canonical chain's in-window blocks by producer coalition.
The canonical chain is the best *arrived* tip's ancestry (honest longest-chain, first-seen
tie-break); past k-finality every node agrees on it, so it is the reward-bearing chain.
Returns ``(adv_blocks, honest_blocks)`` counting blocks with slot in ``[0, T)``.
A withheld block never arrives (``A[:, b] > cutoff`` at every node) yet keeps a valid height, so
it must be **excluded** from tip selection — otherwise a never-propagated coalition block could
be chosen as the canonical tip and credited a phantom reward. Only the *full* matrix carries
withheld columns; the pruned path is never used with withholding, so all blocks arrived there.
"""
nb = tree.n_blocks
if nb <= 1:
return 0, 0
ids = np.arange(nb)
if isinstance(A, np.ndarray):
arrived = (A <= cutoff).any(axis=0) # (nb,) — withheld cols (A=E+1) -> False
else:
arrived = np.ones(nb, dtype=bool) # pruned path never withholds
arrived[0] = True # genesis is known to all
# best arrived tip by (height, -slot, -id); never-arrived blocks pushed below genesis
h = np.where(arrived, tree.height, np.iinfo(np.int64).min)
best = int(np.lexsort((-ids, -tree.slot, h))[-1])
adv = honest = 0
b = best
while b > 0:
s = int(tree.slot[b])
if 0 <= s < T:
if coalition_mask is not None and coalition_mask[int(tree.leader[b])]:
adv += 1
else:
honest += 1
b = int(tree.parent[b])
return adv, honest
def simulate_epoch(
config: SimConfig,
stake: np.ndarray,
d_est: np.ndarray,
path_latency: np.ndarray,
epoch_ss: np.random.SeedSequence,
adversary_mask: np.ndarray | None = None,
coalition_mask: np.ndarray | None = None,
inactive_mask: np.ndarray | None = None,
) -> EpochResult:
"""``adversary_mask`` drives BEHAVIOUR this epoch (None == honest); ``coalition_mask`` is the
fixed coalition identity used only for reward attribution (so a rejoin epoch, mask None, still
credits the coalition's honestly-produced blocks). Defaults to ``adversary_mask`` when unset.
"""
f, T, E = config.f, config.period_T, config.epoch_len
lottery_ss, aux_ss = epoch_ss.spawn(2)
aux_rng = np.random.default_rng(aux_ss)
# per-node lottery: d_est is a VECTOR -> per-node win prob, sparse sampler unchanged
p = lottery.win_probs(stake, d_est, f)
if inactive_mask is not None:
p = np.where(inactive_mask, 0.0, p) # churned-out nodes win no slots this epoch
winner_slots, winner_nodes = lottery.sample_wins(p, E, np.random.default_rng(lottery_ss))
active_slots, groups = lottery.group_by_slot(winner_slots, winner_nodes)
tree, A = build_tree_pernode(active_slots, groups, path_latency, config, aux_rng,
adversary_mask=adversary_mask)
# measurement: each node's own canonical chain, deduped by tip + numba-accelerated
ms = measure(tree, A, active_slots, T, cutoff=E,
legacy_block_count=config.legacy_block_count)
n_active_window = int((active_slots < T).sum())
d_next = tsi.update_D_vec(d_est, ms.m, T, f, config.beta, config.fixed_point)
attribution = coalition_mask if coalition_mask is not None else adversary_mask
adv_blocks, honest_blocks = _canonical_producer_split(tree, A, attribution, T, E)
fork_rate, max_reorg_depth, mean_reorg_depth, p_ref = fork.fork_stats(tree, A, T, cutoff=E)
return EpochResult(
d_next=d_next, m=ms.m, q=ms.q, q_eff=ms.q_eff, n_blocks=tree.n_blocks - 1,
n_active_window=n_active_window,
agreement_window=ms.agreement_window, agreement_tip=ms.agreement_tip,
mean_orphan_rate=float(ms.orphan_rate.mean()),
adv_blocks=adv_blocks, honest_blocks=honest_blocks,
fork_rate=fork_rate, max_reorg_depth=max_reorg_depth, mean_reorg_depth=mean_reorg_depth,
p_ref=p_ref,
)

78
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/fork.py Normal file
View File

@ -0,0 +1,78 @@
"""Fork structure of the global block tree: fork rate and reorg depth.
Both are read off the final canonical chain (the best arrived tip\'s ancestry, the chain all
honest nodes agree on past k-finality):
* ``fork_rate`` = orphaned blocks / total blocks, over blocks with slot in the window. The
share of produced blocks that lost their race and left the canonical chain.
* ``max_reorg_depth`` = the length of the deepest *maximal orphan branch* — the number of
consecutive non-canonical blocks a node that had adopted that branch would discard on
switching to canonical. This is the worst-case reorg (reorganisation) an honest node could
suffer, and its cost is what deep-fork-avoidance optimises.
"""
from __future__ import annotations
import numpy as np
from .blocktree import BlockTree
def fork_stats(tree: BlockTree, A, T: int, cutoff: int) -> tuple[float, int, float, float]:
"""Return ``(fork_rate, max_reorg_depth, mean_reorg_depth, p_ref)`` over in-window blocks.
``p_ref`` is the emergent **reference rate**: the fraction of in-window orphans that some
canonical block references as an uncle — the quantity the §6.8 soft-inclusion argument
assumes is high. ``A`` is the arrival matrix (full ``np.ndarray`` or pruned): only used to
exclude withheld blocks (which reach no node) from canonical-tip selection.
"""
nb = tree.n_blocks
if nb <= 1:
return 0.0, 0, 0.0, 1.0
ids = np.arange(nb)
if isinstance(A, np.ndarray):
arrived = (A <= cutoff).any(axis=0)
else:
arrived = np.ones(nb, dtype=bool)
arrived[0] = True
h = np.where(arrived, tree.height, np.iinfo(np.int64).min)
best = int(np.lexsort((-ids, -tree.slot, h))[-1])
canonical = np.zeros(nb, dtype=bool)
b = best
while b > 0:
canonical[b] = True
b = int(tree.parent[b])
canonical[0] = True
in_win = (tree.slot >= 0) & (tree.slot < T)
total = int(in_win.sum())
if total == 0:
return 0.0, 0, 0.0, 1.0
# depth[b] = length of the non-canonical run ending at b (0 if canonical). Parent-before-child
# holds because a block\'s parent has a strictly smaller id (built earlier).
depth = np.zeros(nb, dtype=np.int64)
for b in range(1, nb):
if not canonical[b]:
depth[b] = depth[int(tree.parent[b])] + 1
orphan_in_win = in_win & ~canonical
n_orphan = int(orphan_in_win.sum())
fork_rate = float(n_orphan) / total
# reorg depth per maximal orphan branch = depth at its deepest block; take branch tips
has_child = np.zeros(nb, dtype=bool)
has_child[tree.parent[1:]] = True
tips = (~has_child) & orphan_in_win
branch_depths = depth[tips]
max_depth = int(branch_depths.max()) if branch_depths.size else 0
mean_depth = float(branch_depths.mean()) if branch_depths.size else 0.0
# p_ref: fraction of in-window orphans referenced as an uncle by a canonical block
referenced = np.zeros(nb, dtype=bool)
for b in np.nonzero(canonical)[0]:
for u in tree.uncles[b]:
referenced[u] = True
ref_orphans = int((orphan_in_win & referenced).sum())
p_ref = ref_orphans / n_orphan if n_orphan else 1.0
return fork_rate, max_depth, mean_depth, p_ref

54
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/latency.py Normal file
View File

@ -0,0 +1,54 @@
"""Network latency models.
Latency ``L`` is the number of slots between a block being produced and it becoming
visible to the rest of the network. ``L`` is deliberately named to avoid clashing with
``D`` (the stake estimate). A leader at slot ``t`` can only build on blocks whose
``visible_at <= t`` (its own block is visible to itself immediately), which is what
produces latency-induced forks.
"""
from __future__ import annotations
from typing import Protocol
import numpy as np
class LatencyModel(Protocol):
def visible_at(self, produced_slot: int, rng: np.random.Generator) -> int:
"""Slot at which a block produced at ``produced_slot`` becomes visible to others."""
...
class FixedSlotLatency:
"""Deterministic integer-slot latency: visible to all others at ``t + L``."""
def __init__(self, latency: int) -> None:
self.latency = int(latency)
def visible_at(self, produced_slot: int, rng: np.random.Generator) -> int:
return produced_slot + self.latency
class RealisticLatency:
"""Stochastic latency with mean ``L`` slots (optional sensitivity model).
Rounds an exponential draw (mean ``L``) up to whole slots. A stand-in for the
reference notebook's blend/broadcast delay model; not used by the primary sweep.
"""
def __init__(self, mean_latency: float) -> None:
self.mean_latency = float(mean_latency)
def visible_at(self, produced_slot: int, rng: np.random.Generator) -> int:
if self.mean_latency <= 0:
return produced_slot
draw = rng.exponential(self.mean_latency)
return produced_slot + int(np.ceil(draw))
def make_latency(config) -> LatencyModel: # noqa: ANN001 - avoid import cycle with config
"""Build the latency model for a config."""
if config.latency_stochastic:
return RealisticLatency(config.latency)
return FixedSlotLatency(config.latency)

114
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/lottery.py Normal file
View File

@ -0,0 +1,114 @@
"""Stake-weighted slot lottery (sparse sampler).
Per node ``i`` and slot, an independent Bernoulli win with probability
``phi_f(alpha_i) = 1 - (1 - f)^alpha_i`` where ``alpha_i = w_i / D_est``. Multiple winners
in a slot are possible (a guaranteed fork).
The number of slots a node wins is exactly ``Binomial(n_slots, p_i)``, and the won slots
are a uniformly-random distinct subset — this is *distributionally identical* to drawing an
independent Bernoulli(p_i) in every slot, but avoids materialising the dense
``(n_nodes, n_slots)`` array (which was ~95% of the whole simulator's runtime). Winners are
returned as sparse ``(winner_slots, winner_nodes)`` coordinates, sorted by slot.
"""
from __future__ import annotations
import numpy as np
from joblib import Parallel, delayed
def phi(f: float, alpha: np.ndarray | float) -> np.ndarray | float:
"""Leader-lottery win probability ``1 - (1 - f)^alpha``."""
return 1.0 - (1.0 - f) ** alpha
def win_probs(stake: np.ndarray, d_est: float, f: float) -> np.ndarray:
"""Per-node win probability ``phi_f(w_i / D_est)``."""
return phi(f, stake / d_est)
def _winners_from_counts(
counts: np.ndarray, offset: int, span: int, rng: np.random.Generator
) -> tuple[list[np.ndarray], list[np.ndarray]]:
"""For each winning node, sample ``counts[i]`` distinct slots in ``[offset, offset+span)``."""
nz = np.nonzero(counts)[0]
slot_parts: list[np.ndarray] = []
node_parts: list[np.ndarray] = []
for i in nz:
c = int(counts[i])
slots_i = rng.choice(span, size=c, replace=False).astype(np.int64) + offset
slot_parts.append(slots_i)
node_parts.append(np.full(c, i, np.int64))
return slot_parts, node_parts
def _finalize(
slot_parts: list[np.ndarray], node_parts: list[np.ndarray]
) -> tuple[np.ndarray, np.ndarray]:
if not slot_parts:
return np.empty(0, np.int64), np.empty(0, np.int64)
winner_slots = np.concatenate(slot_parts)
winner_nodes = np.concatenate(node_parts)
order = np.argsort(winner_slots, kind="stable")
return winner_slots[order], winner_nodes[order]
def sample_wins(
p_win: np.ndarray, n_slots: int, rng: np.random.Generator, chunk: int = 8192
) -> tuple[np.ndarray, np.ndarray]:
"""Sample lottery wins over ``n_slots`` slots (sparse; ``chunk`` kept for API compat)."""
counts = rng.binomial(n_slots, p_win)
slot_parts, node_parts = _winners_from_counts(counts, 0, n_slots, rng)
return _finalize(slot_parts, node_parts)
def sample_wins_chunked(
p_win: np.ndarray,
n_slots: int,
seedseq: np.random.SeedSequence,
n_chunks: int,
n_jobs: int = -1,
) -> tuple[np.ndarray, np.ndarray]:
"""Parallel sparse lottery: partition slots into ``n_chunks`` independent ranges.
Correct because ``Binomial(n_slots, p) = sum_c Binomial(L_c, p)`` and per-chunk distinct
subsets are independent. Deterministic given ``(seedseq, n_chunks)`` — but the exact
winner identities differ from the serial sampler and *change with* ``n_chunks``, so
``n_chunks`` must be a pinned, recorded config parameter, never derived from core count.
Note: after the sparse rewrite the lottery is a small fraction of an epoch, so this has
little ROI versus across-config parallelism; it exists for the rare isolated config with
an enormous ``n_slots`` and no across-config work to fill cores.
"""
if n_chunks <= 1:
return sample_wins(p_win, n_slots, np.random.default_rng(seedseq))
bounds = np.linspace(0, n_slots, n_chunks + 1).astype(np.int64)
children = seedseq.spawn(n_chunks)
def one_chunk(c: int) -> tuple[list[np.ndarray], list[np.ndarray]]:
lo, hi = int(bounds[c]), int(bounds[c + 1])
span = hi - lo
rng = np.random.default_rng(children[c])
counts = rng.binomial(span, p_win)
return _winners_from_counts(counts, lo, span, rng)
results = Parallel(n_jobs=n_jobs, prefer="threads")(
delayed(one_chunk)(c) for c in range(n_chunks)
)
slot_parts: list[np.ndarray] = []
node_parts: list[np.ndarray] = []
for sp, npar in results:
slot_parts.extend(sp)
node_parts.extend(npar)
return _finalize(slot_parts, node_parts)
def group_by_slot(
winner_slots: np.ndarray, winner_nodes: np.ndarray
) -> tuple[np.ndarray, list[np.ndarray]]:
"""Group sorted winner coordinates into ``(active_slots, winners_per_active_slot)``."""
if winner_slots.size == 0:
return np.empty(0, np.int64), []
active_slots, starts = np.unique(winner_slots, return_index=True)
groups = np.split(winner_nodes, starts[1:])
return active_slots, groups

201
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/measure.py Normal file
View File

@ -0,0 +1,201 @@
"""Optimised per-node measurement pass.
The naive loop recomputes each node's canonical chain, density, and agreement fingerprint
independently — O(N x chain) Python and ~95% of an epoch. Two exact optimisations:
The counted density ``m`` is SLOT-based (canonical slots + recovered uncle slots — the
"one count per slot" invariant; ``legacy_block_count`` reproduces the old per-block count).
1. **Dedup by tip.** Nodes sharing a current tip share their whole canonical chain and every
derived quantity, so we compute once per *distinct* tip and broadcast. High node agreement
(the common case) collapses N to a handful of computations.
2. **numba kernel.** Each distinct tip's chain walk (honest count, deduped referenced uncles,
recovered orphan slots, window-prefix fingerprint) runs as one cached, C-speed routine over
flat arrays. A pure-Python fallback keeps the package importable without numba.
Results are identical to the reference loop (``measure_reference``); see test_measure.
"""
from __future__ import annotations
from collections import Counter
from dataclasses import dataclass
import numpy as np
from .blocktree import BlockTree, tips_for_all_nodes
try:
from numba import njit, uint64
_HAVE_NUMBA = True
except ImportError: # pragma: no cover - numba is an optional accelerator
_HAVE_NUMBA = False
@dataclass
class Measurement:
m: np.ndarray # (N,) per-node block count
q: np.ndarray # (N,) honest active-slot fraction
q_eff: np.ndarray # (N,) uncle-recovered fraction
orphan_rate: np.ndarray # (N,)
agreement_window: float
agreement_tip: float
def _uncles_csr(tree: BlockTree) -> tuple[np.ndarray, np.ndarray]:
"""Flatten the ragged per-block uncle lists into CSR (flat ids + offsets)."""
nb = tree.n_blocks
ptr = np.zeros(nb + 1, np.int64)
for b in range(nb):
ptr[b + 1] = ptr[b] + len(tree.uncles[b])
flat = np.empty(int(ptr[-1]), np.int64)
for b in range(nb):
for j, u in enumerate(tree.uncles[b]):
flat[ptr[b] + j] = u
return flat, ptr
def _measure_tips_py(distinct_tips, parent, slot, uncle_flat, uncle_ptr, T,
uncle_stamp, honest_stamp):
"""Pure-Python per-distinct-tip walk (fallback / reference for the kernel)."""
K = distinct_tips.shape[0]
m = np.empty(K, np.int64)
n_honest = np.empty(K, np.int64)
n_rec = np.empty(K, np.int64)
chain_len = np.empty(K, np.int64)
fp = np.empty(K, np.uint64)
for ki in range(K):
# pass 1: chain -> honest count, mark honest slots, fingerprint, chain length
honest = 0
clen = 0
f = np.uint64(0)
b = int(distinct_tips[ki])
while b > 0:
clen += 1
s = int(slot[b])
if 0 <= s < T:
honest += 1
honest_stamp[s] = ki
f ^= _mix_py(np.uint64(b))
b = int(parent[b])
# pass 2: deduped referenced uncles in window + recovered orphan slots
ucnt = 0
rec = 0
b = int(distinct_tips[ki])
while b > 0:
for j in range(int(uncle_ptr[b]), int(uncle_ptr[b + 1])):
u = int(uncle_flat[j])
su = int(slot[u])
if 0 <= su < T and uncle_stamp[u] != ki:
uncle_stamp[u] = ki # dedup uncles by id (m counts blocks)
ucnt += 1
if honest_stamp[su] != ki:
rec += 1 # recovered slots deduped by slot
honest_stamp[su] = ki
b = int(parent[b])
m[ki] = honest + ucnt
n_honest[ki] = honest
n_rec[ki] = rec
chain_len[ki] = clen
fp[ki] = f
return m, n_honest, n_rec, chain_len, fp
def _mix_py(x: np.uint64) -> np.uint64:
with np.errstate(over="ignore"): # splitmix64 intentionally wraps mod 2^64
x = (x ^ (x >> np.uint64(30))) * np.uint64(0xBF58476D1CE4E5B9)
x = (x ^ (x >> np.uint64(27))) * np.uint64(0x94D049BB133111EB)
return x ^ (x >> np.uint64(31))
if _HAVE_NUMBA:
@njit(cache=True)
def _mix(x):
x = (x ^ (x >> uint64(30))) * uint64(0xBF58476D1CE4E5B9)
x = (x ^ (x >> uint64(27))) * uint64(0x94D049BB133111EB)
return x ^ (x >> uint64(31))
@njit(cache=True)
def _measure_tips_nb(distinct_tips, parent, slot, uncle_flat, uncle_ptr, T,
uncle_stamp, honest_stamp):
K = distinct_tips.shape[0]
m = np.empty(K, np.int64)
n_honest = np.empty(K, np.int64)
n_rec = np.empty(K, np.int64)
chain_len = np.empty(K, np.int64)
fp = np.empty(K, np.uint64)
for ki in range(K):
honest = 0
clen = 0
f = uint64(0)
b = distinct_tips[ki]
while b > 0:
clen += 1
s = slot[b]
if 0 <= s < T:
honest += 1
honest_stamp[s] = ki
f ^= _mix(uint64(b))
b = parent[b]
ucnt = 0
rec = 0
b = distinct_tips[ki]
while b > 0:
for j in range(uncle_ptr[b], uncle_ptr[b + 1]):
u = uncle_flat[j]
su = slot[u]
if 0 <= su < T and uncle_stamp[u] != ki:
uncle_stamp[u] = ki # dedup uncles by id
ucnt += 1
if honest_stamp[su] != ki:
rec += 1 # recovered slots deduped by slot
honest_stamp[su] = ki
b = parent[b]
m[ki] = honest + ucnt
n_honest[ki] = honest
n_rec[ki] = rec
chain_len[ki] = clen
fp[ki] = f
return m, n_honest, n_rec, chain_len, fp
def measure(tree: BlockTree, A, active_slots: np.ndarray, T: int, cutoff: int,
use_numba: bool = True, legacy_block_count: bool = False) -> Measurement:
"""Per-node m/q/q_eff + agreement, deduped by tip and (optionally) numba-accelerated.
``A`` is the full ``(N, n_blocks)`` arrival matrix or a pruned ``SlidingArrival`` — only
``tips_for_all_nodes`` reads it, so ``N`` is taken from the returned per-node tips.
"""
tips = tips_for_all_nodes(tree, A, cutoff)
N = tips.shape[0]
n_real = tree.n_blocks - 1
n_active = int((active_slots < T).sum())
distinct_tips, inverse = np.unique(tips, return_inverse=True)
inverse = inverse.ravel()
uncle_flat, uncle_ptr = _uncles_csr(tree)
uncle_stamp = np.full(tree.n_blocks, -1, np.int64)
honest_stamp = np.full(max(T, 1), -1, np.int64)
kernel = _measure_tips_nb if (_HAVE_NUMBA and use_numba) else _measure_tips_py
m_d, nh_d, nrec_d, clen_d, fp_d = kernel(
distinct_tips.astype(np.int64), tree.parent, tree.slot,
uncle_flat, uncle_ptr, np.int64(T), uncle_stamp, honest_stamp)
# correct slot counting: canonical slots + recovered (non-canonical, deduped) uncle slots.
# legacy_block_count reproduces the earlier per-block-id count (kernel's m = honest + ucnt).
m = m_d[inverse] if legacy_block_count else (nh_d + nrec_d)[inverse]
q = (nh_d[inverse] / n_active) if n_active else np.full(N, np.nan)
q_eff = ((nh_d[inverse] + nrec_d[inverse]) / n_active) if n_active else np.full(N, np.nan)
orphan_rate = ((n_real - clen_d[inverse]) / n_real) if n_real else np.zeros(N)
node_counts = np.bincount(inverse, minlength=distinct_tips.shape[0])
agreement_tip = float(node_counts.max()) / N
fp_counts: Counter = Counter()
for ki in range(distinct_tips.shape[0]):
fp_counts[int(fp_d[ki])] += int(node_counts[ki])
agreement_window = max(fp_counts.values()) / N
return Measurement(m=m, q=q, q_eff=q_eff, orphan_rate=orphan_rate,
agreement_window=agreement_window, agreement_tip=agreement_tip)

68
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/memguard.py Normal file
View File

@ -0,0 +1,68 @@
"""Fail-loud memory guards for the per-node engine's large allocations.
The two dominant arrays — the ``(N x n_blocks)`` arrival matrix ``A`` and the ``(N x N)``
``path_latency`` — can each reach tens to hundreds of GB: ``A`` when a low ``genesis_d_factor``
explodes ``n_blocks`` (the collapsed-``D_est`` regime), ``path_latency`` at very large ``N``.
Every worker checks the size *before* allocating and raises ``ArrivalMatrixTooLarge`` if it would
exceed its budget, so an under-estimated config fails with a clear message instead of freezing the
machine.
Budget resolution (``arrival_budget_bytes``):
- ``TSI_ARRIVAL_BYTES_BUDGET`` > 0 -> that many bytes (the sweep sets this to each worker's RAM
share so concurrent workers can't collectively OOM);
- unset / ``0`` / invalid -> a default of ``DEFAULT_BUDGET_FRAC`` of physical RAM, so a
*single* process (a bare ``run_trajectory``, ``tsi-verify``, the calibration probe, or a
``--mem-frac 0`` run) still cannot allocate past what the box physically has.
There is intentionally no "unlimited" setting: no correct run needs to allocate more than physical
RAM, and allowing it is exactly what froze the machine.
"""
from __future__ import annotations
import os
import subprocess
# Absolute per-process ceiling (fraction of physical RAM) used when no explicit budget is set.
DEFAULT_BUDGET_FRAC = 0.9
class ArrivalMatrixTooLarge(MemoryError):
"""A per-node engine array would exceed the memory budget; raised before allocating.
Typically a block-count explosion (a low ``genesis_d_factor`` inflating early-epoch lottery
wins) blowing up the ``(N x n_blocks)`` arrival matrix, or a very large ``N`` blowing up the
``(N x N)`` ``path_latency`` matrix.
"""
def total_ram_bytes() -> int:
"""Best-effort physical RAM in bytes (POSIX sysconf, then Darwin sysctl, then 8 GB)."""
try:
return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
except (ValueError, OSError, AttributeError):
pass
try: # macOS lacks SC_PHYS_PAGES
out = subprocess.run(["sysctl", "-n", "hw.memsize"], capture_output=True, text=True)
return int(out.stdout.strip())
except (OSError, ValueError):
return 8 * 1024**3
def arrival_budget_bytes() -> int:
"""Per-process byte budget for a single big array (see module docstring)."""
try:
explicit = int(os.environ.get("TSI_ARRIVAL_BYTES_BUDGET", "0"))
except ValueError:
explicit = 0
if explicit > 0:
return explicit
return int(DEFAULT_BUDGET_FRAC * total_ram_bytes())
def check_alloc(nbytes: int, label: str, detail: str = "") -> None:
"""Raise ``ArrivalMatrixTooLarge`` if allocating ``nbytes`` would exceed the budget."""
budget = arrival_budget_bytes()
if nbytes > budget:
raise ArrivalMatrixTooLarge(
f"{label} needs {nbytes / 1024**3:.1f} GB > per-process budget "
f"{budget / 1024**3:.1f} GB.{(' ' + detail) if detail else ''}")

82
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/metrics.py Normal file
View File

@ -0,0 +1,82 @@
"""Per-epoch per-node divergence rows and equilibrium summaries."""
from __future__ import annotations
from typing import Any
import numpy as np
from .config import SimConfig
from .epoch import EpochResult
# Config fields recorded on every row for grouping/plotting.
_CONFIG_FIELDS = (
"n_nodes", "stake_dist", "pareto_shape", "latency", "topology", "degree",
"link_latency_mean", "link_latency_dist", "blend_hops", "blend_delay_max",
"init_dest", "init_spread", "uncle_window", "max_uncles", "uncle_strategy",
"f", "beta", "k", "genesis_d_factor", "epochs", "fixed_point", "legacy_block_count",
"replicate",
"adversary_frac", "adversary_strategy", "adversary_period", "adversary_withhold_epochs",
)
def divergence_row(
config: SimConfig, epoch: int, d_in: np.ndarray, er: EpochResult, d_true: float
) -> dict[str, Any]:
"""One row per (config, epoch): per-node D_est spread + chain agreement."""
ratio = np.asarray(er.d_next, dtype=float) / d_true # (N,)
row: dict[str, Any] = {field: getattr(config, field) for field in _CONFIG_FIELDS}
row.update(
epoch=epoch,
mean_ratio=float(ratio.mean()),
median_ratio=float(np.median(ratio)),
std_ratio=float(ratio.std()),
min_ratio=float(ratio.min()),
max_ratio=float(ratio.max()),
range_ratio=float(ratio.max() - ratio.min()), # the headline divergence measure
iqr_ratio=float(np.percentile(ratio, 75) - np.percentile(ratio, 25)),
p10_ratio=float(np.percentile(ratio, 10)),
p90_ratio=float(np.percentile(ratio, 90)),
mean_ratio_in=float((np.asarray(d_in, dtype=float) / d_true).mean()),
range_ratio_in=float(np.ptp(np.asarray(d_in, dtype=float) / d_true)),
mean_m=float(np.mean(er.m)),
mean_q=float(np.nanmean(er.q)),
mean_q_eff=float(np.nanmean(er.q_eff)),
std_q=float(np.nanstd(er.q)),
agreement_window=er.agreement_window,
agreement_tip=er.agreement_tip,
mean_orphan_rate=er.mean_orphan_rate,
n_active_window=er.n_active_window,
n_blocks=er.n_blocks,
adv_blocks=er.adv_blocks,
honest_blocks=er.honest_blocks,
adv_block_share=(
er.adv_blocks / (er.adv_blocks + er.honest_blocks)
if (er.adv_blocks + er.honest_blocks) > 0 else 0.0
),
fork_rate=er.fork_rate,
max_reorg_depth=er.max_reorg_depth,
mean_reorg_depth=er.mean_reorg_depth,
p_ref=er.p_ref,
)
return row
def equilibrium_stats(values: np.ndarray, burn_in: int) -> dict[str, float]:
"""Mean/variance of ``values`` after ``burn_in`` epochs."""
values = np.asarray(values, dtype=float)
if values.size == 0:
return {"mean": float("nan"), "var": float("nan"), "std": float("nan")}
tail = values[burn_in:]
if tail.size == 0:
tail = values[-1:]
return {"mean": float(np.mean(tail)), "var": float(np.var(tail)), "std": float(np.std(tail))}
def epochs_to_within(values: np.ndarray, target: float, eps: float) -> int:
"""First epoch after which ``|values - target| <= eps`` holds for the rest."""
within = np.abs(np.asarray(values, dtype=float) - target) <= eps
if within.size == 0:
return 0
false_idx = np.flatnonzero(~within)
return int(false_idx[-1] + 1) if false_idx.size else 0

1
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/plotting/__init__.py Normal file
View File

@ -0,0 +1 @@
"""Academic-quality figure generation."""

360
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/plotting/figures_pernode.py Normal file
View File

@ -0,0 +1,360 @@
"""Per-node divergence & topology figures. Each takes the results frame, returns a Figure.
Headline finding these visualise: per-node ``D_est`` spread stays ~0 and window agreement
stays ~1 (validating the reduced model) even while nodes disagree on the current tip;
topology/latency instead shift the shared *mean* accuracy, which uncles recover.
"""
from __future__ import annotations
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from . import style
CONFIG_COLS = ["n_nodes", "stake_dist", "topology", "degree", "link_latency_mean",
"blend_hops", "blend_delay_max", "latency", "max_uncles",
"uncle_strategy", "uncle_window", "init_dest", "k"]
# Graph topologies (as opposed to the full_mesh baseline) and the dominant latency knob each
# is plotted against: regular varies the per-link latency, blend varies the per-hop mix delay.
GRAPH_TOPOLOGIES = ("regular", "blend")
def _lat_axis(topo: str) -> tuple[str, str]:
"""(dataframe column, axis label) for the dominant latency knob of a graph topology."""
if topo == "blend":
return "blend_delay_max", "max blending delay per hop (slots)"
return "link_latency_mean", "mean per-link latency (slots)"
def equilibrium(df: pd.DataFrame, burn_frac: float = 0.5) -> pd.DataFrame:
"""Per-(config, replicate) tail means of the summary columns.
Burn-in is a fraction of each trajectory's *observed* last epoch, not the configured
``epochs`` — early-stopped runs (config.early_stop) terminate well before the planned
``epochs``, so thresholding on the configured value would drop every row.
"""
max_epoch = df.groupby([*CONFIG_COLS, "replicate"])["epoch"].transform("max")
tail = df[df["epoch"] >= max_epoch * burn_frac]
agg = {c: (c, "mean") for c in
("mean_ratio", "range_ratio", "iqr_ratio", "agreement_window", "agreement_tip",
"mean_q", "mean_q_eff", "mean_orphan_rate", "max_ratio", "min_ratio")}
return tail.groupby([*CONFIG_COLS, "replicate"], as_index=False).agg(**agg)
def _prov(df: pd.DataFrame) -> str:
k = [int(x) for x in sorted(df["k"].unique())]
n = [int(x) for x in sorted(df["n_nodes"].unique())]
return f"tsi-sim-pernode | k={k} N={n} reps={int(df['replicate'].nunique())}"
# --- Figure 1: headline — D_est spread ~0 & agreement, vs epoch --------------
def divergence_vs_epoch(df: pd.DataFrame, stake_dist: str, topo: str = "regular") -> plt.Figure:
style.apply_style()
sub = df[(df["stake_dist"] == stake_dist) & (df["topology"] == topo)]
if sub.empty:
sub = df[df["stake_dist"] == stake_dist]
g = sub.groupby("epoch").agg(
range_ratio=("range_ratio", "mean"), iqr_ratio=("iqr_ratio", "mean"),
agree_w=("agreement_window", "mean"), agree_t=("agreement_tip", "mean"))
fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, figsize=(6.4, 5.0))
ax1.plot(g.index, g["range_ratio"], "o-", color=style.color_for(1), label="range (max−min)")
ax1.plot(g.index, g["iqr_ratio"], "s--", color=style.color_for(0), label="IQR")
ax1.set_ylabel(r"per-node $\hat D/D_{\mathrm{true}}$ spread")
ax1.set_title(f"Per-node D_est divergence stays ~0 ({stake_dist}, {topo})")
ax1.legend()
ax2.plot(g.index, g["agree_w"], "o-", color=style.color_for(2), label="window prefix")
ax2.plot(g.index, g["agree_t"], "^--", color=style.color_for(3), label="current tip")
ax2.set_ylim(0, 1.05)
ax2.set_xlabel("epoch")
ax2.set_ylabel("node agreement fraction")
ax2.legend(title="agreement on")
return fig
# --- Figure 2: mean accuracy vs latency, per degree --------------------------
def accuracy_vs_link_latency(df: pd.DataFrame, stake_dist: str,
topo: str = "regular") -> plt.Figure:
style.apply_style()
lat_col, lat_label = _lat_axis(topo)
eq = equilibrium(df)
eq = eq[(eq["stake_dist"] == stake_dist) & (eq["topology"] == topo)]
umin = int(eq["max_uncles"].min()) if not eq.empty else 0 # min U WITHIN this (dist, topo)
eq = eq[eq["max_uncles"] == umin]
lls = sorted(eq[lat_col].unique())
fig, ax = plt.subplots()
for i, deg in enumerate(sorted(eq["degree"].unique())):
s = eq[eq["degree"] == deg].groupby(lat_col)["mean_ratio"].mean()
ax.plot(lls, [s.get(x, np.nan) for x in lls], "o-", color=style.color_for(i),
label=f"degree={deg}")
ax.axhline(1.0, color="0.4", lw=1.0, ls="--", zorder=0)
ax.set_xlabel(lat_label)
ax.set_ylabel(r"mean $\hat D / D_{\mathrm{true}}$")
ax.set_title(f"Accuracy vs latency ({stake_dist}, {topo}, U={umin})")
ax.legend(title="peering")
return fig
# --- Figure 3: mean accuracy vs U, per latency (uncle recovery) --------------
def accuracy_vs_u(df: pd.DataFrame, stake_dist: str, topo: str = "regular") -> plt.Figure:
style.apply_style()
lat_col, lat_label = _lat_axis(topo)
eq = equilibrium(df)
eq = eq[(eq["stake_dist"] == stake_dist) & (eq["topology"] == topo)]
uvals = sorted(eq["max_uncles"].unique())
fig, ax = plt.subplots()
for i, ll in enumerate(sorted(eq[lat_col].unique())):
s = eq[eq[lat_col] == ll].groupby("max_uncles")["mean_ratio"].mean()
ax.plot(uvals, [s.get(x, np.nan) for x in uvals], "o-", color=style.color_for(i),
label=f"{ll:g}")
ax.axhline(1.0, color="0.4", lw=1.0, ls="--", zorder=0)
ax.set_xlabel("max uncles per block $U$")
ax.set_ylabel(r"mean $\hat D / D_{\mathrm{true}}$")
ax.set_title(f"Uncle recovery under topology ({stake_dist}, {topo})")
ax.set_xticks(uvals)
# Slot-counting bounds the equilibrium at 1 (it cannot over-count occupied slots); any
# above-1 reading is sampling noise, so cap the view at the bound rather than show headroom.
ax.set_ylim(top=1.01)
ax.legend(title=lat_label)
return fig
# --- Figure 4: tip agreement vs latency, per degree --------------------------
def tip_agreement_vs_latency(df: pd.DataFrame, stake_dist: str,
topo: str = "regular") -> plt.Figure:
style.apply_style()
lat_col, lat_label = _lat_axis(topo)
eq = equilibrium(df)
eq = eq[(eq["stake_dist"] == stake_dist) & (eq["topology"] == topo)]
umin = int(eq["max_uncles"].min()) if not eq.empty else 0 # min U WITHIN this (dist, topo)
eq = eq[eq["max_uncles"] == umin]
lls = sorted(eq[lat_col].unique())
fig, ax = plt.subplots()
for i, deg in enumerate(sorted(eq["degree"].unique())):
s = eq[eq["degree"] == deg].groupby(lat_col)["agreement_tip"].mean()
ax.plot(lls, [s.get(x, np.nan) for x in lls], "o-", color=style.color_for(i),
label=f"degree={deg}")
ax.set_xlabel(lat_label)
ax.set_ylabel("current-tip agreement fraction")
ax.set_title(f"Tip-level fork disagreement vs topology ({stake_dist}, {topo})")
ax.legend(title="peering")
return fig
# --- Figure 6: accuracy heatmap over latency x uncle-cap (per degree) --------
def heatmap_accuracy(df: pd.DataFrame, stake_dist: str, degree: int,
topo: str = "regular") -> plt.Figure:
"""Decision chart: mean D_est/D_true over (latency knob) x (uncle cap) at a degree."""
style.apply_style()
lat_col, lat_label = _lat_axis(topo)
eq = equilibrium(df)
eq = eq[(eq["stake_dist"] == stake_dist) & (eq["topology"] == topo)
& (eq["degree"] == degree) & (eq["init_dest"] == "common")]
piv = eq.groupby([lat_col, "max_uncles"])["mean_ratio"].mean().unstack("max_uncles")
lls = piv.index.to_numpy()
uvals = piv.columns.to_numpy()
data = piv.to_numpy()
fig, ax = plt.subplots()
# Accuracy is bounded by 1 (slot-counting cannot over-count), so the colour scale tops out
# at the true maximum 1.0 rather than treating above-1 noise as a symmetric deviation.
lo = float(np.nanmin(data)) if np.isfinite(data).any() else 0.5
im = ax.imshow(data, origin="lower", aspect="auto", cmap=style.SEQUENTIAL_CMAP,
vmin=lo, vmax=1.0)
ax.set_xticks(range(len(uvals)), uvals)
ax.set_yticks(range(len(lls)), [f"{x:g}" for x in lls])
ax.set_xlabel("max uncles per block $U$")
ax.set_ylabel(lat_label)
ax.set_title(
f"Accuracy $\\hat D/D_{{\\mathrm{{true}}}}$ ({stake_dist}, {topo}, degree={degree})")
for yi in range(len(lls)):
for xi in range(len(uvals)):
v = data[yi, xi]
if np.isfinite(v):
safe = 0.98 <= v <= 1.02
tc = "white" if (v - lo) / max(1e-9, 1.0 - lo) < 0.45 else "black"
ax.text(xi, yi, f"{v:.2f}", ha="center", va="center", fontsize=7,
color=tc, fontweight="bold" if safe else "normal")
fig.colorbar(im, ax=ax, label=r"$\hat D / D_{\mathrm{true}}$")
ax.grid(False)
return fig
# --- Figure 5: heterogeneous-start recovery (spread preserved) ---------------
def heterogeneous_recovery(df: pd.DataFrame, stake_dist: str,
topo: str = "regular") -> plt.Figure | None:
# filter to ONE topology — the injected-spread dynamics differ by propagation model, so
# merging regular + blend would plot a curve neither regime actually follows.
het = df[(df["stake_dist"] == stake_dist) & (df["init_dest"] == "heterogeneous")
& (df["topology"] == topo)]
if het.empty:
return None
style.apply_style()
g = het.groupby("epoch")["range_ratio"].agg(["mean", "std"])
fig, ax = plt.subplots()
ax.errorbar(g.index, g["mean"], yerr=g["std"], marker="o", color=style.color_for(1), capsize=2)
ax.set_xlabel("epoch")
ax.set_ylabel(r"per-node spread range($\hat D/D_{\mathrm{true}}$)")
ax.set_title(f"Heterogeneous start: injected disagreement is preserved ({stake_dist}, {topo})")
return fig
# --- Figure 7: bootstrap — block-production rate stabilisation ----------------
def block_production_stabilization(df: pd.DataFrame, stake_dist: str | None = None) -> plt.Figure:
"""Cold-start dynamics: block production rate + ``D_est`` convergence per ``genesis_d_factor``.
At bootstrap no node knows the true total stake, so the difficulty ``D_est`` is only a guess.
A guess *below* the truth (``genesis_d_factor < 1``) inflates every node's win probability
``phi(w_i / D_est)`` → a block "storm" many times the target rate; TSI reads the high density
and raises ``D_est`` until production settles at the equilibrium rate ``~f`` within a couple of
epochs. A guess *above* the truth under-produces and is corrected up. Top panel: production
rate ``n_blocks / epoch_len`` (log scale) vs epoch; bottom: ``D_est / D_true`` vs epoch.
"""
style.apply_style()
sub = df if stake_dist is None else df[df["stake_dist"] == stake_dist]
if sub.empty:
sub = df
sub = sub.copy()
f = float(sub["f"].iloc[0])
sub["epoch_len"] = np.floor(sub["k"] / sub["f"]) * 10.0 # E = 10*floor(k/f)
sub["blocks_per_slot"] = sub["n_blocks"] / sub["epoch_len"]
fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, figsize=(6.4, 5.6))
for i, g in enumerate(sorted(sub["genesis_d_factor"].unique())):
s = sub[sub["genesis_d_factor"] == g]
rate = s.groupby("epoch")["blocks_per_slot"].mean()
# D_est that DROVE each epoch's production (start-of-epoch estimate), so both panels
# are indexed by the same operating estimate — the gdf=0.01 curve starts at 0.01.
ratio_in = s.groupby("epoch")["mean_ratio_in"].mean()
c = style.color_for(i)
ax1.plot(rate.index, rate.to_numpy(), "o-", color=c, ms=3.5, label=f"{g:g}")
ax2.plot(ratio_in.index, ratio_in.to_numpy(), "o-", color=c, ms=3.5, label=f"{g:g}")
ax1.axhline(f, color="0.4", lw=1.0, ls="--", zorder=0)
ax1.text(0.99, f, r" equilibrium $\approx f$", transform=ax1.get_yaxis_transform(),
va="bottom", ha="right", fontsize=7, color="0.4")
ax1.set_yscale("log")
ax1.set_ylabel("block production\n(blocks / slot)")
ax1.set_title("Bootstrap: block production stabilises to the target rate")
ax1.legend(title=r"genesis $D_{\mathrm{est}}/D_{\mathrm{true}}$", ncol=2, loc="upper right")
ax2.axhline(1.0, color="0.4", lw=1.0, ls="--", zorder=0)
ax2.set_yscale("log")
ax2.set_xlabel("epoch")
ax2.set_ylabel(r"$D_{\mathrm{est}} / D_{\mathrm{true}}$ (start of epoch)")
ax2.set_title(r"...as TSI corrects $D_{\mathrm{est}}$ to the true total stake")
return fig
# --- Figure 8: uncle-window sufficiency — accuracy vs W, per delay -------------
def accuracy_vs_uncle_window(df: pd.DataFrame, stake_dist: str,
topo: str = "regular") -> plt.Figure:
"""Mean ``D_hat/D`` vs the uncle window ``W``, one line per delay (the topology's latency knob).
An uncle can only reference an orphan whose slot is within ``W`` of the referencing block, so
when block visibility is delayed the orphans spread over a wider slot range — a small ``W``
then fails to reach them and the estimate stays low. This shows, at a fixed uncle cap, the
critical ``W`` at which recovery kicks in, and how it grows with the delay.
"""
style.apply_style()
lat_col, lat_label = _lat_axis(topo)
eq = equilibrium(df)
eq = eq[(eq["stake_dist"] == stake_dist) & (eq["topology"] == topo)]
u = int(eq["max_uncles"].max()) if not eq.empty else 0 # this study fixes a single U
eq = eq[eq["max_uncles"] == u]
ws = sorted(eq["uncle_window"].unique())
fig, ax = plt.subplots()
for i, d in enumerate(sorted(eq[lat_col].unique())):
s = eq[eq[lat_col] == d].groupby("uncle_window")["mean_ratio"].mean()
ax.plot(ws, [s.get(x, np.nan) for x in ws], "o-", color=style.color_for(i), label=f"{d:g}")
ax.axhline(1.0, color="0.4", lw=1.0, ls="--", zorder=0)
ax.set_xscale("log")
ax.set_xlabel("uncle window $W$ (slots)")
ax.set_ylabel(r"mean $\hat D / D_{\mathrm{true}}$")
ax.set_title(f"Uncle-window sufficiency ({stake_dist}, {topo}, U={u})")
ax.set_ylim(top=1.01) # bounded by 1 (see accuracy_vs_u); no above-1 headroom
ax.legend(title=lat_label)
return fig
# --- Figure 9: W x delay accuracy heatmap -------------------------------------
def heatmap_window_delay(df: pd.DataFrame, stake_dist: str, topo: str = "regular") -> plt.Figure:
"""Accuracy ``D_hat/D`` over uncle window ``W`` (rows) x delay (cols) at a fixed uncle cap.
Reads off the ``(W, delay)`` relation directly: blue cells are where ``W`` is too small for the
delay (uncles can't reach the orphans) — the boundary is the minimum window a given delay needs.
"""
style.apply_style()
lat_col, lat_label = _lat_axis(topo)
eq = equilibrium(df)
eq = eq[(eq["stake_dist"] == stake_dist) & (eq["topology"] == topo)]
u = int(eq["max_uncles"].max()) if not eq.empty else 0
eq = eq[eq["max_uncles"] == u]
piv = eq.groupby(["uncle_window", lat_col])["mean_ratio"].mean().unstack(lat_col)
ws = piv.index.to_numpy()
delays = piv.columns.to_numpy()
data = piv.to_numpy()
fig, ax = plt.subplots()
# Accuracy is bounded by 1 (slot-counting cannot over-count), so the colour scale tops out
# at the true maximum 1.0 rather than treating above-1 noise as a symmetric deviation.
lo = float(np.nanmin(data)) if np.isfinite(data).any() else 0.5
im = ax.imshow(data, origin="lower", aspect="auto", cmap=style.SEQUENTIAL_CMAP,
vmin=lo, vmax=1.0)
ax.set_xticks(range(len(delays)), [f"{x:g}" for x in delays])
ax.set_yticks(range(len(ws)), [f"{int(x)}" for x in ws])
ax.set_xlabel(lat_label)
ax.set_ylabel("uncle window $W$ (slots)")
ax.set_title(f"Accuracy over (W x delay) ({stake_dist}, {topo}, U={u})")
for yi in range(len(ws)):
for xi in range(len(delays)):
v = data[yi, xi]
if np.isfinite(v):
tc = "white" if (v - lo) / max(1e-9, 1.0 - lo) < 0.45 else "black"
ax.text(xi, yi, f"{v:.2f}", ha="center", va="center", fontsize=7,
color=tc, fontweight="bold" if 0.98 <= v <= 1.02 else "normal")
fig.colorbar(im, ax=ax, label=r"$\hat D / D_{\mathrm{true}}$")
ax.grid(False)
return fig
# --- Figure 10: joint (W x U) safe region at a fixed delay --------------------
def heatmap_window_uncles(df: pd.DataFrame, stake_dist: str, delay: float,
topo: str = "blend") -> plt.Figure:
"""Accuracy ``D_hat/D`` over uncle window ``W`` (rows) x uncle cap ``U`` (cols) at one delay.
Maps the joint ``(W, U)`` safe region: white cells are recovered. Moving right (more uncles)
vs up (wider window) shows which lever matters where — a wider window only helps until one
uncle per block can no longer drain the orphan queue, past which you must add uncles instead.
"""
style.apply_style()
lat_col, lat_label = _lat_axis(topo)
eq = equilibrium(df)
eq = eq[(eq["stake_dist"] == stake_dist) & (eq["topology"] == topo) & (eq[lat_col] == delay)]
piv = eq.groupby(["uncle_window", "max_uncles"])["mean_ratio"].mean().unstack("max_uncles")
ws = piv.index.to_numpy()
us = piv.columns.to_numpy()
data = piv.to_numpy()
fig, ax = plt.subplots()
# Accuracy is bounded by 1 (slot-counting cannot over-count), so the colour scale tops out
# at the true maximum 1.0 rather than treating above-1 noise as a symmetric deviation.
lo = float(np.nanmin(data)) if np.isfinite(data).any() else 0.5
im = ax.imshow(data, origin="lower", aspect="auto", cmap=style.SEQUENTIAL_CMAP,
vmin=lo, vmax=1.0)
ax.set_xticks(range(len(us)), [f"{int(x)}" for x in us])
ax.set_yticks(range(len(ws)), [f"{int(x)}" for x in ws])
ax.set_xlabel("max uncles per block $U$")
ax.set_ylabel("uncle window $W$ (slots)")
knob = lat_label.split("(")[0].strip()
ax.set_title(f"(W x U) accuracy ({stake_dist}, {topo}, {knob}={delay:g})")
for yi in range(len(ws)):
for xi in range(len(us)):
v = data[yi, xi]
if np.isfinite(v):
tc = "white" if (v - lo) / max(1e-9, 1.0 - lo) < 0.45 else "black"
ax.text(xi, yi, f"{v:.2f}", ha="center", va="center", fontsize=7,
color=tc, fontweight="bold" if 0.98 <= v <= 1.02 else "normal")
fig.colorbar(im, ax=ax, label=r"$\hat D / D_{\mathrm{true}}$")
ax.grid(False)
return fig

83
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/plotting/make_figures.py Normal file
View File

@ -0,0 +1,83 @@
"""Render per-node divergence & topology figures (importable + CLI ``tsi-figures``)."""
from __future__ import annotations
import argparse
import datetime as _dt
from pathlib import Path
import pandas as pd
from . import figures_pernode as F
from . import style
def timestamped_figdir(outdir: str | Path, label: str) -> Path:
"""A fresh ``<outdir>/<YYYY-MM-DD_HHMMSS>_<label>`` folder (never overwrites)."""
ts = _dt.datetime.now().strftime("%Y-%m-%d_%H%M%S")
base = Path(outdir) / f"{ts}_{label}"
d, i = base, 2
while d.exists():
d = base.with_name(f"{base.name}_{i}")
i += 1
return d
def render(df: pd.DataFrame, out: str | Path) -> int:
"""Render every applicable figure from ``df`` into directory ``out``; return the count.
Graph figures are produced per graph topology present (``regular`` and/or ``blend``) so the
two propagation models are never merged; each is plotted against its own latency knob
(per-link latency for ``regular``, per-hop mixing delay for ``blend``).
"""
out = Path(out)
prov = F._prov(df)
written: list[Path] = []
present = set(df["topology"].unique())
graph_topos = [t for t in F.GRAPH_TOPOLOGIES if t in present] or ["regular"]
for dist in sorted(df["stake_dist"].unique()):
for topo in graph_topos:
tag = f"{dist}_{topo}"
sub = df[(df["stake_dist"] == dist) & (df["topology"] == topo)]
written += style.save(F.divergence_vs_epoch(df, dist, topo),
out / f"01_divergence_{tag}", prov)
if sub.empty:
continue # latency / heatmap figures need real graph data
written += style.save(F.accuracy_vs_link_latency(df, dist, topo),
out / f"02_accuracy_vs_latency_{tag}", prov)
written += style.save(F.accuracy_vs_u(df, dist, topo),
out / f"03_accuracy_vs_u_{tag}", prov)
written += style.save(F.tip_agreement_vs_latency(df, dist, topo),
out / f"04_tip_agreement_{tag}", prov)
# accuracy heatmap (latency knob x U) for every peering degree
for deg in sorted(int(x) for x in sub["degree"].unique()):
written += style.save(F.heatmap_accuracy(df, dist, deg, topo),
out / f"06_heatmap_{tag}_deg{deg}", prov)
fig = F.heterogeneous_recovery(df, dist, topo) # per topology (never merged)
if fig is not None:
written += style.save(fig, out / f"05_heterogeneous_recovery_{tag}", prov)
# uncle-window study figures — only when W is actually swept
if sub["uncle_window"].nunique() > 1:
written += style.save(F.accuracy_vs_uncle_window(df, dist, topo),
out / f"07_accuracy_vs_window_{tag}", prov)
written += style.save(F.heatmap_window_delay(df, dist, topo),
out / f"08_window_delay_heatmap_{tag}", prov)
return len(written)
def main(argv: list[str] | None = None) -> None:
ap = argparse.ArgumentParser(description="Render per-node TSI figures")
ap.add_argument("--results", required=True)
ap.add_argument("--outdir", default="figures",
help="parent dir; a dated sub-folder is created (never overwrites)")
ap.add_argument("--label", default=None, help="run label (default: results file stem)")
ap.add_argument("--out", default=None, help="explicit output dir (skips the dated folder)")
args = ap.parse_args(argv)
label = args.label or Path(args.results).stem
out = Path(args.out) if args.out else timestamped_figdir(args.outdir, label)
n = render(pd.read_parquet(args.results), out)
print(f"wrote {n} figures -> {out}")
if __name__ == "__main__": # pragma: no cover
main()

92
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/plotting/style.py Normal file
View File

@ -0,0 +1,92 @@
"""Shared academic matplotlib theme, palette, and helpers.
Palette: Okabe-Ito — the standard colorblind-safe qualitative set — for categorical
series (the ``U`` lines); ``cividis`` (perceptually uniform, CVD-safe) for heatmaps.
Figures are saved as vector PDF (primary) plus 300-dpi PNG preview.
"""
from __future__ import annotations
from collections.abc import Sequence
from pathlib import Path
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
# Okabe-Ito colorblind-safe qualitative palette
OKABE_ITO = [
"#0072B2", # blue
"#D55E00", # vermillion
"#009E73", # bluish green
"#CC79A7", # reddish purple
"#E69F00", # orange
"#56B4E9", # sky blue
"#F0E442", # yellow
"#000000", # black
]
SEQUENTIAL_CMAP = "cividis"
# Diverging map centered at ratio = 1 (accuracy heatmaps)
DIVERGING_CMAP = "RdBu_r"
def apply_style() -> None:
"""Install the shared rcParams theme (idempotent)."""
mpl.rcParams.update(
{
"figure.dpi": 150,
"savefig.dpi": 300,
"savefig.bbox": "tight",
"figure.figsize": (6.4, 4.0),
"font.size": 10,
"font.family": "sans-serif",
"axes.titlesize": 11,
"axes.labelsize": 10,
"axes.spines.top": False,
"axes.spines.right": False,
"axes.grid": True,
"grid.alpha": 0.25,
"grid.linewidth": 0.6,
"legend.frameon": False,
"legend.fontsize": 8.5,
"lines.linewidth": 1.8,
"axes.prop_cycle": mpl.cycler(color=OKABE_ITO),
"mathtext.default": "regular",
}
)
def color_for(index: int) -> str:
return OKABE_ITO[index % len(OKABE_ITO)]
def band_plot(
ax: plt.Axes,
x: Sequence[float],
series: np.ndarray,
*,
color: str,
label: str | None = None,
percentiles: tuple[float, float] = (10, 90),
) -> None:
"""Plot the mean of ``series`` (shape ``(n_replicates, len(x))``) with a percentile band."""
x = np.asarray(x, dtype=float)
mean = np.nanmean(series, axis=0)
lo = np.nanpercentile(series, percentiles[0], axis=0)
hi = np.nanpercentile(series, percentiles[1], axis=0)
ax.plot(x, mean, color=color, label=label)
ax.fill_between(x, lo, hi, color=color, alpha=0.18, linewidth=0)
def save(fig: plt.Figure, out_stem: str | Path, provenance: str | None = None) -> list[Path]:
"""Save ``fig`` as a 300-dpi PNG. ``out_stem`` has no suffix. Returns written paths."""
out_stem = Path(out_stem)
out_stem.parent.mkdir(parents=True, exist_ok=True)
if provenance:
# leave headroom so the footnote never collides with the x-axis label
fig.subplots_adjust(bottom=0.16)
fig.text(0.99, 0.005, provenance, fontsize=6, alpha=0.5, va="bottom", ha="right")
p = out_stem.with_suffix(".png")
fig.savefig(p)
plt.close(fig)
return [p]

73
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/reorg.py Normal file
View File

@ -0,0 +1,73 @@
"""Private-chain "deepest-reorg" adversary: how deep a reorganisation an attacker can force.
A coalition holding stake fraction ``alpha`` mines a HIDDEN chain off the current public tip
and releases it to override the public chain — the classic longest-chain private-chain attack,
here run to maximise the *depth* of the override (the number of honest blocks discarded), which
is the cost a reorg imposes, rather than to maximise revenue (that is §6.6's selfish MDP).
Because the density engine reads a window past k-finality where the chain is settled, the race
is a global longest-chain race, so we model it directly (as §6.6 does) rather than in the
per-node loop. Two ingredients tie it to the protocol parameters:
* **Effective adversary share.** Honest blocks orphaned by natural forks (Blend delay) do not
extend the public chain, so they do not count in the race. With honest orphan rate ``o`` (the
engine's ``fork_rate`` at 0 % adversary), the adversary's *effective* share of chain-extending
blocks is ``alpha_eff = alpha / (alpha + (1-alpha)(1-o))`` — deeper honest forking *helps* the
attacker. Keeping the operating point at ``rho < 1`` (low ``o``) is therefore also what keeps
reorgs shallow. Uncle references change what is *counted*, not the longest-chain race, so they
do **not** change reorg depth.
* **Deepest-reorg strategy.** The adversary's lead ``L = adv_len - pub_len`` is a random walk:
``+1`` w.p. ``alpha_eff`` (adversary extends privately), ``-1`` w.p. ``1-alpha_eff`` (honest
extends the public chain). Each excursion above 0 is one attack; the deepest reorg it can force
is the maximum lead reached (release the private chain at its peak, displacing that many public
confirmations). The tail is the Nakamoto/Rosenfeld gambler's-ruin first passage
``P(depth >= d) = (alpha_eff/(1-alpha_eff))**d`` for ``alpha_eff < 1/2``.
"""
from __future__ import annotations
import numpy as np
def alpha_effective(alpha: float, orphan_rate: float) -> float:
"""Adversary share of chain-extending blocks, given honest orphan rate ``o``."""
honest_eff = (1.0 - alpha) * (1.0 - orphan_rate)
denom = alpha + honest_eff
return alpha / denom if denom > 0 else 0.0
def reorg_depth_tail(alpha_eff: float, d: int) -> float:
"""Closed-form ``P(reorg depth >= d)`` (gambler's-ruin). 1.0 for ``alpha_eff >= 1/2``."""
if d <= 0:
return 1.0
if alpha_eff >= 0.5:
return 1.0
if alpha_eff <= 0.0:
return 0.0
return (alpha_eff / (1.0 - alpha_eff)) ** d
def simulate_deepest_reorg(alpha_eff: float, n_events: int,
rng: np.random.Generator) -> np.ndarray:
"""Monte-Carlo the deepest-reorg strategy; return the reorg depth of each attack (excursion).
Walks the lead over ``n_events`` block events. An excursion is one private-chain attempt (from
when the adversary first pulls ahead until the public chain ties it back); its reorg depth is
the maximum lead reached — the public confirmations the adversary displaces by releasing at
the peak. Validates the closed-form tail :func:`reorg_depth_tail`.
"""
steps = rng.random(n_events) < alpha_eff # True = adversary extends
depths = []
lead = 0
peak = 0
for adv in steps:
if adv:
lead += 1
peak = max(peak, lead)
elif lead > 0:
lead -= 1
if lead == 0: # excursion closes: record its deepest reorg
depths.append(peak)
peak = 0
# honest block at lead 0 -> canonical growth, no attack in progress
return np.asarray(depths, dtype=np.int64)

32
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/rng.py Normal file
View File

@ -0,0 +1,32 @@
"""Deterministic, order-independent RNG derivation.
Each ``SimConfig`` maps to an independent root ``SeedSequence`` seeded from a hash of its
identity plus the global root seed. This guarantees a config yields the same random stream
regardless of the order in which parallel workers run it. The engine ``spawn``\\s children
of this root — one per epoch, and independent sub-streams within an epoch — so every draw
(including the optional parallel chunked lottery) is a deterministic function of the root.
"""
from __future__ import annotations
import hashlib
import numpy as np
from .config import SimConfig
def _entropy(config: SimConfig) -> int:
payload = repr((config.root_seed, config.key())).encode()
digest = hashlib.blake2b(payload, digest_size=16).digest()
return int.from_bytes(digest, "big")
def seedseq_for(config: SimConfig) -> np.random.SeedSequence:
"""Return the reproducible root ``SeedSequence`` for this exact config+replicate."""
return np.random.SeedSequence(_entropy(config))
def rng_for(config: SimConfig) -> np.random.Generator:
"""Return the reproducible ``Generator`` for this exact config+replicate."""
return np.random.default_rng(seedseq_for(config))

223
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/selfish.py Normal file
View File

@ -0,0 +1,223 @@
"""Selfish / private-chain block withholding, and its interaction with TSI.
§6.5 modelled withholding as *abstention* — a coalition that discards its own blocks to deflate the
active-stake estimate — and found it strictly unprofitable (the forfeit is a dead loss). The classic
"block withholding" attack is different and stronger: the coalition mines a **private chain** and
**releases** it to orphan honest blocks (Eyal & Sirer, *Majority is not Enough*, 2013), recovering
the forfeit. This module models that adversary and its TSI-specific coupling.
Why a global longest-chain race (not the per-node arrival engine): TSI reads density from a window
buried far past k-finality, where every node provably agrees on the canonical chain (§3.1). The
selfish/honest outcome is decided by that single finalized chain, so the standard block-level race —
with a network tie-break parameter ``gamma`` (fraction of honest miners that build on the
adversary's block in a same-length race) — is the faithful, exact tool for the reward and density.
Two quantities matter:
* **revenue share** ``adv/(adv+hon)`` — the adversary's fraction of *canonical* blocks. Above the
selfish threshold (``alpha > (1-gamma)/(3-2*gamma)``; 1/3 at gamma=0) it exceeds the stake
share ``alpha`` — i.e. private-chain withholding IS profitable, unlike §6.5's abstention.
* **canonical density fraction** ``(adv+hon)/events`` < 1 — orphaned honest blocks go uncounted,
so TSI measures a low density and **deflates ``D̂`` to ``D*``·(density fraction)**. Uncle
references (§6.3–6.4) recover orphaned honest blocks into the count, blunting the deflation.
"""
from __future__ import annotations
from dataclasses import dataclass
import numpy as np
def selfish_revenue_closed_form(alpha: float, gamma: float) -> float:
"""Eyal–Sirer relative revenue of the selfish pool (fraction of canonical blocks).
``R = [a(1-a)^2 (4a + g(1-2a)) - a^3] / [1 - a(1 + (2-a)a)]`` (a=alpha, g=gamma).
Exceeds ``alpha`` above the profitability threshold ``alpha > (1-gamma)/(3-2gamma)``.
"""
a, g = float(alpha), float(gamma)
num = a * (1 - a) ** 2 * (4 * a + g * (1 - 2 * a)) - a ** 3
den = 1 - a * (1 + (2 - a) * a)
return num / den
def selfish_threshold(gamma: float) -> float:
"""Stake fraction above which selfish mining out-earns honest: ``(1-gamma)/(3-2gamma)``."""
return (1 - gamma) / (3 - 2 * gamma)
@dataclass
class RaceResult:
adv: int # adversary blocks on the canonical chain
hon: int # honest blocks on the canonical chain
orphan_hon: int # honest blocks orphaned by adversary overrides (recoverable via uncles)
orphan_adv: int # adversary private blocks that lost a race (dead)
events: int # total block-finding events
@property
def revenue_share(self) -> float:
c = self.adv + self.hon
return self.adv / c if c else 0.0
@property
def density_fraction(self) -> float:
"""Counted canonical blocks / all mined blocks — the factor TSI deflates ``D̂`` by."""
return (self.adv + self.hon) / self.events if self.events else 1.0
def simulate_selfish(is_adv: np.ndarray, gamma: float, rng: np.random.Generator) -> RaceResult:
"""Eyal–Sirer SM1 selfish-mining race over a stream of block-finding events.
``is_adv[i]`` marks whether event ``i`` was found by the adversary pool. Honest miners build on
the public head; the adversary builds a private chain and reveals it per SM1. ``gamma`` is the
tie-race bias. Reward is assigned to the canonical (surviving) branch at each resolution.
State ``k`` = adversary's hidden lead over the public branch; ``tie`` = the 1–1 race state 0'.
"""
k = 0 # adversary private lead
tie = False # in the 0' (1–1 race) state
adv = hon = orphan_hon = orphan_adv = 0
for x in np.asarray(is_adv, dtype=bool):
if tie: # resolving a 1–1 race (adversary revealed 1)
if x: # adversary extends its branch -> it wins
adv += 2
orphan_hon += 1 # the honest matching block is orphaned
elif rng.random() < gamma: # honest builds on adversary block -> adv wins
adv += 1
hon += 1
orphan_hon += 1 # the honest matcher is orphaned by the race
else: # honest builds on honest block -> honest wins
hon += 2
orphan_adv += 1 # the adversary's revealed block is orphaned
tie = False
k = 0
continue
if x: # adversary found a block: extend private chain
k += 1
elif k == 0: # honest found, adversary no lead -> honest wins
hon += 1
elif k == 1: # honest matches a 1-lead -> reveal 1, race 0'
tie = True
k = 0
elif k == 2: # honest chips a 2-lead -> reveal all, override
adv += 2
orphan_hon += 1 # the honest block is orphaned
k = 0
else: # k > 2: reveal one to stay ahead, orphan honest
adv += 1
orphan_hon += 1
k -= 1
# flush: adversary reveals any remaining private lead (longer -> canonical)
if tie:
hon += 1 # unresolved tie: honest keeps its block (H),
orphan_adv += 1 # the adversary's revealed matcher is lost
adv += k
return RaceResult(adv=adv, hon=hon, orphan_hon=orphan_hon, orphan_adv=orphan_adv,
events=int(np.asarray(is_adv).size))
def race_from_alpha(alpha: float, n_events: int, gamma: float,
rng: np.random.Generator) -> RaceResult:
"""Convenience: a Bernoulli(``alpha``) event stream of ``n_events`` blocks through the race."""
is_adv = rng.random(n_events) < alpha
return simulate_selfish(is_adv, gamma, rng)
@dataclass
class RewardParams:
"""A configurable block/uncle reward schedule (all rewards as a fraction of a block reward = 1).
``w_uncle`` — paid to the *producer* of an orphaned block referenced as an uncle (GHOST/Ethereum
style). Compensates a miner whose block was orphaned by latency (§3.2).
``w_nephew`` — paid to the block that *references* an uncle (per uncle). Incentivises inclusion.
The reference game is strategic — who references whom decides who is compensated — so three rate
knobs model it (all fractions in [0,1]):
``p_ref`` — honest orphans that get referenced. Under the SOFT (reward-weighted)
inclusion rule (§6.8) this is EMERGENT and high: an honest orphan is
published, so any honest canonical block within ``W`` references it for the
nephew reward; the attacker only suppresses on its own blocks. Rises toward 1
with ``W``; driven low only by deep reorgs whose orphans age out (residual).
``p_ref_adv`` — the attacker's OWN revealed-but-lost blocks (``orphan_adv``) that it
self-uncles to recover reward. A rational attacker → 1.
``adv_nephew`` — of the honest orphans that are referenced, the fraction whose *nephew* reward
the attacker captures (its canonical block did the referencing — e.g. forced
to under a mandate). 0 without a mandate (honest blocks reference); >0 with.
Safety constraint (§6.7): self-uncling a block pays ``w_uncle + w_nephew``; orphaning a block
you could have made canonical to farm that pays iff ``w_uncle + w_nephew > 1``, so a sound
schedule needs **w_uncle + w_nephew < 1**.
"""
w_uncle: float = 0.0
w_nephew: float = 0.0
p_ref: float = 1.0
p_ref_adv: float = 0.0
adv_nephew: float = 0.0
@classmethod
def mandatory(cls, w_uncle: float, w_nephew: float = 0.0,
adv_region_frac: float = 0.5) -> RewardParams:
"""The ``p_ref = 1`` limit — every in-window uncle referenced. NOT a deployed *hard mandate*
(a validity rule cannot prove what forks a producer saw, §6.8 — rejected for fork-safety);
this is the large-``W`` / full-honest-referencer limit that the SOFT rule *approaches*. The
resulting share is ``≈ α`` with a small residual premium that grows with alpha (~0 near 1/3,
+0.006 at 0.4, +0.014 at 0.46). Two-
type model: no ``W``/``U`` queue, so ``p_ref`` is a knob, not derived — real coverage is
``< 1``, set by ``W``/visibility (see the ``p_ref`` sweep in scripts/reward_mandate.py)."""
return cls(w_uncle=w_uncle, w_nephew=w_nephew, p_ref=1.0, p_ref_adv=1.0,
adv_nephew=adv_region_frac)
@dataclass
class RewardSplit:
adv_reward: float
hon_reward: float
@property
def adv_reward_share(self) -> float:
t = self.adv_reward + self.hon_reward
return self.adv_reward / t if t else 0.0
def reward_shares(race: RaceResult, rp: RewardParams) -> RewardSplit:
"""Reward-weighted split under an uncle-reward schedule (block reward = 1 per canonical block).
Four reference flows (uncle -> producer, nephew -> referencer): honest orphans referenced
(uncle -> honest; nephew -> honest, or -> attacker for the ``adv_nephew`` fraction it must
reference); and the attacker's own lost blocks self-uncled (uncle **and** nephew -> attacker).
``adv_reward_share`` is the attacker's fraction of *total issued reward*.
"""
adv, hon = race.adv, race.hon
if adv + hon == 0:
return RewardSplit(0.0, 0.0)
ref_hon = rp.p_ref * race.orphan_hon # honest orphans compensated (uncle->honest)
ref_adv = rp.p_ref_adv * race.orphan_adv # attacker self-uncles its own lost blocks
hon_reward = (hon + rp.w_uncle * ref_hon # honest uncle producers
+ rp.w_nephew * ref_hon * (1.0 - rp.adv_nephew)) # honest nephews
adv_reward = (adv + rp.w_nephew * ref_hon * rp.adv_nephew # attacker-captured nephews
+ (rp.w_uncle + rp.w_nephew) * ref_adv) # self-uncle: producer + nephew
return RewardSplit(adv_reward=adv_reward, hon_reward=hon_reward)
def honest_reward_recovery(race: RaceResult, rp: RewardParams) -> float:
"""Fraction of honest miners' *mined* value they actually collect (fairness metric).
Without uncle rewards an honest miner loses everything for a latency-orphaned block; with them
it recovers ``w_uncle`` per referenced orphan. Returns
``(hon + w_uncle·p_ref·orphan_hon) / (hon + orphan_hon)`` — 1.0 means fully compensated.
"""
mined = race.hon + race.orphan_hon
if mined == 0:
return 1.0
return (race.hon + rp.w_uncle * rp.p_ref * race.orphan_hon) / mined
def tsi_dhat_ratio(race: RaceResult, uncle_recovery: float) -> float:
"""Equilibrium ``D̂/D*`` under a selfish attack with uncle recovery.
TSI drives the *counted* density to ``f``; the counted density is the canonical fraction plus a
recovered fraction ``uncle_recovery ∈ [0,1]`` of the orphaned HONEST blocks (referenced back as
uncles — adversary blocks reference none, §6.4). ``D̂/D* = (adv + hon + u*orphan_hon)/events``.
"""
u = float(np.clip(uncle_recovery, 0.0, 1.0))
counted = race.adv + race.hon + u * race.orphan_hon
return counted / race.events if race.events else 1.0

153
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/selfish_mdp.py Normal file
View File

@ -0,0 +1,153 @@
"""Optimal selfish-mining strategy via the Sapirshtein–Sompolinsky–Zohar (2016) MDP.
§6.6 measures the *SM1* selfish strategy (Eyal–Sirer). SM1 is not optimal: an adversary can do
better by choosing, in each state, among {adopt, override, match, wait} rather than following the
fixed SM1 rule. SSZ cast this as an MDP over states ``(a, h, fork)`` — adversary secret-chain length
``a``, honest public-chain length ``h`` since the fork, and ``fork ∈ {irrelevant, relevant,
active}`` — and maximise the *relative* revenue ``adv/(adv+hon)``.
Because the objective is a ratio, we use the standard transform: for a candidate value ``rho`` the
per-step reward is ``(1-rho)·adv − rho·hon``; the optimal average reward ``g(rho)`` is decreasing in
``rho``, and the ``rho*`` where ``g(rho*) = 0`` is the optimal relative revenue. We find it by
bisection, solving each inner MDP by relative value iteration.
The optimal revenue is an upper bound on any selfish adversary's take; it lower-bounds the honest
stake threshold above which deviating pays. Used in §6.6 to bracket the real profit frontier.
"""
from __future__ import annotations
import numpy as np
# fork labels
IRRELEVANT, RELEVANT, ACTIVE = 0, 1, 2
# actions
ADOPT, OVERRIDE, MATCH, WAIT = 0, 1, 2, 3
def _build_states(cap: int):
"""Enumerate (a, h, fork) with 0<=a,h<=cap; return index maps."""
states = [(a, h, f) for a in range(cap + 1) for h in range(cap + 1)
for f in (IRRELEVANT, RELEVANT, ACTIVE)]
index = {s: i for i, s in enumerate(states)}
return states, index
def _transitions(a, h, f, action, alpha, gamma, cap):
"""Legal-action transition list: [(prob, (a',h',f'), adv_reward, hon_reward)].
Returns None if the action is illegal in this state. At the cap, only adopt/override remain so
the chain stays bounded (the optimal policy resolves long before the cap in the tested range).
"""
beta = 1.0 - alpha
at_cap = a >= cap or h >= cap
if action == ADOPT:
# abandon the secret chain; the h honest blocks are confirmed, then one block is mined
return [(alpha, (1, 0, IRRELEVANT), 0, h),
(beta, (0, 1, IRRELEVANT), 0, h)]
if action == OVERRIDE:
if a <= h:
return None # need a strictly longer chain to override
# publish h+1 blocks -> they override the public h; a-h-1 stay secret; then one block mined
return [(alpha, (a - h, 0, IRRELEVANT), h + 1, 0),
(beta, (a - h - 1, 1, RELEVANT), h + 1, 0)]
if at_cap:
return None # only adopt/override allowed at the boundary
if action == WAIT:
if f != ACTIVE:
return [(alpha, (a + 1, h, IRRELEVANT), 0, 0),
(beta, (a, h + 1, RELEVANT), 0, 0)]
if a < h:
return None # inconsistent (unreachable) active state: only adopt is valid
# waiting while a fork is active: the same race dynamics as match
return [(alpha, (a + 1, h, ACTIVE), 0, 0),
(gamma * beta, (a - h, 1, RELEVANT), h, 0), # adv's matched branch wins h
((1 - gamma) * beta, (a, h + 1, RELEVANT), 0, 0)]
if action == MATCH:
if not (f == RELEVANT and a >= h):
return None # match needs equal-or-longer chain on a fresh tip
return [(alpha, (a + 1, h, ACTIVE), 0, 0),
(gamma * beta, (a - h, 1, RELEVANT), h, 0),
((1 - gamma) * beta, (a, h + 1, RELEVANT), 0, 0)]
return None
_K = 3 # max successor branches of any (state, action)
def _precompute(alpha, gamma, states, index, cap):
"""Padded transition tensors (A, n, K) for a fully-vectorised value iteration, plus a legality
mask. Independent of rho — built once per (alpha, gamma) and reused across the bisection."""
n = len(states)
probs = np.zeros((4, n, _K))
nxt = np.zeros((4, n, _K), dtype=np.int64)
radv = np.zeros((4, n, _K))
rhon = np.zeros((4, n, _K))
legal = np.zeros((4, n), dtype=bool)
for i, (a, h, f) in enumerate(states):
for action in (ADOPT, OVERRIDE, MATCH, WAIT):
tr = _transitions(a, h, f, action, alpha, gamma, cap)
if tr is None:
continue
legal[action, i] = True
for b, (p, s2, ra, rh) in enumerate(tr):
probs[action, i, b] = p
nxt[action, i, b] = index[s2]
radv[action, i, b] = ra
rhon[action, i, b] = rh
return probs, nxt, radv, rhon, legal
def _solve_mdp(pc, rho, ref, iters, tol):
"""Optimal average gain for the rho-parametrised reward, by *damped* relative value iteration.
The chain is periodic, so undamped VI oscillates and a naive |Δgain| stop can false-trigger as
the gain crosses zero. We damp (``V ← V + τ(TV − V)``) to break periodicity and stop on the
textbook span criterion: at the average-reward fixed point ``TV − V = g·1`` (span → 0), and the
gain ``g`` is that uniform increment. Returns the span-centre of the final Bellman increment.
"""
probs, nxt, radv, rhon, legal = pc
reward = (1.0 - rho) * radv - rho * rhon # (4, n, K), constant across iterations
V = np.zeros(probs.shape[1])
tau = 0.5
d = np.zeros(1)
for _ in range(iters):
q = (probs * (reward + V[nxt])).sum(axis=2) # (4, n)
q[~legal] = -1e18
d = q.max(axis=0) - V # Bellman increment TV − V
if d.max() - d.min() < tol:
break
V = V + tau * d
V -= V[ref] # anchor to keep values bounded
return 0.5 * (d.max() + d.min())
def optimal_selfish_revenue(alpha: float, gamma: float, cap: int = 60,
iters: int = 4000, tol: float = 1e-10) -> float:
"""Optimal relative revenue adv/(adv+hon) for a selfish miner with stake ``alpha``, tie-break
``gamma``. Bisection on ``rho`` (the objective value), inner MDP by relative value iteration.
Always ``>= max(alpha, SM1)``; equals ``alpha`` below the profitability threshold. ``cap``
bounds the tracked lead and ``iters`` the value-iteration budget — both must grow *together*
near ``alpha → 0.5`` (long leads). The defaults are converged to <1e-3 for ``alpha <= 0.46``;
for ``alpha >= 0.47`` raise both (e.g. ``cap=80, iters=6000``) or the optimum is over-estimated
by ~0.01 (cap-truncation + under-iteration). ``cap=16`` suffices for ``alpha <= 0.4``.
"""
states, index = _build_states(cap)
pc = _precompute(alpha, gamma, states, index, cap)
ref = index[(1, 0, IRRELEVANT)]
lo, hi = alpha - 1e-9, 1.0 # relative revenue in [alpha, 1)
for _ in range(44):
mid = 0.5 * (lo + hi)
g = _solve_mdp(pc, mid, ref, iters, tol)
if g > 0: # policy still profits at this rho -> true rho is higher
lo = mid
else:
hi = mid
return 0.5 * (lo + hi)

32
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/stake.py Normal file
View File

@ -0,0 +1,32 @@
"""Stake distribution generation.
Total stake is held FIXED across distributions (via renormalisation) so that accuracy
comparisons between ``uniform`` and ``pareto`` isolate the *shape* effect on the lottery
(winner multiplicity / forking), not a difference in aggregate stake.
"""
from __future__ import annotations
import numpy as np
from .config import SimConfig
def make_stake(config: SimConfig, rng: np.random.Generator) -> np.ndarray:
"""Return an ``(n_nodes,)`` non-negative stake vector summing to ``total_stake``."""
n = config.n_nodes
if config.stake_dist == "uniform":
if config.uniform_random:
w = rng.random(n)
else:
w = np.ones(n)
elif config.stake_dist == "pareto":
# numpy.pareto draws Lomax = Pareto(shape) - 1, heavy-tailed for small shape.
w = rng.pareto(config.pareto_shape, n) + 1.0
else: # pragma: no cover - guarded by Literal typing
raise ValueError(f"unknown stake_dist: {config.stake_dist}")
total = w.sum()
if total <= 0: # pragma: no cover - degenerate
raise ValueError("stake vector summed to zero")
return w * (config.total_stake / total)

341
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/sweep.py Normal file
View File

@ -0,0 +1,341 @@
"""Parameter-sweep expansion, parallel execution, and result persistence.
Across-config parallelism is the main multicore lever: ``run_trajectory`` is a pure
function of an immutable, hash-seeded ``SimConfig``, so results are bitwise
order-independent and the grid is embarrassingly parallel. We use joblib's process-based
**loky** backend (this workload is CPU-bound Python that holds the GIL, so threads would
serialise), and pin each worker to a single BLAS thread to avoid oversubscription.
"""
from __future__ import annotations
import argparse
import datetime as _dt
import multiprocessing as mp
import os
import queue
import sys
import time
from dataclasses import dataclass, replace
from pathlib import Path
import numpy as np
import pandas as pd
import yaml
from joblib import Parallel, delayed
from tqdm import tqdm
from .config import SimConfig, SweepConfig
from .engine import run_trajectory
from .memguard import DEFAULT_BUDGET_FRAC, ArrivalMatrixTooLarge
from .memguard import total_ram_bytes as _total_ram_bytes
# ``calibrate="auto"`` runs a real memory probe when the analytic estimate is either extrapolated
# past the validated network size (N > CALIBRATION_N_THRESHOLD) OR simply large in absolute terms
# (per-worker estimate > CALIBRATION_BYTES_THRESHOLD) — the latter catches a block-count explosion
# from a low genesis_d_factor even at small N, which is exactly what OOM-froze the box at N=1000.
CALIBRATION_N_THRESHOLD = 2000
CALIBRATION_BYTES_THRESHOLD = 1.5 * 1024**3
# Keep numpy/BLAS single-threaded inside each worker process (belt-and-braces alongside
# joblib's inner_max_num_threads); prevents N_workers x N_blas_threads oversubscription.
for _var in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS"):
os.environ.setdefault(_var, "1")
def expected_peak_blocks(config: SimConfig) -> int:
"""Expected block count of the *heaviest* epoch (genesis), which sizes the arrival matrix.
``n_blocks`` is NOT ``~10*k``: it is the number of lottery wins over the epoch, which scales
with ``sum_i phi(w_i / D_est)``. At genesis ``D_est = genesis_d_factor * D_true`` is at its
smallest, so ``sum(stake)/D_est = 1/genesis_d_factor`` is largest and block production peaks
there (a low ``genesis_d_factor`` can inflate it 100x — the collapsed-estimate regime). We
realise the seeded stake and compute the expected genesis-epoch win count exactly; this is
what previously blew the estimate up by ~90x and OOM-ed the box. Capped at ``N*E`` (every
node can win every slot at most once).
"""
from .lottery import win_probs
from .rng import seedseq_for
from .stake import make_stake
child0 = seedseq_for(config).spawn(config.epochs + 3)[0]
stake = make_stake(config, np.random.default_rng(child0))
d_est_genesis = config.genesis_d_factor * float(stake.sum())
p = win_probs(stake, d_est_genesis, config.f)
expected = float(config.epoch_len) * float(p.sum())
return int(min(expected, float(config.n_nodes) * config.epoch_len)) + 1
def _arrival_columns(config: SimConfig, peak_blocks: int) -> int:
"""Estimated stored arrival columns: full ``n_blocks``, or the pruned keep-span window.
With ``prune_arrival`` the arrival buffer keeps only blocks inside ``max(horizon, W)``
slots, so its width is ``~keepspan * blocks_per_slot`` regardless of how far the block count
exploded. We don't have the graph here, so ``horizon`` is approximated generously (dominated by
``uncle_window``, plus the blend mix cascade); the exact buffer size is guarded in-worker.
"""
if not (config.prune_arrival and config.windowed_fork_choice):
return peak_blocks
per_slot = peak_blocks / config.epoch_len if config.epoch_len else peak_blocks
keepspan = float(config.uncle_window)
if config.topology == "blend":
lat = max(config.link_latency_mean, 0.1)
keepspan = max(keepspan, (config.blend_hops + 1) * lat * 4
+ config.blend_hops * config.blend_delay_max)
buf_width = 2.0 * (keepspan * per_slot * 1.5 + 64) # ~ _build_pruned's 2*(cap + slack)
return int(min(peak_blocks, buf_width)) + 1
def estimate_worker_bytes(config: SimConfig) -> int:
"""Rough peak RSS of one worker running ``config``.
The arrival buffer and the dense ``path_latency`` (``N x N`` float64, plus Dijkstra scratch)
dominate; everything else (block-tree 1-D arrays, measurement temp) is smaller. The arrival
term is ``N * columns * 8`` where ``columns`` is the full peak-epoch block count
(``expected_peak_blocks``, capturing a low-``genesis_d_factor`` explosion) or, with
``prune_arrival``, just the bounded keep-span window (``_arrival_columns``). Factors are
generous so the cap errs toward fewer, safe workers.
"""
n = config.n_nodes
cols = _arrival_columns(config, expected_peak_blocks(config))
arrival = 1.35 * n * cols * 8 # buffer + measurement/fork-choice temps
path_latency = 2.2 * n * n * 8 # dense (N,N) + Dijkstra predecessor/scratch
fixed = 250 * 1024**2 # python + numpy + numba loaded per process
return int(arrival + path_latency + fixed)
def _ru_maxrss_bytes(ru_maxrss: int) -> int:
"""Normalise ``getrusage`` peak RSS to bytes (macOS reports bytes, Linux kibibytes)."""
return int(ru_maxrss) if sys.platform == "darwin" else int(ru_maxrss) * 1024
def _calibration_target(config: SimConfig, q: mp.Queue) -> None: # pragma: no cover - subprocess
"""Child entry point: run ONE epoch of ``config`` and report this process's peak RSS.
One epoch reaches the same peak as a full trajectory — the arrival matrix ``A`` and the
``path_latency`` matrix are (re)built every epoch and the measurement temporary peaks within
an epoch — so a single epoch is a faithful, cheap probe of a worker's high-water mark.
The probe bounds ITSELF to a fraction of physical RAM (overriding any inherited budget): it
must be free to allocate the real peak in order to measure it, but must still fail loud rather
than freeze if the heaviest config exceeds the box — in which case the parent falls back to the
(large) analytic estimate.
"""
import resource
from .engine import run_trajectory
from .memguard import total_ram_bytes
os.environ["TSI_ARRIVAL_BYTES_BUDGET"] = str(int(DEFAULT_BUDGET_FRAC * total_ram_bytes()))
run_trajectory(replace(config, epochs=1))
q.put(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
def measure_worker_bytes(config: SimConfig, timeout: float = 900.0) -> int | None:
"""Peak RSS (bytes) of a real worker running ``config``, measured in a fresh spawned
process — the same isolation loky gives each worker, so numba/numpy/scratch are all counted.
Returns ``None`` if the probe cannot start, crashes, or exceeds ``timeout`` (the caller then
falls back to the analytic estimate). ``spawn`` matches loky and keeps the probe independent
of the parent's already-imported modules; it needs an importable ``__main__`` (a real script
or ``-m`` entry point), so from a bare REPL/stdin the child fails to bootstrap and we return
``None`` promptly by polling liveness rather than blocking the full timeout.
"""
ctx = mp.get_context("spawn")
q: mp.Queue = ctx.Queue()
proc = ctx.Process(target=_calibration_target, args=(config, q), daemon=True)
proc.start()
deadline = time.monotonic() + timeout
try:
while True:
try:
ru_maxrss = q.get(timeout=1.0)
return _ru_maxrss_bytes(ru_maxrss)
except queue.Empty:
if not proc.is_alive(): # child died (bootstrap failure / crash / OOM-kill)
return None
if time.monotonic() >= deadline:
return None
finally:
if proc.is_alive():
proc.terminate()
proc.join(timeout=5.0)
@dataclass
class WorkerPlan:
"""The chosen worker count plus the numbers behind it (for logging)."""
n_jobs: int
per_worker_bytes: int
calibrated: bool
ram_bytes: int
mem_frac: float
def plan_workers(requested: int, configs: list[SimConfig], mem_frac: float,
calibrate: str = "auto") -> WorkerPlan:
"""Size the worker pool so ``n_jobs * per-worker peak`` fits ``mem_frac`` of physical RAM.
Per-worker peak is the analytic ``estimate_worker_bytes`` by default, but is replaced by a
**measured** peak RSS (one epoch of the heaviest config, in a spawned process) when the
calibration probe fires:
* ``calibrate="auto"`` (default) — probe when the estimate is extrapolated past the validated
network size (N > ``CALIBRATION_N_THRESHOLD``) OR is large in absolute terms
(> ``CALIBRATION_BYTES_THRESHOLD``); the latter catches a low-``genesis_d_factor`` block
explosion even at small N;
* ``calibrate="always"`` — always probe;
* ``calibrate="never"`` — never probe (analytic estimate only).
``requested`` follows joblib's convention (``-1`` = all logical cores); ``mem_frac <= 0``
disables the cap entirely.
"""
cores = os.cpu_count() or 1
want = cores if requested < 0 else max(1, requested)
ram = _total_ram_bytes()
if mem_frac <= 0.0 or not configs:
return WorkerPlan(want, 0, False, ram, mem_frac)
heaviest = max(configs, key=estimate_worker_bytes)
per_worker = estimate_worker_bytes(heaviest)
calibrated = False
do_probe = calibrate == "always" or (calibrate == "auto" and (
heaviest.n_nodes > CALIBRATION_N_THRESHOLD
or per_worker > CALIBRATION_BYTES_THRESHOLD))
if do_probe:
measured = measure_worker_bytes(heaviest)
if measured is not None:
per_worker = int(measured * 1.1) # 10% headroom over the measured high-water mark
calibrated = True
budget = int(mem_frac * ram)
fit = max(1, budget // per_worker)
return WorkerPlan(min(want, fit), per_worker, calibrated, ram, mem_frac)
def run_sweep(
sweep: SweepConfig, n_jobs: int = -1, progress: bool = True, batch_size: str | int = "auto",
mem_frac: float = 0.7, calibrate: str = "auto",
) -> pd.DataFrame:
"""Expand the grid, run every config across cores (loky), return one big frame.
``n_jobs=-1`` uses all logical cores, but the count is then **capped** so that
``workers * per-worker peak RSS`` stays under ``mem_frac`` of physical RAM (``mem_frac=0``
uncaps concurrency but still keeps the per-process fail-loud guard). This prevents the
``(N x n_blocks)`` arrival matrix from OOM-ing the box when many heavy configs run at once —
including the collapsed-``D_est`` regime where a low ``genesis_d_factor`` explodes ``n_blocks``
(see ``expected_peak_blocks``). Per-worker peak is measured by a **calibration probe** when
heavy (see ``plan_workers`` / ``calibrate``), and each worker additionally enforces an
in-process budget (``TSI_ARRIVAL_BYTES_BUDGET`` on both ``A`` and ``path_latency``) so a
mis-estimated config fails loud instead of freezing the box. ``batch_size`` defaults to joblib
"auto"; pass ``1`` for a small grid of heavy full-scale configs.
"""
configs = sweep.expand()
plan = plan_workers(n_jobs, configs, mem_frac, calibrate)
# Per-worker byte budget for the in-worker guard: each worker's share of the RAM budget.
# Enforced in build_tree_pernode (A) and build_path_latency (N^2) -> ArrivalMatrixTooLarge.
# "0" is NOT "unlimited": memguard resolves it to DEFAULT_BUDGET_FRAC of RAM, so even a
# `mem_frac=0` run keeps an absolute per-process ceiling (no single alloc can freeze the box).
worker_budget = (int(mem_frac * plan.ram_bytes // plan.n_jobs)
if (mem_frac > 0 and plan.n_jobs) else 0)
os.environ["TSI_ARRIVAL_BYTES_BUDGET"] = str(worker_budget)
if progress:
src = "measured" if plan.calibrated else "estimated"
gb = plan.per_worker_bytes / 1024**3
print(f"[sweep] {len(configs)} configs; ~{gb:.2f} GB/worker ({src}); "
f"using {plan.n_jobs} worker(s) of {os.cpu_count()} cores "
f"(RAM {plan.ram_bytes / 1024**3:.0f} GB x cap {mem_frac:g})")
if worker_budget and plan.per_worker_bytes > worker_budget:
print(f"[sweep] WARNING: a single config's estimated peak "
f"({gb:.1f} GB) exceeds its per-worker RAM share "
f"({worker_budget / 1024**3:.1f} GB); it may swap or hit the arrival-matrix "
f"guard. Raise genesis_d_factor, lower n_nodes/k/epochs, or raise --mem-frac.")
runner = Parallel(
n_jobs=plan.n_jobs,
backend="loky",
inner_max_num_threads=1,
batch_size=batch_size,
return_as="generator",
)(delayed(run_trajectory)(c) for c in configs)
if progress:
runner = tqdm(runner, total=len(configs), desc="configs")
rows: list[dict] = []
try:
for traj in runner:
rows.extend(traj)
except ArrivalMatrixTooLarge as exc:
raise RuntimeError(
f"aborting sweep: {exc} (worker memory guard tripped — the per-worker estimate was "
f"too low, likely an under-modelled block explosion)."
) from exc
return pd.DataFrame(rows)
def persist(df: pd.DataFrame, path: str | Path) -> None:
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
if path.suffix == ".parquet":
df.to_parquet(path, index=False)
else:
df.to_csv(path, index=False)
def load_sweep_yaml(path: str | Path) -> SweepConfig:
with open(path) as fh:
data = yaml.safe_load(fh)
return SweepConfig.from_dict(data)
def new_run_dir(outdir: str | Path, label: str) -> Path:
"""Create a fresh timestamped run directory so runs never overwrite each other."""
ts = _dt.datetime.now().strftime("%Y-%m-%d_%H%M%S")
base = Path(outdir) / f"{ts}_{label}"
run_dir, i = base, 2
while run_dir.exists(): # avoid same-second collisions
run_dir = base.with_name(f"{base.name}_{i}")
i += 1
run_dir.mkdir(parents=True)
return run_dir
def main(argv: list[str] | None = None) -> None:
parser = argparse.ArgumentParser(description="Run a TSI parameter sweep + figures")
parser.add_argument("--config", required=True, help="sweep YAML")
parser.add_argument("--outdir", default="runs",
help="parent dir; a dated sub-folder is created per run")
parser.add_argument("--label", default=None, help="run label (default: config name)")
parser.add_argument("--n-jobs", type=int, default=-1, help="-1 = all logical cores")
parser.add_argument("--batch-size", default="auto",
help="'auto' (default) or an int; use 1 for a small heavy grid")
parser.add_argument("--mem-frac", type=float, default=0.7,
help="cap concurrent workers to this fraction of physical RAM "
"(0 disables the cap)")
parser.add_argument("--calibrate", choices=["auto", "always", "never"], default="auto",
help="measure a real worker's peak RSS to size the pool: 'auto' "
"(default) probes when N>2000, 'always', or 'never' (estimate only)")
parser.add_argument("--no-figures", action="store_true",
help="skip auto figure generation")
args = parser.parse_args(argv)
batch_size = int(args.batch_size) if args.batch_size != "auto" else "auto"
label = args.label or Path(args.config).stem
run_dir = new_run_dir(args.outdir, label)
sweep = load_sweep_yaml(args.config)
df = run_sweep(sweep, n_jobs=args.n_jobs, batch_size=batch_size, mem_frac=args.mem_frac,
calibrate=args.calibrate)
results_path = run_dir / "results.parquet"
persist(df, results_path)
key_cols = ["n_nodes", "stake_dist", "topology", "degree", "link_latency_mean",
"latency", "max_uncles", "uncle_strategy", "init_dest", "replicate"]
n_cfg = len(df[key_cols].drop_duplicates())
print(f"wrote {len(df)} rows ({n_cfg} configs) -> {results_path}")
if not args.no_figures:
from .plotting.make_figures import render
n_fig = render(df, run_dir / "figures")
print(f"wrote {n_fig} figures -> {run_dir / 'figures'}")
print(f"run dir: {run_dir}")
if __name__ == "__main__": # pragma: no cover
main()

60
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/theory.py Normal file
View File

@ -0,0 +1,60 @@
"""Closed-form TSI results from ``analysis-total-stake-inference.md``.
Used both as figure overlays and as ground truth for the verification checks. ``q`` is
the honest active-slot utilisation; with uncle references, substitute the effective
``q_eff`` to predict the improved accuracy.
"""
from __future__ import annotations
import numpy as np
ArrayLike = np.ndarray | float
def expected_ratio(f: float, q: ArrayLike) -> ArrayLike:
"""Equilibrium ``E[D_inf] / D_true = log(1-f) / log(1 - f/q)`` for ``q in (f, 1]``."""
q = np.asarray(q, dtype=float)
return np.log(1.0 - f) / np.log(1.0 - f / q)
def block_count_ceiling(f: float) -> float:
"""Equilibrium ratio at *full* uncle recovery for the equal-stake limit.
TSI counts blocks (all lottery wins, rate ``-ln(1-f)`` per slot in the small-stake
limit), whereas ``f`` is the *active-slot* rate. So even with every orphan recovered the
estimate equilibrates at ``-ln(1-f)/f`` (~1.017 for f=1/30), not 1.0. This is a
deterministic overshoot floor, not noise; concentrated (Pareto) stake gives a smaller
value because of the concavity of ``phi``.
"""
return -np.log(1.0 - f) / f
def fixed_point_bias(f: float, precision: int = 1000) -> float:
"""Extra multiplicative bias from the spec's integer f-truncation ``int(f*P)/P``."""
f_p = int(f * precision) / precision
return f / f_p
def variance_ratio(f: float, q: ArrayLike, T: int, beta: float = 1.0) -> ArrayLike:
"""Equilibrium ``Var[D_inf / D_true]``."""
q = np.asarray(q, dtype=float)
er = expected_ratio(f, q)
return (beta / f) ** 2 * (q / T) * er**2 * (1.0 - f) * f
def variance_bound(f: float, T: int, beta: float = 1.0) -> float:
"""Upper bound on ``Var[D_inf / D_true]`` at ``q = 1`` (perfect network)."""
return (beta / f) ** 2 / T * (1.0 - f) * f
def beta_stability_bound(f: float, q: ArrayLike) -> ArrayLike:
"""Stability threshold: convergence requires ``beta < 2f/((q-f) log(1/(1-f/q)))``."""
q = np.asarray(q, dtype=float)
return 2.0 * f / ((q - f) * np.log(1.0 / (1.0 - f / q)))
def optimal_beta(f: float, q: ArrayLike) -> ArrayLike:
"""Convergence-optimal learning rate ``f/((q-f) log(1/(1-f/q)))``."""
q = np.asarray(q, dtype=float)
return f / ((q - f) * np.log(1.0 / (1.0 - f / q)))

189
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/topology.py Normal file
View File

@ -0,0 +1,189 @@
"""Network topology and per-node message-propagation latency.
A block produced by node ``p`` at slot ``t`` becomes usable at node ``j`` after the
shortest **weighted** path latency from ``p`` to ``j`` over the peering graph (gossip
flooding = fastest path). ``path_latency[p, j]`` (in slots) is precomputed once per
trajectory and is invariant across epochs and the stake estimate. Its *shape* is determined by
``(N, topology, degree, link-latency model)``, but the concrete random draw is seeded from the
config's full-key spawn hierarchy (see ``engine.run_trajectory``), so it also re-rolls with any
other ``config.key()`` field (``graph_seed`` is one contributor, not a standalone invariance knob).
Topologies:
- ``full_mesh``: every node one hop away with uniform latency ``L`` — reproduces the
reduced model's ``FixedSlotLatency`` and is the validation baseline.
- ``regular``: a random d-regular peering graph (configurable ``degree``) with per-link
latency drawn from ``link_latency_dist``; distant-in-network nodes see a producer's
blocks later, which is the sole source of per-node view divergence.
- ``blend``: the **same** d-regular graph, but a block is first relayed through
``blend_hops`` random nodes (a mix cascade — each adds a ``Uniform(0, blend_delay_max)``
mixing delay) before a final network-wide gossip makes it visible. Models routing over the
Blend mixnet, where the dominant delay is the per-hop mixing, not the graph transport. The
``path_latency`` matrix is identical to ``regular``; only the arrival law differs
(``arrival_column``).
"""
from __future__ import annotations
import numpy as np
from scipy.sparse import csr_matrix
from scipy.sparse.csgraph import dijkstra
from . import constants
from .config import SimConfig
from .memguard import check_alloc
def _circulant_edges(n: int, degree: int) -> set[tuple[int, int]]:
"""A valid d-regular base graph (ring lattice); randomised later by edge swaps."""
edges: set[tuple[int, int]] = set()
half = degree // 2
for i in range(n):
for off in range(1, half + 1):
j = (i + off) % n
edges.add((min(i, j), max(i, j)))
if degree % 2 == 1: # odd degree needs n even: add the antipodal matching
for i in range(n // 2):
j = i + n // 2
edges.add((min(i, j), max(i, j)))
return edges
def _double_edge_swaps(edges: set[tuple[int, int]], n_swaps: int,
rng: np.random.Generator) -> set[tuple[int, int]]:
"""Randomise a graph while preserving every node's degree (Maslov–Sneppen swaps)."""
elist = list(edges)
eset = set(edges)
for _ in range(n_swaps):
i, j = rng.integers(0, len(elist), size=2)
if i == j:
continue
a, b = elist[i]
c, d = elist[j]
if rng.random() < 0.5:
c, d = d, c
if len({a, b, c, d}) < 4:
continue
e1 = (min(a, c), max(a, c))
e2 = (min(b, d), max(b, d))
if e1 in eset or e2 in eset:
continue
eset.discard(elist[i]) # discard the stored (min-sorted) edges, not the
eset.discard(elist[j]) # possibly-reoriented (c, d)
eset.add(e1)
eset.add(e2)
elist[i] = e1
elist[j] = e2
return eset
def _sample_link_latencies(n_edges: int, config: SimConfig,
rng: np.random.Generator) -> np.ndarray:
"""Per-link one-way latency (slots) for every peering edge.
All four modes have expected value ``link_latency_mean`` so it stays the single mean-latency
control knob. ``geo`` reproduces the real-world geographic spread (short intra-region links,
long inter-continental ones) by drawing each link from ``constants.GEO_LATENCY_BANDS_SLOTS``,
then rescaling the fixed band shape so its mean matches ``link_latency_mean``.
"""
mean = config.link_latency_mean
if config.link_latency_dist == "fixed":
return np.full(n_edges, mean, dtype=float)
if config.link_latency_dist == "uniform":
return rng.uniform(0.0, 2.0 * mean, size=n_edges)
if config.link_latency_dist == "exp":
return rng.exponential(mean, size=n_edges)
if config.link_latency_dist == "geo":
bands = np.asarray(constants.GEO_LATENCY_BANDS_SLOTS, dtype=float)
weights = np.asarray(constants.GEO_LATENCY_WEIGHTS, dtype=float)
idx = rng.choice(bands.shape[0], size=n_edges, p=weights)
scale = mean / constants.GEO_LATENCY_MEAN_SLOTS # rescale so E[latency] == mean
return bands[idx] * scale
raise ValueError(config.link_latency_dist) # pragma: no cover
def build_path_latency(config: SimConfig, rng: np.random.Generator) -> np.ndarray:
"""Return ``path_latency[N, N]`` in **slots** (float, sub-slot capable); 0 on the diagonal.
Latency is measured in slots and a slot is 1 second (``constants.SLOT_SECONDS``), so
real-world inter-node latencies (tens to hundreds of milliseconds) are *fractions* of a
slot. We keep the value as a float rather than rounding to whole slots so that realistic
sub-second latencies are not discarded — a block whose fastest path is 0.3 slots (300 ms)
is delivered mid-slot, not "0 slots" or "1 slot".
"""
n = config.n_nodes
# Guard BEFORE allocating: path_latency is a dense (N x N) float64, built here at the start of
# each trajectory — BEFORE the arrival-matrix guard runs — and grows as N^2 independently of
# n_blocks. The 2.2x covers the Dijkstra predecessor/scratch + finite-mask temporaries.
check_alloc(int(2.2 * n * n * 8), f"path_latency (N={n} x N x 8B, +Dijkstra scratch)",
f"N={n} makes the dense (N x N) latency matrix ~{n * n * 8 / 1024**3:.1f} GB. "
f"Lower n_nodes or raise --mem-frac.")
if config.topology == "full_mesh":
pl = np.full((n, n), float(config.latency), dtype=np.float64)
np.fill_diagonal(pl, 0.0)
return pl
# "regular" and "blend" share the same weighted d-regular graph; blend adds a mix cascade
# on top of it in arrival_column, but the transport-latency matrix is identical.
edges = _circulant_edges(n, config.degree)
edges = _double_edge_swaps(edges, n_swaps=10 * len(edges), rng=rng)
e = np.array(sorted(edges), dtype=np.int64)
w = _sample_link_latencies(e.shape[0], config, rng)
rows = np.concatenate([e[:, 0], e[:, 1]])
cols = np.concatenate([e[:, 1], e[:, 0]])
data = np.concatenate([w, w])
adj = csr_matrix((data, (rows, cols)), shape=(n, n))
dist = dijkstra(adj, directed=False) # (N, N) float slots, inf if disconnected
dist[~np.isfinite(dist)] = float(config.epoch_len) # unreachable -> never arrives
np.fill_diagonal(dist, 0.0)
return dist
def _blend_arrival_column(
path_latency: np.ndarray, producer: int, slot: int, config: SimConfig,
rng: np.random.Generator,
) -> np.ndarray:
"""Sub-slot arrival of a block via the Blend mix cascade (``topology == "blend"``).
The producer picks ``blend_hops`` DISTINCT relay nodes uniformly at random. The block hops
``producer -> r1 -> ... -> r_hops`` over the shortest weighted path, and each relay waits a
``Uniform(0, blend_delay_max)`` mixing delay before forwarding. The last relay's forward is
the final network-wide gossip that makes the block visible. Relays are blind forwarders, so
every node — relays included — first learns the block from that final gossip; only the
producer knows it earlier (handled by the caller).
"""
n = path_latency.shape[0]
hops = int(config.blend_hops)
delay_max = float(config.blend_delay_max)
pool = np.delete(np.arange(n), producer) # distinct relays, never the producer
relays = rng.choice(pool, size=hops, replace=False)
t = float(slot)
src = int(producer)
for r in relays.tolist(): # transport leg, then this relay's mix delay
t += float(path_latency[src, r])
t += float(rng.uniform(0.0, delay_max))
src = r
last = int(relays[-1])
return t + path_latency[last].astype(np.float64) # final gossip floods from the last relay
def arrival_column(
path_latency: np.ndarray, producer: int, slot: int, config: SimConfig,
rng: np.random.Generator,
) -> np.ndarray:
"""Sub-slot arrival time of a block at every node (before the parent clamp)."""
if config.topology == "blend":
col = _blend_arrival_column(path_latency, producer, slot, config, rng)
else:
col = float(slot) + path_latency[producer].astype(np.float64)
if config.jitter_mean > 0.0:
if config.jitter_dist == "poisson":
# Long-tail model: a `jitter_frac` fraction of deliveries straggle by a
# Poisson(jitter_mean) whole-slot delay; the rest arrive on time.
extra = rng.poisson(config.jitter_mean, size=col.shape).astype(np.float64)
if config.jitter_frac < 1.0:
extra = extra * (rng.random(col.shape) < config.jitter_frac)
col = col + extra
else:
col = col + rng.exponential(config.jitter_mean, size=col.shape)
col[producer] = float(slot) # producer sees own block instantly
return col

137
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/tsi.py Normal file
View File

@ -0,0 +1,137 @@
"""Total Stake Inference: density counting and the per-epoch estimate update.
The estimate counts *slots*, preserving the pre-uncle design invariant "one count per
slot" (the slot lottery is calibrated so slots activate at rate ``f``; multiple winners
of one slot must not inflate the count): ``m = canonical-occupied slots in window +
distinct referenced-uncle slots in window that are NOT already canonical-occupied``.
``legacy_block_count=True`` reproduces the earlier (buggy) block-id counting, which
double-counted same-slot co-winners and inflated the equilibrium by c(f); kept only as a
flag, not used by any study.
"""
from __future__ import annotations
from dataclasses import dataclass
import numpy as np
from .blocktree import BlockTree
def referenced_uncle_ids(tree: BlockTree, canonical_ids: list[int]) -> set[int]:
"""Deduplicated set of uncle ids referenced by the canonical chain."""
ref: set[int] = set()
for b in canonical_ids:
ref.update(tree.uncles[b])
return ref
def _in_window(slot: int, T: int) -> bool:
return 0 <= slot < T
def density_m(tree: BlockTree, canonical_ids: list[int], T: int,
legacy_block_count: bool = False) -> int:
"""Slot count ``m`` for the TSI update: canonical slots + recovered uncle slots.
A slot counts at most once: canonical blocks occupy distinct slots by construction, and
a referenced uncle contributes only if its slot is not already canonical-occupied (and
only once per slot, however many same-slot uncles are referenced). ``legacy_block_count``
reproduces the earlier per-block-id counting (double-counts multi-winner slots).
"""
s = tree.slot[canonical_ids]
in_win = (s >= 0) & (s < T)
honest = int(in_win.sum())
ref = referenced_uncle_ids(tree, canonical_ids)
if legacy_block_count:
return honest + sum(1 for u in ref if _in_window(int(tree.slot[u]), T))
canon_slots = set(int(x) for x in s[in_win])
rec_slots = {int(tree.slot[u]) for u in ref
if _in_window(int(tree.slot[u]), T) and int(tree.slot[u]) not in canon_slots}
return honest + len(rec_slots)
# On-chain fixed-point scale for the target rate f. The original spec used 1000 (three decimals),
# which rounds f=1/30 to f_p=0.033 and leaves a ~1% (f/f_p) over-estimate — the sole residual bias
# after the slot-counting fix (Appendix A). Raised to 1_000_000 (six decimals) per the report's
# recommendation: f_p=0.033333, so f/f_p=1.00000, and the residual drops below 10^-5 (negligible).
PRECISION = 1_000_000
def _f_eff(f: float, fixed_point: bool) -> float:
"""Target rate used in the recursion: exact ``f``, or the spec's integer quantisation.
Guards the quantised path against ``f`` so small that ``int(f*PRECISION) == 0`` (e.g. f < .001),
which would make ``f_eff = 0`` and divide-by-zero in the recursion.
"""
if not fixed_point:
return f
q = int(f * PRECISION)
if q == 0:
raise ValueError(
f"fixed_point=True with f={f} quantises the target rate to 0 "
f"(int(f*{PRECISION})==0); use f >= 1/{PRECISION} or fixed_point=False")
return q / PRECISION
def update_D(
d_prev: float, m: int, T: int, f: float, beta: float, fixed_point: bool = False
) -> float:
"""Spec TSI recursion: ``max(1, D_prev * (1 - beta*(f_eff - m/T)/f_eff))``.
With ``fixed_point=True`` the target rate ``f`` is quantised as the on-chain algorithm does
(``f_p = int(f*PRECISION)/PRECISION``). At the raised ``PRECISION = 10**6`` this is
``f_p = 0.033333`` for f=1/30, so ``f/f_p = 1.00001`` and the residual over-estimate is
below 10^-5 (negligible) — the report's f-precision recommendation, applied. (At the
original spec ``PRECISION = 1000`` the offset was ~1%.)
"""
f_eff = _f_eff(f, fixed_point)
measured_density = m / T
d_new = d_prev * (1.0 - beta * (f_eff - measured_density) / f_eff)
return max(d_new, 1.0)
def update_D_vec(
d_prev: np.ndarray, m: np.ndarray, T: int, f: float, beta: float, fixed_point: bool = False,
) -> np.ndarray:
"""Per-node TSI recursion: :func:`update_D` applied elementwise over ``(N,)`` arrays.
Each node updates its OWN estimate ``d_prev[i]`` from its OWN measured slot count
``m[i]``. Identical formula to :func:`update_D`, clamped at 1.
"""
f_eff = _f_eff(f, fixed_point)
measured_density = np.asarray(m, dtype=float) / T
d_new = np.asarray(d_prev, dtype=float) * (1.0 - beta * (f_eff - measured_density) / f_eff)
return np.maximum(d_new, 1.0)
@dataclass
class SlotStats:
n_active: int # active slots (>=1 winner) in window
n_honest: int # honest-chain-occupied slots in window
n_recovered: int # orphan-only slots recovered via referenced uncles
q: float # n_honest / n_active
q_eff: float # (n_honest + n_recovered) / n_active
def slot_stats(
tree: BlockTree,
canonical_ids: list[int],
ref_uncle_ids: set[int],
active_slots: np.ndarray,
T: int,
) -> SlotStats:
"""Slot-based utilisation stats used for theory overlays."""
active_in = active_slots[(active_slots >= 0) & (active_slots < T)]
n_active = int(active_in.size)
honest_slots = {int(tree.slot[b]) for b in canonical_ids if _in_window(int(tree.slot[b]), T)}
recovered: set[int] = set()
for u in ref_uncle_ids:
su = int(tree.slot[u])
if _in_window(su, T) and su not in honest_slots:
recovered.add(su)
n_honest = len(honest_slots)
n_rec = len(recovered)
q = n_honest / n_active if n_active else float("nan")
q_eff = (n_honest + n_rec) / n_active if n_active else float("nan")
return SlotStats(n_active=n_active, n_honest=n_honest, n_recovered=n_rec, q=q, q_eff=q_eff)

135
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/uncles.py Normal file
View File

@ -0,0 +1,135 @@
"""Proposer-local uncle selection.
For each canonical block ``B`` (processed oldest-first so ancestors' references are
known), candidates are orphan (non-canonical) blocks ``U`` with
``0 < slot_B - slot_U <= W`` that have not already been referenced by an ancestor of
``B``. Two strategies match the spec: deterministic oldest-first, and random (oldest-first
order, a coin of probability ``uncle_random_p`` per candidate, capped at ``U``). The spec's
coin is unbiased (``uncle_random_p = 0.5``, the default); other values are a non-spec
sensitivity knob. Dedup across ancestors is enforced by threading a ``referenced`` set down
the canonical chain.
"""
from __future__ import annotations
import numpy as np
from .blocktree import GENESIS, BlockTree
from .config import SimConfig
def _orphans_sorted(tree: BlockTree, canonical_ids: list[int]) -> tuple[np.ndarray, np.ndarray]:
"""Return orphan block ids sorted by (slot, id) and their slots."""
canonical = np.zeros(tree.n_blocks, dtype=bool)
canonical[canonical_ids] = True
all_real = np.arange(1, tree.n_blocks)
orphan_ids = all_real[~canonical[1:]]
orphan_slots = tree.slot[orphan_ids]
order = np.lexsort((orphan_ids, orphan_slots)) # by slot, then id
return orphan_ids[order], orphan_slots[order]
def annotate_uncles(
tree: BlockTree, canonical_ids: list[int], config: SimConfig, rng: np.random.Generator
) -> None:
"""Fill ``tree.uncles[B]`` for every canonical block ``B`` per the selection rule."""
u_max = config.max_uncles
if u_max <= 0:
return
w = config.uncle_window
orphan_ids, orphan_slots = _orphans_sorted(tree, canonical_ids)
if orphan_ids.size == 0:
return
referenced: set[int] = set()
# oldest canonical block first
for b in reversed(canonical_ids):
sb = int(tree.slot[b])
lo = int(np.searchsorted(orphan_slots, sb - w, side="left")) # slot_U >= sb - W
hi = int(np.searchsorted(orphan_slots, sb, side="left")) # slot_U < sb
if hi <= lo:
continue
window_ids = orphan_ids[lo:hi] # already oldest-first
selected = _select(window_ids, referenced, config, rng)
if selected:
tree.uncles[b] = tuple(selected)
referenced.update(selected)
def _select(
window_ids: np.ndarray, referenced: set[int], config: SimConfig, rng: np.random.Generator
) -> list[int]:
u_max = config.max_uncles
out: list[int] = []
if config.uncle_strategy == "oldest":
for bid in window_ids.tolist():
if bid in referenced:
continue
out.append(bid)
if len(out) >= u_max:
break
elif config.uncle_strategy == "random":
p = config.uncle_random_p
for bid in window_ids.tolist():
if bid in referenced:
continue
if rng.random() < p:
out.append(bid)
if len(out) >= u_max:
break
else: # pragma: no cover - guarded by Literal
raise ValueError(config.uncle_strategy)
return out
def select_uncles_at_production(
slot: np.ndarray,
parent: np.ndarray,
uncles: list,
arrival_v: np.ndarray,
nb: int,
parent_id: int,
t: int,
config: SimConfig,
rng: np.random.Generator,
arr_base: int = 0,
) -> tuple[int, ...]:
"""Uncles a block gets when produced by node ``v`` (arrival row ``arrival_v``) at slot ``t``.
Candidates are blocks in ``v``'s view (``arrival_v[b] <= t``) with slot in ``[t-W, t)``
that are NOT on the chain ``v`` extends (ancestors of ``parent_id``) and not already
referenced by that chain. Selected once and baked globally (same for everyone who adopts
the block), so density counting stays view-independent.
``arrival_v`` is indexed by *block id minus ``arr_base``* — ``arr_base=0`` for the full arrival
matrix row ``A[v]``, or the sliding-window buffer's base offset when pruning (every uncle-window
block ``[t-W, t)`` is inside the kept span, so the buffer row covers all candidates).
"""
u_max = config.max_uncles
if u_max <= 0:
return ()
w = config.uncle_window
slot_view = slot[:nb]
lo = int(np.searchsorted(slot_view, t - w, side="left")) # slot >= t-W
hi = int(np.searchsorted(slot_view, t, side="left")) # slot < t
if hi <= lo:
return ()
# blocks in the window that have arrived at v (candidates before excluding own chain)
arrived = np.nonzero(arrival_v[lo - arr_base:hi - arr_base] <= t)[0] + lo
if arrived.size == 0:
return ()
# v's own chain within the window + the uncles it already references (for dedup)
on_chain: set[int] = set()
referenced: set[int] = set()
a = int(parent_id)
while a > GENESIS and int(slot[a]) >= t - w:
on_chain.add(a)
referenced.update(uncles[a])
a = int(parent[a])
cands = np.array(
[b for b in arrived.tolist() if b > GENESIS and b not in on_chain and b not in referenced],
dtype=np.int64,
)
if cands.size == 0:
return ()
return tuple(_select(cands, set(), config, rng)) # cands already oldest-first (slot,id)

95
tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/verify.py Normal file
View File

@ -0,0 +1,95 @@
"""Per-node analytic checks (``tsi-verify``). Exits non-zero if any check fails.
Validates that per-node D_est disagreement collapses (the reduced-model assumption) and
that the full-mesh baseline reproduces the reduced model. Replicates run across cores.
"""
from __future__ import annotations
from dataclasses import replace
import numpy as np
import pandas as pd
from joblib import Parallel, delayed
from .config import SimConfig
from .engine import run_trajectory
from .theory import expected_ratio
F = 1.0 / 30.0
K = 32
EPOCHS = 20
REPS = 8
BURN = 10
def tail(cfg: SimConfig, col: str, reps: int = REPS) -> float:
def one(r: int) -> float:
df = pd.DataFrame(run_trajectory(replace(cfg, replicate=r)))
return float(df[col].iloc[BURN:].mean())
vals = Parallel(n_jobs=-1, backend="loky", inner_max_num_threads=1)(
delayed(one)(r) for r in range(reps)
)
return float(np.mean(vals))
def check(name: str, ok: bool, detail: str) -> bool:
print(f"[{'PASS' if ok else 'FAIL'}] {name}: {detail}")
return ok
def main() -> int:
results = []
common = dict(n_nodes=300, stake_dist="uniform", k=K, epochs=EPOCHS, genesis_d_factor=0.5)
# 1. Full-mesh baseline: zero per-node divergence, full window agreement.
fm = SimConfig(topology="full_mesh", latency=4, max_uncles=0, **common)
rng_range = tail(fm, "range_ratio")
agree = tail(fm, "agreement_window")
results.append(check("full-mesh: zero D_est spread, full window agreement",
rng_range < 1e-9 and agree > 0.999,
f"range={rng_range:.2e} agreement_window={agree:.4f}"))
# 2. Full-mesh mean accuracy ~ reduced theory(q).
mean_r = tail(fm, "mean_ratio")
q = tail(fm, "mean_q")
pred = float(expected_ratio(F, q))
results.append(check("full-mesh mean ratio ~ theory(q)",
abs(mean_r - pred) < 0.03,
f"sim={mean_r:.4f} theory(q={q:.3f})={pred:.4f}"))
# 3. Regular graph: still zero D_est divergence (window settled), but tip forking present.
reg = SimConfig(topology="regular", degree=8, link_latency_mean=2.0, max_uncles=0, **common)
rng_range = tail(reg, "range_ratio")
agree_w = tail(reg, "agreement_window")
agree_t = tail(reg, "agreement_tip")
results.append(check("regular graph: D_est agrees (window) despite tip forks",
rng_range < 1e-9 and agree_w > 0.999 and agree_t < 1.0,
f"range={rng_range:.2e} agree_win={agree_w:.4f} agree_tip={agree_t:.4f}"))
# 4. Topology affects mean accuracy: sparser/slower graph -> lower mean ratio (more forks).
sparse = tail(SimConfig(topology="regular", degree=4, link_latency_mean=6.0,
max_uncles=0, **common), "mean_ratio")
dense = tail(SimConfig(topology="regular", degree=16, link_latency_mean=1.0,
max_uncles=0, **common), "mean_ratio")
results.append(check("topology shifts mean accuracy (sparse < dense)",
sparse < dense,
f"sparse(deg4,ll6)={sparse:.4f} < dense(deg16,ll1)={dense:.4f}"))
# 5. Uncles recover the mean accuracy under a graph (as in the reduced model).
u0 = tail(SimConfig(topology="regular", degree=8, link_latency_mean=4.0,
max_uncles=0, **common), "mean_ratio")
u4 = tail(SimConfig(topology="regular", degree=8, link_latency_mean=4.0,
max_uncles=4, uncle_strategy="oldest", **common), "mean_ratio")
results.append(check("uncles recover mean accuracy under topology",
abs(u4 - 1) < abs(u0 - 1) and abs(u4 - 1) < 0.03,
f"mean ratio U0={u0:.4f} -> U4={u4:.4f}"))
print()
n_pass = sum(results)
print(f"{n_pass}/{len(results)} checks passed")
return 0 if n_pass == len(results) else 1
if __name__ == "__main__": # pragma: no cover
raise SystemExit(main())

54
tools/simulators/tsi/tsi-sim-pernode/tests/test_blocktree.py Normal file
View File

@ -0,0 +1,54 @@
import numpy as np
from tsi_sim.blocktree import build_tree
from tsi_sim.latency import FixedSlotLatency
def _winners(*groups):
return [np.array(g, dtype=np.int64) for g in groups]
def test_latency_induces_fork_and_longest_chain():
# L=2. slot0: node0; slot1: node1 (can't see block1 yet -> forks on genesis);
# slot3: node2 (sees both, builds on the earlier-slot tip -> block1).
active = np.array([0, 1, 3], dtype=np.int64)
winners = _winners([0], [1], [2])
tree = build_tree(active, winners, FixedSlotLatency(2), np.random.default_rng(0))
assert tree.n_blocks == 4 # genesis + 3
assert tree.height.tolist() == [0, 1, 1, 2]
# block 3 built on block 1 (earlier slot wins the height-1 tie)
assert tree.parent[3] == 1
assert tree.canonical_chain() == [3, 1] # tip-first
# block 2 is the orphan
canon = set(tree.canonical_chain())
orphans = [b for b in range(1, tree.n_blocks) if b not in canon]
assert orphans == [2]
def test_same_slot_multiwinner_forks_even_at_zero_latency():
active = np.array([0, 1], dtype=np.int64)
winners = _winners([0, 1], [2]) # two winners in slot 0 -> guaranteed fork
tree = build_tree(active, winners, FixedSlotLatency(0), np.random.default_rng(0))
# blocks 1 and 2 are siblings at height 1 on genesis
assert tree.parent[1] == 0 and tree.parent[2] == 0
assert tree.height[1] == 1 and tree.height[2] == 1
# block 3 at slot 1 extends one of them (height 2)
assert tree.height[3] == 2
assert len(tree.canonical_chain()) == 2
def test_self_extension_within_latency():
# A single node winning consecutive slots builds on its own block despite latency.
active = np.array([0, 1], dtype=np.int64)
winners = _winners([5], [5])
tree = build_tree(active, winners, FixedSlotLatency(10), np.random.default_rng(0))
assert tree.parent[2] == 1 # node 5 self-extends
assert tree.height.tolist() == [0, 1, 2]
def test_ancestors():
active = np.array([0, 1, 2], dtype=np.int64)
winners = _winners([0], [0], [0]) # one node, clean chain
tree = build_tree(active, winners, FixedSlotLatency(0), np.random.default_rng(0))
assert tree.ancestors(3) == [3, 2, 1]

51
tools/simulators/tsi/tsi-sim-pernode/tests/test_churn_pref.py Normal file
View File

@ -0,0 +1,51 @@
"""Honest stake churn (TSI tracks active stake) and the emergent p_ref metric."""
from __future__ import annotations
import pandas as pd
from tsi_sim.config import SimConfig
from tsi_sim.engine import _churn_active_fraction, run_trajectory
BASE = dict(n_nodes=400, stake_dist="pareto", topology="blend", degree=6,
link_latency_mean=0.5, link_latency_dist="geo", blend_hops=3, blend_delay_max=4.0,
max_uncles=2, uncle_window=300, k=256, epochs=16, genesis_d_factor=0.5)
def test_churn_schedule_shapes():
sine = SimConfig(**BASE, churn_amp=0.3, churn_period=4, churn_mode="sine")
assert _churn_active_fraction(sine, 0) == 1.0 # cos(0)=1 -> no drop
assert abs(_churn_active_fraction(sine, 2) - 0.7) < 1e-9 # trough at half period
ramp = SimConfig(**BASE, churn_amp=0.3, churn_period=4, churn_mode="ramp")
assert abs(_churn_active_fraction(ramp, 4) - 0.7) < 1e-9
assert abs(_churn_active_fraction(ramp, 8) - 0.7) < 1e-9 # holds after ramp
step = SimConfig(**BASE, churn_amp=0.3, churn_period=4, churn_mode="step")
assert _churn_active_fraction(step, 3) == 1.0
assert abs(_churn_active_fraction(step, 4) - 0.7) < 1e-9
def test_churn_zero_is_bit_identical():
off = pd.DataFrame(run_trajectory(SimConfig(**BASE)))
z = pd.DataFrame(run_trajectory(SimConfig(**BASE, churn_amp=0.0)))
assert (off.mean_ratio.to_numpy() == z.mean_ratio.to_numpy()).all()
def test_tsi_tracks_active_stake_under_churn():
"""A step drop in active stake: D_hat/D_active stays ~1, D_hat/D_total drops to active_frac."""
cfg = SimConfig(**{**BASE, "epochs": 20}, churn_amp=0.3, churn_period=6, churn_mode="step")
df = pd.DataFrame(run_trajectory(cfg))
tail = df[df.epoch >= 14]
# D_hat/D_total tracks the reduced active fraction (~0.7)
assert 0.6 < tail.mean_ratio.mean() < 0.8
# corrected for active fraction, accuracy is ~1
corrected = (tail.mean_ratio / tail.active_stake_frac).mean()
assert abs(corrected - 1.0) < 0.05
assert tail.range_ratio.max() == 0.0 # churn does not break consensus
def test_p_ref_recorded_and_high_at_recommended_window():
cfg = SimConfig(**{**BASE, "blend_delay_max": 8.0, "uncle_window": 300})
df = pd.DataFrame(run_trajectory(cfg))
assert "p_ref" in df.columns
# at a generous window, most orphans get referenced
assert df[df.epoch >= 8].p_ref.mean() > 0.7

40
tools/simulators/tsi/tsi-sim-pernode/tests/test_concurrency.py Normal file
View File

@ -0,0 +1,40 @@
import numpy as np
from tsi_sim import concurrency
from tsi_sim.config import SimConfig
def test_window_counts_partitions_all_proposals():
ws = np.array([0, 1, 2, 5, 5, 9], dtype=np.int64)
counts = concurrency.window_counts(ws, epoch_len=10, bucket=5)
assert counts.tolist() == [3, 3] # slots 0-4 -> 3, slots 5-9 -> 3
assert counts.sum() == ws.size
def test_window_counts_bucket_one_is_per_slot():
ws = np.array([0, 0, 3], dtype=np.int64)
counts = concurrency.window_counts(ws, epoch_len=4, bucket=1)
assert counts.tolist() == [2, 0, 0, 1] # slot 0 has 2 concurrent, slot 3 has 1
def test_window_counts_empty():
assert concurrency.window_counts(np.empty(0, np.int64), 10, 2).tolist() == [0, 0, 0, 0, 0]
def test_concurrency_stats_scale_with_latency():
# Bigger latency bucket => more proposals per bucket (max/mean grow roughly with L).
small = concurrency.concurrency_stats(SimConfig(n_nodes=1000, latency=2, k=32, epochs=1))
large = concurrency.concurrency_stats(SimConfig(n_nodes=1000, latency=20, k=32, epochs=1))
assert large["mean"] > small["mean"]
assert large["max"] >= small["max"]
# mean per bucket ~ bucket * (-ln(1-f))
expected = large["bucket"] * (-np.log(1 - 1 / 30))
assert abs(large["mean"] - expected) < 0.25 * expected
def test_proposal_slots_reproducible_and_sorted():
cfg = SimConfig(n_nodes=500, latency=4, k=16, epochs=1)
a = concurrency.proposal_slots(cfg, replicate=0)
b = concurrency.proposal_slots(cfg, replicate=0)
np.testing.assert_array_equal(a, b)
assert np.all(np.diff(a) >= 0)

146
tools/simulators/tsi/tsi-sim-pernode/tests/test_config.py Normal file
View File

@ -0,0 +1,146 @@
import dataclasses
import pytest
from tsi_sim.config import SimConfig, SweepConfig
def test_expand_cardinality_and_u0_strategy_dedup():
# full_mesh, where `latency` (L) IS a live axis, so the x2 latency factor applies.
sweep = SweepConfig(
n_nodes=[1000, 2000],
stake_dist=["uniform"],
topology=["full_mesh"],
latency=[0, 4],
max_uncles=[0, 1, 2],
uncle_strategy=["oldest", "random"],
replicates=3,
base={"k": 16, "epochs": 5},
)
configs = sweep.expand()
# U=0 keeps only the first strategy; U>0 keeps both.
# per (n,dist,lat): U0 x1 strat + U1 x2 + U2 x2 = 5 strat-U combos, x3 reps = 15
# x 2 n_nodes x 1 dist x 2 lat = 60
assert len(configs) == 60
u0 = [c for c in configs if c.max_uncles == 0]
assert all(c.uncle_strategy == "oldest" for c in u0)
assert {c.replicate for c in configs} == {0, 1, 2}
def test_latency_collapsed_for_graph_topologies():
# `latency` is the full_mesh-only uniform-L knob; regular/blend ignore it, so sweeping it
# must NOT emit duplicate (seed-shifted) graph cells.
for topo in ("regular", "blend"):
sweep = SweepConfig(
n_nodes=[100], topology=[topo], degree=[4], latency=[0, 2, 4], max_uncles=[0],
uncle_strategy=["oldest"], replicates=1, base={"k": 8, "epochs": 3},
)
configs = sweep.expand()
assert len(configs) == 1, topo # 3 latency values collapse to 1
assert configs[0].latency == 0
def test_base_propagation():
sweep = SweepConfig(n_nodes=[500], stake_dist=["pareto"], latency=[2], max_uncles=[0],
uncle_strategy=["oldest"], replicates=1,
base={"k": 32, "epochs": 7, "fixed_point": True})
(c,) = sweep.expand()
assert c.k == 32 and c.epochs == 7 and c.fixed_point is True and c.stake_dist == "pareto"
@pytest.mark.parametrize("kwargs", [
{"k": 0}, {"epochs": 0}, {"n_nodes": 0}, {"latency": -1}, {"max_uncles": -1},
{"uncle_window": 0}, {"lottery_chunks": 0}, {"uncle_random_p": 1.5}, {"f": 0.0},
{"f": 1.0}, {"beta": 0.0}, {"genesis_d_factor": 0.0}, {"pareto_shape": 0.0},
{"stake_dist": "zipf"}, {"uncle_strategy": "newest"}, {"topology": "star"},
{"blend_hops": 0}, {"blend_delay_max": -1.0},
{"adversary_period": -1}, {"adversary_withhold_epochs": -1},
# a schedule may not withhold for more epochs than its own period
{"adversary_period": 2, "adversary_withhold_epochs": 3},
# blend needs `blend_hops` distinct relays from the non-producer pool (n-1 of them)
{"topology": "blend", "n_nodes": 4, "degree": 2, "blend_hops": 4},
])
def test_validation_rejects_bad_fields(kwargs):
with pytest.raises(ValueError):
SimConfig(**kwargs)
def test_blend_dedup_does_not_multiply_non_blend_configs():
# blend_hops / blend_delay_max only affect blend runs; sweeping them must not duplicate
# the regular / full_mesh cells.
sweep = SweepConfig(
n_nodes=[100], stake_dist=["uniform"], topology=["regular", "blend"],
degree=[4], link_latency_mean=[1.0], link_latency_dist=["fixed"],
blend_hops=[2, 3], blend_delay_max=[1.0, 3.0], max_uncles=[0],
uncle_strategy=["oldest"], init_dest=["common"], replicates=1,
base={"k": 8, "epochs": 3},
)
configs = sweep.expand()
regular = [c for c in configs if c.topology == "regular"]
blend = [c for c in configs if c.topology == "blend"]
assert len(regular) == 1 # blend knobs collapsed for non-blend
assert len(blend) == 4 # 2 hops x 2 delay_max
assert {(c.blend_hops, c.blend_delay_max) for c in blend} == {
(2, 1.0), (2, 3.0), (3, 1.0), (3, 3.0)}
def test_uncle_window_sweeps_and_collapses_for_u0():
# uncle_window is a live axis for U>0, but U=0 references no uncles so it must collapse.
sweep = SweepConfig(
n_nodes=[100], stake_dist=["uniform"], topology=["blend"], degree=[6],
link_latency_mean=[0.5], link_latency_dist=["geo"], blend_hops=[3],
blend_delay_max=[4.0], uncle_window=[10, 100], max_uncles=[0, 1],
uncle_strategy=["oldest"], init_dest=["common"], replicates=1,
base={"k": 8, "epochs": 3},
)
configs = sweep.expand()
u0 = [c for c in configs if c.max_uncles == 0]
u1 = [c for c in configs if c.max_uncles == 1]
assert len(u0) == 1 # W collapsed for U=0
assert {c.uncle_window for c in u1} == {10, 100} # both W kept for U=1
def test_unknown_sweep_key_rejected():
with pytest.raises(ValueError, match="unknown sweep keys"):
SweepConfig.from_dict({"latencies": [0, 1], "base": {}}) # typo: latencies vs latency
def test_from_dict_roundtrip_ok():
sw = SweepConfig.from_dict({"latency": [0, 3], "max_uncles": [0, 2], "replicates": 2,
"base": {"k": 8, "epochs": 4}})
assert sw.latency == [0, 3] and sw.replicates == 2 and sw.base["k"] == 8
def test_key_covers_every_field():
# Guard against the silent shared-RNG bug: key() must reflect all run-affecting fields.
# root_seed enters _entropy separately; windowed_fork_choice is a pure compute
# optimisation (no RNG, identical results) so it is intentionally not in key().
# early_stop is truncation-only (per-epoch RNG streams are pre-spawned, so the epochs
# that DO run are bit-identical to a full run's prefix) — intentionally excluded from key().
ignored = {"root_seed", "windowed_fork_choice", "prune_arrival", "early_stop"}
names = {f.name for f in dataclasses.fields(SimConfig)} - ignored
a = SimConfig()
for name in names:
cur = getattr(a, name)
alt = _perturb(cur)
b = dataclasses.replace(a, **{name: alt})
assert a.key() != b.key(), f"key() does not distinguish field {name!r}"
def _perturb(v):
if isinstance(v, bool):
return not v
if isinstance(v, int):
return v + 1
if isinstance(v, float):
# stay inside [0, 1]-capped fields (e.g. jitter_frac defaults to 1.0)
return v - 0.001 if v >= 1.0 else v + 0.001
flips = {
"uniform": "pareto", "oldest": "random", "full_mesh": "regular",
"fixed": "exp", "common": "heterogeneous", "suppress": "withhold",
"exp": "poisson", # jitter_dist
"sine": "ramp", # churn_mode
}
if isinstance(v, str) and v in flips:
return flips[v]
return v

34
tools/simulators/tsi/tsi-sim-pernode/tests/test_early_stop.py Normal file
View File

@ -0,0 +1,34 @@
"""Early stop: converge-then-measure truncation that preserves equilibrium statistics."""
from __future__ import annotations
import pandas as pd
from tsi_sim.config import SimConfig
from tsi_sim.engine import ES_MEASURE, ES_MIN_EPOCH, run_trajectory
BASE = dict(n_nodes=300, stake_dist="pareto", topology="blend", degree=6,
link_latency_mean=0.5, link_latency_dist="geo", blend_hops=3, blend_delay_max=4.0,
max_uncles=2, uncle_window=300, k=256, epochs=40, genesis_d_factor=0.5)
def test_early_stop_truncates_and_matches_full_run():
full = pd.DataFrame(run_trajectory(SimConfig(**BASE)))
es = pd.DataFrame(run_trajectory(SimConfig(**BASE, early_stop=True)))
# truncation happened, with room for the measurement budget
assert ES_MIN_EPOCH + 1 <= len(es) < len(full)
# bit-identical prefix (same RNG streams; early_stop is excluded from key())
n = len(es)
assert (full.head(n).mean_ratio.to_numpy() == es.mean_ratio.to_numpy()).all()
# equilibrium agrees: early-stop tail (measurement sample) vs full-run tail
es_tail = es.tail(ES_MEASURE).mean_ratio.mean()
full_tail = full[full.epoch >= 20].mean_ratio.mean()
assert abs(es_tail - full_tail) < 0.015
def test_sawtooth_never_stops_early():
cfg = SimConfig(**{**BASE, "epochs": 24}, early_stop=True,
adversary_frac=0.3, adversary_strategy="withhold",
adversary_period=6, adversary_withhold_epochs=3)
df = pd.DataFrame(run_trajectory(cfg))
assert len(df) == 24 # full budget: detector disabled for schedules

56
tools/simulators/tsi/tsi-sim-pernode/tests/test_fork.py Normal file
View File

@ -0,0 +1,56 @@
"""Fork rate and reorg depth from the global tree."""
from __future__ import annotations
import numpy as np
import pandas as pd
from tsi_sim.blocktree import BlockTree
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
from tsi_sim.fork import fork_stats
def make_tree(slots, parents, heights):
n = len(slots)
return BlockTree(
slot=np.array(slots, np.int64), parent=np.array(parents, np.int64),
height=np.array(heights, np.int64), leader=np.zeros(n, np.int64),
uncles=[() for _ in range(n)],
)
def test_no_forks():
# a straight chain 1->2->3, no orphans
tree = make_tree([-1, 0, 1, 2], [-1, 0, 1, 2], [0, 1, 2, 3])
fr, mx, mn, pr = fork_stats(tree, None, T=10, cutoff=100)
assert fr == 0.0 and mx == 0 and mn == 0.0
def test_single_orphan_depth_one():
# canonical 1(s0),2(s1),4(s3); orphan 3(s2) hangs off block1 -> branch depth 1
tree = make_tree([-1, 0, 1, 2, 3], [-1, 0, 1, 1, 2], [0, 1, 2, 2, 3])
fr, mx, mn, pr = fork_stats(tree, None, T=10, cutoff=100)
assert mx == 1
assert abs(fr - 1 / 4) < 1e-9 # 1 orphan of 4 in-window blocks
def test_deep_orphan_branch():
# canonical spine 1..3 (heights 1,2,3); a 2-deep orphan branch 4->5 off block1
# blocks: 0 gen; 1(s0,h1),2(s1,h2),3(s2,h3) canonical; 4(s1,h2)->1, 5(s2,h3)->4 orphan
tree = make_tree([-1, 0, 1, 2, 1, 2], [-1, 0, 1, 2, 1, 4], [0, 1, 2, 3, 2, 3])
fr, mx, mn, pr = fork_stats(tree, None, T=10, cutoff=100)
assert mx == 2 # branch 4->5 is 2 deep
assert abs(fr - 2 / 5) < 1e-9 # 2 orphans of 5
def test_engine_reports_fork_columns():
cfg = SimConfig(n_nodes=300, stake_dist="pareto", topology="blend", degree=6,
link_latency_mean=0.5, link_latency_dist="geo", blend_hops=3,
blend_delay_max=16.0, max_uncles=1, uncle_window=300, k=256, epochs=8)
df = pd.DataFrame(run_trajectory(cfg))
for col in ("fork_rate", "max_reorg_depth", "mean_reorg_depth"):
assert col in df.columns
# heavy delay -> real forks
assert df[df.epoch >= 4].fork_rate.mean() > 0.0
assert df.max_reorg_depth.max() >= 1

63
tools/simulators/tsi/tsi-sim-pernode/tests/test_jitter_poisson.py Normal file
View File

@ -0,0 +1,63 @@
"""Poisson long-tail jitter model (case (b) of the N-scaling study)."""
from __future__ import annotations
import numpy as np
import pandas as pd
import pytest
from tsi_sim import topology as topo
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
BASE = dict(n_nodes=200, stake_dist="pareto", topology="blend", degree=6,
link_latency_mean=0.5, link_latency_dist="geo", blend_hops=3, blend_delay_max=4.0,
max_uncles=1, uncle_window=300, k=64, epochs=8, genesis_d_factor=0.5)
def test_poisson_frac_zero_matches_no_jitter_statistics():
"""jitter_frac=0 hits nobody: consensus holds and accuracy matches the jitter-free run."""
off = pd.DataFrame(run_trajectory(SimConfig(**BASE)))
z = pd.DataFrame(run_trajectory(SimConfig(
**BASE, jitter_mean=3.0, jitter_dist="poisson", jitter_frac=0.0)))
assert z.range_ratio.max() == 0.0
assert z.agreement_window.min() == 1.0
# same equilibrium to MC noise (different RNG stream — key includes jitter fields)
assert abs(z[z.epoch >= 4].mean_ratio.mean() - off[off.epoch >= 4].mean_ratio.mean()) < 0.1
def test_poisson_tail_consensus_survives():
"""10% long-tail stragglers (lambda=3 slots): spread stays 0, agreement stays 1."""
df = pd.DataFrame(run_trajectory(SimConfig(
**BASE, jitter_mean=3.0, jitter_dist="poisson", jitter_frac=0.1)))
assert df.range_ratio.max() == 0.0
assert df.agreement_window.min() == 1.0
def test_jitter_fields_validated_and_keyed():
with pytest.raises(ValueError):
SimConfig(jitter_dist="weibull")
with pytest.raises(ValueError):
SimConfig(jitter_frac=1.5)
a = SimConfig(**BASE, jitter_mean=3.0, jitter_dist="poisson", jitter_frac=0.1)
b = SimConfig(**BASE, jitter_mean=3.0, jitter_dist="poisson", jitter_frac=0.05)
c = SimConfig(**BASE, jitter_mean=3.0, jitter_dist="exp", jitter_frac=0.1)
assert len({a.key(), b.key(), c.key()}) == 3
def test_poisson_only_hits_the_requested_fraction():
"""Direct check on one arrival column: ~frac of nodes delayed, by whole slots."""
big = {**BASE, "n_nodes": 2000}
cfg_j = SimConfig(**big, jitter_mean=3.0, jitter_dist="poisson", jitter_frac=0.1)
cfg_0 = SimConfig(**big)
pl = topo.build_path_latency(cfg_j, np.random.default_rng(0))
# same rng seed: relay/mixing draws happen before jitter, so the columns differ by jitter only
col_j = topo.arrival_column(pl, producer=0, slot=100, config=cfg_j,
rng=np.random.default_rng(1))
col_0 = topo.arrival_column(pl, producer=0, slot=100, config=cfg_0,
rng=np.random.default_rng(1))
extra = col_j - col_0
delayed = extra > 0
frac = delayed[1:].mean() # exclude the producer (clamped to its own slot)
assert 0.04 < frac < 0.16 # ~10%, minus the Poisson(3) zeros (~5%)
assert np.allclose(extra[delayed], np.round(extra[delayed])) # whole-slot stragglers

36
tools/simulators/tsi/tsi-sim-pernode/tests/test_latency.py Normal file
View File

@ -0,0 +1,36 @@
import numpy as np
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
from tsi_sim.latency import FixedSlotLatency, RealisticLatency, make_latency
def test_fixed_slot_latency():
lat = FixedSlotLatency(3)
assert lat.visible_at(10, np.random.default_rng(0)) == 13
def test_realistic_latency_zero_mean_immediate():
lat = RealisticLatency(0.0)
assert lat.visible_at(5, np.random.default_rng(0)) == 5
def test_realistic_latency_positive_delays():
lat = RealisticLatency(4.0)
rng = np.random.default_rng(0)
vals = [lat.visible_at(100, rng) for _ in range(200)]
assert all(v >= 101 for v in vals) # strictly after production
assert np.mean([v - 100 for v in vals]) > 1.0
def test_make_latency_dispatch():
assert isinstance(make_latency(SimConfig(latency=2)), FixedSlotLatency)
assert isinstance(make_latency(SimConfig(latency=2, latency_stochastic=True)), RealisticLatency)
def test_pernode_trajectory_integration_runs():
cfg = SimConfig(n_nodes=300, topology="regular", degree=8, link_latency_mean=3.0,
k=8, epochs=5, max_uncles=2)
rows = run_trajectory(cfg)
assert len(rows) == 5
assert all(np.isfinite(r["mean_ratio"]) for r in rows)

77
tools/simulators/tsi/tsi-sim-pernode/tests/test_lottery.py Normal file
View File

@ -0,0 +1,77 @@
import numpy as np
from tsi_sim import lottery
def test_phi_bounds():
f = 1 / 30
assert lottery.phi(f, 0.0) == 0.0
# a single all-stake node (alpha=1) wins at exactly rate f
assert abs(lottery.phi(f, 1.0) - f) < 1e-12
def test_win_probs_monotone_in_stake():
stake = np.array([1.0, 2.0, 3.0])
p = lottery.win_probs(stake, d_est=6.0, f=1 / 30)
assert np.all(np.diff(p) > 0)
def test_sample_wins_sorted_and_rate():
rng = np.random.default_rng(0)
n, slots = 500, 4000
p = np.full(n, 0.001)
ws, wn = lottery.sample_wins(p, slots, rng, chunk=512)
assert np.all(np.diff(ws) >= 0) # sorted by slot
assert ws.shape == wn.shape
assert np.all((wn >= 0) & (wn < n))
# expected wins ~ n * slots * p
assert abs(ws.size - n * slots * 0.001) < 4 * np.sqrt(n * slots * 0.001)
def test_group_by_slot():
ws = np.array([0, 0, 2, 5, 5, 5])
wn = np.array([3, 7, 1, 2, 4, 9])
active, groups = lottery.group_by_slot(ws, wn)
assert list(active) == [0, 2, 5]
assert [g.tolist() for g in groups] == [[3, 7], [1], [2, 4, 9]]
def test_group_by_slot_empty():
active, groups = lottery.group_by_slot(np.empty(0, int), np.empty(0, int))
assert active.size == 0 and groups == []
def test_sparse_per_node_distinct_slots():
# Each node wins any slot at most once (independent Bernoulli-per-slot invariant).
rng = np.random.default_rng(1)
p = np.full(300, 0.05)
ws, wn = lottery.sample_wins(p, 2000, rng)
for node in np.unique(wn):
slots = ws[wn == node]
assert slots.size == np.unique(slots).size # no duplicate (node, slot)
def test_sparse_preserves_multiwinner_slots():
# With high p, some slots have >1 distinct winner (guaranteed forks) — must be possible.
rng = np.random.default_rng(2)
p = np.full(50, 0.3)
ws, _ = lottery.sample_wins(p, 500, rng)
_, counts = np.unique(ws, return_counts=True)
assert counts.max() >= 2
def test_chunked_matches_serial_distribution():
# Chunked sampler is deterministic given (seedseq, n_chunks) and statistically matches
# serial. NOTE: SeedSequence.spawn is stateful, so each call needs a FRESH SeedSequence.
p = np.full(400, 0.02)
a_s, a_n = lottery.sample_wins_chunked(p, 100000, np.random.SeedSequence(123), 4, n_jobs=1)
b_s, b_n = lottery.sample_wins_chunked(p, 100000, np.random.SeedSequence(123), 4, n_jobs=1)
np.testing.assert_array_equal(a_s, b_s) # deterministic
np.testing.assert_array_equal(a_n, b_n)
assert np.all(np.diff(a_s) >= 0) # sorted
for node in np.unique(a_n): # per-node distinct slots preserved
s = a_s[a_n == node]
assert s.size == np.unique(s).size
serial_n = lottery.sample_wins(p, 100000, np.random.default_rng(7))[0].size
assert abs(a_s.size - serial_n) < 6 * np.sqrt(serial_n) # same rate

79
tools/simulators/tsi/tsi-sim-pernode/tests/test_measure.py Normal file
View File

@ -0,0 +1,79 @@
"""The optimised measurement must be bit-identical to the naive per-node reference."""
from collections import Counter
import numpy as np
import pytest
from tsi_sim import lottery, topology, tsi
from tsi_sim.blocktree import build_tree_pernode, tips_for_all_nodes
from tsi_sim.config import SimConfig
from tsi_sim.measure import measure
def _build(cfg, rep=0):
root = np.random.SeedSequence(abs(hash((cfg.key(), rep))) % (2**63))
ch = root.spawn(4)
stake = np.ones(cfg.n_nodes) * (cfg.total_stake / cfg.n_nodes)
pl = topology.build_path_latency(cfg, np.random.default_rng(ch[1]))
d = np.full(cfg.n_nodes, cfg.genesis_d_factor * cfg.total_stake)
ws, wn = lottery.sample_wins(lottery.win_probs(stake, d, cfg.f), cfg.epoch_len,
np.random.default_rng(ch[2]))
active, groups = lottery.group_by_slot(ws, wn)
tree, A = build_tree_pernode(active, groups, pl, cfg, np.random.default_rng(ch[3]))
return tree, A, active
def _reference(tree, A, active_slots, T, cutoff):
"""Original naive per-node measurement (the ground truth)."""
tips = tips_for_all_nodes(tree, A, cutoff) # A may be a full matrix or a pruned SlidingArrival
n = tips.shape[0]
n_real = tree.n_blocks - 1
m = np.empty(n, np.int64)
q = np.empty(n)
qe = np.empty(n)
orp = np.empty(n)
fps = []
for i in range(n):
canon = tree.ancestors(int(tips[i]))
m[i] = tsi.density_m(tree, canon, T)
ss = tsi.slot_stats(tree, canon, tsi.referenced_uncle_ids(tree, canon), active_slots, T)
q[i], qe[i] = ss.q, ss.q_eff
orp[i] = (n_real - len(canon)) / n_real if n_real else 0.0
fps.append(tuple(sorted(b for b in canon if 0 <= tree.slot[b] < T)))
aw = Counter(fps).most_common(1)[0][1] / n
at = Counter(tips.tolist()).most_common(1)[0][1] / n
return m, q, qe, orp, aw, at
@pytest.mark.parametrize("kw", [
dict(topology="full_mesh", latency=0, max_uncles=0),
dict(topology="full_mesh", latency=4, max_uncles=2),
dict(topology="regular", degree=6, link_latency_mean=3.0, max_uncles=4),
dict(topology="regular", degree=2, link_latency_mean=6.0, max_uncles=2), # low agreement
dict(topology="regular", degree=8, link_latency_mean=1.0, max_uncles=0),
])
def test_measure_matches_reference(kw):
cfg = SimConfig(n_nodes=150, k=10, **kw)
tree, A, active = _build(cfg)
T, E = cfg.period_T, cfg.epoch_len
ref_m, ref_q, ref_qe, ref_orp, ref_aw, ref_at = _reference(tree, A, active, T, E)
got = measure(tree, A, active, T, E)
np.testing.assert_array_equal(got.m, ref_m)
np.testing.assert_allclose(got.q, ref_q, equal_nan=True)
np.testing.assert_allclose(got.q_eff, ref_qe, equal_nan=True)
np.testing.assert_allclose(got.orphan_rate, ref_orp)
assert got.agreement_window == pytest.approx(ref_aw)
assert got.agreement_tip == pytest.approx(ref_at)
def test_numba_and_python_kernels_agree():
cfg = SimConfig(n_nodes=150, topology="regular", degree=4, link_latency_mean=4.0,
max_uncles=3, k=10)
tree, A, active = _build(cfg)
a = measure(tree, A, active, cfg.period_T, cfg.epoch_len, use_numba=True)
b = measure(tree, A, active, cfg.period_T, cfg.epoch_len, use_numba=False)
np.testing.assert_array_equal(a.m, b.m)
np.testing.assert_allclose(a.q_eff, b.q_eff, equal_nan=True)
assert a.agreement_window == b.agreement_window

309
tools/simulators/tsi/tsi-sim-pernode/tests/test_pernode.py Normal file
View File

@ -0,0 +1,309 @@
import numpy as np
import pandas as pd
import pytest
from tsi_sim import constants, lottery, topology, tsi
from tsi_sim.blocktree import GENESIS, build_tree_pernode
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
def _build(cfg, replicate=0):
"""Build one epoch's (tree, A, path_latency) for a config."""
root = np.random.SeedSequence(abs(hash((cfg.key(), replicate))) % (2**63))
ch = root.spawn(4)
stake = np.ones(cfg.n_nodes) * (cfg.total_stake / cfg.n_nodes)
pl = topology.build_path_latency(cfg, np.random.default_rng(ch[1]))
d = np.full(cfg.n_nodes, cfg.genesis_d_factor * cfg.total_stake)
p = lottery.win_probs(stake, d, cfg.f)
ws, wn = lottery.sample_wins(p, cfg.epoch_len, np.random.default_rng(ch[2]))
active, groups = lottery.group_by_slot(ws, wn)
tree, A = build_tree_pernode(active, groups, pl, cfg, np.random.default_rng(ch[3]))
return tree, A, pl
# --- topology ---------------------------------------------------------------
def test_full_mesh_path_latency():
cfg = SimConfig(n_nodes=50, topology="full_mesh", latency=5)
pl = topology.build_path_latency(cfg, np.random.default_rng(0))
assert pl.shape == (50, 50)
assert np.all(np.diag(pl) == 0)
off = pl[~np.eye(50, dtype=bool)]
assert np.all(off == 5)
def test_geo_link_latency_mean_preserved_and_banded():
# "geo" draws real-world geographic latency bands but rescales them so the configured
# link_latency_mean stays the single mean-latency knob.
cfg = SimConfig(n_nodes=300, topology="regular", degree=8,
link_latency_mean=0.08, link_latency_dist="geo")
w = topology._sample_link_latencies(50_000, cfg, np.random.default_rng(3))
assert abs(w.mean() - cfg.link_latency_mean) < 0.02 * cfg.link_latency_mean # E[w] == mean
scale = cfg.link_latency_mean / constants.GEO_LATENCY_MEAN_SLOTS
expected = {round(b * scale, 9) for b in constants.GEO_LATENCY_BANDS_SLOTS}
assert {round(x, 9) for x in np.unique(w)} == expected # exact bands
# realistic direct-gossip: end-to-end (multi-hop) propagation stays a fraction of a slot
pl = topology.build_path_latency(cfg, np.random.default_rng(4))
assert float(pl[~np.eye(cfg.n_nodes, dtype=bool)].mean()) < 1.0
def test_regular_graph_connected_symmetric_and_degree():
cfg = SimConfig(n_nodes=100, topology="regular", degree=6, link_latency_mean=1.0)
pl = topology.build_path_latency(cfg, np.random.default_rng(1))
assert np.all(np.diag(pl) == 0)
assert np.allclose(pl, pl.T) # undirected
assert np.all(np.isfinite(pl)) and pl.max() < cfg.epoch_len # connected
# direct neighbours (latency == link mean 1) — each node has exactly `degree` of them
neigh = (pl == 1).sum(axis=1)
assert np.all(neigh == 6)
# --- blend mixnet topology --------------------------------------------------
def test_blend_reuses_the_regular_graph():
# blend transport runs over the SAME weighted d-regular graph as `regular`.
common = dict(n_nodes=120, degree=6, link_latency_mean=1.0, link_latency_dist="fixed",
graph_seed=7)
plr = topology.build_path_latency(SimConfig(topology="regular", **common),
np.random.default_rng(0))
plb = topology.build_path_latency(SimConfig(topology="blend", **common),
np.random.default_rng(0))
np.testing.assert_array_equal(plr, plb)
def test_blend_arrival_bounded_and_producer_instant():
# inspects the raw (N x n_blocks) arrival matrix, so keep it (prune stores a sliding buffer)
cfg = SimConfig(n_nodes=120, topology="blend", degree=6, link_latency_mean=1.0,
blend_hops=3, blend_delay_max=3.0, k=8, max_uncles=0, prune_arrival=False)
tree, A, pl = _build(cfg)
# hard cascade bound used by the windowed horizon: (hops+1) transport legs + hops mix delays
H = (cfg.blend_hops + 1) * float(pl.max()) + cfg.blend_hops * cfg.blend_delay_max
assert np.all(A[:, GENESIS] == 0)
for b in range(1, tree.n_blocks):
s = float(tree.slot[b])
v = int(tree.leader[b])
assert A[v, b] == max(s, float(A[v, int(tree.parent[b])])) # producer sees own instantly
assert np.all(A[:, b] - s <= H + 1e-9) # nobody exceeds the bound
assert np.all(A[:, b] >= A[:, int(tree.parent[b])]) # never before parent
def test_blend_mixing_delay_increases_arrival():
def mean_rel_arrival(delay_max):
cfg = SimConfig(n_nodes=120, topology="blend", degree=6, link_latency_mean=1.0,
blend_hops=3, blend_delay_max=delay_max, k=8, max_uncles=0,
prune_arrival=False)
tree, A, _ = _build(cfg)
rel = [float((np.delete(A[:, b], int(tree.leader[b])) - float(tree.slot[b])).mean())
for b in range(1, tree.n_blocks)]
return float(np.mean(rel))
# more mixing delay per hop => strictly later visibility on average
assert mean_rel_arrival(0.0) < mean_rel_arrival(6.0)
# --- arrival matrix invariants ----------------------------------------------
def test_arrival_clamp_and_genesis():
cfg = SimConfig(n_nodes=60, topology="regular", degree=6, link_latency_mean=2.0,
k=8, max_uncles=0, prune_arrival=False) # inspects the raw arrival matrix
tree, A, _ = _build(cfg)
assert np.all(A[:, GENESIS] == 0) # genesis seen by all at slot 0
for b in range(1, tree.n_blocks):
p = int(tree.parent[b])
assert np.all(A[:, b] >= A[:, p]) # never arrive before parent
v = int(tree.leader[b])
# producer sees its own block at its slot (sub-slot/float arrivals)
assert A[v, b] == max(float(tree.slot[b]), float(A[v, p]))
def test_full_mesh_zero_divergence_parity():
# Full mesh L=0: identical views -> identical per-node D_est -> range 0, agreement 1.
df = pd.DataFrame(run_trajectory(SimConfig(
n_nodes=200, topology="full_mesh", latency=0, max_uncles=0, k=16, epochs=16)))
assert (df["range_ratio"] == 0.0).all()
assert (df["agreement_window"] == 1.0).all()
assert abs(df["mean_ratio"].iloc[-4:].mean() - 1.0) < 0.1
def test_regular_graph_window_agreement_holds():
# Even under a graph, the settled window agrees -> zero D_est spread.
df = pd.DataFrame(run_trajectory(SimConfig(
n_nodes=200, topology="regular", degree=6, link_latency_mean=3.0,
max_uncles=2, k=12, epochs=10)))
assert (df["range_ratio"] < 1e-9).all()
assert (df["agreement_window"] > 0.999).all()
def test_topology_shifts_mean_accuracy():
def mean_ratio(**kw):
vals = [pd.DataFrame(run_trajectory(SimConfig(
n_nodes=200, topology="regular", max_uncles=0, k=12, epochs=12, replicate=r, **kw)
))["mean_ratio"].iloc[6:].mean() for r in range(4)]
return float(np.mean(vals))
sparse = mean_ratio(degree=2, link_latency_mean=6.0)
dense = mean_ratio(degree=16, link_latency_mean=1.0)
assert sparse < dense # more forks -> lower estimate
# --- windowed fork choice ---------------------------------------------------
@pytest.mark.parametrize("kw", [
dict(topology="regular", degree=6, link_latency_mean=3.0, max_uncles=4),
dict(topology="regular", degree=2, link_latency_mean=8.0, max_uncles=2), # sparse: big H
dict(topology="full_mesh", latency=5, max_uncles=2),
dict(topology="full_mesh", latency=0, max_uncles=0),
# blend: mixing delays are Uniform-bounded, so the windowed horizon stays exact
dict(topology="blend", degree=6, link_latency_mean=1.0, blend_hops=3, blend_delay_max=3.0,
max_uncles=4),
dict(topology="blend", degree=4, link_latency_mean=2.0, blend_hops=2, blend_delay_max=5.0,
max_uncles=2),
])
def test_windowed_fork_choice_matches_full_scan(kw):
# Deterministic latency (jitter=0): the windowed horizon must be BIT-IDENTICAL to a full scan.
# windowed_fork_choice is not in key(), so both share the same seed/inputs. prune_arrival is
# disabled here so both keep a full matrix A (the pruned==full parity is test_prune_* below).
tw, Aw, _ = _build(SimConfig(n_nodes=150, k=10, windowed_fork_choice=True,
prune_arrival=False, **kw))
tf, Af, _ = _build(SimConfig(n_nodes=150, k=10, windowed_fork_choice=False,
prune_arrival=False, **kw))
np.testing.assert_array_equal(tw.parent, tf.parent)
np.testing.assert_array_equal(tw.height, tf.height)
np.testing.assert_array_equal(Aw, Af)
assert tw.uncles == tf.uncles
def test_jitter_warns_under_windowed():
cfg = SimConfig(n_nodes=80, topology="regular", degree=6, link_latency_mean=2.0,
jitter_mean=1.0, k=8)
with pytest.warns(RuntimeWarning, match="windowed_fork_choice"):
_build(cfg)
def test_windowed_never_builds_on_unreceived_block_under_jitter():
# jitter > 0 disables pruning (falls back to the full matrix), whose safety clamp keeps the
# tree valid even under the windowed approximation.
cfg = SimConfig(n_nodes=80, topology="regular", degree=6, link_latency_mean=2.0,
jitter_mean=2.0, k=8, max_uncles=2)
with pytest.warns(RuntimeWarning):
tree, A, _ = _build(cfg)
for b in range(1, tree.n_blocks):
v = int(tree.leader[b])
assert A[v, int(tree.parent[b])] <= int(tree.slot[b]) # producer had the parent
# --- sliding-window prune ---------------------------------------------------
@pytest.mark.parametrize("kw", [
dict(topology="full_mesh", latency=3, max_uncles=2),
dict(topology="regular", degree=8, link_latency_mean=1.0, max_uncles=3),
dict(topology="regular", degree=4, link_latency_mean=0.1, max_uncles=2,
uncle_strategy="random"),
# blend deg4 (small graph horizon, W-dominated keepspan -> heavy compaction) caught an
# off-by-one in the finalize boundary; low gdf forces many compactions.
dict(topology="blend", degree=4, link_latency_mean=0.1, blend_hops=3, blend_delay_max=2.0,
max_uncles=2, genesis_d_factor=0.05, uncle_strategy="oldest"),
dict(topology="blend", degree=4, link_latency_mean=0.1, blend_hops=3, blend_delay_max=2.0,
max_uncles=3, genesis_d_factor=0.05, uncle_strategy="random", init_dest="heterogeneous"),
])
def test_prune_matches_full_matrix(kw):
# prune_arrival is a pure memory optimisation: at jitter=0 the pruned build must reproduce the
# full matrix's per-epoch rows BIT-FOR-BIT (it is excluded from key(), so seeds match).
common = dict(n_nodes=120, k=64, epochs=6, stake_dist="pareto", link_latency_dist="geo",
init_spread=0.5)
df_prune = pd.DataFrame(run_trajectory(SimConfig(**common, **kw, prune_arrival=True)))
df_full = pd.DataFrame(run_trajectory(SimConfig(**common, **kw, prune_arrival=False)))
pd.testing.assert_frame_equal(df_prune, df_full)
def test_prune_returns_small_sliding_buffer():
# The collapsed regime that OOM-froze the full matrix: pruning must keep only a keep-span
# window, not all n_blocks.
from tsi_sim.blocktree import SlidingArrival
cfg = SimConfig(n_nodes=100, k=64, stake_dist="pareto", genesis_d_factor=0.02,
topology="regular", degree=8, link_latency_mean=0.1)
tree, arr, _ = _build(cfg)
assert isinstance(arr, SlidingArrival)
assert arr.buf.shape[1] < tree.n_blocks # buffer far narrower than the block count
# --- update_D_vec -----------------------------------------------------------
def test_update_D_vec_matches_scalar():
f, T = 1 / 30, 3000
d = np.array([1000.0, 500.0, 2000.0])
m = np.array([90, 100, 110])
got = tsi.update_D_vec(d, m, T, f, beta=1.0)
exp = np.array([tsi.update_D(float(d[i]), int(m[i]), T, f, 1.0) for i in range(3)])
np.testing.assert_allclose(got, exp)
# --- adversarial grinding (uncle suppression) -------------------------------
def test_adversary_frac_zero_is_noop():
# adversary_frac=0 -> mask None -> identical to a plain honest run (bit-for-bit rows).
kw = dict(n_nodes=150, k=16, epochs=6, stake_dist="pareto", topology="blend", degree=6,
link_latency_dist="geo", link_latency_mean=0.5, blend_hops=3, blend_delay_max=8.0,
max_uncles=2, genesis_d_factor=0.5)
a = pd.DataFrame(run_trajectory(SimConfig(**kw)))
b = pd.DataFrame(run_trajectory(SimConfig(**kw, adversary_frac=0.0)))
pd.testing.assert_frame_equal(a, b)
def test_adversary_uncle_suppression_deflates_and_is_deterministic():
kw = dict(n_nodes=200, k=32, epochs=8, stake_dist="pareto", topology="blend", degree=6,
link_latency_dist="geo", link_latency_mean=0.5, blend_hops=3, blend_delay_max=24.0,
max_uncles=2, uncle_window=300, genesis_d_factor=0.5)
honest = pd.DataFrame(run_trajectory(SimConfig(**kw)))
adv = pd.DataFrame(run_trajectory(SimConfig(**kw, adversary_frac=0.4)))
def tail(df):
return df[df.epoch >= df.epochs * 0.5].mean_ratio.mean()
assert tail(adv) < tail(honest) - 0.02 # grinding deflates D_hat (raises win rate)
adv2 = pd.DataFrame(run_trajectory(SimConfig(**kw, adversary_frac=0.4)))
pd.testing.assert_frame_equal(adv, adv2) # deterministic
def test_adversary_withhold_deflates_far_more_than_suppress():
# Withholding orphans the adversary's own blocks (won slots wasted), so counted density drops
# ~beta and D_hat deflates toward the active stake (1-beta)*D — much stronger than suppression.
kw = dict(n_nodes=300, k=64, epochs=12, stake_dist="pareto", topology="regular", degree=8,
link_latency_mean=0.3, link_latency_dist="geo", max_uncles=2, uncle_window=300,
genesis_d_factor=0.5, windowed_fork_choice=False)
def settled(**extra):
rows = pd.DataFrame(run_trajectory(SimConfig(**kw, **extra)))
return rows[lambda d: d.epoch >= 6].mean_ratio.mean()
honest = settled()
supp = settled(adversary_frac=0.3, adversary_strategy="suppress")
wh = settled(adversary_frac=0.3, adversary_strategy="withhold")
assert supp > 0.95 * honest # suppression barely moves it on sub-slot direct gossip
assert wh < 0.8 * honest # withholding deflates toward (1-0.3) = 0.7
def test_reward_attribution_backward_compat():
# adversary_frac == 0 leaves the sim bit-identical and credits no adversary blocks.
kw = dict(n_nodes=200, k=32, epochs=6, stake_dist="uniform", topology="regular", degree=6,
link_latency_mean=0.3, link_latency_dist="geo", max_uncles=1)
rows = pd.DataFrame(run_trajectory(SimConfig(**kw)))
assert (rows.adv_blocks == 0).all() and (rows.adv_block_share == 0.0).all()
assert (rows.honest_blocks > 0).all()
def test_dynamic_withhold_schedule_gates_and_is_unprofitable():
# Equal stakes -> coalition_frac == beta exactly. A withhold-then-rejoin grinder deflates D_hat
# only on its withhold epochs and earns canonical blocks only on its rejoin epochs; because it
# forfeits whole epochs to depress a difficulty that helps everyone, it earns strictly LESS
# than its stake share (reward/stake < 1) — dynamic withholding is self-punishing like static.
beta_adv = 0.3
kw = dict(n_nodes=1000, k=64, epochs=20, stake_dist="uniform", topology="regular", degree=8,
link_latency_mean=0.3, link_latency_dist="geo", max_uncles=2, uncle_window=300,
genesis_d_factor=0.5, windowed_fork_choice=False, adversary_frac=beta_adv,
adversary_strategy="withhold")
dyn = pd.DataFrame(run_trajectory(
SimConfig(**kw, adversary_period=2, adversary_withhold_epochs=1)))
tail = dyn[dyn.epoch >= 10]
wh = tail[tail.adversary_withholding]
rj = tail[~tail.adversary_withholding]
# schedule alternates; withhold epochs deflate toward active stake, rejoin epochs recover
assert wh.mean_ratio.mean() < 0.85 and rj.mean_ratio.mean() > 0.90
# coalition earns ~0 while withholding, ~beta while rejoined
assert wh.adv_block_share.mean() < 0.02
assert abs(rj.adv_block_share.mean() - beta_adv) < 0.05
# profitability: realized share of ALL canonical blocks over the run, vs stake share beta
total_blocks = (tail.adv_blocks + tail.honest_blocks).sum()
reward_over_stake = tail.adv_blocks.sum() / (beta_adv * total_blocks)
assert reward_over_stake < 1.0 # strictly unprofitable for a minority coalition

52
tools/simulators/tsi/tsi-sim-pernode/tests/test_reorg.py Normal file
View File

@ -0,0 +1,52 @@
"""Private-chain reorg-depth model: effective share, closed-form tail, MC validation."""
from __future__ import annotations
import numpy as np
from tsi_sim.reorg import alpha_effective, reorg_depth_tail, simulate_deepest_reorg
def test_alpha_effective_monotone_in_orphan_rate():
# honest forks waste honest blocks -> raise the adversary's effective share
assert alpha_effective(0.2, 0.0) == 0.2
assert alpha_effective(0.2, 0.1) > 0.2
assert alpha_effective(0.2, 0.5) > alpha_effective(0.2, 0.25)
assert alpha_effective(0.0, 0.3) == 0.0 # no adversary -> no share
def test_tail_shape():
assert reorg_depth_tail(0.3, 0) == 1.0
assert reorg_depth_tail(0.0, 3) == 0.0
assert reorg_depth_tail(0.6, 5) == 1.0 # majority -> unbounded
# geometric decay: P(>=2)/P(>=1) = beta/(1-beta)
b = 0.3
assert abs(reorg_depth_tail(b, 2) / reorg_depth_tail(b, 1) - b / (1 - b)) < 1e-12
def test_reverse_d_matches_catch_up_from_behind():
"""(beta/(1-beta))**d == P(a walker starting d behind ever reaches 0) — the reorg tail."""
rng = np.random.default_rng(7)
beta, d = 0.3, 3
hits = 0
trials = 40000
horizon = 4000
up = rng.random((trials, horizon)) < beta
for row in up:
pos = -d
for step in row:
pos += 1 if step else -1
if pos >= 0:
hits += 1
break
mc = hits / trials
cf = reorg_depth_tail(beta, d)
assert abs(mc - cf) < 0.02 # 0.0937 closed form
def test_simulate_realized_depths_are_shallow_and_bounded():
rng = np.random.default_rng(1)
d = simulate_deepest_reorg(alpha_effective(0.3, 0.0), 500_000, rng)
assert d.size > 1000
assert d.min() >= 1
assert d.mean() < 2.0 # typical opportunistic reorg is ~1 deep

34
tools/simulators/tsi/tsi-sim-pernode/tests/test_rng.py Normal file
View File

@ -0,0 +1,34 @@
import numpy as np
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
from tsi_sim.rng import rng_for, seedseq_for
def test_seedseq_and_rng_deterministic():
cfg = SimConfig(k=8, epochs=3)
a = np.random.default_rng(seedseq_for(cfg)).random(5)
b = rng_for(cfg).random(5)
np.testing.assert_array_equal(a, b)
def test_distinct_configs_get_distinct_streams():
c0 = SimConfig(k=8, epochs=3, latency=0)
c1 = SimConfig(k=8, epochs=3, latency=1)
assert not np.array_equal(rng_for(c0).random(4), rng_for(c1).random(4))
def test_trajectory_is_order_independent_and_reproducible():
cfg = SimConfig(n_nodes=300, topology="regular", degree=8, k=8, epochs=6,
link_latency_mean=2.0, max_uncles=2)
r1 = run_trajectory(cfg)
r2 = run_trajectory(cfg)
assert [row["mean_ratio"] for row in r1] == [row["mean_ratio"] for row in r2]
def test_replicates_differ():
a = run_trajectory(SimConfig(n_nodes=300, topology="regular", k=8, epochs=6,
link_latency_mean=2.0, replicate=0))
b = run_trajectory(SimConfig(n_nodes=300, topology="regular", k=8, epochs=6,
link_latency_mean=2.0, replicate=1))
assert a[-1]["mean_ratio"] != b[-1]["mean_ratio"]

141
tools/simulators/tsi/tsi-sim-pernode/tests/test_selfish.py Normal file
View File

@ -0,0 +1,141 @@
"""Selfish / private-chain withholding model (§6.6)."""
import numpy as np
import pytest
from tsi_sim.selfish import (
RaceResult,
RewardParams,
honest_reward_recovery,
race_from_alpha,
reward_shares,
selfish_revenue_closed_form,
selfish_threshold,
simulate_selfish,
tsi_dhat_ratio,
)
from tsi_sim.selfish_mdp import optimal_selfish_revenue
@pytest.mark.parametrize("gamma", [0.0, 0.5, 1.0])
@pytest.mark.parametrize("alpha", [0.1, 0.2, 1 / 3, 0.4, 0.45])
def test_race_matches_eyal_sirer_closed_form(alpha, gamma):
# The SM1 simulation must reproduce the analytic relative revenue within MC noise.
rng = np.random.default_rng(12345)
r = race_from_alpha(alpha, 2_000_000, gamma, rng)
assert abs(r.revenue_share - selfish_revenue_closed_form(alpha, gamma)) < 0.004
@pytest.mark.parametrize("gamma", [0.0, 0.5, 1.0])
def test_block_conservation(gamma):
# Every mined block is exactly one of: adversary-canonical, honest-canonical, or orphaned
# (honest or adversary). This must hold at all gamma (the tie race must account for its blocks).
r = race_from_alpha(0.4, 1_000_000, gamma, np.random.default_rng(7))
assert r.adv + r.hon + r.orphan_hon + r.orphan_adv == r.events
def test_honest_only_has_no_orphans_and_stake_share():
# alpha = 0 -> every block honest, none orphaned, adversary share 0.
rng = np.random.default_rng(1)
r = race_from_alpha(0.0, 100_000, 0.0, rng)
assert r.adv == 0 and r.orphan_hon == 0 and r.orphan_adv == 0
assert r.revenue_share == 0.0 and r.density_fraction == 1.0
def test_selfish_is_profitable_above_threshold_only():
# Below the gamma=0 threshold (1/3) selfish under-earns; above it over-earns its stake.
rng = np.random.default_rng(7)
below = race_from_alpha(0.25, 2_000_000, 0.0, rng)
above = race_from_alpha(0.40, 2_000_000, 0.0, rng)
assert below.revenue_share < 0.25 # honest mining is better here
assert above.revenue_share > 0.40 # selfish premium: earns more than its stake
assert abs(selfish_threshold(0.0) - 1 / 3) < 1e-9
def test_selfish_orphans_honest_blocks_and_deflates_density():
# A profitable selfish attack orphans honest blocks, so the counted canonical density < 1,
# which is exactly what deflates D_hat; uncle recovery restores it monotonically toward 1.
rng = np.random.default_rng(3)
r = race_from_alpha(0.4, 1_000_000, 0.0, rng)
assert r.orphan_hon > 0
assert r.density_fraction < 1.0
d0 = tsi_dhat_ratio(r, uncle_recovery=0.0)
d_half = tsi_dhat_ratio(r, uncle_recovery=0.5)
d1 = tsi_dhat_ratio(r, uncle_recovery=1.0)
assert d0 < d_half < d1 <= 1.0 + 1e-9
assert abs(d0 - r.density_fraction) < 1e-9
def test_simulate_selfish_is_deterministic_given_stream_and_seed():
is_adv = np.random.default_rng(0).random(50_000) < 0.35
a = simulate_selfish(is_adv, 0.5, np.random.default_rng(99))
b = simulate_selfish(is_adv, 0.5, np.random.default_rng(99))
assert isinstance(a, RaceResult) and (a.adv, a.hon) == (b.adv, b.hon)
# --- optimal-selfish MDP (Sapirshtein) ---------------------------------------------------------
@pytest.mark.parametrize("alpha,gamma", [(0.1, 0.0), (0.3, 0.0), (0.4, 0.0), (0.4, 0.5)])
def test_optimal_selfish_dominates_sm1_and_honest(alpha, gamma):
# The optimal policy is never worse than SM1 or than honest mining (both are feasible policies).
opt = optimal_selfish_revenue(alpha, gamma, cap=16, iters=1500)
assert opt >= selfish_revenue_closed_form(alpha, gamma) - 3e-3
assert opt >= alpha - 3e-3
def test_optimal_selfish_is_honest_below_threshold():
# Below the gamma=0 threshold (1/3) no deviation beats honest: optimal == alpha.
assert abs(optimal_selfish_revenue(0.25, 0.0, cap=16, iters=1500) - 0.25) < 3e-3
# --- uncle-reward model ------------------------------------------------------------------------
def test_uncle_reward_shrinks_selfish_share_and_compensates_honest():
# A profitable selfish attack orphans honest blocks. Paying uncle rewards to those orphans
# raises honest total reward, so the attacker's REWARD share falls below its block share, and
# honest miners recover more of their mined value.
r = race_from_alpha(0.4, 500_000, 0.0, np.random.default_rng(2))
block_share = r.revenue_share
s0 = reward_shares(r, RewardParams(w_uncle=0.0)).adv_reward_share
s_half = reward_shares(r, RewardParams(w_uncle=0.5)).adv_reward_share
s_full = reward_shares(r, RewardParams(w_uncle=1.0)).adv_reward_share
assert abs(s0 - block_share) < 1e-9 # no uncle reward -> reward share == block share
assert s_full < s_half < s0 # more uncle reward -> smaller attacker share
# honest fairness: recovery rises monotonically with the uncle reward toward 1.0
rec0 = honest_reward_recovery(r, RewardParams(w_uncle=0.0))
rec1 = honest_reward_recovery(r, RewardParams(w_uncle=1.0))
assert rec0 < rec1 <= 1.0 + 1e-9
def test_uncle_reward_is_noop_without_orphans():
# Honest mining (no orphans) -> uncle rewards change nothing; share stays at the stake.
r = race_from_alpha(0.0, 50_000, 0.0, np.random.default_rng(5))
assert reward_shares(r, RewardParams(w_uncle=1.0, w_nephew=0.5)).adv_reward_share == 0.0
assert honest_reward_recovery(r, RewardParams(w_uncle=1.0)) == 1.0
def test_uncle_rewards_backfire_without_mandate_but_mandate_neutralises():
# The strategic result (report §6.7/§6.8): without a mandate the attacker suppresses honest
# references AND self-uncles its own lost blocks, so an uncle reward RAISES its share above the
# block share; a mandatory-inclusion schedule instead drives it down to ~stake (break-even).
r = race_from_alpha(0.4, 1_000_000, 0.0, np.random.default_rng(1))
block = r.revenue_share
rp_supp = RewardParams(w_uncle=1.0, p_ref=0.0, p_ref_adv=1.0)
suppress = reward_shares(r, rp_supp).adv_reward_share
mandate = reward_shares(r, RewardParams.mandatory(w_uncle=1.0)).adv_reward_share
assert suppress > block # self-uncle + suppression -> uncle reward helps the attacker
assert mandate < block # forced honest-orphan compensation -> premium shrinks
assert abs(mandate - 0.40) < 0.03 # pushed to ~stake (break-even)
def test_self_uncle_recovery_is_monotone_in_p_ref_adv():
# Recovering more of the attacker's own lost blocks (higher p_ref_adv) raises its reward share.
r = race_from_alpha(0.42, 800_000, 0.5, np.random.default_rng(4))
s = [reward_shares(r, RewardParams(w_uncle=0.8, p_ref=0.0, p_ref_adv=pa)).adv_reward_share
for pa in (0.0, 0.5, 1.0)]
assert s[0] < s[1] < s[2]
def test_farming_profitable_iff_wu_plus_wn_exceeds_one():
# A self-farmer orphaning a canonical win to uncle it collects w_uncle (producer) + w_nephew
# (self-nephew); the marginal per-slot payoff vs an honest block (=1) is exactly w_u + w_n.
for wu, wn, profitable in [(0.5, 0.3, False), (0.875, 0.03125, False), (0.9, 0.15, True)]:
assert ((wu + wn) > 1.0) == profitable

86
tools/simulators/tsi/tsi-sim-pernode/tests/test_slot_counting.py Normal file
View File

@ -0,0 +1,86 @@
"""Corrected slot-based density counting: one count per slot, never more."""
from __future__ import annotations
import numpy as np
import pandas as pd
from tsi_sim.blocktree import BlockTree
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
from tsi_sim.theory import block_count_ceiling
from tsi_sim.tsi import density_m
def make_tree(slots, parents, heights, uncles):
n = len(slots)
return BlockTree(
slot=np.array(slots, np.int64),
parent=np.array(parents, np.int64),
height=np.array(heights, np.int64),
leader=np.zeros(n, np.int64),
uncles=uncles,
)
def test_same_slot_co_winner_uncle_not_counted():
"""An uncle sharing a canonical block's slot must not add a count (slot already won)."""
# canonical 1(slot0), 3(slot2); orphan 2 ALSO at slot0 (co-winner), referenced by 3.
tree = make_tree(
slots=[-1, 0, 0, 2],
parents=[-1, 0, 0, 1],
heights=[0, 1, 1, 2],
uncles=[(), (), (), (2,)],
)
canonical = [3, 1]
assert density_m(tree, canonical, T=10) == 2 # slots {0, 2} — uncle adds nothing
assert density_m(tree, canonical, T=10, legacy_block_count=True) == 3 # the old bug
def test_multiple_uncles_same_slot_count_once():
"""Two referenced orphans in the same (non-canonical) slot count as one recovered slot."""
# canonical 1(slot0), 4(slot3); orphans 2 and 3 BOTH at slot1, both referenced.
tree = make_tree(
slots=[-1, 0, 1, 1, 3],
parents=[-1, 0, 0, 0, 1],
heights=[0, 1, 1, 1, 2],
uncles=[(), (), (), (), (2, 3)],
)
canonical = [4, 1]
assert density_m(tree, canonical, T=10) == 3 # slots {0, 1, 3}
assert density_m(tree, canonical, T=10, legacy_block_count=True) == 4 # the old bug
def test_distinct_slot_uncles_still_counted():
"""The fix must not lose genuinely distinct recovered slots."""
tree = make_tree(
slots=[-1, 0, 1, 2, 3],
parents=[-1, 0, 0, 0, 1],
heights=[0, 1, 1, 1, 2],
uncles=[(), (), (), (), (2, 3)],
)
canonical = [4, 1]
assert density_m(tree, canonical, T=10) == 4 # slots {0, 1, 2, 3}
def test_zero_delay_equilibrium_is_one_not_ceiling():
"""The c(f) ceiling was the bug: corrected counting equilibrates at 1.0 with uncles."""
base = dict(n_nodes=300, stake_dist="uniform", topology="full_mesh", latency=0,
max_uncles=2, uncle_window=300, k=64, epochs=24, genesis_d_factor=1.0)
tails = []
for rep in range(3):
df = pd.DataFrame(run_trajectory(SimConfig(**base, replicate=rep)))
tails.append(df[df.epoch >= 8].mean_ratio.mean())
assert abs(np.mean(tails) - 1.0) < 0.015
def test_legacy_flag_reproduces_the_ceiling():
base = dict(n_nodes=300, stake_dist="uniform", topology="full_mesh", latency=0,
max_uncles=2, uncle_window=300, k=64, epochs=24, genesis_d_factor=1.0)
tails = []
for rep in range(3):
df = pd.DataFrame(run_trajectory(
SimConfig(**base, legacy_block_count=True, replicate=rep)))
tails.append(df[df.epoch >= 8].mean_ratio.mean())
c = block_count_ceiling(SimConfig().f)
assert abs(np.mean(tails) - c) < 0.015

40
tools/simulators/tsi/tsi-sim-pernode/tests/test_stake.py Normal file
View File

@ -0,0 +1,40 @@
import numpy as np
from tsi_sim.config import SimConfig
from tsi_sim.stake import make_stake
def _rng():
return np.random.default_rng(0)
def test_uniform_equal_and_sum():
cfg = SimConfig(n_nodes=100, stake_dist="uniform", total_stake=1e9)
w = make_stake(cfg, _rng())
assert w.shape == (100,)
assert np.allclose(w, w[0]) # equal weights
assert abs(w.sum() - 1e9) < 1e-3
def test_uniform_random_varies_but_sums():
cfg = SimConfig(n_nodes=200, stake_dist="uniform", uniform_random=True, total_stake=5e8)
w = make_stake(cfg, _rng())
assert w.std() > 0
assert abs(w.sum() - 5e8) < 1e-2
assert np.all(w >= 0)
def test_pareto_sum_fixed_and_heavier_tailed():
n = 5000
uni = make_stake(SimConfig(n_nodes=n, stake_dist="uniform", total_stake=1e9), _rng())
par = make_stake(SimConfig(n_nodes=n, stake_dist="pareto", pareto_shape=1.16,
total_stake=1e9), _rng())
assert abs(par.sum() - 1e9) < 1.0
assert par.max() > uni.max() * 5 # heavy tail: richest holds far more
assert np.all(par >= 0)
def test_total_stake_fixed_across_distributions():
uni = make_stake(SimConfig(n_nodes=1000, stake_dist="uniform", total_stake=7e8), _rng())
par = make_stake(SimConfig(n_nodes=1000, stake_dist="pareto", total_stake=7e8), _rng())
assert abs(uni.sum() - par.sum()) < 1.0 # comparability guarantee

182
tools/simulators/tsi/tsi-sim-pernode/tests/test_sweep.py Normal file
View File

@ -0,0 +1,182 @@
"""Worker-count memory planning: analytic estimate, RAM cap, and the calibration probe."""
from __future__ import annotations
import os
import pytest
from tsi_sim import sweep
from tsi_sim.config import SimConfig
# The explosion/throttle behaviours below are properties of the FULL (unpruned) arrival matrix,
# so they pin prune_arrival=False; the pruned path's much smaller footprint is tested separately.
FULL = dict(prune_arrival=False)
def test_estimate_grows_with_n_and_k():
small = sweep.estimate_worker_bytes(SimConfig(n_nodes=1000, k=256, **FULL))
big_n = sweep.estimate_worker_bytes(SimConfig(n_nodes=4000, k=256, **FULL))
big_k = sweep.estimate_worker_bytes(SimConfig(n_nodes=1000, k=2160, **FULL))
assert big_n > small and big_k > small # both N (via A + N^2) and k (via n_blocks)
def test_low_genesis_d_factor_explodes_block_estimate():
# The collapsed-D_est regime: a 100x-low genesis estimate inflates lottery wins ~100x, so
# the peak-epoch block count explodes. Without pruning that blows up A (the OOM that froze the
# box); the estimate must reflect it.
hi = SimConfig(n_nodes=1000, k=2160, stake_dist="pareto", genesis_d_factor=0.5, **FULL)
lo = SimConfig(n_nodes=1000, k=2160, stake_dist="pareto", genesis_d_factor=0.01, **FULL)
assert sweep.expected_peak_blocks(lo) > 20 * sweep.expected_peak_blocks(hi)
assert sweep.expected_peak_blocks(lo) > 10 * (10 * lo.k) # far past the ~10*k equilibrium
assert sweep.estimate_worker_bytes(lo) > 10 * 1024**3 # unpruned -> tens of GB/worker
def test_prune_shrinks_estimate_and_keeps_all_workers(monkeypatch):
# The whole point of prune_arrival (default on): the same gdf=0.01 config no longer needs a
# huge per-worker matrix, so it stays well under a GB and does NOT throttle the worker pool.
monkeypatch.setattr(sweep, "_total_ram_bytes", lambda: 51 * 1024**3)
lo = SimConfig(n_nodes=1000, k=2160, stake_dist="pareto", genesis_d_factor=0.01) # prune on
assert sweep.estimate_worker_bytes(lo) < 1 * 1024**3 # vs >10 GB unpruned
plan = sweep.plan_workers(requested=-1, configs=[lo], mem_frac=0.7, calibrate="never")
assert plan.n_jobs == (os.cpu_count() or 1) # all cores, no throttle
def test_low_gdf_caps_workers_hard(monkeypatch):
monkeypatch.setattr(sweep, "_total_ram_bytes", lambda: 51 * 1024**3)
lo = SimConfig(n_nodes=1000, k=2160, stake_dist="pareto", genesis_d_factor=0.01, **FULL)
plan = sweep.plan_workers(requested=-1, configs=[lo], mem_frac=0.7, calibrate="never")
assert plan.n_jobs * plan.per_worker_bytes <= int(0.7 * 51 * 1024**3) + plan.per_worker_bytes
assert plan.n_jobs <= 2 # unpruned: was 14 -> ~206 GB; now a couple
def test_auto_calibration_fires_on_bytes_threshold(monkeypatch):
# Even at small N, a heavy per-worker estimate (low gdf, unpruned) must trigger the probe.
seen = {}
def _probe(cfg, **k):
seen["n"] = cfg.n_nodes
return 3 * 1024**3
monkeypatch.setattr(sweep, "_total_ram_bytes", lambda: 64 * 1024**3)
monkeypatch.setattr(sweep, "measure_worker_bytes", _probe)
lo = SimConfig(n_nodes=1000, k=2160, stake_dist="pareto", genesis_d_factor=0.01, **FULL)
plan = sweep.plan_workers(requested=-1, configs=[lo], mem_frac=0.7, calibrate="auto")
assert plan.calibrated and seen.get("n") == 1000 # probed despite N <= 2000
def test_arrival_matrix_guard_raises_before_allocation(monkeypatch):
# Budget chosen so path_latency (100x100 -> ~0.18 MB) fits but the block-exploded A
# (gdf=0.01 -> ~30k blocks -> ~25 MB) does not, so the *arrival-matrix* guard is what fires.
from tsi_sim.engine import run_trajectory
from tsi_sim.memguard import ArrivalMatrixTooLarge
monkeypatch.setenv("TSI_ARRIVAL_BYTES_BUDGET", str(1_000_000))
cfg = SimConfig(n_nodes=100, k=32, epochs=1, topology="regular", degree=4,
stake_dist="pareto", genesis_d_factor=0.01, prune_arrival=False)
with pytest.raises(ArrivalMatrixTooLarge, match="arrival matrix A"):
run_trajectory(cfg)
def test_pruned_arrival_buffer_guard_raises(monkeypatch):
# The pruned buffer is guarded too. Budget (0.4 MB) is chosen so path_latency (~0.18 MB) fits
# but the sliding buffer (~1.4 MB, vs a ~21 MB full matrix here) does not — so the *pruned*
# guard fires, and the message distinguishes it from the full-matrix one.
from tsi_sim.engine import run_trajectory
from tsi_sim.memguard import ArrivalMatrixTooLarge
monkeypatch.setenv("TSI_ARRIVAL_BYTES_BUDGET", str(400_000))
cfg = SimConfig(n_nodes=100, k=32, epochs=1, topology="regular", degree=4,
stake_dist="pareto", genesis_d_factor=0.01, prune_arrival=True)
with pytest.raises(ArrivalMatrixTooLarge, match="pruned arrival buffer"):
run_trajectory(cfg)
def test_path_latency_guard_raises_before_allocation(monkeypatch):
# The (N x N) path_latency is guarded too, BEFORE the arrival matrix is ever reached
# (it is built first in run_trajectory) — so a large-N / small-n_blocks config can't slip past.
from tsi_sim import topology
from tsi_sim.memguard import ArrivalMatrixTooLarge
monkeypatch.setenv("TSI_ARRIVAL_BYTES_BUDGET", "1024")
cfg = SimConfig(n_nodes=300, k=8, topology="full_mesh")
import numpy as np
with pytest.raises(ArrivalMatrixTooLarge):
topology.build_path_latency(cfg, np.random.default_rng(0))
def test_unset_budget_defaults_to_ram_fraction(monkeypatch):
# "0"/unset is NOT unlimited: it resolves to a fraction of physical RAM, so a bare
# run_trajectory / tsi-verify / mem_frac=0 run still has an absolute per-process ceiling.
from tsi_sim import memguard
from tsi_sim.engine import run_trajectory
monkeypatch.setattr(memguard, "total_ram_bytes", lambda: 32 * 1024**3)
monkeypatch.delenv("TSI_ARRIVAL_BYTES_BUDGET", raising=False)
assert memguard.arrival_budget_bytes() == int(memguard.DEFAULT_BUDGET_FRAC * 32 * 1024**3)
monkeypatch.setenv("TSI_ARRIVAL_BYTES_BUDGET", "0")
assert memguard.arrival_budget_bytes() == int(memguard.DEFAULT_BUDGET_FRAC * 32 * 1024**3)
monkeypatch.setenv("TSI_ARRIVAL_BYTES_BUDGET", str(5 * 1024**3))
assert memguard.arrival_budget_bytes() == 5 * 1024**3 # explicit positive wins
monkeypatch.delenv("TSI_ARRIVAL_BYTES_BUDGET", raising=False)
rows = run_trajectory(SimConfig(n_nodes=80, k=8, epochs=1, topology="regular", degree=4))
assert rows # small config well under the ceiling
def test_mem_frac_zero_disables_cap():
plan = sweep.plan_workers(requested=4, configs=[SimConfig(n_nodes=9999, k=2160)],
mem_frac=0.0, calibrate="never")
assert plan.n_jobs == 4 and not plan.calibrated
def test_estimate_caps_workers_when_grid_is_heavy(monkeypatch):
# Fix RAM so the cap is machine-independent: 16 GB budget*0.7 = 11.2 GB, config >> that -> 1.
monkeypatch.setattr(sweep, "_total_ram_bytes", lambda: 16 * 1024**3)
heavy = SimConfig(n_nodes=20200, k=2160, **FULL) # unpruned -> tens of GB/worker
plan = sweep.plan_workers(requested=8, configs=[heavy], mem_frac=0.7, calibrate="never")
assert plan.n_jobs == 1 and plan.per_worker_bytes > 8 * 1024**3
def test_auto_does_not_probe_at_or_below_threshold(monkeypatch):
called = False
def _boom(*a, **k):
nonlocal called
called = True
return 1
monkeypatch.setattr(sweep, "measure_worker_bytes", _boom)
plan = sweep.plan_workers(requested=-1, configs=[SimConfig(n_nodes=2000, k=2160)],
mem_frac=0.7, calibrate="auto")
assert not called and not plan.calibrated # N<=2000 uses the analytic estimate only
def test_auto_probes_above_threshold_and_uses_measurement(monkeypatch):
measured = 2 * 1024**3 # 2 GB peak RSS reported by the probe
ram = 64 * 1024**3
monkeypatch.setattr(sweep, "_total_ram_bytes", lambda: ram)
monkeypatch.setattr(sweep, "measure_worker_bytes", lambda cfg, **k: measured)
plan = sweep.plan_workers(requested=-1, configs=[SimConfig(n_nodes=3000, k=2160)],
mem_frac=0.7, calibrate="auto")
assert plan.calibrated
assert plan.per_worker_bytes == int(measured * 1.1) # 10% headroom over the measurement
fit = int(0.7 * ram // plan.per_worker_bytes)
assert plan.n_jobs == min(os.cpu_count() or 1, fit)
def test_probe_failure_falls_back_to_estimate(monkeypatch):
monkeypatch.setattr(sweep, "measure_worker_bytes", lambda cfg, **k: None) # probe unavailable
cfg = SimConfig(n_nodes=3000, k=2160)
plan = sweep.plan_workers(requested=-1, configs=[cfg], mem_frac=0.7, calibrate="always")
assert not plan.calibrated
assert plan.per_worker_bytes == sweep.estimate_worker_bytes(cfg)
def test_ru_maxrss_unit_normalisation(monkeypatch):
monkeypatch.setattr(sweep.sys, "platform", "darwin")
assert sweep._ru_maxrss_bytes(1000) == 1000 # macOS already bytes
monkeypatch.setattr(sweep.sys, "platform", "linux")
assert sweep._ru_maxrss_bytes(1000) == 1000 * 1024 # Linux reports kibibytes
def test_measure_worker_bytes_real_spawn():
# Exercise the real probe on a tiny config. Spawn needs an importable __main__; if the test
# environment cannot bootstrap the child, the probe returns None (graceful) and we skip.
got = sweep.measure_worker_bytes(SimConfig(n_nodes=40, k=6, epochs=2, degree=4), timeout=180)
if got is None:
pytest.skip("spawn-based calibration probe unavailable in this environment")
assert got > 30 * 1024**2 # any real Python+numpy worker RSS clears tens of MB

40
tools/simulators/tsi/tsi-sim-pernode/tests/test_theory.py Normal file
View File

@ -0,0 +1,40 @@
import numpy as np
from tsi_sim import theory
F = 1 / 30
T = 10000
def test_expected_ratio_unbiased_at_q1():
assert abs(float(theory.expected_ratio(F, 1.0)) - 1.0) < 1e-12
def test_expected_ratio_monotone_in_q():
qs = np.linspace(0.5, 1.0, 20)
er = theory.expected_ratio(F, qs)
assert np.all(np.diff(er) > 0) # accuracy improves as q -> 1
assert np.all(er <= 1.0 + 1e-12) # always an underestimate
def test_variance_bound_matches_at_q1():
v = float(theory.variance_ratio(F, 1.0, T))
assert abs(v - theory.variance_bound(F, T)) < 1e-15
def test_optimal_beta_is_half_stability_bound():
for q in (0.7, 0.85, 0.95):
opt = float(theory.optimal_beta(F, q))
bound = float(theory.beta_stability_bound(F, q))
assert abs(opt - bound / 2) < 1e-12
def test_block_count_ceiling_above_one():
c = theory.block_count_ceiling(F)
assert 1.015 < c < 1.02 # -ln(1-1/30)/(1/30) ~ 1.01705
def test_fixed_point_bias_about_one_percent():
b = theory.fixed_point_bias(F)
assert abs(b - (F / (33 / 1000))) < 1e-12
assert 1.005 < b < 1.02

49
tools/simulators/tsi/tsi-sim-pernode/tests/test_theory_convergence.py Normal file
View File

@ -0,0 +1,49 @@
"""End-to-end per-node statistical checks against closed-form theory (scaled k)."""
import numpy as np
import pandas as pd
import pytest
from tsi_sim.config import SimConfig
from tsi_sim.engine import run_trajectory
from tsi_sim.theory import expected_ratio
F = 1 / 30
def _tail_mean(col, **cfg):
reps = cfg.pop("reps", 6)
burn = cfg["epochs"] // 2
vals = []
for r in range(reps):
df = pd.DataFrame(run_trajectory(SimConfig(replicate=r, **cfg)))
vals.append(df[col].iloc[burn:].mean())
return float(np.mean(vals))
def test_baseline_exact_without_forks():
# Full mesh, no latency, no uncles: active-slot rate == f, so the estimate is unbiased
# and (full mesh) every node agrees exactly.
ratio = _tail_mean("mean_ratio", n_nodes=300, stake_dist="uniform", topology="full_mesh",
latency=0, max_uncles=0, k=48, epochs=30, genesis_d_factor=0.5, reps=6)
assert abs(ratio - 1.0) < 0.02
@pytest.mark.slow
def test_full_mesh_mean_matches_expected_ratio():
cfg = dict(n_nodes=300, stake_dist="uniform", topology="full_mesh", latency=6,
max_uncles=0, k=96, epochs=45, genesis_d_factor=0.5)
ratio = _tail_mean("mean_ratio", reps=8, **cfg)
q = _tail_mean("mean_q", reps=8, **cfg)
assert abs(ratio - float(expected_ratio(F, q))) < 0.03
@pytest.mark.slow
def test_uncles_recover_mean_accuracy_under_graph():
common = dict(n_nodes=300, stake_dist="uniform", topology="regular", degree=8,
link_latency_mean=4.0, uncle_strategy="oldest", k=96, epochs=45,
genesis_d_factor=0.5)
r0 = _tail_mean("mean_ratio", max_uncles=0, reps=8, **common)
r4 = _tail_mean("mean_ratio", max_uncles=4, reps=8, **common)
assert abs(r4 - 1) < abs(r0 - 1) # uncles reduce the error
assert abs(r4 - 1) < 0.03 # residual is the small block-count overshoot

89
tools/simulators/tsi/tsi-sim-pernode/tests/test_tsi_counting.py Normal file
View File

@ -0,0 +1,89 @@
import numpy as np
from tsi_sim.blocktree import BlockTree
from tsi_sim.tsi import density_m, referenced_uncle_ids, slot_stats, update_D
def make_tree(slots, parents, heights, uncles):
n = len(slots)
return BlockTree(
slot=np.array(slots, np.int64),
parent=np.array(parents, np.int64),
height=np.array(heights, np.int64),
leader=np.zeros(n, np.int64),
uncles=uncles,
)
def test_density_counts_honest_plus_deduped_uncles_in_window():
# canonical 1(slot0),4(slot3); orphans 2(slot1),3(slot2). block4 refs uncles 2 and 3.
tree = make_tree(
slots=[-1, 0, 1, 2, 3],
parents=[-1, 0, 0, 0, 1],
heights=[0, 1, 1, 1, 2],
uncles=[(), (), (), (), (2, 3)],
)
canonical = [4, 1] # tip-first
# window T=10 includes all slots
assert density_m(tree, canonical, T=10) == 4 # 2 honest (slots 0,3) + 2 uncles (1,2)
# window T=2 excludes slots 2,3 -> honest {slot0}=1, uncle slot1=1 (slot2 excluded)
assert density_m(tree, canonical, T=2) == 2
def test_uncle_counted_by_own_slot_and_deduped():
tree = make_tree(
slots=[-1, 0, 5, 1],
parents=[-1, 0, 1, 0],
heights=[0, 1, 2, 1],
uncles=[(), (), (3,), ()], # block2 (slot5) references orphan 3 (slot1)
)
canonical = [2, 1]
assert referenced_uncle_ids(tree, canonical) == {3}
# uncle counted by its OWN slot (1), so window T=2 includes it
assert density_m(tree, canonical, T=2) == 2 # honest slot0 + uncle slot1
# window that excludes the uncle's own slot
assert density_m(tree, canonical, T=1) == 1 # only honest slot0
def test_slot_stats_q_and_qeff():
# active slots 0,1,2 in window; honest occupies 0,2; orphan at slot1 recovered by uncle
tree = make_tree(
slots=[-1, 0, 1, 2],
parents=[-1, 0, 0, 1],
heights=[0, 1, 1, 2],
uncles=[(), (), (), (2,)], # block3 refs orphan 2 (slot1)
)
canonical = [3, 1]
active = np.array([0, 1, 2], np.int64)
ref = referenced_uncle_ids(tree, canonical)
ss = slot_stats(tree, canonical, ref, active, T=10)
assert ss.n_active == 3
assert ss.n_honest == 2 # slots 0 and 2
assert ss.n_recovered == 1 # slot 1 recovered via uncle
assert abs(ss.q - 2 / 3) < 1e-9
assert abs(ss.q_eff - 1.0) < 1e-9
def test_update_D_fixed_point():
f, T = 1 / 30, 3000
m = int(round(T * f)) # measured density == f -> D unchanged
assert abs(update_D(1000.0, m, T, f, beta=1.0) - 1000.0) < 1e-6
# measured below f -> estimate drops
assert update_D(1000.0, m - 20, T, f, 1.0) < 1000.0
# clamp at 1
assert update_D(1.0, 0, T, f, 1.0) == 1.0
def test_update_D_fixed_point_mode_targets_truncated_f():
# In fixed-point mode the target rate is int(f*PRECISION)/PRECISION, slightly below f.
from tsi_sim.tsi import PRECISION
f, T = 1 / 30, 1_000_000
f_p = int(f * PRECISION) / PRECISION
m = 34000
# For the same measured density, fixed-point (lower target f_p) raises the estimate
# more than exact-f, i.e. it is systematically higher (now only ~1e-5 at PRECISION=1e6).
assert (update_D(1000.0, m, T, f, 1.0, fixed_point=True)
>= update_D(1000.0, m, T, f, 1.0, fixed_point=False))
# A density of exactly f_p is the fixed-point fixed point (estimate unchanged).
m_trunc = int(round(f_p * T)) # exact when f_p*T is integral (T=1e6)
assert abs(update_D(1000.0, m_trunc, T, f, 1.0, fixed_point=True) - 1000.0) < 1e-6

110
tools/simulators/tsi/tsi-sim-pernode/tests/test_uncles.py Normal file
View File

@ -0,0 +1,110 @@
import numpy as np
from tsi_sim.blocktree import BlockTree
from tsi_sim.config import SimConfig
from tsi_sim.uncles import annotate_uncles
def make_tree(slots, parents, heights, leaders):
n = len(slots)
return BlockTree(
slot=np.array(slots, np.int64),
parent=np.array(parents, np.int64),
height=np.array(heights, np.int64),
leader=np.array(leaders, np.int64),
uncles=[() for _ in range(n)],
)
def _canonical_and_orphan_tree():
# genesis(0); canonical chain 1(slot0)->3(slot3)->4(slot5); orphan 2(slot1)
tree = make_tree(
slots=[-1, 0, 1, 3, 5],
parents=[-1, 0, 0, 1, 3],
heights=[0, 1, 1, 2, 3],
leaders=[-1, 0, 1, 2, 3],
)
canonical = [4, 3, 1] # tip-first
return tree, canonical
def test_oldest_selection_and_window():
tree, canonical = _canonical_and_orphan_tree()
cfg = SimConfig(max_uncles=1, uncle_window=300, uncle_strategy="oldest")
annotate_uncles(tree, canonical, cfg, np.random.default_rng(0))
# orphan 2 (slot1) is within window of block 3 (slot3) -> referenced there
referenced = {u for b in canonical for u in tree.uncles[b]}
assert referenced == {2}
def test_no_uncles_when_u_zero():
tree, canonical = _canonical_and_orphan_tree()
cfg = SimConfig(max_uncles=0)
annotate_uncles(tree, canonical, cfg, np.random.default_rng(0))
assert all(tree.uncles[b] == () for b in canonical)
def test_window_excludes_out_of_range_orphan():
tree, canonical = _canonical_and_orphan_tree()
cfg = SimConfig(max_uncles=1, uncle_window=1, uncle_strategy="oldest")
annotate_uncles(tree, canonical, cfg, np.random.default_rng(0))
# orphan 2 at slot1; nearest canonical after it is block3 at slot3 -> gap 2 > W=1
referenced = {u for b in canonical for u in tree.uncles[b]}
assert referenced == set()
def _wide_orphan_tree():
# canonical 1(0)->6(6); orphans 2,3,4,5 at slots 1,2,3,4 (all within window of block6)
tree = make_tree(
slots=[-1, 0, 1, 2, 3, 4, 6],
parents=[-1, 0, 0, 0, 0, 0, 1],
heights=[0, 1, 1, 1, 1, 1, 2],
leaders=[-1, 0, 1, 2, 3, 4, 0],
)
return tree, [6, 1] # tip-first
def test_random_strategy_deterministic_and_capped():
import numpy as np
tree_a, canon = _wide_orphan_tree()
tree_b, _ = _wide_orphan_tree()
cfg = SimConfig(max_uncles=2, uncle_window=300, uncle_strategy="random", uncle_random_p=0.5)
annotate_uncles(tree_a, canon, cfg, np.random.default_rng(7))
annotate_uncles(tree_b, canon, cfg, np.random.default_rng(7))
assert tree_a.uncles == tree_b.uncles # same seed -> identical
total = sum(len(tree_a.uncles[b]) for b in canon)
assert total <= cfg.max_uncles # capped
def test_random_p_one_matches_oldest():
import numpy as np
tree_r, canon = _wide_orphan_tree()
tree_o, _ = _wide_orphan_tree()
annotate_uncles(tree_r, canon, SimConfig(max_uncles=2, uncle_strategy="random",
uncle_random_p=1.0), np.random.default_rng(1))
annotate_uncles(tree_o, canon, SimConfig(max_uncles=2, uncle_strategy="oldest"),
np.random.default_rng(1))
assert tree_r.uncles == tree_o.uncles # p=1 deterministically takes oldest-first
def test_random_p_zero_selects_nothing():
import numpy as np
tree, canon = _wide_orphan_tree()
annotate_uncles(tree, canon, SimConfig(max_uncles=4, uncle_strategy="random",
uncle_random_p=0.0), np.random.default_rng(1))
assert all(tree.uncles[b] == () for b in canon)
def test_dedup_across_ancestors():
# Two canonical blocks both within window of the single orphan: only one references it.
tree = make_tree(
slots=[-1, 0, 1, 2, 3],
parents=[-1, 0, 0, 1, 3],
heights=[0, 1, 1, 2, 3],
leaders=[-1, 0, 9, 2, 3],
)
canonical = [4, 3, 1]
cfg = SimConfig(max_uncles=4, uncle_window=300, uncle_strategy="oldest")
annotate_uncles(tree, canonical, cfg, np.random.default_rng(0))
counts = sum(len(tree.uncles[b]) for b in canonical)
assert counts == 1 # orphan 2 referenced exactly once despite two eligible blocks