diff --git a/reports/tsi/README.md b/reports/tsi/README.md index bf02513..10866c4 100644 --- a/reports/tsi/README.md +++ b/reports/tsi/README.md @@ -854,7 +854,7 @@ The floor rises by one to two block-intervals, which is what the ~one-block-inte | 14 | 0.9965 / 0.945 | 0.9957 / 0.943 | +0.0016 ± 0.0004 | 3.5 | | 15 | 0.9964 / 0.945 | 0.9963 / 0.944 | +0.0021 ± 0.0005 | 4.3 | -The last two columns are the paired difference against **today's recipe** — uncle-anchored at `W` = 10 — so the question "does this configuration cost anything relative to what the spec does now?" is read straight off them. +The last two columns are the paired difference against **today's recipe** — uncle-anchored at `W` = 10 — so the question "does this configuration cost anything relative to what the spec does now?" is read straight off them. Two of the 32 replicates drew a stake vector on which `adversary_frac` = 0.3 is unreachable (one holder above the target) and ran a 0.09 and a 0.06 coalition instead; because they are weaker they pull every cell toward parity, which would make the crossing look *smaller* than it is. Dropping them moves nothing — the crossing stays at `W` = 12 and the interpolated zero stays at 11.98 (`scripts/w_pairing_analysis.py` reports both subsets). **`W` = 12 is the smallest window at which the parent anchor reaches parity, and the parity is exact.** Every window below it is resolvably worse: even `W` = 11, only one interval short, still costs 0.0024 at `t` = 4.5. At `W` = 12 the paired difference is `+0.00004 ± 0.00050`, `t` = 0.1 — indistinguishable from today's recipe, and `p_ref` agrees (0.938 against 0.939). Beyond it the anchor is *better* than today rather than equal (`t` = 3.5 at `W` = 14), so 12 is a floor and not a knife-edge. **Pair the anchor change with `W` = 12, or take neither**: at `W` = 10 it costs 0.0056 under this adversary, at `t` = 10.4, which is the one regime where the change is not free. diff --git a/tools/simulators/tsi/tsi-sim-pernode/configs/pref-window-anchor-fine.yaml b/tools/simulators/tsi/tsi-sim-pernode/configs/pref-window-anchor-fine.yaml new file mode 100644 index 0000000..920e818 --- /dev/null +++ b/tools/simulators/tsi/tsi-sim-pernode/configs/pref-window-anchor-fine.yaml @@ -0,0 +1,49 @@ +# Where exactly does the parent-anchored window cross today's recipe? (§6.12) — high-power run +# +# The 8-15 integer sweep put the crossing between W = 11 (-0.0024, t = -4.5) and W = 12 +# (+0.00004, t = 0.1), and recommended 12 as the smallest window reaching parity. That is a +# ~0.0005-scale claim resting on a ~0.0005 standard error, so the crossing is located to about one +# block-interval and no better. This run is built to place it more finely and to show the answer +# is not an artefact of replicate noise. +# +# Three changes, each attacking a different variance source: +# * HALF-INTEGER W across the crossing (9 to 14 in steps of 0.5). w_u = floor(W/f) is 285 slots +# at W = 9.5 against 270 at 9 and 300 at 10, so these are genuinely distinct windows and not +# re-labelled duplicates. If the crossing sits at 11.5 rather than 12, this resolves it. +# * 96 REPLICATES (from 32) — cuts the standard error on a paired difference by ~1.7x. +# * early_stop OFF, 30 EPOCHS (from early-stopped 20) — the per-run equilibrium sample goes from +# ~10 measured epochs to a full 15, which attacks the within-run noise the replicate count +# cannot touch. This is the more important of the two: replicates average out between-run +# variance, epochs average out the per-epoch sampling noise of +-0.9% (Appendix B). +# +# paired_streams stays on and is what makes any of this affordable: SimConfig._base_key() excludes +# uncle_window_anchor and window_absorption, so at a given replicate every cell draws the same +# stake vector, peering graph and lottery, and the differences are paired. Unpaired, none of these +# differences would resolve at any replicate count worth running. +# +# 11 windows x 2 anchors x 96 replicates = 2112 runs. Geometry is the capstone's, so the answer +# lands on the recommended configuration: delta_max = 8 (rho ~ 0.47), U = 2 = ceil(rho)+1, +# degree 6, k = 2160, 30% suppression adversary. Latency is in SLOTS (1 slot = 1 s). +n_nodes: [1000] # network size +stake_dist: [pareto] # heavy-tailed (realistic) stake distribution +topology: [blend] # Blend mixnet — the deployment transport +degree: [6] # peering degree of the d-regular graph +link_latency_mean: [0.5] # natural geographic transport (sub-slot) +link_latency_dist: [geo] # real-world geographic band mixture +blend_hops: [3] # the spec's Blend cascade length +blend_delay_max: [8.0] # the capstone's operating point, rho ~ 0.47 +window_absorption: [9, 9.5, 10, 10.5, 11, 11.5, 12, 12.5, 13, 13.5, 14] # half-interval steps +uncle_window_anchor: [uncle, parent] # spec rule vs the §6.12 proposal +max_uncles: [2] # the recommended cap U = ceil(rho)+1 +uncle_strategy: [oldest] # spec Uncle Selection +init_dest: [common] # per-node initial D_est from agreement +replicates: 96 # 3x the integer sweep +base: # per-run settings shared by every cell + k: 2160 # true security parameter + epochs: 30 # 15 equilibrium epochs after the 50% burn + f: 0.03333333333333333 # slot activation coefficient (1/30) + genesis_d_factor: 0.5 # start near true stake (cheap epoch 0) + early_stop: false # OFF: take the full equilibrium sample per run + adversary_frac: 0.3 # the capstone's suppression coalition + adversary_strategy: suppress # references no uncles from its own blocks + paired_streams: true # COMMON RANDOM NUMBERS across anchor and W diff --git a/tools/simulators/tsi/tsi-sim-pernode/scripts/w_pairing_analysis.py b/tools/simulators/tsi/tsi-sim-pernode/scripts/w_pairing_analysis.py new file mode 100644 index 0000000..5996e98 --- /dev/null +++ b/tools/simulators/tsi/tsi-sim-pernode/scripts/w_pairing_analysis.py @@ -0,0 +1,172 @@ +"""Where does the parent-anchored window reach parity with today's recipe? (§6.12) + +Reads a `pref-window-anchor*` sweep and answers one question: the smallest `W` at which the +parent-anchored rule is no longer resolvably worse than what the spec does today (uncle-anchored, +`W` = 10). + +Everything here is PAIRED. The sweep runs under `paired_streams`, so at a given replicate every +cell shares the stake vector, the peering graph and the lottery — the window rule is the only +difference — and the statistic is the per-replicate difference against the reference cell, not a +difference of two independently-noisy means. Unpaired, none of these differences resolve. + +Two things are reported that a bare mean would hide: + + * **the crossing, with its uncertainty** — the smallest `W` whose paired difference is not + resolvably negative (`t > -2`), plus a linear interpolation of where the difference actually + reaches zero, so "12" can be read as a grid point rather than a physical constant; + * **how many replicates ran an off-label adversary** — a Pareto draw can leave `adversary_frac` + unreachable (one holder above the target), which `engine._adversary_mask` warns about. Those + replicates run a WEAKER adversary than the label, which biases an attacked arm toward the + honest baseline. It is conservative, but a sweep quoting levels has to say how many. + +Run: python scripts/w_pairing_analysis.py [run-label-glob] +""" + +from __future__ import annotations + +import sys +import warnings +from pathlib import Path + +import numpy as np +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from tsi_sim.config import SimConfig # noqa: E402 +from tsi_sim.engine import _adversary_mask # noqa: E402 +from tsi_sim.stake import stake_for # noqa: E402 + +HERE = Path(__file__).resolve().parent.parent +RUNS = HERE / "runs" +REFERENCE = ("uncle", 10.0) # today's recipe: the thing a change has to not cost against + + +def load(pattern: str) -> tuple[pd.DataFrame, str]: + src = sorted(RUNS.glob(f"*_{pattern}/results.parquet")) + if not src: + raise SystemExit(f"no run matching *_{pattern}/results.parquet under {RUNS}") + df = pd.read_parquet(src[-1]) + return df[df.epoch >= df.epochs.iloc[0] // 2], src[-1].parent.name + + +def off_label_replicates(df: pd.DataFrame) -> tuple[list[int], dict[int, float]]: + """Replicates whose stake draw cannot realise `adversary_frac` (see module docstring).""" + row = df.iloc[0] + off, got = [], {} + for rep in sorted(df.replicate.unique()): + cfg = SimConfig(n_nodes=int(row.n_nodes), stake_dist=str(row.stake_dist), + topology=str(row.topology), degree=int(row.degree), + link_latency_mean=float(row.link_latency_mean), + link_latency_dist=str(row.link_latency_dist), + blend_hops=int(row.blend_hops), blend_delay_max=float(row.blend_delay_max), + max_uncles=int(row.max_uncles), uncle_strategy=str(row.uncle_strategy), + init_dest=str(row.init_dest), k=int(row.k), epochs=int(row.epochs), + f=float(row.f), genesis_d_factor=float(row.genesis_d_factor), + adversary_frac=float(row.adversary_frac), + adversary_strategy=str(row.adversary_strategy), paired_streams=True, + window_absorption=float(row.window_absorption), replicate=int(rep)) + stake = stake_for(cfg) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + mask = _adversary_mask(cfg, stake) + if any("not reachable" in str(c.message) for c in caught): + off.append(int(rep)) + got[int(rep)] = float(stake[mask].sum() / stake.sum()) + return off, got + + +def main() -> None: + pattern = sys.argv[1] if len(sys.argv) > 1 else "pref-window-anchor-fine" + t, label = load(pattern) + cell = t.groupby(["uncle_window_anchor", "window_absorption", "replicate"]).agg( + r=("mean_ratio", "mean"), p=("p_ref", "mean")) + base = cell.loc[REFERENCE[0]].loc[REFERENCE[1]] + windows = sorted(t.window_absorption.unique()) + reps = t.replicate.nunique() + + print(f"=== {label} ===") + print(f"{reps} replicates, {len(windows)} windows, {int(t.epochs.iloc[0])} epochs, " + f"paired against {REFERENCE[0]}-anchored W = {REFERENCE[1]:.0f}\n") + + off, got = off_label_replicates(t) + if off: + print(f"!! {len(off)}/{reps} replicates ran an OFF-LABEL adversary (unreachable on their " + f"stake draw): {off}") + print(f" realised {[round(got[r], 3) for r in off]} against a " + f"{t.adversary_frac.iloc[0]:.2f} label — weaker, so the attacked arms are " + f"conservative.\n") + else: + print(f"all {reps} replicates on-label " + f"(realised {min(got.values()):.4f}–{max(got.values()):.4f})\n") + + def table(keep: set[int] | None, title: str) -> pd.DataFrame: + """Paired table over a replicate subset; `keep=None` means all of them.""" + rows = [] + print(f"\n--- {title} ---") + print(f"{'anchor':>7} {'W':>6} | {'D_hat/D':>17} | {'paired diff vs today':>24} " + f"{'t':>7} | {'p_ref':>7}") + for anchor in ("uncle", "parent"): + for w in windows: + g = cell.loc[anchor].loc[w] + i = g.index.intersection(base.index) + if keep is not None: + i = i[[r in keep for r in i]] + d = g.r[i] - base.r[i] + tt = d.mean() / d.sem() if d.std(ddof=1) > 0 else float("nan") + rows.append(dict(anchor=anchor, W=float(w), gap=d.mean(), stderr=d.sem(), + tstat=tt, ratio=g.r[i].mean(), p_ref=g.p[i].mean())) + tag = " <- today" if (anchor, w) == REFERENCE else "" + print(f"{anchor:>7} {w:6.1f} | {g.r[i].mean():10.5f}+-{g.r[i].sem():.5f} | " + f"{d.mean():+15.5f}+-{d.sem():.5f} {tt:7.2f} | {g.p[i].mean():7.4f}{tag}") + return pd.DataFrame(rows) + + def parity(res: pd.DataFrame, base_p: float) -> float | None: + """Smallest W not resolvably worse than today, with the interpolated zero crossing. + + Deliberately asymmetric: the claim is "adopting the anchor costs nothing against today", + so the burden is on ruling out a LOSS, not on proving equality. + """ + par = res[res.anchor == "parent"].sort_values("W") + ok = par[par.tstat > -2.0] + if ok.empty: + print(" no window in this grid reaches parity — widen W past the grid.") + return None + first = ok.iloc[0] + print(f" PARITY at W = {first.W:g}: {first.gap:+.5f} +- {first.stderr:.5f} " + f"(t = {first.tstat:.2f}), p_ref {first.p_ref:.4f} vs {base_p:.4f} today.") + below = par[par.W < first.W] + if not below.empty: + last = below.iloc[-1] + print(f" W = {last.W:g} is still resolvably worse: {last.gap:+.5f} +- " + f"{last.stderr:.5f} (t = {last.tstat:.2f}).") + if first.gap != last.gap: + cross = last.W + (0 - last.gap) * (first.W - last.W) / (first.gap - last.gap) + print(f" interpolated zero crossing: W = {cross:.2f}") + return float(first.W) + + all_reps = set(int(r) for r in t.replicate.unique()) + res_all = table(None, f"all {len(all_reps)} replicates") + w_all = parity(res_all, base.p.mean()) + + if off: + # Off-label replicates ran a WEAKER adversary, so their paired difference sits near zero + # and drags every cell toward parity — which would make the crossing look SMALLER than it + # is. The clean subset is the headline; the full set is the robustness check. + keep = all_reps - set(off) + res_clean = table(keep, f"on-label replicates only ({len(keep)} of {len(all_reps)})") + base_clean = base.p[[r in keep for r in base.p.index]].mean() + w_clean = parity(res_clean, base_clean) + if w_all is not None and w_clean is not None and w_clean != w_all: + print(f"\n!! the crossing MOVES when the off-label replicates are dropped: " + f"W = {w_all:g} -> W = {w_clean:g}. Quote the on-label figure.") + else: + print("\n the crossing is unchanged by dropping the off-label replicates.") + + print("\nNote: the uncle-anchored column moves too — widening today's own rule helps it. The " + "parity above is against TODAY'S recipe, which is the decision on the table, not " + "against the same W under both anchors.") + + +if __name__ == "__main__": + main()