diff --git a/.gitignore b/.gitignore index 83972fa..559613b 100644 --- a/.gitignore +++ b/.gitignore @@ -216,3 +216,11 @@ __marimo__/ # Streamlit .streamlit/secrets.toml + +# Editor / local agent state +.obsidian/ +.claude/settings.json +.claude/settings.local.json + +# External upstream clones vendored for benchmarking (own .git, not our history) +tools/benchmarks/original/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..417efeb --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,9 @@ +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/reports/tsi/report-figures/fig30_countable_vs_old.png b/reports/tsi/report-figures/fig30_countable_vs_old.png new file mode 100644 index 0000000..c8bc698 Binary files /dev/null and b/reports/tsi/report-figures/fig30_countable_vs_old.png differ diff --git a/reports/tsi/report-figures/fig31_countable_prediction.png b/reports/tsi/report-figures/fig31_countable_prediction.png new file mode 100644 index 0000000..33b8c0f Binary files /dev/null and b/reports/tsi/report-figures/fig31_countable_prediction.png differ diff --git a/reports/tsi/report-figures/fig32_countable_recovery.png b/reports/tsi/report-figures/fig32_countable_recovery.png new file mode 100644 index 0000000..951ce4f Binary files /dev/null and b/reports/tsi/report-figures/fig32_countable_recovery.png differ diff --git a/reports/tsi/report-figures/fig33_absorption_window.png b/reports/tsi/report-figures/fig33_absorption_window.png new file mode 100644 index 0000000..45c8165 Binary files /dev/null and b/reports/tsi/report-figures/fig33_absorption_window.png differ diff --git a/tools/simulators/tsi/tsi-sim-pernode/README.md b/tools/simulators/tsi/tsi-sim-pernode/README.md index d054f63..9acf057 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/README.md +++ b/tools/simulators/tsi/tsi-sim-pernode/README.md @@ -38,6 +38,16 @@ - **Per-node views:** one global block tree plus an `(N × n_blocks)` **arrival matrix** `A`; each node builds on / measures density over the blocks that have arrived at it. Uncle refs are **baked at production** from the producer's view (faithful — immutable once adopted). +- **Uncle model (`uncle_model`, CLI `--old`):** the default **countable** model implements the + spec's counting-only rules (cryptarchia-v1-protocol.md): only the **first block of a fork** + (parent on the producer's chain) is referenceable/countable, the window is **derived** as + `w_u = window_absorption / f` slots (`W` expected block-intervals, default `W = 10` → 300 + slots, bounded `W ≤ 0.6·k`), selection skips slots already occupied on the producer's chain + and picks one uncle per slot, and the measurement pass re-checks every rule per reference + (rejections tallied as `deep_ref_share`). Passing `--old` to `tsi-sweep`/`tsi-verify` runs + the pre-redesign model unchanged — window = `uncle_window` slots, any-depth orphans + referenceable, every baked reference counted — and **bit-reproduces historical runs** (the + old model's RNG key is byte-identical to the pre-`uncle_model` key). - **Metrics:** per-node `D_est` spread (`range`, `IQR`), canonical-chain **agreement** (window prefix vs current tip), mean accuracy, and — with `init_dest=heterogeneous` — transient re-convergence. @@ -145,6 +155,8 @@ src/tsi_sim/ constants config rng stake lottery topology blocktree(+build_tree uncles(+select_uncles_at_production) tsi(+update_D_vec) epoch engine metrics theory verify plotting/{style, figures_pernode, make_figures} configs/ smoke.yaml default.yaml fullscale.yaml + countable-vs-old.yaml absorption-window.yaml (countable-model studies) tests/ test_{pernode,config,rng,lottery,blocktree,uncles,tsi_counting,stake, - theory,latency,theory_convergence}.py + theory,latency,theory_convergence,countable_counting,...}.py +scripts/ plot_countable_vs_old.py (old-vs-countable comparison figures) ``` diff --git a/tools/simulators/tsi/tsi-sim-pernode/configs/absorption-window.yaml b/tools/simulators/tsi/tsi-sim-pernode/configs/absorption-window.yaml new file mode 100644 index 0000000..38b95e8 --- /dev/null +++ b/tools/simulators/tsi/tsi-sim-pernode/configs/absorption-window.yaml @@ -0,0 +1,26 @@ +# Countable model: sweep the window absorption parameter W (w_u = W/f slots derived) at +# U=1 against the Blend mixing delay. The window-miss contribution to non-recovery is +# (1-f)^(W/f) ~ e^-W (theory.window_miss_prob): recovery should saturate within a few +# expected block-intervals, with the residual set by the delay (orphans spread wider than +# the window) and by the first-fork restriction. The countable counterpart of the old +# model's uncle-window.yaml (which swept uncle_window in raw slots; run that with --old). +# Latency is in SLOTS (1 slot = 1 s). +n_nodes: [1000] # network size +stake_dist: [pareto] # heavy-tailed (realistic) stake distribution +topology: [blend] # Blend mixnet (delay stresses the window) +degree: [6] # peering degree of the d-regular graph +link_latency_mean: [0.5] # natural geographic transport (sub-slot) +link_latency_dist: [geo] # real-world geographic band mixture +blend_hops: [3] # fixed hop count; delay is the swept knob +blend_delay_max: [8.0, 16.0, 32.0] # max per-relay mixing delay (slots) +window_absorption: [1, 2, 3, 5, 7, 10] # W: window in expected block-intervals +max_uncles: [1] # FIXED at one uncle (the question is about W) +uncle_strategy: [oldest] # spec selection: oldest-first fill +init_dest: [common] # per-node initial D_est from agreement +replicates: 5 # independent RNG replicates per grid cell +base: # per-run settings shared by every cell + k: 2160 # true security parameter + epochs: 20 # equilibrium within ~2 epochs; burn 50% + f: 0.03333333333333333 # slot activation coefficient (1/30) + genesis_d_factor: 0.5 # start near true stake (cheap epoch 0) + early_stop: true diff --git a/tools/simulators/tsi/tsi-sim-pernode/configs/countable-vs-old.yaml b/tools/simulators/tsi/tsi-sim-pernode/configs/countable-vs-old.yaml new file mode 100644 index 0000000..3bdd5d6 --- /dev/null +++ b/tools/simulators/tsi/tsi-sim-pernode/configs/countable-vs-old.yaml @@ -0,0 +1,23 @@ +# Headline comparison for the countable uncle model (cryptarchia-v1-protocol.md counting +# rules) vs the old pre-redesign model: accuracy vs Blend mixing delay at U in {0,1,2,4}. +# Run TWICE — default (countable) and with --old — same grid; the countable run also +# yields deep_ref_share (the first-fork restriction's rejection rate) and q/q_eff for the +# q_u = q + (1-q) r theory overlay. Latency is in SLOTS (1 slot = 1 s). +n_nodes: [1000] # network size +stake_dist: [pareto] # heavy-tailed (realistic) stake distribution +topology: [blend] # Blend mixnet (the multi-slot fork regime) +degree: [6] # peering degree of the d-regular graph +link_latency_mean: [0.5] # natural geographic transport (sub-slot) +link_latency_dist: [geo] # real-world geographic band mixture +blend_hops: [3] # fixed hop count; delay is the swept knob +blend_delay_max: [4.0, 8.0, 16.0, 32.0] # max per-relay mixing delay (slots) +max_uncles: [0, 1, 2, 4] # U: 0 baseline, then the recovery levers +uncle_strategy: [oldest] # spec selection: oldest-first fill +init_dest: [common] # per-node initial D_est from agreement +replicates: 5 # independent RNG replicates per grid cell +base: # per-run settings shared by every cell + k: 2160 # true security parameter + epochs: 20 # equilibrium within ~2 epochs; burn 50% + f: 0.03333333333333333 # slot activation coefficient (1/30) + genesis_d_factor: 0.5 # start near true stake (cheap epoch 0) + early_stop: true diff --git a/tools/simulators/tsi/tsi-sim-pernode/scripts/plot_countable_vs_old.py b/tools/simulators/tsi/tsi-sim-pernode/scripts/plot_countable_vs_old.py new file mode 100644 index 0000000..77b32c4 --- /dev/null +++ b/tools/simulators/tsi/tsi-sim-pernode/scripts/plot_countable_vs_old.py @@ -0,0 +1,169 @@ +"""Comparison figures: countable uncle model (spec counting rules) vs the old model. + +Consumes the results of three sweeps: + + tsi-sweep --config configs/countable-vs-old.yaml --label cvo-countable + tsi-sweep --config configs/countable-vs-old.yaml --old --label cvo-old + tsi-sweep --config configs/absorption-window.yaml --label absorption-window + +and renders (into --out): + + cvo_accuracy_vs_delay equilibrium D/D_true vs Blend mixing delay; solid = countable, + dashed = old, one Okabe-Ito hue per U (color follows U). + cvo_prediction_vs_sim predicted log(1-f)/log(1-f/q_u) from the MEASURED q_u vs the + simulated equilibrium — the q -> q_u reduction check. + cvo_recovery_vs_delay measured recovery r = (q_eff - q)/(1 - q) and the first-fork + rejection share (deep_ref_share) vs delay. + absorption_window equilibrium vs the window absorption parameter W per delay. + +Usage: + python scripts/plot_countable_vs_old.py --countable RUNDIR --old RUNDIR \ + --absorption RUNDIR --out figures/countable-vs-old +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + +from tsi_sim.plotting import style +from tsi_sim.plotting.figures_pernode import equilibrium +from tsi_sim.theory import expected_ratio + +DELAY = "blend_delay_max" + + +def _load(run_dir: str | Path) -> pd.DataFrame: + return pd.read_parquet(Path(run_dir) / "results.parquet") + + +def _eq(df: pd.DataFrame, extra_cols: tuple[str, ...] = ()) -> pd.DataFrame: + """Equilibrium (post-burn) means per (delay, U) cell, averaged over replicates.""" + eq = equilibrium(df) + keys = [DELAY, "max_uncles", *extra_cols] + agg = {"mean_ratio": "mean", "mean_q": "mean", "mean_q_eff": "mean"} + if "deep_ref_share" in eq.columns: + agg["deep_ref_share"] = "mean" + return eq.groupby(keys, as_index=False).agg(agg) + + +def fig_accuracy_vs_delay(cnt: pd.DataFrame, old: pd.DataFrame) -> plt.Figure: + fig, ax = plt.subplots() + us = sorted(cnt["max_uncles"].unique()) + for i, u in enumerate(us): + c = style.color_for(i) + a = cnt[cnt.max_uncles == u].sort_values(DELAY) + b = old[old.max_uncles == u].sort_values(DELAY) + ax.plot(a[DELAY], a.mean_ratio, "-o", color=c, label=f"U={u} countable", ms=4) + ax.plot(b[DELAY], b.mean_ratio, "--s", color=c, label=f"U={u} old", ms=4, + alpha=0.75) + ax.axhline(1.0, color="0.4", lw=0.8, ls=":") + ax.set_xlabel("max per-relay mixing delay (slots)") + ax.set_ylabel(r"equilibrium $\hat{D}/D_{true}$") + ax.set_title("Accuracy vs delay: countable (solid) vs old (dashed) uncle model") + ax.legend(ncol=2) + return fig + + +def fig_prediction_vs_sim(cnt: pd.DataFrame, f: float) -> plt.Figure: + fig, ax = plt.subplots(figsize=(4.6, 4.4)) + sub = cnt[cnt.max_uncles > 0] + pred = expected_ratio(f, sub.mean_q_eff.to_numpy()) + us = sorted(sub["max_uncles"].unique()) + for i, u in enumerate(us): + m = (sub.max_uncles == u).to_numpy() + ax.scatter(np.asarray(pred)[m], sub.mean_ratio.to_numpy()[m], + color=style.color_for(i), s=22, label=f"U={u}") + lo = min(float(np.min(pred)), float(sub.mean_ratio.min())) - 0.01 + ax.plot([lo, 1.005], [lo, 1.005], color="0.3", lw=0.9, ls=":", label="prediction = sim") + ax.set_xlabel(r"predicted $\log(1-f)\,/\,\log(1-f/\bar{q}_u)$ (measured $\bar{q}_u$)") + ax.set_ylabel(r"simulated equilibrium $\hat{D}/D_{true}$") + ax.set_title(r"$q \to q_u$ reduction: prediction vs simulation") + ax.legend() + return fig + + +def fig_recovery_vs_delay(cnt: pd.DataFrame) -> plt.Figure: + # Right panel: the non-recovered waste share 1-r (log scale). Under joint countable + # selection+counting the first-fork restriction acts at SELECTION (deep orphans are + # never referenced), so counting-side rejections (deep_ref_share) are 0 and the + # restriction shows up inside 1-r together with capacity losses. + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(9.2, 3.8)) + us = [u for u in sorted(cnt["max_uncles"].unique()) if u > 0] + for i, u in enumerate(us): + a = cnt[cnt.max_uncles == u].sort_values(DELAY) + denom = np.maximum(1.0 - a.mean_q.to_numpy(), 1e-12) + r = (a.mean_q_eff.to_numpy() - a.mean_q.to_numpy()) / denom + ax1.plot(a[DELAY], r, "-o", color=style.color_for(i), label=f"U={u}", ms=4) + ax2.semilogy(a[DELAY], np.maximum(1.0 - r, 1e-4), "-o", + color=style.color_for(i), label=f"U={u}", ms=4) + ax1.set_xlabel("max per-relay mixing delay (slots)") + ax1.set_ylabel(r"measured recovery $r=(\bar{q}_u-\bar{q})/(1-\bar{q})$") + ax1.set_ylim(0, 1.02) + ax1.set_title("Uncle recovery rate") + ax1.legend() + ax2.set_xlabel("max per-relay mixing delay (slots)") + ax2.set_ylabel(r"non-recovered waste share $1-r$") + ax2.set_title("Residual (first-fork + capacity losses)") + ax2.legend() + return fig + + +def fig_absorption_window(absw: pd.DataFrame) -> plt.Figure: + fig, ax = plt.subplots() + eq = equilibrium(absw) + agg = eq.groupby([DELAY, "window_absorption"], as_index=False).mean_ratio.mean() + for i, d in enumerate(sorted(agg[DELAY].unique())): + a = agg[agg[DELAY] == d].sort_values("window_absorption") + ax.plot(a.window_absorption, a.mean_ratio, "-o", color=style.color_for(i), + label=f"delay={d:g}", ms=4) + ax.axhline(1.0, color="0.4", lw=0.8, ls=":") + ax.set_xlabel("window absorption parameter W (expected block-intervals)") + ax.set_ylabel(r"equilibrium $\hat{D}/D_{true}$") + ax.set_title("Accuracy vs the derived uncle window $w_u = W/f$ (U=1)") + ax.legend(title=None) + return fig + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--countable", required=True, help="run dir of cvo-countable") + ap.add_argument("--old", required=True, help="run dir of cvo-old") + ap.add_argument("--absorption", required=True, help="run dir of absorption-window") + ap.add_argument("--out", default="figures/countable-vs-old") + args = ap.parse_args() + style.apply_style() + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + cnt_raw, old_raw = _load(args.countable), _load(args.old) + f = float(cnt_raw["f"].iloc[0]) + cnt, old = _eq(cnt_raw), _eq(old_raw) + prov = "tsi-sim-pernode countable-vs-old.yaml (+--old) / absorption-window.yaml" + + written = [] + written += style.save(fig_accuracy_vs_delay(cnt, old), out / "cvo_accuracy_vs_delay", prov) + written += style.save(fig_prediction_vs_sim(cnt, f), out / "cvo_prediction_vs_sim", prov) + written += style.save(fig_recovery_vs_delay(cnt), out / "cvo_recovery_vs_delay", prov) + written += style.save(fig_absorption_window(_load(args.absorption)), + out / "absorption_window", prov) + # headline numbers for the report / analysis doc + for u in sorted(cnt["max_uncles"].unique()): + for _, row in cnt[cnt.max_uncles == u].sort_values(DELAY).iterrows(): + q, qu = row.mean_q, row.mean_q_eff + r = (qu - q) / max(1.0 - q, 1e-12) + o = old[(old.max_uncles == u) & (old[DELAY] == row[DELAY])] + old_ratio = float(o.mean_ratio.iloc[0]) if len(o) else float("nan") + print(f"U={u} delay={row[DELAY]:>5g} countable={row.mean_ratio:.4f} " + f"old={old_ratio:.4f} q={q:.4f} q_u={qu:.4f} r={r:.4f} " + f"pred={float(expected_ratio(f, qu)):.4f} " + f"deep={row.get('deep_ref_share', float('nan')):.4f}") + print(f"wrote {len(written)} files -> {out}") + + +if __name__ == "__main__": + main() diff --git a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/blocktree.py b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/blocktree.py index 7d9b68b..f20d8d3 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/blocktree.py +++ b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/blocktree.py @@ -364,7 +364,9 @@ def _build_pruned(active_slots, winners_per_slot, path_latency, config, rng, from .uncles import select_uncles_at_production NEG = np.iinfo(np.int64).min - keepspan = max(float(horizon), float(config.uncle_window)) # columns kept within this span + # columns kept within this span; the uncle window is model-dependent (derived W/f for + # countable, uncle_window slots for --old), so use the effective value. + keepspan = max(float(horizon), float(config.effective_uncle_window)) counts = np.array([int(g.shape[0]) for g in winners_per_slot], dtype=np.int64) cap = _max_span_blocks(active_slots, counts, keepspan) # max live blocks at once max_slot = int(counts.max()) if counts.size else 0 diff --git a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/config.py b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/config.py index 76edec3..4f4951c 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/config.py +++ b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/config.py @@ -10,6 +10,16 @@ from . import constants StakeDist = Literal["uniform", "pareto"] UncleStrategy = Literal["oldest", "random"] +# Uncle counting/selection model: +# "countable" (default) — the spec's counting-only model (cryptarchia-v1-protocol.md): +# only the FIRST block of a fork is referenceable/countable (its parent lies on the +# referencing chain), the window is derived as w_u = window_absorption / f slots, +# selection excludes slots already occupied on the producer's chain and picks at most +# one uncle per slot, and counting re-checks every rule per reference. +# "old" — the pre-redesign model (run with --old): window = uncle_window slots directly, +# any orphan in view is referenceable regardless of fork depth, no occupied-slot or +# per-slot exclusion, and every baked reference counts. +UncleModel = Literal["countable", "old"] Topology = Literal["full_mesh", "regular", "blend"] LinkLatencyDist = Literal["fixed", "uniform", "exp", "geo"] JitterDist = Literal["exp", "poisson"] @@ -81,7 +91,14 @@ class SimConfig: jitter_frac: float = 1.0 # fraction of deliveries hit (poisson model; exp uses all) # --- uncle references --- - uncle_window: int = constants.W_DEFAULT # W + uncle_model: UncleModel = "countable" # countable (spec, default) | old (--old) + # Countable model: window absorption parameter W; the uncle reference window is DERIVED + # as w_u = W / f slots (W expected block-intervals), bounded 1 <= W <= 0.6*k + # (constants.W_ABS_MAX_FACTOR). Ignored by the old model. + window_absorption: float = constants.W_ABS_DEFAULT + # Old model only (--old): the uncle reference window w_u in slots, set directly. + # Ignored by the countable model, which derives the window from window_absorption. + uncle_window: int = constants.W_DEFAULT max_uncles: int = 0 # U (0 = baseline, no uncles) uncle_strategy: UncleStrategy = "oldest" # Coin-flip inclusion prob for the "random" strategy. Only 0.5 reproduces the spec's @@ -187,6 +204,25 @@ class SimConfig: raise ValueError(f"stake_dist must be uniform|pareto, got {self.stake_dist!r}") if self.uncle_strategy not in ("oldest", "random"): raise ValueError(f"uncle_strategy must be oldest|random, got {self.uncle_strategy!r}") + if self.uncle_model not in ("countable", "old"): + raise ValueError(f"uncle_model must be countable|old, got {self.uncle_model!r}") + if self.uncle_model == "countable": + if self.window_absorption < 1.0: + raise ValueError( + f"window_absorption W={self.window_absorption} must be >= 1") + if self.window_absorption > constants.W_ABS_MAX_FACTOR * self.k: + # The spec bounds W <= 0.6*k (w_u <= 0.6*k/f, inside the finalization + # window). Scaled-down research geometries (small k) may violate it on + # purpose — warn loudly rather than refuse, but full-scale runs should + # never see this. + import warnings + + warnings.warn( + f"window_absorption W={self.window_absorption} exceeds the spec bound " + f"{constants.W_ABS_MAX_FACTOR}*k = " + f"{constants.W_ABS_MAX_FACTOR * self.k:g} (k={self.k}); the derived " + f"window is outside the finalization window at this geometry", + RuntimeWarning, stacklevel=2) if self.topology not in ("full_mesh", "regular", "blend"): raise ValueError(f"topology must be full_mesh|regular|blend, got {self.topology!r}") if self.link_latency_dist not in ("fixed", "uniform", "exp", "geo"): @@ -264,6 +300,17 @@ class SimConfig: return (epoch % self.adversary_period) < self.adversary_withhold_epochs # derived geometry ------------------------------------------------------- + @property + def effective_uncle_window(self) -> int: + """The uncle reference window ``w_u`` in slots actually used by this run. + + Countable model (default): derived, ``w_u = round(window_absorption / f)``. + Old model (``--old``): ``uncle_window`` taken directly. + """ + if self.uncle_model == "old": + return self.uncle_window + return constants.uncle_window_slots(self.window_absorption, self.f) + @property def epoch_len(self) -> int: return constants.epoch_len(self.k, self.f) @@ -276,9 +323,12 @@ class SimConfig: """Hashable identity used to seed the RNG deterministically. Must include EVERY field that affects the run (guarded by test_rng), otherwise two - distinct configs would share an RNG stream. + distinct configs would share an RNG stream. ``uncle_model`` / + ``window_absorption`` are appended ONLY for the countable model: an ``--old`` run's + key is then byte-identical to the pre-redesign key, so ``--old`` bit-reproduces + historical runs (the two models still get distinct streams from the marker). """ - return ( + base = ( self.n_nodes, self.stake_dist, self.pareto_shape, self.uniform_random, self.total_stake, self.latency, self.latency_stochastic, self.uncle_window, self.max_uncles, self.uncle_strategy, self.uncle_random_p, self.f, self.beta, @@ -295,11 +345,15 @@ class SimConfig: # NOTE: windowed_fork_choice and prune_arrival are deliberately excluded — they are pure # compute/memory optimisations that consume no RNG and (at jitter_mean == 0) change no # result, so pruned and full-matrix runs must share a seed (see test_pernode parity). + if self.uncle_model == "old": + return base # historical (pre-uncle_model) key: --old bit-compat + return base + (self.uncle_model, self.window_absorption) # Axes that can be swept; every SimConfig field is legal here. _SWEEP_AXES = ( "n_nodes", "stake_dist", "latency", "max_uncles", "uncle_strategy", "uncle_window", + "window_absorption", "topology", "degree", "link_latency_mean", "link_latency_dist", "blend_hops", "blend_delay_max", "init_dest", "f", ) @@ -315,6 +369,7 @@ class SweepConfig: max_uncles: list[int] = field(default_factory=lambda: [0, 1, 2, 4]) uncle_strategy: list[UncleStrategy] = field(default_factory=lambda: ["oldest"]) uncle_window: list[int] = field(default_factory=lambda: [constants.W_DEFAULT]) + window_absorption: list[float] = field(default_factory=lambda: [constants.W_ABS_DEFAULT]) topology: list[Topology] = field(default_factory=lambda: ["regular"]) degree: list[int] = field(default_factory=lambda: [8]) link_latency_mean: list[float] = field(default_factory=lambda: [1.0]) @@ -333,11 +388,22 @@ class SweepConfig: axis_values = [getattr(self, ax) for ax in _SWEEP_AXES] for combo in itertools.product(*axis_values): overrides = dict(zip(_SWEEP_AXES, combo, strict=True)) - # U=0 references no uncles, so it is independent of uncle_strategy AND uncle_window; - # keep only the first of each to avoid duplicate (identical) work. + # U=0 references no uncles, so it is independent of uncle_strategy AND the window + # knobs; keep only the first of each to avoid duplicate (identical) work. if overrides["max_uncles"] == 0 and ( overrides["uncle_strategy"] != self.uncle_strategy[0] or overrides["uncle_window"] != self.uncle_window[0] + or overrides["window_absorption"] != self.window_absorption[0] + ): + continue + # each uncle model reads exactly one window knob — collapse the other axis so a + # sweep never emits duplicate cells that differ only in an ignored field. + if base.uncle_model == "countable" and ( + overrides["uncle_window"] != self.uncle_window[0] + ): + continue + if base.uncle_model == "old" and ( + overrides["window_absorption"] != self.window_absorption[0] ): continue # full mesh ignores degree / link-latency model; keep only the first to avoid dupes. diff --git a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/constants.py b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/constants.py index e4554d0..c69dda4 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/constants.py +++ b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/constants.py @@ -10,10 +10,23 @@ from __future__ import annotations # --- True protocol values (full scale) ------------------------------------- K_TRUE = 2160 # security parameter (blocks) F = 1.0 / 30.0 # slot activation coefficient (default; configurable per run) -W_DEFAULT = 300 # uncle reference window w_u (slots) +W_DEFAULT = 300 # old model: uncle reference window w_u (slots), set directly (--old) BETA_DEFAULT = 1.0 # TSI learning rate SLOT_SECONDS = 1 # slot length (seconds) — so 1 slot == 1 s +# --- Countable uncle model (cryptarchia-v1-protocol.md, uncle references) --- +# The spec derives the uncle reference window from the *window absorption parameter* W: +# w_u = W * f^-1 slots, i.e. W expected block-intervals. W is bounded by 1 <= W <= 0.6*k, +# equivalently w_u <= 0.6*k/f = s/5, keeping the window strictly inside the finalization +# window. The default W = 10 reproduces w_u = 300 slots at f = 1/30. +W_ABS_DEFAULT = 10.0 # window absorption parameter W (expected block-intervals) +W_ABS_MAX_FACTOR = 0.6 # bound: W <= W_ABS_MAX_FACTOR * k + + +def uncle_window_slots(w_abs: float, f: float = F) -> int: + """Derived uncle reference window ``w_u = W / f`` in slots (countable model).""" + return max(1, int(round(w_abs / f))) + # --- Real-world inter-node network latency (per gossip link) --------------- # A slot is SLOT_SECONDS = 1 s, so measured internet latencies (tens–hundreds of ms) are diff --git a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/epoch.py b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/epoch.py index 7c1c820..d6e37bf 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/epoch.py +++ b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/epoch.py @@ -30,6 +30,8 @@ class EpochResult: max_reorg_depth: int # deepest maximal orphan branch (blocks a reorg would discard) mean_reorg_depth: float # mean maximal-orphan-branch depth p_ref: float # emergent reference rate: in-window orphans referenced as uncles + deep_ref_share: float # share of examined references rejected by the parent-on-chain + # (first-fork) counting rule; 0 under the old model def _canonical_producer_split( @@ -101,7 +103,9 @@ def simulate_epoch( # measurement: each node's own canonical chain, deduped by tip + numba-accelerated ms = measure(tree, A, active_slots, T, cutoff=E, - legacy_block_count=config.legacy_block_count) + legacy_block_count=config.legacy_block_count, + countable=config.uncle_model != "old", + w=config.effective_uncle_window) n_active_window = int((active_slots < T).sum()) d_next = tsi.update_D_vec(d_est, ms.m, T, f, config.beta, config.fixed_point) @@ -109,6 +113,8 @@ def simulate_epoch( attribution = coalition_mask if coalition_mask is not None else adversary_mask adv_blocks, honest_blocks = _canonical_producer_split(tree, A, attribution, T, E) fork_rate, max_reorg_depth, mean_reorg_depth, p_ref = fork.fork_stats(tree, A, T, cutoff=E) + ref_total = int(ms.ref_total.sum()) + deep_ref_share = (int(ms.ref_deep.sum()) / ref_total) if ref_total else 0.0 return EpochResult( d_next=d_next, m=ms.m, q=ms.q, q_eff=ms.q_eff, n_blocks=tree.n_blocks - 1, @@ -117,5 +123,5 @@ def simulate_epoch( mean_orphan_rate=float(ms.orphan_rate.mean()), adv_blocks=adv_blocks, honest_blocks=honest_blocks, fork_rate=fork_rate, max_reorg_depth=max_reorg_depth, mean_reorg_depth=mean_reorg_depth, - p_ref=p_ref, + p_ref=p_ref, deep_ref_share=deep_ref_share, ) diff --git a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/measure.py b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/measure.py index 9db0c57..80945c2 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/measure.py +++ b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/measure.py @@ -6,6 +6,16 @@ independently — O(N x chain) Python and ~95% of an epoch. Two exact optimisati The counted density ``m`` is SLOT-based (canonical slots + recovered uncle slots — the "one count per slot" invariant; ``legacy_block_count`` reproduces the old per-block count). +Counting models (``countable`` flag; CLI ``--old`` clears it): + +* **countable** (default) — the spec's counting rules are re-checked per reference + (cryptarchia-v1-protocol.md): the reference must be within the window + (``0 < slot_B - slot_U <= w``), the uncle must not lie on the counting chain, and its + **parent must lie on the counting chain** (only the first block of a fork counts). + References failing the parent rule are tallied as ``deep`` diagnostics. +* **old** — every baked reference in the measurement window counts (fork depth ignored), + reproducing the pre-redesign behaviour. + 1. **Dedup by tip.** Nodes sharing a current tip share their whole canonical chain and every derived quantity, so we compute once per *distinct* tip and broadcast. High node agreement (the common case) collapses N to a handful of computations. @@ -41,6 +51,9 @@ class Measurement: q: np.ndarray # (N,) honest active-slot fraction q_eff: np.ndarray # (N,) uncle-recovered fraction orphan_rate: np.ndarray # (N,) + ref_total: np.ndarray # (N,) distinct referenced uncles examined in the window + ref_deep: np.ndarray # (N,) of those, rejected by the parent-on-chain (first-fork) rule; + # always 0 under the old model (no rule to reject on) agreement_window: float agreement_tip: float @@ -58,23 +71,26 @@ def _uncles_csr(tree: BlockTree) -> tuple[np.ndarray, np.ndarray]: return flat, ptr -def _measure_tips_py(distinct_tips, parent, slot, uncle_flat, uncle_ptr, T, - uncle_stamp, honest_stamp): +def _measure_tips_py(distinct_tips, parent, slot, uncle_flat, uncle_ptr, T, w, countable, + uncle_stamp, honest_stamp, chain_stamp): """Pure-Python per-distinct-tip walk (fallback / reference for the kernel).""" K = distinct_tips.shape[0] m = np.empty(K, np.int64) n_honest = np.empty(K, np.int64) n_rec = np.empty(K, np.int64) + n_ref = np.empty(K, np.int64) + n_deep = np.empty(K, np.int64) chain_len = np.empty(K, np.int64) fp = np.empty(K, np.uint64) for ki in range(K): - # pass 1: chain -> honest count, mark honest slots, fingerprint, chain length + # pass 1: chain -> honest count, mark honest slots + chain membership, fingerprint honest = 0 clen = 0 f = np.uint64(0) b = int(distinct_tips[ki]) while b > 0: clen += 1 + chain_stamp[b] = ki # chain membership (any slot, incl. outside T) s = int(slot[b]) if 0 <= s < T: honest += 1 @@ -84,13 +100,27 @@ def _measure_tips_py(distinct_tips, parent, slot, uncle_flat, uncle_ptr, T, # pass 2: deduped referenced uncles in window + recovered orphan slots ucnt = 0 rec = 0 + refs = 0 + deep = 0 b = int(distinct_tips[ki]) while b > 0: for j in range(int(uncle_ptr[b]), int(uncle_ptr[b + 1])): u = int(uncle_flat[j]) su = int(slot[u]) if 0 <= su < T and uncle_stamp[u] != ki: - uncle_stamp[u] = ki # dedup uncles by id (m counts blocks) + uncle_stamp[u] = ki # dedup uncles by id + refs += 1 + if countable: + # spec counting rules, re-checked per reference: + d = int(slot[b]) - su + if d <= 0 or d > w: + continue # outside the reference window + if chain_stamp[u] == ki: + continue # uncle lies on the counting chain + pu = int(parent[u]) + if pu != 0 and chain_stamp[pu] != ki: + deep += 1 # not a first fork block: uncounted + continue ucnt += 1 if honest_stamp[su] != ki: rec += 1 # recovered slots deduped by slot @@ -99,9 +129,11 @@ def _measure_tips_py(distinct_tips, parent, slot, uncle_flat, uncle_ptr, T, m[ki] = honest + ucnt n_honest[ki] = honest n_rec[ki] = rec + n_ref[ki] = refs + n_deep[ki] = deep chain_len[ki] = clen fp[ki] = f - return m, n_honest, n_rec, chain_len, fp + return m, n_honest, n_rec, n_ref, n_deep, chain_len, fp def _mix_py(x: np.uint64) -> np.uint64: @@ -119,12 +151,14 @@ if _HAVE_NUMBA: return x ^ (x >> uint64(31)) @njit(cache=True) - def _measure_tips_nb(distinct_tips, parent, slot, uncle_flat, uncle_ptr, T, - uncle_stamp, honest_stamp): + def _measure_tips_nb(distinct_tips, parent, slot, uncle_flat, uncle_ptr, T, w, countable, + uncle_stamp, honest_stamp, chain_stamp): K = distinct_tips.shape[0] m = np.empty(K, np.int64) n_honest = np.empty(K, np.int64) n_rec = np.empty(K, np.int64) + n_ref = np.empty(K, np.int64) + n_deep = np.empty(K, np.int64) chain_len = np.empty(K, np.int64) fp = np.empty(K, np.uint64) for ki in range(K): @@ -134,6 +168,7 @@ if _HAVE_NUMBA: b = distinct_tips[ki] while b > 0: clen += 1 + chain_stamp[b] = ki # chain membership (any slot, incl. outside T) s = slot[b] if 0 <= s < T: honest += 1 @@ -142,6 +177,8 @@ if _HAVE_NUMBA: b = parent[b] ucnt = 0 rec = 0 + refs = 0 + deep = 0 b = distinct_tips[ki] while b > 0: for j in range(uncle_ptr[b], uncle_ptr[b + 1]): @@ -149,25 +186,45 @@ if _HAVE_NUMBA: su = slot[u] if 0 <= su < T and uncle_stamp[u] != ki: uncle_stamp[u] = ki # dedup uncles by id - ucnt += 1 - if honest_stamp[su] != ki: - rec += 1 # recovered slots deduped by slot - honest_stamp[su] = ki + refs += 1 + ok = True + if countable: + d = slot[b] - su + if d <= 0 or d > w: + ok = False # outside the reference window + elif chain_stamp[u] == ki: + ok = False # uncle lies on the counting chain + else: + pu = parent[u] + if pu != 0 and chain_stamp[pu] != ki: + deep += 1 # not a first fork block: uncounted + ok = False + if ok: + ucnt += 1 + if honest_stamp[su] != ki: + rec += 1 # recovered slots deduped by slot + honest_stamp[su] = ki b = parent[b] m[ki] = honest + ucnt n_honest[ki] = honest n_rec[ki] = rec + n_ref[ki] = refs + n_deep[ki] = deep chain_len[ki] = clen fp[ki] = f - return m, n_honest, n_rec, chain_len, fp + return m, n_honest, n_rec, n_ref, n_deep, chain_len, fp def measure(tree: BlockTree, A, active_slots: np.ndarray, T: int, cutoff: int, - use_numba: bool = True, legacy_block_count: bool = False) -> Measurement: + use_numba: bool = True, legacy_block_count: bool = False, + countable: bool = False, w: int = 0) -> Measurement: """Per-node m/q/q_eff + agreement, deduped by tip and (optionally) numba-accelerated. - ``A`` is the full ``(N, n_blocks)`` arrival matrix or a pruned ``SlidingArrival`` — only - ``tips_for_all_nodes`` reads it, so ``N`` is taken from the returned per-node tips. + ``countable`` applies the spec's per-reference counting rules (window ``w``, + not-on-chain, parent-on-chain); ``countable=False`` reproduces the old model where every + baked reference counts. ``A`` is the full ``(N, n_blocks)`` arrival matrix or a pruned + ``SlidingArrival`` — only ``tips_for_all_nodes`` reads it, so ``N`` is taken from the + returned per-node tips. """ tips = tips_for_all_nodes(tree, A, cutoff) N = tips.shape[0] @@ -179,11 +236,13 @@ def measure(tree: BlockTree, A, active_slots: np.ndarray, T: int, cutoff: int, uncle_flat, uncle_ptr = _uncles_csr(tree) uncle_stamp = np.full(tree.n_blocks, -1, np.int64) honest_stamp = np.full(max(T, 1), -1, np.int64) + chain_stamp = np.full(tree.n_blocks, -1, np.int64) kernel = _measure_tips_nb if (_HAVE_NUMBA and use_numba) else _measure_tips_py - m_d, nh_d, nrec_d, clen_d, fp_d = kernel( + m_d, nh_d, nrec_d, nref_d, ndeep_d, clen_d, fp_d = kernel( distinct_tips.astype(np.int64), tree.parent, tree.slot, - uncle_flat, uncle_ptr, np.int64(T), uncle_stamp, honest_stamp) + uncle_flat, uncle_ptr, np.int64(T), np.int64(w), bool(countable), + uncle_stamp, honest_stamp, chain_stamp) # correct slot counting: canonical slots + recovered (non-canonical, deduped) uncle slots. # legacy_block_count reproduces the earlier per-block-id count (kernel's m = honest + ucnt). @@ -200,4 +259,5 @@ def measure(tree: BlockTree, A, active_slots: np.ndarray, T: int, cutoff: int, agreement_window = max(fp_counts.values()) / N return Measurement(m=m, q=q, q_eff=q_eff, orphan_rate=orphan_rate, + ref_total=nref_d[inverse], ref_deep=ndeep_d[inverse], agreement_window=agreement_window, agreement_tip=agreement_tip) diff --git a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/metrics.py b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/metrics.py index 6d933ab..79587f2 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/metrics.py +++ b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/metrics.py @@ -13,7 +13,8 @@ from .epoch import EpochResult _CONFIG_FIELDS = ( "n_nodes", "stake_dist", "pareto_shape", "latency", "topology", "degree", "link_latency_mean", "link_latency_dist", "blend_hops", "blend_delay_max", - "init_dest", "init_spread", "uncle_window", "max_uncles", "uncle_strategy", + "init_dest", "init_spread", "uncle_model", "window_absorption", + "uncle_window", "max_uncles", "uncle_strategy", "f", "beta", "k", "genesis_d_factor", "epochs", "fixed_point", "legacy_block_count", "replicate", "adversary_frac", "adversary_strategy", "adversary_period", "adversary_withhold_epochs", @@ -58,5 +59,6 @@ def divergence_row( max_reorg_depth=er.max_reorg_depth, mean_reorg_depth=er.mean_reorg_depth, p_ref=er.p_ref, + deep_ref_share=er.deep_ref_share, ) return row diff --git a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/plotting/figures_pernode.py b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/plotting/figures_pernode.py index 9f08222..e7c3164 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/plotting/figures_pernode.py +++ b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/plotting/figures_pernode.py @@ -18,8 +18,8 @@ from . import style # this exhaustive over the recorded config fields (see metrics._CONFIG_FIELDS). CONFIG_COLS = ["n_nodes", "stake_dist", "pareto_shape", "topology", "degree", "link_latency_mean", "link_latency_dist", "blend_hops", "blend_delay_max", - "latency", "max_uncles", "uncle_strategy", "uncle_window", - "init_dest", "init_spread", "genesis_d_factor", + "latency", "uncle_model", "window_absorption", "max_uncles", "uncle_strategy", + "uncle_window", "init_dest", "init_spread", "genesis_d_factor", "f", "beta", "k", "fixed_point", "legacy_block_count"] # Graph topologies (as opposed to the full_mesh baseline) and the dominant latency knob each @@ -41,12 +41,15 @@ def equilibrium(df: pd.DataFrame, burn_frac: float = 0.5) -> pd.DataFrame: ``epochs`` — early-stopped runs (config.early_stop) terminate well before the planned ``epochs``, so thresholding on the configured value would drop every row. """ - max_epoch = df.groupby([*CONFIG_COLS, "replicate"])["epoch"].transform("max") + cfg_cols = [c for c in CONFIG_COLS if c in df.columns] # old parquets lack new fields + max_epoch = df.groupby([*cfg_cols, "replicate"])["epoch"].transform("max") tail = df[df["epoch"] >= max_epoch * burn_frac] agg = {c: (c, "mean") for c in ("mean_ratio", "range_ratio", "iqr_ratio", "agreement_window", "agreement_tip", - "mean_q", "mean_q_eff", "mean_orphan_rate", "max_ratio", "min_ratio")} - return tail.groupby([*CONFIG_COLS, "replicate"], as_index=False).agg(**agg) + "mean_q", "mean_q_eff", "mean_orphan_rate", "max_ratio", "min_ratio", + "deep_ref_share", "p_ref", "fork_rate") + if c in tail.columns} + return tail.groupby([*cfg_cols, "replicate"], as_index=False).agg(**agg) def _prov(df: pd.DataFrame) -> str: diff --git a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/sweep.py b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/sweep.py index ecc3d4a..2f53bd1 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/sweep.py +++ b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/sweep.py @@ -77,7 +77,7 @@ def _arrival_columns(config: SimConfig, peak_blocks: int) -> int: if not (config.prune_arrival and config.windowed_fork_choice): return peak_blocks per_slot = peak_blocks / config.epoch_len if config.epoch_len else peak_blocks - keepspan = float(config.uncle_window) + keepspan = float(config.effective_uncle_window) if config.topology == "blend": lat = max(config.link_latency_mean, 0.1) keepspan = max(keepspan, (config.blend_hops + 1) * lat * 4 @@ -314,19 +314,26 @@ def main(argv: list[str] | None = None) -> None: "(default) probes when N>2000, 'always', or 'never' (estimate only)") parser.add_argument("--no-figures", action="store_true", help="skip auto figure generation") + parser.add_argument("--old", action="store_true", + help="run the old (pre countable redesign) uncle model: window = " + "uncle_window slots, any-depth orphans referenceable, every " + "baked reference counts; bit-reproduces historical runs") args = parser.parse_args(argv) batch_size = int(args.batch_size) if args.batch_size != "auto" else "auto" - label = args.label or Path(args.config).stem + label = args.label or (Path(args.config).stem + ("-old" if args.old else "")) run_dir = new_run_dir(args.outdir, label) sweep = load_sweep_yaml(args.config) + if args.old: + sweep.base["uncle_model"] = "old" df = run_sweep(sweep, n_jobs=args.n_jobs, batch_size=batch_size, mem_frac=args.mem_frac, calibrate=args.calibrate) results_path = run_dir / "results.parquet" persist(df, results_path) key_cols = ["n_nodes", "stake_dist", "topology", "degree", "link_latency_mean", - "latency", "max_uncles", "uncle_strategy", "init_dest", "replicate"] + "latency", "uncle_model", "max_uncles", "uncle_strategy", "init_dest", + "replicate"] n_cfg = len(df[key_cols].drop_duplicates()) print(f"wrote {len(df)} rows ({n_cfg} configs) -> {results_path}") diff --git a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/theory.py b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/theory.py index cb96976..42283a0 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/theory.py +++ b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/theory.py @@ -18,6 +18,30 @@ def expected_ratio(f: float, q: ArrayLike) -> ArrayLike: return np.log(1.0 - f) / np.log(1.0 - f / q) +def q_effective(q: ArrayLike, r: ArrayLike) -> ArrayLike: + """Effective slot utilisation with uncle recovery: ``q_u = q + (1 - q) r``. + + ``r`` is the recovery rate — the probability that a wasted active slot is recovered by a + countable referenced uncle. Recovery is a binomial thinning of the waste + (``n ~ Bin(p, A)`` wasted, ``u | n ~ Bin(r, n)`` recovered, so the residual waste is + ``Bin(p(1-r), A)``), hence every closed-form result above holds verbatim with ``q`` + replaced by ``q_u``. Full recovery (``r = 1``) gives ``q_u = 1`` and an unbiased + equilibrium; ``r = 0`` reduces to the chain-only ``q``. + """ + q = np.asarray(q, dtype=float) + r = np.asarray(r, dtype=float) + return q + (1.0 - q) * r + + +def window_miss_prob(f: float, w_abs: ArrayLike) -> ArrayLike: + """P(no canonical block appears within the uncle window ``w_u = W/f``) — the window's + contribution to non-recovery: ``(1-f)^(W/f) ~ e^-W`` (4.5e-5 at the default W = 10). + The absorption parameter W therefore controls the miss probability directly. + """ + w_abs = np.asarray(w_abs, dtype=float) + return (1.0 - f) ** (w_abs / f) + + def block_count_ceiling(f: float) -> float: """LEGACY-mode ceiling: the equilibrium ratio under ``legacy_block_count=True``. diff --git a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/tsi.py b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/tsi.py index 29d5f25..32d1f45 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/tsi.py +++ b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/tsi.py @@ -19,30 +19,61 @@ from .blocktree import BlockTree def referenced_uncle_ids(tree: BlockTree, canonical_ids: list[int]) -> set[int]: - """Deduplicated set of uncle ids referenced by the canonical chain.""" + """Deduplicated set of uncle ids referenced by the canonical chain (old model: all).""" ref: set[int] = set() for b in canonical_ids: ref.update(tree.uncles[b]) return ref +def countable_refs(tree: BlockTree, canonical_ids: list[int], w: int) -> set[int]: + """Deduplicated set of COUNTABLE referenced uncles (spec counting rules). + + A reference ``u`` of canonical block ``b`` is countable iff ``u`` is not itself + canonical, ``0 < slot_b - slot_u <= w``, and ``u``'s parent lies on the canonical chain + (only the first block of a fork counts) — the per-reference re-check of + cryptarchia-v1-protocol.md's counting rules. Reference implementation for the + measurement kernel (see test_measure / test_tsi_counting). + """ + canon = set(canonical_ids) + out: set[int] = set() + for b in canonical_ids: + sb = int(tree.slot[b]) + for u in tree.uncles[b]: + if u in canon: + continue # uncle lies on the counting chain + du = sb - int(tree.slot[u]) + if not 0 < du <= w: + continue # outside the reference window + p = int(tree.parent[u]) + if p != 0 and p not in canon: + continue # not a first fork block (deep): uncounted + out.add(u) + return out + + def _in_window(slot: int, T: int) -> bool: return 0 <= slot < T def density_m(tree: BlockTree, canonical_ids: list[int], T: int, - legacy_block_count: bool = False) -> int: + legacy_block_count: bool = False, + countable: bool = False, w: int = 0) -> int: """Slot count ``m`` for the TSI update: canonical slots + recovered uncle slots. A slot counts at most once: canonical blocks occupy distinct slots by construction, and a referenced uncle contributes only if its slot is not already canonical-occupied (and - only once per slot, however many same-slot uncles are referenced). ``legacy_block_count`` - reproduces the earlier per-block-id counting (double-counts multi-winner slots). + only once per slot, however many same-slot uncles are referenced). ``countable`` applies + the spec's per-reference counting rules via :func:`countable_refs` (window ``w``); + ``countable=False`` is the old model where every baked reference counts. + ``legacy_block_count`` reproduces the earlier per-block-id counting (double-counts + multi-winner slots). """ s = tree.slot[canonical_ids] in_win = (s >= 0) & (s < T) honest = int(in_win.sum()) - ref = referenced_uncle_ids(tree, canonical_ids) + ref = (countable_refs(tree, canonical_ids, w) if countable + else referenced_uncle_ids(tree, canonical_ids)) if legacy_block_count: return honest + sum(1 for u in ref if _in_window(int(tree.slot[u]), T)) canon_slots = set(int(x) for x in s[in_win]) diff --git a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/uncles.py b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/uncles.py index dfe8b42..04cc065 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/uncles.py +++ b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/uncles.py @@ -1,13 +1,21 @@ """Proposer-local uncle selection. -For each canonical block ``B`` (processed oldest-first so ancestors' references are -known), candidates are orphan (non-canonical) blocks ``U`` with -``0 < slot_B - slot_U <= W`` that have not already been referenced by an ancestor of -``B``. Two strategies match the spec: deterministic oldest-first, and random (oldest-first -order, a coin of probability ``uncle_random_p`` per candidate, capped at ``U``). The spec's -coin is unbiased (``uncle_random_p = 0.5``, the default); other values are a non-spec -sensitivity knob. Dedup across ancestors is enforced by threading a ``referenced`` set down -the canonical chain. +Two models, switched by ``config.uncle_model`` (CLI: ``--old``): + +**countable** (default) — the spec's counting-only model (cryptarchia-v1-protocol.md, +Uncle Selection): candidates are orphan blocks in the producer's view within the DERIVED +window ``w_u = window_absorption / f`` whose **parent lies on the producer's chain** (only +the first block of a fork is countable), excluding candidates whose slot is already +occupied on that chain (by a canonical block or an already-referenced uncle), and picking +at most one uncle per slot, oldest-first (or the ``random`` sensitivity knob). + +**old** (pre-redesign; kept verbatim for ``--old`` reproduction) — candidates are ANY +orphan blocks in view with ``0 < slot_B - slot_U <= uncle_window``, regardless of fork +depth, that are not on the producer's chain and not already referenced by it; dedup is by +block id only (no slot exclusion). + +Selected refs are baked at production and immutable once adopted, so density counting +stays view-independent under both models. """ from __future__ import annotations @@ -32,39 +40,77 @@ def _orphans_sorted(tree: BlockTree, canonical_ids: list[int]) -> tuple[np.ndarr def annotate_uncles( tree: BlockTree, canonical_ids: list[int], config: SimConfig, rng: np.random.Generator ) -> None: - """Fill ``tree.uncles[B]`` for every canonical block ``B`` per the selection rule.""" + """Fill ``tree.uncles[B]`` for every canonical block ``B`` per the selection rule (offline). + + Countable model: candidates are restricted to orphans whose parent is canonical (first + fork blocks), slots already occupied on the chain are excluded, and at most one uncle + per slot is picked. Old model: any orphan in the window, dedup by id only. + """ u_max = config.max_uncles if u_max <= 0: return - w = config.uncle_window + w = config.effective_uncle_window orphan_ids, orphan_slots = _orphans_sorted(tree, canonical_ids) if orphan_ids.size == 0: return + countable = config.uncle_model != "old" + occupied: set[int] = set() + if countable: + canonical = set(canonical_ids) + keep = [i for i in range(orphan_ids.size) + if int(tree.parent[orphan_ids[i]]) == GENESIS + or int(tree.parent[orphan_ids[i]]) in canonical] + orphan_ids, orphan_slots = orphan_ids[keep], orphan_slots[keep] + if orphan_ids.size == 0: + return + occupied = {int(tree.slot[b]) for b in canonical_ids} + referenced: set[int] = set() # oldest canonical block first for b in reversed(canonical_ids): sb = int(tree.slot[b]) - lo = int(np.searchsorted(orphan_slots, sb - w, side="left")) # slot_U >= sb - W + lo = int(np.searchsorted(orphan_slots, sb - w, side="left")) # slot_U >= sb - w hi = int(np.searchsorted(orphan_slots, sb, side="left")) # slot_U < sb if hi <= lo: continue window_ids = orphan_ids[lo:hi] # already oldest-first - selected = _select(window_ids, referenced, config, rng) + if countable: + window_ids = np.array( + [x for x in window_ids.tolist() if int(tree.slot[x]) not in occupied], + dtype=np.int64, + ) + selected = _select(window_ids, referenced, config, rng, + slot=tree.slot, one_per_slot=countable) if selected: tree.uncles[b] = tuple(selected) referenced.update(selected) + if countable: + occupied.update(int(tree.slot[u]) for u in selected) def _select( - window_ids: np.ndarray, referenced: set[int], config: SimConfig, rng: np.random.Generator + window_ids: np.ndarray, + referenced: set[int], + config: SimConfig, + rng: np.random.Generator, + slot: np.ndarray | None = None, + one_per_slot: bool = False, ) -> list[int]: + """Pick up to ``max_uncles`` candidates. ``one_per_slot`` adds the countable model's + per-slot dedup (a second same-slot candidate adds no occupied slot, so it is skipped).""" u_max = config.max_uncles out: list[int] = [] + slots_taken: set[int] = set() if config.uncle_strategy == "oldest": for bid in window_ids.tolist(): if bid in referenced: continue + if one_per_slot: + s = int(slot[bid]) + if s in slots_taken: + continue + slots_taken.add(s) out.append(bid) if len(out) >= u_max: break @@ -73,7 +119,11 @@ def _select( for bid in window_ids.tolist(): if bid in referenced: continue + if one_per_slot and int(slot[bid]) in slots_taken: + continue if rng.random() < p: + if one_per_slot: + slots_taken.add(int(slot[bid])) out.append(bid) if len(out) >= u_max: break @@ -96,21 +146,24 @@ def select_uncles_at_production( ) -> tuple[int, ...]: """Uncles a block gets when produced by node ``v`` (arrival row ``arrival_v``) at slot ``t``. - Candidates are blocks in ``v``'s view (``arrival_v[b] <= t``) with slot in ``[t-W, t)`` - that are NOT on the chain ``v`` extends (ancestors of ``parent_id``) and not already - referenced by that chain. Selected once and baked globally (same for everyone who adopts - the block), so density counting stays view-independent. + Candidates are blocks in ``v``'s view (``arrival_v[b] <= t``) with slot in + ``[t-w_u, t)`` that are NOT on the chain ``v`` extends (ancestors of ``parent_id``) and + not already referenced by that chain; the countable model (default) additionally + requires the candidate's **parent to lie on that chain** (first block of its fork), + excludes candidates whose slot is already occupied on the chain, and picks at most one + per slot. Selected once and baked globally (same for everyone who adopts the block), so + density counting stays view-independent. ``arrival_v`` is indexed by *block id minus ``arr_base``* — ``arr_base=0`` for the full arrival matrix row ``A[v]``, or the sliding-window buffer's base offset when pruning (every uncle-window - block ``[t-W, t)`` is inside the kept span, so the buffer row covers all candidates). + block ``[t-w_u, t)`` is inside the kept span, so the buffer row covers all candidates). """ u_max = config.max_uncles if u_max <= 0: return () - w = config.uncle_window + w = config.effective_uncle_window slot_view = slot[:nb] - lo = int(np.searchsorted(slot_view, t - w, side="left")) # slot >= t-W + lo = int(np.searchsorted(slot_view, t - w, side="left")) # slot >= t-w_u hi = int(np.searchsorted(slot_view, t, side="left")) # slot < t if hi <= lo: return () @@ -118,18 +171,61 @@ def select_uncles_at_production( arrived = np.nonzero(arrival_v[lo - arr_base:hi - arr_base] <= t)[0] + lo if arrived.size == 0: return () - # v's own chain within the window + the uncles it already references (for dedup) - on_chain: set[int] = set() - referenced: set[int] = set() + + if config.uncle_model == "old": + # --- old model (pre countable redesign; kept verbatim for --old) ---------------- + # v's own chain within the window + the uncles it already references (for dedup) + on_chain: set[int] = set() + referenced: set[int] = set() + a = int(parent_id) + while a > GENESIS and int(slot[a]) >= t - w: + on_chain.add(a) + referenced.update(uncles[a]) + a = int(parent[a]) + cands = np.array( + [b for b in arrived.tolist() + if b > GENESIS and b not in on_chain and b not in referenced], + dtype=np.int64, + ) + if cands.size == 0: + return () + return tuple(_select(cands, set(), config, rng)) # cands already oldest-first (slot,id) + + # --- countable model (spec counting rules; the default) ----------------------------- + # Window walk over v's chain: chain blocks, their referenced uncles, and the slots both + # occupy (the spec's occupied-slot exclusion in Uncle Selection). + on_chain = set() + referenced = set() + occupied: set[int] = set() a = int(parent_id) while a > GENESIS and int(slot[a]) >= t - w: on_chain.add(a) - referenced.update(uncles[a]) + occupied.add(int(slot[a])) + for u in uncles[a]: + referenced.add(u) + su = int(slot[u]) + if su >= t - w: + occupied.add(su) a = int(parent[a]) + pre = [b for b in arrived.tolist() + if b > GENESIS and b not in on_chain and b not in referenced + and int(slot[b]) not in occupied] + if not pre: + return () + # Parent-on-chain (only the first block of a fork is countable): the window walk covers + # parents inside the window; extend chain membership exactly far enough below it to + # decide the oldest candidate parent (cheap — parents are typically recent). + pmin = min(int(slot[int(parent[b])]) for b in pre) + below: set[int] = set() + while a > GENESIS and int(slot[a]) >= pmin: + below.add(a) + a = int(parent[a]) + chain_ids = on_chain | below cands = np.array( - [b for b in arrived.tolist() if b > GENESIS and b not in on_chain and b not in referenced], + [b for b in pre if int(parent[b]) == GENESIS or int(parent[b]) in chain_ids], dtype=np.int64, ) if cands.size == 0: return () - return tuple(_select(cands, set(), config, rng)) # cands already oldest-first (slot,id) + # cands already oldest-first (slot, id); one uncle per slot per the spec's selection. + return tuple(_select(cands, set(), config, rng, slot=slot, one_per_slot=True)) diff --git a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/verify.py b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/verify.py index 411a6a6..1f91e3b 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/verify.py +++ b/tools/simulators/tsi/tsi-sim-pernode/src/tsi_sim/verify.py @@ -38,9 +38,18 @@ def check(name: str, ok: bool, detail: str) -> bool: return ok -def main() -> int: +def main(argv: list[str] | None = None) -> int: + import argparse + + ap = argparse.ArgumentParser(description="Per-node TSI analytic checks") + ap.add_argument("--old", action="store_true", + help="run the old (pre countable redesign) uncle model") + args = ap.parse_args(argv) + uncle_model = "old" if args.old else "countable" + results = [] - common = dict(n_nodes=300, stake_dist="uniform", k=K, epochs=EPOCHS, genesis_d_factor=0.5) + common = dict(n_nodes=300, stake_dist="uniform", k=K, epochs=EPOCHS, genesis_d_factor=0.5, + uncle_model=uncle_model) # 1. Full-mesh baseline: zero per-node divergence, full window agreement. fm = SimConfig(topology="full_mesh", latency=4, max_uncles=0, **common) diff --git a/tools/simulators/tsi/tsi-sim-pernode/tests/test_config.py b/tools/simulators/tsi/tsi-sim-pernode/tests/test_config.py index a40f395..26e47fa 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/tests/test_config.py +++ b/tools/simulators/tsi/tsi-sim-pernode/tests/test_config.py @@ -85,13 +85,15 @@ def test_blend_dedup_does_not_multiply_non_blend_configs(): def test_uncle_window_sweeps_and_collapses_for_u0(): - # uncle_window is a live axis for U>0, but U=0 references no uncles so it must collapse. + # OLD model: uncle_window is a live axis for U>0, but U=0 references no uncles so it + # must collapse. (The countable model ignores uncle_window entirely — see the twin + # test below.) sweep = SweepConfig( n_nodes=[100], stake_dist=["uniform"], topology=["blend"], degree=[6], link_latency_mean=[0.5], link_latency_dist=["geo"], blend_hops=[3], blend_delay_max=[4.0], uncle_window=[10, 100], max_uncles=[0, 1], uncle_strategy=["oldest"], init_dest=["common"], replicates=1, - base={"k": 8, "epochs": 3}, + base={"k": 8, "epochs": 3, "uncle_model": "old"}, ) configs = sweep.expand() u0 = [c for c in configs if c.max_uncles == 0] @@ -100,6 +102,24 @@ def test_uncle_window_sweeps_and_collapses_for_u0(): assert {c.uncle_window for c in u1} == {10, 100} # both W kept for U=1 +def test_window_absorption_sweeps_and_ignored_axis_collapses(): + # COUNTABLE model: window_absorption is the live window axis; uncle_window is ignored + # and must collapse. And vice versa for the old model (guarded above). + sweep = SweepConfig( + n_nodes=[100], stake_dist=["uniform"], topology=["blend"], degree=[6], + link_latency_mean=[0.5], link_latency_dist=["geo"], blend_hops=[3], + blend_delay_max=[4.0], uncle_window=[10, 100], window_absorption=[2.0, 4.0], + max_uncles=[0, 1], uncle_strategy=["oldest"], init_dest=["common"], replicates=1, + base={"k": 8, "epochs": 3}, + ) + configs = sweep.expand() + u0 = [c for c in configs if c.max_uncles == 0] + u1 = [c for c in configs if c.max_uncles == 1] + assert len(u0) == 1 # all window knobs collapse + assert {c.window_absorption for c in u1} == {2.0, 4.0} # live axis kept for U=1 + assert {c.uncle_window for c in u1} == {10} # ignored axis collapsed + + def test_unknown_sweep_key_rejected(): with pytest.raises(ValueError, match="unknown sweep keys"): SweepConfig.from_dict({"latencies": [0, 1], "base": {}}) # typo: latencies vs latency @@ -117,7 +137,11 @@ def test_key_covers_every_field(): # optimisation (no RNG, identical results) so it is intentionally not in key(). # early_stop is truncation-only (per-epoch RNG streams are pre-spawned, so the epochs # that DO run are bit-identical to a full run's prefix) — intentionally excluded from key(). - ignored = {"root_seed", "windowed_fork_choice", "prune_arrival", "early_stop"} + # uncle_window is read ONLY by the old model; under the (default) countable model it is + # an ignored field, deliberately left in the base tuple at its old position so that an + # --old run's key stays byte-identical to historical keys. + ignored = {"root_seed", "windowed_fork_choice", "prune_arrival", "early_stop", + "uncle_window"} names = {f.name for f in dataclasses.fields(SimConfig)} - ignored a = SimConfig() for name in names: @@ -125,6 +149,42 @@ def test_key_covers_every_field(): alt = _perturb(cur) b = dataclasses.replace(a, **{name: alt}) assert a.key() != b.key(), f"key() does not distinguish field {name!r}" + # ... and uncle_window IS distinguished under the old model, where it is live. + old = SimConfig(uncle_model="old") + assert old.key() != dataclasses.replace(old, uncle_window=old.uncle_window + 1).key() + + +def test_old_model_key_is_historical(): + # --old must bit-reproduce historical runs: its key is exactly the pre-uncle_model + # tuple (no uncle_model / window_absorption entries), and the countable key extends it. + old = SimConfig(uncle_model="old") + new = SimConfig() + assert new.key()[: len(old.key())] == old.key() + assert new.key()[len(old.key()):] == ("countable", new.window_absorption) + # window_absorption is ignored (and absent from key) under the old model... + assert dataclasses.replace(old, window_absorption=2.0).key() == old.key() + # ...and live under the countable model. + assert dataclasses.replace(new, window_absorption=2.0).key() != new.key() + + +def test_effective_uncle_window(): + # countable: derived w_u = round(W / f); old: uncle_window taken directly. + assert SimConfig(k=2160).effective_uncle_window == 300 # W=10, f=1/30 + assert SimConfig(k=2160, window_absorption=5.0).effective_uncle_window == 150 + assert SimConfig(uncle_model="old", uncle_window=42).effective_uncle_window == 42 + + +def test_window_absorption_bound(): + import warnings + + with pytest.raises(ValueError): + SimConfig(window_absorption=0.5) # W < 1 rejected + with pytest.warns(RuntimeWarning, match="exceeds the spec bound"): + SimConfig(k=8, window_absorption=10.0) # W > 0.6*k warns + with warnings.catch_warnings(): + warnings.simplefilter("error") + SimConfig(k=2160, window_absorption=10.0) # full scale: silent + SimConfig(k=8, uncle_model="old", uncle_window=300) # old model: no bound def _perturb(v): @@ -140,6 +200,7 @@ def _perturb(v): "fixed": "exp", "common": "heterogeneous", "suppress": "withhold", "exp": "poisson", # jitter_dist "sine": "ramp", # churn_mode + "countable": "old", # uncle_model } if isinstance(v, str) and v in flips: return flips[v] diff --git a/tools/simulators/tsi/tsi-sim-pernode/tests/test_countable_counting.py b/tools/simulators/tsi/tsi-sim-pernode/tests/test_countable_counting.py new file mode 100644 index 0000000..748516a --- /dev/null +++ b/tools/simulators/tsi/tsi-sim-pernode/tests/test_countable_counting.py @@ -0,0 +1,84 @@ +"""Countable-model counting: measurement kernels vs the tsi.py reference oracle. + +Hand-built tree exercising every counting rule on baked references: +canonical 1(s0) -> 2(s2) -> 3(s5) -> 4(s10, tip); orphans 5(s1, parent 1, first fork), +6(s6, parent 5, DEEP), 7(s7, parent 1, first fork), 8(s7, parent 2, first fork, same slot +as 7). References: block2 -> (5,), block3 -> (1,) [a canonical block], block4 -> (6, 7, 8). + +With w = 5 and T = 20 the countable verdicts are: 5 counted (d=1); 1 skipped (on chain); +6 deep-rejected (parent is an orphan); 7 counted (d=3); 8 counted by id but its slot is +already recovered by 7 (slot dedup). Old model counts every reference. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from tsi_sim.blocktree import BlockTree +from tsi_sim.measure import measure +from tsi_sim.tsi import countable_refs, density_m + +W = 5 +T = 20 + + +def _tree() -> BlockTree: + tree = BlockTree( + slot=np.array([-1, 0, 2, 5, 10, 1, 6, 7, 7], np.int64), + parent=np.array([-1, 0, 1, 2, 3, 1, 5, 1, 2], np.int64), + height=np.array([0, 1, 2, 3, 4, 2, 3, 2, 3], np.int64), + leader=np.array([-1, 0, 1, 2, 3, 4, 5, 6, 7], np.int64), + uncles=[() for _ in range(9)], + ) + tree.uncles[2] = (5,) + tree.uncles[3] = (1,) + tree.uncles[4] = (6, 7, 8) + return tree + + +CANONICAL = [4, 3, 2, 1] +ACTIVE = np.array([0, 1, 2, 5, 6, 7, 10], np.int64) # 7 distinct active slots in T + + +def test_countable_refs_oracle(): + assert countable_refs(_tree(), CANONICAL, W) == {5, 7, 8} + + +def test_density_m_countable_and_old(): + tree = _tree() + # countable: honest slots {0,2,5,10} + recovered slots {1, 7} -> 6 + assert density_m(tree, CANONICAL, T, countable=True, w=W) == 6 + # old model: every reference counts -> recovered slots {1, 6, 7} -> 7 + assert density_m(tree, CANONICAL, T) == 7 + + +@pytest.mark.parametrize("use_numba", [False, True]) +def test_measure_countable_matches_oracle(use_numba): + tree = _tree() + A = np.zeros((2, tree.n_blocks)) # both nodes received everything + ms = measure(tree, A, ACTIVE, T, cutoff=15, use_numba=use_numba, + countable=True, w=W) + np.testing.assert_array_equal(ms.m, [6, 6]) # = density_m countable + np.testing.assert_allclose(ms.q, 4 / 7) # canonical slots / active + np.testing.assert_allclose(ms.q_eff, 6 / 7) # + recovered slots + np.testing.assert_array_equal(ms.ref_total, [5, 5]) # ids 5,1,6,7,8 examined + np.testing.assert_array_equal(ms.ref_deep, [1, 1]) # id 6 rejected as deep + + +@pytest.mark.parametrize("use_numba", [False, True]) +def test_measure_old_counts_all_refs(use_numba): + tree = _tree() + A = np.zeros((2, tree.n_blocks)) + ms = measure(tree, A, ACTIVE, T, cutoff=15, use_numba=use_numba) + np.testing.assert_array_equal(ms.m, [7, 7]) # = density_m old + np.testing.assert_allclose(ms.q_eff, 7 / 7) + np.testing.assert_array_equal(ms.ref_deep, [0, 0]) # no rule to reject on + + +def test_window_recheck_rejects_stale_reference(): + # A baked reference outside the counting window is uncounted under countable + # (the old model still counts it): shrink w below block4 -> uncle 7 distance (d=3). + tree = _tree() + assert countable_refs(tree, CANONICAL, 2) == {5} # 7, 8 now out of window (d=3) + assert density_m(tree, CANONICAL, T, countable=True, w=2) == 5 diff --git a/tools/simulators/tsi/tsi-sim-pernode/tests/test_slot_counting.py b/tools/simulators/tsi/tsi-sim-pernode/tests/test_slot_counting.py index aa1ca6e..a767610 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/tests/test_slot_counting.py +++ b/tools/simulators/tsi/tsi-sim-pernode/tests/test_slot_counting.py @@ -64,19 +64,31 @@ def test_distinct_slot_uncles_still_counted(): def test_zero_delay_equilibrium_is_one_not_ceiling(): - """The c(f) ceiling was the bug: corrected counting equilibrates at 1.0 with uncles.""" + """The c(f) ceiling was the bug: corrected counting equilibrates at 1.0 with uncles. + + Holds under the (default) countable model too: at zero delay the only orphans are + same-slot co-winners, which countable selection never references (occupied slot) and + which add nothing to the slot count anyway. 5 replicates / 0.02 tolerance because the + countable model's key() draws a different RNG stream than the historical runs the old + 3-rep/0.015 margin was tuned on. + """ base = dict(n_nodes=300, stake_dist="uniform", topology="full_mesh", latency=0, max_uncles=2, uncle_window=300, k=64, epochs=24, genesis_d_factor=1.0) tails = [] - for rep in range(3): + for rep in range(5): df = pd.DataFrame(run_trajectory(SimConfig(**base, replicate=rep))) tails.append(df[df.epoch >= 8].mean_ratio.mean()) - assert abs(np.mean(tails) - 1.0) < 0.015 + assert abs(np.mean(tails) - 1.0) < 0.02 def test_legacy_flag_reproduces_the_ceiling(): + # OLD model on purpose: the c(f) ceiling arises from referencing same-slot co-winners + # and counting them per block id. The countable model never references a same-slot + # co-winner (its slot is already occupied on the chain), so under it the legacy flag + # has nothing to double-count and this historical bug cannot be reproduced. base = dict(n_nodes=300, stake_dist="uniform", topology="full_mesh", latency=0, - max_uncles=2, uncle_window=300, k=64, epochs=24, genesis_d_factor=1.0) + max_uncles=2, uncle_window=300, k=64, epochs=24, genesis_d_factor=1.0, + uncle_model="old") tails = [] for rep in range(3): df = pd.DataFrame(run_trajectory( diff --git a/tools/simulators/tsi/tsi-sim-pernode/tests/test_uncles.py b/tools/simulators/tsi/tsi-sim-pernode/tests/test_uncles.py index 7e8b244..4240cc8 100644 --- a/tools/simulators/tsi/tsi-sim-pernode/tests/test_uncles.py +++ b/tools/simulators/tsi/tsi-sim-pernode/tests/test_uncles.py @@ -45,14 +45,33 @@ def test_no_uncles_when_u_zero(): def test_window_excludes_out_of_range_orphan(): + # OLD model: uncle_window is read directly. (The countable model ignores uncle_window + # and derives the window from window_absorption — see the countable twin below.) tree, canonical = _canonical_and_orphan_tree() - cfg = SimConfig(max_uncles=1, uncle_window=1, uncle_strategy="oldest") + cfg = SimConfig(max_uncles=1, uncle_window=1, uncle_strategy="oldest", uncle_model="old") annotate_uncles(tree, canonical, cfg, np.random.default_rng(0)) # orphan 2 at slot1; nearest canonical after it is block3 at slot3 -> gap 2 > W=1 referenced = {u for b in canonical for u in tree.uncles[b]} assert referenced == set() +def test_countable_window_is_derived_from_absorption(): + # countable: w_u = round(W / f). With f=0.5 and W=1, w_u = 2 slots: the orphan at slot1 + # is out of range of the canonical block at slot5 (gap 4) and of slot3 (gap 2 <= 2 OK). + tree, canonical = _canonical_and_orphan_tree() + cfg = SimConfig(max_uncles=1, f=0.5, window_absorption=1.0) + assert cfg.effective_uncle_window == 2 + annotate_uncles(tree, canonical, cfg, np.random.default_rng(0)) + referenced = {u for b in canonical for u in tree.uncles[b]} + assert referenced == {2} # block3 (slot3) still reaches it + # shrink f so the derived window rounds to 1 slot: gap 2 > 1 -> excluded + tree2, canonical2 = _canonical_and_orphan_tree() + cfg2 = SimConfig(max_uncles=1, f=0.9, window_absorption=1.0) + assert cfg2.effective_uncle_window == 1 + annotate_uncles(tree2, canonical2, cfg2, np.random.default_rng(0)) + assert {u for b in canonical2 for u in tree2.uncles[b]} == set() + + def _wide_orphan_tree(): # canonical 1(0)->6(6); orphans 2,3,4,5 at slots 1,2,3,4 (all within window of block6) tree = make_tree( @@ -108,3 +127,72 @@ def test_dedup_across_ancestors(): annotate_uncles(tree, canonical, cfg, np.random.default_rng(0)) counts = sum(len(tree.uncles[b]) for b in canonical) assert counts == 1 # orphan 2 referenced exactly once despite two eligible blocks + + +# --- countable model (spec counting rules) --------------------------------------------- + + +def _deep_fork_tree(): + # canonical 1(slot0)->5(slot5); orphan branch 2(slot1,parent=1)->3(slot2,parent=2); + # orphan 4(slot3, parent=1). Blocks 2,4 are FIRST fork blocks; 3 is deep. + tree = make_tree( + slots=[-1, 0, 1, 2, 3, 5], + parents=[-1, 0, 1, 2, 1, 1], + heights=[0, 1, 2, 3, 2, 2], + leaders=[-1, 0, 1, 2, 3, 4], + ) + return tree, [5, 1] # tip-first + + +def test_countable_excludes_deep_fork_blocks(): + tree, canonical = _deep_fork_tree() + cfg = SimConfig(max_uncles=4) + annotate_uncles(tree, canonical, cfg, np.random.default_rng(0)) + referenced = {u for b in canonical for u in tree.uncles[b]} + assert referenced == {2, 4} # deep block 3 (parent is an orphan) excluded + + +def test_old_model_still_references_deep_fork_blocks(): + tree, canonical = _deep_fork_tree() + cfg = SimConfig(max_uncles=4, uncle_model="old", uncle_window=300) + annotate_uncles(tree, canonical, cfg, np.random.default_rng(0)) + referenced = {u for b in canonical for u in tree.uncles[b]} + assert referenced == {2, 3, 4} # --old: fork depth ignored + + +def test_countable_excludes_occupied_slots_and_dedups_per_slot(): + # canonical 1(slot0)->5(slot4); orphans: 2 at slot0 (canonical-occupied), 3/4 at slot2. + tree = make_tree( + slots=[-1, 0, 0, 2, 2, 4], + parents=[-1, 0, 0, 1, 1, 1], + heights=[0, 1, 1, 2, 2, 2], + leaders=[-1, 0, 1, 2, 3, 4], + ) + canonical = [5, 1] + cfg = SimConfig(max_uncles=4) + annotate_uncles(tree, canonical, cfg, np.random.default_rng(0)) + referenced = {u for b in canonical for u in tree.uncles[b]} + # slot0 is canonical-occupied -> orphan 2 excluded; slot2 pair -> exactly one picked + assert referenced == {3} + + +def test_production_selection_countable_rules(): + from tsi_sim.uncles import select_uncles_at_production + + # 0 genesis; 1 canonical slot0; 2 first-fork slot1 (parent 1); 3 deep slot2 (parent 2); + # 4/5 same-slot first-forks at slot3 (parent 1). + slot = np.array([-1, 0, 1, 2, 3, 3], np.int64) + parent = np.array([-1, 0, 1, 2, 1, 1], np.int64) + uncles: list = [() for _ in range(6)] + arrival = np.zeros(6) # everything arrived immediately + cfg = SimConfig(max_uncles=4) + sel = select_uncles_at_production( + slot, parent, uncles, arrival, nb=6, parent_id=1, t=5, config=cfg, + rng=np.random.default_rng(0)) + assert sel == (2, 4) # deep 3 excluded; one per slot at slot3 + + old = SimConfig(max_uncles=4, uncle_model="old", uncle_window=300) + sel_old = select_uncles_at_production( + slot, parent, uncles, arrival, nb=6, parent_id=1, t=5, config=old, + rng=np.random.default_rng(0)) + assert sel_old == (2, 3, 4, 5) # --old: depth and slot-dedup ignored