mirror of
https://github.com/logos-blockchain/research.git
synced 2026-08-08 12:13:14 +00:00
69 lines
2.9 KiB
Python
69 lines
2.9 KiB
Python
"""Fail-loud memory guards for the per-node engine's large allocations.
|
|
|
|
The two dominant arrays — the ``(N x n_blocks)`` arrival matrix ``A`` and the ``(N x N)``
|
|
``path_latency`` — can each reach tens to hundreds of GB: ``A`` when a low ``genesis_d_factor``
|
|
explodes ``n_blocks`` (the collapsed-``D_est`` regime), ``path_latency`` at very large ``N``.
|
|
Every worker checks the size *before* allocating and raises ``ArrivalMatrixTooLarge`` if it would
|
|
exceed its budget, so an under-estimated config fails with a clear message instead of freezing the
|
|
machine.
|
|
|
|
Budget resolution (``arrival_budget_bytes``):
|
|
- ``TSI_ARRIVAL_BYTES_BUDGET`` > 0 -> that many bytes (the sweep sets this to each worker's RAM
|
|
share so concurrent workers can't collectively OOM);
|
|
- unset / ``0`` / invalid -> a default of ``DEFAULT_BUDGET_FRAC`` of physical RAM, so a
|
|
*single* process (a bare ``run_trajectory``, ``tsi-verify``, the calibration probe, or a
|
|
``--mem-frac 0`` run) still cannot allocate past what the box physically has.
|
|
There is intentionally no "unlimited" setting: no correct run needs to allocate more than physical
|
|
RAM, and allowing it is exactly what froze the machine.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
|
|
# Absolute per-process ceiling (fraction of physical RAM) used when no explicit budget is set.
|
|
DEFAULT_BUDGET_FRAC = 0.9
|
|
|
|
|
|
class ArrivalMatrixTooLarge(MemoryError):
|
|
"""A per-node engine array would exceed the memory budget; raised before allocating.
|
|
|
|
Typically a block-count explosion (a low ``genesis_d_factor`` inflating early-epoch lottery
|
|
wins) blowing up the ``(N x n_blocks)`` arrival matrix, or a very large ``N`` blowing up the
|
|
``(N x N)`` ``path_latency`` matrix.
|
|
"""
|
|
|
|
|
|
def total_ram_bytes() -> int:
|
|
"""Best-effort physical RAM in bytes (POSIX sysconf, then Darwin sysctl, then 8 GB)."""
|
|
try:
|
|
return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
|
|
except (ValueError, OSError, AttributeError):
|
|
pass
|
|
try: # macOS lacks SC_PHYS_PAGES
|
|
out = subprocess.run(["sysctl", "-n", "hw.memsize"], capture_output=True, text=True)
|
|
return int(out.stdout.strip())
|
|
except (OSError, ValueError):
|
|
return 8 * 1024**3
|
|
|
|
|
|
def arrival_budget_bytes() -> int:
|
|
"""Per-process byte budget for a single big array (see module docstring)."""
|
|
try:
|
|
explicit = int(os.environ.get("TSI_ARRIVAL_BYTES_BUDGET", "0"))
|
|
except ValueError:
|
|
explicit = 0
|
|
if explicit > 0:
|
|
return explicit
|
|
return int(DEFAULT_BUDGET_FRAC * total_ram_bytes())
|
|
|
|
|
|
def check_alloc(nbytes: int, label: str, detail: str = "") -> None:
|
|
"""Raise ``ArrivalMatrixTooLarge`` if allocating ``nbytes`` would exceed the budget."""
|
|
budget = arrival_budget_bytes()
|
|
if nbytes > budget:
|
|
raise ArrivalMatrixTooLarge(
|
|
f"{label} needs {nbytes / 1024**3:.1f} GB > per-process budget "
|
|
f"{budget / 1024**3:.1f} GB.{(' ' + detail) if detail else ''}")
|