logos-blockchain/logos-blockchain-testing
1
0
Fork 0
mirror of https://github.com/logos-blockchain/logos-blockchain-testing.git synced 2026-01-02 13:23:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(testing-framework): rename runners to deployers

Browse Source 
- Update paths and orchestration for deployers (compose/k8s/local/docker)

- Consolidate scripts helpers and refresh book/README docs
This commit is contained in:
andrussal 2025-12-16 21:20:27 +01:00
parent bba0350f58
commit f451fd504d
111 changed files with 3074 additions and 2763 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -6930,6 +6930,7 @@ dependencies = [
"async-trait",
"k8s-openapi",
"kube",
"nomos-tracing-service",
"reqwest",
"serde",
"serde_yaml",
@ -6963,6 +6964,7 @@ dependencies = [
"key-management-system-service",
"nomos-core",
"rand 0.8.5",
"reqwest",
"testing-framework-config",
"testing-framework-core",
"thiserror 2.0.17",

12
Cargo.toml
View File

@ -4,9 +4,9 @@ members = [
"examples/doc-snippets",
"testing-framework/configs",
"testing-framework/core",
"testing-framework/runners/compose",
"testing-framework/runners/k8s",
"testing-framework/runners/local",
"testing-framework/deployers/compose",
"testing-framework/deployers/k8s",
"testing-framework/deployers/local",
"testing-framework/tools/cfgsync",
"testing-framework/workflows",
]
@ -32,9 +32,9 @@ all = "allow"
# Local testing framework crates
testing-framework-config = { default-features = false, path = "testing-framework/configs" }
testing-framework-core = { default-features = false, path = "testing-framework/core" }
testing-framework-runner-compose = { default-features = false, path = "testing-framework/runners/compose" }
testing-framework-runner-k8s = { default-features = false, path = "testing-framework/runners/k8s" }
testing-framework-runner-local = { default-features = false, path = "testing-framework/runners/local" }
testing-framework-runner-compose = { default-features = false, path = "testing-framework/deployers/compose" }
testing-framework-runner-k8s = { default-features = false, path = "testing-framework/deployers/k8s" }
testing-framework-runner-local = { default-features = false, path = "testing-framework/deployers/local" }
testing-framework-workflows = { default-features = false, path = "testing-framework/workflows" }
# Nomos git dependencies (pinned to latest master)

2
README.md
View File

@ -114,7 +114,7 @@ scripts/build-bundle.sh --platform linux
# Use the bundle when building images
export NOMOS_BINARIES_TAR=.tmp/nomos-binaries-linux-v0.3.1.tar.gz
testing-framework/assets/stack/scripts/build_test_image.sh
scripts/build_test_image.sh
```
## Environment Variables

7
book/src/annotated-tree.md
View File

@ -17,7 +17,7 @@ logos-blockchain-testing/
│ ├─ kzgrs_test_params/ # KZG circuit parameters directory
│ │ └─ kzgrs_test_params # Actual proving key file (note repeated name)
│ ├─ monitoring/ # Prometheus config
│ ├─ scripts/ # Container entrypoints, image builder
│ ├─ scripts/ # Container entrypoints
│ └─ cfgsync.yaml # Config sync server template
│
├─ examples/ # PRIMARY ENTRY POINT: runnable binaries
@ -53,12 +53,13 @@ Core library crates providing the testing API.
Docker/K8s deployment assets:
- **`kzgrs_test_params/kzgrs_test_params`**: Circuit parameters file (note repeated name; override via `NOMOS_KZGRS_PARAMS_PATH`)
- **`monitoring/`**: Prometheus config
- **`scripts/`**: Container entrypoints and image builder
- **`scripts/`**: Container entrypoints
### `scripts/`
Convenience utilities:
- **`run-examples.sh`**: All-in-one script for host/compose/k8s modes (recommended)
- **`build-bundle.sh`**: Create prebuilt binaries+circuits bundle for compose/k8s
- **`build_test_image.sh`**: Build the compose/k8s Docker image (bakes in assets)
- **`setup-circuits-stack.sh`**: Fetch KZG parameters for both Linux and host
- **`cfgsync.yaml`**: Configuration sync server template
@ -86,7 +87,7 @@ Helper utilities:
**Logging** controlled by:
- `NOMOS_LOG_DIR` — Write per-node log files
- `NOMOS_LOG_LEVEL` — Global log level (error/warn/info/debug/trace)
- `NOMOS_LOG_FILTER` — Target-specific filtering (e.g., `consensus=trace,da=debug`)
- `NOMOS_LOG_FILTER` — Target-specific filtering (e.g., `cryptarchia=trace,nomos_da_sampling=debug`)
- `NOMOS_TESTS_TRACING` — Enable file logging for local runner
See [Logging and Observability](operations.md#logging-and-observability) for details.

8
book/src/architecture-overview.md
View File

@ -97,7 +97,7 @@ Three deployer implementations:
## Assets and Images
### Docker Image
Built via `testing-framework/assets/stack/scripts/build_test_image.sh`:
Built via `scripts/build_test_image.sh`:
- Embeds KZG circuit parameters and binaries from `testing-framework/assets/stack/kzgrs_test_params/kzgrs_test_params`
- Includes runner scripts: `run_nomos_node.sh`, `run_nomos_executor.sh`
- Tagged as `NOMOS_TESTNET_IMAGE` (default: `logos-blockchain-testing:local`)
@ -123,12 +123,12 @@ Templates and configs in `testing-framework/runners/compose/assets/`:
| Component | Configuration | Output |
|-----------|--------------|--------|
| **Runner binaries** | `RUST_LOG` | Framework orchestration logs |
| **Node processes** | `NOMOS_LOG_LEVEL`, `NOMOS_LOG_FILTER`, `NOMOS_LOG_DIR` | Consensus, DA, mempool logs |
| **Node processes** | `NOMOS_LOG_LEVEL`, `NOMOS_LOG_FILTER` (+ `NOMOS_LOG_DIR` on host runner) | Consensus, DA, mempool logs |
**Node logging:**
- **Local runner:** Writes to temporary directories by default (cleaned up). Set `NOMOS_TESTS_TRACING=true` + `NOMOS_LOG_DIR` for persistent files.
- **Compose runner:** Default logs to container stdout/stderr (`docker logs`). Optional per-node files if `NOMOS_LOG_DIR` is set and mounted.
- **K8s runner:** Logs to pod stdout/stderr (`kubectl logs`). Optional per-node files if `NOMOS_LOG_DIR` is set and mounted.
- **Compose runner:** Default logs to container stdout/stderr (`docker logs`). To write per-node files, set `tracing_settings.logger: !File` in `testing-framework/assets/stack/cfgsync.yaml` (and mount a writable directory).
- **K8s runner:** Logs to pod stdout/stderr (`kubectl logs`). To write per-node files, set `tracing_settings.logger: !File` in `testing-framework/assets/stack/cfgsync.yaml` (and mount a writable directory).
**File naming:** Per-node files use prefix `nomos-node-{index}` or `nomos-executor-{index}` (may include timestamps).

64
book/src/operations.md
View File

@ -102,13 +102,13 @@ Notes:
If you hit Docker build failures, mysterious I/O errors, or are running out of disk space:
```bash
scripts/clean
scripts/clean.sh
```
For extra Docker cache cleanup:
```bash
scripts/clean --docker
scripts/clean.sh --docker
```
### Host Runner (Direct Cargo Run)
@ -127,11 +127,11 @@ cargo run -p runner-examples --bin local_runner
- `NOMOS_DEMO_EXECUTORS=2` — Number of executors (default: 1, or use legacy `LOCAL_DEMO_EXECUTORS`)
- `NOMOS_DEMO_RUN_SECS=120` — Run duration in seconds (default: 60, or use legacy `LOCAL_DEMO_RUN_SECS`)
- `NOMOS_NODE_BIN` / `NOMOS_EXECUTOR_BIN` — Paths to binaries (required for direct run)
- `NOMOS_LOG_DIR=/tmp/logs` — Directory for per-node log files (works across runners)
- `NOMOS_LOG_DIR=/tmp/logs` — Directory for per-node log files (host runner). For compose/k8s, set `tracing_settings.logger: !File` in `testing-framework/assets/stack/cfgsync.yaml`.
- `NOMOS_TESTS_KEEP_LOGS=1` — Keep per-run temporary directories (useful for debugging/CI artifacts)
- `NOMOS_TESTS_TRACING=true` — Enable the debug tracing preset (optional; combine with `NOMOS_LOG_DIR` unless you have external tracing backends configured)
- `NOMOS_LOG_LEVEL=debug` — Set log level (default: info)
- `NOMOS_LOG_FILTER=consensus=trace,da=debug` — Fine-grained module filtering
- `NOMOS_LOG_FILTER="cryptarchia=trace,nomos_da_sampling=debug"` — Fine-grained module filtering
**Note:** Requires circuit assets and host binaries. Use `scripts/run-examples.sh host` to handle setup automatically.
@ -148,7 +148,7 @@ scripts/build-bundle.sh --platform linux
# Build image (embeds bundle assets)
export NOMOS_BINARIES_TAR=.tmp/nomos-binaries-linux-v0.3.1.tar.gz
testing-framework/assets/stack/scripts/build_test_image.sh
scripts/build_test_image.sh
# Run
NOMOS_TESTNET_IMAGE=logos-blockchain-testing:local \
@ -168,7 +168,7 @@ scripts/setup-nomos-circuits.sh v0.3.1 /tmp/nomos-circuits
cp -r /tmp/nomos-circuits/* testing-framework/assets/stack/kzgrs_test_params/
# Build image
testing-framework/assets/stack/scripts/build_test_image.sh
scripts/build_test_image.sh
# Run
NOMOS_TESTNET_IMAGE=logos-blockchain-testing:local \
@ -184,7 +184,8 @@ cargo run -p runner-examples --bin compose_runner
- `TEST_FRAMEWORK_PROMETHEUS_PORT=9091` — Override Prometheus port (default: 9090)
- `COMPOSE_RUNNER_HOST=127.0.0.1` — Host address for port mappings
- `COMPOSE_RUNNER_PRESERVE=1` — Keep containers running after test
- `NOMOS_LOG_DIR=/tmp/compose-logs` — Write logs to files inside containers
- `NOMOS_LOG_LEVEL=debug` / `NOMOS_LOG_FILTER=...` — Control node log verbosity (stdout/stderr)
- `testing-framework/assets/stack/cfgsync.yaml` (`tracing_settings.logger`) — Switch node logs between stdout/stderr and file output
**Compose-specific features:**
- **Node control support**: Only runner that supports chaos testing (`.enable_node_control()` + chaos workloads)
@ -208,7 +209,7 @@ For manual control, you can run the `k8s_runner` binary directly. K8s requires t
# Build image with bundle (recommended)
scripts/build-bundle.sh --platform linux
export NOMOS_BINARIES_TAR=.tmp/nomos-binaries-linux-v0.3.1.tar.gz
testing-framework/assets/stack/scripts/build_test_image.sh
scripts/build_test_image.sh
# Load into cluster
export NOMOS_TESTNET_IMAGE=logos-blockchain-testing:local
@ -228,6 +229,32 @@ cargo run -p runner-examples --bin k8s_runner
- `NOMOS_TESTNET_IMAGE` — Image tag (required)
- `POL_PROOF_DEV_MODE=true` — **Required** for all runners
- `NOMOS_DEMO_VALIDATORS` / `NOMOS_DEMO_EXECUTORS` / `NOMOS_DEMO_RUN_SECS` — Topology overrides
- `K8S_RUNNER_EXTERNAL_PROMETHEUS_URL` (or `NOMOS_EXTERNAL_PROMETHEUS_URL`) — Reuse an existing Prometheus and skip deploying the in-chart Prometheus; also points node OTLP metrics export and the in-cluster Grafana datasource at that Prometheus
**External Prometheus (optional):**
```bash
export K8S_RUNNER_EXTERNAL_PROMETHEUS_URL=http://your-prometheus:9090
cargo run -p runner-examples --bin k8s_runner
```
Notes:
- The runner config expects Prometheus to accept OTLP metrics at `/api/v1/otlp/v1/metrics` (the in-chart Prometheus is started with `--web.enable-otlp-receiver` and `--enable-feature=otlp-write-receiver`).
- Use a URL reachable from inside the cluster (for example a `Service` DNS name like `http://prometheus.monitoring:9090`).
**Via `scripts/run-examples.sh` (optional):**
```bash
scripts/run-examples.sh -t 60 -v 1 -e 1 k8s --external-prometheus http://your-prometheus:9090
```
**In code (optional):**
```rust
use testing_framework_core::scenario::ScenarioBuilder;
use testing_framework_workflows::ObservabilityBuilderExt as _;
let plan = ScenarioBuilder::with_node_counts(1, 1)
.with_external_prometheus_str("http://your-prometheus:9090")
.build();
```
**Important:**
- K8s runner mounts `testing-framework/assets/stack/kzgrs_test_params` as a hostPath volume with file `/kzgrs_test_params/kzgrs_test_params` inside pods
@ -314,7 +341,7 @@ If you see this error, the file `kzgrs_test_params` is missing from the director
| Component | Controlled By | Purpose |
|-----------|--------------|---------|
| **Framework binaries** (`cargo run -p runner-examples --bin local_runner`) | `RUST_LOG` | Runner orchestration, deployment logs |
| **Node processes** (validators, executors spawned by runner) | `NOMOS_LOG_LEVEL`, `NOMOS_LOG_FILTER`, `NOMOS_LOG_DIR` | Consensus, DA, mempool, network logs |
| **Node processes** (validators, executors spawned by runner) | `NOMOS_LOG_LEVEL`, `NOMOS_LOG_FILTER` (+ `NOMOS_LOG_DIR` on host runner) | Consensus, DA, mempool, network logs |
**Common mistake:** Setting `RUST_LOG=debug` only increases verbosity of the runner binary itself. Node logs remain at their default level unless you also set `NOMOS_LOG_LEVEL=debug`.
@ -334,9 +361,9 @@ RUST_LOG=debug NOMOS_LOG_LEVEL=debug cargo run -p runner-examples --bin local_ru
| Variable | Default | Effect |
|----------|---------|--------|
| `NOMOS_LOG_DIR` | None (console only) | Directory for per-node log files. If unset, logs go to stdout/stderr. |
| `NOMOS_LOG_DIR` | None (console only) | Host runner: directory for per-node log files. Compose/k8s: use `testing-framework/assets/stack/cfgsync.yaml` (`tracing_settings.logger: !File`) and mount a writable directory. |
| `NOMOS_LOG_LEVEL` | `info` | Global log level: `error`, `warn`, `info`, `debug`, `trace` |
| `NOMOS_LOG_FILTER` | None | Fine-grained target filtering (e.g., `consensus=trace,da=debug`) |
| `NOMOS_LOG_FILTER` | None | Fine-grained target filtering (e.g., `cryptarchia=trace,nomos_da_sampling=debug`) |
| `NOMOS_TESTS_TRACING` | `false` | Enable the debug tracing preset (optional; combine with `NOMOS_LOG_DIR` unless you have external tracing backends configured) |
| `NOMOS_OTLP_ENDPOINT` | None | OTLP trace endpoint (optional, disables OTLP noise if unset) |
| `NOMOS_OTLP_METRICS_ENDPOINT` | None | OTLP metrics endpoint (optional) |
@ -346,7 +373,7 @@ RUST_LOG=debug NOMOS_LOG_LEVEL=debug cargo run -p runner-examples --bin local_ru
NOMOS_TESTS_TRACING=true \
NOMOS_LOG_DIR=/tmp/test-logs \
NOMOS_LOG_LEVEL=debug \
NOMOS_LOG_FILTER="nomos_consensus=trace,nomos_da_sampling=debug" \
NOMOS_LOG_FILTER="cryptarchia=trace,nomos_da_sampling=debug,nomos_da_dispersal=debug,nomos_da_verifier=debug,nomos_blend=debug,chain_service=info,chain_network=info,chain_leader=info" \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin local_runner
```
@ -367,18 +394,18 @@ Common target prefixes for `NOMOS_LOG_FILTER`:
| Target Prefix | Subsystem |
|---------------|-----------|
| `nomos_consensus` | Consensus (Cryptarchia) |
| `cryptarchia` | Consensus (Cryptarchia) |
| `nomos_da_sampling` | DA sampling service |
| `nomos_da_dispersal` | DA dispersal service |
| `nomos_da_verifier` | DA verification |
| `nomos_mempool` | Transaction mempool |
| `nomos_blend` | Mix network/privacy layer |
| `chain_service` | Chain service (node APIs/state) |
| `chain_network` | P2P networking |
| `chain_leader` | Leader election |
**Example filter:**
```bash
NOMOS_LOG_FILTER="nomos_consensus=trace,nomos_da_sampling=debug,chain_network=info"
NOMOS_LOG_FILTER="cryptarchia=trace,nomos_da_sampling=debug,chain_service=info,chain_network=info,chain_leader=info"
```
### Accessing Logs Per Runner
@ -422,18 +449,19 @@ docker logs -f $(docker ps --filter "name=nomos-compose-.*-validator-0" -q | hea
**Via file collection (advanced):**
Setting `NOMOS_LOG_DIR` writes files **inside the container**. To access them, you must either:
To write per-node log files inside containers, set `tracing_settings.logger: !File` in `testing-framework/assets/stack/cfgsync.yaml` (and ensure the directory is writable). To access them, you must either:
1. **Copy files out after the run:**
```bash
NOMOS_LOG_DIR=/logs \
# Ensure `testing-framework/assets/stack/cfgsync.yaml` is configured to log to `/logs`
# via `tracing_settings.logger: !File`.
NOMOS_TESTNET_IMAGE=logos-blockchain-testing:local \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin compose_runner
# After test, copy files from containers:
docker ps --filter "name=nomos-compose-"
docker cp <container-id>:/logs/nomos-node-0* /tmp/
docker cp <container-id>:/logs/node* /tmp/
```
2. **Mount a host volume** (requires modifying compose template):

2
book/src/quickstart.md
View File

@ -231,7 +231,7 @@ export NOMOS_BINARIES_TAR=.tmp/nomos-binaries-linux-v0.3.1.tar.gz
# Option 2: Manual circuit/image setup (rebuilds during image build)
scripts/setup-nomos-circuits.sh v0.3.1 /tmp/nomos-circuits
cp -r /tmp/nomos-circuits/* testing-framework/assets/stack/kzgrs_test_params/
testing-framework/assets/stack/scripts/build_test_image.sh
scripts/build_test_image.sh
# Run with Compose
NOMOS_TESTNET_IMAGE=logos-blockchain-testing:local \

14
book/src/troubleshooting.md
View File

@ -9,8 +9,8 @@
- **macOS + Docker Desktop (Apple silicon):** prefer `NOMOS_BUNDLE_DOCKER_PLATFORM=linux/arm64` for local compose/k8s runs to avoid slow/fragile amd64 emulation builds.
- **Disk space:** bundle/image builds are storage-heavy. If you see I/O errors or Docker build failures, check free space and prune old artifacts (`.tmp/`, `target/`, and Docker build cache) before retrying.
- **K8s runner scope:** the default Helm chart mounts KZG params via `hostPath` and uses a local image tag (`logos-blockchain-testing:local`). This is intended for local clusters (Docker Desktop / minikube / kind), not remote managed clusters without additional setup.
- Quick cleanup: `scripts/clean` (and `scripts/clean --docker` if needed).
- Destructive cleanup (last resort): `scripts/clean --docker-system --dangerous` (add `--volumes` if you also want to prune Docker volumes).
- Quick cleanup: `scripts/clean.sh` (and `scripts/clean.sh --docker` if needed).
- Destructive cleanup (last resort): `scripts/clean.sh --docker-system --dangerous` (add `--volumes` if you also want to prune Docker volumes).
**Recommended:** Use `scripts/run-examples.sh` which handles all setup automatically.
@ -31,12 +31,12 @@ Common symptoms and likely causes:
| Runner | Default Output | With `NOMOS_LOG_DIR` + Flags | Access Command |
|--------|---------------|------------------------------|----------------|
| **Host** (local) | Per-run temporary directories under the current working directory (removed unless `NOMOS_TESTS_KEEP_LOGS=1`) | Per-node files with prefix `nomos-node-{index}` (set `NOMOS_LOG_DIR`) | `cat $NOMOS_LOG_DIR/nomos-node-0*` |
| **Compose** | Docker container stdout/stderr | Per-node files inside containers (if path is mounted) | `docker ps` then `docker logs <container-id>` |
| **K8s** | Pod stdout/stderr | Per-node files inside pods (if path is mounted) | `kubectl logs -l nomos/logical-role=validator` |
| **Compose** | Docker container stdout/stderr | Set `tracing_settings.logger: !File` in `testing-framework/assets/stack/cfgsync.yaml` (and mount a writable directory) | `docker ps` then `docker logs <container-id>` |
| **K8s** | Pod stdout/stderr | Set `tracing_settings.logger: !File` in `testing-framework/assets/stack/cfgsync.yaml` (and mount a writable directory) | `kubectl logs -l nomos/logical-role=validator` |
**Important Notes:**
- **Host runner** (local processes): Per-run temporary directories are created under the current working directory and removed after the run unless `NOMOS_TESTS_KEEP_LOGS=1`. To write per-node log files to a stable location, set `NOMOS_LOG_DIR=/path/to/logs`.
- **Compose/K8s**: Per-node log files only exist inside containers/pods if `NOMOS_LOG_DIR` is set AND the path is writable inside the container/pod. By default, rely on `docker logs` or `kubectl logs`.
- **Compose/K8s**: Node log destination is controlled by `testing-framework/assets/stack/cfgsync.yaml` (`tracing_settings.logger`). By default, rely on `docker logs` or `kubectl logs`.
- **File naming**: Log files use prefix `nomos-node-{index}*` or `nomos-executor-{index}*` with timestamps, e.g., `nomos-node-0.2024-12-01T10-30-45.log` (NOT just `.log` suffix).
- **Container names**: Compose containers include project UUID, e.g., `nomos-compose-<uuid>-validator-0-1` where `<uuid>` is randomly generated per run
@ -203,7 +203,7 @@ If logs are too sparse, increase verbosity:
```bash
NOMOS_LOG_LEVEL=debug \
NOMOS_LOG_FILTER="nomos_consensus=trace,nomos_da_sampling=debug" \
NOMOS_LOG_FILTER="cryptarchia=trace,nomos_da_sampling=debug" \
cargo run -p runner-examples --bin local_runner
```
@ -294,7 +294,7 @@ Run a minimal baseline test (e.g., 2 validators, consensus liveness only). If it
- **Fix (manual)**:
1. Build bundle: `scripts/build-bundle.sh --platform linux`
2. Set bundle path: `export NOMOS_BINARIES_TAR=.tmp/nomos-binaries-linux-v0.3.1.tar.gz`
3. Build image: `testing-framework/assets/stack/scripts/build_test_image.sh`
3. Build image: `scripts/build_test_image.sh`
4. **kind/minikube:** load the image into the cluster nodes (e.g. `kind load docker-image logos-blockchain-testing:local`, or `minikube image load ...`), or push to a registry and set `NOMOS_TESTNET_IMAGE` accordingly.
### "Failed to load KZG parameters" or "Circuit file not found"

387
scripts/build-bundle.sh
View File

@ -1,14 +1,391 @@
#!/usr/bin/env bash
set -euo pipefail
# Thin wrapper; the actual implementation lives in scripts/lib/build-bundle.sh
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# shellcheck disable=SC1091
. "${ROOT_DIR}/scripts/lib/build-bundle.sh"
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
build_bundle::main "$@"
readonly DOCKER_RUST_IMAGE="rust:1.80-bullseye"
declare -ar DOCKER_APT_PACKAGES=(
clang
llvm-dev
libclang-dev
pkg-config
cmake
libssl-dev
rsync
libgmp10
libgmp-dev
libgomp1
nasm
)
build_bundle::usage() {
cat <<'USAGE'
Usage: scripts/build-bundle.sh [--platform host|linux] [--output PATH]
Options:
--platform Target platform for binaries (default: host)
--output Output path for the tarball (default: .tmp/nomos-binaries-<platform>-<version>.tar.gz)
--rev nomos-node git revision to build (overrides NOMOS_NODE_REV)
--path Use local nomos-node checkout at DIR (skip fetch/checkout)
--features Extra cargo features to enable (comma-separated); base always includes "testing"
--docker-platform Docker platform for Linux bundle when running on non-Linux host (default: auto; linux/arm64 on Apple silicon Docker Desktop, else linux/amd64)
Notes:
- For compose/k8s, use platform=linux. If running on macOS, this script will
run inside a Linux Docker container to produce Linux binaries.
- On Apple silicon, Docker defaults to linux/arm64; for compose/k8s you likely
want linux/amd64 (the default here). Override with --docker-platform.
- VERSION, NOMOS_NODE_REV, and optional NOMOS_NODE_PATH env vars are honored (defaults align with run-examples.sh).
USAGE
}
build_bundle::fail() {
echo "$1" >&2
exit 1
}
build_bundle::apply_nomos_node_patches() {
local node_src="$1"
local apply="${NOMOS_NODE_APPLY_PATCHES:-1}"
if [ "${apply}" = "0" ]; then
return 0
fi
local patch_dir="${NOMOS_NODE_PATCH_DIR:-${ROOT_DIR}/patches/nomos-node}"
if [ ! -d "${patch_dir}" ]; then
return 0
fi
local level="${NOMOS_NODE_PATCH_LEVEL:-}"
if [ -z "${level}" ]; then
level="all"
fi
shopt -s nullglob
local -a patches=("${patch_dir}"/*.patch)
shopt -u nullglob
if [ "${#patches[@]}" -eq 0 ]; then
return 0
fi
echo "==> Applying nomos-node patches from ${patch_dir} (level=${level})"
local patch base phase
for patch in "${patches[@]}"; do
base="$(basename "${patch}")"
phase=""
if [[ "${base}" =~ phase([0-9]+) ]]; then
phase="${BASH_REMATCH[1]}"
fi
if [ "${level}" != "all" ] && [ "${level}" != "ALL" ]; then
if ! [[ "${level}" =~ ^[0-9]+$ ]]; then
build_bundle::fail "Invalid NOMOS_NODE_PATCH_LEVEL: ${level} (expected integer or 'all')"
fi
if [ -n "${phase}" ] && [ "${phase}" -gt "${level}" ]; then
continue
fi
fi
git -C "${node_src}" apply --whitespace=nowarn "${patch}"
done
}
build_bundle::load_env() {
ROOT_DIR="$(common::repo_root)"
export ROOT_DIR
common::require_file "${ROOT_DIR}/versions.env"
# shellcheck disable=SC1091
. "${ROOT_DIR}/versions.env"
DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}"
DEFAULT_NODE_REV="${NOMOS_NODE_REV:-}"
DEFAULT_NODE_PATH="${NOMOS_NODE_PATH:-}"
NOMOS_EXTRA_FEATURES="${NOMOS_EXTRA_FEATURES:-}"
DOCKER_PLATFORM="${NOMOS_BUNDLE_DOCKER_PLATFORM:-${NOMOS_BIN_PLATFORM:-}}"
BUNDLE_RUSTUP_TOOLCHAIN="${BUNDLE_RUSTUP_TOOLCHAIN:-}"
if [ -z "${BUNDLE_RUSTUP_TOOLCHAIN}" ] && command -v rustup >/dev/null 2>&1 && [ -f "${ROOT_DIR}/rust-toolchain.toml" ]; then
BUNDLE_RUSTUP_TOOLCHAIN="$(awk -F '\"' '/^[[:space:]]*channel[[:space:]]*=/{print $2; exit}' "${ROOT_DIR}/rust-toolchain.toml")"
fi
}
build_bundle::default_docker_platform() {
if [ -n "${DOCKER_PLATFORM}" ]; then
return 0
fi
if ! command -v docker >/dev/null 2>&1; then
return 0
fi
local docker_arch
docker_arch="$(docker version --format '{{.Server.Arch}}' 2>/dev/null || true)"
case "${docker_arch}" in
arm64|aarch64) DOCKER_PLATFORM="linux/arm64" ;;
amd64|x86_64) DOCKER_PLATFORM="linux/amd64" ;;
*) DOCKER_PLATFORM="linux/amd64" ;;
esac
}
build_bundle::parse_args() {
PLATFORM="host"
OUTPUT=""
REV_OVERRIDE=""
PATH_OVERRIDE=""
if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
build_bundle::usage
exit 0
fi
while [ "$#" -gt 0 ]; do
case "$1" in
--platform=*|-p=*) PLATFORM="${1#*=}"; shift ;;
--platform|-p) PLATFORM="${2:-}"; shift 2 ;;
--output=*|-o=*) OUTPUT="${1#*=}"; shift ;;
--output|-o) OUTPUT="${2:-}"; shift 2 ;;
--rev=*) REV_OVERRIDE="${1#*=}"; shift ;;
--rev) REV_OVERRIDE="${2:-}"; shift 2 ;;
--path=*) PATH_OVERRIDE="${1#*=}"; shift ;;
--path) PATH_OVERRIDE="${2:-}"; shift 2 ;;
--features=*) NOMOS_EXTRA_FEATURES="${1#*=}"; shift ;;
--features) NOMOS_EXTRA_FEATURES="${2:-}"; shift 2 ;;
--docker-platform=*) DOCKER_PLATFORM="${1#*=}"; shift ;;
--docker-platform) DOCKER_PLATFORM="${2:-}"; shift 2 ;;
*) build_bundle::fail "Unknown argument: $1" ;;
esac
done
}
build_bundle::validate_and_finalize() {
case "${PLATFORM}" in
host|linux) ;;
*) build_bundle::fail "--platform must be host or linux" ;;
esac
VERSION="${DEFAULT_VERSION}"
if [ -n "${REV_OVERRIDE}" ] && [ -n "${PATH_OVERRIDE}" ]; then
build_bundle::fail "Use either --rev or --path, not both"
fi
if [ -z "${REV_OVERRIDE}" ] && [ -z "${PATH_OVERRIDE}" ] && [ -z "${DEFAULT_NODE_REV}" ] && [ -z "${DEFAULT_NODE_PATH}" ]; then
build_bundle::fail "Provide --rev, --path, or set NOMOS_NODE_REV/NOMOS_NODE_PATH in versions.env"
fi
NOMOS_NODE_REV="${REV_OVERRIDE:-${DEFAULT_NODE_REV}}"
NOMOS_NODE_PATH="${PATH_OVERRIDE:-${DEFAULT_NODE_PATH}}"
export NOMOS_NODE_REV NOMOS_NODE_PATH
build_bundle::default_docker_platform
DOCKER_PLATFORM="${DOCKER_PLATFORM:-linux/amd64}"
# Normalize OUTPUT to an absolute path under the workspace.
if [ -z "${OUTPUT}" ]; then
OUTPUT="${ROOT_DIR}/.tmp/nomos-binaries-${PLATFORM}-${VERSION}.tar.gz"
elif [[ "${OUTPUT}" != /* ]]; then
OUTPUT="${ROOT_DIR}/${OUTPUT#./}"
fi
echo "Bundle output: ${OUTPUT}"
}
build_bundle::clean_cargo_linux_cache() {
rm -rf "${ROOT_DIR}/.tmp/cargo-linux/registry" "${ROOT_DIR}/.tmp/cargo-linux/git"
}
build_bundle::maybe_run_linux_build_in_docker() {
# With `set -e`, this function must return 0 when no Docker cross-build is needed.
if [ "${PLATFORM}" != "linux" ] || [ "$(uname -s)" = "Linux" ] || [ -n "${BUNDLE_IN_CONTAINER:-}" ]; then
return 0
fi
command -v docker >/dev/null 2>&1 || build_bundle::fail "Docker is required to build a Linux bundle from non-Linux host"
[ -n "${DOCKER_PLATFORM}" ] || build_bundle::fail "--docker-platform must not be empty"
local node_path_env="${NOMOS_NODE_PATH}"
local -a extra_mounts=()
if [ -n "${NOMOS_NODE_PATH}" ]; then
case "${NOMOS_NODE_PATH}" in
"${ROOT_DIR}"/*)
node_path_env="/workspace${NOMOS_NODE_PATH#"${ROOT_DIR}"}"
;;
/*)
node_path_env="/external/nomos-node"
extra_mounts+=("-v" "${NOMOS_NODE_PATH}:${node_path_env}")
;;
*)
build_bundle::fail "--path must be absolute when cross-building in Docker"
;;
esac
fi
echo "==> Building Linux bundle inside Docker"
local container_output="/workspace${OUTPUT#"${ROOT_DIR}"}"
mkdir -p "${ROOT_DIR}/.tmp/cargo-linux" "${ROOT_DIR}/.tmp/nomos-node-linux-target"
local -a features_args=()
if [ -n "${NOMOS_EXTRA_FEATURES:-}" ]; then
features_args+=(--features "${NOMOS_EXTRA_FEATURES}")
fi
local -a src_args=()
if [ -n "${node_path_env}" ]; then
src_args+=(--path "${node_path_env}")
else
src_args+=(--rev "${NOMOS_NODE_REV}")
fi
docker run --rm --platform "${DOCKER_PLATFORM}" \
-e VERSION="${VERSION}" \
-e NOMOS_NODE_REV="${NOMOS_NODE_REV}" \
-e NOMOS_NODE_PATH="${node_path_env}" \
-e NOMOS_CIRCUITS="/workspace/.tmp/nomos-circuits-linux" \
-e STACK_DIR="/workspace/.tmp/nomos-circuits-linux" \
-e HOST_DIR="/workspace/.tmp/nomos-circuits-linux" \
-e NOMOS_EXTRA_FEATURES="${NOMOS_EXTRA_FEATURES:-}" \
-e BUNDLE_IN_CONTAINER=1 \
-e CARGO_HOME=/workspace/.tmp/cargo-linux \
-e CARGO_TARGET_DIR=/workspace/.tmp/nomos-node-linux-target \
-v "${ROOT_DIR}/.tmp/cargo-linux":/workspace/.tmp/cargo-linux \
-v "${ROOT_DIR}/.tmp/nomos-node-linux-target":/workspace/.tmp/nomos-node-linux-target \
-v "${ROOT_DIR}:/workspace" \
"${extra_mounts[@]}" \
-w /workspace \
"${DOCKER_RUST_IMAGE}" \
bash -c "apt-get update && apt-get install -y ${DOCKER_APT_PACKAGES[*]} && ./scripts/build-bundle.sh --platform linux --output \"${container_output}\" ${src_args[*]} ${features_args[*]}"
exit 0
}
build_bundle::prepare_circuits() {
echo "==> Preparing circuits (version ${VERSION})"
if [ "${PLATFORM}" = "host" ]; then
CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits-host"
NODE_TARGET="${ROOT_DIR}/.tmp/nomos-node-host-target"
else
CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits-linux"
NODE_TARGET="${ROOT_DIR}/.tmp/nomos-node-linux-target"
fi
NODE_SRC_DEFAULT="${ROOT_DIR}/.tmp/nomos-node-${PLATFORM}-src"
NODE_SRC="${NOMOS_NODE_PATH:-${NODE_SRC_DEFAULT}}"
if [ -n "${NOMOS_NODE_PATH}" ]; then
[ -d "${NODE_SRC}" ] || build_bundle::fail "NOMOS_NODE_PATH does not exist: ${NODE_SRC}"
rm -rf "${NODE_SRC_DEFAULT}"
if [ -d "${NODE_TARGET}" ]; then
find "${NODE_TARGET}" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
fi
NODE_TARGET="${NODE_TARGET}-local"
fi
export NOMOS_CIRCUITS="${CIRCUITS_DIR}"
mkdir -p "${ROOT_DIR}/.tmp" "${CIRCUITS_DIR}"
if [ -f "${CIRCUITS_DIR}/${KZG_FILE:-kzgrs_test_params}" ]; then
echo "Circuits already present at ${CIRCUITS_DIR}; skipping download"
else
STACK_DIR="${CIRCUITS_DIR}" HOST_DIR="${CIRCUITS_DIR}" \
"${ROOT_DIR}/scripts/setup-circuits-stack.sh" "${VERSION}" </dev/null
fi
NODE_BIN="${NODE_TARGET}/debug/nomos-node"
EXEC_BIN="${NODE_TARGET}/debug/nomos-executor"
CLI_BIN="${NODE_TARGET}/debug/nomos-cli"
}
build_bundle::build_binaries() {
FEATURES="testing"
if [ -n "${NOMOS_EXTRA_FEATURES:-}" ]; then
FEATURES="${FEATURES},${NOMOS_EXTRA_FEATURES}"
fi
echo "==> Building binaries (platform=${PLATFORM})"
mkdir -p "${NODE_SRC}"
(
cd "${NODE_SRC}"
if [ -n "${NOMOS_NODE_PATH}" ]; then
echo "Using local nomos-node checkout at ${NODE_SRC} (no fetch/checkout)"
else
if [ ! -d "${NODE_SRC}/.git" ]; then
git clone https://github.com/logos-co/nomos-node.git "${NODE_SRC}"
fi
git fetch --depth 1 origin "${NOMOS_NODE_REV}"
git checkout "${NOMOS_NODE_REV}"
git reset --hard
git clean -fdx
fi
if [ -z "${NOMOS_NODE_PATH}" ]; then
build_bundle::apply_nomos_node_patches "${NODE_SRC}"
fi
if [ -n "${BUNDLE_RUSTUP_TOOLCHAIN}" ]; then
RUSTFLAGS='--cfg feature="pol-dev-mode"' NOMOS_CIRCUITS="${CIRCUITS_DIR}" \
RUSTUP_TOOLCHAIN="${BUNDLE_RUSTUP_TOOLCHAIN}" \
cargo build --features "${FEATURES}" \
-p nomos-node -p nomos-executor -p nomos-cli \
--target-dir "${NODE_TARGET}"
else
RUSTFLAGS='--cfg feature="pol-dev-mode"' NOMOS_CIRCUITS="${CIRCUITS_DIR}" \
cargo build --features "${FEATURES}" \
-p nomos-node -p nomos-executor -p nomos-cli \
--target-dir "${NODE_TARGET}"
fi
)
}
build_bundle::package_bundle() {
echo "==> Packaging bundle"
local bundle_dir="${ROOT_DIR}/.tmp/nomos-bundle"
rm -rf "${bundle_dir}"
mkdir -p "${bundle_dir}/artifacts/circuits"
cp -a "${CIRCUITS_DIR}/." "${bundle_dir}/artifacts/circuits/"
mkdir -p "${bundle_dir}/artifacts"
cp "${NODE_BIN}" "${bundle_dir}/artifacts/"
cp "${EXEC_BIN}" "${bundle_dir}/artifacts/"
cp "${CLI_BIN}" "${bundle_dir}/artifacts/"
{
echo "nomos_node_path=${NOMOS_NODE_PATH:-}"
echo "nomos_node_rev=${NOMOS_NODE_REV:-}"
if [ -d "${NODE_SRC}/.git" ] && command -v git >/dev/null 2>&1; then
echo "nomos_node_git_head=$(git -C "${NODE_SRC}" rev-parse HEAD 2>/dev/null || true)"
fi
echo "platform=${PLATFORM}"
echo "features=${FEATURES}"
} > "${bundle_dir}/artifacts/nomos-bundle-meta.env"
mkdir -p "$(dirname "${OUTPUT}")"
if tar --help 2>/dev/null | grep -q -- '--no-mac-metadata'; then
tar --no-mac-metadata --no-xattrs -czf "${OUTPUT}" -C "${bundle_dir}" artifacts
elif tar --help 2>/dev/null | grep -q -- '--no-xattrs'; then
tar --no-xattrs -czf "${OUTPUT}" -C "${bundle_dir}" artifacts
else
tar -czf "${OUTPUT}" -C "${bundle_dir}" artifacts
fi
echo "Bundle created at ${OUTPUT}"
if [[ "${FEATURES}" == *profiling* ]]; then
cat <<'EOF_PROF'
Profiling endpoints (enabled by --features profiling):
CPU pprof (SVG): curl "http://<node-host>:8722/debug/pprof/profile?seconds=15&format=svg" -o profile.svg
CPU pprof (proto): go tool pprof -http=:8080 "http://<node-host>:8722/debug/pprof/profile?seconds=15&format=proto"
EOF_PROF
fi
}
build_bundle::main() {
build_bundle::load_env
build_bundle::clean_cargo_linux_cache
build_bundle::parse_args "$@"
build_bundle::validate_and_finalize
build_bundle::maybe_run_linux_build_in_docker
build_bundle::prepare_circuits
build_bundle::build_binaries
build_bundle::package_bundle
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
build_bundle::main "$@"
fi

172
scripts/build-linux-binaries.sh
View File

@ -1,14 +1,176 @@
#!/usr/bin/env bash
set -euo pipefail
# Thin wrapper; the actual implementation lives in scripts/lib/build-linux-binaries.sh
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# shellcheck disable=SC1091
. "${ROOT_DIR}/scripts/lib/build-linux-binaries.sh"
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
build_linux_binaries::main "$@"
build_linux_binaries::usage() {
cat <<'EOF'
Usage: scripts/build-linux-binaries.sh [options]
Builds a Linux bundle via scripts/build-bundle.sh, then stages artifacts into:
- testing-framework/assets/stack/bin
- testing-framework/assets/stack/kzgrs_test_params (or NOMOS_KZG_DIR_REL)
Options:
--rev REV nomos-node git revision to build (overrides NOMOS_NODE_REV)
--path DIR use local nomos-node checkout (skip fetch/checkout)
--features LIST extra cargo features (comma-separated); base includes "testing"
--docker-platform PLAT docker platform for the Linux build (e.g. linux/amd64, linux/arm64)
--tar PATH stage from an existing bundle tarball (skip build)
--output PATH where to write the bundle tarball when building (default: .tmp/nomos-binaries-linux-<version>.tar.gz)
-h, --help show help
Environment:
VERSION circuits version (default from versions.env)
NOMOS_CIRCUITS_VERSION legacy alias for VERSION (supported)
NOMOS_NODE_REV default nomos-node revision (from versions.env)
NOMOS_KZG_DIR_REL host path for staged circuits dir (default: testing-framework/assets/stack/kzgrs_test_params)
EOF
}
build_linux_binaries::fail_with_usage() {
echo "$1" >&2
build_linux_binaries::usage
exit 1
}
build_linux_binaries::load_env() {
ROOT_DIR="$(common::repo_root)"
export ROOT_DIR
common::require_file "${ROOT_DIR}/versions.env"
# shellcheck disable=SC1091
. "${ROOT_DIR}/versions.env"
common::maybe_source "${ROOT_DIR}/paths.env"
DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}"
VERSION="${VERSION:-${DEFAULT_VERSION}}"
if [ -n "${NOMOS_CIRCUITS_VERSION:-}" ]; then
VERSION="${NOMOS_CIRCUITS_VERSION}"
fi
}
build_linux_binaries::parse_args() {
REV_OVERRIDE=""
PATH_OVERRIDE=""
EXTRA_FEATURES=""
DOCKER_PLATFORM=""
OUTPUT_TAR=""
INPUT_TAR=""
while [ "$#" -gt 0 ]; do
case "$1" in
-h|--help) build_linux_binaries::usage; exit 0 ;;
--rev) REV_OVERRIDE="${2:-}"; shift 2 ;;
--rev=*) REV_OVERRIDE="${1#*=}"; shift ;;
--path) PATH_OVERRIDE="${2:-}"; shift 2 ;;
--path=*) PATH_OVERRIDE="${1#*=}"; shift ;;
--features) EXTRA_FEATURES="${2:-}"; shift 2 ;;
--features=*) EXTRA_FEATURES="${1#*=}"; shift ;;
--docker-platform) DOCKER_PLATFORM="${2:-}"; shift 2 ;;
--docker-platform=*) DOCKER_PLATFORM="${1#*=}"; shift ;;
--tar) INPUT_TAR="${2:-}"; shift 2 ;;
--tar=*) INPUT_TAR="${1#*=}"; shift ;;
--output|-o) OUTPUT_TAR="${2:-}"; shift 2 ;;
--output=*|-o=*) OUTPUT_TAR="${1#*=}"; shift ;;
*) build_linux_binaries::fail_with_usage "Unknown argument: $1" ;;
esac
done
if [ -n "${REV_OVERRIDE}" ] && [ -n "${PATH_OVERRIDE}" ]; then
build_linux_binaries::fail_with_usage "Use either --rev or --path, not both"
fi
if [ -n "${INPUT_TAR}" ] && [ ! -f "${INPUT_TAR}" ]; then
build_linux_binaries::fail_with_usage "Bundle tarball not found: ${INPUT_TAR}"
fi
if [ -z "${OUTPUT_TAR}" ]; then
OUTPUT_TAR="${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz"
elif [[ "${OUTPUT_TAR}" != /* ]]; then
OUTPUT_TAR="${ROOT_DIR}/${OUTPUT_TAR#./}"
fi
}
build_linux_binaries::build_bundle_if_needed() {
if [ -n "${INPUT_TAR}" ]; then
BUNDLE_TAR="${INPUT_TAR}"
return 0
fi
mkdir -p "$(dirname "${OUTPUT_TAR}")"
BUILD_ARGS=(--platform linux --output "${OUTPUT_TAR}")
if [ -n "${REV_OVERRIDE}" ]; then
BUILD_ARGS+=(--rev "${REV_OVERRIDE}")
elif [ -n "${PATH_OVERRIDE}" ]; then
BUILD_ARGS+=(--path "${PATH_OVERRIDE}")
fi
if [ -n "${EXTRA_FEATURES}" ]; then
BUILD_ARGS+=(--features "${EXTRA_FEATURES}")
fi
if [ -n "${DOCKER_PLATFORM}" ]; then
BUILD_ARGS+=(--docker-platform "${DOCKER_PLATFORM}")
fi
echo "==> Building Linux bundle"
VERSION="${VERSION}" "${ROOT_DIR}/scripts/build-bundle.sh" "${BUILD_ARGS[@]}"
BUNDLE_TAR="${OUTPUT_TAR}"
}
build_linux_binaries::stage_from_bundle() {
local tar_path="$1"
local extract_dir
extract_dir="$(common::tmpdir nomos-linux-bundle.XXXXXX)"
cleanup() { rm -rf "${extract_dir}" 2>/dev/null || true; }
trap cleanup EXIT
echo "==> Extracting ${tar_path}"
tar -xzf "${tar_path}" -C "${extract_dir}"
local artifacts="${extract_dir}/artifacts"
[ -f "${artifacts}/nomos-node" ] || common::die "Missing nomos-node in bundle: ${tar_path}"
[ -f "${artifacts}/nomos-executor" ] || common::die "Missing nomos-executor in bundle: ${tar_path}"
[ -f "${artifacts}/nomos-cli" ] || common::die "Missing nomos-cli in bundle: ${tar_path}"
[ -d "${artifacts}/circuits" ] || common::die "Missing circuits/ in bundle: ${tar_path}"
local bin_out="${ROOT_DIR}/testing-framework/assets/stack/bin"
local kzg_dir_rel="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}"
local circuits_out="${ROOT_DIR}/${kzg_dir_rel}"
echo "==> Staging binaries to ${bin_out}"
mkdir -p "${bin_out}"
cp "${artifacts}/nomos-node" "${artifacts}/nomos-executor" "${artifacts}/nomos-cli" "${bin_out}/"
echo "==> Staging circuits to ${circuits_out}"
rm -rf "${circuits_out}"
mkdir -p "${circuits_out}"
if command -v rsync >/dev/null 2>&1; then
rsync -a --delete "${artifacts}/circuits/" "${circuits_out}/"
else
cp -a "${artifacts}/circuits/." "${circuits_out}/"
fi
# If the tarball was produced inside Docker, it might be root-owned on the host.
chown -R "$(id -u)":"$(id -g)" "${bin_out}" "${circuits_out}" 2>/dev/null || true
}
build_linux_binaries::main() {
build_linux_binaries::load_env
build_linux_binaries::parse_args "$@"
build_linux_binaries::build_bundle_if_needed
build_linux_binaries::stage_from_bundle "${BUNDLE_TAR}"
echo
echo "Binaries staged in ${ROOT_DIR}/testing-framework/assets/stack/bin"
echo "Circuits staged in ${ROOT_DIR}/${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}"
echo "Bundle tarball: ${BUNDLE_TAR}"
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
build_linux_binaries::main "$@"
fi

13
scripts/build-rapidsnark.sh
View File

@ -6,6 +6,11 @@
set -euo pipefail
readonly DEFAULT_RAPIDSNARK_REPO="https://github.com/iden3/rapidsnark.git"
readonly DEFAULT_RAPIDSNARK_REF="main"
readonly DEFAULT_FORCE_REBUILD=0
readonly DEFAULT_USE_ASM="ON"
if [ $# -lt 1 ]; then
echo "usage: $0 <circuits_dir>" >&2
exit 1
@ -13,13 +18,13 @@ fi
TARGET_ARCH="$(uname -m)"
CIRCUITS_DIR="$1"
RAPIDSNARK_REPO="${RAPIDSNARK_REPO:-https://github.com/iden3/rapidsnark.git}"
RAPIDSNARK_REF="${RAPIDSNARK_REF:-main}"
FORCE_REBUILD="${RAPIDSNARK_FORCE_REBUILD:-0}"
RAPIDSNARK_REPO="${RAPIDSNARK_REPO:-${DEFAULT_RAPIDSNARK_REPO}}"
RAPIDSNARK_REF="${RAPIDSNARK_REF:-${DEFAULT_RAPIDSNARK_REF}}"
FORCE_REBUILD="${RAPIDSNARK_FORCE_REBUILD:-${DEFAULT_FORCE_REBUILD}}"
BUILD_DIR=""
PACKAGE_DIR=""
CMAKE_TARGET_PLATFORM=""
USE_ASM="${RAPIDSNARK_USE_ASM:-ON}"
USE_ASM="${RAPIDSNARK_USE_ASM:-${DEFAULT_USE_ASM}}"
CMAKE_C_FLAGS="${RAPIDSNARK_C_FLAGS:-}"
CMAKE_CXX_FLAGS="${RAPIDSNARK_CXX_FLAGS:-}"

221
scripts/build_test_image.sh Executable file
View File

@ -0,0 +1,221 @@
#!/usr/bin/env bash
set -euo pipefail
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
# shellcheck disable=SC1091
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
build_test_image::usage() {
cat <<'USAGE'
Usage: scripts/build_test_image.sh [options]
Builds the compose/k8s test image (bakes in binaries + circuit assets).
Options:
--tag TAG Docker image tag (default: logos-blockchain-testing:local; or env IMAGE_TAG)
--version VERSION Circuits release tag (default: versions.env VERSION)
--dockerfile PATH Dockerfile path (default: testing-framework/assets/stack/Dockerfile)
--circuits-override PATH Relative path (within repo) to circuits dir/file to bake (default: testing-framework/assets/stack/kzgrs_test_params)
--circuits-platform NAME Circuits platform identifier for downloads (default: auto; linux-x86_64 or linux-aarch64)
--bundle-tar PATH Bundle tar containing artifacts/{nomos-*,circuits} (default: .tmp/nomos-binaries-linux-<version>.tar.gz; or env NOMOS_BINARIES_TAR)
--no-restore Do not restore binaries/circuits from bundle tar (forces Dockerfile to build/download as needed)
--print-config Print resolved configuration and exit
-h, --help Show this help and exit
Env (legacy/compatible):
IMAGE_TAG, VERSION, CIRCUITS_OVERRIDE, CIRCUITS_PLATFORM, COMPOSE_CIRCUITS_PLATFORM,
NOMOS_BINARIES_TAR, NOMOS_KZG_DIR_REL
USAGE
}
build_test_image::fail() {
common::die "$1"
}
build_test_image::load_env() {
if [ -n "${ROOT_DIR:-}" ] && [ -f "${ROOT_DIR}/versions.env" ]; then
: # Use provided ROOT_DIR.
else
ROOT_DIR="$(common::repo_root)"
fi
export ROOT_DIR
common::require_file "${ROOT_DIR}/versions.env"
# shellcheck disable=SC1091
. "${ROOT_DIR}/versions.env"
common::maybe_source "${ROOT_DIR}/paths.env"
DOCKERFILE_PATH_DEFAULT="${ROOT_DIR}/testing-framework/assets/stack/Dockerfile"
IMAGE_TAG_DEFAULT="logos-blockchain-testing:local"
VERSION_DEFAULT="${VERSION:?Missing VERSION in versions.env}"
NOMOS_NODE_REV="${NOMOS_NODE_REV:?Missing NOMOS_NODE_REV in versions.env}"
}
build_test_image::detect_circuits_platform() {
case "$(uname -m)" in
x86_64) echo "linux-x86_64" ;;
arm64|aarch64) echo "linux-aarch64" ;;
*) echo "linux-x86_64" ;;
esac
}
build_test_image::parse_args() {
IMAGE_TAG="${IMAGE_TAG:-${IMAGE_TAG_DEFAULT}}"
VERSION_OVERRIDE=""
DOCKERFILE_PATH="${DOCKERFILE_PATH_DEFAULT}"
KZG_DIR_REL_DEFAULT="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}"
CIRCUITS_OVERRIDE="${CIRCUITS_OVERRIDE:-${KZG_DIR_REL_DEFAULT}}"
CIRCUITS_PLATFORM="${CIRCUITS_PLATFORM:-${COMPOSE_CIRCUITS_PLATFORM:-}}"
BUNDLE_TAR_PATH="${NOMOS_BINARIES_TAR:-}"
NO_RESTORE=0
PRINT_CONFIG=0
while [ "$#" -gt 0 ]; do
case "$1" in
-h|--help) build_test_image::usage; exit 0 ;;
--tag=*) IMAGE_TAG="${1#*=}"; shift ;;
--tag) IMAGE_TAG="${2:-}"; shift 2 ;;
--version=*) VERSION_OVERRIDE="${1#*=}"; shift ;;
--version) VERSION_OVERRIDE="${2:-}"; shift 2 ;;
--dockerfile=*) DOCKERFILE_PATH="${1#*=}"; shift ;;
--dockerfile) DOCKERFILE_PATH="${2:-}"; shift 2 ;;
--circuits-override=*) CIRCUITS_OVERRIDE="${1#*=}"; shift ;;
--circuits-override) CIRCUITS_OVERRIDE="${2:-}"; shift 2 ;;
--circuits-platform=*) CIRCUITS_PLATFORM="${1#*=}"; shift ;;
--circuits-platform) CIRCUITS_PLATFORM="${2:-}"; shift 2 ;;
--bundle-tar=*) BUNDLE_TAR_PATH="${1#*=}"; shift ;;
--bundle-tar) BUNDLE_TAR_PATH="${2:-}"; shift 2 ;;
--no-restore) NO_RESTORE=1; shift ;;
--print-config) PRINT_CONFIG=1; shift ;;
*) build_test_image::fail "Unknown argument: $1" ;;
esac
done
if [ -n "${VERSION_OVERRIDE}" ]; then
VERSION="${VERSION_OVERRIDE}"
else
VERSION="${VERSION_DEFAULT}"
fi
if [ -z "${CIRCUITS_PLATFORM}" ]; then
CIRCUITS_PLATFORM="$(build_test_image::detect_circuits_platform)"
fi
BIN_DST="${ROOT_DIR}/testing-framework/assets/stack/bin"
KZG_DIR_REL="${KZG_DIR_REL_DEFAULT}"
CIRCUITS_DIR_HOST="${ROOT_DIR}/${KZG_DIR_REL}"
DEFAULT_LINUX_TAR="${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz"
TAR_PATH="${BUNDLE_TAR_PATH:-${DEFAULT_LINUX_TAR}}"
}
build_test_image::print_config() {
echo "Workspace root: ${ROOT_DIR}"
echo "Image tag: ${IMAGE_TAG}"
echo "Dockerfile: ${DOCKERFILE_PATH}"
echo "Nomos node rev: ${NOMOS_NODE_REV}"
echo "Circuits override: ${CIRCUITS_OVERRIDE:-<none>}"
echo "Circuits version (download fallback): ${VERSION}"
echo "Circuits platform: ${CIRCUITS_PLATFORM}"
echo "Host circuits dir: ${CIRCUITS_DIR_HOST}"
echo "Binaries dir: ${BIN_DST}"
echo "Bundle tar (if used): ${TAR_PATH}"
echo "Restore from tar: $([ "${NO_RESTORE}" -eq 1 ] && echo "disabled" || echo "enabled")"
}
build_test_image::have_host_binaries() {
# Preserve existing behavior: only require node+executor on the host.
# If nomos-cli is missing, the Dockerfile can still build it from source.
[ -x "${BIN_DST}/nomos-node" ] && [ -x "${BIN_DST}/nomos-executor" ]
}
build_test_image::restore_from_bundle() {
[ -f "${TAR_PATH}" ] || build_test_image::fail "Prebuilt binaries missing and bundle tar not found at ${TAR_PATH}"
echo "==> Restoring binaries/circuits from ${TAR_PATH}"
local tmp_extract
tmp_extract="$(common::tmpdir nomos-bundle-extract.XXXXXX)"
trap 'rm -rf "${tmp_extract}"' RETURN
tar -xzf "${TAR_PATH}" -C "${tmp_extract}"
local artifacts="${tmp_extract}/artifacts"
for bin in nomos-node nomos-executor nomos-cli; do
[ -f "${artifacts}/${bin}" ] || build_test_image::fail "Bundle ${TAR_PATH} missing artifacts/${bin}"
done
mkdir -p "${BIN_DST}"
cp "${artifacts}/nomos-node" "${artifacts}/nomos-executor" "${artifacts}/nomos-cli" "${BIN_DST}/"
chmod +x "${BIN_DST}/nomos-node" "${BIN_DST}/nomos-executor" "${BIN_DST}/nomos-cli" || true
if [ -d "${artifacts}/circuits" ]; then
mkdir -p "${CIRCUITS_DIR_HOST}"
if command -v rsync >/dev/null 2>&1; then
rsync -a --delete "${artifacts}/circuits/" "${CIRCUITS_DIR_HOST}/"
else
cp -a "${artifacts}/circuits/." "${CIRCUITS_DIR_HOST}/"
fi
fi
}
build_test_image::maybe_restore_assets() {
if [ "${NO_RESTORE}" -eq 1 ]; then
return 0
fi
if build_test_image::have_host_binaries; then
return 0
fi
build_test_image::restore_from_bundle
}
build_test_image::docker_build() {
command -v docker >/dev/null 2>&1 || build_test_image::fail "docker not found in PATH"
[ -f "${DOCKERFILE_PATH}" ] || build_test_image::fail "Dockerfile not found: ${DOCKERFILE_PATH}"
local -a build_args=(
-f "${DOCKERFILE_PATH}"
-t "${IMAGE_TAG}"
--build-arg "NOMOS_NODE_REV=${NOMOS_NODE_REV}"
--build-arg "CIRCUITS_PLATFORM=${CIRCUITS_PLATFORM}"
--build-arg "VERSION=${VERSION}"
"${ROOT_DIR}"
)
if [ -n "${CIRCUITS_OVERRIDE}" ]; then
build_args+=(--build-arg "CIRCUITS_OVERRIDE=${CIRCUITS_OVERRIDE}")
fi
printf "Running:"
printf " %q" docker build "${build_args[@]}"
echo
docker build "${build_args[@]}"
}
build_test_image::main() {
build_test_image::load_env
build_test_image::parse_args "$@"
if [ "${PRINT_CONFIG}" -eq 1 ]; then
build_test_image::print_config
exit 0
fi
build_test_image::print_config
build_test_image::maybe_restore_assets
build_test_image::docker_build
cat <<EOF
Build complete.
- Use this image in k8s/compose by exporting NOMOS_TESTNET_IMAGE=${IMAGE_TAG}
- Circuits source: ${CIRCUITS_OVERRIDE:-download ${VERSION}}
EOF
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
build_test_image::main "$@"
fi

219
scripts/checks
View File

@ -1,219 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
say() { printf "%s\n" "$*"; }
section() { printf "\n==> %s\n" "$*"; }
have() { command -v "$1" >/dev/null 2>&1; }
bytes_to_human() {
local bytes="${1:-0}"
local kib=$((1024))
local mib=$((1024 * 1024))
local gib=$((1024 * 1024 * 1024))
if [ "$bytes" -ge "$gib" ]; then
awk -v b="$bytes" 'BEGIN{printf "%.1fGiB", b/1024/1024/1024}'
elif [ "$bytes" -ge "$mib" ]; then
awk -v b="$bytes" 'BEGIN{printf "%.1fMiB", b/1024/1024}'
elif [ "$bytes" -ge "$kib" ]; then
awk -v b="$bytes" 'BEGIN{printf "%.1fKiB", b/1024}'
else
printf "%sB" "$bytes"
fi
}
warn() { say "WARN: $*"; }
ok() { say "OK: $*"; }
section "Workspace"
say "root: ${ROOT_DIR}"
if [ -f "${ROOT_DIR}/versions.env" ]; then
# shellcheck disable=SC1091
. "${ROOT_DIR}/versions.env"
ok "versions.env present"
say "VERSION=${VERSION:-<unset>}"
say "NOMOS_NODE_REV=${NOMOS_NODE_REV:-<unset>}"
if [ -n "${NOMOS_NODE_PATH:-}" ]; then
say "NOMOS_NODE_PATH=${NOMOS_NODE_PATH}"
fi
else
warn "versions.env missing (scripts depend on it)"
fi
if [ -f "${ROOT_DIR}/paths.env" ]; then
# shellcheck disable=SC1091
. "${ROOT_DIR}/paths.env"
ok "paths.env present"
fi
section "Disk Space"
if have df; then
df -h "${ROOT_DIR}" | sed -n '1,2p'
fi
tmp_dir="${ROOT_DIR}/.tmp"
if [ -d "${tmp_dir}" ]; then
if have du; then
say ".tmp size: $(du -sh "${tmp_dir}" 2>/dev/null | awk '{print $1}')"
fi
else
say ".tmp: <absent>"
fi
if [ -d "${ROOT_DIR}/target" ] && have du; then
say "target size: $(du -sh "${ROOT_DIR}/target" 2>/dev/null | awk '{print $1}')"
fi
section "KZG Params"
KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}"
KZG_FILE="${NOMOS_KZG_FILE:-kzgrs_test_params}"
KZG_CONTAINER_PATH="${NOMOS_KZG_CONTAINER_PATH:-/kzgrs_test_params/kzgrs_test_params}"
HOST_KZG_PATH="${ROOT_DIR}/${KZG_DIR_REL}/${KZG_FILE}"
say "host: ${HOST_KZG_PATH}"
say "container: ${KZG_CONTAINER_PATH}"
if [ -f "${HOST_KZG_PATH}" ]; then
ok "KZG params file exists"
else
warn "KZG params file missing (DA workloads will fail); run: scripts/run-examples.sh <mode> (auto) or scripts/setup-nomos-circuits.sh"
fi
section "Rust Toolchain"
if have rustup; then
ok "rustup: $(rustup --version | head -n1)"
if [ -f "${ROOT_DIR}/rust-toolchain.toml" ]; then
channel="$(awk -F '\"' '/^[[:space:]]*channel[[:space:]]*=/{print $2; exit}' "${ROOT_DIR}/rust-toolchain.toml" 2>/dev/null || true)"
say "rust-toolchain.toml channel: ${channel:-<unknown>}"
fi
elif have rustc; then
ok "rustc: $(rustc --version)"
else
warn "rust toolchain not found (rustup/rustc missing)"
fi
section "Docker (compose/k8s image + linux bundle builds)"
if have docker; then
ok "docker client: $(docker version --format '{{.Client.Version}}' 2>/dev/null || docker --version)"
server_arch="$(docker version --format '{{.Server.Os}}/{{.Server.Arch}}' 2>/dev/null || true)"
if [ -n "${server_arch}" ]; then
say "docker engine: ${server_arch}"
else
warn "could not query docker engine arch (is Docker running?)"
fi
bundle_platform="${NOMOS_BUNDLE_DOCKER_PLATFORM:-${NOMOS_BIN_PLATFORM:-}}"
if [ -z "${bundle_platform}" ]; then
say "NOMOS_BUNDLE_DOCKER_PLATFORM=<auto>"
if [[ "${server_arch}" == *"linux/arm64"* ]]; then
say "bundle docker platform (auto): linux/arm64"
else
say "bundle docker platform (auto): linux/amd64"
fi
bundle_platform="auto"
else
say "NOMOS_BUNDLE_DOCKER_PLATFORM=${bundle_platform}"
fi
if [[ "${server_arch}" == *"linux/arm64"* ]] && [ "${bundle_platform}" = "linux/amd64" ]; then
warn "Docker engine is linux/arm64 but bundle platform is linux/amd64 (emulation). If builds are slow/flaky, set: NOMOS_BUNDLE_DOCKER_PLATFORM=linux/arm64"
fi
image="${NOMOS_TESTNET_IMAGE:-logos-blockchain-testing:local}"
say "NOMOS_TESTNET_IMAGE=${image}"
if docker image inspect "${image}" >/dev/null 2>&1; then
ok "testnet image present locally"
else
warn "testnet image not present locally (compose/k8s runs will rebuild or fail if NOMOS_SKIP_IMAGE_BUILD=1)"
fi
else
warn "docker not found (compose/k8s unavailable; linux bundle build on macOS requires docker)"
fi
section "Docker Compose"
if have docker; then
if docker compose version >/dev/null 2>&1; then
ok "docker compose available"
else
warn "docker compose not available"
fi
fi
section "Kubernetes (k8s runner)"
if have kubectl; then
ok "kubectl: $(kubectl version --client=true --short 2>/dev/null || true)"
ctx="$(kubectl config current-context 2>/dev/null || true)"
if [ -n "${ctx}" ]; then
say "current-context: ${ctx}"
fi
if kubectl cluster-info >/dev/null 2>&1; then
ok "cluster reachable"
kubectl get nodes -o wide 2>/dev/null | sed -n '1,3p' || true
else
warn "cluster not reachable (k8s runner will skip with ClientInit error)"
fi
else
warn "kubectl not found (k8s runner unavailable)"
fi
if have helm; then
ok "helm: $(helm version --short 2>/dev/null || true)"
else
warn "helm not found (k8s runner uses helm)"
fi
section "K8s Image Visibility"
image="${NOMOS_TESTNET_IMAGE:-logos-blockchain-testing:local}"
if [ -n "${ctx:-}" ]; then
case "${ctx}" in
docker-desktop)
ok "docker-desktop context shares local Docker images"
;;
kind-*)
if [[ "${image}" == *":local" ]]; then
warn "kind cluster won't see local Docker images by default"
say "Suggested: kind load docker-image ${image}"
fi
;;
minikube)
if [[ "${image}" == *":local" ]]; then
warn "minikube may not see local Docker images by default"
say "Suggested: minikube image load ${image}"
fi
;;
*)
if [[ "${image}" == *":local" ]]; then
warn "current context is ${ctx}; a :local image tag may not be reachable by cluster nodes"
say "Suggested: push to a registry and set NOMOS_TESTNET_IMAGE, or load into the cluster if supported"
fi
;;
esac
fi
section "Docker Desktop Kubernetes Health (best-effort)"
if have kubectl && [ "${ctx:-}" = "docker-desktop" ]; then
if ! kubectl -n kube-system get pod storage-provisioner >/dev/null 2>&1; then
warn "storage-provisioner pod not found"
else
phase="$(kubectl -n kube-system get pod storage-provisioner -o jsonpath='{.status.phase}' 2>/dev/null || true)"
reason="$(kubectl -n kube-system get pod storage-provisioner -o jsonpath='{.status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || true)"
if [ "${phase}" = "Running" ] || [ "${phase}" = "Succeeded" ]; then
ok "storage-provisioner: ${phase}"
else
warn "storage-provisioner: ${phase:-<unknown>} ${reason}"
fi
fi
fi
section "Runner Debug Flags (optional)"
say "SLOW_TEST_ENV=${SLOW_TEST_ENV:-<unset>} (if true: doubles readiness timeouts)"
say "NOMOS_SKIP_IMAGE_BUILD=${NOMOS_SKIP_IMAGE_BUILD:-<unset>} (compose/k8s)"
say "COMPOSE_RUNNER_PRESERVE=${COMPOSE_RUNNER_PRESERVE:-<unset>} (compose)"
say "K8S_RUNNER_PRESERVE=${K8S_RUNNER_PRESERVE:-<unset>} (k8s)"
say "K8S_RUNNER_DEBUG=${K8S_RUNNER_DEBUG:-<unset>} (k8s helm debug)"
say "COMPOSE_RUNNER_HOST=${COMPOSE_RUNNER_HOST:-<unset>} (compose readiness host override)"
say "K8S_RUNNER_NODE_HOST=${K8S_RUNNER_NODE_HOST:-<unset>} (k8s NodePort host override)"
say "K8S_RUNNER_NAMESPACE=${K8S_RUNNER_NAMESPACE:-<unset>} (k8s fixed namespace)"
section "Done"
say "If something looks off, start with: scripts/run-examples.sh <mode> -t 60 -v 1 -e 1"

303
scripts/checks.sh Executable file
View File

@ -0,0 +1,303 @@
#!/usr/bin/env bash
set -euo pipefail
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
# shellcheck disable=SC1091
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
checks::usage() {
cat <<'USAGE'
Usage: scripts/checks.sh [--help]
Runs a best-effort local environment sanity check for the testing framework
(assets, Rust, Docker, Kubernetes).
Notes:
- This script is informational; it does not modify your system.
- Many checks are best-effort and may be skipped if tools are missing.
USAGE
}
checks::say() { printf "%s\n" "$*"; }
checks::section() { printf "\n==> %s\n" "$*"; }
checks::have() { command -v "$1" >/dev/null 2>&1; }
checks::warn() { checks::say "WARN: $*"; }
checks::ok() { checks::say "OK: $*"; }
checks::load_env() {
ROOT_DIR="$(common::repo_root)"
export ROOT_DIR
if [ -f "${ROOT_DIR}/versions.env" ]; then
# shellcheck disable=SC1091
. "${ROOT_DIR}/versions.env"
fi
if [ -f "${ROOT_DIR}/paths.env" ]; then
# shellcheck disable=SC1091
. "${ROOT_DIR}/paths.env"
fi
}
checks::print_workspace() {
checks::section "Workspace"
checks::say "root: ${ROOT_DIR}"
if [ -f "${ROOT_DIR}/versions.env" ]; then
checks::ok "versions.env present"
checks::say "VERSION=${VERSION:-<unset>}"
checks::say "NOMOS_NODE_REV=${NOMOS_NODE_REV:-<unset>}"
if [ -n "${NOMOS_NODE_PATH:-}" ]; then
checks::say "NOMOS_NODE_PATH=${NOMOS_NODE_PATH}"
fi
else
checks::warn "versions.env missing (scripts depend on it)"
fi
if [ -f "${ROOT_DIR}/paths.env" ]; then
checks::ok "paths.env present"
fi
}
checks::print_disk_space() {
checks::section "Disk Space"
if checks::have df; then
df -h "${ROOT_DIR}" | sed -n '1,2p'
fi
local tmp_dir="${ROOT_DIR}/.tmp"
if [ -d "${tmp_dir}" ]; then
if checks::have du; then
checks::say ".tmp size: $(du -sh "${tmp_dir}" 2>/dev/null | awk '{print $1}')"
fi
else
checks::say ".tmp: <absent>"
fi
if [ -d "${ROOT_DIR}/target" ] && checks::have du; then
checks::say "target size: $(du -sh "${ROOT_DIR}/target" 2>/dev/null | awk '{print $1}')"
fi
}
checks::print_kzg_params() {
checks::section "KZG Params"
local default_kzg_dir_rel="testing-framework/assets/stack/kzgrs_test_params"
local default_kzg_file="kzgrs_test_params"
local default_kzg_container_path="/kzgrs_test_params/kzgrs_test_params"
local kzg_dir_rel="${NOMOS_KZG_DIR_REL:-${default_kzg_dir_rel}}"
local kzg_file="${NOMOS_KZG_FILE:-${default_kzg_file}}"
local kzg_container_path="${NOMOS_KZG_CONTAINER_PATH:-${default_kzg_container_path}}"
local host_kzg_path="${ROOT_DIR}/${kzg_dir_rel}/${kzg_file}"
checks::say "host: ${host_kzg_path}"
checks::say "container: ${kzg_container_path}"
if [ -f "${host_kzg_path}" ]; then
checks::ok "KZG params file exists"
else
checks::warn "KZG params file missing (DA workloads will fail); run: scripts/run-examples.sh <mode> (auto) or scripts/setup-nomos-circuits.sh"
fi
}
checks::print_rust_toolchain() {
checks::section "Rust Toolchain"
if checks::have rustup; then
checks::ok "rustup: $(rustup --version | head -n1)"
if [ -f "${ROOT_DIR}/rust-toolchain.toml" ]; then
local channel
channel="$(awk -F '\"' '/^[[:space:]]*channel[[:space:]]*=/{print $2; exit}' "${ROOT_DIR}/rust-toolchain.toml" 2>/dev/null || true)"
checks::say "rust-toolchain.toml channel: ${channel:-<unknown>}"
fi
elif checks::have rustc; then
checks::ok "rustc: $(rustc --version)"
else
checks::warn "rust toolchain not found (rustup/rustc missing)"
fi
}
checks::print_docker() {
checks::section "Docker (compose/k8s image + linux bundle builds)"
local default_local_image="logos-blockchain-testing:local"
local default_bundle_platform_amd64="linux/amd64"
local default_bundle_platform_arm64="linux/arm64"
if ! checks::have docker; then
checks::warn "docker not found (compose/k8s unavailable; linux bundle build on macOS requires docker)"
return 0
fi
checks::ok "docker client: $(docker version --format '{{.Client.Version}}' 2>/dev/null || docker --version)"
local server_arch
server_arch="$(docker version --format '{{.Server.Os}}/{{.Server.Arch}}' 2>/dev/null || true)"
if [ -n "${server_arch}" ]; then
checks::say "docker engine: ${server_arch}"
else
checks::warn "could not query docker engine arch (is Docker running?)"
fi
local bundle_platform="${NOMOS_BUNDLE_DOCKER_PLATFORM:-${NOMOS_BIN_PLATFORM:-}}"
if [ -z "${bundle_platform}" ]; then
checks::say "NOMOS_BUNDLE_DOCKER_PLATFORM=<auto>"
if [[ "${server_arch}" == *"linux/arm64"* ]]; then
checks::say "bundle docker platform (auto): ${default_bundle_platform_arm64}"
else
checks::say "bundle docker platform (auto): ${default_bundle_platform_amd64}"
fi
bundle_platform="auto"
else
checks::say "NOMOS_BUNDLE_DOCKER_PLATFORM=${bundle_platform}"
fi
if [[ "${server_arch}" == *"linux/arm64"* ]] && [ "${bundle_platform}" = "${default_bundle_platform_amd64}" ]; then
checks::warn "Docker engine is linux/arm64 but bundle platform is ${default_bundle_platform_amd64} (emulation). If builds are slow/flaky, set: NOMOS_BUNDLE_DOCKER_PLATFORM=${default_bundle_platform_arm64}"
fi
local image="${NOMOS_TESTNET_IMAGE:-${default_local_image}}"
checks::say "NOMOS_TESTNET_IMAGE=${image}"
if docker image inspect "${image}" >/dev/null 2>&1; then
checks::ok "testnet image present locally"
else
checks::warn "testnet image not present locally (compose/k8s runs will rebuild or fail if NOMOS_SKIP_IMAGE_BUILD=1)"
fi
}
checks::print_docker_compose() {
checks::section "Docker Compose"
if checks::have docker; then
if docker compose version >/dev/null 2>&1; then
checks::ok "docker compose available"
else
checks::warn "docker compose not available"
fi
fi
}
checks::print_kubernetes() {
checks::section "Kubernetes (k8s runner)"
if checks::have kubectl; then
checks::ok "kubectl: $(kubectl version --client=true --short 2>/dev/null || true)"
KUBE_CONTEXT="$(kubectl config current-context 2>/dev/null || true)"
if [ -n "${KUBE_CONTEXT}" ]; then
checks::say "current-context: ${KUBE_CONTEXT}"
fi
if kubectl cluster-info >/dev/null 2>&1; then
checks::ok "cluster reachable"
kubectl get nodes -o wide 2>/dev/null | sed -n '1,3p' || true
else
checks::warn "cluster not reachable (k8s runner will skip with ClientInit error)"
fi
else
checks::warn "kubectl not found (k8s runner unavailable)"
KUBE_CONTEXT=""
fi
if checks::have helm; then
checks::ok "helm: $(helm version --short 2>/dev/null || true)"
else
checks::warn "helm not found (k8s runner uses helm)"
fi
}
checks::print_k8s_image_visibility() {
checks::section "K8s Image Visibility"
local default_local_image="logos-blockchain-testing:local"
local image="${NOMOS_TESTNET_IMAGE:-${default_local_image}}"
if [ -z "${KUBE_CONTEXT:-}" ]; then
return 0
fi
case "${KUBE_CONTEXT}" in
docker-desktop)
checks::ok "docker-desktop context shares local Docker images"
;;
kind-*)
if [[ "${image}" == *":local" ]]; then
checks::warn "kind cluster won't see local Docker images by default"
checks::say "Suggested: kind load docker-image ${image}"
fi
;;
minikube)
if [[ "${image}" == *":local" ]]; then
checks::warn "minikube may not see local Docker images by default"
checks::say "Suggested: minikube image load ${image}"
fi
;;
*)
if [[ "${image}" == *":local" ]]; then
checks::warn "current context is ${KUBE_CONTEXT}; a :local image tag may not be reachable by cluster nodes"
checks::say "Suggested: push to a registry and set NOMOS_TESTNET_IMAGE, or load into the cluster if supported"
fi
;;
esac
}
checks::print_docker_desktop_kubernetes_health() {
checks::section "Docker Desktop Kubernetes Health (best-effort)"
if ! checks::have kubectl; then
return 0
fi
if [ "${KUBE_CONTEXT:-}" != "docker-desktop" ]; then
return 0
fi
local kube_system_namespace="kube-system"
local storage_provisioner_pod="storage-provisioner"
if ! kubectl -n "${kube_system_namespace}" get pod "${storage_provisioner_pod}" >/dev/null 2>&1; then
checks::warn "${storage_provisioner_pod} pod not found"
return 0
fi
local phase reason
phase="$(kubectl -n "${kube_system_namespace}" get pod "${storage_provisioner_pod}" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
reason="$(kubectl -n "${kube_system_namespace}" get pod "${storage_provisioner_pod}" -o jsonpath='{.status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || true)"
if [ "${phase}" = "Running" ] || [ "${phase}" = "Succeeded" ]; then
checks::ok "${storage_provisioner_pod}: ${phase}"
else
checks::warn "${storage_provisioner_pod}: ${phase:-<unknown>} ${reason}"
fi
}
checks::print_debug_flags() {
checks::section "Runner Debug Flags (optional)"
checks::say "SLOW_TEST_ENV=${SLOW_TEST_ENV:-<unset>} (if true: doubles readiness timeouts)"
checks::say "NOMOS_SKIP_IMAGE_BUILD=${NOMOS_SKIP_IMAGE_BUILD:-<unset>} (compose/k8s)"
checks::say "COMPOSE_RUNNER_PRESERVE=${COMPOSE_RUNNER_PRESERVE:-<unset>} (compose)"
checks::say "K8S_RUNNER_PRESERVE=${K8S_RUNNER_PRESERVE:-<unset>} (k8s)"
checks::say "K8S_RUNNER_DEBUG=${K8S_RUNNER_DEBUG:-<unset>} (k8s helm debug)"
checks::say "COMPOSE_RUNNER_HOST=${COMPOSE_RUNNER_HOST:-<unset>} (compose readiness host override)"
checks::say "K8S_RUNNER_NODE_HOST=${K8S_RUNNER_NODE_HOST:-<unset>} (k8s NodePort host override)"
checks::say "K8S_RUNNER_NAMESPACE=${K8S_RUNNER_NAMESPACE:-<unset>} (k8s fixed namespace)"
}
checks::main() {
case "${1:-}" in
-h|--help) checks::usage; exit 0 ;;
esac
checks::load_env
checks::print_workspace
checks::print_disk_space
checks::print_kzg_params
checks::print_rust_toolchain
checks::print_docker
checks::print_docker_compose
checks::print_kubernetes
checks::print_k8s_image_visibility
checks::print_docker_desktop_kubernetes_health
checks::print_debug_flags
checks::section "Done"
checks::say "If something looks off, start with: scripts/run-examples.sh <mode> -t 60 -v 1 -e 1"
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
checks::main "$@"
fi

90
scripts/clean
View File

@ -1,90 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
usage() {
cat <<'EOF'
Usage: scripts/clean [options]
Removes local build artifacts that commonly cause disk pressure and flaky Docker builds.
Options:
--tmp Remove .tmp (default)
--target Remove target (default)
--docker Prune Docker builder cache (docker builder prune -f)
--docker-system Prune Docker system objects (requires --dangerous)
--volumes With --docker-system, also prune volumes
--dangerous Required for --docker-system (destructive)
--all Equivalent to --tmp --target --docker
-h, --help Show this help
EOF
}
DO_TMP=0
DO_TARGET=0
DO_DOCKER=0
DO_DOCKER_SYSTEM=0
DO_VOLUMES=0
DANGEROUS=0
if [ "$#" -eq 0 ]; then
DO_TMP=1
DO_TARGET=1
fi
while [ "$#" -gt 0 ]; do
case "$1" in
--tmp) DO_TMP=1; shift ;;
--target) DO_TARGET=1; shift ;;
--docker) DO_DOCKER=1; shift ;;
--docker-system) DO_DOCKER_SYSTEM=1; shift ;;
--volumes) DO_VOLUMES=1; shift ;;
--dangerous) DANGEROUS=1; shift ;;
--all) DO_TMP=1; DO_TARGET=1; DO_DOCKER=1; shift ;;
-h|--help) usage; exit 0 ;;
*) echo "Unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
echo "Workspace: ${ROOT_DIR}"
if [ "${DO_TMP}" -eq 1 ]; then
echo "==> Removing ${ROOT_DIR}/.tmp"
rm -rf "${ROOT_DIR}/.tmp"
fi
if [ "${DO_TARGET}" -eq 1 ]; then
echo "==> Removing ${ROOT_DIR}/target"
rm -rf "${ROOT_DIR}/target"
fi
if [ "${DO_DOCKER}" -eq 1 ]; then
if command -v docker >/dev/null 2>&1; then
echo "==> Pruning Docker builder cache"
docker builder prune -f >/dev/null
echo "==> Docker builder cache pruned"
else
echo "WARN: docker not found; skipping Docker prune" >&2
fi
fi
if [ "${DO_DOCKER_SYSTEM}" -eq 1 ]; then
if [ "${DANGEROUS}" -ne 1 ]; then
echo "ERROR: --docker-system requires --dangerous" >&2
exit 2
fi
if command -v docker >/dev/null 2>&1; then
echo "==> Pruning Docker system objects"
if [ "${DO_VOLUMES}" -eq 1 ]; then
docker system prune -af --volumes >/dev/null
else
docker system prune -af >/dev/null
fi
echo "==> Docker system prune complete"
else
echo "WARN: docker not found; skipping Docker system prune" >&2
fi
fi
echo "Done."

122
scripts/clean.sh Executable file
View File

@ -0,0 +1,122 @@
#!/usr/bin/env bash
set -euo pipefail
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
# shellcheck disable=SC1091
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
clean::usage() {
cat <<'USAGE'
Usage: scripts/clean.sh [options]
Removes local build artifacts that commonly cause disk pressure and flaky Docker builds.
Options:
--tmp Remove .tmp (default)
--target Remove target (default)
--docker Prune Docker builder cache (docker builder prune -f)
--docker-system Prune Docker system objects (requires --dangerous)
--volumes With --docker-system, also prune volumes
--dangerous Required for --docker-system (destructive)
--all Equivalent to --tmp --target --docker
-h, --help Show this help
USAGE
}
clean::have() { command -v "$1" >/dev/null 2>&1; }
clean::warn() { printf "WARN: %s\n" "$*" >&2; }
clean::die_usage() { printf "ERROR: %s\n" "$*" >&2; clean::usage >&2; exit 2; }
clean::parse_args() {
DO_TMP=0
DO_TARGET=0
DO_DOCKER=0
DO_DOCKER_SYSTEM=0
DO_VOLUMES=0
DANGEROUS=0
if [ "$#" -eq 0 ]; then
DO_TMP=1
DO_TARGET=1
fi
while [ "$#" -gt 0 ]; do
case "$1" in
--tmp) DO_TMP=1; shift ;;
--target) DO_TARGET=1; shift ;;
--docker) DO_DOCKER=1; shift ;;
--docker-system) DO_DOCKER_SYSTEM=1; shift ;;
--volumes) DO_VOLUMES=1; shift ;;
--dangerous) DANGEROUS=1; shift ;;
--all) DO_TMP=1; DO_TARGET=1; DO_DOCKER=1; shift ;;
-h|--help) clean::usage; exit 0 ;;
*) clean::die_usage "Unknown argument: $1" ;;
esac
done
}
clean::rm_path() {
local path="$1"
if [ -e "${path}" ]; then
echo "==> Removing ${path}"
rm -rf "${path}"
else
echo "==> Skipping missing ${path}"
fi
}
clean::docker_prune_builder() {
if clean::have docker; then
echo "==> Pruning Docker builder cache"
docker builder prune -f >/dev/null
echo "==> Docker builder cache pruned"
else
clean::warn "docker not found; skipping Docker prune"
fi
}
clean::docker_prune_system() {
if [ "${DANGEROUS}" -ne 1 ]; then
clean::die_usage "--docker-system requires --dangerous"
fi
if clean::have docker; then
echo "==> Pruning Docker system objects"
if [ "${DO_VOLUMES}" -eq 1 ]; then
docker system prune -af --volumes >/dev/null
else
docker system prune -af >/dev/null
fi
echo "==> Docker system prune complete"
else
clean::warn "docker not found; skipping Docker system prune"
fi
}
clean::main() {
clean::parse_args "$@"
ROOT_DIR="$(common::repo_root)"
echo "Workspace: ${ROOT_DIR}"
if [ "${DO_TMP}" -eq 1 ]; then
clean::rm_path "${ROOT_DIR}/.tmp"
fi
if [ "${DO_TARGET}" -eq 1 ]; then
clean::rm_path "${ROOT_DIR}/target"
fi
if [ "${DO_DOCKER}" -eq 1 ]; then
clean::docker_prune_builder
fi
if [ "${DO_DOCKER_SYSTEM}" -eq 1 ]; then
clean::docker_prune_system
fi
echo "Done."
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
clean::main "$@"
fi

2
scripts/lib/common.sh → scripts/common.sh Normal file → Executable file
View File

@ -1,6 +1,8 @@
#!/usr/bin/env bash
set -euo pipefail
# Shared helpers for `scripts/*.sh`.
common::ensure_bash() {
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"

369
scripts/lib/build-bundle.sh
View File

@ -1,369 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Intended to be sourced by scripts/build-bundle.sh
# shellcheck disable=SC1091
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
build_bundle::usage() {
cat <<'USAGE'
Usage: scripts/build-bundle.sh [--platform host|linux] [--output PATH]
Options:
--platform Target platform for binaries (default: host)
--output Output path for the tarball (default: .tmp/nomos-binaries-<platform>-<version>.tar.gz)
--rev nomos-node git revision to build (overrides NOMOS_NODE_REV)
--path Use local nomos-node checkout at DIR (skip fetch/checkout)
--features Extra cargo features to enable (comma-separated); base always includes "testing"
--docker-platform Docker platform for Linux bundle when running on non-Linux host (default: auto; linux/arm64 on Apple silicon Docker Desktop, else linux/amd64)
Notes:
- For compose/k8s, use platform=linux. If running on macOS, this script will
run inside a Linux Docker container to produce Linux binaries.
- On Apple silicon, Docker defaults to linux/arm64; for compose/k8s you likely
want linux/amd64 (the default here). Override with --docker-platform.
- VERSION, NOMOS_NODE_REV, and optional NOMOS_NODE_PATH env vars are honored (defaults align with run-examples.sh).
USAGE
}
build_bundle::fail() {
echo "$1" >&2
exit 1
}
build_bundle::apply_nomos_node_patches() {
local node_src="$1"
local apply="${NOMOS_NODE_APPLY_PATCHES:-1}"
if [ "${apply}" = "0" ]; then
return 0
fi
local patch_dir="${NOMOS_NODE_PATCH_DIR:-${ROOT_DIR}/patches/nomos-node}"
if [ ! -d "${patch_dir}" ]; then
return 0
fi
local level="${NOMOS_NODE_PATCH_LEVEL:-}"
if [ -z "${level}" ]; then
level="all"
fi
shopt -s nullglob
local -a patches=("${patch_dir}"/*.patch)
shopt -u nullglob
if [ "${#patches[@]}" -eq 0 ]; then
return 0
fi
echo "==> Applying nomos-node patches from ${patch_dir} (level=${level})"
local patch base phase
for patch in "${patches[@]}"; do
base="$(basename "${patch}")"
phase=""
if [[ "${base}" =~ phase([0-9]+) ]]; then
phase="${BASH_REMATCH[1]}"
fi
if [ "${level}" != "all" ] && [ "${level}" != "ALL" ]; then
if ! [[ "${level}" =~ ^[0-9]+$ ]]; then
build_bundle::fail "Invalid NOMOS_NODE_PATCH_LEVEL: ${level} (expected integer or 'all')"
fi
if [ -n "${phase}" ] && [ "${phase}" -gt "${level}" ]; then
continue
fi
fi
git -C "${node_src}" apply --whitespace=nowarn "${patch}"
done
}
build_bundle::load_env() {
ROOT_DIR="$(common::repo_root)"
export ROOT_DIR
common::require_file "${ROOT_DIR}/versions.env"
# shellcheck disable=SC1091
. "${ROOT_DIR}/versions.env"
DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}"
DEFAULT_NODE_REV="${NOMOS_NODE_REV:-}"
DEFAULT_NODE_PATH="${NOMOS_NODE_PATH:-}"
NOMOS_EXTRA_FEATURES="${NOMOS_EXTRA_FEATURES:-}"
DOCKER_PLATFORM="${NOMOS_BUNDLE_DOCKER_PLATFORM:-${NOMOS_BIN_PLATFORM:-}}"
BUNDLE_RUSTUP_TOOLCHAIN="${BUNDLE_RUSTUP_TOOLCHAIN:-}"
if [ -z "${BUNDLE_RUSTUP_TOOLCHAIN}" ] && command -v rustup >/dev/null 2>&1 && [ -f "${ROOT_DIR}/rust-toolchain.toml" ]; then
BUNDLE_RUSTUP_TOOLCHAIN="$(awk -F '\"' '/^[[:space:]]*channel[[:space:]]*=/{print $2; exit}' "${ROOT_DIR}/rust-toolchain.toml")"
fi
}
build_bundle::default_docker_platform() {
if [ -n "${DOCKER_PLATFORM}" ]; then
return 0
fi
if ! command -v docker >/dev/null 2>&1; then
return 0
fi
local docker_arch
docker_arch="$(docker version --format '{{.Server.Arch}}' 2>/dev/null || true)"
case "${docker_arch}" in
arm64|aarch64) DOCKER_PLATFORM="linux/arm64" ;;
amd64|x86_64) DOCKER_PLATFORM="linux/amd64" ;;
*) DOCKER_PLATFORM="linux/amd64" ;;
esac
}
build_bundle::parse_args() {
PLATFORM="host"
OUTPUT=""
REV_OVERRIDE=""
PATH_OVERRIDE=""
if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
build_bundle::usage
exit 0
fi
while [ "$#" -gt 0 ]; do
case "$1" in
--platform=*|-p=*) PLATFORM="${1#*=}"; shift ;;
--platform|-p) PLATFORM="${2:-}"; shift 2 ;;
--output=*|-o=*) OUTPUT="${1#*=}"; shift ;;
--output|-o) OUTPUT="${2:-}"; shift 2 ;;
--rev=*) REV_OVERRIDE="${1#*=}"; shift ;;
--rev) REV_OVERRIDE="${2:-}"; shift 2 ;;
--path=*) PATH_OVERRIDE="${1#*=}"; shift ;;
--path) PATH_OVERRIDE="${2:-}"; shift 2 ;;
--features=*) NOMOS_EXTRA_FEATURES="${1#*=}"; shift ;;
--features) NOMOS_EXTRA_FEATURES="${2:-}"; shift 2 ;;
--docker-platform=*) DOCKER_PLATFORM="${1#*=}"; shift ;;
--docker-platform) DOCKER_PLATFORM="${2:-}"; shift 2 ;;
*) build_bundle::fail "Unknown argument: $1" ;;
esac
done
}
build_bundle::validate_and_finalize() {
case "${PLATFORM}" in
host|linux) ;;
*) build_bundle::fail "--platform must be host or linux" ;;
esac
VERSION="${DEFAULT_VERSION}"
if [ -n "${REV_OVERRIDE}" ] && [ -n "${PATH_OVERRIDE}" ]; then
build_bundle::fail "Use either --rev or --path, not both"
fi
if [ -z "${REV_OVERRIDE}" ] && [ -z "${PATH_OVERRIDE}" ] && [ -z "${DEFAULT_NODE_REV}" ] && [ -z "${DEFAULT_NODE_PATH}" ]; then
build_bundle::fail "Provide --rev, --path, or set NOMOS_NODE_REV/NOMOS_NODE_PATH in versions.env"
fi
NOMOS_NODE_REV="${REV_OVERRIDE:-${DEFAULT_NODE_REV}}"
NOMOS_NODE_PATH="${PATH_OVERRIDE:-${DEFAULT_NODE_PATH}}"
export NOMOS_NODE_REV NOMOS_NODE_PATH
build_bundle::default_docker_platform
DOCKER_PLATFORM="${DOCKER_PLATFORM:-linux/amd64}"
# Normalize OUTPUT to an absolute path under the workspace.
if [ -z "${OUTPUT}" ]; then
OUTPUT="${ROOT_DIR}/.tmp/nomos-binaries-${PLATFORM}-${VERSION}.tar.gz"
elif [[ "${OUTPUT}" != /* ]]; then
OUTPUT="${ROOT_DIR}/${OUTPUT#./}"
fi
echo "Bundle output: ${OUTPUT}"
}
build_bundle::clean_cargo_linux_cache() {
rm -rf "${ROOT_DIR}/.tmp/cargo-linux/registry" "${ROOT_DIR}/.tmp/cargo-linux/git"
}
build_bundle::maybe_run_linux_build_in_docker() {
# With `set -e`, this function must return 0 when no Docker cross-build is needed.
if [ "${PLATFORM}" != "linux" ] || [ "$(uname -s)" = "Linux" ] || [ -n "${BUNDLE_IN_CONTAINER:-}" ]; then
return 0
fi
command -v docker >/dev/null 2>&1 || build_bundle::fail "Docker is required to build a Linux bundle from non-Linux host"
[ -n "${DOCKER_PLATFORM}" ] || build_bundle::fail "--docker-platform must not be empty"
local node_path_env="${NOMOS_NODE_PATH}"
local -a extra_mounts=()
if [ -n "${NOMOS_NODE_PATH}" ]; then
case "${NOMOS_NODE_PATH}" in
"${ROOT_DIR}"/*)
node_path_env="/workspace${NOMOS_NODE_PATH#"${ROOT_DIR}"}"
;;
/*)
node_path_env="/external/nomos-node"
extra_mounts+=("-v" "${NOMOS_NODE_PATH}:${node_path_env}")
;;
*)
build_bundle::fail "--path must be absolute when cross-building in Docker"
;;
esac
fi
echo "==> Building Linux bundle inside Docker"
local container_output="/workspace${OUTPUT#"${ROOT_DIR}"}"
mkdir -p "${ROOT_DIR}/.tmp/cargo-linux" "${ROOT_DIR}/.tmp/nomos-node-linux-target"
local -a features_args=()
if [ -n "${NOMOS_EXTRA_FEATURES:-}" ]; then
features_args+=(--features "${NOMOS_EXTRA_FEATURES}")
fi
local -a src_args=()
if [ -n "${node_path_env}" ]; then
src_args+=(--path "${node_path_env}")
else
src_args+=(--rev "${NOMOS_NODE_REV}")
fi
docker run --rm --platform "${DOCKER_PLATFORM}" \
-e VERSION="${VERSION}" \
-e NOMOS_NODE_REV="${NOMOS_NODE_REV}" \
-e NOMOS_NODE_PATH="${node_path_env}" \
-e NOMOS_CIRCUITS="/workspace/.tmp/nomos-circuits-linux" \
-e STACK_DIR="/workspace/.tmp/nomos-circuits-linux" \
-e HOST_DIR="/workspace/.tmp/nomos-circuits-linux" \
-e NOMOS_EXTRA_FEATURES="${NOMOS_EXTRA_FEATURES:-}" \
-e BUNDLE_IN_CONTAINER=1 \
-e CARGO_HOME=/workspace/.tmp/cargo-linux \
-e CARGO_TARGET_DIR=/workspace/.tmp/nomos-node-linux-target \
-v "${ROOT_DIR}/.tmp/cargo-linux":/workspace/.tmp/cargo-linux \
-v "${ROOT_DIR}/.tmp/nomos-node-linux-target":/workspace/.tmp/nomos-node-linux-target \
-v "${ROOT_DIR}:/workspace" \
"${extra_mounts[@]}" \
-w /workspace \
rust:1.80-bullseye \
bash -c "apt-get update && apt-get install -y clang llvm-dev libclang-dev pkg-config cmake libssl-dev rsync libgmp10 libgmp-dev libgomp1 nasm && ./scripts/build-bundle.sh --platform linux --output \"${container_output}\" ${src_args[*]} ${features_args[*]}"
exit 0
}
build_bundle::prepare_circuits() {
echo "==> Preparing circuits (version ${VERSION})"
if [ "${PLATFORM}" = "host" ]; then
CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits-host"
NODE_TARGET="${ROOT_DIR}/.tmp/nomos-node-host-target"
else
CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits-linux"
NODE_TARGET="${ROOT_DIR}/.tmp/nomos-node-linux-target"
fi
NODE_SRC_DEFAULT="${ROOT_DIR}/.tmp/nomos-node-${PLATFORM}-src"
NODE_SRC="${NOMOS_NODE_PATH:-${NODE_SRC_DEFAULT}}"
if [ -n "${NOMOS_NODE_PATH}" ]; then
[ -d "${NODE_SRC}" ] || build_bundle::fail "NOMOS_NODE_PATH does not exist: ${NODE_SRC}"
rm -rf "${NODE_SRC_DEFAULT}"
if [ -d "${NODE_TARGET}" ]; then
find "${NODE_TARGET}" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
fi
NODE_TARGET="${NODE_TARGET}-local"
fi
export NOMOS_CIRCUITS="${CIRCUITS_DIR}"
mkdir -p "${ROOT_DIR}/.tmp" "${CIRCUITS_DIR}"
if [ -f "${CIRCUITS_DIR}/${KZG_FILE:-kzgrs_test_params}" ]; then
echo "Circuits already present at ${CIRCUITS_DIR}; skipping download"
else
STACK_DIR="${CIRCUITS_DIR}" HOST_DIR="${CIRCUITS_DIR}" \
"${ROOT_DIR}/scripts/setup-circuits-stack.sh" "${VERSION}" </dev/null
fi
NODE_BIN="${NODE_TARGET}/debug/nomos-node"
EXEC_BIN="${NODE_TARGET}/debug/nomos-executor"
CLI_BIN="${NODE_TARGET}/debug/nomos-cli"
}
build_bundle::build_binaries() {
FEATURES="testing"
if [ -n "${NOMOS_EXTRA_FEATURES:-}" ]; then
FEATURES="${FEATURES},${NOMOS_EXTRA_FEATURES}"
fi
echo "==> Building binaries (platform=${PLATFORM})"
mkdir -p "${NODE_SRC}"
(
cd "${NODE_SRC}"
if [ -n "${NOMOS_NODE_PATH}" ]; then
echo "Using local nomos-node checkout at ${NODE_SRC} (no fetch/checkout)"
else
if [ ! -d "${NODE_SRC}/.git" ]; then
git clone https://github.com/logos-co/nomos-node.git "${NODE_SRC}"
fi
git fetch --depth 1 origin "${NOMOS_NODE_REV}"
git checkout "${NOMOS_NODE_REV}"
git reset --hard
git clean -fdx
fi
if [ -z "${NOMOS_NODE_PATH}" ]; then
build_bundle::apply_nomos_node_patches "${NODE_SRC}"
fi
if [ -n "${BUNDLE_RUSTUP_TOOLCHAIN}" ]; then
RUSTFLAGS='--cfg feature="pol-dev-mode"' NOMOS_CIRCUITS="${CIRCUITS_DIR}" \
RUSTUP_TOOLCHAIN="${BUNDLE_RUSTUP_TOOLCHAIN}" \
cargo build --features "${FEATURES}" \
-p nomos-node -p nomos-executor -p nomos-cli \
--target-dir "${NODE_TARGET}"
else
RUSTFLAGS='--cfg feature="pol-dev-mode"' NOMOS_CIRCUITS="${CIRCUITS_DIR}" \
cargo build --features "${FEATURES}" \
-p nomos-node -p nomos-executor -p nomos-cli \
--target-dir "${NODE_TARGET}"
fi
)
}
build_bundle::package_bundle() {
echo "==> Packaging bundle"
local bundle_dir="${ROOT_DIR}/.tmp/nomos-bundle"
rm -rf "${bundle_dir}"
mkdir -p "${bundle_dir}/artifacts/circuits"
cp -a "${CIRCUITS_DIR}/." "${bundle_dir}/artifacts/circuits/"
mkdir -p "${bundle_dir}/artifacts"
cp "${NODE_BIN}" "${bundle_dir}/artifacts/"
cp "${EXEC_BIN}" "${bundle_dir}/artifacts/"
cp "${CLI_BIN}" "${bundle_dir}/artifacts/"
{
echo "nomos_node_path=${NOMOS_NODE_PATH:-}"
echo "nomos_node_rev=${NOMOS_NODE_REV:-}"
if [ -d "${NODE_SRC}/.git" ] && command -v git >/dev/null 2>&1; then
echo "nomos_node_git_head=$(git -C "${NODE_SRC}" rev-parse HEAD 2>/dev/null || true)"
fi
echo "platform=${PLATFORM}"
echo "features=${FEATURES}"
} > "${bundle_dir}/artifacts/nomos-bundle-meta.env"
mkdir -p "$(dirname "${OUTPUT}")"
if tar --help 2>/dev/null | grep -q -- '--no-mac-metadata'; then
tar --no-mac-metadata --no-xattrs -czf "${OUTPUT}" -C "${bundle_dir}" artifacts
elif tar --help 2>/dev/null | grep -q -- '--no-xattrs'; then
tar --no-xattrs -czf "${OUTPUT}" -C "${bundle_dir}" artifacts
else
tar -czf "${OUTPUT}" -C "${bundle_dir}" artifacts
fi
echo "Bundle created at ${OUTPUT}"
if [[ "${FEATURES}" == *profiling* ]]; then
cat <<'EOF_PROF'
Profiling endpoints (enabled by --features profiling):
CPU pprof (SVG): curl "http://<node-host>:8722/debug/pprof/profile?seconds=15&format=svg" -o profile.svg
CPU pprof (proto): go tool pprof -http=:8080 "http://<node-host>:8722/debug/pprof/profile?seconds=15&format=proto"
EOF_PROF
fi
}
build_bundle::main() {
build_bundle::load_env
build_bundle::clean_cargo_linux_cache
build_bundle::parse_args "$@"
build_bundle::validate_and_finalize
build_bundle::maybe_run_linux_build_in_docker
build_bundle::prepare_circuits
build_bundle::build_binaries
build_bundle::package_bundle
}

170
scripts/lib/build-linux-binaries.sh
View File

@ -1,170 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Intended to be sourced by scripts/build-linux-binaries.sh
# shellcheck disable=SC1091
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
build_linux_binaries::usage() {
cat <<'EOF'
Usage: scripts/build-linux-binaries.sh [options]
Builds a Linux bundle via scripts/build-bundle.sh, then stages artifacts into:
- testing-framework/assets/stack/bin
- testing-framework/assets/stack/kzgrs_test_params (or NOMOS_KZG_DIR_REL)
Options:
--rev REV nomos-node git revision to build (overrides NOMOS_NODE_REV)
--path DIR use local nomos-node checkout (skip fetch/checkout)
--features LIST extra cargo features (comma-separated); base includes "testing"
--docker-platform PLAT docker platform for the Linux build (e.g. linux/amd64, linux/arm64)
--tar PATH stage from an existing bundle tarball (skip build)
--output PATH where to write the bundle tarball when building (default: .tmp/nomos-binaries-linux-<version>.tar.gz)
-h, --help show help
Environment:
VERSION circuits version (default from versions.env)
NOMOS_CIRCUITS_VERSION legacy alias for VERSION (supported)
NOMOS_NODE_REV default nomos-node revision (from versions.env)
NOMOS_KZG_DIR_REL host path for staged circuits dir (default: testing-framework/assets/stack/kzgrs_test_params)
EOF
}
build_linux_binaries::fail_with_usage() {
echo "$1" >&2
build_linux_binaries::usage
exit 1
}
build_linux_binaries::load_env() {
ROOT_DIR="$(common::repo_root)"
export ROOT_DIR
common::require_file "${ROOT_DIR}/versions.env"
# shellcheck disable=SC1091
. "${ROOT_DIR}/versions.env"
common::maybe_source "${ROOT_DIR}/paths.env"
DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}"
VERSION="${VERSION:-${DEFAULT_VERSION}}"
if [ -n "${NOMOS_CIRCUITS_VERSION:-}" ]; then
VERSION="${NOMOS_CIRCUITS_VERSION}"
fi
}
build_linux_binaries::parse_args() {
REV_OVERRIDE=""
PATH_OVERRIDE=""
EXTRA_FEATURES=""
DOCKER_PLATFORM=""
OUTPUT_TAR=""
INPUT_TAR=""
while [ "$#" -gt 0 ]; do
case "$1" in
-h|--help) build_linux_binaries::usage; exit 0 ;;
--rev) REV_OVERRIDE="${2:-}"; shift 2 ;;
--rev=*) REV_OVERRIDE="${1#*=}"; shift ;;
--path) PATH_OVERRIDE="${2:-}"; shift 2 ;;
--path=*) PATH_OVERRIDE="${1#*=}"; shift ;;
--features) EXTRA_FEATURES="${2:-}"; shift 2 ;;
--features=*) EXTRA_FEATURES="${1#*=}"; shift ;;
--docker-platform) DOCKER_PLATFORM="${2:-}"; shift 2 ;;
--docker-platform=*) DOCKER_PLATFORM="${1#*=}"; shift ;;
--tar) INPUT_TAR="${2:-}"; shift 2 ;;
--tar=*) INPUT_TAR="${1#*=}"; shift ;;
--output|-o) OUTPUT_TAR="${2:-}"; shift 2 ;;
--output=*|-o=*) OUTPUT_TAR="${1#*=}"; shift ;;
*) build_linux_binaries::fail_with_usage "Unknown argument: $1" ;;
esac
done
if [ -n "${REV_OVERRIDE}" ] && [ -n "${PATH_OVERRIDE}" ]; then
build_linux_binaries::fail_with_usage "Use either --rev or --path, not both"
fi
if [ -n "${INPUT_TAR}" ] && [ ! -f "${INPUT_TAR}" ]; then
build_linux_binaries::fail_with_usage "Bundle tarball not found: ${INPUT_TAR}"
fi
if [ -z "${OUTPUT_TAR}" ]; then
OUTPUT_TAR="${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz"
elif [[ "${OUTPUT_TAR}" != /* ]]; then
OUTPUT_TAR="${ROOT_DIR}/${OUTPUT_TAR#./}"
fi
}
build_linux_binaries::build_bundle_if_needed() {
if [ -n "${INPUT_TAR}" ]; then
BUNDLE_TAR="${INPUT_TAR}"
return 0
fi
mkdir -p "$(dirname "${OUTPUT_TAR}")"
BUILD_ARGS=(--platform linux --output "${OUTPUT_TAR}")
if [ -n "${REV_OVERRIDE}" ]; then
BUILD_ARGS+=(--rev "${REV_OVERRIDE}")
elif [ -n "${PATH_OVERRIDE}" ]; then
BUILD_ARGS+=(--path "${PATH_OVERRIDE}")
fi
if [ -n "${EXTRA_FEATURES}" ]; then
BUILD_ARGS+=(--features "${EXTRA_FEATURES}")
fi
if [ -n "${DOCKER_PLATFORM}" ]; then
BUILD_ARGS+=(--docker-platform "${DOCKER_PLATFORM}")
fi
echo "==> Building Linux bundle"
VERSION="${VERSION}" "${ROOT_DIR}/scripts/build-bundle.sh" "${BUILD_ARGS[@]}"
BUNDLE_TAR="${OUTPUT_TAR}"
}
build_linux_binaries::stage_from_bundle() {
local tar_path="$1"
local extract_dir
extract_dir="$(common::tmpdir nomos-linux-bundle.XXXXXX)"
cleanup() { rm -rf "${extract_dir}" 2>/dev/null || true; }
trap cleanup EXIT
echo "==> Extracting ${tar_path}"
tar -xzf "${tar_path}" -C "${extract_dir}"
local artifacts="${extract_dir}/artifacts"
[ -f "${artifacts}/nomos-node" ] || common::die "Missing nomos-node in bundle: ${tar_path}"
[ -f "${artifacts}/nomos-executor" ] || common::die "Missing nomos-executor in bundle: ${tar_path}"
[ -f "${artifacts}/nomos-cli" ] || common::die "Missing nomos-cli in bundle: ${tar_path}"
[ -d "${artifacts}/circuits" ] || common::die "Missing circuits/ in bundle: ${tar_path}"
local bin_out="${ROOT_DIR}/testing-framework/assets/stack/bin"
local kzg_dir_rel="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}"
local circuits_out="${ROOT_DIR}/${kzg_dir_rel}"
echo "==> Staging binaries to ${bin_out}"
mkdir -p "${bin_out}"
cp "${artifacts}/nomos-node" "${artifacts}/nomos-executor" "${artifacts}/nomos-cli" "${bin_out}/"
echo "==> Staging circuits to ${circuits_out}"
rm -rf "${circuits_out}"
mkdir -p "${circuits_out}"
if command -v rsync >/dev/null 2>&1; then
rsync -a --delete "${artifacts}/circuits/" "${circuits_out}/"
else
cp -a "${artifacts}/circuits/." "${circuits_out}/"
fi
# If the tarball was produced inside Docker, it might be root-owned on the host.
chown -R "$(id -u)":"$(id -g)" "${bin_out}" "${circuits_out}" 2>/dev/null || true
}
build_linux_binaries::main() {
build_linux_binaries::load_env
build_linux_binaries::parse_args "$@"
build_linux_binaries::build_bundle_if_needed
build_linux_binaries::stage_from_bundle "${BUNDLE_TAR}"
echo
echo "Binaries staged in ${ROOT_DIR}/testing-framework/assets/stack/bin"
echo "Circuits staged in ${ROOT_DIR}/${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}"
echo "Bundle tarball: ${BUNDLE_TAR}"
}

517
scripts/lib/run-examples.sh
View File

@ -1,517 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# This file is meant to be sourced by scripts/run-examples.sh (and possibly others).
# shellcheck disable=SC1091
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
run_examples::usage() {
cat <<'EOF'
Usage: scripts/run-examples.sh [options] [compose|host|k8s]
Modes:
compose Run examples/src/bin/compose_runner.rs (default)
host Run examples/src/bin/local_runner.rs
k8s Run examples/src/bin/k8s_runner.rs
Options:
-t, --run-seconds N Duration to run the demo (required)
-v, --validators N Number of validators (required)
-e, --executors N Number of executors (required)
--bundle PATH Convenience alias for setting NOMOS_BINARIES_TAR=PATH
--local Use a local Docker image tag (default for docker-desktop k8s)
--ecr Use an ECR image reference (default for non-docker-desktop k8s)
--no-image-build Skip rebuilding the compose/k8s image (sets NOMOS_SKIP_IMAGE_BUILD=1)
Environment:
VERSION Circuits version (default from versions.env)
NOMOS_TESTNET_IMAGE Image reference (overridden by --local/--ecr selection)
ECR_IMAGE Full image reference for --ecr (overrides ECR_REGISTRY/ECR_REPO/TAG)
ECR_REGISTRY Registry hostname for --ecr (default public.ecr.aws/r4s5t9y4)
ECR_REPO Repository path for --ecr (default logos/logos-blockchain)
TAG Tag for --ecr (default test)
NOMOS_TESTNET_IMAGE_PULL_POLICY K8s imagePullPolicy (default IfNotPresent; set to Always for --ecr)
NOMOS_BINARIES_TAR Path to prebuilt binaries/circuits tarball (default .tmp/nomos-binaries-<platform>-<version>.tar.gz)
NOMOS_SKIP_IMAGE_BUILD Set to 1 to skip rebuilding the compose/k8s image
EOF
}
run_examples::fail_with_usage() {
echo "$1" >&2
run_examples::usage
exit 1
}
run_examples::load_env() {
ROOT_DIR="$(common::repo_root)"
export ROOT_DIR
common::require_file "${ROOT_DIR}/versions.env"
# shellcheck disable=SC1091
. "${ROOT_DIR}/versions.env"
common::maybe_source "${ROOT_DIR}/paths.env"
DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}"
VERSION="${VERSION:-${DEFAULT_VERSION}}"
KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}"
KZG_FILE="${NOMOS_KZG_FILE:-kzgrs_test_params}"
KZG_CONTAINER_PATH="${NOMOS_KZG_CONTAINER_PATH:-/kzgrs_test_params/kzgrs_test_params}"
HOST_KZG_DIR="${ROOT_DIR}/${KZG_DIR_REL}"
HOST_KZG_FILE="${HOST_KZG_DIR}/${KZG_FILE}"
}
run_examples::select_bin() {
case "${MODE}" in
compose) BIN="compose_runner" ;;
host) BIN="local_runner" ;;
k8s) BIN="k8s_runner" ;;
*) common::die "Unknown mode '${MODE}' (use compose|host|k8s)" ;;
esac
}
run_examples::parse_args() {
MODE="compose"
RUN_SECS_RAW=""
DEMO_VALIDATORS=""
DEMO_EXECUTORS=""
IMAGE_SELECTION_MODE="auto"
RUN_SECS_RAW_SPECIFIED=""
while [ "$#" -gt 0 ]; do
case "$1" in
-h|--help)
run_examples::usage
exit 0
;;
-t|--run-seconds)
RUN_SECS_RAW_SPECIFIED=1
RUN_SECS_RAW="${2:-}"
shift 2
;;
--run-seconds=*)
RUN_SECS_RAW_SPECIFIED=1
RUN_SECS_RAW="${1#*=}"
shift
;;
-v|--validators)
DEMO_VALIDATORS="${2:-}"
shift 2
;;
--validators=*)
DEMO_VALIDATORS="${1#*=}"
shift
;;
-e|--executors)
DEMO_EXECUTORS="${2:-}"
shift 2
;;
--executors=*)
DEMO_EXECUTORS="${1#*=}"
shift
;;
--bundle)
NOMOS_BINARIES_TAR="${2:-}"
export NOMOS_BINARIES_TAR
shift 2
;;
--bundle=*)
NOMOS_BINARIES_TAR="${1#*=}"
export NOMOS_BINARIES_TAR
shift
;;
--local)
if [ "${IMAGE_SELECTION_MODE}" = "ecr" ]; then
run_examples::fail_with_usage "--local and --ecr are mutually exclusive"
fi
IMAGE_SELECTION_MODE="local"
shift
;;
--ecr)
if [ "${IMAGE_SELECTION_MODE}" = "local" ]; then
run_examples::fail_with_usage "--local and --ecr are mutually exclusive"
fi
IMAGE_SELECTION_MODE="ecr"
shift
;;
--no-image-build)
NOMOS_SKIP_IMAGE_BUILD=1
export NOMOS_SKIP_IMAGE_BUILD
shift
;;
compose|host|k8s)
MODE="$1"
shift
;;
*)
# Positional run-seconds fallback for legacy usage.
if [ -z "${RUN_SECS_RAW_SPECIFIED}" ] && common::is_uint "$1"; then
RUN_SECS_RAW="$1"
shift
else
run_examples::fail_with_usage "Unknown argument: $1"
fi
;;
esac
done
if [ -n "${NOMOS_BINARIES_TAR:-}" ] && [ ! -f "${NOMOS_BINARIES_TAR}" ]; then
run_examples::fail_with_usage "NOMOS_BINARIES_TAR is set but missing: ${NOMOS_BINARIES_TAR}"
fi
if ! common::is_uint "${RUN_SECS_RAW}" || [ "${RUN_SECS_RAW}" -le 0 ]; then
run_examples::fail_with_usage "run-seconds must be a positive integer (pass -t/--run-seconds)"
fi
RUN_SECS="${RUN_SECS_RAW}"
if [ -z "${DEMO_VALIDATORS}" ] || [ -z "${DEMO_EXECUTORS}" ]; then
run_examples::fail_with_usage "validators and executors must be provided via -v/--validators and -e/--executors"
fi
if ! common::is_uint "${DEMO_VALIDATORS}" ; then
run_examples::fail_with_usage "validators must be a non-negative integer (pass -v/--validators)"
fi
if ! common::is_uint "${DEMO_EXECUTORS}" ; then
run_examples::fail_with_usage "executors must be a non-negative integer (pass -e/--executors)"
fi
}
run_examples::select_image() {
local selection="${IMAGE_SELECTION_MODE}"
local context=""
if [ "${selection}" = "auto" ]; then
if [ "${MODE}" = "k8s" ] && command -v kubectl >/dev/null 2>&1; then
context="$(kubectl config current-context 2>/dev/null || true)"
if [ "${context}" = "docker-desktop" ]; then
selection="local"
else
selection="ecr"
fi
else
selection="local"
fi
fi
if [ "${selection}" = "local" ]; then
IMAGE="${NOMOS_TESTNET_IMAGE:-logos-blockchain-testing:local}"
export NOMOS_TESTNET_IMAGE_PULL_POLICY="${NOMOS_TESTNET_IMAGE_PULL_POLICY:-IfNotPresent}"
elif [ "${selection}" = "ecr" ]; then
local tag="${TAG:-test}"
if [ -n "${ECR_IMAGE:-}" ]; then
IMAGE="${ECR_IMAGE}"
elif [ -n "${ECR_REGISTRY:-}" ]; then
local registry="${ECR_REGISTRY}"
local repo="${ECR_REPO:-logos/logos-blockchain}"
IMAGE="${registry}/${repo}:${tag}"
elif [ -n "${AWS_ACCOUNT_ID:-}" ]; then
local aws_region="${AWS_REGION:-ap-southeast-2}"
local aws_account_id="${AWS_ACCOUNT_ID}"
local repo="${ECR_REPO:-logos-blockchain-testing}"
IMAGE="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com/${repo}:${tag}"
else
local registry="public.ecr.aws/r4s5t9y4"
local repo="${ECR_REPO:-logos/logos-blockchain}"
IMAGE="${registry}/${repo}:${tag}"
fi
export NOMOS_TESTNET_IMAGE_PULL_POLICY="${NOMOS_TESTNET_IMAGE_PULL_POLICY:-Always}"
else
run_examples::fail_with_usage "Unknown image selection mode: ${selection}"
fi
export IMAGE_TAG="${IMAGE}"
export NOMOS_TESTNET_IMAGE="${IMAGE}"
if [ "${MODE}" = "k8s" ]; then
if [ "${selection}" = "ecr" ]; then
export NOMOS_KZG_MODE="${NOMOS_KZG_MODE:-inImage}"
else
export NOMOS_KZG_MODE="${NOMOS_KZG_MODE:-hostPath}"
fi
fi
}
run_examples::default_tar_path() {
if [ -n "${NOMOS_BINARIES_TAR:-}" ]; then
echo "${NOMOS_BINARIES_TAR}"
return
fi
case "${MODE}" in
host) echo "${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz" ;;
compose|k8s)
if [ "${NOMOS_SKIP_IMAGE_BUILD:-}" = "1" ]; then
echo "${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz"
else
echo "${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz"
fi
;;
*) echo "${ROOT_DIR}/.tmp/nomos-binaries-${VERSION}.tar.gz" ;;
esac
}
run_examples::bundle_matches_expected() {
local tar_path="$1"
[ -f "${tar_path}" ] || return 1
[ -z "${NOMOS_NODE_REV:-}" ] && return 0
local meta tar_rev tar_head
meta="$(tar -xOzf "${tar_path}" artifacts/nomos-bundle-meta.env 2>/dev/null || true)"
if [ -z "${meta}" ]; then
echo "Bundle meta missing in ${tar_path}; treating as stale and rebuilding." >&2
return 1
fi
tar_rev="$(echo "${meta}" | sed -n 's/^nomos_node_rev=//p' | head -n 1)"
tar_head="$(echo "${meta}" | sed -n 's/^nomos_node_git_head=//p' | head -n 1)"
if [ -n "${tar_rev}" ] && [ "${tar_rev}" != "${NOMOS_NODE_REV}" ]; then
echo "Bundle ${tar_path} is for nomos-node rev ${tar_rev}, expected ${NOMOS_NODE_REV}; rebuilding." >&2
return 1
fi
if [ -n "${tar_head}" ] && [ "${tar_head}" != "${NOMOS_NODE_REV}" ]; then
echo "Bundle ${tar_path} is for nomos-node git head ${tar_head}, expected ${NOMOS_NODE_REV}; rebuilding." >&2
return 1
fi
return 0
}
run_examples::host_bin_matches_arch() {
local bin_path="$1"
[ -x "${bin_path}" ] || return 1
command -v file >/dev/null 2>&1 || return 0
local info expected
info="$(file -b "${bin_path}" 2>/dev/null || true)"
case "$(uname -m)" in
x86_64) expected="x86-64|x86_64" ;;
aarch64|arm64) expected="arm64|aarch64" ;;
*) expected="" ;;
esac
[ -n "${expected}" ] && echo "${info}" | grep -Eqi "${expected}"
}
run_examples::restore_binaries_from_tar() {
local tar_path="${1:-}"
if [ -z "${tar_path}" ]; then
tar_path="$(run_examples::default_tar_path)"
fi
run_examples::bundle_matches_expected "${tar_path}" || return 1
[ -f "${tar_path}" ] || return 1
local extract_dir="${ROOT_DIR}/.tmp/nomos-binaries"
echo "==> Restoring binaries from ${tar_path}"
rm -rf "${extract_dir}"
mkdir -p "${extract_dir}"
tar -xzf "${tar_path}" -C "${extract_dir}" || common::die "Failed to extract ${tar_path}"
local src="${extract_dir}/artifacts"
local bin_dst="${ROOT_DIR}/testing-framework/assets/stack/bin"
local circuits_src="${src}/circuits"
local circuits_dst="${HOST_KZG_DIR}"
RESTORED_BIN_DIR="${src}"
export RESTORED_BIN_DIR
if [ ! -f "${src}/nomos-node" ] || [ ! -f "${src}/nomos-executor" ] || [ ! -f "${src}/nomos-cli" ]; then
echo "Binaries missing in ${tar_path}; provide a prebuilt binaries tarball." >&2
return 1
fi
local copy_bins=1
if [ "${MODE}" != "host" ] && ! run_examples::host_bin_matches_arch "${src}/nomos-node"; then
echo "Bundled binaries do not match host arch; skipping copy so containers rebuild from source."
copy_bins=0
rm -f "${bin_dst}/nomos-node" "${bin_dst}/nomos-executor" "${bin_dst}/nomos-cli"
fi
if [ "${copy_bins}" -eq 1 ]; then
mkdir -p "${bin_dst}"
cp "${src}/nomos-node" "${src}/nomos-executor" "${src}/nomos-cli" "${bin_dst}/"
fi
if [ -d "${circuits_src}" ] && [ -f "${circuits_src}/${KZG_FILE}" ]; then
rm -rf "${circuits_dst}"
mkdir -p "${circuits_dst}"
if command -v rsync >/dev/null 2>&1; then
rsync -a --delete "${circuits_src}/" "${circuits_dst}/"
else
rm -rf "${circuits_dst:?}/"*
cp -a "${circuits_src}/." "${circuits_dst}/"
fi
else
echo "Circuits missing in ${tar_path}; provide a prebuilt binaries/circuits tarball." >&2
return 1
fi
RESTORED_BINARIES=1
export RESTORED_BINARIES
}
run_examples::ensure_binaries_tar() {
local platform="$1"
local tar_path="$2"
echo "==> Building fresh binaries bundle (${platform}) at ${tar_path}"
"${ROOT_DIR}/scripts/build-bundle.sh" --platform "${platform}" --output "${tar_path}" --rev "${NOMOS_NODE_REV}"
}
run_examples::prepare_bundles() {
RESTORED_BINARIES=0
NEED_HOST_RESTORE_AFTER_IMAGE=0
HOST_TAR="${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz"
LINUX_TAR="${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz"
if [ -n "${NOMOS_NODE_BIN:-}" ] && [ -x "${NOMOS_NODE_BIN}" ] && [ -n "${NOMOS_EXECUTOR_BIN:-}" ] && [ -x "${NOMOS_EXECUTOR_BIN}" ]; then
echo "==> Using pre-specified host binaries (NOMOS_NODE_BIN/NOMOS_EXECUTOR_BIN); skipping tarball restore"
return 0
fi
# On non-Linux compose/k8s runs, use the Linux bundle for image build, then restore host bundle for the runner.
if [ "${MODE}" != "host" ] && [ "$(uname -s)" != "Linux" ] && [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "0" ] && [ -f "${LINUX_TAR}" ]; then
NEED_HOST_RESTORE_AFTER_IMAGE=1
run_examples::restore_binaries_from_tar "${LINUX_TAR}" || {
run_examples::ensure_binaries_tar linux "${LINUX_TAR}"
run_examples::restore_binaries_from_tar "${LINUX_TAR}"
}
fi
if ! run_examples::restore_binaries_from_tar; then
local tar_path
tar_path="$(run_examples::default_tar_path)"
case "${MODE}" in
host) run_examples::ensure_binaries_tar host "${tar_path}" ;;
compose|k8s)
if [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "1" ]; then
run_examples::ensure_binaries_tar host "${tar_path}"
else
run_examples::ensure_binaries_tar linux "${tar_path}"
fi
;;
*) run_examples::ensure_binaries_tar host "${tar_path}" ;;
esac
run_examples::restore_binaries_from_tar "${tar_path}" || common::die \
"Missing or invalid binaries tarball. Provide it via --bundle/NOMOS_BINARIES_TAR or place it at $(run_examples::default_tar_path)."
fi
}
run_examples::maybe_rebuild_image() {
if [ "${MODE}" = "host" ]; then
return 0
fi
if [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "1" ]; then
echo "==> Skipping testnet image rebuild (NOMOS_SKIP_IMAGE_BUILD=1)"
return 0
fi
echo "==> Rebuilding testnet image (${IMAGE})"
IMAGE_TAG="${IMAGE}" COMPOSE_CIRCUITS_PLATFORM="${COMPOSE_CIRCUITS_PLATFORM:-}" \
"${ROOT_DIR}/testing-framework/assets/stack/scripts/build_test_image.sh"
}
run_examples::maybe_restore_host_after_image() {
if [ "${NEED_HOST_RESTORE_AFTER_IMAGE}" != "1" ]; then
return 0
fi
echo "==> Restoring host bundle for runner (${HOST_TAR})"
if [ ! -f "${HOST_TAR}" ]; then
run_examples::ensure_binaries_tar host "${HOST_TAR}"
fi
run_examples::restore_binaries_from_tar "${HOST_TAR}" || common::die "Failed to restore host bundle from ${HOST_TAR}"
}
run_examples::validate_restored_bundle() {
HOST_BUNDLE_PATH="${HOST_KZG_DIR}"
KZG_HOST_PATH="${HOST_BUNDLE_PATH}/${KZG_FILE}"
if [ ! -x "${HOST_BUNDLE_PATH}/zksign/witness_generator" ]; then
common::die "Missing zksign/witness_generator in restored bundle; ensure the tarball contains host-compatible circuits."
fi
if [ ! -f "${KZG_HOST_PATH}" ]; then
common::die "KZG params missing at ${KZG_HOST_PATH}; ensure the tarball contains circuits."
fi
if [ "${MODE}" = "host" ] && ! { [ -n "${NOMOS_NODE_BIN:-}" ] && [ -x "${NOMOS_NODE_BIN:-}" ] && [ -n "${NOMOS_EXECUTOR_BIN:-}" ] && [ -x "${NOMOS_EXECUTOR_BIN:-}" ]; }; then
local tar_node tar_exec
tar_node="${RESTORED_BIN_DIR:-${ROOT_DIR}/testing-framework/assets/stack/bin}/nomos-node"
tar_exec="${RESTORED_BIN_DIR:-${ROOT_DIR}/testing-framework/assets/stack/bin}/nomos-executor"
[ -x "${tar_node}" ] && [ -x "${tar_exec}" ] || common::die \
"Restored tarball missing host executables; provide a host-compatible binaries tarball."
run_examples::host_bin_matches_arch "${tar_node}" && run_examples::host_bin_matches_arch "${tar_exec}" || common::die \
"Restored executables do not match host architecture; provide a host-compatible binaries tarball."
echo "==> Using restored host binaries from tarball"
NOMOS_NODE_BIN="${tar_node}"
NOMOS_EXECUTOR_BIN="${tar_exec}"
export NOMOS_NODE_BIN NOMOS_EXECUTOR_BIN
fi
}
run_examples::kzg_path_for_mode() {
if [ "${MODE}" = "compose" ] || [ "${MODE}" = "k8s" ]; then
if [ "${MODE}" = "k8s" ] && [ "${NOMOS_KZG_MODE:-hostPath}" = "inImage" ]; then
echo "${NOMOS_KZG_IN_IMAGE_PARAMS_PATH:-/opt/nomos/kzg-params/kzgrs_test_params}"
else
echo "${KZG_CONTAINER_PATH}"
fi
else
echo "${KZG_HOST_PATH}"
fi
}
run_examples::ensure_compose_circuits_platform_default() {
if [ "${MODE}" != "compose" ] || [ -n "${COMPOSE_CIRCUITS_PLATFORM:-}" ]; then
return 0
fi
local arch
arch="$(uname -m)"
case "${arch}" in
x86_64) COMPOSE_CIRCUITS_PLATFORM="linux-x86_64" ;;
arm64|aarch64) COMPOSE_CIRCUITS_PLATFORM="linux-aarch64" ;;
*) COMPOSE_CIRCUITS_PLATFORM="linux-x86_64" ;;
esac
export COMPOSE_CIRCUITS_PLATFORM
}
run_examples::run() {
local kzg_path
kzg_path="$(run_examples::kzg_path_for_mode)"
export NOMOS_DEMO_RUN_SECS="${RUN_SECS}"
export NOMOS_DEMO_VALIDATORS="${DEMO_VALIDATORS}"
export NOMOS_DEMO_EXECUTORS="${DEMO_EXECUTORS}"
echo "==> Running ${BIN} for ${RUN_SECS}s (mode=${MODE}, image=${IMAGE})"
cd "${ROOT_DIR}"
POL_PROOF_DEV_MODE=true \
TESTNET_PRINT_ENDPOINTS=1 \
NOMOS_TESTNET_IMAGE="${IMAGE}" \
NOMOS_CIRCUITS="${HOST_BUNDLE_PATH}" \
NOMOS_KZGRS_PARAMS_PATH="${kzg_path}" \
NOMOS_NODE_BIN="${NOMOS_NODE_BIN:-}" \
NOMOS_EXECUTOR_BIN="${NOMOS_EXECUTOR_BIN:-}" \
COMPOSE_CIRCUITS_PLATFORM="${COMPOSE_CIRCUITS_PLATFORM:-}" \
cargo run -p runner-examples --bin "${BIN}"
}
run_examples::main() {
run_examples::load_env
run_examples::parse_args "$@"
run_examples::select_bin
run_examples::select_image
run_examples::prepare_bundles
echo "==> Using restored circuits/binaries bundle"
SETUP_OUT="$(common::tmpfile nomos-setup-output.XXXXXX)"
cleanup() { rm -f "${SETUP_OUT}" 2>/dev/null || true; }
trap cleanup EXIT
run_examples::maybe_rebuild_image
run_examples::maybe_restore_host_after_image
run_examples::validate_restored_bundle
run_examples::ensure_compose_circuits_platform_default
run_examples::run
}

162
scripts/lib/setup-circuits-stack.sh
View File

@ -1,162 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Intended to be sourced by scripts/setup-circuits-stack.sh
# shellcheck disable=SC1091
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
setup_circuits_stack::usage() {
cat <<'EOF'
Usage: scripts/setup-circuits-stack.sh [VERSION]
Prepares circuits for both the Docker image (Linux/x86_64) and the host (for
witness generators).
Env overrides:
STACK_DIR Where to place the Linux bundle (default: testing-framework/assets/stack/kzgrs_test_params)
HOST_DIR Where to place the host bundle (default: .tmp/nomos-circuits-host)
LINUX_STAGE_DIR Optional staging dir for the Linux bundle (default: .tmp/nomos-circuits-linux)
NOMOS_CIRCUITS_PLATFORM Force host platform (e.g., macos-aarch64)
NOMOS_CIRCUITS_REBUILD_RAPIDSNARK Set to 1 to force rebuild (host bundle only)
EOF
}
setup_circuits_stack::fail_with_usage() {
echo "$1" >&2
setup_circuits_stack::usage
exit 1
}
setup_circuits_stack::realpath_py() {
python3 - "$1" <<'PY'
import os, sys
print(os.path.realpath(sys.argv[1]))
PY
}
setup_circuits_stack::detect_platform() {
local os arch
case "$(uname -s)" in
Linux*) os="linux" ;;
Darwin*) os="macos" ;;
MINGW*|MSYS*|CYGWIN*) os="windows" ;;
*) common::die "Unsupported OS: $(uname -s)" ;;
esac
case "$(uname -m)" in
x86_64) arch="x86_64" ;;
aarch64|arm64) arch="aarch64" ;;
*) common::die "Unsupported arch: $(uname -m)" ;;
esac
echo "${os}-${arch}"
}
setup_circuits_stack::fetch_bundle() {
local platform="$1"
local dest="$2"
local rebuild="${3:-0}"
rm -rf "${dest}"
mkdir -p "${dest}"
NOMOS_CIRCUITS_PLATFORM="${platform}" \
NOMOS_CIRCUITS_REBUILD_RAPIDSNARK="${rebuild}" \
"${ROOT_DIR}/scripts/setup-nomos-circuits.sh" "${VERSION}" "${dest}"
}
setup_circuits_stack::fetch_kzg_params() {
local dest_dir="$1"
local dest_file="${dest_dir}/${KZG_FILE}"
local url="https://raw.githubusercontent.com/logos-co/nomos-node/${NOMOS_NODE_REV}/tests/kzgrs/kzgrs_test_params"
echo "Fetching KZG parameters from ${url}"
curl -fsSL "${url}" -o "${dest_file}"
}
setup_circuits_stack::load_env() {
ROOT_DIR="$(common::repo_root)"
export ROOT_DIR
common::require_file "${ROOT_DIR}/versions.env"
# shellcheck disable=SC1091
. "${ROOT_DIR}/versions.env"
common::maybe_source "${ROOT_DIR}/paths.env"
KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}"
KZG_FILE="${NOMOS_KZG_FILE:-kzgrs_test_params}"
HOST_DIR_REL_DEFAULT="${NOMOS_CIRCUITS_HOST_DIR_REL:-.tmp/nomos-circuits-host}"
LINUX_DIR_REL_DEFAULT="${NOMOS_CIRCUITS_LINUX_DIR_REL:-.tmp/nomos-circuits-linux}"
VERSION="${VERSION:-v0.3.1}"
STACK_DIR="${STACK_DIR:-${ROOT_DIR}/${KZG_DIR_REL}}"
HOST_DIR="${HOST_DIR:-${ROOT_DIR}/${HOST_DIR_REL_DEFAULT}}"
LINUX_STAGE_DIR="${LINUX_STAGE_DIR:-${ROOT_DIR}/${LINUX_DIR_REL_DEFAULT}}"
NOMOS_NODE_REV="${NOMOS_NODE_REV:?Missing NOMOS_NODE_REV in versions.env or env}"
# Force non-interactive installs so repeated runs do not prompt.
export NOMOS_CIRCUITS_NONINTERACTIVE=1
}
setup_circuits_stack::main() {
if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
setup_circuits_stack::usage
exit 0
fi
setup_circuits_stack::load_env
if [ -n "${1:-}" ]; then
VERSION="$1"
fi
echo "Preparing circuits (version ${VERSION})"
echo "Workspace: ${ROOT_DIR}"
local linux_platform="linux-x86_64"
echo "Installing Linux bundle for Docker image into ${STACK_DIR}"
local stage_real stack_real
stage_real="$(setup_circuits_stack::realpath_py "${LINUX_STAGE_DIR}")"
stack_real="$(setup_circuits_stack::realpath_py "${STACK_DIR}")"
if [ "${stage_real}" = "${stack_real}" ]; then
rm -rf "${STACK_DIR}"
setup_circuits_stack::fetch_bundle "${linux_platform}" "${STACK_DIR}" 0
setup_circuits_stack::fetch_kzg_params "${STACK_DIR}"
else
rm -rf "${LINUX_STAGE_DIR}"
mkdir -p "${LINUX_STAGE_DIR}"
setup_circuits_stack::fetch_bundle "${linux_platform}" "${LINUX_STAGE_DIR}" 0
rm -rf "${STACK_DIR}"
mkdir -p "${STACK_DIR}"
cp -R "${LINUX_STAGE_DIR}/." "${STACK_DIR}/"
setup_circuits_stack::fetch_kzg_params "${STACK_DIR}"
fi
echo "Linux bundle ready at ${STACK_DIR}"
local host_platform
host_platform="${NOMOS_CIRCUITS_PLATFORM:-$(setup_circuits_stack::detect_platform)}"
if [[ "${host_platform}" == "${linux_platform}" ]]; then
echo "Host platform ${host_platform} matches Linux bundle; host can reuse ${STACK_DIR}"
echo "Export if you want to be explicit:"
echo " export NOMOS_CIRCUITS=\"${STACK_DIR}\""
else
echo "Host platform detected: ${host_platform}; installing host-native bundle into ${HOST_DIR}"
setup_circuits_stack::fetch_bundle "${host_platform}" "${HOST_DIR}" "${NOMOS_CIRCUITS_REBUILD_RAPIDSNARK:-0}"
setup_circuits_stack::fetch_kzg_params "${HOST_DIR}"
echo "Host bundle ready at ${HOST_DIR}"
echo
echo "Set for host runs:"
echo " export NOMOS_CIRCUITS=\"${HOST_DIR}\""
fi
cat <<'EOF'
Done.
- For Docker/compose: rebuild the image to bake the Linux bundle:
testing-framework/assets/stack/scripts/build_test_image.sh
- For host runs (e.g., compose_runner): ensure NOMOS_CIRCUITS points to the host bundle above.
EOF
}

254
scripts/lib/setup-nomos-circuits.sh
View File

@ -1,254 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
setup_nomos_circuits::usage() {
cat <<'EOF'
Usage: scripts/setup-nomos-circuits.sh [VERSION] [INSTALL_DIR]
Arguments:
VERSION Optional. Version to install (default: v0.3.1)
INSTALL_DIR Optional. Installation directory (default: $HOME/.nomos-circuits)
Environment:
NOMOS_CIRCUITS_PLATFORM Override platform (e.g. linux-x86_64, macos-aarch64)
NOMOS_CIRCUITS_NONINTERACTIVE Set to 1 to auto-overwrite without prompt
NOMOS_CIRCUITS_REBUILD_RAPIDSNARK Set to 1 to force rapidsnark rebuild
GITHUB_TOKEN Optional token for GitHub releases download
EOF
}
setup_nomos_circuits::init_vars() {
VERSION="${1:-v0.3.1}"
DEFAULT_INSTALL_DIR="${HOME}/.nomos-circuits"
INSTALL_DIR="${2:-${DEFAULT_INSTALL_DIR}}"
REPO="logos-co/nomos-circuits"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
NONINTERACTIVE="${NOMOS_CIRCUITS_NONINTERACTIVE:-0}"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
}
setup_nomos_circuits::print_info() { echo -e "${BLUE}ℹ${NC} $1"; }
setup_nomos_circuits::print_success() { echo -e "${GREEN}✓${NC} $1"; }
setup_nomos_circuits::print_warning() { echo -e "${YELLOW}⚠${NC} $1"; }
setup_nomos_circuits::print_error() { echo -e "${RED}✗${NC} $1"; }
setup_nomos_circuits::detect_platform() {
local os="" arch=""
case "$(uname -s)" in
Linux*) os="linux" ;;
Darwin*) os="macos" ;;
MINGW*|MSYS*|CYGWIN*) os="windows" ;;
*) setup_nomos_circuits::print_error "Unsupported operating system: $(uname -s)"; exit 1 ;;
esac
case "$(uname -m)" in
x86_64) arch="x86_64" ;;
aarch64|arm64) arch="aarch64" ;;
*) setup_nomos_circuits::print_error "Unsupported architecture: $(uname -m)"; exit 1 ;;
esac
echo "${os}-${arch}"
}
setup_nomos_circuits::check_existing_installation() {
if [ -d "${INSTALL_DIR}" ]; then
setup_nomos_circuits::print_warning "Installation directory already exists: ${INSTALL_DIR}"
if [ -f "${INSTALL_DIR}/VERSION" ]; then
local current_version
current_version="$(cat "${INSTALL_DIR}/VERSION")"
setup_nomos_circuits::print_info "Currently installed version: ${current_version}"
fi
if [ "${NONINTERACTIVE}" = "1" ] || [ ! -t 0 ]; then
setup_nomos_circuits::print_info "Non-interactive environment detected, automatically overwriting..."
else
echo
read -p "Do you want to overwrite it? (y/N): " -n 1 -r
echo
if [[ ! ${REPLY} =~ ^[Yy]$ ]]; then
setup_nomos_circuits::print_info "Installation cancelled."
exit 0
fi
fi
setup_nomos_circuits::print_info "Removing existing installation..."
rm -rf "${INSTALL_DIR}"
fi
}
setup_nomos_circuits::download_release() {
local platform="$1"
local artifact="nomos-circuits-${VERSION}-${platform}.tar.gz"
local url="https://github.com/${REPO}/releases/download/${VERSION}/${artifact}"
local temp_dir
temp_dir="$(mktemp -d)"
setup_nomos_circuits::print_info "Downloading nomos-circuits ${VERSION} for ${platform}..."
setup_nomos_circuits::print_info "URL: ${url}"
local curl_cmd="curl -fL --retry 5 --retry-delay 2 --retry-all-errors"
if [ -n "${GITHUB_TOKEN:-}" ]; then
curl_cmd="${curl_cmd} --header 'authorization: Bearer ${GITHUB_TOKEN}'"
fi
curl_cmd="${curl_cmd} -o ${temp_dir}/${artifact} ${url}"
if ! eval "${curl_cmd}"; then
setup_nomos_circuits::print_error "Failed to download release artifact"
setup_nomos_circuits::print_error "Please check that version ${VERSION} exists for platform ${platform}"
setup_nomos_circuits::print_error "Available releases: https://github.com/${REPO}/releases"
rm -rf "${temp_dir}"
return 1
fi
setup_nomos_circuits::print_success "Download complete"
if ! tar -tzf "${temp_dir}/${artifact}" >/dev/null 2>&1; then
setup_nomos_circuits::print_error "Downloaded archive is not a valid tar.gz: ${temp_dir}/${artifact}"
rm -rf "${temp_dir}"
return 1
fi
setup_nomos_circuits::print_info "Extracting to ${INSTALL_DIR}..."
mkdir -p "${INSTALL_DIR}"
if ! tar -xzf "${temp_dir}/${artifact}" -C "${INSTALL_DIR}" --strip-components=1; then
setup_nomos_circuits::print_error "Failed to extract archive"
rm -rf "${temp_dir}"
return 1
fi
rm -rf "${temp_dir}"
setup_nomos_circuits::print_success "Extraction complete"
}
setup_nomos_circuits::handle_macos_quarantine() {
setup_nomos_circuits::print_info "macOS detected: Removing quarantine attributes from executables..."
if find "${INSTALL_DIR}" -type f -perm -111 -exec xattr -d com.apple.quarantine {} \; 2>/dev/null; then
setup_nomos_circuits::print_success "Quarantine attributes removed"
else
setup_nomos_circuits::print_warning "Could not remove quarantine attributes (they may not exist)"
fi
}
setup_nomos_circuits::print_circuits() {
setup_nomos_circuits::print_info "The following circuits are available:"
local dir circuit_name
for dir in "${INSTALL_DIR}"/*/; do
if [ -d "${dir}" ]; then
circuit_name="$(basename "${dir}")"
if [ -f "${dir}/witness_generator" ]; then
echo " • ${circuit_name}"
fi
fi
done
}
setup_nomos_circuits::resolve_platform() {
local platform_override="${NOMOS_CIRCUITS_PLATFORM:-}"
if [ -n "${platform_override}" ]; then
PLATFORM="${platform_override}"
setup_nomos_circuits::print_info "Using overridden platform: ${PLATFORM}"
else
PLATFORM="$(setup_nomos_circuits::detect_platform)"
setup_nomos_circuits::print_info "Detected platform: ${PLATFORM}"
fi
}
setup_nomos_circuits::download_with_fallbacks() {
# Outputs:
# PLATFORM - platform used for the downloaded bundle
# REBUILD_REQUIRED - 0/1
REBUILD_REQUIRED="${NOMOS_CIRCUITS_REBUILD_RAPIDSNARK:-0}"
if setup_nomos_circuits::download_release "${PLATFORM}"; then
return 0
fi
if [[ "${PLATFORM}" == "linux-aarch64" ]]; then
setup_nomos_circuits::print_warning "Falling back to linux-x86_64 circuits bundle; will rebuild prover for aarch64."
rm -rf "${INSTALL_DIR}"
PLATFORM="linux-x86_64"
setup_nomos_circuits::download_release "${PLATFORM}" || return 1
REBUILD_REQUIRED=1
return 0
fi
if [[ "${PLATFORM}" == "macos-x86_64" ]]; then
setup_nomos_circuits::print_warning "No macOS x86_64 bundle; falling back to macOS aarch64 circuits bundle and rebuilding prover."
rm -rf "${INSTALL_DIR}"
PLATFORM="macos-aarch64"
if ! setup_nomos_circuits::download_release "${PLATFORM}"; then
setup_nomos_circuits::print_warning "macOS aarch64 bundle unavailable; trying linux-x86_64 bundle and rebuilding prover."
rm -rf "${INSTALL_DIR}"
PLATFORM="linux-x86_64"
setup_nomos_circuits::download_release "${PLATFORM}" || return 1
fi
REBUILD_REQUIRED=1
return 0
fi
return 1
}
setup_nomos_circuits::maybe_handle_quarantine() {
if [[ "${PLATFORM}" == macos-* ]]; then
echo
setup_nomos_circuits::handle_macos_quarantine
fi
}
setup_nomos_circuits::maybe_rebuild_rapidsnark() {
if [[ "${REBUILD_REQUIRED}" == "1" ]]; then
echo
setup_nomos_circuits::print_info "Rebuilding rapidsnark prover for ${PLATFORM}..."
"${SCRIPT_DIR}/build-rapidsnark.sh" "${INSTALL_DIR}"
else
setup_nomos_circuits::print_info "Skipping rapidsnark rebuild (set NOMOS_CIRCUITS_REBUILD_RAPIDSNARK=1 to force)."
fi
}
setup_nomos_circuits::print_summary() {
echo
setup_nomos_circuits::print_success "Installation complete!"
echo
setup_nomos_circuits::print_info "nomos-circuits ${VERSION} is now installed at: ${INSTALL_DIR}"
setup_nomos_circuits::print_circuits
if [ "${INSTALL_DIR}" != "${DEFAULT_INSTALL_DIR}" ]; then
echo
setup_nomos_circuits::print_info "Since you're using a custom installation directory, set the environment variable:"
setup_nomos_circuits::print_info " export NOMOS_CIRCUITS=${INSTALL_DIR}"
echo
fi
}
setup_nomos_circuits::main() {
if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
setup_nomos_circuits::usage
exit 0
fi
setup_nomos_circuits::init_vars "${1:-}" "${2:-}"
setup_nomos_circuits::print_info "Setting up nomos-circuits ${VERSION}"
setup_nomos_circuits::print_info "Installation directory: ${INSTALL_DIR}"
echo
setup_nomos_circuits::resolve_platform
setup_nomos_circuits::check_existing_installation
setup_nomos_circuits::download_with_fallbacks || exit 1
setup_nomos_circuits::maybe_handle_quarantine
setup_nomos_circuits::maybe_rebuild_rapidsnark
setup_nomos_circuits::print_summary
}

225
scripts/lib/update-nomos-rev.sh
View File

@ -1,225 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Intended to be sourced by scripts/update-nomos-rev.sh
# shellcheck disable=SC1091
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
update_nomos_rev::usage() {
cat <<'EOF'
Usage:
scripts/update-nomos-rev.sh --rev <git_rev>
scripts/update-nomos-rev.sh --path <local_dir>
scripts/update-nomos-rev.sh --unskip-worktree
Notes:
--rev sets NOMOS_NODE_REV and updates Cargo.toml revs
--path sets NOMOS_NODE_PATH (clears NOMOS_NODE_REV) and patches Cargo.toml to use a local nomos-node checkout
--unskip-worktree clears any skip-worktree flag for Cargo.toml
Only one may be used at a time.
EOF
}
update_nomos_rev::fail_with_usage() {
echo "$1" >&2
update_nomos_rev::usage
exit 1
}
update_nomos_rev::maybe_unskip_worktree() {
local file="$1"
if git -C "${ROOT_DIR}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
git -C "${ROOT_DIR}" update-index --no-skip-worktree "${file}" >/dev/null 2>&1 || true
fi
}
update_nomos_rev::maybe_skip_worktree() {
local file="$1"
if git -C "${ROOT_DIR}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
git -C "${ROOT_DIR}" update-index --skip-worktree "${file}" >/dev/null 2>&1 || true
fi
}
update_nomos_rev::ensure_env_key() {
local key="$1" default_value="$2"
if ! grep -Eq "^#?[[:space:]]*${key}=" "${ROOT_DIR}/versions.env"; then
echo "${default_value}" >> "${ROOT_DIR}/versions.env"
fi
}
update_nomos_rev::parse_args() {
REV=""
LOCAL_PATH=""
UNSKIP_WORKTREE=0
while [ "$#" -gt 0 ]; do
case "$1" in
--rev) REV="${2:-}"; shift 2 ;;
--path) LOCAL_PATH="${2:-}"; shift 2 ;;
--unskip-worktree) UNSKIP_WORKTREE=1; shift ;;
-h|--help) update_nomos_rev::usage; exit 0 ;;
*) update_nomos_rev::fail_with_usage "Unknown arg: $1" ;;
esac
done
if [ "${UNSKIP_WORKTREE}" -eq 1 ] && { [ -n "${REV}" ] || [ -n "${LOCAL_PATH}" ]; }; then
update_nomos_rev::fail_with_usage "Use --unskip-worktree alone."
fi
if [ -n "${REV}" ] && [ -n "${LOCAL_PATH}" ]; then
update_nomos_rev::fail_with_usage "Use either --rev or --path, not both"
fi
if [ -z "${REV}" ] && [ -z "${LOCAL_PATH}" ] && [ "${UNSKIP_WORKTREE}" -eq 0 ]; then
update_nomos_rev::usage
exit 1
fi
}
update_nomos_rev::load_env() {
ROOT_DIR="$(common::repo_root)"
export ROOT_DIR
common::require_file "${ROOT_DIR}/versions.env"
}
update_nomos_rev::update_to_rev() {
local rev="$1"
echo "Updating nomos-node rev to ${rev}"
sed -i.bak -E \
-e "s/^#?[[:space:]]*NOMOS_NODE_REV=.*/NOMOS_NODE_REV=${rev}/" \
-e "s/^#?[[:space:]]*NOMOS_NODE_PATH=.*/# NOMOS_NODE_PATH=/" \
"${ROOT_DIR}/versions.env"
rm -f "${ROOT_DIR}/versions.env.bak"
python3 - "${ROOT_DIR}" "${rev}" <<'PY'
import pathlib, re, sys
root = pathlib.Path(sys.argv[1])
rev = sys.argv[2]
cargo_toml = root / "Cargo.toml"
txt = cargo_toml.read_text()
txt = txt.replace("\\n", "\n")
txt = re.sub(
r'(?ms)^\[patch\."https://github\.com/logos-co/nomos-node"\].*?(?=^\[|\Z)',
"",
txt,
)
txt = re.sub(
r'(git = "https://github\.com/logos-co/nomos-node\.git", rev = ")[^"]+(")',
r"\g<1>" + rev + r"\2",
txt,
)
cargo_toml.write_text(txt.rstrip() + "\n")
PY
update_nomos_rev::maybe_unskip_worktree "Cargo.toml"
}
update_nomos_rev::update_to_path() {
local node_path="$1"
echo "Pointing to local nomos-node at ${node_path}"
[ -d "${node_path}" ] || common::die "path does not exist: ${node_path}"
local current_rev escaped_path
current_rev="$(grep -E '^[#[:space:]]*NOMOS_NODE_REV=' "${ROOT_DIR}/versions.env" | head -n1 | sed -E 's/^#?[[:space:]]*NOMOS_NODE_REV=//')"
escaped_path="${node_path//\//\\/}"
sed -i.bak -E \
-e "s/^#?[[:space:]]*NOMOS_NODE_PATH=.*/NOMOS_NODE_PATH=${escaped_path}/" \
-e "s/^#?[[:space:]]*NOMOS_NODE_REV=.*/# NOMOS_NODE_REV=${current_rev}/" \
"${ROOT_DIR}/versions.env"
rm -f "${ROOT_DIR}/versions.env.bak"
local python_bin="${PYTHON_BIN:-python3}"
command -v "${python_bin}" >/dev/null 2>&1 || common::die "python3 is required to patch Cargo.toml for local paths"
"${python_bin}" - "${ROOT_DIR}" "${node_path}" <<'PY'
import json
import pathlib
import re
import subprocess
import sys
root = pathlib.Path(sys.argv[1])
node_path = pathlib.Path(sys.argv[2])
targets = [
"broadcast-service", "chain-leader", "chain-network", "chain-service",
"common-http-client", "cryptarchia-engine", "cryptarchia-sync",
"executor-http-client", "groth16", "key-management-system-service",
"kzgrs", "kzgrs-backend", "nomos-api", "nomos-blend-message",
"nomos-blend-service", "nomos-core", "nomos-da-dispersal",
"nomos-da-network-core", "nomos-da-network-service", "nomos-da-sampling",
"nomos-da-verifier", "nomos-executor", "nomos-http-api-common",
"nomos-ledger", "nomos-libp2p", "nomos-network", "nomos-node",
"nomos-sdp", "nomos-time", "nomos-tracing", "nomos-tracing-service",
"nomos-utils", "nomos-wallet", "poc", "pol", "subnetworks-assignations",
"tests", "tx-service", "wallet", "zksign",
]
try:
meta = subprocess.check_output(
["cargo", "metadata", "--format-version", "1", "--no-deps"],
cwd=node_path,
)
except subprocess.CalledProcessError as exc:
sys.stderr.write(f"Failed to run cargo metadata in {node_path}: {exc}\n")
sys.exit(1)
data = json.loads(meta)
paths = {}
for pkg in data.get("packages", []):
paths[pkg["name"]] = str(pathlib.Path(pkg["manifest_path"]).parent)
patch_lines = ['[patch."https://github.com/logos-co/nomos-node"]']
missing = []
for name in targets:
if name in paths:
patch_lines.append(f'{name} = {{ path = "{paths[name]}" }}')
else:
missing.append(name)
cargo_toml = root / "Cargo.toml"
txt = cargo_toml.read_text()
txt = txt.replace("\\n", "\n")
txt = re.sub(
r'(?ms)^\[patch\."https://github\.com/logos-co/nomos-node"\].*?(?=^\[|\Z)',
"",
txt,
)
txt = txt.rstrip() + "\n\n" + "\n".join(patch_lines) + "\n"
cargo_toml.write_text(txt)
if missing:
sys.stderr.write(
"Warning: missing crates in local nomos-node checkout: "
+ ", ".join(missing)
+ "\n"
)
PY
update_nomos_rev::maybe_skip_worktree "Cargo.toml"
echo "Local nomos-node patch applied; Cargo.toml marked skip-worktree (run --unskip-worktree to clear)."
}
update_nomos_rev::main() {
update_nomos_rev::load_env
update_nomos_rev::parse_args "$@"
update_nomos_rev::ensure_env_key "NOMOS_NODE_REV" "# NOMOS_NODE_REV="
update_nomos_rev::ensure_env_key "NOMOS_NODE_PATH" "# NOMOS_NODE_PATH="
if [ "${UNSKIP_WORKTREE}" -eq 1 ]; then
update_nomos_rev::maybe_unskip_worktree "Cargo.toml"
echo "Cleared skip-worktree on Cargo.toml (if it was set)."
exit 0
fi
if [ -n "${REV}" ]; then
update_nomos_rev::update_to_rev "${REV}"
else
update_nomos_rev::update_to_path "${LOCAL_PATH}"
fi
echo "Done. Consider updating Cargo.lock if needed (cargo fetch)."
}

28
scripts/push-ecr-test.sh
View File

@ -3,6 +3,14 @@ set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
readonly DEFAULT_TAG="test"
readonly DEFAULT_ECR_IMAGE_REPO="public.ecr.aws/r4s5t9y4/logos/logos-blockchain"
readonly DEFAULT_AWS_REGION="us-east-1"
readonly DEFAULT_LOCAL_IMAGE_REPO="logos-blockchain-testing"
readonly DEFAULT_DOCKER_PLATFORM="linux/amd64"
readonly DEFAULT_CIRCUITS_PLATFORM="linux-x86_64"
readonly PUBLIC_ECR_HOST="public.ecr.aws"
# Publishes the testnet image to ECR Public by default.
#
# Env overrides:
@ -13,25 +21,25 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# Legacy (private ECR) overrides:
# AWS_ACCOUNT_ID - if set, uses private ECR login/push unless ECR_IMAGE_REPO points at public.ecr.aws
TAG="${TAG:-test}"
ECR_IMAGE_REPO="${ECR_IMAGE_REPO:-public.ecr.aws/r4s5t9y4/logos/logos-blockchain}"
AWS_REGION="${AWS_REGION:-us-east-1}"
TAG="${TAG:-${DEFAULT_TAG}}"
ECR_IMAGE_REPO="${ECR_IMAGE_REPO:-${DEFAULT_ECR_IMAGE_REPO}}"
AWS_REGION="${AWS_REGION:-${DEFAULT_AWS_REGION}}"
LOCAL_IMAGE="${LOCAL_IMAGE:-logos-blockchain-testing:${TAG}}"
LOCAL_IMAGE="${LOCAL_IMAGE:-${DEFAULT_LOCAL_IMAGE_REPO}:${TAG}}"
REMOTE_IMAGE="${ECR_IMAGE_REPO}:${TAG}"
export DOCKER_DEFAULT_PLATFORM="linux/amd64"
export CIRCUITS_PLATFORM="${CIRCUITS_PLATFORM:-linux-x86_64}"
export DOCKER_DEFAULT_PLATFORM="${DEFAULT_DOCKER_PLATFORM}"
export CIRCUITS_PLATFORM="${CIRCUITS_PLATFORM:-${DEFAULT_CIRCUITS_PLATFORM}}"
export IMAGE_TAG="${REMOTE_IMAGE}"
"${ROOT_DIR}/testing-framework/assets/stack/scripts/build_test_image.sh"
"${ROOT_DIR}/scripts/build_test_image.sh"
if [[ "${ECR_IMAGE_REPO}" == public.ecr.aws/* ]]; then
if [[ "${ECR_IMAGE_REPO}" == ${PUBLIC_ECR_HOST}/* ]]; then
aws ecr-public get-login-password --region "${AWS_REGION}" \
| docker login --username AWS --password-stdin "public.ecr.aws"
| docker login --username AWS --password-stdin "${PUBLIC_ECR_HOST}"
else
if [ -z "${AWS_ACCOUNT_ID:-}" ]; then
echo "ERROR: AWS_ACCOUNT_ID must be set for private ECR pushes (or set ECR_IMAGE_REPO=public.ecr.aws/...)" >&2
echo "ERROR: AWS_ACCOUNT_ID must be set for private ECR pushes (or set ECR_IMAGE_REPO=${PUBLIC_ECR_HOST}/...)" >&2
exit 1
fi
ECR_REGISTRY="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com"

63
scripts/query-prom-metrics.sh
View File

@ -1,63 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Query all metric names from a Prometheus endpoint and print one sample value
# per metric (if present).
#
# Usage:
# PROM_URL=http://127.0.0.1:9090 ./scripts/query-prom-metrics.sh
# ./scripts/query-prom-metrics.sh http://127.0.0.1:59804
PROM_URL="${1:-${PROM_URL:-http://127.0.0.1:9090}}"
require() { command -v "$1" >/dev/null 2>&1 || { echo "$1 is required but not installed" >&2; exit 1; }; }
require jq
require python3
echo "Querying Prometheus at ${PROM_URL}"
python3 - <<'PY'
import os, sys, json, urllib.parse, urllib.request
prom = os.environ.get("PROM_URL")
if not prom:
sys.exit("PROM_URL is not set")
def fetch(path, params=None):
url = prom + path
if params:
url += "?" + urllib.parse.urlencode(params)
with urllib.request.urlopen(url, timeout=10) as resp:
return json.load(resp)
names = fetch("/api/v1/label/__name__/values").get("data", [])
if not names:
sys.exit("No metrics found or failed to reach Prometheus")
jobs = fetch("/api/v1/label/job/values").get("data", [])
if jobs:
print("Jobs seen:", ", ".join(sorted(jobs)))
else:
print("Jobs seen: <none>")
by_job = {j: [] for j in jobs} if jobs else {}
for name in sorted(names):
data = fetch("/api/v1/query", {"query": name}).get("data", {}).get("result", [])
for series in data:
labels = series.get("metric", {})
value = series.get("value", ["", "N/A"])[1]
job = labels.get("job", "<no-job>")
by_job.setdefault(job, []).append((name, value))
if not by_job:
sys.exit("No metric samples returned")
for job in sorted(by_job):
print(f"{job}:")
samples = by_job[job]
if not samples:
print(" <no samples>")
else:
for name, value in sorted(samples):
print(f" {name}: {value}")
PY

558
scripts/run-examples.sh
View File

@ -1,15 +1,561 @@
#!/usr/bin/env bash
set -euo pipefail
# Thin wrapper; the actual implementation lives in scripts/lib/run-examples.sh
# so it can be shared/tested more easily.
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# shellcheck disable=SC1091
. "${ROOT_DIR}/scripts/lib/run-examples.sh"
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
run_examples::main "$@"
readonly DEFAULT_KZG_DIR_REL="testing-framework/assets/stack/kzgrs_test_params"
readonly DEFAULT_KZG_FILE="kzgrs_test_params"
readonly DEFAULT_KZG_CONTAINER_PATH="/kzgrs_test_params/kzgrs_test_params"
readonly DEFAULT_KZG_IN_IMAGE_PARAMS_PATH="/opt/nomos/kzg-params/kzgrs_test_params"
readonly DEFAULT_LOCAL_IMAGE="logos-blockchain-testing:local"
readonly DEFAULT_PUBLIC_ECR_REGISTRY="public.ecr.aws/r4s5t9y4"
readonly DEFAULT_PUBLIC_ECR_REPO="logos/logos-blockchain"
readonly DEFAULT_PRIVATE_ECR_REPO="logos-blockchain-testing"
readonly DEFAULT_ECR_TAG="test"
readonly DEFAULT_PRIVATE_AWS_REGION="ap-southeast-2"
readonly DEFAULT_PULL_POLICY_LOCAL="IfNotPresent"
readonly DEFAULT_PULL_POLICY_ECR="Always"
readonly DOCKER_DESKTOP_CONTEXT="docker-desktop"
run_examples::usage() {
cat <<EOF
Usage: scripts/run-examples.sh [options] [compose|host|k8s]
Modes:
compose Run examples/src/bin/compose_runner.rs (default)
host Run examples/src/bin/local_runner.rs
k8s Run examples/src/bin/k8s_runner.rs
Options:
-t, --run-seconds N Duration to run the demo (required)
-v, --validators N Number of validators (required)
-e, --executors N Number of executors (required)
--bundle PATH Convenience alias for setting NOMOS_BINARIES_TAR=PATH
--external-prometheus URL (k8s) Reuse existing Prometheus; skips Helm Prometheus
--local Use a local Docker image tag (default for docker-desktop k8s)
--ecr Use an ECR image reference (default for non-docker-desktop k8s)
--no-image-build Skip rebuilding the compose/k8s image (sets NOMOS_SKIP_IMAGE_BUILD=1)
Environment:
VERSION Circuits version (default from versions.env)
NOMOS_TESTNET_IMAGE Image reference (overridden by --local/--ecr selection)
ECR_IMAGE Full image reference for --ecr (overrides ECR_REGISTRY/ECR_REPO/TAG)
ECR_REGISTRY Registry hostname for --ecr (default ${DEFAULT_PUBLIC_ECR_REGISTRY})
ECR_REPO Repository path for --ecr (default ${DEFAULT_PUBLIC_ECR_REPO})
TAG Tag for --ecr (default ${DEFAULT_ECR_TAG})
NOMOS_TESTNET_IMAGE_PULL_POLICY K8s imagePullPolicy (default ${DEFAULT_PULL_POLICY_LOCAL}; set to ${DEFAULT_PULL_POLICY_ECR} for --ecr)
NOMOS_BINARIES_TAR Path to prebuilt binaries/circuits tarball (default .tmp/nomos-binaries-<platform>-<version>.tar.gz)
NOMOS_SKIP_IMAGE_BUILD Set to 1 to skip rebuilding the compose/k8s image
K8S_RUNNER_EXTERNAL_PROMETHEUS_URL Reuse existing Prometheus; skips Helm Prometheus
EOF
}
run_examples::fail_with_usage() {
echo "$1" >&2
run_examples::usage
exit 1
}
run_examples::load_env() {
ROOT_DIR="$(common::repo_root)"
export ROOT_DIR
common::require_file "${ROOT_DIR}/versions.env"
# shellcheck disable=SC1091
. "${ROOT_DIR}/versions.env"
common::maybe_source "${ROOT_DIR}/paths.env"
DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}"
VERSION="${VERSION:-${DEFAULT_VERSION}}"
KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-${DEFAULT_KZG_DIR_REL}}"
KZG_FILE="${NOMOS_KZG_FILE:-${DEFAULT_KZG_FILE}}"
KZG_CONTAINER_PATH="${NOMOS_KZG_CONTAINER_PATH:-${DEFAULT_KZG_CONTAINER_PATH}}"
HOST_KZG_DIR="${ROOT_DIR}/${KZG_DIR_REL}"
HOST_KZG_FILE="${HOST_KZG_DIR}/${KZG_FILE}"
}
run_examples::select_bin() {
case "${MODE}" in
compose) BIN="compose_runner" ;;
host) BIN="local_runner" ;;
k8s) BIN="k8s_runner" ;;
*) common::die "Unknown mode '${MODE}' (use compose|host|k8s)" ;;
esac
}
run_examples::parse_args() {
MODE="compose"
RUN_SECS_RAW=""
DEMO_VALIDATORS=""
DEMO_EXECUTORS=""
IMAGE_SELECTION_MODE="auto"
EXTERNAL_PROMETHEUS_URL=""
RUN_SECS_RAW_SPECIFIED=""
while [ "$#" -gt 0 ]; do
case "$1" in
-h|--help)
run_examples::usage
exit 0
;;
-t|--run-seconds)
RUN_SECS_RAW_SPECIFIED=1
RUN_SECS_RAW="${2:-}"
shift 2
;;
--run-seconds=*)
RUN_SECS_RAW_SPECIFIED=1
RUN_SECS_RAW="${1#*=}"
shift
;;
-v|--validators)
DEMO_VALIDATORS="${2:-}"
shift 2
;;
--validators=*)
DEMO_VALIDATORS="${1#*=}"
shift
;;
-e|--executors)
DEMO_EXECUTORS="${2:-}"
shift 2
;;
--executors=*)
DEMO_EXECUTORS="${1#*=}"
shift
;;
--bundle)
NOMOS_BINARIES_TAR="${2:-}"
export NOMOS_BINARIES_TAR
shift 2
;;
--bundle=*)
NOMOS_BINARIES_TAR="${1#*=}"
export NOMOS_BINARIES_TAR
shift
;;
--external-prometheus)
EXTERNAL_PROMETHEUS_URL="${2:-}"
shift 2
;;
--external-prometheus=*)
EXTERNAL_PROMETHEUS_URL="${1#*=}"
shift
;;
--local)
if [ "${IMAGE_SELECTION_MODE}" = "ecr" ]; then
run_examples::fail_with_usage "--local and --ecr are mutually exclusive"
fi
IMAGE_SELECTION_MODE="local"
shift
;;
--ecr)
if [ "${IMAGE_SELECTION_MODE}" = "local" ]; then
run_examples::fail_with_usage "--local and --ecr are mutually exclusive"
fi
IMAGE_SELECTION_MODE="ecr"
shift
;;
--no-image-build)
NOMOS_SKIP_IMAGE_BUILD=1
export NOMOS_SKIP_IMAGE_BUILD
shift
;;
compose|host|k8s)
MODE="$1"
shift
;;
*)
# Positional run-seconds fallback for legacy usage.
if [ -z "${RUN_SECS_RAW_SPECIFIED}" ] && common::is_uint "$1"; then
RUN_SECS_RAW="$1"
shift
else
run_examples::fail_with_usage "Unknown argument: $1"
fi
;;
esac
done
if [ -n "${NOMOS_BINARIES_TAR:-}" ] && [ ! -f "${NOMOS_BINARIES_TAR}" ]; then
run_examples::fail_with_usage "NOMOS_BINARIES_TAR is set but missing: ${NOMOS_BINARIES_TAR}"
fi
if ! common::is_uint "${RUN_SECS_RAW}" || [ "${RUN_SECS_RAW}" -le 0 ]; then
run_examples::fail_with_usage "run-seconds must be a positive integer (pass -t/--run-seconds)"
fi
RUN_SECS="${RUN_SECS_RAW}"
if [ -z "${DEMO_VALIDATORS}" ] || [ -z "${DEMO_EXECUTORS}" ]; then
run_examples::fail_with_usage "validators and executors must be provided via -v/--validators and -e/--executors"
fi
if ! common::is_uint "${DEMO_VALIDATORS}" ; then
run_examples::fail_with_usage "validators must be a non-negative integer (pass -v/--validators)"
fi
if ! common::is_uint "${DEMO_EXECUTORS}" ; then
run_examples::fail_with_usage "executors must be a non-negative integer (pass -e/--executors)"
fi
if [ -n "${EXTERNAL_PROMETHEUS_URL}" ] && [ "${MODE}" != "k8s" ]; then
echo "Warning: --external-prometheus is only used in k8s mode; ignoring." >&2
EXTERNAL_PROMETHEUS_URL=""
fi
}
run_examples::select_image() {
local selection="${IMAGE_SELECTION_MODE}"
local context=""
if [ "${selection}" = "auto" ]; then
if [ "${MODE}" = "k8s" ] && command -v kubectl >/dev/null 2>&1; then
context="$(kubectl config current-context 2>/dev/null || true)"
if [ "${context}" = "${DOCKER_DESKTOP_CONTEXT}" ]; then
selection="local"
else
selection="ecr"
fi
else
selection="local"
fi
fi
if [ "${selection}" = "local" ]; then
IMAGE="${NOMOS_TESTNET_IMAGE:-${DEFAULT_LOCAL_IMAGE}}"
export NOMOS_TESTNET_IMAGE_PULL_POLICY="${NOMOS_TESTNET_IMAGE_PULL_POLICY:-${DEFAULT_PULL_POLICY_LOCAL}}"
elif [ "${selection}" = "ecr" ]; then
local tag="${TAG:-${DEFAULT_ECR_TAG}}"
if [ -n "${ECR_IMAGE:-}" ]; then
IMAGE="${ECR_IMAGE}"
elif [ -n "${ECR_REGISTRY:-}" ]; then
local registry="${ECR_REGISTRY}"
local repo="${ECR_REPO:-${DEFAULT_PUBLIC_ECR_REPO}}"
IMAGE="${registry}/${repo}:${tag}"
elif [ -n "${AWS_ACCOUNT_ID:-}" ]; then
local aws_region="${AWS_REGION:-${DEFAULT_PRIVATE_AWS_REGION}}"
local aws_account_id="${AWS_ACCOUNT_ID}"
local repo="${ECR_REPO:-${DEFAULT_PRIVATE_ECR_REPO}}"
IMAGE="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com/${repo}:${tag}"
else
local registry="${DEFAULT_PUBLIC_ECR_REGISTRY}"
local repo="${ECR_REPO:-${DEFAULT_PUBLIC_ECR_REPO}}"
IMAGE="${registry}/${repo}:${tag}"
fi
export NOMOS_TESTNET_IMAGE_PULL_POLICY="${NOMOS_TESTNET_IMAGE_PULL_POLICY:-${DEFAULT_PULL_POLICY_ECR}}"
else
run_examples::fail_with_usage "Unknown image selection mode: ${selection}"
fi
export IMAGE_TAG="${IMAGE}"
export NOMOS_TESTNET_IMAGE="${IMAGE}"
if [ "${MODE}" = "k8s" ]; then
if [ "${selection}" = "ecr" ]; then
export NOMOS_KZG_MODE="${NOMOS_KZG_MODE:-inImage}"
else
export NOMOS_KZG_MODE="${NOMOS_KZG_MODE:-hostPath}"
fi
fi
}
run_examples::default_tar_path() {
if [ -n "${NOMOS_BINARIES_TAR:-}" ]; then
echo "${NOMOS_BINARIES_TAR}"
return
fi
case "${MODE}" in
host) echo "${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz" ;;
compose|k8s)
if [ "${NOMOS_SKIP_IMAGE_BUILD:-}" = "1" ]; then
echo "${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz"
else
echo "${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz"
fi
;;
*) echo "${ROOT_DIR}/.tmp/nomos-binaries-${VERSION}.tar.gz" ;;
esac
}
run_examples::bundle_matches_expected() {
local tar_path="$1"
[ -f "${tar_path}" ] || return 1
[ -z "${NOMOS_NODE_REV:-}" ] && return 0
local meta tar_rev tar_head
meta="$(tar -xOzf "${tar_path}" artifacts/nomos-bundle-meta.env 2>/dev/null || true)"
if [ -z "${meta}" ]; then
echo "Bundle meta missing in ${tar_path}; treating as stale and rebuilding." >&2
return 1
fi
tar_rev="$(echo "${meta}" | sed -n 's/^nomos_node_rev=//p' | head -n 1)"
tar_head="$(echo "${meta}" | sed -n 's/^nomos_node_git_head=//p' | head -n 1)"
if [ -n "${tar_rev}" ] && [ "${tar_rev}" != "${NOMOS_NODE_REV}" ]; then
echo "Bundle ${tar_path} is for nomos-node rev ${tar_rev}, expected ${NOMOS_NODE_REV}; rebuilding." >&2
return 1
fi
if [ -n "${tar_head}" ] && [ "${tar_head}" != "${NOMOS_NODE_REV}" ]; then
echo "Bundle ${tar_path} is for nomos-node git head ${tar_head}, expected ${NOMOS_NODE_REV}; rebuilding." >&2
return 1
fi
return 0
}
run_examples::host_bin_matches_arch() {
local bin_path="$1"
[ -x "${bin_path}" ] || return 1
command -v file >/dev/null 2>&1 || return 0
local info expected
info="$(file -b "${bin_path}" 2>/dev/null || true)"
case "$(uname -m)" in
x86_64) expected="x86-64|x86_64" ;;
aarch64|arm64) expected="arm64|aarch64" ;;
*) expected="" ;;
esac
[ -n "${expected}" ] && echo "${info}" | grep -Eqi "${expected}"
}
run_examples::restore_binaries_from_tar() {
local tar_path="${1:-}"
if [ -z "${tar_path}" ]; then
tar_path="$(run_examples::default_tar_path)"
fi
run_examples::bundle_matches_expected "${tar_path}" || return 1
[ -f "${tar_path}" ] || return 1
local extract_dir="${ROOT_DIR}/.tmp/nomos-binaries"
echo "==> Restoring binaries from ${tar_path}"
rm -rf "${extract_dir}"
mkdir -p "${extract_dir}"
tar -xzf "${tar_path}" -C "${extract_dir}" || common::die "Failed to extract ${tar_path}"
local src="${extract_dir}/artifacts"
local bin_dst="${ROOT_DIR}/testing-framework/assets/stack/bin"
local circuits_src="${src}/circuits"
local circuits_dst="${HOST_KZG_DIR}"
RESTORED_BIN_DIR="${src}"
export RESTORED_BIN_DIR
if [ ! -f "${src}/nomos-node" ] || [ ! -f "${src}/nomos-executor" ] || [ ! -f "${src}/nomos-cli" ]; then
echo "Binaries missing in ${tar_path}; provide a prebuilt binaries tarball." >&2
return 1
fi
local copy_bins=1
if [ "${MODE}" != "host" ] && ! run_examples::host_bin_matches_arch "${src}/nomos-node"; then
echo "Bundled binaries do not match host arch; skipping copy so containers rebuild from source."
copy_bins=0
rm -f "${bin_dst}/nomos-node" "${bin_dst}/nomos-executor" "${bin_dst}/nomos-cli"
fi
if [ "${copy_bins}" -eq 1 ]; then
mkdir -p "${bin_dst}"
cp "${src}/nomos-node" "${src}/nomos-executor" "${src}/nomos-cli" "${bin_dst}/"
fi
if [ -d "${circuits_src}" ] && [ -f "${circuits_src}/${KZG_FILE}" ]; then
rm -rf "${circuits_dst}"
mkdir -p "${circuits_dst}"
if command -v rsync >/dev/null 2>&1; then
rsync -a --delete "${circuits_src}/" "${circuits_dst}/"
else
rm -rf "${circuits_dst:?}/"*
cp -a "${circuits_src}/." "${circuits_dst}/"
fi
else
echo "Circuits missing in ${tar_path}; provide a prebuilt binaries/circuits tarball." >&2
return 1
fi
RESTORED_BINARIES=1
export RESTORED_BINARIES
}
run_examples::ensure_binaries_tar() {
local platform="$1"
local tar_path="$2"
echo "==> Building fresh binaries bundle (${platform}) at ${tar_path}"
"${ROOT_DIR}/scripts/build-bundle.sh" --platform "${platform}" --output "${tar_path}" --rev "${NOMOS_NODE_REV}"
}
run_examples::prepare_bundles() {
RESTORED_BINARIES=0
NEED_HOST_RESTORE_AFTER_IMAGE=0
HOST_TAR="${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz"
LINUX_TAR="${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz"
if [ -n "${NOMOS_NODE_BIN:-}" ] && [ -x "${NOMOS_NODE_BIN}" ] && [ -n "${NOMOS_EXECUTOR_BIN:-}" ] && [ -x "${NOMOS_EXECUTOR_BIN}" ]; then
echo "==> Using pre-specified host binaries (NOMOS_NODE_BIN/NOMOS_EXECUTOR_BIN); skipping tarball restore"
return 0
fi
# On non-Linux compose/k8s runs, use the Linux bundle for image build, then restore host bundle for the runner.
if [ "${MODE}" != "host" ] && [ "$(uname -s)" != "Linux" ] && [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "0" ] && [ -f "${LINUX_TAR}" ]; then
NEED_HOST_RESTORE_AFTER_IMAGE=1
run_examples::restore_binaries_from_tar "${LINUX_TAR}" || {
run_examples::ensure_binaries_tar linux "${LINUX_TAR}"
run_examples::restore_binaries_from_tar "${LINUX_TAR}"
}
fi
if ! run_examples::restore_binaries_from_tar; then
local tar_path
tar_path="$(run_examples::default_tar_path)"
case "${MODE}" in
host) run_examples::ensure_binaries_tar host "${tar_path}" ;;
compose|k8s)
if [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "1" ]; then
run_examples::ensure_binaries_tar host "${tar_path}"
else
run_examples::ensure_binaries_tar linux "${tar_path}"
fi
;;
*) run_examples::ensure_binaries_tar host "${tar_path}" ;;
esac
run_examples::restore_binaries_from_tar "${tar_path}" || common::die \
"Missing or invalid binaries tarball. Provide it via --bundle/NOMOS_BINARIES_TAR or place it at $(run_examples::default_tar_path)."
fi
}
run_examples::maybe_rebuild_image() {
if [ "${MODE}" = "host" ]; then
return 0
fi
if [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "1" ]; then
echo "==> Skipping testnet image rebuild (NOMOS_SKIP_IMAGE_BUILD=1)"
return 0
fi
echo "==> Rebuilding testnet image (${IMAGE})"
IMAGE_TAG="${IMAGE}" COMPOSE_CIRCUITS_PLATFORM="${COMPOSE_CIRCUITS_PLATFORM:-}" \
bash "${ROOT_DIR}/scripts/build_test_image.sh"
}
run_examples::maybe_restore_host_after_image() {
if [ "${NEED_HOST_RESTORE_AFTER_IMAGE}" != "1" ]; then
return 0
fi
echo "==> Restoring host bundle for runner (${HOST_TAR})"
if [ ! -f "${HOST_TAR}" ]; then
run_examples::ensure_binaries_tar host "${HOST_TAR}"
fi
run_examples::restore_binaries_from_tar "${HOST_TAR}" || common::die "Failed to restore host bundle from ${HOST_TAR}"
}
run_examples::validate_restored_bundle() {
HOST_BUNDLE_PATH="${HOST_KZG_DIR}"
KZG_HOST_PATH="${HOST_BUNDLE_PATH}/${KZG_FILE}"
if [ ! -x "${HOST_BUNDLE_PATH}/zksign/witness_generator" ]; then
common::die "Missing zksign/witness_generator in restored bundle; ensure the tarball contains host-compatible circuits."
fi
if [ ! -f "${KZG_HOST_PATH}" ]; then
common::die "KZG params missing at ${KZG_HOST_PATH}; ensure the tarball contains circuits."
fi
if [ "${MODE}" = "host" ] && ! { [ -n "${NOMOS_NODE_BIN:-}" ] && [ -x "${NOMOS_NODE_BIN:-}" ] && [ -n "${NOMOS_EXECUTOR_BIN:-}" ] && [ -x "${NOMOS_EXECUTOR_BIN:-}" ]; }; then
local tar_node tar_exec
tar_node="${RESTORED_BIN_DIR:-${ROOT_DIR}/testing-framework/assets/stack/bin}/nomos-node"
tar_exec="${RESTORED_BIN_DIR:-${ROOT_DIR}/testing-framework/assets/stack/bin}/nomos-executor"
[ -x "${tar_node}" ] && [ -x "${tar_exec}" ] || common::die \
"Restored tarball missing host executables; provide a host-compatible binaries tarball."
run_examples::host_bin_matches_arch "${tar_node}" && run_examples::host_bin_matches_arch "${tar_exec}" || common::die \
"Restored executables do not match host architecture; provide a host-compatible binaries tarball."
echo "==> Using restored host binaries from tarball"
NOMOS_NODE_BIN="${tar_node}"
NOMOS_EXECUTOR_BIN="${tar_exec}"
export NOMOS_NODE_BIN NOMOS_EXECUTOR_BIN
fi
}
run_examples::kzg_path_for_mode() {
if [ "${MODE}" = "compose" ] || [ "${MODE}" = "k8s" ]; then
if [ "${MODE}" = "k8s" ] && [ "${NOMOS_KZG_MODE:-hostPath}" = "inImage" ]; then
echo "${NOMOS_KZG_IN_IMAGE_PARAMS_PATH:-${DEFAULT_KZG_IN_IMAGE_PARAMS_PATH}}"
else
echo "${KZG_CONTAINER_PATH}"
fi
else
echo "${KZG_HOST_PATH}"
fi
}
run_examples::ensure_compose_circuits_platform_default() {
if [ "${MODE}" != "compose" ] || [ -n "${COMPOSE_CIRCUITS_PLATFORM:-}" ]; then
return 0
fi
local arch
arch="$(uname -m)"
case "${arch}" in
x86_64) COMPOSE_CIRCUITS_PLATFORM="linux-x86_64" ;;
arm64|aarch64) COMPOSE_CIRCUITS_PLATFORM="linux-aarch64" ;;
*) COMPOSE_CIRCUITS_PLATFORM="linux-x86_64" ;;
esac
export COMPOSE_CIRCUITS_PLATFORM
}
run_examples::run() {
local kzg_path
kzg_path="$(run_examples::kzg_path_for_mode)"
export NOMOS_DEMO_RUN_SECS="${RUN_SECS}"
export NOMOS_DEMO_VALIDATORS="${DEMO_VALIDATORS}"
export NOMOS_DEMO_EXECUTORS="${DEMO_EXECUTORS}"
if [ "${MODE}" = "k8s" ] && [ -n "${EXTERNAL_PROMETHEUS_URL}" ]; then
export K8S_RUNNER_EXTERNAL_PROMETHEUS_URL="${EXTERNAL_PROMETHEUS_URL}"
export NOMOS_EXTERNAL_PROMETHEUS_URL="${EXTERNAL_PROMETHEUS_URL}"
fi
echo "==> Running ${BIN} for ${RUN_SECS}s (mode=${MODE}, image=${IMAGE})"
cd "${ROOT_DIR}"
POL_PROOF_DEV_MODE=true \
TESTNET_PRINT_ENDPOINTS=1 \
NOMOS_TESTNET_IMAGE="${IMAGE}" \
NOMOS_CIRCUITS="${HOST_BUNDLE_PATH}" \
NOMOS_KZGRS_PARAMS_PATH="${kzg_path}" \
NOMOS_NODE_BIN="${NOMOS_NODE_BIN:-}" \
NOMOS_EXECUTOR_BIN="${NOMOS_EXECUTOR_BIN:-}" \
COMPOSE_CIRCUITS_PLATFORM="${COMPOSE_CIRCUITS_PLATFORM:-}" \
cargo run -p runner-examples --bin "${BIN}"
}
run_examples::main() {
run_examples::load_env
run_examples::parse_args "$@"
run_examples::select_bin
run_examples::select_image
run_examples::prepare_bundles
echo "==> Using restored circuits/binaries bundle"
SETUP_OUT="$(common::tmpfile nomos-setup-output.XXXXXX)"
cleanup() { rm -f "${SETUP_OUT}" 2>/dev/null || true; }
trap cleanup EXIT
run_examples::maybe_rebuild_image
run_examples::maybe_restore_host_after_image
run_examples::validate_restored_bundle
run_examples::ensure_compose_circuits_platform_default
run_examples::run
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
run_examples::main "$@"
fi

172
scripts/setup-circuits-stack.sh
View File

@ -1,14 +1,176 @@
#!/usr/bin/env bash
set -euo pipefail
# Thin wrapper; the actual implementation lives in scripts/lib/setup-circuits-stack.sh
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# shellcheck disable=SC1091
. "${ROOT_DIR}/scripts/lib/setup-circuits-stack.sh"
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
setup_circuits_stack::main "$@"
readonly DEFAULT_CIRCUITS_VERSION="v0.3.1"
readonly DEFAULT_LINUX_PLATFORM="linux-x86_64"
readonly DEFAULT_KZG_DIR_REL="testing-framework/assets/stack/kzgrs_test_params"
readonly DEFAULT_KZG_FILE="kzgrs_test_params"
readonly DEFAULT_KZG_PARAMS_RELPATH="tests/kzgrs/kzgrs_test_params"
readonly RAW_GITHUB_BASE_URL="https://raw.githubusercontent.com"
setup_circuits_stack::usage() {
cat <<'EOF'
Usage: scripts/setup-circuits-stack.sh [VERSION]
Prepares circuits for both the Docker image (Linux/x86_64) and the host (for
witness generators).
Env overrides:
STACK_DIR Where to place the Linux bundle (default: testing-framework/assets/stack/kzgrs_test_params)
HOST_DIR Where to place the host bundle (default: .tmp/nomos-circuits-host)
LINUX_STAGE_DIR Optional staging dir for the Linux bundle (default: .tmp/nomos-circuits-linux)
NOMOS_CIRCUITS_PLATFORM Force host platform (e.g., macos-aarch64)
NOMOS_CIRCUITS_REBUILD_RAPIDSNARK Set to 1 to force rebuild (host bundle only)
EOF
}
setup_circuits_stack::fail_with_usage() {
echo "$1" >&2
setup_circuits_stack::usage
exit 1
}
setup_circuits_stack::realpath_py() {
python3 - "$1" <<'PY'
import os, sys
print(os.path.realpath(sys.argv[1]))
PY
}
setup_circuits_stack::detect_platform() {
local os arch
case "$(uname -s)" in
Linux*) os="linux" ;;
Darwin*) os="macos" ;;
MINGW*|MSYS*|CYGWIN*) os="windows" ;;
*) common::die "Unsupported OS: $(uname -s)" ;;
esac
case "$(uname -m)" in
x86_64) arch="x86_64" ;;
aarch64|arm64) arch="aarch64" ;;
*) common::die "Unsupported arch: $(uname -m)" ;;
esac
echo "${os}-${arch}"
}
setup_circuits_stack::fetch_bundle() {
local platform="$1"
local dest="$2"
local rebuild="${3:-0}"
rm -rf "${dest}"
mkdir -p "${dest}"
NOMOS_CIRCUITS_PLATFORM="${platform}" \
NOMOS_CIRCUITS_REBUILD_RAPIDSNARK="${rebuild}" \
"${ROOT_DIR}/scripts/setup-nomos-circuits.sh" "${VERSION}" "${dest}"
}
setup_circuits_stack::fetch_kzg_params() {
local dest_dir="$1"
local dest_file="${dest_dir}/${KZG_FILE}"
local url="${RAW_GITHUB_BASE_URL}/logos-co/nomos-node/${NOMOS_NODE_REV}/${DEFAULT_KZG_PARAMS_RELPATH}"
echo "Fetching KZG parameters from ${url}"
curl -fsSL "${url}" -o "${dest_file}"
}
setup_circuits_stack::load_env() {
ROOT_DIR="$(common::repo_root)"
export ROOT_DIR
common::require_file "${ROOT_DIR}/versions.env"
# shellcheck disable=SC1091
. "${ROOT_DIR}/versions.env"
common::maybe_source "${ROOT_DIR}/paths.env"
KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-${DEFAULT_KZG_DIR_REL}}"
KZG_FILE="${NOMOS_KZG_FILE:-${DEFAULT_KZG_FILE}}"
HOST_DIR_REL_DEFAULT="${NOMOS_CIRCUITS_HOST_DIR_REL:-.tmp/nomos-circuits-host}"
LINUX_DIR_REL_DEFAULT="${NOMOS_CIRCUITS_LINUX_DIR_REL:-.tmp/nomos-circuits-linux}"
VERSION="${VERSION:-${DEFAULT_CIRCUITS_VERSION}}"
STACK_DIR="${STACK_DIR:-${ROOT_DIR}/${KZG_DIR_REL}}"
HOST_DIR="${HOST_DIR:-${ROOT_DIR}/${HOST_DIR_REL_DEFAULT}}"
LINUX_STAGE_DIR="${LINUX_STAGE_DIR:-${ROOT_DIR}/${LINUX_DIR_REL_DEFAULT}}"
NOMOS_NODE_REV="${NOMOS_NODE_REV:?Missing NOMOS_NODE_REV in versions.env or env}"
# Force non-interactive installs so repeated runs do not prompt.
export NOMOS_CIRCUITS_NONINTERACTIVE=1
}
setup_circuits_stack::main() {
if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
setup_circuits_stack::usage
exit 0
fi
setup_circuits_stack::load_env
if [ -n "${1:-}" ]; then
VERSION="$1"
fi
echo "Preparing circuits (version ${VERSION})"
echo "Workspace: ${ROOT_DIR}"
local linux_platform="${DEFAULT_LINUX_PLATFORM}"
echo "Installing Linux bundle for Docker image into ${STACK_DIR}"
local stage_real stack_real
stage_real="$(setup_circuits_stack::realpath_py "${LINUX_STAGE_DIR}")"
stack_real="$(setup_circuits_stack::realpath_py "${STACK_DIR}")"
if [ "${stage_real}" = "${stack_real}" ]; then
rm -rf "${STACK_DIR}"
setup_circuits_stack::fetch_bundle "${linux_platform}" "${STACK_DIR}" 0
setup_circuits_stack::fetch_kzg_params "${STACK_DIR}"
else
rm -rf "${LINUX_STAGE_DIR}"
mkdir -p "${LINUX_STAGE_DIR}"
setup_circuits_stack::fetch_bundle "${linux_platform}" "${LINUX_STAGE_DIR}" 0
rm -rf "${STACK_DIR}"
mkdir -p "${STACK_DIR}"
cp -R "${LINUX_STAGE_DIR}/." "${STACK_DIR}/"
setup_circuits_stack::fetch_kzg_params "${STACK_DIR}"
fi
echo "Linux bundle ready at ${STACK_DIR}"
local host_platform
host_platform="${NOMOS_CIRCUITS_PLATFORM:-$(setup_circuits_stack::detect_platform)}"
if [[ "${host_platform}" == "${linux_platform}" ]]; then
echo "Host platform ${host_platform} matches Linux bundle; host can reuse ${STACK_DIR}"
echo "Export if you want to be explicit:"
echo " export NOMOS_CIRCUITS=\"${STACK_DIR}\""
else
echo "Host platform detected: ${host_platform}; installing host-native bundle into ${HOST_DIR}"
setup_circuits_stack::fetch_bundle "${host_platform}" "${HOST_DIR}" "${NOMOS_CIRCUITS_REBUILD_RAPIDSNARK:-0}"
setup_circuits_stack::fetch_kzg_params "${HOST_DIR}"
echo "Host bundle ready at ${HOST_DIR}"
echo
echo "Set for host runs:"
echo " export NOMOS_CIRCUITS=\"${HOST_DIR}\""
fi
cat <<'EOF'
Done.
- For Docker/compose: rebuild the image to bake the Linux bundle:
scripts/build_test_image.sh
- For host runs (e.g., compose_runner): ensure NOMOS_CIRCUITS points to the host bundle above.
EOF
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
setup_circuits_stack::main "$@"
fi

278
scripts/setup-nomos-circuits.sh
View File

@ -1,15 +1,283 @@
#!/usr/bin/env bash
set -euo pipefail
# Thin wrapper; the actual implementation lives in scripts/lib/setup-nomos-circuits.sh
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
readonly DEFAULT_CIRCUITS_VERSION="v0.3.1"
readonly DEFAULT_INSTALL_SUBDIR=".nomos-circuits"
readonly DEFAULT_CIRCUITS_REPO="logos-co/nomos-circuits"
# shellcheck disable=SC1091
. "${ROOT_DIR}/scripts/lib/setup-nomos-circuits.sh"
readonly DEFAULT_NONINTERACTIVE=0
readonly DEFAULT_REBUILD_RAPIDSNARK=0
setup_nomos_circuits::main "$@"
readonly CURL_RETRY_COUNT=5
readonly CURL_RETRY_DELAY_SECONDS=2
readonly ANSI_RED=$'\033[0;31m'
readonly ANSI_GREEN=$'\033[0;32m'
readonly ANSI_YELLOW=$'\033[1;33m'
readonly ANSI_BLUE=$'\033[0;34m'
readonly ANSI_RESET=$'\033[0m'
readonly ICON_INFO="ℹ"
readonly ICON_OK="✓"
readonly ICON_WARN="⚠"
readonly ICON_ERR="✗"
setup_nomos_circuits::usage() {
cat <<EOF
Usage: scripts/setup-nomos-circuits.sh [VERSION] [INSTALL_DIR]
Arguments:
VERSION Optional. Version to install (default: ${DEFAULT_CIRCUITS_VERSION})
INSTALL_DIR Optional. Installation directory (default: \$HOME/${DEFAULT_INSTALL_SUBDIR})
Environment:
NOMOS_CIRCUITS_PLATFORM Override platform (e.g. linux-x86_64, macos-aarch64)
NOMOS_CIRCUITS_NONINTERACTIVE Set to 1 to auto-overwrite without prompt
NOMOS_CIRCUITS_REBUILD_RAPIDSNARK Set to 1 to force rapidsnark rebuild
GITHUB_TOKEN Optional token for GitHub releases download
EOF
}
setup_nomos_circuits::init_vars() {
VERSION="${1:-${DEFAULT_CIRCUITS_VERSION}}"
DEFAULT_INSTALL_DIR="${HOME}/${DEFAULT_INSTALL_SUBDIR}"
INSTALL_DIR="${2:-${DEFAULT_INSTALL_DIR}}"
REPO="${DEFAULT_CIRCUITS_REPO}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
NONINTERACTIVE="${NOMOS_CIRCUITS_NONINTERACTIVE:-${DEFAULT_NONINTERACTIVE}}"
# Colors for output
RED="${ANSI_RED}"
GREEN="${ANSI_GREEN}"
YELLOW="${ANSI_YELLOW}"
BLUE="${ANSI_BLUE}"
NC="${ANSI_RESET}"
}
setup_nomos_circuits::print_info() { echo -e "${BLUE}${ICON_INFO}${NC} $1"; }
setup_nomos_circuits::print_success() { echo -e "${GREEN}${ICON_OK}${NC} $1"; }
setup_nomos_circuits::print_warning() { echo -e "${YELLOW}${ICON_WARN}${NC} $1"; }
setup_nomos_circuits::print_error() { echo -e "${RED}${ICON_ERR}${NC} $1"; }
setup_nomos_circuits::detect_platform() {
local os="" arch=""
case "$(uname -s)" in
Linux*) os="linux" ;;
Darwin*) os="macos" ;;
MINGW*|MSYS*|CYGWIN*) os="windows" ;;
*) setup_nomos_circuits::print_error "Unsupported operating system: $(uname -s)"; exit 1 ;;
esac
case "$(uname -m)" in
x86_64) arch="x86_64" ;;
aarch64|arm64) arch="aarch64" ;;
*) setup_nomos_circuits::print_error "Unsupported architecture: $(uname -m)"; exit 1 ;;
esac
echo "${os}-${arch}"
}
setup_nomos_circuits::check_existing_installation() {
if [ -d "${INSTALL_DIR}" ]; then
setup_nomos_circuits::print_warning "Installation directory already exists: ${INSTALL_DIR}"
if [ -f "${INSTALL_DIR}/VERSION" ]; then
local current_version
current_version="$(cat "${INSTALL_DIR}/VERSION")"
setup_nomos_circuits::print_info "Currently installed version: ${current_version}"
fi
if [ "${NONINTERACTIVE}" = "1" ] || [ ! -t 0 ]; then
setup_nomos_circuits::print_info "Non-interactive environment detected, automatically overwriting..."
else
echo
read -p "Do you want to overwrite it? (y/N): " -n 1 -r
echo
if [[ ! ${REPLY} =~ ^[Yy]$ ]]; then
setup_nomos_circuits::print_info "Installation cancelled."
exit 0
fi
fi
setup_nomos_circuits::print_info "Removing existing installation..."
rm -rf "${INSTALL_DIR}"
fi
}
setup_nomos_circuits::download_release() {
local platform="$1"
local artifact="nomos-circuits-${VERSION}-${platform}.tar.gz"
local url="https://github.com/${REPO}/releases/download/${VERSION}/${artifact}"
local temp_dir
temp_dir="$(mktemp -d)"
setup_nomos_circuits::print_info "Downloading nomos-circuits ${VERSION} for ${platform}..."
setup_nomos_circuits::print_info "URL: ${url}"
local -a curl_args=(curl -fL --retry "${CURL_RETRY_COUNT}" --retry-delay "${CURL_RETRY_DELAY_SECONDS}" --retry-all-errors)
if [ -n "${GITHUB_TOKEN:-}" ]; then
curl_args+=(--header "authorization: Bearer ${GITHUB_TOKEN}")
fi
curl_args+=(-o "${temp_dir}/${artifact}" "${url}")
if ! "${curl_args[@]}"; then
setup_nomos_circuits::print_error "Failed to download release artifact"
setup_nomos_circuits::print_error "Please check that version ${VERSION} exists for platform ${platform}"
setup_nomos_circuits::print_error "Available releases: https://github.com/${REPO}/releases"
rm -rf "${temp_dir}"
return 1
fi
setup_nomos_circuits::print_success "Download complete"
if ! tar -tzf "${temp_dir}/${artifact}" >/dev/null 2>&1; then
setup_nomos_circuits::print_error "Downloaded archive is not a valid tar.gz: ${temp_dir}/${artifact}"
rm -rf "${temp_dir}"
return 1
fi
setup_nomos_circuits::print_info "Extracting to ${INSTALL_DIR}..."
mkdir -p "${INSTALL_DIR}"
if ! tar -xzf "${temp_dir}/${artifact}" -C "${INSTALL_DIR}" --strip-components=1; then
setup_nomos_circuits::print_error "Failed to extract archive"
rm -rf "${temp_dir}"
return 1
fi
rm -rf "${temp_dir}"
setup_nomos_circuits::print_success "Extraction complete"
}
setup_nomos_circuits::handle_macos_quarantine() {
setup_nomos_circuits::print_info "macOS detected: Removing quarantine attributes from executables..."
if find "${INSTALL_DIR}" -type f -perm -111 -exec xattr -d com.apple.quarantine {} \; 2>/dev/null; then
setup_nomos_circuits::print_success "Quarantine attributes removed"
else
setup_nomos_circuits::print_warning "Could not remove quarantine attributes (they may not exist)"
fi
}
setup_nomos_circuits::print_circuits() {
setup_nomos_circuits::print_info "The following circuits are available:"
local dir circuit_name
for dir in "${INSTALL_DIR}"/*/; do
if [ -d "${dir}" ]; then
circuit_name="$(basename "${dir}")"
if [ -f "${dir}/witness_generator" ]; then
echo " • ${circuit_name}"
fi
fi
done
}
setup_nomos_circuits::resolve_platform() {
local platform_override="${NOMOS_CIRCUITS_PLATFORM:-}"
if [ -n "${platform_override}" ]; then
PLATFORM="${platform_override}"
setup_nomos_circuits::print_info "Using overridden platform: ${PLATFORM}"
else
PLATFORM="$(setup_nomos_circuits::detect_platform)"
setup_nomos_circuits::print_info "Detected platform: ${PLATFORM}"
fi
}
setup_nomos_circuits::download_with_fallbacks() {
# Outputs:
# PLATFORM - platform used for the downloaded bundle
# REBUILD_REQUIRED - 0/1
REBUILD_REQUIRED="${NOMOS_CIRCUITS_REBUILD_RAPIDSNARK:-${DEFAULT_REBUILD_RAPIDSNARK}}"
if setup_nomos_circuits::download_release "${PLATFORM}"; then
return 0
fi
if [[ "${PLATFORM}" == "linux-aarch64" ]]; then
setup_nomos_circuits::print_warning "Falling back to linux-x86_64 circuits bundle; will rebuild prover for aarch64."
rm -rf "${INSTALL_DIR}"
PLATFORM="linux-x86_64"
setup_nomos_circuits::download_release "${PLATFORM}" || return 1
REBUILD_REQUIRED=1
return 0
fi
if [[ "${PLATFORM}" == "macos-x86_64" ]]; then
setup_nomos_circuits::print_warning "No macOS x86_64 bundle; falling back to macOS aarch64 circuits bundle and rebuilding prover."
rm -rf "${INSTALL_DIR}"
PLATFORM="macos-aarch64"
if ! setup_nomos_circuits::download_release "${PLATFORM}"; then
setup_nomos_circuits::print_warning "macOS aarch64 bundle unavailable; trying linux-x86_64 bundle and rebuilding prover."
rm -rf "${INSTALL_DIR}"
PLATFORM="linux-x86_64"
setup_nomos_circuits::download_release "${PLATFORM}" || return 1
fi
REBUILD_REQUIRED=1
return 0
fi
return 1
}
setup_nomos_circuits::maybe_handle_quarantine() {
if [[ "${PLATFORM}" == macos-* ]]; then
echo
setup_nomos_circuits::handle_macos_quarantine
fi
}
setup_nomos_circuits::maybe_rebuild_rapidsnark() {
if [[ "${REBUILD_REQUIRED}" == "1" ]]; then
echo
setup_nomos_circuits::print_info "Rebuilding rapidsnark prover for ${PLATFORM}..."
"${SCRIPT_DIR}/build-rapidsnark.sh" "${INSTALL_DIR}"
else
setup_nomos_circuits::print_info "Skipping rapidsnark rebuild (set NOMOS_CIRCUITS_REBUILD_RAPIDSNARK=1 to force)."
fi
}
setup_nomos_circuits::print_summary() {
echo
setup_nomos_circuits::print_success "Installation complete!"
echo
setup_nomos_circuits::print_info "nomos-circuits ${VERSION} is now installed at: ${INSTALL_DIR}"
setup_nomos_circuits::print_circuits
if [ "${INSTALL_DIR}" != "${DEFAULT_INSTALL_DIR}" ]; then
echo
setup_nomos_circuits::print_info "Since you're using a custom installation directory, set the environment variable:"
setup_nomos_circuits::print_info " export NOMOS_CIRCUITS=${INSTALL_DIR}"
echo
fi
}
setup_nomos_circuits::main() {
if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
setup_nomos_circuits::usage
exit 0
fi
setup_nomos_circuits::init_vars "${1:-}" "${2:-}"
setup_nomos_circuits::print_info "Setting up nomos-circuits ${VERSION}"
setup_nomos_circuits::print_info "Installation directory: ${INSTALL_DIR}"
echo
setup_nomos_circuits::resolve_platform
setup_nomos_circuits::check_existing_installation
setup_nomos_circuits::download_with_fallbacks || exit 1
setup_nomos_circuits::maybe_handle_quarantine
setup_nomos_circuits::maybe_rebuild_rapidsnark
setup_nomos_circuits::print_summary
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
setup_nomos_circuits::main "$@"
fi

126
scripts/update-grafana-panel-titles.py
View File

@ -1,126 +0,0 @@
#!/usr/bin/env python3
import glob
import json
import os
import re
from typing import Any, Optional
DASH_DIR = "testing-framework/assets/stack/monitoring/grafana/dashboards"
TITLE_SEP = " — "
METRIC_RE = re.compile(r"\b[a-zA-Z_:][a-zA-Z0-9_:]*\b")
def _collect_exprs(panel: dict[str, Any]) -> list[str]:
exprs: list[str] = []
for target in panel.get("targets") or []:
expr = target.get("expr")
if isinstance(expr, str) and expr.strip():
exprs.append(expr.strip())
return exprs
def _descriptor_from_exprs(title: str, exprs: list[str]) -> Optional[str]:
if not exprs:
return None
all_expr = "\n".join(exprs)
if "histogram_quantile" in all_expr:
return "p95 latency"
if "time() - on() (" in all_expr:
return "time since last"
if "consensus_tip_height - consensus_finalized_height" in all_expr:
return "finalization gap"
if any("rate(" in e for e in exprs) or any("irate(" in e for e in exprs):
return "events/sec"
lower_title = title.lower()
if "throughput" in lower_title or "tps" in lower_title:
return "tx/sec"
if "errors" in lower_title or "fail" in lower_title:
return "error rate"
if "peers" in lower_title:
return "peer count"
if "connections" in lower_title:
return "conn count"
if "queue" in lower_title or "pending" in lower_title:
return "queue depth"
# If the title didn't help, infer from obvious metric names.
metrics = {m for m in METRIC_RE.findall(all_expr) if "_" in m or ":" in m}
if any(m.endswith("_pending") for m in metrics):
return "queue depth"
if any(m.endswith("_height") for m in metrics):
return "height"
if any(m.endswith("_slot") for m in metrics):
return "slot"
if any(m.endswith("_epoch") for m in metrics):
return "epoch"
if any("connections" in m for m in metrics):
return "conn count"
return "current"
def _update_panel_title(panel: dict[str, Any]) -> bool:
if panel.get("type") == "row":
return False
title = panel.get("title")
if not isinstance(title, str) or not title.strip():
return False
if TITLE_SEP in title:
return False
exprs = _collect_exprs(panel)
desc = _descriptor_from_exprs(title, exprs)
if not desc:
return False
panel["title"] = f"{title}{TITLE_SEP}{desc}"
return True
def main() -> int:
paths = sorted(glob.glob(os.path.join(DASH_DIR, "*.json")))
if not paths:
raise SystemExit(f"No dashboards found at {DASH_DIR}")
changed_files = 0
changed_panels = 0
for path in paths:
with open(path) as f:
dash = json.load(f)
changed = False
for panel in dash.get("panels") or []:
if _update_panel_title(panel):
changed = True
changed_panels += 1
if changed:
with open(path, "w") as f:
json.dump(dash, f, indent=2, sort_keys=False)
f.write("\n")
changed_files += 1
print(f"updated {changed_panels} panels across {changed_files} dashboards")
return 0
if __name__ == "__main__":
raise SystemExit(main())

227
scripts/update-nomos-rev.sh
View File

@ -1,14 +1,231 @@
#!/usr/bin/env bash
set -euo pipefail
# Thin wrapper; the actual implementation lives in scripts/lib/update-nomos-rev.sh
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# shellcheck disable=SC1091
. "${ROOT_DIR}/scripts/lib/update-nomos-rev.sh"
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
update_nomos_rev::main "$@"
update_nomos_rev::usage() {
cat <<'EOF'
Usage:
scripts/update-nomos-rev.sh --rev <git_rev>
scripts/update-nomos-rev.sh --path <local_dir>
scripts/update-nomos-rev.sh --unskip-worktree
Notes:
--rev sets NOMOS_NODE_REV and updates Cargo.toml revs
--path sets NOMOS_NODE_PATH (clears NOMOS_NODE_REV) and patches Cargo.toml to use a local nomos-node checkout
--unskip-worktree clears any skip-worktree flag for Cargo.toml
Only one may be used at a time.
EOF
}
update_nomos_rev::fail_with_usage() {
echo "$1" >&2
update_nomos_rev::usage
exit 1
}
update_nomos_rev::maybe_unskip_worktree() {
local file="$1"
if git -C "${ROOT_DIR}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
git -C "${ROOT_DIR}" update-index --no-skip-worktree "${file}" >/dev/null 2>&1 || true
fi
}
update_nomos_rev::maybe_skip_worktree() {
local file="$1"
if git -C "${ROOT_DIR}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
git -C "${ROOT_DIR}" update-index --skip-worktree "${file}" >/dev/null 2>&1 || true
fi
}
update_nomos_rev::ensure_env_key() {
local key="$1" default_value="$2"
if ! grep -Eq "^#?[[:space:]]*${key}=" "${ROOT_DIR}/versions.env"; then
echo "${default_value}" >> "${ROOT_DIR}/versions.env"
fi
}
update_nomos_rev::parse_args() {
REV=""
LOCAL_PATH=""
UNSKIP_WORKTREE=0
while [ "$#" -gt 0 ]; do
case "$1" in
--rev) REV="${2:-}"; shift 2 ;;
--path) LOCAL_PATH="${2:-}"; shift 2 ;;
--unskip-worktree) UNSKIP_WORKTREE=1; shift ;;
-h|--help) update_nomos_rev::usage; exit 0 ;;
*) update_nomos_rev::fail_with_usage "Unknown arg: $1" ;;
esac
done
if [ "${UNSKIP_WORKTREE}" -eq 1 ] && { [ -n "${REV}" ] || [ -n "${LOCAL_PATH}" ]; }; then
update_nomos_rev::fail_with_usage "Use --unskip-worktree alone."
fi
if [ -n "${REV}" ] && [ -n "${LOCAL_PATH}" ]; then
update_nomos_rev::fail_with_usage "Use either --rev or --path, not both"
fi
if [ -z "${REV}" ] && [ -z "${LOCAL_PATH}" ] && [ "${UNSKIP_WORKTREE}" -eq 0 ]; then
update_nomos_rev::usage
exit 1
fi
}
update_nomos_rev::load_env() {
ROOT_DIR="$(common::repo_root)"
export ROOT_DIR
common::require_file "${ROOT_DIR}/versions.env"
}
update_nomos_rev::update_to_rev() {
local rev="$1"
echo "Updating nomos-node rev to ${rev}"
sed -i.bak -E \
-e "s/^#?[[:space:]]*NOMOS_NODE_REV=.*/NOMOS_NODE_REV=${rev}/" \
-e "s/^#?[[:space:]]*NOMOS_NODE_PATH=.*/# NOMOS_NODE_PATH=/" \
"${ROOT_DIR}/versions.env"
rm -f "${ROOT_DIR}/versions.env.bak"
python3 - "${ROOT_DIR}" "${rev}" <<'PY'
import pathlib, re, sys
root = pathlib.Path(sys.argv[1])
rev = sys.argv[2]
cargo_toml = root / "Cargo.toml"
txt = cargo_toml.read_text()
txt = txt.replace("\\n", "\n")
txt = re.sub(
r'(?ms)^\[patch\."https://github\.com/logos-co/nomos-node"\].*?(?=^\[|\Z)',
"",
txt,
)
txt = re.sub(
r'(git = "https://github\.com/logos-co/nomos-node\.git", rev = ")[^"]+(")',
r"\g<1>" + rev + r"\2",
txt,
)
cargo_toml.write_text(txt.rstrip() + "\n")
PY
update_nomos_rev::maybe_unskip_worktree "Cargo.toml"
}
update_nomos_rev::update_to_path() {
local node_path="$1"
echo "Pointing to local nomos-node at ${node_path}"
[ -d "${node_path}" ] || common::die "path does not exist: ${node_path}"
local current_rev escaped_path
current_rev="$(grep -E '^[#[:space:]]*NOMOS_NODE_REV=' "${ROOT_DIR}/versions.env" | head -n1 | sed -E 's/^#?[[:space:]]*NOMOS_NODE_REV=//')"
escaped_path="${node_path//\//\\/}"
sed -i.bak -E \
-e "s/^#?[[:space:]]*NOMOS_NODE_PATH=.*/NOMOS_NODE_PATH=${escaped_path}/" \
-e "s/^#?[[:space:]]*NOMOS_NODE_REV=.*/# NOMOS_NODE_REV=${current_rev}/" \
"${ROOT_DIR}/versions.env"
rm -f "${ROOT_DIR}/versions.env.bak"
local python_bin="${PYTHON_BIN:-python3}"
command -v "${python_bin}" >/dev/null 2>&1 || common::die "python3 is required to patch Cargo.toml for local paths"
"${python_bin}" - "${ROOT_DIR}" "${node_path}" <<'PY'
import json
import pathlib
import re
import subprocess
import sys
root = pathlib.Path(sys.argv[1])
node_path = pathlib.Path(sys.argv[2])
targets = [
"broadcast-service", "chain-leader", "chain-network", "chain-service",
"common-http-client", "cryptarchia-engine", "cryptarchia-sync",
"executor-http-client", "groth16", "key-management-system-service",
"kzgrs", "kzgrs-backend", "nomos-api", "nomos-blend-message",
"nomos-blend-service", "nomos-core", "nomos-da-dispersal",
"nomos-da-network-core", "nomos-da-network-service", "nomos-da-sampling",
"nomos-da-verifier", "nomos-executor", "nomos-http-api-common",
"nomos-ledger", "nomos-libp2p", "nomos-network", "nomos-node",
"nomos-sdp", "nomos-time", "nomos-tracing", "nomos-tracing-service",
"nomos-utils", "nomos-wallet", "poc", "pol", "subnetworks-assignations",
"tests", "tx-service", "wallet", "zksign",
]
try:
meta = subprocess.check_output(
["cargo", "metadata", "--format-version", "1", "--no-deps"],
cwd=node_path,
)
except subprocess.CalledProcessError as exc:
sys.stderr.write(f"Failed to run cargo metadata in {node_path}: {exc}\n")
sys.exit(1)
data = json.loads(meta)
paths = {}
for pkg in data.get("packages", []):
paths[pkg["name"]] = str(pathlib.Path(pkg["manifest_path"]).parent)
patch_lines = ['[patch."https://github.com/logos-co/nomos-node"]']
missing = []
for name in targets:
if name in paths:
patch_lines.append(f'{name} = {{ path = "{paths[name]}" }}')
else:
missing.append(name)
cargo_toml = root / "Cargo.toml"
txt = cargo_toml.read_text()
txt = txt.replace("\\n", "\n")
txt = re.sub(
r'(?ms)^\[patch\."https://github\.com/logos-co/nomos-node"\].*?(?=^\[|\Z)',
"",
txt,
)
txt = txt.rstrip() + "\n\n" + "\n".join(patch_lines) + "\n"
cargo_toml.write_text(txt)
if missing:
sys.stderr.write(
"Warning: missing crates in local nomos-node checkout: "
+ ", ".join(missing)
+ "\n"
)
PY
update_nomos_rev::maybe_skip_worktree "Cargo.toml"
echo "Local nomos-node patch applied; Cargo.toml marked skip-worktree (run --unskip-worktree to clear)."
}
update_nomos_rev::main() {
update_nomos_rev::load_env
update_nomos_rev::parse_args "$@"
update_nomos_rev::ensure_env_key "NOMOS_NODE_REV" "# NOMOS_NODE_REV="
update_nomos_rev::ensure_env_key "NOMOS_NODE_PATH" "# NOMOS_NODE_PATH="
if [ "${UNSKIP_WORKTREE}" -eq 1 ]; then
update_nomos_rev::maybe_unskip_worktree "Cargo.toml"
echo "Cleared skip-worktree on Cargo.toml (if it was set)."
exit 0
fi
if [ -n "${REV}" ]; then
update_nomos_rev::update_to_rev "${REV}"
else
update_nomos_rev::update_to_path "${LOCAL_PATH}"
fi
echo "Done. Consider updating Cargo.lock if needed (cargo fetch)."
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
update_nomos_rev::main "$@"
fi

7
testing-framework/assets/stack/cfgsync.yaml
View File

@ -33,10 +33,9 @@ retry_commitments_limit: 5
# Tracing
tracing_settings:
# Write node logs to disk for debugging (avoid noisy stdout/trace DNS spam).
logger: !File
directory: /var/log/nomos
prefix: node
# Default to stdout so `docker logs` / `kubectl logs` shows node output.
# Switch to `!File` if you want per-node log files inside the container/pod.
logger: Stdout
# Disable OTLP traces to remove DNS errors; metrics stay enabled below.
tracing: None
filter: !EnvFilter

99
testing-framework/assets/stack/scripts/build_test_image.sh
View File

@ -1,99 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Builds the testnet image with circuits. Prefers a local circuits bundle
# (tests/kzgrs/kzgrs_test_params) or a custom override; otherwise downloads
# from logos-co/nomos-circuits.
# Always run under bash; bail out if someone invokes via sh.
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
# shellcheck disable=SC1091
. "${ROOT_DIR}/scripts/lib/common.sh"
common::require_file "${ROOT_DIR}/versions.env"
# shellcheck disable=SC1091
. "${ROOT_DIR}/versions.env"
common::maybe_source "${ROOT_DIR}/paths.env"
DOCKERFILE_PATH="${ROOT_DIR}/testing-framework/assets/stack/Dockerfile"
IMAGE_TAG="${IMAGE_TAG:-logos-blockchain-testing:local}"
DEFAULT_VERSION="${VERSION:-v0.3.1}"
VERSION="${VERSION:-${DEFAULT_VERSION}}"
KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}"
CIRCUITS_DIR_HOST="${ROOT_DIR}/${KZG_DIR_REL}"
CIRCUITS_OVERRIDE="${CIRCUITS_OVERRIDE:-${KZG_DIR_REL}}"
CIRCUITS_PLATFORM="${CIRCUITS_PLATFORM:-${COMPOSE_CIRCUITS_PLATFORM:-}}"
if [ -z "${CIRCUITS_PLATFORM}" ]; then
case "$(uname -m)" in
x86_64) CIRCUITS_PLATFORM="linux-x86_64" ;;
arm64|aarch64) CIRCUITS_PLATFORM="linux-aarch64" ;;
*) CIRCUITS_PLATFORM="linux-x86_64" ;;
esac
fi
NOMOS_NODE_REV="${NOMOS_NODE_REV:?Missing NOMOS_NODE_REV in versions.env or env}"
echo "Workspace root: ${ROOT_DIR}"
echo "Image tag: ${IMAGE_TAG}"
echo "Circuits override: ${CIRCUITS_OVERRIDE:-<none>}"
echo "Circuits version (fallback download): ${VERSION}"
echo "Circuits platform: ${CIRCUITS_PLATFORM}"
echo "Bundle tar (if used): ${NOMOS_BINARIES_TAR:-<default> ${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz}"
# If prebuilt binaries are missing, restore them from a bundle tarball instead of
# rebuilding nomos inside the image.
BIN_DST="${ROOT_DIR}/testing-framework/assets/stack/bin"
DEFAULT_LINUX_TAR="${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz"
TAR_PATH="${NOMOS_BINARIES_TAR:-${DEFAULT_LINUX_TAR}}"
if [ ! -x "${BIN_DST}/nomos-node" ] || [ ! -x "${BIN_DST}/nomos-executor" ]; then
if [ -f "${TAR_PATH}" ]; then
echo "Restoring binaries/circuits from ${TAR_PATH}"
tmp_extract="$(common::tmpdir nomos-bundle-extract.XXXXXX)"
tar -xzf "${TAR_PATH}" -C "${tmp_extract}"
if [ -f "${tmp_extract}/artifacts/nomos-node" ] && [ -f "${tmp_extract}/artifacts/nomos-executor" ]; then
mkdir -p "${BIN_DST}"
cp "${tmp_extract}/artifacts/nomos-node" "${tmp_extract}/artifacts/nomos-executor" "${tmp_extract}/artifacts/nomos-cli" "${BIN_DST}/"
else
common::die "Bundle ${TAR_PATH} missing binaries under artifacts/"
fi
if [ -d "${tmp_extract}/artifacts/circuits" ]; then
mkdir -p "${CIRCUITS_DIR_HOST}"
if command -v rsync >/dev/null 2>&1; then
rsync -a --delete "${tmp_extract}/artifacts/circuits/" "${CIRCUITS_DIR_HOST}/"
else
cp -a "${tmp_extract}/artifacts/circuits/." "${CIRCUITS_DIR_HOST}/"
fi
fi
rm -rf "${tmp_extract}"
else
common::die "Prebuilt binaries missing and bundle tar not found at ${TAR_PATH}"
fi
fi
build_args=(
-f "${DOCKERFILE_PATH}"
-t "${IMAGE_TAG}"
--build-arg "NOMOS_NODE_REV=${NOMOS_NODE_REV}"
--build-arg "CIRCUITS_PLATFORM=${CIRCUITS_PLATFORM}"
"${ROOT_DIR}"
)
# Pass override/version args to the Docker build.
if [ -n "${CIRCUITS_OVERRIDE}" ]; then
build_args+=(--build-arg "CIRCUITS_OVERRIDE=${CIRCUITS_OVERRIDE}")
fi
build_args+=(--build-arg "VERSION=${VERSION}")
echo "Running: docker build ${build_args[*]}"
docker build "${build_args[@]}"
cat <<EOF
Build complete.
- Use this image in k8s/compose by exporting NOMOS_TESTNET_IMAGE=${IMAGE_TAG}
- Circuits source: ${CIRCUITS_OVERRIDE:-download ${VERSION}}
EOF

23
testing-framework/assets/stack/scripts/setup-nomos-circuits.sh
View File

@ -13,10 +13,17 @@
set -euo pipefail
VERSION="${1:-v0.3.1}"
DEFAULT_INSTALL_DIR="$HOME/.nomos-circuits"
INSTALL_DIR="${2:-$DEFAULT_INSTALL_DIR}"
REPO="logos-co/nomos-circuits"
readonly DEFAULT_CIRCUITS_VERSION="v0.3.1"
readonly DEFAULT_INSTALL_SUBDIR=".nomos-circuits"
readonly DEFAULT_CIRCUITS_REPO="logos-co/nomos-circuits"
readonly CURL_RETRY_COUNT=5
readonly CURL_RETRY_DELAY_SECONDS=2
VERSION="${1:-${DEFAULT_CIRCUITS_VERSION}}"
DEFAULT_INSTALL_DIR="${HOME}/${DEFAULT_INSTALL_SUBDIR}"
INSTALL_DIR="${2:-${DEFAULT_INSTALL_DIR}}"
REPO="${DEFAULT_CIRCUITS_REPO}"
detect_platform() {
local os=""
@ -43,13 +50,13 @@ download_release() {
temp_dir=$(mktemp -d)
echo "Downloading nomos-circuits ${VERSION} for ${platform}..."
local -a curl_args=(curl -fL --retry "${CURL_RETRY_COUNT}" --retry-delay "${CURL_RETRY_DELAY_SECONDS}" --retry-all-errors)
if [ -n "${GITHUB_TOKEN:-}" ]; then
auth_header="Authorization: Bearer ${GITHUB_TOKEN}"
else
auth_header=""
curl_args+=(-H "Authorization: Bearer ${GITHUB_TOKEN}")
fi
curl_args+=(-o "${temp_dir}/${artifact}" "${url}")
if ! curl -L ${auth_header:+-H "$auth_header"} -o "${temp_dir}/${artifact}" "${url}"; then
if ! "${curl_args[@]}"; then
echo "Failed to download release artifact from ${url}" >&2
rm -rf "${temp_dir}"
exit 1

14
testing-framework/core/src/scenario/capabilities.rs
View File

@ -1,4 +1,5 @@
use async_trait::async_trait;
use reqwest::Url;
use super::DynError;
@ -6,6 +7,15 @@ use super::DynError;
#[derive(Clone, Copy, Debug, Default)]
pub struct NodeControlCapability;
/// Optional observability settings attached to a scenario.
///
/// Runners may use this to decide whether to provision in-cluster Prometheus or
/// reuse an existing endpoint.
#[derive(Clone, Debug, Default)]
pub struct ObservabilityCapability {
pub external_prometheus: Option<Url>,
}
/// Trait implemented by scenario capability markers to signal whether node
/// control is required.
pub trait RequiresNodeControl {
@ -20,6 +30,10 @@ impl RequiresNodeControl for NodeControlCapability {
const REQUIRED: bool = true;
}
impl RequiresNodeControl for ObservabilityCapability {
const REQUIRED: bool = false;
}
/// Interface exposed by runners that can restart nodes at runtime.
#[async_trait]
pub trait NodeControlHandle: Send + Sync {

4
testing-framework/core/src/scenario/mod.rs
View File

@ -10,7 +10,9 @@ mod workload;
pub type DynError = Box<dyn std::error::Error + Send + Sync + 'static>;
pub use capabilities::{NodeControlCapability, NodeControlHandle, RequiresNodeControl};
pub use capabilities::{
NodeControlCapability, NodeControlHandle, ObservabilityCapability, RequiresNodeControl,
};
pub use definition::{Builder, Scenario, ScenarioBuilder, TopologyConfigurator};
pub use expectation::Expectation;
pub use runtime::{

0
testing-framework/runners/compose/Cargo.toml → testing-framework/deployers/compose/Cargo.toml
View File

0
testing-framework/runners/compose/assets/docker-compose.yml.tera → testing-framework/deployers/compose/assets/docker-compose.yml.tera
View File

0
testing-framework/runners/compose/src/deployer/clients.rs → testing-framework/deployers/compose/src/deployer/clients.rs
View File

0
testing-framework/runners/compose/src/deployer/mod.rs → testing-framework/deployers/compose/src/deployer/mod.rs
View File

0
testing-framework/runners/compose/src/deployer/orchestrator.rs → testing-framework/deployers/compose/src/deployer/orchestrator.rs
View File

0
testing-framework/runners/compose/src/deployer/ports.rs → testing-framework/deployers/compose/src/deployer/ports.rs
View File

0
testing-framework/runners/compose/src/deployer/readiness.rs → testing-framework/deployers/compose/src/deployer/readiness.rs
View File

0
testing-framework/runners/compose/src/deployer/setup.rs → testing-framework/deployers/compose/src/deployer/setup.rs
View File

0
testing-framework/runners/compose/src/descriptor/mod.rs → testing-framework/deployers/compose/src/descriptor/mod.rs
View File

0
testing-framework/runners/compose/src/descriptor/node.rs → testing-framework/deployers/compose/src/descriptor/node.rs
View File

0
testing-framework/runners/compose/src/docker/commands.rs → testing-framework/deployers/compose/src/docker/commands.rs
View File

0
testing-framework/runners/compose/src/docker/control.rs → testing-framework/deployers/compose/src/docker/control.rs
View File

2
testing-framework/runners/compose/src/docker/mod.rs → testing-framework/deployers/compose/src/docker/mod.rs
View File

@ -104,7 +104,7 @@ pub async fn build_local_image(
) -> Result<(), ComposeRunnerError> {
let repo_root =
repository_root().map_err(|source| ComposeRunnerError::ImageBuild { source })?;
let dockerfile = repo_root.join("testing-framework/runners/docker/runner.Dockerfile");
let dockerfile = repo_root.join("../../../docker/runner.Dockerfile");
tracing::info!(image, "building compose runner docker image");

0
testing-framework/runners/compose/src/docker/platform.rs → testing-framework/deployers/compose/src/docker/platform.rs
View File

0
testing-framework/runners/compose/src/docker/workspace.rs → testing-framework/deployers/compose/src/docker/workspace.rs
View File

0
testing-framework/runners/compose/src/errors.rs → testing-framework/deployers/compose/src/errors.rs
View File

0
testing-framework/runners/compose/src/infrastructure/cfgsync.rs → testing-framework/deployers/compose/src/infrastructure/cfgsync.rs
View File

0
testing-framework/runners/compose/src/infrastructure/environment.rs → testing-framework/deployers/compose/src/infrastructure/environment.rs
View File

0
testing-framework/runners/compose/src/infrastructure/mod.rs → testing-framework/deployers/compose/src/infrastructure/mod.rs
View File

0
testing-framework/runners/compose/src/infrastructure/ports.rs → testing-framework/deployers/compose/src/infrastructure/ports.rs
View File

0
testing-framework/runners/compose/src/infrastructure/template.rs → testing-framework/deployers/compose/src/infrastructure/template.rs
View File

0
testing-framework/runners/compose/src/lib.rs → testing-framework/deployers/compose/src/lib.rs
View File

0
testing-framework/runners/compose/src/lifecycle/block_feed.rs → testing-framework/deployers/compose/src/lifecycle/block_feed.rs
View File

0
testing-framework/runners/compose/src/lifecycle/cleanup.rs → testing-framework/deployers/compose/src/lifecycle/cleanup.rs
View File

0
testing-framework/runners/compose/src/lifecycle/mod.rs → testing-framework/deployers/compose/src/lifecycle/mod.rs
View File

0
testing-framework/runners/compose/src/lifecycle/readiness.rs → testing-framework/deployers/compose/src/lifecycle/readiness.rs
View File

0
testing-framework/runners/compose/src/lifecycle/wait.rs → testing-framework/deployers/compose/src/lifecycle/wait.rs
View File

0
testing-framework/runners/docker/.dockerignore → testing-framework/deployers/docker/.dockerignore
View File

0
testing-framework/runners/docker/runner.Dockerfile → testing-framework/deployers/docker/runner.Dockerfile
View File

1
testing-framework/runners/k8s/Cargo.toml → testing-framework/deployers/k8s/Cargo.toml
View File

@ -17,6 +17,7 @@ anyhow = "1"
async-trait = { workspace = true }
k8s-openapi = { version = "0.20", features = ["latest"] }
kube = { version = "0.87", default-features = false, features = ["client", "runtime", "rustls-tls"] }
nomos-tracing-service = { workspace = true }
reqwest = { workspace = true, features = ["json"] }
serde = { version = "1", features = ["derive"] }
serde_yaml = { workspace = true }

0
testing-framework/runners/k8s/helm/nomos-runner/Chart.yaml → testing-framework/deployers/k8s/helm/nomos-runner/Chart.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/grafana/dashboards/.gitignore → testing-framework/deployers/k8s/helm/nomos-runner/grafana/dashboards/.gitignore vendored
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/_helpers.tpl → testing-framework/deployers/k8s/helm/nomos-runner/templates/_helpers.tpl
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/cfgsync-deployment.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/cfgsync-deployment.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/cfgsync-service.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/cfgsync-service.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/configmap.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/configmap.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/executor-deployments.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/executor-deployments.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/executor-services.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/executor-services.yaml
View File

2
testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-configmap.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/grafana-configmap.yaml
View File

@ -13,7 +13,7 @@ data:
isDefault: true
uid: PBFA97CFB590B2093
orgId: 1
url: http://prometheus:9090
url: {{ if .Values.prometheus.enabled }}http://prometheus:9090{{ else }}{{ required "prometheus.externalUrl must be set when prometheus.enabled=false" .Values.prometheus.externalUrl }}{{ end }}
editable: true
dashboards.yaml: |
apiVersion: 1

0
testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-deployment.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/grafana-deployment.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-service.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/grafana-service.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/prometheus-configmap.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/prometheus-configmap.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/prometheus-deployment.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/prometheus-deployment.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/prometheus-service.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/prometheus-service.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/pv.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/pv.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/pvc.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/pvc.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/validator-deployments.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/validator-deployments.yaml
View File

0
testing-framework/runners/k8s/helm/nomos-runner/templates/validator-services.yaml → testing-framework/deployers/k8s/helm/nomos-runner/templates/validator-services.yaml
View File

1
testing-framework/runners/k8s/helm/nomos-runner/values.yaml → testing-framework/deployers/k8s/helm/nomos-runner/values.yaml
View File

@ -29,6 +29,7 @@ kzg:
prometheus:
enabled: true
externalUrl: ""
image: "prom/prometheus:v3.0.1"
imagePullPolicy: IfNotPresent
retention: "7d"

0
testing-framework/runners/k8s/src/deployer/mod.rs → testing-framework/deployers/k8s/src/deployer/mod.rs
View File

406
testing-framework/deployers/k8s/src/deployer/orchestrator.rs Normal file
View File

@ -0,0 +1,406 @@
use anyhow::Error;
use async_trait::async_trait;
use kube::Client;
use reqwest::Url;
use testing_framework_core::{
scenario::{
BlockFeedTask, CleanupGuard, Deployer, MetricsError, ObservabilityCapability, RunContext,
Runner, Scenario,
},
topology::generation::GeneratedTopology,
};
use tracing::{error, info};
use crate::{
infrastructure::{
assets::{AssetsError, prepare_assets},
cluster::{
ClusterEnvironment, NodeClientError, PortSpecs, RemoteReadinessError,
build_node_clients, cluster_identifiers, collect_port_specs, ensure_cluster_readiness,
install_stack, kill_port_forwards, metrics_handle_from_endpoint,
metrics_handle_from_url, wait_for_ports_or_cleanup,
},
helm::HelmError,
},
lifecycle::{block_feed::spawn_block_feed_with, cleanup::RunnerCleanup},
wait::{ClusterWaitError, HostPort, PortForwardHandle},
};
/// Deploys a scenario into Kubernetes using Helm charts and port-forwards.
#[derive(Clone, Copy)]
pub struct K8sDeployer {
readiness_checks: bool,
}
impl Default for K8sDeployer {
fn default() -> Self {
Self::new()
}
}
impl K8sDeployer {
#[must_use]
/// Create a k8s deployer with readiness checks enabled.
pub const fn new() -> Self {
Self {
readiness_checks: true,
}
}
#[must_use]
/// Enable/disable readiness probes before handing control to workloads.
pub const fn with_readiness(mut self, enabled: bool) -> Self {
self.readiness_checks = enabled;
self
}
}
#[derive(Debug, thiserror::Error)]
/// High-level runner failures returned to the scenario harness.
pub enum K8sRunnerError {
#[error(
"kubernetes runner requires at least one validator and one executor (validators={validators}, executors={executors})"
)]
UnsupportedTopology { validators: usize, executors: usize },
#[error("failed to initialise kubernetes client: {source}")]
ClientInit {
#[source]
source: kube::Error,
},
#[error(transparent)]
Assets(#[from] AssetsError),
#[error(transparent)]
Helm(#[from] HelmError),
#[error(transparent)]
Cluster(#[from] Box<ClusterWaitError>),
#[error(transparent)]
Readiness(#[from] RemoteReadinessError),
#[error(transparent)]
NodeClients(#[from] NodeClientError),
#[error(transparent)]
Telemetry(#[from] MetricsError),
#[error("k8s runner requires at least one node client to follow blocks")]
BlockFeedMissing,
#[error("failed to initialize block feed: {source}")]
BlockFeed {
#[source]
source: Error,
},
}
#[async_trait]
impl Deployer for K8sDeployer {
type Error = K8sRunnerError;
async fn deploy(&self, scenario: &Scenario) -> Result<Runner, Self::Error> {
deploy_with_prometheus(self, scenario, None).await
}
}
#[async_trait]
impl Deployer<ObservabilityCapability> for K8sDeployer {
type Error = K8sRunnerError;
async fn deploy(
&self,
scenario: &Scenario<ObservabilityCapability>,
) -> Result<Runner, Self::Error> {
deploy_with_prometheus(
self,
scenario,
scenario.capabilities().external_prometheus.clone(),
)
.await
}
}
fn cluster_prometheus_endpoint(cluster: &Option<ClusterEnvironment>) -> Option<&HostPort> {
cluster
.as_ref()
.expect("cluster must be available")
.prometheus_endpoint()
}
fn cluster_grafana_endpoint(cluster: &Option<ClusterEnvironment>) -> Option<&HostPort> {
cluster
.as_ref()
.expect("cluster must be available")
.grafana_endpoint()
}
async fn fail_cluster(cluster: &mut Option<ClusterEnvironment>, reason: &str) {
if let Some(env) = cluster.as_mut() {
env.fail(reason).await;
}
}
impl From<ClusterWaitError> for K8sRunnerError {
fn from(value: ClusterWaitError) -> Self {
Self::Cluster(Box::new(value))
}
}
fn ensure_supported_topology(descriptors: &GeneratedTopology) -> Result<(), K8sRunnerError> {
let validators = descriptors.validators().len();
let executors = descriptors.executors().len();
if validators == 0 || executors == 0 {
return Err(K8sRunnerError::UnsupportedTopology {
validators,
executors,
});
}
Ok(())
}
async fn deploy_with_prometheus<Caps>(
deployer: &K8sDeployer,
scenario: &Scenario<Caps>,
external_prometheus: Option<Url>,
) -> Result<Runner, K8sRunnerError> {
let external_prometheus = match external_prometheus {
Some(url) => Some(url),
None => match std::env::var("K8S_RUNNER_EXTERNAL_PROMETHEUS_URL")
.ok()
.or_else(|| std::env::var("NOMOS_EXTERNAL_PROMETHEUS_URL").ok())
{
Some(raw) if !raw.trim().is_empty() => Some(Url::parse(raw.trim()).map_err(|err| {
MetricsError::new(format!("invalid external prometheus url: {err}"))
})?),
_ => None,
},
};
let descriptors = scenario.topology().clone();
let validator_count = descriptors.validators().len();
let executor_count = descriptors.executors().len();
ensure_supported_topology(&descriptors)?;
let client = Client::try_default()
.await
.map_err(|source| K8sRunnerError::ClientInit { source })?;
info!(
validators = validator_count,
executors = executor_count,
duration_secs = scenario.duration().as_secs(),
readiness_checks = deployer.readiness_checks,
external_prometheus = external_prometheus.as_ref().map(|u| u.as_str()),
"starting k8s deployment"
);
let port_specs = collect_port_specs(&descriptors);
let mut cluster = Some(
setup_cluster(
&client,
&port_specs,
&descriptors,
deployer.readiness_checks,
external_prometheus.as_ref(),
)
.await?,
);
info!("building node clients");
let node_clients = match build_node_clients(
cluster
.as_ref()
.expect("cluster must be available while building clients"),
) {
Ok(clients) => clients,
Err(err) => {
fail_cluster(&mut cluster, "failed to construct node api clients").await;
error!(error = ?err, "failed to build k8s node clients");
return Err(err.into());
}
};
let telemetry = match external_prometheus.clone() {
Some(url) => metrics_handle_from_url(url),
None => cluster
.as_ref()
.and_then(|cluster| cluster.prometheus_endpoint())
.ok_or_else(|| MetricsError::new("prometheus endpoint unavailable"))
.and_then(metrics_handle_from_endpoint),
};
let telemetry = match telemetry {
Ok(handle) => handle,
Err(err) => {
fail_cluster(
&mut cluster,
"failed to configure prometheus metrics handle",
)
.await;
error!(error = ?err, "failed to configure prometheus metrics handle");
return Err(err.into());
}
};
let (block_feed, block_feed_guard) = match spawn_block_feed_with(&node_clients).await {
Ok(pair) => pair,
Err(err) => {
fail_cluster(&mut cluster, "failed to initialize block feed").await;
error!(error = ?err, "failed to initialize block feed");
return Err(err);
}
};
if let Some(url) = external_prometheus.as_ref() {
info!(prometheus_url = %url.as_str(), "using external prometheus endpoint");
} else if let Some(prometheus) = cluster_prometheus_endpoint(&cluster) {
info!(
prometheus_url = %format!("http://{}:{}/", prometheus.host, prometheus.port),
"prometheus endpoint available on host"
);
}
if let Some(grafana) = cluster_grafana_endpoint(&cluster) {
info!(
grafana_url = %format!("http://{}:{}/", grafana.host, grafana.port),
"grafana dashboard available on host"
);
}
if std::env::var("TESTNET_PRINT_ENDPOINTS").is_ok() {
let prometheus = external_prometheus
.as_ref()
.map(|u| u.as_str().to_string())
.or_else(|| {
cluster_prometheus_endpoint(&cluster)
.map(|endpoint| format!("http://{}:{}/", endpoint.host, endpoint.port))
})
.unwrap_or_else(|| "<disabled>".to_string());
let grafana = cluster_grafana_endpoint(&cluster);
println!(
"TESTNET_ENDPOINTS prometheus={} grafana={}",
prometheus,
grafana
.map(|endpoint| format!("http://{}:{}/", endpoint.host, endpoint.port))
.unwrap_or_else(|| "<disabled>".to_string())
);
for (idx, client) in node_clients.validator_clients().iter().enumerate() {
println!(
"TESTNET_PPROF validator_{}={}/debug/pprof/profile?seconds=15&format=proto",
idx,
client.base_url()
);
}
for (idx, client) in node_clients.executor_clients().iter().enumerate() {
println!(
"TESTNET_PPROF executor_{}={}/debug/pprof/profile?seconds=15&format=proto",
idx,
client.base_url()
);
}
}
let (cleanup, port_forwards) = cluster
.take()
.expect("cluster should still be available")
.into_cleanup();
let cleanup_guard: Box<dyn CleanupGuard> = Box::new(K8sCleanupGuard::new(
cleanup,
block_feed_guard,
port_forwards,
));
let context = RunContext::new(
descriptors,
None,
node_clients,
scenario.duration(),
telemetry,
block_feed,
None,
);
info!(
validators = validator_count,
executors = executor_count,
duration_secs = scenario.duration().as_secs(),
"k8s deployment ready; handing control to scenario runner"
);
Ok(Runner::new(context, Some(cleanup_guard)))
}
async fn setup_cluster(
client: &Client,
specs: &PortSpecs,
descriptors: &GeneratedTopology,
readiness_checks: bool,
external_prometheus: Option<&Url>,
) -> Result<ClusterEnvironment, K8sRunnerError> {
let assets = prepare_assets(descriptors, external_prometheus)?;
let validators = descriptors.validators().len();
let executors = descriptors.executors().len();
let (namespace, release) = cluster_identifiers();
info!(%namespace, %release, validators, executors, "preparing k8s assets and namespace");
let mut cleanup_guard =
Some(install_stack(client, &assets, &namespace, &release, validators, executors).await?);
info!("waiting for helm-managed services to become ready");
let cluster_ready = wait_for_ports_or_cleanup(
client,
&namespace,
&release,
specs,
external_prometheus.is_none(),
&mut cleanup_guard,
)
.await?;
if let Some(prometheus) = cluster_ready.ports.prometheus.as_ref() {
info!(prometheus = ?prometheus, "discovered prometheus endpoint");
}
let environment = ClusterEnvironment::new(
client.clone(),
namespace,
release,
cleanup_guard
.take()
.expect("cleanup guard must exist after successful cluster startup"),
&cluster_ready.ports,
cluster_ready.port_forwards,
);
if readiness_checks {
info!("probing cluster readiness");
ensure_cluster_readiness(descriptors, &environment).await?;
info!("cluster readiness probes passed");
}
Ok(environment)
}
struct K8sCleanupGuard {
cleanup: RunnerCleanup,
block_feed: Option<BlockFeedTask>,
port_forwards: Vec<PortForwardHandle>,
}
impl K8sCleanupGuard {
const fn new(
cleanup: RunnerCleanup,
block_feed: BlockFeedTask,
port_forwards: Vec<PortForwardHandle>,
) -> Self {
Self {
cleanup,
block_feed: Some(block_feed),
port_forwards,
}
}
}
impl CleanupGuard for K8sCleanupGuard {
fn cleanup(mut self: Box<Self>) {
if let Some(block_feed) = self.block_feed.take() {
CleanupGuard::cleanup(Box::new(block_feed));
}
kill_port_forwards(&mut self.port_forwards);
CleanupGuard::cleanup(Box::new(self.cleanup));
}
}

0
testing-framework/runners/k8s/src/host.rs → testing-framework/deployers/k8s/src/host.rs
View File

46
testing-framework/runners/k8s/src/infrastructure/assets.rs → testing-framework/deployers/k8s/src/infrastructure/assets.rs
View File

@ -5,6 +5,8 @@ use std::{
};
use anyhow::{Context as _, Result as AnyResult};
use nomos_tracing_service::MetricsLayer;
use reqwest::Url;
use serde::Serialize;
use tempfile::TempDir;
use testing_framework_core::{
@ -88,7 +90,10 @@ fn kzg_mode() -> KzgMode {
/// Render cfgsync config, Helm values, and locate scripts/KZG assets for a
/// topology.
pub fn prepare_assets(topology: &GeneratedTopology) -> Result<RunnerAssets, AssetsError> {
pub fn prepare_assets(
topology: &GeneratedTopology,
external_prometheus: Option<&Url>,
) -> Result<RunnerAssets, AssetsError> {
info!(
validators = topology.validators().len(),
executors = topology.executors().len(),
@ -97,7 +102,7 @@ pub fn prepare_assets(topology: &GeneratedTopology) -> Result<RunnerAssets, Asse
let root = workspace_root().map_err(|source| AssetsError::WorkspaceRoot { source })?;
let kzg_mode = kzg_mode();
let cfgsync_yaml = render_cfgsync_config(&root, topology, kzg_mode)?;
let cfgsync_yaml = render_cfgsync_config(&root, topology, kzg_mode, external_prometheus)?;
let tempdir = tempfile::Builder::new()
.prefix("nomos-helm-")
@ -112,7 +117,7 @@ pub fn prepare_assets(topology: &GeneratedTopology) -> Result<RunnerAssets, Asse
};
let chart_path = helm_chart_path()?;
sync_grafana_dashboards(&root, &chart_path)?;
let values_yaml = render_values_yaml(topology)?;
let values_yaml = render_values_yaml(topology, external_prometheus)?;
let values_file = write_temp_file(tempdir.path(), "values.yaml", values_yaml)?;
let image = env::var("NOMOS_TESTNET_IMAGE")
.unwrap_or_else(|_| String::from("public.ecr.aws/r4s5t9y4/logos/logos-blockchain:test"));
@ -220,6 +225,7 @@ fn render_cfgsync_config(
root: &Path,
topology: &GeneratedTopology,
kzg_mode: KzgMode,
external_prometheus: Option<&Url>,
) -> Result<String, AssetsError> {
let cfgsync_template_path = stack_assets_root(root).join("cfgsync.yaml");
debug!(path = %cfgsync_template_path.display(), "loading cfgsync template");
@ -231,6 +237,18 @@ fn render_cfgsync_config(
.ok()
.unwrap_or_else(|| DEFAULT_IN_IMAGE_KZG_PARAMS_PATH.to_string());
}
if let Some(external_prometheus) = external_prometheus {
let base = external_prometheus.as_str().trim_end_matches('/');
let otlp_metrics = format!("{base}/api/v1/otlp/v1/metrics");
let endpoint = Url::parse(&otlp_metrics).map_err(|source| AssetsError::Cfgsync {
source: anyhow::anyhow!(
"invalid OTLP metrics endpoint derived from external Prometheus url '{base}': {source}"
),
})?;
if let MetricsLayer::Otlp(ref mut config) = cfg.tracing_settings.metrics {
config.endpoint = endpoint;
}
}
cfg.timeout = cfg.timeout.max(CFGSYNC_K8S_TIMEOUT_SECS);
render_cfgsync_yaml(&cfg).map_err(|source| AssetsError::Cfgsync { source })
}
@ -292,8 +310,11 @@ fn helm_chart_path() -> Result<PathBuf, AssetsError> {
}
}
fn render_values_yaml(topology: &GeneratedTopology) -> Result<String, AssetsError> {
let values = build_values(topology);
fn render_values_yaml(
topology: &GeneratedTopology,
external_prometheus: Option<&Url>,
) -> Result<String, AssetsError> {
let values = build_values(topology, external_prometheus);
serde_yaml::to_string(&values).map_err(|source| AssetsError::Values { source })
}
@ -349,6 +370,7 @@ struct HelmValues {
cfgsync: CfgsyncValues,
validators: NodeGroup,
executors: NodeGroup,
prometheus: PrometheusValues,
grafana: GrafanaValues,
}
@ -372,6 +394,13 @@ struct NodeValues {
env: BTreeMap<String, String>,
}
#[derive(Serialize)]
struct PrometheusValues {
enabled: bool,
#[serde(rename = "externalUrl", skip_serializing_if = "Option::is_none")]
external_url: Option<String>,
}
#[derive(Serialize)]
struct GrafanaValues {
enabled: bool,
@ -393,7 +422,7 @@ struct GrafanaServiceValues {
node_port: Option<u16>,
}
fn build_values(topology: &GeneratedTopology) -> HelmValues {
fn build_values(topology: &GeneratedTopology, external_prometheus: Option<&Url>) -> HelmValues {
let cfgsync = CfgsyncValues {
port: cfgsync_port(),
};
@ -420,6 +449,10 @@ fn build_values(topology: &GeneratedTopology) -> HelmValues {
node_port: grafana_node_port,
},
};
let prometheus = PrometheusValues {
enabled: external_prometheus.is_none(),
external_url: external_prometheus.map(|url| url.as_str().trim_end_matches('/').to_string()),
};
debug!(pol_mode, "rendering Helm values for k8s stack");
let validators = topology
.validators()
@ -506,6 +539,7 @@ fn build_values(topology: &GeneratedTopology) -> HelmValues {
count: topology.executors().len(),
nodes: executors,
},
prometheus,
grafana,
}
}

12
testing-framework/runners/k8s/src/infrastructure/cluster.rs → testing-framework/deployers/k8s/src/infrastructure/cluster.rs
View File

@ -38,7 +38,7 @@ pub struct ClusterEnvironment {
validator_testing_ports: Vec<u16>,
executor_api_ports: Vec<u16>,
executor_testing_ports: Vec<u16>,
prometheus: HostPort,
prometheus: Option<HostPort>,
grafana: Option<HostPort>,
port_forwards: Vec<PortForwardHandle>,
}
@ -105,8 +105,8 @@ impl ClusterEnvironment {
&self.release
}
pub fn prometheus_endpoint(&self) -> &HostPort {
&self.prometheus
pub fn prometheus_endpoint(&self) -> Option<&HostPort> {
self.prometheus.as_ref()
}
pub fn grafana_endpoint(&self) -> Option<&HostPort> {
@ -235,6 +235,10 @@ pub fn metrics_handle_from_endpoint(endpoint: &HostPort) -> Result<Metrics, Metr
Metrics::from_prometheus(url)
}
pub fn metrics_handle_from_url(url: Url) -> Result<Metrics, MetricsError> {
Metrics::from_prometheus(url)
}
pub async fn ensure_cluster_readiness(
descriptors: &GeneratedTopology,
cluster: &ClusterEnvironment,
@ -320,6 +324,7 @@ pub async fn wait_for_ports_or_cleanup(
namespace: &str,
release: &str,
specs: &PortSpecs,
prometheus_enabled: bool,
cleanup_guard: &mut Option<RunnerCleanup>,
) -> Result<ClusterReady, crate::deployer::K8sRunnerError> {
info!(
@ -335,6 +340,7 @@ pub async fn wait_for_ports_or_cleanup(
release,
&specs.validators,
&specs.executors,
prometheus_enabled,
)
.await
{

0
testing-framework/runners/k8s/src/infrastructure/helm.rs → testing-framework/deployers/k8s/src/infrastructure/helm.rs
View File

0
testing-framework/runners/k8s/src/infrastructure/mod.rs → testing-framework/deployers/k8s/src/infrastructure/mod.rs
View File

0
testing-framework/runners/k8s/src/lib.rs → testing-framework/deployers/k8s/src/lib.rs
View File

0
testing-framework/runners/k8s/src/lifecycle/block_feed.rs → testing-framework/deployers/k8s/src/lifecycle/block_feed.rs
View File

0
testing-framework/runners/k8s/src/lifecycle/cleanup.rs → testing-framework/deployers/k8s/src/lifecycle/cleanup.rs
View File

0
testing-framework/runners/k8s/src/lifecycle/logs.rs → testing-framework/deployers/k8s/src/lifecycle/logs.rs
View File

0
testing-framework/runners/k8s/src/lifecycle/mod.rs → testing-framework/deployers/k8s/src/lifecycle/mod.rs
View File

0
testing-framework/runners/k8s/src/lifecycle/wait/deployment.rs → testing-framework/deployers/k8s/src/lifecycle/wait/deployment.rs
View File

0
testing-framework/runners/k8s/src/lifecycle/wait/forwarding.rs → testing-framework/deployers/k8s/src/lifecycle/wait/forwarding.rs
View File

0
testing-framework/runners/k8s/src/lifecycle/wait/grafana.rs → testing-framework/deployers/k8s/src/lifecycle/wait/grafana.rs
View File

0
testing-framework/runners/k8s/src/lifecycle/wait/http_probe.rs → testing-framework/deployers/k8s/src/lifecycle/wait/http_probe.rs
View File

Some files were not shown because too many files have changed in this diff Show More