Add optional persist_dir

wip

lower block expectancy
This commit is contained in:
copilot-swe-agent[bot] 2026-02-04 16:19:55 +00:00 committed by hansieodendaal
parent ec112a2449
commit cee6b1a8a2
No known key found for this signature in database
GPG Key ID: 4B3B15868823687C
40 changed files with 249 additions and 274 deletions

View File

@ -239,7 +239,6 @@ jobs:
host_smoke: host_smoke:
runs-on: ubuntu-latest runs-on: ubuntu-latest
env: env:
POL_PROOF_DEV_MODE: true
LOCAL_DEMO_RUN_SECS: 120 LOCAL_DEMO_RUN_SECS: 120
LOCAL_DEMO_VALIDATORS: 1 LOCAL_DEMO_VALIDATORS: 1
LOGOS_BLOCKCHAIN_CIRCUITS: ${{ github.workspace }}/.tmp/logos-blockchain-circuits LOGOS_BLOCKCHAIN_CIRCUITS: ${{ github.workspace }}/.tmp/logos-blockchain-circuits
@ -500,7 +499,6 @@ jobs:
- name: Run compose mixed workload binary - name: Run compose mixed workload binary
env: env:
POL_PROOF_DEV_MODE: "true"
COMPOSE_NODE_PAIRS: "1x1" COMPOSE_NODE_PAIRS: "1x1"
LOGOS_BLOCKCHAIN_TESTNET_IMAGE: ${{ env.LOGOS_BLOCKCHAIN_TESTNET_IMAGE }} LOGOS_BLOCKCHAIN_TESTNET_IMAGE: ${{ env.LOGOS_BLOCKCHAIN_TESTNET_IMAGE }}
COMPOSE_RUNNER_HOST: "127.0.0.1" COMPOSE_RUNNER_HOST: "127.0.0.1"

View File

@ -123,7 +123,6 @@ Key environment variables for customization:
| Variable | Purpose | Default | | Variable | Purpose | Default |
|----------|---------|---------| |----------|---------|---------|
| `POL_PROOF_DEV_MODE=true` | **Required** — Disable expensive proof generation (set automatically by `scripts/run/run-examples.sh`) | (none) |
| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | Docker image tag for compose/k8s | `logos-blockchain-testing:local` | | `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | Docker image tag for compose/k8s | `logos-blockchain-testing:local` |
| `LOGOS_BLOCKCHAIN_DEMO_NODES` | Number of nodes | Varies by example | | `LOGOS_BLOCKCHAIN_DEMO_NODES` | Number of nodes | Varies by example |
| `LOGOS_BLOCKCHAIN_LOG_DIR` | Directory for persistent log files | (temporary) | | `LOGOS_BLOCKCHAIN_LOG_DIR` | Directory for persistent log files | (temporary) |

View File

@ -65,9 +65,7 @@ Convenience utilities:
- `compose_runner.rs` — Docker Compose (requires `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` built) - `compose_runner.rs` — Docker Compose (requires `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` built)
- `k8s_runner.rs` — Kubernetes (requires cluster + image) - `k8s_runner.rs` — Kubernetes (requires cluster + image)
**Run with:** `POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin <name>` **Run with:** `cargo run -p runner-examples --bin <name>`
**All runners require `POL_PROOF_DEV_MODE=true`** to avoid expensive proof generation.
### `scripts/` ### `scripts/`
Helper utilities: Helper utilities:

View File

@ -150,11 +150,9 @@ This handles circuit setup, binary building/bundling, image building, and execut
**Alternative:** Direct cargo run (requires manual setup): **Alternative:** Direct cargo run (requires manual setup):
```bash ```bash
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin <name> cargo run -p runner-examples --bin <name>
``` ```
**Important:** All runners require `POL_PROOF_DEV_MODE=true` to avoid expensive Groth16 proof generation that causes timeouts.
These binaries use the framework API (`ScenarioBuilder`) to construct and execute scenarios. These binaries use the framework API (`ScenarioBuilder`) to construct and execute scenarios.
## Builder API ## Builder API

View File

@ -153,7 +153,7 @@ async fn hello_consensus_liveness() -> Result<()> {
**Run it:** **Run it:**
```bash ```bash
POL_PROOF_DEV_MODE=true cargo test hello_consensus_liveness cargo test hello_consensus_liveness
``` ```
--- ---

View File

@ -128,13 +128,9 @@ strategy:
## Anti-Patterns to Avoid ## Anti-Patterns to Avoid
**DON'T: Run without POL_PROOF_DEV_MODE**
```bash ```bash
# BAD: Will hang/timeout on proof generation # BAD: Will hang/timeout on proof generation
cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
# GOOD: Fast mode for testing
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
``` ```
**DON'T: Use tiny durations** **DON'T: Use tiny durations**

View File

@ -39,7 +39,6 @@ on:
branches: [main] branches: [main]
env: env:
POL_PROOF_DEV_MODE: true
CARGO_TERM_COLOR: always CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1 RUST_BACKTRACE: 1
@ -243,17 +242,6 @@ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
## Best Practices ## Best Practices
### Required: Set POL_PROOF_DEV_MODE
**Always set `POL_PROOF_DEV_MODE=true` globally** in your workflow env:
```yaml
env:
POL_PROOF_DEV_MODE: true # REQUIRED!
```
Without this, tests will hang due to expensive proof generation.
### Use Helper Scripts ### Use Helper Scripts
Prefer `scripts/run/run-examples.sh` which handles all setup automatically: Prefer `scripts/run/run-examples.sh` which handles all setup automatically:

View File

@ -2,27 +2,6 @@
Complete reference of environment variables used by the testing framework, organized by category. Complete reference of environment variables used by the testing framework, organized by category.
## Critical Variables
These MUST be set for successful test runs:
| Variable | Required | Default | Effect |
|----------|----------|---------|--------|
| `POL_PROOF_DEV_MODE` | **YES** | — | **REQUIRED for all runners**. Set to `true` to use fast dev-mode proving instead of expensive Groth16. Without this, tests will hang/timeout. |
**Example:**
```bash
export POL_PROOF_DEV_MODE=true
```
Or add to your shell profile (`~/.bashrc`, `~/.zshrc`):
```bash
# Required for nomos-testing framework
export POL_PROOF_DEV_MODE=true
```
--- ---
## Runner Selection & Topology ## Runner Selection & Topology
@ -138,7 +117,6 @@ Control node log output (not framework runner logs):
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \ LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \ LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \ LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
# Inspect logs # Inspect logs
@ -303,7 +281,6 @@ Node-level configuration passed through to logos-blockchain-node:
# Faster block production # Faster block production
CONSENSUS_SLOT_TIME=5 \ CONSENSUS_SLOT_TIME=5 \
CONSENSUS_ACTIVE_SLOT_COEFF=0.9 \ CONSENSUS_ACTIVE_SLOT_COEFF=0.9 \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
``` ```
@ -350,14 +327,12 @@ Variables used by helper scripts (`scripts/run/run-examples.sh`, etc.):
### Minimal Host Run ### Minimal Host Run
```bash ```bash
POL_PROOF_DEV_MODE=true \
scripts/run/run-examples.sh -t 60 -n 3 host scripts/run/run-examples.sh -t 60 -n 3 host
``` ```
### Debug Logging (Host) ### Debug Logging (Host)
```bash ```bash
POL_PROOF_DEV_MODE=true \
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/logs \ LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/logs \
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \ LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \ LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \
@ -367,7 +342,6 @@ scripts/run/run-examples.sh -t 60 -n 3 host
### Compose with Observability ### Compose with Observability
```bash ```bash
POL_PROOF_DEV_MODE=true \
LOGOS_BLOCKCHAIN_METRICS_QUERY_URL=http://localhost:9090 \ LOGOS_BLOCKCHAIN_METRICS_QUERY_URL=http://localhost:9090 \
LOGOS_BLOCKCHAIN_GRAFANA_URL=http://localhost:3000 \ LOGOS_BLOCKCHAIN_GRAFANA_URL=http://localhost:3000 \
scripts/run/run-examples.sh -t 60 -n 3 compose scripts/run/run-examples.sh -t 60 -n 3 compose
@ -376,7 +350,6 @@ scripts/run/run-examples.sh -t 60 -n 3 compose
### K8s with Debug ### K8s with Debug
```bash ```bash
POL_PROOF_DEV_MODE=true \
K8S_RUNNER_NAMESPACE=nomos-debug \ K8S_RUNNER_NAMESPACE=nomos-debug \
K8S_RUNNER_DEBUG=1 \ K8S_RUNNER_DEBUG=1 \
K8S_RUNNER_PRESERVE=1 \ K8S_RUNNER_PRESERVE=1 \
@ -387,7 +360,6 @@ scripts/run/run-examples.sh -t 60 -n 3 k8s
```yaml ```yaml
env: env:
POL_PROOF_DEV_MODE: true
RUST_BACKTRACE: 1 RUST_BACKTRACE: 1
LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS: 1 LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS: 1
``` ```

View File

@ -15,9 +15,7 @@ and expectations.
**Recommended:** Use `scripts/run/run-examples.sh -t <duration> -n <nodes> <mode>` where mode is `host`, `compose`, or `k8s`. **Recommended:** Use `scripts/run/run-examples.sh -t <duration> -n <nodes> <mode>` where mode is `host`, `compose`, or `k8s`.
**Alternative:** Direct cargo run: `POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin <name>` **Alternative:** Direct cargo run: `cargo run -p runner-examples --bin <name>`
**All runners require `POL_PROOF_DEV_MODE=true`** to avoid expensive proof generation.
**Code patterns** below show how to build scenarios. Wrap these in `#[tokio::test]` functions for integration tests, or `#[tokio::main]` for binaries. **Code patterns** below show how to build scenarios. Wrap these in `#[tokio::test]` functions for integration tests, or `#[tokio::main]` for binaries.

View File

@ -38,10 +38,6 @@
Also called "correctness expectations." Also called "correctness expectations."
- **Mantle transaction**: transaction type in Logos that can contain UTXO transfers - **Mantle transaction**: transaction type in Logos that can contain UTXO transfers
(LedgerTx) and operations (Op). (LedgerTx) and operations (Op).
- **POL_PROOF_DEV_MODE**: environment variable that disables expensive Groth16 zero-knowledge
proof generation for leader election. **Required for all runners** (local, compose, k8s)
for practical testing—without it, proof generation causes timeouts. Should never be
used in production environments.
--- ---

View File

@ -46,7 +46,6 @@ LOGOS_BLOCKCHAIN_TESTS_TRACING=true \
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \ LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \ LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace,chain_service=info,chain_network=info" \ LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace,chain_service=info,chain_network=info" \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
``` ```
@ -90,7 +89,7 @@ LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace,chain_service=info,chain_network=
**Default (temporary directories, auto-cleanup):** **Default (temporary directories, auto-cleanup):**
```bash ```bash
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
# Logs written to temporary directories in working directory # Logs written to temporary directories in working directory
# Automatically cleaned up after test completes # Automatically cleaned up after test completes
``` ```
@ -99,7 +98,6 @@ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
```bash ```bash
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/local-logs \ LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/local-logs \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
# After test completes: # After test completes:
@ -137,7 +135,6 @@ To write per-node log files inside containers, set `tracing_settings.logger: !Fi
```bash ```bash
# Ensure cfgsync.yaml is configured to log to /logs # Ensure cfgsync.yaml is configured to log to /logs
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \ LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin compose_runner cargo run -p runner-examples --bin compose_runner
# After test, copy files from containers: # After test, copy files from containers:
@ -257,7 +254,6 @@ scripts/setup/setup-observability.sh compose up
eval $(scripts/setup/setup-observability.sh compose env) eval $(scripts/setup/setup-observability.sh compose env)
# Run scenario with metrics # Run scenario with metrics
POL_PROOF_DEV_MODE=true \
scripts/run/run-examples.sh -t 60 -n 3 compose scripts/run/run-examples.sh -t 60 -n 3 compose
``` ```
@ -275,7 +271,7 @@ scripts/setup/setup-observability.sh compose up
eval $(scripts/setup/setup-observability.sh compose env) eval $(scripts/setup/setup-observability.sh compose env)
export LOGOS_BLOCKCHAIN_GRAFANA_URL=http://localhost:3000 export LOGOS_BLOCKCHAIN_GRAFANA_URL=http://localhost:3000
POL_PROOF_DEV_MODE=true scripts/run/run-examples.sh -t 60 -n 3 compose scripts/run/run-examples.sh -t 60 -n 3 compose
``` ```
**Default bundled Grafana login:** `admin` / `admin` (see `scripts/observability/compose/docker-compose.yml`). **Default bundled Grafana login:** `admin` / `admin` (see `scripts/observability/compose/docker-compose.yml`).
@ -322,7 +318,6 @@ flowchart TD
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/logs \ LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/logs \
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \ LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \ LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \
POL_PROOF_DEV_MODE=true \
scripts/run/run-examples.sh -t 60 -n 3 host scripts/run/run-examples.sh -t 60 -n 3 host
``` ```
@ -334,7 +329,6 @@ scripts/setup/setup-observability.sh compose up
eval $(scripts/setup/setup-observability.sh compose env) eval $(scripts/setup/setup-observability.sh compose env)
# Run with metrics # Run with metrics
POL_PROOF_DEV_MODE=true \
scripts/run/run-examples.sh -t 60 -n 3 compose scripts/run/run-examples.sh -t 60 -n 3 compose
# Access Grafana at http://localhost:3000 # Access Grafana at http://localhost:3000
@ -346,7 +340,6 @@ scripts/run/run-examples.sh -t 60 -n 3 compose
K8S_RUNNER_NAMESPACE=nomos-debug \ K8S_RUNNER_NAMESPACE=nomos-debug \
K8S_RUNNER_DEBUG=1 \ K8S_RUNNER_DEBUG=1 \
K8S_RUNNER_PRESERVE=1 \ K8S_RUNNER_PRESERVE=1 \
POL_PROOF_DEV_MODE=true \
scripts/run/run-examples.sh -t 60 -n 3 k8s scripts/run/run-examples.sh -t 60 -n 3 k8s
# Inspect logs # Inspect logs

View File

@ -375,7 +375,6 @@ async fn external_driver_example() -> Result<()> {
```bash ```bash
# Required: dev mode for fast proofs # Required: dev mode for fast proofs
POL_PROOF_DEV_MODE=true \
cargo test -p runner-examples -- --ignored external_driver_example cargo test -p runner-examples -- --ignored external_driver_example
``` ```
@ -385,7 +384,6 @@ cargo test -p runner-examples -- --ignored external_driver_example
# Preserve logs after test # Preserve logs after test
LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1 \ LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1 \
RUST_LOG=info \ RUST_LOG=info \
POL_PROOF_DEV_MODE=true \
cargo test -p runner-examples -- --ignored external_driver_example cargo test -p runner-examples -- --ignored external_driver_example
``` ```

View File

@ -23,7 +23,6 @@ Operational readiness focuses on prerequisites, environment fit, and clear signa
- Binary bundles for reproducible builds - Binary bundles for reproducible builds
**Environment Configuration:** **Environment Configuration:**
- `POL_PROOF_DEV_MODE=true` is **REQUIRED for all runners** to avoid expensive proof generation
- Logging configured via `LOGOS_BLOCKCHAIN_LOG_*` variables - Logging configured via `LOGOS_BLOCKCHAIN_LOG_*` variables
- Observability endpoints (Prometheus, Grafana) optional but useful - Observability endpoints (Prometheus, Grafana) optional but useful

View File

@ -194,30 +194,6 @@ minikube image load logos-blockchain-testing:local
- Resource isolation - Resource isolation
- Large topologies - Large topologies
## Critical Environment Variable
**`POL_PROOF_DEV_MODE=true` is REQUIRED for ALL runners!**
Without this, proof generation uses expensive Groth16 proving, causing:
- Tests "hang" for minutes
- CPU spikes to 100%
- Timeouts and failures
**Always set:**
```bash
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
POL_PROOF_DEV_MODE=true scripts/run/run-examples.sh -t 60 -n 3 compose
# etc.
```
**Or add to your shell profile:**
```bash
# ~/.bashrc or ~/.zshrc
export POL_PROOF_DEV_MODE=true
```
## Quick Setup Check ## Quick Setup Check
Run this checklist before your first scenario: Run this checklist before your first scenario:
@ -229,16 +205,13 @@ cat versions.env
# 2. Check circuit assets # 2. Check circuit assets
ls -lh "${HOME}/.logos-blockchain-circuits" ls -lh "${HOME}/.logos-blockchain-circuits"
# 3. Verify POL_PROOF_DEV_MODE is set # 3. For compose/k8s: verify Docker is running
echo $POL_PROOF_DEV_MODE # Should print: true
# 4. For compose/k8s: verify Docker is running
docker ps docker ps
# 5. For compose/k8s: verify image exists # 4. For compose/k8s: verify image exists
docker images | grep logos-blockchain-testing docker images | grep logos-blockchain-testing
# 6. For host runner: verify node binaries (if not using scripts) # 5. For host runner: verify node binaries (if not using scripts)
$LOGOS_BLOCKCHAIN_NODE_BIN --version $LOGOS_BLOCKCHAIN_NODE_BIN --version
``` ```

View File

@ -16,7 +16,7 @@ git clone https://github.com/logos-blockchain/logos-blockchain-testing.git
cd logos-blockchain-testing cd logos-blockchain-testing
# 3. Run your first scenario (downloads dependencies automatically) # 3. Run your first scenario (downloads dependencies automatically)
POL_PROOF_DEV_MODE=true scripts/run/run-examples.sh -t 60 -n 1 host scripts/run/run-examples.sh -t 60 -n 1 host
``` ```
**First run takes 5-10 minutes** (downloads ~120MB circuit assets, builds binaries). **First run takes 5-10 minutes** (downloads ~120MB circuit assets, builds binaries).
@ -56,7 +56,7 @@ This handles circuit setup, binary building, and runs a complete scenario: 1 nod
```bash ```bash
# Requires circuits in place and LOGOS_BLOCKCHAIN_NODE_BIN set # Requires circuits in place and LOGOS_BLOCKCHAIN_NODE_BIN set
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
``` ```
**Core API Pattern** (simplified example): **Core API Pattern** (simplified example):
@ -92,8 +92,6 @@ pub async fn run_local_demo() -> Result<()> {
**Note:** The examples are binaries with `#[tokio::main]`, not test functions. If you want to write integration tests, wrap this pattern in `#[tokio::test]` functions in your own test suite. **Note:** The examples are binaries with `#[tokio::main]`, not test functions. If you want to write integration tests, wrap this pattern in `#[tokio::test]` functions in your own test suite.
**Important:** `POL_PROOF_DEV_MODE=true` disables expensive Groth16 zero-knowledge proof generation for leader election. Without it, proof generation is CPU-intensive and tests will timeout. **This is required for all runners** (local, compose, k8s) for practical testing. Never use in production.
**What you should see:** **What you should see:**
- Nodes spawn as local processes - Nodes spawn as local processes
- Consensus starts producing blocks - Consensus starts producing blocks
@ -213,7 +211,6 @@ scripts/run/run-examples.sh -t 120 -n 3 host
# Uses LOGOS_BLOCKCHAIN_DEMO_* env vars (or legacy *_DEMO_* vars) # Uses LOGOS_BLOCKCHAIN_DEMO_* env vars (or legacy *_DEMO_* vars)
LOGOS_BLOCKCHAIN_DEMO_NODES=3 \ LOGOS_BLOCKCHAIN_DEMO_NODES=3 \
LOGOS_BLOCKCHAIN_DEMO_RUN_SECS=120 \ LOGOS_BLOCKCHAIN_DEMO_RUN_SECS=120 \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
``` ```
@ -246,7 +243,6 @@ scripts/build/build_test_image.sh
# Run with Compose # Run with Compose
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \ LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin compose_runner cargo run -p runner-examples --bin compose_runner
``` ```

View File

@ -4,8 +4,6 @@ Runners turn a scenario plan into a live environment while keeping the plan
unchanged. Choose based on feedback speed, reproducibility, and fidelity. For unchanged. Choose based on feedback speed, reproducibility, and fidelity. For
environment and operational considerations, see [Operations Overview](operations-overview.md). environment and operational considerations, see [Operations Overview](operations-overview.md).
**Important:** All runners require `POL_PROOF_DEV_MODE=true` to avoid expensive Groth16 proof generation that causes timeouts.
## Host runner (local processes) ## Host runner (local processes)
- Launches node processes directly on the host (via `LocalDeployer`). - Launches node processes directly on the host (via `LocalDeployer`).
- Binary: `local_runner.rs`, script mode: `host` - Binary: `local_runner.rs`, script mode: `host`

View File

@ -99,7 +99,6 @@ scripts/ops/clean.sh --docker
For manual control, run the `local_runner` binary directly: For manual control, run the `local_runner` binary directly:
```bash ```bash
POL_PROOF_DEV_MODE=true \
LOGOS_BLOCKCHAIN_NODE_BIN=/path/to/logos-blockchain-node \ LOGOS_BLOCKCHAIN_NODE_BIN=/path/to/logos-blockchain-node \
cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
``` ```
@ -116,7 +115,6 @@ cargo run -p runner-examples --bin local_runner
| `LOGOS_BLOCKCHAIN_TESTS_TRACING` | false | Enable debug tracing preset | | `LOGOS_BLOCKCHAIN_TESTS_TRACING` | false | Enable debug tracing preset |
| `LOGOS_BLOCKCHAIN_LOG_LEVEL` | info | Global log level: error, warn, info, debug, trace | | `LOGOS_BLOCKCHAIN_LOG_LEVEL` | info | Global log level: error, warn, info, debug, trace |
| `LOGOS_BLOCKCHAIN_LOG_FILTER` | None | Fine-grained module filtering (e.g., `cryptarchia=trace`) | | `LOGOS_BLOCKCHAIN_LOG_FILTER` | None | Fine-grained module filtering (e.g., `cryptarchia=trace`) |
| `POL_PROOF_DEV_MODE` | — | **REQUIRED**: Set to `true` for all runners |
**Note:** Requires circuit assets and host binaries. Use `scripts/run/run-examples.sh host` to handle setup automatically. **Note:** Requires circuit assets and host binaries. Use `scripts/run/run-examples.sh host` to handle setup automatically.
@ -139,7 +137,6 @@ scripts/build/build_test_image.sh
# 3. Run # 3. Run
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \ LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin compose_runner cargo run -p runner-examples --bin compose_runner
``` ```
@ -154,7 +151,6 @@ scripts/build/build_test_image.sh
# Run # Run
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \ LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin compose_runner cargo run -p runner-examples --bin compose_runner
``` ```
@ -169,7 +165,6 @@ cargo run -p runner-examples --bin compose_runner
| Variable | Default | Effect | | Variable | Default | Effect |
|----------|---------|--------| |----------|---------|--------|
| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | — | Image tag (required, must match built image) | | `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | — | Image tag (required, must match built image) |
| `POL_PROOF_DEV_MODE` | — | **REQUIRED**: Set to `true` for all runners |
| `LOGOS_BLOCKCHAIN_DEMO_NODES` | 1 | Number of nodes | | `LOGOS_BLOCKCHAIN_DEMO_NODES` | 1 | Number of nodes |
| `LOGOS_BLOCKCHAIN_DEMO_RUN_SECS` | 60 | Run duration in seconds | | `LOGOS_BLOCKCHAIN_DEMO_RUN_SECS` | 60 | Run duration in seconds |
| `COMPOSE_NODE_PAIRS` | — | Alternative topology format: "nodes" (e.g., `3`) | | `COMPOSE_NODE_PAIRS` | — | Alternative topology format: "nodes" (e.g., `3`) |
@ -232,7 +227,6 @@ export LOGOS_BLOCKCHAIN_TESTNET_IMAGE=your-registry/logos-blockchain-testing:lat
```bash ```bash
export LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local export LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local
export POL_PROOF_DEV_MODE=true
cargo run -p runner-examples --bin k8s_runner cargo run -p runner-examples --bin k8s_runner
``` ```
@ -241,7 +235,6 @@ cargo run -p runner-examples --bin k8s_runner
| Variable | Default | Effect | | Variable | Default | Effect |
|----------|---------|--------| |----------|---------|--------|
| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | — | Image tag (required) | | `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | — | Image tag (required) |
| `POL_PROOF_DEV_MODE` | — | **REQUIRED**: Set to `true` for all runners |
| `LOGOS_BLOCKCHAIN_DEMO_NODES` | 1 | Number of nodes | | `LOGOS_BLOCKCHAIN_DEMO_NODES` | 1 | Number of nodes |
| `LOGOS_BLOCKCHAIN_DEMO_RUN_SECS` | 60 | Run duration in seconds | | `LOGOS_BLOCKCHAIN_DEMO_RUN_SECS` | 60 | Run duration in seconds |
| `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` | None | Prometheus-compatible base URL for runner to query (PromQL) | | `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` | None | Prometheus-compatible base URL for runner to query (PromQL) |

View File

@ -69,7 +69,6 @@ Notes:
Run the built-in local examples: Run the built-in local examples:
```bash ```bash
POL_PROOF_DEV_MODE=true \
scripts/run/run-examples.sh -t 60 -n 3 host scripts/run/run-examples.sh -t 60 -n 3 host
``` ```
@ -82,7 +81,6 @@ scripts/run/run-examples.sh -t 60 -n 3 host
Run the built-in compose examples: Run the built-in compose examples:
```bash ```bash
POL_PROOF_DEV_MODE=true \
scripts/run/run-examples.sh -t 60 -n 3 compose scripts/run/run-examples.sh -t 60 -n 3 compose
``` ```
@ -95,7 +93,6 @@ scripts/run/run-examples.sh -t 60 -n 3 compose
Run the built-in k8s examples: Run the built-in k8s examples:
```bash ```bash
POL_PROOF_DEV_MODE=true \
scripts/run/run-examples.sh -t 60 -n 3 k8s scripts/run/run-examples.sh -t 60 -n 3 k8s
``` ```

View File

@ -2,7 +2,6 @@
**Prerequisites for All Runners:** **Prerequisites for All Runners:**
- **`versions.env` file** at repository root (required by helper scripts) - **`versions.env` file** at repository root (required by helper scripts)
- **`POL_PROOF_DEV_MODE=true`** MUST be set for all runners (host, compose, k8s) to avoid expensive Groth16 proof generation that causes timeouts
- **Circuit assets** must be present and `LOGOS_BLOCKCHAIN_CIRCUITS` must point to a directory that contains them - **Circuit assets** must be present and `LOGOS_BLOCKCHAIN_CIRCUITS` must point to a directory that contains them
**Platform/Environment Notes:** **Platform/Environment Notes:**
@ -18,7 +17,6 @@
Common symptoms and likely causes: Common symptoms and likely causes:
- **No or slow block progression**: missing `POL_PROOF_DEV_MODE=true`, missing circuit assets, too-short run window, port conflicts, or resource exhaustion—set required env vars, verify assets exist, extend duration, check node logs for startup errors.
- **Transactions not included**: unfunded or misconfigured wallets (check `.wallets(N)` vs `.users(M)`), transaction rate exceeding block capacity, or rates exceeding block production speed—reduce rate, increase wallet count, verify wallet setup in logs. - **Transactions not included**: unfunded or misconfigured wallets (check `.wallets(N)` vs `.users(M)`), transaction rate exceeding block capacity, or rates exceeding block production speed—reduce rate, increase wallet count, verify wallet setup in logs.
- **Chaos stalls the run**: chaos (node control) only works with ComposeDeployer; host runner (LocalDeployer) and K8sDeployer don't support it (won't "stall", just can't execute chaos workloads). With compose, aggressive restart cadence can prevent consensus recovery—widen restart intervals. - **Chaos stalls the run**: chaos (node control) only works with ComposeDeployer; host runner (LocalDeployer) and K8sDeployer don't support it (won't "stall", just can't execute chaos workloads). With compose, aggressive restart cadence can prevent consensus recovery—widen restart intervals.
- **Observability gaps**: metrics or logs unreachable because ports clash or services are not exposed—adjust observability ports and confirm runner wiring. - **Observability gaps**: metrics or logs unreachable because ports clash or services are not exposed—adjust observability ports and confirm runner wiring.
@ -28,40 +26,7 @@ Common symptoms and likely causes:
This section shows what you'll actually see when common issues occur. Each example includes realistic console output and the fix. This section shows what you'll actually see when common issues occur. Each example includes realistic console output and the fix.
### 1. Missing `POL_PROOF_DEV_MODE=true` (Most Common!) ### 1. Missing `versions.env` File
**Symptoms:**
- Test "hangs" with no visible progress
- CPU usage spikes to 100%
- Eventually hits timeout after several minutes
- Nodes appear to start but blocks aren't produced
**What you'll see:**
```text
$ cargo run -p runner-examples --bin local_runner
Finished dev [unoptimized + debuginfo] target(s) in 0.48s
Running `target/debug/local_runner`
[INFO runner_examples::local_runner] Starting local runner scenario
[INFO testing_framework_runner_local] Launching 3 nodes
[INFO testing_framework_runner_local] Waiting for node readiness...
(hangs here for 5+ minutes, CPU at 100%)
thread 'main' panicked at 'readiness timeout expired'
```
**Root Cause:** Groth16 proof generation is extremely slow without dev mode. The system tries to compute real cryptographic proofs, which can take minutes per block.
**Fix:**
```bash
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
```
**Prevention:** Set this in your shell profile or `.env` file so you never forget it.
---
### 2. Missing `versions.env` File
**Symptoms:** **Symptoms:**
- Helper scripts fail immediately - Helper scripts fail immediately
@ -93,7 +58,7 @@ cat versions.env
--- ---
### 3. Missing Circuit Assets ### 2. Missing Circuit Assets
**Symptoms:** **Symptoms:**
- Node startup fails early - Node startup fails early
@ -102,7 +67,7 @@ cat versions.env
**What you'll see:** **What you'll see:**
```text ```text
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner $ cargo run -p runner-examples --bin local_runner
[INFO testing_framework_runner_local] Starting local runner scenario [INFO testing_framework_runner_local] Starting local runner scenario
Error: circuit assets directory missing or invalid Error: circuit assets directory missing or invalid
thread 'main' panicked at 'workload init failed' thread 'main' panicked at 'workload init failed'
@ -129,7 +94,7 @@ export LOGOS_BLOCKCHAIN_CIRCUITS=$HOME/.logos-blockchain-circuits
--- ---
### 4. Node Binaries Not Found ### 3. Node Binaries Not Found
**Symptoms:** **Symptoms:**
- Error about missing `logos-blockchain-node` binary - Error about missing `logos-blockchain-node` binary
@ -139,7 +104,7 @@ export LOGOS_BLOCKCHAIN_CIRCUITS=$HOME/.logos-blockchain-circuits
**What you'll see:** **What you'll see:**
```text ```text
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner $ cargo run -p runner-examples --bin local_runner
[INFO testing_framework_runner_local] Spawning node 0 [INFO testing_framework_runner_local] Spawning node 0
Error: Os { code: 2, kind: NotFound, message: "No such file or directory" } Error: Os { code: 2, kind: NotFound, message: "No such file or directory" }
thread 'main' panicked at 'failed to spawn logos-blockchain-node process' thread 'main' panicked at 'failed to spawn logos-blockchain-node process'
@ -166,12 +131,12 @@ export LOGOS_BLOCKCHAIN_NODE_BIN=$PWD/target/release/logos-blockchain-node
# Return to testing framework # Return to testing framework
cd ../nomos-testing cd ../nomos-testing
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
``` ```
--- ---
### 5. Docker Daemon Not Running (Compose) ### 4. Docker Daemon Not Running (Compose)
**Symptoms:** **Symptoms:**
- Compose tests fail immediately - Compose tests fail immediately
@ -208,7 +173,7 @@ sudo usermod -aG docker $USER
--- ---
### 6. Image Not Found (Compose/K8s) ### 5. Image Not Found (Compose/K8s)
**Symptoms:** **Symptoms:**
- Compose/K8s tests fail during deployment - Compose/K8s tests fail during deployment
@ -218,7 +183,7 @@ sudo usermod -aG docker $USER
**What you'll see:** **What you'll see:**
```text ```text
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin compose_runner $ cargo run -p runner-examples --bin compose_runner
[INFO testing_framework_runner_compose] Starting compose deployment [INFO testing_framework_runner_compose] Starting compose deployment
Error: Failed to pull image 'logos-blockchain-testing:local': No such image Error: Failed to pull image 'logos-blockchain-testing:local': No such image
thread 'main' panicked at 'compose deployment failed' thread 'main' panicked at 'compose deployment failed'
@ -255,7 +220,7 @@ kind load docker-image logos-blockchain-testing:local
--- ---
### 7. Port Conflicts ### 6. Port Conflicts
**Symptoms:** **Symptoms:**
- "Address already in use" errors - "Address already in use" errors
@ -265,7 +230,7 @@ kind load docker-image logos-blockchain-testing:local
**What you'll see:** **What you'll see:**
```text ```text
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner $ cargo run -p runner-examples --bin local_runner
[INFO testing_framework_runner_local] Launching node 0 on port 18080 [INFO testing_framework_runner_local] Launching node 0 on port 18080
Error: Os { code: 48, kind: AddrInUse, message: "Address already in use" } Error: Os { code: 48, kind: AddrInUse, message: "Address already in use" }
thread 'main' panicked at 'failed to bind port 18080' thread 'main' panicked at 'failed to bind port 18080'
@ -305,7 +270,7 @@ vim scripts/observability/compose/docker-compose.yml
--- ---
### 8. Wallet Seeding Failed (Insufficient Funds) ### 7. Wallet Seeding Failed (Insufficient Funds)
**Symptoms:** **Symptoms:**
- Transaction workload reports wallet issues - Transaction workload reports wallet issues
@ -315,7 +280,7 @@ vim scripts/observability/compose/docker-compose.yml
**What you'll see:** **What you'll see:**
```text ```text
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner $ cargo run -p runner-examples --bin local_runner
[INFO testing_framework_workflows] Starting transaction workload with 10 users [INFO testing_framework_workflows] Starting transaction workload with 10 users
[ERROR testing_framework_workflows] Wallet seeding failed: requested 10 users but only 3 wallets available [ERROR testing_framework_workflows] Wallet seeding failed: requested 10 users but only 3 wallets available
thread 'main' panicked at 'workload init failed: insufficient wallets' thread 'main' panicked at 'workload init failed: insufficient wallets'
@ -340,7 +305,7 @@ let scenario = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3))
--- ---
### 9. Resource Exhaustion (OOM / CPU) ### 8. Resource Exhaustion (OOM / CPU)
**Symptoms:** **Symptoms:**
- Nodes crash randomly - Nodes crash randomly
@ -385,7 +350,7 @@ ulimit -n 4096
--- ---
### 10. Logs Disappear After Run ### 9. Logs Disappear After Run
**Symptoms:** **Symptoms:**
- Test completes but no logs on disk - Test completes but no logs on disk
@ -395,7 +360,7 @@ ulimit -n 4096
**What you'll see:** **What you'll see:**
```text ```text
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner $ cargo run -p runner-examples --bin local_runner
[INFO runner_examples] Test complete, cleaning up [INFO runner_examples] Test complete, cleaning up
[INFO testing_framework_runner_local] Removing temporary directories [INFO testing_framework_runner_local] Removing temporary directories
$ ls .tmp/ $ ls .tmp/
@ -410,7 +375,6 @@ $ ls .tmp/
# Persist logs to a specific directory # Persist logs to a specific directory
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \ LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \
LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1 \ LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1 \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
# Logs persist after run # Logs persist after run
@ -422,7 +386,7 @@ ls /tmp/test-logs/
--- ---
### 11. Consensus Timing Too Tight / Run Duration Too Short ### 10. Consensus Timing Too Tight / Run Duration Too Short
**Symptoms:** **Symptoms:**
- "Consensus liveness expectation failed" - "Consensus liveness expectation failed"
@ -432,7 +396,7 @@ ls /tmp/test-logs/
**What you'll see:** **What you'll see:**
```text ```text
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner $ cargo run -p runner-examples --bin local_runner
[INFO testing_framework_core] Starting workloads [INFO testing_framework_core] Starting workloads
[INFO testing_framework_core] Run window: 10 seconds [INFO testing_framework_core] Run window: 10 seconds
[INFO testing_framework_core] Evaluating expectations [INFO testing_framework_core] Evaluating expectations
@ -463,7 +427,6 @@ let scenario = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3))
# Faster block production (shorter slot time) # Faster block production (shorter slot time)
CONSENSUS_SLOT_TIME=5 \ CONSENSUS_SLOT_TIME=5 \
CONSENSUS_ACTIVE_SLOT_COEFF=0.9 \ CONSENSUS_ACTIVE_SLOT_COEFF=0.9 \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
``` ```
@ -473,17 +436,16 @@ cargo run -p runner-examples --bin local_runner
When a test fails, check these in order: When a test fails, check these in order:
1. **`POL_PROOF_DEV_MODE=true` is set** (REQUIRED for all runners) 1. **`versions.env` exists at repo root**
2. **`versions.env` exists at repo root** 2. **Circuit assets present** (`LOGOS_BLOCKCHAIN_CIRCUITS` points to a valid directory)
3. **Circuit assets present** (`LOGOS_BLOCKCHAIN_CIRCUITS` points to a valid directory) 3. **Node binaries available** (`LOGOS_BLOCKCHAIN_NODE_BIN` set, or using `run-examples.sh`)
4. **Node binaries available** (`LOGOS_BLOCKCHAIN_NODE_BIN` set, or using `run-examples.sh`) 4. **Docker daemon running** (for compose/k8s)
5. **Docker daemon running** (for compose/k8s) 5. **Docker image built** (`logos-blockchain-testing:local` exists for compose/k8s)
6. **Docker image built** (`logos-blockchain-testing:local` exists for compose/k8s) 6. **No port conflicts** (`lsof -i :18080`, kill orphaned processes)
7. **No port conflicts** (`lsof -i :18080`, kill orphaned processes) 7. **Sufficient wallets** (`.wallets(N)``.users(M)`)
8. **Sufficient wallets** (`.wallets(N)``.users(M)`) 8. **Enough resources** (Docker memory 8GB+, ulimit -n 4096)
9. **Enough resources** (Docker memory 8GB+, ulimit -n 4096) 9. **Run duration appropriate** (long enough for consensus timing)
10. **Run duration appropriate** (long enough for consensus timing) 10. **Logs persisted** (`LOGOS_BLOCKCHAIN_LOG_DIR` + `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1` if needed)
11. **Logs persisted** (`LOGOS_BLOCKCHAIN_LOG_DIR` + `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1` if needed)
**Still stuck?** Check node logs (see [Where to Find Logs](#where-to-find-logs)) for the actual error. **Still stuck?** Check node logs (see [Where to Find Logs](#where-to-find-logs)) for the actual error.
@ -509,14 +471,13 @@ When a test fails, check these in order:
**Console output (default):** **Console output (default):**
```bash ```bash
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner 2>&1 | tee test.log cargo run -p runner-examples --bin local_runner 2>&1 | tee test.log
``` ```
**Persistent file output:** **Persistent file output:**
```bash ```bash
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/debug-logs \ LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/debug-logs \
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \ LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin local_runner cargo run -p runner-examples --bin local_runner
# Inspect logs (note: filenames include timestamps): # Inspect logs (note: filenames include timestamps):
@ -546,7 +507,6 @@ docker logs --tail 100 <container-id>
```bash ```bash
COMPOSE_RUNNER_PRESERVE=1 \ COMPOSE_RUNNER_PRESERVE=1 \
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \ LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
POL_PROOF_DEV_MODE=true \
cargo run -p runner-examples --bin compose_runner cargo run -p runner-examples --bin compose_runner
# OR: Use run-examples.sh (handles setup automatically) # OR: Use run-examples.sh (handles setup automatically)
@ -651,7 +611,6 @@ Focus on the first node that exhibited problems or the node with the highest ind
- "ERROR: versions.env missing" → missing required `versions.env` file at repository root - "ERROR: versions.env missing" → missing required `versions.env` file at repository root
- "Failed to bind address" → port conflict - "Failed to bind address" → port conflict
- "Connection refused" → peer not ready or network issue - "Connection refused" → peer not ready or network issue
- "Proof verification failed" or "Proof generation timeout" → missing `POL_PROOF_DEV_MODE=true` (REQUIRED for all runners)
- "Circuit file not found" → missing circuit assets at the path in `LOGOS_BLOCKCHAIN_CIRCUITS` - "Circuit file not found" → missing circuit assets at the path in `LOGOS_BLOCKCHAIN_CIRCUITS`
- "Insufficient funds" → wallet seeding issue (increase `.wallets(N)` or reduce `.users(M)`) - "Insufficient funds" → wallet seeding issue (increase `.wallets(N)` or reduce `.users(M)`)
@ -689,16 +648,14 @@ Run a minimal baseline test (e.g., 2 nodes, consensus liveness only). If it pass
### "Consensus liveness expectation failed" ### "Consensus liveness expectation failed"
- **Cause**: Not enough blocks produced during the run window, missing - **Cause**: Not enough blocks produced during the run window, missing circuit
`POL_PROOF_DEV_MODE=true` (causes slow proof generation), or missing circuit
assets. assets.
- **Fix**: - **Fix**:
1. Verify `POL_PROOF_DEV_MODE=true` is set (REQUIRED for all runners). 1. Verify circuit assets exist at the path referenced by
2. Verify circuit assets exist at the path referenced by
`LOGOS_BLOCKCHAIN_CIRCUITS`. `LOGOS_BLOCKCHAIN_CIRCUITS`.
3. Extend `with_run_duration()` to allow more blocks. 2. Extend `with_run_duration()` to allow more blocks.
4. Check node logs for proof generation or circuit asset errors. 3. Check node logs for proof generation or circuit asset errors.
5. Reduce transaction rate if nodes are overwhelmed. 4. Reduce transaction rate if nodes are overwhelmed.
### "Wallet seeding failed" ### "Wallet seeding failed"
@ -720,11 +677,10 @@ Run a minimal baseline test (e.g., 2 nodes, consensus liveness only). If it pass
- **Cause**: Nodes didn't become responsive within expected time (often due to - **Cause**: Nodes didn't become responsive within expected time (often due to
missing prerequisites). missing prerequisites).
- **Fix**: - **Fix**:
1. **Verify `POL_PROOF_DEV_MODE=true` is set** (REQUIRED for all runners—without
it, proof generation is too slow). it, proof generation is too slow).
2. Check node logs for startup errors (port conflicts, missing assets). 1. Check node logs for startup errors (port conflicts, missing assets).
3. Verify network connectivity between nodes. 2. Verify network connectivity between nodes.
4. Ensure circuit assets are present and `LOGOS_BLOCKCHAIN_CIRCUITS` points to them. 3. Ensure circuit assets are present and `LOGOS_BLOCKCHAIN_CIRCUITS` points to them.
### "ERROR: versions.env missing" ### "ERROR: versions.env missing"

View File

@ -78,13 +78,7 @@ ScenarioBuilder::topology_with(|t| t.network_star().nodes(3))
``` ```
The workload will fail during `init()` if no wallets are configured. The workload will fail during `init()` if no wallets are configured.
2. **Proof generation must be fast:** 2. **Circuit artifacts must be available:**
```bash
export POL_PROOF_DEV_MODE=true
```
Without this, proof generation takes ~30-60 seconds per transaction, causing timeouts.
3. **Circuit artifacts must be available:**
- Automatically staged by `scripts/run/run-examples.sh` - Automatically staged by `scripts/run/run-examples.sh`
- Or manually via `scripts/setup/setup-logos-blockchain-circuits.sh` (recommended) / `scripts/setup/setup-logos-blockchain-circuits.sh` - Or manually via `scripts/setup/setup-logos-blockchain-circuits.sh` (recommended) / `scripts/setup/setup-logos-blockchain-circuits.sh`
@ -108,7 +102,6 @@ Error: Expectation failed: TxInclusionExpectation
Observed: 127 transactions Observed: 127 transactions
Possible causes: Possible causes:
- POL_PROOF_DEV_MODE not set (proof generation too slow)
- Duration too short (nodes still syncing) - Duration too short (nodes still syncing)
- Node crashes (check logs for panics/OOM) - Node crashes (check logs for panics/OOM)
- Wallet accounts not seeded (check topology config) - Wallet accounts not seeded (check topology config)
@ -119,9 +112,8 @@ Error: Expectation failed: TxInclusionExpectation
```bash ```bash
grep "proof generation" $LOGOS_BLOCKCHAIN_LOG_DIR/*/*.log grep "proof generation" $LOGOS_BLOCKCHAIN_LOG_DIR/*/*.log
``` ```
2. Verify `POL_PROOF_DEV_MODE=true` was set 2. Increase duration: `.with_run_duration(Duration::from_secs(120))`
3. Increase duration: `.with_run_duration(Duration::from_secs(120))` 3. Reduce rate: `.rate(5)` instead of `.rate(10)`
4. Reduce rate: `.rate(5)` instead of `.rate(10)`
--- ---
@ -382,12 +374,11 @@ ScenarioBuilder::topology_with(|t| t.network_star().nodes(3))
When a workload or expectation fails: When a workload or expectation fails:
1. Check logs: `$LOGOS_BLOCKCHAIN_LOG_DIR/*/` or `docker compose logs` or `kubectl logs` 1. Check logs: `$LOGOS_BLOCKCHAIN_LOG_DIR/*/` or `docker compose logs` or `kubectl logs`
2. Verify environment variables: `POL_PROOF_DEV_MODE`, `LOGOS_BLOCKCHAIN_NODE_BIN`, etc. 2. Check prerequisites: wallets, node control, circuits
3. Check prerequisites: wallets, node control, circuits 3. Increase duration: Double the run duration and retry
4. Increase duration: Double the run duration and retry 4. Reduce rates: Half the traffic rates and retry
5. Reduce rates: Half the traffic rates and retry 5. Check metrics: Prometheus queries for block height and tx count
6. Check metrics: Prometheus queries for block height and tx count 6. Reproduce locally: Use local runner for faster iteration
7. Reproduce locally: Use local runner for faster iteration
--- ---

View File

@ -1,4 +1,4 @@
use std::{env, process, time::Duration}; use std::{process, time::Duration};
use anyhow::{Context as _, Result}; use anyhow::{Context as _, Result};
use runner_examples::{DeployerKind, ScenarioBuilderExt as _, demo, read_env_any}; use runner_examples::{DeployerKind, ScenarioBuilderExt as _, demo, read_env_any};
@ -17,11 +17,6 @@ async fn main() {
tracing_subscriber::fmt::init(); tracing_subscriber::fmt::init();
if env::var("POL_PROOF_DEV_MODE").is_err() {
warn!("POL_PROOF_DEV_MODE=true is required for the local runner demo");
process::exit(1);
}
let nodes = read_env_any(&["LOGOS_BLOCKCHAIN_DEMO_NODES"], demo::DEFAULT_NODES); let nodes = read_env_any(&["LOGOS_BLOCKCHAIN_DEMO_NODES"], demo::DEFAULT_NODES);
let run_secs = read_env_any(&["LOGOS_BLOCKCHAIN_DEMO_RUN_SECS"], demo::DEFAULT_RUN_SECS); let run_secs = read_env_any(&["LOGOS_BLOCKCHAIN_DEMO_RUN_SECS"], demo::DEFAULT_RUN_SECS);

View File

@ -21,7 +21,6 @@ fn set_default_env(key: &str, value: &str) {
} }
pub fn init_logging_defaults() { pub fn init_logging_defaults() {
set_default_env("POL_PROOF_DEV_MODE", "true");
set_default_env("LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS", "1"); set_default_env("LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS", "1");
set_default_env("LOGOS_BLOCKCHAIN_LOG_LEVEL", "info"); set_default_env("LOGOS_BLOCKCHAIN_LOG_LEVEL", "info");
set_default_env("RUST_LOG", "info"); set_default_env("RUST_LOG", "info");

View File

@ -86,6 +86,7 @@ impl Workload for JoinNodeWithPeersWorkload {
let options = StartNodeOptions { let options = StartNodeOptions {
peers: PeerSelection::Named(self.peers.clone()), peers: PeerSelection::Named(self.peers.clone()),
config_patch: None, config_patch: None,
persist_dir: None,
}; };
let node = handle.start_node_with(&self.name, options).await?; let node = handle.start_node_with(&self.name, options).await?;
let client = node.api; let client = node.api;

View File

@ -18,7 +18,6 @@ const CONVERGENCE_POLL: Duration = Duration::from_secs(2);
async fn manual_cluster_two_clusters_merge() -> Result<()> { async fn manual_cluster_two_clusters_merge() -> Result<()> {
let _ = try_init(); let _ = try_init();
// Required env vars (set on the command line when running this test): // Required env vars (set on the command line when running this test):
// - `POL_PROOF_DEV_MODE=true`
// - `RUST_LOG=info` (optional) // - `RUST_LOG=info` (optional)
let config = TopologyConfig::with_node_numbers(2); let config = TopologyConfig::with_node_numbers(2);
let deployer = LocalDeployer::new(); let deployer = LocalDeployer::new();
@ -33,6 +32,7 @@ async fn manual_cluster_two_clusters_merge() -> Result<()> {
StartNodeOptions { StartNodeOptions {
peers: PeerSelection::None, peers: PeerSelection::None,
config_patch: None, config_patch: None,
persist_dir: None,
}, },
) )
.await? .await?
@ -48,6 +48,7 @@ async fn manual_cluster_two_clusters_merge() -> Result<()> {
StartNodeOptions { StartNodeOptions {
peers: PeerSelection::Named(vec!["node-a".to_owned()]), peers: PeerSelection::Named(vec!["node-a".to_owned()]),
config_patch: None, config_patch: None,
persist_dir: None,
}, },
) )
.await? .await?
@ -82,3 +83,57 @@ async fn manual_cluster_two_clusters_merge() -> Result<()> {
sleep(CONVERGENCE_POLL).await; sleep(CONVERGENCE_POLL).await;
} }
} }
#[tokio::test]
#[ignore = "run manually with `cargo test -p runner-examples -- --ignored manual_cluster_with_persist_dir`"]
async fn manual_cluster_with_persist_dir() -> Result<()> {
use std::path::PathBuf;
let _ = try_init();
// Required env vars (set on the command line when running this test):
// - `RUST_LOG=info` (optional)
let config = TopologyConfig::with_node_numbers(1);
let deployer = LocalDeployer::new();
let cluster = deployer.manual_cluster(config)?;
let persist_dir = PathBuf::from("/tmp/test-node-persist-dir");
println!("starting validator with persist_dir: {:?}", persist_dir);
let _node = cluster
.start_node_with(
"test",
StartNodeOptions {
peers: PeerSelection::None,
config_patch: None,
persist_dir: Some(persist_dir.clone()),
},
)
.await?
.api;
println!("validator started, waiting briefly");
sleep(Duration::from_secs(5)).await;
// Drop the cluster to trigger the persist logic
drop(cluster);
println!("cluster dropped, checking if persist_dir exists");
// Verify the persist_dir was created
if !persist_dir.exists() {
return Err(anyhow::anyhow!(
"persist_dir was not created: {:?}",
persist_dir
));
}
println!("persist_dir verified: {:?}", persist_dir);
// Clean up
if persist_dir.exists() {
std::fs::remove_dir_all(&persist_dir)?;
}
Ok(())
}

View File

@ -17,7 +17,6 @@ use tracing_subscriber::fmt::try_init;
async fn manual_cluster_api_port_override() -> Result<()> { async fn manual_cluster_api_port_override() -> Result<()> {
let _ = try_init(); let _ = try_init();
// Required env vars (set on the command line when running this test): // Required env vars (set on the command line when running this test):
// - `POL_PROOF_DEV_MODE=true`
// - `LOGOS_BLOCKCHAIN_NODE_BIN=...` // - `LOGOS_BLOCKCHAIN_NODE_BIN=...`
// - `LOGOS_BLOCKCHAIN_CIRCUITS=...` // - `LOGOS_BLOCKCHAIN_CIRCUITS=...`
// - `RUST_LOG=info` (optional) // - `RUST_LOG=info` (optional)
@ -33,6 +32,7 @@ async fn manual_cluster_api_port_override() -> Result<()> {
StartNodeOptions { StartNodeOptions {
peers: PeerSelection::None, peers: PeerSelection::None,
config_patch: None, config_patch: None,
persist_dir: None,
} }
.create_patch(move |mut config| { .create_patch(move |mut config| {
println!("overriding API port to {api_port}"); println!("overriding API port to {api_port}");
@ -62,7 +62,6 @@ async fn manual_cluster_api_port_override() -> Result<()> {
async fn scenario_builder_api_port_override() -> Result<()> { async fn scenario_builder_api_port_override() -> Result<()> {
let _ = try_init(); let _ = try_init();
// Required env vars (set on the command line when running this test): // Required env vars (set on the command line when running this test):
// - `POL_PROOF_DEV_MODE=true`
// - `LOGOS_BLOCKCHAIN_NODE_BIN=...` // - `LOGOS_BLOCKCHAIN_NODE_BIN=...`
// - `LOGOS_BLOCKCHAIN_CIRCUITS=...` // - `LOGOS_BLOCKCHAIN_CIRCUITS=...`
// - `RUST_LOG=info` (optional) // - `RUST_LOG=info` (optional)

View File

@ -25,7 +25,6 @@ const POLL_INTERVAL: Duration = Duration::from_secs(1);
async fn orphan_manual_cluster() -> Result<()> { async fn orphan_manual_cluster() -> Result<()> {
let _ = try_init(); let _ = try_init();
// Required env vars (set on the command line when running this test): // Required env vars (set on the command line when running this test):
// - `POL_PROOF_DEV_MODE=true`
// - `LOGOS_BLOCKCHAIN_NODE_BIN=...` // - `LOGOS_BLOCKCHAIN_NODE_BIN=...`
// - `NOMOS_KZGRS_PARAMS_PATH=...` (path to KZG params directory/file) // - `NOMOS_KZGRS_PARAMS_PATH=...` (path to KZG params directory/file)
// - `RUST_LOG=info` (optional; better visibility) // - `RUST_LOG=info` (optional; better visibility)

View File

@ -515,7 +515,6 @@ run_examples::run() {
echo "==> Running ${BIN} for ${RUN_SECS}s (mode=${MODE}, image=${IMAGE})" echo "==> Running ${BIN} for ${RUN_SECS}s (mode=${MODE}, image=${IMAGE})"
cd "${ROOT_DIR}" cd "${ROOT_DIR}"
POL_PROOF_DEV_MODE=true \
TESTNET_PRINT_ENDPOINTS=1 \ TESTNET_PRINT_ENDPOINTS=1 \
LOGOS_BLOCKCHAIN_TESTNET_IMAGE="${IMAGE}" \ LOGOS_BLOCKCHAIN_TESTNET_IMAGE="${IMAGE}" \
LOGOS_BLOCKCHAIN_NODE_BIN="${LOGOS_BLOCKCHAIN_NODE_BIN:-}" \ LOGOS_BLOCKCHAIN_NODE_BIN="${LOGOS_BLOCKCHAIN_NODE_BIN:-}" \

View File

@ -47,8 +47,7 @@ export CFG_FILE_PATH="/config.yaml" \
CFG_HOST_KIND="${CFG_HOST_KIND:-$role}" \ CFG_HOST_KIND="${CFG_HOST_KIND:-$role}" \
CFG_HOST_IDENTIFIER="${CFG_HOST_IDENTIFIER:-$host_identifier_default}" \ CFG_HOST_IDENTIFIER="${CFG_HOST_IDENTIFIER:-$host_identifier_default}" \
LOGOS_BLOCKCHAIN_TIME_BACKEND="${LOGOS_BLOCKCHAIN_TIME_BACKEND:-monotonic}" \ LOGOS_BLOCKCHAIN_TIME_BACKEND="${LOGOS_BLOCKCHAIN_TIME_BACKEND:-monotonic}" \
LOG_LEVEL="${LOG_LEVEL:-INFO}" \ LOG_LEVEL="${LOG_LEVEL:-INFO}"``"
POL_PROOF_DEV_MODE="${POL_PROOF_DEV_MODE:-true}"
# Ensure recovery directory exists to avoid early crashes in services that # Ensure recovery directory exists to avoid early crashes in services that
# persist state. # persist state.

View File

@ -13,7 +13,8 @@ use std::cmp;
use blend::GeneralBlendConfig; use blend::GeneralBlendConfig;
use consensus::{ use consensus::{
ConsensusConfigError, GeneralConsensusConfig, ProviderInfo, create_genesis_tx_with_declarations, ConsensusConfigError, GeneralConsensusConfig, ProviderInfo,
create_genesis_tx_with_declarations, sync_utxos_with_genesis,
}; };
use key_management_system_service::{backend::preload::PreloadKMSBackendSettings, keys::Key}; use key_management_system_service::{backend::preload::PreloadKMSBackendSettings, keys::Key};
use network::GeneralNetworkConfig; use network::GeneralNetworkConfig;
@ -203,7 +204,7 @@ fn apply_consensus_genesis_tx(
) -> Result<(), ConsensusConfigError> { ) -> Result<(), ConsensusConfigError> {
for c in consensus_configs { for c in consensus_configs {
c.genesis_tx = genesis_tx.clone(); c.genesis_tx = genesis_tx.clone();
consensus::sync_utxos_with_genesis(&mut c.utxos, genesis_tx)?; sync_utxos_with_genesis(&mut c.utxos, genesis_tx)?;
} }
Ok(()) Ok(())

View File

@ -71,15 +71,23 @@ pub struct NodeHandle<T> {
pub(crate) tempdir: TempDir, pub(crate) tempdir: TempDir,
pub(crate) config: T, pub(crate) config: T,
pub(crate) api: ApiClient, pub(crate) api: ApiClient,
pub(crate) persist_dir: Option<PathBuf>,
} }
impl<T> NodeHandle<T> { impl<T> NodeHandle<T> {
pub fn new(child: Child, tempdir: TempDir, config: T, api: ApiClient) -> Self { pub fn new(
child: Child,
tempdir: TempDir,
config: T,
api: ApiClient,
persist_dir: Option<PathBuf>,
) -> Self {
Self { Self {
child, child,
tempdir, tempdir,
config, config,
api, api,
persist_dir,
} }
} }
@ -154,6 +162,7 @@ pub async fn spawn_node<C>(
config_filename: &str, config_filename: &str,
binary_path: PathBuf, binary_path: PathBuf,
enable_logging: bool, enable_logging: bool,
persist_dir: Option<PathBuf>,
) -> Result<NodeHandle<C>, SpawnNodeError> ) -> Result<NodeHandle<C>, SpawnNodeError>
where where
C: NodeConfigCommon + Serialize, C: NodeConfigCommon + Serialize,
@ -168,7 +177,13 @@ where
let child = spawn_node_process(&binary_path, &config_path, dir.path())?; let child = spawn_node_process(&binary_path, &config_path, dir.path())?;
let mut handle = NodeHandle::new(child, dir, config, ApiClient::new(addr, testing_addr)); let mut handle = NodeHandle::new(
child,
dir,
config,
ApiClient::new(addr, testing_addr),
persist_dir,
);
// Wait for readiness via consensus_info // Wait for readiness via consensus_info
let ready = wait_for_consensus_readiness(&handle.api).await; let ready = wait_for_consensus_readiness(&handle.api).await;

View File

@ -7,6 +7,7 @@ use std::sync::LazyLock;
pub use api_client::{ApiClient, ApiClientError}; pub use api_client::{ApiClient, ApiClientError};
use tempfile::TempDir; use tempfile::TempDir;
use testing_framework_env as tf_env; use testing_framework_env as tf_env;
use tracing::info;
pub(crate) const LOGS_PREFIX: &str = "__logs"; pub(crate) const LOGS_PREFIX: &str = "__logs";
static KEEP_NODE_TEMPDIRS: LazyLock<bool> = LazyLock::new(tf_env::nomos_tests_keep_logs); static KEEP_NODE_TEMPDIRS: LazyLock<bool> = LazyLock::new(tf_env::nomos_tests_keep_logs);
@ -30,6 +31,55 @@ fn persist_tempdir(tempdir: &mut TempDir, label: &str) -> std::io::Result<()> {
Ok(()) Ok(())
} }
pub(crate) fn persist_tempdir_to(
tempdir: &mut TempDir,
target_dir: &std::path::Path,
label: &str,
) -> std::io::Result<()> {
use std::fs;
info!(
label,
from = %tempdir.path().display(),
to = %target_dir.display(),
"persisting directory"
);
// Create parent directory if it doesn't exist
if let Some(parent) = target_dir.parent() {
fs::create_dir_all(parent)?;
}
// Copy tempdir contents to target directory
if target_dir.exists() {
fs::remove_dir_all(target_dir)?;
}
/// Recursively copies all contents from src directory to dst directory.
///
/// # Arguments
/// * `src` - Source directory path to copy from
/// * `dst` - Destination directory path to copy to
fn copy_dir_all(src: &std::path::Path, dst: &std::path::Path) -> std::io::Result<()> {
use std::fs;
fs::create_dir_all(dst)?;
for entry in fs::read_dir(src)? {
let entry = entry?;
let ty = entry.file_type()?;
if ty.is_dir() {
copy_dir_all(&entry.path(), &dst.join(entry.file_name()))?;
} else {
fs::copy(entry.path(), dst.join(entry.file_name()))?;
}
}
Ok(())
}
copy_dir_all(tempdir.path(), target_dir)?;
Ok(())
}
pub(crate) fn should_persist_tempdir() -> bool { pub(crate) fn should_persist_tempdir() -> bool {
std::thread::panicking() || *KEEP_NODE_TEMPDIRS std::thread::panicking() || *KEEP_NODE_TEMPDIRS
} }

View File

@ -5,7 +5,7 @@ use nomos_tracing_service::LoggerLayer;
pub use testing_framework_config::nodes::node::create_node_config; pub use testing_framework_config::nodes::node::create_node_config;
use tracing::{debug, info}; use tracing::{debug, info};
use super::{persist_tempdir, should_persist_tempdir}; use super::{persist_tempdir, persist_tempdir_to, should_persist_tempdir};
use crate::{ use crate::{
IS_DEBUG_TRACING, IS_DEBUG_TRACING,
nodes::{ nodes::{
@ -67,10 +67,20 @@ impl Deref for Node {
impl Drop for Node { impl Drop for Node {
fn drop(&mut self) { fn drop(&mut self) {
if should_persist_tempdir() if should_persist_tempdir() {
&& let Err(e) = persist_tempdir(&mut self.handle.tempdir, "logos-blockchain-node") if let Some(ref persist_dir) = self.handle.persist_dir {
{ if let Err(e) = persist_tempdir_to(
debug!(error = ?e, "failed to persist node tempdir"); &mut self.handle.tempdir,
persist_dir,
"logos-blockchain-node",
) {
debug!(error = ?e, persist_dir = %persist_dir.display(), "failed to persist node tempdir to custom directory");
}
} else {
if let Err(e) = persist_tempdir(&mut self.handle.tempdir, "logos-blockchain-node") {
debug!(error = ?e, "failed to persist node tempdir");
}
}
} }
debug!("stopping node process"); debug!("stopping node process");
@ -96,7 +106,11 @@ impl Node {
self.handle.wait_for_exit(timeout).await self.handle.wait_for_exit(timeout).await
} }
pub async fn spawn(config: RunConfig, label: &str) -> Result<Self, SpawnNodeError> { pub async fn spawn(
config: RunConfig,
label: &str,
persist_dir: Option<PathBuf>,
) -> Result<Self, SpawnNodeError> {
let log_prefix = format!("{LOGS_PREFIX}-{label}"); let log_prefix = format!("{LOGS_PREFIX}-{label}");
let handle = spawn_node( let handle = spawn_node(
config, config,
@ -104,6 +118,7 @@ impl Node {
"node.yaml", "node.yaml",
binary_path(), binary_path(),
!*IS_DEBUG_TRACING, !*IS_DEBUG_TRACING,
persist_dir,
) )
.await?; .await?;

View File

@ -1,4 +1,4 @@
use std::sync::Arc; use std::{path::PathBuf, sync::Arc};
use reqwest::Url; use reqwest::Url;
@ -41,6 +41,8 @@ pub struct StartNodeOptions {
pub peers: PeerSelection, pub peers: PeerSelection,
/// Optional node config patch applied before spawn. /// Optional node config patch applied before spawn.
pub config_patch: Option<NodeConfigPatch>, pub config_patch: Option<NodeConfigPatch>,
/// Optional directory to persist node's tempdir to on stop.
pub persist_dir: Option<PathBuf>,
} }
impl Default for StartNodeOptions { impl Default for StartNodeOptions {
@ -48,6 +50,7 @@ impl Default for StartNodeOptions {
Self { Self {
peers: PeerSelection::DefaultLayout, peers: PeerSelection::DefaultLayout,
config_patch: None, config_patch: None,
persist_dir: None,
} }
} }
} }

View File

@ -1,4 +1,4 @@
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, path::PathBuf, sync::Arc};
use nomos_core::{ use nomos_core::{
mantle::GenesisTx as _, mantle::GenesisTx as _,
@ -11,7 +11,7 @@ use testing_framework_config::topology::{
base::{BaseConfigError, BaseConfigs, build_base_configs}, base::{BaseConfigError, BaseConfigs, build_base_configs},
consensus::{ consensus::{
ConsensusConfigError, ConsensusParams, ProviderInfo, ConsensusConfigError, ConsensusParams, ProviderInfo,
create_genesis_tx_with_declarations, create_genesis_tx_with_declarations, sync_utxos_with_genesis,
}, },
network::{Libp2pNetworkLayout, NetworkParams}, network::{Libp2pNetworkLayout, NetworkParams},
tracing::create_tracing_configs, tracing::create_tracing_configs,
@ -65,6 +65,7 @@ pub struct TopologyConfig {
pub network_params: NetworkParams, pub network_params: NetworkParams,
pub wallet_config: WalletConfig, pub wallet_config: WalletConfig,
pub node_config_patches: HashMap<usize, NodeConfigPatch>, pub node_config_patches: HashMap<usize, NodeConfigPatch>,
pub persist_dirs: HashMap<usize, PathBuf>,
} }
impl TopologyConfig { impl TopologyConfig {
@ -77,6 +78,7 @@ impl TopologyConfig {
network_params: NetworkParams::default(), network_params: NetworkParams::default(),
wallet_config: WalletConfig::default(), wallet_config: WalletConfig::default(),
node_config_patches: HashMap::new(), node_config_patches: HashMap::new(),
persist_dirs: HashMap::new(),
} }
} }
@ -89,6 +91,7 @@ impl TopologyConfig {
network_params: NetworkParams::default(), network_params: NetworkParams::default(),
wallet_config: WalletConfig::default(), wallet_config: WalletConfig::default(),
node_config_patches: HashMap::new(), node_config_patches: HashMap::new(),
persist_dirs: HashMap::new(),
} }
} }
@ -103,6 +106,7 @@ impl TopologyConfig {
network_params: NetworkParams::default(), network_params: NetworkParams::default(),
wallet_config: WalletConfig::default(), wallet_config: WalletConfig::default(),
node_config_patches: HashMap::new(), node_config_patches: HashMap::new(),
persist_dirs: HashMap::new(),
} }
} }
@ -121,6 +125,17 @@ impl TopologyConfig {
self.node_config_patches.insert(index, patch); self.node_config_patches.insert(index, patch);
self self
} }
#[must_use]
pub fn persist_dir(&self, index: usize) -> Option<&PathBuf> {
self.persist_dirs.get(&index)
}
#[must_use]
pub fn with_persist_dir(mut self, index: usize, dir: PathBuf) -> Self {
self.persist_dirs.insert(index, dir);
self
}
} }
/// Builder that produces `GeneratedTopology` instances from a `TopologyConfig`. /// Builder that produces `GeneratedTopology` instances from a `TopologyConfig`.
@ -163,6 +178,13 @@ impl TopologyBuilder {
self self
} }
#[must_use]
/// Apply a persist dir for a specific node index.
pub fn with_persist_dir(mut self, index: usize, dir: PathBuf) -> Self {
self.config.persist_dirs.insert(index, dir);
self
}
#[must_use] #[must_use]
/// Set node counts. /// Set node counts.
pub const fn with_node_count(mut self, nodes: usize) -> Self { pub const fn with_node_count(mut self, nodes: usize) -> Self {
@ -240,6 +262,7 @@ impl TopologyBuilder {
&kms_configs, &kms_configs,
&time_config, &time_config,
&config.node_config_patches, &config.node_config_patches,
&config.persist_dirs,
)?; )?;
Ok(GeneratedTopology { config, nodes }) Ok(GeneratedTopology { config, nodes })
@ -310,10 +333,7 @@ fn apply_consensus_genesis_tx(
) -> Result<(), TopologyBuildError> { ) -> Result<(), TopologyBuildError> {
for c in consensus_configs { for c in consensus_configs {
c.genesis_tx = genesis_tx.clone(); c.genesis_tx = genesis_tx.clone();
testing_framework_config::topology::configs::consensus::sync_utxos_with_genesis( sync_utxos_with_genesis(&mut c.utxos, genesis_tx)?;
&mut c.utxos,
genesis_tx,
)?;
} }
Ok(()) Ok(())
} }
@ -333,6 +353,7 @@ fn build_node_descriptors(
kms_configs: &[key_management_system_service::backend::preload::PreloadKMSBackendSettings], kms_configs: &[key_management_system_service::backend::preload::PreloadKMSBackendSettings],
time_config: &testing_framework_config::topology::configs::time::GeneralTimeConfig, time_config: &testing_framework_config::topology::configs::time::GeneralTimeConfig,
node_config_patches: &HashMap<usize, NodeConfigPatch>, node_config_patches: &HashMap<usize, NodeConfigPatch>,
persist_dirs: &HashMap<usize, PathBuf>,
) -> Result<Vec<GeneratedNodeConfig>, TopologyBuildError> { ) -> Result<Vec<GeneratedNodeConfig>, TopologyBuildError> {
let mut nodes = Vec::with_capacity(config.n_nodes); let mut nodes = Vec::with_capacity(config.n_nodes);
@ -367,6 +388,7 @@ fn build_node_descriptors(
general, general,
blend_port, blend_port,
config_patch: node_config_patches.get(&i).cloned(), config_patch: node_config_patches.get(&i).cloned(),
persist_dir: persist_dirs.get(&i).cloned(),
}; };
nodes.push(descriptor); nodes.push(descriptor);

View File

@ -1,4 +1,4 @@
use std::{collections::HashSet, time::Duration}; use std::{collections::HashSet, path::PathBuf, time::Duration};
use reqwest::{Client, Url}; use reqwest::{Client, Url};
@ -16,6 +16,7 @@ pub struct GeneratedNodeConfig {
pub general: GeneralConfig, pub general: GeneralConfig,
pub blend_port: u16, pub blend_port: u16,
pub config_patch: Option<NodeConfigPatch>, pub config_patch: Option<NodeConfigPatch>,
pub persist_dir: Option<PathBuf>,
} }
impl GeneratedNodeConfig { impl GeneratedNodeConfig {

View File

@ -119,12 +119,10 @@ fn default_extra_hosts() -> Vec<String> {
} }
fn base_environment(cfgsync_port: u16) -> Vec<EnvEntry> { fn base_environment(cfgsync_port: u16) -> Vec<EnvEntry> {
let pol_mode = tf_env::pol_proof_dev_mode().unwrap_or_else(|| "true".to_string());
let rust_log = tf_env::rust_log().unwrap_or_else(|| "info".to_string()); let rust_log = tf_env::rust_log().unwrap_or_else(|| "info".to_string());
let nomos_log_level = tf_env::nomos_log_level().unwrap_or_else(|| "info".to_string()); let nomos_log_level = tf_env::nomos_log_level().unwrap_or_else(|| "info".to_string());
let time_backend = tf_env::nomos_time_backend().unwrap_or_else(|| "monotonic".into()); let time_backend = tf_env::nomos_time_backend().unwrap_or_else(|| "monotonic".into());
vec![ vec![
EnvEntry::new("POL_PROOF_DEV_MODE", pol_mode),
EnvEntry::new("RUST_LOG", rust_log), EnvEntry::new("RUST_LOG", rust_log),
EnvEntry::new("LOGOS_BLOCKCHAIN_LOG_LEVEL", nomos_log_level), EnvEntry::new("LOGOS_BLOCKCHAIN_LOG_LEVEL", nomos_log_level),
EnvEntry::new("LOGOS_BLOCKCHAIN_TIME_BACKEND", time_backend), EnvEntry::new("LOGOS_BLOCKCHAIN_TIME_BACKEND", time_backend),

View File

@ -291,11 +291,9 @@ fn build_values(topology: &GeneratedTopology) -> HelmValues {
let cfgsync = CfgsyncValues { let cfgsync = CfgsyncValues {
port: cfgsync_port(), port: cfgsync_port(),
}; };
let pol_mode = pol_proof_mode();
let image_pull_policy = let image_pull_policy =
tf_env::nomos_testnet_image_pull_policy().unwrap_or_else(|| "IfNotPresent".into()); tf_env::nomos_testnet_image_pull_policy().unwrap_or_else(|| "IfNotPresent".into());
debug!(pol_mode, "rendering Helm values for k8s stack"); let nodes = build_node_group("node", topology.nodes());
let nodes = build_node_group("node", topology.nodes(), &pol_mode);
HelmValues { HelmValues {
image_pull_policy, image_pull_policy,
@ -307,12 +305,11 @@ fn build_values(topology: &GeneratedTopology) -> HelmValues {
fn build_node_group( fn build_node_group(
kind: &'static str, kind: &'static str,
nodes: &[testing_framework_core::topology::generation::GeneratedNodeConfig], nodes: &[testing_framework_core::topology::generation::GeneratedNodeConfig],
pol_mode: &str,
) -> NodeGroup { ) -> NodeGroup {
let node_values = nodes let node_values = nodes
.iter() .iter()
.enumerate() .enumerate()
.map(|(index, node)| build_node_values(kind, index, node, pol_mode)) .map(|(index, node)| build_node_values(kind, index, node))
.collect(); .collect();
NodeGroup { NodeGroup {
@ -325,10 +322,8 @@ fn build_node_values(
kind: &'static str, kind: &'static str,
index: usize, index: usize,
node: &testing_framework_core::topology::generation::GeneratedNodeConfig, node: &testing_framework_core::topology::generation::GeneratedNodeConfig,
pol_mode: &str,
) -> NodeValues { ) -> NodeValues {
let mut env = BTreeMap::new(); let mut env = BTreeMap::new();
env.insert("POL_PROOF_DEV_MODE".into(), pol_mode.to_string());
env.insert("CFG_NETWORK_PORT".into(), node.network_port().to_string()); env.insert("CFG_NETWORK_PORT".into(), node.network_port().to_string());
env.insert("CFG_BLEND_PORT".into(), node.blend_port.to_string()); env.insert("CFG_BLEND_PORT".into(), node.blend_port.to_string());
env.insert( env.insert(
@ -352,7 +347,3 @@ fn build_node_values(
env, env,
} }
} }
fn pol_proof_mode() -> String {
tf_env::pol_proof_dev_mode().unwrap_or_else(|| "true".to_string())
}

View File

@ -110,7 +110,7 @@ impl LocalNodeManager {
for node in descriptors.nodes() { for node in descriptors.nodes() {
let label = Self::default_label(node.index()); let label = Self::default_label(node.index());
let config = create_node_config(node.general.clone()); let config = create_node_config(node.general.clone());
let spawned = Node::spawn(config, &label).await?; let spawned = Node::spawn(config, &label, node.persist_dir.clone()).await?;
nodes.push(spawned); nodes.push(spawned);
} }
@ -335,7 +335,7 @@ impl LocalNodeManager {
)?; )?;
let api_client = self let api_client = self
.spawn_and_register_node(&node_name, network_port, config) .spawn_and_register_node(&node_name, network_port, config, options.persist_dir)
.await?; .await?;
Ok(StartedNode { Ok(StartedNode {
@ -428,8 +428,9 @@ impl LocalNodeManager {
node_name: &str, node_name: &str,
network_port: u16, network_port: u16,
config: RunConfig, config: RunConfig,
persist_dir: Option<std::path::PathBuf>,
) -> Result<ApiClient, LocalNodeManagerError> { ) -> Result<ApiClient, LocalNodeManagerError> {
let node = Node::spawn(config, node_name) let node = Node::spawn(config, node_name, persist_dir)
.await .await
.map_err(|source| LocalNodeManagerError::Spawn { source })?; .map_err(|source| LocalNodeManagerError::Spawn { source })?;
let client = node.api().clone(); let client = node.api().clone();

View File

@ -52,11 +52,6 @@ pub fn nomos_testnet_image_pull_policy() -> Option<String> {
env::var("LOGOS_BLOCKCHAIN_TESTNET_IMAGE_PULL_POLICY").ok() env::var("LOGOS_BLOCKCHAIN_TESTNET_IMAGE_PULL_POLICY").ok()
} }
#[must_use]
pub fn pol_proof_dev_mode() -> Option<String> {
env::var("POL_PROOF_DEV_MODE").ok()
}
#[must_use] #[must_use]
pub fn rust_log() -> Option<String> { pub fn rust_log() -> Option<String> {
env::var("RUST_LOG").ok() env::var("RUST_LOG").ok()

View File

@ -25,7 +25,7 @@ impl Default for ConsensusLiveness {
} }
const LAG_ALLOWANCE: u64 = 2; const LAG_ALLOWANCE: u64 = 2;
const MIN_PROGRESS_BLOCKS: u64 = 5; const MIN_PROGRESS_BLOCKS: u64 = 3;
const REQUEST_RETRIES: usize = 15; const REQUEST_RETRIES: usize = 15;
const REQUEST_RETRY_DELAY: Duration = Duration::from_secs(2); const REQUEST_RETRY_DELAY: Duration = Duration::from_secs(2);
const MAX_LAG_ALLOWANCE: u64 = 5; const MAX_LAG_ALLOWANCE: u64 = 5;
@ -67,7 +67,7 @@ enum ConsensusLivenessIssue {
#[derive(Debug, Error)] #[derive(Debug, Error)]
enum ConsensusLivenessError { enum ConsensusLivenessError {
#[error("consensus liveness requires at least one node")] #[error("consensus liveness requires at least one validator")]
MissingParticipants, MissingParticipants,
#[error("consensus liveness violated (target={target}):\n{details}")] #[error("consensus liveness violated (target={target}):\n{details}")]
Violations { Violations {