mirror of
https://github.com/logos-blockchain/logos-blockchain-testing.git
synced 2026-02-17 11:43:12 +00:00
Add optional persist_dir
wip lower block expectancy
This commit is contained in:
parent
ec112a2449
commit
cee6b1a8a2
2
.github/workflows/lint.yml
vendored
2
.github/workflows/lint.yml
vendored
@ -239,7 +239,6 @@ jobs:
|
|||||||
host_smoke:
|
host_smoke:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
env:
|
env:
|
||||||
POL_PROOF_DEV_MODE: true
|
|
||||||
LOCAL_DEMO_RUN_SECS: 120
|
LOCAL_DEMO_RUN_SECS: 120
|
||||||
LOCAL_DEMO_VALIDATORS: 1
|
LOCAL_DEMO_VALIDATORS: 1
|
||||||
LOGOS_BLOCKCHAIN_CIRCUITS: ${{ github.workspace }}/.tmp/logos-blockchain-circuits
|
LOGOS_BLOCKCHAIN_CIRCUITS: ${{ github.workspace }}/.tmp/logos-blockchain-circuits
|
||||||
@ -500,7 +499,6 @@ jobs:
|
|||||||
|
|
||||||
- name: Run compose mixed workload binary
|
- name: Run compose mixed workload binary
|
||||||
env:
|
env:
|
||||||
POL_PROOF_DEV_MODE: "true"
|
|
||||||
COMPOSE_NODE_PAIRS: "1x1"
|
COMPOSE_NODE_PAIRS: "1x1"
|
||||||
LOGOS_BLOCKCHAIN_TESTNET_IMAGE: ${{ env.LOGOS_BLOCKCHAIN_TESTNET_IMAGE }}
|
LOGOS_BLOCKCHAIN_TESTNET_IMAGE: ${{ env.LOGOS_BLOCKCHAIN_TESTNET_IMAGE }}
|
||||||
COMPOSE_RUNNER_HOST: "127.0.0.1"
|
COMPOSE_RUNNER_HOST: "127.0.0.1"
|
||||||
|
|||||||
@ -123,7 +123,6 @@ Key environment variables for customization:
|
|||||||
|
|
||||||
| Variable | Purpose | Default |
|
| Variable | Purpose | Default |
|
||||||
|----------|---------|---------|
|
|----------|---------|---------|
|
||||||
| `POL_PROOF_DEV_MODE=true` | **Required** — Disable expensive proof generation (set automatically by `scripts/run/run-examples.sh`) | (none) |
|
|
||||||
| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | Docker image tag for compose/k8s | `logos-blockchain-testing:local` |
|
| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | Docker image tag for compose/k8s | `logos-blockchain-testing:local` |
|
||||||
| `LOGOS_BLOCKCHAIN_DEMO_NODES` | Number of nodes | Varies by example |
|
| `LOGOS_BLOCKCHAIN_DEMO_NODES` | Number of nodes | Varies by example |
|
||||||
| `LOGOS_BLOCKCHAIN_LOG_DIR` | Directory for persistent log files | (temporary) |
|
| `LOGOS_BLOCKCHAIN_LOG_DIR` | Directory for persistent log files | (temporary) |
|
||||||
|
|||||||
@ -65,9 +65,7 @@ Convenience utilities:
|
|||||||
- `compose_runner.rs` — Docker Compose (requires `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` built)
|
- `compose_runner.rs` — Docker Compose (requires `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` built)
|
||||||
- `k8s_runner.rs` — Kubernetes (requires cluster + image)
|
- `k8s_runner.rs` — Kubernetes (requires cluster + image)
|
||||||
|
|
||||||
**Run with:** `POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin <name>`
|
**Run with:** `cargo run -p runner-examples --bin <name>`
|
||||||
|
|
||||||
**All runners require `POL_PROOF_DEV_MODE=true`** to avoid expensive proof generation.
|
|
||||||
|
|
||||||
### `scripts/`
|
### `scripts/`
|
||||||
Helper utilities:
|
Helper utilities:
|
||||||
|
|||||||
@ -150,11 +150,9 @@ This handles circuit setup, binary building/bundling, image building, and execut
|
|||||||
**Alternative:** Direct cargo run (requires manual setup):
|
**Alternative:** Direct cargo run (requires manual setup):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin <name>
|
cargo run -p runner-examples --bin <name>
|
||||||
```
|
```
|
||||||
|
|
||||||
**Important:** All runners require `POL_PROOF_DEV_MODE=true` to avoid expensive Groth16 proof generation that causes timeouts.
|
|
||||||
|
|
||||||
These binaries use the framework API (`ScenarioBuilder`) to construct and execute scenarios.
|
These binaries use the framework API (`ScenarioBuilder`) to construct and execute scenarios.
|
||||||
|
|
||||||
## Builder API
|
## Builder API
|
||||||
|
|||||||
@ -153,7 +153,7 @@ async fn hello_consensus_liveness() -> Result<()> {
|
|||||||
|
|
||||||
**Run it:**
|
**Run it:**
|
||||||
```bash
|
```bash
|
||||||
POL_PROOF_DEV_MODE=true cargo test hello_consensus_liveness
|
cargo test hello_consensus_liveness
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@ -128,13 +128,9 @@ strategy:
|
|||||||
|
|
||||||
## Anti-Patterns to Avoid
|
## Anti-Patterns to Avoid
|
||||||
|
|
||||||
**DON'T: Run without POL_PROOF_DEV_MODE**
|
|
||||||
```bash
|
```bash
|
||||||
# BAD: Will hang/timeout on proof generation
|
# BAD: Will hang/timeout on proof generation
|
||||||
cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
|
|
||||||
# GOOD: Fast mode for testing
|
|
||||||
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**DON'T: Use tiny durations**
|
**DON'T: Use tiny durations**
|
||||||
|
|||||||
@ -39,7 +39,6 @@ on:
|
|||||||
branches: [main]
|
branches: [main]
|
||||||
|
|
||||||
env:
|
env:
|
||||||
POL_PROOF_DEV_MODE: true
|
|
||||||
CARGO_TERM_COLOR: always
|
CARGO_TERM_COLOR: always
|
||||||
RUST_BACKTRACE: 1
|
RUST_BACKTRACE: 1
|
||||||
|
|
||||||
@ -243,17 +242,6 @@ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
|||||||
|
|
||||||
## Best Practices
|
## Best Practices
|
||||||
|
|
||||||
### Required: Set POL_PROOF_DEV_MODE
|
|
||||||
|
|
||||||
**Always set `POL_PROOF_DEV_MODE=true` globally** in your workflow env:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
env:
|
|
||||||
POL_PROOF_DEV_MODE: true # REQUIRED!
|
|
||||||
```
|
|
||||||
|
|
||||||
Without this, tests will hang due to expensive proof generation.
|
|
||||||
|
|
||||||
### Use Helper Scripts
|
### Use Helper Scripts
|
||||||
|
|
||||||
Prefer `scripts/run/run-examples.sh` which handles all setup automatically:
|
Prefer `scripts/run/run-examples.sh` which handles all setup automatically:
|
||||||
|
|||||||
@ -2,27 +2,6 @@
|
|||||||
|
|
||||||
Complete reference of environment variables used by the testing framework, organized by category.
|
Complete reference of environment variables used by the testing framework, organized by category.
|
||||||
|
|
||||||
## Critical Variables
|
|
||||||
|
|
||||||
These MUST be set for successful test runs:
|
|
||||||
|
|
||||||
| Variable | Required | Default | Effect |
|
|
||||||
|----------|----------|---------|--------|
|
|
||||||
| `POL_PROOF_DEV_MODE` | **YES** | — | **REQUIRED for all runners**. Set to `true` to use fast dev-mode proving instead of expensive Groth16. Without this, tests will hang/timeout. |
|
|
||||||
|
|
||||||
**Example:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
export POL_PROOF_DEV_MODE=true
|
|
||||||
```
|
|
||||||
|
|
||||||
Or add to your shell profile (`~/.bashrc`, `~/.zshrc`):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Required for nomos-testing framework
|
|
||||||
export POL_PROOF_DEV_MODE=true
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Runner Selection & Topology
|
## Runner Selection & Topology
|
||||||
@ -138,7 +117,6 @@ Control node log output (not framework runner logs):
|
|||||||
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \
|
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \
|
||||||
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
|
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
|
||||||
LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \
|
LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
|
|
||||||
# Inspect logs
|
# Inspect logs
|
||||||
@ -303,7 +281,6 @@ Node-level configuration passed through to logos-blockchain-node:
|
|||||||
# Faster block production
|
# Faster block production
|
||||||
CONSENSUS_SLOT_TIME=5 \
|
CONSENSUS_SLOT_TIME=5 \
|
||||||
CONSENSUS_ACTIVE_SLOT_COEFF=0.9 \
|
CONSENSUS_ACTIVE_SLOT_COEFF=0.9 \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -350,14 +327,12 @@ Variables used by helper scripts (`scripts/run/run-examples.sh`, etc.):
|
|||||||
### Minimal Host Run
|
### Minimal Host Run
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
scripts/run/run-examples.sh -t 60 -n 3 host
|
scripts/run/run-examples.sh -t 60 -n 3 host
|
||||||
```
|
```
|
||||||
|
|
||||||
### Debug Logging (Host)
|
### Debug Logging (Host)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/logs \
|
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/logs \
|
||||||
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
|
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
|
||||||
LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \
|
LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \
|
||||||
@ -367,7 +342,6 @@ scripts/run/run-examples.sh -t 60 -n 3 host
|
|||||||
### Compose with Observability
|
### Compose with Observability
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
LOGOS_BLOCKCHAIN_METRICS_QUERY_URL=http://localhost:9090 \
|
LOGOS_BLOCKCHAIN_METRICS_QUERY_URL=http://localhost:9090 \
|
||||||
LOGOS_BLOCKCHAIN_GRAFANA_URL=http://localhost:3000 \
|
LOGOS_BLOCKCHAIN_GRAFANA_URL=http://localhost:3000 \
|
||||||
scripts/run/run-examples.sh -t 60 -n 3 compose
|
scripts/run/run-examples.sh -t 60 -n 3 compose
|
||||||
@ -376,7 +350,6 @@ scripts/run/run-examples.sh -t 60 -n 3 compose
|
|||||||
### K8s with Debug
|
### K8s with Debug
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
K8S_RUNNER_NAMESPACE=nomos-debug \
|
K8S_RUNNER_NAMESPACE=nomos-debug \
|
||||||
K8S_RUNNER_DEBUG=1 \
|
K8S_RUNNER_DEBUG=1 \
|
||||||
K8S_RUNNER_PRESERVE=1 \
|
K8S_RUNNER_PRESERVE=1 \
|
||||||
@ -387,7 +360,6 @@ scripts/run/run-examples.sh -t 60 -n 3 k8s
|
|||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
env:
|
env:
|
||||||
POL_PROOF_DEV_MODE: true
|
|
||||||
RUST_BACKTRACE: 1
|
RUST_BACKTRACE: 1
|
||||||
LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS: 1
|
LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS: 1
|
||||||
```
|
```
|
||||||
|
|||||||
@ -15,9 +15,7 @@ and expectations.
|
|||||||
|
|
||||||
**Recommended:** Use `scripts/run/run-examples.sh -t <duration> -n <nodes> <mode>` where mode is `host`, `compose`, or `k8s`.
|
**Recommended:** Use `scripts/run/run-examples.sh -t <duration> -n <nodes> <mode>` where mode is `host`, `compose`, or `k8s`.
|
||||||
|
|
||||||
**Alternative:** Direct cargo run: `POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin <name>`
|
**Alternative:** Direct cargo run: `cargo run -p runner-examples --bin <name>`
|
||||||
|
|
||||||
**All runners require `POL_PROOF_DEV_MODE=true`** to avoid expensive proof generation.
|
|
||||||
|
|
||||||
**Code patterns** below show how to build scenarios. Wrap these in `#[tokio::test]` functions for integration tests, or `#[tokio::main]` for binaries.
|
**Code patterns** below show how to build scenarios. Wrap these in `#[tokio::test]` functions for integration tests, or `#[tokio::main]` for binaries.
|
||||||
|
|
||||||
|
|||||||
@ -38,10 +38,6 @@
|
|||||||
Also called "correctness expectations."
|
Also called "correctness expectations."
|
||||||
- **Mantle transaction**: transaction type in Logos that can contain UTXO transfers
|
- **Mantle transaction**: transaction type in Logos that can contain UTXO transfers
|
||||||
(LedgerTx) and operations (Op).
|
(LedgerTx) and operations (Op).
|
||||||
- **POL_PROOF_DEV_MODE**: environment variable that disables expensive Groth16 zero-knowledge
|
|
||||||
proof generation for leader election. **Required for all runners** (local, compose, k8s)
|
|
||||||
for practical testing—without it, proof generation causes timeouts. Should never be
|
|
||||||
used in production environments.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@ -46,7 +46,6 @@ LOGOS_BLOCKCHAIN_TESTS_TRACING=true \
|
|||||||
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \
|
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \
|
||||||
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
|
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
|
||||||
LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace,chain_service=info,chain_network=info" \
|
LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace,chain_service=info,chain_network=info" \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -90,7 +89,7 @@ LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace,chain_service=info,chain_network=
|
|||||||
**Default (temporary directories, auto-cleanup):**
|
**Default (temporary directories, auto-cleanup):**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
# Logs written to temporary directories in working directory
|
# Logs written to temporary directories in working directory
|
||||||
# Automatically cleaned up after test completes
|
# Automatically cleaned up after test completes
|
||||||
```
|
```
|
||||||
@ -99,7 +98,6 @@ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/local-logs \
|
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/local-logs \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
|
|
||||||
# After test completes:
|
# After test completes:
|
||||||
@ -137,7 +135,6 @@ To write per-node log files inside containers, set `tracing_settings.logger: !Fi
|
|||||||
```bash
|
```bash
|
||||||
# Ensure cfgsync.yaml is configured to log to /logs
|
# Ensure cfgsync.yaml is configured to log to /logs
|
||||||
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
|
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin compose_runner
|
cargo run -p runner-examples --bin compose_runner
|
||||||
|
|
||||||
# After test, copy files from containers:
|
# After test, copy files from containers:
|
||||||
@ -257,7 +254,6 @@ scripts/setup/setup-observability.sh compose up
|
|||||||
eval $(scripts/setup/setup-observability.sh compose env)
|
eval $(scripts/setup/setup-observability.sh compose env)
|
||||||
|
|
||||||
# Run scenario with metrics
|
# Run scenario with metrics
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
scripts/run/run-examples.sh -t 60 -n 3 compose
|
scripts/run/run-examples.sh -t 60 -n 3 compose
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -275,7 +271,7 @@ scripts/setup/setup-observability.sh compose up
|
|||||||
eval $(scripts/setup/setup-observability.sh compose env)
|
eval $(scripts/setup/setup-observability.sh compose env)
|
||||||
|
|
||||||
export LOGOS_BLOCKCHAIN_GRAFANA_URL=http://localhost:3000
|
export LOGOS_BLOCKCHAIN_GRAFANA_URL=http://localhost:3000
|
||||||
POL_PROOF_DEV_MODE=true scripts/run/run-examples.sh -t 60 -n 3 compose
|
scripts/run/run-examples.sh -t 60 -n 3 compose
|
||||||
```
|
```
|
||||||
|
|
||||||
**Default bundled Grafana login:** `admin` / `admin` (see `scripts/observability/compose/docker-compose.yml`).
|
**Default bundled Grafana login:** `admin` / `admin` (see `scripts/observability/compose/docker-compose.yml`).
|
||||||
@ -322,7 +318,6 @@ flowchart TD
|
|||||||
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/logs \
|
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/logs \
|
||||||
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
|
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
|
||||||
LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \
|
LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
scripts/run/run-examples.sh -t 60 -n 3 host
|
scripts/run/run-examples.sh -t 60 -n 3 host
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -334,7 +329,6 @@ scripts/setup/setup-observability.sh compose up
|
|||||||
eval $(scripts/setup/setup-observability.sh compose env)
|
eval $(scripts/setup/setup-observability.sh compose env)
|
||||||
|
|
||||||
# Run with metrics
|
# Run with metrics
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
scripts/run/run-examples.sh -t 60 -n 3 compose
|
scripts/run/run-examples.sh -t 60 -n 3 compose
|
||||||
|
|
||||||
# Access Grafana at http://localhost:3000
|
# Access Grafana at http://localhost:3000
|
||||||
@ -346,7 +340,6 @@ scripts/run/run-examples.sh -t 60 -n 3 compose
|
|||||||
K8S_RUNNER_NAMESPACE=nomos-debug \
|
K8S_RUNNER_NAMESPACE=nomos-debug \
|
||||||
K8S_RUNNER_DEBUG=1 \
|
K8S_RUNNER_DEBUG=1 \
|
||||||
K8S_RUNNER_PRESERVE=1 \
|
K8S_RUNNER_PRESERVE=1 \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
scripts/run/run-examples.sh -t 60 -n 3 k8s
|
scripts/run/run-examples.sh -t 60 -n 3 k8s
|
||||||
|
|
||||||
# Inspect logs
|
# Inspect logs
|
||||||
|
|||||||
@ -375,7 +375,6 @@ async fn external_driver_example() -> Result<()> {
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Required: dev mode for fast proofs
|
# Required: dev mode for fast proofs
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo test -p runner-examples -- --ignored external_driver_example
|
cargo test -p runner-examples -- --ignored external_driver_example
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -385,7 +384,6 @@ cargo test -p runner-examples -- --ignored external_driver_example
|
|||||||
# Preserve logs after test
|
# Preserve logs after test
|
||||||
LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1 \
|
LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1 \
|
||||||
RUST_LOG=info \
|
RUST_LOG=info \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo test -p runner-examples -- --ignored external_driver_example
|
cargo test -p runner-examples -- --ignored external_driver_example
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@ -23,7 +23,6 @@ Operational readiness focuses on prerequisites, environment fit, and clear signa
|
|||||||
- Binary bundles for reproducible builds
|
- Binary bundles for reproducible builds
|
||||||
|
|
||||||
**Environment Configuration:**
|
**Environment Configuration:**
|
||||||
- `POL_PROOF_DEV_MODE=true` is **REQUIRED for all runners** to avoid expensive proof generation
|
|
||||||
- Logging configured via `LOGOS_BLOCKCHAIN_LOG_*` variables
|
- Logging configured via `LOGOS_BLOCKCHAIN_LOG_*` variables
|
||||||
- Observability endpoints (Prometheus, Grafana) optional but useful
|
- Observability endpoints (Prometheus, Grafana) optional but useful
|
||||||
|
|
||||||
|
|||||||
@ -194,30 +194,6 @@ minikube image load logos-blockchain-testing:local
|
|||||||
- Resource isolation
|
- Resource isolation
|
||||||
- Large topologies
|
- Large topologies
|
||||||
|
|
||||||
## Critical Environment Variable
|
|
||||||
|
|
||||||
**`POL_PROOF_DEV_MODE=true` is REQUIRED for ALL runners!**
|
|
||||||
|
|
||||||
Without this, proof generation uses expensive Groth16 proving, causing:
|
|
||||||
- Tests "hang" for minutes
|
|
||||||
- CPU spikes to 100%
|
|
||||||
- Timeouts and failures
|
|
||||||
|
|
||||||
**Always set:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
|
||||||
POL_PROOF_DEV_MODE=true scripts/run/run-examples.sh -t 60 -n 3 compose
|
|
||||||
# etc.
|
|
||||||
```
|
|
||||||
|
|
||||||
**Or add to your shell profile:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# ~/.bashrc or ~/.zshrc
|
|
||||||
export POL_PROOF_DEV_MODE=true
|
|
||||||
```
|
|
||||||
|
|
||||||
## Quick Setup Check
|
## Quick Setup Check
|
||||||
|
|
||||||
Run this checklist before your first scenario:
|
Run this checklist before your first scenario:
|
||||||
@ -229,16 +205,13 @@ cat versions.env
|
|||||||
# 2. Check circuit assets
|
# 2. Check circuit assets
|
||||||
ls -lh "${HOME}/.logos-blockchain-circuits"
|
ls -lh "${HOME}/.logos-blockchain-circuits"
|
||||||
|
|
||||||
# 3. Verify POL_PROOF_DEV_MODE is set
|
# 3. For compose/k8s: verify Docker is running
|
||||||
echo $POL_PROOF_DEV_MODE # Should print: true
|
|
||||||
|
|
||||||
# 4. For compose/k8s: verify Docker is running
|
|
||||||
docker ps
|
docker ps
|
||||||
|
|
||||||
# 5. For compose/k8s: verify image exists
|
# 4. For compose/k8s: verify image exists
|
||||||
docker images | grep logos-blockchain-testing
|
docker images | grep logos-blockchain-testing
|
||||||
|
|
||||||
# 6. For host runner: verify node binaries (if not using scripts)
|
# 5. For host runner: verify node binaries (if not using scripts)
|
||||||
$LOGOS_BLOCKCHAIN_NODE_BIN --version
|
$LOGOS_BLOCKCHAIN_NODE_BIN --version
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@ -16,7 +16,7 @@ git clone https://github.com/logos-blockchain/logos-blockchain-testing.git
|
|||||||
cd logos-blockchain-testing
|
cd logos-blockchain-testing
|
||||||
|
|
||||||
# 3. Run your first scenario (downloads dependencies automatically)
|
# 3. Run your first scenario (downloads dependencies automatically)
|
||||||
POL_PROOF_DEV_MODE=true scripts/run/run-examples.sh -t 60 -n 1 host
|
scripts/run/run-examples.sh -t 60 -n 1 host
|
||||||
```
|
```
|
||||||
|
|
||||||
**First run takes 5-10 minutes** (downloads ~120MB circuit assets, builds binaries).
|
**First run takes 5-10 minutes** (downloads ~120MB circuit assets, builds binaries).
|
||||||
@ -56,7 +56,7 @@ This handles circuit setup, binary building, and runs a complete scenario: 1 nod
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Requires circuits in place and LOGOS_BLOCKCHAIN_NODE_BIN set
|
# Requires circuits in place and LOGOS_BLOCKCHAIN_NODE_BIN set
|
||||||
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
```
|
```
|
||||||
|
|
||||||
**Core API Pattern** (simplified example):
|
**Core API Pattern** (simplified example):
|
||||||
@ -92,8 +92,6 @@ pub async fn run_local_demo() -> Result<()> {
|
|||||||
|
|
||||||
**Note:** The examples are binaries with `#[tokio::main]`, not test functions. If you want to write integration tests, wrap this pattern in `#[tokio::test]` functions in your own test suite.
|
**Note:** The examples are binaries with `#[tokio::main]`, not test functions. If you want to write integration tests, wrap this pattern in `#[tokio::test]` functions in your own test suite.
|
||||||
|
|
||||||
**Important:** `POL_PROOF_DEV_MODE=true` disables expensive Groth16 zero-knowledge proof generation for leader election. Without it, proof generation is CPU-intensive and tests will timeout. **This is required for all runners** (local, compose, k8s) for practical testing. Never use in production.
|
|
||||||
|
|
||||||
**What you should see:**
|
**What you should see:**
|
||||||
- Nodes spawn as local processes
|
- Nodes spawn as local processes
|
||||||
- Consensus starts producing blocks
|
- Consensus starts producing blocks
|
||||||
@ -213,7 +211,6 @@ scripts/run/run-examples.sh -t 120 -n 3 host
|
|||||||
# Uses LOGOS_BLOCKCHAIN_DEMO_* env vars (or legacy *_DEMO_* vars)
|
# Uses LOGOS_BLOCKCHAIN_DEMO_* env vars (or legacy *_DEMO_* vars)
|
||||||
LOGOS_BLOCKCHAIN_DEMO_NODES=3 \
|
LOGOS_BLOCKCHAIN_DEMO_NODES=3 \
|
||||||
LOGOS_BLOCKCHAIN_DEMO_RUN_SECS=120 \
|
LOGOS_BLOCKCHAIN_DEMO_RUN_SECS=120 \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -246,7 +243,6 @@ scripts/build/build_test_image.sh
|
|||||||
|
|
||||||
# Run with Compose
|
# Run with Compose
|
||||||
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
|
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin compose_runner
|
cargo run -p runner-examples --bin compose_runner
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@ -4,8 +4,6 @@ Runners turn a scenario plan into a live environment while keeping the plan
|
|||||||
unchanged. Choose based on feedback speed, reproducibility, and fidelity. For
|
unchanged. Choose based on feedback speed, reproducibility, and fidelity. For
|
||||||
environment and operational considerations, see [Operations Overview](operations-overview.md).
|
environment and operational considerations, see [Operations Overview](operations-overview.md).
|
||||||
|
|
||||||
**Important:** All runners require `POL_PROOF_DEV_MODE=true` to avoid expensive Groth16 proof generation that causes timeouts.
|
|
||||||
|
|
||||||
## Host runner (local processes)
|
## Host runner (local processes)
|
||||||
- Launches node processes directly on the host (via `LocalDeployer`).
|
- Launches node processes directly on the host (via `LocalDeployer`).
|
||||||
- Binary: `local_runner.rs`, script mode: `host`
|
- Binary: `local_runner.rs`, script mode: `host`
|
||||||
|
|||||||
@ -99,7 +99,6 @@ scripts/ops/clean.sh --docker
|
|||||||
For manual control, run the `local_runner` binary directly:
|
For manual control, run the `local_runner` binary directly:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
LOGOS_BLOCKCHAIN_NODE_BIN=/path/to/logos-blockchain-node \
|
LOGOS_BLOCKCHAIN_NODE_BIN=/path/to/logos-blockchain-node \
|
||||||
cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
```
|
```
|
||||||
@ -116,7 +115,6 @@ cargo run -p runner-examples --bin local_runner
|
|||||||
| `LOGOS_BLOCKCHAIN_TESTS_TRACING` | false | Enable debug tracing preset |
|
| `LOGOS_BLOCKCHAIN_TESTS_TRACING` | false | Enable debug tracing preset |
|
||||||
| `LOGOS_BLOCKCHAIN_LOG_LEVEL` | info | Global log level: error, warn, info, debug, trace |
|
| `LOGOS_BLOCKCHAIN_LOG_LEVEL` | info | Global log level: error, warn, info, debug, trace |
|
||||||
| `LOGOS_BLOCKCHAIN_LOG_FILTER` | None | Fine-grained module filtering (e.g., `cryptarchia=trace`) |
|
| `LOGOS_BLOCKCHAIN_LOG_FILTER` | None | Fine-grained module filtering (e.g., `cryptarchia=trace`) |
|
||||||
| `POL_PROOF_DEV_MODE` | — | **REQUIRED**: Set to `true` for all runners |
|
|
||||||
|
|
||||||
**Note:** Requires circuit assets and host binaries. Use `scripts/run/run-examples.sh host` to handle setup automatically.
|
**Note:** Requires circuit assets and host binaries. Use `scripts/run/run-examples.sh host` to handle setup automatically.
|
||||||
|
|
||||||
@ -139,7 +137,6 @@ scripts/build/build_test_image.sh
|
|||||||
|
|
||||||
# 3. Run
|
# 3. Run
|
||||||
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
|
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin compose_runner
|
cargo run -p runner-examples --bin compose_runner
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -154,7 +151,6 @@ scripts/build/build_test_image.sh
|
|||||||
|
|
||||||
# Run
|
# Run
|
||||||
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
|
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin compose_runner
|
cargo run -p runner-examples --bin compose_runner
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -169,7 +165,6 @@ cargo run -p runner-examples --bin compose_runner
|
|||||||
| Variable | Default | Effect |
|
| Variable | Default | Effect |
|
||||||
|----------|---------|--------|
|
|----------|---------|--------|
|
||||||
| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | — | Image tag (required, must match built image) |
|
| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | — | Image tag (required, must match built image) |
|
||||||
| `POL_PROOF_DEV_MODE` | — | **REQUIRED**: Set to `true` for all runners |
|
|
||||||
| `LOGOS_BLOCKCHAIN_DEMO_NODES` | 1 | Number of nodes |
|
| `LOGOS_BLOCKCHAIN_DEMO_NODES` | 1 | Number of nodes |
|
||||||
| `LOGOS_BLOCKCHAIN_DEMO_RUN_SECS` | 60 | Run duration in seconds |
|
| `LOGOS_BLOCKCHAIN_DEMO_RUN_SECS` | 60 | Run duration in seconds |
|
||||||
| `COMPOSE_NODE_PAIRS` | — | Alternative topology format: "nodes" (e.g., `3`) |
|
| `COMPOSE_NODE_PAIRS` | — | Alternative topology format: "nodes" (e.g., `3`) |
|
||||||
@ -232,7 +227,6 @@ export LOGOS_BLOCKCHAIN_TESTNET_IMAGE=your-registry/logos-blockchain-testing:lat
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
export LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local
|
export LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local
|
||||||
export POL_PROOF_DEV_MODE=true
|
|
||||||
cargo run -p runner-examples --bin k8s_runner
|
cargo run -p runner-examples --bin k8s_runner
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -241,7 +235,6 @@ cargo run -p runner-examples --bin k8s_runner
|
|||||||
| Variable | Default | Effect |
|
| Variable | Default | Effect |
|
||||||
|----------|---------|--------|
|
|----------|---------|--------|
|
||||||
| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | — | Image tag (required) |
|
| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | — | Image tag (required) |
|
||||||
| `POL_PROOF_DEV_MODE` | — | **REQUIRED**: Set to `true` for all runners |
|
|
||||||
| `LOGOS_BLOCKCHAIN_DEMO_NODES` | 1 | Number of nodes |
|
| `LOGOS_BLOCKCHAIN_DEMO_NODES` | 1 | Number of nodes |
|
||||||
| `LOGOS_BLOCKCHAIN_DEMO_RUN_SECS` | 60 | Run duration in seconds |
|
| `LOGOS_BLOCKCHAIN_DEMO_RUN_SECS` | 60 | Run duration in seconds |
|
||||||
| `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` | None | Prometheus-compatible base URL for runner to query (PromQL) |
|
| `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` | None | Prometheus-compatible base URL for runner to query (PromQL) |
|
||||||
|
|||||||
@ -69,7 +69,6 @@ Notes:
|
|||||||
Run the built-in local examples:
|
Run the built-in local examples:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
scripts/run/run-examples.sh -t 60 -n 3 host
|
scripts/run/run-examples.sh -t 60 -n 3 host
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -82,7 +81,6 @@ scripts/run/run-examples.sh -t 60 -n 3 host
|
|||||||
Run the built-in compose examples:
|
Run the built-in compose examples:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
scripts/run/run-examples.sh -t 60 -n 3 compose
|
scripts/run/run-examples.sh -t 60 -n 3 compose
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -95,7 +93,6 @@ scripts/run/run-examples.sh -t 60 -n 3 compose
|
|||||||
Run the built-in k8s examples:
|
Run the built-in k8s examples:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
scripts/run/run-examples.sh -t 60 -n 3 k8s
|
scripts/run/run-examples.sh -t 60 -n 3 k8s
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@ -2,7 +2,6 @@
|
|||||||
|
|
||||||
**Prerequisites for All Runners:**
|
**Prerequisites for All Runners:**
|
||||||
- **`versions.env` file** at repository root (required by helper scripts)
|
- **`versions.env` file** at repository root (required by helper scripts)
|
||||||
- **`POL_PROOF_DEV_MODE=true`** MUST be set for all runners (host, compose, k8s) to avoid expensive Groth16 proof generation that causes timeouts
|
|
||||||
- **Circuit assets** must be present and `LOGOS_BLOCKCHAIN_CIRCUITS` must point to a directory that contains them
|
- **Circuit assets** must be present and `LOGOS_BLOCKCHAIN_CIRCUITS` must point to a directory that contains them
|
||||||
|
|
||||||
**Platform/Environment Notes:**
|
**Platform/Environment Notes:**
|
||||||
@ -18,7 +17,6 @@
|
|||||||
|
|
||||||
Common symptoms and likely causes:
|
Common symptoms and likely causes:
|
||||||
|
|
||||||
- **No or slow block progression**: missing `POL_PROOF_DEV_MODE=true`, missing circuit assets, too-short run window, port conflicts, or resource exhaustion—set required env vars, verify assets exist, extend duration, check node logs for startup errors.
|
|
||||||
- **Transactions not included**: unfunded or misconfigured wallets (check `.wallets(N)` vs `.users(M)`), transaction rate exceeding block capacity, or rates exceeding block production speed—reduce rate, increase wallet count, verify wallet setup in logs.
|
- **Transactions not included**: unfunded or misconfigured wallets (check `.wallets(N)` vs `.users(M)`), transaction rate exceeding block capacity, or rates exceeding block production speed—reduce rate, increase wallet count, verify wallet setup in logs.
|
||||||
- **Chaos stalls the run**: chaos (node control) only works with ComposeDeployer; host runner (LocalDeployer) and K8sDeployer don't support it (won't "stall", just can't execute chaos workloads). With compose, aggressive restart cadence can prevent consensus recovery—widen restart intervals.
|
- **Chaos stalls the run**: chaos (node control) only works with ComposeDeployer; host runner (LocalDeployer) and K8sDeployer don't support it (won't "stall", just can't execute chaos workloads). With compose, aggressive restart cadence can prevent consensus recovery—widen restart intervals.
|
||||||
- **Observability gaps**: metrics or logs unreachable because ports clash or services are not exposed—adjust observability ports and confirm runner wiring.
|
- **Observability gaps**: metrics or logs unreachable because ports clash or services are not exposed—adjust observability ports and confirm runner wiring.
|
||||||
@ -28,40 +26,7 @@ Common symptoms and likely causes:
|
|||||||
|
|
||||||
This section shows what you'll actually see when common issues occur. Each example includes realistic console output and the fix.
|
This section shows what you'll actually see when common issues occur. Each example includes realistic console output and the fix.
|
||||||
|
|
||||||
### 1. Missing `POL_PROOF_DEV_MODE=true` (Most Common!)
|
### 1. Missing `versions.env` File
|
||||||
|
|
||||||
**Symptoms:**
|
|
||||||
- Test "hangs" with no visible progress
|
|
||||||
- CPU usage spikes to 100%
|
|
||||||
- Eventually hits timeout after several minutes
|
|
||||||
- Nodes appear to start but blocks aren't produced
|
|
||||||
|
|
||||||
**What you'll see:**
|
|
||||||
|
|
||||||
```text
|
|
||||||
$ cargo run -p runner-examples --bin local_runner
|
|
||||||
Finished dev [unoptimized + debuginfo] target(s) in 0.48s
|
|
||||||
Running `target/debug/local_runner`
|
|
||||||
[INFO runner_examples::local_runner] Starting local runner scenario
|
|
||||||
[INFO testing_framework_runner_local] Launching 3 nodes
|
|
||||||
[INFO testing_framework_runner_local] Waiting for node readiness...
|
|
||||||
(hangs here for 5+ minutes, CPU at 100%)
|
|
||||||
thread 'main' panicked at 'readiness timeout expired'
|
|
||||||
```
|
|
||||||
|
|
||||||
**Root Cause:** Groth16 proof generation is extremely slow without dev mode. The system tries to compute real cryptographic proofs, which can take minutes per block.
|
|
||||||
|
|
||||||
**Fix:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
|
||||||
```
|
|
||||||
|
|
||||||
**Prevention:** Set this in your shell profile or `.env` file so you never forget it.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 2. Missing `versions.env` File
|
|
||||||
|
|
||||||
**Symptoms:**
|
**Symptoms:**
|
||||||
- Helper scripts fail immediately
|
- Helper scripts fail immediately
|
||||||
@ -93,7 +58,7 @@ cat versions.env
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 3. Missing Circuit Assets
|
### 2. Missing Circuit Assets
|
||||||
|
|
||||||
**Symptoms:**
|
**Symptoms:**
|
||||||
- Node startup fails early
|
- Node startup fails early
|
||||||
@ -102,7 +67,7 @@ cat versions.env
|
|||||||
**What you'll see:**
|
**What you'll see:**
|
||||||
|
|
||||||
```text
|
```text
|
||||||
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
$ cargo run -p runner-examples --bin local_runner
|
||||||
[INFO testing_framework_runner_local] Starting local runner scenario
|
[INFO testing_framework_runner_local] Starting local runner scenario
|
||||||
Error: circuit assets directory missing or invalid
|
Error: circuit assets directory missing or invalid
|
||||||
thread 'main' panicked at 'workload init failed'
|
thread 'main' panicked at 'workload init failed'
|
||||||
@ -129,7 +94,7 @@ export LOGOS_BLOCKCHAIN_CIRCUITS=$HOME/.logos-blockchain-circuits
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 4. Node Binaries Not Found
|
### 3. Node Binaries Not Found
|
||||||
|
|
||||||
**Symptoms:**
|
**Symptoms:**
|
||||||
- Error about missing `logos-blockchain-node` binary
|
- Error about missing `logos-blockchain-node` binary
|
||||||
@ -139,7 +104,7 @@ export LOGOS_BLOCKCHAIN_CIRCUITS=$HOME/.logos-blockchain-circuits
|
|||||||
**What you'll see:**
|
**What you'll see:**
|
||||||
|
|
||||||
```text
|
```text
|
||||||
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
$ cargo run -p runner-examples --bin local_runner
|
||||||
[INFO testing_framework_runner_local] Spawning node 0
|
[INFO testing_framework_runner_local] Spawning node 0
|
||||||
Error: Os { code: 2, kind: NotFound, message: "No such file or directory" }
|
Error: Os { code: 2, kind: NotFound, message: "No such file or directory" }
|
||||||
thread 'main' panicked at 'failed to spawn logos-blockchain-node process'
|
thread 'main' panicked at 'failed to spawn logos-blockchain-node process'
|
||||||
@ -166,12 +131,12 @@ export LOGOS_BLOCKCHAIN_NODE_BIN=$PWD/target/release/logos-blockchain-node
|
|||||||
|
|
||||||
# Return to testing framework
|
# Return to testing framework
|
||||||
cd ../nomos-testing
|
cd ../nomos-testing
|
||||||
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 5. Docker Daemon Not Running (Compose)
|
### 4. Docker Daemon Not Running (Compose)
|
||||||
|
|
||||||
**Symptoms:**
|
**Symptoms:**
|
||||||
- Compose tests fail immediately
|
- Compose tests fail immediately
|
||||||
@ -208,7 +173,7 @@ sudo usermod -aG docker $USER
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 6. Image Not Found (Compose/K8s)
|
### 5. Image Not Found (Compose/K8s)
|
||||||
|
|
||||||
**Symptoms:**
|
**Symptoms:**
|
||||||
- Compose/K8s tests fail during deployment
|
- Compose/K8s tests fail during deployment
|
||||||
@ -218,7 +183,7 @@ sudo usermod -aG docker $USER
|
|||||||
**What you'll see:**
|
**What you'll see:**
|
||||||
|
|
||||||
```text
|
```text
|
||||||
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin compose_runner
|
$ cargo run -p runner-examples --bin compose_runner
|
||||||
[INFO testing_framework_runner_compose] Starting compose deployment
|
[INFO testing_framework_runner_compose] Starting compose deployment
|
||||||
Error: Failed to pull image 'logos-blockchain-testing:local': No such image
|
Error: Failed to pull image 'logos-blockchain-testing:local': No such image
|
||||||
thread 'main' panicked at 'compose deployment failed'
|
thread 'main' panicked at 'compose deployment failed'
|
||||||
@ -255,7 +220,7 @@ kind load docker-image logos-blockchain-testing:local
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 7. Port Conflicts
|
### 6. Port Conflicts
|
||||||
|
|
||||||
**Symptoms:**
|
**Symptoms:**
|
||||||
- "Address already in use" errors
|
- "Address already in use" errors
|
||||||
@ -265,7 +230,7 @@ kind load docker-image logos-blockchain-testing:local
|
|||||||
**What you'll see:**
|
**What you'll see:**
|
||||||
|
|
||||||
```text
|
```text
|
||||||
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
$ cargo run -p runner-examples --bin local_runner
|
||||||
[INFO testing_framework_runner_local] Launching node 0 on port 18080
|
[INFO testing_framework_runner_local] Launching node 0 on port 18080
|
||||||
Error: Os { code: 48, kind: AddrInUse, message: "Address already in use" }
|
Error: Os { code: 48, kind: AddrInUse, message: "Address already in use" }
|
||||||
thread 'main' panicked at 'failed to bind port 18080'
|
thread 'main' panicked at 'failed to bind port 18080'
|
||||||
@ -305,7 +270,7 @@ vim scripts/observability/compose/docker-compose.yml
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 8. Wallet Seeding Failed (Insufficient Funds)
|
### 7. Wallet Seeding Failed (Insufficient Funds)
|
||||||
|
|
||||||
**Symptoms:**
|
**Symptoms:**
|
||||||
- Transaction workload reports wallet issues
|
- Transaction workload reports wallet issues
|
||||||
@ -315,7 +280,7 @@ vim scripts/observability/compose/docker-compose.yml
|
|||||||
**What you'll see:**
|
**What you'll see:**
|
||||||
|
|
||||||
```text
|
```text
|
||||||
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
$ cargo run -p runner-examples --bin local_runner
|
||||||
[INFO testing_framework_workflows] Starting transaction workload with 10 users
|
[INFO testing_framework_workflows] Starting transaction workload with 10 users
|
||||||
[ERROR testing_framework_workflows] Wallet seeding failed: requested 10 users but only 3 wallets available
|
[ERROR testing_framework_workflows] Wallet seeding failed: requested 10 users but only 3 wallets available
|
||||||
thread 'main' panicked at 'workload init failed: insufficient wallets'
|
thread 'main' panicked at 'workload init failed: insufficient wallets'
|
||||||
@ -340,7 +305,7 @@ let scenario = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3))
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 9. Resource Exhaustion (OOM / CPU)
|
### 8. Resource Exhaustion (OOM / CPU)
|
||||||
|
|
||||||
**Symptoms:**
|
**Symptoms:**
|
||||||
- Nodes crash randomly
|
- Nodes crash randomly
|
||||||
@ -385,7 +350,7 @@ ulimit -n 4096
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 10. Logs Disappear After Run
|
### 9. Logs Disappear After Run
|
||||||
|
|
||||||
**Symptoms:**
|
**Symptoms:**
|
||||||
- Test completes but no logs on disk
|
- Test completes but no logs on disk
|
||||||
@ -395,7 +360,7 @@ ulimit -n 4096
|
|||||||
**What you'll see:**
|
**What you'll see:**
|
||||||
|
|
||||||
```text
|
```text
|
||||||
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
$ cargo run -p runner-examples --bin local_runner
|
||||||
[INFO runner_examples] Test complete, cleaning up
|
[INFO runner_examples] Test complete, cleaning up
|
||||||
[INFO testing_framework_runner_local] Removing temporary directories
|
[INFO testing_framework_runner_local] Removing temporary directories
|
||||||
$ ls .tmp/
|
$ ls .tmp/
|
||||||
@ -410,7 +375,6 @@ $ ls .tmp/
|
|||||||
# Persist logs to a specific directory
|
# Persist logs to a specific directory
|
||||||
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \
|
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \
|
||||||
LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1 \
|
LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1 \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
|
|
||||||
# Logs persist after run
|
# Logs persist after run
|
||||||
@ -422,7 +386,7 @@ ls /tmp/test-logs/
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 11. Consensus Timing Too Tight / Run Duration Too Short
|
### 10. Consensus Timing Too Tight / Run Duration Too Short
|
||||||
|
|
||||||
**Symptoms:**
|
**Symptoms:**
|
||||||
- "Consensus liveness expectation failed"
|
- "Consensus liveness expectation failed"
|
||||||
@ -432,7 +396,7 @@ ls /tmp/test-logs/
|
|||||||
**What you'll see:**
|
**What you'll see:**
|
||||||
|
|
||||||
```text
|
```text
|
||||||
$ POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner
|
$ cargo run -p runner-examples --bin local_runner
|
||||||
[INFO testing_framework_core] Starting workloads
|
[INFO testing_framework_core] Starting workloads
|
||||||
[INFO testing_framework_core] Run window: 10 seconds
|
[INFO testing_framework_core] Run window: 10 seconds
|
||||||
[INFO testing_framework_core] Evaluating expectations
|
[INFO testing_framework_core] Evaluating expectations
|
||||||
@ -463,7 +427,6 @@ let scenario = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3))
|
|||||||
# Faster block production (shorter slot time)
|
# Faster block production (shorter slot time)
|
||||||
CONSENSUS_SLOT_TIME=5 \
|
CONSENSUS_SLOT_TIME=5 \
|
||||||
CONSENSUS_ACTIVE_SLOT_COEFF=0.9 \
|
CONSENSUS_ACTIVE_SLOT_COEFF=0.9 \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -473,17 +436,16 @@ cargo run -p runner-examples --bin local_runner
|
|||||||
|
|
||||||
When a test fails, check these in order:
|
When a test fails, check these in order:
|
||||||
|
|
||||||
1. **`POL_PROOF_DEV_MODE=true` is set** (REQUIRED for all runners)
|
1. **`versions.env` exists at repo root**
|
||||||
2. **`versions.env` exists at repo root**
|
2. **Circuit assets present** (`LOGOS_BLOCKCHAIN_CIRCUITS` points to a valid directory)
|
||||||
3. **Circuit assets present** (`LOGOS_BLOCKCHAIN_CIRCUITS` points to a valid directory)
|
3. **Node binaries available** (`LOGOS_BLOCKCHAIN_NODE_BIN` set, or using `run-examples.sh`)
|
||||||
4. **Node binaries available** (`LOGOS_BLOCKCHAIN_NODE_BIN` set, or using `run-examples.sh`)
|
4. **Docker daemon running** (for compose/k8s)
|
||||||
5. **Docker daemon running** (for compose/k8s)
|
5. **Docker image built** (`logos-blockchain-testing:local` exists for compose/k8s)
|
||||||
6. **Docker image built** (`logos-blockchain-testing:local` exists for compose/k8s)
|
6. **No port conflicts** (`lsof -i :18080`, kill orphaned processes)
|
||||||
7. **No port conflicts** (`lsof -i :18080`, kill orphaned processes)
|
7. **Sufficient wallets** (`.wallets(N)` ≥ `.users(M)`)
|
||||||
8. **Sufficient wallets** (`.wallets(N)` ≥ `.users(M)`)
|
8. **Enough resources** (Docker memory 8GB+, ulimit -n 4096)
|
||||||
9. **Enough resources** (Docker memory 8GB+, ulimit -n 4096)
|
9. **Run duration appropriate** (long enough for consensus timing)
|
||||||
10. **Run duration appropriate** (long enough for consensus timing)
|
10. **Logs persisted** (`LOGOS_BLOCKCHAIN_LOG_DIR` + `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1` if needed)
|
||||||
11. **Logs persisted** (`LOGOS_BLOCKCHAIN_LOG_DIR` + `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1` if needed)
|
|
||||||
|
|
||||||
**Still stuck?** Check node logs (see [Where to Find Logs](#where-to-find-logs)) for the actual error.
|
**Still stuck?** Check node logs (see [Where to Find Logs](#where-to-find-logs)) for the actual error.
|
||||||
|
|
||||||
@ -509,14 +471,13 @@ When a test fails, check these in order:
|
|||||||
|
|
||||||
**Console output (default):**
|
**Console output (default):**
|
||||||
```bash
|
```bash
|
||||||
POL_PROOF_DEV_MODE=true cargo run -p runner-examples --bin local_runner 2>&1 | tee test.log
|
cargo run -p runner-examples --bin local_runner 2>&1 | tee test.log
|
||||||
```
|
```
|
||||||
|
|
||||||
**Persistent file output:**
|
**Persistent file output:**
|
||||||
```bash
|
```bash
|
||||||
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/debug-logs \
|
LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/debug-logs \
|
||||||
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
|
LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin local_runner
|
cargo run -p runner-examples --bin local_runner
|
||||||
|
|
||||||
# Inspect logs (note: filenames include timestamps):
|
# Inspect logs (note: filenames include timestamps):
|
||||||
@ -546,7 +507,6 @@ docker logs --tail 100 <container-id>
|
|||||||
```bash
|
```bash
|
||||||
COMPOSE_RUNNER_PRESERVE=1 \
|
COMPOSE_RUNNER_PRESERVE=1 \
|
||||||
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
|
LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
cargo run -p runner-examples --bin compose_runner
|
cargo run -p runner-examples --bin compose_runner
|
||||||
|
|
||||||
# OR: Use run-examples.sh (handles setup automatically)
|
# OR: Use run-examples.sh (handles setup automatically)
|
||||||
@ -651,7 +611,6 @@ Focus on the first node that exhibited problems or the node with the highest ind
|
|||||||
- "ERROR: versions.env missing" → missing required `versions.env` file at repository root
|
- "ERROR: versions.env missing" → missing required `versions.env` file at repository root
|
||||||
- "Failed to bind address" → port conflict
|
- "Failed to bind address" → port conflict
|
||||||
- "Connection refused" → peer not ready or network issue
|
- "Connection refused" → peer not ready or network issue
|
||||||
- "Proof verification failed" or "Proof generation timeout" → missing `POL_PROOF_DEV_MODE=true` (REQUIRED for all runners)
|
|
||||||
- "Circuit file not found" → missing circuit assets at the path in `LOGOS_BLOCKCHAIN_CIRCUITS`
|
- "Circuit file not found" → missing circuit assets at the path in `LOGOS_BLOCKCHAIN_CIRCUITS`
|
||||||
- "Insufficient funds" → wallet seeding issue (increase `.wallets(N)` or reduce `.users(M)`)
|
- "Insufficient funds" → wallet seeding issue (increase `.wallets(N)` or reduce `.users(M)`)
|
||||||
|
|
||||||
@ -689,16 +648,14 @@ Run a minimal baseline test (e.g., 2 nodes, consensus liveness only). If it pass
|
|||||||
|
|
||||||
### "Consensus liveness expectation failed"
|
### "Consensus liveness expectation failed"
|
||||||
|
|
||||||
- **Cause**: Not enough blocks produced during the run window, missing
|
- **Cause**: Not enough blocks produced during the run window, missing circuit
|
||||||
`POL_PROOF_DEV_MODE=true` (causes slow proof generation), or missing circuit
|
|
||||||
assets.
|
assets.
|
||||||
- **Fix**:
|
- **Fix**:
|
||||||
1. Verify `POL_PROOF_DEV_MODE=true` is set (REQUIRED for all runners).
|
1. Verify circuit assets exist at the path referenced by
|
||||||
2. Verify circuit assets exist at the path referenced by
|
|
||||||
`LOGOS_BLOCKCHAIN_CIRCUITS`.
|
`LOGOS_BLOCKCHAIN_CIRCUITS`.
|
||||||
3. Extend `with_run_duration()` to allow more blocks.
|
2. Extend `with_run_duration()` to allow more blocks.
|
||||||
4. Check node logs for proof generation or circuit asset errors.
|
3. Check node logs for proof generation or circuit asset errors.
|
||||||
5. Reduce transaction rate if nodes are overwhelmed.
|
4. Reduce transaction rate if nodes are overwhelmed.
|
||||||
|
|
||||||
### "Wallet seeding failed"
|
### "Wallet seeding failed"
|
||||||
|
|
||||||
@ -720,11 +677,10 @@ Run a minimal baseline test (e.g., 2 nodes, consensus liveness only). If it pass
|
|||||||
- **Cause**: Nodes didn't become responsive within expected time (often due to
|
- **Cause**: Nodes didn't become responsive within expected time (often due to
|
||||||
missing prerequisites).
|
missing prerequisites).
|
||||||
- **Fix**:
|
- **Fix**:
|
||||||
1. **Verify `POL_PROOF_DEV_MODE=true` is set** (REQUIRED for all runners—without
|
|
||||||
it, proof generation is too slow).
|
it, proof generation is too slow).
|
||||||
2. Check node logs for startup errors (port conflicts, missing assets).
|
1. Check node logs for startup errors (port conflicts, missing assets).
|
||||||
3. Verify network connectivity between nodes.
|
2. Verify network connectivity between nodes.
|
||||||
4. Ensure circuit assets are present and `LOGOS_BLOCKCHAIN_CIRCUITS` points to them.
|
3. Ensure circuit assets are present and `LOGOS_BLOCKCHAIN_CIRCUITS` points to them.
|
||||||
|
|
||||||
### "ERROR: versions.env missing"
|
### "ERROR: versions.env missing"
|
||||||
|
|
||||||
|
|||||||
@ -78,13 +78,7 @@ ScenarioBuilder::topology_with(|t| t.network_star().nodes(3))
|
|||||||
```
|
```
|
||||||
The workload will fail during `init()` if no wallets are configured.
|
The workload will fail during `init()` if no wallets are configured.
|
||||||
|
|
||||||
2. **Proof generation must be fast:**
|
2. **Circuit artifacts must be available:**
|
||||||
```bash
|
|
||||||
export POL_PROOF_DEV_MODE=true
|
|
||||||
```
|
|
||||||
Without this, proof generation takes ~30-60 seconds per transaction, causing timeouts.
|
|
||||||
|
|
||||||
3. **Circuit artifacts must be available:**
|
|
||||||
- Automatically staged by `scripts/run/run-examples.sh`
|
- Automatically staged by `scripts/run/run-examples.sh`
|
||||||
- Or manually via `scripts/setup/setup-logos-blockchain-circuits.sh` (recommended) / `scripts/setup/setup-logos-blockchain-circuits.sh`
|
- Or manually via `scripts/setup/setup-logos-blockchain-circuits.sh` (recommended) / `scripts/setup/setup-logos-blockchain-circuits.sh`
|
||||||
|
|
||||||
@ -108,7 +102,6 @@ Error: Expectation failed: TxInclusionExpectation
|
|||||||
Observed: 127 transactions
|
Observed: 127 transactions
|
||||||
|
|
||||||
Possible causes:
|
Possible causes:
|
||||||
- POL_PROOF_DEV_MODE not set (proof generation too slow)
|
|
||||||
- Duration too short (nodes still syncing)
|
- Duration too short (nodes still syncing)
|
||||||
- Node crashes (check logs for panics/OOM)
|
- Node crashes (check logs for panics/OOM)
|
||||||
- Wallet accounts not seeded (check topology config)
|
- Wallet accounts not seeded (check topology config)
|
||||||
@ -119,9 +112,8 @@ Error: Expectation failed: TxInclusionExpectation
|
|||||||
```bash
|
```bash
|
||||||
grep "proof generation" $LOGOS_BLOCKCHAIN_LOG_DIR/*/*.log
|
grep "proof generation" $LOGOS_BLOCKCHAIN_LOG_DIR/*/*.log
|
||||||
```
|
```
|
||||||
2. Verify `POL_PROOF_DEV_MODE=true` was set
|
2. Increase duration: `.with_run_duration(Duration::from_secs(120))`
|
||||||
3. Increase duration: `.with_run_duration(Duration::from_secs(120))`
|
3. Reduce rate: `.rate(5)` instead of `.rate(10)`
|
||||||
4. Reduce rate: `.rate(5)` instead of `.rate(10)`
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@ -382,12 +374,11 @@ ScenarioBuilder::topology_with(|t| t.network_star().nodes(3))
|
|||||||
When a workload or expectation fails:
|
When a workload or expectation fails:
|
||||||
|
|
||||||
1. Check logs: `$LOGOS_BLOCKCHAIN_LOG_DIR/*/` or `docker compose logs` or `kubectl logs`
|
1. Check logs: `$LOGOS_BLOCKCHAIN_LOG_DIR/*/` or `docker compose logs` or `kubectl logs`
|
||||||
2. Verify environment variables: `POL_PROOF_DEV_MODE`, `LOGOS_BLOCKCHAIN_NODE_BIN`, etc.
|
2. Check prerequisites: wallets, node control, circuits
|
||||||
3. Check prerequisites: wallets, node control, circuits
|
3. Increase duration: Double the run duration and retry
|
||||||
4. Increase duration: Double the run duration and retry
|
4. Reduce rates: Half the traffic rates and retry
|
||||||
5. Reduce rates: Half the traffic rates and retry
|
5. Check metrics: Prometheus queries for block height and tx count
|
||||||
6. Check metrics: Prometheus queries for block height and tx count
|
6. Reproduce locally: Use local runner for faster iteration
|
||||||
7. Reproduce locally: Use local runner for faster iteration
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
use std::{env, process, time::Duration};
|
use std::{process, time::Duration};
|
||||||
|
|
||||||
use anyhow::{Context as _, Result};
|
use anyhow::{Context as _, Result};
|
||||||
use runner_examples::{DeployerKind, ScenarioBuilderExt as _, demo, read_env_any};
|
use runner_examples::{DeployerKind, ScenarioBuilderExt as _, demo, read_env_any};
|
||||||
@ -17,11 +17,6 @@ async fn main() {
|
|||||||
|
|
||||||
tracing_subscriber::fmt::init();
|
tracing_subscriber::fmt::init();
|
||||||
|
|
||||||
if env::var("POL_PROOF_DEV_MODE").is_err() {
|
|
||||||
warn!("POL_PROOF_DEV_MODE=true is required for the local runner demo");
|
|
||||||
process::exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
let nodes = read_env_any(&["LOGOS_BLOCKCHAIN_DEMO_NODES"], demo::DEFAULT_NODES);
|
let nodes = read_env_any(&["LOGOS_BLOCKCHAIN_DEMO_NODES"], demo::DEFAULT_NODES);
|
||||||
let run_secs = read_env_any(&["LOGOS_BLOCKCHAIN_DEMO_RUN_SECS"], demo::DEFAULT_RUN_SECS);
|
let run_secs = read_env_any(&["LOGOS_BLOCKCHAIN_DEMO_RUN_SECS"], demo::DEFAULT_RUN_SECS);
|
||||||
|
|
||||||
|
|||||||
@ -21,7 +21,6 @@ fn set_default_env(key: &str, value: &str) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn init_logging_defaults() {
|
pub fn init_logging_defaults() {
|
||||||
set_default_env("POL_PROOF_DEV_MODE", "true");
|
|
||||||
set_default_env("LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS", "1");
|
set_default_env("LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS", "1");
|
||||||
set_default_env("LOGOS_BLOCKCHAIN_LOG_LEVEL", "info");
|
set_default_env("LOGOS_BLOCKCHAIN_LOG_LEVEL", "info");
|
||||||
set_default_env("RUST_LOG", "info");
|
set_default_env("RUST_LOG", "info");
|
||||||
|
|||||||
@ -86,6 +86,7 @@ impl Workload for JoinNodeWithPeersWorkload {
|
|||||||
let options = StartNodeOptions {
|
let options = StartNodeOptions {
|
||||||
peers: PeerSelection::Named(self.peers.clone()),
|
peers: PeerSelection::Named(self.peers.clone()),
|
||||||
config_patch: None,
|
config_patch: None,
|
||||||
|
persist_dir: None,
|
||||||
};
|
};
|
||||||
let node = handle.start_node_with(&self.name, options).await?;
|
let node = handle.start_node_with(&self.name, options).await?;
|
||||||
let client = node.api;
|
let client = node.api;
|
||||||
|
|||||||
@ -18,7 +18,6 @@ const CONVERGENCE_POLL: Duration = Duration::from_secs(2);
|
|||||||
async fn manual_cluster_two_clusters_merge() -> Result<()> {
|
async fn manual_cluster_two_clusters_merge() -> Result<()> {
|
||||||
let _ = try_init();
|
let _ = try_init();
|
||||||
// Required env vars (set on the command line when running this test):
|
// Required env vars (set on the command line when running this test):
|
||||||
// - `POL_PROOF_DEV_MODE=true`
|
|
||||||
// - `RUST_LOG=info` (optional)
|
// - `RUST_LOG=info` (optional)
|
||||||
let config = TopologyConfig::with_node_numbers(2);
|
let config = TopologyConfig::with_node_numbers(2);
|
||||||
let deployer = LocalDeployer::new();
|
let deployer = LocalDeployer::new();
|
||||||
@ -33,6 +32,7 @@ async fn manual_cluster_two_clusters_merge() -> Result<()> {
|
|||||||
StartNodeOptions {
|
StartNodeOptions {
|
||||||
peers: PeerSelection::None,
|
peers: PeerSelection::None,
|
||||||
config_patch: None,
|
config_patch: None,
|
||||||
|
persist_dir: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
@ -48,6 +48,7 @@ async fn manual_cluster_two_clusters_merge() -> Result<()> {
|
|||||||
StartNodeOptions {
|
StartNodeOptions {
|
||||||
peers: PeerSelection::Named(vec!["node-a".to_owned()]),
|
peers: PeerSelection::Named(vec!["node-a".to_owned()]),
|
||||||
config_patch: None,
|
config_patch: None,
|
||||||
|
persist_dir: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
@ -82,3 +83,57 @@ async fn manual_cluster_two_clusters_merge() -> Result<()> {
|
|||||||
sleep(CONVERGENCE_POLL).await;
|
sleep(CONVERGENCE_POLL).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore = "run manually with `cargo test -p runner-examples -- --ignored manual_cluster_with_persist_dir`"]
|
||||||
|
async fn manual_cluster_with_persist_dir() -> Result<()> {
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
let _ = try_init();
|
||||||
|
// Required env vars (set on the command line when running this test):
|
||||||
|
// - `RUST_LOG=info` (optional)
|
||||||
|
let config = TopologyConfig::with_node_numbers(1);
|
||||||
|
let deployer = LocalDeployer::new();
|
||||||
|
let cluster = deployer.manual_cluster(config)?;
|
||||||
|
|
||||||
|
let persist_dir = PathBuf::from("/tmp/test-node-persist-dir");
|
||||||
|
|
||||||
|
println!("starting validator with persist_dir: {:?}", persist_dir);
|
||||||
|
|
||||||
|
let _node = cluster
|
||||||
|
.start_node_with(
|
||||||
|
"test",
|
||||||
|
StartNodeOptions {
|
||||||
|
peers: PeerSelection::None,
|
||||||
|
config_patch: None,
|
||||||
|
persist_dir: Some(persist_dir.clone()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
.api;
|
||||||
|
|
||||||
|
println!("validator started, waiting briefly");
|
||||||
|
sleep(Duration::from_secs(5)).await;
|
||||||
|
|
||||||
|
// Drop the cluster to trigger the persist logic
|
||||||
|
drop(cluster);
|
||||||
|
|
||||||
|
println!("cluster dropped, checking if persist_dir exists");
|
||||||
|
|
||||||
|
// Verify the persist_dir was created
|
||||||
|
if !persist_dir.exists() {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"persist_dir was not created: {:?}",
|
||||||
|
persist_dir
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("persist_dir verified: {:?}", persist_dir);
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
if persist_dir.exists() {
|
||||||
|
std::fs::remove_dir_all(&persist_dir)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|||||||
@ -17,7 +17,6 @@ use tracing_subscriber::fmt::try_init;
|
|||||||
async fn manual_cluster_api_port_override() -> Result<()> {
|
async fn manual_cluster_api_port_override() -> Result<()> {
|
||||||
let _ = try_init();
|
let _ = try_init();
|
||||||
// Required env vars (set on the command line when running this test):
|
// Required env vars (set on the command line when running this test):
|
||||||
// - `POL_PROOF_DEV_MODE=true`
|
|
||||||
// - `LOGOS_BLOCKCHAIN_NODE_BIN=...`
|
// - `LOGOS_BLOCKCHAIN_NODE_BIN=...`
|
||||||
// - `LOGOS_BLOCKCHAIN_CIRCUITS=...`
|
// - `LOGOS_BLOCKCHAIN_CIRCUITS=...`
|
||||||
// - `RUST_LOG=info` (optional)
|
// - `RUST_LOG=info` (optional)
|
||||||
@ -33,6 +32,7 @@ async fn manual_cluster_api_port_override() -> Result<()> {
|
|||||||
StartNodeOptions {
|
StartNodeOptions {
|
||||||
peers: PeerSelection::None,
|
peers: PeerSelection::None,
|
||||||
config_patch: None,
|
config_patch: None,
|
||||||
|
persist_dir: None,
|
||||||
}
|
}
|
||||||
.create_patch(move |mut config| {
|
.create_patch(move |mut config| {
|
||||||
println!("overriding API port to {api_port}");
|
println!("overriding API port to {api_port}");
|
||||||
@ -62,7 +62,6 @@ async fn manual_cluster_api_port_override() -> Result<()> {
|
|||||||
async fn scenario_builder_api_port_override() -> Result<()> {
|
async fn scenario_builder_api_port_override() -> Result<()> {
|
||||||
let _ = try_init();
|
let _ = try_init();
|
||||||
// Required env vars (set on the command line when running this test):
|
// Required env vars (set on the command line when running this test):
|
||||||
// - `POL_PROOF_DEV_MODE=true`
|
|
||||||
// - `LOGOS_BLOCKCHAIN_NODE_BIN=...`
|
// - `LOGOS_BLOCKCHAIN_NODE_BIN=...`
|
||||||
// - `LOGOS_BLOCKCHAIN_CIRCUITS=...`
|
// - `LOGOS_BLOCKCHAIN_CIRCUITS=...`
|
||||||
// - `RUST_LOG=info` (optional)
|
// - `RUST_LOG=info` (optional)
|
||||||
|
|||||||
@ -25,7 +25,6 @@ const POLL_INTERVAL: Duration = Duration::from_secs(1);
|
|||||||
async fn orphan_manual_cluster() -> Result<()> {
|
async fn orphan_manual_cluster() -> Result<()> {
|
||||||
let _ = try_init();
|
let _ = try_init();
|
||||||
// Required env vars (set on the command line when running this test):
|
// Required env vars (set on the command line when running this test):
|
||||||
// - `POL_PROOF_DEV_MODE=true`
|
|
||||||
// - `LOGOS_BLOCKCHAIN_NODE_BIN=...`
|
// - `LOGOS_BLOCKCHAIN_NODE_BIN=...`
|
||||||
// - `NOMOS_KZGRS_PARAMS_PATH=...` (path to KZG params directory/file)
|
// - `NOMOS_KZGRS_PARAMS_PATH=...` (path to KZG params directory/file)
|
||||||
// - `RUST_LOG=info` (optional; better visibility)
|
// - `RUST_LOG=info` (optional; better visibility)
|
||||||
|
|||||||
@ -515,7 +515,6 @@ run_examples::run() {
|
|||||||
echo "==> Running ${BIN} for ${RUN_SECS}s (mode=${MODE}, image=${IMAGE})"
|
echo "==> Running ${BIN} for ${RUN_SECS}s (mode=${MODE}, image=${IMAGE})"
|
||||||
cd "${ROOT_DIR}"
|
cd "${ROOT_DIR}"
|
||||||
|
|
||||||
POL_PROOF_DEV_MODE=true \
|
|
||||||
TESTNET_PRINT_ENDPOINTS=1 \
|
TESTNET_PRINT_ENDPOINTS=1 \
|
||||||
LOGOS_BLOCKCHAIN_TESTNET_IMAGE="${IMAGE}" \
|
LOGOS_BLOCKCHAIN_TESTNET_IMAGE="${IMAGE}" \
|
||||||
LOGOS_BLOCKCHAIN_NODE_BIN="${LOGOS_BLOCKCHAIN_NODE_BIN:-}" \
|
LOGOS_BLOCKCHAIN_NODE_BIN="${LOGOS_BLOCKCHAIN_NODE_BIN:-}" \
|
||||||
|
|||||||
@ -47,8 +47,7 @@ export CFG_FILE_PATH="/config.yaml" \
|
|||||||
CFG_HOST_KIND="${CFG_HOST_KIND:-$role}" \
|
CFG_HOST_KIND="${CFG_HOST_KIND:-$role}" \
|
||||||
CFG_HOST_IDENTIFIER="${CFG_HOST_IDENTIFIER:-$host_identifier_default}" \
|
CFG_HOST_IDENTIFIER="${CFG_HOST_IDENTIFIER:-$host_identifier_default}" \
|
||||||
LOGOS_BLOCKCHAIN_TIME_BACKEND="${LOGOS_BLOCKCHAIN_TIME_BACKEND:-monotonic}" \
|
LOGOS_BLOCKCHAIN_TIME_BACKEND="${LOGOS_BLOCKCHAIN_TIME_BACKEND:-monotonic}" \
|
||||||
LOG_LEVEL="${LOG_LEVEL:-INFO}" \
|
LOG_LEVEL="${LOG_LEVEL:-INFO}"``"
|
||||||
POL_PROOF_DEV_MODE="${POL_PROOF_DEV_MODE:-true}"
|
|
||||||
|
|
||||||
# Ensure recovery directory exists to avoid early crashes in services that
|
# Ensure recovery directory exists to avoid early crashes in services that
|
||||||
# persist state.
|
# persist state.
|
||||||
|
|||||||
@ -13,7 +13,8 @@ use std::cmp;
|
|||||||
|
|
||||||
use blend::GeneralBlendConfig;
|
use blend::GeneralBlendConfig;
|
||||||
use consensus::{
|
use consensus::{
|
||||||
ConsensusConfigError, GeneralConsensusConfig, ProviderInfo, create_genesis_tx_with_declarations,
|
ConsensusConfigError, GeneralConsensusConfig, ProviderInfo,
|
||||||
|
create_genesis_tx_with_declarations, sync_utxos_with_genesis,
|
||||||
};
|
};
|
||||||
use key_management_system_service::{backend::preload::PreloadKMSBackendSettings, keys::Key};
|
use key_management_system_service::{backend::preload::PreloadKMSBackendSettings, keys::Key};
|
||||||
use network::GeneralNetworkConfig;
|
use network::GeneralNetworkConfig;
|
||||||
@ -203,7 +204,7 @@ fn apply_consensus_genesis_tx(
|
|||||||
) -> Result<(), ConsensusConfigError> {
|
) -> Result<(), ConsensusConfigError> {
|
||||||
for c in consensus_configs {
|
for c in consensus_configs {
|
||||||
c.genesis_tx = genesis_tx.clone();
|
c.genesis_tx = genesis_tx.clone();
|
||||||
consensus::sync_utxos_with_genesis(&mut c.utxos, genesis_tx)?;
|
sync_utxos_with_genesis(&mut c.utxos, genesis_tx)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
@ -71,15 +71,23 @@ pub struct NodeHandle<T> {
|
|||||||
pub(crate) tempdir: TempDir,
|
pub(crate) tempdir: TempDir,
|
||||||
pub(crate) config: T,
|
pub(crate) config: T,
|
||||||
pub(crate) api: ApiClient,
|
pub(crate) api: ApiClient,
|
||||||
|
pub(crate) persist_dir: Option<PathBuf>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<T> NodeHandle<T> {
|
impl<T> NodeHandle<T> {
|
||||||
pub fn new(child: Child, tempdir: TempDir, config: T, api: ApiClient) -> Self {
|
pub fn new(
|
||||||
|
child: Child,
|
||||||
|
tempdir: TempDir,
|
||||||
|
config: T,
|
||||||
|
api: ApiClient,
|
||||||
|
persist_dir: Option<PathBuf>,
|
||||||
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
child,
|
child,
|
||||||
tempdir,
|
tempdir,
|
||||||
config,
|
config,
|
||||||
api,
|
api,
|
||||||
|
persist_dir,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -154,6 +162,7 @@ pub async fn spawn_node<C>(
|
|||||||
config_filename: &str,
|
config_filename: &str,
|
||||||
binary_path: PathBuf,
|
binary_path: PathBuf,
|
||||||
enable_logging: bool,
|
enable_logging: bool,
|
||||||
|
persist_dir: Option<PathBuf>,
|
||||||
) -> Result<NodeHandle<C>, SpawnNodeError>
|
) -> Result<NodeHandle<C>, SpawnNodeError>
|
||||||
where
|
where
|
||||||
C: NodeConfigCommon + Serialize,
|
C: NodeConfigCommon + Serialize,
|
||||||
@ -168,7 +177,13 @@ where
|
|||||||
|
|
||||||
let child = spawn_node_process(&binary_path, &config_path, dir.path())?;
|
let child = spawn_node_process(&binary_path, &config_path, dir.path())?;
|
||||||
|
|
||||||
let mut handle = NodeHandle::new(child, dir, config, ApiClient::new(addr, testing_addr));
|
let mut handle = NodeHandle::new(
|
||||||
|
child,
|
||||||
|
dir,
|
||||||
|
config,
|
||||||
|
ApiClient::new(addr, testing_addr),
|
||||||
|
persist_dir,
|
||||||
|
);
|
||||||
|
|
||||||
// Wait for readiness via consensus_info
|
// Wait for readiness via consensus_info
|
||||||
let ready = wait_for_consensus_readiness(&handle.api).await;
|
let ready = wait_for_consensus_readiness(&handle.api).await;
|
||||||
|
|||||||
@ -7,6 +7,7 @@ use std::sync::LazyLock;
|
|||||||
pub use api_client::{ApiClient, ApiClientError};
|
pub use api_client::{ApiClient, ApiClientError};
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
use testing_framework_env as tf_env;
|
use testing_framework_env as tf_env;
|
||||||
|
use tracing::info;
|
||||||
|
|
||||||
pub(crate) const LOGS_PREFIX: &str = "__logs";
|
pub(crate) const LOGS_PREFIX: &str = "__logs";
|
||||||
static KEEP_NODE_TEMPDIRS: LazyLock<bool> = LazyLock::new(tf_env::nomos_tests_keep_logs);
|
static KEEP_NODE_TEMPDIRS: LazyLock<bool> = LazyLock::new(tf_env::nomos_tests_keep_logs);
|
||||||
@ -30,6 +31,55 @@ fn persist_tempdir(tempdir: &mut TempDir, label: &str) -> std::io::Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn persist_tempdir_to(
|
||||||
|
tempdir: &mut TempDir,
|
||||||
|
target_dir: &std::path::Path,
|
||||||
|
label: &str,
|
||||||
|
) -> std::io::Result<()> {
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
info!(
|
||||||
|
label,
|
||||||
|
from = %tempdir.path().display(),
|
||||||
|
to = %target_dir.display(),
|
||||||
|
"persisting directory"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Create parent directory if it doesn't exist
|
||||||
|
if let Some(parent) = target_dir.parent() {
|
||||||
|
fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy tempdir contents to target directory
|
||||||
|
if target_dir.exists() {
|
||||||
|
fs::remove_dir_all(target_dir)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recursively copies all contents from src directory to dst directory.
|
||||||
|
///
|
||||||
|
/// # Arguments
|
||||||
|
/// * `src` - Source directory path to copy from
|
||||||
|
/// * `dst` - Destination directory path to copy to
|
||||||
|
fn copy_dir_all(src: &std::path::Path, dst: &std::path::Path) -> std::io::Result<()> {
|
||||||
|
use std::fs;
|
||||||
|
fs::create_dir_all(dst)?;
|
||||||
|
for entry in fs::read_dir(src)? {
|
||||||
|
let entry = entry?;
|
||||||
|
let ty = entry.file_type()?;
|
||||||
|
if ty.is_dir() {
|
||||||
|
copy_dir_all(&entry.path(), &dst.join(entry.file_name()))?;
|
||||||
|
} else {
|
||||||
|
fs::copy(entry.path(), dst.join(entry.file_name()))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
copy_dir_all(tempdir.path(), target_dir)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn should_persist_tempdir() -> bool {
|
pub(crate) fn should_persist_tempdir() -> bool {
|
||||||
std::thread::panicking() || *KEEP_NODE_TEMPDIRS
|
std::thread::panicking() || *KEEP_NODE_TEMPDIRS
|
||||||
}
|
}
|
||||||
|
|||||||
@ -5,7 +5,7 @@ use nomos_tracing_service::LoggerLayer;
|
|||||||
pub use testing_framework_config::nodes::node::create_node_config;
|
pub use testing_framework_config::nodes::node::create_node_config;
|
||||||
use tracing::{debug, info};
|
use tracing::{debug, info};
|
||||||
|
|
||||||
use super::{persist_tempdir, should_persist_tempdir};
|
use super::{persist_tempdir, persist_tempdir_to, should_persist_tempdir};
|
||||||
use crate::{
|
use crate::{
|
||||||
IS_DEBUG_TRACING,
|
IS_DEBUG_TRACING,
|
||||||
nodes::{
|
nodes::{
|
||||||
@ -67,10 +67,20 @@ impl Deref for Node {
|
|||||||
|
|
||||||
impl Drop for Node {
|
impl Drop for Node {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if should_persist_tempdir()
|
if should_persist_tempdir() {
|
||||||
&& let Err(e) = persist_tempdir(&mut self.handle.tempdir, "logos-blockchain-node")
|
if let Some(ref persist_dir) = self.handle.persist_dir {
|
||||||
{
|
if let Err(e) = persist_tempdir_to(
|
||||||
debug!(error = ?e, "failed to persist node tempdir");
|
&mut self.handle.tempdir,
|
||||||
|
persist_dir,
|
||||||
|
"logos-blockchain-node",
|
||||||
|
) {
|
||||||
|
debug!(error = ?e, persist_dir = %persist_dir.display(), "failed to persist node tempdir to custom directory");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if let Err(e) = persist_tempdir(&mut self.handle.tempdir, "logos-blockchain-node") {
|
||||||
|
debug!(error = ?e, "failed to persist node tempdir");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("stopping node process");
|
debug!("stopping node process");
|
||||||
@ -96,7 +106,11 @@ impl Node {
|
|||||||
self.handle.wait_for_exit(timeout).await
|
self.handle.wait_for_exit(timeout).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn spawn(config: RunConfig, label: &str) -> Result<Self, SpawnNodeError> {
|
pub async fn spawn(
|
||||||
|
config: RunConfig,
|
||||||
|
label: &str,
|
||||||
|
persist_dir: Option<PathBuf>,
|
||||||
|
) -> Result<Self, SpawnNodeError> {
|
||||||
let log_prefix = format!("{LOGS_PREFIX}-{label}");
|
let log_prefix = format!("{LOGS_PREFIX}-{label}");
|
||||||
let handle = spawn_node(
|
let handle = spawn_node(
|
||||||
config,
|
config,
|
||||||
@ -104,6 +118,7 @@ impl Node {
|
|||||||
"node.yaml",
|
"node.yaml",
|
||||||
binary_path(),
|
binary_path(),
|
||||||
!*IS_DEBUG_TRACING,
|
!*IS_DEBUG_TRACING,
|
||||||
|
persist_dir,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
use std::sync::Arc;
|
use std::{path::PathBuf, sync::Arc};
|
||||||
|
|
||||||
use reqwest::Url;
|
use reqwest::Url;
|
||||||
|
|
||||||
@ -41,6 +41,8 @@ pub struct StartNodeOptions {
|
|||||||
pub peers: PeerSelection,
|
pub peers: PeerSelection,
|
||||||
/// Optional node config patch applied before spawn.
|
/// Optional node config patch applied before spawn.
|
||||||
pub config_patch: Option<NodeConfigPatch>,
|
pub config_patch: Option<NodeConfigPatch>,
|
||||||
|
/// Optional directory to persist node's tempdir to on stop.
|
||||||
|
pub persist_dir: Option<PathBuf>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for StartNodeOptions {
|
impl Default for StartNodeOptions {
|
||||||
@ -48,6 +50,7 @@ impl Default for StartNodeOptions {
|
|||||||
Self {
|
Self {
|
||||||
peers: PeerSelection::DefaultLayout,
|
peers: PeerSelection::DefaultLayout,
|
||||||
config_patch: None,
|
config_patch: None,
|
||||||
|
persist_dir: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
use std::{collections::HashMap, sync::Arc};
|
use std::{collections::HashMap, path::PathBuf, sync::Arc};
|
||||||
|
|
||||||
use nomos_core::{
|
use nomos_core::{
|
||||||
mantle::GenesisTx as _,
|
mantle::GenesisTx as _,
|
||||||
@ -11,7 +11,7 @@ use testing_framework_config::topology::{
|
|||||||
base::{BaseConfigError, BaseConfigs, build_base_configs},
|
base::{BaseConfigError, BaseConfigs, build_base_configs},
|
||||||
consensus::{
|
consensus::{
|
||||||
ConsensusConfigError, ConsensusParams, ProviderInfo,
|
ConsensusConfigError, ConsensusParams, ProviderInfo,
|
||||||
create_genesis_tx_with_declarations,
|
create_genesis_tx_with_declarations, sync_utxos_with_genesis,
|
||||||
},
|
},
|
||||||
network::{Libp2pNetworkLayout, NetworkParams},
|
network::{Libp2pNetworkLayout, NetworkParams},
|
||||||
tracing::create_tracing_configs,
|
tracing::create_tracing_configs,
|
||||||
@ -65,6 +65,7 @@ pub struct TopologyConfig {
|
|||||||
pub network_params: NetworkParams,
|
pub network_params: NetworkParams,
|
||||||
pub wallet_config: WalletConfig,
|
pub wallet_config: WalletConfig,
|
||||||
pub node_config_patches: HashMap<usize, NodeConfigPatch>,
|
pub node_config_patches: HashMap<usize, NodeConfigPatch>,
|
||||||
|
pub persist_dirs: HashMap<usize, PathBuf>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TopologyConfig {
|
impl TopologyConfig {
|
||||||
@ -77,6 +78,7 @@ impl TopologyConfig {
|
|||||||
network_params: NetworkParams::default(),
|
network_params: NetworkParams::default(),
|
||||||
wallet_config: WalletConfig::default(),
|
wallet_config: WalletConfig::default(),
|
||||||
node_config_patches: HashMap::new(),
|
node_config_patches: HashMap::new(),
|
||||||
|
persist_dirs: HashMap::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -89,6 +91,7 @@ impl TopologyConfig {
|
|||||||
network_params: NetworkParams::default(),
|
network_params: NetworkParams::default(),
|
||||||
wallet_config: WalletConfig::default(),
|
wallet_config: WalletConfig::default(),
|
||||||
node_config_patches: HashMap::new(),
|
node_config_patches: HashMap::new(),
|
||||||
|
persist_dirs: HashMap::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -103,6 +106,7 @@ impl TopologyConfig {
|
|||||||
network_params: NetworkParams::default(),
|
network_params: NetworkParams::default(),
|
||||||
wallet_config: WalletConfig::default(),
|
wallet_config: WalletConfig::default(),
|
||||||
node_config_patches: HashMap::new(),
|
node_config_patches: HashMap::new(),
|
||||||
|
persist_dirs: HashMap::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -121,6 +125,17 @@ impl TopologyConfig {
|
|||||||
self.node_config_patches.insert(index, patch);
|
self.node_config_patches.insert(index, patch);
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn persist_dir(&self, index: usize) -> Option<&PathBuf> {
|
||||||
|
self.persist_dirs.get(&index)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn with_persist_dir(mut self, index: usize, dir: PathBuf) -> Self {
|
||||||
|
self.persist_dirs.insert(index, dir);
|
||||||
|
self
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Builder that produces `GeneratedTopology` instances from a `TopologyConfig`.
|
/// Builder that produces `GeneratedTopology` instances from a `TopologyConfig`.
|
||||||
@ -163,6 +178,13 @@ impl TopologyBuilder {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
/// Apply a persist dir for a specific node index.
|
||||||
|
pub fn with_persist_dir(mut self, index: usize, dir: PathBuf) -> Self {
|
||||||
|
self.config.persist_dirs.insert(index, dir);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
/// Set node counts.
|
/// Set node counts.
|
||||||
pub const fn with_node_count(mut self, nodes: usize) -> Self {
|
pub const fn with_node_count(mut self, nodes: usize) -> Self {
|
||||||
@ -240,6 +262,7 @@ impl TopologyBuilder {
|
|||||||
&kms_configs,
|
&kms_configs,
|
||||||
&time_config,
|
&time_config,
|
||||||
&config.node_config_patches,
|
&config.node_config_patches,
|
||||||
|
&config.persist_dirs,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
Ok(GeneratedTopology { config, nodes })
|
Ok(GeneratedTopology { config, nodes })
|
||||||
@ -310,10 +333,7 @@ fn apply_consensus_genesis_tx(
|
|||||||
) -> Result<(), TopologyBuildError> {
|
) -> Result<(), TopologyBuildError> {
|
||||||
for c in consensus_configs {
|
for c in consensus_configs {
|
||||||
c.genesis_tx = genesis_tx.clone();
|
c.genesis_tx = genesis_tx.clone();
|
||||||
testing_framework_config::topology::configs::consensus::sync_utxos_with_genesis(
|
sync_utxos_with_genesis(&mut c.utxos, genesis_tx)?;
|
||||||
&mut c.utxos,
|
|
||||||
genesis_tx,
|
|
||||||
)?;
|
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@ -333,6 +353,7 @@ fn build_node_descriptors(
|
|||||||
kms_configs: &[key_management_system_service::backend::preload::PreloadKMSBackendSettings],
|
kms_configs: &[key_management_system_service::backend::preload::PreloadKMSBackendSettings],
|
||||||
time_config: &testing_framework_config::topology::configs::time::GeneralTimeConfig,
|
time_config: &testing_framework_config::topology::configs::time::GeneralTimeConfig,
|
||||||
node_config_patches: &HashMap<usize, NodeConfigPatch>,
|
node_config_patches: &HashMap<usize, NodeConfigPatch>,
|
||||||
|
persist_dirs: &HashMap<usize, PathBuf>,
|
||||||
) -> Result<Vec<GeneratedNodeConfig>, TopologyBuildError> {
|
) -> Result<Vec<GeneratedNodeConfig>, TopologyBuildError> {
|
||||||
let mut nodes = Vec::with_capacity(config.n_nodes);
|
let mut nodes = Vec::with_capacity(config.n_nodes);
|
||||||
|
|
||||||
@ -367,6 +388,7 @@ fn build_node_descriptors(
|
|||||||
general,
|
general,
|
||||||
blend_port,
|
blend_port,
|
||||||
config_patch: node_config_patches.get(&i).cloned(),
|
config_patch: node_config_patches.get(&i).cloned(),
|
||||||
|
persist_dir: persist_dirs.get(&i).cloned(),
|
||||||
};
|
};
|
||||||
|
|
||||||
nodes.push(descriptor);
|
nodes.push(descriptor);
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
use std::{collections::HashSet, time::Duration};
|
use std::{collections::HashSet, path::PathBuf, time::Duration};
|
||||||
|
|
||||||
use reqwest::{Client, Url};
|
use reqwest::{Client, Url};
|
||||||
|
|
||||||
@ -16,6 +16,7 @@ pub struct GeneratedNodeConfig {
|
|||||||
pub general: GeneralConfig,
|
pub general: GeneralConfig,
|
||||||
pub blend_port: u16,
|
pub blend_port: u16,
|
||||||
pub config_patch: Option<NodeConfigPatch>,
|
pub config_patch: Option<NodeConfigPatch>,
|
||||||
|
pub persist_dir: Option<PathBuf>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl GeneratedNodeConfig {
|
impl GeneratedNodeConfig {
|
||||||
|
|||||||
@ -119,12 +119,10 @@ fn default_extra_hosts() -> Vec<String> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn base_environment(cfgsync_port: u16) -> Vec<EnvEntry> {
|
fn base_environment(cfgsync_port: u16) -> Vec<EnvEntry> {
|
||||||
let pol_mode = tf_env::pol_proof_dev_mode().unwrap_or_else(|| "true".to_string());
|
|
||||||
let rust_log = tf_env::rust_log().unwrap_or_else(|| "info".to_string());
|
let rust_log = tf_env::rust_log().unwrap_or_else(|| "info".to_string());
|
||||||
let nomos_log_level = tf_env::nomos_log_level().unwrap_or_else(|| "info".to_string());
|
let nomos_log_level = tf_env::nomos_log_level().unwrap_or_else(|| "info".to_string());
|
||||||
let time_backend = tf_env::nomos_time_backend().unwrap_or_else(|| "monotonic".into());
|
let time_backend = tf_env::nomos_time_backend().unwrap_or_else(|| "monotonic".into());
|
||||||
vec![
|
vec![
|
||||||
EnvEntry::new("POL_PROOF_DEV_MODE", pol_mode),
|
|
||||||
EnvEntry::new("RUST_LOG", rust_log),
|
EnvEntry::new("RUST_LOG", rust_log),
|
||||||
EnvEntry::new("LOGOS_BLOCKCHAIN_LOG_LEVEL", nomos_log_level),
|
EnvEntry::new("LOGOS_BLOCKCHAIN_LOG_LEVEL", nomos_log_level),
|
||||||
EnvEntry::new("LOGOS_BLOCKCHAIN_TIME_BACKEND", time_backend),
|
EnvEntry::new("LOGOS_BLOCKCHAIN_TIME_BACKEND", time_backend),
|
||||||
|
|||||||
@ -291,11 +291,9 @@ fn build_values(topology: &GeneratedTopology) -> HelmValues {
|
|||||||
let cfgsync = CfgsyncValues {
|
let cfgsync = CfgsyncValues {
|
||||||
port: cfgsync_port(),
|
port: cfgsync_port(),
|
||||||
};
|
};
|
||||||
let pol_mode = pol_proof_mode();
|
|
||||||
let image_pull_policy =
|
let image_pull_policy =
|
||||||
tf_env::nomos_testnet_image_pull_policy().unwrap_or_else(|| "IfNotPresent".into());
|
tf_env::nomos_testnet_image_pull_policy().unwrap_or_else(|| "IfNotPresent".into());
|
||||||
debug!(pol_mode, "rendering Helm values for k8s stack");
|
let nodes = build_node_group("node", topology.nodes());
|
||||||
let nodes = build_node_group("node", topology.nodes(), &pol_mode);
|
|
||||||
|
|
||||||
HelmValues {
|
HelmValues {
|
||||||
image_pull_policy,
|
image_pull_policy,
|
||||||
@ -307,12 +305,11 @@ fn build_values(topology: &GeneratedTopology) -> HelmValues {
|
|||||||
fn build_node_group(
|
fn build_node_group(
|
||||||
kind: &'static str,
|
kind: &'static str,
|
||||||
nodes: &[testing_framework_core::topology::generation::GeneratedNodeConfig],
|
nodes: &[testing_framework_core::topology::generation::GeneratedNodeConfig],
|
||||||
pol_mode: &str,
|
|
||||||
) -> NodeGroup {
|
) -> NodeGroup {
|
||||||
let node_values = nodes
|
let node_values = nodes
|
||||||
.iter()
|
.iter()
|
||||||
.enumerate()
|
.enumerate()
|
||||||
.map(|(index, node)| build_node_values(kind, index, node, pol_mode))
|
.map(|(index, node)| build_node_values(kind, index, node))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
NodeGroup {
|
NodeGroup {
|
||||||
@ -325,10 +322,8 @@ fn build_node_values(
|
|||||||
kind: &'static str,
|
kind: &'static str,
|
||||||
index: usize,
|
index: usize,
|
||||||
node: &testing_framework_core::topology::generation::GeneratedNodeConfig,
|
node: &testing_framework_core::topology::generation::GeneratedNodeConfig,
|
||||||
pol_mode: &str,
|
|
||||||
) -> NodeValues {
|
) -> NodeValues {
|
||||||
let mut env = BTreeMap::new();
|
let mut env = BTreeMap::new();
|
||||||
env.insert("POL_PROOF_DEV_MODE".into(), pol_mode.to_string());
|
|
||||||
env.insert("CFG_NETWORK_PORT".into(), node.network_port().to_string());
|
env.insert("CFG_NETWORK_PORT".into(), node.network_port().to_string());
|
||||||
env.insert("CFG_BLEND_PORT".into(), node.blend_port.to_string());
|
env.insert("CFG_BLEND_PORT".into(), node.blend_port.to_string());
|
||||||
env.insert(
|
env.insert(
|
||||||
@ -352,7 +347,3 @@ fn build_node_values(
|
|||||||
env,
|
env,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn pol_proof_mode() -> String {
|
|
||||||
tf_env::pol_proof_dev_mode().unwrap_or_else(|| "true".to_string())
|
|
||||||
}
|
|
||||||
|
|||||||
@ -110,7 +110,7 @@ impl LocalNodeManager {
|
|||||||
for node in descriptors.nodes() {
|
for node in descriptors.nodes() {
|
||||||
let label = Self::default_label(node.index());
|
let label = Self::default_label(node.index());
|
||||||
let config = create_node_config(node.general.clone());
|
let config = create_node_config(node.general.clone());
|
||||||
let spawned = Node::spawn(config, &label).await?;
|
let spawned = Node::spawn(config, &label, node.persist_dir.clone()).await?;
|
||||||
nodes.push(spawned);
|
nodes.push(spawned);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -335,7 +335,7 @@ impl LocalNodeManager {
|
|||||||
)?;
|
)?;
|
||||||
|
|
||||||
let api_client = self
|
let api_client = self
|
||||||
.spawn_and_register_node(&node_name, network_port, config)
|
.spawn_and_register_node(&node_name, network_port, config, options.persist_dir)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
Ok(StartedNode {
|
Ok(StartedNode {
|
||||||
@ -428,8 +428,9 @@ impl LocalNodeManager {
|
|||||||
node_name: &str,
|
node_name: &str,
|
||||||
network_port: u16,
|
network_port: u16,
|
||||||
config: RunConfig,
|
config: RunConfig,
|
||||||
|
persist_dir: Option<std::path::PathBuf>,
|
||||||
) -> Result<ApiClient, LocalNodeManagerError> {
|
) -> Result<ApiClient, LocalNodeManagerError> {
|
||||||
let node = Node::spawn(config, node_name)
|
let node = Node::spawn(config, node_name, persist_dir)
|
||||||
.await
|
.await
|
||||||
.map_err(|source| LocalNodeManagerError::Spawn { source })?;
|
.map_err(|source| LocalNodeManagerError::Spawn { source })?;
|
||||||
let client = node.api().clone();
|
let client = node.api().clone();
|
||||||
|
|||||||
5
testing-framework/env/src/lib.rs
vendored
5
testing-framework/env/src/lib.rs
vendored
@ -52,11 +52,6 @@ pub fn nomos_testnet_image_pull_policy() -> Option<String> {
|
|||||||
env::var("LOGOS_BLOCKCHAIN_TESTNET_IMAGE_PULL_POLICY").ok()
|
env::var("LOGOS_BLOCKCHAIN_TESTNET_IMAGE_PULL_POLICY").ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use]
|
|
||||||
pub fn pol_proof_dev_mode() -> Option<String> {
|
|
||||||
env::var("POL_PROOF_DEV_MODE").ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn rust_log() -> Option<String> {
|
pub fn rust_log() -> Option<String> {
|
||||||
env::var("RUST_LOG").ok()
|
env::var("RUST_LOG").ok()
|
||||||
|
|||||||
@ -25,7 +25,7 @@ impl Default for ConsensusLiveness {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const LAG_ALLOWANCE: u64 = 2;
|
const LAG_ALLOWANCE: u64 = 2;
|
||||||
const MIN_PROGRESS_BLOCKS: u64 = 5;
|
const MIN_PROGRESS_BLOCKS: u64 = 3;
|
||||||
const REQUEST_RETRIES: usize = 15;
|
const REQUEST_RETRIES: usize = 15;
|
||||||
const REQUEST_RETRY_DELAY: Duration = Duration::from_secs(2);
|
const REQUEST_RETRY_DELAY: Duration = Duration::from_secs(2);
|
||||||
const MAX_LAG_ALLOWANCE: u64 = 5;
|
const MAX_LAG_ALLOWANCE: u64 = 5;
|
||||||
@ -67,7 +67,7 @@ enum ConsensusLivenessIssue {
|
|||||||
|
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
enum ConsensusLivenessError {
|
enum ConsensusLivenessError {
|
||||||
#[error("consensus liveness requires at least one node")]
|
#[error("consensus liveness requires at least one validator")]
|
||||||
MissingParticipants,
|
MissingParticipants,
|
||||||
#[error("consensus liveness violated (target={target}):\n{details}")]
|
#[error("consensus liveness violated (target={target}):\n{details}")]
|
||||||
Violations {
|
Violations {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user