From e2df6ed46542e151b5c713b8d42881e4691bbf18 Mon Sep 17 00:00:00 2001 From: andrussal Date: Mon, 20 Jul 2026 12:25:00 +0200 Subject: [PATCH] docs(book): rewrite testing framework guide --- README.md | 287 ++++--- book/COMPREHENSIVE_REPO_SYNC_REVIEW.md | 70 -- book/README.md | 531 +----------- book/book.toml | 7 +- book/src/SUMMARY.md | 91 +- book/src/annotated-tree.md | 101 --- book/src/api-levels.md | 119 --- book/src/app-backend-scope.md | 70 ++ book/src/app-deployment.md | 145 ++++ book/src/app-host.md | 125 +++ book/src/application-model.md | 139 ++++ book/src/architecture-overview.md | 256 ------ book/src/authoring-scenarios.md | 383 --------- book/src/best-practices.md | 237 ------ book/src/binary-providers.md | 123 +++ book/src/boundaries.md | 87 ++ book/src/capabilities.md | 117 +++ book/src/capability-matrix.md | 46 + book/src/cfgsync.md | 92 ++ book/src/chaos.md | 173 +++- book/src/ci-integration.md | 360 -------- book/src/ci.md | 77 ++ book/src/cluster-provisioning.md | 121 +++ book/src/composing-stacks.md | 154 ++++ book/src/crate-map.md | 107 +++ book/src/custom-workload-example.md | 134 --- book/src/deployer-compose.md | 79 ++ book/src/deployer-k8s.md | 97 +++ book/src/deployer-local.md | 109 +++ book/src/deployment-policies.md | 90 ++ book/src/design-rationale.md | 7 - book/src/diagnostics.md | 76 ++ book/src/dsl-cheat-sheet.md | 188 ----- book/src/entry-patterns.md | 95 +++ book/src/environment-variables.md | 411 ++------- book/src/examples-advanced.md | 328 -------- book/src/examples.md | 118 --- book/src/expectations.md | 171 ++++ book/src/extending.md | 360 -------- book/src/extension-points.md | 202 +++++ book/src/external-clusters.md | 122 +++ book/src/faq.md | 32 - book/src/framework-in-brief.md | 971 ++++++++++++++++++++++ book/src/framework-map.svg | 313 +++++++ book/src/glossary.md | 85 +- book/src/handles-teardown.md | 115 +++ book/src/implementing-application.md | 155 ++++ book/src/internal-crate-reference.md | 174 ---- book/src/introduction.md | 156 +++- book/src/local-app-cluster.md | 122 +++ book/src/local-process-app.md | 128 +++ book/src/logging-observability.md | 356 -------- book/src/manual-cluster.md | 471 +++-------- book/src/node-config.md | 126 +++ book/src/node-control.md | 391 --------- book/src/observation.md | 154 ++++ book/src/operations-overview.md | 79 -- book/src/part-i.md | 12 +- book/src/part-ii.md | 15 +- book/src/part-iii.md | 16 +- book/src/part-iv.md | 51 +- book/src/part-v.md | 36 +- book/src/part-vi.md | 7 + book/src/part-vii.md | 10 + book/src/persistence.md | 111 +++ book/src/prerequisites.md | 244 ------ book/src/project-context-primer.md | 156 ---- book/src/quickstart.md | 298 ++----- book/src/runners.md | 147 ---- book/src/running-examples.md | 329 ++------ book/src/running-scenarios.md | 115 --- book/src/runtime-extensions.md | 129 +++ book/src/scenario-builder-ext-patterns.md | 19 - book/src/scenario-lifecycle.md | 133 --- book/src/scenario-model.md | 129 ++- book/src/seeds.md | 94 +++ book/src/telemetry.md | 104 +++ book/src/testing-philosophy.md | 179 ---- book/src/tf-boundaries.md | 92 ++ book/src/topology-chaos.md | 36 - book/src/topology.md | 112 +++ book/src/troubleshooting.md | 731 +--------------- book/src/usage-patterns.md | 16 - book/src/verb-layer.md | 110 +++ book/src/what-you-will-learn.md | 63 -- book/src/workloads.md | 504 +++-------- book/src/workspace-layout.md | 21 - book/theme/tour-v2.css | 822 ++++++++++++++++++ book/theme/tour-v2.js | 409 +++++++++ docs/book-maintenance.md | 135 +++ 90 files changed, 7802 insertions(+), 8016 deletions(-) delete mode 100644 book/COMPREHENSIVE_REPO_SYNC_REVIEW.md delete mode 100644 book/src/annotated-tree.md delete mode 100644 book/src/api-levels.md create mode 100644 book/src/app-backend-scope.md create mode 100644 book/src/app-deployment.md create mode 100644 book/src/app-host.md create mode 100644 book/src/application-model.md delete mode 100644 book/src/architecture-overview.md delete mode 100644 book/src/authoring-scenarios.md delete mode 100644 book/src/best-practices.md create mode 100644 book/src/binary-providers.md create mode 100644 book/src/boundaries.md create mode 100644 book/src/capabilities.md create mode 100644 book/src/capability-matrix.md create mode 100644 book/src/cfgsync.md delete mode 100644 book/src/ci-integration.md create mode 100644 book/src/ci.md create mode 100644 book/src/cluster-provisioning.md create mode 100644 book/src/composing-stacks.md create mode 100644 book/src/crate-map.md delete mode 100644 book/src/custom-workload-example.md create mode 100644 book/src/deployer-compose.md create mode 100644 book/src/deployer-k8s.md create mode 100644 book/src/deployer-local.md create mode 100644 book/src/deployment-policies.md delete mode 100644 book/src/design-rationale.md create mode 100644 book/src/diagnostics.md delete mode 100644 book/src/dsl-cheat-sheet.md create mode 100644 book/src/entry-patterns.md delete mode 100644 book/src/examples-advanced.md delete mode 100644 book/src/examples.md create mode 100644 book/src/expectations.md delete mode 100644 book/src/extending.md create mode 100644 book/src/extension-points.md create mode 100644 book/src/external-clusters.md delete mode 100644 book/src/faq.md create mode 100644 book/src/framework-in-brief.md create mode 100644 book/src/framework-map.svg create mode 100644 book/src/handles-teardown.md create mode 100644 book/src/implementing-application.md delete mode 100644 book/src/internal-crate-reference.md create mode 100644 book/src/local-app-cluster.md create mode 100644 book/src/local-process-app.md delete mode 100644 book/src/logging-observability.md create mode 100644 book/src/node-config.md delete mode 100644 book/src/node-control.md create mode 100644 book/src/observation.md delete mode 100644 book/src/operations-overview.md create mode 100644 book/src/part-vi.md create mode 100644 book/src/part-vii.md create mode 100644 book/src/persistence.md delete mode 100644 book/src/prerequisites.md delete mode 100644 book/src/project-context-primer.md delete mode 100644 book/src/runners.md delete mode 100644 book/src/running-scenarios.md create mode 100644 book/src/runtime-extensions.md delete mode 100644 book/src/scenario-builder-ext-patterns.md delete mode 100644 book/src/scenario-lifecycle.md create mode 100644 book/src/seeds.md create mode 100644 book/src/telemetry.md delete mode 100644 book/src/testing-philosophy.md create mode 100644 book/src/tf-boundaries.md delete mode 100644 book/src/topology-chaos.md create mode 100644 book/src/topology.md delete mode 100644 book/src/usage-patterns.md create mode 100644 book/src/verb-layer.md delete mode 100644 book/src/what-you-will-learn.md delete mode 100644 book/src/workspace-layout.md create mode 100644 book/theme/tour-v2.css create mode 100644 book/theme/tour-v2.js create mode 100644 docs/book-maintenance.md diff --git a/README.md b/README.md index a3aea03..7ab6e57 100644 --- a/README.md +++ b/README.md @@ -1,155 +1,184 @@ -# Logos Blockchain Testing Framework +# Logos Testing Framework -A comprehensive testing framework for the Logos blockchain implementation, providing declarative scenario definitions, multiple deployment backends, and production-grade observability. +A Rust framework for system-level tests of networked applications. It can start +local processes and node clusters, deploy uniform clusters with Docker Compose +or Kubernetes, connect to existing deployments, run test activity, evaluate +outcomes, and clean up the resources it started. -## Overview +The framework is application-agnostic. Application repositories provide their +node configuration, clients, readiness checks, and backend-specific launch +settings. -This framework enables you to define, deploy, and execute integration tests for Logos blockchain scenarios across different environments—from local processes to containerized Kubernetes deployments—using a unified API. +## Start Here -**Key capabilities:** -- **Declarative scenario model** — Define topology, workloads, and success criteria using a fluent builder API -- **Multiple deployment backends** — Local processes, Docker Compose, or Kubernetes -- **Built-in workloads** — Transaction injection, DA (Data Availability) traffic, and restart-based chaos (requires node control; compose runner supported) -- **Observability-first** — Integrated Prometheus metrics, structured logging, and OpenTelemetry support -- **Production-ready** — Used in CI/CD pipelines with reproducible containerized environments - -## Quick Start - -### Prerequisites - -- Rust toolchain (nightly) -- `versions.env` file at repository root (included) -- For Docker Compose: Docker daemon -- For Kubernetes: Cluster access, `kubectl`, and `helm` - -### Run Your First Test +The workspace pins its Rust toolchain in `rust-toolchain.toml`. The local +kvstore example needs no additional setup; its node binary is built on first +use: ```bash -# Host mode (local processes) - fastest iteration -scripts/run/run-examples.sh -t 60 -v 1 -e 1 host - -# Compose mode (Docker containers) - reproducible environment -scripts/run/run-examples.sh -t 60 -v 1 -e 1 compose - -# K8s mode (Kubernetes cluster) - production-like fidelity -scripts/run/run-examples.sh -t 60 -v 1 -e 1 k8s +cargo run -p kvstore-examples --bin kvstore_app_host_convergence ``` -The script handles circuit setup, binary building, image preparation, and scenario execution automatically. +This starts three local node processes, writes data, restarts a node, checks +the result, and removes the processes and temporary directories. + +The composed-application example runs as integration tests: + +```bash +cargo test -p multi-app-e2e +``` + +It covers a queue cluster, worker process, and result-store cluster through a +runner-driven scenario and direct imperative tests. + +For Compose examples, run a Docker daemon and prepare the image named by the +example; the Compose deployer checks that it exists locally but does not build +or pull it. Kubernetes examples require a reachable cluster, `kubectl`, Helm, +and a node image available to that cluster. + +See [Quickstart](book/src/quickstart.md) and +[Running the Examples](book/src/running-examples.md) for the complete commands +and requirements. + +## Ways to Write Tests + +### Scenarios + +A scenario records the system to deploy, workloads to run, expectations to +evaluate, runtime limits, and enabled capabilities. The runner performs +deployment, readiness checks, concurrent workloads, cooldown, expectation +evaluation, and cleanup. + +```rust +let mut scenario = AppHost::scenario() + .with_app(KvLocalApp::nodes(3)) + .with_workload(KvAppHostConvergence::new(3)) + .with_run_duration(Duration::from_secs(5)) + .build()?; + +let runner = AppHostLocalDeployer::default() + .deploy(&scenario) + .await?; + +runner.run(&mut scenario).await?; +``` + +### Imperative Tests + +`ManualCluster` gives ordinary Rust or a BDD harness direct control of one +uniform cluster. Tests can start, stop, restart, and wait for nodes without +using workloads or expectations. + +A composed application can also be deployed directly through `DeployContext` +when test code needs to control the complete stack step by step. + +### Composed Applications + +`AppDeployment` describes how application components are started and connected. +`AppHost` runs one root deployment as part of a scenario. Child deployments can +start uniform clusters through `LocalAppCluster` and standalone binaries through +`LocalProcessApp`, then expose typed handles to workloads and expectations. + +App composition currently runs only with the local process deployer. Compose +and Kubernetes support uniform application clusters, not an `AppDeployment` +tree containing several application types. + +### Existing Deployments + +Scenarios can use managed nodes, attach to an existing Compose project or +Kubernetes deployment, or construct clients for external endpoints. Available +node control depends on the selected source and backend. + +## Deployment Backends + +Uniform scenarios use the same scenario runtime on all three backends. Each +application supplies a thin backend adapter containing details such as the +binary or image, config location, and service ports. + +| Capability | Local | Compose | Kubernetes | +|---|---|---|---| +| Uniform managed scenarios | Yes | Yes | Yes | +| Managed node control | Start, stop, restart | Restart | Use Kubernetes `ManualCluster` | +| Existing clusters | No | Compose project or services | Label selector and namespace | +| External endpoints | Yes | Yes | Yes | +| `AppHost` composition | Yes | No | No | +| Config delivery | Files in node working directories | cfgsync | cfgsync | + +The local deployer resolves executable paths through path, environment, build, +or download providers. Compose generates a project and services. Kubernetes +installs a Helm release in a per-run namespace. Container backends deliver +generated per-node configuration and other static files through cfgsync. + +See the [Capability Matrix](book/src/capability-matrix.md), +[Local Deployer](book/src/deployer-local.md), +[Compose Deployer](book/src/deployer-compose.md), and +[Kubernetes Deployer](book/src/deployer-k8s.md). + +## Repository Layout + +```text +testing-framework/ +├── core/ scenario runtime, topology, provisioning, control +├── app/ AppHost, AppDeployment, typed handles, composition +└── deployers/ + ├── local/ local processes and binary providers + ├── compose/ generated Docker Compose projects + └── k8s/ Helm and Kubernetes deployment + +cfgsync/ +├── artifacts/ backend-neutral per-node files +├── core/ protocol, server, client, and rendering +├── adapter/ application config materialization +└── runtime/ cfgsync server and client binaries + +examples/ self-contained example applications and tests +book/ mdBook source and presentation theme +scripts/ checks, cleanup, and observability helpers +``` + +The example applications include uniform clusters, composed stacks, consensus +failover, queues, WebSocket pub/sub, metrics, and unmodified NATS and Redis +servers. See [examples/README.md](examples/README.md) for the recommended entry +points. ## Documentation -**Complete documentation available at:** https://logos-blockchain.github.io/logos-blockchain-testing/ +- [The Framework in Brief](book/src/framework-in-brief.md) +- [Quickstart](book/src/quickstart.md) +- [Application and Environment Model](book/src/application-model.md) +- [Composing Applications](book/src/part-ii.md) +- [Scenario Runtime](book/src/part-iii.md) +- [Uniform Clusters and Configuration](book/src/part-iv.md) +- [Deployers and Sources](book/src/part-v.md) +- [Environment Variables](book/src/environment-variables.md) +- [Troubleshooting](book/src/troubleshooting.md) -### Essential Guides +Published book: -| Topic | Link | -|-------|------| -| **Getting Started** | [Quickstart Guide](https://logos-blockchain.github.io/logos-blockchain-testing/quickstart.html) | -| **Core Concepts** | [Testing Philosophy](https://logos-blockchain.github.io/logos-blockchain-testing/testing-philosophy.html) | -| **Examples** | [Basic](https://logos-blockchain.github.io/logos-blockchain-testing/examples.html) \| [Advanced](https://logos-blockchain.github.io/logos-blockchain-testing/examples-advanced.html) | -| **Deployment Options** | [Runners Overview](https://logos-blockchain.github.io/logos-blockchain-testing/runners.html) | -| **API Reference** | [Builder API](https://logos-blockchain.github.io/logos-blockchain-testing/dsl-cheat-sheet.html) | -| **Operations** | [Setup & Configuration](https://logos-blockchain.github.io/logos-blockchain-testing/operations.html) | -| **Troubleshooting** | [Common Issues](https://logos-blockchain.github.io/logos-blockchain-testing/troubleshooting.html) | +Build and test it locally with: -## Repository Structure - -``` -logos-blockchain-testing/ -├── testing-framework/ # Core library crates -│ ├── core/ # Scenario model, runtime orchestration -│ ├── workflows/ # Workloads (tx, DA, chaos) and expectations -│ ├── configs/ # Node configuration builders -│ ├── runners/ # Deployment backends (local, compose, k8s) -│ └── assets/stack/ # Docker/K8s deployment assets -├── examples/ # Runnable demo binaries -│ └── src/bin/ # local_runner, compose_runner, k8s_runner -├── scripts/ # Helper utilities (run-examples.sh, build-bundle.sh) -└── book/ # Documentation sources (mdBook) +```bash +mdbook build book +mdbook test book ``` -## Architecture - -The framework follows a clear separation of concerns: - -**Scenario Definition** → **Topology Builder** → **Deployer** → **Runner** → **Workloads** → **Expectations** - -- **Scenario**: Declarative description of test intent (topology + workloads + success criteria) -- **Deployer**: Provisions infrastructure on chosen backend (host/compose/k8s) -- **Runner**: Orchestrates execution, manages lifecycle, collects observability -- **Workloads**: Generate traffic and conditions (transactions, DA blobs, chaos) -- **Expectations**: Evaluate success/failure based on observed behavior +Install `mdbook` first if it is not already available. ## Development -### Building the Documentation +Useful focused checks from the workspace root: ```bash -# Install mdBook -cargo install mdbook mdbook-mermaid - -# Build and serve locally -cd book && mdbook serve -# Open http://localhost:3000 +cargo fmt --all -- --check +cargo test -p testing-framework-core +cargo test -p testing-framework-app +cargo test -p multi-app-e2e +cargo clippy --all --all-targets --all-features -- -D warnings ``` -### Running Tests - -```bash -# Run framework unit tests -cargo test - -# Run integration examples -scripts/run/run-examples.sh -t 60 -v 2 -e 1 host -``` - -### Creating Prebuilt Bundles - -For compose/k8s deployments, you can create prebuilt bundles to speed up image builds: - -```bash -# Build Linux bundle (required for compose/k8s) -scripts/build/build-bundle.sh --platform linux - -# Use the bundle when building images -export LOGOS_BLOCKCHAIN_BINARIES_TAR=.tmp/nomos-binaries-linux-v0.3.1.tar.gz -scripts/build/build_test_image.sh -``` - -## Environment Variables - -Key environment variables for customization: - -| Variable | Purpose | Default | -|----------|---------|---------| -| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | Docker image tag for compose/k8s | `logos-blockchain-testing:local` | -| `LOGOS_BLOCKCHAIN_DEMO_NODES` | Number of nodes | Varies by example | -| `LOGOS_BLOCKCHAIN_LOG_DIR` | Directory for persistent log files | (temporary) | -| `LOGOS_BLOCKCHAIN_LOG_LEVEL` | Logging verbosity | `info` | - -See [Operations Guide](https://logos-blockchain.github.io/logos-blockchain-testing/operations.html) for complete configuration reference. - -## CI/CD Integration - -The framework is designed for CI/CD pipelines: - -- **Host runner**: Fast smoke tests with minimal overhead -- **Compose runner**: Reproducible containerized environment with Prometheus -- **K8s runner**: Production-like cluster validation - -Example CI workflow: `.github/workflows/lint.yml` (see `compose_smoke` job) +The lint workflow also checks dependency policy with `cargo-deny`, unused +dependencies with `cargo-machete`, and TOML formatting with Taplo. ## License -This project is part of the Logos blockchain implementation. - -## Links - -- **Documentation**: https://logos-blockchain.github.io/logos-blockchain-testing/ -- **Logos Project**: https://github.com/logos-co -- **Logos Node (repo: logos-blockchain-node)**: https://github.com/logos-co/logos-blockchain-node - -## Support - -For issues, questions, or contributions, please refer to the [Troubleshooting Guide](https://logos-blockchain.github.io/logos-blockchain-testing/troubleshooting.html) or file an issue in this repository. +MIT OR Apache-2.0. diff --git a/book/COMPREHENSIVE_REPO_SYNC_REVIEW.md b/book/COMPREHENSIVE_REPO_SYNC_REVIEW.md deleted file mode 100644 index e90b979..0000000 --- a/book/COMPREHENSIVE_REPO_SYNC_REVIEW.md +++ /dev/null @@ -1,70 +0,0 @@ -# Book → Repo Sync Review (2025-12-20) - -Reviewed against `git rev-parse HEAD` at the time of writing, plus local working tree changes. - -## Checks Run - -- `mdbook build book` -- `mdbook test book` -- `cargo build -p doc-snippets` -- Verified `book/src/SUMMARY.md` covers all pages in `book/src/` (no orphaned pages) -- Verified all `scripts/...` paths referenced from the book exist -- Compared `NOMOS_*` environment variables used in `scripts/`, `testing-framework/`, and `examples/` vs. `book/src/environment-variables.md` - -## Findings / Fixes Applied - -- `book/src/environment-variables.md` was not a complete reference: it missed multiple `NOMOS_*` variables used by the repo (scripts + framework). Added the missing variables and corrected a misleading note about `RUST_LOG` vs node logging. -- `book/src/running-examples.md` “Quick Smoke Matrix” section didn’t reflect current `scripts/run/run-test-matrix.sh` flags. Added the commonly used options and clarified the relationship to `LOGOS_BLOCKCHAIN_SKIP_IMAGE_BUILD`. -- `book/src/part-iv.md` existed but was not in `book/src/SUMMARY.md`. Removed it so the rendered book doesn’t silently diverge from the filesystem. -- `mdbook test book` was failing because: - - Many Rust examples were written as ` ```rust` (doctested by default) but depend on workspace crates; they aren’t standalone doctest snippets. - - Several unlabeled code blocks (e.g. tree/log output) were treated as Rust by rustdoc. - - Updated code fences to ` ```rust,ignore` for non-standalone Rust examples and to ` ```text` for non-Rust output blocks so `mdbook test book` succeeds. - -## Pages Reviewed (No Skips) - -All pages under `book/src/` currently included by `book/src/SUMMARY.md`: - -- `annotated-tree.md` -- `api-levels.md` -- `architecture-overview.md` -- `authoring-scenarios.md` -- `best-practices.md` -- `chaos.md` -- `ci-integration.md` -- `custom-workload-example.md` -- `design-rationale.md` -- `dsl-cheat-sheet.md` -- `environment-variables.md` -- `examples-advanced.md` -- `examples.md` -- `extending.md` -- `faq.md` -- `glossary.md` -- `internal-crate-reference.md` -- `introduction.md` -- `logging-observability.md` -- `node-control.md` -- `operations-overview.md` -- `part-i.md` -- `part-ii.md` -- `part-iii.md` -- `part-v.md` -- `part-vi.md` -- `prerequisites.md` -- `project-context-primer.md` -- `quickstart.md` -- `runners.md` -- `running-examples.md` -- `running-scenarios.md` -- `scenario-builder-ext-patterns.md` -- `scenario-lifecycle.md` -- `scenario-model.md` -- `testing-philosophy.md` -- `topology-chaos.md` -- `troubleshooting.md` -- `usage-patterns.md` -- `what-you-will-learn.md` -- `workloads.md` -- `workspace-layout.md` - diff --git a/book/README.md b/book/README.md index ca17ccc..33c872b 100644 --- a/book/README.md +++ b/book/README.md @@ -1,533 +1,18 @@ -# Documentation Maintenance Guide +# Testing Framework Book -This guide helps maintainers keep the book synchronized with code changes. Use these checklists when modifying the framework. +The book is an mdBook rooted at `book/src/SUMMARY.md`. Its current editing conventions and verification checklist live in [`../docs/book-maintenance.md`](../docs/book-maintenance.md). -**Key Tool:** The `examples/doc-snippets/` crate contains compilable versions of code examples from the book. Always run `cargo build -p doc-snippets` after API changes to catch broken examples early. - -## Quick Reference: What to Update When - -| Change Type | Pages to Check | Estimated Time | -|-------------|----------------|----------------| -| API method renamed/changed | [API Changes](#api-changes) | 1-2 hours | -| New workload/expectation added | [New Features](#new-features) | 30 minutes | -| Environment variable added/changed | [Environment Variables](#environment-variables) | 15 minutes | -| Script path/interface changed | [Scripts & Tools](#scripts--tools) | 30 minutes | -| New runner/deployer added | [New Runner](#new-runner) | 2-3 hours | -| Trait signature changed | [Trait Changes](#trait-changes) | 1-2 hours | - ---- - -## Detailed Checklists - -### API Changes - -**When:** Builder API methods, trait methods, or core types change - -**Examples:** -- Rename: `.transactions_with()` → `.with_transactions()` -- New method: `.with_timeout()` -- Parameter change: `.validators(3)` → `.validators(count, config)` - -**Update these pages:** +Build and test it from the repository root: ```bash -# 1. Search for affected API usage -rg "old_method_name" book/src/ - -# 2. Update these files: -- [ ] src/dsl-cheat-sheet.md # Builder API reference (highest priority) -- [ ] src/quickstart.md # First example users see -- [ ] src/examples.md # 4 complete scenarios -- [ ] src/examples-advanced.md # 3 advanced scenarios -- [ ] src/introduction.md # "A Scenario in 20 Lines" example -- [ ] src/project-context-primer.md # Quick example section -- [ ] src/authoring-scenarios.md # Scenario patterns -- [ ] src/best-practices.md # Code organization examples -- [ ] src/custom-workload-example.md # Complete implementation -- [ ] src/extending.md # Trait implementation examples +mdbook build book +mdbook test book ``` -**Verification:** -```bash -# Compile doc-snippets to catch API breakage -cargo build -p doc-snippets - -# Check if book links are valid -mdbook test -``` - ---- - -### New Features - -#### New Workload or Expectation - -**When:** Adding a new traffic generator or success criterion - -**Examples:** -- New workload: `MemoryPressureWorkload` -- New expectation: `ExpectZeroDroppedTransactions` - -**Update these pages:** +Preview it locally with: ```bash -- [ ] src/workloads.md # Add to built-in workloads section -- [ ] src/dsl-cheat-sheet.md # Add DSL helper if provided -- [ ] src/examples-advanced.md # Consider adding example usage -- [ ] src/glossary.md # Add term definition -- [ ] src/internal-crate-reference.md # Document crate location +mdbook serve book ``` -**Optional (if significant feature):** -```bash -- [ ] src/what-you-will-learn.md # Add to learning outcomes -- [ ] src/best-practices.md # Add usage guidance -``` - -#### New Runner/Deployer - -**When:** Adding support for a new deployment target (e.g., AWS ECS) - -**Update these pages:** - -```bash -# Core documentation -- [ ] src/runners.md # Add to comparison table and decision guide -- [ ] src/operations-overview.md # Update runner-agnostic matrix -- [ ] src/architecture-overview.md # Update deployer list and diagram -- [ ] src/running-examples.md # Add runner-specific section - -# Reference pages -- [ ] src/dsl-cheat-sheet.md # Add deployer import/usage -- [ ] src/internal-crate-reference.md # Document new crate -- [ ] src/glossary.md # Add runner type definition - -# Potentially affected -- [ ] src/ci-integration.md # Add CI example if applicable -- [ ] src/troubleshooting.md # Add common issues -- [ ] src/faq.md # Add FAQ entries -``` - -#### New Topology Helper - -**When:** Adding topology generation helpers (e.g., `.network_mesh()`) - -**Update these pages:** - -```bash -- [ ] src/dsl-cheat-sheet.md # Add to topology section -- [ ] src/authoring-scenarios.md # Add usage pattern -- [ ] src/topology-chaos.md # Add topology description -- [ ] src/examples.md # Consider adding example -``` - ---- - -### Trait Changes - -**When:** Core trait signatures change (breaking changes) - -**Examples:** -- `Workload::init()` adds new parameter -- `Expectation::evaluate()` changes return type -- `Deployer::deploy()` signature update - -**Update these pages:** - -```bash -# Critical - these show full trait definitions -- [ ] src/extending.md # Complete trait outlines (6+ examples) -- [ ] src/custom-workload-example.md # Full implementation example -- [ ] src/scenario-model.md # Core model documentation - -# Important - these reference traits -- [ ] src/api-levels.md # Trait usage patterns -- [ ] src/architecture-overview.md # Extension points diagram -- [ ] src/internal-crate-reference.md # Trait locations -``` - -**Verification:** -```bash -# Ensure trait examples would compile -cargo doc --no-deps --document-private-items -``` - ---- - -### Environment Variables - -**When:** New environment variable added, changed, or removed - -**Examples:** -- New: `LOGOS_BLOCKCHAIN_NEW_FEATURE_ENABLED` -- Changed: `LOGOS_BLOCKCHAIN_LOG_LEVEL` accepts new values -- Deprecated: `OLD_FEATURE_FLAG` - -**Update these pages:** - -```bash -# Primary location (single source of truth) -- [ ] src/environment-variables.md # Add to appropriate category table - -# Secondary mentions -- [ ] src/prerequisites.md # If affects setup -- [ ] src/running-examples.md # If affects runner usage -- [ ] src/troubleshooting.md # If commonly misconfigured -- [ ] src/glossary.md # If significant/commonly referenced -``` - -**Environment Variables Table Location:** -``` -src/environment-variables.md - ├─ Runner Configuration - ├─ Node Binary & Paths - ├─ Circuit Assets - ├─ Logging & Tracing - ├─ Observability & Metrics - ├─ Proof System - ├─ Docker & Images - ├─ Testing Behavior - └─ CI/CD -``` - ---- - -### Scripts & Tools - -**When:** Helper scripts move, rename, or change interface - -**Examples:** -- Script moved: `scripts/run-examples.sh` → `scripts/run/run-examples.sh` -- New script: `scripts/clean-all.sh` -- Interface change: `run-examples.sh` adds new required flag - -**Update these pages:** - -```bash -# High impact -- [ ] src/quickstart.md # Uses run-examples.sh prominently -- [ ] src/running-examples.md # Documents all scripts -- [ ] src/prerequisites.md # References setup scripts -- [ ] src/examples.md # Script recommendations -- [ ] src/examples-advanced.md # Script recommendations - -# Moderate impact -- [ ] src/ci-integration.md # May reference scripts in workflows -- [ ] src/troubleshooting.md # Cleanup scripts -- [ ] src/architecture-overview.md # Asset preparation scripts -``` - -**Find all script references:** -```bash -rg "scripts/" book/src/ --no-heading -``` - ---- - -### Operational Changes - -#### Docker Image Changes - -**When:** Image build process, tag names, or embedded assets change - -**Update these pages:** - -```bash -- [ ] src/prerequisites.md # Image build instructions -- [ ] src/runners.md # Compose/K8s prerequisites -- [ ] src/environment-variables.md # LOGOS_BLOCKCHAIN_TESTNET_IMAGE, LOGOS_BLOCKCHAIN_BINARIES_TAR -- [ ] src/architecture-overview.md # Assets and Images section -``` - -#### Observability Stack Changes - -**When:** Prometheus, Grafana, OTLP, or metrics configuration changes - -**Update these pages:** - -```bash -- [ ] src/logging-observability.md # Primary documentation -- [ ] src/environment-variables.md # LOGOS_BLOCKCHAIN_METRICS_*, LOGOS_BLOCKCHAIN_OTLP_* -- [ ] src/architecture-overview.md # Observability section -- [ ] src/runners.md # Runner observability support -``` - -#### CI/CD Changes - -**When:** CI workflow changes, new actions, or integration patterns - -**Update these pages:** - -```bash -- [ ] src/ci-integration.md # Complete workflow examples -- [ ] src/best-practices.md # CI recommendations -- [ ] src/operations-overview.md # CI mentioned in runner matrix -``` - ---- - -### Node Protocol Changes - -**When:** Changes to Logos blockchain protocol or node behavior - -**Examples:** -- New consensus parameter -- DA protocol change -- Network layer update - -**Update these pages:** - -```bash -# Context pages (high-level only) -- [ ] src/project-context-primer.md # Protocol overview -- [ ] src/glossary.md # Protocol terms -- [ ] src/faq.md # May need protocol updates - -# Usually NOT affected (framework is protocol-agnostic) -- Testing framework abstracts protocol details -- Only update if change affects testing methodology -``` - ---- - -### Crate Structure Changes - -**When:** Crate reorganization, renames, or new crates added - -**Examples:** -- New crate: `testing-framework-metrics` -- Crate rename: `runner-examples` → `examples` -- Module moved: `core::scenario` → `core::model` - -**Update these pages:** - -```bash -# Critical -- [ ] src/internal-crate-reference.md # Complete crate listing -- [ ] src/architecture-overview.md # Crate dependency diagram -- [ ] src/workspace-layout.md # Directory structure -- [ ] src/annotated-tree.md # File tree with annotations - -# Code examples (update imports) -- [ ] src/dsl-cheat-sheet.md # Import statements -- [ ] src/extending.md # use statements in examples -- [ ] src/custom-workload-example.md # Full imports -``` - -**Find all import statements:** -```bash -rg "^use testing_framework" book/src/ -``` - ---- - -## Testing Documentation Changes - -### Build the Book - -```bash -cd book -mdbook build - -# Output: ../target/book/ -``` - -### Test Documentation - -```bash -# Check for broken links -mdbook test - -# Preview locally -mdbook serve -# Open http://localhost:3000 -``` - -### Test Code Examples (Doc Snippets) - -**The `examples/doc-snippets/` crate contains compilable versions of code examples from the book.** - -This ensures examples stay synchronized with the actual API and don't break when code changes. - -**Why doc-snippets exist:** -- Code examples in the book (73 blocks across 18 files) can drift from reality -- Compilation failures catch API breakage immediately -- Single source of truth for code examples - -**Current coverage:** 40+ snippet files corresponding to examples in: -- `quickstart.md` (7 snippets) -- `examples.md` (4 scenarios) -- `examples-advanced.md` (3 scenarios) -- `dsl-cheat-sheet.md` (11 DSL examples) -- `custom-workload-example.md` (2 trait implementations) -- `internal-crate-reference.md` (6 extension examples) -- And more... - -**Testing snippets:** - -```bash -# Compile all doc snippets -cargo build -p doc-snippets - -# Run with full warnings -cargo build -p doc-snippets --all-features - -# Check during CI -cargo check -p doc-snippets -``` - -**When to update snippets:** - -1. **API method changed** → Update corresponding snippet file - ```bash - # Example: If .transactions_with() signature changes - # Update: examples/doc-snippets/src/examples_transaction_workload.rs - ``` - -2. **New code example added to book** → Create new snippet file - ```bash - # Example: Adding new topology pattern - # Create: examples/doc-snippets/src/topology_mesh_example.rs - ``` - -3. **Trait signature changed** → Update trait implementation snippets - ```bash - # Update: custom_workload_example_*.rs - # Update: internal_crate_reference_add_*.rs - ``` - -**Snippet naming convention:** -``` -book/src/examples.md → examples_*.rs -book/src/quickstart.md → quickstart_*.rs -book/src/dsl-cheat-sheet.md → dsl_cheat_sheet_*.rs -``` - -**Best practice:** -When updating code examples in markdown, update the corresponding snippet file first, verify it compiles, then copy to the book. This ensures examples are always valid. - -### Check for Common Issues - -```bash -# Find outdated API references -rg "old_deprecated_api" src/ - -# Find broken GitHub links -rg "github.com.*404" src/ - -# Find TODO/FIXME markers -rg "(TODO|FIXME|XXX)" src/ - -# Check for inconsistent terminology -rg "(Nomos node|nomos blockchain)" src/ # Should be "Logos node|Logos blockchain" -``` - ---- - -## Maintenance Schedule - -### On Every PR - -- [ ] Check if changes affect documented APIs -- [ ] Update relevant pages per checklist above -- [ ] Update corresponding doc-snippets if code examples changed -- [ ] Run `cargo build -p doc-snippets` to verify examples compile -- [ ] Build book to verify no broken links -- [ ] Verify code examples still make sense - -### Monthly - -- [ ] Review recent PRs for documentation impact -- [ ] Update environment variables table -- [ ] Check script references are current -- [ ] Verify GitHub source links are not 404 - -### Quarterly - -- [ ] Full audit of code examples against latest API -- [ ] Verify all doc-snippets still compile with latest dependencies -- [ ] Check for code examples in book that don't have corresponding snippets -- [ ] Review troubleshooting for new patterns -- [ ] Update FAQ with common questions -- [ ] Check all Mermaid diagrams render correctly - -### Major Release - -- [ ] Complete review of all technical content -- [ ] Verify all version-specific references -- [ ] Update "What You Will Learn" outcomes -- [ ] Add release notes for documentation changes - ---- - -## Content Organization - -### Stability Tiers (Change Frequency) - -**Stable (Rarely Change)** -- Part I — Foundations (philosophy, architecture, design rationale) -- Part VI — Appendix (glossary, FAQ, troubleshooting symptoms) -- Front matter (project context, introduction) - -**Semi-Stable (Occasional Changes)** -- Part II — User Guide (usage patterns, best practices, examples) -- Part V — Operations (prerequisites, CI, logging) - -**High Volatility (Frequent Changes)** -- API references (dsl-cheat-sheet.md, extending.md) -- Code examples (73 blocks across 18 files) -- Environment variables (50+ documented) -- Runner comparisons (features evolve) - -### Page Dependency Map - -**Core pages** (many other pages reference these): -- `dsl-cheat-sheet.md` ← Referenced by examples, quickstart, authoring -- `environment-variables.md` ← Referenced by operations, troubleshooting, runners -- `runners.md` ← Referenced by operations, quickstart, examples -- `glossary.md` ← Referenced throughout the book - -**When updating core pages, check for broken cross-references.** - ---- - -## Common Patterns - -### Adding a Code Example - -```markdown -# 1. Add the code block -```rust -use testing_framework_core::scenario::ScenarioBuilder; -// ... example code -``` - -# 2. Add context -**When to use:** [explain use case] - -# 3. Link to complete source (if applicable) -[View in source](https://github.com/logos-blockchain/logos-blockchain-testing/blob/master/examples/src/bin/example.rs) -``` - -### Adding a Cross-Reference - -```markdown -See [Environment Variables](environment-variables.md) for complete configuration reference. -``` - -### Adding a "When to Read" Callout - -```markdown -> **When should I read this?** [guidance on when this content is relevant] -``` - ---- - -## Contact & Questions - -When in doubt: -1. Check this README for guidance -2. Review recent similar changes in git history -3. Ask the team in technical documentation discussions - -**Remember:** Documentation quality directly impacts framework adoption and user success. Taking time to update docs properly is an investment in the project's future. +Generated output goes to `target/book/`. diff --git a/book/book.toml b/book/book.toml index bb2fe50..5444cbd 100644 --- a/book/book.toml +++ b/book/book.toml @@ -2,12 +2,13 @@ authors = ["Logos Testing"] language = "en" src = "src" -title = "Logos Blockchain Testing Framework Book" +title = "Testing Framework Book" [build] # Keep book output in target/ to avoid polluting the workspace root. build-dir = "../target/book" [output.html] -additional-js = ["theme/mermaid-init.js"] -default-theme = "light" +additional-css = ["theme/tour-v2.css"] +additional-js = ["theme/mermaid-init.js", "theme/tour-v2.js"] +default-theme = "light" diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index cc63a44..c42681b 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -1,44 +1,55 @@ # Summary -- [Project Context Primer](project-context-primer.md) -- [What You Will Learn](what-you-will-learn.md) + +- [Introduction](introduction.md) - [Quickstart](quickstart.md) -- [Part I — Foundations](part-i.md) - - [Introduction](introduction.md) - - [Architecture Overview](architecture-overview.md) - - [Testing Philosophy](testing-philosophy.md) - - [Scenario Lifecycle](scenario-lifecycle.md) - - [Design Rationale](design-rationale.md) -- [Part II — User Guide](part-ii.md) - - [Workspace Layout](workspace-layout.md) - - [Annotated Tree](annotated-tree.md) - - [Authoring Scenarios](authoring-scenarios.md) - - [Core Content: Workloads & Expectations](workloads.md) - - [Core Content: ScenarioBuilderExt Patterns](scenario-builder-ext-patterns.md) - - [Best Practices](best-practices.md) - - [Usage Patterns](usage-patterns.md) - - [Examples](examples.md) - - [Advanced & Artificial Examples](examples-advanced.md) - - [Running Scenarios](running-scenarios.md) - - [Runners](runners.md) - - [RunContext: BlockFeed & Node Control](node-control.md) - - [Chaos Workloads](chaos.md) - - [Topology & Chaos Patterns](topology-chaos.md) - - [Manual Clusters: Imperative Control](manual-cluster.md) -- [Part III — Developer Reference](part-iii.md) - - [Scenario Model (Developer Level)](scenario-model.md) - - [API Levels: Builder DSL vs. Direct](api-levels.md) - - [Extending the Framework](extending.md) - - [Example: New Workload & Expectation (Rust)](custom-workload-example.md) - - [Internal Crate Reference](internal-crate-reference.md) -- [Part IV — Operations & Deployment](part-iv.md) - - [Overview](operations-overview.md) - - [Prerequisites & Setup](prerequisites.md) - - [Running Examples](running-examples.md) - - [CI Integration](ci-integration.md) +- [The Framework in Brief](framework-in-brief.md) +- [Part I — Mental Model](part-i.md) + - [Application, AppDeployment, and Environments](application-model.md) + - [Scenario Model and Lifecycle](scenario-model.md) + - [Choosing an Entry Pattern](entry-patterns.md) + - [Ownership and Design Boundaries](boundaries.md) +- [Part II — Composing Applications](part-ii.md) + - [AppHost and with_app](app-host.md) + - [AppDeployment and DeployContext](app-deployment.md) + - [Handle Ownership and Teardown](handles-teardown.md) + - [One Binary: LocalProcessApp](local-process-app.md) + - [Uniform Child Clusters: LocalAppCluster](local-app-cluster.md) + - [Composing Heterogeneous Stacks](composing-stacks.md) + - [Backend Scope](app-backend-scope.md) +- [Part III — Scenario Runtime](part-iii.md) + - [Workloads and Concurrency](workloads.md) + - [Expectations and Evaluation](expectations.md) + - [The Verb Layer](verb-layer.md) + - [Scenario Capabilities](capabilities.md) + - [Chaos and Controlled Failure](chaos.md) + - [Runtime Extensions](runtime-extensions.md) + - [Continuous Observation](observation.md) + - [Telemetry and External Observability](telemetry.md) +- [Part IV — Uniform Clusters and Configuration](part-iv.md) + - [Implementing Application](implementing-application.md) + - [Topology and Deployment Plans](topology.md) + - [Ports, Peers, Node Config, and Readiness](node-config.md) + - [Static Artifacts and cfgsync](cfgsync.md) + - [Seeds and Reproducibility](seeds.md) + - [ManualCluster: Imperative Node Control](manual-cluster.md) + - [Persistence, Snapshots, and Recovery Testing](persistence.md) +- [Part V — Deployers and Sources](part-v.md) + - [Capability Matrix](capability-matrix.md) + - [Local Deployer](deployer-local.md) + - [Compose Deployer](deployer-compose.md) + - [Kubernetes Deployer](deployer-k8s.md) + - [Shared Cluster Provisioning](cluster-provisioning.md) + - [Existing and External Clusters](external-clusters.md) + - [Binary Providers](binary-providers.md) + - [Readiness, Retry, and Artifact Preservation](deployment-policies.md) +- [Part VI — Extending and Reference](part-vi.md) + - [Public Extension Points](extension-points.md) + - [Crate and API Map](crate-map.md) + - [Framework vs Application Boundaries](tf-boundaries.md) +- [Part VII — Operations](part-vii.md) + - [Running the Examples](running-examples.md) + - [Continuous Integration](ci.md) + - [Diagnostics and Retained Artifacts](diagnostics.md) - [Environment Variables](environment-variables.md) - - [Logging & Observability](logging-observability.md) -- [Part V — Appendix](part-v.md) - - [Builder API Quick Reference](dsl-cheat-sheet.md) - - [Troubleshooting Scenarios](troubleshooting.md) - - [FAQ](faq.md) + - [Troubleshooting](troubleshooting.md) - [Glossary](glossary.md) diff --git a/book/src/annotated-tree.md b/book/src/annotated-tree.md deleted file mode 100644 index d4d05e9..0000000 --- a/book/src/annotated-tree.md +++ /dev/null @@ -1,101 +0,0 @@ -# Annotated Tree - -Directory structure with key paths annotated: - -```text -logos-blockchain-testing/ -├─ testing-framework/ # Core library crates -│ ├─ configs/ # Node config builders, topology generation, tracing/logging config -│ ├─ core/ # Scenario model (ScenarioBuilder), runtime (Runner, Deployer), topology, node spawning -│ ├─ workflows/ # Workloads (transactions, chaos), expectations (liveness), builder DSL extensions -│ ├─ deployers/ # Deployment backends -│ │ ├─ local/ # LocalDeployer (spawns local processes) -│ │ ├─ compose/ # ComposeDeployer (Docker Compose + Prometheus) -│ │ └─ k8s/ # K8sDeployer (Kubernetes Helm) -│ └─ assets/ # Docker/K8s stack assets -│ └─ stack/ -│ ├─ monitoring/ # Prometheus config -│ ├─ scripts/ # Container entrypoints -│ └─ cfgsync.yaml # Config sync server template -│ -├─ examples/ # PRIMARY ENTRY POINT: runnable binaries -│ └─ src/bin/ -│ ├─ local_runner.rs # Host processes demo (LocalDeployer) -│ ├─ compose_runner.rs # Docker Compose demo (ComposeDeployer) -│ └─ k8s_runner.rs # Kubernetes demo (K8sDeployer) -│ -├─ scripts/ # Helper utilities -│ ├─ run-examples.sh # Convenience script (handles setup + runs examples) -│ ├─ build-bundle.sh # Build prebuilt binaries+circuits bundle -│ └─ setup-logos-blockchain-circuits.sh # Fetch circuit assets (Linux + host) -│ -└─ book/ # This documentation (mdBook) -``` - -## Key Directories Explained - -### `testing-framework/` -Core library crates providing the testing API. - -| Crate | Purpose | Key Exports | -|-------|---------|-------------| -| `configs` | Node configuration builders | Topology generation, tracing config | -| `core` | Scenario model & runtime | `ScenarioBuilder`, `Deployer`, `Runner` | -| `workflows` | Workloads & expectations | `ScenarioBuilderExt`, `ChaosBuilderExt` | -| `deployers/local` | Local process deployer | `LocalDeployer` | -| `deployers/compose` | Docker Compose deployer | `ComposeDeployer` | -| `deployers/k8s` | Kubernetes deployer | `K8sDeployer` | - -### `testing-framework/assets/stack/` -Docker/K8s deployment assets: -- **`monitoring/`**: Prometheus config -- **`scripts/`**: Container entrypoints - -### `scripts/` -Convenience utilities: -- **`run-examples.sh`**: All-in-one script for host/compose/k8s modes (recommended) -- **`build-bundle.sh`**: Create prebuilt binaries+circuits bundle for compose/k8s -- **`build_test_image.sh`**: Build the compose/k8s Docker image (bakes in assets) -- **`setup-logos-blockchain-circuits.sh`**: Fetch circuit assets for both Linux and host -- **`cfgsync.yaml`**: Configuration sync server template - -### `examples/` (Start Here!) -**Runnable binaries** demonstrating framework usage: -- `local_runner.rs` — Local processes -- `compose_runner.rs` — Docker Compose (requires `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` built) -- `k8s_runner.rs` — Kubernetes (requires cluster + image) - -**Run with:** `cargo run -p runner-examples --bin ` - -### `scripts/` -Helper utilities: -- **`setup-logos-blockchain-circuits.sh`**: Fetch circuit assets from releases - -## Observability - -**Compose runner** includes: -- **Prometheus** at `http://localhost:9090` (metrics scraping) -- Node metrics exposed per node -- Access in expectations: `ctx.telemetry().prometheus().map(|p| p.base_url())` - -**Logging** controlled by: -- `LOGOS_BLOCKCHAIN_LOG_DIR` — Write per-node log files -- `LOGOS_BLOCKCHAIN_LOG_LEVEL` — Global log level (error/warn/info/debug/trace) -- `LOGOS_BLOCKCHAIN_LOG_FILTER` — Target-specific filtering (e.g., `cryptarchia=trace`) -- `LOGOS_BLOCKCHAIN_TESTS_TRACING` — Enable file logging for local runner - -See [Logging & Observability](logging-observability.md) for details. - -## Navigation Guide - -| To Do This | Go Here | -|------------|---------| -| **Run an example** | `examples/src/bin/` → `cargo run -p runner-examples --bin ` | -| **Write a custom scenario** | `testing-framework/core/` → Implement using `ScenarioBuilder` | -| **Add a new workload** | `testing-framework/workflows/src/workloads/` → Implement `Workload` trait | -| **Add a new expectation** | `testing-framework/workflows/src/expectations/` → Implement `Expectation` trait | -| **Modify node configs** | `testing-framework/configs/src/topology/configs/` | -| **Extend builder DSL** | `testing-framework/workflows/src/builder/` → Add trait methods | -| **Add a new deployer** | `testing-framework/deployers/` → Implement `Deployer` trait | - -For detailed guidance, see [Internal Crate Reference](internal-crate-reference.md). diff --git a/book/src/api-levels.md b/book/src/api-levels.md deleted file mode 100644 index 6a14ef0..0000000 --- a/book/src/api-levels.md +++ /dev/null @@ -1,119 +0,0 @@ -# API Levels: Builder DSL vs. Direct Instantiation - -The framework supports two styles for constructing scenarios: - -1. **High-level Builder DSL** (recommended): fluent helper methods (e.g. `.transactions_with(...)`) -2. **Low-level direct instantiation**: construct workload/expectation types explicitly, then attach them - -Both styles produce the same runtime behavior because they ultimately call the same core builder APIs. - -## High-Level Builder DSL (Recommended) - -The DSL is implemented as extension traits (primarily `testing_framework_workflows::ScenarioBuilderExt`) on the core scenario builder. - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -let plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .wallets(5) - .transactions_with(|txs| txs.rate(5).users(3)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(60)) - .build(); -``` - -**When to use:** -- Most test code (smoke, regression, CI) -- When you want sensible defaults and minimal boilerplate - -## Low-Level Direct Instantiation - -Direct instantiation gives you explicit control over the concrete types you attach: - -```rust,ignore -use std::{ - num::NonZeroUsize, - time::Duration, -}; - -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::{ - expectations::ConsensusLiveness, - workloads::transaction, -}; - -let tx_workload = transaction::Workload::with_rate(5) - .expect("transaction rate must be non-zero") - .with_user_limit(NonZeroUsize::new(3)); - -let plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .wallets(5) - .with_workload(tx_workload) - .with_expectation(ConsensusLiveness::default()) - .with_run_duration(Duration::from_secs(60)) - .build(); -``` - -**When to use:** -- Custom workload/expectation implementations -- Reusing preconfigured workload instances across multiple scenarios -- Debugging / exploring the underlying workload types - -## Method Correspondence - -| High-Level DSL | Low-Level Direct | -|----------------|------------------| -| `.transactions_with(\|txs\| txs.rate(5).users(3))` | `.with_workload(transaction::Workload::with_rate(5).expect(...).with_user_limit(...))` | -| `.expect_consensus_liveness()` | `.with_expectation(ConsensusLiveness::default())` | - -## Bundled Expectations (Important) - -Workloads can bundle expectations by implementing `Workload::expectations()`. - -These bundled expectations are attached automatically whenever you call `.with_workload(...)` (including when you use the DSL), because the core builder expands workload expectations during attachment. - -## Mixing Both Styles - -Mixing is common: use the DSL for built-ins, and direct instantiation for custom pieces. - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::{ScenarioBuilderExt, workloads::transaction}; - -let tx_workload = transaction::Workload::with_rate(5) - .expect("transaction rate must be non-zero"); - -let plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .wallets(5) - .with_workload(tx_workload) // direct instantiation - .expect_consensus_liveness() // DSL - .with_run_duration(Duration::from_secs(60)) - .build(); -``` - -## Implementation Detail (How the DSL Works) - -The DSL methods are thin wrappers. For example: - -`builder.transactions_with(|txs| txs.rate(5).users(3))` - -is roughly equivalent to: - -`builder.transactions().rate(5).users(3).apply()` - -## Troubleshooting - -**DSL method not found** -- Ensure the extension traits are in scope, e.g. `use testing_framework_workflows::ScenarioBuilderExt;` -- Cross-check method names in [Builder API Quick Reference](dsl-cheat-sheet.md) - -## See Also - -- [Builder API Quick Reference](dsl-cheat-sheet.md) -- [Example: New Workload & Expectation (Rust)](custom-workload-example.md) -- [Extending the Framework](extending.md) diff --git a/book/src/app-backend-scope.md b/book/src/app-backend-scope.md new file mode 100644 index 0000000..665694e --- /dev/null +++ b/book/src/app-backend-scope.md @@ -0,0 +1,70 @@ +# Backend Scope + +The app layer currently deploys components only through the local backend. Compose and Kubernetes support uniform single-application scenarios. + +This chapter lists the supported combinations and the APIs missing from the container backends. + +--- + +## What Works Where + +| Scenario shape | Local | Compose | Kubernetes | +|----------------|-------|---------|------------| +| Uniform cluster (`ScenarioBuilder` over a topology) | yes | yes | yes | +| AppHost composed stack (`AppHost::scenario().with_app(...)`) | yes | no | no | +| `with_app` presets over an existing uniform scenario | yes | yes | yes | + +An app preset that deploys nothing, such as the "existing cluster" presets in [AppHost and with_app](app-host.md), works on every backend because it only wraps `ctx.deployment()` and `ctx.node_clients()` in a typed handle. Deploying new components through the app layer is local-only: `LocalProcessApp` and `LocalAppCluster` use the local deployer's process primitives (`ProcessNode`, `ManualCluster`, `ProcessDeployer`), and `AppHostLocalDeployer` is a local process deployer. + +Single-app Compose and Kubernetes deployers are unchanged by the app layer. The kvstore and OpenRaft examples keep dedicated bins for them (`kvstore_compose_convergence`, `kvstore_k8s_convergence`, `openraft_kv_compose_failover`, `openraft_kv_k8s_failover`); see [Compose Deployer](deployer-compose.md) and [Kubernetes Deployer](deployer-k8s.md). + +--- + +## Why the Gap Exists + +The app layer starts, addresses, and stops individual units and can run application code between those starts, for example to check readiness or pass an address to a dependent component. The local deployer provides per-unit APIs. Compose and Kubernetes currently render and deploy a complete uniform scenario as one planned unit. Supporting `AppDeployment` on those backends requires corresponding per-unit planning and deployment APIs. + +--- + +## Choosing a Shape Today + +```mermaid +flowchart TD + Q{System under test} -->|one uniform cluster| U[ScenarioBuilder over a topology] + Q -->|composed stack| A[AppHost + root AppDeployment] + U --> B{Backend} + B --> L1[Local] + B --> C1[Compose] + B --> K1[Kubernetes] + A --> L2[Local only] +``` + +- **Composed stacks: run locally.** The local backend provides direct process control and per-unit restarts for heterogeneous stacks (see [Composing Heterogeneous Stacks](composing-stacks.md)). +- **Uniform single-app scenarios: use any supported backend.** A topology containing one application can run locally, with Compose, or on Kubernetes, subject to the capability matrix. +- **Split suites by shape, not by app.** If one system needs both a composed integration stack and a large containerized soak test of its main cluster, express them as two scenarios: an AppHost stack running locally, and a uniform scenario of the main app running on Compose/K8s. The kvstore example uses one environment crate with separate binaries per shape and backend. + +--- + +## Keep Workloads Backend-Independent + +Write workloads against typed handles and clients, not against backend details. A workload that requires a `StoreHandle` does not care whether the store came from a `LocalAppCluster` today or a future containerized unit: + +```rust,ignore +async fn start(&self, ctx: &RunContext) -> Result<(), DynError> { + let store = ctx.require_app::()?; // no backend visible here + store.put("/kv/scope-check", "ok").await?; + Ok(()) +} +``` + +If another backend later supports app composition, backend-specific changes should remain in the root deployment and its child adapters. Workloads and expectations can continue using the same handles. The OpenRaft "existing cluster" preset already works with Compose and Kubernetes because it only reads `ctx.deployment()` and `ctx.node_clients()`. + +**Note:** node-control-style fault injection inside a composed stack (restart one child-cluster node) is a handle method on [`LocalAppCluster`](local-app-cluster.md), so it is local-only by construction. Fault injection on containerized uniform scenarios goes through the scenario-level node control capability instead; see the openraft `openraft_kv_k8s_failover` bin. + +--- + +## See Also + +- [Capability Matrix](capability-matrix.md): the full feature-by-backend table. +- [Local Deployer](deployer-local.md): the backend the app layer builds on. +- [AppHost and with_app](app-host.md): the entry point this scope applies to. diff --git a/book/src/app-deployment.md b/book/src/app-deployment.md new file mode 100644 index 0000000..b01d905 --- /dev/null +++ b/book/src/app-deployment.md @@ -0,0 +1,145 @@ +# AppDeployment and DeployContext + +Application repositories implement `AppDeployment` for deployable components. The implementation uses `DeployContext` to deploy children, expose handles, and register managed resources with scenario cleanup. + +The framework runs deployments, stores handles, and performs teardown without defining application binaries or clients. The application crate decides which components start and which typed handles workloads receive. + +--- + +## The Trait Contract + +```rust,ignore +#[async_trait] +pub trait AppDeployment: Send + 'static { + type Handle: AppHandle; + + async fn deploy(self, ctx: &mut DeployContext) -> Result; +} +``` + +The trait has these properties: + +- **`deploy` consumes `self`.** A deployment value describes one preparation attempt and returns its runtime access handle. +- **The handle is typed.** `Handle` can be any `Clone + Send + Sync + 'static` type. Managed lifetime is registered separately; see [Handle Ownership and Teardown](handles-teardown.md). +- **`Clone` is required by the factory.** `with_app` needs `A: AppDeployment + Clone + Sync` because `AppDeploymentFactory` clones the description on each `prepare`. Deployment structs should contain configuration such as node counts, ports, and paths rather than live resources. + +A minimal implementation, from the kvstore example: + +```rust,ignore +// examples/kvstore/testing/integration/src/app.rs +#[derive(Clone)] +pub struct KvLocalApp { + deployment: KvTopology, +} + +#[async_trait] +impl AppDeployment for KvLocalApp { + type Handle = LocalAppCluster; + + async fn deploy(self, ctx: &mut DeployContext) -> Result { + ctx.deploy_local_cluster::(self.deployment).await + } +} +``` + +--- + +## DeployContext API + +One context belongs to one scenario preparation. It carries the active cluster provisioner, outer deployment and clients, exposed handles, and a cleanup stack. Routing every managed child through this context registers cleanup as soon as the resource is acquired, including when deployment fails partway. + +| Method | Purpose | +|--------|---------| +| `deploy(app)` | Runs a child deployment, returns its handle. Does **not** expose it. | +| `deploy_and_expose(app)` | Runs a child deployment and exposes a clone of its handle. | +| `expose(handle)` | Registers the default (unnamed) handle for its concrete type. | +| `expose_named(name, handle)` | Registers a named handle; allows several instances of one type. | +| `get::()` / `get_named::(name)` | `Option` clone of an exposed handle. | +| `require::()` / `require_named::(name)` | `Result` — typed missing-handle error. | +| `contains::()` | Whether a default handle for `T` is exposed. | +| `handles()` | Borrows the registry of handles exposed so far. | +| `deployment()` | The outer scenario deployment descriptor (`E::Deployment`). | +| `node_clients()` | Clients for nodes owned by the outer scenario (`NodeClients`). | +| `deploy_cluster::(request)` | Provisions a managed, attached, or external cluster through the active provisioner. | +| `deploy_local_cluster::(deployment)` | Convenience for an eager managed cluster with the active provisioner. | + +`deploy` does not expose its returned handle. Use it when only the parent needs the child handle. Use `deploy_and_expose` when workloads should also be able to request the child directly. Both `expose` and `expose_named` return `AppDeployError::DuplicateHandle` if the type or type/name pair is already registered. + +--- + +## Nested Deployments + +A deployment composes children by calling `ctx.deploy(...)` or `ctx.deploy_and_expose(...)` on other `AppDeployment` values. The parent decides what is visible: + +```rust,ignore +#[async_trait] +impl AppDeployment for ParentApp { + type Handle = ParentHandle; + + async fn deploy(self, ctx: &mut DeployContext) -> Result { + let child = ctx.deploy(ChildApp).await?; // child handle NOT exposed + let parent = ParentHandle { child }; + + ctx.expose(parent.clone())?; // only the parent is visible + + Ok(parent) + } +} +``` + +Workloads can then require `ParentHandle` but not `ChildHandle`: the child stays an implementation detail. Its managed resources remain registered with scenario cleanup whether or not the returned handle is exposed or embedded. Expose the child too when workloads legitimately need it. + +```mermaid +flowchart TD + Root[Root AppDeployment] -->|deploy| C1[Child A] + Root -->|deploy_and_expose| C2[Child B] + Root -->|expose| RH[Root handle] + C2 --> BH[Child B handle] + RH --> W[Workloads] + BH --> W + RH:::hd + BH:::hd + W:::sc + classDef hd stroke:#4caf7d,stroke-width:2.5px; + classDef sc stroke:#9b6dd6,stroke-width:2.5px; +``` + +--- + +## The Outer Scenario: deployment() and node_clients() + +For `AppHost` scenarios, `deployment()` is the empty `AppHostTopology` and `node_clients()` is empty; everything lives in your handles. On a regular uniform-cluster scenario, they are how an app preset wraps the managed cluster itself: + +```rust,ignore +// examples/kvstore/testing/integration/src/app.rs +#[async_trait] +impl AppDeployment for KvExistingClusterApp { + type Handle = KvStoreCluster; + + async fn deploy(self, ctx: &mut DeployContext) -> Result { + Ok(KvStoreCluster::new( + ctx.deployment().clone(), + ctx.node_clients().clone(), + )) + } +} +``` + +This preset does not launch nodes. It returns typed access to the nodes already managed by the scenario. + +--- + +## Root-Handle Auto-Exposure + +After the root deployment returns, `AppDeploymentFactory` checks `ctx.contains::()`. If the root handle type is not already exposed, the factory exposes the returned handle as the default for its type. So: + +- A simple root app can just `return Ok(handle)` and workloads can `require_app::()` with no explicit `expose`. +- A root app that already exposed its own handle (like the stack apps in [Composing Heterogeneous Stacks](composing-stacks.md)) is left alone, so there is no duplicate error. + +--- + +## See Also + +- [AppHost and with_app](app-host.md): how a deployment gets registered and prepared. +- [Handle Ownership and Teardown](handles-teardown.md): what exposure means for resource lifetime. +- [One Binary: LocalProcessApp](local-process-app.md), [Uniform Child Clusters: LocalAppCluster](local-app-cluster.md): ready-made deployments to compose. diff --git a/book/src/app-host.md b/book/src/app-host.md new file mode 100644 index 0000000..eccbc39 --- /dev/null +++ b/book/src/app-host.md @@ -0,0 +1,125 @@ +# AppHost and with_app + +`AppHost` creates a scenario whose system under test is supplied by application deployments instead of an outer managed node topology. + +The core scenario engine models one `Application` and a uniform cluster of its nodes. For a composed stack containing a binary, an additional cluster, or several applications, start from `AppHost::scenario()` and register the stack with `.with_app(...)`. Workloads, expectations, run duration, and teardown follow the lifecycle in [Scenario Model and Lifecycle](scenario-model.md). + +--- + +## The Zero-Node Scenario + +`AppHost::scenario()` returns a `ScenarioBuilder` seeded with `AppHostTopology`: + +| Type | Role | +|------|------| +| `AppHostTopology` | Deployment descriptor with `node_count() == 0`. The outer scenario manages no nodes. | +| `AppHostEnv` | Null environment: `NodeClient = ()`, and `build_node_client` always errors. Clients come from app handles instead. | +| `AppHostScenarioBuilder` | Alias for `ScenarioBuilder`. | +| `AppHostLocalDeployer` | Alias for `ProcessDeployer` — the local deployer that executes the scenario. | + +Because the outer topology is empty, app deployments create the processes and clusters used by the run. + +```rust,ignore +use testing_framework_app::{AppHost, AppHostLocalDeployer, AppScenarioBuilderExt}; +use testing_framework_core::scenario::Deployer; + +let mut scenario = AppHost::scenario() + .with_app(KvLocalApp::nodes(3)) + .with_run_duration(Duration::from_secs(5)) + .with_workload(KvAppHostConvergence::new(3)) + .build()?; + +let deployer = AppHostLocalDeployer::default(); +let runner = deployer.deploy(&scenario).await?; +runner.run(&mut scenario).await?; +``` + +The runnable `kvstore_app_host_convergence` binary uses this structure: + +```bash +cargo run -p kvstore-examples --bin kvstore_app_host_convergence +``` + +--- + +## How with_app Runs + +`AppScenarioBuilderExt::with_app(app)` wraps your [`AppDeployment`](app-deployment.md) in an `AppDeploymentFactory` and registers it as a runtime extension factory, the same lifecycle hook covered in [Runtime Extensions](runtime-extensions.md). Going through the extension mechanism ties managed deployment cleanup to the scenario lifetime and makes exposed handles available during the run. + +```mermaid +flowchart LR + B["with_app(app)"] --> F[AppDeploymentFactory] + F -->|prepare| C[DeployContext] + C -->|"deploy(root app)"| H["handles + cleanup"] + H --> R[AppRuntime extension] + R -->|require_app| W[Workloads] + H:::hd + R:::hd + W:::sc + classDef hd stroke:#4caf7d,stroke-width:2.5px; + classDef sc stroke:#9b6dd6,stroke-width:2.5px; +``` + +During scenario preparation the factory: + +1. Clones your app (this is why the factory requires `Clone`) and builds a fresh `DeployContext`. +2. Runs the root deployment's `deploy`, which may deploy and expose child apps. +3. Auto-exposes the returned root handle if the deployment did not expose one of that type itself (`!ctx.contains::()`). +4. Transfers the handle registry and cleanup stack into an `AppRuntime` extension. + +If any step fails, the partially built context is dropped and every resource deployed so far is released (see [Handle Ownership and Teardown](handles-teardown.md)). + +A scenario accepts one `with_app` registration. Every `AppDeploymentFactory` produces the same extension type (`AppRuntime`), and the runtime rejects duplicate extension types. A second registration fails during preparation with `duplicate runtime extension type registered: AppRuntime`. Compose several applications inside one root `AppDeployment` and expose the child handles from there, as shown in [Composing Heterogeneous Stacks](composing-stacks.md). + +--- + +## with_app Outside AppHost + +`with_app` is defined for every scenario builder, not only `AppHostScenarioBuilder`. On a regular uniform-cluster scenario, an "existing cluster" preset can wrap the outer scenario's deployment and node clients in a typed handle without deploying another resource. The OpenRaft example uses this pattern: + +```rust,ignore +// examples/openraft_kv/testing/integration/src/scenario.rs +fn with_existing_openraft_kv_app(app: OpenRaftKvExistingClusterApp) -> Self { + OpenRaftKvScenarioBuilder::with_deployment(app.topology()) + .with_app(app) + .with_cluster_observer() +} +``` + +Here the scenario still manages a uniform OpenRaft cluster, and the app layer just gives workloads a typed `OpenRaftKvCluster` handle over it. + +--- + +## Retrieving Handles in Workloads + +Workloads never see the deploy context. They retrieve exposed handles through `AppRunContextExt`, implemented on `RunContext`: + +| Method | Returns | +|--------|---------| +| `app::()` | `Option` — default handle for `T`, if exposed | +| `app_named::(name)` | `Option` — named handle for `T` | +| `require_app::()` | `Result` — errors if missing | +| `require_app_named::(name)` | `Result` — errors if missing | + +```rust,ignore +use testing_framework_app::AppRunContextExt; + +async fn start(&self, ctx: &RunContext) -> Result<(), DynError> { + let cluster = ctx.require_app::>()?; + cluster.restart_node("node-0").await?; + cluster.wait_node_ready("node-0").await?; + Ok(()) +} +``` + +Workloads normally use the `require_*` variants so that a missing handle produces a typed error containing the requested handle type. + +Every retrieval clones the handle. Handles are normally small access values backed by `Arc`; scenario cleanup still determines managed resource lifetime. + +--- + +## Where to Go Next + +- [AppDeployment and DeployContext](app-deployment.md): implementing the deployment itself. +- [One Binary: LocalProcessApp](local-process-app.md) and [Uniform Child Clusters: LocalAppCluster](local-app-cluster.md): the two built-in building blocks. +- [Backend Scope](app-backend-scope.md): why AppHost scenarios run on the local deployer today. diff --git a/book/src/application-model.md b/book/src/application-model.md new file mode 100644 index 0000000..a8e5f59 --- /dev/null +++ b/book/src/application-model.md @@ -0,0 +1,139 @@ +# Application, AppDeployment, and Environments + +This chapter distinguishes the `Application` trait, the `AppDeployment` trait, and the concrete environment types that implement `Application`. + +--- + +## The Application Trait + +`Application` is the contract between the scenario engine and whatever system you are testing. It bundles the backend-specific types the engine needs, without the engine ever knowing what your application does: + +```rust,ignore +use testing_framework_core::scenario::Application; + +pub trait Application: Send + Sync + 'static { + type Deployment: DeploymentDescriptor + Clone + 'static; + type NodeClient: Clone + Send + Sync + 'static; + type NodeConfig: Clone + Send + Sync + 'static; + + fn external_node_client(source: &ExternalNodeSource) -> Result; + fn build_node_client(access: &NodeAccess) -> Result; + fn node_readiness_path() -> &'static str; // default: "/" +} +``` + +The associated types and methods are: + +- **`Deployment`**: the topology descriptor, i.e. how many nodes exist and how they relate. +- **`NodeClient`**: the typed client workloads use to talk to one node. +- **`NodeConfig`**: the per-node configuration your binary consumes. +- **Client constructors**: `build_node_client` turns deployer-provided `NodeAccess` into a client; `external_node_client` does the same for nodes the framework did not start. Both return an "unsupported" error by default. An environment must implement the operations it supports ([Ownership and Design Boundaries](boundaries.md)). +- **`node_readiness_path`**: the HTTP path deployers probe during readiness gating. + +An implementation of `Application` is called an **environment**. Everything generic in the framework (`ScenarioBuilder`, `Workload`, `Expectation`, `RunContext`) is parameterized over one. + +Source: `testing-framework/core/src/env.rs`. + +--- + +## The AppDeployment Trait + +`Application` describes a *uniform* node population. A system containing a cluster plus another process, or several different clusters, is represented through `AppDeployment` in `testing-framework-app`: + +```rust,ignore +use testing_framework_app::{AppDeployment, AppHandle, DeployContext}; + +pub trait AppDeployment: Send + 'static { + type Handle: AppHandle; + + async fn deploy(self, ctx: &mut DeployContext) -> Result; +} +``` + +An `AppDeployment` is a deployable unit: it consumes its description, prepares whatever it represents, and returns a typed runtime handle. `AppHandle` is a blanket implementation, so any `Clone + Send + Sync + 'static` type qualifies. The handle provides access and control; managed resources acquired through framework adapters are owned separately by scenario cleanup. + +Deployments compose: inside `deploy`, the `DeployContext` lets a parent deployment call `ctx.deploy(child)` or `ctx.deploy_and_expose(child)`, then `ctx.expose(handle)` to publish typed handles to workloads. See [AppDeployment and DeployContext](app-deployment.md) for the full context API. + +An `AppDeployment` registered with `.with_app(...)` runs during scenario preparation. It participates in the scenario lifecycle; it does not replace that lifecycle. + +Source: `testing-framework/app/src/deployment.rs`. + +--- + +## Concrete Environments + +### AppHostEnv: an environment without outer nodes + +`AppHostEnv` is an environment with no outer nodes at all. Its topology, `AppHostTopology`, reports a node count of zero; its `NodeClient` and `NodeConfig` are both `()`; asking it for a node client is an error. It exists so that a scenario can be composed *entirely* from application deployments: + +```rust,ignore +use testing_framework_app::{AppHost, AppScenarioBuilderExt}; + +let builder = AppHost::scenario() // ScenarioBuilder, zero nodes + .with_app(KvLocalApp::nodes(3)); // apps provide all processes +``` + +The system is supplied by `with_app` deployments, and workloads access it through typed handles instead of outer node clients. See [AppHost and with_app](app-host.md). + +Source: `testing-framework/app/src/host.rs`. + +### KvEnv: a uniform node environment + +The kvstore example shows a full environment for a real binary: + +```rust,ignore +pub struct KvEnv; + +impl Application for KvEnv { + type Deployment = KvTopology; // ClusterTopology + type NodeClient = KvHttpClient; + type NodeConfig = KvNodeConfig; + + fn build_node_client(access: &NodeAccess) -> Result { + Ok(KvHttpClient::new(access.api_base_url()?)) + } + + fn node_readiness_path() -> &'static str { + "/health/ready" + } +} +``` + +`KvEnv` additionally implements `LocalBinaryApp` (in `examples/kvstore/testing/integration/src/local_env.rs`) to tell the local deployer which binary to run, how to render per-node configs, and which port serves the HTTP API. The same environment type also backs `AppDeployment` presets like `KvLocalApp`, whose handle is a whole child cluster. [Implementing Application](implementing-application.md) walks through this in detail. + +Source: `examples/kvstore/testing/integration/src/app.rs`. + +--- + +## How the Three Relate + +```mermaid +graph TD + SB["ScenarioBuilder<E>"] -->|"E: Application"| APP["Application
(env contract)"] + KV["KvEnv"] -.->|implements| APP + AH["AppHostEnv
(zero-node env)"] -.->|implements| APP + AD["AppDeployment<E, P>"] -->|"deploys via"| DC["DeployContext<E, P>"] + AD -->|returns| H["typed Handle"] + SB -->|".with_app(...)"| AD + SB:::sc + H:::hd + classDef sc stroke:#9b6dd6,stroke-width:2.5px; + classDef hd stroke:#4caf7d,stroke-width:2.5px; +``` + +| Role | What it is | What it answers | Example | +|---|---|---|---| +| `Application` | Trait bundling `Deployment`, `NodeClient`, `NodeConfig` for the scenario engine | "What types does the engine plumb around?" | `KvEnv` | +| `AppDeployment` | Trait for one deployable unit returning a typed handle | "How is this piece prepared, and what can test code access?" | `KvLocalApp` | +| Concrete environment | A type implementing `Application` | "Which system am I testing, uniform or zero-node?" | `KvEnv`, `AppHostEnv` | + +`Application` and `AppDeployment` are not alternatives. Every scenario has one environment type `E`, and app deployments are registered inside that scenario. A uniform kvstore cluster uses `KvEnv` directly; a heterogeneous stack uses `AppHostEnv` and supplies its components as app deployments. + +--- + +## Where to Go Next + +- [Scenario Model and Lifecycle](scenario-model.md): what a scenario is and how it runs. +- [Choosing an Entry Pattern](entry-patterns.md): which combination of these pieces fits your system. +- [Part II — Composing Applications](part-ii.md): the app layer in depth. +- [Part IV — Uniform Clusters and Configuration](part-iv.md): implementing an environment for your own binary. diff --git a/book/src/architecture-overview.md b/book/src/architecture-overview.md deleted file mode 100644 index 73c8759..0000000 --- a/book/src/architecture-overview.md +++ /dev/null @@ -1,256 +0,0 @@ -# Architecture Overview - -The framework follows a clear flow: **Topology → Scenario → Deployer → Runner → Workloads → Expectations**. - -## Core Flow - -```mermaid -flowchart LR - A(Topology
shape cluster) --> B(Scenario
plan) - B --> C(Deployer
provision & readiness) - C --> D(Runner
orchestrate execution) - D --> E(Workloads
drive traffic) - E --> F(Expectations
verify outcomes) -``` - -## Crate Architecture - -```mermaid -flowchart TB - subgraph Examples["Runner Examples"] - LocalBin[local_runner.rs] - ComposeBin[compose_runner.rs] - K8sBin[k8s_runner.rs] - end - - subgraph Workflows["Workflows (Batteries Included)"] - DSL[ScenarioBuilderExt
Fluent API] - TxWorkload[Transaction Workload] - ChaosWorkload[Chaos Workload] - Expectations[Built-in Expectations] - end - - subgraph Core["Core Framework"] - ScenarioModel[Scenario Model] - Traits[Deployer + Runner Traits] - BlockFeed[BlockFeed] - NodeClients[Node Clients] - Topology[Topology Generation] - end - - subgraph Deployers["Runner Implementations"] - LocalDeployer[LocalDeployer] - ComposeDeployer[ComposeDeployer] - K8sDeployer[K8sDeployer] - end - - subgraph Support["Supporting Crates"] - Configs[Configs & Topology] - Nodes[Node API Clients] - end - - Examples --> Workflows - Examples --> Deployers - Workflows --> Core - Deployers --> Core - Deployers --> Support - Core --> Support - Workflows --> Support - - style Examples fill:#e1f5ff - style Workflows fill:#e1ffe1 - style Core fill:#fff4e1 - style Deployers fill:#ffe1f5 - style Support fill:#f0f0f0 -``` - -### Layer Responsibilities - -**Runner Examples (Entry Points)** -- Executable binaries that demonstrate framework usage -- Wire together deployers, scenarios, and execution -- Provide CLI interfaces for different modes - -**Workflows (High-Level API)** -- `ScenarioBuilderExt` trait provides fluent DSL -- Built-in workloads (transactions, chaos) -- Common expectations (liveness, inclusion) -- Simplifies scenario authoring - -**Core Framework (Foundation)** -- `Scenario` model and lifecycle orchestration -- `Deployer` and `Runner` traits (extension points) -- `BlockFeed` for real-time block observation -- `RunContext` providing node clients and metrics -- Topology generation and validation - -**Runner Implementations** -- `LocalDeployer` - spawns processes on host -- `ComposeDeployer` - orchestrates Docker Compose -- `K8sDeployer` - deploys to Kubernetes cluster -- Each implements `Deployer` trait - -**Supporting Crates** -- `configs` - Topology configuration and generation -- `nodes` - HTTP/RPC client for node APIs - -### Extension Points - -```mermaid -flowchart LR - Custom[Your Code] -.implements.-> Workload[Workload Trait] - Custom -.implements.-> Expectation[Expectation Trait] - Custom -.implements.-> Deployer[Deployer Trait] - - Workload --> Core[Core Framework] - Expectation --> Core - Deployer --> Core - - style Custom fill:#ffe1f5 - style Core fill:#fff4e1 -``` - -**Extend by implementing:** -- `Workload` - Custom traffic generation patterns -- `Expectation` - Custom success criteria -- `Deployer` - Support for new deployment targets - -See [Extending the Framework](extending.md) for details. - -### Components - -- **Topology** describes the cluster: how many nodes and the high-level network parameters they should follow. -- **Scenario** combines that topology with the activities to run and the checks to perform, forming a single plan. -- **Deployer** provisions infrastructure on the chosen backend (local processes, Docker Compose, or Kubernetes), waits for readiness, and returns a Runner. -- **Runner** orchestrates scenario execution: starts workloads, observes signals, evaluates expectations, and triggers cleanup. -- **Workloads** generate traffic and conditions that exercise the system. -- **Expectations** observe the run and judge success or failure once activity completes. - -Each layer has a narrow responsibility so that cluster shape, deployment choice, -traffic generation, and health checks can evolve independently while fitting -together predictably. - -## Entry Points - -The framework is consumed via **runnable example binaries** in `examples/src/bin/`: - -- `local_runner.rs` — Spawns nodes as host processes -- `compose_runner.rs` — Deploys via Docker Compose (requires `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` built) -- `k8s_runner.rs` — Deploys via Kubernetes Helm (requires cluster + image) - -**Recommended:** Use the convenience script: - -```bash -scripts/run/run-examples.sh -t -n -# mode: host, compose, or k8s -``` - -This handles circuit setup, binary building/bundling, image building, and execution. - -**Alternative:** Direct cargo run (requires manual setup): - -```bash -cargo run -p runner-examples --bin -``` - -These binaries use the framework API (`ScenarioBuilder`) to construct and execute scenarios. - -## Builder API - -Scenarios are defined using a fluent builder pattern: - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn scenario_plan() -> testing_framework_core::scenario::Scenario<()> { - ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .wallets(50) - .transactions_with(|txs| txs.rate(5).users(20)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(90)) - .build() -} -``` - -**Key API Points:** -- Topology uses `.topology_with(|t| { t.nodes(N) })` closure pattern -- Workloads are configured via `_with` closures (`transactions_with`, `chaos_with`) -- Chaos workloads require `.enable_node_control()` and a compatible runner - -## Deployers - -Three deployer implementations: - -| Deployer | Backend | Prerequisites | Node Control | -|----------|---------|---------------|--------------| -| `LocalDeployer` | Host processes | Binaries (built on demand or via bundle) | No | -| `ComposeDeployer` | Docker Compose | Image with embedded assets/binaries | Yes | -| `K8sDeployer` | Kubernetes Helm | Cluster + image loaded | Not yet | - -**Compose-specific features:** -- Observability is external (set `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` / `LOGOS_BLOCKCHAIN_METRICS_OTLP_INGEST_URL` / `LOGOS_BLOCKCHAIN_GRAFANA_URL` as needed) -- Optional OTLP trace/metrics endpoints (`LOGOS_BLOCKCHAIN_OTLP_ENDPOINT`, `LOGOS_BLOCKCHAIN_OTLP_METRICS_ENDPOINT`) -- Node control for chaos testing (restart nodes) - -## Assets and Images - -### Docker Image -Built via `scripts/build/build_test_image.sh`: -- Embeds circuit assets and binaries -- Includes runner scripts: `run_nomos_node.sh` -- Tagged as `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` (default: `logos-blockchain-testing:local`) -- **Recommended:** Use prebuilt bundle via `scripts/build/build-bundle.sh --platform linux` and set `LOGOS_BLOCKCHAIN_BINARIES_TAR` before building image - -### Circuit Assets -Circuit assets required by the node binary: -- **Host path:** `~/.logos-blockchain-circuits` (default) -- **Container path:** `/opt/circuits` (for compose/k8s) -- **Override:** `LOGOS_BLOCKCHAIN_CIRCUITS=/custom/path/to/dir` (must point to a directory) -- **Fetch via:** `scripts/setup/setup-logos-blockchain-circuits.sh v0.3.1 ~/.logos-blockchain-circuits` or use `scripts/run/run-examples.sh` - -### Compose Stack -Templates and configs in `testing-framework/runners/compose/assets/`: -- `docker-compose.yml.tera` — Stack template (nodes) -- Cfgsync config: `testing-framework/assets/stack/cfgsync.yaml` -- Monitoring assets (not deployed by the framework): `testing-framework/assets/stack/monitoring/` - -## Logging Architecture - -**Two separate logging pipelines:** - -| Component | Configuration | Output | -|-----------|--------------|--------| -| **Runner binaries** | `RUST_LOG` | Framework orchestration logs | -| **Node processes** | `LOGOS_BLOCKCHAIN_LOG_LEVEL`, `LOGOS_BLOCKCHAIN_LOG_FILTER` (+ `LOGOS_BLOCKCHAIN_LOG_DIR` on host runner) | Consensus, mempool, network logs | - -**Node logging:** -- **Local runner:** Writes to temporary directories by default (cleaned up). Set `LOGOS_BLOCKCHAIN_TESTS_TRACING=true` + `LOGOS_BLOCKCHAIN_LOG_DIR` for persistent files. -- **Compose runner:** Default logs to container stdout/stderr (`docker logs`). To write per-node files, set `tracing_settings.logger: !File` in `testing-framework/assets/stack/cfgsync.yaml` (and mount a writable directory). -- **K8s runner:** Logs to pod stdout/stderr (`kubectl logs`). To write per-node files, set `tracing_settings.logger: !File` in `testing-framework/assets/stack/cfgsync.yaml` (and mount a writable directory). - -**File naming:** Per-node files use prefix `logos-blockchain-node-{index}` (may include timestamps). - -## Observability - -**Prometheus-compatible metrics querying (optional):** -- The framework does **not** deploy Prometheus/Grafana. -- Provide a Prometheus-compatible base URL (PromQL API) via `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL`. -- Accessible in expectations when configured: `ctx.telemetry().prometheus().map(|p| p.base_url())` - -**Grafana dashboards (optional):** -- Dashboards live in `testing-framework/assets/stack/monitoring/grafana/dashboards/` and can be imported into your Grafana. -- If you set `LOGOS_BLOCKCHAIN_GRAFANA_URL`, the deployer prints it in `TESTNET_ENDPOINTS`. - -**Node APIs:** -- HTTP endpoints per node for consensus info and network status -- Accessible in expectations: `ctx.node_clients().node_clients().get(0)` - -**OTLP (optional):** -- Trace endpoint: `LOGOS_BLOCKCHAIN_OTLP_ENDPOINT=http://localhost:4317` -- Metrics endpoint: `LOGOS_BLOCKCHAIN_OTLP_METRICS_ENDPOINT=http://localhost:4318` -- Disabled by default (no noise if unset) - -For detailed logging configuration, see [Logging & Observability](logging-observability.md). diff --git a/book/src/authoring-scenarios.md b/book/src/authoring-scenarios.md deleted file mode 100644 index 95e9ef9..0000000 --- a/book/src/authoring-scenarios.md +++ /dev/null @@ -1,383 +0,0 @@ -# Authoring Scenarios - -Creating a scenario is a declarative exercise. This page walks you through the core authoring loop with concrete examples, explains the units and timing model, and shows how to structure scenarios in Rust test suites. - ---- - -## The Core Authoring Loop - -Every scenario follows the same pattern: - -```mermaid -flowchart LR - A[1. Topology] --> B[2. Workloads] - B --> C[3. Expectations] - C --> D[4. Duration] - D --> E[5. Deploy & Run] -``` - -1. **Shape the topology** — How many nodes, what network shape -2. **Attach workloads** — What traffic to generate (transactions, chaos) -3. **Define expectations** — What success looks like (liveness, inclusion, recovery) -4. **Set duration** — How long to run the experiment -5. **Choose a runner** — Where to execute (local, compose, k8s) - ---- - -## Hello Scenario: Your First Test - -Let's build a minimal consensus liveness test step-by-step. - -### Step 1: Shape the Topology - -```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -let scenario = ScenarioBuilder::topology_with(|t| { - t.network_star() // Star network (one gateway + nodes) - .nodes(3) // 3 nodes -}) -``` - -**What goes in topology?** -- Node counts (nodes) -- Network shape (`network_star()` is currently the only built-in layout) - -**What does NOT go in topology?** -- Traffic rates (that's workloads) -- Success criteria (that's expectations) -- Runtime configuration (that's duration/runner) - -### Step 2: Attach Workloads - -```rust,ignore -.wallets(20) // Seed funded wallet accounts for transaction workloads -.transactions_with(|tx| { - tx.rate(10) // 10 transactions per block - .users(5) // distributed across 5 wallets -}) -``` - -**What goes in workloads?** -- Transaction traffic (rate, users) -- Chaos injection (restarts, delays) - -**Units explained:** -- `.rate(10)` = **10 transactions per block** (not per second!) -- `.users(5)` = use 5 distinct wallet accounts -- The framework adapts to block time automatically - -### Step 3: Define Expectations - -```rust,ignore -.expect_consensus_liveness() -``` - -**What goes in expectations?** -- Health checks that run after the scenario completes -- Liveness (blocks produced) -- Inclusion (workload activity landed on-chain) -- Recovery (system survived chaos) - -**When do expectations run?** -After the duration window ends, during the **evaluation phase** of the scenario lifecycle. - -### Step 4: Set Duration - -```rust,ignore -use std::time::Duration; - -.with_run_duration(Duration::from_secs(60)) -``` - -**How long is enough?** -- Minimum: 2× the expected block time × number of blocks you want -- For consensus liveness: 30-60 seconds -- For transaction inclusion: 60-120 seconds -- For chaos recovery: 2-5 minutes - -**What happens during this window?** -- Nodes are running -- Workloads generate traffic -- Metrics/logs are collected -- BlockFeed broadcasts observations in real-time - -### Step 5: Build and Deploy - -```rust,ignore -.build(); - -// Choose a runner -use testing_framework_core::scenario::Deployer; -use testing_framework_runner_local::LocalDeployer; - -let deployer = LocalDeployer::default(); -let runner = deployer.deploy(&scenario).await?; -let _result = runner.run(&mut scenario).await?; -``` - ---- - -## Complete "Hello Scenario" - -Putting it all together: - -```rust,ignore -use std::time::Duration; - -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_local::LocalDeployer; -use testing_framework_workflows::ScenarioBuilderExt; - -#[tokio::test] -async fn hello_consensus_liveness() -> Result<()> { - let mut scenario = ScenarioBuilder::topology_with(|t| { - t.network_star() - .nodes(3) - }) - .wallets(20) - .transactions_with(|tx| tx.rate(10).users(5)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(60)) - .build(); - - let deployer = LocalDeployer::default(); - let runner = deployer.deploy(&scenario).await?; - runner.run(&mut scenario).await?; - - Ok(()) -} -``` - -**Run it:** -```bash -cargo test hello_consensus_liveness -``` - ---- - -## Understanding Units & Timing - -### Transaction Rate: Per-Block, Not Per-Second - -**Wrong mental model:** `.rate(10)` = 10 tx/second - -**Correct mental model:** `.rate(10)` = 10 tx/block - -**Why?** The blockchain produces blocks at variable rates depending on consensus timing. The framework submits the configured rate **per block** to ensure predictable load regardless of block time. - -**Example:** -- Block time = 2 seconds -- `.rate(10)` → 10 tx/block → 5 tx/second average -- Block time = 5 seconds -- `.rate(10)` → 10 tx/block → 2 tx/second average - -### Duration: Wall-Clock Time - -`.with_run_duration(Duration::from_secs(60))` means the scenario runs for **60 seconds of real time**, not 60 blocks. - -**How many blocks will be produced?** -Depends on consensus timing (slot time, active slot coefficient). Typical: 1-2 seconds per block. - -**Rule of thumb:** -- 60 seconds → ~30-60 blocks -- 120 seconds → ~60-120 blocks - ---- - -## Structuring Scenarios in a Test Suite - -### Pattern 1: Integration Test Module - -```rust,ignore -// tests/integration_test.rs -use std::time::Duration; - -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_local::LocalDeployer; -use testing_framework_workflows::ScenarioBuilderExt; - -#[tokio::test] -async fn test_consensus_liveness() -> Result<()> { - let mut scenario = ScenarioBuilder::topology_with(|t| { - t.network_star().nodes(3) - }) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(30)) - .build(); - - let deployer = LocalDeployer::default(); - let runner = deployer.deploy(&scenario).await?; - runner.run(&mut scenario).await?; - Ok(()) -} - -#[tokio::test] -async fn test_transaction_inclusion() -> Result<()> { - let mut scenario = ScenarioBuilder::topology_with(|t| { - t.network_star().nodes(2) - }) - .wallets(10) - .transactions_with(|tx| tx.rate(5).users(5)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(60)) - .build(); - - let deployer = LocalDeployer::default(); - let runner = deployer.deploy(&scenario).await?; - runner.run(&mut scenario).await?; - Ok(()) -} -``` - -### Pattern 2: Shared Scenario Builders - -Extract common topology patterns: - -```rust,ignore -// tests/helpers.rs -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn minimal_topology() -> ScenarioBuilder { - ScenarioBuilder::topology_with(|t| { - t.network_star().nodes(2) - }) -} - -pub fn production_like_topology() -> ScenarioBuilder { - ScenarioBuilder::topology_with(|t| { - t.network_star().nodes(7) - }) -} - -// tests/consensus_tests.rs -use std::time::Duration; - -use helpers::*; - -#[tokio::test] -async fn small_cluster_liveness() -> anyhow::Result<()> { - let mut scenario = minimal_topology() - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(30)) - .build(); - // ... deploy and run - Ok(()) -} - -#[tokio::test] -async fn large_cluster_liveness() -> anyhow::Result<()> { - let mut scenario = production_like_topology() - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(60)) - .build(); - // ... deploy and run - Ok(()) -} -``` - -### Pattern 3: Parameterized Scenarios - -Test the same behavior across different scales: - -```rust,ignore -use std::time::Duration; - -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_local::LocalDeployer; -use testing_framework_workflows::ScenarioBuilderExt; - -async fn test_liveness_with_topology(nodes: usize) -> Result<()> { - let mut scenario = ScenarioBuilder::topology_with(|t| { - t.network_star() - .nodes(nodes) - }) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(60)) - .build(); - - let deployer = LocalDeployer::default(); - let runner = deployer.deploy(&scenario).await?; - runner.run(&mut scenario).await?; - Ok(()) -} - -#[tokio::test] -async fn liveness_small() -> Result<()> { - test_liveness_with_topology(2, 1).await -} - -#[tokio::test] -async fn liveness_medium() -> Result<()> { - test_liveness_with_topology(5, 2).await -} - -#[tokio::test] -async fn liveness_large() -> Result<()> { - test_liveness_with_topology(10, 3).await -} -``` - ---- - -## What Belongs Where? - -### Topology - -**Do include:** -- Node counts (`.nodes(3)`) -- Network shape (`.network_star()`) - -**Don't include:** -- Traffic rates (workload concern) -- Expected outcomes (expectation concern) -- Runtime behavior (runner/duration concern) - -### Workloads - -**Do include:** -- Transaction traffic (`.transactions_with(|tx| ...)`) -- Chaos traffic (`.chaos().restart()` or `RandomRestartWorkload`) -- Chaos injection (`.with_workload(RandomRestartWorkload::new(...))`) -- Rates, users, timing - -**Don't include:** -- Node configuration (topology concern) -- Success criteria (expectation concern) - -### Expectations - -**Do include:** -- Health checks (`.expect_consensus_liveness()`) -- Inclusion verification (built-in to workloads) -- Custom assertions (`.with_expectation(MyExpectation::new())`) - -**Don't include:** -- Traffic generation (workload concern) -- Cluster shape (topology concern) - ---- - -## Best Practices - -1. **Keep scenarios focused**: One scenario = one behavior under test -2. **Start small**: 2-3 nodes, 30-60 seconds -3. **Use descriptive names**: `test_consensus_survives_node_restart` not `test_1` -4. **Extract common patterns**: Shared topology builders, helper functions -5. **Document intent**: Add comments explaining what you're testing and why -6. **Mind the units**: `.rate(N)` is per-block, `.with_run_duration()` is wall-clock -7. **Set realistic durations**: Allow enough time for multiple blocks + workload effects - ---- - -## Next Steps - -- **[Core Content: Workloads & Expectations](workloads.md)** — Comprehensive reference for built-in workloads and expectations -- **[Examples](examples.md)** — More scenario patterns (chaos, advanced topologies) -- **[Running Scenarios](running-scenarios.md)** — How execution works, artifacts produced, per-runner details -- **[API Levels](api-levels.md)** — When to use builder DSL vs. direct instantiation diff --git a/book/src/best-practices.md b/book/src/best-practices.md deleted file mode 100644 index 6bc802a..0000000 --- a/book/src/best-practices.md +++ /dev/null @@ -1,237 +0,0 @@ -# Best Practices - -This page collects proven patterns for authoring, running, and maintaining test scenarios that are reliable, maintainable, and actionable. - -## Scenario Design - -**State your intent** -- Document the goal of each scenario (throughput, resilience) so expectation choices are obvious -- Use descriptive variable names that explain topology purpose (e.g., `star_topology_3val_2exec` vs `topology`) -- Add comments explaining why specific rates or durations were chosen - -**Keep runs meaningful** -- Choose durations that allow multiple blocks and make timing-based assertions trustworthy -- Use [FAQ: Run Duration Calculator](faq.md#how-long-should-a-scenario-run) to estimate minimum duration -- Avoid runs shorter than 30 seconds unless testing startup behavior specifically - -**Separate concerns** -- Start with deterministic workloads for functional checks -- Add chaos in dedicated resilience scenarios to avoid noisy failures -- Don't mix high transaction load with aggressive chaos in the same test (hard to debug) - -**Start small, scale up** -- Begin with minimal topology (1-2 nodes) to validate scenario logic -- Gradually increase topology size and workload rates -- Use Host runner for fast iteration, then validate on Compose before production - -## Code Organization - -**Reuse patterns** -- Standardize on shared topology and workload presets so results are comparable across environments and teams -- Extract common topology builders into helper functions -- Create workspace-level constants for standard rates and durations - -**Example: Topology preset** - -```rust,ignore -pub fn standard_topology() -> GeneratedTopology { - TopologyBuilder::new() - .network_star() - .nodes(3) - .generate() -} -``` - -**Example: Shared constants** - -```rust,ignore -pub const STANDARD_TX_RATE: f64 = 10.0; -pub const SHORT_RUN_DURATION: Duration = Duration::from_secs(60); -pub const LONG_RUN_DURATION: Duration = Duration::from_secs(300); -``` - -## Debugging & Observability - -**Observe first, tune second** -- Rely on liveness and inclusion signals to interpret outcomes before tweaking rates or topology -- Enable detailed logging (`RUST_LOG=debug`, `LOGOS_BLOCKCHAIN_LOG_LEVEL=debug`) only after initial failure -- Use `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1` to persist logs when debugging failures - -**Use BlockFeed effectively** -- Subscribe to BlockFeed in expectations for real-time block monitoring -- Track block production rate to detect liveness issues early -- Use block statistics (`block_feed.stats().total_transactions()`) to verify inclusion - -**Collect metrics** -- Set up Prometheus/Grafana via `scripts/setup/setup-observability.sh compose up` for visualizing node behavior -- Use metrics to identify bottlenecks before adding more load -- Monitor mempool size, block size, and consensus timing - -## Environment & Runner Selection - -**Environment fit** -- Pick runners that match the feedback loop you need: - - **Host**: Fast iteration during development, quick CI smoke tests - - **Compose**: Reproducible environments (recommended for CI), chaos testing - - **K8s**: Production-like fidelity, large topologies (10+ nodes) - -**Runner-specific considerations** - -| Runner | When to Use | When to Avoid | -|--------|-------------|---------------| -| Host | Development iteration, fast feedback | Chaos testing, container-specific issues | -| Compose | CI pipelines, chaos tests, reproducibility | Very large topologies (>10 nodes) | -| K8s | Production-like testing, cluster behaviors | Local development, fast iteration | - -**Minimal surprises** -- Seed only necessary wallets and keep configuration deltas explicit when moving between CI and developer machines -- Use `versions.env` to pin node versions consistently across environments -- Document non-default environment variables in scenario comments or README - -## CI/CD Integration - -**Use matrix builds** - -```yaml -strategy: - matrix: - runner: [host, compose] - topology: [small, medium] -``` - -**Cache aggressively** -- Cache Rust build artifacts (`target/`) -- Cache circuit parameters (`~/.logos-blockchain-circuits/`) -- Cache Docker layers (use BuildKit cache) - -**Collect logs on failure** - -```yaml -- name: Collect logs on failure - if: failure() - run: | - mkdir -p test-logs - find /tmp -name "nomos-*.log" -exec cp {} test-logs/ \; -- uses: actions/upload-artifact@v3 - if: failure() - with: - name: test-logs-${{ matrix.runner }} - path: test-logs/ -``` - -**Time limits** -- Set job timeout to prevent hung runs: `timeout-minutes: 30` -- Use shorter durations in CI (60s) vs local testing (300s) -- Run expensive tests (k8s, large topologies) only on main branch or release tags - -**See also:** [CI Integration](ci-integration.md) for complete workflow examples - -## Anti-Patterns to Avoid - -```bash -# BAD: Will hang/timeout on proof generation -cargo run -p runner-examples --bin local_runner -``` - -**DON'T: Use tiny durations** -```rust,ignore -// BAD: Not enough time for blocks to propagate -.with_run_duration(Duration::from_secs(5)) - -// GOOD: Allow multiple consensus rounds -.with_run_duration(Duration::from_secs(60)) -``` - -**DON'T: Ignore cleanup failures** -```rust,ignore -// BAD: Next run inherits leaked state -runner.run(&mut scenario).await?; -// forgot to call cleanup or use CleanupGuard - -// GOOD: Cleanup via guard (automatic on panic) -let _cleanup = CleanupGuard::new(runner.clone()); -runner.run(&mut scenario).await?; -``` - -**DON'T: Mix concerns in one scenario** -```rust,ignore -// BAD: Hard to debug when it fails -.transactions_with(|tx| tx.rate(50).users(100)) // high load -.chaos_with(|c| c.restart().min_delay(...)) // AND chaos - -// GOOD: Separate tests for each concern -// Test 1: High transaction load only -// Test 2: Chaos resilience only -``` - -**DON'T: Hardcode paths or ports** -```rust,ignore -// BAD: Breaks on different machines -let path = PathBuf::from("/home/user/circuits"); -let port = 9000; // might conflict - -// GOOD: Use env vars and dynamic allocation -let path = std::env::var("LOGOS_BLOCKCHAIN_CIRCUITS") - .unwrap_or_else(|_| "~/.logos-blockchain-circuits".to_string()); -let port = get_available_tcp_port(); -``` - -**DON'T: Ignore resource limits** -```bash -# BAD: Large topology without checking resources -scripts/run/run-examples.sh -n 20 compose -# (might OOM or exhaust ulimits) - -# GOOD: Scale gradually and monitor resources -scripts/run/run-examples.sh -n 3 compose # start small -docker stats # monitor resource usage -# then increase if resources allow -``` - -## Scenario Design Heuristics - -**Minimal viable topology** -- Consensus: 3 nodes (minimum for Byzantine fault tolerance) -- Network: Star topology (simplest for debugging) - -**Workload rate selection** -- Start with 1-5 tx/s per user, then increase -- Chaos: 30s+ intervals between restarts (allow recovery) - -**Duration guidelines** - -| Test Type | Minimum Duration | Typical Duration | -|-----------|------------------|------------------| -| Smoke test | 30s | 60s | -| Integration test | 60s | 120s | -| Load test | 120s | 300s | -| Resilience test | 120s | 300s | -| Soak test | 600s (10m) | 3600s (1h) | - -**Expectation selection** - -| Test Goal | Expectations | -|-----------|--------------| -| Basic functionality | `expect_consensus_liveness()` | -| Transaction handling | `expect_consensus_liveness()` + custom inclusion check | -| Resilience | `expect_consensus_liveness()` + recovery time measurement | - -## Testing the Tests - -**Validate scenarios before committing** -1. Run on Host runner first (fast feedback) -2. Run on Compose runner (reproducibility check) -3. Check logs for warnings or errors -4. Verify cleanup (no leaked processes/containers) -5. Run 2-3 times to check for flakiness - -**Handling flaky tests** -- Increase run duration (timing-sensitive assertions need longer runs) -- Reduce workload rates (might be saturating nodes) -- Check resource limits (CPU/RAM/ulimits) -- Add debugging output to identify race conditions -- Consider if test is over-specified (too strict expectations) - -**See also:** -- [Troubleshooting](troubleshooting.md) for common failure patterns -- [FAQ](faq.md) for design decisions and gotchas diff --git a/book/src/binary-providers.md b/book/src/binary-providers.md new file mode 100644 index 0000000..11b7fbb --- /dev/null +++ b/book/src/binary-providers.md @@ -0,0 +1,123 @@ +# Binary Providers + +Binary providers resolve the executable a local node process runs. The source can be a path, an env var, a build command, a download, or an ordered fallback chain. + +Every local node launch has exactly one provider selected on its `LocalProcessSpec`. Providers live in `testing_framework_runner_local::binary` and are re-exported from the crate root. They apply to the [Local Deployer](deployer-local.md) only; compose and k8s nodes run container images instead (see the [Capability Matrix](capability-matrix.md)). + +--- + +## The Trait + +```rust,ignore +pub trait BinaryProvider: Send + Sync { + fn try_resolve(&self) -> Result, BinaryProviderError>; + fn display(&self) -> String; + fn cache_key(&self) -> String; + + // Provided: cache lookup, then resolve_uncached. + fn resolve(&self) -> Result { /* ... */ } + fn resolve_uncached(&self) -> Result { /* ... */ } +} + +pub type BinaryProviderRef = Arc; +``` + +`try_resolve` returns `Ok(None)` when the provider is valid but cannot produce a binary in the current environment, which is not an error. Standalone, `resolve` turns `None` into `BinaryProviderError::NotFound`; inside a `FallbackBinaryProvider`, `None` means "try the next provider". Other errors (a failed build, a checksum mismatch) abort resolution immediately. + +--- + +## The Providers + +| Provider | Resolves from | Unresolved (`None`) when | +|---|---|---| +| `PathBinaryProvider` | A fixed absolute path | Path is not a file (relative paths are an error) | +| `EnvBinaryProvider` | An env var containing a path | Var unset or not pointing at a file | +| `BuildBinaryProvider` | Running a build command | Never — build failure is an error | +| `DownloadBinaryProvider` | Fetching a URL into a cache | Never — download failure is an error | +| `FallbackBinaryProvider` | First chain member to resolve | Every member returned `None` | + +**`PathBinaryProvider::new(path)`**: a deterministic explicit path. No filesystem search, no `PATH` lookup. + +**`EnvBinaryProvider::new("MY_NODE_BIN")`**: the standard override hook. `LocalProcessSpec::new(env_var)` installs one of these by default. + +**`BuildBinaryProvider`** delegates to any command: + +```rust,ignore +BuildBinaryProvider { + command: BuildCommand::new("cargo").with_args(["build", "-p", "kvstore-node"]), + output_path: "target/debug/kvstore-node".into(), // relative to working_dir + working_dir: Some(workspace_root), // default: current dir + lock_dir: None, // default: /.tf-binaries +} +``` + +The command is not Cargo-specific; it can invoke a Make target, shell script, or cache fetch. After the command succeeds, the configured `output_path` must exist or resolution fails with `MissingBuildOutput`. + +**`DownloadBinaryProvider`** fetches into a cache directory (default `target/.tf-binaries` under the current directory): + +- `DownloadUrl::Fixed(url)` or `DownloadUrl::Env(var)` selects the source. +- `DownloadChecksum::Fixed(sha256)` or `DownloadChecksum::Env(var)` enables SHA-256 verification; mismatches fail with `ChecksumMismatch` before anything is written to the final path. +- A `DownloadProcessor` post-processes artifacts that are not directly executable (archives, bundles). It receives the verified download and must materialize the executable at the output path. `DownloadProcessorFn::new(cache_key, closure)` (or `.with_processor_fn(...)`) is the lightweight adapter; the `cache_key` is part of cache identity, so changing your extraction logic invalidates previously prepared binaries. +- On Unix, the result is marked executable (`0o755`). Downloads are staged through temporary `.download`/`.part` files and renamed into place. + +**`FallbackBinaryProvider::new([a, b, ...])`**: an ordered chain, tried first to last. From the launch spec's perspective it is still a single provider. + +--- + +## Caching and Cache Identity + +Successful resolutions are cached **per process** in a global map keyed by `cache_key()`, so repeated node starts with the same provider config do not rebuild, redownload, or re-scan. Cache identity encodes the full request: + +- `path:`, `env:` +- `build:::` +- `download::::` +- fallback: the members' keys joined with commas + +Change any component and you get a fresh resolution. The download provider also caches on disk: the cached file name hashes the URL, resolved checksum, and processor key, so an already-downloaded binary is reused across processes without refetching. + +--- + +## Concurrent Resolution Locking + +Builds and downloads may be triggered by several test processes at once (e.g. `cargo nextest` running integration tests in parallel). Providers that materialize files take a **cross-process file lock** before doing work: a lock file created with `create_new` under `.tf-binaries` (build) or the download cache dir, retried every 200 ms for up to 10 minutes, then `BinaryProviderError::LockTimeout`. The lock file is removed when the guard drops. + +A killed test process can leave a stale lock file behind. If resolution hangs and then times out, look for leftover `*.lock` files under `.tf-binaries` and delete them. + +--- + +## Worked Example: kvstore's Fallback Chain + +From `examples/kvstore/testing/integration/src/local_env.rs`, which prefers an explicit env override and otherwise builds from source: + +```rust,ignore +use std::{path::PathBuf, sync::Arc}; +use testing_framework_runner_local::{ + BinaryProviderRef, BuildBinaryProvider, BuildCommand, EnvBinaryProvider, + FallbackBinaryProvider, LocalProcessSpec, +}; + +fn kvstore_binary_provider() -> FallbackBinaryProvider { + let providers: [BinaryProviderRef; 2] = [ + Arc::new(EnvBinaryProvider::new("KVSTORE_NODE_BIN")), + Arc::new(BuildBinaryProvider { + command: BuildCommand::new("cargo") + .with_args(["build", "-p", "kvstore-node", "--bin", "kvstore-node"]), + output_path: PathBuf::from(format!( + "target/debug/kvstore-node{}", + std::env::consts::EXE_SUFFIX + )), + working_dir: Some(workspace_root()), + lock_dir: None, + }), + ]; + FallbackBinaryProvider::new(providers) +} + +fn local_process_spec() -> LocalProcessSpec { + LocalProcessSpec::new("KVSTORE_NODE_BIN") + .with_binary_provider(kvstore_binary_provider()) + .with_rust_log("kvstore_node=info") +} +``` + +First run: `KVSTORE_NODE_BIN` is unset, the env provider yields `None`, the build provider compiles the node under the workspace lock, and the path is cached for the rest of the process. Set `KVSTORE_NODE_BIN=/path/to/kvstore-node` to skip the build entirely, which is useful for prebuilt release binaries or mixed-version clusters (via `local_process_spec_for_node`, see [Local Deployer](deployer-local.md)). diff --git a/book/src/boundaries.md b/book/src/boundaries.md new file mode 100644 index 0000000..3b9554a --- /dev/null +++ b/book/src/boundaries.md @@ -0,0 +1,87 @@ +# Ownership and Design Boundaries + +This chapter lists the responsibilities of the framework and of an application repository. + +--- + +## The Boundary + +The scenario engine never names a concrete application. Its only coupling point is the `Application` trait: a bundle of associated types (`Deployment`, `NodeClient`, `NodeConfig`) that the engine plumbs around generically. Everything that knows what your system *is* (its binary, its config format, its client, its notion of "healthy") lives on your side of that trait. + +| Concern | Owner | +|---|---| +| Process lifetime (spawn, stop, restart, PIDs) | Framework | +| Working directories and temp dirs | Framework | +| Cleanup guards and teardown ordering | Framework | +| Topology mechanics (ports, peers, node names) | Framework | +| Readiness probing, retry, and gating | Framework | +| Handle storage and lookup (`HandleRegistry`, `AppRuntime`, `RunContext`) | Framework | +| Workload/expectation scheduling and aggregation | Framework | +| Node binaries and how to obtain them | Application repo | +| `NodeConfig` shape and rendering | Application repo | +| Typed node clients | Application repo | +| Readiness endpoints and app-specific checks | Application repo | +| Domain handles (`StoreHandle`, `WalletHandle`, ...) | Application repo | +| Meaningful workloads, expectations, scenarios | Application repo | + +--- + +## What the Framework Owns + +**Process lifetime and working directories.** Deployers spawn node processes into per-run working directories, track PIDs, and stop everything on teardown. Artifact retention is policy (`CleanupPolicy::preserve_artifacts`), not something scenarios hand-roll. + +**Cleanup.** Teardown is guard-based and automatic: cleanup guards chain and run in reverse registration order when the `RunHandle` drops, and the same guards run on the failure path. App-layer adapters register managed resources in a LIFO cleanup stack so dependants stop before dependencies, independently of exposed handle clones ([Handle Ownership and Teardown](handles-teardown.md)). + +**Topology mechanics.** Port allocation, peer wiring, node naming, and readiness gating with retry are all generic over `E: Application`. The engine asks your environment *what* to render and probe, never *why*. + +**Handle storage and lookup.** `DeployContext` collects typed handles during preparation; `AppRuntime` carries them into the run; `AppRunContextExt` returns clones to workloads. Duplicate exposure of a type/name pair is an error, never a silent replacement. + +--- + +## What the Application Repository Owns + +The kvstore example is the template. Its integration crate supplies, in its own repository: + +- **The binary and how to get it**: a `FallbackBinaryProvider` chain that uses `KVSTORE_NODE_BIN` if set and otherwise builds `kvstore-node` with cargo ([Binary Providers](binary-providers.md)). +- **Config**: `KvNodeConfig`, built per node from the framework's port/peer views and rendered to YAML. +- **Client**: `KvHttpClient`, constructed in `Application::build_node_client`. +- **Readiness**: `node_readiness_path()` returning `/health/ready`. +- **Domain handles and presets**: `KvStoreCluster`, `KvLocalApp`, `KvExistingClusterApp`. +- **Scenarios that mean something**: write workloads, convergence expectations, runnable bins. + +```rust,ignore +// Application side: domain knowledge, no orchestration. +fn node_readiness_path() -> &'static str { + "/health/ready" +} + +// Framework side: orchestration, no domain knowledge. +// It only ever sees E::NodeClient, E::NodeConfig, E::Deployment. +``` + +Sources: `examples/kvstore/testing/integration/src/{app,local_env}.rs`, `testing-framework/app/src/lib.rs`. + +--- + +## How the Boundary Is Enforced + +**Unsupported defaults.** `Application::build_node_client` and `external_node_client` return an "unsupported" error by default. Capabilities are available only when the environment implements them. + +**Generic application types.** There is no global list of known applications or framework config file naming their binaries. `testing-framework-core` compiles against `E: Application`, so it does not depend on adopter types or their dependencies. The same runtime can therefore be instantiated with kvstore, openraft_kv, nats, or an application from another repository. + +**Application-owned composition.** Application repositories implement `AppDeployment`, compose children through `DeployContext`, and expose typed handles. The framework supplies the context, registry, and lifecycle without defining the application stack. + +**CI boundary check.** `scripts/run/check-boundaries.sh` checks an application-side topology crate for framework-extension symbols (`cfgsync`, `ComposeDeployEnv`, `K8sDeployEnv`, `runner-compose`, `runner-k8s`). This detects backend dependencies in topology code. The compiler enforces the reverse direction because core crates do not reference concrete application types. + +> **External example:** the current boundary script targets logos-blockchain's `lb-topology` crate (in its own checkout), which keeps that adopter's topology code local/topology-focused. The pattern generalizes: point the same grep at your own integration crates. + +Application-specific config formats and startup rules belong in the environment implementation or an `AppDeployment`, not in framework crates. + +--- + +## Where to Go Next + +- [Application, AppDeployment, and Environments](application-model.md): the trait that defines the boundary. +- [Implementing Application](implementing-application.md): building your side of it. +- [Framework vs Application Boundaries](tf-boundaries.md): the reference treatment in Part VI. +- [Public Extension Points](extension-points.md): the sanctioned ways to extend the framework itself. diff --git a/book/src/capabilities.md b/book/src/capabilities.md new file mode 100644 index 0000000..d80fefe --- /dev/null +++ b/book/src/capabilities.md @@ -0,0 +1,117 @@ +# Scenario Capabilities + +Capabilities record, in the type system, which deployer services a scenario requests, such as node control or external telemetry. Unsupported combinations fail during construction, compilation, or deployment rather than during a workload. + +--- + +## The Capability Type Parameter + +The core builder is generic over a capability marker: `Builder` with `Caps = ()` by default. Building a scenario produces `Scenario`, and deployers are typed as `Deployer`, so a deployer that cannot provide a capability does not accept scenarios that demand it. + +The public wrappers (`testing-framework/core/src/scenario/definition/builder.rs`): + +| Builder type | Capability | Entered via | +|--------------|-----------|-------------| +| `ScenarioBuilder` | `()` | `ScenarioBuilder::with_deployment(...)` / `::new(provider)` | +| `NodeControlScenarioBuilder` | `NodeControlCapability` | `.with_node_control()` (alias `.enable_node_control()`) | +| `ObservabilityScenarioBuilder` | `ObservabilityCapability` | `.with_observability()` or any `ObservabilityBuilderExt` method | + +All three expose the same fluent surface (`with_workload`, `with_expectation`, `with_run_duration`, ...), so the capability switch can happen anywhere in the chain: + +```rust,ignore +let scenario = ScenarioBuilder::with_deployment(topology) + .with_node_control() // () -> NodeControlCapability + .with_workload(my_restart_workload) + .with_run_duration(Duration::from_secs(60)) + .build()?; +``` + +`RequiresNodeControl` (`testing-framework/core/src/scenario/capabilities.rs`) is how `build()` and deployers reason about the marker: + +```rust,ignore +pub trait RequiresNodeControl { + const REQUIRED: bool; +} +// (): false NodeControlCapability: true ObservabilityCapability: false +``` + +`build()` uses it to validate the source configuration: a scenario that requires node control but only has external, uncontrolled nodes fails with a `SourceConfiguration` error ("node control is not available for cluster mode 'external-only' ..."). See [Existing and External Clusters](external-clusters.md). + +--- + +## Node Control Without ManualCluster + +Restarting nodes from a declarative workload does **not** require `ManualCluster`. The node-control capability provides access instead: + +1. Call `.with_node_control()` on the builder. +2. Deploy with a deployer that supports the capability (local ships full node control, compose supports restart; the k8s deployer wires no node control handle into managed scenarios, so use its `ManualCluster` mode instead). +3. Inside a workload, take the handle from the context: + +```rust,ignore +let Some(control) = ctx.node_control() else { + return Err("this workload requires node control".into()); +}; + +control.restart_node("node-1").await?; +``` + +`ManualCluster` is the imperative API for tests that control the entire node lifecycle themselves; see [ManualCluster: Imperative Node Control](manual-cluster.md). The scenario form above runs workloads, expectations, and teardown through the scenario runtime. [Chaos and Controlled Failure](chaos.md) shows a full failover scenario built this way. + +### NodeControlHandle + +`NodeControlHandle` (`testing-framework/core/src/scenario/control.rs`) is the deployer-agnostic control surface. Every method has a default implementation returning a "not supported by this deployer" error, so partial support is explicit at run time: + +| Method | Effect | +|--------|--------| +| `restart_node(name)` | Stop and start a named node | +| `restart_node_with(name, options)` | Restart with `StartNodeOptions` overrides | +| `start_node(name)` | Start a node, returning `StartedNode` | +| `start_node_with(name, options)` | Start with overrides | +| `stop_node(name)` | Stop a named node | +| `wait_node_ready(name)` | Wait for one named node's readiness gate | +| `node_client(name)` | Current client for a node, if any | +| `node_pid(name)` | OS pid where applicable | + +`StartedNode` is a plain pair: the node `name` and a fresh `E::NodeClient`. + +`ClusterWaitHandle` is the matching wait surface: a single `wait_network_ready()` used for readiness gates. It is exposed publicly on the runner as `Runner::wait_network_ready()` (before `run` starts) and on `ManualCluster`; inside workloads, prefer waiting on observed application state instead. + +### StartNodeOptions + +`StartNodeOptions` customizes a dynamic start or restart. Overview (full treatment in Part IV: [Ports, Peers, Node Config, and Readiness](node-config.md) and [Persistence, Snapshots, and Recovery Testing](persistence.md)): + +| Field | Builder method | Purpose | +|-------|----------------|---------| +| `peers: Option` | `with_peers` | `DefaultLayout`, `None`, or `Named(vec)` | +| `config_override: Option` | `with_config_override` | Replace the generated config | +| `config_patch` | `create_patch(fn)` | Transform the generated config before spawn | +| `persist_dir: Option` | `with_persist_dir` | Place the working directory at a findable location ([Persistence](persistence.md)) | +| `snapshot_dir: Option` | `with_snapshot_dir` | Seed the working dir from a snapshot | +| `args: Vec` | `with_args` | Extra process arguments | +| `runtime.start_timeout` | `with_runtime` / `with_start_timeout` | Readiness timeout override | + +--- + +## The Observability Capability + +`ObservabilityCapability` carries optional telemetry endpoints (Prometheus query URL, OTLP ingest URL, Grafana URL). It does not require node control and is populated through `ObservabilityBuilderExt` (`testing-framework/core/src/scenario/builder_ext.rs`): + +```rust,ignore +use testing_framework_core::scenario::ObservabilityBuilderExt; + +let builder = ScenarioBuilder::with_deployment(topology) + .with_metrics_query_url_str("http://127.0.0.1:9090"); +``` + +Each method transitions `ScenarioBuilder` into `ObservabilityScenarioBuilder` (and is a plain setter if you are already there). `Url`-typed, `_str` (panicking), and `try_..._str` (fallible) variants exist for all three endpoints. Deployers merge these values with environment variables; the details, including what telemetry is and is not, are in [Telemetry and External Observability](telemetry.md). + +Capabilities use one marker per scenario, not a set. Choosing `with_node_control()` gives the scenario node control; choosing an observability method supplies telemetry endpoints. Each deployer declares which `Caps` it supports; the [Capability Matrix](capability-matrix.md) lists the available combinations. + +--- + +## See Also + +- [Chaos and Controlled Failure](chaos.md) — node control from workloads +- [ManualCluster: Imperative Node Control](manual-cluster.md) — the imperative alternative +- [Telemetry and External Observability](telemetry.md) — the observability capability in use +- [Capability Matrix](capability-matrix.md) — deployer support by capability diff --git a/book/src/capability-matrix.md b/book/src/capability-matrix.md new file mode 100644 index 0000000..358bd52 --- /dev/null +++ b/book/src/capability-matrix.md @@ -0,0 +1,46 @@ +# Capability Matrix + +This page records what each deployer backend currently supports, based on the deployer implementations. + +The framework ships three deployers: `ProcessDeployer` (local processes), `ComposeDeployer` (Docker Compose), and `K8sDeployer` (Kubernetes/Helm). All three drive the same scenario runtime; they differ in where nodes run and which capabilities they wire into it. + +| Feature | Local | Compose | K8s | +|---|---|---|---| +| Uniform managed scenarios | Yes | Yes | Yes | +| Node control (`with_node_control`) | Yes — start, stop, restart | Restart only (managed); restart + stop (attached) | No — use `ManualCluster` | +| Observability / telemetry inputs | No — telemetry is empty | Yes | Yes | +| Attach / existing clusters | No — rejected | Yes — compose project/services | Yes — label selector | +| External nodes | Yes | Yes | Yes | +| App layer / AppHost composition | Yes (only backend) | No | No | +| Binary providers | Yes | No — container images | No — container images | +| cfgsync artifacts | No — direct config files | Yes | Yes | + +--- + +## Row-by-Row + +**Uniform managed scenarios.** All three deployers implement the `Deployer` trait for scenarios built with `ScenarioBuilder` over a topology: `deployer.deploy(&scenario).await` returns a `Runner`. This is the common path shown in the [Local](deployer-local.md), [Compose](deployer-compose.md), and [Kubernetes](deployer-k8s.md) chapters. + +**Node control.** The local deployer implements `Deployer` and backs it with a `NodeManager` that can start, stop, and restart node processes, including `StartNodeOptions` (peer selection, config overrides, persist/snapshot dirs). The compose deployer wires a `ComposeNodeControl` handle that supports `restart_node` via `docker compose restart`; in attached (existing-cluster) mode it also supports `stop_node` via `docker container stop`. The k8s deployer does not wire a node control handle into managed scenario deployments at all; node lifecycle control on Kubernetes goes through the k8s `ManualCluster` (see [Kubernetes Deployer](deployer-k8s.md) and [ManualCluster](manual-cluster.md)). + +**Observability / telemetry inputs.** Compose and k8s resolve `ObservabilityInputs` from `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` / `LOGOS_BLOCKCHAIN_METRICS_OTLP_INGEST_URL` / `LOGOS_BLOCKCHAIN_GRAFANA_URL` env vars merged with the scenario's `ObservabilityCapability`, pass the OTLP ingest URL into workspace preparation, and build the run's `Metrics` telemetry handle from the query URL. The local orchestrator constructs its runtime with `Metrics::empty()` and never resolves observability inputs. See [Telemetry and External Observability](telemetry.md). + +**Attach / existing clusters.** `with_existing_cluster(...)` switches the scenario to `ClusterMode::ExistingCluster`. Compose accepts descriptors built with `ExistingCluster::for_compose_project` / `for_compose_services`; k8s accepts `for_k8s_selector` / `for_k8s_selector_in_namespace`. The local deployer explicitly rejects existing-cluster mode with a source-orchestration error. Details in [Existing and External Clusters](external-clusters.md). + +**External nodes.** All three deployers resolve `with_external_node(s)` sources into node clients through `Application::external_node_client`. The local deployer additionally falls back to a generic endpoint parser (`build_external_client`) when the application does not override that hook. + +**App layer / AppHost composition.** The app layer is local-only today: `AppHostLocalDeployer` is a type alias for `ProcessDeployer`. There is no compose or k8s AppHost deployer. See [Backend Scope](app-backend-scope.md). + +**Binary providers.** Binary resolution (`PathBinaryProvider`, `EnvBinaryProvider`, `BuildBinaryProvider`, `DownloadBinaryProvider`, `FallbackBinaryProvider`) lives in the local deployer crate and feeds `LocalProcessSpec`. Compose and k8s nodes run container images instead, so image selection happens through descriptor specs and env-var overrides, not binary providers. See [Binary Providers](binary-providers.md). + +**cfgsync artifacts.** The compose deployer writes a `cfgsync.yaml` into its workspace and can launch a Docker-backed cfgsync config server sidecar (`ComposeConfigServerMode::Docker`); the k8s deployer supports cfgsync-backed config overrides in manual-cluster flows and cfgsync-rendered bootstrap assets in chart values. The local deployer materializes rendered config files directly into each node's working directory with no cfgsync involvement. See [Static Artifacts and cfgsync](cfgsync.md). + +--- + +## Backend Selection + +The local backend runs node processes directly and provides full node control. It requires no infrastructure beyond the node binary, which a [binary provider](binary-providers.md) can build. + +Use Compose for container images, container networking, or telemetry endpoints. Use Kubernetes to exercise charts, NodePort or port-forward access paths, and cluster infrastructure. Attach to an already-running stack when the cluster outlives the test (see [Existing and External Clusters](external-clusters.md)). + +Readiness gating, deploy retries, and artifact preservation are controlled uniformly through `DeploymentPolicy`; see [Readiness, Retry, and Artifact Preservation](deployment-policies.md). diff --git a/book/src/cfgsync.md b/book/src/cfgsync.md new file mode 100644 index 0000000..00b7462 --- /dev/null +++ b/book/src/cfgsync.md @@ -0,0 +1,92 @@ +# Static Artifacts and cfgsync + +This chapter covers how typed app configs become per-node file artifacts and how containerized backends deliver them to nodes that cannot see your filesystem. + +--- + +## Why cfgsync Exists + +The local deployer writes each node's rendered config into the node's working directory. Compose and Kubernetes nodes run in containers without access to the host directory where the framework generated those configs. cfgsync transfers the generated per-node files at startup and when a node is restarted with overridden options. + +cfgsync consists of a typed artifact model and an HTTP service. A node container starts, registers with the cfgsync server, fetches its artifact set, writes the files locally, and then starts the application. The same artifact types support runtime config overrides through `replace_node_artifacts` for dynamic node starts on Kubernetes. + +```mermaid +sequenceDiagram + participant R as Runner + participant S as cfgsync server + participant C as node container + R->>S: render config + artifacts, start server + C->>S: POST /register (identifier, ip, metadata) + C->>S: POST /node + S-->>C: NodeArtifactsPayload (files) + C->>C: write files, exec node binary +``` + +--- + +## The Crates + +| Crate | Role | +|---|---| +| `cfgsync-artifacts` | Pure data model: `ArtifactFile { path, content }`, `ArtifactSet` (with `ensure_unique_paths`) | +| `cfgsync-core` | Protocol types, HTTP server/router, protocol client, config sources, render helpers | +| `cfgsync-adapter` | App-facing materialization: registrations in, artifacts out | +| `cfgsync-runtime` | Runnable server/client: `cfgsync-server` and `cfgsync-client` binaries, env-driven client | + +**Protocol (`cfgsync-core`).** `NodeRegistration` carries a stable `identifier`, an IPv4 address, and an opaque `RegistrationPayload`, adapter-owned JSON metadata the framework never interprets (`with_metadata(&T)` / `from_json_str`). The server answers `/node` with a `NodeArtifactsPayload` (schema version + files) or a structured error: `MissingConfig` (unknown node), `NotReady` (registered, artifacts pending), `Internal`. `Client` wraps the endpoints: `register_node`, `fetch_node_config`, `fetch_node_config_status` (→ `ConfigFetchStatus::{Ready, NotReady, Missing}`), and the administrative `ReplaceNodeArtifactsRequest` for swapping one node's served files. + +**Sources.** A server serves whatever its `NodeConfigSource` resolves: + +- `StaticConfigSource`: an in-memory map of identifier → payload, built from payloads or from a `NodeArtifactsBundle` (per-node entries plus `shared_files` served to everyone). Registration succeeds only for known identifiers. Supports `replace_node_artifacts`. +- `RegistrationConfigSource` (`cfgsync-adapter`) is registration-aware: it records registrations, snapshots them, and asks a materializer for artifacts on every resolve. Per-node overrides installed via `replace_node_artifacts` win over materialized files (shared files are still appended). + +**Materialization (`cfgsync-adapter`).** The adapter contract is one trait: + +```rust,ignore +pub trait RegistrationSnapshotMaterializer: Send + Sync { + fn materialize_snapshot( + &self, + registrations: &RegistrationSnapshot, + ) -> Result; +} +``` + +`RegistrationSnapshot` is the current registration set, sorted by identifier for determinism. The result is `NotReady` (keep polling) or `Ready(MaterializedArtifacts)`: per-node `ArtifactSet`s keyed by identifier plus a shared set appended to every node (`resolve(identifier)` merges them). Wrappers: `CachedSnapshotMaterializer` caches results per snapshot, `PersistingSnapshotMaterializer` additionally pushes ready artifacts into a `MaterializedArtifactsSink`. A prebuilt `MaterializedArtifacts` value is itself a materializer that is always ready. + +**Runtime (`cfgsync-runtime`).** `ServerConfig { port, source }` loads from YAML; `ServerSource` is `static` (serve precomputed artifacts, no registration required) or `registration` (require registration first). The runtime `Client` adds local materialization: `OutputMap` routes artifact paths to disk (`OutputMap::under(root)`, `config_and_shared(config_path, shared_dir)`, or explicit `route(...)`), and `run_client_from_env` drives the whole register-fetch-write loop from `CFG_SERVER_ADDR`, `CFG_HOST_IDENTIFIER`, `CFG_HOST_IP`, `CFG_REGISTRATION_METADATA_JSON`, and output paths (`CFG_FILE_PATH`, `CFG_DEPLOYMENT_PATH`). This is what runs inside node containers before the app binary starts. + +--- + +## From Typed Config to Artifacts + +The boundary between your typed `NodeConfig` and cfgsync lives in `testing-framework/core/src/cfgsync/mod.rs`. + +Apps that implement `ClusterNodeConfigApplication` (see [Implementing Application](implementing-application.md)) get `StaticNodeConfigProvider` for free: build a config for node `i`, rewrite it for backend hostnames (`node-0.svc` instead of `127.0.0.1`), and serialize it. On top of that: + +- `build_static_artifacts::(deployment, hostnames)` produces a `MaterializedArtifacts` with one `/config.yaml` per `node-` identifier. Hostname count must match the node count. +- `render_and_write_registration_server::(...)` renders both the cfgsync server config YAML and the precomputed artifacts YAML to disk, with an `enrich_artifacts` hook for app-specific extras (shared files, additional per-node files). +- `build_node_artifact_override::(deployment, index, hostnames, options)` builds the replacement artifact set for a node started with non-default `StartNodeOptions`; `PeerSelection`, `config_override`, and `config_patch` are interpreted for container backends here (see [node-config.md](node-config.md)). + +The backends consume these directly: the Compose deployer calls `write_registration_server_compose_configs` to render the server config and artifacts into the generated stack directory before `docker compose up` ([Compose Deployer](deployer-compose.md)); the K8s deployer exposes `cfgsync_service`, `cfgsync_hostnames`, and `build_cfgsync_override_artifacts` hooks on its environment trait and pushes override artifacts through `replace_node_artifacts` when its manual cluster starts nodes with options ([Kubernetes Deployer](deployer-k8s.md)). + +```mermaid +graph LR + A["ClusterNodeConfigApplication
(typed NodeConfig)"] --> B["build_static_artifacts
(MaterializedArtifacts)"] + B --> C[cfgsync server] + C --> D["cfgsync client in container
(writes files)"] + D --> E[node process] + C:::pr + D:::pr + E:::pr + classDef pr stroke:#e08a3c,stroke-width:2.5px; +``` + +--- + +## Choosing a Shape + +**Precomputed (used by the framework deployers):** all registrations are known up front, so the deployer materializes every artifact before the stack starts and serves them through a `registration`-kind source. The rendered `cfgsync.artifacts.yaml` remains in the stack directory for inspection. + +**Registration-driven:** when artifacts depend on runtime facts (e.g. which IPs registered), implement `RegistrationSnapshotMaterializer` yourself and return `NotReady` until the snapshot is complete. The runnable examples in `cfgsync/runtime/examples/` (`minimal_cfgsync.rs`, `precomputed_registration_cfgsync.rs`, `wait_for_registrations_cfgsync.rs`) show both shapes end to end. + +The framework deployers currently use the precomputed path. The registration-driven materializer is public API for integrations whose per-node configs cannot be finalized before nodes start. diff --git a/book/src/chaos.md b/book/src/chaos.md index 5a63917..9427c5b 100644 --- a/book/src/chaos.md +++ b/book/src/chaos.md @@ -1,57 +1,138 @@ -# Chaos Workloads +# Chaos and Controlled Failure -> **When should I read this?** You don't need chaos testing to be productive with the framework. Focus on basic scenarios first—chaos is for resilience validation and operational readiness drills once your core tests are stable. +Chaos scenarios deliberately stop or restart nodes and then check recovery. Ordinary workloads perform these operations through the node-control capability. -Chaos in the framework uses node control to introduce failures and validate -recovery. The built-in restart workload lives in -`testing_framework_workflows::workloads::chaos::RandomRestartWorkload`. +--- -## How it works -- Requires `NodeControlCapability` (`enable_node_control()` in the scenario - builder) and a runner that provides a `NodeControlHandle`. -- Randomly selects nodes to restart based on your - include/exclude flags. -- Respects min/max delay between restarts and a target cooldown to avoid - flapping the same node too frequently. -- Runs alongside other workloads; expectations should account for the added - disruption. -- Support varies by runner: node control is not provided by the local runner - and is not yet implemented for the k8s runner. Use a runner that advertises - `NodeControlHandle` support (e.g., compose) for chaos workloads. +## The Shape of a Chaos Scenario + +A chaos test is three ordinary pieces wired together: + +1. A scenario built with `.with_node_control()` (see [Scenario Capabilities](capabilities.md)). +2. A workload that drives traffic, disrupts a node via `ctx.node_control()`, waits for recovery, and drives traffic again. +3. An expectation that verifies the end state converged despite the disruption. + +```mermaid +flowchart LR + T[Drive traffic]:::sc --> D[Disrupt
restart node]:::sc + D --> W[Wait for recovery
observed state]:::sc + W --> T2[Drive traffic again]:::sc + T2 --> V[Expectation:
state converged]:::sc + + classDef sc stroke:#9b6dd6,stroke-width:2.5px; +``` + +No `ManualCluster` is involved: the deployer provides a `NodeControlHandle` because the scenario declared the capability. + +--- + +## Worked Example: OpenRaft Leader Failover + +The openraft_kv failover test bootstraps a three-node Raft cluster, expands it to three voters, writes a batch, restarts the *leader*, then writes a second batch through the node elected next. Its workload is in `examples/openraft_kv/testing/workloads/src/failover.rs`: -## Usage ```rust,ignore -use std::time::Duration; +#[async_trait] +impl Workload for OpenRaftKvFailoverWorkload { + fn name(&self) -> &str { + "openraft_kv_failover_workload" + } -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::{ScenarioBuilderExt, workloads::chaos::RandomRestartWorkload}; + async fn start(&self, ctx: &RunContext) -> Result<(), DynError> { + let clients = ctx.node_clients().snapshot(); + let observer = ctx.require_extension::>()?; -pub fn random_restart_plan() -> testing_framework_core::scenario::Scenario< - testing_framework_core::scenario::NodeControlCapability, -> { - ScenarioBuilder::topology_with(|t| t.network_star().nodes(2)) - .enable_node_control() - .with_workload(RandomRestartWorkload::new( - Duration::from_secs(45), // min delay - Duration::from_secs(75), // max delay - Duration::from_secs(120), // target cooldown - true, // include nodes - )) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(150)) - .build() + ensure_cluster_size(&clients, 3)?; + self.bootstrap_cluster(&clients).await?; + + let initial_leader = wait_for_observed_leader(&observer, self.timeout, None).await?; + let membership = OpenRaftMembership::discover(&clients).await?; + + self.promote_cluster(&observer, &clients, initial_leader, &membership).await?; + self.write_initial_batch(&clients, initial_leader).await?; + + let new_leader = self + .restart_leader_and_wait_for_failover(ctx, &observer, initial_leader) + .await?; + self.write_second_batch(&clients, new_leader).await?; + + Ok(()) + } } ``` -## Expectations to pair -- **Consensus liveness**: ensure blocks keep progressing despite restarts. -- **Height convergence**: optionally check all nodes converge after the chaos - window. -- Any workload-specific inclusion checks if you’re also driving transactions. +The disruption itself is a few lines: -## Best practices -- Keep delays/cooldowns realistic; avoid back-to-back restarts that would never - happen in production. -- Limit chaos scope: toggle nodes based on what you want to - test. -- Combine with observability: monitor metrics/logs to explain failures. +```rust,ignore +let Some(control) = ctx.node_control() else { + return Err("openraft failover workload requires node control".into()); +}; + +control.restart_node(&format!("node-{leader_id}")).await?; + +let new_leader = wait_for_observed_leader(observer, self.timeout, Some(leader_id)).await?; +``` + +The guard returns a clear error if the capability is missing. + +### Assembling the Scenario + +`build_failover_scenario` (`examples/openraft_kv/examples/src/lib.rs`) puts workload, expectation, and capability together: + +```rust,ignore +pub fn build_failover_scenario( + run_duration: Duration, + workload_timeout: Duration, +) -> anyhow::Result> { + Ok(OpenRaftKvScenarioBuilder::with_existing_openraft_kv_app( + OpenRaftKvExistingClusterApp::nodes(3), + ) + .enable_node_control() + .with_run_duration(run_duration) + .with_workload(OpenRaftKvClusterAccessible::new(3)) + .with_workload( + OpenRaftKvFailoverWorkload::new() + .first_batch(INITIAL_WRITE_BATCH) + .second_batch(SECOND_WRITE_BATCH) + .timeout(workload_timeout) + .key_prefix(RAFT_KEY_PREFIX), + ) + .with_expectation( + OpenRaftKvConverges::new(TOTAL_WRITES) + .timeout(run_duration) + .key_prefix(RAFT_KEY_PREFIX), + ) + .build()?) +} +``` + +The return type is `Scenario`, so only deployers that provide node control will accept it. Run it locally or on compose: + +```bash +cargo run -p openraft-kv-examples --bin openraft_kv_basic_failover +cargo run -p openraft-kv-examples --bin openraft_kv_compose_failover +``` + +The `openraft_kv_k8s_failover` bin executes the same failover flow on Kubernetes, but uses `ManualCluster` imperatively (`start_node` per node, `restart_node`, `wait_network_ready`). See [ManualCluster: Imperative Node Control](manual-cluster.md). + +--- + +## Patterns + +**Restart-and-verify.** The minimal chaos loop: write known data, `restart_node`, wait for readiness, verify the data survived. Restarts reuse the node's existing working directory, so on-disk state survives them by default; use `with_snapshot_dir` to seed a restore from saved state; details are in [Persistence, Snapshots, and Recovery Testing](persistence.md). + +**Leader failover.** Restart the node that currently holds a distinguished role. The failover workload discovers the leader from observed cluster state instead of assuming a node index. Passing the old identity to the wait (`different_from: Some(leader_id)` above) verifies that leadership changed rather than accepting the old leader after it restarts. + +**Readiness waits after disruption.** Wait before sending traffic after a restart. The example waits on *observed application state* (an agreed leader across all nodes) via an [observation handle](observation.md), which checks more than an HTTP readiness probe. In imperative flows, `ManualCluster::wait_network_ready()` covers transport-level readiness (the k8s bin calls it right after `restart_node`); inside a declarative workload, wait on observed state. + +**Pair chaos with continuous observation.** A background observer polls every node through the disruption, so waits read stored snapshots and can report the last observation (`timed out waiting for observed leader agreement ...; last observation: node=0 leader=None ...`). A workload can also poll clients directly, but must then track the polling state itself. + +Managed clusters get a minimum 30-second cooldown window after the workload phase before expectations run, allowing post-chaos state to settle; see [Expectations and Evaluation](expectations.md). + +--- + +## See Also + +- [Scenario Capabilities](capabilities.md) — `with_node_control` and `StartNodeOptions` +- [Continuous Observation](observation.md) — the observer used for recovery waits +- [Persistence, Snapshots, and Recovery Testing](persistence.md) — restart with retained state +- [ManualCluster: Imperative Node Control](manual-cluster.md) — the imperative variant diff --git a/book/src/ci-integration.md b/book/src/ci-integration.md deleted file mode 100644 index a258d2c..0000000 --- a/book/src/ci-integration.md +++ /dev/null @@ -1,360 +0,0 @@ -# CI Integration - -Both **LocalDeployer** and **ComposeDeployer** work well in CI environments. Choose based on your tradeoffs. - -## Runner Comparison for CI - -**LocalDeployer (Host Runner):** -- Faster startup (no Docker overhead) -- Good for quick smoke tests -- **Trade-off:** Less isolation (processes share host resources) - -**ComposeDeployer (Recommended for CI):** -- Better isolation (containerized) -- Reproducible environment -- Can integrate with external Prometheus/Grafana (optional) -- **Trade-offs:** Slower startup (Docker image build), requires Docker daemon - -**K8sDeployer:** -- Production-like environment -- Full resource isolation -- **Trade-offs:** Slowest (cluster setup + image loading), requires cluster access -- Best for nightly/weekly runs or production validation - -**Existing Examples:** - -See `.github/workflows/lint.yml` (jobs: `host_smoke`, `compose_smoke`) for CI examples running the demo scenarios in this repository. - -## Complete CI Workflow Example - -Here's a comprehensive GitHub Actions workflow demonstrating host and compose runners with caching, matrix testing, and log collection: - -```yaml -name: Testing Framework CI - -on: - push: - branches: [main, develop] - pull_request: - branches: [main] - -env: - CARGO_TERM_COLOR: always - RUST_BACKTRACE: 1 - -jobs: - # Quick smoke test with host runner (no Docker) - host_smoke: - name: Host Runner Smoke Test - runs-on: ubuntu-latest - timeout-minutes: 15 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Rust toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - - - name: Cache Rust dependencies - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-host-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-host- - - - name: Cache logos-blockchain-node build - uses: actions/cache@v3 - with: - path: | - ../logos-blockchain-node/target/release/logos-blockchain-node - key: ${{ runner.os }}-nomos-${{ hashFiles('../logos-blockchain-node/**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-nomos- - - - name: Run host smoke test - run: | - # Use run-examples.sh which handles setup automatically - scripts/run/run-examples.sh -t 120 -n 3 host - - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@v3 - with: - name: host-runner-logs - path: | - .tmp/ - *.log - retention-days: 7 - - # Compose runner matrix (with Docker) - compose_matrix: - name: Compose Runner (${{ matrix.topology }}) - runs-on: ubuntu-latest - timeout-minutes: 25 - - strategy: - fail-fast: false - matrix: - topology: - - "3v1e" - - "5v1e" - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Rust toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Rust dependencies - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-compose-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-compose- - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile', 'scripts/build/build_test_image.sh') }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Run compose test - env: - TOPOLOGY: ${{ matrix.topology }} - run: | - # Build and run with the specified topology - scripts/run/run-examples.sh -t 120 -n ${TOPOLOGY:0:1} compose - - - name: Collect Docker logs on failure - if: failure() - run: | - mkdir -p logs - for container in $(docker ps -a --filter "name=nomos-compose-" -q); do - docker logs $container > logs/$(docker inspect --format='{{.Name}}' $container).log 2>&1 - done - - - name: Upload logs and artifacts - if: failure() - uses: actions/upload-artifact@v3 - with: - name: compose-${{ matrix.topology }}-logs - path: | - logs/ - .tmp/ - retention-days: 7 - - - name: Clean up Docker resources - if: always() - run: | - docker compose down -v 2>/dev/null || true - docker ps -a --filter "name=nomos-compose-" -q | xargs -r docker rm -f - - # Summary job (requires all tests to pass) - ci_success: - name: CI Success - needs: [host_smoke, compose_matrix] - runs-on: ubuntu-latest - if: always() - - steps: - - name: Check all jobs - run: | - if [[ "${{ needs.host_smoke.result }}" != "success" ]] || \ - [[ "${{ needs.compose_matrix.result }}" != "success" ]]; then - echo "One or more CI jobs failed" - exit 1 - fi - echo "All CI jobs passed!" -``` - -## Workflow Features - -1. **Matrix Testing:** Runs compose tests with different topologies (`3v1e`, `5v1e`) -2. **Caching:** Caches Rust dependencies, Docker layers, and logos-blockchain-node builds for faster runs -3. **Log Collection:** Automatically uploads logs and artifacts when tests fail -4. **Timeout Protection:** Reasonable timeouts prevent jobs from hanging indefinitely -6. **Clean Teardown:** Ensures Docker resources are cleaned up even on failure - -## Customization Points - -**Topology Matrix:** - -Add more topologies for comprehensive testing: - -```yaml -matrix: - topology: - - "3v1e" - - "5v1e" - - "10v2e" # Larger scale -``` - -**Timeout Adjustments:** - -Increase `timeout-minutes` for longer-running scenarios or slower environments: - -```yaml -timeout-minutes: 30 # Instead of 15 -``` - -**Artifact Retention:** - -Change `retention-days` based on your storage needs: - -```yaml -retention-days: 14 # Keep logs for 2 weeks -``` - -**Conditional Execution:** - -Run expensive tests only on merge to main: - -```yaml -if: github.event_name == 'push' && github.ref == 'refs/heads/main' -``` - -## Best Practices - -### Use Helper Scripts - -Prefer `scripts/run/run-examples.sh` which handles all setup automatically: - -```bash -scripts/run/run-examples.sh -t 120 -n 3 host -``` - -This is more reliable than manual `cargo run` commands. - -### Cache Aggressively - -Cache Rust dependencies, logos-blockchain-node builds, and Docker layers to speed up CI: - -```yaml -- name: Cache Rust dependencies - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} -``` - -### Collect Logs on Failure - -Always upload logs when tests fail for easier debugging: - -```yaml -- name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@v3 - with: - name: test-logs - path: | - .tmp/ - *.log - retention-days: 7 -``` - -### Split Workflows for Faster Iteration - -For large projects, split host/compose/k8s into separate workflow files: - -- `.github/workflows/test-host.yml` — Fast smoke tests -- `.github/workflows/test-compose.yml` — Reproducible integration tests -- `.github/workflows/test-k8s.yml` — Production-like validation (nightly) - -### Run K8s Tests Less Frequently - -K8s tests are slower. Consider running them only on main branch or scheduled: - -```yaml -on: - push: - branches: [main] - schedule: - - cron: '0 2 * * *' # Daily at 2 AM -``` - -## Platform-Specific Notes - -### Ubuntu Runners - -- Docker pre-installed and running -- Best for compose/k8s runners -- Most common choice - -### macOS Runners - -- Docker Desktop not installed by default -- Slower and more expensive -- Use only if testing macOS-specific issues - -### Self-Hosted Runners - -- Cache Docker images locally for faster builds -- Set resource limits (`SLOW_TEST_ENV=true` if needed) -- Ensure cleanup scripts run (`docker system prune`) - -## Debugging CI Failures - -### Enable Debug Logging - -Add debug environment variables temporarily: - -```yaml -env: - RUST_LOG: debug - LOGOS_BLOCKCHAIN_LOG_LEVEL: debug -``` - -### Preserve Containers (Compose) - -Set `COMPOSE_RUNNER_PRESERVE=1` to keep containers running for inspection: - -```yaml -- name: Run compose test (preserve on failure) - env: - COMPOSE_RUNNER_PRESERVE: 1 - run: scripts/run/run-examples.sh -t 120 -n 3 compose -``` - -### Access Artifacts - -Download uploaded artifacts from the GitHub Actions UI to inspect logs locally. - -## Next Steps - -- [Running Examples](running-examples.md) — Manual execution for local development -- [Environment Variables](environment-variables.md) — Full variable reference -- [Troubleshooting](troubleshooting.md) — Common CI-specific issues - diff --git a/book/src/ci.md b/book/src/ci.md new file mode 100644 index 0000000..341bc1a --- /dev/null +++ b/book/src/ci.md @@ -0,0 +1,77 @@ +# Continuous Integration + +This chapter covers how this repository checks itself, and patterns for running framework-based tests in your own CI. + +--- + +## Workflows in This Repository + +Two GitHub Actions workflows live in `.github/workflows/`. + +### `lint.yml` + +Runs on every push and pull request, with per-ref concurrency cancellation. All jobs pin the `nightly-2025-09-14` toolchain and cache `~/.cargo/registry`, `~/.cargo/git`, and `target/` keyed on `Cargo.lock`. + +| Job | Command | Checks | +|---|---|---| +| `fmt` | `cargo +nightly-2025-09-14 fmt --all -- --check` | formatting | +| `clippy` | `cargo clippy --all --all-targets --all-features -- -D warnings` | lints, warnings as errors | +| `deny` | `cargo deny check -c .cargo-deny.toml --show-stats -D warnings` | licenses, advisories, bans | +| `taplo` | `taplo fmt --check` and `taplo lint` | TOML formatting and lints | +| `machete` | `cargo machete` | unused dependencies | + +### `deploy-pages.yml` + +Builds this book with `mdbook build book` and publishes `target/book` to GitHub Pages. It triggers on pushes to `master` that touch `book/**`, or manually via `workflow_dispatch`. + +**Note:** CI currently covers linting and the book. The unit tests and example scenarios run via `cargo test` / `cargo run` on developer machines; there is no test workflow yet. + +--- + +## Helper Scripts + +- `scripts/run/checks.sh` is an informational, best-effort environment sanity check. It reports workspace and disk state, the Rust toolchain, Docker and Docker Compose availability, the Kubernetes context (including whether a `:local` image tag will be visible to `kind`, `minikube`, or `docker-desktop` clusters), and the current values of runner debug flags such as `COMPOSE_RUNNER_PRESERVE` and `K8S_RUNNER_PRESERVE`. Run it first when a backend misbehaves. +- `scripts/run/check-boundaries.sh` is a boundary guard for an adopter checkout living next to this repository. It fails if the adopter's topology crate references extension-specific symbols (`cfgsync`, compose/k8s deployer types), keeping the [framework/application boundary](tf-boundaries.md) enforceable by grep. + +--- + +## Patterns for Consumers + +If your repository builds tests on this framework, the following translate directly into CI configuration. + +### Cache the build for `BuildBinaryProvider` + +Local scenarios that resolve node binaries through a `BuildBinaryProvider` invoke `cargo build` at deploy time. On a cold runner this can dominate the job. Cache `~/.cargo/registry`, `~/.cargo/git`, and `target/` keyed on `Cargo.lock`, as `lint.yml` does, so the deploy-time build is incremental. Resolution is also cached in-process and serialized across concurrent test processes with a file lock, so parallel test binaries do not race the same build; see [Binary Providers](binary-providers.md). + +### Ensure Docker for compose tests + +Compose scenarios need a working Docker daemon and the node images already present: the runner verifies images with `docker image inspect` and fails with `MissingImage` rather than building or pulling. Add an image build/pull step before the test step. Decide your skip policy explicitly: the in-repo example binaries treat `ComposeRunnerError::DockerUnavailable` as a graceful skip, which is convenient locally but silently masks coverage loss in CI. In a pipeline, prefer failing (or gating the job on a Docker-capable runner). + +### Slow runners + +Set `SLOW_TEST_ENV=true` on constrained runners; the framework doubles its internal timeouts (`testing_framework_core::adjust_timeout`). The k8s deployer's wait timeouts can also be tuned individually; see [Environment Variables](environment-variables.md). + +### Preserve artifacts on failure + +By default every backend tears down and deletes its working state. To retain evidence from failed CI runs, preserve and upload the artifacts: + +```yaml +- name: Run scenarios + run: cargo test -p my-scenarios + env: + TF_KEEP_LOGS: "1" # keep local per-node working directories + COMPOSE_RUNNER_PRESERVE: "1" # keep the compose workspace and containers +- name: Upload artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: scenario-artifacts + path: | + **/.tmp* +``` + +Local node directories are created under the test process's working directory; note that panicking tests preserve their node directories automatically. The equivalent in code is `CleanupPolicy { preserve_artifacts: true }` via `with_deployment_policy`. What lands in those directories and how to read them is covered in [Diagnostics and Retained Artifacts](diagnostics.md). + +### Reproduce failures + +Log or fix the deployment seed (`with_deployment_seed`) so a failing CI run can be replayed locally with the same generated deployment; see [Seeds and Reproducibility](seeds.md). diff --git a/book/src/cluster-provisioning.md b/book/src/cluster-provisioning.md new file mode 100644 index 0000000..b60cff8 --- /dev/null +++ b/book/src/cluster-provisioning.md @@ -0,0 +1,121 @@ +# Shared Cluster Provisioning + +App composition and uniform scenarios share a cluster-provisioning model. A request describes the cluster source and required behavior. A provisioner returns a backend-independent `ClusterHandle` and registers any managed lifetime with the app cleanup stack. + +--- + +## One Request, Three Sources + +`ClusterRequest` separates what the test needs from how a backend supplies it: + +```rust,ignore +let managed = ClusterRequest::::managed(QueueTopology::new(3)); +let attached = ClusterRequest::::attached(existing_cluster); +let external = ClusterRequest::::external(node_sources); +``` + +| Source | Nodes started by the framework | Clients | Node control | Framework teardown | +|---|---:|---:|---:|---:| +| `Managed` | Yes, unless start mode is on demand | Yes | When requested and supported | Yes | +| `Attached` | No | Yes | According to the attached cluster's control profile | Only resources the framework itself acquires | +| `External` | No | Yes | No | No | + +Managed and attached sources can also include external nodes with `with_external_nodes(...)`. This is useful when one logical cluster combines framework-visible nodes from more than one source. + +--- + +## Requesting Behavior + +The request carries requirements that are meaningful across backends: + +| Method | Meaning | +|---|---| +| `with_policy(policy)` | Apply readiness, retry, cleanup, and network-control policy. | +| `with_start_mode(Eager)` | Start managed nodes while provisioning. This is the default. | +| `with_start_mode(OnDemand)` | Prepare a managed cluster but let test code start nodes explicitly. | +| `with_control(Full)` | Require the node-control surface on the returned handle. | +| `with_network_control()` | Require backend network control. | +| `with_network_recovery(recovery)` | Register application recovery after a network effect is released; also requests network control. | + +Backends may support different combinations of these requirements. The [Capability Matrix](capability-matrix.md) records current coverage. + +--- + +## Provisioning Inside an AppDeployment + +`DeployContext` is parameterized by a `ClusterProvisioner`. Its `deploy_cluster` method is the app-layer entry point: + +```rust,ignore +#[async_trait] +impl AppDeployment for QueueApp { + type Handle = ClusterHandle; + + async fn deploy( + self, + ctx: &mut DeployContext, + ) -> Result { + ctx.deploy_cluster(ClusterRequest::::managed(self.topology)) + .await + } +} +``` + +`DeployContext::deploy_cluster` requests the full common node-control surface because app workloads receive the returned cluster handle directly. The local convenience `deploy_local_cluster` expresses the common managed, eager case. Use `deploy_cluster` when ownership mode, start mode, or policy must be visible in the app definition. + +`with_app(app)` selects the default local provisioner. `with_app_using(app, provisioner)` supplies another provisioner. The root deployment must implement `AppDeployment` for that provisioner type; code written only as `AppDeployment` uses the default local type. + +--- + +## The Returned Handle + +`ClusterHandle` presents the common runtime surface: + +- clients: `node_clients`, `clients`, `first_client`, `node_client`; +- cluster description: `deployment`, `node_count`, `control_profile`; +- node operations when present: `start_node`, `stop_node`, `restart_node`, `wait_node_ready`; +- cluster readiness: `wait_network_ready`; +- network effects when present: `network_control`. + +Unavailable operations return an error or `None`; callers can inspect `control_profile()` when behavior depends on ownership mode. + +The handle does not own managed lifetime. The provisioner returns a private cleanup guard alongside the runtime surfaces. `DeployContext` moves that guard into the scenario cleanup stack, which runs in reverse acquisition order on normal completion and partial deployment failure. + +--- + +## Backend Boundary + +`ClusterProvisioner` has one operation: + +```rust,ignore +#[async_trait] +pub trait ClusterProvisioner: Clone + Send + Sync + 'static { + async fn provision_cluster( + &self, + request: ClusterRequest, + ) -> Result, DynError>; +} +``` + +A backend implementation translates the request into concrete resources, clients, control adapters, readiness, and cleanup. `ClusterUnit` carries these values from the provisioner; applications normally use the resulting `ClusterHandle`. + +The local provisioner currently supports managed and external sources. Attached support and equivalent Compose/Kubernetes app provisioners require backend implementations, but not another application-composition model. + +--- + +## Relation to Other Entry Patterns + +- A uniform scenario asks its deployer to provision the scenario's primary cluster. +- A composed stack asks its `DeployContext` to provision one or more child clusters. +- An attached or external test changes `ClusterSource`, while workloads keep using clients and available controls. +- `ManualCluster` uses local provisioning machinery directly and gives imperative code responsibility for sequencing. + +The entry patterns differ in who describes and drives the test. They do not need separate definitions of what a cluster is, which controls it exposes, or who tears it down. + +--- + +## See Also + +- [Existing and External Clusters](external-clusters.md): declaring non-managed sources in ordinary scenarios. +- [AppDeployment and DeployContext](app-deployment.md): composing cluster and process children. +- [Handle Ownership and Teardown](handles-teardown.md): the lifetime boundary in detail. +- [Readiness, Retry, and Cleanup](deployment-policies.md): the policies carried by a request. diff --git a/book/src/composing-stacks.md b/book/src/composing-stacks.md new file mode 100644 index 0000000..42248b4 --- /dev/null +++ b/book/src/composing-stacks.md @@ -0,0 +1,154 @@ +# Composing Heterogeneous Stacks + +A root `AppDeployment` deploys the components, passes dependency addresses between them, and exposes typed handles. + +A scenario accepts one `with_app` registration (see [AppHost and with_app](app-host.md)), so the root deployment composes its children. It exposes component handles needed by workloads and may also expose an aggregate stack handle. The `examples/multi_app` fixture contains a queue cluster, job-worker process, and kv result-store cluster in one job-processing pipeline. + +--- + +## The Root App + +```rust,ignore +// examples/multi_app/fixture/src/lib.rs +#[derive(Clone)] +struct JobStackApp { + queue_nodes: usize, + result_nodes: usize, +} + +impl JobStackApp { + fn new() -> Self { + Self { + queue_nodes: 2, + result_nodes: 2, + } + } +} + +#[async_trait] +impl AppDeployment for JobStackApp { + type Handle = JobStackHandle; + + async fn deploy(self, ctx: &mut DeployContext) -> Result { + let queue = ctx + .deploy_and_expose(QueueLocalApp::nodes(self.queue_nodes)) + .await?; + let results = ctx + .deploy_and_expose(KvLocalApp::nodes(self.result_nodes)) + .await?; + + let queue_url = queue + .first_client() + .ok_or("queue cluster has no clients")? + .base_url() + .clone(); + let results_url = results + .first_client() + .ok_or("result store has no clients")? + .base_url() + .clone(); + let worker = ctx + .deploy_and_expose(JobWorkerApp::new(queue_url, results_url)) + .await?; + + let stack = JobStackHandle { queue, results, worker }; + ctx.expose(stack.clone())?; + + Ok(stack) + } +} +``` + +The example establishes these relationships: + +- **Children are deployed through the context** (`deploy_and_expose`), so each cluster and the worker process are owned by the runtime for the whole run. +- **Dependencies are constructor arguments.** The worker receives the queue and result-store URLs from the already-deployed clusters, so the root deployment shows the dependency graph. +- **Both levels are exposed**: each component handle and the aggregate `JobStackHandle`, allowing workloads to request the smallest handle they need. + +```mermaid +flowchart TD + Root[JobStackApp] --> Q["queue cluster x2
LocalAppCluster<QueueEnv>"] + Root --> R["result store x2
LocalAppCluster<KvEnv>"] + Q --> W["job worker
LocalProcessApp"] + R --> W + Q --> St[JobStackHandle] + R --> St + W --> St + Q:::cl + R:::cl + W:::pr + St:::hd + classDef cl stroke:#4a90d9,stroke-width:2.5px; + classDef pr stroke:#e08a3c,stroke-width:2.5px; + classDef hd stroke:#4caf7d,stroke-width:2.5px; +``` + +--- + +## Workloads Require What They Need + +Each workload asks for exactly the handles it uses, the whole stack or one component: + +```rust,ignore +async fn start(&self, ctx: &RunContext) -> Result<(), DynError> { + let stack = ctx.require_app::()?; + let queue = stack.queue.first_client().ok_or("queue cluster has no clients")?; + + for index in 0..self.count { + let response: EnqueueResponse = queue + .post("/queue/enqueue", &EnqueueRequest { payload: job_key(index) }) + .await?; + if !response.accepted { + return Err(format!("queue rejected job {index}").into()); + } + } + Ok(()) +} +``` + +Assembling and running the scenario is unchanged from any other AppHost run: + +```rust,ignore +let mut scenario = AppHost::scenario() + .with_app(JobStackApp::new()) + .with_run_duration(Duration::from_secs(10)) + .with_workload(EnqueueJobs::new(10)) + .with_expectation(AllJobsCompleted::new(10)) + .build()?; + +let deployer = AppHostLocalDeployer::default(); +let runner = deployer.deploy(&scenario).await?; +runner.run(&mut scenario).await?; +``` + +```bash +cargo test -p multi-app-e2e +``` + +--- + +## Wiring Dependencies Between Components + +**Pass dependencies through constructors.** Deploy the dependency first and pass its handle or address into the dependent's constructor, as `JobWorkerApp::new(queue_url, results_url)` does above. This records the dependency graph and acquisition order in the root deployment. A child can call `ctx.require::()`, but then it depends on another deployment having exposed `T` earlier. If that did not happen, deployment fails at run time with `HandleMissing`. + +The same rule applies to process-level wiring. The job worker is a [`LocalProcessApp`](local-process-app.md) whose `LaunchSpec` receives the queue and store URLs as command-line arguments: deploy the dependency, read its client's `base_url()`, and feed the address into the process. Do not have the process guess. + +**Use named handles for two instances of one type.** The registry allows one default handle per concrete type; a second `expose` of the same type is a duplicate error (see [Handle Ownership and Teardown](handles-teardown.md)). Two kv clusters in one stack therefore need names: + +```rust,ignore +ctx.expose_named("primary", primary)?; +ctx.expose_named("replica", replica)?; + +// in the workload: +let primary = ctx.require_app_named::>("primary")?; +``` + +**Expose components as well as the stack when both are used.** A workload that touches one component can request its handle directly, while stack-level workloads can request the aggregate handle. + +--- + +## See Also + +- [AppDeployment and DeployContext](app-deployment.md): the composition API in detail. +- [Uniform Child Clusters: LocalAppCluster](local-app-cluster.md): the child clusters used here. +- [Backend Scope](app-backend-scope.md): where composed stacks can run today. diff --git a/book/src/crate-map.md b/book/src/crate-map.md new file mode 100644 index 0000000..5473b50 --- /dev/null +++ b/book/src/crate-map.md @@ -0,0 +1,107 @@ +# Crate and API Map + +This chapter maps which crate owns which concept, what each one exports, and how they depend on each other. + +The workspace splits into three layers: the app-agnostic core, the deployment backends, and the cfgsync configuration pipeline. Example applications live in their own workspace layout under `examples/` and depend on the framework, never the other way around. + +```mermaid +graph BT + art[cfgsync-artifacts] + cc[cfgsync-core] --> art + ca[cfgsync-adapter] --> cc + ca --> art + cr[cfgsync-runtime] --> ca + core[testing-framework-core] --> ca + local[testing-framework-runner-local] --> core + compose[testing-framework-runner-compose] --> core + k8s[testing-framework-runner-k8s] --> core + k8s --> cc + k8s --> art + app[testing-framework-app] --> core + app --> local +``` + +--- + +## testing-framework-core + +Path: `testing-framework/core`. The scenario engine and everything app-agnostic: builder, runtime, topology, observation, sources, capabilities. Every other framework crate depends on it. + +| Module | Contents | +|---|---| +| `env` | The `Application` trait (re-exported from `scenario`) | +| `scenario` | `ScenarioBuilder`, `Scenario`, `Workload`, `Expectation`, `RunContext`, `RunHandle`, `Runner`, `Deployer`, `RuntimeExtensionFactory`, `DeploymentPolicy`, cluster provisioning (`ClusterRequest`, `ClusterSource`, `ClusterHandle`, `ClusterProvisioner`), control traits, capability markers, sources, observability inputs | +| `topology` | `DeploymentDescriptor`, `DeploymentProvider`, `FixedDeploymentProvider`, `DeploymentSeed`, `DeploymentPlan`, `TopologyShapeBuilder`, `ClusterTopology`, `NodeCountTopology` | +| `observation` | `Observer`, `SourceProvider`, `StaticSourceProvider`, `SourceProviderFactory`, `ObservationExtensionFactory`, `ObservationRuntime`, `ObservationHandle`, `ObservationConfig` | +| `workloads` | Generic reusable workloads and verbs: `ChaosBuilderExt`, `RestartChaosBuilderExt`, `RandomRestartWorkload`, `NetworkPartitionWorkload` | +| `runtime` | `manual` (the `ManualClusterHandle` interface), `process`, `retry` | +| `cfgsync` | Bridges deployments to the cfgsync pipeline (re-exports `cfgsync-adapter`, rendering output types) | + +Key builder entry points: `ScenarioBuilder::with_deployment`, `::new(provider)`, and the capability-gated variants `with_node_control()` and `with_observability()`. `ObservabilityBuilderExt` and `CoreBuilderExt` live here too. + +--- + +## testing-framework-app + +Path: `testing-framework/app`. The app layer for heterogeneous stacks: singleton processes, extra clusters, or several applications composed into one system. Depends on core plus the local deployer; the app layer is local-only today (see [Backend Scope](app-backend-scope.md)). + +| Export | Role | +|---|---| +| `AppHost`, `AppHostEnv`, `AppHostTopology`, `AppHostScenarioBuilder`, `AppHostLocalDeployer` | Zero-node scenario entry point: `AppHost::scenario().with_app(...)` | +| `AppDeployment`, `AppHandle` | The composition trait and its blanket handle bound | +| `DeployContext` | Deploy children, expose typed/named handles, provision clusters through `deploy_cluster` | +| `AppDeploymentFactory`, `AppScenarioBuilderExt`, `AppRunContextExt` | Builder registration (`with_app`) and workload-side handle lookup (`app`, `require_app`, ...) | +| `LocalProcessApp`, `LocalProcessHandle` | One supervised local process as an app | +| `LocalAppCluster` | Alias for the common `ClusterHandle` used by local child clusters | +| `AppRuntime`, `HandleRegistry`, `AppDeployError` | Runtime handle storage and errors; managed cleanup is kept separately | + +--- + +## Deployment Backends + +Each backend implements `Deployer` for its environment trait and returns the same core `Runner`. + +**`testing-framework-runner-local`** (`testing-framework/deployers/local`) spawns nodes as local processes. Exports `ProcessDeployer`, `ManualCluster`, `NodeManager`, the `LocalDeployerEnv` / `LocalBinaryApp` environment traits with config/port helpers (`LocalProcessSpec`, `LocalNodePorts`, `build_local_cluster_node_config`, ...), process primitives (`LaunchSpec`, `NodeEndpoints`, `ProcessNode`), and the whole `binary` module (`BinaryProvider` and its implementations). Honors `TF_KEEP_LOGS` for tempdir retention. + +**`testing-framework-runner-compose`** (`.../compose`) renders a Docker Compose stack. Exports `ComposeDeployer`, `ComposeDeployEnv`, descriptor builders (`ComposeDescriptor`, `NodeDescriptor`), compose lifecycle commands (`compose_up`, `compose_down`, `dump_compose_logs`), and the Docker config-server support used to serve cfgsync artifacts to containers. + +**`testing-framework-runner-k8s`** (`.../k8s`) installs a Helm release. Exports `K8sDeployer`, `K8sDeployEnv`, `ManualCluster` (K8s variant), Helm/chart-value infrastructure (`HelmInstallSpec`, `RunnerChartValues`, `render_binary_config_node_chart_assets`, ...), and wait/cleanup helpers. Depends directly on `cfgsync-core` and `cfgsync-artifacts` for artifact delivery. + +--- + +## cfgsync + +cfgsync is the typed pipeline that turns app config into per-node files: app config → registration snapshot → per-node artifact sets → backend rendering. Consumed by the compose and k8s deployers (locally, configs are written straight to disk). See [Static Artifacts and cfgsync](cfgsync.md). + +| Crate | Responsibility | Key exports | +|---|---|---| +| `cfgsync-artifacts` | App-agnostic artifact model | `ArtifactFile`, `ArtifactSet` | +| `cfgsync-core` | Protocol, client/server, template rendering, bundles | `Client`, `serve_cfgsync`, `NodeRegistration`, `NodeArtifactsPayload`, `RenderedCfgsync`, `NodeArtifactsBundle`, config sources | +| `cfgsync-adapter` | Materializing registration snapshots into artifacts | `RegistrationSnapshotMaterializer`, `CachedSnapshotMaterializer`, `PersistingSnapshotMaterializer`, `MaterializedArtifacts`, `RegistrationConfigSource` | +| `cfgsync-runtime` | Standalone server/client binaries-facing runtime | `serve_from_config`, `run_client_from_env`, `ServerConfig` | + +--- + +## Examples Workspace Layout + +Every example app follows the same four-part shape under `examples//`: + +```text +examples/kvstore/ +├── kvstore-node/ # the application binary under test +├── testing/ +│ ├── integration/ # crate kvstore-runtime-ext: Application impl, +│ │ # local/compose/k8s env impls, observation +│ └── workloads/ # crate kvstore-runtime-workloads: Workloads + Expectations +└── examples/ # crate kvstore-examples: runnable bins +``` + +The naming is uniform: `-runtime-ext`, `-runtime-workloads`, `-examples`. `nats` and `redis_streams` have no node crate because they run upstream binaries or images. `multi_app` uses an acceptance-suite layout instead: a `job-worker/` binary crate, a `fixture/` crate (`multi-app-fixture`: the stack deployment, handles, workload, and expectation), and an `e2e/` crate (`multi-app-e2e`) whose integration tests drive the fixture. It demonstrates application composition. + +Run any example bin with: + +```bash +cargo run -p kvstore-examples --bin kvstore_basic_convergence +``` + +**Note:** the dependency arrows only ever point from examples toward the framework and from backends toward core. If you find yourself wanting an arrow in the other direction, read [Framework vs Application Boundaries](tf-boundaries.md). The trait-level view of the same surface is in [Public Extension Points](extension-points.md). diff --git a/book/src/custom-workload-example.md b/book/src/custom-workload-example.md deleted file mode 100644 index c6d9439..0000000 --- a/book/src/custom-workload-example.md +++ /dev/null @@ -1,134 +0,0 @@ -# Example: New Workload & Expectation (Rust) - -A minimal, end-to-end illustration of adding a custom workload and matching -expectation. This shows the shape of the traits and where to plug into the -framework; expand the logic to fit your real test. - -## Workload: simple reachability probe - -Key ideas: -- **name**: identifies the workload in logs. -- **expectations**: workloads can bundle defaults so callers don’t forget checks. -- **init**: derive inputs from the generated topology (e.g., pick a target node). -- **start**: drive async activity using the shared `RunContext`. - -```rust,ignore -use async_trait::async_trait; -use testing_framework_core::{ - scenario::{DynError, Expectation, RunContext, RunMetrics, Workload}, - topology::generation::GeneratedTopology, -}; - -pub struct ReachabilityWorkload { - target_idx: usize, -} - -impl ReachabilityWorkload { - pub fn new(target_idx: usize) -> Self { - Self { target_idx } - } -} - -#[async_trait] -impl Workload for ReachabilityWorkload { - fn name(&self) -> &str { - "reachability_workload" - } - - fn expectations(&self) -> Vec> { - vec![Box::new( - crate::custom_workload_example_expectation::ReachabilityExpectation::new( - self.target_idx, - ), - )] - } - - fn init( - &mut self, - topology: &GeneratedTopology, - _run_metrics: &RunMetrics, - ) -> Result<(), DynError> { - if topology.nodes().get(self.target_idx).is_none() { - return Err(Box::new(std::io::Error::new( - std::io::ErrorKind::Other, - "no node at requested index", - ))); - } - Ok(()) - } - - async fn start(&self, ctx: &RunContext) -> Result<(), DynError> { - let client = ctx - .node_clients() - .node_clients() - .get(self.target_idx) - .ok_or_else(|| { - Box::new(std::io::Error::new( - std::io::ErrorKind::Other, - "missing target client", - )) as DynError - })?; - - // Lightweight API call to prove reachability. - client - .consensus_info() - .await - .map(|_| ()) - .map_err(|e| e.into()) - } -} -``` - -## Expectation: confirm the target stayed reachable - -Key ideas: -- **start_capture**: snapshot baseline if needed (not used here). -- **evaluate**: assert the condition after workloads finish. - -```rust,ignore -use async_trait::async_trait; -use testing_framework_core::scenario::{DynError, Expectation, RunContext}; - -pub struct ReachabilityExpectation { - target_idx: usize, -} - -impl ReachabilityExpectation { - pub fn new(target_idx: usize) -> Self { - Self { target_idx } - } -} - -#[async_trait] -impl Expectation for ReachabilityExpectation { - fn name(&self) -> &str { - "target_reachable" - } - - async fn evaluate(&mut self, ctx: &RunContext) -> Result<(), DynError> { - let client = ctx - .node_clients() - .node_clients() - .get(self.target_idx) - .ok_or_else(|| { - Box::new(std::io::Error::new( - std::io::ErrorKind::Other, - "missing target client", - )) as DynError - })?; - - client - .consensus_info() - .await - .map(|_| ()) - .map_err(|e| e.into()) - } -} -``` - -## How to wire it -- Build your scenario as usual and call `.with_workload(ReachabilityWorkload::new(0))`. -- The bundled expectation is attached automatically; you can add more with - `.with_expectation(...)` if needed. -- Keep the logic minimal and fast for smoke tests; grow it into richer probes - for deeper scenarios. diff --git a/book/src/deployer-compose.md b/book/src/deployer-compose.md new file mode 100644 index 0000000..16ea134 --- /dev/null +++ b/book/src/deployer-compose.md @@ -0,0 +1,79 @@ +# Compose Deployer + +`ComposeDeployer` runs each node as a Docker Compose service generated from your deployment descriptor. + +The compose deployer lives in the `testing-framework-runner-compose` crate. It generates a compose file per run, brings the stack up, discovers the host ports Docker assigned, probes readiness, and hands control to the scenario runner. It requires a running Docker daemon; otherwise deployment returns `ComposeRunnerError::DockerUnavailable`. + +```rust,ignore +use kvstore_runtime_ext::KvComposeDeployer; // = ComposeDeployer +use testing_framework_core::scenario::Deployer; +use testing_framework_runner_compose::ComposeRunnerError; + +let deployer = KvComposeDeployer::new(); +let runner = match deployer.deploy(&scenario).await { + Ok(runner) => runner, + Err(ComposeRunnerError::DockerUnavailable) => return Ok(()), // skip without Docker + Err(error) => return Err(error.into()), +}; +runner.run(&mut scenario).await?; +``` + +Run the demonstration binary with `cargo run -p kvstore-examples --bin kvstore_compose_convergence`. + +--- + +## Deployment Pipeline + +```mermaid +flowchart LR + A[Workspace
tempdir] --> B[Write configs
+ cfgsync.yaml] + B --> C[Render
compose.generated.yml] + C --> D[docker compose
create + up] + D --> E[Port discovery
docker compose port] + E --> F[Readiness
probes] + F --> G[Node clients
+ Runner] +``` + +1. **Workspace.** A temporary `ComposeWorkspace` is created; the app's `ComposeDeployEnv::prepare_compose_configs` writes per-node config files (for `ComposeBinaryApp` environments, one static config per node under `stack/configs/`, rewritten for service hostnames `node-0`, `node-1`, ...). +2. **cfgsync.** If the environment enables `ComposeConfigServerMode::Docker`, a cfgsync config server container is started on an ephemeral port and the deployer waits for it to accept TCP connections before proceeding. The default mode is `Disabled`. See [Static Artifacts and cfgsync](cfgsync.md). +3. **Compose file.** The env's `compose_descriptor` (image, entrypoint, volumes, ports, environment, optional platform per service) is rendered through the Tera template at `testing-framework/deployers/compose/assets/docker-compose.yml.tera` into `compose.generated.yml`. The template is resolved relative to the repository root (`CARGO_WORKSPACE_DIR` override respected). Required images are checked with `docker image inspect` up front. The deployer never builds or pulls them; a missing image fails the deploy with `MissingImage`. +4. **Bring-up.** `docker compose create` and `docker compose up` run under a unique project name (`compose-stack-`). On failure, container logs are dumped before cleanup. +5. **Ports.** Container ports map to ephemeral host ports; the deployer resolves each with `docker compose port` and records them as `NodeHostPorts { api, testing }`. The host defaults to `127.0.0.1` and can be overridden with `COMPOSE_RUNNER_HOST`. +6. **Readiness.** Per the env's `ComposeReadinessProbe`: HTTP GET against `Application::node_readiness_path()` on each mapped API port, or raw TCP reachability. Gated by `DeploymentPolicy.readiness_enabled` and the deployer's own `with_readiness(bool)` switch; when disabled, the stack gets a short fixed grace period instead. See [Readiness, Retry, and Artifact Preservation](deployment-policies.md). +7. **Clients.** `build_node_client` runs against the discovered host/port pairs, producing the scenario's typed node clients. + +--- + +## Node Control + +With `with_node_control()` on the builder, the deployer installs a `ComposeNodeControl` handle bound to the generated compose file and project. It supports **restart only**: `restart_node(name)` shells out to `docker compose restart `. Start and stop of individual services are not wired for managed compose scenarios. The openraft_kv failover scenario runs on this backend: `cargo run -p openraft-kv-examples --bin openraft_kv_compose_failover`. + +--- + +## Attaching to an Existing Stack + +The compose deployer fully supports existing-cluster mode. A scenario built with `with_existing_cluster(ExistingCluster::for_compose_project("my-project"))` skips workspace generation entirely: services are discovered from the running project (or taken from `for_compose_services`), each container's labeled API port is inspected, and clients are built through `Application::external_node_client`. In this mode node control gains `stop_node` in addition to `restart_node`, implemented with `docker container stop` / `docker container restart` against discovered container IDs. + +`deploy_with_metadata` returns `ComposeDeploymentMetadata` alongside the runner; its `existing_cluster()` / `IntoExistingCluster` impl lets a later scenario attach to the stack this one deployed. See [Existing and External Clusters](external-clusters.md). + +--- + +## Observability + +Compose resolves `ObservabilityInputs` by merging `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL`, `LOGOS_BLOCKCHAIN_METRICS_OTLP_INGEST_URL`, and `LOGOS_BLOCKCHAIN_GRAFANA_URL` env vars with the scenario's observability capability (capability values win). The OTLP ingest URL is passed into config preparation so node configs can point at your collector; the metrics query URL becomes the run's Prometheus-backed `Metrics` handle. Setting `TESTNET_PRINT_ENDPOINTS` prints Prometheus/Grafana endpoints and per-node pprof profile URLs to stdout. See [Telemetry and External Observability](telemetry.md). + +--- + +## Cleanup + +The runner's cleanup guard runs `docker compose down`, shuts down the cfgsync container if one was started, and removes the workspace. Setting `COMPOSE_RUNNER_PRESERVE` (or `TESTNET_RUNNER_PRESERVE`) keeps the stack running and persists the workspace directory for post-mortem inspection; the preserved path is logged. + +--- + +**Requirements recap:** + +| Requirement | Why | +|---|---| +| Docker daemon running | `ensure_docker_available` gates every deploy | +| Node container images | Must exist locally before deploy; missing images fail with `MissingImage` | +| Repository checkout | The compose Tera template is read from the repo tree | diff --git a/book/src/deployer-k8s.md b/book/src/deployer-k8s.md new file mode 100644 index 0000000..2255377 --- /dev/null +++ b/book/src/deployer-k8s.md @@ -0,0 +1,97 @@ +# Kubernetes Deployer + +`K8sDeployer` installs each scenario as a Helm release in a throwaway namespace and reaches nodes through NodePorts or port-forwards. + +The k8s deployer lives in the `testing-framework-runner-k8s` crate. It talks to whatever cluster your current kubeconfig context points at (`kube::Client::try_default()`), installs a Helm release, waits for the workloads, and builds node clients against externally reachable ports. + +```rust,ignore +use kvstore_runtime_ext::KvK8sDeployer; // = K8sDeployer +use testing_framework_core::scenario::Deployer; +use testing_framework_runner_k8s::K8sRunnerError; + +let deployer = KvK8sDeployer::new(); +let runner = match deployer.deploy(&scenario).await { + Ok(runner) => runner, + Err(K8sRunnerError::ClientInit { .. }) => return Ok(()), // no cluster available + Err(error) => return Err(error.into()), +}; +runner.run(&mut scenario).await?; +``` + +Run the demonstration binary with `cargo run -p kvstore-examples --bin kvstore_k8s_convergence`. + +--- + +## Charts and Values + +The environment trait `K8sDeployEnv` produces installable assets via `prepare_assets`, returning a `PreparedK8sStack`. Two asset shapes exist: + +- **Generated single-template charts.** Apps implementing `K8sBinaryApp` get the standard shape for free: `render_binary_config_node_manifest` renders one ConfigMap (the serialized node config), one Deployment (single replica, `--config` arg, config mounted from the ConfigMap), and one NodePort Service per node, then `render_manifest_chart_assets` wraps them in a minimal chart (`RenderedHelmChartAssets`). +- **Real chart directories.** `NodeRuntimeSpec` builds `RunnerChartValues` (node image, pull policy, fullname override, asset mount layout, node group, optional shared bootstrap service with cfgsync configs and start scripts) and a `HelmReleaseBundle` with `--set` values and `--set-file` entries for start scripts and bootstrap configs. `RunnerAssetLayout` fixes where bootstrap configs and runner scripts land inside the chart's mount path. + +Node images are resolved from env vars: for a conventional `BinaryConfigK8sSpec` the primary override is `_K8S_IMAGE`, the fallback `_IMAGE`, and the default `:local` with `imagePullPolicy: IfNotPresent`. + +Each run installs into fresh identifiers: namespace `tf-testnet--`, release `tf-runner` (override via `K8sDeployEnv::cluster_identifiers`). + +--- + +## Lifecycle Waits + +After `helm install`, the deployer waits in stages: + +1. **Deployment readiness**: each node Deployment must report ready replicas (timeout `K8S_RUNNER_DEPLOYMENT_TIMEOUT_SECS`, default 180 s). +2. **Port discovery**: each node Service must have allocated NodePorts for the API and auxiliary ports declared by `collect_port_specs`. +3. **HTTP readiness**: nodes are probed over their NodePorts at `node_readiness_path()`. The probe host is `K8S_RUNNER_NODE_HOST` if set, else `KUBERNETES_SERVICE_HOST`, else `127.0.0.1`. If NodePort probing fails (common when the cluster's node IPs are not routable from the runner), the deployer transparently falls back to `kubectl port-forward` per service and probes over `127.0.0.1`. +4. **Policy-gated cluster readiness**: a final probe pass controlled by `DeploymentPolicy.readiness_enabled` / `readiness_requirement` and the deployer's `with_readiness(bool)` switch. See [Readiness, Retry, and Artifact Preservation](deployment-policies.md). + +HTTP wait tuning: `K8S_RUNNER_HTTP_TIMEOUT_SECS` (default 240), `K8S_RUNNER_HTTP_PROBE_TIMEOUT_SECS` (default 30), `K8S_RUNNER_HTTP_POLL_INTERVAL_SECS` (default 1). + +--- + +## Node Control + +The Kubernetes deployer does not wire a node-control handle into managed scenario deployments. A scenario built with `with_node_control()` compiles against this backend, but runtime restart calls fail. For node lifecycle control on Kubernetes, use the Kubernetes `ManualCluster` below. + +--- + +## Manual Mode + +`K8sDeployer::manual_cluster_from_descriptors(descriptors)` (or `ManualCluster::from_topology`) installs the same Helm release, discovers every node's ports, then **scales all node Deployments to zero** so your code decides when each node starts: + +```rust,ignore +let deployer = OpenRaftKvK8sDeployer::new(); +let cluster = deployer + .manual_cluster_from_descriptors(OpenRaftKvTopology::new(3)) + .await?; + +cluster.start_node("node-0").await?; +cluster.start_node("node-1").await?; +cluster.wait_network_ready().await?; +cluster.restart_node("node-0").await?; +cluster.stop_all(); +``` + +Start, stop, and restart are implemented by patching Deployment replicas between 0 and 1 and waiting for the rollout. `start_node_with` accepts `StartNodeOptions`, with two k8s-specific limits: `persist_dir` / `snapshot_dir` are rejected, and peer selection or config overrides require the environment to implement cfgsync override artifacts (`cfgsync_service` + `build_cfgsync_override_artifacts`); the override is pushed to the in-cluster cfgsync service through a temporary port-forward before the node starts. The failover demonstration uses this path end to end: `cargo run -p openraft-kv-examples --bin openraft_kv_k8s_failover`. Contrast with the declarative local variant in [ManualCluster: Imperative Node Control](manual-cluster.md). + +--- + +## Attaching to an Existing Cluster + +Existing-cluster mode is supported with a k8s descriptor: `with_existing_cluster(ExistingCluster::for_k8s_selector("app.kubernetes.io/instance=tf-runner"))` (optionally namespaced with `for_k8s_selector_in_namespace`). Services matching the selector are listed, each service's single TCP NodePort (or the port named `http`/`api`) becomes the node endpoint, and clients are built via `Application::external_node_client`. `deploy_with_metadata` returns `K8sDeploymentMetadata` (namespace + label selector) so a later scenario can attach to the stack this one installed. See [Existing and External Clusters](external-clusters.md). + +--- + +## Observability and Cleanup + +Observability inputs resolve exactly as in compose (`LOGOS_BLOCKCHAIN_*` env vars merged with the scenario capability), and `TESTNET_PRINT_ENDPOINTS` prints Prometheus/Grafana and per-node pprof endpoints. Cleanup uninstalls the Helm release and deletes the namespace (Kubernetes API first, `kubectl delete namespace` fallback), after killing any port-forward processes. Set `K8S_RUNNER_PRESERVE` to keep the release and namespace for inspection. + +--- + +**Requirements recap:** + +| Requirement | Why | +|---|---| +| Reachable cluster in current kubeconfig context | `Client::try_default()` at deploy time | +| `helm` on PATH | Release install/uninstall | +| `kubectl` on PATH | Port-forward fallback, namespace-delete fallback | +| Node images loadable by the cluster | `_K8S_IMAGE` / `_IMAGE` / `:local` | diff --git a/book/src/deployer-local.md b/book/src/deployer-local.md new file mode 100644 index 0000000..7d84241 --- /dev/null +++ b/book/src/deployer-local.md @@ -0,0 +1,109 @@ +# Local Deployer + +`ProcessDeployer` runs every node as a local OS process. It is the default backend. + +The local deployer lives in the `testing-framework-runner-local` crate. It requires no Docker daemon and no cluster: it resolves a node binary, writes each node's config into a private working directory, spawns the processes, probes readiness, and hands the running cluster to the scenario runner. + +```rust,ignore +use kvstore_runtime_ext::KvLocalDeployer; // = ProcessDeployer +use testing_framework_core::scenario::Deployer; + +let deployer = KvLocalDeployer::default(); +let runner = deployer.deploy(&scenario).await?; +runner.run(&mut scenario).await?; +``` + +Run the demonstration binary with `cargo run -p kvstore-examples --bin kvstore_basic_convergence`. No manual binary setup is needed, because kvstore's fallback provider chain builds the node binary on first use (see [Binary Providers](binary-providers.md)). + +--- + +## What deploy Does + +For a managed scenario, `ProcessDeployer::deploy`: + +1. Validates the cluster mode: the local deployer rejects `ClusterMode::ExistingCluster` (attach is a compose/k8s feature, see [Existing and External Clusters](external-clusters.md)). +2. Builds the source orchestration plan and spawns one `ProcessNode` per topology entry. +3. Probes readiness and retries the whole spawn on failure (see below). +4. Merges external node clients into the managed set. +5. Assembles the runtime and returns a `Runner` whose cleanup guard owns the node processes. + +The main runtime types are: + +- **`ProcessDeployer`**: the deployer. `E` implements `LocalDeployerEnv` (full-control hooks) or the compact `LocalBinaryApp` trait (one binary + one config file + one HTTP port per node). +- **`LaunchSpec`**: the launch plan for one process: binary path, files to materialize, CLI args, env vars. +- **`ProcessNode`**: a spawned child process plus its tempdir, endpoints, and typed client. + +--- + +## Working Directories and Logs + +Each node gets its own temporary working directory, created under the current directory (or under a caller-supplied persist path). Config files and any other `LaunchFile` entries are written there before spawn, and the process starts with that directory as its cwd. + +Node stdout and stderr are inherited from the test process, so node logs interleave with your test output; control verbosity with the `RUST_LOG` value configured on the app's `LocalProcessSpec` (for example `.with_rust_log("kvstore_node=info")`). + +On drop, each `ProcessNode` kills its child and removes the tempdir. Two things preserve working directories instead of deleting them: + +- `DeploymentPolicy` with `cleanup_policy.preserve_artifacts = true` (see [Readiness, Retry, and Artifact Preservation](deployment-policies.md)), or the `TF_KEEP_LOGS=1` env var. +- A panicking test thread, in which case directories are kept automatically for inspection. + +Nodes started with a `persist_dir` or seeded from a `snapshot_dir` (via `StartNodeOptions`) copy or place state accordingly before spawn (see [Persistence, Snapshots, and Recovery Testing](persistence.md)). + +--- + +## Ports + +The deployer reserves real OS ports up front: `allocate_available_port()` binds an ephemeral listener and releases it, and `reserve_local_node_ports` reserves the network port plus any app-named extra ports for each node. Endpoints are surfaced as `NodeEndpoints` (an API socket address plus named extra ports), from which the app builds its typed `NodeClient`. + +--- + +## Readiness and Retry + +Readiness is governed by the scenario's `DeploymentPolicy` combined with the deployer's own switch: + +| Control | Effect | +|---|---| +| `ProcessDeployer::with_membership_check(false)` | Disables local readiness probing entirely | +| `DeploymentPolicy.readiness_enabled` | Must also be true for probes to run | +| `DeploymentPolicy.readiness_requirement` | `AllNodesReady`, `AnyNodeReady`, or `AtLeast(n)` | +| `DeploymentPolicy.retry_policy` | Attempts and backoff; defaults to 3 attempts, 250 ms base, 2 s max | + +The probe shape comes from the environment: `LocalReadinessProbe::HttpGet { path }` (default, using `Application::node_readiness_path()`) or `LocalReadinessProbe::Tcp`. If spawn or readiness fails, all nodes from that attempt are dropped and the entire cluster is respawned with exponential backoff and jitter, up to the retry budget. + +--- + +## Node Control + +The local deployer supports the complete node-control surface. Building the scenario with `with_node_control()` deploys through `Deployer`, which wraps the spawned nodes in a `NodeManager`. Workloads can then start, stop, and restart nodes by name, with full `StartNodeOptions` support (peer selection, config overrides and patches, persist and snapshot directories, extra args, start timeouts). The openraft_kv failover scenario uses this path: + +```bash +cargo run -p openraft-kv-examples --bin openraft_kv_basic_failover +``` + +See [Scenario Capabilities](capabilities.md) for the capability-gated builder. + +--- + +## Manual Clusters + +For orchestration outside the scenario runner, such as Cucumber steps or another test harness, the deployer provides an imperative cluster: + +```rust,ignore +let deployer = ProcessDeployer::::new(); +let cluster = deployer.manual_cluster_from_descriptors(descriptors); + +cluster.start_node("node-0").await?; +cluster.wait_network_ready().await?; +cluster.stop_all(); +``` + +`ManualCluster` exposes `start_node(_with)`, `stop_node`, `restart_node(_with)`, `wait_node_ready`, `wait_network_ready`, `node_client`, `node_pid`, `node_clients`, and `add_external_sources` / `add_external_clients`. It is covered in depth in [ManualCluster: Imperative Node Control](manual-cluster.md). + +--- + +## Binary Resolution + +Every local node needs an executable. `LocalProcessSpec::new("MY_NODE_BIN")` defaults to an env-var provider; `with_binary_provider` swaps in any `BinaryProvider`, including fallback chains that try an env override first and build with Cargo otherwise. Resolution is cached per process and locked across processes. Full detail in [Binary Providers](binary-providers.md). + +--- + +The local deployer supports external node sources (`with_external_node`) but not attached existing clusters. If `Application::external_node_client` is not implemented, it falls back to parsing the endpoint (`http://host:port`) and building a client from the resolved socket address. diff --git a/book/src/deployment-policies.md b/book/src/deployment-policies.md new file mode 100644 index 0000000..6cf66d8 --- /dev/null +++ b/book/src/deployment-policies.md @@ -0,0 +1,90 @@ +# Readiness, Retry, and Artifact Preservation + +`DeploymentPolicy` is the single policy struct that controls readiness gating, deploy retries, and artifact retention across all deployers. + +--- + +## The Policy + +From `testing-framework-core` (`core/src/scenario/deployment_policy.rs`): + +```rust,ignore +pub struct DeploymentPolicy { + pub readiness_enabled: bool, + pub readiness_requirement: HttpReadinessRequirement, + pub retry_policy: Option, + pub cleanup_policy: CleanupPolicy, +} + +pub struct RetryPolicy { + pub max_attempts: usize, + pub base_delay: Duration, + pub max_delay: Duration, +} + +pub struct CleanupPolicy { + pub preserve_artifacts: bool, +} +``` + +Defaults: `readiness_enabled: true`, `readiness_requirement: HttpReadinessRequirement::AllNodesReady`, `retry_policy: None`, `preserve_artifacts: false`. `HttpReadinessRequirement` is `AllNodesReady`, `AnyNodeReady`, or `AtLeast(usize)`. + +Set it on the builder: + +```rust,ignore +use std::time::Duration; +use testing_framework_core::scenario::{ + CleanupPolicy, DeploymentPolicy, HttpReadinessRequirement, RetryPolicy, +}; + +let scenario = KvScenarioBuilder::deployment_with(|_| KvTopology::new(3)) + .with_deployment_policy(DeploymentPolicy { + readiness_enabled: true, + readiness_requirement: HttpReadinessRequirement::AtLeast(2), + retry_policy: Some(RetryPolicy::new( + 5, + Duration::from_millis(500), + Duration::from_secs(5), + )), + cleanup_policy: CleanupPolicy::new(true), + }) + .build()?; +``` + +To adjust only the requirement, `with_http_readiness_requirement(...)` is the shortcut. + +--- + +## Readiness + +`readiness_enabled` and `readiness_requirement` gate the post-spawn probe pass in every deployer. Each backend also has its own deployer-level switch that must agree (`ProcessDeployer::with_membership_check(bool)`, `ComposeDeployer::with_readiness(bool)`, `K8sDeployer::with_readiness(bool)`), so effective readiness is `deployer switch && policy.readiness_enabled`. The probe shape (HTTP path vs TCP) comes from the application environment; see the per-deployer chapters ([Local](deployer-local.md), [Compose](deployer-compose.md), [K8s](deployer-k8s.md)). + +--- + +## Retry + +`retry_policy` drives the local deployer's spawn-and-readiness loop: on failure, all nodes from the attempt are dropped and the cluster is respawned with exponential backoff (from `base_delay`, capped at `max_delay`, with jitter) up to `max_attempts`. When `retry_policy` is `None`, the local deployer falls back to its built-in default of 3 attempts, 250 ms base delay, 2 s max delay. + +The Compose and Kubernetes deployers currently honor the readiness fields of the policy but do not repeat deployment on failure; `retry_policy` has no effect on those backends today. + +--- + +## Artifact Preservation + +`cleanup_policy.preserve_artifacts` controls **artifact and tempdir retention, not teardown ordering**. Teardown itself always follows the runner's cleanup-guard chain (see [Handle Ownership and Teardown](handles-teardown.md)); this flag only decides whether per-node working directories survive it. + +The local orchestrator computes retention as: + +```rust,ignore +policy.cleanup_policy.preserve_artifacts || keep_tempdir_from_env() // TF_KEEP_LOGS +``` + +so either the policy flag or `TF_KEEP_LOGS=1` (also `true`/`yes`) keeps every node's working directory (configs, on-disk state, anything the process wrote) after the run. Panicking tests preserve working directories regardless. + +The container deployers preserve through env vars rather than the policy: `COMPOSE_RUNNER_PRESERVE` / `TESTNET_RUNNER_PRESERVE` keep the compose stack and workspace, `K8S_RUNNER_PRESERVE` keeps the Helm release and namespace. See [Diagnostics and Retained Artifacts](diagnostics.md). + +| Backend | Policy `preserve_artifacts` | Env var | +|---|---|---| +| Local | Yes — keeps node tempdirs | `TF_KEEP_LOGS` | +| Compose | No effect | `COMPOSE_RUNNER_PRESERVE` / `TESTNET_RUNNER_PRESERVE` | +| K8s | No effect | `K8S_RUNNER_PRESERVE` | diff --git a/book/src/design-rationale.md b/book/src/design-rationale.md deleted file mode 100644 index 94961b6..0000000 --- a/book/src/design-rationale.md +++ /dev/null @@ -1,7 +0,0 @@ -# Design Rationale - -- **Modular crates** keep configuration, orchestration, workloads, and runners decoupled so each can evolve without breaking the others. -- **Pluggable runners** let the same scenario run on a laptop, a Docker host, or a Kubernetes cluster, making validation portable across environments. -- **Separated workloads and expectations** clarify intent: what traffic to generate versus how to judge success. This simplifies review and reuse. -- **Declarative topology** makes cluster shape explicit and repeatable, reducing surprise when moving between CI and developer machines. -- **Maintainability through predictability**: a clear flow from plan to deployment to verification lowers the cost of extending the framework and interpreting failures. diff --git a/book/src/diagnostics.md b/book/src/diagnostics.md new file mode 100644 index 0000000..a429f46 --- /dev/null +++ b/book/src/diagnostics.md @@ -0,0 +1,76 @@ +# Diagnostics and Retained Artifacts + +This chapter explains where node output and generated files go, how to keep them after a run, and how to turn a failed run into a diagnosis. + +--- + +## Where Output Goes + +**Process output.** Local node processes are spawned with inherited stdout/stderr (`testing-framework/deployers/local/src/process.rs`). Node logs interleave with your test's own output on the terminal; they are not redirected to files by the framework. Control node verbosity with the process env, e.g. `LocalProcessSpec::with_rust_log("my_node=debug")` or `with_env("RUST_LOG", ...)`. + +**Files.** Every local node runs inside its own temporary working directory, created per node per run **under the current working directory of the test process** (a `TempDir` with a random `.tmp*` name). The directory contains: + +- the materialized launch files: for the standard spec, the rendered config (`config.yaml` by default, `LocalProcessSpec::with_config_file` to change it) plus any extra `LaunchFile` entries; +- anything the node itself writes, since the process is spawned with the directory as its `current_dir` (databases, application logs, snapshots); +- state seeded before start: `with_snapshot_dir(path)` copies a snapshot into the directory before spawn. + +A typical kvstore node directory looks like: + +```text +.tmpAbC123/ +├── config.yaml # rendered by the framework before spawn +└── data/ # whatever the node itself created +``` + +If a persistent location was requested (`with_persist_dir(path)` on `LocalProcessApp`, or `persist_dir` in `StartNodeOptions`), the directory is instead created next to `path` with a `_` prefix, so restarts and recovery tests can find it; see [Persistence, Snapshots, and Recovery Testing](persistence.md). + +Compose runs render their whole stack (compose file, per-node configs under `stack/configs/`, cfgsync artifacts) into a `compose-stack-*` workspace in the system temp directory. Kubernetes runs render Helm charts into temporary chart directories and install them into a per-run namespace. + +--- + +## Keeping Artifacts + +By default all of the above is deleted at teardown. Three mechanisms retain it: + +| Mechanism | Scope | How | +|---|---|---| +| `CleanupPolicy` | one scenario | `with_deployment_policy(DeploymentPolicy { cleanup_policy: CleanupPolicy::new(true), .. })` | +| `keep_tempdir` | one process | `LocalProcessApp::keep_tempdir(true)` at build time, or `handle.keep_tempdir().await` at run time | +| `TF_KEEP_LOGS` | whole process | env var, no code change | + +The local orchestrator preserves node directories when **either** the policy or the env var asks for it: `policy.cleanup_policy.preserve_artifacts || keep_tempdir_from_env()`. `TF_KEEP_LOGS` accepts `1`, `true`, or `yes` (case-insensitive) and is also honored by `ManualCluster` node starts. See [Readiness, Retry, and Artifact Preservation](deployment-policies.md) for the full policy type. + +A panicking test thread preserves its node working directories automatically (`thread::panicking()` is checked in the process drop path), so a failed assertion usually leaves the directory behind without an additional flag. + +The container backends have their own preserve switches: `COMPOSE_RUNNER_PRESERVE` (or `TESTNET_RUNNER_PRESERVE`) keeps the compose workspace and skips `docker compose down`; `K8S_RUNNER_PRESERVE` skips Helm uninstall and namespace deletion; `K8S_RUNNER_DEBUG` additionally logs Helm install output. All are listed in [Environment Variables](environment-variables.md). + +--- + +## Teardown Ordering (What Preservation Does Not Change) + +Preservation only controls file deletion; it does not change stop order. At the end of a run the runner executes its cleanup guards. App-layer managed resources form one LIFO guard stack within that chain, so dependants acquired later stop before their dependencies. Handle-registry release is separate and does not own process or cluster lifetime. Details in [Handle Ownership and Teardown](handles-teardown.md). + +```mermaid +flowchart LR + A[run ends] --> B[cleanup guard chain] + B --> C[app cleanup stack
reverse acquisition order] + C --> D{preserve?} + D -- no --> E[tempdirs deleted] + D -- yes --> F[tempdirs kept on disk] +``` + +--- + +## Post-Mortem Workflow + +1. **Reproduce with preservation.** Re-run the failing binary or test with `TF_KEEP_LOGS=1` (plus `COMPOSE_RUNNER_PRESERVE=1` for compose). On panic, artifacts are often already there from the first failure. + +2. **Locate the directories.** Local node dirs are the `.tmp*` entries under the directory you launched from (named `_*` when a persist dir was set). The `working_dir()` accessor on `LocalProcessHandle` and the spawn-time log lines give exact paths. + +3. **Inspect configs first.** Many deploy-time failures come from configuration. Check the rendered `config.yaml` for the ports, peer lists, and paths the framework generated. For Compose, diff the rendered files under the preserved workspace's `stack/` directory; for Kubernetes, re-run with `K8S_RUNNER_DEBUG=1` to see Helm output. + +4. **Read the node's own output.** Scroll the interleaved terminal output for the failing node's log lines, or raise its `RUST_LOG` and re-run. Anything the node writes to files is in its working directory. + +5. **Re-run deterministically.** If the deployment was generated from a seed, replay it with the same one via `with_deployment_seed` so the topology and generated identities match the failing run exactly; see [Seeds and Reproducibility](seeds.md). Combined with preserved state and `with_snapshot_dir`, you can restart a node from the exact bytes it crashed with. + +6. **Use imperative control when needed.** To inspect the cluster interactively, start one node at a time, or restart with modified options, rebuild the situation with [ManualCluster](manual-cluster.md). It uses the same working-directory and `TF_KEEP_LOGS` behavior. diff --git a/book/src/dsl-cheat-sheet.md b/book/src/dsl-cheat-sheet.md deleted file mode 100644 index 85262b3..0000000 --- a/book/src/dsl-cheat-sheet.md +++ /dev/null @@ -1,188 +0,0 @@ -# Builder API Quick Reference - -Quick reference for the scenario builder DSL. All methods are chainable. - -## Imports - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_compose::ComposeDeployer; -use testing_framework_runner_k8s::K8sDeployer; -use testing_framework_runner_local::LocalDeployer; -use testing_framework_workflows::{ChaosBuilderExt, ScenarioBuilderExt}; -``` - -## Topology - -```rust,ignore -use testing_framework_core::scenario::{Builder, ScenarioBuilder}; - -pub fn topology() -> Builder<()> { - ScenarioBuilder::topology_with(|t| { - t.network_star() // Star topology (all connect to seed node) - .nodes(3) // Number of nodes - }) -} -``` - -## Wallets - -```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn wallets_plan() -> testing_framework_core::scenario::Scenario<()> { - ScenarioBuilder::topology_with(|t| t.network_star().nodes(1)) - .wallets(50) // Seed 50 funded wallet accounts - .build() -} -``` - -## Transaction Workload - -```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn transactions_plan() -> testing_framework_core::scenario::Scenario<()> { - ScenarioBuilder::topology_with(|t| t.network_star().nodes(1)) - .wallets(50) - .transactions_with(|txs| { - txs.rate(5) // 5 transactions per block - .users(20) // Use 20 of the seeded wallets - }) // Finish transaction workload config - .build() -} -``` - -## Chaos Workload (Requires `enable_node_control()`) - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::{NodeControlCapability, ScenarioBuilder}; -use testing_framework_workflows::{ChaosBuilderExt, ScenarioBuilderExt}; - -pub fn chaos_plan() -> testing_framework_core::scenario::Scenario { - ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .enable_node_control() // Enable node control capability - .chaos_with(|c| { - c.restart() // Random restart chaos - .min_delay(Duration::from_secs(30)) // Min time between restarts - .max_delay(Duration::from_secs(60)) // Max time between restarts - .target_cooldown(Duration::from_secs(45)) // Cooldown after restart - .apply() // Required for chaos configuration - }) - .build() -} -``` - -## Expectations - -```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn expectations_plan() -> testing_framework_core::scenario::Scenario<()> { - ScenarioBuilder::topology_with(|t| t.network_star().nodes(1)) - .expect_consensus_liveness() // Assert blocks are produced continuously - .build() -} -``` - -## Run Duration - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn run_duration_plan() -> testing_framework_core::scenario::Scenario<()> { - ScenarioBuilder::topology_with(|t| t.network_star().nodes(1)) - .with_run_duration(Duration::from_secs(120)) // Run for 120 seconds - .build() -} -``` - -## Build - -```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn build_plan() -> testing_framework_core::scenario::Scenario<()> { - ScenarioBuilder::topology_with(|t| t.network_star().nodes(1)).build() // Construct the final Scenario -} -``` - -## Deployers - -```rust,ignore -use testing_framework_runner_compose::ComposeDeployer; -use testing_framework_runner_k8s::K8sDeployer; -use testing_framework_runner_local::LocalDeployer; - -pub fn deployers() { - // Local processes - let _deployer = LocalDeployer::default(); - - // Docker Compose - let _deployer = ComposeDeployer::default(); - - // Kubernetes - let _deployer = K8sDeployer::default(); -} -``` - -## Execution - -```rust,ignore -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_local::LocalDeployer; -use testing_framework_workflows::ScenarioBuilderExt; - -pub async fn execution() -> Result<()> { - let mut plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(1)) - .expect_consensus_liveness() - .build(); - - let deployer = LocalDeployer::default(); - let runner = deployer.deploy(&plan).await?; - let _handle = runner.run(&mut plan).await?; - - Ok(()) -} -``` - -## Complete Example - -```rust,ignore -use std::time::Duration; - -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_local::LocalDeployer; -use testing_framework_workflows::ScenarioBuilderExt; - -pub async fn run_test() -> Result<()> { - let mut plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .wallets(50) - .transactions_with(|txs| { - txs.rate(5) // 5 transactions per block - .users(20) - }) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(90)) - .build(); - - let deployer = LocalDeployer::default(); - let runner = deployer.deploy(&plan).await?; - let _handle = runner.run(&mut plan).await?; - - Ok(()) -} -``` diff --git a/book/src/entry-patterns.md b/book/src/entry-patterns.md new file mode 100644 index 0000000..fe15288 --- /dev/null +++ b/book/src/entry-patterns.md @@ -0,0 +1,95 @@ +# Choosing an Entry Pattern + +The framework supports three scenario-based entry patterns and one imperative entry pattern. This chapter compares them. + +--- + +## The Four Patterns + +**1. Uniform managed cluster.** Your system is N identical nodes of one application. Implement `Application` (and the deployer-specific traits), describe a topology, and build a `ScenarioBuilder` over it. The framework spawns, gates, and tears down every node. + +```rust,ignore +let mut scenario = KvScenarioBuilder::deployment_with(|t| t) // 3-node default topology + .with_run_duration(Duration::from_secs(30)) + .with_workload(KvWriteWorkload::new().operations(300)) + .with_expectation(KvConverges::new("demo", 30)) + .build()?; + +let runner = KvLocalDeployer::default().deploy(&scenario).await?; +runner.run(&mut scenario).await?; +``` + +(The shipped `kvstore_basic_convergence` binary wraps the same topology in the `with_existing_kvstore_app` convenience preset; that hybrid is covered in [AppHost and with_app](app-host.md).) + +**2. Composed application stack.** Your system is heterogeneous: several clusters, singleton processes, or both. Start from `AppHost::scenario()` (a zero-node `ScenarioBuilder`) and register one root `AppDeployment` with `.with_app(...)`. The deployment composes children through `DeployContext` and exposes typed handles that workloads retrieve with `AppRunContextExt`. + +```rust,ignore +let mut scenario = AppHost::scenario() + .with_app(JobStackApp::new()) // queue cluster + worker + result store + .with_run_duration(Duration::from_secs(10)) + .with_workload(EnqueueJobs::new(10)) + .with_expectation(AllJobsCompleted::new(10)) + .build()?; + +let runner = AppHostLocalDeployer::default().deploy(&scenario).await?; +runner.run(&mut scenario).await?; +``` + +A scenario accepts one `with_app` registration. A second registration fails at prepare time with a duplicate-runtime-extension error; compose multiple apps inside one root deployment instead ([Composing Heterogeneous Stacks](composing-stacks.md)). + +**3. Attached and external nodes.** The system already runs somewhere else: a staging network, a long-lived cluster, another team's deployment. You plug it in as a source instead of deploying it: `with_existing_cluster(...)` / `with_existing_cluster_from(...)` attach a cluster description, `with_external_node(...)` / `with_external_nodes(...)` add endpoint-only nodes, and `with_external_only_nodes(...)` declares a scenario with no framework-managed nodes at all. `Application::external_node_client` turns each `ExternalNodeSource` into a typed client. Workloads and expectations are unchanged. See [Existing and External Clusters](external-clusters.md). + +**4. ManualCluster: imperative control.** Your code decides when nodes start, stop, and restart, step by step. `ManualCluster::from_topology(descriptors)` (or `ProcessDeployer::manual_cluster_from_descriptors`) gives you `start_node`, `start_node_with(StartNodeOptions)`, `stop_node`, `restart_node`, `wait_network_ready`, `wait_node_ready`, and `node_client`, but no workloads, no expectations, no runner. See [ManualCluster: Imperative Node Control](manual-cluster.md). + +**Note:** needing to restart nodes does *not* push you to ManualCluster. Declarative scenarios gain restart-capable workloads via `with_node_control()` on the builder ([Scenario Capabilities](capabilities.md)), and app-layer child clusters expose `restart_node` on their handles. + +--- + +## One Runtime, Three Declarative Patterns + +```mermaid +flowchart TD + U["Uniform cluster
ScenarioBuilder::with_deployment"] --> S["Scenario"] + A["Composed stack
AppHost::scenario().with_app(...)"] --> S + X["Attached / external
with_existing_cluster,
with_external_nodes"] --> S + S --> R["Deployer::deploy → Runner::run
(one lifecycle, see Scenario Model)"] + M["ManualCluster
managed nodes, you drive"] -.->|"bypasses the runner"| C["imperative node control"] + S:::sc + R:::sc + classDef sc stroke:#9b6dd6,stroke-width:2.5px; +``` + +All three declarative patterns produce a `Scenario` and use the same [lifecycle](scenario-model.md), so workloads and expectations can be reused across them when their required clients and capabilities are available. `ManualCluster` uses the node-startup implementation without the scenario runtime. Its nodes remain framework-managed while test code controls the sequence. + +--- + +## Decision Table + +| Shape of the system under test | Pattern | Read next | +|---|---|---| +| N identical nodes of one binary, framework-managed | Uniform managed cluster | [Part IV](part-iv.md) | +| Several apps or clusters composed into one stack | `AppHost` + `with_app` | [Part II](part-ii.md) | +| Already-running nodes you must not deploy | Attached / external sources | [Part V](part-v.md) | +| An external driver dictates every step | `ManualCluster` | [ManualCluster](manual-cluster.md) | + +--- + +## Choosing by Example + +**"Three kvstore nodes, write traffic, convergence check."** Uniform managed cluster. `KvEnv` already models the node; the framework owns the whole population. This is `cargo run -p kvstore-examples --bin kvstore_basic_convergence`. + +**"A queue cluster, a worker process, and a result-store cluster forming one pipeline."** Composed stack. One root `AppDeployment` deploys both clusters, wires the worker to them by URL, and exposes each handle plus a stack handle; a workload enqueues jobs and an expectation verifies the results. The `multi-app-e2e` acceptance test covers this shape; run it with `cargo test -p multi-app-e2e`. + +**"Run our smoke workload against the live staging network."** Attach. There is nothing to deploy: declare the endpoints with `with_external_only_nodes`, let `external_node_client` build clients, and keep the exact same workloads and expectations you use locally. + +**"A Gherkin suite where each step starts or kills a node."** ManualCluster. The BDD runner owns sequencing, and its steps call `start_node_with`, `stop_node`, and `wait_node_ready` directly. + +> **External example:** logos-blockchain's cucumber suite is a real example of the fourth pattern: Gherkin steps drive `ManualCluster` for dependency-ordered starts, restarts, and snapshot/restore flows, all in its own repository. + +--- + +## Where to Go Next + +- [Scenario Model and Lifecycle](scenario-model.md): the runtime every declarative pattern converges on. +- [Application, AppDeployment, and Environments](application-model.md): the types behind patterns 1 and 2. +- [Ownership and Design Boundaries](boundaries.md): what stays yours regardless of pattern. diff --git a/book/src/environment-variables.md b/book/src/environment-variables.md index 6896261..236f609 100644 --- a/book/src/environment-variables.md +++ b/book/src/environment-variables.md @@ -1,375 +1,104 @@ -# Environment Variables Reference +# Environment Variables -Complete reference of environment variables used by the testing framework, organized by category. +This chapter is the complete, audited list of environment variables the framework reads, and where each read happens. + +This chapter was produced by auditing the source (`grep -rn "env::var" testing-framework/ cfgsync/ --include="*.rs"`), not by convention. If a variable is not listed here, the framework does not read it. Re-run the grep after upgrading. --- -## Runner Selection & Topology +## Core (`testing-framework-core`) -Control which runner to use and the test topology: +| Variable | Purpose | Read in | When unset | +|---|---|---|---| +| `SLOW_TEST_ENV` | When exactly `true`, `adjust_timeout` doubles framework timeouts (slow CI runners) | `core/src/lib.rs` | normal timeouts | +| `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` | Prometheus-compatible query endpoint for `ObservabilityInputs::from_env` | `core/src/scenario/observability.rs` | metrics queries disabled (`Metrics::empty()`) | +| `LOGOS_BLOCKCHAIN_METRICS_OTLP_INGEST_URL` | OTLP metrics ingest endpoint | `core/src/scenario/observability.rs` | none | +| `LOGOS_BLOCKCHAIN_GRAFANA_URL` | Grafana base URL surfaced alongside run output | `core/src/scenario/observability.rs` | none | -| Variable | Default | Effect | -|----------|---------|--------| -| `LOGOS_BLOCKCHAIN_DEMO_NODES` | 1 | Number of nodes (all runners) | -| `LOGOS_BLOCKCHAIN_DEMO_RUN_SECS` | 60 | Run duration in seconds (all runners) | -| `LOCAL_DEMO_NODES` | — | Legacy: Number of nodes (host runner only) | -| `LOCAL_DEMO_RUN_SECS` | — | Legacy: Run duration (host runner only) | -| `COMPOSE_NODE_PAIRS` | — | Compose-specific topology format: "nodes" (e.g., `3`) | - -**Example:** - -```bash -# Run with 5 nodes, for 120 seconds -LOGOS_BLOCKCHAIN_DEMO_NODES=5 \ -LOGOS_BLOCKCHAIN_DEMO_RUN_SECS=120 \ -scripts/run/run-examples.sh -t 120 -n 5 host -``` +The three `LOGOS_BLOCKCHAIN_*` names are historical; they are only consulted when telemetry inputs come from the environment rather than from an `ObservabilityCapability`; see [Telemetry and External Observability](telemetry.md). --- -## Node Binaries (Host Runner) +## Local Deployer (`testing-framework-runner-local`) -Required for host runner when not using helper scripts: +| Variable | Purpose | Read in | When unset | +|---|---|---|---| +| `TF_KEEP_LOGS` | Preserve per-node working directories (`1`/`true`/`yes`) | `deployers/local/src/lib.rs`, honored by the orchestrator and `ManualCluster` | directories deleted at teardown (unless the deployment policy preserves them) | -| Variable | Required | Default | Effect | -|----------|----------|---------|--------| -| `LOGOS_BLOCKCHAIN_NODE_BIN` | Yes (host) | — | Path to `logos-blockchain-node` binary | -| `LOGOS_BLOCKCHAIN_NODE_PATH` | No | — | Path to logos-blockchain-node git checkout (dev workflow) | +Two provider types read **caller-named** variables, where the framework defines the mechanism and the application names the variable: -**Example:** +- `EnvBinaryProvider::new("MY_NODE_BIN")` reads that variable as an explicit executable path. Unset or not-a-file counts as unresolved, letting a `FallbackBinaryProvider` continue to the next provider. +- `DownloadUrl::Env(var)` / `DownloadChecksum::Env(var)` on `DownloadBinaryProvider` read the download URL and expected SHA-256 from the named variables. A missing URL variable is a hard error (`MissingDownloadUrl`); a missing checksum variable disables verification. -```bash -export LOGOS_BLOCKCHAIN_NODE_BIN=/path/to/logos-blockchain-node/target/release/logos-blockchain-node -``` +See [Binary Providers](binary-providers.md). --- -## Docker Images (Compose / K8s) +## Compose Deployer (`testing-framework-runner-compose`) -Required for compose and k8s runners: +| Variable | Purpose | Read in | When unset | +|---|---|---|---| +| `COMPOSE_RUNNER_PRESERVE` | Skip `docker compose down`, keep the workspace | `lifecycle/cleanup.rs` | full teardown | +| `TESTNET_RUNNER_PRESERVE` | Alias for the above | `lifecycle/cleanup.rs` | full teardown | +| `COMPOSE_RUNNER_HOST` | Host used to reach published container ports | `infrastructure/ports.rs` | `127.0.0.1` | +| `COMPOSE_RUNNER_HOST_GATEWAY` | Explicit `extra_hosts` gateway entry; `disable` or empty removes it | `docker/platform.rs` | falls through to `DOCKER_HOST_GATEWAY` | +| `DOCKER_HOST_GATEWAY` | Gateway IP mapped as `host.docker.internal:` | `docker/platform.rs` | `host.docker.internal:host-gateway` | +| `TESTNET_PRINT_ENDPOINTS` | If set (any value), print discovered endpoints after deploy | `deployer/orchestrator.rs` | silent | +| `REPO_ROOT_OVERRIDE_DIR` | Override repository-root detection for stack assets | `docker/workspace.rs` | falls through to `CARGO_WORKSPACE_DIR`, then manifest-relative detection | +| `CARGO_WORKSPACE_DIR` | Workspace root override (also used by template rendering) | `docker/workspace.rs`, `infrastructure/template.rs` | manifest-relative detection | +| `REL_ASSETS_STACK_DIR` | Alternative stack-assets directory (absolute, or relative to repo root) | `docker/workspace.rs` | bundled default assets | -| Variable | Required | Default | Effect | -|----------|----------|---------|--------| -| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | Yes (compose/k8s) | `logos-blockchain-testing:local` | Docker image tag for node containers | -| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE_PULL_POLICY` | No | `IfNotPresent` (local) / `Always` (ECR) | K8s `imagePullPolicy` used by the runner | -| `LOGOS_BLOCKCHAIN_BINARIES_TAR` | No | — | Path to prebuilt bundle (`.tar.gz`) for image build | -| `LOGOS_BLOCKCHAIN_SKIP_IMAGE_BUILD` | No | 0 | Skip image rebuild (compose/k8s); assumes image already exists | -| `LOGOS_BLOCKCHAIN_FORCE_IMAGE_BUILD` | No | 0 | Force rebuilding the image even when the script would normally skip it (e.g. non-local k8s) | - -**Example:** - -```bash -# Using prebuilt bundle -export LOGOS_BLOCKCHAIN_BINARIES_TAR=.tmp/nomos-binaries-linux-v0.3.1.tar.gz -export LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local -scripts/build/build_test_image.sh - -# Using pre-existing image (skip build) -export LOGOS_BLOCKCHAIN_SKIP_IMAGE_BUILD=1 -scripts/run/run-examples.sh -t 60 -n 3 compose -``` +Per-application image selection is again a mechanism with caller-derived names: `BinaryConfigNodeSpec::conventional("/usr/local/bin/kvstore-node", ...)` derives the prefix `KVSTORE` and reads `KVSTORE_IMAGE` (default `kvstore-node:local`) and `KVSTORE_PLATFORM` (`descriptor/node.rs`). --- -## Circuit Assets +## Kubernetes Deployer (`testing-framework-runner-k8s`) -Circuit asset configuration used by local runs and image builds: +| Variable | Purpose | Read in | When unset | +|---|---|---|---| +| `K8S_RUNNER_NODE_HOST` | Host used to reach NodePort services | `host.rs` | `KUBERNETES_SERVICE_HOST`, then `127.0.0.1` | +| `KUBERNETES_SERVICE_HOST` | Standard fallback for the above (e.g. Docker Desktop) | `host.rs` | `127.0.0.1` | +| `K8S_RUNNER_PRESERVE` | Skip Helm uninstall and namespace deletion | `env.rs` | full teardown | +| `K8S_RUNNER_DEBUG` | Log Helm install stdout/stderr | `infrastructure/helm.rs` | Helm output suppressed | +| `K8S_RUNNER_DEPLOYMENT_TIMEOUT_SECS` | Deployment readiness timeout (integer seconds) | `lifecycle/wait/mod.rs` | built-in default | +| `K8S_RUNNER_HTTP_TIMEOUT_SECS` | Node HTTP readiness timeout | `lifecycle/wait/mod.rs` | built-in default | +| `K8S_RUNNER_HTTP_PROBE_TIMEOUT_SECS` | Per-probe HTTP timeout | `lifecycle/wait/mod.rs` | built-in default | +| `K8S_RUNNER_HTTP_POLL_INTERVAL_SECS` | Readiness poll interval | `lifecycle/wait/mod.rs` | built-in default | +| `TESTNET_PRINT_ENDPOINTS` | If set, print Prometheus/Grafana/pprof endpoints after deploy | `deployer/orchestrator.rs` | silent | -| Variable | Default | Effect | -|----------|---------|--------| -| `LOGOS_BLOCKCHAIN_CIRCUITS` | `~/.logos-blockchain-circuits` | Directory containing circuit assets | -| `VERSION` | From `versions.env` | Circuit release tag (used by helper scripts) | -| `LOGOS_BLOCKCHAIN_CIRCUITS_VERSION` | — | Legacy alias for `VERSION` (supported by some build scripts) | -| `LOGOS_BLOCKCHAIN_CIRCUITS_PLATFORM` | Auto-detected | Override circuits platform (e.g. `linux-x86_64`, `macos-aarch64`) | -| `LOGOS_BLOCKCHAIN_CIRCUITS_HOST_DIR_REL` | `.tmp/logos-blockchain-circuits-host` | Output dir for host circuit bundle (relative to repo root) | -| `LOGOS_BLOCKCHAIN_CIRCUITS_LINUX_DIR_REL` | `.tmp/logos-blockchain-circuits-linux` | Output dir for linux circuit bundle (relative to repo root) | -| `LOGOS_BLOCKCHAIN_CIRCUITS_NONINTERACTIVE` | 0 | Set to `1` to overwrite outputs without prompting in setup scripts | -| `LOGOS_BLOCKCHAIN_CIRCUITS_REBUILD_RAPIDSNARK` | 0 | Set to `1` to force rebuilding rapidsnark (host bundle only) | - -**Example:** - -```bash -# Use custom circuit assets -LOGOS_BLOCKCHAIN_CIRCUITS=/custom/path/to/circuits \ -cargo run -p runner-examples --bin local_runner -``` +Image selection mirrors compose with a k8s-specific override first: `BinaryConfigK8sSpec::conventional` reads `_K8S_IMAGE`, then `_IMAGE`, then the `:local` default (`env.rs`). `workspace.rs` additionally exposes `resolve_workspace_root` / `resolve_optional_relative_dir` helpers that read a variable **named by the caller**. --- -## Node Logging +## cfgsync Runtime (`cfgsync-runtime`) -Control node log output (not framework runner logs): +These are read by the cfgsync **client inside node containers** at startup, not by your test process; the deployers set them when rendering the stack. See [Static Artifacts and cfgsync](cfgsync.md). -| Variable | Default | Effect | -|----------|---------|--------| -| `LOGOS_BLOCKCHAIN_LOG_LEVEL` | `info` | Global log level: `error`, `warn`, `info`, `debug`, `trace` | -| `LOGOS_BLOCKCHAIN_LOG_FILTER` | — | Fine-grained module filtering (e.g., `cryptarchia=trace`) | -| `LOGOS_BLOCKCHAIN_LOG_DIR` | — | Host runner: directory for per-node log files (persistent). Compose/k8s: use `cfgsync.yaml` for file logging. | -| `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS` | 0 | Keep per-run temporary directories (useful for debugging/CI artifacts) | -| `LOGOS_BLOCKCHAIN_TESTS_TRACING` | false | Enable debug tracing preset (combine with `LOGOS_BLOCKCHAIN_LOG_DIR` unless external tracing backends configured) | - -**Important:** Node logging ignores `RUST_LOG`; use `LOGOS_BLOCKCHAIN_LOG_LEVEL` and `LOGOS_BLOCKCHAIN_LOG_FILTER` for node logs. - -**Example:** - -```bash -# Debug logging to files -LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \ -LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \ -LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \ -cargo run -p runner-examples --bin local_runner - -# Inspect logs -ls /tmp/test-logs/ -# logos-blockchain-node-0.2024-12-18T14-30-00.log -# logos-blockchain-node-1.2024-12-18T14-30-00.log -``` - -**Common filter targets:** - -| Target Prefix | Subsystem | -|---------------------------|-----------| -| `lb_cryptarchia` | Consensus (Cryptarchia) | -| `lb_blend` | Mix network/privacy layer | -| `lb_chain_service` | Chain service (node APIs/state) | -| `lb_chain_network` | P2P networking | -| `lb_chain_leader_service` | Leader election | +| Variable | Purpose | When unset | +|---|---|---| +| `CFG_SERVER_ADDR` | cfgsync server URL | `http://127.0.0.1:` | +| `CFG_HOST_IP` | This node's IPv4 address for registration | `127.0.0.1` | +| `CFG_HOST_IDENTIFIER` | Node identifier for registration | `unidentified-node` | +| `CFG_REGISTRATION_METADATA_JSON` | Extra registration payload (JSON) | empty payload | +| `CFG_FILE_PATH` | Where to write the fetched `config.yaml` | config output not routed | +| `CFG_DEPLOYMENT_PATH` | Where to write the fetched deployment settings | deployment output not routed | +| `LOGOS_BLOCKCHAIN_CFGSYNC_PORT` | Default server port for the `cfgsync-client` binary | `4400` | --- -## Observability & Metrics +## Example-App Variables (Not Framework Variables) -Optional observability integration: +The example applications define their own variables through the mechanisms above. **These belong to the examples**: `KVSTORE_NODE_BIN` is defined by the kvstore example's environment implementation, not by the framework; your application will define its own equivalents. Found by auditing `examples/`: -| Variable | Default | Effect | -|----------|---------|--------| -| `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` | — | Prometheus-compatible base URL for runner to query (e.g., `http://localhost:9090`) | -| `LOGOS_BLOCKCHAIN_METRICS_OTLP_INGEST_URL` | — | Full OTLP HTTP ingest URL for node metrics export (e.g., `http://localhost:9090/api/v1/otlp/v1/metrics`) | -| `LOGOS_BLOCKCHAIN_GRAFANA_URL` | — | Grafana base URL for printing/logging (e.g., `http://localhost:3000`) | -| `LOGOS_BLOCKCHAIN_OTLP_ENDPOINT` | — | OTLP trace endpoint (optional) | -| `LOGOS_BLOCKCHAIN_OTLP_METRICS_ENDPOINT` | — | OTLP metrics endpoint (optional) | +| Variable | Example | Purpose | +|---|---|---| +| `KVSTORE_NODE_BIN`, `OPENRAFT_KV_NODE_BIN` | kvstore, openraft_kv | optional binary override (fallback builds with Cargo) | +| `QUEUE_NODE_BIN`, `PUBSUB_NODE_BIN`, `METRICS_COUNTER_NODE_BIN` | queue, pubsub, metrics_counter | required node binary path for local runs | +| `NATS_SERVER_BIN` | nats | path to an upstream `nats-server` executable | +| `NATS_IMAGE` / `NATS_PLATFORM` | nats | compose image override (default `nats:2.10`) | +| `REDIS_STREAMS_IMAGE` / `REDIS_STREAMS_PLATFORM` | redis_streams | compose image override (default `redis:7`) | +| `KVSTORE_IMAGE`, `QUEUE_IMAGE`, … (`_IMAGE`/`_PLATFORM`/`_K8S_IMAGE`) | all node apps | derived image overrides via the conventional specs | +| `METRICS_COUNTER_K8S_PROMETHEUS_NODE_PORT` | metrics_counter | fixed NodePort for the Prometheus service | +| `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` | metrics_counter | also consulted by the example to locate Prometheus | -**Example:** - -```bash -# Enable Prometheus querying -export LOGOS_BLOCKCHAIN_METRICS_QUERY_URL=http://localhost:9090 -export LOGOS_BLOCKCHAIN_METRICS_OTLP_INGEST_URL=http://localhost:9090/api/v1/otlp/v1/metrics -export LOGOS_BLOCKCHAIN_GRAFANA_URL=http://localhost:3000 - -scripts/run/run-examples.sh -t 60 -n 3 compose -``` - ---- - -## Compose Runner Specific - -Variables specific to Docker Compose deployment: - -| Variable | Default | Effect | -|----------|---------|--------| -| `COMPOSE_RUNNER_HOST` | `127.0.0.1` | Host address for port mappings | -| `COMPOSE_RUNNER_PRESERVE` | 0 | Keep containers running after test (for debugging) | -| `COMPOSE_RUNNER_HTTP_TIMEOUT_SECS` | — | Override HTTP readiness timeout (seconds) | -| `COMPOSE_RUNNER_HOST_GATEWAY` | `host.docker.internal:host-gateway` | Controls `extra_hosts` entry injected into compose (set to `disable` to omit) | -| `TESTNET_RUNNER_PRESERVE` | — | Alias for `COMPOSE_RUNNER_PRESERVE` | - -**Example:** - -```bash -# Keep containers after test for debugging -COMPOSE_RUNNER_PRESERVE=1 \ -scripts/run/run-examples.sh -t 60 -n 3 compose - -# Containers remain running -docker ps --filter "name=nomos-compose-" -docker logs -``` - ---- - -## K8s Runner Specific - -Variables specific to Kubernetes deployment: - -| Variable | Default | Effect | -|----------|---------|--------| -| `K8S_RUNNER_NAMESPACE` | Random UUID | Kubernetes namespace (pin for debugging) | -| `K8S_RUNNER_RELEASE` | Random UUID | Helm release name (pin for debugging) | -| `K8S_RUNNER_NODE_HOST` | — | NodePort host resolution for non-local clusters | -| `K8S_RUNNER_DEBUG` | 0 | Log Helm stdout/stderr for install commands | -| `K8S_RUNNER_PRESERVE` | 0 | Keep namespace/release after run (for debugging) | -| `K8S_RUNNER_DEPLOYMENT_TIMEOUT_SECS` | — | Override deployment readiness timeout | -| `K8S_RUNNER_HTTP_TIMEOUT_SECS` | — | Override HTTP readiness timeout (port-forwards) | -| `K8S_RUNNER_HTTP_PROBE_TIMEOUT_SECS` | — | Override HTTP readiness timeout (NodePort probes) | -| `K8S_RUNNER_PROMETHEUS_HTTP_TIMEOUT_SECS` | — | Override Prometheus readiness timeout | -| `K8S_RUNNER_PROMETHEUS_HTTP_PROBE_TIMEOUT_SECS` | — | Override Prometheus NodePort probe timeout | - -**Example:** - -```bash -# Pin namespace for debugging -K8S_RUNNER_NAMESPACE=nomos-test-debug \ -K8S_RUNNER_PRESERVE=1 \ -K8S_RUNNER_DEBUG=1 \ -scripts/run/run-examples.sh -t 60 -n 3 k8s - -# Inspect resources -kubectl get pods -n nomos-test-debug -kubectl logs -n nomos-test-debug -l nomos/logical-role=node -``` - ---- - -## Platform & Build Configuration - -Platform-specific build configuration: - -| Variable | Default | Effect | -|----------|---------|--------| -| `LOGOS_BLOCKCHAIN_BUNDLE_DOCKER_PLATFORM` | Host arch | Docker platform for bundle builds: `linux/arm64` or `linux/amd64` (macOS/Windows hosts) | -| `LOGOS_BLOCKCHAIN_BIN_PLATFORM` | — | Legacy alias for `LOGOS_BLOCKCHAIN_BUNDLE_DOCKER_PLATFORM` | -| `COMPOSE_CIRCUITS_PLATFORM` | Host arch | Circuits platform for image builds: `linux-aarch64` or `linux-x86_64` | -| `LOGOS_BLOCKCHAIN_EXTRA_FEATURES` | — | Extra cargo features to enable when building bundles (used by `scripts/build/build-bundle.sh`) | - -**macOS / Apple Silicon:** - -```bash -# Native performance (recommended for local testing) -export LOGOS_BLOCKCHAIN_BUNDLE_DOCKER_PLATFORM=linux/arm64 - -# Or target amd64 (slower via emulation) -export LOGOS_BLOCKCHAIN_BUNDLE_DOCKER_PLATFORM=linux/amd64 -``` - ---- - -## Timeouts & Performance - -Timeout and performance tuning: - -| Variable | Default | Effect | -|----------|---------|--------| -| `SLOW_TEST_ENV` | false | Doubles built-in readiness timeouts (useful in CI / constrained laptops) | -| `TESTNET_PRINT_ENDPOINTS` | 0 | Print `TESTNET_ENDPOINTS` / `TESTNET_PPROF` lines during deploy (set automatically by `scripts/run/run-examples.sh`) | - -**Example:** - -```bash -# Increase timeouts for slow environments -SLOW_TEST_ENV=true \ -scripts/run/run-examples.sh -t 120 -n 5 compose -``` - ---- - -## Node Configuration (Advanced) - -Node-level configuration passed through to logos-blockchain-node: - -| Variable | Default | Effect | -|----------|---------|--------| -| `CONSENSUS_SLOT_TIME` | — | Consensus slot time (seconds) | -| `CONSENSUS_ACTIVE_SLOT_COEFF` | — | Active slot coefficient (0.0-1.0) | -| `LOGOS_BLOCKCHAIN_USE_AUTONAT` | Unset | If set, use AutoNAT instead of a static loopback address for libp2p NAT settings | -| `LOGOS_BLOCKCHAIN_CFGSYNC_PORT` | 4400 | Port used for cfgsync service inside the stack | -| `LOGOS_BLOCKCHAIN_TIME_BACKEND` | `monotonic` | Select time backend (used by compose/k8s stack scripts and deployers) | - -**Example:** - -```bash -# Faster block production -CONSENSUS_SLOT_TIME=5 \ -CONSENSUS_ACTIVE_SLOT_COEFF=0.9 \ -cargo run -p runner-examples --bin local_runner -``` - ---- - -## Framework Runner Logging (Not Node Logs) - -Control framework runner process logs (uses `RUST_LOG`, not `NOMOS_*`): - -| Variable | Default | Effect | -|----------|---------|--------| -| `RUST_LOG` | — | Framework runner log level (e.g., `debug`, `info`) | -| `RUST_BACKTRACE` | — | Enable Rust backtraces on panic (`1` or `full`) | -| `CARGO_TERM_COLOR` | — | Cargo output color (`always`, `never`, `auto`) | - -**Example:** - -```bash -# Debug framework runner (not nodes) -RUST_LOG=debug \ -RUST_BACKTRACE=1 \ -cargo run -p runner-examples --bin local_runner -``` - ---- - -## Helper Script Variables - -Variables used by helper scripts (`scripts/run/run-examples.sh`, etc.): - -| Variable | Default | Effect | -|----------|---------|--------| -| `LOGOS_BLOCKCHAIN_NODE_REV` | From `versions.env` | logos-blockchain-node git revision to build/fetch | -| `LOGOS_BLOCKCHAIN_BUNDLE_VERSION` | From `versions.env` | Bundle schema version | -| `LOGOS_BLOCKCHAIN_IMAGE_SELECTION` | — | Internal: image selection mode set by `run-examples.sh` (`local`/`ecr`/`auto`) | -| `LOGOS_BLOCKCHAIN_NODE_APPLY_PATCHES` | 1 | Set to `0` to disable applying local patches when building bundles | -| `LOGOS_BLOCKCHAIN_NODE_PATCH_DIR` | `patches/logos-blockchain-node` | Patch directory applied to logos-blockchain-node checkout during bundle builds | -| `LOGOS_BLOCKCHAIN_NODE_PATCH_LEVEL` | — | Patch application level (`all` or an integer) for bundle builds | - ---- - -## Quick Reference Examples - -### Minimal Host Run - -```bash -scripts/run/run-examples.sh -t 60 -n 3 host -``` - -### Debug Logging (Host) - -```bash -LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/logs \ -LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \ -LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \ -scripts/run/run-examples.sh -t 60 -n 3 host -``` - -### Compose with Observability - -```bash -LOGOS_BLOCKCHAIN_METRICS_QUERY_URL=http://localhost:9090 \ -LOGOS_BLOCKCHAIN_GRAFANA_URL=http://localhost:3000 \ -scripts/run/run-examples.sh -t 60 -n 3 compose -``` - -### K8s with Debug - -```bash -K8S_RUNNER_NAMESPACE=nomos-debug \ -K8S_RUNNER_DEBUG=1 \ -K8S_RUNNER_PRESERVE=1 \ -scripts/run/run-examples.sh -t 60 -n 3 k8s -``` - -### CI Environment - -```yaml -env: - RUST_BACKTRACE: 1 - LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS: 1 -``` - ---- - -## See Also - -- [Prerequisites & Setup](prerequisites.md) — Required files and setup -- [Running Examples](running-examples.md) — How to run scenarios -- [Logging & Observability](logging-observability.md) — Log collection details -- [CI Integration](ci-integration.md) — CI-specific variables -- [Troubleshooting](troubleshooting.md) — Common issues with variables +See [Running the Examples](running-examples.md) for how these fit each binary. diff --git a/book/src/examples-advanced.md b/book/src/examples-advanced.md deleted file mode 100644 index f737ca4..0000000 --- a/book/src/examples-advanced.md +++ /dev/null @@ -1,328 +0,0 @@ -# Advanced Examples - -> **When should I read this?** Skim now to see what's possible, revisit later when you need load testing, chaos scenarios, or custom extensions. Start with [basic examples](examples.md) first. - -Realistic advanced scenarios demonstrating framework capabilities for production testing. - -**Adapt from Complete Source:** -- [compose_runner.rs](https://github.com/logos-blockchain/logos-blockchain-testing/blob/master/examples/src/bin/compose_runner.rs) — Compose examples with workloads -- [k8s_runner.rs](https://github.com/logos-blockchain/logos-blockchain-testing/blob/master/examples/src/bin/k8s_runner.rs) — K8s production patterns -- [Chaos testing patterns](https://github.com/logos-blockchain/logos-blockchain-testing/blob/master/testing-framework/workflows/src/workloads/chaos.rs) — Node control implementation - -## Summary - -| Example | Topology | Workloads | Deployer | Key Feature | -|---------|----------|-----------|----------|-------------| -| Load Progression | 3 nodes | Increasing tx rate | Compose | Dynamic load testing | -| Sustained Load | 4 nodes | High tx rate | Compose | Stress testing | -| Aggressive Chaos | 4 nodes | Frequent restarts + traffic | Compose | Resilience validation | - -## Load Progression Test - -Test consensus under progressively increasing transaction load: - -```rust,ignore -use std::time::Duration; - -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_compose::ComposeDeployer; -use testing_framework_workflows::ScenarioBuilderExt; - -pub async fn load_progression_test() -> Result<()> { - for rate in [5, 10, 20, 30] { - println!("Testing with rate: {}", rate); - - let mut plan = - ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .wallets(50) - .transactions_with(|txs| txs.rate(rate).users(20)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(60)) - .build(); - - let deployer = ComposeDeployer::default(); - let runner = deployer.deploy(&plan).await?; - let _handle = runner.run(&mut plan).await?; - } - - Ok(()) -} -``` - -**When to use:** Finding the maximum sustainable transaction rate for a given topology. - -## Sustained Load Test - -Run high transaction load for extended duration: - -```rust,ignore -use std::time::Duration; - -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_compose::ComposeDeployer; -use testing_framework_workflows::ScenarioBuilderExt; - -pub async fn sustained_load_test() -> Result<()> { - let mut plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(4)) - .wallets(100) - .transactions_with(|txs| txs.rate(15).users(50)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(300)) - .build(); - - let deployer = ComposeDeployer::default(); - let runner = deployer.deploy(&plan).await?; - let _handle = runner.run(&mut plan).await?; - - Ok(()) -} -``` - -**When to use:** Validating stability under continuous high load over extended periods. - -## Aggressive Chaos Test - -Frequent node restarts with active traffic: - -```rust,ignore -use std::time::Duration; - -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_compose::ComposeDeployer; -use testing_framework_workflows::{ChaosBuilderExt, ScenarioBuilderExt}; - -pub async fn aggressive_chaos_test() -> Result<()> { - let mut plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(4)) - .enable_node_control() - .wallets(50) - .transactions_with(|txs| txs.rate(10).users(20)) - .chaos_with(|c| { - c.restart() - .min_delay(Duration::from_secs(10)) - .max_delay(Duration::from_secs(20)) - .target_cooldown(Duration::from_secs(15)) - .apply() - }) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(180)) - .build(); - - let deployer = ComposeDeployer::default(); - let runner = deployer.deploy(&plan).await?; - let _handle = runner.run(&mut plan).await?; - - Ok(()) -} -``` - -**When to use:** Validating recovery and liveness under aggressive failure conditions. - -**Note:** Requires `ComposeDeployer` for node control support. - -## Extension Ideas - -These scenarios require custom implementations but demonstrate framework extensibility: - -### Mempool & Transaction Handling - -#### Transaction Propagation & Inclusion Test - -**Concept:** Submit the same batch of independent transactions to different nodes in randomized order/offsets, then verify all transactions are included and final state matches across nodes. - -**Requirements:** -- **Custom workload:** Generates a fixed batch of transactions and submits the same set to different nodes via `ctx.node_clients()`, with randomized submission order and timing offsets per node -- **Custom expectation:** Verifies all transactions appear in blocks (order may vary), final state matches across all nodes (compare balances or state roots), and no transactions are dropped - -**Why useful:** Exercises mempool propagation, proposer fairness, and transaction inclusion guarantees under realistic race conditions. Tests that the protocol maintains consistency regardless of which node receives transactions first. - -**Implementation notes:** Requires both a custom `Workload` implementation (to submit same transactions to multiple nodes with jitter) and a custom `Expectation` implementation (to verify inclusion and state consistency). - -#### Cross-Validator Mempool Divergence & Convergence - -**Concept:** Drive different transaction subsets into different nodes (or differing arrival orders) to create temporary mempool divergence, then verify mempools/blocks converge to contain the union (no permanent divergence). - -**Requirements:** -- **Custom workload:** Targets specific nodes via `ctx.node_clients()` with disjoint or jittered transaction batches -- **Custom expectation:** After a convergence window, verifies that all transactions appear in blocks (order may vary) or that mempool contents converge across nodes -- Run normal workloads during convergence period - -**Expectations:** -- Temporary mempool divergence is acceptable (different nodes see different transactions initially) -- After convergence window, all transactions appear in blocks or mempools converge -- No transactions are permanently dropped despite initial divergence -- Mempool gossip/reconciliation mechanisms work correctly - -**Why useful:** Exercises mempool gossip and reconciliation under uneven input or latency. Ensures no node "drops" transactions seen elsewhere, validating that mempool synchronization mechanisms correctly propagate transactions across the network even when they arrive at different nodes in different orders. - -**Implementation notes:** Requires both a custom `Workload` implementation (to inject disjoint/jittered batches per node) and a custom `Expectation` implementation (to verify mempool convergence or block inclusion). Uses existing `ctx.node_clients()` capability—no new infrastructure needed. - -#### Adaptive Mempool Pressure Test - -**Concept:** Ramp transaction load over time to observe mempool growth, fee prioritization/eviction, and block saturation behavior, detecting performance regressions and ensuring backpressure/eviction work under increasing load. - -**Requirements:** -- **Custom workload:** Steadily increases transaction rate over time (optional: use fee tiers) -- **Custom expectation:** Monitors mempool size, evictions, and throughput (blocks/txs per slot), flagging runaway growth or stalls -- Run for extended duration to observe pressure buildup - -**Expectations:** -- Mempool size grows predictably with load (not runaway growth) -- Fee prioritization/eviction mechanisms activate under pressure -- Block saturation behavior is acceptable (blocks fill appropriately) -- Throughput (blocks/txs per slot) remains stable or degrades gracefully -- No stalls or unbounded mempool growth - -**Why useful:** Detects performance regressions in mempool management. Ensures backpressure and eviction mechanisms work correctly under increasing load, preventing memory exhaustion or unbounded growth. Validates that fee prioritization correctly selects high-value transactions when mempool is full. - -**Implementation notes:** Can be built with current workload model (ramping rate). Requires custom `Expectation` implementation that reads mempool metrics (via node HTTP APIs or Prometheus) and monitors throughput to judge behavior. No new infrastructure needed—uses existing observability capabilities. - -#### Invalid Transaction Fuzzing - -**Concept:** Submit malformed transactions and verify they're rejected properly. - -**Implementation approach:** -- Custom workload that generates invalid transactions (bad signatures, insufficient funds, malformed structure) -- Expectation verifies mempool rejects them and they never appear in blocks -- Test mempool resilience and filtering - -**Why useful:** Ensures mempool doesn't crash or include invalid transactions under fuzzing. - -### Network & Gossip - -#### Gossip Latency Gradient Scenario - -**Concept:** Test consensus robustness under skewed gossip delays by partitioning nodes into latency tiers (tier A ≈10ms, tier B ≈100ms, tier C ≈300ms) and observing propagation lag, fork rate, and eventual convergence. - -**Requirements:** -- Partition nodes into three groups (tiers) -- Apply per-group network delay via chaos: `netem`/`iptables` in compose; NetworkPolicy + `netem` sidecar in k8s -- Run standard workload (transactions/block production) -- Optional: Remove delays at end to check recovery - -**Expectations:** -- **Propagation:** Messages reach all tiers within acceptable bounds -- **Safety:** No divergent finalized heads; fork rate stays within tolerance -- **Liveness:** Chain keeps advancing; convergence after delays relaxed (if healed) - -**Why useful:** Real networks have heterogeneous latency. This stress-tests proposer selection and fork resolution when some peers are "far" (high latency), validating that consensus remains safe and live under realistic network conditions. - -**Current blocker:** Runner support for per-group delay injection (network delay via `netem`/`iptables`) is not present today. Would require new chaos plumbing in compose/k8s deployers to inject network delays per node group. - -#### Byzantine Gossip Flooding (libp2p Peer) - -**Concept:** Spin up a custom workload/sidecar that runs a libp2p host, joins the cluster's gossip mesh, and publishes a high rate of syntactically valid but useless/stale messages to selected topics, testing gossip backpressure, scoring, and queue handling under a "malicious" peer. - -**Requirements:** -- Custom workload/sidecar that implements a libp2p host -- Join the cluster's gossip mesh as a peer -- Publish high-rate syntactically valid but useless/stale messages to selected gossip topics -- Run alongside normal workloads (transactions/block production) - -**Expectations:** -- Gossip backpressure mechanisms prevent message flooding from overwhelming nodes -- Peer scoring correctly identifies and penalizes the malicious peer -- Queue handling remains stable under flood conditions -- Normal consensus operation continues despite malicious peer - -**Why useful:** Tests Byzantine behavior (malicious peer) which is critical for consensus protocol robustness. More realistic than RPC spam since it uses the actual gossip protocol. Validates that gossip backpressure, peer scoring, and queue management correctly handle adversarial peers without disrupting consensus. - -**Current blocker:** Requires adding gossip-capable helper (libp2p integration) to the framework. Would need a custom workload/sidecar implementation that can join the gossip mesh and inject messages. The rest of the scenario can use existing runners/workloads. - -#### Network Partition Recovery - -**Concept:** Test consensus recovery after network partitions. - -**Requirements:** -- Needs `block_peer()` / `unblock_peer()` methods in `NodeControlHandle` -- Partition subsets of nodes, wait, then restore connectivity -- Verify chain convergence after partition heals - -**Why useful:** Tests the most realistic failure mode in distributed systems. - -**Current blocker:** Node control doesn't yet support network-level actions (only process restarts). - -### Time & Timing - -#### Time-Shifted Blocks (Clock Skew Test) - -**Concept:** Test consensus and timestamp handling when nodes run with skewed clocks (e.g., +1s, −1s, +200ms jitter) to surface timestamp validation issues, reorg sensitivity, and clock drift handling. - -**Requirements:** -- Assign per-node time offsets (e.g., +1s, −1s, +200ms jitter) -- Run normal workload (transactions/block production) -- Observe whether blocks are accepted/propagated and the chain stays consistent - -**Expectations:** -- Blocks with skewed timestamps are handled correctly (accepted or rejected per protocol rules) -- Chain remains consistent across nodes despite clock differences -- No unexpected reorgs or chain splits due to timestamp validation issues - -**Why useful:** Clock skew is a common real-world issue in distributed systems. This validates that consensus correctly handles timestamp validation and maintains safety/liveness when nodes have different clock offsets, preventing timestamp-based attacks or failures. - -**Current blocker:** Runner ability to skew per-node clocks (e.g., privileged containers with `libfaketime`/`chrony` or time-offset netns) is not available today. Would require a new chaos/time-skew hook in deployers to inject clock offsets per node. - -#### Block Timing Consistency - -**Concept:** Verify block production intervals stay within expected bounds. - -**Implementation approach:** -- Custom expectation that consumes `BlockFeed` -- Collect block timestamps during run -- Assert intervals are within `(slot_duration * active_slot_coeff) ± tolerance` - -**Why useful:** Validates consensus timing under various loads. - -### Topology & Membership - -#### Dynamic Topology (Churn) Scenario - -**Concept:** Nodes join and leave mid-run (new identities/addresses added; some nodes permanently removed) to exercise peer discovery, bootstrapping, reputation, and load balancing under churn. - -**Requirements:** -- Runner must be able to spin up new nodes with fresh keys/addresses at runtime -- Update peer lists and bootstraps dynamically as nodes join/leave -- Optionally tear down nodes permanently (not just restart) -- Run normal workloads (transactions/block production) during churn - -**Expectations:** -- New nodes successfully discover and join the network -- Peer discovery mechanisms correctly handle dynamic topology changes -- Reputation systems adapt to new/removed peers -- Load balancing adjusts to changing node set -- Consensus remains safe and live despite topology churn - -**Why useful:** Real networks experience churn (nodes joining/leaving). Unlike restarts (which preserve topology), churn changes the actual topology size and peer set, testing how the protocol handles dynamic membership. This exercises peer discovery, bootstrapping, reputation systems, and load balancing under realistic conditions. - -**Current blocker:** Runner support for dynamic node addition/removal at runtime is not available today. Chaos today only restarts existing nodes; churn would require the ability to spin up new nodes with fresh identities/addresses, update peer lists/bootstraps dynamically, and permanently remove nodes. Would need new topology management capabilities in deployers. - -### API & External Interfaces - -#### API DoS/Stress Test - -**Concept:** Adversarial workload floods node HTTP/WS APIs with high QPS and malformed/bursty requests; expectation checks nodes remain responsive or rate-limit without harming consensus. - -**Requirements:** -- **Custom workload:** Targets node HTTP/WS API endpoints with mixed valid/invalid requests at high rate -- **Custom expectation:** Monitors error rates, latency, and confirms block production/liveness unaffected -- Run alongside normal workloads (transactions/block production) - -**Expectations:** -- Nodes remain responsive or correctly rate-limit under API flood -- Error rates/latency are acceptable (rate limiting works) -- Block production/liveness unaffected by API abuse -- Consensus continues normally despite API stress - -**Why useful:** Validates API hardening under abuse and ensures control/telemetry endpoints don't destabilize the node. Tests that API abuse is properly isolated from consensus operations, preventing DoS attacks on API endpoints from affecting blockchain functionality. - -**Implementation notes:** Requires custom `Workload` implementation that directs high-QPS traffic to node APIs (via `ctx.node_clients()` or direct HTTP clients) and custom `Expectation` implementation that monitors API responsiveness metrics and consensus liveness. Uses existing node API access—no new infrastructure needed. - -### State & Correctness - -#### Wallet Balance Verification - -**Concept:** Track wallet balances and verify state consistency. - -**Description:** After transaction workload completes, query all wallet balances via node API and verify total supply is conserved. Requires tracking initial state, submitted transactions, and final balances. Validates that the ledger maintains correctness under load (no funds lost or created). This is a **state assertion** expectation that checks correctness, not just liveness. diff --git a/book/src/examples.md b/book/src/examples.md deleted file mode 100644 index 6f696a2..0000000 --- a/book/src/examples.md +++ /dev/null @@ -1,118 +0,0 @@ -# Examples - -Concrete scenario shapes that illustrate how to combine topologies, workloads, -and expectations. - -**View Complete Source Code:** -- [local_runner.rs](https://github.com/logos-blockchain/logos-blockchain-testing/blob/master/examples/src/bin/local_runner.rs) — Host processes (local) -- [compose_runner.rs](https://github.com/logos-blockchain/logos-blockchain-testing/blob/master/examples/src/bin/compose_runner.rs) — Docker Compose -- [k8s_runner.rs](https://github.com/logos-blockchain/logos-blockchain-testing/blob/master/examples/src/bin/k8s_runner.rs) — Kubernetes - -**Runnable examples:** The repo includes complete binaries in `examples/src/bin/`: -- `local_runner.rs` — Host processes (local) -- `compose_runner.rs` — Docker Compose (requires image built) -- `k8s_runner.rs` — Kubernetes (requires cluster access and image loaded) - -**Recommended:** Use `scripts/run/run-examples.sh -t -n ` where mode is `host`, `compose`, or `k8s`. - -**Alternative:** Direct cargo run: `cargo run -p runner-examples --bin ` - -**Code patterns** below show how to build scenarios. Wrap these in `#[tokio::test]` functions for integration tests, or `#[tokio::main]` for binaries. - -## Simple consensus liveness - -Minimal test that validates basic block production: - -```rust,ignore -use std::time::Duration; - -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_local::LocalDeployer; -use testing_framework_workflows::ScenarioBuilderExt; - -pub async fn simple_consensus() -> Result<()> { - let mut plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(30)) - .build(); - - let deployer = LocalDeployer::default(); - let runner = deployer.deploy(&plan).await?; - let _handle = runner.run(&mut plan).await?; - - Ok(()) -} -``` - -**When to use**: smoke tests for consensus on minimal hardware. - -## Transaction workload - -Test consensus under transaction load: - -```rust,ignore -use std::time::Duration; - -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_local::LocalDeployer; -use testing_framework_workflows::ScenarioBuilderExt; - -pub async fn transaction_workload() -> Result<()> { - let mut plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(2)) - .wallets(20) - .transactions_with(|txs| txs.rate(5).users(10)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(60)) - .build(); - - let deployer = LocalDeployer::default(); - let runner = deployer.deploy(&plan).await?; - let _handle = runner.run(&mut plan).await?; - - Ok(()) -} -``` - -**When to use**: validate transaction submission and inclusion. - -## Chaos resilience - -Test system resilience under node restarts: - -```rust,ignore -use std::time::Duration; - -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_compose::ComposeDeployer; -use testing_framework_workflows::{ChaosBuilderExt, ScenarioBuilderExt}; - -pub async fn chaos_resilience() -> Result<()> { - let mut plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(4)) - .enable_node_control() - .wallets(20) - .transactions_with(|txs| txs.rate(3).users(10)) - .chaos_with(|c| { - c.restart() - .min_delay(Duration::from_secs(20)) - .max_delay(Duration::from_secs(40)) - .target_cooldown(Duration::from_secs(30)) - .apply() - }) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(120)) - .build(); - - let deployer = ComposeDeployer::default(); - let runner = deployer.deploy(&plan).await?; - let _handle = runner.run(&mut plan).await?; - - Ok(()) -} -``` - -**When to use**: resilience validation and operational readiness drills. - -**Note**: Chaos tests require `ComposeDeployer` or another runner with node control support. diff --git a/book/src/expectations.md b/book/src/expectations.md new file mode 100644 index 0000000..7af7951 --- /dev/null +++ b/book/src/expectations.md @@ -0,0 +1,171 @@ +# Expectations and Evaluation + +Expectations define success conditions. They can capture state before workloads start, check invariants while traffic runs, and evaluate the final state after the run settles. + +--- + +## The Expectation Trait + +`Expectation` lives in `testing-framework/core/src/scenario/expectation.rs`: + +```rust,ignore +use async_trait::async_trait; +use testing_framework_core::scenario::{DynError, Expectation, RunContext}; + +#[async_trait] +pub trait Expectation: Send + Sync { + fn name(&self) -> &str; + + fn init( + &mut self, + _descriptors: &E::Deployment, + _run_metrics: &RunMetrics, + ) -> Result<(), DynError> { + Ok(()) + } + + async fn start_capture(&mut self, _ctx: &RunContext) -> Result<(), DynError> { + Ok(()) + } + + /// Optional periodic check used by fail-fast expectation mode. + async fn check_during_capture(&mut self, _ctx: &RunContext) -> Result<(), DynError> { + Ok(()) + } + + async fn evaluate(&mut self, ctx: &RunContext) -> Result<(), DynError>; +} +``` + +The trait methods are: +- **`init`** runs at `build()` time with the resolved deployment and run metrics; a failure aborts the build. +- **`start_capture`** runs once per expectation *before any workload starts*. Use it to record a baseline (initial counters, starting state). A failure here is `ScenarioError::ExpectationCapture` and stops the run before traffic begins. +- **`check_during_capture`** is a fail-fast hook. The runner calls it on every expectation roughly once per second for the whole workload window (and the cooldown window). The default is a no-op, so existing end-of-run expectations are unaffected. The first check that returns `Err` aborts the run immediately with `ScenarioError::ExpectationFailedDuringCapture`. Use it for invariants that must hold throughout the run. +- **`evaluate`** checks the final condition after the run settles. It takes `&mut self`, so it can consume state accumulated during capture. + +--- + +## Registration + +Two paths feed the scenario's expectation list: + +1. **Explicit**: `.with_expectation(exp)` or `.with_expectation_boxed(boxed)` on any builder. +2. **Workload-attached**: when you call `.with_workload(w)`, the builder also collects `w.expectations()` (see [Workloads and Concurrency](workloads.md)). The default implementation returns none. + +Both end up in the same list and are treated identically at run time. + +Workload-attached expectations let a workload register the checks associated with its own traffic. Adding the workload also adds those checks. + +--- + +## Evaluation and Failure Aggregation + +```mermaid +flowchart LR + SC[start_capture]:::sc --> W[Workload window
+ periodic checks]:::sc + W --> CD[Cooldown + settle]:::sc + CD --> EV[evaluate all]:::sc + EV --> R{failures?} + R -->|no| OK[run passes] + R -->|yes| AGG[aggregated report] + + classDef sc stroke:#9b6dd6,stroke-width:2.5px; +``` + +At the end of the run the runner evaluates **every** registered expectation, even after failures. Each failure is recorded as `name: error`, and the results are joined into a single `ScenarioError::Expectations` report: + +```text +expectations failed: +kv_converges: kv convergence not reached within 20s for 20 keys +openraft_kv_converges: timed out waiting for observed replicated state convergence ... +``` + +This is different from workload failures and capture-check failures, which abort immediately. + +--- + +## Cooldown: with_expectation_cooldown + +Workload traffic may need time to settle before evaluation: replication lags, queues drain, and restarted nodes rejoin. The builder exposes: + +```rust,ignore +.with_expectation_cooldown(Duration::from_secs(20)) +``` + +Verified behavior (`runner.rs` and `definition/validation.rs`): + +- If you never call it, the cooldown defaults to **10 seconds**. (`build()` also enforces a minimum run duration of 10 seconds.) +- After the workload window, the runner keeps the run alive for the cooldown window, still joining unfinished workloads and still running `check_during_capture` ticks. +- When the framework owns the node lifecycle (managed clusters), the cooldown window is raised to a **minimum of 30 seconds** so restarted or freshly deployed nodes stabilize. +- Before calling `evaluate`, the runner additionally sleeps a short settle wait derived from the same setting (at least 2 seconds when a cooldown is configured or node control is active) so runtime extensions such as [observers](observation.md) catch up. + +Set the cooldown to zero only for scenarios without managed nodes where staleness cannot matter. + +--- + +## Worked Example: Convergence Checks + +The kvstore example's `KvConverges` (`examples/kvstore/testing/workloads/src/expectations.rs`) is a plain polling expectation. It only implements `evaluate` and does its own retry loop against the node clients: + +```rust,ignore +use async_trait::async_trait; +use kvstore_runtime_ext::KvEnv; +use testing_framework_core::scenario::{DynError, Expectation, RunContext}; + +#[async_trait] +impl Expectation for KvConverges { + fn name(&self) -> &str { + "kv_converges" + } + + async fn evaluate(&mut self, ctx: &RunContext) -> Result<(), DynError> { + let clients = ctx.node_clients().snapshot(); + if clients.is_empty() { + return Err("no kv node clients available".into()); + } + + let deadline = tokio::time::Instant::now() + self.timeout; + while tokio::time::Instant::now() < deadline { + if self.is_converged(&clients).await? { + return Ok(()); + } + tokio::time::sleep(self.poll_interval).await; + } + + Err(format!( + "kv convergence not reached within {:?} for {} keys", + self.timeout, self.key_count + ) + .into()) + } +} +``` + +The example follows two conventions: + +- **Poll with a deadline inside `evaluate`.** Eventual consistency is the common case; a one-shot read makes flaky tests. +- **Make the error message carry the diagnosis.** State what was expected, how long you waited, and (where available) what was last observed. + +The openraft_kv variant, `OpenRaftKvConverges` (`examples/openraft_kv/testing/workloads/src/convergence.rs`), reads the cluster observer registered as a runtime extension instead of querying nodes directly: + +```rust,ignore +async fn evaluate(&mut self, ctx: &RunContext) -> Result<(), DynError> { + let expected = expected_kv(&self.key_prefix, self.total_writes); + let observer = ctx.require_extension::>()?; + + wait_for_observed_replication(&observer, &expected, self.timeout).await?; + + Ok(()) +} +``` + +The observer polls every node in the background, and the expectation waits for a matching snapshot without maintaining its own client polling state. See [Continuous Observation](observation.md) for the mechanism. + +--- + +## See Also + +- [Workloads and Concurrency](workloads.md) — the traffic these checks judge +- [Continuous Observation](observation.md) — snapshot-based state for expectations +- [Runtime Extensions](runtime-extensions.md) — how extension handles reach `evaluate` +- [Telemetry and External Observability](telemetry.md) — asserting on Prometheus metrics diff --git a/book/src/extending.md b/book/src/extending.md deleted file mode 100644 index 547e584..0000000 --- a/book/src/extending.md +++ /dev/null @@ -1,360 +0,0 @@ -# Extending the Framework - -This guide shows how to extend the framework with custom workloads, expectations, runners, and topology helpers. Each section includes the trait outline and a minimal code example. - -## Adding a Workload - -**Steps:** -1. Implement `testing_framework_core::scenario::Workload` -2. Provide a name and any bundled expectations -3. Use `init` to derive inputs from topology/metrics; fail fast if prerequisites missing -4. Use `start` to drive async traffic using `RunContext` clients -5. Expose from `testing-framework/workflows` and optionally add a DSL helper - -**Trait outline:** - -```rust,ignore -use async_trait::async_trait; -use testing_framework_core::scenario::{ - DynError, Expectation, RunContext, RunMetrics, Workload, -}; -use testing_framework_core::topology::generation::GeneratedTopology; - -struct MyExpectation; - -#[async_trait] -impl Expectation for MyExpectation { - fn name(&self) -> &str { - "my_expectation" - } - - async fn evaluate(&mut self, _ctx: &RunContext) -> Result<(), DynError> { - Ok(()) - } -} - -pub struct MyWorkload { - // Configuration fields - target_rate: u64, -} - -impl MyWorkload { - pub fn new(target_rate: u64) -> Self { - Self { target_rate } - } -} - -#[async_trait] -impl Workload for MyWorkload { - fn name(&self) -> &str { - "my_workload" - } - - fn expectations(&self) -> Vec> { - // Return bundled expectations that should run with this workload - vec![Box::new(MyExpectation)] - } - - fn init( - &mut self, - topology: &GeneratedTopology, - _run_metrics: &RunMetrics, - ) -> Result<(), DynError> { - // Validate prerequisites (e.g., enough nodes, wallet data present) - if topology.nodes().is_empty() { - return Err("no nodes available".into()); - } - Ok(()) - } - - async fn start(&self, ctx: &RunContext) -> Result<(), DynError> { - // Drive async activity: submit transactions, query nodes, etc. - let clients = ctx.node_clients().node_clients(); - - for client in clients { - let info = client.consensus_info().await?; - tracing::info!(height = info.height, "workload queried node"); - } - - Ok(()) - } -} -``` - -**Key points:** -- `name()` identifies the workload in logs -- `expectations()` bundles default checks (can be empty) -- `init()` validates topology before run starts -- `start()` executes concurrently with other workloads; it should complete before run duration expires - -See [Example: New Workload & Expectation](custom-workload-example.md) for a complete, runnable example. - -## Adding an Expectation - -**Steps:** -1. Implement `testing_framework_core::scenario::Expectation` -2. Use `start_capture` to snapshot baseline metrics (optional) -3. Use `evaluate` to assert outcomes after workloads finish -4. Return descriptive errors; the runner aggregates them -5. Export from `testing-framework/workflows` if reusable - -**Trait outline:** - -```rust,ignore -use async_trait::async_trait; -use testing_framework_core::scenario::{DynError, Expectation, RunContext}; - -pub struct MyExpectation { - expected_value: u64, - captured_baseline: Option, -} - -impl MyExpectation { - pub fn new(expected_value: u64) -> Self { - Self { - expected_value, - captured_baseline: None, - } - } -} - -#[async_trait] -impl Expectation for MyExpectation { - fn name(&self) -> &str { - "my_expectation" - } - - async fn start_capture(&mut self, ctx: &RunContext) -> Result<(), DynError> { - // Optional: capture baseline state before workloads start - let client = ctx.node_clients().node_clients().first() - .ok_or("no nodes")?; - - let info = client.consensus_info().await?; - self.captured_baseline = Some(info.height); - - tracing::info!(baseline = self.captured_baseline, "captured baseline"); - Ok(()) - } - - async fn evaluate(&mut self, ctx: &RunContext) -> Result<(), DynError> { - // Assert the expected condition holds after workloads finish - let client = ctx.node_clients().node_clients().first() - .ok_or("no nodes")?; - - let info = client.consensus_info().await?; - let final_height = info.height; - - let baseline = self.captured_baseline.unwrap_or(0); - let delta = final_height.saturating_sub(baseline); - - if delta < self.expected_value { - return Err(format!( - "expected at least {} blocks, got {}", - self.expected_value, delta - ).into()); - } - - tracing::info!(delta, "expectation passed"); - Ok(()) - } -} -``` - -**Key points:** -- `name()` identifies the expectation in logs -- `start_capture()` runs before workloads start (optional) -- `evaluate()` runs after workloads finish; return descriptive errors -- Expectations run sequentially; keep them fast - -## Adding a Runner (Deployer) - -**Steps:** -1. Implement `testing_framework_core::scenario::Deployer` for your capability type -2. Deploy infrastructure and return a `Runner` -3. Construct `NodeClients` and spawn a `BlockFeed` -4. Build a `RunContext` and provide a `CleanupGuard` for teardown - -**Trait outline:** - -```rust,ignore -use async_trait::async_trait; -use testing_framework_core::scenario::{ - CleanupGuard, Deployer, DynError, Metrics, NodeClients, RunContext, Runner, Scenario, - spawn_block_feed, -}; -use testing_framework_core::topology::deployment::Topology; - -pub struct MyDeployer { - // Configuration: cluster connection details, etc. -} - -impl MyDeployer { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait] -impl Deployer<()> for MyDeployer { - type Error = DynError; - - async fn deploy(&self, scenario: &Scenario<()>) -> Result { - // 1. Launch nodes using scenario.topology() - // 2. Wait for readiness (e.g., consensus info endpoint responds) - // 3. Build NodeClients for nodes - // 4. Spawn a block feed for expectations (optional but recommended) - // 5. Create NodeControlHandle if you support restarts (optional) - // 6. Return a Runner wrapping RunContext + CleanupGuard - - tracing::info!("deploying scenario with MyDeployer"); - - let topology: Option = None; // Some(topology) if you spawned one - let node_clients = NodeClients::default(); // Or NodeClients::from_topology(...) - - let client = node_clients - .any_client() - .ok_or("no api clients available")? - .clone(); - let (block_feed, block_feed_guard) = spawn_block_feed(client).await?; - - let telemetry = Metrics::empty(); // or Metrics::from_prometheus(...) - let node_control = None; // or Some(Arc) - - let context = RunContext::new( - scenario.topology().clone(), - topology, - node_clients, - scenario.duration(), - telemetry, - block_feed, - node_control, - ); - - // If you also have other resources to clean up (containers/pods/etc), - // wrap them in your own CleanupGuard implementation and call - // CleanupGuard::cleanup(Box::new(block_feed_guard)) inside it. - Ok(Runner::new(context, Some(Box::new(block_feed_guard)))) - } -} -``` - -**Key points:** -- `deploy()` must return a fully prepared `Runner` -- Block until nodes are ready before returning (avoid false negatives) -- Use a `CleanupGuard` to tear down resources on failure (and on `RunHandle` drop) -- If you want chaos workloads, also provide a `NodeControlHandle` via `RunContext` - -## Adding Topology Helpers - -**Steps:** -1. Extend `testing_framework_core::topology::config::TopologyBuilder` with new layouts -2. Keep defaults safe: ensure at least one participant, clamp dispersal factors -3. Consider adding configuration presets for specialized parameters - -**Example:** - -```rust,ignore -use testing_framework_core::topology::{ - config::TopologyBuilder, - configs::network::Libp2pNetworkLayout, -}; - -pub trait TopologyBuilderExt { - fn network_full(self) -> Self; -} - -impl TopologyBuilderExt for TopologyBuilder { - fn network_full(self) -> Self { - self.with_network_layout(Libp2pNetworkLayout::Full) - } -} -``` - -**Key points:** -- Maintain method chaining (return `&mut Self`) -- Validate inputs: clamp factors, enforce minimums -- Document assumptions (e.g., "requires at least 4 nodes") - -## Adding a DSL Helper - -To expose your custom workload through the high-level DSL, add a trait extension: - -```rust,ignore -use async_trait::async_trait; -use testing_framework_core::scenario::{DynError, RunContext, ScenarioBuilder, Workload}; - -#[derive(Default)] -pub struct MyWorkloadBuilder { - target_rate: u64, - some_option: bool, -} - -impl MyWorkloadBuilder { - pub const fn target_rate(mut self, target_rate: u64) -> Self { - self.target_rate = target_rate; - self - } - - pub const fn some_option(mut self, some_option: bool) -> Self { - self.some_option = some_option; - self - } - - pub const fn build(self) -> MyWorkload { - MyWorkload { - target_rate: self.target_rate, - some_option: self.some_option, - } - } -} - -pub struct MyWorkload { - target_rate: u64, - some_option: bool, -} - -#[async_trait] -impl Workload for MyWorkload { - fn name(&self) -> &str { - "my_workload" - } - - async fn start(&self, _ctx: &RunContext) -> Result<(), DynError> { - Ok(()) - } -} - -pub trait MyWorkloadDsl { - fn my_workload_with( - self, - f: impl FnOnce(MyWorkloadBuilder) -> MyWorkloadBuilder, - ) -> Self; -} - -impl MyWorkloadDsl for ScenarioBuilder { - fn my_workload_with( - self, - f: impl FnOnce(MyWorkloadBuilder) -> MyWorkloadBuilder, - ) -> Self { - let builder = f(MyWorkloadBuilder::default()); - self.with_workload(builder.build()) - } -} -``` - -Users can then call: - -```rust,ignore -ScenarioBuilder::topology_with(|t| t.network_star().nodes(1)) - .my_workload_with(|w| { - w.target_rate(10) - .some_option(true) - }) - .build() -``` - -## See Also - -- [API Levels: Builder DSL vs. Direct](api-levels.md) - Understanding the two API levels -- [Custom Workload Example](custom-workload-example.md) - Complete runnable example -- [Internal Crate Reference](internal-crate-reference.md) - Where to add new code diff --git a/book/src/extension-points.md b/book/src/extension-points.md new file mode 100644 index 0000000..f5a3daf --- /dev/null +++ b/book/src/extension-points.md @@ -0,0 +1,202 @@ +# Public Extension Points + +This chapter lists every trait you implement to plug your application into the framework. + +The framework never imports your application. It defines public traits that your integration crate implements and calls them at defined points in the run lifecycle. Each entry below links to the corresponding chapter. + +| Trait | Defined in | You implement it to... | Taught in | +|---|---|---|---| +| `Application` | `testing-framework-core` (`env.rs`) | Bundle your deployment, client, and config types | [Implementing Application](implementing-application.md) | +| `DeploymentProvider` | `testing-framework-core` (`topology`) | Build a deployment plan, optionally from a seed | [Topology and Deployment Plans](topology.md) | +| `Workload` | `testing-framework-core` (`scenario`) | Drive traffic against the running system | [Workloads and Concurrency](workloads.md) | +| `Expectation` | `testing-framework-core` (`scenario`) | Define what success means | [Expectations and Evaluation](expectations.md) | +| `RuntimeExtensionFactory` | `testing-framework-core` (`scenario`) | Prepare a shared runtime value before workloads start | [Runtime Extensions](runtime-extensions.md) | +| `Observer` | `testing-framework-core` (`observation`) | Continuously materialize app state | [Continuous Observation](observation.md) | +| `SourceProvider` | `testing-framework-core` (`observation`) | Supply the current observation source set | [Continuous Observation](observation.md) | +| `SourceProviderFactory` | `testing-framework-core` (`observation`) | Build a source provider once node clients exist | [Continuous Observation](observation.md) | +| `AppDeployment` | `testing-framework-app` | Prepare one composable application preset | [AppDeployment and DeployContext](app-deployment.md) | +| `Deployer` | `testing-framework-core` (`scenario::runtime`) | Provision a scenario into a target environment | [Part V](part-v.md) | +| `NodeControlHandle` | `testing-framework-core` (`scenario`) | Expose start/stop/restart of nodes at runtime | [Scenario Capabilities](capabilities.md) | +| `ClusterWaitHandle` | `testing-framework-core` (`scenario`) | Expose cluster readiness waits | [Scenario Capabilities](capabilities.md) | +| `ObservabilityCapabilityProvider` | `testing-framework-core` (`scenario`) | Surface telemetry endpoints from capability markers | [Telemetry and External Observability](telemetry.md) | +| `BinaryProvider` | `testing-framework-runner-local` (`binary`) | Resolve the node executable for local processes | [Binary Providers](binary-providers.md) | +| `DownloadProcessor` | `testing-framework-runner-local` (`binary`) | Turn a downloaded artifact into an executable | [Binary Providers](binary-providers.md) | +| `IntoExistingCluster` | `testing-framework-core` (`scenario::sources`) | Convert a value into an existing-cluster descriptor | [Existing and External Clusters](external-clusters.md) | + +--- + +## Environment and Topology + +**`Application`** is the root of every integration. It bundles the backend-specific types the scenario engine is generic over: a deployment descriptor, a node client, and a node config. The three methods have working defaults: override `external_node_client` to support external sources, `build_node_client` to support deployer-discovered nodes, and `node_readiness_path` when your health endpoint is not `/`. + +```rust,ignore +#[async_trait] +pub trait Application: Send + Sync + 'static { + type Deployment: DeploymentDescriptor + Clone + 'static; + type NodeClient: Clone + Send + Sync + 'static; + type NodeConfig: Clone + Send + Sync + 'static; + + fn external_node_client(source: &ExternalNodeSource) -> Result; + fn build_node_client(access: &NodeAccess) -> Result; + fn node_readiness_path() -> &'static str; // default "/" +} +``` + +Plugs in as the `E` type parameter of `ScenarioBuilder`, `Workload`, `Expectation`, and every deployer. + +**`DeploymentProvider`** builds the deployment descriptor a scenario runs against, optionally driven by a `DeploymentSeed` for reproducible generation. `ScenarioBuilder::new` accepts one; `ScenarioBuilder::with_deployment` wraps a fixed value in the built-in `FixedDeploymentProvider`. + +```rust,ignore +pub trait DeploymentProvider: Send + Sync { + fn build(&self, seed: Option<&DeploymentSeed>) -> Result; +} +``` + +--- + +## Scenario Behavior + +**`Workload`** describes an action sequence executed during the run. `start` receives the `RunContext` (node clients, extensions, run metrics) and runs concurrently with other workloads. A workload can bundle its own checks via `expectations()`. + +```rust,ignore +#[async_trait] +pub trait Workload: Send + Sync { + fn name(&self) -> &str; + fn expectations(&self) -> Vec>> { Vec::new() } + fn init(&mut self, descriptors: &E::Deployment, metrics: &RunMetrics) -> Result<(), DynError> { Ok(()) } + async fn start(&self, ctx: &RunContext) -> Result<(), DynError>; +} +``` + +Registered with `with_workload` / `with_workload_boxed` on the builder. + +**`Expectation`** defines a check evaluated during or after the run. `start_capture` records a baseline, `check_during_capture` is the optional fail-fast hook polled during the run, and `evaluate` delivers the verdict at the end. + +```rust,ignore +#[async_trait] +pub trait Expectation: Send + Sync { + fn name(&self) -> &str; + fn init(&mut self, descriptors: &E::Deployment, metrics: &RunMetrics) -> Result<(), DynError> { Ok(()) } + async fn start_capture(&mut self, ctx: &RunContext) -> Result<(), DynError> { Ok(()) } + async fn check_during_capture(&mut self, ctx: &RunContext) -> Result<(), DynError> { Ok(()) } + async fn evaluate(&mut self, ctx: &RunContext) -> Result<(), DynError>; +} +``` + +Registered with `with_expectation` / `with_expectation_boxed`. + +**`RuntimeExtensionFactory`** prepares one typed value after deployment (node clients are available) and before workloads start. The value is stored by `TypeId` and retrieved in workloads via `ctx.extension::()` / `ctx.require_extension::()`. Return `PreparedRuntimeExtension::new(value)`, `::with_cleanup(value, guard)`, or `::from_task(value, join_handle)` to tie a background task's lifetime to the run. + +```rust,ignore +#[async_trait] +pub trait RuntimeExtensionFactory: Send + Sync { + async fn prepare( + &self, + deployment: &E::Deployment, + node_clients: NodeClients, + ) -> Result; +} +``` + +Registered with `with_runtime_extension_factory`. Registering two factories that produce the same extension type fails at prepare time with `duplicate runtime extension type registered`. + +--- + +## Observation + +**`Observer`** owns the app-side logic of the continuous observation runtime: `init` builds retained state from the source set, `poll` advances it each cycle and emits delta events, `snapshot` renders the current view. The runtime handles scheduling, history, and error tracking. + +```rust,ignore +#[async_trait] +pub trait Observer: Send + Sync + 'static { + type Source: Clone + Send + Sync + 'static; + type State: Send + Sync + 'static; + type Snapshot: Clone + Send + Sync + 'static; + type Event: Clone + Send + Sync + 'static; + + async fn init(&self, sources: &[ObservedSource]) -> Result; + async fn poll(&self, sources: &[ObservedSource], state: &mut Self::State) + -> Result, DynError>; + fn snapshot(&self, state: &Self::State) -> Self::Snapshot; +} +``` + +**`SourceProvider`** returns the current source set before each cycle, which lets the observed population change mid-run. Use `StaticSourceProvider` for a fixed set. + +```rust,ignore +#[async_trait] +pub trait SourceProvider: Send + Sync + 'static { + async fn sources(&self) -> Result>, DynError>; +} +``` + +**`SourceProviderFactory`** builds the provider once node clients exist. Any `Fn(&E::Deployment, NodeClients) -> Result, DynError>` closure implements it. All three plug into a scenario through `ObservationExtensionFactory`, which is itself a `RuntimeExtensionFactory`; see `examples/openraft_kv/testing/integration/src/observation.rs` for a complete implementation. + +--- + +## Application Composition + +**`AppDeployment`** prepares one reusable application preset, such as a process, child cluster, or composed stack, and returns a typed access or control handle. Framework adapters register managed resource lifetime separately with scenario cleanup. `AppHandle` is blanket-implemented for any `Clone + Send + Sync + 'static` type. + +```rust,ignore +#[async_trait] +pub trait AppDeployment: Send + 'static +where + E: Application, +{ + type Handle: AppHandle; + async fn deploy(self, ctx: &mut DeployContext) -> Result; +} +``` + +Registered with `AppScenarioBuilderExt::with_app`, which wraps it in an `AppDeploymentFactory` (a `RuntimeExtensionFactory`). Compose children inside `deploy` via `ctx.deploy(...)` / `ctx.deploy_and_expose(...)`. See [Handle Ownership and Teardown](handles-teardown.md) for handle access and cleanup semantics. + +--- + +## Deployment Backends + +**`Deployer`** is the contract every backend implements: turn a built `Scenario` into a `Runner`. `ProcessDeployer` (local), `ComposeDeployer`, and `K8sDeployer` are the in-repo implementations; `Caps` carries capability markers such as `NodeControlCapability`. + +```rust,ignore +#[async_trait] +pub trait Deployer: Send + Sync { + type Error; + async fn deploy(&self, scenario: &Scenario) -> Result, Self::Error>; +} +``` + +**`NodeControlHandle`** is the deployer-agnostic control surface behind node-control scenarios: `start_node(_with)`, `stop_node`, `restart_node(_with)`, `wait_node_ready`, `node_client`, and `node_pid`. Every method has a default that returns a "not supported by this deployer" error, so backends implement only what they support. **`ClusterWaitHandle`** provides the cluster-wide `wait_network_ready` operation. Both are combined by `ManualClusterHandle` in `core::runtime::manual`, the interface behind [ManualCluster](manual-cluster.md). + +**`ObservabilityCapabilityProvider`** lets deployers read telemetry endpoints out of whatever capability marker a scenario was built with; it is implemented for `()`, `NodeControlCapability`, and `ObservabilityCapability`. You only implement it when defining a new capability marker type. + +--- + +## Local Binary Resolution + +**`BinaryProvider`** resolves the executable path for a locally spawned node process. Implementations return `Ok(None)` when valid but unable to resolve, which is how `FallbackBinaryProvider` chains providers. The default `resolve` caches per process by `cache_key`. + +```rust,ignore +pub trait BinaryProvider: Send + Sync { + fn try_resolve(&self) -> Result, BinaryProviderError>; + fn display(&self) -> String; + fn cache_key(&self) -> String; + // provided: resolve(), resolve_uncached() +} +``` + +Built-in implementations: `PathBinaryProvider`, `EnvBinaryProvider`, `BuildBinaryProvider`, `DownloadBinaryProvider`, `FallbackBinaryProvider`. **`DownloadProcessor`** post-processes a checksum-verified download (for example, unpacking an archive) into the executable; `DownloadProcessorFn` adapts a closure with a stable `cache_key` so changed preparation logic invalidates the cache. + +```rust,ignore +pub trait DownloadProcessor: Send + Sync { + fn process(&self, artifact: &Path, output: &Path) -> Result<(), DownloadProcessorError>; + fn cache_key(&self) -> &str; +} +``` + +--- + +## Attaching Sources + +**`IntoExistingCluster`** converts a value into the typed `ExistingCluster` descriptor accepted by `with_existing_cluster_from`. It is implemented for `ExistingCluster` and `&ExistingCluster`; implement it for your own environment-selection types to keep attach logic in one place. External endpoints use `ExternalNodeSource` values directly and pair with `Application::external_node_client`. + +The required extension points depend on the entry pattern: a uniform managed cluster needs `Application` and the scenario traits, an AppHost stack adds `AppDeployment`, and attached clusters add the source traits. See [Choosing an Entry Pattern](entry-patterns.md). For where each implementation should live, see [Framework vs Application Boundaries](tf-boundaries.md) and the crate-level view in [Crate and API Map](crate-map.md). diff --git a/book/src/external-clusters.md b/book/src/external-clusters.md new file mode 100644 index 0000000..550c146 --- /dev/null +++ b/book/src/external-clusters.md @@ -0,0 +1,122 @@ +# Existing and External Clusters + +Scenarios can run against nodes the framework did not deploy: an attached existing cluster, standalone external endpoints, or a mix. + +Every scenario draws its node clients from three source classes: **managed** nodes the deployer spawns, **attached** nodes discovered in an existing cluster, and **external** nodes named by static endpoints. The builder records which sources you want; the deployer resolves them into one `NodeClients` inventory at deploy time. + +--- + +## The Source Model + +The source model uses these types from `testing-framework-core`: + +| Type | Shape | +|---|---| +| `ExistingCluster` | Typed descriptor of a cluster to attach to — a k8s label selector (optionally namespaced) or a compose project (optionally with explicit services) | +| `IntoExistingCluster` | Conversion trait; implemented by `ExistingCluster` itself and by deployer metadata types | +| `ExternalNodeSource` | A label plus an endpoint string, e.g. `http://10.0.0.5:8080` | +| `ClusterMode` | `Managed`, `ExistingCluster`, or `ExternalOnly` | +| `ClusterControlProfile` | `FrameworkManaged`, `ExistingClusterAttached`, `ExternalUncontrolled`, `ManualControlled` | + +`ExistingCluster` is constructed with `for_k8s_selector(selector)`, `for_k8s_selector_in_namespace(namespace, selector)`, `for_compose_project(project)`, or `for_compose_services(project, services)`. `ExternalNodeSource::new(label, endpoint)` wraps a plain endpoint string. + +The mode is derived, not set: a scenario with only a topology is `Managed`; adding an existing cluster makes it `ExistingCluster`; `with_external_only` makes it `ExternalOnly`. Invalid combinations (managed **and** attached at once) are unrepresentable. Each mode maps to a `ClusterControlProfile`, which workloads can consult to know whether the framework owns node lifecycles (`framework_owns_lifecycle()` is true only for `FrameworkManaged`). + +--- + +## Builder Methods + +```rust,ignore +use testing_framework_core::scenario::{ExistingCluster, ExternalNodeSource}; + +// Attach to a running compose project instead of deploying nodes. +let scenario = KvScenarioBuilder::deployment_with(|_| KvTopology::new(3)) + .with_existing_cluster(ExistingCluster::for_compose_project("compose-stack-1234".into())) + .with_workload(KvWriteWorkload::new().operations(100)) + .build()?; + +// Add a standalone external endpoint alongside managed nodes. +let scenario = KvScenarioBuilder::deployment_with(|_| KvTopology::new(2)) + .with_external_node(ExternalNodeSource::new( + "staging-gateway".into(), + "http://staging.example.net:8080".into(), + )) + .build()?; +``` + +| Method | Effect | +|---|---| +| `with_existing_cluster(cluster)` | Switch to existing-cluster mode with this descriptor | +| `with_existing_cluster_from(value)` | Same, converting through `IntoExistingCluster` (fallible) | +| `with_attach_source(attach)` | Alias for `with_existing_cluster` | +| `with_external_node(node)` | Add one external endpoint to the current mode | +| `with_external_nodes(nodes)` | Add several | +| `with_external_only()` | Drop the managed topology; keep only external nodes | +| `with_external_only_nodes(nodes)` | `with_external_nodes` + `with_external_only` in one call | + +External nodes compose with every mode: managed + external and attached + external are both valid hybrids. + +--- + +## From Source to Typed Client + +External and attached sources become typed clients through one hook on the `Application` trait: + +```rust,ignore +fn external_node_client(source: &ExternalNodeSource) -> Result; +``` + +The default implementation errors with "external node sources are not supported"; an application opts in by parsing `source.endpoint()` and constructing its client. The local deployer additionally falls back to a generic parser that resolves `http://host:port` endpoints and builds the client from the socket address when the app has not overridden the hook. + +--- + +## Resolution at Runtime + +At deploy time the scenario's sources become a `SourceOrchestrationPlan`, and each deployer supplies a `SourceProviders` set: a managed provider (the clients it just deployed), an attach provider, and an external provider. `orchestrate_sources_with_providers` resolves the plan: + +```mermaid +flowchart LR + P[SourceOrchestrationPlan] --> M[managed provider
deployer-spawned clients] + P --> A[attach provider
discover existing cluster] + P --> X[external provider
external_node_client] + M --> N[NodeClients] + A --> N + X --> N +``` + +The final inventory is ordered managed, then attached, then external. Managed mode with zero managed nodes is rejected; existing-cluster and external-only modes require at least one resolved client overall. + +**Per-deployer attach support:** + +- **Local**: no attach. `ProcessDeployer` rejects `ClusterMode::ExistingCluster` outright; external nodes are supported. +- **Compose**: requires a compose descriptor. Services are taken from the descriptor or discovered from the running project; each container's labeled API port is inspected and turned into an `ExternalNodeSource` fed to `external_node_client`. Attached mode also wires restart/stop node control. See [Compose Deployer](deployer-compose.md). +- **K8s**: requires a k8s descriptor. Services matching the label selector are listed in the namespace (default `default`); each service's single TCP NodePort (preferring ports named `http` or `api`) becomes the endpoint. See [Kubernetes Deployer](deployer-k8s.md). + +--- + +## Deploy-Then-Attach + +Both container deployers return metadata that converts back into an attach descriptor, so one process can deploy a stack and a second scenario can attach to it: + +```rust,ignore +let (runner, metadata) = ComposeDeployer::::new() + .deploy_with_metadata(&scenario) + .await?; + +// Later, or elsewhere: attach to the same project. +let attached = KvScenarioBuilder::deployment_with(|_| KvTopology::new(3)) + .with_existing_cluster_from(&metadata)? + .build()?; +``` + +`K8sDeployer::deploy_with_metadata` provides the equivalent `K8sDeploymentMetadata` (namespace + label selector). + +--- + +## Use Cases + +- **Staging and live networks.** Point `with_external_only_nodes` at long-lived endpoints and run workloads and expectations against them; the framework never touches their lifecycle (`ExternalUncontrolled`). +- **Shared test stacks.** Deploy a compose or k8s stack once, attach many fast scenarios to it, and preserve the stack between runs with the deployer preserve env vars (see [Readiness, Retry, and Artifact Preservation](deployment-policies.md)). +- **Hybrid scenarios.** Combine managed nodes with an external dependency, for example a locally deployed cluster that must interoperate with a fixed remote peer. + +Manual clusters have their own external hooks: `add_external_sources` and `add_external_clients` on `ManualCluster` merge external endpoints into an imperatively driven cluster (see [ManualCluster](manual-cluster.md)). diff --git a/book/src/faq.md b/book/src/faq.md deleted file mode 100644 index 7986074..0000000 --- a/book/src/faq.md +++ /dev/null @@ -1,32 +0,0 @@ -# FAQ - -**Why block-oriented timing?** -Slots advance at a fixed rate (NTP-synchronized, 2s by default), so reasoning -about blocks and consensus intervals keeps assertions aligned with protocol -behavior rather than arbitrary wall-clock durations. - -**Can I reuse the same scenario across runners?** -Yes. The plan stays the same; swap runners (local, compose, k8s) to target -different environments. - -**When should I enable chaos workloads?** -Only when testing resilience or operational recovery; keep functional smoke -tests deterministic. - -**How long should runs be?** -The framework enforces a minimum of **2× slot duration** (4 seconds with default 2s slots), but practical recommendations: - -- **Smoke tests**: 30s minimum (~14 blocks with default 2s slots, 0.9 coefficient) -- **Transaction workloads**: 60s+ (~27 blocks) to observe inclusion patterns -- **Chaos tests**: 120s+ (~54 blocks) to allow recovery after restarts - -Very short runs (< 30s) risk false confidence—one or two lucky blocks don't prove liveness. - -**Do I always need seeded wallets?** -Only for transaction scenarios. Pure chaos scenarios may not require them, but -liveness checks still need nodes producing blocks. - -**What if expectations fail but workloads “look fine”?** -Trust expectations first—they capture the intended success criteria. Use the -observability signals and runner logs to pinpoint why the system missed the -target. diff --git a/book/src/framework-in-brief.md b/book/src/framework-in-brief.md new file mode 100644 index 0000000..75c9b06 --- /dev/null +++ b/book/src/framework-in-brief.md @@ -0,0 +1,971 @@ +
+ +# The Framework in Brief + +
+

the framework

+

The testing framework runs system-level tests against multi-process and multi-node deployments, from ordinary Rust code.

+

every test has four parts

+
start the systemlocal · Compose · Kubernetes→drive trafficworkloads→verify outcomesexpectations→tear downautomatic, reverse
+

the system under test can be

+
uniform clustersN nodes of one binarysingle binariesyours or third-partycomposed stacksclusters + processes, wiredalready runningattached / external
+

two ways to drive the test, using the same deployment code

+
a scenariodeclarative — the runner drivesoryour own codeimperative — ManualCluster
+

an Application defines the config, client, and deployment shape for one node kind

+ +
+ +The testing framework runs system-level tests against multi-process and multi-node deployments. A test starts the system — as local processes, a Compose project, or a Kubernetes deployment — drives traffic against it, verifies outcomes, and tears everything down, all from ordinary Rust code. The sections below explain the APIs used for each part. + +This page summarizes the main concepts and links each one to a full chapter. It uses the **job-processing stack** from `examples/multi_app` throughout: jobs enter a queue cluster, a worker process consumes them, and results are written to a result-store cluster. + +```mermaid +flowchart LR + WL["workload
enqueue 10 jobs"]:::sc --> Q["queue cluster
2 nodes"]:::cl + Q --> W["worker
one process"]:::pr + W --> R["result store
2 nodes"]:::cl + R --> EX["expectation
10 results visible"]:::sc + classDef cl stroke:#4a90d9,stroke-width:2.5px; + classDef pr stroke:#e08a3c,stroke-width:2.5px; + classDef sc stroke:#9b6dd6,stroke-width:2.5px; +``` + +
+
+ +
+

six terms

+

A Builder creates a Scenario. A Deployer starts the system. The Runner starts its Workloads and evaluates its Expectations.

+ +
+
+describe the test +
Scenariodeployment and test plan
+
Builderassembles the scenario
+
+
+act and check +
Workloadcreates activity
+
Expectationverifies an outcome
+
+
+execute it +
Deployerstarts the system
+
Runnerruns it end to end
+
+
+ +

these six terms appear throughout the examples below

+ +
+ +
+Scenariodeployment and test planwhat to deploy, what activity to run, what to verify, and for how long +Builderassembles the scenariothe chain of with_* calls + +Workloadcreates activitycode that runs against the live system: send jobs, restart nodes, cut the network +Expectationverifies an outcomecode that checks the result after the activity: all results present, cluster converged + +Deployerstarts the systemfor real, as local processes, a Compose project, or a Kubernetes deployment +Runnerruns it end to endwait until ready, run workloads, evaluate expectations, tear down +
+ +
+
+ +
+ +This map shows how the concepts on the page relate. Each §N badge links the concept to the section that explains it. + +
+ +{{#include framework-map.svg}} + +
+ +

Click an empty area to enlarge the map. Drag to pan; press Escape or use Close to return.

+ +
+ +
+

the whole test

+

This is the main body of the multi-app-e2e acceptance test. Run it with cargo test -p multi-app-e2e.

+

example used throughout: enqueue ten jobs in the examples/multi_app stack and check that ten results are stored

+ +```rust,ignore +let mut scenario = AppHost::scenario() // ① + .with_app(JobStackApp::new()) // ② + .with_run_duration(Duration::from_secs(10)) // ③ + .with_workload(EnqueueJobs::new(10)) // ④ + .with_expectation(AllJobsCompleted::new(10)) // ⑤ + .build()?; + +let runner = AppHostLocalDeployer::default() + .deploy(&scenario) // ⑥ + .await?; + +runner.run(&mut scenario).await?; // ⑦ +``` + +

① create the scenario · ② add the stack · ③ set the run limit · ④ add traffic · ⑤ add a check · ⑥ start locally · ⑦ run and clean up

+ +
+ +
    +
  • ① a scenario with no framework-managed nodes of its own — the composed stack provides the system → section 1
    • +
  • ② deploy the stack: two clusters and a process, wired together → section 3
    • +
  • ③ the run window (a maximum, not a timer you must fill) → section 4
    • +
  • ④ ⑤ create activity, verify outcomes → section 4
    • +
  • ⑥ where it runs: local processes here; other backends → section 10
    • +
  • ⑦ the runner order: readiness → workloads → cooldown → evaluate → teardown → section 1
    • +
+ +
+
+ +
+

the same builder, further

+

The helper API expresses a partition, random restarts, and a convergence check in one chain. Runnable as cargo run -p queue-examples --bin queue_dsl_demo.

+
produce400 jobs at 40/s→group A ✂ group Bsplit 20 s, then heal+⚡ random restartsevery 5–15 s→expect convergenceall 5 nodes at 400
+ +```rust,ignore +QueueScenario::nodes(5) + .produce(400).rate_per_sec(40).done() + .restart_nodes_randomly().every_secs(5, 15).done() + .partition(["node-0", "node-1"], ["node-2", "node-3", "node-4"]).hold_secs(20).done() + .expect_converged(400).within_secs(60) + .run_secs(120) + .await?; +``` + +

each helper adds ordinary workloads, expectations, and the capabilities they require. Tests can also use the explicit API

+ +
+ +The next two blocks use a second, simpler system, because chaos reads clearest on a uniform cluster: one five-node queue cluster, no worker or store. The scenario produces jobs against it while restarting random nodes and cutting the network in two, then checks that every node still converges: + +```mermaid +flowchart LR + WL["produce
400 jobs at 40/s"]:::sc --> A + subgraph A["partition group A"] + N0["node-0"]:::cl + N1["node-1"]:::cl + end + subgraph B["partition group B"] + N2["node-2"]:::cl + N3["node-3"]:::cl + N4["node-4"]:::cl + end + A -. "✂ split 20s, then heal" .- B + RR["⚡ random restarts
every 5–15s"]:::pr -.-> A + RR -.-> B + B --> EX["expect
all 5 nodes converge at 400"]:::sc + classDef cl stroke:#4a90d9,stroke-width:2.5px; + classDef pr stroke:#e08a3c,stroke-width:2.5px; + classDef sc stroke:#9b6dd6,stroke-width:2.5px; +``` + +First in the explicit API, compile-checked: + +```rust,ignore +let mut scenario = QueueScenarioBuilder::deployment_with(|_| QueueTopology::new(5)) + .enable_node_control() // restarts allowed + .with_network_control() // partitions allowed + .with_workload( + QueueProduceWorkload::new() // steady traffic + .operations(400) + .rate_per_sec(40) + .payload_prefix("soak"), + ) + .with_workload(RandomRestartWorkload::new( // random node restarts + Duration::from_secs(5), + Duration::from_secs(15), + Duration::from_secs(10), + )) + .with_workload(NetworkPartitionWorkload::new( // split, hold, heal + NetworkPartitionSpec::new(vec![ + vec!["node-0", "node-1"], + vec!["node-2", "node-3", "node-4"], + ]), + Duration::from_secs(20), + Duration::from_secs(20), + )) + .with_expectation(QueueConverges::new(400).timeout(Duration::from_secs(60))) + .with_run_duration(Duration::from_secs(120)) + .build()?; + +let runner = QueueLocalDeployer::default().deploy(&scenario).await?; + +runner.run(&mut scenario).await?; +``` + +Each helper in the shorter chain adds these same workloads, expectations, and capabilities. Tests can use the explicit API whenever the helpers do not cover what they need. + +The framework began with an API sketch in this style. Its current implementation separates that idea into scenarios, workloads, expectations, and deployment backends. The sections below show how those parts fit together. + +
+
+ +The numbered sections first explain this test, then cover manual control, state, existing deployments, backends, and observation. + +--- + +## 1 · Mental Model + +

lines ① and ⑦: scenario contents and runner order.

+ +
+

A scenario records what to deploy, what to run, what to check, and the runtime settings.

+
deploy→readiness→run workloads→cooldown→evaluate→reverse teardown
+

the runner executes the same order for uniform clusters, composed stacks, and existing deployments

+
+ +

The framework does not contain queue- or blockchain-specific node logic. An Application supplies the deployment shape, client type, config type, and readiness contract for one node kind. A scenario combines the system to deploy, the test behavior, and the runtime settings.

+ +The framework sees an application only through those four things, and `Application` captures exactly that (deploying is the deployer's job): + +```rust,ignore +pub trait Application: Send + Sync + 'static { + type Deployment: DeploymentDescriptor + Clone; // cluster shape + type NodeClient: Clone + Send + Sync; // how tests reach a node + type NodeConfig: Clone + Send + Sync; // per-node config type +} +``` + +Running a scenario (line ⑦) always follows the one lifecycle shown above. + +`Application` and `AppDeployment` answer different questions: + +| Concept | Describes | Example | +|---|---|---| +| `Application` | one node kind: topology, client, config, readiness | `QueueEnv` | +| `AppDeployment` | how one component or composed stack is prepared and exposed | `JobStackApp` | + +A uniform scenario is parameterized directly by an `Application`. A composed scenario uses `AppDeployment` values, which may provision clusters of several application types plus standalone processes. + +Tests can also target nodes the framework did not start. A cluster is *managed* when TF starts and removes it, *attached* when TF connects to it and has some control, or *external* when TF only has clients. The example uses managed clusters. Section 8 shows all three modes. + +

Next: the available ways to run a test.

+ +

Application, AppDeployment, and Environments · Scenario Model and Lifecycle

+ +--- + +## 2 · Entry Patterns + +

runner-driven scenarios and direct, step-by-step control.

+ +
+

The runner can drive a uniform cluster, a composed stack, or an existing deployment. ManualCluster leaves the test sequence to your code.

+
uniform cluster·composed stack·attached / external→runner
+

ManualCluster still starts and cleans up managed nodes. Your test code replaces the scenario runner

+
+ +

Most tests let the runner perform deployment, readiness checks, workloads, evaluation, and teardown. Tests that need step-by-step control can perform those actions directly. This choice is independent of ownership: ManualCluster, for example, gives your code control of the sequence while TF still starts and removes the nodes.

+ +
+ +```mermaid +flowchart TD + U["Uniform cluster
N identical nodes"]:::cl --> S["Scenario"]:::sc + A["Composed stack
the job stack — line ②"]:::sc --> S + X["Attached / external
clusters you already run"]:::cl --> S + S --> R["Runner
one lifecycle for all three"]:::sc + M["ManualCluster
managed nodes, you drive"] -.->|bypasses the runner| C["step-by-step node control"] + classDef cl stroke:#4a90d9,stroke-width:2.5px; + classDef sc stroke:#9b6dd6,stroke-width:2.5px; +``` + +
+ +Bypassing the runner changes who drives the nodes, not who owns them: `ManualCluster` nodes are still framework-managed. + +
+Decision table: which pattern fits which system + +| Shape of the system under test | Pattern | Read | +|---|---|---| +| N identical nodes of one binary | Uniform managed cluster | [Part IV](part-iv.md) | +| Several apps composed into one stack | `AppHost` + `AppDeployment` | [Part II](part-ii.md) | +| Already-running nodes you must not deploy | Attached / external sources | [section 8](#8--sources-and-ownership) | +| An external driver dictates every step | `ManualCluster`, or direct `DeployContext` for a composed stack | [section 5](#5--imperative-control) | + +
+ +

Next: what line ② deploys for the job-processing example.

+ +

Choosing an Entry Pattern

+ +--- + +## 3 · Composed Applications: the Job Stack + +

line ②: .with_app(JobStackApp::new()).

+ +
+

JobStackApp starts the queue and result store, passes their addresses to the worker, and returns access to all three.

+
root deployment→deploy dependencies→inject addresses→deploy dependants→stack handle
+

each resource is registered for cleanup as soon as it starts. Teardown stops the worker before its dependencies

+
+ +

JobStackApp implements AppDeployment. Its deploy method starts the two clusters, reads their runtime addresses, then starts the worker with both addresses.

+ +```rust,ignore +async fn deploy(self, ctx: &mut DeployContext) -> Result { + let queue = ctx + .deploy_and_expose(QueueLocalApp::nodes(self.queue_nodes)) // ① + .await?; + + let results = ctx + .deploy_and_expose(KvLocalApp::nodes(self.result_nodes)) + .await?; + + let queue_url = queue.first_client().ok_or("queue cluster has no clients")?.base_url().clone(); + let results_url = results.first_client().ok_or("result store has no clients")?.base_url().clone(); + + let worker = ctx + .deploy_and_expose(JobWorkerApp::new(queue_url, results_url)) // ② + .await?; + + let stack = JobStackHandle { queue, results, worker }; // ③ + ctx.expose(stack.clone())?; + + Ok(stack) +} +``` + +The aggregate returned to test code contains two uniform-cluster handles and one process handle: + +```rust,ignore +struct JobStackHandle { + queue: LocalAppCluster, + results: LocalAppCluster, + worker: LocalProcessHandle, +} +``` + +
    +
  • ① deploy_and_expose starts a child and publishes its handle for test code. Registering a second unnamed handle of the same type returns an error.
    • +
  • ② dependencies travel by constructor: the worker receives the URLs of the already-running clusters. The dependency endpoints are passed explicitly.
    • +
  • ③ the stack handle contains all three members. A test can retrieve the stack or retrieve an exposed child by type.
    • +
+ +The worker is the single-binary member. A `LaunchSpec` declares the process; a readiness closure gates it: + +```rust,ignore +let launch = LaunchSpec { + binary: worker_binary_provider().resolve()?, // section 9 + args: vec!["--queue-url".to_owned(), queue_url.to_string(), /* … */], + ..LaunchSpec::default() +}; + +let process = LocalProcessApp::new("job-worker", launch, endpoints, client) + .with_readiness(|_, client| async move { client.wait_ready().await }); +``` + +The deployment APIs provide the following lifecycle behavior: + +- Managed clusters use their configured HTTP or TCP readiness probe. A process uses its readiness closure. A custom deployment must not return from `deploy` until it is usable. +- Managed resources register for cleanup when they start. Cleanup runs in reverse order, so this example stops the worker before either cluster. If `deploy` fails partway through, resources already started are still removed. + +
+Which lifecycle operations each deployment path provides + +| Deployment path | Automatic teardown | Explicit control | +|---|---|---| +| uniform cluster | yes | `start_node`, `stop_node`, `restart_node`, readiness waits | +| `LocalProcessApp` | yes | `start`, `stop`, `restart`, `is_running` | +| custom deployment | when it composes managed adapters (they register with scenario cleanup) | only methods its handle implements | +| external | no | none without an adapter | + +
+ +

Next: how lines ③④⑤ send work through the deployed stack and check the result.

+ +

AppDeployment and DeployContext · One Binary: LocalProcessApp · Handle Ownership and Teardown · Composing Heterogeneous Stacks

+ +--- + +## 4 · Test Behavior + +

lines ③ ④ ⑤: duration, workload, and expectation.

+ +
+

The runner starts workloads, waits for the cooldown, then evaluates expectations.

+
workload⌁ through handles→deployed stack→expectation⌁ through handles
+

lines ③④⑤: workloads start concurrently, expectation failures are collected, and duration is a maximum

+
+ +

A workload sends requests or performs other activity against the deployed system. An expectation checks the resulting state. Both receive the scenario's typed handles, but the runner executes them in separate phases.

+ +The scenario registers both objects. `runner.run` calls them at the appropriate phases: + +```rust,ignore +let mut scenario = AppHost::scenario() + .with_app(JobStackApp::new()) + .with_run_duration(Duration::from_secs(10)) + .with_workload(EnqueueJobs::new(10)) // register activity + .with_expectation(AllJobsCompleted::new(10)) // register the check + .build()?; + +let runner = AppHostLocalDeployer::default().deploy(&scenario).await?; + +runner.run(&mut scenario).await?; // TF invokes both +``` + +
deploy + readiness→Workload::start(ctx)→cooldown→Expectation::evaluate(ctx)→cleanup
+ +The runner supplies the same `RunContext` to both callbacks. In an `AppHost` scenario, they use it to retrieve the typed handles exposed by `JobStackApp`. + +### The Workload + +`EnqueueJobs` implements TF's `Workload` trait. During the workload phase, the runner calls `start`; returning an error fails the run. + +```rust,ignore +#[async_trait] +impl Workload for EnqueueJobs { + fn name(&self) -> &str { + "enqueue_jobs" + } + + async fn start(&self, ctx: &RunContext) -> Result<(), DynError> { + let stack = ctx.require_app::()?; + let queue = stack + .queue() + .first_client() + .ok_or("queue cluster has no clients")?; + + for index in 0..self.count { + let response: EnqueueResponse = queue + .post("/queue/enqueue", &EnqueueRequest { payload: job_key(index) }) + .await?; + + if !response.accepted { + return Err(format!("queue rejected job {index}").into()); + } + } + + Ok(()) + } +} +``` + +### The Expectation + +`AllJobsCompleted` implements TF's `Expectation` trait. After workloads and cooldown, the runner calls `evaluate`; `Ok(())` passes this check and `Err(...)` reports an expectation failure. + +```rust,ignore +#[async_trait] +impl Expectation for AllJobsCompleted { + fn name(&self) -> &str { + "all_jobs_completed" + } + + async fn evaluate(&mut self, ctx: &RunContext) -> Result<(), DynError> { + let stack = ctx.require_app::()?; + let clients = stack.results().clients(); + let deadline = Instant::now() + self.timeout; + + while Instant::now() < deadline { + if all_results_are_visible(&clients, self.count).await? { + if !stack.worker().is_running().await { + return Err("job worker stopped before evaluation".into()); + } + + return Ok(()); + } + + tokio::time::sleep(Duration::from_millis(200)).await; + } + + Err(format!("job results did not converge within {:?}", self.timeout).into()) + } +} +``` + +This expectation polls every result-store node until all ten keys read `completed`, and checks that the worker is still running. + +The runner behaves as follows: + +- All workloads start **concurrently**; a panic is reported as a workload failure; an error ends the run immediately. +- The duration is a **maximum**: when every workload finishes early, cooldown starts early. +- Expectations have four phases: `init`, `start_capture`, `check_during_capture` (~1 s tick), and `evaluate` after cooldown. Failures **aggregate** rather than short-circuit. +- **Cooldown** is the settle window between traffic and evaluation. +- **Runtime extensions** are typed scenario-lifetime services prepared after readiness; the app layer is one, which is why `with_app` is once per scenario. + +### Responsibility Split + +| TF behavior | Application or test responsibility | +|---|---| +| Nodes are ready before any workload starts | Readiness paths and probes are correct for your node | +| Workloads run concurrently and panics are reported as failures | Every workload terminates; an unbounded one blocks the run | +| Every expectation evaluates and failures aggregate | Expectations poll with their own deadline instead of assuming fresh state | +| Managed resources release in reverse acquisition order, also on partial failure | Custom adapters register cleanup immediately after acquiring a resource | +| Artifacts survive a panic or an explicit preservation setting | Compose images are built beforehand; env-provider binaries are pointed at real files | + +

Next: running the same deployments without workloads and expectations.

+ +

Workloads and Concurrency · Expectations and Evaluation · Runtime Extensions

+ +--- + +## 5 · Imperative Control + +

direct control from a Rust test or an external harness.

+ +
+

Tests can control a uniform cluster or deploy a composed stack directly, without using the scenario runner.

+
your test→start→call + assert→restart→cleanup
+

ManualCluster for one uniform cluster · DeployContext for an AppDeployment tree

+
+ +

A BDD harness, debugging tool, or ordinary Rust test can control a uniform cluster through ManualCluster. It can also deploy an existing composed stack through DeployContext. In both cases the test code decides when to call, stop, or restart each component.

+ +Runner-driven and manually driven tests use the same cluster and process deployment code: + +```mermaid +flowchart TB + R["runner sequences the test"]:::driver --> SETUP["system setup"]:::setup + U["your Rust code / BDD steps
sequence the test"]:::driver --> SETUP + + SETUP --> C["one uniform cluster"]:::shape + SETUP --> S["composed stack"]:::shape + S --> CC["uniform child clusters"]:::shape + S --> P["standalone processes"]:::process + + C --> N["node bring-up
topology → ports + peers → config → binary → start → readiness"]:::engine + CC --> N + P --> SP["process bring-up
launch settings → binary → start → readiness"]:::engine + + N --> H["running resources
clients · lifecycle control · reverse cleanup"]:::runtime + SP --> H + + classDef driver stroke:#9b6dd6,stroke-width:2.5px; + classDef setup stroke:#777,stroke-width:2px,stroke-dasharray:4 3; + classDef shape stroke:#4a90d9,stroke-width:2.5px; + classDef process stroke:#e08a3c,stroke-width:2.5px; + classDef engine stroke:#777,stroke-width:2px; + classDef runtime stroke:#4caf7d,stroke-width:2.5px; +``` + +A uniform cluster can be the whole system or one child of a composed stack. Both use the same node startup path. A composed stack can also contain standalone processes. The runner, ManualCluster, and DeployContext call these shared deployment APIs in different ways. + +In API terms, `ManualCluster` reuses the `Application` definition and local cluster implementation for `QueueEnv`. It does not execute `AppDeployment::deploy` or create a `DeployContext`; direct composed-stack deployment is the separate path shown later in this section. + +### One Uniform Cluster: `ManualCluster` + +This test uses the queue from the same job-stack example. It is a normal async test: TF starts and owns the processes, while the test owns the sequence and assertions. + +```rust,ignore +#[tokio::test] +async fn drives_queue_cluster_without_a_scenario() -> Result<(), DynError> { + let cluster = ManualCluster::::from_topology(QueueTopology::new(2)); + + let node0 = cluster.start_node("node-0").await?.client; + let node1 = cluster.start_node("node-1").await?.client; + cluster.wait_network_ready().await?; + + enqueue(&node0, "manual-job").await?; + wait_for_queue_len(&[node0, node1], 1).await?; + + cluster.restart_node("node-1").await?; + cluster.wait_node_ready("node-1").await?; + + let restarted = cluster + .node_client("node-1") + .ok_or("node-1 client missing after restart")?; + wait_for_queue_len(&[restarted], 1).await?; + + Ok(()) +} +``` + +Dropping the cluster stops every child process, including on an early `?` or panic. `StartNodeOptions` adds peer selection, config overrides and patches, persistent or snapshot directories, extra arguments, and per-start timeouts. + +### A Composed Stack: Direct `AppDeployment` + +The same `JobStackApp` recipe used by `.with_app(...)` can be deployed directly. The returned aggregate exposes every component handle, so ordinary Rust can use and control the queue cluster, result-store cluster, and worker process: + +```rust,ignore +let mut deployment = + DeployContext::::new(AppHostTopology, NodeClients::default()); +let stack = deployment.deploy(JobStackApp::new()).await?; + +assert_eq!(stack.queue().node_count(), 2); +assert_eq!(stack.results().node_count(), 2); + +let queue = stack.queue().first_client().ok_or("queue has no clients")?; +let results = stack.results().clients(); +let worker = stack.worker().clone(); + +worker.restart().await?; +enqueue(&queue, "imperative-job").await?; +wait_for_completed_result(&results, "imperative-job").await?; + +drop(deployment); // reverse cleanup for the whole stack +assert!(!worker.is_running().await); +``` + +In this form, `DeployContext` keeps the child `AppDeployment`s, their typed handles, and the cleanup callbacks. Dropping it runs cleanup in reverse order, just as scenario teardown does. + +| | Declarative scenario | `ManualCluster` | Direct `AppDeployment` | +|---|---|---|---| +| Who sequences behavior? | TF's runner | Your Rust code or external harness | Your Rust code or external harness | +| System shape | Uniform cluster or composed stack | One uniform cluster | One component or composed stack | +| Reusable definition | `Application`, optionally `AppDeployment` | `Application` | `AppDeployment` and its child apps | +| Test behavior | Workloads and expectations | Client calls, helpers, assertions | Handle calls, helpers, assertions | +| Cleanup owner | Scenario runtime | `ManualCluster` | `DeployContext` | + +Manual control is also available without abandoning a scenario. A scenario can opt into node control with `with_node_control()`, and app deployments return `ClusterHandle` / `LocalAppCluster` and `LocalProcessHandle` values with direct lifecycle methods. + +

Next: how TF assigns ports and how applications produce node configuration.

+ +

Scenario Capabilities · Chaos and Controlled Failure · ManualCluster: Imperative Node Control

+ +--- + +## 6 · Configuration and Deployment Policy + +

ports, peer addresses, node config, readiness, and retry.

+ +
+

TF assigns ports and peers. Application code builds the node config. The selected backend writes the files and starts the process.

+
framework inputs→typed app config→launched + ready node
+

deployment policy controls readiness, retry, and retained artifacts without changing the application config

+
+ +

TF allocates collision-free ports and prepares each node's peer list. Application code converts those values into the config and command expected by its binary. The local, Compose, or Kubernetes backend delivers the files, starts the binary, and applies the requested readiness and retry policy.

+ +```mermaid +flowchart TB + T["1 · topology
the test asks for three queue nodes"]:::input + V["2 · framework prepares node 1
identity · reserved port · peer addresses"]:::framework + C["3 · application builds queue configuration
node id · HTTP port · peers · sync interval"]:::app + B["4 · backend launches the node
write config file · resolve binary · pass args and environment · start"]:::backend + H["5 · ready running resource
typed client · lifecycle control · registered cleanup"]:::runtime + + T --> V --> C --> B --> H + + POL["deployment policy
readiness · retry · retained artifacts"]:::policy -. "governs launch" .-> B + POL -. "gates access" .-> H + + classDef input stroke:#777,stroke-width:2px; + classDef framework stroke:#4a90d9,stroke-width:2.5px; + classDef app stroke:#9b6dd6,stroke-width:2.5px; + classDef backend stroke:#777,stroke-width:2px,stroke-dasharray:4 3; + classDef runtime stroke:#4caf7d,stroke-width:2.5px; + classDef policy stroke:#c89b3c,stroke-width:2.5px; +``` + +### Application-Owned Configuration + +The queue's real config builder receives one framework-generated node view plus all peer views and returns the value understood by the queue binary: + +```rust,ignore +fn build_cluster_node_config( + node: &ClusterNodeView, + peers: &[ClusterPeerView], +) -> Result { + Ok(QueueNodeConfig { + node_id: node.index() as u64, + http_port: node.network_port(), + peers: peers + .iter() + .map(|peer| QueuePeerInfo { + node_id: peer.index() as u64, + http_address: peer.authority(), + }) + .collect(), + sync_interval_ms: 500, + }) +} +``` + +The local adapter then says where the binary comes from, how to serialize that typed config, and which port is its API: + +```rust,ignore +fn local_process_spec() -> LocalProcessSpec { + LocalProcessSpec::new("QUEUE_NODE_BIN") + .with_binary_provider(queue_binary_provider()) + .with_rust_log("queue_node=info") +} + +fn render_local_config(config: &QueueNodeConfig) -> Result, DynError> { + yaml_node_config(config) +} + +fn http_api_port(config: &QueueNodeConfig) -> u16 { + config.http_port +} +``` + +Scenario deployment, `ManualCluster`, and uniform child clusters all call these same application functions. + +### Delivering Configuration: Local Files and cfgsync + +TF renders the same per-node artifacts for each backend. The local backend writes them directly, while container backends deliver them through cfgsync: + +```mermaid +flowchart TB + C["typed per-node configuration"]:::app --> A["rendered per-node artifacts
config file + any additional files"]:::artifact + A --> L["local backend
write directly into the node working directory"]:::local + A --> S["container backends
serve artifacts through cfgsync"]:::container + S --> F["cfgsync client in each container
register · fetch · write files"]:::container + L --> N["start node binary"]:::process + F --> N + + classDef app stroke:#9b6dd6,stroke-width:2.5px; + classDef artifact stroke:#777,stroke-width:2px,stroke-dasharray:4 3; + classDef local stroke:#4a90d9,stroke-width:2.5px; + classDef container stroke:#c89b3c,stroke-width:2.5px; + classDef process stroke:#e08a3c,stroke-width:2.5px; +``` + +Locally, TF writes files into the process working directory. Compose and Kubernetes nodes cannot see that host directory. For those backends, a cfgsync server holds each node's artifacts, and a client inside the container fetches and writes them before executing the node. cfgsync only transports generated configuration. It does not preserve application state or create snapshots. + +### Test-Side Changes for One Start + +Tests normally keep the generated ports and peers and patch only the behavior they care about. The Section 5 manual-cluster test really starts its second node with a faster synchronization interval: + +```rust,ignore +let node1 = cluster + .start_node_with( + "node-1", + StartNodeOptions::::default().create_patch(|mut config| { + config.sync_interval_ms = 50; + Ok(config) + }), + ) + .await? + .client; +``` + +Use `config_override` only when the test intends to replace the complete generated config. `config_patch` preserves framework-assigned values unless the callback deliberately changes them. + +### Deployment Policy + +The node config is passed to the application binary. `DeploymentPolicy` separately controls TF's readiness checks, retry behavior, cleanup, and artifact retention: + +```rust,ignore +let policy = DeploymentPolicy { + readiness_enabled: true, + readiness_requirement: HttpReadinessRequirement::AtLeast(2), + retry_policy: Some(RetryPolicy::new( + 5, + Duration::from_millis(500), + Duration::from_secs(5), + )), + cleanup_policy: CleanupPolicy::new(true), + ..DeploymentPolicy::default() +}; +``` + +For the primary scenario cluster, set this through `.with_deployment_policy(policy)`. A child cluster created by an `AppDeployment` carries policy on its `ClusterRequest`. `deploy_local_cluster(...)` uses the default policy. + +
+Readinessrequirement (all nodes / quorum) + probe (HTTP path or TCP) with retry budgets; SLOW_TEST_ENV doubles timeouts +Retrythe local backend respawns a failed cluster attempt with backoff; Compose and Kubernetes currently do not repeat deployment +Artifactslocal files live in node working directories; container backends receive rendered config through cfgsync +Retentionpreserve_artifacts, TF_KEEP_LOGS, or a panic keep local working directories for post-mortems +
+ +

Next: what happens to node state during restart and restore.

+ +

Ports, Peers, Node Config, and Readiness · Static Artifacts and cfgsync · Readiness, Retry, and Artifact Preservation · Diagnostics and Retained Artifacts

+ +--- + +## 7 · State and Reproducibility + +

working directories, snapshot input, config changes, and deterministic deployment seeds.

+ +
+

A restart keeps the node's working directory. A snapshot starts a new node from copied state.

+
+
persist_dirchoose a stable working-directory location
+
snapshot_dirsaved state copied in at spawn
+
repeatable runsconfig override / patch · deterministic seeds
+
+

ordinary restart: same working directory · restore: new working directory seeded from a snapshot

+
+ +
+Restartstop and respawn the same node in its existing working directory, including state written there +persist_dirplace that working directory at a stable, findable path rather than a framework temporary directory +snapshot_dircopy saved state into the fresh working directory at spawn — the base for stop → snapshot → restore tests +Configconfig_override replaces the generated per-node config; config_patch transforms it +Seedswith_deployment_seed feeds deterministic deployment providers +
+ +

Next: connecting the same test to clusters that TF did not start.

+ +

Persistence, Snapshots, and Recovery Testing · Seeds and Reproducibility

+ +--- + +## 8 · Cluster Sources and Ownership + +

managed, attached, and external clusters use one request API but provide different levels of control.

+ +
+

You request every cluster through the same API. Available control depends on whether TF started it or connected to it.

+
+
manageddeployed and torn down
+
attachedpartially driven
+
externalclients only
+
+

deploy_cluster(ClusterRequest::…) returns clients in all three modes; only managed clusters always provide full lifecycle control

+
+ +

The job-stack example asks TF to start both clusters. A test can instead connect to an existing deployment. deploy_cluster handles all three cases and returns node clients for each one. Full start, stop, and restart control is guaranteed only when TF manages the nodes.

+ +```rust,ignore +let cluster = ctx.deploy_cluster(ClusterRequest::managed(deployment)).await?; +let attached = ctx.deploy_cluster(ClusterRequest::attached(existing)).await?; +let external = ctx.deploy_cluster(ClusterRequest::external(endpoints)).await?; +``` + +| | Managed | Attached | External | +|---|---|---|---| +| Clients | ✓ | ✓ | ✓ | +| Node control | ✓ | per backend | — | +| Readiness waits | ✓ | ✓ | — | +| Torn down by the framework | ✓ | — | — | + +The scenario builder exposes the same modes through `with_existing_cluster`, `with_external_nodes`, and `with_external_only_nodes`. Workloads and expectations use node clients, so they do not need to change when a test moves from a locally managed cluster to an existing deployment. + +

Next: how TF finds the binaries it has been asked to start.

+ +

Shared Cluster Provisioning · Existing and External Clusters

+ +--- + +## 9 · Binary Resolution + +

the worker_binary_provider() call inside line ②.

+ +
+

A binary provider returns an executable path. Providers can try an override, a local build, or a download in order.

+
explicit path·env var·local build·checksummed download→binary
+

the worker_binary_provider() call inside line ② — FallbackBinaryProvider tries sources in order, with a cache and cross-process locking

+
+ +

Every process TF starts needs an executable path. A binary provider can return an explicit path, read one from an environment variable, build the binary locally, or download an artifact. A fallback provider tries several providers in order.

+ +The job worker's real provider chain tries an env var override and falls back to a local build: + +```rust,ignore +FallbackBinaryProvider::new([ + Arc::new(EnvBinaryProvider::new("MULTI_APP_JOB_WORKER_BIN")), + Arc::new(BuildBinaryProvider { + command: BuildCommand::new("cargo") + .with_args(["build", "-p", "multi-app-job-worker", "--bin", "multi-app-job-worker"]), + output_path: "target/debug/multi-app-job-worker".into(), + working_dir: Some(workspace), + lock_dir: None, + }), +]) +``` + +The available providers are explicit path, environment variable, local build, and checksummed download with post-processing. `FallbackBinaryProvider` chains them, with a resolution cache and cross-process locking. + +

Next: selecting the local, Compose, or Kubernetes backend.

+ +

Binary Providers

+ +--- + +## 10 · Deployment Backends + +

line ⑥: local, Compose, and Kubernetes deployment.

+ +
+

Uniform scenarios can run locally, with Compose, or on Kubernetes. Backend capabilities currently differ.

+
+
localprocesses · full node control · app composition
+
Composecontainers · cfgsync · restart
+
KubernetesHelm · cfgsync · manual mode
+
+

line ⑥ picks the backend; app composition is local-only today

+
+ +

Line ⑥ selects the local backend. Uniform scenarios can also use the Compose and Kubernetes deployers. The table lists the deployment and control features currently implemented by each backend.

+ +| | Local | Compose | Kubernetes | +|---|---|---|---| +| Node startup | processes + temp dirs | generated compose file | Helm chart + values | +| Config delivery | filesystem | cfgsync artifacts | cfgsync artifacts | +| Node control | full | restart | manual mode only | +| App composition | ✓ | — | — | +| Attach / external | external nodes | ✓ | ✓ | + +App composition currently runs only on the local backend. Uniform scenarios run on all three. Local working directories are temporary and removed after a successful run unless `TF_KEEP_LOGS` or `preserve_artifacts` is set. They are also retained after a panic. + +

Next: reading changing application state during a test.

+ +

Capability Matrix · Local · Compose · Kubernetes · Diagnostics

+ +--- + +## 11 · Observability + +

continuous state capture for tests, plus external metrics, logs, and traces.

+ +
+

Tests read application state through observation. Metrics, logs, and traces are exported through telemetry.

+
+
observationan Observer polls on a cadence — snapshots · history · subscriptions
+
telemetrymetrics · logs · tracing → Grafana / OTLP
+
+

observation is a runtime extension; telemetry is a backend capability

+
+ +
+
+

Continuous observation

+ +An `Observer` polls application state on a cadence; tests read `latest_snapshot()`, `history()`, or `subscribe()` from an `ObservationHandle`. Sources can be dynamic, re-queried as nodes come and go. + +
+
+

Telemetry

+ +Metrics, logs, tracing, and Grafana/OTLP endpoints are configured through the observability capability and environment variables. They serve external monitoring, not test logic. + +
+
+ +Continuous observation is implemented as a **runtime extension** (section 4); telemetry is a backend capability configured on the scenario, not an extension. + +

Next: matching common test cases to the APIs covered above.

+ +

Continuous Observation · Telemetry and External Observability · Runtime Extensions

+ +--- + +## 12 · Choosing What to Test + +

common test cases and the APIs normally used for them.

+ +
+

The table below shows which APIs are normally used for each kind of test.

+
convergencerestart recoverysnapshot restorefailoverchaos under loadload / soakthird-party binarieslive networks
+
+ +
+ +| Test kind | Framework tools | Read | +|---|---|---| +| Convergence / consistency | traffic workload + expectation polling every node client | [Workloads](workloads.md), [Expectations](expectations.md) | +| Recovery across a restart | `restart_node` or process `restart()`; working directories survive restarts | [Imperative Control](#5--imperative-control), [Persistence](persistence.md) | +| Restore from saved state | `snapshot_dir` seeding + an expectation on the restored data | [Persistence](persistence.md) | +| Role failover | find the role through observation, restart it via node control, expect a new holder | [Chaos](chaos.md), [Observation](observation.md) | +| Chaos under load | traffic workload + `RandomRestartWorkload` / the chaos builder in one scenario | [Chaos](chaos.md) | +| Load / soak | bounded traffic workloads paced across the run window | [Workloads](workloads.md) | +| Deployment and config validation | the same uniform scenario per backend, plus readiness policy | [Backends](#10--deployment-backends), [Config](#6--configuration-and-deployment-policy) | +| Behavior of a third-party binary | `LocalProcessApp` + `LaunchSpec` around the unmodified executable | [Section 3](#3--composed-applications-the-job-stack) | +| Against a live network | attached or external sources with unchanged workloads and expectations | [Sources](#8--cluster-sources-and-ownership) | + +
+ +
diff --git a/book/src/framework-map.svg b/book/src/framework-map.svg new file mode 100644 index 0000000..25cd58a --- /dev/null +++ b/book/src/framework-map.svg @@ -0,0 +1,313 @@ + + + + + + + + + + +1 · DEFINE — you write this; the builder assembles it into one artifact + + +Verb DSL +one verb per line — eachexpands to builder calls andenables what it needs + +expands to + +Builder +the with_*() chain —validates and assemblesthe whole test + + +build()? + +§3 +impl AppDeployment +your composition code —deploys children in order,exposes typed handles + + +becomes the +root of + + + +Scenario §1 — the whole test, described as data + +only one with_app per scenario + + +system under test §2 +what to deploy — one of three shapes + +Uniform cluster +N identical nodes of one binary + +§3 +Composed stack — root AppDeployment + +queue +cluster + +worker +process + +custom +component + + + + +handle + +handle + +handle +children start in dependency order,readiness-gated; each exposes a typedhandle that test code fetches later + +§8 +Attached / external sources +already running — never deployed by the framework +ownership mode — decides available operations and teardown §8 + +managed +full ops · torn down + +attached +partial ops · kept + +external +clients only · kept + + +test behavior §4 +what the run does, and how it is judged + +Workloads +create activity through handles —send jobs · restart nodes · cut the net + +Expectations +verify outcomes; failures aggregate + + +runtime policy §6 +duration — the run window; a maximum, +not a timer to fill §4 +readiness — all / quorum + HTTP or TCP probe +retry — managed spawns retry with backoff +capabilities the test requests up front: +with_node_control() §5 +with_network_control() §5 + + + +connected straight through to the run — +never deployed, never torn down + + +deploy(&scenario) — provision +the system on one backend §10 + + +2 · DEPLOY — the framework starts the system for real + +§9 +Binary providers +every launched executable —path · env var · cargo build ·download; chained fallbacks,cached, cross-process locked + +supply + +§7 +cfgsync +typed app config renderedinto per-node artifacts forthe container backends + +delivers +configs + + + +Deployer §10 — one scenario model, three substrates + +local — processes in temp dirs +full node control · the only backend runningapp composition today §10 + +compose — generated project +containers; configs via cfgsync ·restart-level node control + +kubernetes — Helm chart + values +configs via cfgsync · node control only inmanual mode §10 + + +§7 +Per-node working dirs +ports · peer view · typed config,written before launch §6persist_dir — pin and reusesnapshot_dir — seed saved statekept on panic / TF_KEEP_LOGS + + +writes + + +§5 +ManualCluster +same provisioning, no Runner —your code sequences start_node ·stop_node · restart_node ·wait_node_ready; nodes stillframework-managed and torn down + + +provisions + + +starts nodes and processes + +returns a Runner + + +drives nodes step-by-step — no Runner, no lifecycle §5 + + +3 · RUN — a fixed lifecycle; test code reaches the system only through the RunContext + + +The running system +started by the deployer — reached via the surfaces on the right + +managed — framework-owned §8 + +node-0 +app state + +node-1 +app state + + +✂ + +node-2 +app state + +worker +process +each node: own working dir · ports · typed config §6 §7 +✂ = a partition cut point (network control) + +attached / external — yours §8 +left running at teardown · attached = partialcontrol, external = clients only + + +§11 +Telemetry +Grafana / OTLP — externalmonitoring, not test logic + +emits metrics · logs · traces + + +RunContext +handed to every workload and +expectation — the only door in + +node clients +typed HTTP client per node + +app handles §3 +require_app::<T>() + +node control §5 +start · stop · restart a node + +network control §5 +partition · heal + reconnect assist + +observer §11 +polls app state on a cadence + + +call + +drive + +restart + +cut · heal + +poll + + + +Runner §1 +runner.run(&mut scenario) + +1 · wait until ready +probe per readiness policy —all nodes or quorum §6 + + +2 · run workloads +all start concurrently §4 —a panic becomes a clean failure + + +3 · cooldown +settle window — starts earlywhen workloads finish early + + +4 · evaluate expectations +every one runs — failuresaggregate, no short-circuit + + +5 · tear down +managed only, in reverseacquisition order §3 + + +builds & +hands over + + +steps 2 and 4 run the workloads and expectations you defined above §4 + + +releases managed resources in reverse acquisition order — attached / external left running + + +READING THE MAP +top → bottom = time (define → deploy → run) · solid arrow = builds / owns · dashed arrow = runtime access · §N = the section of this page that unpacks it + +cluster of nodes + +single process + +typed handle / access + +scenario and runtime machinery + +dashed box = not owned by the framework + diff --git a/book/src/glossary.md b/book/src/glossary.md index 4e5a55b..bdee25f 100644 --- a/book/src/glossary.md +++ b/book/src/glossary.md @@ -1,46 +1,51 @@ # Glossary -- **Node**: process that participates in consensus and produces blocks. -- **Deployer**: component that provisions infrastructure (spawns processes, - creates containers, or launches pods), waits for readiness, and returns a - Runner. Examples: LocalDeployer, ComposeDeployer, K8sDeployer. -- **Runner**: component returned by deployers that orchestrates scenario - execution—starts workloads, observes signals, evaluates expectations, and - triggers cleanup. -- **Workload**: traffic or behavior generator that exercises the system during a - scenario run. -- **Expectation**: post-run assertion that judges whether the system met the - intended success criteria. -- **Topology**: declarative description of the cluster shape, roles, and - high-level parameters for a scenario. -- **Scenario**: immutable plan combining topology, workloads, expectations, and - run duration. -- **Blockfeed**: stream of block observations used for liveness or inclusion - signals during a run. -- **Control capability**: the ability for a runner to start, stop, or restart - nodes, used by chaos workloads. -- **Slot duration**: time interval between consensus rounds in Cryptarchia. Blocks - are produced at multiples of the slot duration based on lottery outcomes. -- **Block cadence**: observed rate of block production in a live network, measured - in blocks per second or seconds per block. -- **Cooldown**: waiting period after a chaos action (e.g., node restart) before - triggering the next action, allowing the system to stabilize. -- **Run window**: total duration a scenario executes, specified via - `with_run_duration()`. Framework auto-extends to at least 2× slot duration. -- **Readiness probe**: health check performed by runners to ensure nodes are - reachable and responsive before starting workloads. Prevents false negatives - from premature traffic. -- **Liveness**: property that the system continues making progress (producing - blocks) under specified conditions. Contrasts with safety/correctness which - verifies that state transitions are accurate. -- **State assertion**: expectation that verifies specific values in the system - state (e.g., wallet balances, UTXO sets) rather than just progress signals. - Also called "correctness expectations." -- **Mantle transaction**: transaction type in Logos that can contain UTXO transfers - (LedgerTx) and operations (Op). +This glossary gives short definitions of the terms used throughout this book, with a link to the chapter that owns each. --- -## External Resources +**AppDeployment**: the trait a composable application implements: `deploy(self, ctx)` builds the application (processes, clusters, wiring) and returns its handle. Deployments consume themselves and must be `Clone` so the factory can re-run them. See [AppDeployment and DeployContext](app-deployment.md). -- **[Logos Project Documentation](https://nomos-tech.notion.site/project)** — Protocol specifications, node internals, and architecture details +**AppHost**: the app-layer entry point. `AppHost::scenario()` returns a scenario builder over a zero-node environment (`AppHostEnv`) so the composed application, not a managed topology, is the system under test. See [AppHost and with_app](app-host.md). + +**Application (environment)**: the trait that defines one application's deployment descriptor, node client, node config, and readiness path for the framework. Often called the environment; implemented once per application. See [Application, AppDeployment, and Environments](application-model.md). + +**Binary Provider**: the local deployer's strategy for producing a node executable: explicit path, env-var override, build command, checksum-verified download, or an ordered fallback chain. Resolution is cached and cross-process locked. See [Binary Providers](binary-providers.md). + +**cfgsync artifact**: a per-node configuration file rendered from typed app config by the cfgsync pipeline and served to nodes at container startup; how the compose and k8s deployers get configs into containers. See [Static Artifacts and cfgsync](cfgsync.md). + +**Cleanup Guard**: the core runner's teardown hook (`CleanupGuard`). Guards are registered as resources are acquired and run when the scenario runtime is released; the app layer groups its managed resources in a LIFO cleanup stack. See [Handle Ownership and Teardown](handles-teardown.md). + +**Deployer**: the object that turns a scenario definition into running infrastructure (`deployer.deploy(&scenario)` → runner): local processes, a compose stack, or a Kubernetes namespace. See [Capability Matrix](capability-matrix.md). + +**Deployment Plan / Topology**: the application-defined descriptor of what to deploy (node count and layout), owned by the `Application::Deployment` type and consumed by every backend. See [Topology and Deployment Plans](topology.md). + +**Deployment Policy**: per-scenario knobs for deploy behavior: readiness on/off and requirement, optional retry with backoff, and artifact preservation (`CleanupPolicy`). Set with `with_deployment_policy`. See [Readiness, Retry, and Artifact Preservation](deployment-policies.md). + +**Entry Pattern**: one of the three declarative ways into the scenario runtime (uniform managed cluster, AppHost composed stack, attached/external nodes), or imperative control through `ManualCluster`. See [Choosing an Entry Pattern](entry-patterns.md). + +**Existing Cluster / External Node**: sources that plug already-running infrastructure into a scenario instead of deploying it: `ExistingCluster` for a whole cluster, `ExternalNodeSource` for a single endpoint. See [Existing and External Clusters](external-clusters.md). + +**Expectation**: a post-run (and cooldown-aware) assertion about the system's end state, registered with `with_expectation`; expectations decide whether the scenario passed. See [Expectations and Evaluation](expectations.md). + +**Handle (typed / named)**: a cheaply clonable access or control value exposed by a deployment and fetched by workloads (`require_app::()`), keyed by concrete type plus an optional instance name. Managed lifetime belongs to scenario cleanup rather than handle clones. See [Handle Ownership and Teardown](handles-teardown.md). + +**Cluster Provisioner**: a backend adapter that turns a managed, attached, or external `ClusterRequest` into common clients, controls, readiness, and optional cleanup. See [Shared Cluster Provisioning](cluster-provisioning.md). + +**Verb Layer**: optional typed syntax that expands domain actions into ordinary workloads, expectations, and capability requests. See [The Verb Layer](verb-layer.md). + +**ManualCluster**: imperative node orchestration that bypasses the scenario runner: start, stop, restart, and probe named nodes directly. Use it for interactive debugging and bespoke lifecycles. See [ManualCluster: Imperative Node Control](manual-cluster.md). + +**Observation**: the continuous observation runtime: named `ObservedSource`s polled into snapshots and history that workloads and expectations read through an `ObservationHandle`. Test-visible application state, as opposed to Telemetry. See [Continuous Observation](observation.md). + +**Runner**: what a deployer returns after a successful deploy; `runner.run(&mut scenario)` executes workloads, evaluates expectations, and tears the run down. See [Scenario Model and Lifecycle](scenario-model.md). + +**Runtime Extension**: a typed value prepared before workloads start and shared through the `RunContext` (one instance per type). The app layer's `AppRuntime` is a runtime extension. See [Runtime Extensions](runtime-extensions.md). + +**Scenario**: the complete declarative test definition produced by a `ScenarioBuilder`: deployment, workloads, expectations, run duration, policies, and extensions, all evaluated by one runtime regardless of entry pattern. See [Scenario Model and Lifecycle](scenario-model.md). + +**Seed**: the value (`DeploymentSeed`, set via `with_deployment_seed`) that makes generated deployments deterministic, so a failing run can be replayed exactly. See [Seeds and Reproducibility](seeds.md). + +**Telemetry**: metrics, logs, and tracing reached through external endpoints (`ObservabilityInputs`, Prometheus, Grafana); operational visibility, as opposed to the Observation runtime's test-visible state. See [Telemetry and External Observability](telemetry.md). + +**Workload**: active behavior during the run: a named task (`trait Workload`) started against the `RunContext` that drives traffic or chaos while the scenario clock runs. See [Workloads and Concurrency](workloads.md). diff --git a/book/src/handles-teardown.md b/book/src/handles-teardown.md new file mode 100644 index 0000000..af74f09 --- /dev/null +++ b/book/src/handles-teardown.md @@ -0,0 +1,115 @@ +# Handle Ownership and Teardown + +Handles provide typed access to deployed applications. The scenario runtime owns managed resource lifetime separately through a cleanup stack. + +Cloning a handle preserves access to its shared state, but does not extend a process or cluster beyond the run that created it. + +--- + +## Typed and Named Handles + +The registry keys every handle by concrete type plus name (`TypeId` and a string). An empty name is the default handle for that type. + +| Operation | Key | On conflict | +|---|---|---| +| `expose(handle)` | `(TypeId::of::(), "")` | `AppDeployError::DuplicateHandle` | +| `expose_named(name, handle)` | `(TypeId::of::(), name)` | `AppDeployError::DuplicateHandle` | + +Duplicate exposure is an error rather than a replacement. Use distinct names when a scenario exposes multiple values of one handle type, then retrieve them with `app_named` or `require_app_named`. + +Missing handles are typed runtime errors. `require::()` and `require_named::(name)` return `AppDeployError::HandleMissing` with the requested Rust type and instance name. + +--- + +## What a Handle Means + +```rust,ignore +pub trait AppHandle: Clone + Send + Sync + 'static {} + +impl AppHandle for T +where + T: Clone + Send + Sync + 'static, +{} +``` + +An app handle can be a client, a control surface, or a domain aggregate such as `JobStackHandle`. Retrieval clones it so workloads can use it without borrowing the registry. + +The registry lookup uses `TypeId`. Requesting the wrong concrete type is therefore a runtime miss, not a compile-time error. Prefer specific handle types or domain newtypes over primitives whose role is unclear. + +Cloneability is about access, not ownership of the deployment. `LocalProcessHandle` clones share process state and controls; `ClusterHandle` clones share clients and control adapters. Scenario cleanup remains authoritative for managed lifetime. + +--- + +## Managed Lifetime + +Every framework adapter that acquires a managed resource registers a cleanup guard immediately. `DeployContext` collects those guards in acquisition order and transfers the stack to the scenario runtime after successful preparation. + +```mermaid +flowchart LR + D["deploy child"] --> G["register cleanup guard"] + G --> H["return and optionally expose handle"] + H --> R["scenario runs"] + R --> C["cleanup stack: last acquired, first released"] +``` + +This produces two parallel structures: + +| Structure | Contains | Purpose | Release order | +|---|---|---|---| +| Handle registry | Cloneable typed access values | Workload and expectation lookup | Reverse exposure order | +| Cleanup stack | Private managed-resource guards | Stop processes, clusters, and other acquired resources | Reverse acquisition order | + +The cleanup stack decides when managed resources stop. A handle clone retained outside the registry does not postpone cleanup; after cleanup, operations on a `LocalProcessHandle` fail because the run no longer owns the process. + +--- + +## Dependency-Ordered Teardown + +Deploy dependencies before dependents: + +```rust,ignore +let queue = ctx.deploy_and_expose(QueueLocalApp::nodes(2)).await?; +let results = ctx.deploy_and_expose(KvLocalApp::nodes(2)).await?; +let worker = ctx + .deploy_and_expose(JobWorkerApp::new(queue_url, results_url)) + .await?; +``` + +Each deployment registers cleanup as soon as it acquires its resource. LIFO cleanup therefore stops the worker first, then the result store, then the queue. The order is independent of which handles the final `JobStackHandle` embeds or how many clones workloads retain. + +Exposure order usually follows acquisition order, but it is not the ownership mechanism. Expose a handle when test code needs to find it; register cleanup when the framework acquires a managed resource. + +--- + +## Partial-Deployment Failure + +If deployment fails halfway through, dropping `DeployContext` runs every cleanup guard already registered. The same LIFO rule applies, so successfully started dependents stop before their dependencies even though no scenario runner was created. + +Readiness belongs inside deployment for the same reason. `LocalProcessApp::with_readiness` stops its just-started process if the check fails, while the context cleans up all earlier children. + +Custom `AppDeployment` implementations should acquire managed resources through framework adapters such as `LocalProcessApp` and `deploy_cluster`. A raw process started directly by application code has no cleanup guard unless that code implements and registers an adapter. + +--- + +## Manual Control During a Run + +Automatic teardown does not prevent explicit control. A workload can call `stop`, `start`, or `restart` on a process handle, or the corresponding node methods on a cluster handle. Cleanup remains registered and idempotently closes whatever is still active when the run ends. + +Managed applications therefore support both properties: + +- test code can deliberately change runtime state; +- every exit path still has a final owner that cleans up. + +--- + +## Keeping Artifacts + +Managed cleanup normally removes generated working directories. Use `LocalProcessApp::keep_tempdir(true)` or `LocalProcessHandle::keep_tempdir()` for a process. Primary-cluster artifact retention is controlled by the deployment policy described in [Readiness, Retry, and Cleanup](deployment-policies.md). + +--- + +## See Also + +- [AppDeployment and DeployContext](app-deployment.md): where children, handles, and cleanup are assembled. +- [One Binary: LocalProcessApp](local-process-app.md): a managed process and its control handle. +- [Shared Cluster Provisioning](cluster-provisioning.md): cluster handles across ownership modes. diff --git a/book/src/implementing-application.md b/book/src/implementing-application.md new file mode 100644 index 0000000..211769c --- /dev/null +++ b/book/src/implementing-application.md @@ -0,0 +1,155 @@ +# Implementing Application + +This chapter shows how to put your own node binary behind the framework so deployers can launch it as a uniform cluster. + +--- + +## The Application Trait + +Every environment starts with `Application` (`testing-framework/core/src/env.rs`). It bundles the backend-agnostic types the scenario engine needs: + +```rust,ignore +pub trait Application: Send + Sync + 'static { + type Deployment: DeploymentDescriptor + Clone + 'static; + type NodeClient: Clone + Send + Sync + 'static; + type NodeConfig: Clone + Send + Sync + 'static; + + fn external_node_client(source: &ExternalNodeSource) -> Result; + fn build_node_client(access: &NodeAccess) -> Result; + fn node_readiness_path() -> &'static str; +} +``` + +The associated types and methods are: + +| Member | Role | Default | +|---|---|---| +| `Deployment` | Cluster shape descriptor (see [Topology](topology.md)) | required | +| `NodeClient` | Cheap-to-clone client handed to workloads and expectations | required | +| `NodeConfig` | Per-node configuration value the deployer materializes | required | +| `external_node_client` | Builds a client from a static external endpoint | errors ("not supported") | +| `build_node_client` | Builds a client from deployer-provided `NodeAccess` (host, API port, named ports) | errors ("not supported") | +| `node_readiness_path` | Path probed during default HTTP readiness checks | `"/"` | + +Workloads and expectations use only these types, so the same scenario code can run against local processes, Compose containers, and Kubernetes services. Each backend still requires its corresponding deployment integration. + +`Application` does not specify how nodes run. Each deployer adds a backend-specific integration trait. + +--- + +## Local Integration: Two Paths + +The local deployer (`testing-framework/deployers/local/src/env/mod.rs`) offers two traits. + +**`LocalBinaryApp`** covers apps that launch one binary per node, write one config file per node, and expose one HTTP API port. You implement five methods; a blanket implementation supplies `LocalDeployerEnv`: + +| Method | Purpose | +|---|---| +| `initial_node_name_prefix()` | Prefix for generated config/artifact names (`kv-node-0`, ...); control APIs always address nodes as `node-` | +| `build_local_node_config_with_peers(...)` | Produce a `NodeConfig` from reserved ports and peer views | +| `local_process_spec()` | Binary provider, config file name/flag, env vars, extra args | +| `render_local_config(config)` | Serialize the config into the file written next to the process | +| `http_api_port(config)` | Main HTTP port used for discovery and readiness | + +Optional overrides: `initial_local_port_names()` (extra named ports reserved per node), `readiness_endpoint_path()`, `readiness_probe()` (HTTP GET or plain TCP), and `wait_readiness_stable(nodes)` for app-specific stabilization after the port probe succeeds. + +**`LocalDeployerEnv`** exposes the deployer-facing hooks directly: `build_node_config_from_template`, `build_initial_node_configs`, `build_launch_spec`, `node_endpoints`, `node_client`, `node_peer_port`, `local_process_spec_for_node` (per-node binary selection for mixed-version clusters), and `initial_persist_dir` / `initial_snapshot_dir` (see [Persistence](persistence.md)). Implement it directly when `LocalBinaryApp` does not cover the application's launch requirements. + +```mermaid +graph LR + A[Application] --> B[LocalBinaryApp] + B -- blanket impl --> C[LocalDeployerEnv] + C --> D["ProcessDeployer<E>"] +``` + +--- + +## Worked Example: kvstore + +The kvstore integration lives in `examples/kvstore/testing/integration/src/`. The environment type is an empty struct: + +```rust,ignore +pub struct KvEnv; + +#[async_trait] +impl Application for KvEnv { + type Deployment = KvTopology; // = ClusterTopology + type NodeClient = KvHttpClient; + type NodeConfig = KvNodeConfig; + + fn build_node_client(access: &NodeAccess) -> Result { + Ok(KvHttpClient::new(access.api_base_url()?)) + } + + fn node_readiness_path() -> &'static str { + "/health/ready" + } +} +``` + +**Client construction.** `build_node_client` receives `NodeAccess`, a host plus API port (and optional testing/named ports) discovered by the deployer, and wraps its base URL in the app's HTTP client. The same function serves every backend: local processes, Compose containers, and K8s services all resolve to a `NodeAccess`. + +**Readiness path.** `node_readiness_path` returns `/health/ready`. Deployers append it to `http://:` and poll until the node answers 2xx. See [Ports, Peers, Node Config, and Readiness](node-config.md) for the probe implementation. + +The local side (`local_env.rs`) implements `LocalBinaryApp`: + +```rust,ignore +impl LocalBinaryApp for KvEnv { + fn initial_node_name_prefix() -> &'static str { + "kv-node" + } + + fn build_local_node_config_with_peers( + _topology: &Self::Deployment, + index: usize, + ports: &LocalNodePorts, + peers: &[LocalPeerNode], + _peer_ports_by_name: &HashMap, + _options: &StartNodeOptions, + _template_config: Option<&KvNodeConfig>, + ) -> Result { + build_local_cluster_node_config::(index, ports, peers) + } + + fn local_process_spec() -> LocalProcessSpec { + LocalProcessSpec::new("KVSTORE_NODE_BIN") + .with_binary_provider(kvstore_binary_provider()) + .with_rust_log("kvstore_node=info") + } + + fn render_local_config(config: &KvNodeConfig) -> Result, DynError> { + yaml_node_config(config) + } + + fn http_api_port(config: &KvNodeConfig) -> u16 { + config.http_port + } +} +``` + +**Config generation.** kvstore delegates to `build_local_cluster_node_config::`, which works because `KvEnv` also implements `ClusterNodeConfigApplication` (`app.rs`): a backend-neutral hook that builds a `NodeConfig` from a `ClusterNodeView` (own index, host, ports) and `ClusterPeerView` list. Implementing that one trait gives kvstore local config generation *and* the static-artifact path used by Compose/K8s; see [Static Artifacts and cfgsync](cfgsync.md). + +**Binary provider.** `kvstore_binary_provider()` returns a `FallbackBinaryProvider` chain: first `EnvBinaryProvider::new("KVSTORE_NODE_BIN")` (use a prebuilt binary if the env var is set), then `BuildBinaryProvider` running `cargo build -p kvstore-node --bin kvstore-node` in the workspace root. This is why kvstore examples need no manual setup. Providers are covered in [Binary Providers](binary-providers.md). + +**Launch and config rendering.** At spawn time the framework renders the config with `render_local_config`, writes it as `config.yaml` into the node's working directory, and launches ` --config config.yaml` with the spec's env vars. `LocalProcessSpec` supports different file names, positional config arguments, and extra args (see [node-config.md](node-config.md)). + +Finally, `lib.rs` exports ready-made deployer aliases: + +```rust,ignore +pub type KvLocalDeployer = testing_framework_runner_local::ProcessDeployer; +pub type KvComposeDeployer = testing_framework_runner_compose::ComposeDeployer; +pub type KvK8sDeployer = testing_framework_runner_k8s::K8sDeployer; +``` + +--- + +## Other Backends + +The same `KvEnv` gains container support in two short files: + +- `compose_env.rs` implements `ComposeBinaryApp`: a `BinaryConfigNodeSpec` naming the in-container binary path, config path, and exposed ports. See [Compose Deployer](deployer-compose.md). +- `k8s_env.rs` implements `K8sBinaryApp`: a `BinaryConfigK8sSpec` with release name, node-name prefix, binary and config paths, and service ports. See [Kubernetes Deployer](deployer-k8s.md). + +Both backends deliver generated configs through cfgsync rather than the local filesystem ([Static Artifacts and cfgsync](cfgsync.md)). + +Implement traits only for the backends you use. An `Application` implementation plus `LocalBinaryApp` is sufficient for local scenarios. diff --git a/book/src/internal-crate-reference.md b/book/src/internal-crate-reference.md deleted file mode 100644 index 50ed515..0000000 --- a/book/src/internal-crate-reference.md +++ /dev/null @@ -1,174 +0,0 @@ -# Internal Crate Reference - -High-level roles of the crates that make up the framework: - -- **Configs** (`testing-framework/configs/`): Prepares reusable configuration primitives for nodes, networking, tracing, and wallets, shared by all scenarios and runners. Includes topology generation and circuit asset resolution. - -- **Core scenario orchestration** (`testing-framework/core/`): Houses the topology and scenario model, runtime coordination, node clients, and readiness/health probes. Defines `Deployer` and `Runner` traits, `ScenarioBuilder`, and `RunContext`. - -- **Workflows** (`testing-framework/workflows/`): Packages workloads (transaction, chaos) and expectations (consensus liveness) into reusable building blocks. Offers fluent DSL extensions (`ScenarioBuilderExt`, `ChaosBuilderExt`). - -- **Deployers** (`testing-framework/deployers/{local,compose,k8s}/`): Implements deployment backends (local host, Docker Compose, Kubernetes) that all consume the same scenario plan. Each provides a `Deployer` implementation (`LocalDeployer`, `ComposeDeployer`, `K8sDeployer`). - -- **Runner Examples** (crate name: `runner-examples`, path: `examples/`): Runnable binaries demonstrating framework usage and serving as living documentation. These are the **primary entry point** for running scenarios (`examples/src/bin/local_runner.rs`, `examples/src/bin/compose_runner.rs`, `examples/src/bin/k8s_runner.rs`). - -## Where to Add New Capabilities - -| What You're Adding | Where It Goes | Examples | -|-------------------|---------------|----------| -| **Node config parameter** | `testing-framework/configs/src/topology/configs/` | Slot duration, log levels | -| **Topology feature** | `testing-framework/core/src/topology/` | New network layouts | -| **Scenario capability** | `testing-framework/core/src/scenario/` | New capabilities, context methods | -| **Workload** | `testing-framework/workflows/src/workloads/` | New traffic generators | -| **Expectation** | `testing-framework/workflows/src/expectations/` | New success criteria | -| **Builder API** | `testing-framework/workflows/src/builder/` | DSL extensions, fluent methods | -| **Deployer** | `testing-framework/deployers/` | New deployment backends | -| **Example scenario** | `examples/src/bin/` | Demonstration binaries | - -## Extension Workflow - -### Adding a New Workload - -1. **Define the workload** in `testing-framework/workflows/src/workloads/your_workload.rs`: -```rust,ignore -use async_trait::async_trait; -use testing_framework_core::scenario::{DynError, RunContext, Workload}; - -pub struct YourWorkload; - -#[async_trait] -impl Workload for YourWorkload { - fn name(&self) -> &'static str { - "your_workload" - } - - async fn start(&self, _ctx: &RunContext) -> Result<(), DynError> { - // implementation - Ok(()) - } -} -``` - -2. **Add builder extension** in `testing-framework/workflows/src/builder/mod.rs`: -```rust,ignore -pub struct YourWorkloadBuilder; - -impl YourWorkloadBuilder { - pub fn some_config(self) -> Self { - self - } -} - -pub trait ScenarioBuilderExt: Sized { - fn your_workload(self) -> YourWorkloadBuilder; -} -``` - -3. **Use in examples** in `examples/src/bin/your_scenario.rs`: -```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; - -pub struct YourWorkloadBuilder; - -impl YourWorkloadBuilder { - pub fn some_config(self) -> Self { - self - } -} - -pub trait YourWorkloadDslExt: Sized { - fn your_workload_with(self, configurator: F) -> Self - where - F: FnOnce(YourWorkloadBuilder) -> YourWorkloadBuilder; -} - -impl YourWorkloadDslExt for testing_framework_core::scenario::Builder { - fn your_workload_with(self, configurator: F) -> Self - where - F: FnOnce(YourWorkloadBuilder) -> YourWorkloadBuilder, - { - let _ = configurator(YourWorkloadBuilder); - self - } -} - -pub fn use_in_examples() { - let _plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .your_workload_with(|w| w.some_config()) - .build(); -} -``` - -### Adding a New Expectation - -1. **Define the expectation** in `testing-framework/workflows/src/expectations/your_expectation.rs`: -```rust,ignore -use async_trait::async_trait; -use testing_framework_core::scenario::{DynError, Expectation, RunContext}; - -pub struct YourExpectation; - -#[async_trait] -impl Expectation for YourExpectation { - fn name(&self) -> &'static str { - "your_expectation" - } - - async fn evaluate(&mut self, _ctx: &RunContext) -> Result<(), DynError> { - // implementation - Ok(()) - } -} -``` - -2. **Add builder extension** in `testing-framework/workflows/src/builder/mod.rs`: -```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; - -pub trait YourExpectationDslExt: Sized { - fn expect_your_condition(self) -> Self; -} - -impl YourExpectationDslExt for testing_framework_core::scenario::Builder { - fn expect_your_condition(self) -> Self { - self - } -} - -pub fn use_in_examples() { - let _plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .expect_your_condition() - .build(); -} -``` - -### Adding a New Deployer - -1. **Implement `Deployer` trait** in `testing-framework/runners/your_runner/src/deployer.rs`: -```rust,ignore -use async_trait::async_trait; -use testing_framework_core::scenario::{Deployer, Runner, Scenario}; - -#[derive(Debug)] -pub struct YourError; - -pub struct YourDeployer; - -#[async_trait] -impl Deployer for YourDeployer { - type Error = YourError; - - async fn deploy(&self, _scenario: &Scenario<()>) -> Result { - // Provision infrastructure - // Wait for readiness - // Return Runner - todo!() - } -} -``` - -2. **Provide cleanup** and handle node control if supported. - -3. **Add example** in `examples/src/bin/your_runner.rs`. - -For detailed examples, see [Extending the Framework](extending.md) and [Custom Workload Example](custom-workload-example.md). diff --git a/book/src/introduction.md b/book/src/introduction.md index 2c75179..ca669f2 100644 --- a/book/src/introduction.md +++ b/book/src/introduction.md @@ -1,44 +1,138 @@ -# Introduction +# Testing Framework -The Logos Testing Framework is a purpose-built toolkit for exercising Logos in -realistic, multi-node environments. It solves the gap between small, isolated -tests and full-system validation by letting teams describe a cluster layout, -drive meaningful traffic, and assert the outcomes in one coherent plan. +System-level testing for networked applications from Rust. -It is for protocol engineers, infrastructure operators, and QA teams who need -repeatable confidence that node components work together under network and -timing constraints. +The testing framework deploys and controls processes, containers, and clusters. Tests can run several nodes over network connections for a bounded period. Application-specific configuration and clients stay outside the framework, so the runtime can be used with a key-value store, a Raft cluster, a message queue, or a blockchain. -Multi-node integration testing is required because many Logos behaviors—block -progress and liveness under churn—only emerge when several nodes interact over -real networking and time. This framework makes those checks -declarative, observable, and portable across environments. +[**Get Started**](quickstart.md) -## A Scenario in 20 Lines +--- -Here's the conceptual shape of every test you'll write: +## Scenarios + +A declarative test is represented by a `Scenario` containing: + +- **Topology** — the system under test (a uniform cluster, a composed application stack, or attached external nodes) +- **Workloads** — traffic and conditions that exercise the system +- **Expectations** — success criteria verified after execution +- **Duration** — the time window for the experiment + +```mermaid +flowchart LR + subgraph SC["Scenario"] + T["topology
the system under test"]:::cl + W["workloads
drive traffic"]:::sc + EX["expectations
verify outcomes"]:::sc + D["duration
the run window"]:::sc + end + SC --> RN["Runner
deploy · run · evaluate · teardown"]:::sc + classDef cl stroke:#4a90d9,stroke-width:2.5px; + classDef sc stroke:#9b6dd6,stroke-width:2.5px; +``` + +The scenario runtime executes these parts in the same order for each declarative entry pattern. The entry pattern determines how the system is supplied. + +--- + +## Entry Patterns + +```mermaid +flowchart LR + A[Uniform managed cluster]:::cl --> S[Scenario] + B[AppHost composed stack] --> S + C[Attached / external nodes]:::cl --> S + S:::sc --> R[Runner: workloads + expectations]:::sc + M[ManualCluster] --> I[Imperative orchestration] + classDef cl stroke:#4a90d9,stroke-width:2.5px; + classDef hd stroke:#4caf7d,stroke-width:2.5px; + classDef sc stroke:#9b6dd6,stroke-width:2.5px; +``` + +Three entry patterns use the scenario runtime: + +1. **Uniform managed cluster** — the framework generates configs and launches N identical nodes from a topology. See [Part IV](part-iv.md). +2. **AppHost composed stack** — the app layer deploys heterogeneous components (processes, child clusters, in-process services) as one system and exposes typed handles to workloads. See [Part II](part-ii.md). +3. **Attached and external nodes** — the scenario targets clusters you already run, or plain URLs. See [Existing and External Clusters](external-clusters.md). + +**[ManualCluster](manual-cluster.md)** is the imperative alternative. It provides direct start, stop, restart, and readiness operations without the scenario runner, including for step-driven BDD harnesses. + +If you are not sure which to use, read [Choosing an Entry Pattern](entry-patterns.md). + +--- + +## Provided APIs + +**Declarative API** +- Express tests as topology + workloads + expectations +- Reuse the same definition across local, Compose, and Kubernetes deployers +- Compose stacks from reusable application deployments + +**Application layer** +- Deploy heterogeneous systems as one root `AppDeployment` +- Typed, named handles connect workloads to components +- Deterministic cleanup, including on partial-deployment failure + +**Runtime capabilities** +- Capability-gated node control: restart nodes from workloads, portably +- Continuous observation: snapshots, history, and event streams of application state +- Telemetry: metrics, logs, and tracing endpoints + +**Operations** +- Binary providers resolve node binaries from paths, env vars, builds, or downloads +- Reproducible deployments via seeds +- Artifact preservation for post-mortem debugging + +--- + +## Quick Example ```rust,ignore -// 1. Define the cluster -let scenario = ScenarioBuilder::topology_with(|t| { - t.network_star() - .nodes(3) -}) -// 2. Add workloads (traffic) -.transactions_with(|tx| tx.rate(10).users(5)) +use testing_framework_app::{AppHost, AppHostLocalDeployer, AppScenarioBuilderExt as _}; +use testing_framework_core::scenario::Deployer as _; -// 3. Define success criteria -.expect_consensus_liveness() +let mut scenario = AppHost::scenario() + .with_app(KvLocalApp::nodes(3)) + .with_workload(KvAppHostConvergence::new(3)) + .build()?; -// 4. Set experiment duration -.with_run_duration(Duration::from_secs(60)) -.build(); - -// 5. Deploy and run -let runner = deployer.deploy(&scenario).await?; +let runner = AppHostLocalDeployer::default().deploy(&scenario).await?; runner.run(&mut scenario).await?; ``` -This pattern—topology, workloads, expectations, duration—repeats across all scenarios in this book. +This deploys a three-node key-value store cluster, runs a convergence workload against it (including a node restart), and tears everything down. The remaining chapters cover each part of this pattern in detail. -**Learn more:** For protocol-level documentation and node internals, see the [Logos Project Documentation](https://nomos-tech.notion.site/project). +[View the example apps](running-examples.md) + +--- + +## The Example Apps + +The repository includes small applications under `examples/` that exercise the framework APIs: + +| App | Demonstrates | +|-----|--------------| +| `kvstore` | Uniform clusters, app hosting, convergence testing, all three deployers | +| `openraft_kv` | Node control, failover, continuous observation | +| `multi_app` | Composing heterogeneous stacks with typed handles | +| `nats`, `redis_streams` | Testing third-party binaries you did not write | +| `pubsub`, `queue`, `metrics_counter` | Additional workload and expectation patterns | + +Some chapters also link to adopter repositories. The examples listed in this table run from this workspace. + +--- + +## Documentation Structure + +| Section | Description | +|---------|-------------| +| **[Part I — Mental Model](part-i.md)** | The core abstractions and how to choose between entry patterns | +| **[Part II — Composing Applications](part-ii.md)** | The app layer: deployments, handles, teardown | +| **[Part III — Scenario Runtime](part-iii.md)** | Workloads, expectations, capabilities, observation | +| **[Part IV — Uniform Clusters](part-iv.md)** | Implementing `Application`, topology, config, manual control | +| **[Part V — Deployers and Sources](part-v.md)** | Local, Compose, Kubernetes, external clusters, binaries | +| **[Part VI — Extending](part-vi.md)** | Extension points, crate map, boundaries | +| **[Part VII — Operations](part-vii.md)** | Running examples, CI, diagnostics, troubleshooting | + +--- + +Start with the **[Quickstart](quickstart.md)**. diff --git a/book/src/local-app-cluster.md b/book/src/local-app-cluster.md new file mode 100644 index 0000000..c91866e --- /dev/null +++ b/book/src/local-app-cluster.md @@ -0,0 +1,122 @@ +# Uniform Child Clusters: LocalAppCluster + +`LocalAppCluster` runs an additional uniform cluster of local processes as one child of a composed stack. + +For N identical nodes of one binary with peer wiring and per-node clients, use `ScenarioBuilder` when the cluster is the system under test. When the cluster is one component of a larger stack, deploy it as a `LocalAppCluster` inside the root deployment. + +The environment `E` must implement `LocalDeployerEnv` (config rendering, ports, process spec; see [Local Deployer](deployer-local.md)). That work is the same whether the app runs standalone or as a child, so a cluster env written for uniform scenarios is reusable here unchanged. + +--- + +## Starting a Child Cluster + +Inside an `AppDeployment`, `DeployContext::deploy_local_cluster` launches every node described by the deployment (`node-0`, `node-1`, ...), waits for network readiness, registers cleanup, and returns the cluster handle: + +```rust,ignore +// examples/kvstore/testing/integration/src/app.rs +#[derive(Clone)] +pub struct KvLocalApp { + deployment: KvTopology, +} + +impl KvLocalApp { + #[must_use] + pub fn nodes(nodes: usize) -> Self { + Self { deployment: KvTopology::new(nodes) } + } +} + +#[async_trait] +impl AppDeployment for KvLocalApp { + type Handle = LocalAppCluster; + + async fn deploy(self, ctx: &mut DeployContext) -> Result { + ctx.deploy_local_cluster::(self.deployment).await + } +} +``` + +The kvstore preset delegates cluster provisioning to `deploy_local_cluster`, which registers a cleanup guard and returns a cloneable access and control handle. Scenario cleanup stops any remaining nodes independently of handle clones. + +--- + +## The Handle API + +
+LocalAppCluster handle method reference + +| Method | Purpose | +|--------|---------| +| `deployment()` / `node_count()` | The cluster's deployment descriptor and node count. | +| `node_clients()` | Shared `NodeClients` collection. | +| `clients()` | Snapshot of all currently available clients. | +| `first_client()` | First available client, if any. | +| `node_client(name)` | Client for one node, if started. | +| `node_pid(name)` | OS process id for one node, if running. | +| `start_node(name)` / `start_node_with(name, options)` | Start a node, optionally with `StartNodeOptions` (config overrides, persist/snapshot dirs, args). | +| `stop_node(name)` | Stop a node. | +| `restart_node(name)` / `restart_node_with(name, options)` | Restart with existing or explicit options. | +| `wait_network_ready()` | Wait for the cluster-level readiness condition. | +| `wait_node_ready(name)` | Wait for one node to report ready. | + +
+ +Node names follow the `node-{index}` convention used at startup. `LocalAppCluster` is the backend-independent `ClusterHandle` alias; it exposes the supported common control surface rather than an underlying `ManualCluster`. + +Per-node control is provided by the cluster handle. A workload restarting a child-cluster node does not need the scenario-level `with_node_control()` capability. + +--- + +## Worked Example: kvstore Convergence Across a Restart + +The `kvstore_app_host_convergence` bin runs a three-node kv cluster as an app, then exercises convergence across a node restart: + +```rust,ignore +// examples/kvstore/examples/src/bin/app_host_convergence.rs +let mut scenario = AppHost::scenario() + .with_app(KvLocalApp::nodes(3)) + .with_run_duration(Duration::from_secs(5)) + .with_workload(KvAppHostConvergence::new(3)) + .build()?; + +let deployer = AppHostLocalDeployer::default(); +let runner = deployer.deploy(&scenario).await?; +runner.run(&mut scenario).await?; +``` + +The workload requires the cluster handle and drives it directly: + +```rust,ignore +async fn start(&self, ctx: &RunContext) -> Result<(), DynError> { + let cluster = ctx.require_app::>()?; + + ensure_cluster_shape(&cluster, self.expected_nodes)?; + put_value(&cluster, "before-restart").await?; + cluster.restart_node("node-0").await?; + cluster.wait_node_ready("node-0").await?; + put_value(&cluster, "after-restart").await?; + + Ok(()) +} +``` + +`put_value` writes through `cluster.first_client()`; `ensure_cluster_shape` checks `node_count()`, `clients()`, `node_client("node-0")`, and `node_pid("node-0")`. Run it with: + +```bash +cargo run -p kvstore-examples --bin kvstore_app_host_convergence +``` + +The kvstore environment resolves its node binary through a fallback provider chain, so this example does not require a manually configured binary path (see [Binary Providers](binary-providers.md)). + +--- + +## Exposing the Cluster to Workloads + +`KvLocalApp` returns the raw `LocalAppCluster` as its handle, and the factory auto-exposes it, so workloads request `LocalAppCluster` directly. In a composed stack you can either expose the raw cluster handle (as the job stack does), wrap it in a domain newtype (`StoreHandle`) for clearer requirements, or use named handles when two child clusters share an environment type (see [Composing Heterogeneous Stacks](composing-stacks.md)). + +--- + +## See Also + +- [One Binary: LocalProcessApp](local-process-app.md): the single-process counterpart. +- [Backend Scope](app-backend-scope.md): why child clusters are local-only today. diff --git a/book/src/local-process-app.md b/book/src/local-process-app.md new file mode 100644 index 0000000..05d9910 --- /dev/null +++ b/book/src/local-process-app.md @@ -0,0 +1,128 @@ +# One Binary: LocalProcessApp + +`LocalProcessApp` deploys one local binary with a typed client, without modeling it as a node topology. + +Application code supplies the launch files, client type, and readiness check. The framework manages the process lifetime, working directory, and teardown. This is used for third-party infrastructure such as a message broker or database, and for singleton services such as a sequencer or indexer inside a composed stack. + +--- + +## Construction + +```rust,ignore +LocalProcessApp::new(label, launch, endpoints, client) +``` + +| Argument | Type | Meaning | +|----------|------|---------| +| `label` | `impl Into` | Name used for the process working directory and logs. | +| `launch` | `LaunchSpec` | How to start the binary. | +| `endpoints` | `NodeEndpoints` | Addresses the process will listen on. | +| `client` | `C: Clone + Send + Sync + 'static` | The typed client returned through the handle. | + +`LaunchSpec` (from `testing_framework_runner_local`) is a plain launch plan: + +| Field | Type | Purpose | +|-------|------|---------| +| `binary` | `PathBuf` | Executable path. | +| `files` | `Vec` | Files written into the working directory before spawn (`relative_path` + `contents`). | +| `args` | `Vec` | Command-line arguments. | +| `env` | `Vec` | Environment variables (`LaunchEnvVar::new(key, value)`). | + +`NodeEndpoints` describes where the process listens: an `api: SocketAddr` plus `extra_ports` keyed by `NodeEndpointPort` (`TestingApi`, `Network`, or `Custom(String)`). Build one with `NodeEndpoints::from_api_port(port)` and `insert_port`. + +Endpoints are *declared*, not allocated. The generic process layer does not select ports. The launch configuration and the supplied endpoints must use the same values. + +--- + +## Builder Options + +| Method | Effect | +|--------|--------| +| `with_readiness(closure)` | Async check run after spawn. The closure receives `(NodeEndpoints, C)`. **On failure the process is stopped and the deploy fails.** | +| `keep_tempdir(bool)` | Keep the generated working directory after teardown. | +| `with_persist_dir(path)` | Place the working directory next to `path` (as `_`); nothing is copied — see [Persistence](persistence.md). | +| `with_snapshot_dir(path)` | Copy the snapshot directory's contents into the fresh working directory before start. | + +If the readiness closure fails, deployment returns an error, stops the new process, and cleans up children deployed earlier (see [Handle Ownership and Teardown](handles-teardown.md)). + +--- + +## Example: A Single nats-server Process + +The nats example normally runs as a uniform cluster, but its `NatsClient` works just as well against one broker started as a process app: + +```rust,ignore +use std::time::Duration; + +use nats_runtime_ext::NatsClient; +use testing_framework_app::LocalProcessApp; +use testing_framework_runner_local::{LaunchSpec, NodeEndpoints}; + +let launch = LaunchSpec { + binary: std::env::var("NATS_SERVER_BIN")?.into(), + args: vec!["-p".into(), "4222".into(), "-m".into(), "8222".into()], + ..LaunchSpec::default() +}; + +let client = NatsClient::new( + "nats://127.0.0.1:4222".to_owned(), + "http://127.0.0.1:8222".parse()?, +); + +let broker = LocalProcessApp::new("nats", launch, NodeEndpoints::from_api_port(8222), client) + .with_readiness(|_endpoints, client| async move { + for _ in 0..50 { + if client.is_healthy().await.unwrap_or(false) { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err("nats-server did not become healthy".into()) + }); +``` + +`broker` is an `AppDeployment` for any environment, so a root deployment composes it like any other child: + +```rust,ignore +let nats = ctx.deploy_and_expose(broker).await?; +``` + +--- + +## The Handle: LocalProcessHandle + +`deploy` returns `LocalProcessHandle`. Clones share access to the same process state, while scenario cleanup owns the process lifetime. Cleanup stops the process even if a handle clone still exists. + +
+LocalProcessHandle method reference + +| Method | Returns | Notes | +|--------|---------|-------| +| `client()` | `C` | Clone of the typed client. | +| `endpoints()` | `&NodeEndpoints` | The endpoints supplied at deployment. | +| `pid()` | `u32` | OS process id (async). | +| `is_running()` | `bool` | Whether the child is still alive (async). | +| `working_dir()` | `PathBuf` | The generated working directory (async). | +| `start()` | `Result<(), DynError>` | Start again after an explicit stop, using the original `LaunchSpec`. | +| `restart()` | `Result<(), DynError>` | Restart with the original `LaunchSpec`. | +| `stop()` | — | Stop now, without waiting for drop. | +| `keep_tempdir()` | `io::Result<()>` | Retain the working directory at teardown. | + +
+ +A workload retrieves the handle like any other (see [AppHost and with_app](app-host.md)): + +```rust,ignore +let broker = ctx.require_app::>()?; +broker.restart().await?; +assert!(broker.is_running().await); +``` + +Tests can use `start`, `restart`, and `stop` for process lifecycle and fault injection. These operations fail after scenario cleanup has closed the managed resource. + +--- + +## See Also + +- [AppDeployment and DeployContext](app-deployment.md): composing a process app under a root deployment. +- [Composing Heterogeneous Stacks](composing-stacks.md): mixing single processes with child clusters. diff --git a/book/src/logging-observability.md b/book/src/logging-observability.md deleted file mode 100644 index eb0a3bd..0000000 --- a/book/src/logging-observability.md +++ /dev/null @@ -1,356 +0,0 @@ -# Logging & Observability - -Comprehensive guide to log collection, metrics, and debugging across all runners. - -## Node Logging vs Framework Logging - -**Critical distinction:** Node logs and framework logs use different configuration mechanisms. - -| Component | Controlled By | Purpose | -|-----------|--------------|---------| -| **Framework binaries** (`cargo run -p runner-examples --bin local_runner`) | `RUST_LOG` | Runner orchestration, deployment logs | -| **Node processes** (nodes spawned by runner) | `LOGOS_BLOCKCHAIN_LOG_LEVEL`, `LOGOS_BLOCKCHAIN_LOG_FILTER` (+ `LOGOS_BLOCKCHAIN_LOG_DIR` on host runner) | Consensus, mempool, network logs | - -**Common mistake:** Setting `RUST_LOG=debug` only increases verbosity of the runner binary itself. Node logs remain at their default level unless you also set `LOGOS_BLOCKCHAIN_LOG_LEVEL=debug`. - -**Example:** - -```bash -# This only makes the RUNNER verbose, not the nodes: -RUST_LOG=debug cargo run -p runner-examples --bin local_runner - -# This makes the NODES verbose: -LOGOS_BLOCKCHAIN_LOG_LEVEL=debug cargo run -p runner-examples --bin local_runner - -# Both verbose (typically not needed): -RUST_LOG=debug LOGOS_BLOCKCHAIN_LOG_LEVEL=debug cargo run -p runner-examples --bin local_runner -``` - -## Logging Environment Variables - -See [Environment Variables Reference](environment-variables.md) for complete details. Quick summary: - -| Variable | Default | Effect | -|----------|---------|--------| -| `LOGOS_BLOCKCHAIN_LOG_DIR` | None (console only) | Host runner: directory for per-node log files. Compose/k8s: use `cfgsync.yaml` | -| `LOGOS_BLOCKCHAIN_LOG_LEVEL` | `info` | Global log level: `error`, `warn`, `info`, `debug`, `trace` | -| `LOGOS_BLOCKCHAIN_LOG_FILTER` | None | Fine-grained target filtering (e.g., `cryptarchia=trace`) | -| `LOGOS_BLOCKCHAIN_TESTS_TRACING` | false | Enable debug tracing preset | -| `LOGOS_BLOCKCHAIN_OTLP_ENDPOINT` | None | OTLP trace endpoint (optional) | -| `LOGOS_BLOCKCHAIN_OTLP_METRICS_ENDPOINT` | None | OTLP metrics endpoint (optional) | - -**Example:** Full debug logging to files: - -```bash -LOGOS_BLOCKCHAIN_TESTS_TRACING=true \ -LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \ -LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \ -LOGOS_BLOCKCHAIN_LOG_FILTER="lb_cryptarchia=trace,lb_chain_service=info,lb_chain_network=info" \ -cargo run -p runner-examples --bin local_runner -``` - -## Per-Node Log Files - -When `LOGOS_BLOCKCHAIN_LOG_DIR` is set, each node writes logs to separate files: - -**File naming pattern:** -- **Validators**: Prefix `logos-blockchain-node-0`, `logos-blockchain-node-1`, etc. (may include timestamp suffix) - -**Example filenames:** -- `logos-blockchain-node-0.2024-12-18T14-30-00.log` -- `logos-blockchain-node-1.2024-12-18T14-30-00.log` - -**Local runner note:** The local runner uses per-run temporary directories under the current working directory and removes them after the run unless `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1`. Use `LOGOS_BLOCKCHAIN_LOG_DIR=/path/to/logs` to write per-node log files to a stable location. - -## Filter Target Names - -Common target prefixes for `LOGOS_BLOCKCHAIN_LOG_FILTER`: - -| Target Prefix | Subsystem | -|---------------------------|-----------| -| `lb_cryptarchia` | Consensus (Cryptarchia) | -| `lb_blend` | Mix network/privacy layer | -| `lb_chain_service` | Chain service (node APIs/state) | -| `lb_chain_network` | P2P networking | -| `lb_chain_leader_service` | Leader election | - -**Example filter:** - -```bash -LOGOS_BLOCKCHAIN_LOG_FILTER="lb_cryptarchia=trace,lb_chain_service=info,lb_chain_network=info" -``` - ---- - -## Accessing Logs by Runner - -### Local Runner (Host Processes) - -**Default (temporary directories, auto-cleanup):** - -```bash -cargo run -p runner-examples --bin local_runner -# Logs written to temporary directories in working directory -# Automatically cleaned up after test completes -``` - -**Persistent file output:** - -```bash -LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/local-logs \ -cargo run -p runner-examples --bin local_runner - -# After test completes: -ls /tmp/local-logs/ -# Files with prefix: logos-blockchain-node-0*, logos-blockchain-node-1* -# May include timestamps in filename -``` - -**Tip:** Use `LOGOS_BLOCKCHAIN_LOG_DIR` for persistent per-node log files, and `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1` if you want to keep the per-run temporary directories (configs/state) for post-mortem inspection. - -### Compose Runner (Docker Containers) - -**Via Docker logs (default, recommended):** - -```bash -# List containers (note the UUID prefix in names) -docker ps --filter "name=nomos-compose-" - -# Stream logs from specific container -docker logs -f - -# Or use name pattern matching: -docker logs -f $(docker ps --filter "name=nomos-compose-.*-node-0" -q | head -1) - -# Show last 100 lines -docker logs --tail 100 -``` - -**Via file collection (advanced):** - -To write per-node log files inside containers, set `tracing_settings.logger: !File` in `testing-framework/assets/stack/cfgsync.yaml` (and ensure the directory is writable). To access them, you must either: - -1. **Copy files out after the run:** - -```bash -# Ensure cfgsync.yaml is configured to log to /logs -LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \ -cargo run -p runner-examples --bin compose_runner - -# After test, copy files from containers: -docker ps --filter "name=nomos-compose-" -docker cp :/logs/node* /tmp/ -``` - -2. **Mount a host volume** (requires modifying compose template): - -```yaml -volumes: - - /tmp/host-logs:/logs # Add to docker-compose.yml.tera -``` - -**Recommendation:** Use `docker logs` by default. File collection inside containers is complex and rarely needed. - -**Keep containers for debugging:** - -```bash -COMPOSE_RUNNER_PRESERVE=1 \ -LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \ -cargo run -p runner-examples --bin compose_runner -# Containers remain running after test—inspect with docker logs or docker exec -``` - -**Compose debugging variables:** -- `COMPOSE_RUNNER_HOST=127.0.0.1` — host used for readiness probes -- `COMPOSE_RUNNER_HOST_GATEWAY=host.docker.internal:host-gateway` — controls `extra_hosts` entry (set to `disable` to omit) -- `TESTNET_RUNNER_PRESERVE=1` — alias for `COMPOSE_RUNNER_PRESERVE=1` -- `COMPOSE_RUNNER_HTTP_TIMEOUT_SECS=` — override HTTP readiness timeout - -**Note:** Container names follow pattern `nomos-compose-{uuid}-node-{index}-1` where `{uuid}` changes per run. - -### K8s Runner (Kubernetes Pods) - -**Via kubectl logs (use label selectors):** - -```bash -# List pods -kubectl get pods - -# Stream logs using label selectors (recommended) -# Helm chart labels: -# - nomos/logical-role=node -# - nomos/node-index -kubectl logs -l nomos/logical-role=node -f - -# Stream logs from specific pod -kubectl logs -f logos-blockchain-node-0 - -# Previous logs from crashed pods -kubectl logs --previous -l nomos/logical-role=node -``` - -**Download logs for offline analysis:** - -```bash -# Using label selectors -kubectl logs -l nomos/logical-role=node --tail=1000 > all-nodes.log - -# Specific pods -kubectl logs logos-blockchain-node-0 > node-0.log -``` - -**K8s debugging variables:** -- `K8S_RUNNER_DEBUG=1` — logs Helm stdout/stderr for install commands -- `K8S_RUNNER_PRESERVE=1` — keep namespace/release after run -- `K8S_RUNNER_NODE_HOST=` — override NodePort host resolution -- `K8S_RUNNER_NAMESPACE=` / `K8S_RUNNER_RELEASE=` — pin namespace/release (useful for debugging) - -**Specify namespace (if not using default):** - -```bash -kubectl logs -n my-namespace -l nomos/logical-role=node -f -``` - -**Note:** K8s runner is optimized for local clusters (Docker Desktop K8s, minikube, kind). Remote clusters require additional setup. - ---- - -## OTLP and Telemetry - -**OTLP exporters are optional.** If you see errors about unreachable OTLP endpoints, it's safe to ignore them unless you're actively collecting traces/metrics. - -**To enable OTLP:** - -```bash -LOGOS_BLOCKCHAIN_OTLP_ENDPOINT=http://localhost:4317 \ -LOGOS_BLOCKCHAIN_OTLP_METRICS_ENDPOINT=http://localhost:4318 \ -cargo run -p runner-examples --bin local_runner -``` - -**To silence OTLP errors:** Simply leave these variables unset (the default). - ---- - -## Observability: Prometheus and Node APIs - -Runners expose metrics and node HTTP endpoints for expectation code and debugging. - -### Prometheus-Compatible Metrics Querying (Optional) - -- Runners do **not** provision Prometheus automatically -- For a ready-to-run stack, use `scripts/setup/setup-observability.sh`: - - Compose: `scripts/setup/setup-observability.sh compose up` then `scripts/setup/setup-observability.sh compose env` - - K8s: `scripts/setup/setup-observability.sh k8s install` then `scripts/setup/setup-observability.sh k8s env` -- Provide `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` (PromQL base URL) to enable `ctx.telemetry()` queries -- Access from expectations when configured: `ctx.telemetry().prometheus().map(|p| p.base_url())` - -**Example:** - -```bash -# Start observability stack (Compose) -scripts/setup/setup-observability.sh compose up - -# Get environment variables -eval $(scripts/setup/setup-observability.sh compose env) - -# Run scenario with metrics -scripts/run/run-examples.sh -t 60 -n 3 compose -``` - -### Grafana (Optional) - -- Runners do **not** provision Grafana automatically (but `scripts/setup/setup-observability.sh` can) -- If you set `LOGOS_BLOCKCHAIN_GRAFANA_URL`, the deployer prints it in `TESTNET_ENDPOINTS` -- Dashboards live in `testing-framework/assets/stack/monitoring/grafana/dashboards/` (the bundled stack auto-provisions them) - -**Example:** - -```bash -# Bring up the bundled Prometheus+Grafana stack (optional) -scripts/setup/setup-observability.sh compose up -eval $(scripts/setup/setup-observability.sh compose env) - -export LOGOS_BLOCKCHAIN_GRAFANA_URL=http://localhost:3000 -scripts/run/run-examples.sh -t 60 -n 3 compose -``` - -**Default bundled Grafana login:** `admin` / `admin` (see `scripts/observability/compose/docker-compose.yml`). - -### Node APIs - -- Access from expectations: `ctx.node_clients().node_clients().get(0)` -- Endpoints: consensus info, network info, etc. -- See `testing-framework/core/src/nodes/api_client.rs` for available methods - -**Example usage in expectations:** - -```rust,ignore -use testing_framework_core::scenario::{DynError, RunContext}; - -async fn evaluate(ctx: &RunContext) -> Result<(), DynError> { - let client = &ctx.node_clients().node_clients()[0]; - - let info = client.consensus_info().await?; - tracing::info!(height = info.height, "consensus info from node 0"); - - Ok(()) -} -``` - ---- - -## Observability Flow - -```mermaid -flowchart TD - Expose[Runner exposes endpoints/ports] --> Collect[Runtime collects block/health signals] - Collect --> Consume[Expectations consume signals
decide pass/fail] - Consume --> Inspect[Operators inspect logs/metrics
when failures arise] -``` - ---- - -## Quick Reference - -### Debug Logging (Host) - -```bash -LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/logs \ -LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \ -LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \ -scripts/run/run-examples.sh -t 60 -n 3 host -``` - -### Compose with Observability - -```bash -# Start observability stack -scripts/setup/setup-observability.sh compose up -eval $(scripts/setup/setup-observability.sh compose env) - -# Run with metrics -scripts/run/run-examples.sh -t 60 -n 3 compose - -# Access Grafana at http://localhost:3000 -``` - -### K8s with Debug - -```bash -K8S_RUNNER_NAMESPACE=nomos-debug \ -K8S_RUNNER_DEBUG=1 \ -K8S_RUNNER_PRESERVE=1 \ -scripts/run/run-examples.sh -t 60 -n 3 k8s - -# Inspect logs -kubectl logs -n nomos-debug -l nomos/logical-role=node -``` - ---- - -## See Also - -- [Environment Variables](environment-variables.md) — Complete variable reference -- [Troubleshooting](troubleshooting.md) — Log-related debugging (see "Where to Find Logs") -- [Running Examples](running-examples.md) — Runner-specific logging details -- [Prerequisites & Setup](prerequisites.md) — Setup before running diff --git a/book/src/manual-cluster.md b/book/src/manual-cluster.md index d42b2da..7e76200 100644 --- a/book/src/manual-cluster.md +++ b/book/src/manual-cluster.md @@ -1,397 +1,116 @@ -# Manual Clusters: Imperative Control +# ManualCluster: Imperative Node Control -**When should I read this?** You're integrating external test drivers (like Cucumber/BDD frameworks) that need imperative node orchestration. This is an escape hatch for when the test orchestration must live outside the framework—most tests should use the standard scenario approach. +`ManualCluster` provides imperative node lifecycle control. Your code starts, stops, and restarts nodes directly without using the scenario runner. --- -## Overview +## When to Use It -**Manual clusters** provide imperative, on-demand node control for scenarios that don't fit the declarative `ScenarioBuilder` pattern: +Use `ManualCluster` when orchestration lives outside the scenario runtime. Scenarios can also start and restart nodes from workloads by requesting `with_node_control()`; see [Scenario Capabilities](capabilities.md) and [Chaos and Controlled Failure](chaos.md). -```rust -use testing_framework_core::topology::config::TopologyConfig; -use testing_framework_core::scenario::{PeerSelection, StartNodeOptions}; -use testing_framework_runner_local::LocalDeployer; +- **Step-driven flows**: an external driver decides when each node starts and what happens next. +- **BDD harnesses**: Gherkin steps map naturally onto imperative start/stop/wait calls. +- **Exploratory debugging**: poke at a live cluster from a `main` function without writing workloads or expectations. -let config = TopologyConfig::with_node_numbers(3); -let deployer = LocalDeployer::new(); -let cluster = deployer.manual_cluster(config)?; +There are no workloads, expectations, or `RunContext`; you call methods and assert with your own code. -// Start nodes on demand with explicit peer selection -let node_a = cluster.start_node_with( - "a", - StartNodeOptions { - peers: PeerSelection::None, // Start isolated - } -).await?.api; +--- -let node_b = cluster.start_node_with( - "b", - StartNodeOptions { - peers: PeerSelection::Named(vec!["node-a".to_owned()]), // Connect to A - } -).await?.api; +## Creating a Cluster -// Wait for network readiness +Two equivalent entry points on the local backend (`testing-framework/deployers/local/src/manual/mod.rs`): + +```rust,ignore +use testing_framework_runner_local::{ManualCluster, ProcessDeployer}; + +// Directly from a deployment descriptor… +let cluster = ManualCluster::::from_topology(KvTopology::new(3)); + +// …or via the deployer +let deployer = ProcessDeployer::::new(); +let cluster = deployer.manual_cluster_from_descriptors(KvTopology::new(3)); +``` + +The descriptor defines capacity and indexing, not initial state: no processes exist until you call `start_node`. `E` must implement `LocalDeployerEnv` (see [Implementing Application](implementing-application.md)). + +**Naming:** requested names are normalized to a `node-` prefix: `start_node("a")` registers `node-a`; names already starting with `node-` pass through; an empty name becomes `node-`. Each started node needs a fresh name; reusing a registered name is an error. + +--- + +## API + +| Method | What it does | +|---|---| +| `start_node(name)` | Start a node with default options | +| `start_node_with(name, options)` | Start with `StartNodeOptions` (below); returns `StartedNode { name, client }` | +| `stop_node(name)` | Kill the process; the node stays registered | +| `stop_all()` | Stop every node and reset registration state (also runs on drop) | +| `restart_node(name)` | Stop and respawn in the same working directory | +| `restart_node_with(name, options)` | Restart with extra `args` / `runtime`; other overrides rejected | +| `wait_network_ready()` | Poll every started node's readiness endpoint (`AllNodesReady`) | +| `wait_node_ready(name)` | Poll one node, honoring its `start_timeout` | +| `node_client(name)` / `node_clients()` | Look up one client / the shared `NodeClients` collection | +| `node_pid(name)` | OS pid, `None` if the process is not running | +| `add_external_sources(sources)` | Build clients for `ExternalNodeSource`s and add them to the client set | +| `add_external_clients(clients)` | Add prebuilt clients to the client set | + +`ManualCluster` also implements the core `NodeControlHandle` and `ClusterWaitHandle` traits, so it can stand behind code written against those abstractions. An app-layer child cluster exposes the same common operations through `ClusterHandle`, without exposing the backend-specific `ManualCluster` object. + +--- + +## StartNodeOptions + +The full options struct (`core/src/scenario/capabilities.rs`): + +| Field | Type | Builder | Meaning | +|---|---|---|---| +| `peers` | `Option` | `with_peers(sel)` | `DefaultLayout`, `None`, or `Named(names)` — see [node-config.md](node-config.md) for where each path honors it | +| `config_override` | `Option` | `with_config_override(cfg)` | Replace the generated config wholesale | +| `config_patch` | patch closure | `create_patch(fn)` | Transform the generated config before spawn | +| `persist_dir` | `Option` | `with_persist_dir(path)` | Place the working directory predictably — see [Persistence](persistence.md) | +| `snapshot_dir` | `Option` | `with_snapshot_dir(path)` | Seed the working directory from saved state — see [Persistence](persistence.md) | +| `args` | `Vec` | `with_args(args)` | Extra CLI args appended on launch | +| `runtime` | `NodeRuntimeOptions` | `with_runtime(opts)` / `with_start_timeout(dur)` | Per-node readiness timeout | + +`restart_node_with` accepts only `args` and `runtime`. Passing `peers`, `config_override`, `config_patch`, `persist_dir`, or `snapshot_dir` to a restart returns an `InvalidArgument` error, because a restart reuses the node's existing config and working directory. To change those, stop the node and start a new one. + +--- + +## Example: Convergence Under Restart + +Adapted from the in-repo example `cargo run -p kvstore-examples --bin kvstore_k8s_manual_convergence` (`examples/kvstore/examples/src/bin/k8s_manual_convergence.rs`): + +```rust,ignore +let deployer = KvK8sDeployer::new(); +let cluster = deployer + .manual_cluster_from_descriptors(KvTopology::new(3)) + .await?; + +let node0 = cluster.start_node("node-0").await?.client; +let node1 = cluster.start_node("node-1").await?.client; +let node2 = cluster.start_node("node-2").await?.client; cluster.wait_network_ready().await?; -// Custom validation logic -let info_a = node_a.consensus_info().await?; -let info_b = node_b.consensus_info().await?; -assert!(info_a.height.abs_diff(info_b.height) <= 5); -``` +write_keys(&node0, "kv-manual", 12).await?; +wait_for_convergence(&[node0.clone(), node1.clone(), node2.clone()], "kv-manual", 12).await?; -**Key difference from scenarios:** -- **External orchestration:** Your code (or an external driver like Cucumber) controls the execution flow step-by-step -- **Imperative model:** You call `start_node()`, `sleep()`, poll APIs directly in test logic -- **No framework execution:** The scenario runner doesn't drive workloads—you do - -Note: Scenarios with node control can also start nodes dynamically, control peer selection, and orchestrate timing—but via **workloads** within the framework's execution model. Use manual clusters only when the orchestration must be external (e.g., Cucumber steps). - ---- - -## When to Use Manual Clusters - -**Manual clusters are an escape hatch for when orchestration must live outside the framework.** - -Prefer workloads for scenario logic; use manual clusters only when an external system needs to control node lifecycle—for example: - -**Cucumber/BDD integration** -Gherkin steps control when nodes start, which peers they connect to, and when to verify state. The test driver (Cucumber) orchestrates the scenario step-by-step. - -**Custom test harnesses** -External scripts or tools that need programmatic control over node lifecycle as part of a larger testing pipeline. - ---- - -## Core API - -### Starting the Cluster - -```rust -use testing_framework_core::topology::config::TopologyConfig; -use testing_framework_runner_local::LocalDeployer; - -// Define capacity (preallocates ports/configs for N nodes) -let config = TopologyConfig::with_node_numbers(5); - -let deployer = LocalDeployer::new(); -let cluster = deployer.manual_cluster(config)?; -// Nodes are stopped automatically when cluster is dropped -``` - -**Important:** The `TopologyConfig` defines the **maximum capacity**, not the initial state. Nodes are started on-demand via API calls. - -### Starting Nodes - -**Default peers (topology layout):** - -```rust -let node = cluster.start_node("seed").await?; -``` - -**No peers (isolated):** - -```rust -use testing_framework_core::scenario::{PeerSelection, StartNodeOptions}; - -let node = cluster.start_node_with( - "isolated", - StartNodeOptions { - peers: PeerSelection::None, - } -).await?; -``` - -**Explicit peers (named):** - -```rust -let node = cluster.start_node_with( - "follower", - StartNodeOptions { - peers: PeerSelection::Named(vec![ - "node-seed".to_owned(), - "node-isolated".to_owned(), - ]), - } -).await?; -``` - -**Note:** Node names are prefixed with `node-` internally. If you start a node with name `"a"`, reference it as `"node-a"` in peer lists. - -### Getting Node Clients - -```rust -// From start result -let started = cluster.start_node("my-node").await?; -let client = started.api; - -// Or lookup by name -if let Some(client) = cluster.node_client("node-my-node") { - let info = client.consensus_info().await?; - println!("Height: {}", info.height); -} -``` - -### Waiting for Readiness - -```rust -// Waits until all started nodes have connected to their expected peers +cluster.restart_node("node-2").await?; cluster.wait_network_ready().await?; + +let node2 = cluster.node_client("node-2").expect("client after restart"); +wait_for_convergence(&[node0, node1, node2], "kv-manual", 12).await?; + +cluster.stop_all(); ``` -**Behavior:** -- Single-node clusters always ready (no peers to verify) -- Multi-node clusters wait for peer counts to match expectations -- Timeout after 60 seconds (120 seconds if `SLOW_TEST_ENV=true`) with diagnostic message +The driver determines which nodes exist, when writes happen, what convergence means, and when to inject the restart. `write_keys` and `wait_for_convergence` are plain functions over the application's HTTP client. + +That example runs on Kubernetes: the Kubernetes deployer supplies a manual cluster with the same method surface (`manual_cluster_from_descriptors` there is `async` and fallible because it must install the stack first). The local `ManualCluster` documented in this chapter starts processes directly and needs no external infrastructure. --- -## Complete Example: External Test Driver Pattern +## Lifecycle and Cleanup -This shows how an external test driver (like Cucumber) might use manual clusters to control node lifecycle: +Dropping the `ManualCluster` calls `stop_all()`: every child process is killed and waited on. Node working directories are temporary and removed with the processes unless retained. Set `TF_KEEP_LOGS=1` (or `true`/`yes`) to keep them for inspection, and see [Persistence](persistence.md) for deliberate state retention. -```rust -use std::time::Duration; -use anyhow::Result; -use testing_framework_core::{ - scenario::{PeerSelection, StartNodeOptions}, - topology::config::TopologyConfig, -}; -use testing_framework_runner_local::LocalDeployer; -use tokio::time::sleep; - -#[tokio::test] -async fn external_driver_example() -> Result<()> { - // Step 1: Create cluster with capacity for 3 nodes - let config = TopologyConfig::with_node_numbers(3); - let deployer = LocalDeployer::new(); - let cluster = deployer.manual_cluster(config)?; - - // Step 2: External driver decides to start 2 nodes initially - println!("Starting initial topology..."); - let node_a = cluster.start_node("a").await?.api; - let node_b = cluster - .start_node_with( - "b", - StartNodeOptions { - peers: PeerSelection::Named(vec!["node-a".to_owned()]), - }, - ) - .await? - .api; - - cluster.wait_network_ready().await?; - - // Step 3: External driver runs some protocol operations - let info = node_a.consensus_info().await?; - println!("Initial cluster height: {}", info.height); - - // Step 4: Later, external driver decides to add third node - println!("External driver adding third node..."); - let node_c = cluster - .start_node_with( - "c", - StartNodeOptions { - peers: PeerSelection::Named(vec!["node-a".to_owned()]), - }, - ) - .await? - .api; - - cluster.wait_network_ready().await?; - - // Step 5: External driver validates final state - let heights = vec![ - node_a.consensus_info().await?.height, - node_b.consensus_info().await?.height, - node_c.consensus_info().await?.height, - ]; - println!("Final heights: {:?}", heights); - - Ok(()) -} -``` - -**Key pattern:** -The external driver controls **when** nodes start and **which peers** they connect to, allowing test frameworks like Cucumber to orchestrate scenarios step-by-step based on Gherkin steps or other external logic. - ---- - -## Peer Selection Strategies - -**`PeerSelection::DefaultLayout`** -Uses the topology's network layout (star/chain/full). Default behavior. - -```rust -let node = cluster.start_node_with( - "normal", - StartNodeOptions { - peers: PeerSelection::DefaultLayout, - } -).await?; -``` - -**`PeerSelection::None`** -Node starts with no initial peers. Use when an external driver needs to build topology incrementally. - -```rust -let isolated = cluster.start_node_with( - "isolated", - StartNodeOptions { - peers: PeerSelection::None, - } -).await?; -``` - -**`PeerSelection::Named(vec!["node-a", "node-b"])`** -Explicit peer list. Use when an external driver needs to construct specific peer relationships. - -```rust -let follower = cluster.start_node_with( - "follower", - StartNodeOptions { - peers: PeerSelection::Named(vec![ - "node-seed".to_owned(), - "node-seed".to_owned(), - ]), - } -).await?; -``` - -**Remember:** Node names are automatically prefixed with `node-`. If you call `start_node("a")`, reference it as `"node-a"` in peer lists. - ---- - -## Custom Validation Patterns - -Manual clusters don't have built-in expectations—you write validation logic directly: - -### Height Convergence - -```rust -use tokio::time::{sleep, Duration}; - -let start = tokio::time::Instant::now(); -loop { - let heights: Vec = vec![ - node_a.consensus_info().await?.height, - node_b.consensus_info().await?.height, - node_c.consensus_info().await?.height, - ]; - - let max_diff = heights.iter().max().unwrap() - heights.iter().min().unwrap(); - if max_diff <= 5 { - println!("Converged: heights={:?}", heights); - break; - } - - if start.elapsed() > Duration::from_secs(60) { - return Err(anyhow::anyhow!("Convergence timeout: heights={:?}", heights)); - } - - sleep(Duration::from_secs(2)).await; -} -``` - -### Peer Count Verification - -```rust -let info = node.network_info().await?; -assert_eq!( - info.n_peers, 3, - "Expected 3 peers, found {}", - info.n_peers -); -``` - -### Block Production - -```rust -// Verify node is producing blocks -let initial_height = node_a.consensus_info().await?.height; - -sleep(Duration::from_secs(10)).await; - -let current_height = node_a.consensus_info().await?.height; -assert!( - current_height > initial_height, - "Node should have produced blocks: initial={}, current={}", - initial_height, - current_height -); -``` - ---- - -## Limitations - -**Local deployer only** -Manual clusters currently only work with `LocalDeployer`. Compose and K8s support is not available. - -**No built-in workloads** -You must manually submit transactions via node API clients. The framework's transaction workloads are scenario-specific. - -**No automatic expectations** -You wire validation yourself. The `.expect_*()` methods from scenarios are not automatically attached—you write custom validation loops. - -**No RunContext** -Manual clusters don't provide `RunContext`, so features like `BlockFeed` and metrics queries require manual setup. - ---- - -## Relationship to Node Control - -Manual clusters and [node control](node-control.md) share the same underlying infrastructure (`LocalDynamicNodes`), but serve different purposes: - -| Feature | Manual Cluster | Node Control (Scenario) | -|---------|---------------|-------------------------| -| **Orchestration** | External (your code/Cucumber) | Framework (workloads) | -| **Programming model** | Imperative (step-by-step) | Declarative (plan + execute) | -| **Node lifecycle** | Manual `start_node()` calls | Automatic + workload-driven | -| **Traffic generation** | Manual API calls | Built-in workloads (tx, chaos) | -| **Validation** | Manual polling loops | Built-in expectations + custom | -| **Use case** | Cucumber/BDD integration | Standard testing & chaos | - -**When to use which:** -- **Scenarios with node control** → Standard testing (built-in workloads drive node control) -- **Manual clusters** → External drivers (Cucumber/BDD where external logic drives node control) - ---- - -## Running Manual Cluster Tests - -Manual cluster tests are typically marked with `#[ignore]` to prevent accidental runs: - -```rust -#[tokio::test] -#[ignore = "run manually with: cargo test -- --ignored external_driver_example"] -async fn external_driver_example() -> Result<()> { - // ... -} -``` - -**To run:** - -```bash -# Required: dev mode for fast proofs -cargo test -p runner-examples -- --ignored external_driver_example -``` - -**Logs:** - -```bash -# Preserve logs after test -LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1 \ -RUST_LOG=info \ -cargo test -p runner-examples -- --ignored external_driver_example -``` - ---- - -## See Also - -- [Testing Philosophy](testing-philosophy.md) — Why the framework is declarative by default -- [RunContext: BlockFeed & Node Control](node-control.md) — Node control within scenarios -- [Chaos Testing](chaos.md) — Restart-based chaos (scenario approach) -- [Scenario Builder Extensions](scenario-builder-ext-patterns.md) — Extending the declarative model +> **External example:** logos-blockchain's cucumber suite drives `ManualCluster` from Gherkin steps in its own repository, including dependency-ordered starts, targeted restarts, snapshot-on-stop, and restore-from-snapshot. diff --git a/book/src/node-config.md b/book/src/node-config.md new file mode 100644 index 0000000..4529e80 --- /dev/null +++ b/book/src/node-config.md @@ -0,0 +1,126 @@ +# Ports, Peers, Node Config, and Readiness + +This chapter describes how the local deployer allocates ports, wires peers, materializes per-node configs, and decides when a cluster is ready. + +--- + +## Port Allocation + +All local ports come from the OS. `preallocate_ports` (in `testing-framework/deployers/local/src/env/helpers.rs`) binds `127.0.0.1:0`, records the assigned port, and releases the listener. `reserve_local_node_ports(count, names, label)` does this for every node up front and returns one `LocalNodePorts` per node: + +| `LocalNodePorts` method | Returns | +|---|---| +| `network_port()` | The main reserved port for peer traffic | +| `get(name)` / `require(name)` | A reserved *named* port (`Option` / `Result`) | +| `iter()` | All named ports | + +Named ports exist for apps that need more than one listener per node. Declare them via `LocalBinaryApp::initial_local_port_names()` (or `LocalDeployerEnv::local_port_names()`); the deployer reserves one port per name per node. + +Ports are reserved by bind-and-release, so they are free at reservation time but are not deterministic across runs. See [Seeds and Reproducibility](seeds.md). + +--- + +## Peer Wiring + +Peer wiring is why up-front port reservation matters: because every node's ports are reserved before any config is built, each node's config can reference the real addresses of all its peers before a single process starts. + +For each node index the deployer builds peer views of every *other* node: + +- `LocalPeerNode`: `index()`, `network_port()`, `http_address()` / `authority()` (`127.0.0.1:`). +- `build_local_peer_nodes(peer_ports, self_index)`: full peer views, skipping self. +- `build_indexed_http_peers(node_count, self_index, peer_ports, build_peer)`: map peers through your own constructor. + +These flow into the app's config hook together with the node's own ports: + +```rust,ignore +fn build_local_node_config_with_peers( + topology: &Self::Deployment, + index: usize, + ports: &LocalNodePorts, + peers: &[LocalPeerNode], + peer_ports_by_name: &HashMap, + options: &StartNodeOptions, + template_config: Option<&Self::NodeConfig>, +) -> Result; +``` + +For initial cluster startup the deployer calls this once per index with every other node as a peer (a full mesh view); the layout your nodes actually form is up to the config your app builds from those views. `LocalBuildContext` carries the same fields when you customize `build_initial_node_configs` on the full `LocalDeployerEnv` path. Apps that implement `ClusterNodeConfigApplication` can delegate the whole hook to `build_local_cluster_node_config::(index, ports, peers)`, the same abstraction the container backends reuse (see [cfgsync](cfgsync.md)). + +--- + +## Config Templates: LocalProcessSpec + +`LocalProcessSpec` describes how one rendered config becomes a running process: + +| Field / builder | Meaning | +|---|---| +| `LocalProcessSpec::new(env_var)` | Start from an `EnvBinaryProvider` for `env_var` | +| `with_binary_path(path)` / `with_binary_provider(p)` / `with_binary_provider_ref(p)` | Choose the binary source ([Binary Providers](binary-providers.md)) | +| `config_file_name` (default `config.yaml`) | File written into the node working directory | +| `with_config_file(name, arg)` | Pass as a flag pair, e.g. `--config app.yaml` | +| `with_positional_config_file(name)` | Pass the path as a positional argument (`LocalConfigArgMode::Positional`) | +| `with_env(key, value)` / `with_rust_log(value)` | Child process environment | +| `with_args(args)` | Extra CLI args appended after the config argument | + +Rendering helpers: `yaml_node_config` (serialize to YAML bytes), `text_node_config` (already-rendered text), `yaml_config_launch_spec` / `text_config_launch_spec` / `default_yaml_launch_spec` (build a full `LaunchSpec` in one call). The resulting `LaunchSpec` lists the binary, the files to materialize, args, and env; the deployer writes the files into the node's working directory and spawns the process there. + +--- + +## StartNodeOptions: Overrides at Start Time + +Dynamically started nodes (node-control workloads and [ManualCluster](manual-cluster.md)) accept `StartNodeOptions`; the full field table lives in the [ManualCluster chapter](manual-cluster.md). The two config-shaping fields deserve care: + +**`config_override`** replaces the complete generated config. **`config_patch`** (set via `create_patch(|config| ...)`) transforms the generated config, retaining framework-assigned ports and peers unless the callback changes them. A full override must provide every required port itself. + +Where they are honored differs by path: + +- **Local dynamic starts** (`NodeManager::start_node_with`): the framework builds the config through the env hooks (which receive the full `options`) and then applies `config_patch` itself. `config_override` and `peers` are visible to your `build_local_node_config_with_peers` implementation but are not interpreted centrally by the local path. +- **Static-artifact path** (used by the container backends through `StaticNodeConfigProvider::build_node_artifacts_for_options`, `core/src/scenario/config.rs`): the framework interprets everything: `PeerSelection` picks the peer set, then `config_override` replaces, then `config_patch` transforms, and the result is served as an override artifact. + +**`PeerSelection`** variants (`core/src/scenario/capabilities.rs`): + +| Variant | Effect (static-artifact path) | +|---|---| +| `DefaultLayout` | Peer view of all other nodes (same as omitting `peers`) | +| `None` | Start with an empty peer list | +| `Named(vec!["node-0", ...])` | Only the named nodes (names follow the `node-` convention) | + +--- + +## Readiness + +**Per-node probe.** The local deployer probes each node's API port using `LocalDeployerEnv::readiness_probe()`: + +- `LocalReadinessProbe::HttpGet { path }` (default): GET `http://127.0.0.1:` until it returns 2xx. The path defaults to `Application::node_readiness_path()` (`"/"` unless overridden; kvstore uses `/health/ready`). +- `LocalReadinessProbe::Tcp`: the port merely accepts TCP connections. Use for nodes without an HTTP surface. + +**Cluster requirement.** `HttpReadinessRequirement` (`core/src/scenario/runtime/readiness.rs`) decides how many nodes must pass: + +| Variant | Ready when | +|---|---| +| `AllNodesReady` | Every node answers (default) | +| `AnyNodeReady` | At least one node answers | +| `AtLeast(n)` | At least `n` nodes answer | + +Set it on the scenario with `ScenarioBuilder::with_http_readiness_requirement(requirement)`, or as part of a full `DeploymentPolicy`; see [Readiness, Retry, and Artifact Preservation](deployment-policies.md). + +**Waiting imperatively.** `ManualCluster` (and `LocalAppCluster`) expose: + +- `wait_network_ready()`: polls every started node's API port with `AllNodesReady`. +- `wait_node_ready(name)`: polls one node, honoring that node's `NodeRuntimeOptions::start_timeout` if one was set via `StartNodeOptions::with_start_timeout`. + +Default probe timeout is 60 seconds with a 200 ms poll interval; setting `SLOW_TEST_ENV=true` doubles timeouts. Timeouts fail with a message listing the endpoints that never answered. + +**App-specific stabilization.** After the port probe succeeds during deployment, the local deployer calls `wait_readiness_stable(nodes)`, a hook where an app can wait for cluster-level convergence (membership settled, leader elected) before workloads start. The default is a no-op. + +```mermaid +sequenceDiagram + participant D as Deployer + participant N as Node process + D->>N: spawn (config materialized in working dir) + loop until 2xx or timeout + D->>N: GET /health/ready + end + D->>D: requirement satisfied? (All/Any/AtLeast) + D->>N: wait_readiness_stable(...) +``` diff --git a/book/src/node-control.md b/book/src/node-control.md deleted file mode 100644 index 7b9df61..0000000 --- a/book/src/node-control.md +++ /dev/null @@ -1,391 +0,0 @@ -# RunContext: BlockFeed & Node Control - -The deployer supplies a `RunContext` that workloads and expectations share. It -provides: - -- Topology descriptors (`GeneratedTopology`) -- Client handles (`NodeClients` / `ClusterClient`) for HTTP/RPC calls -- Metrics (`RunMetrics`, `Metrics`) and block feed -- Optional `NodeControlHandle` for managing nodes - -## BlockFeed: Observing Block Production - -The `BlockFeed` is a broadcast stream of block observations that allows workloads and expectations to monitor blockchain progress in real-time. It polls a node continuously and broadcasts new blocks to all subscribers. - -### What BlockFeed Provides - -**Real-time block stream:** -- Subscribe to receive `BlockRecord` notifications as blocks are produced -- Each record includes the block header (`HeaderId`) and full block payload -- Backed by a background task that polls node storage every second - -**Block statistics:** -- Track total transactions across all observed blocks -- Access via `block_feed.stats().total_transactions()` - -**Broadcast semantics:** -- Multiple subscribers can receive the same blocks independently -- Late subscribers start receiving from current block (no history replay) -- Lagged subscribers skip missed blocks automatically - -### Accessing BlockFeed - -BlockFeed is available through `RunContext`: - -```rust,ignore -let block_feed = ctx.block_feed(); -``` - -### Usage in Expectations - -Expectations typically use BlockFeed to verify block production and inclusion of transactions/data. - -**Example: Counting blocks during a run** - -```rust,ignore -use std::sync::{ - Arc, - atomic::{AtomicU64, Ordering}, -}; - -use async_trait::async_trait; -use testing_framework_core::scenario::{DynError, Expectation, RunContext}; - -struct MinimumBlocksExpectation { - min_blocks: u64, - captured_blocks: Option>, -} - -#[async_trait] -impl Expectation for MinimumBlocksExpectation { - fn name(&self) -> &'static str { - "minimum_blocks" - } - - async fn start_capture(&mut self, ctx: &RunContext) -> Result<(), DynError> { - let block_count = Arc::new(AtomicU64::new(0)); - let block_count_task = Arc::clone(&block_count); - - // Subscribe to block feed - let mut receiver = ctx.block_feed().subscribe(); - - // Spawn a task to count blocks - tokio::spawn(async move { - loop { - match receiver.recv().await { - Ok(_record) => { - block_count_task.fetch_add(1, Ordering::Relaxed); - } - Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { - tracing::debug!(skipped, "receiver lagged, skipping blocks"); - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => { - tracing::debug!("block feed closed"); - break; - } - } - } - }); - - self.captured_blocks = Some(block_count); - Ok(()) - } - - async fn evaluate(&mut self, ctx: &RunContext) -> Result<(), DynError> { - let blocks = self.captured_blocks - .as_ref() - .expect("start_capture must be called first") - .load(Ordering::Relaxed); - - if blocks < self.min_blocks { - return Err(format!( - "expected at least {} blocks, observed {}", - self.min_blocks, blocks - ).into()); - } - - tracing::info!(blocks, min = self.min_blocks, "minimum blocks expectation passed"); - Ok(()) - } -} -``` - -**Example: Inspecting block contents** - -```rust,ignore -use testing_framework_core::scenario::{DynError, RunContext}; - -async fn start_capture(ctx: &RunContext) -> Result<(), DynError> { - let mut receiver = ctx.block_feed().subscribe(); - - tokio::spawn(async move { - loop { - match receiver.recv().await { - Ok(record) => { - // Access block header - let header_id = &record.header; - - // Access full block - let tx_count = record.block.transactions().len(); - - tracing::debug!( - ?header_id, - tx_count, - "observed block" - ); - - // Process transactions or other block data. - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => break, - Err(_) => continue, - } - } - }); - - Ok(()) -} -``` - -### Usage in Workloads - -Workloads can use BlockFeed to coordinate timing or wait for specific conditions before proceeding. - -**Example: Wait for N blocks before starting** - -```rust,ignore -use async_trait::async_trait; -use testing_framework_core::scenario::{DynError, RunContext, Workload}; - -struct DelayedWorkload { - wait_blocks: usize, -} - -#[async_trait] -impl Workload for DelayedWorkload { - fn name(&self) -> &str { - "delayed_workload" - } - - async fn start(&self, ctx: &RunContext) -> Result<(), DynError> { - tracing::info!(wait_blocks = self.wait_blocks, "waiting for blocks before starting"); - - // Subscribe to block feed - let mut receiver = ctx.block_feed().subscribe(); - let mut count = 0; - - // Wait for N blocks - while count < self.wait_blocks { - match receiver.recv().await { - Ok(_) => count += 1, - Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, - Err(tokio::sync::broadcast::error::RecvError::Closed) => { - return Err("block feed closed before reaching target".into()); - } - } - } - - tracing::info!("warmup complete, starting actual workload"); - - // Now do the actual work - // ... - - Ok(()) - } -} -``` - -**Example: Rate limiting based on block production** - -```rust,ignore -use testing_framework_core::scenario::{DynError, RunContext}; - -async fn generate_request() -> Option<()> { - None -} - -async fn start(ctx: &RunContext) -> Result<(), DynError> { - let clients = ctx.node_clients().node_clients(); - let mut receiver = ctx.block_feed().subscribe(); - let mut pending_requests: Vec<()> = Vec::new(); - - loop { - tokio::select! { - // Issue a batch on each new block. - Ok(_record) = receiver.recv() => { - if !pending_requests.is_empty() { - tracing::debug!(count = pending_requests.len(), "issuing requests on new block"); - for _req in pending_requests.drain(..) { - let _info = clients[0].consensus_info().await?; - } - } - } - - // Generate work continuously. - Some(req) = generate_request() => { - pending_requests.push(req); - } - } - } -} -``` - -### BlockFeed vs Direct Polling - -**Use BlockFeed when:** -- You need to react to blocks as they're produced -- Multiple components need to observe the same blocks -- You want automatic retry/reconnect logic -- You're tracking statistics across many blocks - -**Use direct polling when:** -- You need to query specific historical blocks -- You're checking final state after workloads complete -- You need transaction receipts or other indexed data -- You're implementing a one-time health check - -Example direct polling in expectations: - -```rust,ignore -use testing_framework_core::scenario::{DynError, RunContext}; - -async fn evaluate(ctx: &RunContext) -> Result<(), DynError> { - let client = &ctx.node_clients().node_clients()[0]; - - // Poll current height once - let info = client.consensus_info().await?; - tracing::info!(height = info.height, "final block height"); - - // This is simpler than BlockFeed for one-time checks - Ok(()) -} -``` - -### Block Statistics - -Access aggregated statistics without subscribing to the feed: - -```rust,ignore -use testing_framework_core::scenario::{DynError, RunContext}; - -async fn evaluate(ctx: &RunContext, expected_min: u64) -> Result<(), DynError> { - let stats = ctx.block_feed().stats(); - let total_txs = stats.total_transactions(); - - tracing::info!(total_txs, "transactions observed across all blocks"); - - if total_txs < expected_min { - return Err(format!( - "expected at least {} transactions, observed {}", - expected_min, total_txs - ).into()); - } - - Ok(()) -} -``` - -### Important Notes - -**Subscription timing:** -- Subscribe in `start_capture()` for expectations -- Subscribe in `start()` for workloads -- Late subscribers miss historical blocks (no replay) - -**Lagged receivers:** -- If your subscriber is too slow, it may lag behind -- Handle `RecvError::Lagged(skipped)` gracefully -- Consider increasing processing speed or reducing block rate - -**Feed lifetime:** -- BlockFeed runs for the entire scenario duration -- Automatically cleaned up when the run completes -- Closed channels signal graceful shutdown - -**Performance:** -- BlockFeed polls nodes every 1 second -- Broadcasts to all subscribers with minimal overhead -- Suitable for scenarios with hundreds of blocks - -### Real-World Examples - -The framework's built-in expectations use BlockFeed extensively: - -- **`ConsensusLiveness`**: Doesn't directly subscribe but uses block feed stats to verify progress -- **`TransactionInclusion`**: Subscribes to find specific transactions in blocks - -See [Examples](examples.md) and [Workloads & Expectations](workloads.md) for more patterns. - ---- - -## Current Chaos Capabilities and Limitations - -The framework currently supports **process-level chaos** (node restarts) for -resilience testing: - -**Supported:** -- Restart nodes (`restart_node`) -- Random restart workload via `.chaos().restart()` - -**Not Yet Supported:** -- Network partitions (blocking peers, packet loss) -- Resource constraints (CPU throttling, memory limits) -- Byzantine behavior injection (invalid blocks, bad signatures) -- Selective peer blocking/unblocking - -For network partition testing, see [Extension Ideas](examples-advanced.md#extension-ideas) -which describes the proposed `block_peer`/`unblock_peer` API (not yet implemented). - -## Accessing node control in workloads/expectations - -Check for control support and use it conditionally: - -```rust,ignore -use async_trait::async_trait; -use testing_framework_core::scenario::{DynError, RunContext, Workload}; - -struct RestartWorkload; - -#[async_trait] -impl Workload for RestartWorkload { - fn name(&self) -> &str { - "restart_workload" - } - - async fn start(&self, ctx: &RunContext) -> Result<(), DynError> { - if let Some(control) = ctx.node_control() { - // Restart the first node (index 0) if supported. - control.restart_node(0).await?; - } - Ok(()) - } -} -``` - -When chaos workloads need control, require `enable_node_control()` in the -scenario builder and deploy with a runner that supports it. - -## Current API surface - -The `NodeControlHandle` trait currently provides: - -```rust,ignore -use async_trait::async_trait; -use testing_framework_core::scenario::DynError; - -#[async_trait] -pub trait NodeControlHandle: Send + Sync { - async fn restart_node(&self, index: usize) -> Result<(), DynError>; -} -``` - -Future extensions may include peer blocking/unblocking or other control -operations. For now, focus on restart-based chaos patterns as shown in the -chaos workload examples. - -## Considerations - -- Always guard control usage: not all runners expose `NodeControlHandle`. -- Treat control as best-effort: failures should surface as test failures, but - workloads should degrade gracefully when control is absent. -- Combine control actions with expectations (e.g., restart then assert height - convergence) to keep scenarios meaningful. diff --git a/book/src/observation.md b/book/src/observation.md new file mode 100644 index 0000000..c92e2a3 --- /dev/null +++ b/book/src/observation.md @@ -0,0 +1,154 @@ +# Continuous Observation + +The observation runtime polls application state in the background and stores snapshots, histories, and event streams for workloads and expectations. It provides typed state inside the test process rather than external telemetry. + +--- + +## Shared Polling Runtime + +Chaos and convergence tests repeatedly query state such as the current leader, whether every node has seen a key, or what changed after a restart. The observation runtime (`testing-framework/core/src/observation/`) runs one background polling task with shared error and staleness tracking. Workloads and expectations read the stored state. + +[Telemetry](telemetry.md) exports metrics, logs, and traces to external endpoints. Observation instead keeps typed application state inside the test and makes it synchronously queryable during the run. + +--- + +## The Observer Trait + +An application defines how to poll and interpret its state; the runtime schedules the polling: + +```rust,ignore +#[async_trait] +pub trait Observer: Send + Sync + 'static { + type Source: Clone + Send + Sync + 'static; // app-owned source handle + type State: Send + Sync + 'static; // retained materialized state + type Snapshot: Clone + Send + Sync + 'static; // current view + type Event: Clone + Send + Sync + 'static; // delta emitted per cycle + + async fn init(&self, sources: &[ObservedSource]) -> Result; + + async fn poll( + &self, + sources: &[ObservedSource], + state: &mut Self::State, + ) -> Result, DynError>; + + fn snapshot(&self, state: &Self::State) -> Self::Snapshot; +} +``` + +Each cycle the runtime refreshes the source set, calls `poll` to advance `State` and collect delta `Event`s, then derives a `Snapshot` from the state. `ObservedSource` is just a `name` plus the app-owned source value (`ObservedSource::new(name, source)`), typically a node client. + +**Sources are re-queried every cycle** through `SourceProvider`: + +```rust,ignore +#[async_trait] +pub trait SourceProvider: Send + Sync + 'static { + async fn sources(&self) -> Result>, DynError>; +} +``` + +`StaticSourceProvider::new(sources)` covers the common fixed-cluster case. A custom provider makes sources *dynamic*: it can return a different set each cycle, which lets observation stay correct across node restarts. `SourceProviderFactory` builds the provider once node clients exist; any closure `Fn(&E::Deployment, NodeClients) -> Result, DynError>` qualifies. + +--- + +## Plugging Into a Scenario + +`ObservationExtensionFactory` is a [runtime extension factory](runtime-extensions.md): at prepare time it builds the source provider, starts the runtime, and stores the read handle in the `RunContext` (background task registered for abort-on-teardown via `PreparedRuntimeExtension::from_task`). The builder has convenience methods for it (`CoreBuilderExt`): + +```rust,ignore +// Clonable observer: +builder.with_observer(MyObserver, my_source_provider_fn, ObservationConfig::default()) +// Observer built lazily per run: +builder.with_observer_factory(|| MyObserver::new(), my_source_provider_fn, config) +``` + +`ObservationConfig` has two fields: `interval` (time between cycles, default 1 s, must be non-zero) and `history_limit` (retained non-empty event batches, default 64). + +Outside scenarios, for example around a `ManualCluster`, start it directly: `ObservationRuntime::start(provider, observer, config)`, then `handle()`, `into_parts()` (handle + `JoinHandle`), or `abort()`. Dropping the runtime aborts the task. + +--- + +## Reading: the ObservationHandle + +Workloads and expectations retrieve the handle by type and read four things: + +| Method | Returns | +|--------|---------| +| `latest_snapshot()` | `Option>` — most recent successful view | +| `history()` | Retained non-empty `ObservationBatch`s, oldest first, bounded by `history_limit` | +| `last_error()` | `Option` — the most recent failed cycle | +| `subscribe()` | `broadcast::Receiver` of future non-empty batches | + +**Snapshots vs batches vs events:** a *snapshot* is the whole current view (`cycle`, `observed_at`, `source_count`, `value`); an *event* is one delta discovered during a cycle; a *batch* groups the events of one cycle. Cycles that produce no events produce no batch. `history()` and `subscribe()` only ever see non-empty batches, while `latest_snapshot()` is refreshed on every successful cycle. + +**Freshness and failures.** On a failed cycle, the runtime records an `ObservationFailure` (with `stage: SourceRefresh` if source discovery failed, `stage: Poll` if the observer failed) and retains the last successful snapshot. The next successful cycle clears `last_error`. To check staleness, compare `snapshot.cycle` or `observed_at` across reads, and inspect `last_error()` when a wait times out; it usually names the source that stopped answering. + +--- + +## Worked Example: the OpenRaft Cluster Observer + +`examples/openraft_kv/testing/integration/src/observation.rs` observes a Raft cluster. State and snapshot are the same type (the latest per-node states plus any per-source failures), and no delta events are emitted (`Event = ()`): + +```rust,ignore +#[derive(Clone, Debug, Default)] +pub struct OpenRaftClusterObserver; + +#[async_trait] +impl Observer for OpenRaftClusterObserver { + type Source = OpenRaftKvClient; + type State = OpenRaftClusterSnapshot; + type Snapshot = OpenRaftClusterSnapshot; + type Event = (); + + async fn init(&self, sources: &[ObservedSource]) -> Result { + Ok(capture_cluster_snapshot(sources).await) + } + + async fn poll( + &self, + sources: &[ObservedSource], + state: &mut Self::State, + ) -> Result, DynError> { + *state = capture_cluster_snapshot(sources).await; + Ok(Vec::new()) + } + + fn snapshot(&self, state: &Self::State) -> Self::Snapshot { + state.clone() + } +} +``` + +`capture_cluster_snapshot` queries each source's `/state` endpoint and records per-node errors as `OpenRaftSourceFailure` values instead of failing the cycle. A node restarting therefore appears as a named failure inside the snapshot. The snapshot type provides `agreed_leader(different_from)`, `all_voters_match(...)`, `all_kv_match(...)`, and `summary()` for timeout messages. + +Two source providers accompany it: + +```rust,ignore +// Fixed: scenario runs, sources from the run's node clients. +pub fn openraft_cluster_source_provider( + _deployment: &::Deployment, + node_clients: NodeClients, +) -> Result, DynError> { + Ok(Box::new(StaticSourceProvider::new(named_sources(node_clients.snapshot())))) +} +``` + +and `OpenRaftManualClusterSourceProvider`, a dynamic provider that re-resolves clients from a `ManualCluster` on every cycle so observation follows manual restarts. The scenario builder wires the fixed one in via `with_observer(OpenRaftClusterObserver, openraft_cluster_source_provider, OpenRaftClusterObserver::config())`. + +The [failover scenario](chaos.md) waits on this observed state: + +```rust,ignore +let observer = ctx.require_extension::>()?; +let leader = wait_for_observed_leader(&observer, timeout, None).await?; +``` + +> **External example:** logos-blockchain's `BlockFeed` is an adopter-side analog of this pattern: an observer in its own repository materializes block records, per-node head snapshots, and transaction statistics, using the same `Observer`/`ObservationHandle` mechanism. + +--- + +## See Also + +- [Runtime Extensions](runtime-extensions.md) — the mechanism observation plugs into +- [Chaos and Controlled Failure](chaos.md) — observation-driven recovery waits +- [Expectations and Evaluation](expectations.md) — snapshot-based verdicts +- [Telemetry and External Observability](telemetry.md) — the external counterpart diff --git a/book/src/operations-overview.md b/book/src/operations-overview.md deleted file mode 100644 index 5df7156..0000000 --- a/book/src/operations-overview.md +++ /dev/null @@ -1,79 +0,0 @@ -# Operations & Deployment Overview - -Operational readiness focuses on prerequisites, environment fit, and clear signals that ensure your test scenarios run reliably across different deployment targets. - -## Core Principles - -- **Prerequisites First**: Ensure all required files, binaries, and assets are in place before attempting to run scenarios -- **Environment Fit**: Choose the right deployment target (host, compose, k8s) based on your isolation, reproducibility, and resource needs -- **Clear Signals**: Verify runners report node readiness before starting workloads to avoid false negatives -- **Failure Triage**: Map failures to specific causes—missing prerequisites, platform issues, or unmet expectations - -## Key Operational Concerns - -**Prerequisites:** -- `versions.env` file at repository root (required by helper scripts) -- Node binaries (`logos-blockchain-node`) available or built on demand -- Platform requirements met (Docker for compose, cluster access for k8s) -- Circuit assets for proof generation - -**Artifacts:** -- Circuit parameters required by the node binary -- Docker images for compose/k8s deployments -- Binary bundles for reproducible builds - -**Environment Configuration:** -- Logging configured via `LOGOS_BLOCKCHAIN_LOG_*` variables -- Observability endpoints (Prometheus, Grafana) optional but useful - -**Readiness & Health:** -- Runners verify node readiness before starting workloads -- Health checks prevent premature workload execution -- Consensus liveness expectations validate basic operation - -## Runner-Agnostic Design - -The framework is intentionally **runner-agnostic**: the same scenario plan runs across all deployment targets. Understanding which operational concerns apply to each runner helps you choose the right fit. - -| Concern | Host | Compose | Kubernetes | -|---------|------|---------|------------| -| **Topology** | Full support | Full support | Full support | -| **Workloads** | All workloads | All workloads | All workloads | -| **Expectations** | All expectations | All expectations | All expectations | -| **Chaos / Node Control** | Not supported | Supported | Not yet | -| **Metrics / Observability** | Manual setup | External stack | Cluster-wide | -| **Log Collection** | Temp files | Container logs | Pod logs | -| **Isolation** | Process-level | Container | Pod + namespace | -| **Setup Time** | < 1 min | 2-5 min | 5-10 min | -| **CI Recommended?** | Smoke tests | Primary | Large-scale only | - -**Key insight:** Operational concerns (prerequisites, environment variables) are largely **consistent** across runners, while deployment-specific concerns (isolation, chaos support) vary by backend. - -## Operational Workflow - -```mermaid -flowchart LR - Setup[Prerequisites & Setup] --> Run[Run Scenarios] - Run --> Monitor[Monitor & Observe] - Monitor --> Debug{Success?} - Debug -->|No| Triage[Failure Triage] - Triage --> Setup - Debug -->|Yes| Done[Complete] -``` - -1. **Setup**: Verify prerequisites, configure environment, prepare assets -2. **Run**: Execute scenarios using appropriate runner (host/compose/k8s) -3. **Monitor**: Collect logs, metrics, and observability signals -4. **Triage**: When failures occur, map to root causes and fix prerequisites - -## Documentation Structure - -This Operations & Deployment section covers: - -- [Prerequisites & Setup](prerequisites.md) — Required files, binaries, and environment setup -- [Running Examples](running-examples.md) — How to run scenarios across different runners -- [CI Integration](ci-integration.md) — Automating tests in continuous integration pipelines -- [Environment Variables](environment-variables.md) — Complete reference of configuration variables -- [Logging & Observability](logging-observability.md) — Log collection, metrics, and debugging - -**Philosophy:** Treat operational hygiene—assets present, prerequisites satisfied, observability reachable—as the first step to reliable scenario outcomes. diff --git a/book/src/part-i.md b/book/src/part-i.md index 74e4ac6..a9ea51e 100644 --- a/book/src/part-i.md +++ b/book/src/part-i.md @@ -1,4 +1,10 @@ -# Part I — Foundations +# Part I — Mental Model -Conceptual chapters that establish the mental model for the framework and how -it approaches multi-node testing. +This part defines the main framework abstractions. + +This part explains what the framework's core types mean, how a scenario executes from build to teardown, and how to pick the right entry pattern for a given test before writing any code. + +- [Application, AppDeployment, and Environments](application-model.md) — the three roles a "thing under test" can play +- [Scenario Model and Lifecycle](scenario-model.md) — what a scenario is and every phase it passes through +- [Choosing an Entry Pattern](entry-patterns.md) — uniform cluster, composed stack, attached nodes, or manual control +- [Ownership and Design Boundaries](boundaries.md) — what the framework owns versus what your application repository owns diff --git a/book/src/part-ii.md b/book/src/part-ii.md index 36eb205..af43993 100644 --- a/book/src/part-ii.md +++ b/book/src/part-ii.md @@ -1,4 +1,13 @@ -# Part II — User Guide +# Part II — Composing Applications -Practical guidance for shaping scenarios, combining workloads and expectations, -and running them across different environments. +The app layer deploys heterogeneous systems as one unit and exposes typed handles to workloads. + +Use this entry pattern when the system under test is not a single uniform cluster. A root `AppDeployment` deploys children such as processes, uniform child clusters, and in-process services. It exposes their handles, while the scenario runtime schedules test behavior and cleanup. + +- [AppHost and with_app](app-host.md) — hosting a composed app inside a scenario +- [AppDeployment and DeployContext](app-deployment.md) — the deployment contract and its context +- [Handle Ownership and Teardown](handles-teardown.md) — typed access, managed lifetime, and reverse cleanup +- [One Binary: LocalProcessApp](local-process-app.md) — the smallest building block +- [Uniform Child Clusters: LocalAppCluster](local-app-cluster.md) — a managed cluster as one component +- [Composing Heterogeneous Stacks](composing-stacks.md) — the root-app pattern, end to end +- [Backend Scope](app-backend-scope.md) — what the app layer supports today diff --git a/book/src/part-iii.md b/book/src/part-iii.md index 107c890..b287079 100644 --- a/book/src/part-iii.md +++ b/book/src/part-iii.md @@ -1,4 +1,14 @@ -# Part III — Developer Reference +# Part III — Scenario Runtime -Deep dives for contributors who extend the framework, evolve its abstractions, -or maintain the crate set. +These chapters describe what happens while a scenario runs: traffic, verification, capabilities, and observation. + +The same runtime serves all three declarative entry patterns: uniform clusters, composed app stacks, and attached external nodes. + +- [Workloads and Concurrency](workloads.md) — driving the system under test +- [Expectations and Evaluation](expectations.md) — verifying outcomes +- [The Verb Layer](verb-layer.md) — concise domain actions over the explicit builder API +- [Scenario Capabilities](capabilities.md) — capability-gated features such as node control +- [Chaos and Controlled Failure](chaos.md) — restarts and failover from workloads +- [Runtime Extensions](runtime-extensions.md) — typed scenario-lifetime services +- [Continuous Observation](observation.md) — snapshots, history, and event streams for test logic +- [Telemetry and External Observability](telemetry.md) — metrics, logs, and tracing diff --git a/book/src/part-iv.md b/book/src/part-iv.md index 2b61617..12ff6ff 100644 --- a/book/src/part-iv.md +++ b/book/src/part-iv.md @@ -1,44 +1,13 @@ -# Part IV — Operations & Deployment +# Part IV — Uniform Clusters and Configuration -This section covers operational aspects of running the testing framework: prerequisites, deployment configuration, continuous integration, and observability. +This part shows how to put your own node behind the framework and control how clusters are configured and driven. -## What You'll Learn - -- **Prerequisites & Setup**: Required files, binaries, circuit assets, and environment configuration -- **Running Examples**: How to execute scenarios across host, compose, and k8s runners -- **CI Integration**: Automating tests in continuous integration pipelines with caching and matrix testing -- **Environment Variables**: Complete reference of all configuration variables -- **Logging & Observability**: Log collection strategies, metrics integration, and debugging techniques - -## Who This Section Is For - -- **Operators** setting up the framework for the first time -- **DevOps Engineers** integrating tests into CI/CD pipelines -- **Developers** debugging test failures or performance issues -- **Platform Engineers** deploying across different environments (local, Docker, Kubernetes) - -## Navigation - -This section is organized for progressive depth: - -1. Start with [Operations Overview](operations-overview.md) for the big picture -2. Follow [Prerequisites & Setup](prerequisites.md) to prepare your environment -3. Use [Running Examples](running-examples.md) to execute your first scenarios -4. Integrate with [CI Integration](ci-integration.md) for automated testing -5. Reference [Environment Variables](environment-variables.md) for complete configuration options -6. Debug with [Logging & Observability](logging-observability.md) when issues arise - -## Key Principles - -**Operational Hygiene:** Assets present, prerequisites satisfied, observability reachable - -**Environment Fit:** Choose the right deployment target based on isolation, reproducibility, and resource needs - -**Clear Signals:** Verify runners report node readiness before starting workloads - -**Failure Triage:** Map failures to specific causes—missing prerequisites, platform issues, or unmet expectations - ---- - -Ready to get started? Begin with [Operations Overview](operations-overview.md) → +It covers the uniform-cluster entry pattern in depth: implementing `Application`, describing topologies, generating per-node configuration, and driving nodes imperatively outside the scenario runtime. +- [Implementing Application](implementing-application.md) — the environment contract for your node +- [Topology and Deployment Plans](topology.md) — describing cluster shape +- [Ports, Peers, Node Config, and Readiness](node-config.md) — per-node configuration mechanics +- [Static Artifacts and cfgsync](cfgsync.md) — typed app config to per-node artifacts to backend rendering +- [Seeds and Reproducibility](seeds.md) — deterministic deployments +- [ManualCluster: Imperative Node Control](manual-cluster.md) — direct node lifecycle control +- [Persistence, Snapshots, and Recovery Testing](persistence.md) — state across restarts diff --git a/book/src/part-v.md b/book/src/part-v.md index 2ff5245..292e561 100644 --- a/book/src/part-v.md +++ b/book/src/part-v.md @@ -1,28 +1,14 @@ -# Part V — Appendix +# Part V — Deployers and Sources -Quick reference materials, troubleshooting guides, and supplementary information. +This part covers where scenarios run and where their nodes come from. -## Contents - -- **Builder API Quick Reference**: Cheat sheet for DSL methods -- **Troubleshooting Scenarios**: Common issues and their solutions, including "What Failure Looks Like" with realistic examples -- **FAQ**: Frequently asked questions -- **Glossary**: Terminology reference - -## When to Use This Section - -- **Quick lookups**: Find DSL method signatures without reading full guides -- **Debugging failures**: Match symptoms to known issues and fixes -- **Clarifying concepts**: Look up unfamiliar terms in the glossary -- **Common questions**: Check FAQ before asking for help - -This section complements the main documentation with practical reference materials that you'll return to frequently during development and operations. - ---- - -Jump to: -- [Builder API Quick Reference](dsl-cheat-sheet.md) -- [Troubleshooting Scenarios](troubleshooting.md) -- [FAQ](faq.md) -- [Glossary](glossary.md) +Uniform scenarios deploy to local processes, Docker Compose, or Kubernetes; scenarios can also attach to clusters you already operate. The app layer uses the same cluster request and handle model, with local as its implemented provisioning backend today. +- [Capability Matrix](capability-matrix.md) — feature support per deployer +- [Local Deployer](deployer-local.md) — processes on your machine +- [Compose Deployer](deployer-compose.md) — containerized clusters +- [Kubernetes Deployer](deployer-k8s.md) — Helm releases in isolated namespaces +- [Shared Cluster Provisioning](cluster-provisioning.md) — one request and handle model across cluster sources +- [Existing and External Clusters](external-clusters.md) — attaching to live systems +- [Binary Providers](binary-providers.md) — resolving node binaries: path, env, build, download +- [Readiness, Retry, and Artifact Preservation](deployment-policies.md) — deployment policy diff --git a/book/src/part-vi.md b/book/src/part-vi.md new file mode 100644 index 0000000..97269a4 --- /dev/null +++ b/book/src/part-vi.md @@ -0,0 +1,7 @@ +# Part VI — Extending and Reference + +This part documents the extension points, the crate map, and the boundary rules that keep the framework application-agnostic. + +- [Public Extension Points](extension-points.md) — the traits you implement to plug in +- [Crate and API Map](crate-map.md) — which concept lives in which crate +- [Framework vs Application Boundaries](tf-boundaries.md) — what belongs where, and how it is enforced diff --git a/book/src/part-vii.md b/book/src/part-vii.md new file mode 100644 index 0000000..2578545 --- /dev/null +++ b/book/src/part-vii.md @@ -0,0 +1,10 @@ +# Part VII — Operations + +These chapters cover running, integrating, and debugging the framework day to day. + +- [Running the Examples](running-examples.md) — every runnable binary and what it exercises +- [Continuous Integration](ci.md) — how this repository tests itself, and patterns for yours +- [Diagnostics and Retained Artifacts](diagnostics.md) — logs, working directories, post-mortems +- [Environment Variables](environment-variables.md) — the complete, audited reference +- [Troubleshooting](troubleshooting.md) — common failures and their causes +- [Glossary](glossary.md) — terms used throughout the book diff --git a/book/src/persistence.md b/book/src/persistence.md new file mode 100644 index 0000000..9e45bdb --- /dev/null +++ b/book/src/persistence.md @@ -0,0 +1,111 @@ +# Persistence, Snapshots, and Recovery Testing + +This chapter explains how node working directories behave, what `persist_dir` and `snapshot_dir` actually do, and how to build stop/restore recovery tests on top of them. + +--- + +## The Working Directory + +Every locally spawned node runs inside a framework-created directory (`testing-framework/deployers/local/src/process.rs`): + +- Launch files (the rendered config) are written into it before spawn, and the process starts with it as its current directory. A node that writes relative paths (a database under `./db`, logs under `./logs`) keeps all its state there. +- By default the directory is a random-named temp directory created **in the current working directory** of the test process, and it is deleted when the node is dropped. +- Deletion is skipped when the owning thread is panicking, when the node was spawned with `keep_tempdir`, when `TF_KEEP_LOGS=1` is set, or when the deployment policy sets `cleanup_policy.preserve_artifacts` (see [Readiness, Retry, and Artifact Preservation](deployment-policies.md)). + +`restart` (used by `ManualCluster::restart_node` and `LocalProcessHandle::restart`) kills the child and respawns it **in the same directory** with the same launch spec. Launch files are rewritten; everything else is untouched, so state in the working directory survives the restart. + +--- + +## persist_dir: a Predictable Location + +`persist_dir` does **not** reuse the given path as-is. Verified semantics from `create_tempdir`: + +- The working directory is created as `_` **inside the parent** of the path you pass. `with_persist_dir("/tmp/kv-run/node-0")` yields a working directory like `/tmp/kv-run/node-0_a1B2c3/`. The parent is created if missing. +- Nothing is copied into it; it starts empty apart from launch files. +- It is still a managed temp directory: deleted on drop unless one of the retention switches above applies. + +Use `persist_dir` when a test (or a human) must *find* the node's state. Pair it with `TF_KEEP_LOGS=1` or `preserve_artifacts` to keep the directory after the run, then feed it back in as a snapshot later. + +--- + +## snapshot_dir: Seeding State at Start + +`snapshot_dir` copies saved state into the fresh working directory before the process spawns. This is the restore half of a recovery test: a fresh node starts from state captured in an earlier run instead of an empty directory. Verified semantics from `copy_snapshot_dir`: + +- The directory you pass is copied **as a subdirectory of the working directory, named after its final path component**, with overwrite enabled. `with_snapshot_dir("/snapshots/run1/db")` produces `/db/...`. +- Consequently, the snapshot source's basename must match the relative path the node expects. If your node reads `./db`, snapshot a directory literally named `db`. +- The copy happens once, at spawn. Restarts do not re-apply it. +- A failed copy fails the spawn (`ProcessSpawnError::Snapshot`). + +The framework copies the supplied directory byte-for-byte without interpreting its contents. The caller determines what constitutes a consistent snapshot, including which directories to copy, whether the node must be stopped first, and whether its on-disk state is crash-consistent. + +--- + +## Where the Options Live + +**Per dynamic node**: `StartNodeOptions` (full table in [ManualCluster](manual-cluster.md)): + +```rust,ignore +let node = cluster.start_node_with( + "restored", + StartNodeOptions::default() + .with_snapshot_dir(PathBuf::from("/snapshots/run1/db")) + .with_start_timeout(Duration::from_secs(90)), +).await?; +``` + +Note `restart_node_with` rejects `persist_dir`/`snapshot_dir` overrides, since restarts keep the existing directory. Start a new node to restore from a snapshot. + +**Per initial node**: `LocalDeployerEnv::initial_persist_dir(topology, node_name, index)` and `initial_snapshot_dir(...)` (default `None`). Override these to mount state under the whole initial cluster, e.g. restore every node of a 3-node cluster from a saved dataset before the scenario begins. + +**Per composed process**: `LocalProcessApp` in the app layer (`testing-framework/app/src/process.rs`) exposes the same three switches for one-binary apps: + +| Builder | Effect | +|---|---| +| `.with_persist_dir(path)` | Same placement rule as above | +| `.with_snapshot_dir(path)` | Same copy-as-subdirectory rule as above | +| `.keep_tempdir(true)` | Retain the working directory on teardown | + +Its `LocalProcessHandle` offers `working_dir()`, `restart()`, `stop()`, `is_running()`, `pid()`, and `keep_tempdir()`; the process stops when the last handle clone drops (see [One Binary: LocalProcessApp](local-process-app.md)). + +--- + +## Recovery-Testing Patterns + +**Restart in place.** State persists because the node reuses its working directory. + +```rust,ignore +write_data(&client).await?; +cluster.restart_node("node-1").await?; +cluster.wait_node_ready("node-1").await?; +assert_data_recovered(&cluster.node_client("node-1").unwrap()).await?; +``` + +**Stop, snapshot, restore.** Full recovery drill via [ManualCluster](manual-cluster.md): + +```rust,ignore +// 1. Run a node whose working dir you can locate. +let node = cluster.start_node_with( + "primary", + StartNodeOptions::default().with_persist_dir(PathBuf::from("/tmp/kv-run/primary")), +).await?; +write_data(&node.client).await?; + +// 2. Stop it, then copy its state out yourself (caller-owned step): +cluster.stop_node("node-primary").await?; +// e.g. cp -r /tmp/kv-run/primary_*/db /snapshots/case1/db + +// 3. Start a fresh node seeded from the snapshot. +let restored = cluster.start_node_with( + "restored", + StartNodeOptions::default().with_snapshot_dir(PathBuf::from("/snapshots/case1/db")), +).await?; +cluster.wait_node_ready("node-restored").await?; +assert_data_recovered(&restored.client).await?; +``` + +Step 2 is caller-owned because the framework neither snapshots on stop nor knows which files constitute application state. Set `TF_KEEP_LOGS=1` so stopped nodes' directories remain available for copying. + +**Config continuity.** The first node started with a `snapshot_dir` has its generated config recorded as a template, and that template is passed to the config-build hooks of later dynamic starts (the `template_config` parameter). An env that honors it can keep restored nodes consistent with the configs their state was produced under. + +**Cross-run state.** Combine `persist_dir` (findable location) with retention (`TF_KEEP_LOGS` / `preserve_artifacts`), archive the directory after run A, and hand it to run B via `snapshot_dir` or the `initial_snapshot_dir` hook. Upgrade tests, long-lived-ledger tests, and crash-recovery matrices all reduce to this loop. diff --git a/book/src/prerequisites.md b/book/src/prerequisites.md deleted file mode 100644 index 83c3415..0000000 --- a/book/src/prerequisites.md +++ /dev/null @@ -1,244 +0,0 @@ -# Prerequisites & Setup - -This page covers everything you need before running your first scenario. - -## Required Files - -### `versions.env` (Required) - -All helper scripts require a `versions.env` file at the repository root: - -```bash -VERSION=v0.3.1 -LOGOS_BLOCKCHAIN_NODE_REV=abc123def456789 -LOGOS_BLOCKCHAIN_BUNDLE_VERSION=v1 -``` - -**What it defines:** -- `VERSION` — Circuit assets release tag -- `LOGOS_BLOCKCHAIN_NODE_REV` — Git revision of logos-blockchain-node to build/fetch -- `LOGOS_BLOCKCHAIN_BUNDLE_VERSION` — Bundle schema version - -**Where it's used:** -- `scripts/run/run-examples.sh` -- `scripts/build/build-bundle.sh` -- `scripts/setup/setup-logos-blockchain-circuits.sh` -- CI workflows - -**Error if missing:** -```text -ERROR: versions.env not found at repository root -This file is required and should define: - VERSION= - LOGOS_BLOCKCHAIN_NODE_REV= - LOGOS_BLOCKCHAIN_BUNDLE_VERSION= -``` - -**Fix:** Ensure you're in the repository root. The file should already exist in the checked-out repo. - -## Node Binaries - -Scenarios need compiled `logos-blockchain-node` binaries. - -### Option 1: Use Helper Scripts (Recommended) - -```bash -scripts/run/run-examples.sh -t 60 -n 3 host -``` - -This automatically: -- Clones/updates logos-blockchain-node checkout -- Builds required binaries -- Sets `LOGOS_BLOCKCHAIN_NODE_BIN` - -### Option 2: Manual Build - -If you have a sibling `logos-blockchain-node` checkout: - -```bash -cd ../logos-blockchain-node -cargo build --release --bin logos-blockchain-node - -# Set environment variables -export LOGOS_BLOCKCHAIN_NODE_BIN=$PWD/target/release/logos-blockchain-node - -# Return to testing framework -cd ../nomos-testing -``` - -### Option 3: Prebuilt Bundles (CI) - -CI workflows use prebuilt artifacts: - -```yaml -- name: Download nomos binaries - uses: actions/download-artifact@v3 - with: - name: nomos-binaries-linux - path: .tmp/ - -- name: Extract bundle - run: | - tar -xzf .tmp/nomos-binaries-linux-*.tar.gz -C .tmp/ - export LOGOS_BLOCKCHAIN_NODE_BIN=$PWD/.tmp/logos-blockchain-node -``` - -## Circuit Assets - -Nodes require circuit assets for proof generation. The framework expects a -directory containing the circuits, not a single file. - -### Asset Location - -**Default path:** `~/.logos-blockchain-circuits` - -**Container path (compose/k8s):** `/opt/circuits` (set during image build) - -### Getting Assets - -**Option 1: Use helper script** (recommended): - -```bash -scripts/setup/setup-logos-blockchain-circuits.sh v0.3.1 ~/.logos-blockchain-circuits -``` - -**Option 2: Let `run-examples.sh` handle it**: - -```bash -scripts/run/run-examples.sh -t 60 -n 3 host -``` - -### Override Path - -Set `LOGOS_BLOCKCHAIN_CIRCUITS` to use a custom location: - -```bash -LOGOS_BLOCKCHAIN_CIRCUITS=/custom/path/to/circuits \ -cargo run -p runner-examples --bin local_runner -``` - -### When Are Assets Needed? - -| Runner | When Required | -|--------|---------------| -| **Host (local)** | Always | -| **Compose** | During image build (baked into image) | -| **K8s** | During image build | - -**Error without assets:** - -```text -Error: circuits directory not found (LOGOS_BLOCKCHAIN_CIRCUITS) -``` - -## Platform Requirements - -### Host Runner (Local Processes) - -**Requires:** -- Rust nightly toolchain -- Node binaries built -- Circuit assets for proof generation -- Available ports (18080+, 3100+, etc.) - -**No Docker required.** - -**Best for:** -- Quick iteration -- Development -- Smoke tests - -### Compose Runner (Docker Compose) - -**Requires:** -- Docker daemon running -- Docker image built: `logos-blockchain-testing:local` -- Circuit assets baked into image -- Docker Desktop (macOS) or Docker Engine (Linux) - -**Platform notes (macOS / Apple silicon):** -- Prefer `LOGOS_BLOCKCHAIN_BUNDLE_DOCKER_PLATFORM=linux/arm64` for native performance -- Use `linux/amd64` only if targeting amd64 environments (slower via emulation) - -**Best for:** -- Reproducible environments -- CI testing -- Chaos workloads (node control support) - -### K8s Runner (Kubernetes) - -**Requires:** -- Kubernetes cluster (Docker Desktop K8s, minikube, kind, or remote) -- `kubectl` configured -- Docker image built and loaded/pushed -- Circuit assets baked into image - -**Local cluster setup:** - -```bash -# Docker Desktop: Enable Kubernetes in settings - -# OR: Use kind -kind create cluster -kind load docker-image logos-blockchain-testing:local - -# OR: Use minikube -minikube start -minikube image load logos-blockchain-testing:local -``` - -**Remote cluster:** Push image to registry and set `LOGOS_BLOCKCHAIN_TESTNET_IMAGE`. - -**Best for:** -- Production-like testing -- Resource isolation -- Large topologies - -## Quick Setup Check - -Run this checklist before your first scenario: - -```bash -# 1. Verify versions.env exists -cat versions.env - -# 2. Check circuit assets -ls -lh "${HOME}/.logos-blockchain-circuits" - -# 3. For compose/k8s: verify Docker is running -docker ps - -# 4. For compose/k8s: verify image exists -docker images | grep logos-blockchain-testing - -# 5. For host runner: verify node binaries (if not using scripts) -$LOGOS_BLOCKCHAIN_NODE_BIN --version -``` - -## Recommended: Use Helper Scripts - -The easiest path is to let the helper scripts handle everything: - -```bash -# Host runner -scripts/run/run-examples.sh -t 60 -n 3 host - -# Compose runner -scripts/run/run-examples.sh -t 60 -n 3 compose - -# K8s runner -scripts/run/run-examples.sh -t 60 -n 3 k8s -``` - -These scripts: -- Verify `versions.env` exists -- Clone/build logos-blockchain-node if needed -- Fetch circuit assets if missing -- Build Docker images (compose/k8s) -- Load images into cluster (k8s) -- Run the scenario with proper environment - -**Next Steps:** -- [Running Examples](running-examples.md) — Learn how to run scenarios -- [Environment Variables](environment-variables.md) — Full variable reference -- [Troubleshooting](troubleshooting.md) — Common issues and fixes diff --git a/book/src/project-context-primer.md b/book/src/project-context-primer.md deleted file mode 100644 index 8cce2ad..0000000 --- a/book/src/project-context-primer.md +++ /dev/null @@ -1,156 +0,0 @@ -# Logos Testing Framework - -**Declarative, multi-node blockchain testing for the Logos network** - -The Logos Testing Framework enables you to test consensus and transaction workloads across local processes, Docker Compose, and Kubernetes deployments—all with a unified scenario API. - -[**Get Started**](quickstart.md) - ---- - -## Core Concept - -**Everything in this framework is a Scenario.** - -A Scenario is a controlled experiment over time, composed of: -- **Topology** — The cluster shape (nodes, network layout) -- **Workloads** — Traffic and conditions that exercise the system (transactions, chaos) -- **Expectations** — Success criteria verified after execution (liveness, inclusion, recovery) -- **Duration** — The time window for the experiment - -This single abstraction makes tests declarative, portable, and composable. - ---- - -## How It Works - -```mermaid -flowchart LR - Build[Define Scenario] --> Deploy[Deploy Topology] - Deploy --> Execute[Run Workloads] - Execute --> Evaluate[Check Expectations] - - style Build fill:#e1f5ff - style Deploy fill:#fff4e1 - style Execute fill:#ffe1f5 - style Evaluate fill:#e1ffe1 -``` - -1. **Define Scenario** — Describe your test: topology, workloads, and success criteria -2. **Deploy Topology** — Launch nodes using host, compose, or k8s runners -3. **Run Workloads** — Drive transactions and chaos operations -4. **Check Expectations** — Verify consensus liveness, inclusion, and system health - ---- - -## Key Features - -**Declarative API** -- Express scenarios as topology + workloads + expectations -- Reuse the same test definition across different deployment targets -- Compose complex tests from modular components - -**Multiple Deployment Modes** -- **Host Runner**: Local processes for fast iteration -- **Compose Runner**: Containerized environments with node control -- **Kubernetes Runner**: Production-like cluster testing - -**Built-in Workloads** -- Transaction submission with configurable rates -- Chaos testing with controlled node restarts - -**Comprehensive Observability** -- Real-time block feed for monitoring consensus progress -- Prometheus/Grafana integration for metrics -- Per-node log collection and debugging - ---- - -## Quick Example - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_core::scenario::Deployer as _; -use testing_framework_runner_local::LocalDeployer; -use testing_framework_workflows::ScenarioBuilderExt; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let mut scenario = ScenarioBuilder::topology_with(|t| { - t.network_star() - .nodes(3) - }) - .transactions_with(|tx| tx.rate(10).users(5)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(60)) - .build(); - - let deployer = LocalDeployer::default(); - let runner = deployer.deploy(&scenario).await?; - runner.run(&mut scenario).await?; - - Ok(()) -} -``` - -[View complete examples](examples.md) - ---- - -## Choose Your Path - -### New to the Framework? - -Start with the **[Quickstart Guide](quickstart.md)** for a hands-on introduction that gets you running tests in minutes. - -### Ready to Write Tests? - -Explore the **[User Guide](part-ii.md)** to learn about authoring scenarios, workloads, expectations, and deployment strategies. - -### Setting Up CI/CD? - -Jump to **[Operations & Deployment](part-v.md)** for prerequisites, environment configuration, and continuous integration patterns. - -### Extending the Framework? - -Check the **[Developer Reference](part-iii.md)** to implement custom workloads, expectations, and runners. - ---- - -## Project Context - -**Logos** is a modular blockchain protocol composed of nodes that participate in consensus and produce blocks. - -Meaningful testing must be performed in multi-node environments that include real networking and timing behavior. - -The Logos Testing Framework provides the infrastructure to orchestrate these multi-node scenarios reliably across development, CI, and production-like environments. - -**Learn more about the protocol:** [Logos Project Documentation](https://nomos-tech.notion.site/project) - ---- - -## Documentation Structure - -| Section | Description | -|---------|-------------| -| **[Foundations](part-i.md)** | Architecture, philosophy, and design principles | -| **[User Guide](part-ii.md)** | Writing and running scenarios, workloads, and expectations | -| **[Developer Reference](part-iii.md)** | Extending the framework with custom components | -| **[Operations & Deployment](part-iv.md)** | Setup, CI integration, and environment configuration | -| **[Appendix](part-v.md)** | Quick reference, troubleshooting, FAQ, and glossary | - ---- - -## Quick Links - -- **[What You Will Learn](what-you-will-learn.md)** — Overview of book contents and learning path -- **[Quickstart](quickstart.md)** — Get up and running in 10 minutes -- **[Examples](examples.md)** — Concrete scenario patterns -- **[Troubleshooting](troubleshooting.md)** — Common issues and solutions -- **[Environment Variables](environment-variables.md)** — Complete configuration reference - ---- - -**Ready to start?** Head to the **[Quickstart](quickstart.md)** diff --git a/book/src/quickstart.md b/book/src/quickstart.md index 991fd89..7360221 100644 --- a/book/src/quickstart.md +++ b/book/src/quickstart.md @@ -1,291 +1,91 @@ # Quickstart -Get a working example running quickly. - -## From Scratch (Complete Setup) - -If you're starting from zero, here's everything you need: - -```bash -# 1. Install Rust nightly -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -rustup default nightly - -# 2. Clone the repository -git clone https://github.com/logos-blockchain/logos-blockchain-testing.git -cd logos-blockchain-testing - -# 3. Run your first scenario (downloads dependencies automatically) -scripts/run/run-examples.sh -t 60 -n 1 host -``` - -**First run takes 5-10 minutes** (downloads ~120MB circuit assets, builds binaries). - -**Windows users:** Use WSL2 (Windows Subsystem for Linux). Native Windows is not supported. +Run a complete multi-node test in one command. --- ## Prerequisites -If you already have the repository cloned: - -- Rust toolchain (nightly) +- Rust toolchain (the workspace pins its version via `rust-toolchain.toml`) - Unix-like system (tested on Linux and macOS) -- For Docker Compose examples: Docker daemon running -- For Docker Desktop on Apple silicon (compose/k8s): set `LOGOS_BLOCKCHAIN_BUNDLE_DOCKER_PLATFORM=linux/arm64` to avoid slow/fragile amd64 emulation builds -- **`versions.env` file** at repository root (defines VERSION, LOGOS_BLOCKCHAIN_NODE_REV, LOGOS_BLOCKCHAIN_BUNDLE_VERSION) +- For Compose examples: a running Docker daemon +- For Kubernetes examples: a reachable cluster context -**Note:** `logos-blockchain-node` binaries are built automatically on demand or can be provided via prebuilt bundles. +No other setup. Example node binaries are resolved automatically; the kvstore example builds its node with Cargo on first run if no prebuilt binary is available. -**Important:** The `versions.env` file is required by helper scripts. If missing, the scripts will fail with an error. The file should already exist in the repository root. +--- ## Your First Test -The framework ships with runnable example binaries in `examples/src/bin/`. - -**Recommended:** Use the convenience script: - ```bash -# From the logos-blockchain-testing directory -scripts/run/run-examples.sh -t 60 -n 1 host +git clone +cd +cargo run -p kvstore-examples --bin kvstore_app_host_convergence ``` -This handles circuit setup, binary building, and runs a complete scenario: 1 node, transaction workload (5 tx/block), 60s duration. +**First run takes a few minutes** (builds the framework and the `kvstore-node` binary). -**Alternative:** Direct cargo run (requires manual setup): +**What happens:** -```bash -# Requires circuits in place and LOGOS_BLOCKCHAIN_NODE_BIN set -cargo run -p runner-examples --bin local_runner -``` - -**Core API Pattern** (simplified example): - -```rust,ignore -use std::time::Duration; - -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_local::LocalDeployer; -use testing_framework_workflows::ScenarioBuilderExt; - -pub async fn run_local_demo() -> Result<()> { - // Define the scenario (1 node, tx workload) - let mut plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(1)) - .wallets(1_000) - .transactions_with(|txs| { - txs.rate(5) // 5 transactions per block - .users(500) // use 500 of the seeded wallets - }) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(60)) - .build(); - - // Deploy and run - let deployer = LocalDeployer::default(); - let runner = deployer.deploy(&plan).await?; - let _handle = runner.run(&mut plan).await?; - - Ok(()) -} -``` - -**Note:** The examples are binaries with `#[tokio::main]`, not test functions. If you want to write integration tests, wrap this pattern in `#[tokio::test]` functions in your own test suite. +1. `AppHost::scenario()` builds a scenario around a composed application instead of a managed node topology. +2. `with_app(KvLocalApp::nodes(3))` deploys a three-node kvstore cluster as local processes. +3. The convergence workload writes a value, restarts `node-0`, waits for readiness, and writes again. +4. The runner evaluates the outcome and tears the cluster down. **What you should see:** -- Nodes spawn as local processes -- Consensus starts producing blocks -- Scenario runs for the configured duration -- Node state/logs written under a temporary per-run directory in the current working directory (removed after the run unless `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1`) -- To write per-node log files to a stable location: set `LOGOS_BLOCKCHAIN_LOG_DIR=/path/to/logs` (files will have prefix like `logos-blockchain-node-0*`, may include timestamps) -## What Just Happened? +- Three `kvstore-node` processes spawn with generated configs in per-run temporary directories +- The workload logs a successful write before and after the restart +- The command exits successfully and removes the temporary directories -Let's unpack the code: +--- -### 1. Topology Configuration +## The Code Behind It + +The binary is short enough to read in full at `examples/kvstore/examples/src/bin/app_host_convergence.rs`. Its core is: ```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; +let mut scenario = AppHost::scenario() + .with_app(KvLocalApp::nodes(3)) + .with_run_duration(Duration::from_secs(5)) + .with_workload(KvAppHostConvergence::new(3)) + .build()?; -pub fn step_1_topology() -> testing_framework_core::scenario::Builder<()> { - ScenarioBuilder::topology_with(|t| { - t.network_star() // Star topology: all nodes connect to seed - .nodes(1) // 1 node - }) -} +let deployer = AppHostLocalDeployer::default(); +let runner = deployer.deploy(&scenario).await?; +runner.run(&mut scenario).await?; ``` -This defines **what** your test network looks like. - -### 2. Wallet Seeding +The workload reaches the deployed cluster through a typed handle (`RunContext` is the object every workload receives at run time; see [Part III](part-iii.md)): ```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn step_2_wallets() -> testing_framework_core::scenario::Builder<()> { - ScenarioBuilder::with_node_counts(1).wallets(1_000) // Seed 1,000 funded wallet accounts -} -``` - -Provides funded accounts for transaction submission. - -### 3. Workloads - -```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn step_3_workloads() -> testing_framework_core::scenario::Builder<()> { - ScenarioBuilder::with_node_counts(1) - .wallets(1_000) - .transactions_with(|txs| { - txs.rate(5) // 5 transactions per block - .users(500) // Use 500 of the 1,000 wallets - }) -} -``` - -Generates transaction traffic to stress the inclusion pipeline. - -### 4. Expectation - -```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn step_4_expectation() -> testing_framework_core::scenario::Builder<()> { - ScenarioBuilder::with_node_counts(1).expect_consensus_liveness() // This says what success means: blocks must be produced continuously. -} -``` - -This says **what success means**: blocks must be produced continuously. - -### 5. Run Duration - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::ScenarioBuilder; - -pub fn step_5_run_duration() -> testing_framework_core::scenario::Builder<()> { - ScenarioBuilder::with_node_counts(1).with_run_duration(Duration::from_secs(60)) -} -``` - -Run for 60 seconds (~27 blocks with default 2s slots, 0.9 coefficient). Framework ensures this is at least 2× the consensus slot duration. Adjust consensus timing via `CONSENSUS_SLOT_TIME` and `CONSENSUS_ACTIVE_SLOT_COEFF`. - -### 6. Deploy and Execute - -```rust,ignore -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_local::LocalDeployer; - -pub async fn step_6_deploy_and_execute() -> Result<()> { - let mut plan = ScenarioBuilder::with_node_counts(1).build(); - - let deployer = LocalDeployer::default(); // Use local process deployer - let runner = deployer.deploy(&plan).await?; // Provision infrastructure - let _handle = runner.run(&mut plan).await?; // Execute workloads & expectations +async fn start(&self, ctx: &RunContext) -> Result<(), DynError> { + let cluster = ctx.require_app::>()?; + put_value(&cluster, "before-restart").await?; + cluster.restart_node("node-0").await?; + cluster.wait_node_ready("node-0").await?; + put_value(&cluster, "after-restart").await?; Ok(()) } ``` -**Deployer** provisions the infrastructure. **Runner** orchestrates execution. - -## Adjust the Topology - -**With run-examples.sh** (recommended): +The same pattern can run in `#[tokio::test]` functions. The composition acceptance suite does this: ```bash -# Scale up to 3 nodes, run for 2 minutes -scripts/run/run-examples.sh -t 120 -n 3 host +cargo test -p multi-app-e2e ``` -**With direct cargo run:** +It uses a reusable fixture crate for the stack, workload, and expectation, then drives them from ordinary integration tests. -```bash -# Uses LOGOS_BLOCKCHAIN_DEMO_* env vars (or legacy *_DEMO_* vars) -LOGOS_BLOCKCHAIN_DEMO_NODES=3 \ -LOGOS_BLOCKCHAIN_DEMO_RUN_SECS=120 \ -cargo run -p runner-examples --bin local_runner -``` +--- -## Try Docker Compose +## Where to Go Next -Use the same API with a different deployer for reproducible containerized environment. - -**Recommended:** Use the convenience script (handles everything): - -```bash -scripts/run/run-examples.sh -t 60 -n 1 compose -``` - -This automatically: -- Fetches circuit assets (to `~/.logos-blockchain-circuits` by default) -- Builds/uses prebuilt binaries (via `LOGOS_BLOCKCHAIN_BINARIES_TAR` if available) -- Builds the Docker image -- Runs the compose scenario - -**Alternative:** Direct cargo run with manual setup: - -```bash -# Option 1: Use prebuilt bundle (recommended for compose/k8s) -scripts/build/build-bundle.sh --platform linux # Creates .tmp/nomos-binaries-linux-v0.3.1.tar.gz -export LOGOS_BLOCKCHAIN_BINARIES_TAR=.tmp/nomos-binaries-linux-v0.3.1.tar.gz - -# Option 2: Manual circuit/image setup (rebuilds during image build) -scripts/setup/setup-logos-blockchain-circuits.sh v0.3.1 /tmp/logos-blockchain-circuits -scripts/build/build_test_image.sh - -# Run with Compose -LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \ -cargo run -p runner-examples --bin compose_runner -``` - -**Benefit:** Reproducible containerized environment (Dockerized nodes, repeatable deployments). - -**Optional: Prometheus + Grafana** - -The runner can integrate with external observability endpoints. For a ready-to-run local stack: - -```bash -scripts/setup/setup-observability.sh compose up -eval "$(scripts/setup/setup-observability.sh compose env)" -``` - -Then run your compose scenario as usual (the environment variables enable PromQL querying and node OTLP metrics export). - -**Note:** Compose expects circuits at `/opt/circuits` inside containers (set by the image build). - -**In code:** Just swap the deployer: - -```rust,ignore -use anyhow::Result; -use testing_framework_core::scenario::{Deployer, ScenarioBuilder}; -use testing_framework_runner_compose::ComposeDeployer; - -pub async fn run_with_compose_deployer() -> Result<()> { - // ... same scenario definition ... - let mut plan = ScenarioBuilder::with_node_counts(1).build(); - - let deployer = ComposeDeployer::default(); // Use Docker Compose - let runner = deployer.deploy(&plan).await?; - let _handle = runner.run(&mut plan).await?; - - Ok(()) -} -``` - -## Next Steps - -Now that you have a working test: - -- **Understand the philosophy**: [Testing Philosophy](testing-philosophy.md) -- **Learn the architecture**: [Architecture Overview](architecture-overview.md) -- **See more examples**: [Examples](examples.md) -- **API reference**: [Builder API Quick Reference](dsl-cheat-sheet.md) -- **Debug failures**: [Troubleshooting](troubleshooting.md) +| Goal | Read | +|------|------| +| Understand the abstractions you just used | [Part I — Mental Model](part-i.md) | +| Compose your own application stack | [Part II — Composing Applications](part-ii.md) | +| Write workloads and expectations | [Part III — Scenario Runtime](part-iii.md) | +| Put your own node behind the framework | [Part IV — Uniform Clusters](part-iv.md) | +| Run against Compose, Kubernetes, or a live network | [Part V — Deployers and Sources](part-v.md) | diff --git a/book/src/runners.md b/book/src/runners.md deleted file mode 100644 index 40f72b0..0000000 --- a/book/src/runners.md +++ /dev/null @@ -1,147 +0,0 @@ -# Runners - -Runners turn a scenario plan into a live environment while keeping the plan -unchanged. Choose based on feedback speed, reproducibility, and fidelity. For -environment and operational considerations, see [Operations Overview](operations-overview.md). - -## Host runner (local processes) -- Launches node processes directly on the host (via `LocalDeployer`). -- Binary: `local_runner.rs`, script mode: `host` -- Fastest feedback loop and minimal orchestration overhead. -- Best for development-time iteration and debugging. -- **Can run in CI** for fast smoke tests. -- **Node control:** Not supported (chaos workloads not available) - -**Run with:** `scripts/run/run-examples.sh -t 60 -n 1 host` - -## Docker Compose runner -- Starts nodes in containers to provide a reproducible multi-node stack on a - single machine (via `ComposeDeployer`). -- Binary: `compose_runner.rs`, script mode: `compose` -- Discovers service ports and wires observability for convenient inspection. -- Good balance between fidelity and ease of setup. -- **Recommended for CI pipelines** (isolated environment, reproducible). -- **Node control:** Supported (can restart nodes for chaos testing) - -**Run with:** `scripts/run/run-examples.sh -t 60 -n 1 compose` - -## Kubernetes runner -- Deploys nodes onto a cluster for higher-fidelity, longer-running scenarios (via `K8sDeployer`). -- Binary: `k8s_runner.rs`, script mode: `k8s` -- Suits CI with cluster access or shared test environments where cluster behavior - and scheduling matter. -- **Node control:** Not supported yet (chaos workloads not available) - -**Run with:** `scripts/run/run-examples.sh -t 60 -n 1 k8s` - -### Common expectations -- All runners require at least one node and, for transaction scenarios, - access to seeded wallets. -- Readiness probes gate workload start so traffic begins only after nodes are - reachable. -- Environment flags can relax timeouts or increase tracing when diagnostics are - needed. - -## Runner Comparison - -```mermaid -flowchart TB - subgraph Host["Host Runner (Local)"] - H1["Speed: Fast"] - H2["Isolation: Shared host"] - H3["Setup: Minimal"] - H4["Chaos: Not supported"] - H5["CI: Quick smoke tests"] - end - - subgraph Compose["Compose Runner (Docker)"] - C1["Speed: Medium"] - C2["Isolation: Containerized"] - C3["Setup: Image build required"] - C4["Chaos: Supported"] - C5["CI: Recommended"] - end - - subgraph K8s["K8s Runner (Cluster)"] - K1["Speed: Slower"] - K2["Isolation: Pod-level"] - K3["Setup: Cluster + image"] - K4["Chaos: Not yet supported"] - K5["CI: Large-scale tests"] - end - - Decision{Choose Based On} - Decision -->|Fast iteration| Host - Decision -->|Reproducibility| Compose - Decision -->|Production-like| K8s - - style Host fill:#e1f5ff - style Compose fill:#e1ffe1 - style K8s fill:#ffe1f5 -``` - -## Detailed Feature Matrix - -| Feature | Host | Compose | K8s | -|---------|------|---------|-----| -| **Speed** | Fastest | Medium | Slowest | -| **Setup Time** | < 1 min | 2-5 min | 5-10 min | -| **Isolation** | Process-level | Container | Pod + namespace | -| **Node Control** | No | Yes | Not yet | -| **Observability** | Basic | External stack | Cluster-wide | -| **CI Integration** | Smoke tests | Recommended | Heavy tests | -| **Resource Usage** | Low | Medium | High | -| **Reproducibility** | Environment-dependent | High | Highest | -| **Network Fidelity** | Localhost only | Virtual network | Real cluster | -| **Parallel Runs** | Port conflicts | Isolated | Namespace isolation | - -## Decision Guide - -```mermaid -flowchart TD - Start[Need to run tests?] --> Q1{Local development?} - Q1 -->|Yes| Q2{Testing chaos?} - Q1 -->|No| Q5{Have cluster access?} - - Q2 -->|Yes| UseCompose[Use Compose] - Q2 -->|No| Q3{Need isolation?} - - Q3 -->|Yes| UseCompose - Q3 -->|No| UseHost[Use Host] - - Q5 -->|Yes| Q6{Large topology?} - Q5 -->|No| Q7{CI pipeline?} - - Q6 -->|Yes| UseK8s[Use K8s] - Q6 -->|No| UseCompose - - Q7 -->|Yes| Q8{Docker available?} - Q7 -->|No| UseHost - - Q8 -->|Yes| UseCompose - Q8 -->|No| UseHost - - style UseHost fill:#e1f5ff - style UseCompose fill:#e1ffe1 - style UseK8s fill:#ffe1f5 -``` - -### Quick Recommendations - -**Use Host Runner when:** -- Iterating rapidly during development -- Running quick smoke tests -- Testing on a laptop with limited resources -- Don't need chaos testing - -**Use Compose Runner when:** -- Need reproducible test environments -- Testing chaos scenarios (node restarts) -- Running in CI pipelines -- Want containerized isolation - -**Use K8s Runner when:** -- Testing large-scale topologies (10+ nodes) -- Need production-like environment -- Have cluster access in CI -- Testing cluster-specific behaviors diff --git a/book/src/running-examples.md b/book/src/running-examples.md index 5cab4eb..9461841 100644 --- a/book/src/running-examples.md +++ b/book/src/running-examples.md @@ -1,300 +1,103 @@ -# Running Examples +# Running the Examples -The framework provides three runner modes: **host** (local processes), **compose** (Docker Compose), and **k8s** (Kubernetes). - -## Quick Start (Recommended) - -Use `scripts/run/run-examples.sh` for all modes—it handles all setup automatically: - -```bash -# Host mode (local processes) -scripts/run/run-examples.sh -t 60 -n 3 host - -# Compose mode (Docker Compose) -scripts/run/run-examples.sh -t 60 -n 3 compose - -# K8s mode (Kubernetes) -scripts/run/run-examples.sh -t 60 -n 3 k8s -``` - -**Parameters:** -- `-t 60` — Run duration in seconds -- `-n 3` — Number of nodes -- `host|compose|k8s` — Deployment mode - -This script handles: -- Circuit asset setup -- Binary building/bundling -- Image building (compose/k8s) -- Image loading into cluster (k8s) -- Execution with proper environment - -**Note:** For `k8s` runs against non-local clusters (e.g. EKS), the cluster pulls images from a registry. In that case, build + push your image separately (see `scripts/build/build_test_image.sh`) and set `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` to the pushed reference. - -## Quick Smoke Matrix - -For a small "does everything still run?" matrix across all runners: - -```bash -scripts/run/run-test-matrix.sh -t 120 -n 1 -``` - -This runs host, compose, and k8s modes with various image-build configurations. Useful after making runner/image/script changes. Forwards `--metrics-*` options through to `scripts/run/run-examples.sh`. - -**Common options:** -- `--modes host,compose,k8s` — Restrict which modes run -- `--no-clean` — Skip `scripts/ops/clean.sh` step -- `--no-bundles` — Skip `scripts/build/build-bundle.sh` (reuses existing `.tmp` tarballs) -- `--no-image-build` — Skip the “rebuild image” variants in the matrix (compose/k8s) -- `--allow-nonzero-progress` — Soft-pass expectation failures if logs show non-zero progress (local iteration only) -- `--force-k8s-image-build` — Allow the k8s image-build variant even on non-docker-desktop clusters - -**Environment overrides:** -- `VERSION=v0.3.1` — Circuit version -- `LOGOS_BLOCKCHAIN_NODE_REV=` — logos-blockchain-node git revision -- `LOGOS_BLOCKCHAIN_BINARIES_TAR=path/to/bundle.tar.gz` — Use prebuilt bundle -- `LOGOS_BLOCKCHAIN_SKIP_IMAGE_BUILD=1` — Skip image rebuild inside `run-examples.sh` (compose/k8s) -- `LOGOS_BLOCKCHAIN_BUNDLE_DOCKER_PLATFORM=linux/arm64|linux/amd64` — Docker platform for bundle builds (macOS/Windows) -- `COMPOSE_CIRCUITS_PLATFORM=linux-aarch64|linux-x86_64` — Circuits platform for image builds -- `SLOW_TEST_ENV=true` — Doubles built-in readiness timeouts (useful in CI / constrained laptops) -- `TESTNET_PRINT_ENDPOINTS=1` — Print `TESTNET_ENDPOINTS` / `TESTNET_PPROF` lines during deploy - -## Dev Workflow: Updating logos-blockchain-node Revision - -The repo pins a `logos-blockchain-node` revision in `versions.env` for reproducible builds. To update it or point to a local checkout: - -```bash -# Pin to a new git revision (updates versions.env + Cargo.toml git revs) -scripts/ops/update-nomos-rev.sh --rev - -# Use a local logos-blockchain-node checkout instead (for development) -scripts/ops/update-nomos-rev.sh --path /path/to/logos-blockchain-node - -# If Cargo.toml was marked skip-worktree, clear it -scripts/ops/update-nomos-rev.sh --unskip-worktree -``` - -**Notes:** -- Don't commit absolute `LOGOS_BLOCKCHAIN_NODE_PATH` values; prefer `--rev` for shared history/CI -- After changing rev/path, expect `Cargo.lock` to update on the next `cargo build`/`cargo test` - -## Cleanup Helper - -If you hit Docker build failures, I/O errors, or disk space issues: - -```bash -scripts/ops/clean.sh -``` - -For extra Docker cache cleanup: - -```bash -scripts/ops/clean.sh --docker -``` +This chapter lists every runnable example binary, the exact command to launch it, and what it needs from your machine. --- -## Host Runner (Direct Cargo Run) +## Conventions -For manual control, run the `local_runner` binary directly: +All examples are ordinary binaries run with: ```bash -LOGOS_BLOCKCHAIN_NODE_BIN=/path/to/logos-blockchain-node \ -cargo run -p runner-examples --bin local_runner +cargo run -p --bin ``` -### Host Runner Environment Variables +Naming encodes the backend: `*_basic_*` and `*_app_host_*` run as local processes, `*_compose_*` need a running Docker daemon, and `*_k8s_*` need a reachable Kubernetes cluster context (the k8s deployer drives Helm and the cluster API). Compose binaries exit gracefully with a warning when Docker is unavailable, and the k8s binaries skip when the cluster cannot be reached (`K8sRunnerError::ClientInit`). -| Variable | Default | Effect | -|----------|---------|--------| -| `LOGOS_BLOCKCHAIN_DEMO_NODES` | 1 | Number of nodes (legacy: `LOCAL_DEMO_NODES`) | -| `LOGOS_BLOCKCHAIN_DEMO_RUN_SECS` | 60 | Run duration in seconds (legacy: `LOCAL_DEMO_RUN_SECS`) | -| `LOGOS_BLOCKCHAIN_NODE_BIN` | — | Path to logos-blockchain-node binary (required) | -| `LOGOS_BLOCKCHAIN_LOG_DIR` | None | Directory for per-node log files | -| `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS` | 0 | Keep per-run temporary directories (useful for debugging/CI) | -| `LOGOS_BLOCKCHAIN_TESTS_TRACING` | false | Enable debug tracing preset | -| `LOGOS_BLOCKCHAIN_LOG_LEVEL` | info | Global log level: error, warn, info, debug, trace | -| `LOGOS_BLOCKCHAIN_LOG_FILTER` | None | Fine-grained module filtering (e.g., `cryptarchia=trace`) | - -**Note:** Requires circuit assets and host binaries. Use `scripts/run/run-examples.sh host` to handle setup automatically. +Logging uses `tracing_subscriber` with an env filter; set `RUST_LOG` to adjust verbosity. --- -## Compose Runner (Direct Cargo Run) +## Summary -For manual control, run the `compose_runner` binary directly. Compose requires a Docker image with embedded assets. - -### Option 1: Prebuilt Bundle (Recommended) - -```bash -# 1. Build a Linux bundle (includes binaries + circuits) -scripts/build/build-bundle.sh --platform linux -# Creates .tmp/nomos-binaries-linux-v0.3.1.tar.gz - -# 2. Build image (embeds bundle assets) -export LOGOS_BLOCKCHAIN_BINARIES_TAR=.tmp/nomos-binaries-linux-v0.3.1.tar.gz -scripts/build/build_test_image.sh - -# 3. Run -LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \ -cargo run -p runner-examples --bin compose_runner -``` - -### Option 2: Manual Circuit/Image Setup - -```bash -# Fetch circuits -scripts/setup/setup-logos-blockchain-circuits.sh v0.3.1 ~/.logos-blockchain-circuits - -# Build image -scripts/build/build_test_image.sh - -# Run -LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \ -cargo run -p runner-examples --bin compose_runner -``` - -### Platform Note (macOS / Apple Silicon) - -- Docker Desktop runs a `linux/arm64` engine by default -- For native performance: `LOGOS_BLOCKCHAIN_BUNDLE_DOCKER_PLATFORM=linux/arm64` (recommended for local testing) -- For amd64 targets: `LOGOS_BLOCKCHAIN_BUNDLE_DOCKER_PLATFORM=linux/amd64` (slower via emulation) - -### Compose Runner Environment Variables - -| Variable | Default | Effect | -|----------|---------|--------| -| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | — | Image tag (required, must match built image) | -| `LOGOS_BLOCKCHAIN_DEMO_NODES` | 1 | Number of nodes | -| `LOGOS_BLOCKCHAIN_DEMO_RUN_SECS` | 60 | Run duration in seconds | -| `COMPOSE_NODE_PAIRS` | — | Alternative topology format: "nodes" (e.g., `3`) | -| `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` | None | Prometheus-compatible base URL for runner to query | -| `LOGOS_BLOCKCHAIN_METRICS_OTLP_INGEST_URL` | None | Full OTLP HTTP ingest URL for node metrics export | -| `LOGOS_BLOCKCHAIN_GRAFANA_URL` | None | Grafana base URL for printing/logging | -| `COMPOSE_RUNNER_HOST` | 127.0.0.1 | Host address for port mappings | -| `COMPOSE_RUNNER_PRESERVE` | 0 | Keep containers running after test | -| `LOGOS_BLOCKCHAIN_LOG_LEVEL` | info | Node log level (stdout/stderr) | -| `LOGOS_BLOCKCHAIN_LOG_FILTER` | None | Fine-grained module filtering | - -**Config file option:** `testing-framework/assets/stack/cfgsync.yaml` (`tracing_settings.logger`) — Switch node logs between stdout/stderr and file output - -### Compose-Specific Features - -- **Node control support**: Only runner that supports chaos testing (`.enable_node_control()` + chaos workloads) -- **External observability**: Set `LOGOS_BLOCKCHAIN_METRICS_*` / `LOGOS_BLOCKCHAIN_GRAFANA_URL` to enable telemetry links and querying - - Quickstart: `scripts/setup/setup-observability.sh compose up` then `scripts/setup/setup-observability.sh compose env` - -**Important:** -- Containers expect circuits at `/opt/circuits` (set by the image build) -- Use `scripts/run/run-examples.sh compose` to handle all setup automatically +| Binary | Package | Backend | Requirements | +|---|---|---|---| +| `kvstore_app_host_convergence` | `kvstore-examples` | local (AppHost) | none — node auto-built | +| `kvstore_basic_convergence` | `kvstore-examples` | local | none — node auto-built | +| `kvstore_compose_convergence` | `kvstore-examples` | compose | Docker + `kvstore-node:local` image | +| `kvstore_k8s_convergence` | `kvstore-examples` | k8s | cluster context, Helm, image | +| `kvstore_k8s_manual_convergence` | `kvstore-examples` | k8s (manual) | cluster context, Helm, image | +| `openraft_kv_app_host_smoke` | `openraft-kv-examples` | local (AppHost) | none — node auto-built | +| `openraft_kv_basic_failover` | `openraft-kv-examples` | local | none — node auto-built | +| `openraft_kv_compose_failover` | `openraft-kv-examples` | compose | Docker + `openraft-kv-node:local` image | +| `openraft_kv_k8s_failover` | `openraft-kv-examples` | k8s | cluster context, Helm, image | +| `processes_queued_jobs_and_converges_results` | `multi-app-e2e` (test, not a bin) | local (AppHost) | none — nodes and worker auto-built | +| `nats_basic_roundtrip` | `nats-examples` | local | `nats-server` binary via `NATS_SERVER_BIN` | +| `nats_compose_roundtrip` | `nats-examples` | compose | Docker + `nats:2.10` image present | +| `nats_parity_check` | `nats-examples` | compose + local | Docker; local leg needs `nats-server` | +| `redis_streams_compose_roundtrip` | `redis-streams-examples` | compose | Docker + `redis:7` image present | +| `redis_streams_compose_failover` | `redis-streams-examples` | compose | Docker + `redis:7` image present | +| `pubsub_basic_ws_roundtrip` | `pubsub-examples` | local | `PUBSUB_NODE_BIN` | +| `pubsub_basic_ws_reconnect` | `pubsub-examples` | local | `PUBSUB_NODE_BIN` | +| `pubsub_compose_ws_roundtrip` | `pubsub-examples` | compose | Docker + `pubsub-node:local` image | +| `pubsub_compose_ws_reconnect` | `pubsub-examples` | compose | Docker + `pubsub-node:local` image | +| `pubsub_k8s_ws_roundtrip` | `pubsub-examples` | k8s | cluster context, Helm, image | +| `pubsub_k8s_manual_ws_roundtrip` | `pubsub-examples` | k8s (manual) | cluster context, Helm, image | +| `queue_basic_convergence` | `queue-examples` | local | `QUEUE_NODE_BIN` | +| `queue_basic_restart_chaos` | `queue-examples` | local | `QUEUE_NODE_BIN` | +| `queue_basic_roundtrip` | `queue-examples` | local | `QUEUE_NODE_BIN` | +| `queue_compose_convergence` | `queue-examples` | compose | Docker + `queue-node:local` image | +| `queue_compose_roundtrip` | `queue-examples` | compose | Docker + `queue-node:local` image | +| `metrics_counter_compose_prometheus_expectation` | `metrics-counter-examples` | compose | Docker + `metrics-counter-node:local` image | +| `metrics_counter_k8s_prometheus_expectation` | `metrics-counter-examples` | k8s | cluster context, Helm, image | +| `metrics_counter_k8s_manual_prometheus` | `metrics-counter-examples` | k8s (manual) | cluster context, Helm, image | --- -## K8s Runner (Direct Cargo Run) +## Binary Resolution for Local Runs -For manual control, run the `k8s_runner` binary directly. K8s requires the same image setup as Compose. +Local examples resolve their node binary through a [Binary Provider](binary-providers.md): -### Prerequisites - -1. **Kubernetes cluster** with `kubectl` configured -2. **Test image built** (same as Compose, preferably with prebuilt bundle) -3. **Image available in cluster** (loaded or pushed to registry) - -### Build and Load Image +- **kvstore and openraft_kv** use a `FallbackBinaryProvider`: an explicit `KVSTORE_NODE_BIN` / `OPENRAFT_KV_NODE_BIN` override wins, otherwise a `BuildBinaryProvider` runs `cargo build -p ` for you. No setup needed. +- **queue, pubsub, and metrics_counter** use a plain `EnvBinaryProvider`: you must build the node and point the env var at it: ```bash -# 1. Build image with bundle (recommended) -scripts/build/build-bundle.sh --platform linux -export LOGOS_BLOCKCHAIN_BINARIES_TAR=.tmp/nomos-binaries-linux-v0.3.1.tar.gz -scripts/build/build_test_image.sh - -# 2. Load into cluster (choose one) -export LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local - -# For kind: -kind load docker-image logos-blockchain-testing:local - -# For minikube: -minikube image load logos-blockchain-testing:local - -# For remote cluster (push to registry): -docker tag logos-blockchain-testing:local your-registry/logos-blockchain-testing:latest -docker push your-registry/logos-blockchain-testing:latest -export LOGOS_BLOCKCHAIN_TESTNET_IMAGE=your-registry/logos-blockchain-testing:latest +cargo build -p queue-node +QUEUE_NODE_BIN=target/debug/queue-node cargo run -p queue-examples --bin queue_basic_convergence ``` -### Run the Example +- **nats** launches the upstream `nats-server` executable. Point `NATS_SERVER_BIN` at one (for example from a package manager install). `nats_parity_check` probes for it (env var or `PATH`) and skips the local leg when it is missing. + +--- + +## Compose Images + +The compose deployer checks images with `docker image inspect` and does **not** build or pull them (`MissingImage` error otherwise; see [Troubleshooting](troubleshooting.md)): + +- In-repo node apps default to `:local` (override via `_IMAGE`). Build them from the repository root, e.g.: ```bash -export LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local -cargo run -p runner-examples --bin k8s_runner +docker build -f examples/kvstore/Dockerfile -t kvstore-node:local . ``` -### K8s Runner Environment Variables +Dockerfiles exist for kvstore, openraft_kv, queue, pubsub, and metrics_counter. -| Variable | Default | Effect | -|----------|---------|--------| -| `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` | — | Image tag (required) | -| `LOGOS_BLOCKCHAIN_DEMO_NODES` | 1 | Number of nodes | -| `LOGOS_BLOCKCHAIN_DEMO_RUN_SECS` | 60 | Run duration in seconds | -| `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` | None | Prometheus-compatible base URL for runner to query (PromQL) | -| `LOGOS_BLOCKCHAIN_METRICS_OTLP_INGEST_URL` | None | Full OTLP HTTP ingest URL for node metrics export | -| `LOGOS_BLOCKCHAIN_GRAFANA_URL` | None | Grafana base URL for printing/logging | -| `K8S_RUNNER_NAMESPACE` | Random | Kubernetes namespace (pin for debugging) | -| `K8S_RUNNER_RELEASE` | Random | Helm release name (pin for debugging) | -| `K8S_RUNNER_NODE_HOST` | — | NodePort host resolution for non-local clusters | -| `K8S_RUNNER_DEBUG` | 0 | Log Helm stdout/stderr for install commands | -| `K8S_RUNNER_PRESERVE` | 0 | Keep namespace/release after run (for debugging) | +- **nats and redis_streams have no node crate at all**: they run the upstream images `nats:2.10` and `redis:7` (override via `NATS_IMAGE` / `REDIS_STREAMS_IMAGE`, platform via `NATS_PLATFORM` / `REDIS_STREAMS_PLATFORM`). Pull them once with `docker pull nats:2.10` / `docker pull redis:7`. -### K8s + Observability (Optional) +--- -```bash -export LOGOS_BLOCKCHAIN_METRICS_QUERY_URL=http://your-prometheus:9090 -# Prometheus OTLP receiver example: -export LOGOS_BLOCKCHAIN_METRICS_OTLP_INGEST_URL=http://your-prometheus:9090/api/v1/otlp/v1/metrics -# Optional: print Grafana link in TESTNET_ENDPOINTS -export LOGOS_BLOCKCHAIN_GRAFANA_URL=http://your-grafana:3000 -cargo run -p runner-examples --bin k8s_runner -``` +## What Each Group Exercises -**Notes:** -- `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` must be reachable from the runner process (often via `kubectl port-forward`) -- `LOGOS_BLOCKCHAIN_METRICS_OTLP_INGEST_URL` must be reachable from nodes (pods/containers) and is backend-specific - - Quickstart installer: `scripts/setup/setup-observability.sh k8s install` then `scripts/setup/setup-observability.sh k8s env` - - Optional dashboards: `scripts/setup/setup-observability.sh k8s dashboards` +**kvstore** demonstrates a uniform cluster. `kvstore_app_host_convergence` deploys a local cluster through `AppHost::scenario().with_app(...)` and drives a write/restart/write convergence workload ([Quickstart](quickstart.md) walks it line by line). `kvstore_basic_convergence` is the same coverage through a direct `ScenarioBuilder`. The Compose and Kubernetes variants run the same scenario against those backends; `kvstore_k8s_manual_convergence` bypasses the scenario runner and drives the cluster imperatively via `manual_cluster_from_descriptors` ([ManualCluster](manual-cluster.md)). -### Via `scripts/run/run-examples.sh` (Recommended) +**openraft_kv** demonstrates consensus and leader failover. `openraft_kv_app_host_smoke` is the AppHost entry point. `openraft_kv_basic_failover` and `openraft_kv_compose_failover` share one scenario built with `.enable_node_control()`: write a batch, restart the Raft leader through the node-control capability, write again, and expect convergence ([Scenario Capabilities](capabilities.md)). `openraft_kv_k8s_failover` runs the same failover flow imperatively through the Kubernetes `ManualCluster`, because the Kubernetes deployer wires no node control into managed scenarios ([ManualCluster](manual-cluster.md)). -```bash -scripts/run/run-examples.sh -t 60 -n 3 k8s \ - --metrics-query-url http://your-prometheus:9090 \ - --metrics-otlp-ingest-url http://your-prometheus:9090/api/v1/otlp/v1/metrics -``` +**multi_app** demonstrates application composition and runs as an acceptance test rather than a binary: `cargo test -p multi-app-e2e`. The `multi-app-fixture` crate deploys a queue cluster and a key-value result-store cluster inside one root `AppDeployment` and launches the `multi-app-job-worker` binary between them (resolved via `MULTI_APP_JOB_WORKER_BIN`, else built by Cargo); the test enqueues ten jobs and expects ten results on every store node ([Composing Heterogeneous Stacks](composing-stacks.md)). -### In Code (Optional) +**nats / redis_streams** test unmodified third-party servers. Round-trip workloads publish and consume messages; `redis_streams_compose_failover` runs a consumer-group failover where a second consumer reclaims another's pending stream entries. `nats_parity_check` runs the same scenario against Compose and local backends in one binary. -```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ObservabilityBuilderExt as _; +**pubsub / queue** exercise WebSocket fan-out and work-queue semantics on small in-repo nodes; `queue_basic_restart_chaos` enables node control for restart chaos under load ([Chaos and Controlled Failure](chaos.md)). -let plan = ScenarioBuilder::with_node_counts(1) - .with_metrics_query_url_str("http://your-prometheus:9090") - .with_metrics_otlp_ingest_url_str("http://your-prometheus:9090/api/v1/otlp/v1/metrics") - .build(); -``` +**metrics_counter** is the telemetry demonstration. The compose variant deploys nodes plus a Prometheus container and asserts on scraped metrics through a Prometheus-backed expectation; it honors `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` as a query-endpoint override ([Telemetry and External Observability](telemetry.md)). -### Important K8s Notes - -- K8s runner uses circuits baked into the image -- File path inside pods: `/opt/circuits` -- **No node control support yet**: Chaos workloads (`.enable_node_control()`) will fail -- Optimized for local clusters (Docker Desktop K8s / minikube / kind) - - Remote clusters require additional setup (registry push, PV/CSI for assets, etc.) -- Use `scripts/run/run-examples.sh k8s` to handle all setup automatically - -## Next Steps - -- [CI Integration](ci-integration.md) — Automate tests in continuous integration -- [Environment Variables](environment-variables.md) — Full variable reference -- [Logging & Observability](logging-observability.md) — Log collection and metrics -- [Troubleshooting](troubleshooting.md) — Common issues and fixes +The app-layer examples (`*_app_host_*`, the `multi-app-e2e` tests) show composed systems. The direct-builder binaries provide backend-specific coverage; see `examples/README.md`. diff --git a/book/src/running-scenarios.md b/book/src/running-scenarios.md deleted file mode 100644 index d648ade..0000000 --- a/book/src/running-scenarios.md +++ /dev/null @@ -1,115 +0,0 @@ -# Running Scenarios - -This page focuses on how scenarios are executed (deploy → run → evaluate → cleanup), what artifacts you get back, and how that differs across runners. - -For “just run something that works” commands, see [Running Examples](running-examples.md). - ---- - -## Execution Flow (High Level) - -When you run a built scenario via a deployer, the run follows the same shape: - -```mermaid -flowchart TD - Build[Scenario built] --> Deploy[Deploy] - Deploy --> Capture[Capture] - Capture --> Execute[Execute] - Execute --> Evaluate[Evaluate] - Evaluate --> Cleanup[Cleanup] -``` - -- **Deploy**: provision infrastructure and start nodes (processes/containers/pods) -- **Capture**: establish clients/observability and capture initial state -- **Execute**: run workloads for the configured wall-clock duration -- **Evaluate**: run expectations (after the execution window ends) -- **Cleanup**: stop resources and finalize artifacts - ---- - -## The Core API - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::{Deployer as _, ScenarioBuilder}; -use testing_framework_runner_local::LocalDeployer; -use testing_framework_workflows::ScenarioBuilderExt; - -async fn run_once() -> anyhow::Result<()> { - let mut scenario = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .wallets(20) - .transactions_with(|tx| tx.rate(1).users(5)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(60)) - .build()?; - - let runner = LocalDeployer::default().deploy(&scenario).await?; - runner.run(&mut scenario).await?; - - Ok(()) -} -``` - -Notes: -- `with_run_duration(...)` is wall-clock time, not “number of blocks”. -- `.transactions_with(...)` rates are per-block. -- Most users should run scenarios via `scripts/run/run-examples.sh` unless they are embedding the framework in their own test crate. - ---- - -## Runner Differences - -### Local (Host) Runner - -- **Best for**: fast iteration and debugging -- **Logs/state**: stored under a temporary run directory unless you set `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1` and/or `LOGOS_BLOCKCHAIN_LOG_DIR=...` -- **Limitations**: no node-control capability (chaos workflows that require node control won’t work here) - -Run the built-in local examples: - -```bash -scripts/run/run-examples.sh -t 60 -n 3 host -``` - -### Compose Runner - -- **Best for**: reproducible multi-node environments and node control -- **Logs**: primarily via `docker compose logs` (and any node-level log configuration you apply) -- **Debugging**: set `COMPOSE_RUNNER_PRESERVE=1` to keep the environment up after a run - -Run the built-in compose examples: - -```bash -scripts/run/run-examples.sh -t 60 -n 3 compose -``` - -### K8s Runner - -- **Best for**: production-like behavior, cluster scheduling/networking -- **Logs**: `kubectl logs ...` -- **Debugging**: set `K8S_RUNNER_PRESERVE=1` and `K8S_RUNNER_NAMESPACE=...` to keep resources around - -Run the built-in k8s examples: - -```bash -scripts/run/run-examples.sh -t 60 -n 3 k8s -``` - ---- - -## Artifacts & Where to Look - -- **Node logs**: configure via `LOGOS_BLOCKCHAIN_LOG_DIR`, `LOGOS_BLOCKCHAIN_LOG_LEVEL`, `LOGOS_BLOCKCHAIN_LOG_FILTER` (see [Logging & Observability](logging-observability.md)) -- **Runner logs**: controlled by `RUST_LOG` (runner process only) -- **Keep run directories**: set `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1` -- **Compose environment preservation**: set `COMPOSE_RUNNER_PRESERVE=1` -- **K8s environment preservation**: set `K8S_RUNNER_PRESERVE=1` - ---- - -## See Also - -- [Scenario Lifecycle](scenario-lifecycle.md) -- [Running Examples](running-examples.md) -- [Troubleshooting Scenarios](troubleshooting.md) diff --git a/book/src/runtime-extensions.md b/book/src/runtime-extensions.md new file mode 100644 index 0000000..2bc8a93 --- /dev/null +++ b/book/src/runtime-extensions.md @@ -0,0 +1,129 @@ +# Runtime Extensions + +Runtime extensions are typed values prepared once per run (after nodes exist, before workloads start) and handed to workloads and expectations through the `RunContext`. The app layer and the observation runtime are built on them. + +--- + +## The Mechanism + +The implementation is in `testing-framework/core/src/scenario/runtime/extensions.rs` and has three parts: + +**1. A factory registered on the builder.** `RuntimeExtensionFactory` is called by the deployer during preparation, when the deployment is resolved and node clients are available: + +```rust,ignore +#[async_trait] +pub trait RuntimeExtensionFactory: Send + Sync { + async fn prepare( + &self, + deployment: &E::Deployment, + node_clients: NodeClients, + ) -> Result; +} +``` + +Register it with `.with_runtime_extension_factory(Box::new(factory))`. Factories run in registration order; any `prepare` error aborts the deployment. + +The factory runs after the deployment is resolved and node clients exist. It prepares the value once so every workload and expectation can share it instead of rebuilding the same clients or polling loops. + +**2. A prepared value, optionally with cleanup.** `PreparedRuntimeExtension` wraps one value of any `Clone + Send + Sync + 'static` type, with three constructors: + +| Constructor | Use when | +|-------------|----------| +| `PreparedRuntimeExtension::new(value)` | The value needs no teardown | +| `PreparedRuntimeExtension::with_cleanup(value, guard)` | Custom teardown via a `CleanupGuard` | +| `PreparedRuntimeExtension::from_task(value, join_handle)` | The value is fed by a background Tokio task; the task is aborted at teardown | + +Cleanup guards are collected into the run's cleanup chain and execute at teardown in reverse registration order; see [Handle Ownership and Teardown](handles-teardown.md) for how the app layer separates those guards from handle access. + +**3. Typed retrieval from the context.** The prepared values land in a type-indexed store inside `RunContext`: + +```rust,ignore +// Somewhere in a workload or expectation: +let handle: MyHandle = ctx.require_extension::()?; +// or, tolerating absence: +let maybe: Option = ctx.extension::(); +``` + +`extension::()` returns a *clone* of the stored value. Extension values should therefore be cheap to clone, typically by wrapping shared state in an `Arc`. + +--- + +## One Value Per Type + +The store is keyed by `TypeId`. Registering two extensions that prepare the same type is a hard error at prepare time: + +```text +duplicate runtime extension type registered: +``` + +Because `ctx.extension::()` returns one value by type, each type may be registered only once. To register several values with the same underlying shape, wrap them in distinct newtypes or, in the app layer, use named handles instead (see [AppDeployment and DeployContext](app-deployment.md)). + +This rule is why a scenario allows only one `with_app(...)`: the app layer registers its `AppRuntime` extension per call, and a second registration collides. Compose multiple applications inside one root `AppDeployment` instead; see [AppHost and with_app](app-host.md). + +--- + +## Writing a Factory + +A minimal factory that shares a client wrapper with all workloads: + +```rust,ignore +use async_trait::async_trait; +use testing_framework_core::scenario::{ + DynError, NodeClients, PreparedRuntimeExtension, RuntimeExtensionFactory, +}; + +#[derive(Clone)] +struct FrontDoor(MyNodeClient); + +struct FrontDoorFactory; + +#[async_trait] +impl RuntimeExtensionFactory for FrontDoorFactory { + async fn prepare( + &self, + _deployment: &::Deployment, + node_clients: NodeClients, + ) -> Result { + let client = node_clients + .snapshot() + .first() + .cloned() + .ok_or("no nodes available")?; + + Ok(PreparedRuntimeExtension::new(FrontDoor(client))) + } +} + +// Registration: +let builder = builder.with_runtime_extension_factory(Box::new(FrontDoorFactory)); +``` + +For an extension backed by a polling loop, spawn the task in `prepare` and return `from_task(handle, join_handle)`. The runner aborts the task when the run tears down, so the loop cannot outlive the cluster. The observation runtime works this way. The pubsub example registers its feed this way (`examples/pubsub/testing/integration/src/scenario.rs`): + +```rust,ignore +self.with_runtime_extension_factory(Box::new(PubSubTopicFeedFactory::new(topic))) +``` + +--- + +## What Is Built on This + +The following layers use runtime extension factories: + +| Layer | Factory | Extension value in `RunContext` | +|-------|---------|--------------------------------| +| App layer | `AppDeploymentFactory` (via `with_app`) | `AppRuntime` + exposed app handles | +| Observation | `ObservationExtensionFactory` (via `with_observer`) | `ObservationHandle` | + +So when a workload calls `ctx.require_app::()` or `ctx.require_extension::>()`, it is walking the same type-indexed store described above. + +- App layer: [AppHost and with_app](app-host.md) +- Observation runtime: [Continuous Observation](observation.md) + +--- + +## See Also + +- [Workloads and Concurrency](workloads.md) — consuming extensions from workloads +- [Continuous Observation](observation.md) — an extension backed by a polling task +- [Handle Ownership and Teardown](handles-teardown.md) — cleanup ordering in depth diff --git a/book/src/scenario-builder-ext-patterns.md b/book/src/scenario-builder-ext-patterns.md deleted file mode 100644 index 96e0837..0000000 --- a/book/src/scenario-builder-ext-patterns.md +++ /dev/null @@ -1,19 +0,0 @@ -# Core Content: ScenarioBuilderExt Patterns - -> **When should I read this?** After writing 2-3 scenarios. This page documents patterns that emerge from real usage—come back when you're refactoring or standardizing your test suite. - -Patterns that keep scenarios readable and reusable: - -- **Topology-first**: start by shaping the cluster (counts, layout) so later - steps inherit a clear foundation. -- **Bundle defaults**: use the DSL helpers to attach common expectations (like - liveness) whenever you add a matching workload, reducing forgotten checks. -- **Intentional rates**: express traffic in per-block terms to align with - protocol timing rather than wall-clock assumptions. -- **Opt-in chaos**: enable restart patterns only in scenarios meant to probe - resilience; keep functional smoke tests deterministic. -- **Wallet clarity**: seed only the number of actors you need; it keeps - transaction scenarios deterministic and interpretable. - -These patterns make scenario definitions self-explanatory while staying aligned -with the framework’s block-oriented timing model. diff --git a/book/src/scenario-lifecycle.md b/book/src/scenario-lifecycle.md deleted file mode 100644 index 121f271..0000000 --- a/book/src/scenario-lifecycle.md +++ /dev/null @@ -1,133 +0,0 @@ -# Scenario Lifecycle - -A scenario progresses through six distinct phases, each with a specific responsibility: - -```mermaid -flowchart TB - subgraph Phase1["1. Build Phase"] - Build[Define Scenario] - BuildDetails["• Declare topology
• Attach workloads
• Add expectations
• Set run duration"] - Build --> BuildDetails - end - - subgraph Phase2["2. Deploy Phase"] - Deploy[Provision Environment] - DeployDetails["• Launch nodes
• Wait for readiness
• Establish connectivity
• Return Runner"] - Deploy --> DeployDetails - end - - subgraph Phase3["3. Capture Phase"] - Capture[Baseline Metrics] - CaptureDetails["• Snapshot initial state
• Start BlockFeed
• Initialize expectations"] - Capture --> CaptureDetails - end - - subgraph Phase4["4. Execution Phase"] - Execute[Drive Workloads] - ExecuteDetails["• Submit transactions
• Trigger chaos events
• Run for duration"] - Execute --> ExecuteDetails - end - - subgraph Phase5["5. Evaluation Phase"] - Evaluate[Check Expectations] - EvaluateDetails["• Verify liveness
• Check inclusion
• Validate outcomes
• Aggregate results"] - Evaluate --> EvaluateDetails - end - - subgraph Phase6["6. Cleanup Phase"] - Cleanup[Teardown] - CleanupDetails["• Stop nodes
• Remove containers
• Collect logs
• Release resources"] - Cleanup --> CleanupDetails - end - - Phase1 --> Phase2 - Phase2 --> Phase3 - Phase3 --> Phase4 - Phase4 --> Phase5 - Phase5 --> Phase6 - - style Phase1 fill:#e1f5ff - style Phase2 fill:#fff4e1 - style Phase3 fill:#f0ffe1 - style Phase4 fill:#ffe1f5 - style Phase5 fill:#e1ffe1 - style Phase6 fill:#ffe1e1 -``` - -## Phase Details - -### 1. Build the Plan - -Declare a topology, attach workloads and expectations, and set the run window. The plan is the single source of truth for what will happen. - -**Key actions:** -- Define cluster shape (nodes, network topology) -- Configure workloads (transaction rate, chaos patterns) -- Attach expectations (liveness, inclusion, custom checks) -- Set timing parameters (run duration, cooldown period) - -**Output:** Immutable `Scenario` plan - -### 2. Deploy - -Hand the plan to a deployer. It provisions the environment on the chosen backend, waits for nodes to signal readiness, and returns a runner. - -**Key actions:** -- Provision infrastructure (processes, containers, or pods) -- Launch nodes -- Wait for readiness probes (HTTP endpoints respond) -- Establish node connectivity and metrics endpoints -- Spawn BlockFeed for real-time block observation - -**Output:** `Runner` + `RunContext` (with node clients, metrics, control handles) - -### 3. Capture Baseline - -Expectations snapshot initial state before workloads begin. - -**Key actions:** -- Record starting block height -- Initialize counters and trackers -- Subscribe to BlockFeed -- Capture baseline metrics - -**Output:** Captured state for later comparison - -### 4. Drive Workloads - -The runner starts traffic and behaviors for the planned duration. - -**Key actions:** -- Submit transactions at configured rates -- Trigger chaos events (node restarts) -- Run concurrently for the specified duration -- Observe blocks and metrics in real-time - -**Note:** Network partitions/peer blocking are not yet supported by node control; today chaos is restart-based. See [RunContext: BlockFeed & Node Control](node-control.md). - -**Duration:** Controlled by `with_run_duration()` - -### 5. Evaluate Expectations - -Once activity stops (and optional cooldown completes), the runner checks liveness and workload-specific outcomes. - -**Key actions:** -- Verify consensus liveness (minimum block production) -- Check transaction inclusion rates -- Assess system recovery after chaos events -- Aggregate pass/fail results - -**Output:** Success or detailed failure report - -### 6. Cleanup - -Tear down resources so successive runs start fresh and do not inherit leaked state. - -**Key actions:** -- Stop all node processes/containers/pods -- Remove temporary directories and volumes -- Collect and archive logs (if `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1`) -- Release ports and network resources -- Cleanup observability stack (if spawned) - -**Guarantee:** Runs even on panic via `CleanupGuard` diff --git a/book/src/scenario-model.md b/book/src/scenario-model.md index ea742c0..22f99f8 100644 --- a/book/src/scenario-model.md +++ b/book/src/scenario-model.md @@ -1,23 +1,112 @@ -# Scenario Model (Developer Level) +# Scenario Model and Lifecycle -The scenario model defines clear, composable responsibilities: +A scenario records a topology, workloads, expectations, runtime settings, and deployment policy. The runner executes the phases described below. -- **Topology**: a declarative description of the cluster—how many nodes, their - roles, and the broad network and data-availability characteristics. It - represents the intended shape of the system under test. -- **Scenario**: a plan combining topology, workloads, expectations, and a run - window. Building a scenario validates prerequisites (like seeded wallets) and - ensures the run lasts long enough to observe meaningful block progression. -- **Workloads**: asynchronous tasks that generate traffic or conditions. They - use shared context to interact with the deployed cluster and may bundle - default expectations. -- **Expectations**: post-run assertions. They can capture baselines before - workloads start and evaluate success once activity stops. -- **Runtime**: coordinates workloads and expectations for the configured - duration, enforces cooldowns when control actions occur, and ensures cleanup - so runs do not leak resources. +--- -Developers extending the model should keep these boundaries strict: topology -describes, scenarios assemble, deployers provision, runners orchestrate, -workloads drive, and expectations judge outcomes. For guidance on adding new -capabilities, see [Extending the Framework](extending.md). +## What a Scenario Is + +You assemble a scenario with `ScenarioBuilder` and hand it to a deployer. The essential ingredients: + +| Ingredient | Builder method | Meaning | +|---|---|---| +| Topology | `with_deployment(...)` / `new(provider)` | Which nodes exist and how they relate | +| Workloads | `with_workload(...)` | Traffic and actions driven during the run | +| Expectations | `with_expectation(...)` | What success means, checked against the run | +| Duration | `with_run_duration(...)` | How long workloads get to run | +| Cooldown | `with_expectation_cooldown(...)` | Extra settle window before evaluation | +| Policy | `with_deployment_policy(...)` | Readiness gating, retries, artifact retention | + +Because the whole plan is declared up front, `build()` can validate it and fail before any process is spawned. + +Workloads implement `Workload` (`name()`, `init(...)`, `async start(&self, ctx)`); expectations implement `Expectation` (`start_capture`, optional `check_during_capture`, `evaluate`). A workload can also contribute its own expectations; `with_workload` collects them automatically. Both receive the shared `RunContext`, which carries the deployment descriptor, node clients, telemetry, and typed runtime extensions. See [Workloads and Concurrency](workloads.md) and [Expectations and Evaluation](expectations.md). + +--- + +## The Lifecycle + +```mermaid +flowchart TD + B["build()"] --> D["deployer.deploy(&scenario)"] + D --> RG["spawn + readiness gating (retry per policy)"] + RG --> PX["prepare runtime extensions (with_app runs here)"] + PX --> RUN["runner.run(&mut scenario)"] + RUN --> W["workloads start concurrently"] + W --> CD["cooldown window"] + CD --> EV["evaluate all expectations (aggregate failures)"] + EV --> H["RunHandle"] + H --> T["drop → cleanup guards"] + W -- "failure" --> T + EV -- "failure" --> T + RUN:::sc + W:::sc + EV:::sc + H:::hd + classDef sc stroke:#9b6dd6,stroke-width:2.5px; + classDef hd stroke:#4caf7d,stroke-width:2.5px; +``` + +### 1. Build + +`build()` finalizes the plan. It resolves the deployment from the topology provider (honoring `with_deployment_seed`), validates the source configuration (for example, external-only scenarios must declare at least one external node, and node control is rejected for uncontrolled external clusters), and calls `init` on every workload and expectation. Failures surface as `ScenarioBuildError` before anything is deployed. + +**Note:** `build()` enforces a minimum run duration of 10 seconds and defaults the expectation cooldown to 10 seconds when you have not set one. + +### 2. Deploy + +`deployer.deploy(&scenario)` provisions the environment and returns a `Runner`. For the local deployer this means spawning node processes, then **readiness gating**: each node's readiness probe (HTTP path or plain TCP, per the app) is retried until the policy's readiness requirement holds, with retry and backoff per `DeploymentPolicy`. Only after the cluster is ready are **runtime extensions** (typed services prepared once per run and handed to workloads, see [Runtime Extensions](runtime-extensions.md)) prepared; `with_app` deployments deploy at this point. Registering two extensions of the same type fails here with a "duplicate runtime extension type registered" error. Failure-path cleanup already applies at this stage through ownership: when deployment errors partway, partially deployed app resources are released as their handles drop, and spawned node processes stop when their process handles drop. + +### 3. Run: workloads + +`runner.run(&mut scenario)` first calls `start_capture` on every expectation, then spawns **all workloads concurrently**, each in its own task. Workload panics are caught and converted into workload errors instead of aborting the process. The run window lasts for the configured duration, during which the runner also ticks `check_during_capture` on every expectation once per second, so an expectation can fail during the run instead of waiting for final evaluation. + +A workload returning early with `Ok(())` is fine; the window keeps running while other workloads are still active. The duration is a **maximum**: once every workload has finished, the window ends early and cooldown begins. A workload error ends the run immediately with `ScenarioError::Workload`. + +### 4. Cooldown and settle + +When the duration elapses, workloads are not cut off abruptly. The runner keeps the run alive through a **cooldown window** derived from `with_expectation_cooldown`; clusters whose lifecycle the framework owns get a 30-second minimum so restarted nodes and runtime extensions observe stabilized state. Remaining workload tasks are then drained, and a short settle wait (at least 2 seconds when a cooldown or node control is in play) runs before evaluation. + +### 5. Evaluation + +Every expectation's `evaluate` runs, including after another expectation fails. Failures are aggregated into one `ScenarioError::Expectations` report with one line per failed expectation. + +### 6. Teardown + +A successful run returns a `RunHandle`. Teardown is guard-based. When the handle drops, its `CleanupGuard` chain runs, stopping node processes, aborting extension tasks, and executing app cleanup stacks (see [Handle Ownership and Teardown](handles-teardown.md)). The same guards run **on the failure path**: any step that errors inside `run` triggers immediate cleanup before the error is returned, so failed runs do not leak managed processes or temp directories. + +```rust,ignore +let mut scenario = KvScenarioBuilder::deployment_with(|t| t) + .with_run_duration(Duration::from_secs(30)) + .with_expectation_cooldown(Duration::from_secs(5)) + .with_workload(KvWriteWorkload::new().operations(300)) + .with_expectation(KvConverges::new("demo", 30)) + .build()?; + +let deployer = KvLocalDeployer::default(); +let runner = deployer.deploy(&scenario).await?; +let _handle = runner.run(&mut scenario).await?; +// dropping _handle tears the cluster down +``` + +Source: `testing-framework/core/src/scenario/runtime/runner.rs` and `runtime/context.rs`. + +--- + +## Errors by Phase + +| Phase | Error | Typical cause | +|---|---|---| +| Build | `ScenarioBuildError` | Bad source configuration, workload/expectation `init` failure | +| Deploy | Deployer error | Spawn failure, readiness timeout, duplicate extension, app deploy failure | +| Run | `ScenarioError::Workload` | Workload error or panic | +| Run | `ScenarioError::ExpectationFailedDuringCapture` | Fail-fast check tripped mid-run | +| Run | `ScenarioError::Expectations` | Aggregated end-of-run evaluation failures | + +--- + +## Where to Go Next + +- [Application, AppDeployment, and Environments](application-model.md): the type parameter behind `ScenarioBuilder`. +- [Choosing an Entry Pattern](entry-patterns.md): the ways to reach this one lifecycle. +- [Readiness, Retry, and Artifact Preservation](deployment-policies.md): tuning the deploy phase. +- [Part III — Scenario Runtime](part-iii.md): workloads, expectations, and capabilities in depth. diff --git a/book/src/seeds.md b/book/src/seeds.md new file mode 100644 index 0000000..03d0b68 --- /dev/null +++ b/book/src/seeds.md @@ -0,0 +1,94 @@ +# Seeds and Reproducibility + +`DeploymentSeed` controls only part of a run's variability. This chapter lists what it seeds, what it does not, and what that means for reproducing a run. + +--- + +## DeploymentSeed + +`DeploymentSeed` (`testing-framework/core/src/topology/mod.rs`) is a 32-byte value: + +```rust,ignore +let seed = DeploymentSeed::new([7u8; 32]); +let bytes: &[u8; 32] = seed.bytes(); +``` + +The seed exists so generated deployments can be reproduced: record it when a run fails, and the same seed makes the provider return the identical deployment. + +You attach it to a scenario with `with_deployment_seed`: + +```rust,ignore +let scenario = ScenarioBuilder::::new(provider) + .with_deployment_seed(DeploymentSeed::new([7u8; 32])) + .with_run_duration(Duration::from_secs(30)) + .build()?; +``` + +The seed has exactly one consumer: when `build()` resolves the deployment, it calls the deployment provider with it: + +```rust,ignore +pub trait DeploymentProvider: Send + Sync { + fn build(&self, seed: Option<&DeploymentSeed>) -> Result; +} +``` + +A provider that generates topologies (random shapes, sampled node parameters, derived node ids) should draw all of its randomness from the seed, so the same seed always yields the same deployment. See [Topology and Deployment Plans](topology.md) for how providers feed the builder. + +--- + +## What Is Actually Seeded + +The current behavior is: + +| Concern | Seeded? | +|---|---| +| Deployment generation by a *custom* `DeploymentProvider` | Yes — the seed is passed to `build()` | +| `FixedDeploymentProvider` (the `with_deployment(...)` path) | No — the seed is accepted and ignored | +| Local port assignment | No — ports come from the OS (`bind 127.0.0.1:0`; see [node-config.md](node-config.md)) | +| Node working directories | No — temp directories get random suffixes (see [Persistence](persistence.md)) | +| Workload timing, scheduling, network behavior | No | + +No in-repo deployment provider currently consumes the seed: `FixedDeploymentProvider` is the only provider shipped, and the example apps all use concrete `ClusterTopology` values. `DeploymentSeed` is available to custom generating providers; setting a seed on a fixed deployment has no effect. + +--- + +## Writing a Seeded Provider + +A provider that wants reproducible generation reads all of its variability from the seed bytes: + +```rust,ignore +use testing_framework_core::topology::{ + ClusterTopology, DeploymentProvider, DeploymentSeed, DynTopologyError, +}; + +struct SizedFromSeed { + min_nodes: usize, + max_nodes: usize, +} + +impl DeploymentProvider for SizedFromSeed { + fn build(&self, seed: Option<&DeploymentSeed>) -> Result { + let first = seed.map_or(0, |seed| seed.bytes()[0] as usize); + let span = self.max_nodes - self.min_nodes + 1; + Ok(ClusterTopology::new(self.min_nodes + first % span)) + } +} + +let scenario = ScenarioBuilder::::new(Box::new(SizedFromSeed { + min_nodes: 3, + max_nodes: 7, + })) + .with_deployment_seed(DeploymentSeed::new([7u8; 32])) + .with_run_duration(Duration::from_secs(30)) + .build()?; +``` + +The same seed always produces the same cluster size; omitting the seed uses the provider's default (`seed` is `None`). A generating provider can feed the 32 bytes into a seeded RNG and derive node parameters or `NodePlan` ids from it. + +--- + +## Practical Reproducibility + +- If your provider is seeded, record the seed alongside failures and replay with `with_deployment_seed` to get the identical deployment. +- Everything downstream of the deployment (ports, PIDs, timing) still varies run to run. Determinism ends at the descriptor; treat expectations accordingly. +- For state-level reproduction (replaying a node from captured state rather than regenerating a topology), use snapshot directories instead; see [Persistence, Snapshots, and Recovery Testing](persistence.md). diff --git a/book/src/telemetry.md b/book/src/telemetry.md new file mode 100644 index 0000000..13ec29b --- /dev/null +++ b/book/src/telemetry.md @@ -0,0 +1,104 @@ +# Telemetry and External Observability + +Telemetry connects a scenario to external observability infrastructure such as Prometheus, an OTLP collector, and Grafana. It supports PromQL queries and external dashboards. For typed application state inside a test, use the [observation runtime](observation.md). + +--- + +## Observation and Telemetry + +| | Observation runtime | Telemetry | +|---|---|---| +| What | Typed app state (leaders, keys, heads) | Metrics/logs/traces on external endpoints | +| Where it lives | Inside the test process | Prometheus / OTLP collector / Grafana | +| Consumed by | Workloads and expectations, synchronously | PromQL queries, dashboards, humans | +| Chapter | [Continuous Observation](observation.md) | this one | + +Telemetry endpoints are optional in every deployer. Without telemetry configuration, the scenario still runs and `RunContext::telemetry()` has no Prometheus backend. + +--- + +## Declaring Endpoints on the Builder + +`ObservabilityCapability` (`testing-framework/core/src/scenario/capabilities.rs`) carries three optional URLs: + +| Field | Meaning | +|-------|---------| +| `metrics_query_url` | Base URL the runner uses to query Prometheus | +| `metrics_otlp_ingest_url` | OTLP HTTP endpoint nodes export metrics to | +| `grafana_url` | Grafana base URL, for logs/output convenience | + +You populate it with `ObservabilityBuilderExt` (`testing-framework/core/src/scenario/builder_ext.rs`), which transitions a plain `ScenarioBuilder` into an `ObservabilityScenarioBuilder`, the capability-typed builder described in [Scenario Capabilities](capabilities.md): + +```rust,ignore +use testing_framework_core::scenario::ObservabilityBuilderExt; + +let scenario = ScenarioBuilder::with_deployment(topology) + .with_metrics_query_url_str("http://127.0.0.1:9090") + .with_metrics_otlp_ingest_url_str("http://127.0.0.1:4318") + .with_run_duration(Duration::from_secs(60)) + .build()?; +``` + +Each endpoint has three setter flavors: `with_..._url(Url)`, `with_..._url_str(&str)` (panics on an invalid URL), and `try_with_..._url_str(&str)` (returns `BuilderInputError`). + +--- + +## ObservabilityInputs: Capability Plus Environment + +Deployers do not read the capability directly; they resolve an `ObservabilityInputs` (`testing-framework/core/src/scenario/observability.rs`) that merges two sources: + +```rust,ignore +let env_inputs = ObservabilityInputs::from_env()?; +let cap_inputs = observability + .observability_capability() // via ObservabilityCapabilityProvider + .map(ObservabilityInputs::from_capability) + .unwrap_or_default(); +let inputs = env_inputs.with_overrides(cap_inputs); +``` + +The compose and k8s orchestrators use this merge in `testing-framework/deployers/{compose,k8s}/src/deployer/orchestrator.rs`: environment values form the base, and any endpoint set on the scenario capability overrides the corresponding environment value. + +This allows the environment to supply infrastructure-specific endpoints while the scenario can override individual URLs on the builder. + +**What `from_env` reads.** Verified against the source, it reads exactly three environment variables, each parsed as a URL (empty or unset values are skipped; an unparsable value is an error): + +| Env var | Feeds | +|---------|-------| +| `LOGOS_BLOCKCHAIN_METRICS_QUERY_URL` | `metrics_query_url` | +| `LOGOS_BLOCKCHAIN_METRICS_OTLP_INGEST_URL` | `metrics_otlp_ingest_url` | +| `LOGOS_BLOCKCHAIN_GRAFANA_URL` | `grafana_url` | + +`ObservabilityInputs` also offers `from_capability(&cap)`, `with_overrides(other)` (field-wise, `Some` wins), and `telemetry_handle()`, which builds the `Metrics` value stored in the `RunContext`: `Metrics::from_prometheus(url)` when `metrics_query_url` is set, `Metrics::empty()` otherwise. + +**Deployer support today:** compose and k8s resolve env + capability as above and wire the OTLP ingest URL into node configuration. The local deployer currently builds its runtime with `Metrics::empty()` and does not wire telemetry endpoints. See the [Capability Matrix](capability-matrix.md). + +--- + +## Querying Metrics in a Run + +`RunContext::telemetry()` returns the `Metrics` handle. Backed by Prometheus it evaluates instant queries: + +```rust,ignore +let telemetry = ctx.telemetry(); +let values = telemetry.instant_values("up")?; // all sample values +let total = telemetry.counter_value("requests_total")?; // summed counter +``` + +Without a configured `metrics_query_url` these calls return a `MetricsError` ("prometheus endpoint unavailable"). Expectations that assert on metrics therefore require a configured telemetry endpoint. + +Telemetry queries depend on scrape intervals, exporter lag, and external infrastructure. Observation polls application state from the test process and reports failures by source. Correctness checks can use observation when they require current typed state; performance checks and post-run analysis can use telemetry. + +--- + +## A Local Stack for Development + +To use a local Prometheus, OTLP collector, and Grafana stack, export the three environment variables above or set the URLs on the builder. The same scenario binary can then run with or without a metrics backend. + +--- + +## See Also + +- [Continuous Observation](observation.md) — test-visible state, the in-process counterpart +- [Scenario Capabilities](capabilities.md) — how the observability capability is typed +- [Capability Matrix](capability-matrix.md) — per-deployer telemetry support +- [Environment Variables](environment-variables.md) — the full audited env var list diff --git a/book/src/testing-philosophy.md b/book/src/testing-philosophy.md deleted file mode 100644 index 455c6cb..0000000 --- a/book/src/testing-philosophy.md +++ /dev/null @@ -1,179 +0,0 @@ -# Testing Philosophy - -This framework embodies specific principles that shape how you author and run -scenarios. Understanding these principles helps you write effective tests and -interpret results correctly. - -## Declarative over Imperative - -Describe **what** you want to test, not **how** to orchestrate it: - -```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn declarative_over_imperative() { - // Good: declarative - let _plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(2)) - .transactions_with(|txs| { - txs.rate(5) // 5 transactions per block - }) - .expect_consensus_liveness() - .build(); - - // Bad: imperative (framework doesn't work this way) - // spawn_node(); - // loop { submit_tx(); check_block(); } -} -``` - -**Why it matters:** The framework handles deployment, readiness, and cleanup. -You focus on test intent, not infrastructure orchestration. - -**Exception:** For advanced network scenarios (split-brain, late joins, network healing) that can't be expressed declaratively, see [Manual Clusters](manual-cluster.md) for imperative control. - -## Protocol Time, Not Wall Time - -Reason in **blocks** and **consensus intervals**, not wall-clock seconds. - -**Consensus defaults:** -- Slot duration: 2 seconds (NTP-synchronized, configurable via `CONSENSUS_SLOT_TIME`) -- Active slot coefficient: 0.9 (90% block probability per slot, configurable via `CONSENSUS_ACTIVE_SLOT_COEFF`) -- Expected rate: ~27 blocks per minute - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn protocol_time_not_wall_time() { - // Good: protocol-oriented thinking - let _plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(2)) - .transactions_with(|txs| { - txs.rate(5) // 5 transactions per block - }) - .with_run_duration(Duration::from_secs(60)) // Let framework calculate expected blocks - .expect_consensus_liveness() // "Did we produce the expected blocks?" - .build(); - - // Bad: wall-clock assumptions - // "I expect exactly 30 blocks in 60 seconds" - // This breaks on slow CI where slot timing might drift -} -``` - -**Why it matters:** Slot timing is fixed (2s by default, NTP-synchronized), so the -expected number of blocks is predictable: ~27 blocks in 60s with the default -0.9 active slot coefficient. The framework calculates expected blocks from slot -duration and run window, making assertions protocol-based rather than tied to -specific wall-clock expectations. Assert on "blocks produced relative to slots" -not "blocks produced in exact wall-clock seconds". - -## Determinism First, Chaos When Needed - -**Default scenarios are repeatable:** -- Fixed topology -- Predictable traffic rates -- Deterministic checks - -**Chaos is opt-in:** -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::{ChaosBuilderExt, ScenarioBuilderExt}; - -pub fn determinism_first() { - // Separate: functional test (deterministic) - let _plan = ScenarioBuilder::topology_with(|t| t.network_star().nodes(2)) - .transactions_with(|txs| { - txs.rate(5) // 5 transactions per block - }) - .expect_consensus_liveness() - .build(); - - // Separate: chaos test (introduces randomness) - let _chaos_plan = - ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .enable_node_control() - .chaos_with(|c| { - c.restart() - .min_delay(Duration::from_secs(30)) - .max_delay(Duration::from_secs(60)) - .target_cooldown(Duration::from_secs(45)) - .apply() - }) - .transactions_with(|txs| { - txs.rate(5) // 5 transactions per block - }) - .expect_consensus_liveness() - .build(); -} -``` - -**Why it matters:** Mixing determinism with chaos creates noisy, hard-to-debug -failures. Separate concerns make failures actionable. - -## Observable Health Signals - -Prefer **user-facing signals** over internal state: - -**Good checks:** -- Blocks progressing at expected rate (liveness) -- Transactions included within N blocks (inclusion) -- Transactions included within N blocks (inclusion) - -**Avoid internal checks:** -- Memory pool size -- Internal service state -- Cache hit rates - -**Why it matters:** User-facing signals reflect actual system health. -Internal state can be "healthy" while the system is broken from a user -perspective. - -## Minimum Run Windows - -Always run long enough for **meaningful block production**: - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -pub fn minimum_run_windows() { - // Bad: too short (~2 blocks with default 2s slots, 0.9 coeff) - let _too_short = ScenarioBuilder::with_node_counts(1) - .with_run_duration(Duration::from_secs(5)) - .expect_consensus_liveness() - .build(); - - // Good: enough blocks for assertions (~27 blocks with default 2s slots, 0.9 - // coeff) - let _good = ScenarioBuilder::with_node_counts(1) - .with_run_duration(Duration::from_secs(60)) - .expect_consensus_liveness() - .build(); -} -``` - -**Note:** Block counts assume default consensus parameters: -- Slot duration: 2 seconds (configurable via `CONSENSUS_SLOT_TIME`) -- Active slot coefficient: 0.9 (90% block probability per slot, configurable via `CONSENSUS_ACTIVE_SLOT_COEFF`) -- Formula: `blocks ≈ (duration / slot_duration) × active_slot_coeff` - -If upstream changes these parameters, adjust your duration expectations accordingly. - -The framework enforces minimum durations (at least 2× slot duration), but be explicit. Very short runs risk false confidence—one lucky block doesn't prove liveness. - -## Summary - -These principles keep scenarios: -- **Portable** across environments (protocol time, declarative) -- **Debuggable** (determinism, separation of concerns) -- **Meaningful** (observable signals, sufficient duration) - -When authoring scenarios, ask: "Does this test the protocol behavior or -my local environment quirks?" diff --git a/book/src/tf-boundaries.md b/book/src/tf-boundaries.md new file mode 100644 index 0000000..d73b8a7 --- /dev/null +++ b/book/src/tf-boundaries.md @@ -0,0 +1,92 @@ +# Framework vs Application Boundaries + +This chapter covers the rules that keep the framework application-agnostic and how they are enforced in practice. + +[Ownership and Design Boundaries](boundaries.md) explains the ownership split. This chapter covers the concrete rules, enforcement mechanisms, and signs that code is in the wrong layer. + +--- + +## The Rule + +Dependencies point in exactly one direction: application repositories depend on framework crates, never the reverse. The framework knows applications only through the traits in [Public Extension Points](extension-points.md); everything app-specific (node configs, HTTP clients, readiness semantics, observers, workloads) lives in the application's own integration crates. + +The in-repo `examples/` workspace models the application side: each app has an integration crate (`-runtime-ext`) implementing `Application` and the per-backend environment traits, and a workloads crate (`-runtime-workloads`) implementing `Workload` and `Expectation`. No framework crate names an example app in its `Cargo.toml`; see the dependency diagram in [Crate and API Map](crate-map.md). + +```mermaid +graph LR + subgraph Application side + ext["<app>-runtime-ext
Application, env impls, Observer"] + wl["<app>-runtime-workloads
Workload, Expectation"] + end + subgraph Framework side + core[testing-framework-core] + appl[testing-framework-app] + runners[deployers] + end + ext --> core + ext --> runners + wl --> core + appl --> core + runners --> core +``` + +Traits cross the boundary; concrete application types never do. + +**What belongs where:** + +| Concern | Framework | Application repo | +|---|---|---| +| Process supervision, port allocation, tempdirs, teardown ordering | yes | — | +| Scenario scheduling, expectations lifecycle, readiness/retry policy | yes | — | +| Observation runtime (cycles, history, failure tracking) | yes | — | +| `Application` impl, node config types, config rendering | — | yes | +| Domain node clients and typed app handles | — | yes | +| Readiness closures with domain semantics (leader elected, stream exists) | — | yes | +| `Observer` impls and their snapshot/event types | — | yes | +| Binary provider *configuration* (env var names, build commands) | — | yes | +| Config templates for a specific application | — | yes | + +--- + +## Enforcement Mechanisms + +**The type system.** The scenario engine is generic over `Application`, so core code physically cannot reference your node client or config, because there is no concrete type to name. The app layer goes further: `AppHostEnv` sets `NodeClient = ()` and its `build_node_client` returns an error, forcing application clients to travel as typed handles owned by the app side rather than leaking into the environment. + +**Runtime registration errors.** Two rules are enforced with hard errors instead of silent replacement: + +- Registering two runtime extension factories that produce the same type fails at prepare time with `duplicate runtime extension type registered: ...`. This is also why a scenario takes exactly one `with_app`; compose multiple apps inside one root `AppDeployment` instead. +- Exposing a handle twice under the same type and name fails with `app handle is already exposed: ...` (`AppDeployError::DuplicateHandle`). + +**The boundary check script.** `scripts/run/check-boundaries.sh` guards the adopter side of the line. What it actually does: + +- resolves a sibling adopter checkout at `../nomos-node/tests/testing_framework/lb-topology` and fails if it is missing; +- greps that crate's `src/` and `Cargo.toml` for extension-specific identifiers (`cfgsync`, `ComposeDeployEnv`, `K8sDeployEnv`, `runner-compose`, `runner-k8s`, `DEFAULT_CFGSYNC_PORT`, `DEFAULT_ASSETS_STACK_DIR`) and fails on any hit. + +The same rule can be applied to other integration crates: a topology-level crate remains backend-agnostic, so references to a specific deployer or cfgsync internals are treated as violations. Backend names belong in the per-backend environment modules; compare `local_env.rs`, `compose_env.rs`, and `k8s_env.rs` in `examples/openraft_kv/testing/integration/src/`. + +**Crate docs as contract.** `testing-framework-app` states the ownership boundary in its crate docs: implement `AppDeployment` in the application repository, compose children through `DeployContext`, and let handles own deployed resources. The `multi_app` README says the same from the other direction: for composed systems, prefer the app-layer shape "instead of building a fake outer cluster or adding app-specific code to TF". + +--- + +## Signs Your Code Is on the Wrong Side + +Symptoms that application code has leaked into the framework: + +- A config template, launch flag, or port convention for one specific application sitting in `testing-framework/` or `cfgsync/`. +- A framework crate importing an example (or adopter) crate, or matching on an application name. +- A "generic" helper in core whose only caller is one app and whose parameters mirror that app's config fields. + +Symptoms that framework mechanics are being re-implemented in the application repo: + +- Hand-rolled process spawn/kill/teardown code where `LocalProcessApp` or `LocalAppCluster` would do. +- A custom polling loop with history and error tracking that duplicates the observation runtime; implement `Observer` instead. +- Re-implementing binary resolution, caching, or fallback chains instead of configuring `BinaryProvider` types. +- A bespoke "wait until cluster healthy" loop instead of readiness closures plus `DeploymentPolicy` (see [Readiness, Retry, and Artifact Preservation](deployment-policies.md)). + +A framework addition should compile and make sense with a different application plugged in. Application-specific code belongs in the application repository. + +--- + +## Backend Scope of the App Layer + +The composition layer is local-only today, and this is visible in the dependency graph: `testing-framework-app` depends on core and `testing-framework-runner-local` only, `AppHostLocalDeployer` is an alias for `ProcessDeployer`, and `DeployContext::deploy_local_cluster` / `LocalAppCluster` require `LocalDeployerEnv`. The compose and k8s deployers remain single-application. Do not work around this by teaching the framework about your app's containers: run composed stacks locally, and use the [Compose](deployer-compose.md) or [Kubernetes](deployer-k8s.md) deployer for uniform clusters. Details in [Backend Scope](app-backend-scope.md). diff --git a/book/src/topology-chaos.md b/book/src/topology-chaos.md deleted file mode 100644 index c4cb9ac..0000000 --- a/book/src/topology-chaos.md +++ /dev/null @@ -1,36 +0,0 @@ -# Topology & Chaos Patterns - -This page focuses on cluster manipulation: node control, chaos patterns, and -what the tooling supports today. - -## Node control availability -- **Supported**: restart control via `NodeControlHandle` (compose runner). -- **Not supported**: local runner does not expose node control; k8s runner does - not support it yet. -- **Not yet supported**: peer blocking/unblocking and network partitions. - -See also: [RunContext: BlockFeed & Node Control](node-control.md) for the current node-control API surface and limitations. - -## Chaos patterns to consider -- **Restarts**: random restarts with minimum delay/cooldown to test recovery. -- **Partitions (planned)**: block/unblock peers to simulate partial isolation, then assert - height convergence after healing. -- **Node churn (planned)**: stop one node and start another (new key) mid-run to - test membership changes; expect convergence. -- **Load SLOs**: push transaction rates and assert inclusion/latency budgets - instead of only liveness. -- **API probes**: poll HTTP/RPC endpoints during chaos to ensure external - contracts stay healthy (shape + latency). - -## Expectations to pair -- **Liveness/height convergence** after chaos windows. -- **SLO checks**: inclusion latency, API latency/shape. -- **Recovery checks**: ensure nodes that were isolated or restarted catch up to - cluster height within a timeout. - -## Guidance -- Keep chaos realistic: avoid flapping or patterns you wouldn't operate in prod. -- Scope chaos: choose nodes intentionally; don't restart all - nodes at once unless you're testing full outages. -- Combine chaos with observability: capture block feed/metrics and API health so - failures are diagnosable. diff --git a/book/src/topology.md b/book/src/topology.md new file mode 100644 index 0000000..69e0572 --- /dev/null +++ b/book/src/topology.md @@ -0,0 +1,112 @@ +# Topology and Deployment Plans + +This chapter explains how a scenario describes cluster shape and how a deployment provider turns that description into the concrete deployment the runner uses. + +--- + +## DeploymentDescriptor + +The core contract is defined in `testing-framework/core/src/topology/mod.rs`: + +```rust,ignore +pub trait DeploymentDescriptor: Send + Sync { + fn node_count(&self) -> usize; +} +``` + +Every `Application::Deployment` implements it. The scenario engine itself only needs the node count; everything richer (per-node configs, ids, network layout) belongs to the app's own deployment type and to the deployer that interprets it. + +--- + +## Built-in Topology Types + +The `topology` module ships a few concrete building blocks. Verify against `testing-framework/core/src/topology/`: + +| Type | File | What it is | +|---|---|---| +| `ClusterTopology` | `simple.rs` | Uniform cluster of `node_count` indexed nodes; `node_indices()` returns `[0..n)` | +| `DeploymentPlan` | `generated.rs` | Shape plus one `NodePlan` per node | +| `NodePlan` | `generated.rs` | `index`, a 32-byte `id`, and a `general` config value | +| `RuntimeTopology` | `generated.rs` | Runtime container of already-built node values | +| `SharedTopology` | `generated.rs` | Alias for `Arc` | +| `TopologyShapeBuilder` | `shape.rs` | Accumulates shape choices: `with_nodes(count)`, `with_star_network()`, read back via `node_count_or(fallback)` / `star_network_enabled()` | +| `DeploymentSeed` | `mod.rs` | 32-byte seed passed to providers (see [Seeds](seeds.md)) | + +Every example app that runs as a uniform cluster aliases `ClusterTopology`: + +```rust,ignore +pub type KvTopology = testing_framework_core::topology::ClusterTopology; + +let topology = KvTopology::new(3); // 3 nodes, indices 0..3 +``` + +`DeploymentPlan` and `NodePlan` implement `DeploymentDescriptor` too, for apps whose deployment must carry a prebuilt per-node config (`plans[i].general`) instead of deriving configs at spawn time. `TopologyShapeBuilder` and `DeploymentPlan` are available building blocks; the in-repo example apps currently build on `ClusterTopology` directly. + +--- + +## Deployment Providers + +A scenario does not have to hold a finished deployment. It holds a *provider*: + +```rust,ignore +pub trait DeploymentProvider: Send + Sync +where + D: DeploymentDescriptor, +{ + fn build(&self, seed: Option<&DeploymentSeed>) -> Result; +} +``` + +`FixedDeploymentProvider` wraps a concrete deployment and clones it on every `build`, ignoring the seed. A custom provider can generate the deployment lazily: sized from the environment, randomized from the seed, or derived from an external inventory. + +--- + +## Feeding the Builder + +`ScenarioBuilder` accepts a deployment in three ways (`core/src/scenario/definition/builder.rs`): + +| Method | Use when | +|---|---| +| `ScenarioBuilder::with_deployment(deployment)` | You already have the concrete value; wraps it in `FixedDeploymentProvider` | +| `ScenarioBuilder::new(provider)` | You start from a boxed `DeploymentProvider` | +| `with_deployment_provider(provider)` | Replace the provider, keeping all accumulated builder state | +| `map_deployment_provider(f)` | Transform the current provider (wrap, decorate) without losing state | +| `with_deployment_seed(seed)` | Store a `DeploymentSeed` handed to the provider at build time | + +Resolution happens once, inside `build()`: the builder calls `provider.build(seed)` and bakes the resulting deployment into the `Scenario`. Deployers and workloads then see a fixed descriptor for the rest of the run. + +```mermaid +graph LR + P[DeploymentProvider] -- "build(seed)" --> D[E::Deployment] + S[with_deployment_seed] -. optional .-> P + D --> SC["Scenario<E>"] + SC --> R[Deployer / Runner] + D:::cl + SC:::sc + classDef cl stroke:#4a90d9,stroke-width:2.5px; + classDef sc stroke:#9b6dd6,stroke-width:2.5px; +``` + +The typical example flow, from kvstore (`examples/kvstore/testing/integration/src/scenario.rs`): + +```rust,ignore +pub trait KvBuilderExt: Sized { + fn deployment_with(f: impl FnOnce(KvTopology) -> KvTopology) -> Self; +} + +impl KvBuilderExt for KvScenarioBuilder { + fn deployment_with(f: impl FnOnce(KvTopology) -> KvTopology) -> Self { + KvScenarioBuilder::with_deployment(f(KvTopology::new(3))) + } +} +``` + +`map_deployment_provider` and `with_deployment_provider` exist on all three builder forms (`ScenarioBuilder`, `NodeControlScenarioBuilder`, `ObservabilityScenarioBuilder`) and on the shared `CoreBuilderExt` used by app-specific builders. Wrapper builders can forward them through that shared extension. + +--- + +## What the Deployment Does Downstream + +- The **local deployer** reads `node_count()` and asks the environment to reserve ports and build one config per index; see [Ports, Peers, Node Config, and Readiness](node-config.md). +- The **container backends** iterate indices to produce per-node static artifacts delivered through cfgsync; see [Static Artifacts and cfgsync](cfgsync.md). +- **`ManualCluster`** treats the deployment as capacity: nodes are started on demand against the descriptor. See [ManualCluster](manual-cluster.md). diff --git a/book/src/troubleshooting.md b/book/src/troubleshooting.md index be18f5c..fdd0269 100644 --- a/book/src/troubleshooting.md +++ b/book/src/troubleshooting.md @@ -1,729 +1,78 @@ -# Troubleshooting Scenarios +# Troubleshooting -**Prerequisites for All Runners:** -- **`versions.env` file** at repository root (required by helper scripts) -- **Circuit assets** must be present and `LOGOS_BLOCKCHAIN_CIRCUITS` must point to a directory that contains them +This chapter collects common failure modes, the exact error text, and what to change. -**Platform/Environment Notes:** -- **macOS + Docker Desktop (Apple silicon):** prefer `LOGOS_BLOCKCHAIN_BUNDLE_DOCKER_PLATFORM=linux/arm64` for local compose/k8s runs to avoid slow/fragile amd64 emulation builds. -- **Disk space:** bundle/image builds are storage-heavy. If you see I/O errors or Docker build failures, check free space and prune old artifacts (`.tmp/`, `target/`, and Docker build cache) before retrying. -- **K8s runner scope:** the default Helm chart mounts circuit assets via `hostPath` and uses a local image tag (`logos-blockchain-testing:local`). This is intended for local clusters (Docker Desktop / minikube / kind), not remote managed clusters without additional setup. - - Quick cleanup: `scripts/ops/clean.sh` (and `scripts/ops/clean.sh --docker` if needed). - - Destructive cleanup (last resort): `scripts/ops/clean.sh --docker-system --dangerous` (add `--volumes` if you also want to prune Docker volumes). - -**Recommended:** Use `scripts/run/run-examples.sh` which handles all setup automatically. - -## Quick Symptom Guide - -Common symptoms and likely causes: - -- **Transactions not included**: unfunded or misconfigured wallets (check `.wallets(N)` vs `.users(M)`), transaction rate exceeding block capacity, or rates exceeding block production speed—reduce rate, increase wallet count, verify wallet setup in logs. -- **Chaos stalls the run**: chaos (node control) only works with ComposeDeployer; host runner (LocalDeployer) and K8sDeployer don't support it (won't "stall", just can't execute chaos workloads). With compose, aggressive restart cadence can prevent consensus recovery—widen restart intervals. -- **Observability gaps**: metrics or logs unreachable because ports clash or services are not exposed—adjust observability ports and confirm runner wiring. -- **Flaky behavior across runs**: mixing chaos with functional smoke tests or inconsistent topology between environments—separate deterministic and chaos scenarios and standardize topology presets. - -## What Failure Looks Like - -This section shows what you'll actually see when common issues occur. Each example includes realistic console output and the fix. - -### 1. Missing `versions.env` File - -**Symptoms:** -- Helper scripts fail immediately -- Error about missing file at repo root -- Scripts can't determine which circuit/node versions to use - -**What you'll see:** - -```text -$ scripts/run/run-examples.sh -t 60 -n 1 host -ERROR: versions.env not found at repository root -This file is required and should define: - VERSION= - LOGOS_BLOCKCHAIN_NODE_REV= - LOGOS_BLOCKCHAIN_BUNDLE_VERSION= -``` - -**Root Cause:** Helper scripts need `versions.env` to know which versions to build/fetch. - -**Fix:** Ensure you're in the repository root directory. The `versions.env` file should already exist—verify it's present: - -```bash -cat versions.env -# Should show: -# VERSION=v0.3.1 -# LOGOS_BLOCKCHAIN_NODE_REV=abc123def456 -# LOGOS_BLOCKCHAIN_BUNDLE_VERSION=v1 -``` +Every error message quoted here comes from an error type in the current source. When in doubt, preserve the run and read the generated configs first; see [Diagnostics and Retained Artifacts](diagnostics.md). --- -### 2. Missing Circuit Assets +## "duplicate runtime extension type registered: … AppRuntime" -**Symptoms:** -- Node startup fails early -- Error messages about missing circuit files +**Symptom:** scenario preparation fails immediately with this message (raised in `core/src/scenario/runtime/extensions.rs`). -**What you'll see:** +**Cause:** two `with_app(...)` calls on one scenario builder. Each `with_app` installs an `AppDeploymentFactory`, and every factory produces the same runtime extension type (`AppRuntime`); the second registration is rejected. The same error appears for any other runtime extension type registered twice. -```text -$ cargo run -p runner-examples --bin local_runner -[INFO testing_framework_runner_local] Starting local runner scenario -Error: circuit assets directory missing or invalid -thread 'main' panicked at 'workload init failed' -``` - -**Root Cause:** Circuit assets are required for proof-related paths. The runner expects `LOGOS_BLOCKCHAIN_CIRCUITS` to point to a directory containing the assets. - -**Fix (recommended):** - -```bash -# Use run-examples.sh which handles setup automatically -scripts/run/run-examples.sh -t 60 -n 1 host -``` - -**Fix (manual):** - -```bash -# Fetch circuits -scripts/setup/setup-logos-blockchain-circuits.sh v0.3.1 ~/.logos-blockchain-circuits - -# Set the environment variable -export LOGOS_BLOCKCHAIN_CIRCUITS=$HOME/.logos-blockchain-circuits -``` +**Fix:** a scenario has one `with_app`. To deploy several applications, compose them inside one root `AppDeployment` that deploys and exposes each child through the `DeployContext`, as the multi_app fixture's `JobStackApp` does; see [Composing Heterogeneous Stacks](composing-stacks.md). --- -### 3. Node Binaries Not Found +## Readiness Timeout on Deploy -**Symptoms:** -- Error about missing `logos-blockchain-node` binary -- "file not found" or "no such file or directory" -- Environment variables `LOGOS_BLOCKCHAIN_NODE_BIN` not set +**Symptom:** deploy fails with `readiness probe timed out: …` (`ReadinessError::ProbeTimeout`), or `cluster stabilization timed out after …`. The processes may have spawned; they just never answered. -**What you'll see:** +**Causes, in observed order of likelihood:** -```text -$ cargo run -p runner-examples --bin local_runner -[INFO testing_framework_runner_local] Spawning node 0 -Error: Os { code: 2, kind: NotFound, message: "No such file or directory" } -thread 'main' panicked at 'failed to spawn logos-blockchain-node process' -``` +1. **Wrong binary.** The binary env var points at a stale or wrong executable, so the process starts and exits (or listens on nothing). Check the interleaved process output for an immediate crash. +2. **Wrong readiness path.** The HTTP probe hits `Application::node_readiness_path()` (default `/`). If your node serves health on `/health` and you did not override the path, the probe 404s forever; see [Ports, Peers, Node Config, and Readiness](node-config.md). +3. **Port conflicts.** Local ports are preallocated by binding port 0, but another process can grab a port between reservation and spawn, or the node config may hardcode a busy port. Preserve the run and check the ports in the rendered `config.yaml`. +4. **Slow machine.** On loaded CI runners, set `SLOW_TEST_ENV=true` to double timeouts, or attach a `RetryPolicy` / relax the requirement to `HttpReadinessRequirement::AnyNodeReady` via [deployment policies](deployment-policies.md). -**Root Cause:** The local runner needs compiled `logos-blockchain-node` binaries, but doesn't know where they are. - -**Fix (recommended):** - -```bash -# Use run-examples.sh which builds binaries automatically -scripts/run/run-examples.sh -t 60 -n 1 host -``` - -**Fix (manual - set paths explicitly):** - -```bash -# Build binaries first -cd ../logos-blockchain-node # or wherever your logos-blockchain-node checkout is -cargo build --release --bin logos-blockchain-node - -# Set environment variables -export LOGOS_BLOCKCHAIN_NODE_BIN=$PWD/target/release/logos-blockchain-node - -# Return to testing framework -cd ../nomos-testing -cargo run -p runner-examples --bin local_runner -``` +For a `LocalProcessApp` with `.with_readiness(...)`, a readiness failure stops the process and fails the deployment with your closure's error, and the same diagnosis applies. --- -### 4. Docker Daemon Not Running (Compose) +## Binary Resolution Failures -**Symptoms:** -- Compose tests fail immediately -- "Cannot connect to Docker daemon" -- Docker commands don't work +All variants live in `BinaryProviderError` (`deployers/local/src/binary/types.rs`); see [Binary Providers](binary-providers.md). -**What you'll see:** - -```text -$ scripts/run/run-examples.sh -t 60 -n 1 compose -[INFO runner_examples::compose_runner] Starting compose deployment -Error: Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running? -thread 'main' panicked at 'compose deployment failed' -``` - -**Root Cause:** Docker Desktop isn't running, or your user doesn't have permission to access Docker. - -**Fix:** - -```bash -# macOS: Start Docker Desktop application -open -a Docker - -# Linux: Start Docker daemon -sudo systemctl start docker - -# Verify Docker is working -docker ps - -# If permission denied, add your user to docker group (Linux) -sudo usermod -aG docker $USER -# Then log out and log back in -``` +| Message | Meaning | Fix | +|---|---|---| +| `binary could not be resolved by provider …` | `NotFound` — no provider in the chain produced a path. For a bare `EnvBinaryProvider` this means the env var is unset **or does not point at an existing file** | set the variable to a real executable path, or add a build/download fallback | +| `build command failed with status …` | `BuildFailed` — the `BuildCommand` exited non-zero | run the command by hand from the provider's `working_dir` | +| `build command did not produce configured binary output …` | `MissingBuildOutput` — build succeeded but `output_path` is missing | fix the `output_path` (profile/target dir mismatch is typical) | +| `download provider requires env var … to contain a binary URL` | `MissingDownloadUrl` — `DownloadUrl::Env` variable unset | export the URL variable | +| `failed to download binary from …` | `Download` — HTTP failure | check URL and network | +| `downloaded binary sha256 mismatch for …: expected …, got …` | `ChecksumMismatch` — bytes did not match the pinned SHA-256 | update the pinned checksum or investigate the source | +| `download processor … failed` / `… did not produce binary output …` | processor error after a verified download | debug the `DownloadProcessor` (archive layout changed?) | +| `binary path must be absolute: …` | `RelativePath` — `PathBinaryProvider` got a relative path | pass an absolute path | +| `timed out waiting for binary provider lock …` | `LockTimeout` — another process held the cross-process lock for over 10 minutes | if no other test run is alive, a crashed process left a stale lock file (under `.tf-binaries` / `target/.tf-binaries`); delete it | --- -### 5. Image Not Found (Compose/K8s) +## Docker and Compose -**Symptoms:** -- Compose/K8s tests fail during deployment -- "Image not found: logos-blockchain-testing:local" -- Containers fail to start +**`docker does not appear to be available on this host`** (`ComposeRunnerError::DockerUnavailable`): the runner probes `docker info` before deploying. Start the Docker daemon. The example binaries treat this as a graceful skip; your CI should probably not (see [Continuous Integration](ci.md)). -**What you'll see:** +**`docker image '' is not available; build or load it locally`** (`MissingImage`): the deployer checks every node image with `docker image inspect` and never builds or pulls. Build the app image (e.g. `docker build -f examples/kvstore/Dockerfile -t kvstore-node:local .`) or `docker pull` the upstream one, or point the `_IMAGE` variable at an image you have. -```text -$ cargo run -p runner-examples --bin compose_runner -[INFO testing_framework_runner_compose] Starting compose deployment -Error: Failed to pull image 'logos-blockchain-testing:local': No such image -thread 'main' panicked at 'compose deployment failed' -``` +**`docker compose up exited with status …` / `… timed out after …`** (`ComposeCommandError`): the stack itself failed to start. Re-run with `COMPOSE_RUNNER_PRESERVE=1` and inspect the preserved workspace and `docker compose logs` for the project. -**Root Cause:** The Docker image hasn't been built yet, or was pruned. - -**Fix (recommended):** - -```bash -# Use run-examples.sh which builds the image automatically -scripts/run/run-examples.sh -t 60 -n 1 compose -``` - -**Fix (manual):** - -```bash -# 1. Build Linux bundle -scripts/build/build-bundle.sh --platform linux - -# 2. Set bundle path -export LOGOS_BLOCKCHAIN_BINARIES_TAR=$(ls -t .tmp/nomos-binaries-linux-*.tar.gz | head -1) - -# 3. Build Docker image -scripts/build/build_test_image.sh - -# 4. Verify image exists -docker images | grep logos-blockchain-testing - -# 5. For kind/minikube: load image into cluster -kind load docker-image logos-blockchain-testing:local -# OR: minikube image load logos-blockchain-testing:local -``` +For Kubernetes, an unreachable cluster surfaces as `K8sRunnerError::ClientInit` at deploy time; `scripts/run/checks.sh` diagnoses context, Helm, and image visibility (a `:local` tag is not visible inside `kind`/`minikube` without loading it). --- -### 6. Port Conflicts +## App Handles -**Symptoms:** -- "Address already in use" errors -- Tests fail during node startup -- Observability stack (Prometheus/Grafana) won't start +**`app handle is not exposed: [named "…"]`** (`AppDeployError::HandleMissing`): a workload called `require_app::()` (or a deployment called `require`) for a handle that was not exposed. `ctx.deploy(app)` returns a handle without exposing it, which allows intermediate handles. Use `deploy_and_expose`, or call `ctx.expose(handle)` explicitly. Only the root deployment's own handle is auto-exposed, and only when nothing of that type was exposed already. For named lookups, the name must match the `expose_named` string exactly. See [AppDeployment and DeployContext](app-deployment.md). -**What you'll see:** - -```text -$ cargo run -p runner-examples --bin local_runner -[INFO testing_framework_runner_local] Launching node 0 on port 18080 -Error: Os { code: 48, kind: AddrInUse, message: "Address already in use" } -thread 'main' panicked at 'failed to bind port 18080' -``` - -**Root Cause:** Previous test didn't clean up properly, or another service is using the port. - -**Fix:** - -```bash -# Find processes using the port -lsof -i :18080 # macOS/Linux -netstat -ano | findstr :18080 # Windows - -# Kill orphaned nomos processes -pkill logos-blockchain-node - -# For compose: ensure containers are stopped -docker compose down -docker ps -a --filter "name=nomos-compose-" -q | xargs docker rm -f - -# Check if port is now free -lsof -i :18080 # Should return nothing -``` - -**For Observability Stack Port Conflicts:** - -```bash -# Edit ports in observability compose file -vim scripts/observability/compose/docker-compose.yml - -# Change conflicting port mappings: -# ports: -# - "9090:9090" # Prometheus - change to "19090:9090" if needed -# - "3000:3000" # Grafana - change to "13000:3000" if needed -``` +**`app handle is already exposed: [named "…"]`** (`AppDeployError::DuplicateHandle`): one unnamed handle per concrete type. Duplicate exposure is always an error, never a silent replacement. For two instances of the same type (two kvstore clusters), expose each under a distinct name with `expose_named`, and fetch with `require_app_named`. --- -### 7. Wallet Seeding Failed (Insufficient Funds) +## Teardown Surprises -**Symptoms:** -- Transaction workload reports wallet issues -- "Insufficient funds" errors -- Transactions aren't being submitted +App-layer resources acquired through framework adapters are owned by scenario cleanup, not by handle clone counts. If a managed process or cluster survives a test, check whether custom deployment code started it outside `LocalProcessApp` or `deploy_cluster`, or whether backend cleanup logged a failure. Managed app cleanup runs in reverse acquisition order; see [Handle Ownership and Teardown](handles-teardown.md). -**What you'll see:** +Backend cleanup failures do not fail an otherwise green run: the compose and k8s deployers log them as `warn!` events with context fields (e.g. `docker compose down failed`, `helm uninstall failed during cleanup` with `release` and `namespace`). If containers or namespaces accumulate, scan your logs for those warnings and clean up manually. -```text -$ cargo run -p runner-examples --bin local_runner -[INFO testing_framework_workflows] Starting transaction workload with 10 users -[ERROR testing_framework_workflows] Wallet seeding failed: requested 10 users but only 3 wallets available -thread 'main' panicked at 'workload init failed: insufficient wallets' -``` - -**Root Cause:** Topology configured fewer wallets than the workload needs. Transaction workload has `.users(M)` but topology only has `.wallets(N)` where N < M. - -**Fix:** - -```rust,ignore -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -let scenario = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .wallets(20) // ← Increase wallet count - .transactions_with(|tx| { - tx.users(10) // ← Must be ≤ wallets(20) - .rate(5) - }) - .build(); -``` - ---- - -### 8. Resource Exhaustion (OOM / CPU) - -**Symptoms:** -- Nodes crash randomly -- "OOM Killed" messages -- Test becomes flaky under load -- Docker containers restart repeatedly - -**What you'll see:** - -```text -$ docker ps --filter "name=nomos-compose-" -CONTAINER ID STATUS -abc123def456 Restarting (137) 30 seconds ago # 137 = OOM killed - -$ docker logs abc123def456 -[INFO nomos_node] Starting node -[INFO consensus] Processing block -Killed # ← OOM killer terminated the process -``` - -**Root Cause:** Too many nodes, too much workload traffic, or insufficient Docker resources. - -**Fix:** - -```bash -# 1. Reduce topology size -# In your scenario: -# .topology(Topology::preset_3v1e()) # Instead of preset_10v2e() - -# 2. Reduce workload rates -# .workload(TransactionWorkload::new().rate(5.0)) # Instead of rate(100.0) - -# 3. Increase Docker resources (Docker Desktop) -# Settings → Resources → Memory: 8GB minimum (12GB+ recommended for large topologies) -# Settings → Resources → CPUs: 4+ cores recommended - -# 4. Increase file descriptor limits (Linux/macOS) -ulimit -n 4096 - -# 5. Close other heavy applications (browsers, IDEs, etc.) -``` - ---- - -### 9. Logs Disappear After Run - -**Symptoms:** -- Test completes but no logs on disk -- Can't debug failures because logs are gone -- Temporary directories cleaned up automatically - -**What you'll see:** - -```text -$ cargo run -p runner-examples --bin local_runner -[INFO runner_examples] Test complete, cleaning up -[INFO testing_framework_runner_local] Removing temporary directories -$ ls .tmp/ -# Empty or missing -``` - -**Root Cause:** Framework cleans up temporary directories by default to avoid disk bloat. - -**Fix:** - -```bash -# Persist logs to a specific directory -LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/test-logs \ -LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1 \ -cargo run -p runner-examples --bin local_runner - -# Logs persist after run -ls /tmp/test-logs/ -# logos-blockchain-node-0.2024-12-18T14-30-00.log -# logos-blockchain-node-1.2024-12-18T14-30-00.log -# ... -``` - ---- - -### 10. Consensus Timing Too Tight / Run Duration Too Short - -**Symptoms:** -- "Consensus liveness expectation failed" -- Only 1-2 blocks produced (or zero) -- Nodes appear healthy but not making progress - -**What you'll see:** - -```text -$ cargo run -p runner-examples --bin local_runner -[INFO testing_framework_core] Starting workloads -[INFO testing_framework_core] Run window: 10 seconds -[INFO testing_framework_core] Evaluating expectations -[ERROR testing_framework_core] Consensus liveness expectation failed: expected min 5 blocks, got 1 -thread 'main' panicked at 'expectations failed' -``` - -**Root Cause:** Run duration too short for consensus parameters. If `CONSENSUS_SLOT_TIME=20s` but run duration is only `10s`, you can't produce many blocks. - -**Fix:** - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::ScenarioBuilderExt; - -// Increase run duration to allow more blocks. -let scenario = ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(120)) // ← Give more time - .build(); -``` - -**Or adjust consensus timing (if you control node config):** - -```bash -# Faster block production (shorter slot time) -CONSENSUS_SLOT_TIME=5 \ -CONSENSUS_ACTIVE_SLOT_COEFF=0.9 \ -cargo run -p runner-examples --bin local_runner -``` - ---- - -## Summary: Quick Checklist for Failed Runs - -When a test fails, check these in order: - -1. **`versions.env` exists at repo root** -2. **Circuit assets present** (`LOGOS_BLOCKCHAIN_CIRCUITS` points to a valid directory) -3. **Node binaries available** (`LOGOS_BLOCKCHAIN_NODE_BIN` set, or using `run-examples.sh`) -4. **Docker daemon running** (for compose/k8s) -5. **Docker image built** (`logos-blockchain-testing:local` exists for compose/k8s) -6. **No port conflicts** (`lsof -i :18080`, kill orphaned processes) -7. **Sufficient wallets** (`.wallets(N)` ≥ `.users(M)`) -8. **Enough resources** (Docker memory 8GB+, ulimit -n 4096) -9. **Run duration appropriate** (long enough for consensus timing) -10. **Logs persisted** (`LOGOS_BLOCKCHAIN_LOG_DIR` + `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1` if needed) - -**Still stuck?** Check node logs (see [Where to Find Logs](#where-to-find-logs)) for the actual error. - -## Where to Find Logs - -### Log Location Quick Reference - -| Runner | Default Output | With `LOGOS_BLOCKCHAIN_LOG_DIR` + Flags | Access Command | -|--------|---------------|------------------------------|----------------| -| **Host** (local) | Per-run temporary directories under the current working directory (removed unless `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1`) | Per-node files with prefix `logos-blockchain-node-{index}` (set `LOGOS_BLOCKCHAIN_LOG_DIR`) | `cat $LOGOS_BLOCKCHAIN_LOG_DIR/logos-blockchain-node-0*` | -| **Compose** | Docker container stdout/stderr | Set `tracing_settings.logger: !File` in `testing-framework/assets/stack/cfgsync.yaml` (and mount a writable directory) | `docker ps` then `docker logs ` | -| **K8s** | Pod stdout/stderr | Set `tracing_settings.logger: !File` in `testing-framework/assets/stack/cfgsync.yaml` (and mount a writable directory) | `kubectl logs -l nomos/logical-role=node` | - -**Important Notes:** -- **Host runner** (local processes): Per-run temporary directories are created under the current working directory and removed after the run unless `LOGOS_BLOCKCHAIN_TESTS_KEEP_LOGS=1`. To write per-node log files to a stable location, set `LOGOS_BLOCKCHAIN_LOG_DIR=/path/to/logs`. -- **Compose/K8s**: Node log destination is controlled by `testing-framework/assets/stack/cfgsync.yaml` (`tracing_settings.logger`). By default, rely on `docker logs` or `kubectl logs`. -- **File naming**: Log files use prefix `logos-blockchain-node-{index}*` with timestamps, e.g., `logos-blockchain-node-0.2024-12-01T10-30-45.log` (NOT just `.log` suffix). -- **Container names**: Compose containers include project UUID, e.g., `nomos-compose--node-0-1` where `` is randomly generated per run - -### Accessing Node Logs by Runner - -#### Local Runner - -**Console output (default):** -```bash -cargo run -p runner-examples --bin local_runner 2>&1 | tee test.log -``` - -**Persistent file output:** -```bash -LOGOS_BLOCKCHAIN_LOG_DIR=/tmp/debug-logs \ -LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \ -cargo run -p runner-examples --bin local_runner - -# Inspect logs (note: filenames include timestamps): -ls /tmp/debug-logs/ -# Example: logos-blockchain-node-0.2024-12-01T10-30-45.log -tail -f /tmp/debug-logs/logos-blockchain-node-0* # Use wildcard to match timestamp -``` - -#### Compose Runner - -**Stream live logs:** -```bash -# List running containers (note the UUID prefix in names) -docker ps --filter "name=nomos-compose-" - -# Find your container ID or name from the list, then: -docker logs -f - -# Or filter by name pattern: -docker logs -f $(docker ps --filter "name=nomos-compose-.*-node-0" -q | head -1) - -# Show last 100 lines -docker logs --tail 100 -``` - -**Keep containers for post-mortem debugging:** -```bash -COMPOSE_RUNNER_PRESERVE=1 \ -LOGOS_BLOCKCHAIN_TESTNET_IMAGE=logos-blockchain-testing:local \ -cargo run -p runner-examples --bin compose_runner - -# OR: Use run-examples.sh (handles setup automatically) -COMPOSE_RUNNER_PRESERVE=1 scripts/run/run-examples.sh -t 60 -n 1 compose - -# After test failure, containers remain running: -docker ps --filter "name=nomos-compose-" -docker exec -it /bin/sh -docker logs > debug.log -``` - -**Note:** Container names follow the pattern `nomos-compose-{uuid}-node-{index}-1`, where `{uuid}` is randomly generated per run. - -#### K8s Runner - -**Important:** Always verify your namespace and use label selectors instead of assuming pod names. - -**Stream pod logs (use label selectors):** - -```bash -# Check your namespace first -kubectl config view --minify | grep namespace - -# All node pods (add -n if not using default) -kubectl logs -l nomos/logical-role=node -f - -# Specific pod by name (find exact name first) -kubectl get pods -l nomos/logical-role=node # Find the exact pod name -kubectl logs -f # Then use it - -# With explicit namespace -kubectl logs -n my-namespace -l nomos/logical-role=node -f -``` - -**Download logs from crashed pods:** - -```bash -# Previous logs from crashed pod -kubectl get pods -l nomos/logical-role=node # Find crashed pod name first -kubectl logs --previous > crashed-node.log - -# Or use label selector for all crashed nodes -for pod in $(kubectl get pods -l nomos/logical-role=node -o name); do - kubectl logs --previous $pod > $(basename $pod)-previous.log 2>&1 -done -``` - -**Access logs from all pods:** - -```bash -# All pods in current namespace -for pod in $(kubectl get pods -o name); do - echo "=== $pod ===" - kubectl logs $pod -done > all-logs.txt - -# Or use label selectors (recommended) -kubectl logs -l nomos/logical-role=node --tail=500 > nodes.log - -# With explicit namespace -kubectl logs -n my-namespace -l nomos/logical-role=node --tail=500 > nodes.log -``` - -## Debugging Workflow - -When a test fails, follow this sequence: - -### 1. Check Framework Output - -Start with the test harness output—did expectations fail? Was there a deployment error? - -**Look for:** - -- Expectation failure messages -- Timeout errors -- Deployment/readiness failures - -### 2. Verify Node Readiness - -Ensure all nodes started successfully and became ready before workloads began. - -**Commands:** - -```bash -# Local: check process list -ps aux | grep nomos - -# Compose: check container status (note UUID in names) -docker ps -a --filter "name=nomos-compose-" - -# K8s: check pod status (use label selectors, add -n if needed) -kubectl get pods -l nomos/logical-role=node -kubectl describe pod # Get name from above first -``` - -### 3. Inspect Node Logs - -Focus on the first node that exhibited problems or the node with the highest index (often the last to start). - -**Common error patterns:** - -- "ERROR: versions.env missing" → missing required `versions.env` file at repository root -- "Failed to bind address" → port conflict -- "Connection refused" → peer not ready or network issue -- "Circuit file not found" → missing circuit assets at the path in `LOGOS_BLOCKCHAIN_CIRCUITS` -- "Insufficient funds" → wallet seeding issue (increase `.wallets(N)` or reduce `.users(M)`) - -### 4. Check Log Levels - -If logs are too sparse, increase verbosity: - -```bash -LOGOS_BLOCKCHAIN_LOG_LEVEL=debug \ -LOGOS_BLOCKCHAIN_LOG_FILTER="cryptarchia=trace" \ -cargo run -p runner-examples --bin local_runner -``` - -If metric updates are polluting your logs (fields like `counter.*` / `gauge.*`), move those events to a dedicated `tracing` target (e.g. `target: "nomos_metrics"`) and set `LOGOS_BLOCKCHAIN_LOG_FILTER="nomos_metrics=off,..."` so they don’t get formatted into log output. - -### 5. Verify Observability Endpoints - -If expectations report observability issues: - -**Prometheus (Compose):** -```bash -curl http://localhost:9090/-/healthy -``` - -**Node HTTP APIs:** -```bash -curl http://localhost:18080/consensus/info # Adjust port per node -``` - -### 6. Compare with Known-Good Scenario - -Run a minimal baseline test (e.g., 2 nodes, consensus liveness only). If it passes, the issue is in your workload or topology configuration. - -## Common Error Messages - -### "Consensus liveness expectation failed" - -- **Cause**: Not enough blocks produced during the run window, missing circuit - assets. -- **Fix**: - 1. Verify circuit assets exist at the path referenced by - `LOGOS_BLOCKCHAIN_CIRCUITS`. - 2. Extend `with_run_duration()` to allow more blocks. - 3. Check node logs for proof generation or circuit asset errors. - 4. Reduce transaction rate if nodes are overwhelmed. - -### "Wallet seeding failed" - -- **Cause**: Topology doesn't have enough funded wallets for the workload. -- **Fix**: Increase `.wallets(N)` count or reduce `.users(M)` in the transaction - workload (ensure N ≥ M). - -### "Node control not available" - -- **Cause**: Runner doesn't support node control (only ComposeDeployer does), or - `enable_node_control()` wasn't called. -- **Fix**: - 1. Use ComposeDeployer for chaos tests (LocalDeployer and K8sDeployer don't - support node control). - 2. Ensure `.enable_node_control()` is called in the scenario before `.chaos()`. - -### "Readiness timeout" - -- **Cause**: Nodes didn't become responsive within expected time (often due to - missing prerequisites). -- **Fix**: - it, proof generation is too slow). - 1. Check node logs for startup errors (port conflicts, missing assets). - 2. Verify network connectivity between nodes. - 3. Ensure circuit assets are present and `LOGOS_BLOCKCHAIN_CIRCUITS` points to them. - -### "ERROR: versions.env missing" - -- **Cause**: Helper scripts (`run-examples.sh`, `build-bundle.sh`, `setup-logos-blockchain-circuits.sh`) require `versions.env` file at repository root. -- **Fix**: Ensure you're running from the repository root directory. The `versions.env` file should already exist and contains: -```text - VERSION= - LOGOS_BLOCKCHAIN_NODE_REV= - LOGOS_BLOCKCHAIN_BUNDLE_VERSION= - ``` - Use the checked-in `versions.env` at the repository root as the source of truth. - -### "Port already in use" - -- **Cause**: Previous test didn't clean up, or another process holds the port. -- **Fix**: Kill orphaned processes (`pkill logos-blockchain-node`), wait for Docker cleanup - (`docker compose down`), or restart Docker. - -### "Image not found: logos-blockchain-testing:local" - -- **Cause**: Docker image not built for Compose/K8s runners, or circuit assets not - baked into the image. -- **Fix (recommended)**: Use run-examples.sh which handles everything: - ```bash - scripts/run/run-examples.sh -t 60 -n 1 compose - ``` -- **Fix (manual)**: - 1. Build bundle: `scripts/build/build-bundle.sh --platform linux` - 2. Set bundle path: `export LOGOS_BLOCKCHAIN_BINARIES_TAR=.tmp/nomos-binaries-linux-v0.3.1.tar.gz` - 3. Build image: `scripts/build/build_test_image.sh` - 4. **kind/minikube:** load the image into the cluster nodes (e.g. `kind load docker-image logos-blockchain-testing:local`, or `minikube image load ...`), or push to a registry and set `LOGOS_BLOCKCHAIN_TESTNET_IMAGE` accordingly. - -### "Circuit file not found" - -- **Cause**: Circuit assets are missing or `LOGOS_BLOCKCHAIN_CIRCUITS` points to a non-existent directory. Inside containers, assets are expected at `/opt/circuits`. -- **Fix (recommended)**: Use run-examples.sh which handles setup: - ```bash - scripts/run/run-examples.sh -t 60 -n 1 - ``` -- **Fix (manual)**: - 1. Fetch assets: `scripts/setup/setup-logos-blockchain-circuits.sh v0.3.1 ~/.logos-blockchain-circuits` - 2. Set `LOGOS_BLOCKCHAIN_CIRCUITS=$HOME/.logos-blockchain-circuits` - 3. Verify directory exists: `ls -lh $LOGOS_BLOCKCHAIN_CIRCUITS` - 4. For Compose/K8s: rebuild image with assets baked in - -For detailed logging configuration and observability setup, see [Logging & Observability](logging-observability.md). +When preservation is enabled, nodes or their directories remain after teardown. If they accumulate, check `TF_KEEP_LOGS`, `COMPOSE_RUNNER_PRESERVE`, and `K8S_RUNNER_PRESERVE` in your shell; `scripts/run/checks.sh` prints their current values. diff --git a/book/src/usage-patterns.md b/book/src/usage-patterns.md deleted file mode 100644 index 76dddba..0000000 --- a/book/src/usage-patterns.md +++ /dev/null @@ -1,16 +0,0 @@ -# Usage Patterns - -- **Shape a topology, pick a runner**: choose local for quick iteration, compose - for reproducible multi-node stacks with observability, or k8s for cluster-grade - validation. -- **Compose workloads deliberately**: pair transactions and data-availability - traffic for end-to-end coverage; add chaos only when assessing recovery and - resilience. -- **Align expectations with goals**: use liveness-style checks to confirm the - system keeps up with planned activity, and add workload-specific assertions for - inclusion or availability. -- **Reuse plans across environments**: keep the scenario constant while swapping - runners to compare behavior between developer machines and CI clusters. -- **Iterate with clear signals**: treat expectation outcomes as the primary - pass/fail indicator, and adjust topology or workloads based on what those - signals reveal. diff --git a/book/src/verb-layer.md b/book/src/verb-layer.md new file mode 100644 index 0000000..56f1b74 --- /dev/null +++ b/book/src/verb-layer.md @@ -0,0 +1,110 @@ +# The Verb Layer + +The verb layer provides optional, domain-specific helpers for recurring test actions. It uses the same scenario builder, workloads, expectations, and capabilities described in the preceding chapters. + +--- + +## Two Equivalent Levels + +The explicit API names the objects being assembled: + +```rust,ignore +let scenario = QueueScenarioBuilder::with_deployment(QueueTopology::new(5)) + .with_node_control() + .with_network_control() + .with_workload(QueueProduceWorkload::new().operations(400).rate_per_sec(40)) + .with_workload(RandomRestartWorkload::new( + Duration::from_secs(5), + Duration::from_secs(15), + Duration::from_secs(15), + )) + .with_workload(NetworkPartitionWorkload::new( + NetworkPartitionSpec::new([ + vec!["node-0", "node-1"], + vec!["node-2", "node-3", "node-4"], + ]), + Duration::from_secs(20), + Duration::from_secs(60), + )) + .with_expectation(QueueConverges::new(400).timeout(Duration::from_secs(60))) + .with_run_duration(Duration::from_secs(120)) + .build()?; +``` + +The verb API lowers to those same operations: + +```rust,ignore +QueueScenario::nodes(5) + .produce(400).rate_per_sec(40).done() + .restart_nodes_randomly().every_secs(5, 15).done() + .partition(["node-0", "node-1"], ["node-2", "node-3", "node-4"]) + .hold_secs(20).done() + .expect_converged(400).within_secs(60) + .run_secs(120) + .await?; +``` + +The explicit API remains available at every point. Use it for one-off workloads, unusual policies, or operations that do not have a domain verb. + +--- + +## How Verbs Map to the Builder + +A verb does not introduce a second runtime. Its sub-builder stores an ordinary workload or expectation and adds it when `done()` or a terminal method is called: + +```rust,ignore +pub trait QueueDslExt: CoreBuilderAccess + Sized { + fn produce(self, operations: usize) -> QueueProduceBuilder { + QueueProduceBuilder { + builder: self, + workload: QueueProduceWorkload::new().operations(operations), + } + } +} + +impl> QueueProduceBuilder { + pub fn done(self) -> B { + self.builder.map_core_builder(|builder| { + builder.with_workload(self.workload) + }) + } +} +``` + +Both forms therefore use the same execution, failure aggregation, and teardown. Generic and application-specific verbs can extend the same builder chain. + +--- + +## Capability-Aware Verbs + +Some actions require a runtime capability. The verb should request it when the requirement follows directly from the action: + +- `restart_nodes_randomly()` transitions a plain builder to a node-control builder. +- `partition(...).done()` requests network control before adding the partition workload. +- A data-plane verb such as `produce(...)` needs no control capability. + +The resulting Rust type records the capability transition. If a deployer cannot supply the requested capability, deployment fails before the workload starts. + +Do not hide an unrelated policy choice inside a verb. A verb may request what its action necessarily needs; retry policy, cleanup policy, backend selection, and other test-wide decisions remain explicit. + +--- + +## Designing Application Verbs + +Put vocabulary shared by applications in the framework and vocabulary specific to one protocol in that application's testing crate. A verb should: + +1. names an operation in the application's domain; +2. configures one workload or expectation, or a small fixed combination; +3. exposes meaningful options through a short sub-builder; +4. returns the underlying builder through `done()` or a clear terminal method; +5. preserves access to `with_workload` and `with_expectation` for uncommon cases. + +The queue example keeps `partition` and random restarts generic, while `produce` and `expect_converged` live with the queue integration. Other applications can then reuse chaos behavior without depending on queue terminology. + +--- + +## See Also + +- [Workloads and Concurrency](workloads.md) and [Expectations and Evaluation](expectations.md): the objects verbs add. +- [Scenario Capabilities](capabilities.md): the requirements capability-aware verbs request. +- [Chaos and Controlled Failure](chaos.md): the generic restart and partition workloads. diff --git a/book/src/what-you-will-learn.md b/book/src/what-you-will-learn.md deleted file mode 100644 index 044c82e..0000000 --- a/book/src/what-you-will-learn.md +++ /dev/null @@ -1,63 +0,0 @@ -# What You Will Learn - -This book gives you a clear mental model for Logos multi-node testing, shows how -to author scenarios that pair realistic workloads with explicit expectations, -and guides you to run them across local, containerized, and cluster environments -without changing the plan. - -## By the End of This Book, You Will Be Able To: - -**Understand the Framework** -- Explain the six-phase scenario lifecycle (Build, Deploy, Capture, Execute, Evaluate, Cleanup) -- Describe how Deployers, Runners, Workloads, and Expectations work together -- Navigate the crate architecture and identify extension points -- Understand when to use each runner (Host, Compose, Kubernetes) - -**Author and Run Scenarios** -- Define multi-node topologies with nodes -- Configure transaction workloads with appropriate rates -- Add consensus liveness and inclusion expectations -- Run scenarios across all three deployment modes -- Use BlockFeed to monitor block production in real-time -- Implement chaos testing with node restarts - -**Operate in Production** -- Set up prerequisites and dependencies correctly -- Configure environment variables for different runners -- Integrate tests into CI/CD pipelines (GitHub Actions) -- Troubleshoot common failure scenarios -- Collect and analyze logs from multi-node runs -- Optimize test durations and resource usage - -**Extend the Framework** -- Implement custom Workload traits for new traffic patterns -- Create custom Expectation traits for domain-specific checks -- Add new Deployer implementations for different backends -- Contribute topology helpers and DSL extensions - -## Learning Path - -**Beginner** (0-2 hours) -- Read [Quickstart](quickstart.md) and run your first scenario -- Review [Examples](examples.md) to see common patterns -- Understand [Scenario Lifecycle](scenario-lifecycle.md) phases - -**Intermediate** (2-8 hours) -- Study [Runners](runners.md) comparison and choose appropriate mode -- Learn [Workloads & Expectations](workloads.md) in depth -- Review [Prerequisites & Setup](prerequisites.md) for your environment -- Practice with [Advanced Examples](examples-advanced.md) - -**Advanced** (8+ hours) -- Master [Environment Variables](environment-variables.md) configuration -- Implement [Custom Workloads](extending.md) for your use cases -- Set up [CI Integration](ci-integration.md) for automated testing -- Explore [Internal Crate Reference](internal-crate-reference.md) for deep dives - -## What This Book Does NOT Cover - -- **Logos node internals** — This book focuses on testing infrastructure, not the blockchain protocol implementation. See the Logos node repository (`logos-blockchain-node`) for protocol documentation. -- **Consensus algorithm theory** — We assume familiarity with basic blockchain concepts (nodes, blocks, transactions). -- **Rust language basics** — Examples use Rust, but we don't teach the language. See [The Rust Book](https://doc.rust-lang.org/book/) if you're new to Rust. -- **Kubernetes administration** — We show how to use the K8s runner, but don't cover cluster setup, networking, or operations. -- **Docker fundamentals** — We assume basic Docker/Compose knowledge for the Compose runner. diff --git a/book/src/workloads.md b/book/src/workloads.md index 5712ca3..e4891f4 100644 --- a/book/src/workloads.md +++ b/book/src/workloads.md @@ -1,391 +1,161 @@ -# Core Content: Workloads & Expectations +# Workloads and Concurrency -Workloads describe the activity a scenario generates; expectations describe the signals that must hold when that activity completes. This page is the **canonical reference** for all built-in workloads and expectations, including configuration knobs, defaults, prerequisites, and debugging guidance. +Workloads describe the activity a scenario generates: every workload runs as its own concurrent task against the shared `RunContext`, and the runner decides when the run window ends. --- -## Overview +## The Workload Trait -```mermaid -flowchart TD - I[Inputs
topology + wallets + rates] --> Init[Workload init] - Init --> Drive[Drive traffic] - Drive --> Collect[Collect signals] - Collect --> Eval[Expectations evaluate] -``` - -**Key concepts:** -- **Workloads** run during the **execution phase** (generate traffic) -- **Expectations** run during the **evaluation phase** (check health signals) -- Each workload can attach its own expectations automatically -- Expectations can also be added explicitly - ---- - -## Built-in Workloads - -### 1. Transaction Workload - -Submits user-level transactions at a configurable rate to exercise transaction processing and inclusion paths. - -**Import:** -```rust,ignore -use testing_framework_workflows::workloads::transaction::Workload; -``` - -#### Configuration - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `rate` | `u64` | **Required** | Transactions per block (not per second!) | -| `users` | `Option` | All wallets | Number of distinct wallet accounts to use | - -#### DSL Usage +A workload is any type implementing `Workload` from `testing-framework-core` (`testing-framework/core/src/scenario/workload.rs`): ```rust,ignore -use testing_framework_workflows::ScenarioBuilderExt; - -ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .wallets(20) // Seed 20 wallet accounts - .transactions_with(|tx| { - tx.rate(10) // 10 transactions per block - .users(5) // Use only 5 of the 20 wallets - }) - .with_run_duration(Duration::from_secs(60)) - .build(); -``` - -#### Direct Instantiation - -```rust,ignore -use testing_framework_workflows::workloads::transaction; - -let tx_workload = transaction::Workload::with_rate(10) - .expect("transaction rate must be non-zero"); - -ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .wallets(20) - .with_workload(tx_workload) - .with_run_duration(Duration::from_secs(60)) - .build(); -``` - -#### Prerequisites - -1. **Wallet accounts must be seeded:** - ```rust,ignore - .wallets(N) // Before .transactions_with() - ``` - The workload will fail during `init()` if no wallets are configured. - -2. **Circuit artifacts must be available:** - - Automatically staged by `scripts/run/run-examples.sh` - - Or manually via `scripts/setup/setup-logos-blockchain-circuits.sh` (recommended) / `scripts/setup/setup-logos-blockchain-circuits.sh` - -#### Attached Expectation - -**TxInclusionExpectation** — Verifies that submitted transactions were included in blocks. - -**What it checks:** -- At least `N` transactions were included on-chain (where N = rate × user count × expected block count) -- Uses BlockFeed to count transactions across all observed blocks - -**Failure modes:** -- "Expected >= X transactions, observed Y" (Y < X) -- Common causes: proof generation timeouts, node crashes, insufficient duration - -#### What Failure Looks Like - -```text -Error: Expectation failed: TxInclusionExpectation - Expected: >= 600 transactions (10 tx/block × 60 blocks) - Observed: 127 transactions - - Possible causes: - - Duration too short (nodes still syncing) - - Node crashes (check logs for panics/OOM) - - Wallet accounts not seeded (check topology config) -``` - -**How to debug:** -1. Check logs for proof generation timing: - ```bash - grep "proof generation" $LOGOS_BLOCKCHAIN_LOG_DIR/*/*.log - ``` -2. Increase duration: `.with_run_duration(Duration::from_secs(120))` -3. Reduce rate: `.rate(5)` instead of `.rate(10)` - ---- - -### 2. Chaos Workload (Random Restart) - -Triggers controlled node restarts to test resilience and recovery behaviors. - -**Import:** -```rust,ignore -use testing_framework_workflows::workloads::chaos::RandomRestartWorkload; -``` - -#### Configuration - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `min_delay` | `Duration` | **Required** | Minimum time between restart attempts | -| `max_delay` | `Duration` | **Required** | Maximum time between restart attempts | -| `target_cooldown` | `Duration` | **Required** | Minimum time before restarting same node again | -| `include_nodes` | `bool` | **Required** | Whether to restart nodes | - -#### Usage - -```rust,ignore -use std::time::Duration; - -use testing_framework_core::scenario::ScenarioBuilder; -use testing_framework_workflows::{ScenarioBuilderExt, workloads::chaos::RandomRestartWorkload}; - -let scenario = ScenarioBuilder::topology_with(|t| { - t.network_star().nodes(3) -}) -.enable_node_control() // REQUIRED for chaos -.with_workload(RandomRestartWorkload::new( - Duration::from_secs(45), // min_delay - Duration::from_secs(75), // max_delay - Duration::from_secs(120), // target_cooldown - true, // include_nodes -)) -.expect_consensus_liveness() -.with_run_duration(Duration::from_secs(180)) -.build(); -``` - -#### Prerequisites - -1. **Node control must be enabled:** - ```rust,ignore - .enable_node_control() - ``` - This adds `NodeControlCapability` to the scenario. - -2. **Runner must support node control:** - - **Compose runner:** Supported - - **Local runner:** Not supported - - **K8s runner:** Not yet implemented - -3. **Sufficient topology:** - - For nodes: Need >1 node (workload skips if only 1) - -4. **Realistic timing:** - - Total duration should be 2-3× the max_delay + cooldown - - Example: max_delay=75s, cooldown=120s → duration >= 180s - -#### Attached Expectation - -None. You must explicitly add expectations (typically `.expect_consensus_liveness()`). - -**Why?** Chaos workloads are about testing recovery under disruption. The appropriate expectation depends on what you're testing: -- Consensus survives restarts → `.expect_consensus_liveness()` -- Height converges after chaos → Custom expectation checking BlockFeed - -#### What Failure Looks Like - -```text -Error: Workload failed: chaos_restart - Cause: NodeControlHandle not available - - Possible causes: - - Forgot .enable_node_control() in scenario builder - - Using local runner (doesn't support node control) - - Using k8s runner (doesn't support node control) -``` - -**Or:** - -```text -Error: Expectation failed: ConsensusLiveness - Expected: >= 20 blocks - Observed: 8 blocks - - Possible causes: - - Restart frequency too high (nodes can't recover) - - Consensus timing too slow (increase duration) - - Too many nodes restarted simultaneously - - Nodes crashed after restart (check logs) -``` - -**How to debug:** -1. Check restart events in logs: - ```bash - grep "restarting\|restart complete" $LOGOS_BLOCKCHAIN_LOG_DIR/*/*.log - ``` -2. Verify node control is enabled: - ```bash - grep "NodeControlHandle" $LOGOS_BLOCKCHAIN_LOG_DIR/*/*.log - ``` -3. Increase cooldown: `Duration::from_secs(180)` -4. Increase duration: `.with_run_duration(Duration::from_secs(300))` - ---- - -## Built-in Expectations - -### 1. Consensus Liveness - -Verifies the system continues to produce blocks during the execution window. - -**Import:** -```rust,ignore -use testing_framework_workflows::ScenarioBuilderExt; -``` - -#### DSL Usage - -```rust,ignore -ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(60)) - .build(); -``` - -#### What It Checks - -- At least `N` blocks were produced (where N = duration / expected_block_time) -- Uses BlockFeed to count observed blocks -- Compares against a minimum threshold (typically 50% of theoretical max) - -#### Failure Modes - -```text -Error: Expectation failed: ConsensusLiveness - Expected: >= 30 blocks - Observed: 3 blocks - - Possible causes: - - Nodes crashed or never started (check logs) - - Consensus timing misconfigured (CONSENSUS_SLOT_TIME too high) - - Insufficient nodes (need >= 2 for BFT consensus) - - Duration too short (nodes still syncing) -``` - -#### How to Debug - -1. Check if nodes started: - ```bash - grep "node started\|listening on" $LOGOS_BLOCKCHAIN_LOG_DIR/*/*.log - ``` -2. Check block production: - ```bash - grep "block.*height" $LOGOS_BLOCKCHAIN_LOG_DIR/node-*/*.log - ``` -3. Check consensus participation: - ```bash - grep "consensus.*slot\|proposal" $LOGOS_BLOCKCHAIN_LOG_DIR/node-*/*.log - ``` -4. Increase duration: `.with_run_duration(Duration::from_secs(120))` -5. Check env vars: `echo $CONSENSUS_SLOT_TIME $CONSENSUS_ACTIVE_SLOT_COEFF` - ---- - -### 2. Workload-Specific Expectations - -Each workload automatically attaches its own expectation: - -| Workload | Expectation | What It Checks | -|----------|-------------|----------------| -| Transaction | `TxInclusionExpectation` | Transactions were included in blocks | -| Chaos | (None) | Add `.expect_consensus_liveness()` explicitly | - -These expectations are added automatically when using the DSL (`.transactions_with()`). - ---- - -## Configuration Quick Reference - -### Transaction Workload - -```rust,ignore -.wallets(20) -.transactions_with(|tx| tx.rate(10).users(5)) -``` - -| What | Value | Unit | -|------|-------|------| -| Rate | 10 | tx/block | -| Users | 5 | wallet accounts | -| Wallets | 20 | total seeded | - -### Chaos Workload - -```rust,ignore -.enable_node_control() -.with_workload(RandomRestartWorkload::new( - Duration::from_secs(45), // min - Duration::from_secs(75), // max - Duration::from_secs(120), // cooldown - true, // nodes -)) -``` - ---- - -## Common Patterns - -### Pattern 1: Multiple Workloads - -```rust,ignore -ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .wallets(20) - .transactions_with(|tx| tx.rate(5).users(10)) - .expect_consensus_liveness() - .with_run_duration(Duration::from_secs(120)) - .build(); -``` - -All workloads run concurrently. Expectations for each workload run after the execution window ends. - -### Pattern 2: Custom Expectation - -```rust,ignore -use testing_framework_core::scenario::Expectation; - -struct MyCustomExpectation; +use async_trait::async_trait; +use testing_framework_core::scenario::{DynError, Expectation, RunContext, Workload}; #[async_trait] -impl Expectation for MyCustomExpectation { - async fn evaluate(&self, ctx: &RunContext) -> Result<(), DynError> { - // Access BlockFeed, metrics, topology, etc. - let block_count = ctx.block_feed()?.count(); - if block_count < 10 { - return Err("Not enough blocks".into()); +pub trait Workload: Send + Sync { + fn name(&self) -> &str; + + fn expectations(&self) -> Vec>> { + Vec::new() + } + + fn init( + &mut self, + _descriptors: &E::Deployment, + _run_metrics: &RunMetrics, + ) -> Result<(), DynError> { + Ok(()) + } + + async fn start(&self, ctx: &RunContext) -> Result<(), DynError>; +} +``` + +The trait methods are: +- **`name`** identifies the workload in logs and failure reports. +- **`expectations`** lets a workload attach its own checks. `with_workload` collects them into the scenario alongside explicitly added expectations (see [Expectations and Evaluation](expectations.md)). +- **`init`** runs synchronously at `build()` time, before anything is deployed. It receives the resolved deployment descriptors and the `RunMetrics` (run duration). A failing `init` aborts the build with a `WorkloadInit` error. +- **`start`** is the async body. It runs once per scenario run and must return when its work is done. + +The runner schedules every workload and applies the same concurrency, panic capture, and run-window behavior described below. + +Register workloads on any builder with `.with_workload(w)` or `.with_workload_boxed(boxed)`. + +--- + +## How the Runner Executes Workloads + +The runner (`testing-framework/core/src/scenario/runtime/runner.rs`) drives a run in fixed phases: + +```mermaid +flowchart LR + P[start_capture
expectations]:::sc --> W[Workload window
run_duration]:::sc + W --> C[Cooldown window]:::sc + C --> D[Drain remaining
workloads]:::sc + D --> S[Settle wait]:::sc + S --> E[Evaluate
expectations]:::sc + + classDef sc stroke:#9b6dd6,stroke-width:2.5px; +``` + +The runner uses the following concurrency rules: + +- **All workloads run concurrently.** Each workload is spawned into its own Tokio task via a `JoinSet`; there is no ordering between them. +- **Panics become errors.** A panicking workload does not abort the process; the panic is caught and reported as `workload panicked: `. +- **One failure fails the run.** The runner joins workload tasks as they finish. The first workload that returns `Err` (or panics) ends the run immediately with `ScenarioError::Workload`; expectations are not evaluated. +- **Finishing early ends the window early.** If every workload returns `Ok` before `with_run_duration` elapses, the workload phase completes without waiting out the timer. +- **The run duration sets the maximum workload window but does not cancel workloads.** When every workload finishes early, the window ends early and cooldown begins. When the timer expires while workloads are still running, the runner keeps driving them through the cooldown window and then *waits for them to finish* (`drain_workloads`). A workload that never returns blocks the run indefinitely. + +Treat `with_run_duration` as the guaranteed run window, not as a workload timeout. A long-running workload should bound its own work, either by operation count or by reading `ctx.run_duration()` and stopping at the deadline. + +After the workload window, managed deployments get a cooldown window (minimum 30 seconds when the framework owns the node lifecycle) plus a short settle wait so runtime extensions catch up before evaluation. Both are tuned with `with_expectation_cooldown`; see [Expectations and Evaluation](expectations.md). + +--- + +## Worked Example: a Key/Value Write Workload + +The kvstore example ships `KvWriteWorkload` (`examples/kvstore/testing/workloads/src/write.rs`), a rate-limited writer over the node HTTP clients: + +```rust,ignore +use async_trait::async_trait; +use kvstore_runtime_ext::KvEnv; +use testing_framework_core::scenario::{DynError, RunContext, Workload}; + +#[async_trait] +impl Workload for KvWriteWorkload { + fn name(&self) -> &str { + "kv_write_workload" + } + + async fn start(&self, ctx: &RunContext) -> Result<(), DynError> { + let clients = ctx.node_clients().snapshot(); + let Some(leader) = clients.first() else { + return Err("no kv node clients available".into()); + }; + + for idx in 0..self.operations { + let key = format!("{}-{}", self.key_prefix, idx % self.key_count); + let response: PutResponse = leader + .put(&format!("/kv/{key}"), &PutRequest { value: format!("value-{idx}"), expected_version: None }) + .await?; + + if !response.applied { + return Err(format!("leader rejected write for key {key}").into()); + } + + if let Some(delay) = interval { + tokio::time::sleep(delay).await; + } } + Ok(()) } } - -ScenarioBuilder::topology_with(|t| t.network_star().nodes(3)) - .with_expectation(MyCustomExpectation) - .with_run_duration(Duration::from_secs(60)) - .build(); ``` +This workload takes one client snapshot, runs a bounded number of operations, controls its rate with a sleep, and returns `Err` for an unexpected response so the runner stops the run. + +The workload is bounded by `self.operations`, so it terminates on its own; the run duration only decides how long the scenario stays up around it. + --- -## Debugging Checklist +## Accessing the RunContext -When a workload or expectation fails: +`RunContext` (`testing-framework/core/src/scenario/runtime/context.rs`) gives a workload access to: -1. Check logs: `$LOGOS_BLOCKCHAIN_LOG_DIR/*/` or `docker compose logs` or `kubectl logs` -2. Check prerequisites: wallets, node control, circuits -3. Increase duration: Double the run duration and retry -4. Reduce rates: Half the traffic rates and retry -5. Check metrics: Prometheus queries for block height and tx count -6. Reproduce locally: Use local runner for faster iteration +| Accessor | Returns | Use for | +|----------|---------|---------| +| `ctx.node_clients()` | `&NodeClients` | Typed API clients for every node | +| `ctx.random_node_client()` | `Option` | Spraying traffic across nodes | +| `ctx.cluster_client()` | `ClusterClient<'_, E>` | Fan-out queries over all clients | +| `ctx.descriptors()` | `&E::Deployment` | The resolved deployment plan | +| `ctx.run_duration()` | `Duration` | Bounding your own loop | +| `ctx.extension::()` / `ctx.require_extension::()` | `Option` / `Result` | Typed runtime extensions | +| `ctx.node_control()` | `Option>>` | Restarting/stopping nodes | +| `ctx.telemetry()` | `&Metrics` | PromQL queries against external telemetry | + +Notes on the client surface: + +- `node_clients().snapshot()` clones the current client vector so you can iterate across `.await` points. Use `with_clients(|clients| ...)` for synchronous reads without the clone. +- `extension::()` returns a *clone* of a value registered by a [runtime extension factory](runtime-extensions.md), for example an `ObservationHandle` from [Continuous Observation](observation.md). +- `node_control()` is only populated when the scenario was built with the node-control capability; see [Scenario Capabilities](capabilities.md) and [Chaos and Controlled Failure](chaos.md). + +Workloads in app-layer scenarios additionally use `AppRunContextExt` (from `testing-framework-app`) to reach composed application handles: + +```rust,ignore +use testing_framework_app::AppRunContextExt; + +let cluster = ctx.require_app::()?; +``` + +`OpenRaftKvClusterAccessible` (`examples/openraft_kv/testing/workloads/src/handle_access.rs`) uses only `require_app` to assert that the exposed cluster handle matches the expected topology. See [AppHost and with_app](app-host.md) for the app layer itself. --- ## See Also -- **[Authoring Scenarios](authoring-scenarios.md)** — Step-by-step tutorial for building scenarios -- **[RunContext: BlockFeed & Node Control](node-control.md)** — Learn how to use BlockFeed in expectations and access node control -- **[Examples](examples.md)** — Concrete scenario patterns combining workloads and expectations -- **[Extending the Framework](extending.md)** — Implement custom workloads and expectations -- **[Troubleshooting](troubleshooting.md)** — Common failure scenarios and fixes +- [Expectations and Evaluation](expectations.md) — the checks that run after your traffic +- [Runtime Extensions](runtime-extensions.md) — sharing typed values with workloads +- [Chaos and Controlled Failure](chaos.md) — workloads that restart nodes +- [Continuous Observation](observation.md) — polling application state while workloads run diff --git a/book/src/workspace-layout.md b/book/src/workspace-layout.md deleted file mode 100644 index e18fd35..0000000 --- a/book/src/workspace-layout.md +++ /dev/null @@ -1,21 +0,0 @@ -# Workspace Layout - -The workspace focuses on multi-node integration testing and sits alongside a -`logos-blockchain-node` checkout. Its crates separate concerns to keep scenarios -repeatable and portable: - -- **Configs**: prepares high-level node, network, tracing, and wallet settings - used across test environments. -- **Core scenario orchestration**: the engine that holds topology descriptions, - scenario plans, runtimes, workloads, and expectations. -- **Workflows**: ready-made workloads (transactions, data-availability, chaos) - and reusable expectations assembled into a user-facing DSL. -- **Runners**: deployment backends for local processes, Docker Compose, and - Kubernetes, all consuming the same scenario plan. -- **Runner Examples** (crate name: `runner-examples`, path: `examples/`): - runnable binaries (`examples/src/bin/local_runner.rs`, - `examples/src/bin/compose_runner.rs`, `examples/src/bin/k8s_runner.rs`) that - demonstrate complete scenario execution with each deployer. - -This split keeps configuration, orchestration, reusable traffic patterns, and -deployment adapters loosely coupled while sharing one mental model for tests. diff --git a/book/theme/tour-v2.css b/book/theme/tour-v2.css new file mode 100644 index 0000000..333ff1b --- /dev/null +++ b/book/theme/tour-v2.css @@ -0,0 +1,822 @@ +/* Presentation-style layout for the Whirlwind Tour chapter only. + Every rule is scoped under .tour so the rest of the book is unaffected. + Colors use mdBook theme variables to stay correct in light and dark themes. */ + +.tour { + font-size: 1em; + line-height: 1.65; +} + +/* Each H2 opens a "slide" */ +.tour h2 { + margin-top: 2.2em; + padding-top: 1.2em; + border-top: 3px solid var(--quote-border, #d0d0d0); + font-size: 2em; + letter-spacing: -0.01em; +} + +.tour .slide-kicker { + display: block; + text-transform: uppercase; + letter-spacing: 0.12em; + font-size: 0.5em; + opacity: 0.6; + margin-bottom: 0.4em; +} + +/* Big opening statement of a slide */ +.tour .lead { + line-height: 1.6; + margin: 0.8em 0 1em 0; +} + +.tour .lead strong { + font-weight: 600; +} + +/* Oversized single-line takeaway */ +.tour .takeaway { + font-size: 1em; + font-weight: 600; + text-align: center; + margin: 1.2em auto; + padding: 0.7em 1em; + background: var(--quote-bg, rgba(128, 128, 128, 0.08)); + border-radius: 12px; + max-width: 34em; +} + +/* Footer of each slide: where the book goes deeper */ +.tour .goes-deeper { + margin-top: 1.4em; + font-size: 0.85em; + opacity: 0.75; +} + +.tour .goes-deeper::before { + content: "Goes deeper \2192 "; + font-weight: 600; +} + +/* Center mermaid diagrams and give them presentation margins */ +.tour pre.mermaid, +.tour .mermaid { + display: flex; + justify-content: center; + margin: 1.4em auto; +} + +/* Relationship edges are the point of the diagrams — keep them visible + against both themes (mermaid's defaults nearly vanish on dark). + Book-wide on purpose: every chapter's diagrams share this problem. */ +.mermaid svg path.flowchart-link, +.mermaid svg .edgePath path, +.mermaid svg .messageLine0, +.mermaid svg .messageLine1 { + stroke: #8a9099 !important; + stroke-width: 1.8px !important; +} + +.mermaid svg marker path { + fill: #8a9099 !important; + stroke: #8a9099 !important; +} + +.mermaid svg .edgeLabel { + color: #8a9099; +} + +/* Collapsible diagrams: tour.js turns each .fold into a toggle + body. + Closed folds keep layout width (height collapse, no display:none) so + mermaid renders correctly while hidden. */ +.tour .fold { + margin: 1.2em 0; +} + +.tour .fold__toggle { + display: block; + width: 100%; + text-align: left; + background: var(--quote-bg, rgba(128, 128, 128, 0.08)); + color: inherit; + border: none; + border-radius: 10px; + padding: 0.55em 1em; + font-family: inherit; + font-size: 0.92em; + font-weight: 600; + cursor: pointer; +} + +.tour .fold__toggle:hover { + background: rgba(128, 128, 128, 0.18); +} + +.tour .fold--closed .fold__body { + height: 0; + overflow: hidden; +} + +/* Hiding the sidebar (the ☰ toggle) widens the reading panel on pages that + carry the tour deck; prose chapters keep the default 750px measure. */ +html.sidebar-hidden .content main:has(.tour) { + max-width: 1250px; +} + +/* Slide panels. A .slide is a bordered page-colored canvas: kicker, one + headline sentence, a visual (nodes flow or tiles row), a small note. + Standalone .slide--top panels carry their own chrome; inside a .sec the + panel chrome sits on the .sec card instead. */ +.tour .slide--top, +.tour .sec { + position: relative; + background: var(--bg, #fff); + border: 1px solid var(--quote-border, #ccc); + border-radius: 12px; + margin: 1em 0; + overflow: hidden; +} + +.tour .slide--top { + padding: 1em 1.3em 0.8em; +} + +.tour .slide-kick { + font-size: 0.82em; + font-weight: 600; + letter-spacing: 0.05em; + opacity: 0.55; + margin: 0 0 0.2em; +} + +.tour .slide-line { + font-size: 1.14em; + font-weight: 600; + margin: 0.1em 0 0.5em; +} + +.tour .slide-note { + font-size: 0.86em; + opacity: 0.75; + margin: 0.5em 0 0.2em; +} + +/* Node flows: bordered concept chips joined by arrows */ +.tour .nodes { + display: flex; + flex-wrap: wrap; + gap: 0.45em; + align-items: center; + justify-content: center; + margin: 0.9em 0; +} + +.tour .nd { + display: inline-block; + border: 1.5px solid var(--quote-border, #bbb); + border-radius: 8px; + padding: 0.38em 0.95em; + font-size: 0.95em; + white-space: nowrap; +} + +/* Concept-colored nodes carry a tint fill; solidity marks the concepts the + eye should land on. Neutral nodes stay outline. */ +.tour .nd-cluster { color: #4a90d9; border-color: #4a90d9; background: rgba(74, 144, 217, 0.10); } +.tour .nd-process { color: #e08a3c; border-color: #e08a3c; background: rgba(224, 138, 60, 0.10); } +.tour .nd-handle { color: #4caf7d; border-color: #4caf7d; background: rgba(76, 175, 125, 0.10); } +.tour .nd-scenario { color: #9b6dd6; border-color: #9b6dd6; background: rgba(155, 109, 214, 0.10); } + +/* Enumeration rows: small quiet chips, no fill — reads as a menu, not a flow */ +.tour .nodes--list { + gap: 0.4em; +} + +.tour .nodes--list .nd { + font-size: 0.86em; + padding: 0.16em 0.55em; + border-width: 1px; + background: transparent; +} + +.tour .ndw { + display: inline-flex; + flex-direction: column; + align-items: center; + gap: 0.15em; +} + +.tour .nd-tag { + color: #4caf7d; + font-size: 0.78em; + white-space: nowrap; +} + +.tour .nd-sub { + font-size: 0.78em; + opacity: 0.65; + white-space: nowrap; +} + +.tour .nlabel { + font-size: 0.78em; + font-weight: 600; + letter-spacing: 0.04em; + opacity: 0.55; + margin: 0.9em 0 -0.4em; +} + +.tour .nda { + opacity: 0.6; + font-weight: 600; + font-size: 1.12em; +} + +.tour .nodes--list .nda { + font-size: 1em; + opacity: 0.5; +} + +.tour .nd--dash { + border-style: dashed; + background: transparent; +} + +/* Tile rows: bordered alternatives; border style encodes ownership */ +.tour .tiles { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + gap: 0.6em; + margin: 0.9em 0; +} + +.tour .tile { + border: 1.5px solid var(--quote-border, #bbb); + border-radius: 8px; + padding: 0.55em 0.75em; + text-align: center; +} + +.tour .tile b { + display: block; + font-size: 0.95em; +} + +.tour .tile > span { + display: block; + font-size: 0.82em; + opacity: 0.75; + margin-top: 0.15em; +} + +.tour .tile-cluster { border-color: #4a90d9; background: rgba(74, 144, 217, 0.08); } +.tour .tile-cluster b { color: #4a90d9; } +.tour .tile-process { border-color: #e08a3c; background: rgba(224, 138, 60, 0.08); } +.tour .tile-process b { color: #e08a3c; } +.tour .tile-handle { border-color: #4caf7d; background: rgba(76, 175, 125, 0.08); } +.tour .tile-handle b { color: #4caf7d; } +.tour .tile-scenario { border-color: #9b6dd6; background: rgba(155, 109, 214, 0.08); } +.tour .tile-scenario b { color: #9b6dd6; } + +/* Hollow variants: the tint drains as framework ownership decreases */ +.tour .tile--dash { border-style: dashed; background: transparent; } +.tour .tile--dot { border-style: dotted; background: transparent; } + +/* Section deck: each section is a slide card — small kicker heading, the + .slide summary, and the body collapsed until clicked (height:0, same + mermaid-width reason as .fold). */ +.tour .sec-num { + position: absolute; + top: -0.18em; + right: 0.3em; + font-size: 3.8em; + font-weight: 600; + opacity: 0.06; + pointer-events: none; +} + +.tour .sec-head { + display: flex; + align-items: baseline; + gap: 0.9em; + padding: 0.85em 1.3em 0; + cursor: pointer; +} + +.tour .sec-head h2 { + margin: 0; + padding: 0; + border: none; + font-size: 0.82em; + letter-spacing: 0.05em; + opacity: 0.6; + flex: 1; +} + +.tour .sec-head .unpacks { + display: none; +} + +.tour .sec-chev { + opacity: 0.55; + font-weight: 600; +} + +.tour .sec-head:hover .sec-chev { + opacity: 1; +} + +.tour .sec > .slide { + padding: 0 1.3em 0.7em; + cursor: pointer; +} + +.tour .sec--closed .sec-body { + height: 0; + overflow: hidden; +} + +.tour .sec-body { + padding: 0 1.3em 0.9em; +} + +.tour .sec:not(.sec--closed) .sec-body { + border-top: 1px solid var(--quote-border, #d0d0d0); + padding-top: 0.8em; + margin: 0.3em 1.3em 0; + padding-left: 0; + padding-right: 0; +} + +.tour .sec-controls { + text-align: right; + margin: 1.4em 0 0.4em; +} + +.tour .sec-controls__toggle { + background: none; + border: none; + color: inherit; + opacity: 0.7; + font-size: 0.88em; + cursor: pointer; + padding: 0.2em 0.4em; +} + +.tour .sec-controls__toggle:hover { + opacity: 1; + text-decoration: underline; +} + +@media print { + .tour .sec--closed .sec-body { + height: auto; + overflow: visible; + } + + .tour .sec-chev, + .tour .sec-controls { + display: none; + } +} + +/* Architecture map: hand-laid SVG poster; scrolls sideways when the + column is narrower than its legibility floor. */ +.tour .arch-map { + overflow-x: auto; + margin: 1.6em 0; +} + +.tour .arch-map svg { + display: block; + width: 100%; + min-width: 740px; + height: auto; + margin: 0 auto; +} + +.tour .map-help { + margin: -0.8em 0 1.2em; + text-align: center; + font-size: 0.84em; + opacity: 0.68; +} + +/* Zoomed overlay for the architecture map: click the map to open it large, + scrolled to the clicked spot; drag pans; click or Escape closes. The zoom + width is carried by a variable set from tour.js. */ +.arch-map-overlay { + position: fixed; + inset: 0; + background: rgba(0, 0, 0, 0.6); + display: flex; + align-items: center; + justify-content: center; + z-index: 9999; + cursor: zoom-out; +} + +.arch-map-overlay__content { + background: var(--bg, #fff); + padding: 16px; + max-width: 95vw; + max-height: 95vh; + overflow: auto; + border-radius: 8px; + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.35); + cursor: grab; +} + +.arch-map-overlay__content svg { + width: var(--fw-zoom-width, 1700px); + max-width: none; + height: auto; + display: block; +} + +.arch-map-overlay__controls { + position: fixed; + top: 1.25rem; + right: 1.25rem; + z-index: 1; + display: flex; + align-items: center; + gap: 0.8rem; + padding: 0.5rem 0.6rem 0.5rem 0.8rem; + color: #fff; + background: rgba(20, 20, 20, 0.86); + border-radius: 8px; + font-size: 0.84rem; + cursor: default; +} + +.arch-map-overlay__controls button { + border: 1px solid rgba(255, 255, 255, 0.45); + border-radius: 5px; + padding: 0.3rem 0.6rem; + color: #fff; + background: transparent; + cursor: pointer; +} + +.arch-map-overlay__controls button:hover, +.arch-map-overlay__controls button:focus-visible { + background: rgba(255, 255, 255, 0.15); +} + +/* Slightly larger code on tour slides */ +.tour pre > code { + font-size: 0.95em; + line-height: 1.5; +} + +/* Two-column fact grid */ +.tour .duo { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 1em; + margin: 1.2em 0; +} + +.tour .duo > div { + background: var(--quote-bg, rgba(128, 128, 128, 0.08)); + border-radius: 12px; + padding: 1em 1.2em; +} + +.tour .duo h4 { + margin: 0 0 0.5em 0; + font-size: 1.05em; +} + +@media (max-width: 700px) { + .tour .duo { + grid-template-columns: 1fr; + } +} + +/* Horizontal sequence strip: boxed items joined by arrows */ +.tour .seq { + display: flex; + flex-wrap: wrap; + gap: 0.45em; + align-items: center; + justify-content: center; + margin: 1.2em 0; +} + +.tour .seq span { + background: var(--quote-bg, rgba(128, 128, 128, 0.08)); + border-radius: 8px; + padding: 0.35em 0.8em; + font-size: 0.95em; + white-space: nowrap; +} + +.tour .seq .arr { + background: none; + padding: 0; + opacity: 0.55; + font-weight: 600; +} + +/* Tight, scannable bullet lists */ +.tour ul li, +.tour ol li { + margin: 0.25em 0; +} + +/* ---- Concept color tokens ---------------------------------------------- + Four hues used consistently across chips, diagrams, and code accents: + cluster #4a90d9 · process #e08a3c · handle #4caf7d · scenario #9b6dd6 */ + +.tour .tk { + border: 1.5px solid; + border-radius: 6px; + padding: 0.05em 0.45em; + font-size: 0.92em; + white-space: nowrap; +} + +.tour .tk-cluster { color: #4a90d9; border-color: #4a90d9; } +.tour .tk-process { color: #e08a3c; border-color: #e08a3c; } +.tour .tk-handle { color: #4caf7d; border-color: #4caf7d; } +.tour .tk-scenario { color: #9b6dd6; border-color: #9b6dd6; } + +/* "Unpacks line N" chip under a section heading */ +.tour .unpacks { + font-size: 0.85em; + opacity: 0.8; + margin: -0.4em 0 1em 0; +} + +.tour .unpacks::before { + content: "Unpacks "; + font-weight: 600; +} + +/* Recap strip between sections: closure + forward hook */ +.tour .recap { + background: var(--quote-bg, rgba(128, 128, 128, 0.08)); + border-left: 4px solid #9b6dd6; + border-radius: 0 10px 10px 0; + padding: 0.6em 1em; + font-size: 0.92em; + margin: 1.6em 0 0 0; +} + +/* Notes list matching ①②③ markers inside a code block */ +.tour .code-notes { + list-style: none; + padding-left: 0; + font-size: 0.92em; + margin-top: 0.6em; +} + +.tour .code-notes li { + margin: 0.3em 0; + padding-left: 2em; + text-indent: -2em; +} + +/* Collapsed depth layers */ +.tour details { + background: var(--quote-bg, rgba(128, 128, 128, 0.08)); + border-radius: 10px; + padding: 0.5em 1em; + margin: 1em 0; +} + +.tour details summary { + cursor: pointer; + font-weight: 600; + font-size: 0.95em; +} + +.tour details[open] summary { + margin-bottom: 0.6em; +} + +/* ---- Section progress rail (built by tour.js) -------------------------- */ +.tour-rail { + position: fixed; + right: 16px; + top: 50%; + transform: translateY(-50%); + display: flex; + flex-direction: column; + gap: 10px; + z-index: 50; +} + +.tour-rail a { + position: relative; + width: 18px; + height: 18px; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + font-size: 10px; + line-height: 1; + text-decoration: none !important; + color: inherit; + border: 1.5px solid currentColor; + opacity: 0.35; + transition: opacity 0.2s ease, transform 0.2s ease; +} + +.tour-rail a:hover { + opacity: 0.75; +} + +.tour-rail a.on { + background: #9b6dd6; + border-color: #9b6dd6; + color: #fff; + opacity: 1; + transform: scale(1.15); +} + +.tour-rail a::after { + content: attr(data-label); + position: absolute; + right: 20px; + top: 50%; + transform: translateY(-50%); + white-space: nowrap; + font-size: 0.78rem; + background: var(--bg, #fff); + border: 1px solid var(--quote-border, #ccc); + border-radius: 6px; + padding: 0.15em 0.6em; + opacity: 0; + pointer-events: none; + transition: opacity 0.15s ease; +} + +.tour-rail a:hover::after, +.tour-rail a:focus-visible::after { + opacity: 1; +} + +@media (max-width: 1150px) { + .tour-rail { + display: none; + } +} + +/* ---- Reveal-on-scroll (tour.js; disabled under reduced motion) --------- */ +.tour .reveal { + opacity: 0; + transform: translateY(10px); + transition: opacity 0.35s ease, transform 0.35s ease; +} + +.tour .reveal.on { + opacity: 1; + transform: none; +} + +@media (prefers-reduced-motion: reduce) { + .tour .reveal { + opacity: 1; + transform: none; + transition: none; + } +} + +/* Compact label:value fact rows */ +.tour .facts { + display: grid; + grid-template-columns: max-content 1fr; + gap: 0.35em 1em; + margin: 1em 0; +} + +.tour .facts b { + white-space: nowrap; +} + +/* One-sentence relation of the glossary terms; neutral chips match .tk */ +.tour .spine { + font-size: 1.14em; + line-height: 2.1; + text-align: center; + margin: 1.3em 0; +} + +.tour .spine b { + border: 1.5px solid var(--quote-border, #ccc); + border-radius: 6px; + padding: 0.05em 0.45em; + font-size: 0.92em; + white-space: nowrap; +} + +/* Glossary term cards: three pairs, icon + term + short gloss per row */ +.tour .gcards { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 0.8em; + margin: 1.2em 0; +} + +.tour .gcard { + background: var(--quote-bg, rgba(128, 128, 128, 0.08)); + border-radius: 10px; + padding: 0.9em 1.1em 1em; +} + +.tour .gcard .gcard-label { + display: block; + font-size: 0.8em; + letter-spacing: 0.04em; + opacity: 0.6; + margin-bottom: 0.75em; +} + +.tour .gcard .gterm { + display: flex; + gap: 0.6em; + align-items: flex-start; + margin-top: 0.6em; +} + +.tour .gcard .gterm svg { + width: 21px; + height: 21px; + flex: none; + margin-top: 0.1em; + opacity: 0.75; +} + +.tour .gcard .gterm b { + display: block; + line-height: 1.35; +} + +.tour .gcard .gterm .tk { + display: inline-block; + margin-bottom: 0.15em; +} + +.tour .gcard .gterm .ggloss { + display: block; + font-size: 0.9em; + opacity: 0.72; + line-height: 1.4; +} + +@media (max-width: 640px) { + .tour .gcards { + grid-template-columns: 1fr; + } +} + +/* Glossary grid: term | short gloss | full definition, rows grouped in pairs */ +.tour .gloss { + display: grid; + grid-template-columns: max-content max-content 1fr; + gap: 0.45em 1.2em; + align-items: baseline; + margin: 1em 0; +} + +.tour .gloss .g-term { + font-weight: 600; + white-space: nowrap; +} + +.tour .gloss .g-gloss { + font-weight: 600; + white-space: nowrap; +} + +.tour .gloss .g-def { + opacity: 0.72; + font-size: 0.92em; +} + +.tour .gloss .brk { + grid-column: 1 / -1; + height: 0; + border-top: 1px solid var(--quote-border, #d0d0d0); + opacity: 0.45; +} + +@media (max-width: 640px) { + .tour .gloss { + grid-template-columns: max-content 1fr; + } + + .tour .gloss .g-gloss { + white-space: normal; + } + + .tour .gloss .g-def { + grid-column: 2; + margin-top: -0.35em; + } +} diff --git a/book/theme/tour-v2.js b/book/theme/tour-v2.js new file mode 100644 index 0000000..1da59a1 --- /dev/null +++ b/book/theme/tour-v2.js @@ -0,0 +1,409 @@ +// Progressive enhancement for .tour pages (framework-in-brief): +// a section progress rail and a subtle reveal-on-scroll. +// No-JS or reduced-motion readers get the full static page. +// Deliberately avoids requestAnimationFrame and IntersectionObserver so it +// also behaves in throttled/automation environments and print pipelines. +(function () { + "use strict"; + + var tour = document.querySelector(".tour"); + if (!tour) { + return; + } + + var reducedMotion = window.matchMedia("(prefers-reduced-motion: reduce)").matches; + + // ---- Section deck ----------------------------------------------------- + // Every section whose heading is followed by an .unpacks chip collapses + // to a header strip (heading + chip); the body opens on click. Bodies + // collapse via height:0, not display:none, for the same mermaid-width + // reason as .fold. Anchor navigation (rail dots, "section N" links, + // search) auto-expands the target section. No-JS readers get the full + // static page. + var secs = []; + + Array.prototype.slice + .call(tour.querySelectorAll("h2[id]")) + .forEach(function (h2) { + var chip = h2.nextElementSibling; + if (!chip || !chip.classList.contains("unpacks")) { + return; + } + var sec = document.createElement("section"); + sec.className = "sec sec--closed"; + h2.parentNode.insertBefore(sec, h2); + + var head = document.createElement("div"); + head.className = "sec-head"; + head.setAttribute("role", "button"); + head.setAttribute("tabindex", "0"); + var chev = document.createElement("span"); + chev.className = "sec-chev"; + head.appendChild(h2); + head.appendChild(chip); + head.appendChild(chev); + + var numMatch = h2.textContent.match(/^\s*(\d+)/); + if (numMatch) { + var num = document.createElement("span"); + num.className = "sec-num"; + num.textContent = numMatch[1]; + sec.appendChild(num); + } + + var body = document.createElement("div"); + body.className = "sec-body"; + sec.appendChild(head); + var node = sec.nextSibling; + while (node && !(node.nodeType === 1 && node.tagName === "H2")) { + var next = node.nextSibling; + if (node.nodeType === 1 && node.classList.contains("slide") && !body.children.length) { + sec.appendChild(node); + } else { + body.appendChild(node); + } + node = next; + } + sec.appendChild(body); + + function render() { + var open = !sec.classList.contains("sec--closed"); + chev.textContent = open ? "▾" : "▸"; + head.setAttribute("aria-expanded", String(open)); + } + function toggle() { + sec.classList.toggle("sec--closed"); + render(); + renderDeckControl(); + onScroll(); + } + function clickToggle(event) { + if (event.target.closest && event.target.closest("a")) { + return; + } + toggle(); + } + head.addEventListener("click", clickToggle); + head.addEventListener("keydown", function (event) { + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + toggle(); + } + }); + var slide = sec.querySelector(".slide"); + if (slide) { + slide.addEventListener("click", clickToggle); + } + render(); + secs.push({ sec: sec, render: render }); + }); + + var deckControl = null; + function renderDeckControl() { + if (!deckControl) { + return; + } + var anyClosed = secs.some(function (s) { + return s.sec.classList.contains("sec--closed"); + }); + deckControl.textContent = anyClosed ? "▸ expand all sections" : "▾ collapse all sections"; + } + + if (secs.length) { + var controls = document.createElement("div"); + controls.className = "sec-controls"; + deckControl = document.createElement("button"); + deckControl.type = "button"; + deckControl.className = "sec-controls__toggle"; + deckControl.addEventListener("click", function () { + var anyClosed = secs.some(function (s) { + return s.sec.classList.contains("sec--closed"); + }); + secs.forEach(function (s) { + s.sec.classList.toggle("sec--closed", !anyClosed); + s.render(); + }); + renderDeckControl(); + onScroll(); + }); + controls.appendChild(deckControl); + secs[0].sec.parentNode.insertBefore(controls, secs[0].sec); + renderDeckControl(); + } + + function openSectionFor(hash) { + if (!hash || hash.length < 2) { + return; + } + var target; + try { + target = document.getElementById(decodeURIComponent(hash.slice(1))); + } catch (e) { + return; + } + if (!target) { + return; + } + secs.forEach(function (s) { + if (s.sec.contains(target) && s.sec.classList.contains("sec--closed")) { + s.sec.classList.remove("sec--closed"); + s.render(); + renderDeckControl(); + window.setTimeout(function () { + target.scrollIntoView(); + onScroll(); + }, 0); + } + }); + } + window.addEventListener("hashchange", function () { + openSectionFor(window.location.hash); + }); + openSectionFor(window.location.hash); + + // ---- Section progress rail ------------------------------------------- + var headings = Array.prototype.slice.call(tour.querySelectorAll("h2[id]")); + var dots = []; + if (headings.length >= 4) { + var rail = document.createElement("nav"); + rail.className = "tour-rail"; + rail.setAttribute("aria-label", "Sections"); + + dots = headings.map(function (heading, index) { + var dot = document.createElement("a"); + dot.href = "#" + heading.id; + var label = heading.textContent.trim(); + dot.textContent = String(index + 1); + dot.setAttribute("data-label", label); + dot.setAttribute("aria-label", label); + rail.appendChild(dot); + return dot; + }); + + document.body.appendChild(rail); + } + + var active = -1; + function updateRail() { + if (!dots.length) { + return; + } + var cutoff = window.innerHeight * 0.34; + var current = -1; + for (var i = 0; i < headings.length; i += 1) { + if (headings[i].getBoundingClientRect().top <= cutoff) { + current = i; + } + } + if (current === active) { + return; + } + if (active >= 0) { + dots[active].classList.remove("on"); + } + active = current; + if (active >= 0) { + dots[active].classList.add("on"); + } + } + + // ---- Reveal-on-scroll ------------------------------------------------- + var pending = []; + if (!reducedMotion) { + pending = Array.prototype.slice.call(tour.children).filter(function (el) { + var tag = el.tagName; + return tag === "PRE" || tag === "TABLE" || tag === "UL" || tag === "OL" || + el.classList.contains("duo") || el.classList.contains("facts") || + el.classList.contains("seq") || el.classList.contains("mermaid"); + }); + pending.forEach(function (el) { + el.classList.add("reveal"); + }); + } + + function updateReveals() { + if (!pending.length) { + return; + } + var limit = window.innerHeight * 0.96; + pending = pending.filter(function (el) { + var rect = el.getBoundingClientRect(); + if (rect.top <= limit && rect.bottom >= 0) { + el.classList.add("on"); + return false; + } + return true; + }); + } + + // ---- Collapsible diagrams --------------------------------------------- + // Each .fold wraps one diagram; a toggle button replaces it until clicked. + // Collapsed via height:0 (not display:none) so mermaid still renders at + // real width while hidden. Without JS the diagrams stay fully visible. + Array.prototype.slice.call(tour.querySelectorAll(".fold")).forEach(function (fold) { + var label = fold.getAttribute("data-label") || "diagram"; + var body = document.createElement("div"); + body.className = "fold__body"; + while (fold.firstChild) { + body.appendChild(fold.firstChild); + } + var toggle = document.createElement("button"); + toggle.type = "button"; + toggle.className = "fold__toggle"; + fold.appendChild(toggle); + fold.appendChild(body); + fold.classList.add("fold--closed"); + + function render() { + var open = !fold.classList.contains("fold--closed"); + toggle.textContent = (open ? "▾ " : "▸ ") + label; + toggle.setAttribute("aria-expanded", String(open)); + } + toggle.addEventListener("click", function () { + fold.classList.toggle("fold--closed"); + render(); + onScroll(); + }); + render(); + }); + + // ---- Architecture map: click opens a zoomed, pannable overlay --------- + // Clicking a §N badge navigates to its section instead; a drag pans the + // zoomed map without closing it. + var archMap = tour.querySelector(".arch-map svg"); + if (archMap) { + archMap.style.cursor = "zoom-in"; + archMap.addEventListener("click", function (event) { + if (event.target.closest && event.target.closest("a")) { + return; + } + openArchMapOverlay(archMap, event); + }); + } + + function openArchMapOverlay(svg, event) { + var overlay = document.createElement("div"); + overlay.className = "arch-map-overlay"; + overlay.setAttribute("role", "dialog"); + overlay.setAttribute("aria-modal", "true"); + overlay.setAttribute("aria-label", "Architecture map"); + var content = document.createElement("div"); + content.className = "arch-map-overlay__content"; + var controls = document.createElement("div"); + controls.className = "arch-map-overlay__controls"; + controls.innerHTML = "Drag to pan · Esc to close"; + var closeButton = document.createElement("button"); + closeButton.type = "button"; + closeButton.textContent = "Close"; + closeButton.setAttribute("aria-label", "Close architecture map"); + controls.appendChild(closeButton); + + var zoomWidth = Math.min(1700, Math.round(window.innerWidth * 1.9)); + overlay.style.setProperty("--fw-zoom-width", zoomWidth + "px"); + + var clone = svg.cloneNode(true); + clone.style.display = "block"; + content.appendChild(clone); + overlay.appendChild(controls); + overlay.appendChild(content); + document.body.appendChild(overlay); + var previousFocus = document.activeElement; + closeButton.focus(); + + // Scroll the overlay so the clicked point sits centered. + var rect = svg.getBoundingClientRect(); + var fx = (event.clientX - rect.left) / rect.width; + var fy = (event.clientY - rect.top) / rect.height; + var zoomHeight = zoomWidth * (rect.height / rect.width); + content.scrollLeft = Math.max(0, fx * zoomWidth - content.clientWidth / 2); + content.scrollTop = Math.max(0, fy * zoomHeight - content.clientHeight / 2); + + var dragging = false; + var moved = false; + var startX = 0; + var startY = 0; + var startLeft = 0; + var startTop = 0; + overlay.addEventListener("pointerdown", function (e) { + dragging = true; + moved = false; + startX = e.clientX; + startY = e.clientY; + startLeft = content.scrollLeft; + startTop = content.scrollTop; + }); + overlay.addEventListener("pointermove", function (e) { + if (!dragging) { + return; + } + var dx = e.clientX - startX; + var dy = e.clientY - startY; + if (Math.abs(dx) + Math.abs(dy) > 6) { + moved = true; + } + content.scrollLeft = startLeft - dx; + content.scrollTop = startTop - dy; + }); + overlay.addEventListener("pointerup", function () { + dragging = false; + }); + + function onKey(e) { + if (e.key === "Escape") { + close(); + } + } + function close() { + overlay.remove(); + document.removeEventListener("keydown", onKey); + if (previousFocus && previousFocus.focus) { + previousFocus.focus(); + } + } + controls.addEventListener("pointerdown", function (e) { + e.stopPropagation(); + }); + controls.addEventListener("click", function (e) { + e.stopPropagation(); + }); + closeButton.addEventListener("click", function (e) { + e.stopPropagation(); + close(); + }); + overlay.addEventListener("click", function () { + if (moved) { + moved = false; + return; + } + close(); + }); + document.addEventListener("keydown", onKey); + } + + // ---- One throttled driver for both ----------------------------------- + var ticking = false; + function update() { + updateRail(); + updateReveals(); + } + + function onScroll() { + if (ticking) { + return; + } + ticking = true; + window.setTimeout(function () { + ticking = false; + update(); + }, 60); + } + + window.addEventListener("scroll", onScroll, { passive: true }); + window.addEventListener("resize", onScroll, { passive: true }); + + // Initial pass: mark what is already in view (mermaid renders async and + // shifts layout, so run again shortly after load). + update(); + window.setTimeout(update, 400); + window.setTimeout(update, 1500); +})(); diff --git a/docs/book-maintenance.md b/docs/book-maintenance.md new file mode 100644 index 0000000..5514840 --- /dev/null +++ b/docs/book-maintenance.md @@ -0,0 +1,135 @@ +# Book Maintenance Guide + +Working rules for editing the book. Read this before any book change: layout, +wording, new sections, diagrams. It consolidates the conventions the rewrite +was built under so later sessions do not rediscover them. + +## Ground truth and workflow + +- The book lives in `book/` on this repository's main line (`master`/`dev`). + There is no separate book branch; edit here. A leftover worktree + (`../nomos-testing-book`, branch `book-rewrite`) may still exist — it is + retired; do not edit there. +- Build with `mdbook build book`; preview with `mdbook serve book` (default + port 3000). Output goes to `target/book/`. +- Deploys: pushing `master` with `book/**` changes publishes to GitHub Pages + automatically. `dev` pushes do not deploy. +- Commits: title only, no body, no bullets. The taplo pre-commit hook fails + offline (schema catalog fetch) — commit with `--no-verify` when it does. + Never push. + +## Voice (the most-corrected area; follow strictly) + +Factual reference voice, like a senior engineer's internal wiki page. Andrus +has rejected drafts twice over this. + +- No marketing or lecture language: no "powerful", "seamless", taglines, + aphorisms ("teardown design is exposure design"), rhetorical questions, or + keynote framing. Headings name topics, not hooks. +- No maturity inflation: an uncommitted experiment is a "prototype + integration", never a "real adopter" or "production". +- No LLM-typical phrasing. Em-dashes at most ~1 per paragraph (table cells and + nav lists exempt). Every sentence has a finite main verb — the banned + fingerprint is the verbless colon-fronted appositive with participial tails: + "A compact mini-book: the framework's main concepts in one coherent read, + each section linking to its full chapter." Write instead: "This page + summarizes the framework's main concepts. Each section links to a chapter + that covers its topic in full." No "X is not Y — it is Z" setup-payoff, no + habitual triads, no coined metaphors (machinery, plumbing, "the X story"), + no glossy compressions ("at a glance", "in one coherent read"). +- Established terms are fixed: entry pattern, imperative side door, ownership + mode, handle, run window, cooldown, exposure order. "Deployer" is the + backend; "Runner" is what `deploy()` returns — never mix. + +## Chapter template + +H1, then a one-sentence summary with a main verb. `---` between H2 sections. +Title Case headings. `**Note:**` / `**Important:**` bold admonitions; +blockquotes only for external-project callouts, labeled `> **External +example:**`. Code fences: `rust,ignore` / `bash` / `mermaid` / `text`. +Cross-links as relative `[Title](file.md)`. External projects (currently only +logos-blockchain) appear only in the labeled callouts; everything else teaches +through the in-repo example apps. LEZ callouts were removed 2026-07-20 as +outdated — do not re-add them. Chapters run roughly 80–250 lines. + +## The Brief (framework-in-brief.md) specifics + +Presentation-styled page scoped under `
`. Its CSS lives in +`book/theme/tour-v2.css`, its JS (section rail, folds, map zoom) in +`book/theme/tour-v2.js`; both are registered in `book/book.toml`. Available classes: +`lead` (opening paragraph of a section), `unpacks` (chip under a heading), +`recap` (strip between sections), `seq` (arrow strip), `facts` (label:value +grid), `spine` (the one-sentence glossary relation; plain `` renders as a +neutral chip), `gcards` (three glossary pair cards; each `gcard` holds a +`gcard-label` and `gterm` rows of inline SVG icon + term + `ggloss` phrase, +stacks to one column under 640px), `gloss` (term | short gloss | definition +grid with `g-term` / `g-gloss` / `g-def` cells and `brk` pair separators, +collapses to two columns under 640px; lives inside the "full definitions" +fold), `duo` (two cards), `code-notes` (①②③ list matching code markers), +`tk tk-cluster|tk-process|tk-handle|tk-scenario` (concept chips), `details` +styling. Concept hues used everywhere (chips, mermaid classDefs, code +accents): cluster `#4a90d9`, process `#e08a3c`, handle `#4caf7d`, +scenario/runtime `#9b6dd6`. Mermaid edges are forced visible book-wide via +`tour.css`; markdown inside raw HTML blocks is not processed — use `` +inside `.facts`/`.seq` divs, never backticks. The page is a slide deck. Slide +anatomy classes: `slide` (the panel), `slide--top` (standalone hand-authored +panels at the page top: the framework, six terms, the whole test, the DSL), +`slide-kick` (small kicker), `slide-line` (one-sentence headline), +`slide-note` (small caption), `nodes`/`nd`/`ndw`/`nd-tag`/`nda` (concept-chip +flow diagrams; `nd-cluster|process|handle|scenario` hue variants), +`tiles`/`tile` (alternative rows; same hue variants plus `tile--dash`/ +`tile--dot` border styles encoding attached/external). Visual hierarchy rule: +hued nodes/tiles carry a ~10% tint fill (solidity = the concept to look at; +dashed/dotted variants stay hollow — the tint drains as framework ownership +decreases); enumeration rows take `nodes--list` (small hollow chips) so only +true flows read as flows. Sections: `tour-v2.js` +turns every `## N ·` heading followed by an `.unpacks` chip into a `.sec` +slide card — heading restyled as kicker, chip hidden (still required as the +deck gate), corner `.sec-num` numeral, the authored `.slide` visible, and the +body collapsed via height:0 until clicked; an expand-all control sits above +the deck, anchor navigation (rail, "section N" links, search) opens the +target section, and print forces everything open. Every section needs both +an `.unpacks` chip and a `.slide` (headline + nodes or tiles + note). Sections are numbered "N ·" and +cross-referenced as "section N" — renumber ALL references when inserting a +section (previous miss: a capitalized "Section N" escaped a lowercase-only +sweep). + +## Accuracy discipline (drift is the #1 recurring failure) + +The codebase moves fast; quoted code rots in days. Rules: + +- Document nothing before it lands on the main line. Aspirational API lives + only in `docs/*-plan.md` files as labeled targets. +- Verify every API name against the current source before writing it. For + dense snippets, compile them (temp bin under an example crate, then delete — + this caught real bugs twice). +- Prefer quoting from tested code: acceptance tests (`multi-app-e2e`, + `queue-e2e`) and example bins are the source for snippets. State the run + command with each quoted example. +- After code lands that the book mentions, run a sync sweep: grep the book for + the old names; check `framework-in-brief.md`, `running-examples.md`, + `crate-map.md`, `troubleshooting.md`, `composing-stacks.md` first — they + concentrate cross-references. + +## Verification routine before committing a book round + +1. `mdbook build book` passes. +2. Link check: every `(*.md)` target in SUMMARY and chapters exists. +3. Banned-token sweep: stale example names, dead API names, "Adopter note", + marketing words, first-person headings. +4. If diagrams changed: load the page in the browser preview and confirm every + mermaid block rendered to SVG (parse errors fail silently to raw text); + check light and dark themes for new colors. +5. If snippets changed: compile-check them. + +## Current known state (2026-07-20; re-verify, do not trust blindly) + +- The Brief uses the hand-authored `framework-map.svg`, a section deck, and + the queue verb DSL as its compact worked example. +- The full book documents the verb layer in `verb-layer.md` and shared app + cluster provisioning in `cluster-provisioning.md`. +- App handles are access surfaces. Managed app lifetime belongs to the LIFO + cleanup stack; do not reintroduce clone-count ownership language. +- App-layer provisioning has a backend seam, but the only implementation that + starts composed child resources today is local. Keep Compose and Kubernetes + claims aligned with `app-backend-scope.md` and `capability-matrix.md`.