From 20b9868acef7671bb0c2da34606804e0eede1b14 Mon Sep 17 00:00:00 2001 From: Moudy Date: Tue, 19 May 2026 00:36:31 +0200 Subject: [PATCH] feat: add e2e_bench tool for end-to-end scenario latency, block, and tx-byte measurements --- Cargo.lock | 27 ++ Cargo.toml | 2 + docs/benchmarks/README.md | 1 + docs/benchmarks/e2e_bench.md | 125 +++++++++ tools/e2e_bench/Cargo.toml | 33 +++ tools/e2e_bench/README.md | 33 +++ tools/e2e_bench/src/bedrock_handle.rs | 147 +++++++++++ tools/e2e_bench/src/bench_context.rs | 205 +++++++++++++++ tools/e2e_bench/src/harness.rs | 297 ++++++++++++++++++++++ tools/e2e_bench/src/main.rs | 229 +++++++++++++++++ tools/e2e_bench/src/scenarios/amm.rs | 200 +++++++++++++++ tools/e2e_bench/src/scenarios/fanout.rs | 90 +++++++ tools/e2e_bench/src/scenarios/mod.rs | 7 + tools/e2e_bench/src/scenarios/parallel.rs | 188 ++++++++++++++ tools/e2e_bench/src/scenarios/private.rs | 150 +++++++++++ tools/e2e_bench/src/scenarios/token.rs | 127 +++++++++ 16 files changed, 1861 insertions(+) create mode 100644 docs/benchmarks/e2e_bench.md create mode 100644 tools/e2e_bench/Cargo.toml create mode 100644 tools/e2e_bench/README.md create mode 100644 tools/e2e_bench/src/bedrock_handle.rs create mode 100644 tools/e2e_bench/src/bench_context.rs create mode 100644 tools/e2e_bench/src/harness.rs create mode 100644 tools/e2e_bench/src/main.rs create mode 100644 tools/e2e_bench/src/scenarios/amm.rs create mode 100644 tools/e2e_bench/src/scenarios/fanout.rs create mode 100644 tools/e2e_bench/src/scenarios/mod.rs create mode 100644 tools/e2e_bench/src/scenarios/parallel.rs create mode 100644 tools/e2e_bench/src/scenarios/private.rs create mode 100644 tools/e2e_bench/src/scenarios/token.rs diff --git a/Cargo.lock b/Cargo.lock index 57135709..96afcef6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2378,6 +2378,33 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "e2e_bench" +version = "0.1.0" +dependencies = [ + "amm_core", + "anyhow", + "borsh", + "chrono", + "clap", + "common", + "indexer_service", + "indexer_service_rpc", + "integration_tests", + "jsonrpsee", + "log", + "nssa", + "nssa_core", + "sequencer_service", + "sequencer_service_rpc", + "serde", + "serde_json", + "tempfile", + "token_core", + "tokio", + "wallet", +] + [[package]] name = "ecdsa" version = "0.16.9" diff --git a/Cargo.toml b/Cargo.toml index 27f3564b..d75e26c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ members = [ "indexer/ffi", "tools/cycle_bench", "tools/crypto_primitives_bench", + "tools/e2e_bench", ] [workspace.dependencies] @@ -75,6 +76,7 @@ faucet_core = { path = "programs/faucet/core" } vault_core = { path = "programs/vault/core" } test_program_methods = { path = "test_program_methods" } testnet_initial_state = { path = "testnet_initial_state" } +integration_tests = { path = "integration_tests" } tokio = { version = "1.50", features = [ "net", diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index 9289d2bf..db539966 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -6,5 +6,6 @@ Bench tools live under `tools/` with READMEs for how to run each one. This direc |---|---| | cycle_bench | [cycle_bench.md](cycle_bench.md) | | crypto_primitives_bench | [crypto_primitives_bench.md](crypto_primitives_bench.md) | +| e2e_bench | [e2e_bench.md](e2e_bench.md) | All numbers are from a single M2 Pro dev box unless noted otherwise. diff --git a/docs/benchmarks/e2e_bench.md b/docs/benchmarks/e2e_bench.md new file mode 100644 index 00000000..2f2a0a7a --- /dev/null +++ b/docs/benchmarks/e2e_bench.md @@ -0,0 +1,125 @@ +# e2e_bench + +End-to-end LEZ scenarios driven through the wallet against an in-process sequencer + indexer wired to an external Bedrock node. Times each step and records borsh sizes per block, split by tx variant. + +## Machine + +| Field | Value | +|---|---| +| Chip | Apple M2 Pro (8P+4E) | +| RAM | 16 GB | +| OS | macOS 15.5 | +| Rust | 1.94.0 | +| Risc0 zkVM | 3.0.5 | +| Profile | release | + +## Scenarios + +| Scenario | Description | +|---|---| +| token | Sequential public token Send + one shielded recipient setup. | +| amm | Pool create, add liquidity, swap, remove liquidity. All public. | +| fanout | One sender → N recipients, sequential. All public. | +| private | Shielded, deshielded, private→private chained private flow. | +| parallel | N senders submit concurrently into one block. All public. | + +## Dev-mode vs real-proving + +`RISC0_DEV_MODE=1` makes the prover emit stub receipts instead of running the recursive STARK pipeline. The table compares each quantity in **dev mode vs real proving** for the two classes of scenarios: + +| Quantity | Public-only scenarios (dev → real) | PPE-bearing scenarios (dev → real) | +|---|---|---| +| Wall time per step | same in both modes | real adds ~100 s per PPE step | +| `public_tx_bytes` | same in both modes | same in both modes | +| `ppe_tx_bytes` | n/a | dev ≈ 2 KB stub → real ≈ 225 KB (matches `S_agg` from cycle_bench) | +| `block_bytes` | same in both modes | real adds ~225 KB per PPE tx in the block | +| `bedrock_finality_ms` | same in both modes | same in both modes (L1 cadence, not LEZ prover) | +| Blocks captured | similar in both modes | real captures more empty clock-only ticks that fill prove wall-time | + +Tables below report dev-mode for all five scenarios. Real-proving numbers are included for `amm_swap_flow` (representative all-public) and `private_chained_flow` (representative chained-private flow); the public-only scenarios converge between modes within run-to-run jitter, so a full real-proving sweep is not run here. + +## Step latencies — dev mode (`RISC0_DEV_MODE=1`) + +Per-scenario wall time and Bedrock L1-finality latency for the closing tip. + +| Scenario | total_ms | total_s | bedrock_finality_ms | bedrock_finality_s | +|---|---:|---:|---:|---:| +| token_onboarding | 60,808 | 60.81 | 24,593 | 24.59 | +| amm_swap_flow | 162,058 | 162.06 | 19,210 | 19.21 | +| multi_recipient_fanout | 222,206 | 222.21 | 16,020 | 16.02 | +| private_chained_flow | 80,700 | 80.70 | 23,963 | 23.96 | +| parallel_fanout | 244,387 | 244.39 | 23,770 | 23.77 | + +Total dev-mode wall time across all five: 912.9 s. + +## Step latencies — real proving (selected scenarios) + +| Scenario | total_ms | total_s | bedrock_finality_ms | bedrock_finality_s | Δ vs dev | +|---|---:|---:|---:|---:|---:| +| amm_swap_flow | 162,437 | 162.44 | ~19,210 | ~19.21 | ~0 (all-public) | +| private_chained_flow | 354,843 | 354.84 | 23,778 | 23.78 | +274.14 s (≈ 91 s per PPE step × 3) | + +Per-step breakdown for `private_chained_flow` in real proving: + +| Step | submit_ms | inclusion_ms | total_ms | total_s | +|---|---:|---:|---:|---:| +| token_new_fungible (public) | 1.1 | 20,276.0 | 20,291.2 | 20.29 | +| shielded_transfer (PPE) | 111,683.3 | 1.0 | 111,730.4 | 111.73 | +| deshielded_transfer (PPE) | 111,454.7 | 1.1 | 111,511.2 | 111.51 | +| private_to_private (PPE) | 111,237.0 | 1.1 | 111,293.0 | 111.29 | + +PPE steps move the cost from `inclusion_ms` (waiting for the next sealed block) to `submit_ms` (the wallet itself proving the PPE circuit before sending). Each PPE prove is ≈ 111 s on this CPU. + +## Block + tx sizes (borsh) — dev mode + +Per scenario, every produced block is fetched via `getBlock(BlockId)` and serialized with `borsh::to_vec(&Block)`. Each transaction is serialized individually and counted by variant. The empty clock-only ticks at `min` give the per-block fixed-cost baseline (≈ 334 bytes across all scenarios). + +| Scenario | blocks | block_bytes (mean) | block_bytes (min..max) | public_tx (mean / n) | ppe_tx (mean / n) | +|---|---:|---:|---|---:|---:| +| token_onboarding | 6 | 881 | 334..2,890 | 206 / 8 | 2,556 / 1 | +| amm_swap_flow | 16 | 553 | 334..1,011 | 248 / 24 | n/a | +| multi_recipient_fanout | 22 | 513 | 334..707 | 221 / 33 | n/a | +| private_chained_flow | 8 | 1,399 | 334..3,565 | 177 / 9 | 2,715 / 3 | +| parallel_fanout | 24 | 646 | 334..3,904 | 248 / 45 | n/a | + +## Block + tx sizes (borsh) — real proving + +| Scenario | blocks | block_bytes (mean) | block_bytes (min..max) | public_tx (mean / n) | ppe_tx (mean / n) | +|---|---:|---:|---|---:|---:| +| amm_swap_flow | 16 | 553 | 334..1,011 | 248 / 24 | n/a | +| private_chained_flow | 35 | 19,692 | 334..226,578 | 159 / 36 | 225,728 / 3 | + +`amm_swap_flow` is byte-identical between dev and real (no proof payload). `private_chained_flow`'s `ppe_tx_bytes` matches the cycle_bench `S_agg` measurement (≈ 225 KB borsh InnerReceipt). The `block_bytes` max (226,578) is the block containing the largest PPE transaction. + +## Findings + +- Public-only scenarios converge between dev mode and real proving in both latency and byte counts. Either mode is suitable to characterize them. +- PPE transactions are ≈ 225 KB on the wire in real proving, dominated by the outer succinct proof. Dev mode emits a ≈ 2 KB stub that does not represent the L1 payload — fee-model storage gas inputs must come from a real-proving run. +- Per-PPE-step prove cost on M2 Pro CPU is ≈ 110-120 s, paid on the wallet side at submit time (not on the sequencer). For a single-program chained flow the cost stacks linearly. +- Empty clock-only ticks set the per-block fixed-cost baseline at ≈ 334 bytes across all scenarios and both modes. +- Bedrock L1 finality stays around 20 s regardless of proving mode, because finality is paced by L1 cadence, not the LEZ prover. + +## Reproduce + +```sh +export LEZ_BEDROCK_BIN=/path/to/logos-blockchain/target/release/logos-blockchain-node +export LEZ_BEDROCK_CONFIG_DIR=/path/to/bedrock/configs + +# Dev-mode sweep (fast, ~16 min for all five scenarios) +RISC0_DEV_MODE=1 cargo run --release -p e2e_bench -- --scenario all + +# Real-proving for representative private flow (~6 min on M2 Pro CPU) +cargo run --release -p e2e_bench -- --scenario private + +# Real-proving for representative public flow (~3 min) +cargo run --release -p e2e_bench -- --scenario amm +``` + +JSON output: `target/e2e_bench_dev.json` / `target/e2e_bench_prove.json` (suffix toggled by `RISC0_DEV_MODE`). + +## Caveats + +- Dev-mode `ppe_tx_bytes` and PPE-step latencies are not representative of production; use real-proving numbers for any fee-model input that touches the storage or prover-cost components. +- Single-host run, no GPU acceleration. Real-proving on production prover hardware will move per-step latencies by orders of magnitude; byte counts will not change. +- Bedrock running locally; no real network latency between sequencer and Bedrock. +- Some scenarios share account state via the same wallet; this is intentional (mirrors `integration_tests::TestContext`) and not a realistic multi-wallet workload. diff --git a/tools/e2e_bench/Cargo.toml b/tools/e2e_bench/Cargo.toml new file mode 100644 index 00000000..e3d7fd7a --- /dev/null +++ b/tools/e2e_bench/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "e2e_bench" +version = "0.1.0" +edition = "2024" +license = { workspace = true } +publish = false + +[lints] +workspace = true + +[dependencies] +integration_tests.workspace = true +wallet.workspace = true +nssa.workspace = true +nssa_core = { workspace = true, features = ["host"] } +sequencer_service.workspace = true +sequencer_service_rpc = { workspace = true, features = ["client"] } +indexer_service.workspace = true +indexer_service_rpc = { workspace = true, features = ["client"] } +jsonrpsee = { workspace = true, features = ["ws-client"] } +token_core.workspace = true +amm_core.workspace = true +common.workspace = true +tempfile.workspace = true +borsh.workspace = true +chrono.workspace = true + +anyhow.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } +clap.workspace = true +serde.workspace = true +serde_json.workspace = true +log.workspace = true diff --git a/tools/e2e_bench/README.md b/tools/e2e_bench/README.md new file mode 100644 index 00000000..37d6b175 --- /dev/null +++ b/tools/e2e_bench/README.md @@ -0,0 +1,33 @@ +# e2e_bench + +End-to-end LEZ scenarios driven through the wallet against an in-process sequencer + indexer wired to an external Bedrock node. Times each step (submit, inclusion, wallet sync) and records borsh sizes for every block produced, split into per-tx-variant counts. + +## Run + +Required env vars (no defaults): + +```sh +export LEZ_BEDROCK_BIN=/path/to/logos-blockchain/target/release/logos-blockchain-node +export LEZ_BEDROCK_CONFIG_DIR=/path/to/bedrock/configs +# optional: LEZ_BEDROCK_PORT (default 18080) +``` + +The config dir must contain `node-config.yaml` and a `deployment-settings.yaml` template with the literal string `PLACEHOLDER_CHAIN_START_TIME` (rewritten per launch). + +```sh +# All scenarios, dev-mode proving (fast) +RISC0_DEV_MODE=1 cargo run --release -p e2e_bench -- --scenario all + +# One scenario, real proving (slow) +cargo run --release -p e2e_bench -- --scenario amm +``` + +Scenarios: `token`, `amm`, `fanout`, `private`, `parallel`, `all`. + +## What you'll see + +Per scenario: a step table (`submit_ms`, `inclusion_ms`, `sync_ms`, `total_ms`) and a size summary covering every block captured during the scenario (block_bytes total/mean/min/max; per-tx-variant sizes for public, PPE, and program-deployment transactions). + +The fanout, parallel, and private scenarios are the most representative for L1-payload-size measurements since they put multiple txs per block. + +JSON output is written to `target/e2e_bench.json`. diff --git a/tools/e2e_bench/src/bedrock_handle.rs b/tools/e2e_bench/src/bedrock_handle.rs new file mode 100644 index 00000000..ef59d8ad --- /dev/null +++ b/tools/e2e_bench/src/bedrock_handle.rs @@ -0,0 +1,147 @@ +//! Manages an external `logos-blockchain-node` process as a child of the bench. +//! Launches a fresh Bedrock instance per scenario so the indexer never has to +//! catch up a large finalization backlog. +//! +//! Required env vars (no defaults — path layouts differ per developer): +//! - `LEZ_BEDROCK_BIN` — absolute path to the `logos-blockchain-node` binary. +//! - `LEZ_BEDROCK_CONFIG_DIR` — directory containing `node-config.yaml` and +//! `deployment-settings.yaml` (template with `PLACEHOLDER_CHAIN_START_TIME`). +//! +//! Optional: +//! - `LEZ_BEDROCK_PORT` (default: 18080) + +use std::{ + env, + net::SocketAddr, + path::PathBuf, + process::{Child, Command, Stdio}, + time::{Duration, Instant}, +}; + +use anyhow::{Context as _, Result, bail}; + +pub struct BedrockHandle { + child: Option, + addr: SocketAddr, + workdir: PathBuf, +} + +impl BedrockHandle { + /// Launch a fresh Bedrock node. Cleans `state/` in the working dir, rewrites + /// `deployment-settings.yaml` with the current UTC `chain_start_time`, spawns + /// the binary, and polls the HTTP port until ready. + pub async fn launch_fresh() -> Result { + let bin = env::var("LEZ_BEDROCK_BIN").map_err(|err| { + anyhow::anyhow!( + "LEZ_BEDROCK_BIN is required ({err}). Set it to the absolute path of the \ + logos-blockchain-node binary (e.g. \ + `export LEZ_BEDROCK_BIN=/path/to/logos-blockchain/target/release/logos-blockchain-node`)." + ) + })?; + let config_dir = env::var("LEZ_BEDROCK_CONFIG_DIR").map_err(|err| { + anyhow::anyhow!( + "LEZ_BEDROCK_CONFIG_DIR is required ({err}). Set it to the directory containing \ + node-config.yaml and deployment-settings.yaml \ + (see tools/e2e_bench/README.md for the expected layout)." + ) + })?; + let port: u16 = env::var("LEZ_BEDROCK_PORT") + .ok() + .and_then(|p| p.parse().ok()) + .unwrap_or(18080); + + let bin_path = PathBuf::from(&bin); + if !bin_path.is_file() { + bail!( + "LEZ_BEDROCK_BIN does not point at a file: {bin}. Build it via \ + `cargo build -p logos-blockchain-node --release` in logos-blockchain." + ); + } + let config_dir = PathBuf::from(config_dir); + let node_config = config_dir.join("node-config.yaml"); + let dep_template = config_dir.join("deployment-settings.yaml"); + if !node_config.is_file() || !dep_template.is_file() { + bail!( + "LEZ_BEDROCK_CONFIG_DIR is missing node-config.yaml or \ + deployment-settings.yaml at {}", + config_dir.display() + ); + } + + let workdir = tempfile::tempdir() + .context("create bedrock workdir")? + .keep(); + let dep_runtime = workdir.join("deployment-settings.yaml"); + let raw = std::fs::read_to_string(&dep_template).context("read deployment template")?; + let timestamp = chrono_now_utc_string(); + let filled = raw.replace("PLACEHOLDER_CHAIN_START_TIME", ×tamp); + std::fs::write(&dep_runtime, filled).context("write deployment-settings runtime")?; + + let log_path = workdir.join("bedrock.log"); + let log_file = std::fs::File::create(&log_path).context("create bedrock log")?; + let log_err = log_file.try_clone().context("clone bedrock log")?; + + eprintln!( + "BedrockHandle: launching {} (workdir {})", + bin, + workdir.display() + ); + let child = Command::new(&bin_path) + .current_dir(&workdir) + .arg("--deployment") + .arg(&dep_runtime) + .arg(&node_config) + .env("POL_PROOF_DEV_MODE", "true") + .stdout(Stdio::from(log_file)) + .stderr(Stdio::from(log_err)) + .spawn() + .context("spawn logos-blockchain-node")?; + + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + wait_for_http(addr, Duration::from_secs(60)) + .await + .context("bedrock HTTP did not come up in 60s")?; + + eprintln!("BedrockHandle: stdout/stderr at {}", log_path.display()); + Ok(Self { + child: Some(child), + addr, + workdir, + }) + } + + pub const fn addr(&self) -> SocketAddr { + self.addr + } +} + +impl Drop for BedrockHandle { + fn drop(&mut self) { + if let Some(mut child) = self.child.take() { + eprintln!("BedrockHandle: stopping bedrock pid {}", child.id()); + let _ = child.kill(); + let _ = child.wait(); + } + let _ = std::fs::remove_dir_all(&self.workdir); + } +} + +async fn wait_for_http(addr: SocketAddr, timeout: Duration) -> Result<()> { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if tokio::net::TcpStream::connect(addr).await.is_ok() { + // TCP accepts; give Bedrock a moment to finish chain bootstrap. + tokio::time::sleep(Duration::from_secs(2)).await; + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + bail!("Bedrock at {addr} did not accept TCP within {timeout:?}"); +} + +fn chrono_now_utc_string() -> String { + // Format: YYYY-MM-DD HH:MM:SS.000000 +00:00:00 (matches the deployment-settings template). + chrono::Utc::now() + .format("%Y-%m-%d %H:%M:%S%.6f +00:00:00") + .to_string() +} diff --git a/tools/e2e_bench/src/bench_context.rs b/tools/e2e_bench/src/bench_context.rs new file mode 100644 index 00000000..43376811 --- /dev/null +++ b/tools/e2e_bench/src/bench_context.rs @@ -0,0 +1,205 @@ +//! BenchContext: wires sequencer + indexer + wallet in-process against an +//! externally-running Bedrock node. Mirrors the surface of +//! `integration_tests::TestContext` for the methods the scenarios need +//! (`wallet_mut()`, `sequencer_client()`), but skips the docker setup. +//! +//! The external Bedrock URL defaults to 127.0.0.1:18080 and can be overridden +//! with the `LEZ_BEDROCK_ADDR` env var. + +use std::{env, net::SocketAddr, path::Path}; + +use anyhow::{Context as _, Result}; +use indexer_service::IndexerHandle; +use integration_tests::config::{ + SequencerPartialConfig, UrlProtocol, addr_to_url, default_private_accounts_for_wallet, + default_public_accounts_for_wallet, genesis_from_accounts, indexer_config, sequencer_config, + wallet_config, +}; +use sequencer_service::SequencerHandle; +use sequencer_service_rpc::{SequencerClient, SequencerClientBuilder}; +use serde::Serialize; +use tempfile::TempDir; +use wallet::{WalletCore, config::WalletConfigOverrides}; + +const DEFAULT_BEDROCK_ADDR: &str = "127.0.0.1:18080"; + +#[expect( + clippy::partial_pub_fields, + reason = "Internal TempDirs are kept alive via private fields for RAII; \ + client and wallet are public for scenarios to drive." +)] +pub struct BenchContext { + pub sequencer_client: SequencerClient, + pub wallet: WalletCore, + #[expect( + dead_code, + reason = "Retained for parity with TestContext; may be needed later." + )] + pub wallet_password: String, + sequencer_handle: Option, + indexer_handle: IndexerHandle, + temp_indexer_dir: TempDir, + temp_sequencer_dir: TempDir, + temp_wallet_dir: TempDir, +} + +impl BenchContext { + pub async fn new() -> Result { + let bedrock_addr_str = + env::var("LEZ_BEDROCK_ADDR").unwrap_or_else(|_| DEFAULT_BEDROCK_ADDR.to_owned()); + let bedrock_addr: SocketAddr = bedrock_addr_str + .parse() + .with_context(|| format!("invalid LEZ_BEDROCK_ADDR `{bedrock_addr_str}`"))?; + + eprintln!("BenchContext: using external bedrock at {bedrock_addr}"); + + let initial_public_accounts = default_public_accounts_for_wallet(); + let initial_private_accounts = default_private_accounts_for_wallet(); + let genesis_transactions = + genesis_from_accounts(&initial_public_accounts, &initial_private_accounts); + let sequencer_partial = SequencerPartialConfig::default(); + + let temp_indexer_dir = tempfile::tempdir().context("indexer temp dir")?; + let indexer_cfg = indexer_config(bedrock_addr, temp_indexer_dir.path().to_owned()) + .context("indexer config")?; + let indexer_handle = indexer_service::run_server(indexer_cfg, 0) + .await + .context("indexer run_server")?; + + let temp_sequencer_dir = tempfile::tempdir().context("sequencer temp dir")?; + let sequencer_cfg = sequencer_config( + sequencer_partial, + temp_sequencer_dir.path().to_owned(), + bedrock_addr, + genesis_transactions, + ) + .context("sequencer config")?; + let sequencer_handle = sequencer_service::run(sequencer_cfg, 0) + .await + .context("sequencer run")?; + + let temp_wallet_dir = tempfile::tempdir().context("wallet temp dir")?; + let mut wallet_cfg = wallet_config(sequencer_handle.addr()).context("wallet config")?; + // The default 30s poll interval is far too slow for a measurement run; + // shrink so the wallet sees new blocks within ~1s. + wallet_cfg.seq_poll_timeout = std::time::Duration::from_secs(1); + let wallet_cfg_str = + serde_json::to_string_pretty(&wallet_cfg).context("serialize wallet config")?; + let wallet_cfg_path = temp_wallet_dir.path().join("wallet_config.json"); + std::fs::write(&wallet_cfg_path, wallet_cfg_str).context("write wallet config")?; + let storage_path = temp_wallet_dir.path().join("storage.json"); + let password = "bench_pass".to_owned(); + let (mut wallet, _mnemonic) = WalletCore::new_init_storage( + wallet_cfg_path, + storage_path, + Some(WalletConfigOverrides::default()), + &password, + ) + .context("wallet init")?; + // Mirror integration_tests::setup_wallet: import the initial accounts + // produced above so the wallet can reference them by AccountId in scenarios. + for (private_key, _balance) in &initial_public_accounts { + wallet + .storage_mut() + .key_chain_mut() + .add_imported_public_account(private_key.clone()); + } + for private_account in &initial_private_accounts { + wallet + .storage_mut() + .key_chain_mut() + .add_imported_private_account( + private_account.key_chain.clone(), + None, + private_account.identifier, + nssa::Account::default(), + ); + } + wallet + .store_persistent_data() + .context("wallet store persistent")?; + + let sequencer_url = + addr_to_url(UrlProtocol::Http, sequencer_handle.addr()).context("sequencer url")?; + let sequencer_client = SequencerClientBuilder::default() + .build(sequencer_url) + .context("build sequencer client")?; + + Ok(Self { + sequencer_client, + wallet, + wallet_password: password, + sequencer_handle: Some(sequencer_handle), + indexer_handle, + temp_indexer_dir, + temp_sequencer_dir, + temp_wallet_dir, + }) + } + + pub const fn wallet_mut(&mut self) -> &mut WalletCore { + &mut self.wallet + } + + pub const fn sequencer_client(&self) -> &SequencerClient { + &self.sequencer_client + } + + pub fn indexer_addr(&self) -> SocketAddr { + self.indexer_handle.addr() + } + + /// Recursively-sized bytes on disk for sequencer + indexer + wallet tempdirs. + pub fn disk_sizes(&self) -> DiskSizes { + DiskSizes { + sequencer_bytes: dir_size_bytes(self.temp_sequencer_dir.path()), + indexer_bytes: dir_size_bytes(self.temp_indexer_dir.path()), + wallet_bytes: dir_size_bytes(self.temp_wallet_dir.path()), + } + } +} + +#[derive(Debug, Clone, Copy, Default, Serialize)] +#[expect( + clippy::struct_field_names, + reason = "The `_bytes` suffix carries the unit and is preserved verbatim in JSON output." +)] +pub struct DiskSizes { + pub sequencer_bytes: u64, + pub indexer_bytes: u64, + pub wallet_bytes: u64, +} + +fn dir_size_bytes(path: &Path) -> u64 { + let mut total = 0_u64; + let Ok(entries) = std::fs::read_dir(path) else { + return 0; + }; + for entry in entries.flatten() { + let Ok(metadata) = entry.metadata() else { + continue; + }; + if metadata.is_file() { + total = total.saturating_add(metadata.len()); + } else if metadata.is_dir() { + total = total.saturating_add(dir_size_bytes(&entry.path())); + } else { + // Sockets, FIFOs, block/char devices: ignore. Symlinks are + // already followed by `is_file()` / `is_dir()`. + } + } + total +} + +impl Drop for BenchContext { + fn drop(&mut self) { + if let Some(handle) = self.sequencer_handle.take() + && !handle.is_healthy() + { + eprintln!("BenchContext drop: sequencer handle was unhealthy"); + } + if !self.indexer_handle.is_healthy() { + eprintln!("BenchContext drop: indexer handle was unhealthy"); + } + } +} diff --git a/tools/e2e_bench/src/harness.rs b/tools/e2e_bench/src/harness.rs new file mode 100644 index 00000000..bcdff7a7 --- /dev/null +++ b/tools/e2e_bench/src/harness.rs @@ -0,0 +1,297 @@ +//! Step / scenario timing primitives shared across scenarios. + +use std::time::{Duration, Instant}; + +use anyhow::{Result, bail}; +use common::transaction::NSSATransaction; +use sequencer_service_rpc::RpcClient as _; +use serde::Serialize; +use wallet::cli::SubcommandReturnValue; + +use crate::bench_context::BenchContext; + +const TX_INCLUSION_POLL_INTERVAL: Duration = Duration::from_millis(250); +const TX_INCLUSION_TIMEOUT: Duration = Duration::from_secs(120); + +/// Borsh-serialized sizes for one zone block fetched after a step. `block_bytes` +/// is the full Block (header + body + bedrock metadata) and is the closest +/// proxy we have to the L1 payload posted per block. `tx_bytes` is each contained +/// transaction split by variant — this is what the fee model's S_tx slot covers. +#[derive(Debug, Serialize, Clone, Default)] +pub struct BlockSize { + pub block_id: u64, + pub block_bytes: usize, + pub public_tx_bytes: Vec, + pub ppe_tx_bytes: Vec, + pub deploy_tx_bytes: Vec, +} + +#[derive(Debug, Serialize, Clone)] +pub struct StepResult { + pub label: String, + pub submit_ms: f64, + pub inclusion_ms: Option, + pub wallet_sync_ms: Option, + pub total_ms: f64, + pub tx_hash: Option, + /// Borsh sizes for every zone block produced during this step. + /// Empty for steps that don't advance the chain (e.g. RegisterAccount). + pub blocks: Vec, +} + +#[derive(Debug, Serialize, Default)] +pub struct ScenarioResult { + pub name: String, + pub setup_ms: f64, + pub steps: Vec, + pub total_ms: f64, + /// Disk sizes (sequencer / indexer / wallet tempdirs) sampled at scenario start. + pub disk_before: Option, + /// Disk sizes sampled at scenario end. + pub disk_after: Option, + /// Bedrock-finality latency: time from final-step inclusion to the indexer + /// reporting the sequencer tip as L1-finalised. Effectively measures the + /// sequencer→Bedrock posting + Bedrock finalisation + indexer L1 ingest path. + /// A value at the timeout (60s) means finalisation did not happen within the bench window. + pub bedrock_finality_ms: Option, +} + +impl ScenarioResult { + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + ..Default::default() + } + } + + pub fn push(&mut self, step: StepResult) { + self.total_ms += step.total_ms; + self.steps.push(step); + } +} + +/// Finish a timed wallet step. Records submit (the time between `started` +/// being captured and `ret` being received) and, if `ret` is a +/// [`SubcommandReturnValue::PrivacyPreservingTransfer`], polls the sequencer +/// for inclusion and records the inclusion latency. Returns a [`StepResult`]. +/// +/// Usage: +/// ```ignore +/// let started = Instant::now(); +/// let ret = wallet::cli::execute_subcommand(ctx.wallet_mut(), cmd).await?; +/// let step = finalize_step("label", started, ret, ctx).await?; +/// ``` +/// Begin a timed step. Capture this *before* submitting the wallet operation +/// so we can later subtract it from the post-submit block height to detect +/// when the chain has advanced past the tx's block. +pub async fn begin_step(ctx: &BenchContext) -> Result { + Ok(ctx.sequencer_client().get_last_block_id().await?) +} + +pub async fn finalize_step( + label: impl Into, + started: Instant, + pre_block_id: u64, + ret: &SubcommandReturnValue, + ctx: &mut BenchContext, +) -> Result { + let label = label.into(); + let submit_ms = started.elapsed().as_secs_f64() * 1_000.0; + + let mut tx_hash_str = None; + let mut inclusion_ms = None; + let mut wallet_sync_ms = None; + let mut blocks: Vec = Vec::new(); + + // For non-account-create steps (anything that produces a tx_hash, or even + // `Empty` for public Token Send), wait for the chain to advance past the + // submission block so state is applied before the next step. We use + // get_last_block_id as the canonical "block has been produced and + // recorded" signal. + let should_wait_for_chain = !matches!(ret, SubcommandReturnValue::RegisterAccount { .. }); + if should_wait_for_chain { + if let SubcommandReturnValue::PrivacyPreservingTransfer { tx_hash } = ret { + tx_hash_str = Some(format!("{tx_hash}")); + } + let started_inclusion = Instant::now(); + wait_for_chain_advance(ctx, pre_block_id, 2).await?; + inclusion_ms = Some(started_inclusion.elapsed().as_secs_f64() * 1_000.0); + + let started_sync = Instant::now(); + sync_wallet_to_tip(ctx).await?; + wallet_sync_ms = Some(started_sync.elapsed().as_secs_f64() * 1_000.0); + + // Capture block-byte and per-tx-byte sizes for every block produced + // during this step. We intentionally capture all blocks, including + // empty clock-only ticks: the empty-block baseline lets the fee model + // back out the per-tx contribution. + let tip = ctx.sequencer_client().get_last_block_id().await?; + for block_id in (pre_block_id.saturating_add(1))..=tip { + if let Some(block) = ctx.sequencer_client().get_block(block_id).await? { + let block_bytes = borsh::to_vec(&block).map_or(0, |v| v.len()); + let mut sz = BlockSize { + block_id, + block_bytes, + public_tx_bytes: Vec::new(), + ppe_tx_bytes: Vec::new(), + deploy_tx_bytes: Vec::new(), + }; + for tx in &block.body.transactions { + let n = borsh::to_vec(tx).map_or(0, |v| v.len()); + match tx { + NSSATransaction::Public(_) => sz.public_tx_bytes.push(n), + NSSATransaction::PrivacyPreserving(_) => sz.ppe_tx_bytes.push(n), + NSSATransaction::ProgramDeployment(_) => sz.deploy_tx_bytes.push(n), + } + } + blocks.push(sz); + } + } + } + + Ok(StepResult { + label, + submit_ms, + inclusion_ms, + wallet_sync_ms, + total_ms: started.elapsed().as_secs_f64() * 1_000.0, + tx_hash: tx_hash_str, + blocks, + }) +} + +/// Wait for `get_last_block_id` to advance by at least `min_blocks` from `from_block_id`. +pub async fn wait_for_chain_advance( + ctx: &BenchContext, + from_block_id: u64, + min_blocks: u64, +) -> Result<()> { + let target = from_block_id.saturating_add(min_blocks); + let deadline = Instant::now() + TX_INCLUSION_TIMEOUT; + loop { + match ctx.sequencer_client().get_last_block_id().await { + Ok(current) if current >= target => return Ok(()), + Ok(_) => {} + Err(err) => eprintln!("get_last_block_id error (continuing poll): {err:#}"), + } + if Instant::now() > deadline { + bail!( + "chain did not advance from {from_block_id} to at least {target} within {TX_INCLUSION_TIMEOUT:?}" + ); + } + tokio::time::sleep(TX_INCLUSION_POLL_INTERVAL).await; + } +} + +async fn sync_wallet_to_tip(ctx: &mut BenchContext) -> Result<()> { + let last_block = ctx.sequencer_client().get_last_block_id().await?; + ctx.wallet_mut().sync_to_block(last_block).await?; + Ok(()) +} + +pub fn print_table(result: &ScenarioResult) { + let label_width = result + .steps + .iter() + .map(|s| s.label.len()) + .max() + .unwrap_or(0) + .max("step".len()); + + println!( + "\nScenario: {} (setup {:.1} ms ({:.2}s), total {:.1} ms ({:.2}s))", + result.name, + result.setup_ms, + result.setup_ms / 1_000.0, + result.total_ms, + result.total_ms / 1_000.0, + ); + println!( + "{:10} {:>12} {:>10} {:>16}", + "step", + "submit_ms", + "inclusion_ms", + "sync_ms", + "total_ms (s)", + lw = label_width, + ); + println!("{}", "-".repeat(label_width + 62)); + for s in &result.steps { + let inclusion = s + .inclusion_ms + .map_or_else(|| "-".to_owned(), |v| format!("{v:.1}")); + let sync = s + .wallet_sync_ms + .map_or_else(|| "-".to_owned(), |v| format!("{v:.1}")); + let total = format!("{:.1} ({:.2}s)", s.total_ms, s.total_ms / 1_000.0); + println!( + "{:10.1} {:>12} {:>10} {:>16}", + s.label, + s.submit_ms, + inclusion, + sync, + total, + lw = label_width, + ); + } + + print_size_summary(result); +} + +/// Aggregate borsh sizes per scenario: total/mean/min/max block bytes, and +/// per-tx bytes split by variant. Empty if no blocks were captured. +fn print_size_summary(result: &ScenarioResult) { + let blocks: Vec<&BlockSize> = result.steps.iter().flat_map(|s| s.blocks.iter()).collect(); + if blocks.is_empty() { + return; + } + + let block_bytes: Vec = blocks.iter().map(|b| b.block_bytes).collect(); + let total_block_bytes: usize = block_bytes.iter().sum(); + let mean_block = mean_usize(&block_bytes); + let min_block = block_bytes.iter().copied().min().unwrap_or(0); + let max_block = block_bytes.iter().copied().max().unwrap_or(0); + + let public: Vec = blocks + .iter() + .flat_map(|b| b.public_tx_bytes.iter().copied()) + .collect(); + let ppe: Vec = blocks + .iter() + .flat_map(|b| b.ppe_tx_bytes.iter().copied()) + .collect(); + let deploy: Vec = blocks + .iter() + .flat_map(|b| b.deploy_tx_bytes.iter().copied()) + .collect(); + + println!( + "\nBlock + tx size summary ({} blocks captured):", + blocks.len() + ); + println!( + " block_bytes: total={total_block_bytes}, mean={mean_block}, min={min_block}, max={max_block}", + ); + print_tx_line("public_tx_bytes ", &public); + print_tx_line("ppe_tx_bytes ", &ppe); + print_tx_line("deploy_tx_bytes ", &deploy); +} + +fn print_tx_line(label: &str, samples: &[usize]) { + if samples.is_empty() { + println!(" {label}: (none)"); + return; + } + let total: usize = samples.iter().sum(); + let mean = mean_usize(samples); + let min = samples.iter().copied().min().unwrap_or(0); + let max = samples.iter().copied().max().unwrap_or(0); + println!( + " {label}: n={}, total={total}, mean={mean}, min={min}, max={max}", + samples.len() + ); +} + +fn mean_usize(xs: &[usize]) -> usize { + xs.iter().sum::().checked_div(xs.len()).unwrap_or(0) +} diff --git a/tools/e2e_bench/src/main.rs b/tools/e2e_bench/src/main.rs new file mode 100644 index 00000000..31ea1189 --- /dev/null +++ b/tools/e2e_bench/src/main.rs @@ -0,0 +1,229 @@ +//! End-to-end LEZ scenario bench. +//! +//! Spins up the full stack (native Bedrock node launched per-scenario via +//! `BedrockHandle` + in-process sequencer + indexer + wallet via +//! `BenchContext`) and drives the wallet through configurable scenarios that +//! mirror real user flows. Times each step and records borsh-serialized +//! block + tx sizes per scenario. +//! +//! Required env vars (no defaults; see `tools/e2e_bench/README.md`): +//! LEZ_BEDROCK_BIN absolute path to logos-blockchain-node. +//! LEZ_BEDROCK_CONFIG_DIR directory with node-config.yaml + deployment template. +//! +//! Run examples: +//! RISC0_DEV_MODE=1 cargo run --release -p e2e_bench -- --scenario all. +//! cargo run --release -p e2e_bench -- --scenario amm. +//! +//! `RISC0_DEV_MODE=1` skips proving and produces latency-only numbers in +//! ~minutes; omitting it produces realistic proving-inclusive numbers but +//! the run takes much longer. + +#![expect( + clippy::arbitrary_source_item_ordering, + clippy::arithmetic_side_effects, + clippy::as_conversions, + clippy::doc_markdown, + clippy::float_arithmetic, + clippy::let_underscore_must_use, + clippy::let_underscore_untyped, + clippy::missing_const_for_fn, + clippy::print_stderr, + clippy::print_stdout, + clippy::single_call_fn, + clippy::single_match_else, + clippy::std_instead_of_core, + clippy::too_many_lines, + clippy::wildcard_enum_match_arm, + reason = "Bench tool: matches test-style fixture code" +)] + +use std::{path::PathBuf, time::Duration}; + +use anyhow::{Context as _, Result}; +use bedrock_handle::BedrockHandle; +use bench_context::BenchContext; +use clap::{Parser, ValueEnum}; +use harness::ScenarioResult; +use serde::Serialize; + +mod bedrock_handle; +mod bench_context; +mod harness; +mod scenarios; + +#[derive(Copy, Clone, Debug, ValueEnum)] +enum ScenarioName { + Token, + Amm, + Fanout, + Private, + Parallel, + All, +} + +#[derive(Parser, Debug)] +#[command(about = "End-to-end LEZ scenario bench")] +struct Cli { + /// Which scenario(s) to run. + #[arg(long, value_enum, default_value_t = ScenarioName::All)] + scenario: ScenarioName, + + /// Optional JSON output path. Defaults to /target/e2e_bench.json. + #[arg(long)] + json_out: Option, +} + +#[derive(Debug, Serialize)] +struct BenchRunReport { + risc0_dev_mode: bool, + scenarios: Vec, + total_wall_seconds: f64, +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<()> { + // integration_tests initializes env_logger via a LazyLock, so we leave logger + // setup to it. Set RUST_LOG=info before running to see logs. + + let cli = Cli::parse(); + let risc0_dev_mode = std::env::var("RISC0_DEV_MODE").is_ok_and(|v| !v.is_empty() && v != "0"); + + eprintln!( + "e2e_bench: scenario={:?}, RISC0_DEV_MODE={}", + cli.scenario, + if risc0_dev_mode { "1" } else { "unset/0" } + ); + + let to_run: Vec = match cli.scenario { + ScenarioName::All => vec![ + ScenarioName::Token, + ScenarioName::Amm, + ScenarioName::Fanout, + ScenarioName::Private, + ScenarioName::Parallel, + ], + other => vec![other], + }; + + let overall_started = std::time::Instant::now(); + let mut all_results = Vec::with_capacity(to_run.len()); + + for name in to_run { + eprintln!("\n=== running scenario: {name:?} ==="); + let setup_started = std::time::Instant::now(); + // Spawn a fresh Bedrock node for this scenario. Each scenario therefore + // starts with an empty chain so the indexer never has a backlog from a + // prior scenario. + let bedrock = BedrockHandle::launch_fresh() + .await + .with_context(|| format!("failed to spawn Bedrock for scenario {name:?}"))?; + let bedrock_addr_string = format!("{}", bedrock.addr()); + // Safety: we restore the previous LEZ_BEDROCK_ADDR value (if any) at scenario teardown. + // SAFETY: this happens before any threaded setup that reads env. + unsafe { + std::env::set_var("LEZ_BEDROCK_ADDR", &bedrock_addr_string); + } + + let mut ctx = BenchContext::new() + .await + .with_context(|| format!("failed to setup BenchContext for scenario {name:?}"))?; + let setup_ms = elapsed_ms(setup_started); + eprintln!("setup: {setup_ms:.1} ms"); + + let disk_before = ctx.disk_sizes(); + let mut result = run_scenario(name, setup_ms, &mut ctx).await?; + result.disk_before = Some(disk_before); + result.disk_after = Some(ctx.disk_sizes()); + result.bedrock_finality_ms = Some(measure_bedrock_finality(&ctx).await?); + harness::print_table(&result); + all_results.push(result); + + drop(ctx); + drop(bedrock); + // Give Bedrock a moment to shut down before the next scenario. + tokio::time::sleep(Duration::from_secs(2)).await; + } + + let total_wall_seconds = overall_started.elapsed().as_secs_f64(); + eprintln!("\nTotal wall time: {total_wall_seconds:.1}s"); + + let report = BenchRunReport { + risc0_dev_mode, + scenarios: all_results, + total_wall_seconds, + }; + + let out_path = match cli.json_out { + Some(p) => p, + None => { + let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .canonicalize()?; + let suffix = if risc0_dev_mode { "dev" } else { "prove" }; + workspace_root + .join("target") + .join(format!("e2e_bench_{suffix}.json")) + } + }; + if let Some(parent) = out_path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&out_path, serde_json::to_string_pretty(&report)?)?; + eprintln!("\nJSON written to {}", out_path.display()); + + Ok(()) +} + +async fn run_scenario( + name: ScenarioName, + setup_ms: f64, + ctx: &mut BenchContext, +) -> Result { + let result = match name { + ScenarioName::Token => scenarios::token::run(ctx).await?, + ScenarioName::Amm => scenarios::amm::run(ctx).await?, + ScenarioName::Fanout => scenarios::fanout::run(ctx).await?, + ScenarioName::Private => scenarios::private::run(ctx).await?, + ScenarioName::Parallel => scenarios::parallel::run(ctx).await?, + ScenarioName::All => unreachable!("dispatched above"), + }; + Ok(ScenarioResult { setup_ms, ..result }) +} + +fn elapsed_ms(t: std::time::Instant) -> f64 { + t.elapsed().as_secs_f64() * 1_000.0 +} + +/// Poll the indexer's L1-finalised block id until it catches up with the +/// sequencer's last block id. This is effectively the sequencer→Bedrock posting +/// plus Bedrock finalisation plus indexer ingest latency. +async fn measure_bedrock_finality(ctx: &BenchContext) -> Result { + use indexer_service_rpc::RpcClient as _; + use jsonrpsee::ws_client::WsClientBuilder; + use sequencer_service_rpc::RpcClient as _; + + let indexer_url = format!("ws://{}", ctx.indexer_addr()); + let indexer_ws = WsClientBuilder::default() + .build(&indexer_url) + .await + .context("connect indexer WS")?; + let sequencer_tip = ctx.sequencer_client().get_last_block_id().await?; + + let started = std::time::Instant::now(); + let deadline = started + Duration::from_secs(60); + loop { + match indexer_ws.get_last_finalized_block_id().await { + Ok(Some(b)) if b >= sequencer_tip => { + return Ok(started.elapsed().as_secs_f64() * 1_000.0); + } + Ok(_) => {} + Err(err) => eprintln!("indexer last_synced poll error: {err:#}"), + } + if std::time::Instant::now() > deadline { + eprintln!("indexer did not catch up to {sequencer_tip} within 60s"); + return Ok(started.elapsed().as_secs_f64() * 1_000.0); + } + tokio::time::sleep(Duration::from_millis(200)).await; + } +} diff --git a/tools/e2e_bench/src/scenarios/amm.rs b/tools/e2e_bench/src/scenarios/amm.rs new file mode 100644 index 00000000..b92bd823 --- /dev/null +++ b/tools/e2e_bench/src/scenarios/amm.rs @@ -0,0 +1,200 @@ +//! AMM swap flow: setup two tokens, create pool, swap, add liquidity, remove liquidity. + +use std::time::Instant; + +use anyhow::{Result, bail}; +use integration_tests::public_mention; +use wallet::cli::{ + Command, SubcommandReturnValue, + account::{AccountSubcommand, NewSubcommand}, + programs::{amm::AmmProgramAgnosticSubcommand, token::TokenProgramAgnosticSubcommand}, +}; + +use crate::harness::{ScenarioResult, finalize_step}; + +pub async fn run(ctx: &mut crate::bench_context::BenchContext) -> Result { + let mut result = ScenarioResult::new("amm_swap_flow"); + + let def_a = new_public_account(ctx, &mut result, "create_acc_def_a").await?; + let supply_a = new_public_account(ctx, &mut result, "create_acc_supply_a").await?; + let user_a = new_public_account(ctx, &mut result, "create_acc_user_a").await?; + + let def_b = new_public_account(ctx, &mut result, "create_acc_def_b").await?; + let supply_b = new_public_account(ctx, &mut result, "create_acc_supply_b").await?; + let user_b = new_public_account(ctx, &mut result, "create_acc_user_b").await?; + + let user_lp = new_public_account(ctx, &mut result, "create_acc_user_lp").await?; + + timed_token_new(ctx, &mut result, "token_a_new", def_a, supply_a, "TokA").await?; + timed_token_send( + ctx, + &mut result, + "token_a_fund_user", + supply_a, + user_a, + 1_000, + ) + .await?; + + timed_token_new(ctx, &mut result, "token_b_new", def_b, supply_b, "TokB").await?; + timed_token_send( + ctx, + &mut result, + "token_b_fund_user", + supply_b, + user_b, + 1_000, + ) + .await?; + + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::AMM(AmmProgramAgnosticSubcommand::New { + user_holding_a: public_mention(user_a), + user_holding_b: public_mention(user_b), + user_holding_lp: public_mention(user_lp), + balance_a: 300, + balance_b: 300, + }), + ) + .await?; + let step = finalize_step("amm_new_pool", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::AMM(AmmProgramAgnosticSubcommand::SwapExactInput { + user_holding_a: public_mention(user_a), + user_holding_b: public_mention(user_b), + amount_in: 50, + min_amount_out: 1, + token_definition: def_a, + }), + ) + .await?; + let step = finalize_step("amm_swap_exact_input", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::AMM(AmmProgramAgnosticSubcommand::AddLiquidity { + user_holding_a: public_mention(user_a), + user_holding_b: public_mention(user_b), + user_holding_lp: public_mention(user_lp), + min_amount_lp: 1, + max_amount_a: 100, + max_amount_b: 100, + }), + ) + .await?; + let step = finalize_step("amm_add_liquidity", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::AMM(AmmProgramAgnosticSubcommand::RemoveLiquidity { + user_holding_a: public_mention(user_a), + user_holding_b: public_mention(user_b), + user_holding_lp: public_mention(user_lp), + balance_lp: 50, + min_amount_a: 1, + min_amount_b: 1, + }), + ) + .await?; + let step = finalize_step("amm_remove_liquidity", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + Ok(result) +} + +async fn new_public_account( + ctx: &mut crate::bench_context::BenchContext, + result: &mut ScenarioResult, + label: &str, +) -> Result { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Account(AccountSubcommand::New(NewSubcommand::Public { + cci: None, + label: None, + })), + ) + .await?; + let step = finalize_step(label, started, pre_block, &ret, ctx).await?; + result.push(step); + match ret { + SubcommandReturnValue::RegisterAccount { account_id } => Ok(account_id), + other => bail!("expected RegisterAccount, got {other:?}"), + } +} + +async fn timed_token_new( + ctx: &mut crate::bench_context::BenchContext, + result: &mut ScenarioResult, + label: &str, + def_id: nssa::AccountId, + supply_id: nssa::AccountId, + name: &str, +) -> Result<()> { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::New { + definition_account_id: public_mention(def_id), + supply_account_id: public_mention(supply_id), + name: name.to_owned(), + total_supply: 10_000, + }), + ) + .await?; + let step = finalize_step(label, started, pre_block, &ret, ctx).await?; + result.push(step); + Ok(()) +} + +async fn timed_token_send( + ctx: &mut crate::bench_context::BenchContext, + result: &mut ScenarioResult, + label: &str, + from_id: nssa::AccountId, + to_id: nssa::AccountId, + amount: u128, +) -> Result<()> { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::Send { + from: public_mention(from_id), + to: Some(public_mention(to_id)), + to_npk: None, + to_vpk: None, + to_identifier: Some(0), + amount, + }), + ) + .await?; + let step = finalize_step(label, started, pre_block, &ret, ctx).await?; + result.push(step); + Ok(()) +} diff --git a/tools/e2e_bench/src/scenarios/fanout.rs b/tools/e2e_bench/src/scenarios/fanout.rs new file mode 100644 index 00000000..adede185 --- /dev/null +++ b/tools/e2e_bench/src/scenarios/fanout.rs @@ -0,0 +1,90 @@ +//! Multi-recipient fanout: one funded supply pays 10 distinct recipients. + +use std::time::Instant; + +use anyhow::{Result, bail}; +use integration_tests::public_mention; +use wallet::cli::{ + Command, SubcommandReturnValue, + account::{AccountSubcommand, NewSubcommand}, + programs::token::TokenProgramAgnosticSubcommand, +}; + +use crate::harness::{ScenarioResult, finalize_step}; + +const FANOUT_COUNT: usize = 10; +const AMOUNT_PER_TRANSFER: u128 = 100; + +pub async fn run(ctx: &mut crate::bench_context::BenchContext) -> Result { + let mut result = ScenarioResult::new("multi_recipient_fanout"); + + let def_id = new_public_account(ctx, &mut result, "create_acc_def").await?; + let supply_id = new_public_account(ctx, &mut result, "create_acc_supply").await?; + + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::New { + definition_account_id: public_mention(def_id), + supply_account_id: public_mention(supply_id), + name: "FanoutToken".to_owned(), + total_supply: 10_000_000, + }), + ) + .await?; + let step = finalize_step("token_new_fungible", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + let mut recipients = Vec::with_capacity(FANOUT_COUNT); + for i in 0..FANOUT_COUNT { + let id = new_public_account(ctx, &mut result, &format!("create_recipient_{i:02}")).await?; + recipients.push(id); + } + + for (i, recipient_id) in recipients.iter().enumerate() { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::Send { + from: public_mention(supply_id), + to: Some(public_mention(*recipient_id)), + to_npk: None, + to_vpk: None, + to_identifier: Some(0), + amount: AMOUNT_PER_TRANSFER, + }), + ) + .await?; + let step = finalize_step(format!("transfer_{i:02}"), started, pre_block, &ret, ctx).await?; + result.push(step); + } + + Ok(result) +} + +async fn new_public_account( + ctx: &mut crate::bench_context::BenchContext, + result: &mut ScenarioResult, + label: &str, +) -> Result { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Account(AccountSubcommand::New(NewSubcommand::Public { + cci: None, + label: None, + })), + ) + .await?; + let step = finalize_step(label, started, pre_block, &ret, ctx).await?; + result.push(step); + match ret { + SubcommandReturnValue::RegisterAccount { account_id } => Ok(account_id), + other => bail!("expected RegisterAccount, got {other:?}"), + } +} diff --git a/tools/e2e_bench/src/scenarios/mod.rs b/tools/e2e_bench/src/scenarios/mod.rs new file mode 100644 index 00000000..0536a250 --- /dev/null +++ b/tools/e2e_bench/src/scenarios/mod.rs @@ -0,0 +1,7 @@ +//! Scenarios driven by the e2e bench. + +pub mod amm; +pub mod fanout; +pub mod parallel; +pub mod private; +pub mod token; diff --git a/tools/e2e_bench/src/scenarios/parallel.rs b/tools/e2e_bench/src/scenarios/parallel.rs new file mode 100644 index 00000000..43bd25ac --- /dev/null +++ b/tools/e2e_bench/src/scenarios/parallel.rs @@ -0,0 +1,188 @@ +//! Parallel-fanout throughput scenario. N distinct senders each transfer one token +//! to one recipient. Submission is serialised through the single wallet but does +//! not wait for chain advance between submits, so all N txs land in the same +//! block (up to `max_num_tx_in_block`). Measures observed throughput. + +use std::time::Instant; + +use anyhow::{Result, bail}; +use common::transaction::NSSATransaction; +use integration_tests::public_mention; +use sequencer_service_rpc::RpcClient as _; +use wallet::cli::{ + Command, SubcommandReturnValue, + account::{AccountSubcommand, NewSubcommand}, + programs::token::TokenProgramAgnosticSubcommand, +}; + +use crate::{ + bench_context::BenchContext, + harness::{BlockSize, ScenarioResult, StepResult, finalize_step}, +}; + +const PARALLEL_FANOUT_N: usize = 10; +const AMOUNT_PER_TRANSFER: u128 = 100; + +pub async fn run(ctx: &mut BenchContext) -> Result { + let mut result = ScenarioResult::new("parallel_fanout"); + + // Setup: definition, master supply, N parallel supplies, N recipients. + let def_id = new_public_account(ctx, &mut result, "create_acc_def").await?; + let master_id = new_public_account(ctx, &mut result, "create_acc_master").await?; + + let mut senders = Vec::with_capacity(PARALLEL_FANOUT_N); + for i in 0..PARALLEL_FANOUT_N { + let id = new_public_account(ctx, &mut result, &format!("create_sender_{i:02}")).await?; + senders.push(id); + } + let mut recipients = Vec::with_capacity(PARALLEL_FANOUT_N); + for i in 0..PARALLEL_FANOUT_N { + let id = new_public_account(ctx, &mut result, &format!("create_recipient_{i:02}")).await?; + recipients.push(id); + } + + // Mint full supply into master. + let total_mint: u128 = (PARALLEL_FANOUT_N as u128) * AMOUNT_PER_TRANSFER * 10; + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::New { + definition_account_id: public_mention(def_id), + supply_account_id: public_mention(master_id), + name: "ParToken".to_owned(), + total_supply: total_mint, + }), + ) + .await?; + let step = finalize_step("token_new_fungible", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + // Fund each sender from master. Serial; this is setup, not measured throughput. + for (i, sender_id) in senders.iter().enumerate() { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::Send { + from: public_mention(master_id), + to: Some(public_mention(*sender_id)), + to_npk: None, + to_vpk: None, + to_identifier: Some(0), + amount: AMOUNT_PER_TRANSFER * 5, + }), + ) + .await?; + let step = + finalize_step(format!("fund_sender_{i:02}"), started, pre_block, &ret, ctx).await?; + result.push(step); + } + + // The measured phase: submit N transfers as fast as possible, do not wait + // for chain advance between submits. The sequencer batches whatever lands in + // its mempool before block_create_timeout. + let pre_block_burst = ctx.sequencer_client().get_last_block_id().await?; + let burst_started = Instant::now(); + + // Submit all N back-to-back. Wallet serialises through `wallet_mut()`, but + // each sender has its own nonce so there are no collisions. + for (sender_id, recipient_id) in senders.iter().zip(recipients.iter()) { + wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::Send { + from: public_mention(*sender_id), + to: Some(public_mention(*recipient_id)), + to_npk: None, + to_vpk: None, + to_identifier: Some(0), + amount: AMOUNT_PER_TRANSFER, + }), + ) + .await?; + } + let all_submitted_at = Instant::now(); + let submit_duration_ms = (all_submitted_at - burst_started).as_secs_f64() * 1_000.0; + + // Wait for the chain to advance by at least 2 blocks past pre_block_burst. + // That guarantees the block holding our burst is sealed and applied. + crate::harness::wait_for_chain_advance(ctx, pre_block_burst, 2).await?; + let inclusion_done_at = Instant::now(); + let inclusion_after_submit_ms = (inclusion_done_at - all_submitted_at).as_secs_f64() * 1_000.0; + let burst_total_ms = (inclusion_done_at - burst_started).as_secs_f64() * 1_000.0; + + eprintln!( + "parallel_fanout: submitted {} txs in {:.1} ms, inclusion in {:.1} ms, total {:.1} ms", + senders.len(), + submit_duration_ms, + inclusion_after_submit_ms, + burst_total_ms, + ); + + // Capture every block produced during the burst window. This is the + // scenario where one block holds many txs, so block_bytes here is the + // most representative L1-payload-equivalent measurement we have. + let tip = ctx.sequencer_client().get_last_block_id().await?; + let mut blocks: Vec = Vec::new(); + for block_id in (pre_block_burst.saturating_add(1))..=tip { + if let Some(block) = ctx.sequencer_client().get_block(block_id).await? { + let block_bytes = borsh::to_vec(&block).map_or(0, |v| v.len()); + let mut sz = BlockSize { + block_id, + block_bytes, + public_tx_bytes: Vec::new(), + ppe_tx_bytes: Vec::new(), + deploy_tx_bytes: Vec::new(), + }; + for tx in &block.body.transactions { + let n = borsh::to_vec(tx).map_or(0, |v| v.len()); + match tx { + NSSATransaction::Public(_) => sz.public_tx_bytes.push(n), + NSSATransaction::PrivacyPreserving(_) => sz.ppe_tx_bytes.push(n), + NSSATransaction::ProgramDeployment(_) => sz.deploy_tx_bytes.push(n), + } + } + blocks.push(sz); + } + } + + // Synthesise a single summary "step" for the burst. Use the submit time + // for `submit_ms` and the inclusion-wait time for `inclusion_ms`. + let burst_step = StepResult { + label: format!("burst_{}_transfers", senders.len()), + submit_ms: submit_duration_ms, + inclusion_ms: Some(inclusion_after_submit_ms), + wallet_sync_ms: None, + total_ms: burst_total_ms, + tx_hash: None, + blocks, + }; + result.push(burst_step); + + Ok(result) +} + +async fn new_public_account( + ctx: &mut BenchContext, + result: &mut ScenarioResult, + label: &str, +) -> Result { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Account(AccountSubcommand::New(NewSubcommand::Public { + cci: None, + label: None, + })), + ) + .await?; + let step = finalize_step(label, started, pre_block, &ret, ctx).await?; + result.push(step); + match ret { + SubcommandReturnValue::RegisterAccount { account_id } => Ok(account_id), + other => bail!("expected RegisterAccount, got {other:?}"), + } +} diff --git a/tools/e2e_bench/src/scenarios/private.rs b/tools/e2e_bench/src/scenarios/private.rs new file mode 100644 index 00000000..2a154673 --- /dev/null +++ b/tools/e2e_bench/src/scenarios/private.rs @@ -0,0 +1,150 @@ +//! Private chained flow: shielded, deshielded, and private-to-private transfers. + +use std::time::Instant; + +use anyhow::{Result, bail}; +use integration_tests::{private_mention, public_mention}; +use wallet::cli::{ + Command, SubcommandReturnValue, + account::{AccountSubcommand, NewSubcommand}, + programs::token::TokenProgramAgnosticSubcommand, +}; + +use crate::harness::{ScenarioResult, finalize_step}; + +pub async fn run(ctx: &mut crate::bench_context::BenchContext) -> Result { + let mut result = ScenarioResult::new("private_chained_flow"); + + let def_id = new_public_account(ctx, &mut result, "create_acc_def").await?; + let supply_id = new_public_account(ctx, &mut result, "create_acc_supply").await?; + let public_recipient_id = + new_public_account(ctx, &mut result, "create_acc_pub_recipient").await?; + let private_a = new_private_account(ctx, &mut result, "create_acc_priv_a").await?; + let private_b = new_private_account(ctx, &mut result, "create_acc_priv_b").await?; + + // Mint into public supply. + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::New { + definition_account_id: public_mention(def_id), + supply_account_id: public_mention(supply_id), + name: "PrivToken".to_owned(), + total_supply: 1_000_000, + }), + ) + .await?; + let step = finalize_step("token_new_fungible", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + // Shielded transfer: public supply -> private_a. + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::Send { + from: public_mention(supply_id), + to: Some(private_mention(private_a)), + to_npk: None, + to_vpk: None, + to_identifier: Some(0), + amount: 1_000, + }), + ) + .await?; + let step = finalize_step("shielded_transfer", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + // Deshielded transfer: private_a -> public_recipient. + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::Send { + from: private_mention(private_a), + to: Some(public_mention(public_recipient_id)), + to_npk: None, + to_vpk: None, + to_identifier: Some(0), + amount: 100, + }), + ) + .await?; + let step = finalize_step("deshielded_transfer", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + // Private-to-private transfer: private_a -> private_b. + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::Send { + from: private_mention(private_a), + to: Some(private_mention(private_b)), + to_npk: None, + to_vpk: None, + to_identifier: Some(0), + amount: 200, + }), + ) + .await?; + let step = finalize_step("private_to_private", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + Ok(result) +} + +async fn new_public_account( + ctx: &mut crate::bench_context::BenchContext, + result: &mut ScenarioResult, + label: &str, +) -> Result { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Account(AccountSubcommand::New(NewSubcommand::Public { + cci: None, + label: None, + })), + ) + .await?; + let step = finalize_step(label, started, pre_block, &ret, ctx).await?; + result.push(step); + match ret { + SubcommandReturnValue::RegisterAccount { account_id } => Ok(account_id), + other => bail!("expected RegisterAccount, got {other:?}"), + } +} + +async fn new_private_account( + ctx: &mut crate::bench_context::BenchContext, + result: &mut ScenarioResult, + label: &str, +) -> Result { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Account(AccountSubcommand::New(NewSubcommand::Private { + cci: None, + label: None, + })), + ) + .await?; + let step = finalize_step(label, started, pre_block, &ret, ctx).await?; + result.push(step); + match ret { + SubcommandReturnValue::RegisterAccount { account_id } => Ok(account_id), + other => bail!("expected RegisterAccount, got {other:?}"), + } +} diff --git a/tools/e2e_bench/src/scenarios/token.rs b/tools/e2e_bench/src/scenarios/token.rs new file mode 100644 index 00000000..c81bd8c1 --- /dev/null +++ b/tools/e2e_bench/src/scenarios/token.rs @@ -0,0 +1,127 @@ +//! Token onboarding scenario: create accounts, mint, public transfer, private transfer. + +use std::time::Instant; + +use anyhow::{Result, bail}; +use integration_tests::{private_mention, public_mention}; +use wallet::cli::{ + Command, SubcommandReturnValue, + account::{AccountSubcommand, NewSubcommand}, + programs::token::TokenProgramAgnosticSubcommand, +}; + +use crate::harness::{ScenarioResult, finalize_step}; + +pub async fn run(ctx: &mut crate::bench_context::BenchContext) -> Result { + let mut result = ScenarioResult::new("token_onboarding"); + + let definition_id = new_public_account(ctx, &mut result, "create_pub_definition").await?; + let supply_id = new_public_account(ctx, &mut result, "create_pub_supply").await?; + let recipient_id = new_public_account(ctx, &mut result, "create_pub_recipient").await?; + + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::New { + definition_account_id: public_mention(definition_id), + supply_account_id: public_mention(supply_id), + name: "BenchToken".to_owned(), + total_supply: 1_000_000, + }), + ) + .await?; + let step = finalize_step("token_new_fungible", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::Send { + from: public_mention(supply_id), + to: Some(public_mention(recipient_id)), + to_npk: None, + to_vpk: None, + to_identifier: Some(0), + amount: 1_000, + }), + ) + .await?; + let step = finalize_step("token_public_transfer", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + let private_recipient_id = + new_private_account(ctx, &mut result, "create_priv_recipient").await?; + + { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Token(TokenProgramAgnosticSubcommand::Send { + from: public_mention(supply_id), + to: Some(private_mention(private_recipient_id)), + to_npk: None, + to_vpk: None, + to_identifier: Some(0), + amount: 500, + }), + ) + .await?; + let step = finalize_step("token_shielded_transfer", started, pre_block, &ret, ctx).await?; + result.push(step); + } + + Ok(result) +} + +async fn new_public_account( + ctx: &mut crate::bench_context::BenchContext, + result: &mut ScenarioResult, + label: &str, +) -> Result { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Account(AccountSubcommand::New(NewSubcommand::Public { + cci: None, + label: None, + })), + ) + .await?; + let step = finalize_step(label, started, pre_block, &ret, ctx).await?; + result.push(step); + match ret { + SubcommandReturnValue::RegisterAccount { account_id } => Ok(account_id), + other => bail!("expected RegisterAccount, got {other:?}"), + } +} + +async fn new_private_account( + ctx: &mut crate::bench_context::BenchContext, + result: &mut ScenarioResult, + label: &str, +) -> Result { + let pre_block = crate::harness::begin_step(ctx).await?; + let started = Instant::now(); + let ret = wallet::cli::execute_subcommand( + ctx.wallet_mut(), + Command::Account(AccountSubcommand::New(NewSubcommand::Private { + cci: None, + label: None, + })), + ) + .await?; + let step = finalize_step(label, started, pre_block, &ret, ctx).await?; + result.push(step); + match ret { + SubcommandReturnValue::RegisterAccount { account_id } => Ok(account_id), + other => bail!("expected RegisterAccount, got {other:?}"), + } +}