diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88be5dab..1a075d77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -353,3 +353,34 @@ jobs: exit 1 fi echo "βœ… Artifacts are up to date" + + # The dashboard generator is plain Rust (no Docker), so this runs directly in + # the CI image container rather than through `run-in-ci-image`. + dashboards: + needs: ci-image + runs-on: ubuntu-latest + container: + image: ${{ needs.ci-image.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + timeout-minutes: 30 + + name: dashboards + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.head.sha || github.head_ref }} + + - name: Regenerate dashboards + run: just regenerate-dashboards + + - name: Check if dashboards match repository + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + if ! git diff --exit-code monitoring/grafana/dashboards/; then + echo "❌ Dashboards in the repository are out of date!" + echo "Please run 'just regenerate-dashboards' and commit the changes." + exit 1 + fi + echo "βœ… Dashboards are up to date" diff --git a/.gitignore b/.gitignore index f32b258c..3befdb9c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ data/ rocksdb* sequencer/service/data/ storage.json +statistics.json result diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b15693b8..68843859 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,6 +6,10 @@ This document describes the guidelines for contributing to the project. We will If you have any questions, come say hi to our [Discord](https://discord.gg/tGJwgGrSPN)! +## Metrics + +We have guidelines about metrics, for more information refer to [metrics](docs/metrics/metrics.md). + ## Commit title format We use [Conventional Commits](https://www.conventionalcommits.org/). @@ -59,7 +63,7 @@ Could be squashed to an empty commit if they belong to the same PR. ## Default branch -By default all PRs must be directed into the `dev` branch. This helps us to keep releases stable. +By default all PRs must be directed into the `dev` branch. This helps us to keep releases stable. ## Branch workflow diff --git a/Cargo.lock b/Cargo.lock index e4eab65f..634cd99e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -794,6 +794,29 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-lc-rs" +version = "1.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "axum" version = "0.7.9" @@ -1589,6 +1612,15 @@ dependencies = [ "lee_core", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cmov" version = "0.5.4" @@ -2259,6 +2291,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dashboard_gen" +version = "0.1.0" +dependencies = [ + "clap", + "json-pretty-compact", + "sequencer_core_metrics", + "sequencer_service_metrics", + "serde", + "serde_json", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -2554,6 +2598,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -2858,6 +2908,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "evmap" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8" +dependencies = [ + "hashbag", + "left-right", + "smallvec", +] + [[package]] name = "example_program_deployment_methods" version = "0.1.0" @@ -3036,6 +3097,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.5.0" @@ -3072,6 +3139,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "2.0.0" @@ -3231,6 +3304,21 @@ dependencies = [ "num-traits", ] +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -3436,6 +3524,12 @@ dependencies = [ "byteorder", ] +[[package]] +name = "hashbag" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" + [[package]] name = "hashbrown" version = "0.12.3" @@ -3459,7 +3553,16 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", ] [[package]] @@ -3817,6 +3920,7 @@ dependencies = [ "hyper-util", "log", "rustls", + "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", @@ -4503,6 +4607,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-pretty-compact" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62a9c1f06b173b0da0ccc8cae00599d7b3ceda6e76d68be9b6bc2c941adafe0e" +dependencies = [ + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "jsonrpsee" version = "0.26.0" @@ -4890,6 +5004,17 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "left-right" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc015ded5d9b3054dbbdb63332cdd6ee42352ccef19e911e25117490e2f48ee" +dependencies = [ + "crossbeam-utils", + "loom", + "slab", +] + [[package]] name = "leptos" version = "0.8.19" @@ -5896,7 +6021,7 @@ dependencies = [ "multiaddr", "num-bigint 0.4.6", "serde", - "strum", + "strum 0.27.2", "thiserror 2.0.18", "time", "tracing", @@ -6435,6 +6560,19 @@ dependencies = [ "prost-types 0.13.5", ] +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber 0.3.23", +] + [[package]] name = "lru" version = "0.12.5" @@ -6613,6 +6751,7 @@ dependencies = [ name = "mempool" version = "0.1.0" dependencies = [ + "futures", "tokio", ] @@ -6643,6 +6782,56 @@ dependencies = [ "paste", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" +dependencies = [ + "base64 0.22.1", + "evmap", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "indexmap 2.14.0", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "rustls", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.16.1", + "metrics", + "quanta", + "rand 0.9.4", + "rand_xoshiro", + "rapidhash", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -7965,6 +8154,21 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quick-protobuf" version = "0.8.1" @@ -8177,6 +8381,33 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.12.1", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -8978,6 +9209,7 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -9042,6 +9274,7 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -9091,7 +9324,7 @@ dependencies = [ "serde", "serde_with", "sha2 0.10.9", - "strum", + "strum 0.27.2", "tempfile", "thiserror 2.0.18", "toml 0.8.23", @@ -9162,6 +9395,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -9271,6 +9510,7 @@ dependencies = [ "programs", "rand 0.8.6", "risc0-zkvm", + "sequencer_core_metrics", "serde", "serde_json", "storage", @@ -9285,6 +9525,15 @@ dependencies = [ "vault_core", ] +[[package]] +name = "sequencer_core_metrics" +version = "0.1.0" +dependencies = [ + "common", + "metrics", + "strum 0.28.0", +] + [[package]] name = "sequencer_service" version = "0.1.0" @@ -9301,14 +9550,23 @@ dependencies = [ "lee", "log", "mempool", + "metrics-exporter-prometheus", "programs", "sequencer_core", + "sequencer_service_metrics", "sequencer_service_protocol", "sequencer_service_rpc", "tokio", "tokio-util", ] +[[package]] +name = "sequencer_service_metrics" +version = "0.1.0" +dependencies = [ + "metrics", +] + [[package]] name = "sequencer_service_protocol" version = "0.1.0" @@ -9724,6 +9982,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -9923,7 +10187,16 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros", + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", ] [[package]] @@ -9938,6 +10211,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "subtle" version = "2.6.1" diff --git a/Cargo.toml b/Cargo.toml index 397eb965..d5ebbe48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,9 +18,11 @@ members = [ "lez/system_accounts", "lez/chain_state", "lez/sequencer/core", + "lez/sequencer/core/metrics", "lez/sequencer/service", "lez/sequencer/service/protocol", "lez/sequencer/service/rpc", + "lez/sequencer/service/metrics", "lez/indexer/core", "lez/indexer/service", "lez/indexer/service/protocol", @@ -67,6 +69,7 @@ members = [ "tools/crypto_primitives_bench", "tools/integration_bench", "tools/cross_zone_chat", + "tools/dashboard_gen", ] [workspace.dependencies] @@ -78,8 +81,10 @@ mempool = { path = "lez/mempool" } storage = { path = "lez/storage" } key_protocol = { path = "lee/key_protocol" } sequencer_core = { path = "lez/sequencer/core" } +sequencer_core_metrics = { path = "lez/sequencer/core/metrics" } sequencer_service_protocol = { path = "lez/sequencer/service/protocol" } sequencer_service_rpc = { path = "lez/sequencer/service/rpc" } +sequencer_service_metrics = { path = "lez/sequencer/service/metrics" } sequencer_service = { path = "lez/sequencer/service" } indexer_core = { path = "lez/indexer/core" } indexer_service = { path = "lez/indexer/service" } @@ -142,6 +147,8 @@ actix-rt = "*" lazy_static = "1.5.0" env_logger = "0.11" log = "0.4.28" +metrics = "0.24.6" +metrics-exporter-prometheus = "0.18.3" lru = "0.16.3" thiserror = "2.0" sha2 = "0.10.8" @@ -173,6 +180,7 @@ url = { version = "2.5.4", features = ["serde"] } tokio-retry = "0.3.0" schemars = "1.2" async-stream = "0.3.6" +strum = { version = "0.28.0", features = ["derive"] } logos-blockchain-common-http-client = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" } logos-blockchain-key-management-system-service = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" } diff --git a/Justfile b/Justfile index d436689d..4741f6c8 100644 --- a/Justfile +++ b/Justfile @@ -53,6 +53,13 @@ regenerate-test-fixture: @echo "πŸ§ͺ Regenerating test fixture" RISC0_DEV_MODE=1 cargo run -p test_fixtures --bin regenerate_test_fixture +# Regenerate the committed Grafana dashboards from the Rust generator +# (tools/dashboard_gen) and commit the result. CI checks these are up to date. +regenerate-dashboards: + @echo "πŸ“Š Regenerating Grafana dashboards" + @cargo build -q -p dashboard_gen + @cargo run -q -p dashboard_gen -- sequencer > monitoring/grafana/dashboards/sequencer.json + # Run criterion benches: fast crypto primitives, then the slow PPE verify (real proving setup). bench: @echo "πŸ“Š Running criterion benches" @@ -65,19 +72,24 @@ run-bedrock: @echo "⛓️ Running bedrock" docker compose up -# Run Sequencer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. -# Optional home/port let a second instance run off the same config, e.g. -# `just run-sequencer "" "$TMPDIR/lez-sequencer2" 3041` for the multi-sequencer demo. +# Run Prometheus + Grafana in docker. Grafana: http://localhost:3000 (anonymous +# admin), Prometheus: http://localhost:9090. Scrapes the sequencer's /metrics. +[working-directory: 'monitoring'] +run-monitoring: + @echo "πŸ“Š Running Prometheus (http://localhost:9090) + Grafana (http://localhost:3000)" + docker compose up + +# Run Sequencer. Extra args are forwarded to the binary. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. [working-directory: 'lez/sequencer/service'] -run-sequencer standalone="" home="" port="3040": +run-sequencer *args: @echo "🧠 Running sequencer" - @if [ "{{standalone}}" = "standalone" ]; then \ - echo "πŸ§ͺ Running in standalone mode"; \ - RUST_LOG=info cargo run --features standalone --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \ - else \ - echo "πŸš€ Running in normal mode"; \ - RUST_LOG=info cargo run --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \ - fi + RUST_LOG=info cargo run --release -p sequencer_service -- configs/debug/sequencer_config.json {{args}} + +# Run Sequencer with mocked Bedrock clients. Takes the same args as `run-sequencer`. +[working-directory: 'lez/sequencer/service'] +run-sequencer-standalone *args: + @echo "πŸ§ͺ Running sequencer in standalone mode" + RUST_LOG=info cargo run --features standalone --release -p sequencer_service -- configs/debug/sequencer_config.json {{args}} # Run Indexer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. [working-directory: 'lez/indexer/service'] @@ -103,6 +115,11 @@ run-wallet +args: @echo "πŸ”‘ Running wallet" LEE_WALLET_HOME_DIR=$(pwd)/configs/debug cargo run --release -p wallet -- {{args}} +# Query sequencer metrics in raw format. Useful for quick debugging. For a more detailed view, use `just run-monitoring`. +get-sequencer-metrics: + @echo "πŸ“Š Querying sequencer's metrics" + curl http://localhost:9000/metrics + # Import test accounts supplied in sequencer configuration. wallet-import-test-accounts: @echo "βš™οΈ Initializing accounts" @@ -143,4 +160,5 @@ clean: rm -rf lez/wallet/configs/debug/storage.json rm -rf lez/wallet/configs/debug/statistics.json rm -rf rocksdb* - cd bedrock && docker compose down -v + cd bedrock && docker compose down -v && cd .. + cd monitoring && docker compose down -v && cd .. diff --git a/docker-compose.yml b/docker-compose.yml index 3644b2aa..4da0d04c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,3 +11,5 @@ include: lez/indexer/service/docker-compose.yml - path: lez/explorer_service/docker-compose.yml + - path: + monitoring/docker-compose.yml diff --git a/docs/metrics/metrics.md b/docs/metrics/metrics.md new file mode 100644 index 00000000..4d758157 --- /dev/null +++ b/docs/metrics/metrics.md @@ -0,0 +1,113 @@ +# Metrics + +Services expose Prometheus metrics; Grafana dashboards are generated from Rust so panel queries and metric names cannot drift apart. + +## Metrics crates + +Every crate that emits metrics gets a sibling `metrics` crate β€” `lez/sequencer/core/metrics` β†’ `sequencer_core_metrics`. Each has two halves: + +| Module | Gated by | Contents | +|---|---|---| +| `names` | always compiled | `pub const BLOCKS_PRODUCED_TOTAL: &str = "blocks_produced_total";` β€” one const per metric | +| `record` | `record` feature | `record_*` / `increment_*` functions, plus `init()` | + +The emitting crate depends on it with `features = ["record"]`; consumers that only need the names (i.e. `dashboard_gen`) take the default features and pull in nothing. Dashboards reference the same consts the recording code does, so **renaming a metric is a compile error rather than a silently empty panel**. + +## Naming + +The recorder runs with `with_recommended_naming(true)`, which enforces Prometheus convention: + +| Kind | Suffix | Example | +|---|---|---| +| Counter | `_total` | `blocks_produced_total`, `submitted_transactions_total` | +| Histogram | unit | `block_creation_time_seconds` | +| Gauge | none | `mempool_size` | + +**Spell the suffix in the const.** The exporter appends a missing unit suffix to the *rendered* name, but bucket matchers (below) run against the registered name β€” a duration metric named without `_seconds` renders correctly yet silently gets the wrong buckets. + +## Metric types + +| Type | Use for | Example | +|---|---|---| +| Counter | monotonically increasing event counts | `mempool_failed_transactions_total` | +| Gauge | a value that moves both ways | `mempool_size`, `chain_height` (a reorg lowers it) | +| Histogram | distributions β€” latencies, sizes, per-batch counts | `mempool_transaction_application_time_seconds` | + +Each metric gets a private constructor plus a public recording wrapper, so its description, unit and labels are declared once: + +```rust +fn blocks_produced_total_counter() -> Counter { + counter!( + description: "Number of blocks produced by this sequencer and applied to the head", + unit: Unit::Count, + names::BLOCKS_PRODUCED_TOTAL + ) +} + +pub fn increment_blocks_produced_total() { + blocks_produced_total_counter().increment(1); +} +``` + +Labels are passed as `"origin" => <&'static str>::from(origin)`; keep them low-cardinality (enums, never IDs or hashes). + +## `init()` + +Each `record` module exposes `init()`, called once at startup after the recorder is installed. It publishes every metric at zero. + +This is not cosmetic. A metric only materialises when first touched, and `rate()`/`increase()` need a sample from *before* an increment to see it β€” a series that springs into existence at `1` reads as `0` until the second event, so the first one is lost forever. Zero-publishing also means an idle service exports `0` instead of nothing at all. + +For histograms, creating the handle publishes zeroed buckets without recording an observation (recording a fake `0` would skew the distribution). Label combinations must each be registered, so `init()` iterates the label enums via `strum::EnumIter`. + +## Metrics in libraries + +**Yes, record metrics from library crates.** The `metrics` facade is a no-op until a recorder is installed, so a library that records costs nothing to a consumer that never installs one β€” including tests. Libraries record; only the binary installs the exporter. + +## Exporter setup + +`sequencer_service`'s `main.rs` installs the Prometheus recorder on the config's `metrics_address` (default `0.0.0.0:9000`) with **explicit histogram buckets**. This matters: without buckets, `metrics-exporter-prometheus` renders histograms as rolling-window summaries whose quantiles **reset to `0`** once the window (default 60 s) drains β€” an idle period reads as "took 0 s" rather than "no data". With buckets you get real `_bucket`/`_sum`/`_count` counters that never decay, are aggregatable, and honour the dashboard's time range. + +Ladders are matched by name suffix, so a new timing metric is covered automatically: + +```rust +.set_buckets(COUNT_BUCKETS) // fallback +.set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), LATENCY_BUCKETS) +``` + +## `dashboard_gen` + +`tools/dashboard_gen` is a small Grafana dashboard builder plus the dashboard definitions. It prints JSON to stdout; the result is committed under `monitoring/grafana/dashboards/` and CI fails if it is stale. + +``` +src/lib.rs, schema.rs, styling.rs, unit.rs the builder library +src/dashboards/.rs one dashboard per module +src/main.rs CLI: `dashboard_gen sequencer` +``` + +Panels are built fluently, and every query is composed from the `names` consts: + +```rust +Panel::timeseries("Block production rate") + .width(18) + .target(rate_per_min(sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL, "blocks/min")) +``` + +Query helpers: `rate_per_min` for counters, `avg` for histograms, and `selected_percentile` for percentile lines β€” the latter reads a `percentile` dashboard dropdown created by `percentile_variable`, so one panel serves p50/p90/p95/p99 instead of drawing all four. Rate windows use `$__rate_interval`, which tracks the panel's zoom. + +**Extending it:** + +| Goal | Change | +|---|---| +| New panel | Add a `Panel::…` to a row in the dashboard module | +| New dashboard | `src/dashboards/.rs` with `pub fn dashboard()`, a `pub mod` line, a `DashboardKind` variant, and a `just regenerate-dashboards` line | +| Grafana option we don't model yet | Add the field to `schema.rs` and a setter on `Panel` (styling setters live in `styling.rs` and panic when handed a redundant default) | + +The builder deliberately models only the subset of Grafana's schema we use. + +## Justfile + +| Recipe | Purpose | +|---|---| +| `just regenerate-dashboards` | Rebuild the committed dashboard JSON. Run after touching metric names or dashboard code β€” CI checks it is current. | +| `just run-monitoring` | Prometheus (`:9090`) + Grafana (`:3000`, anonymous admin) in docker, scraping the sequencer every 5 s | +| `just get-sequencer-metrics` | `curl` the raw `/metrics` endpoint β€” quickest way to confirm a metric name and value | diff --git a/lez/common/src/transaction.rs b/lez/common/src/transaction.rs index 13b2ada5..9970bf7d 100644 --- a/lez/common/src/transaction.rs +++ b/lez/common/src/transaction.rs @@ -35,6 +35,15 @@ impl LeeTransaction { }) } + #[must_use] + pub const fn kind(&self) -> TxKind { + match self { + Self::Public(_) => TxKind::Public, + Self::PrivacyPreserving(_) => TxKind::PrivacyPreserving, + Self::ProgramDeployment(_) => TxKind::ProgramDeployment, + } + } + #[must_use] pub fn affected_public_account_ids(&self) -> Vec { match self { diff --git a/lez/mempool/Cargo.toml b/lez/mempool/Cargo.toml index a2f51bc0..47550fc9 100644 --- a/lez/mempool/Cargo.toml +++ b/lez/mempool/Cargo.toml @@ -12,3 +12,4 @@ tokio = { workspace = true, features = ["sync"] } [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +futures.workspace = true diff --git a/lez/mempool/src/lib.rs b/lez/mempool/src/lib.rs index 0006f2c3..c081e9e7 100644 --- a/lez/mempool/src/lib.rs +++ b/lez/mempool/src/lib.rs @@ -18,6 +18,19 @@ impl MemPool { (mem_pool, sender) } + /// Returns the total number of items in the mempool, including both the front buffer and the + /// channel. + #[must_use] + pub fn len(&self) -> usize { + self.front_buffer.len().saturating_add(self.receiver.len()) + } + + /// Returns true if the mempool is empty, false otherwise. + #[must_use] + pub fn is_empty(&self) -> bool { + self.front_buffer.is_empty() && self.receiver.is_empty() + } + /// Pop an item from the mempool first checking the front buffer (LIFO) then the channel (FIFO). pub fn pop(&mut self) -> Option { use tokio::sync::mpsc::error::TryRecvError; @@ -74,6 +87,7 @@ impl MemPoolHandle { #[cfg(test)] mod tests { + use futures::FutureExt as _; use tokio::test; use super::*; @@ -82,6 +96,7 @@ mod tests { async fn mempool_new() { let (mut pool, _handle): (MemPool, _) = MemPool::new(10); assert_eq!(pool.pop(), None); + assert_eq!(pool.len(), 0); } #[test] @@ -89,10 +104,12 @@ mod tests { let (mut pool, handle) = MemPool::new(10); handle.push(1).await.unwrap(); + assert_eq!(pool.len(), 1); let item = pool.pop(); assert_eq!(item, Some(1)); assert_eq!(pool.pop(), None); + assert_eq!(pool.len(), 0); } #[test] @@ -103,29 +120,23 @@ mod tests { handle.push(2).await.unwrap(); handle.push(3).await.unwrap(); + assert_eq!(pool.len(), 3); assert_eq!(pool.pop(), Some(1)); assert_eq!(pool.pop(), Some(2)); assert_eq!(pool.pop(), Some(3)); assert_eq!(pool.pop(), None); - } - - #[test] - async fn pop_empty() { - let (mut pool, _handle): (MemPool, _) = MemPool::new(10); - assert_eq!(pool.pop(), None); + assert_eq!(pool.len(), 0); } #[test] async fn max_size() { - let (mut pool, handle) = MemPool::new(2); + let (_pool, handle) = MemPool::new(2); handle.push(1).await.unwrap(); handle.push(2).await.unwrap(); - // This should block if buffer is full, but we'll use try_send in a real scenario - // For now, just verify we can pop items - assert_eq!(pool.pop(), Some(1)); - assert_eq!(pool.pop(), Some(2)); + // This should block if buffer is full + assert_eq!(handle.push(3).now_or_never(), None); } #[test] diff --git a/lez/sequencer/core/Cargo.toml b/lez/sequencer/core/Cargo.toml index 64b154f4..a2d8a21c 100644 --- a/lez/sequencer/core/Cargo.toml +++ b/lez/sequencer/core/Cargo.toml @@ -12,6 +12,7 @@ lee.workspace = true lee_core.workspace = true chain_state.workspace = true common.workspace = true +sequencer_core_metrics = { workspace = true, features = ["record"] } storage.workspace = true mempool.workspace = true logos-blockchain-zone-sdk.workspace = true diff --git a/lez/sequencer/core/metrics/Cargo.toml b/lez/sequencer/core/metrics/Cargo.toml new file mode 100644 index 00000000..c67496d3 --- /dev/null +++ b/lez/sequencer/core/metrics/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "sequencer_core_metrics" +version = "0.1.0" +edition = "2024" +license = { workspace = true } + +[lints] +workspace = true + +[features] +default = [] +# Enable metrics record +record = ["dep:common", "dep:metrics", "dep:strum"] + +[dependencies] +common = { workspace = true, optional = true } + +metrics = { workspace = true, optional = true } +strum = { workspace = true, optional = true } diff --git a/lez/sequencer/core/metrics/src/lib.rs b/lez/sequencer/core/metrics/src/lib.rs new file mode 100644 index 00000000..b8884fa2 --- /dev/null +++ b/lez/sequencer/core/metrics/src/lib.rs @@ -0,0 +1,9 @@ +//! This crate provides all metrics exposed by the sequencer core crate. + +#[cfg(feature = "record")] +pub use record::*; + +pub mod names; + +#[cfg(feature = "record")] +pub mod record; diff --git a/lez/sequencer/core/metrics/src/names.rs b/lez/sequencer/core/metrics/src/names.rs new file mode 100644 index 00000000..92b1595e --- /dev/null +++ b/lez/sequencer/core/metrics/src/names.rs @@ -0,0 +1,9 @@ +pub const BLOCK_CREATION_TIME: &str = "block_creation_time_seconds"; +pub const CHAIN_HEIGHT: &str = "chain_height"; +pub const BLOCKS_PRODUCED_TOTAL: &str = "blocks_produced_total"; +pub const MEMPOOL_SIZE: &str = "mempool_size"; +pub const MEMPOOL_MAX_SIZE: &str = "mempool_max_size"; +pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = + "mempool_transaction_application_time_seconds"; +pub const TRANSACTIONS_PER_BLOCK: &str = "transactions_per_block"; +pub const MEMPOOL_FAILED_TRANSACTIONS_TOTAL: &str = "mempool_failed_transactions_total"; diff --git a/lez/sequencer/core/metrics/src/record.rs b/lez/sequencer/core/metrics/src/record.rs new file mode 100644 index 00000000..96b3304e --- /dev/null +++ b/lez/sequencer/core/metrics/src/record.rs @@ -0,0 +1,167 @@ +#![expect( + clippy::cast_precision_loss, + clippy::as_conversions, + reason = "It's okay for metrics" +)] + +use std::time::Duration; + +use metrics::{Counter, Histogram, Unit, counter, gauge, histogram}; +use strum::IntoEnumIterator as _; + +use crate::names; + +#[derive(Debug, Clone, Copy, strum::IntoStaticStr, strum::EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum TransactionOrigin { + User, + Sequencer, +} + +#[derive(Debug, Clone, Copy, strum::IntoStaticStr, strum::EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum TxKind { + Public, + PrivacyPreserving, + ProgramDeployment, +} + +/// Whether applying a transaction to the block's working state succeeded. +#[derive(Debug, Clone, Copy, strum::IntoStaticStr, strum::EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum ApplyStatus { + Applied, + Failed, +} + +impl From for TxKind { + fn from(kind: common::transaction::TxKind) -> Self { + match kind { + common::transaction::TxKind::Public => Self::Public, + common::transaction::TxKind::PrivacyPreserving => Self::PrivacyPreserving, + common::transaction::TxKind::ProgramDeployment => Self::ProgramDeployment, + } + } +} + +/// Initialize metrics. +pub fn init() { + blocks_produced_total_counter().increment(0); + mempool_failed_transactions_total_counter().increment(0); + record_mempool_size(0); + record_chain_height(0); + + drop(block_creation_time_histogram()); + drop(transactions_per_block_histogram()); + for origin in TransactionOrigin::iter() { + for kind in TxKind::iter() { + for status in ApplyStatus::iter() { + drop(mempool_transaction_application_time_histogram( + origin, kind, status, + )); + } + } + } +} + +fn block_creation_time_histogram() -> Histogram { + histogram!( + description: "Time taken to create a block", + unit: Unit::Seconds, + names::BLOCK_CREATION_TIME + ) +} + +pub fn record_block_creation_time(duration: Duration) { + block_creation_time_histogram().record(duration.as_secs_f64()); +} + +/// Height of the chain head, which moves backwards on a reorg, hence a gauge. +pub fn record_chain_height(height: u64) { + gauge!( + description: "Height of the chain head", + unit: Unit::Count, + names::CHAIN_HEIGHT + ) + .set(height as f64); +} + +fn blocks_produced_total_counter() -> Counter { + counter!( + description: "Number of blocks produced by this sequencer and applied to the head", + unit: Unit::Count, + names::BLOCKS_PRODUCED_TOTAL + ) +} + +pub fn increment_blocks_produced_total() { + blocks_produced_total_counter().increment(1); +} + +pub fn record_mempool_size(size: usize) { + gauge!( + description: "Size of the mempool", + unit: Unit::Count, + names::MEMPOOL_SIZE + ) + .set(u64::try_from(size).expect("Mempool size should fit into u64") as f64); +} + +pub fn record_mempool_max_size(size: usize) { + gauge!( + description: "Configured maximum size of the mempool", + unit: Unit::Count, + names::MEMPOOL_MAX_SIZE + ) + .set(u64::try_from(size).expect("Mempool max size should fit into u64") as f64); +} + +fn mempool_transaction_application_time_histogram( + origin: TransactionOrigin, + kind: TxKind, + status: ApplyStatus, +) -> Histogram { + histogram!( + description: "Time taken to apply a mempool transaction", + unit: Unit::Seconds, + names::MEMPOOL_TRANSACTION_APPLICATION_TIME, + "origin" => <&'static str>::from(origin), + "kind" => <&'static str>::from(kind), + "status" => <&'static str>::from(status), + ) +} + +pub fn record_mempool_transaction_application_time( + origin: TransactionOrigin, + kind: TxKind, + status: ApplyStatus, + duration: Duration, +) { + mempool_transaction_application_time_histogram(origin, kind, status) + .record(duration.as_secs_f64()); +} + +fn transactions_per_block_histogram() -> Histogram { + histogram!( + description: "Number of transactions from mempool included in block", + unit: Unit::Count, + names::TRANSACTIONS_PER_BLOCK + ) +} + +pub fn record_transactions_per_block(count: usize) { + transactions_per_block_histogram() + .record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64); +} + +fn mempool_failed_transactions_total_counter() -> Counter { + counter!( + description: "Number of transactions from mempool that failed to be included in blocks", + unit: Unit::Count, + names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL + ) +} + +pub fn increment_mempool_failed_transactions_total() { + mempool_failed_transactions_total_counter().increment(1); +} diff --git a/lez/sequencer/core/src/config.rs b/lez/sequencer/core/src/config.rs index 60bd8502..35f790dc 100644 --- a/lez/sequencer/core/src/config.rs +++ b/lez/sequencer/core/src/config.rs @@ -1,6 +1,7 @@ use std::{ fs::File, io::BufReader, + net::{IpAddr, Ipv4Addr, SocketAddr}, path::{Path, PathBuf}, time::Duration, }; @@ -63,6 +64,9 @@ pub struct SequencerConfig { /// Cross-zone messaging configuration. `None` disables the watcher. #[serde(default)] pub cross_zone: Option, + /// Address the Prometheus metrics exporter binds to. + #[serde(default = "default_metrics_address")] + pub metrics_address: Option, } #[derive(Clone, Serialize, Deserialize)] @@ -77,6 +81,10 @@ pub struct BedrockConfig { } impl SequencerConfig { + /// Address [`Self::metrics_address`] falls back to when the config omits it. + pub const DEFAULT_METRICS_ADDRESS: SocketAddr = + SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 9000); + pub fn from_path(config_home: &Path) -> Result { let file = File::open(config_home)?; let reader = BufReader::new(file); @@ -88,3 +96,8 @@ impl SequencerConfig { const fn default_max_block_size() -> ByteSize { ByteSize::mib(1) } + +#[expect(clippy::unnecessary_wraps, reason = "Required by serde")] +const fn default_metrics_address() -> Option { + Some(SequencerConfig::DEFAULT_METRICS_ADDRESS) +} diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 34ca24a5..e38a19ca 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -80,6 +80,15 @@ pub enum TransactionOrigin { Sequencer, } +impl From for sequencer_core_metrics::TransactionOrigin { + fn from(origin: TransactionOrigin) -> Self { + match origin { + TransactionOrigin::User => Self::User, + TransactionOrigin::Sequencer => Self::Sequencer, + } + } +} + #[derive(Clone, Debug, BorshDeserialize)] struct DepositMetadata { recipient_id: lee::AccountId, @@ -144,6 +153,9 @@ impl SequencerCore { ) .expect("Failed to create database with genesis block"); + // Incrementing count for genesis. + sequencer_core_metrics::increment_blocks_produced_total(); + (store, genesis_state) } } @@ -195,6 +207,8 @@ impl SequencerCore { pub async fn start_from_config( config: SequencerConfig, ) -> (Self, MemPoolHandle<(TransactionOrigin, LeeTransaction)>) { + sequencer_core_metrics::init(); + let bedrock_signing_key = load_or_create_signing_key(&config.home.join("bedrock_signing_key")) .expect("Failed to load or create bedrock signing key"); @@ -215,6 +229,7 @@ impl SequencerCore { let is_fresh_start = initial_checkpoint.is_none(); let (mempool, mempool_handle) = MemPool::new(config.mempool_max_size); + sequencer_core_metrics::record_mempool_max_size(config.mempool_max_size); let block_publisher = BP::new( &config.bedrock_config, @@ -299,6 +314,8 @@ impl SequencerCore { watchers, }; + sequencer_core_metrics::record_chain_height(sequencer_core.chain_height()); + (sequencer_core, mempool_handle) } @@ -596,6 +613,9 @@ impl SequencerCore { chain.head_state(), Some(&checkpoint_bytes), )?; + + sequencer_core_metrics::increment_blocks_produced_total(); + sequencer_core_metrics::record_chain_height(block.header.block_id); } // Neither branch persists anything, checkpoint included: the // inscription it holds as pending belongs to a block that is not @@ -802,6 +822,7 @@ impl SequencerCore { let clock_tx = clock_invocation(new_block_timestamp); let clock_lee_tx = LeeTransaction::Public(clock_tx.clone()); + sequencer_core_metrics::record_mempool_size(self.mempool.len()); // Everything drained from the store first, then user work. `from_store` // is not the same as a `Sequencer` origin: it says the transaction has a // record behind it and so needs no requeue, where the origin only says @@ -872,16 +893,31 @@ impl SequencerCore { break; } - if Self::apply_mempool_transaction( + let before_tx_apply = Instant::now(); + let applied = Self::apply_mempool_transaction( &mut working_state, origin, &tx, new_block_height, new_block_timestamp, &mut withdrawals, - ) { + ); + if applied { + sequencer_core_metrics::record_mempool_transaction_application_time( + origin.into(), + tx.kind().into(), + sequencer_core_metrics::ApplyStatus::Applied, + before_tx_apply.elapsed(), + ); valid_transactions.push(tx); } else { + sequencer_core_metrics::increment_mempool_failed_transactions_total(); + sequencer_core_metrics::record_mempool_transaction_application_time( + origin.into(), + tx.kind().into(), + sequencer_core_metrics::ApplyStatus::Failed, + before_tx_apply.elapsed(), + ); // A failed transaction is simply left out of the block, except a // dispatch: that one is re-fed from the store every turn, so one // that can never execute would fail on every block for ever. @@ -897,6 +933,7 @@ impl SequencerCore { .transition_from_public_transaction(&clock_tx, new_block_height, new_block_timestamp) .context("Clock transaction failed. Aborting block production.")?; valid_transactions.push(clock_lee_tx); + sequencer_core_metrics::record_transactions_per_block(valid_transactions.len()); let hashable_data = HashableBlockData { block_id: new_block_height, @@ -915,6 +952,8 @@ impl SequencerCore { now.elapsed().as_secs() ); + sequencer_core_metrics::record_block_creation_time(now.elapsed()); + Ok(BlockWithMeta { block, withdrawals }) } @@ -1159,7 +1198,7 @@ fn apply_follow_update( // The lock is held across the persist below so disk writes land in apply // order β€” the produce path persists under this same lock. - let (resubmit_txs, outcome) = { + let (resubmit_txs, outcome, head_height) = { let mut chain = chain.lock().expect("chain state mutex poisoned"); // Outcomes align with `adopted`. @@ -1259,9 +1298,11 @@ fn apply_follow_update( }) .unwrap_or_else(|err| panic!("Failed to persist follow update: {err:#}")); - (resubmit_txs, outcome) + (resubmit_txs, outcome, head_tip.map_or(0, |tip| tip.id)) }; + sequencer_core_metrics::record_chain_height(head_height); + if outcome.accepted_deposits > 0 { info!( "Recorded {} Bedrock Deposit event(s); their mints are drained from the store on our next turn", diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 863bb99e..00f78c4d 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -90,6 +90,7 @@ fn setup_sequencer_config() -> SequencerConfig { retry_pending_blocks_timeout: Duration::from_mins(4), genesis: vec![], cross_zone: None, + metrics_address: None, } } diff --git a/lez/sequencer/service/Cargo.toml b/lez/sequencer/service/Cargo.toml index 338aac0d..396fbe6b 100644 --- a/lez/sequencer/service/Cargo.toml +++ b/lez/sequencer/service/Cargo.toml @@ -15,6 +15,7 @@ mempool.workspace = true sequencer_core = { workspace = true, features = ["testnet"] } sequencer_service_protocol.workspace = true sequencer_service_rpc = { workspace = true, features = ["server"] } +sequencer_service_metrics = { workspace = true, features = ["record"] } programs.workspace = true clap = { workspace = true, features = ["derive", "env"] } @@ -22,6 +23,7 @@ anyhow.workspace = true env_logger.workspace = true hex.workspace = true log.workspace = true +metrics-exporter-prometheus.workspace = true tokio.workspace = true tokio-util.workspace = true jsonrpsee.workspace = true diff --git a/lez/sequencer/service/Dockerfile b/lez/sequencer/service/Dockerfile index 1919f775..2e0b89a9 100644 --- a/lez/sequencer/service/Dockerfile +++ b/lez/sequencer/service/Dockerfile @@ -63,8 +63,9 @@ COPY --from=builder --chown=sequencer_service_user:sequencer_service_user /usr/l VOLUME /var/lib/sequencer_service -# Expose default port +# Expose default ports EXPOSE 3040 +EXPOSE 9000 # Health check (TODO #244: Replace when a real health endpoint is available) HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ diff --git a/lez/sequencer/service/docker-compose.yml b/lez/sequencer/service/docker-compose.yml index 477072ad..d9c573d3 100644 --- a/lez/sequencer/service/docker-compose.yml +++ b/lez/sequencer/service/docker-compose.yml @@ -18,6 +18,7 @@ services: container_name: sequencer_service ports: - "3040:3040" + - "9000:9000" volumes: # Mount configuration file - ./configs/docker/sequencer_config.json:/etc/sequencer_service/sequencer_config.json diff --git a/lez/sequencer/service/metrics/Cargo.toml b/lez/sequencer/service/metrics/Cargo.toml new file mode 100644 index 00000000..46dd2d5a --- /dev/null +++ b/lez/sequencer/service/metrics/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "sequencer_service_metrics" +version = "0.1.0" +edition = "2024" +license = { workspace = true } + +[lints] +workspace = true + +[features] +default = [] +# Enable metrics record +record = ["dep:metrics"] + +[dependencies] +metrics = { workspace = true, optional = true } diff --git a/lez/sequencer/service/metrics/src/lib.rs b/lez/sequencer/service/metrics/src/lib.rs new file mode 100644 index 00000000..f375ff1b --- /dev/null +++ b/lez/sequencer/service/metrics/src/lib.rs @@ -0,0 +1,9 @@ +//! This crate provides all metrics exposed by the sequencer service crate. + +#[cfg(feature = "record")] +pub use record::*; + +pub mod names; + +#[cfg(feature = "record")] +pub mod record; diff --git a/lez/sequencer/service/metrics/src/names.rs b/lez/sequencer/service/metrics/src/names.rs new file mode 100644 index 00000000..1ac19948 --- /dev/null +++ b/lez/sequencer/service/metrics/src/names.rs @@ -0,0 +1,3 @@ +pub const SUBMITTED_TRANSACTIONS_TOTAL: &str = "submitted_transactions_total"; +pub const BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL: &str = + "before_mempool_failed_transactions_total"; diff --git a/lez/sequencer/service/metrics/src/record.rs b/lez/sequencer/service/metrics/src/record.rs new file mode 100644 index 00000000..05eaac1a --- /dev/null +++ b/lez/sequencer/service/metrics/src/record.rs @@ -0,0 +1,32 @@ +use metrics::{Counter, Unit, counter}; + +use crate::names; + +pub fn init() { + submitted_transactions_total_counter().increment(0); + before_mempool_failed_transactions_total_counter().increment(0); +} + +fn submitted_transactions_total_counter() -> Counter { + counter!( + description: "Number of transactions submitted", + unit: Unit::Count, + names::SUBMITTED_TRANSACTIONS_TOTAL + ) +} + +pub fn increment_submitted_transactions_total() { + submitted_transactions_total_counter().increment(1); +} + +fn before_mempool_failed_transactions_total_counter() -> Counter { + counter!( + description: "Number of transactions that failed before reaching the mempool", + unit: Unit::Count, + names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL + ) +} + +pub fn increment_before_mempool_failed_transactions_total() { + before_mempool_failed_transactions_total_counter().increment(1); +} diff --git a/lez/sequencer/service/src/lib.rs b/lez/sequencer/service/src/lib.rs index ba5b68ec..3073823a 100644 --- a/lez/sequencer/service/src/lib.rs +++ b/lez/sequencer/service/src/lib.rs @@ -211,6 +211,8 @@ async fn wait_for_store_release(store: &StoreRelease) { } pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result { + sequencer_service_metrics::init(); + let block_timeout = config.block_create_timeout; let max_block_size = config.max_block_size; diff --git a/lez/sequencer/service/src/main.rs b/lez/sequencer/service/src/main.rs index 95d02e16..b3d5bf71 100644 --- a/lez/sequencer/service/src/main.rs +++ b/lez/sequencer/service/src/main.rs @@ -3,9 +3,10 @@ use std::{ path::PathBuf, }; -use anyhow::Result; +use anyhow::{Context as _, Result}; use clap::Parser; use log::{error, info}; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder}; use tokio::signal::unix::{SignalKind, signal}; use tokio_util::sync::CancellationToken; @@ -24,6 +25,10 @@ struct Args { /// so multiple instances can share one config file. #[clap(long)] home: Option, + /// Override the config's `metrics_address`, so multiple instances can share + /// one config file without fighting over the exporter port. + #[clap(long)] + metrics_address: Option, } #[tokio::main] @@ -34,21 +39,18 @@ struct Args { async fn main() -> Result<()> { env_logger::init(); - let Args { - config_path, - port, - listen_address, - home, - } = Args::parse(); + let args = Args::parse(); let cancellation_token = listen_for_shutdown_signal(); - let mut config = sequencer_service::SequencerConfig::from_path(&config_path)?; - if let Some(home) = home { - config.home = home; + let mut config = sequencer_service::SequencerConfig::from_path(&args.config_path)?; + apply_config_overrides(&args, &mut config); + + if let Some(metrics_address) = config.metrics_address { + install_prometheus_recorder(metrics_address)?; } let mut sequencer_handle = - sequencer_service::run(config, SocketAddr::new(listen_address, port)).await?; + sequencer_service::run(config, SocketAddr::new(args.listen_address, args.port)).await?; tokio::select! { () = cancellation_token.cancelled() => { @@ -71,6 +73,45 @@ async fn main() -> Result<()> { Ok(()) } +fn apply_config_overrides(args: &Args, config: &mut sequencer_service::SequencerConfig) { + let Args { + home, + metrics_address, + config_path: _, + port: _, + listen_address: _, + } = args; + + if let Some(home) = home { + config.home.clone_from(home); + } + if let Some(metrics_address) = metrics_address { + config.metrics_address = Some(*metrics_address); + } +} + +/// Installs the recorder on `metrics_address`. +fn install_prometheus_recorder(metrics_address: SocketAddr) -> Result<()> { + /// Ladder for `*_seconds` histograms, densest across the 1–100 ms band where + /// block production and transaction application actually land. + const LATENCY_BUCKETS: &[f64] = &[ + 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ]; + + /// Fallback ladder for histograms that count things rather than measure time. + const COUNT_BUCKETS: &[f64] = &[1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0]; + + PrometheusBuilder::new() + .with_http_listener(metrics_address) + .with_recommended_naming(true) + .set_buckets(COUNT_BUCKETS) + .context("Failed to set default histogram buckets")? + .set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), LATENCY_BUCKETS) + .context("Failed to set latency histogram buckets")? + .install() + .context("Failed to install Prometheus recorder") +} + /// Cancelled on Ctrl-C or `SIGTERM`. /// /// `SIGTERM` is what a container runtime sends first, so without it every diff --git a/lez/sequencer/service/src/service.rs b/lez/sequencer/service/src/service.rs index 7ab9ed3c..e55735c0 100644 --- a/lez/sequencer/service/src/service.rs +++ b/lez/sequencer/service/src/service.rs @@ -6,7 +6,7 @@ use jsonrpsee::{ types::{ErrorCode, ErrorObjectOwned}, }; use lee; -use log::warn; +use log::{error, warn}; use mempool::MemPoolHandle; use sequencer_core::{ DbError, SequencerCore, TransactionOrigin, block_publisher::BlockPublisherTrait, @@ -44,49 +44,62 @@ impl sequencer_service_rpc::Rpc for SequencerService { async fn send_transaction(&self, tx: LeeTransaction) -> Result { - // Reserve ~200 bytes for block header overhead - const BLOCK_HEADER_OVERHEAD: u64 = 200; + sequencer_service_metrics::increment_submitted_transactions_total(); let tx_hash = tx.hash(); - let encoded_tx = - borsh::to_vec(&tx).expect("Transaction borsh serialization should not fail"); - let tx_size = u64::try_from(encoded_tx.len()).expect("Transaction size should fit in u64"); + let res = async move { + // Reserve ~200 bytes for block header overhead + const BLOCK_HEADER_OVERHEAD: u64 = 200; - let max_tx_size = self.max_block_size.saturating_sub(BLOCK_HEADER_OVERHEAD); + let encoded_tx = + borsh::to_vec(&tx).expect("Transaction borsh serialization should not fail"); + let tx_size = + u64::try_from(encoded_tx.len()).expect("Transaction size should fit in u64"); - if tx_size > max_tx_size { - return Err(ErrorObjectOwned::owned( - ErrorCode::InvalidParams.code(), - format!("Transaction too large: size {tx_size}, max {max_tx_size}"), - None::<()>, - )); - } + let max_tx_size = self.max_block_size.saturating_sub(BLOCK_HEADER_OVERHEAD); - let authenticated_tx = tx - .transaction_stateless_check() - .inspect_err(|err| warn!("Error at pre_check {err:#?}")) - .map_err(|err| { - ErrorObjectOwned::owned( + if tx_size > max_tx_size { + return Err(ErrorObjectOwned::owned( ErrorCode::InvalidParams.code(), - format!("{err:?}"), + format!("Transaction too large: size {tx_size}, max {max_tx_size}"), None::<()>, - ) - })?; + )); + } - // Sequencer-only programs (the cross-zone inbox) are injected by the - // watcher; a user must not invoke them top-level, or anyone could forge - // an inbound cross-zone delivery. Chained user calls are already rejected - // by the inbox guest's caller-is-none assertion. - if let LeeTransaction::Public(public_tx) = &authenticated_tx - && sequencer_core::is_sequencer_only_program(public_tx.message().program_id) - { - return Err(ErrorObjectOwned::owned( - ErrorCode::InvalidParams.code(), - "Program is sequencer-only and cannot be invoked by a user transaction".to_owned(), - None::<()>, - )); - } + let authenticated_tx = tx + .transaction_stateless_check() + .inspect_err(|err| warn!("Error at pre_check {err:#?}")) + .map_err(|err| { + ErrorObjectOwned::owned( + ErrorCode::InvalidParams.code(), + format!("{err:?}"), + None::<()>, + ) + })?; + + // Sequencer-only programs (the cross-zone inbox) are injected by the + // watcher; a user must not invoke them top-level, or anyone could forge + // an inbound cross-zone delivery. Chained user calls are already rejected + // by the inbox guest's caller-is-none assertion. + if let LeeTransaction::Public(public_tx) = &authenticated_tx + && sequencer_core::is_sequencer_only_program(public_tx.message().program_id) + { + return Err(ErrorObjectOwned::owned( + ErrorCode::InvalidParams.code(), + "Program is sequencer-only and cannot be invoked by a user transaction" + .to_owned(), + None::<()>, + )); + } + + Ok(authenticated_tx) + }; + + let authenticated_tx = res.await.inspect_err(|err| { + sequencer_service_metrics::increment_before_mempool_failed_transactions_total(); + error!("Transaction failed before reaching mempool: {err:#?}"); + })?; self.mempool_handle .push((TransactionOrigin::User, authenticated_tx)) diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml new file mode 100644 index 00000000..5d6ab935 --- /dev/null +++ b/monitoring/docker-compose.yml @@ -0,0 +1,36 @@ +# Prometheus + Grafana monitoring stack. +services: + prometheus: + image: prom/prometheus:v3.13.1 + container_name: prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + # Lets Prometheus reach services running natively on the host. + extra_hosts: + - "host.docker.internal:host-gateway" + + grafana: + image: grafana/grafana:13.1.1 + container_name: grafana + ports: + - "3000:3000" + environment: + # Dev-only: open Grafana with no login, full access. + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + depends_on: + - prometheus + +volumes: + prometheus_data: + grafana_data: diff --git a/monitoring/grafana/dashboards/sequencer.json b/monitoring/grafana/dashboards/sequencer.json new file mode 100644 index 00000000..ede0ae4b --- /dev/null +++ b/monitoring/grafana/dashboards/sequencer.json @@ -0,0 +1,377 @@ +{ + "annotations": { "list": [ ] }, + "editable": true, + "graphTooltip": 1, + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "color": { "mode": "fixed", "fixedColor": "blue" }, "unit": "short", "decimals": 0 }, + "overrides": [ ] + }, + "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "chain_height", + "legendFormat": "height", + "refId": "A" + } + ], + "title": "Chain height", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "color": { "mode": "fixed", "fixedColor": "green" }, "unit": "short", "decimals": 0 }, + "overrides": [ ] + }, + "gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "blocks_produced_total", + "legendFormat": "produced", + "refId": "A" + } + ], + "title": "Blocks produced by this sequencer since startup", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "produced Β· blocks/min" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] + } + ] + }, + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, + "id": 3, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [ "last", "max" ] }, + "tooltip": { "mode": "single" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(blocks_produced_total[$__rate_interval]) * 60", + "legendFormat": "produced Β· blocks/min", + "refId": "A" + } + ], + "title": "Block production rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "s" }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "avg" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "dash": [ 8, 4 ], "fill": "dash" } }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } } + ] + } + ] + }, + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 7 }, + "id": 4, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(${percentile}, sum by (le) (rate(block_creation_time_seconds_bucket[$__rate_interval])))", + "legendFormat": "${percentile:text}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(block_creation_time_seconds_sum[$__rate_interval]) / rate(block_creation_time_seconds_count[$__rate_interval])", + "legendFormat": "avg", + "refId": "B" + } + ], + "title": "Block creation time", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "s" }, + "overrides": [ ] + }, + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 16 }, + "id": 5, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(${percentile}, sum by (le, kind, origin, status) (rate(mempool_transaction_application_time_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{kind}} Β· {{origin}} Β· {{status}}", + "refId": "A" + } + ], + "title": "Transaction application time", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "avg" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "dash": [ 8, 4 ], "fill": "dash" } }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } } + ] + } + ] + }, + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 }, + "id": 6, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(${percentile}, sum by (le) (rate(transactions_per_block_bucket[$__rate_interval])))", + "legendFormat": "${percentile:text}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(transactions_per_block_sum[$__rate_interval]) / rate(transactions_per_block_count[$__rate_interval])", + "legendFormat": "avg", + "refId": "B" + } + ], + "title": "Transactions per block", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "min": 0.0, + "max": 100.0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 70.0 }, + { "color": "red", "value": 90.0 } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 25 }, + "id": 7, + "options": { + "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "100 * mempool_size / mempool_max_size", + "legendFormat": "utilization", + "refId": "A" + } + ], + "title": "Mempool utilization", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "spanNulls": true }, + "unit": "short", + "min": 0.0 + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "capacity" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "dash": [ 8, 4 ], "fill": "dash" } }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } + ] + }, + { + "matcher": { "id": "byName", "options": "queued" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] + } + ] + }, + "gridPos": { "h": 8, "w": 18, "x": 6, "y": 25 }, + "id": 8, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_size", + "legendFormat": "queued", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_max_size", + "legendFormat": "capacity", + "refId": "B" + } + ], + "title": "Mempool size vs capacity", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 2, + "min": 0.0, + "max": 100.0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1.0 }, + { "color": "red", "value": 5.0 } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 33 }, + "id": 9, + "options": { + "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "100 * (increase(before_mempool_failed_transactions_total[$__range]) + increase(mempool_failed_transactions_total[$__range])) / clamp_min(increase(submitted_transactions_total[$__range]), 1)", + "legendFormat": "failed", + "refId": "A" + } + ], + "title": "Failed transactions share", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 35, "gradientMode": "opacity" }, + "unit": "short", + "min": 0.0 + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "submitted" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] + }, + { + "matcher": { "id": "byName", "options": "failed Β· before mempool" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] + }, + { + "matcher": { "id": "byName", "options": "failed Β· in mempool" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] + } + ] + }, + "gridPos": { "h": 8, "w": 18, "x": 6, "y": 33 }, + "id": 10, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(submitted_transactions_total[$__rate_interval]) * 60", + "legendFormat": "submitted", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(before_mempool_failed_transactions_total[$__rate_interval]) * 60", + "legendFormat": "failed Β· before mempool", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(mempool_failed_transactions_total[$__rate_interval]) * 60", + "legendFormat": "failed Β· in mempool", + "refId": "C" + } + ], + "title": "Submitted vs failed transactions (per minute)", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [ "sequencer" ], + "templating": { + "list": [ + { + "current": { "selected": true, "text": "p95", "value": "0.95" }, + "includeAll": false, + "label": "Percentile", + "multi": false, + "name": "percentile", + "options": [ + { "selected": false, "text": "p50", "value": "0.5" }, + { "selected": false, "text": "p90", "value": "0.9" }, + { "selected": true, "text": "p95", "value": "0.95" }, + { "selected": false, "text": "p99", "value": "0.99" } + ], + "query": "p50 : 0.5, p90 : 0.9, p95 : 0.95, p99 : 0.99", + "type": "custom" + } + ] + }, + "time": { "from": "now-15m", "to": "now" }, + "timezone": "", + "title": "Sequencer", + "uid": "sequencer" +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 00000000..e26d8f8c --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: sequencer + type: file + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..9bf28c50 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + jsonData: + # Must match prometheus.yml's `scrape_interval` + timeInterval: 5s diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 00000000..a4793aba --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,10 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: sequencer + metrics_path: /metrics + static_configs: + - targets: + - host.docker.internal:9000 diff --git a/test_fixtures/src/config.rs b/test_fixtures/src/config.rs index 8df57fb3..9a8bf53f 100644 --- a/test_fixtures/src/config.rs +++ b/test_fixtures/src/config.rs @@ -110,6 +110,7 @@ pub fn sequencer_config( auth: None, }, cross_zone, + metrics_address: Some(SequencerConfig::DEFAULT_METRICS_ADDRESS), }) } diff --git a/tools/dashboard_gen/Cargo.toml b/tools/dashboard_gen/Cargo.toml new file mode 100644 index 00000000..4bc73e86 --- /dev/null +++ b/tools/dashboard_gen/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "dashboard_gen" +version = "0.1.0" +edition = "2024" +license = { workspace = true } + +[lints] +workspace = true + +[dependencies] +sequencer_core_metrics.workspace = true +sequencer_service_metrics.workspace = true + +clap = { workspace = true, features = ["derive"] } +serde = { workspace = true, features = ["derive", "alloc"] } +serde_json.workspace = true +json-pretty-compact = "0.1.2" diff --git a/tools/dashboard_gen/src/dashboards.rs b/tools/dashboard_gen/src/dashboards.rs new file mode 100644 index 00000000..39ed51ab --- /dev/null +++ b/tools/dashboard_gen/src/dashboards.rs @@ -0,0 +1,3 @@ +//! One module per dashboard, each exposing a `dashboard()` builder. + +pub mod sequencer; diff --git a/tools/dashboard_gen/src/dashboards/sequencer.rs b/tools/dashboard_gen/src/dashboards/sequencer.rs new file mode 100644 index 00000000..0520251c --- /dev/null +++ b/tools/dashboard_gen/src/dashboards/sequencer.rs @@ -0,0 +1,195 @@ +//! The sequencer dashboard: chain progress, block timings, mempool and +//! transaction outcomes. + +#![expect( + clippy::non_ascii_literal, + reason = "legend separators use `Β·` intentionally, matching the rendered Grafana labels" +)] + +use dashboard_gen::{ + Color, Dashboard, FieldOverride, GradientMode, Panel, Target, Thresholds, Unit, avg, + percentile_legend, percentile_variable, rate_per_min, selected_percentile, +}; + +const PERCENTILES: &[u32] = &[50, 90, 95, 99]; +const DEFAULT_PERCENTILE: u32 = 95; + +pub fn dashboard() -> Dashboard { + Dashboard::new("Sequencer", "sequencer") + .tag("sequencer") + .variable(percentile_variable(PERCENTILES, DEFAULT_PERCENTILE)) + .row( + 7, + [ + Panel::stat("Chain height") + .width(6) + .unit(Unit::Short) + .decimals(0) + .color(Color::fixed("blue")) + .target( + Target::new(sequencer_core_metrics::names::CHAIN_HEIGHT).legend("height"), + ), + Panel::stat("Blocks produced by this sequencer since startup") + .width(6) + .unit(Unit::Short) + .decimals(0) + .color(Color::fixed("green")) + .target( + Target::new(sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL) + .legend("produced"), + ), + Panel::timeseries("Block production rate") + .width(12) + .unit(Unit::Short) + .target(rate_per_min( + sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL, + "produced Β· blocks/min", + )) + .with_override( + FieldOverride::by_name("produced Β· blocks/min").color(Color::fixed("green")), + ), + ], + ) + .row( + 9, + [Panel::timeseries("Block creation time") + .width(24) + .unit(Unit::Seconds) + .target(selected_percentile( + sequencer_core_metrics::names::BLOCK_CREATION_TIME, + &[], + &percentile_legend(), + )) + .target(avg(sequencer_core_metrics::names::BLOCK_CREATION_TIME)) + .with_override( + FieldOverride::by_name("avg") + .dashed_line() + .color(Color::fixed("text")), + )], + ) + .row( + 9, + [ + Panel::timeseries("Transaction application time") + .width(12) + .unit(Unit::Seconds) + .target(selected_percentile( + sequencer_core_metrics::names::MEMPOOL_TRANSACTION_APPLICATION_TIME, + &["kind", "origin", "status"], + "{{kind}} Β· {{origin}} Β· {{status}}", + )), + Panel::timeseries("Transactions per block") + .width(12) + .unit(Unit::Short) + .target(selected_percentile( + sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK, + &[], + &percentile_legend(), + )) + .target(avg(sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK)) + .with_override( + FieldOverride::by_name("avg") + .dashed_line() + .color(Color::fixed("text")), + ), + ], + ) + .row( + 8, + [ + Panel::gauge("Mempool utilization") + .width(6) + .unit(Unit::Percent) + .decimals(1) + .min(0.0) + .max(100.0) + .thresholds( + Thresholds::base("green") + .step(70.0, "orange") + .step(90.0, "red"), + ) + .target( + Target::new(format!( + "100 * {size} / {max_size}", + size = sequencer_core_metrics::names::MEMPOOL_SIZE, + max_size = sequencer_core_metrics::names::MEMPOOL_MAX_SIZE, + )) + .legend("utilization"), + ), + Panel::timeseries("Mempool size vs capacity") + .width(18) + .unit(Unit::Short) + .span_nulls() + .min(0.0) + .target( + Target::new(sequencer_core_metrics::names::MEMPOOL_SIZE).legend("queued"), + ) + .target( + Target::new(sequencer_core_metrics::names::MEMPOOL_MAX_SIZE) + .legend("capacity"), + ) + .with_override( + FieldOverride::by_name("capacity") + .dashed_line() + .color(Color::fixed("red")), + ) + .with_override(FieldOverride::by_name("queued").color(Color::fixed("blue"))), + ], + ) + .row( + 8, + [ + Panel::gauge("Failed transactions share") + .width(6) + .unit(Unit::Percent) + .decimals(2) + .min(0.0) + .max(100.0) + .thresholds( + Thresholds::base("green") + .step(1.0, "orange") + .step(5.0, "red"), + ) + .target( + Target::new(format!( + // Both failure stages against the same submission base; + // `clamp_min` keeps an idle window (nothing submitted) + // reading as 0% instead of a division by zero. + "100 * (increase({before_mempool}[$__range]) + increase({in_mempool}[$__range])) / clamp_min(increase({submitted}[$__range]), 1)", + before_mempool = sequencer_service_metrics::names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL, + in_mempool = sequencer_core_metrics::names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL, + submitted = sequencer_service_metrics::names::SUBMITTED_TRANSACTIONS_TOTAL, + )) + .legend("failed"), + ), + Panel::timeseries("Submitted vs failed transactions (per minute)") + .width(18) + .unit(Unit::Short) + .min(0.0) + .fill_opacity(35) + .gradient_mode(GradientMode::Opacity) + .target(rate_per_min( + sequencer_service_metrics::names::SUBMITTED_TRANSACTIONS_TOTAL, + "submitted", + )) + .target(rate_per_min( + sequencer_service_metrics::names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL, + "failed Β· before mempool", + )) + .target(rate_per_min( + sequencer_core_metrics::names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL, + "failed Β· in mempool", + )) + .with_override( + FieldOverride::by_name("submitted").color(Color::fixed("green")), + ) + .with_override( + FieldOverride::by_name("failed Β· before mempool") + .color(Color::fixed("orange")), + ) + .with_override( + FieldOverride::by_name("failed Β· in mempool").color(Color::fixed("red")), + ), + ], + ) +} diff --git a/tools/dashboard_gen/src/lib.rs b/tools/dashboard_gen/src/lib.rs new file mode 100644 index 00000000..113592c0 --- /dev/null +++ b/tools/dashboard_gen/src/lib.rs @@ -0,0 +1,603 @@ +//! A tiny, hand-rolled Grafana dashboard builder. +//! +//! This is a deliberately small subset of what the (Rust-less) Grafana +//! Foundation SDK does: model only the panel types and options we actually use, +//! expose a fluent builder, and render to the same dashboard JSON Grafana +//! provisions. The payoff over hand-written JSON: +//! +//! * metric names live in Rust `const`s, so a rename is a compile error here; +//! * repetitive structure (percentile targets, grid layout, legend/tooltip defaults) collapses into +//! a single call instead of copy-pasted JSON. +//! +//! Build a [`Dashboard`] and serialize it directly. + +// Styling vocabularies passed to the optional `styling` setters are part of the +// public API (and are used by this module's `Panel` fields and `finalize`). +pub use schema::{ + AxisPlacement, Color, GradientMode, LineInterpolation, ShowPoints, StackingMode, Thresholds, + Variable, +}; +use schema::{ + Calc, Custom, Datasource, Defaults, DrawStyle, EmptyList, FieldConfig, Fill, GaugeOptions, + GraphMode, GridPos, Legend, LegendDisplay, LineStyle, Matcher, MatcherKind, Options, + OverrideProperty, PanelModel, PanelType, Placement, PropertyId, PropertyValue, ReduceOptions, + SortOrder, Stacking, StatColorMode, StatOptions, Templating, TimeRange, TimeSeriesOptions, + Tooltip, TooltipMode, VariableKind, VariableOption, +}; +use serde::Serialize; +pub use unit::Unit; + +mod schema; +mod styling; +mod unit; + +/// Datasource uid every panel/target points at. Dashboards stay portable across +/// environments because they reference the datasource by this stable uid rather +/// than by a per-environment URL. +pub const DATASOURCE_UID: &str = "prometheus"; + +/// Window every rate query β€” counter and histogram alike β€” rates over. +/// `$__rate_interval` tracks the panel's zoom, so the window always covers the +/// step Grafana samples at: a fixed one shorter than the step leaves gaps the +/// query never looks at, dropping events from the graph entirely. +const RATE_WINDOW: &str = "$__rate_interval"; + +/// Dashboard variable holding the quantile every percentile query reads. +const PERCENTILE_VAR: &str = "percentile"; + +/// Area fill under a timeseries line, as a percentage. Enough to read a series' +/// shape at a glance without drowning the ones stacked behind it. +pub(crate) const DEFAULT_FILL_OPACITY: u32 = 10; + +/// A single Prometheus query within a panel. +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Target { + datasource: Datasource, + expr: String, + #[serde(rename = "legendFormat")] + legend: String, + ref_id: String, +} + +impl Target { + pub fn new(expr: impl Into) -> Self { + Self { + datasource: Datasource::prometheus(), + expr: expr.into(), + legend: String::new(), + ref_id: ref_letter(0), + } + } + + #[must_use] + pub fn legend(mut self, legend: impl Into) -> Self { + self.legend = legend.into(); + self + } +} + +/// A per-series style override, matched by series name. +#[derive(Serialize)] +pub struct FieldOverride { + matcher: Matcher, + properties: Vec, +} + +impl FieldOverride { + pub fn by_name(name: impl Into) -> Self { + Self { + matcher: Matcher { + id: MatcherKind::ByName, + options: name.into(), + }, + properties: Vec::new(), + } + } + + #[must_use] + pub fn color(mut self, color: Color) -> Self { + self.properties.push(OverrideProperty { + id: PropertyId::Color, + value: PropertyValue::Color(color), + }); + self + } + + #[must_use] + pub fn dashed_line(mut self) -> Self { + self.properties.push(OverrideProperty { + id: PropertyId::LineStyle, + value: PropertyValue::LineStyle(LineStyle { + dash: [8, 4], + fill: Fill::Dash, + }), + }); + self + } +} + +#[derive(Clone, Copy)] +enum Kind { + Stat, + TimeSeries, + Gauge, +} + +/// A dashboard panel builder. Grid position and panel id are assigned by +/// [`Dashboard::row`]; everything else is set here. +pub struct Panel { + title: String, + kind: Kind, + targets: Vec, + width: u32, + unit: Option, + decimals: Option, + color: Option, + min: Option, + max: Option, + thresholds: Option, + span_nulls: bool, + overrides: Vec, + // Optional timeseries styling, set via the `styling` setters. + fill_opacity: Option, + line_interpolation: Option, + show_points: Option, + gradient_mode: Option, + stacking: Option, + axis_placement: Option, + axis_label: Option, +} + +impl Panel { + fn new(title: impl Into, kind: Kind) -> Self { + Self { + title: title.into(), + kind, + targets: Vec::new(), + width: 0, + unit: None, + decimals: None, + color: None, + min: None, + max: None, + thresholds: None, + span_nulls: false, + overrides: Vec::new(), + fill_opacity: None, + line_interpolation: None, + show_points: None, + gradient_mode: None, + stacking: None, + axis_placement: None, + axis_label: None, + } + } + + /// A single big-number panel. + pub fn stat(title: impl Into) -> Self { + Self::new(title, Kind::Stat) + } + + /// A time-series line panel. + pub fn timeseries(title: impl Into) -> Self { + Self::new(title, Kind::TimeSeries) + } + + /// A radial gauge panel. Pair it with [`Panel::min`]/[`Panel::max`] β€” the + /// dial needs a range to fill β€” and [`Panel::thresholds`] for its coloring. + pub fn gauge(title: impl Into) -> Self { + Self::new(title, Kind::Gauge) + } + + /// Grid width in Grafana's 24-column units. Unset panels split the row's + /// remaining width evenly. + #[must_use] + pub const fn width(mut self, width: u32) -> Self { + self.width = width; + self + } + + #[must_use] + pub fn unit(mut self, unit: Unit) -> Self { + self.unit = Some(unit); + self + } + + #[must_use] + pub const fn decimals(mut self, decimals: u32) -> Self { + self.decimals = Some(decimals); + self + } + + #[must_use] + pub fn color(mut self, color: Color) -> Self { + self.color = Some(color); + self + } + + /// Lower bound of the value scale. + #[must_use] + pub const fn min(mut self, min: f64) -> Self { + self.min = Some(min); + self + } + + /// Upper bound of the value scale. + #[must_use] + pub const fn max(mut self, max: f64) -> Self { + self.max = Some(max); + self + } + + #[must_use] + pub fn thresholds(mut self, thresholds: Thresholds) -> Self { + self.thresholds = Some(thresholds); + self + } + + #[must_use] + pub const fn span_nulls(mut self) -> Self { + self.span_nulls = true; + self + } + + #[must_use] + pub fn target(mut self, target: Target) -> Self { + self.targets.push(target); + self + } + + #[must_use] + pub fn targets(mut self, targets: impl IntoIterator) -> Self { + self.targets.extend(targets); + self + } + + #[must_use] + pub fn with_override(mut self, over: FieldOverride) -> Self { + self.overrides.push(over); + self + } + + fn finalize(self, id: u32, grid_pos: GridPos) -> PanelModel { + let targets: Vec = self + .targets + .into_iter() + .enumerate() + .map(|(i, mut t)| { + t.ref_id = ref_letter(i); + t + }) + .collect(); + + let unit = self.unit.unwrap_or_else(default_unit); + let (defaults, options, panel_type) = match self.kind { + Kind::Stat => { + let defaults = Defaults { + color: self.color, + custom: None, + unit, + decimals: self.decimals, + min: self.min, + max: self.max, + thresholds: self.thresholds, + }; + let options = Options::Stat(StatOptions { + color_mode: StatColorMode::Value, + graph_mode: GraphMode::Area, + reduce_options: ReduceOptions { + calcs: vec![Calc::LastNotNull], + fields: String::new(), + values: false, + }, + }); + (defaults, options, PanelType::Stat) + } + Kind::Gauge => { + let defaults = Defaults { + color: self.color, + custom: None, + unit, + decimals: self.decimals, + min: self.min, + max: self.max, + thresholds: self.thresholds, + }; + let options = Options::Gauge(GaugeOptions { + reduce_options: ReduceOptions { + calcs: vec![Calc::LastNotNull], + fields: String::new(), + values: false, + }, + show_threshold_labels: false, + show_threshold_markers: true, + }); + (defaults, options, PanelType::Gauge) + } + Kind::TimeSeries => { + let defaults = Defaults { + color: None, + custom: Some(Custom { + draw_style: DrawStyle::Line, + line_width: 1, + fill_opacity: self.fill_opacity.unwrap_or(DEFAULT_FILL_OPACITY), + span_nulls: self.span_nulls.then_some(true), + line_interpolation: self.line_interpolation, + show_points: self.show_points, + gradient_mode: self.gradient_mode, + stacking: self.stacking.map(|mode| Stacking { + mode, + group: "A".to_owned(), + }), + axis_placement: self.axis_placement, + axis_label: self.axis_label, + }), + unit, + decimals: None, + min: self.min, + max: self.max, + thresholds: self.thresholds, + }; + // Panels with several series read better as a sortable table with + // a multi-series tooltip; single-series panels stay compact. A + // `{{label}}` legend fans one target out into a series per label + // value, so it counts as several too. + let multi = + targets.len() > 1 || targets.iter().any(|target| target.legend.contains("{{")); + let options = Options::TimeSeries(TimeSeriesOptions { + legend: Legend { + display_mode: if multi { + LegendDisplay::Table + } else { + LegendDisplay::List + }, + placement: Placement::Bottom, + calcs: vec![Calc::Last, Calc::Max], + }, + tooltip: if multi { + Tooltip { + mode: TooltipMode::Multi, + sort: Some(SortOrder::Desc), + } + } else { + Tooltip { + mode: TooltipMode::Single, + sort: None, + } + }, + }); + (defaults, options, PanelType::Timeseries) + } + }; + + PanelModel { + datasource: Datasource::prometheus(), + field_config: FieldConfig { + defaults, + overrides: self.overrides, + }, + grid_pos, + id, + options, + targets, + title: self.title, + panel_type, + } + } +} + +/// A dashboard, built row by row. Serialize it directly to get the JSON. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Dashboard { + annotations: EmptyList, + editable: bool, + graph_tooltip: u32, + panels: Vec, + refresh: String, + schema_version: u32, + tags: Vec, + templating: Templating, + time: TimeRange, + timezone: String, + title: String, + uid: String, + + // Layout cursor β€” not part of the dashboard schema. + #[serde(skip)] + next_id: u32, + #[serde(skip)] + cursor_y: u32, +} + +impl Dashboard { + pub fn new(title: impl Into, uid: impl Into) -> Self { + Self { + annotations: EmptyList::default(), + editable: true, + graph_tooltip: 1, + panels: Vec::new(), + refresh: "5s".to_owned(), + schema_version: 39, + tags: Vec::new(), + templating: Templating::default(), + time: TimeRange { + from: "now-15m".to_owned(), + to: "now".to_owned(), + }, + timezone: String::new(), + title: title.into(), + uid: uid.into(), + next_id: 1, + cursor_y: 0, + } + } + + #[must_use] + pub fn tag(mut self, tag: impl Into) -> Self { + self.tags.push(tag.into()); + self + } + + #[must_use] + pub fn refresh(mut self, refresh: impl Into) -> Self { + self.refresh = refresh.into(); + self + } + + /// Add a dropdown to the dashboard's top bar, e.g. [`percentile_variable`]. + #[must_use] + pub fn variable(mut self, variable: Variable) -> Self { + self.templating.list.push(variable); + self + } + + /// Place a horizontal row of panels at the current vertical cursor. Panel + /// ids, x offsets and y are assigned here; unset widths split the remaining + /// 24 columns evenly. + #[must_use] + pub fn row(mut self, height: u32, panels: impl IntoIterator) -> Self { + let panels: Vec = panels.into_iter().collect(); + let specified: u32 = panels.iter().map(|p| p.width).sum(); + let auto_count = u32::try_from(panels.iter().filter(|p| p.width == 0).count()).unwrap_or(0); + // `checked_div` yields `None` when there are no auto-width panels; the + // fallback width is unused in that case. + let auto_width = 24_u32 + .saturating_sub(specified) + .checked_div(auto_count) + .unwrap_or(0); + + let mut x = 0; + for panel in panels { + let w = if panel.width == 0 { + auto_width + } else { + panel.width + }; + let grid_pos = GridPos { + h: height, + w, + x, + y: self.cursor_y, + }; + let id = self.next_id; + self.next_id = self.next_id.saturating_add(1); + x = x.saturating_add(w); + self.panels.push(panel.finalize(id, grid_pos)); + } + self.cursor_y = self.cursor_y.saturating_add(height); + self + } +} + +/// The dropdown driving every [`selected_percentile`] query, offering +/// `percentiles` (e.g. `[50, 90, 95, 99]`) with `default` pre-selected. +/// +/// Panics if `default` is not one of `percentiles`, or if any of them falls +/// outside `p1..=p99`. +#[must_use] +pub fn percentile_variable(percentiles: &[u32], default: u32) -> Variable { + assert!( + percentiles.contains(&default), + "default p{default} is not one of the offered percentiles {percentiles:?}", + ); + + let options: Vec = percentiles + .iter() + .map(|&p| VariableOption { + selected: p == default, + text: format!("p{p}"), + value: quantile(p), + }) + .collect(); + let current = options + .iter() + .find(|option| option.selected) + .cloned() + .expect("`default` is one of `percentiles`, asserted above"); + let query = options + .iter() + .map(|option| format!("{} : {}", option.text, option.value)) + .collect::>() + .join(", "); + + Variable { + current, + include_all: false, + label: "Percentile".to_owned(), + // A quantile is a scalar argument to `histogram_quantile`, so exactly + // one may be selected. + multi: false, + name: PERCENTILE_VAR.to_owned(), + options, + query, + kind: VariableKind::Custom, + } +} + +/// The legend fragment that renders the dropdown's current choice, e.g. `p95`. +#[must_use] +pub fn percentile_legend() -> String { + format!("${{{PERCENTILE_VAR}:text}}") +} + +/// A [`histogram_quantile`] line over `metric`'s buckets, at whatever quantile +/// [`percentile_variable`] currently holds. `labels` stay split out into their +/// own series; every other label is summed away. +/// +/// [`histogram_quantile`]: https://prometheus.io/docs/prometheus/latest/querying/functions/#histogram_quantile +#[must_use] +pub fn selected_percentile(metric: &str, labels: &[&str], legend: &str) -> Target { + // `le` carries the bucket boundary, so it must survive the aggregation. + let grouping = std::iter::once("le") + .chain(labels.iter().copied()) + .collect::>() + .join(", "); + + Target::new(format!( + "histogram_quantile(${{{PERCENTILE_VAR}}}, sum by ({grouping}) (rate({metric}_bucket[{RATE_WINDOW}])))" + )) + .legend(legend) +} + +/// A percentile as its `histogram_quantile` argument, derived without float +/// math: zero-pad to two digits then drop trailing zeros (50 β†’ `0.5`). +/// +/// Panics outside `1..=99`, the only range two digits render: p100 would come +/// out as `0.1` and p0 as `0.`, neither of them loudly. +fn quantile(percentile: u32) -> String { + assert!( + (1..=99).contains(&percentile), + "p{percentile} is outside the supported range p1..=p99", + ); + + let quantile = format!("0.{percentile:02}"); + quantile.trim_end_matches('0').to_owned() +} + +/// An `avg` target for a histogram metric: `rate(sum) / rate(count)`. +#[must_use] +pub fn avg(metric: &str) -> Target { + Target::new(format!( + "rate({metric}_sum[{RATE_WINDOW}]) / rate({metric}_count[{RATE_WINDOW}])" + )) + .legend("avg") +} + +/// A per-minute rate target for a counter metric. +#[must_use] +pub fn rate_per_min(metric: &str, legend: &str) -> Target { + //`rate` is per-second whatever the window, so `* 60` is per-minute regardless of the panel's + //`rate` zoom. + Target::new(format!("rate({metric}[{RATE_WINDOW}]) * 60")).legend(legend) +} + +const fn default_unit() -> Unit { + Unit::Short +} + +fn ref_letter(index: usize) -> String { + let offset = u8::try_from(index).unwrap_or(0); + char::from(b'A'.saturating_add(offset)).to_string() +} diff --git a/tools/dashboard_gen/src/main.rs b/tools/dashboard_gen/src/main.rs new file mode 100644 index 00000000..b428817d --- /dev/null +++ b/tools/dashboard_gen/src/main.rs @@ -0,0 +1,45 @@ +//! Builds one of the Grafana dashboards and prints its JSON to stdout. + +#![expect( + clippy::print_stdout, + reason = "CLI tool: emitting the dashboard JSON on stdout is the deliverable" +)] + +use clap::{Parser, ValueEnum}; +use dashboard_gen::Dashboard; +use json_pretty_compact::PrettyCompactFormatter; +use serde::Serialize as _; + +mod dashboards; + +#[derive(Debug, Parser)] +#[clap(version)] +struct Args { + /// Which dashboard to build. + dashboard: DashboardKind, +} + +#[derive(Debug, Clone, Copy, ValueEnum)] +enum DashboardKind { + Sequencer, +} + +impl DashboardKind { + fn build(self) -> Dashboard { + match self { + Self::Sequencer => dashboards::sequencer::dashboard(), + } + } +} + +fn main() { + let Args { dashboard } = Args::parse(); + + let formatter = PrettyCompactFormatter::new(); + let mut output = Vec::new(); + let mut ser = serde_json::Serializer::with_formatter(&mut output, formatter); + dashboard.build().serialize(&mut ser).unwrap(); + + let json = String::from_utf8(output).unwrap(); + println!("{json}"); +} diff --git a/tools/dashboard_gen/src/schema.rs b/tools/dashboard_gen/src/schema.rs new file mode 100644 index 00000000..d04b6bad --- /dev/null +++ b/tools/dashboard_gen/src/schema.rs @@ -0,0 +1,434 @@ +//! The serializable Grafana dashboard schema β€” the internal data model the +//! public builders assemble into. +//! +//! Every type here is `Serialize`-only: the model is sized for what we emit. + +use serde::Serialize; + +use crate::{DATASOURCE_UID, FieldOverride, Target, unit::Unit}; + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum DatasourceKind { + Prometheus, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Fill { + Dash, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum DrawStyle { + Line, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum MatcherKind { + ByName, +} + +#[derive(Clone, Copy, Serialize)] +pub enum PropertyId { + #[serde(rename = "color")] + Color, + #[serde(rename = "custom.lineStyle")] + LineStyle, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum Calc { + LastNotNull, + Last, + Max, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum StatColorMode { + Value, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum GraphMode { + Area, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum LegendDisplay { + Table, + List, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Placement { + Bottom, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum TooltipMode { + Single, + Multi, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum SortOrder { + Desc, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum PanelType { + Stat, + Timeseries, + Gauge, +} + +#[derive(Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum ThresholdMode { + Absolute, + /// Steps expressed as a percentage of the min–max range. The builder never + /// emits this; it exists so Grafana exports using it still parse. + Percentage, +} + +/// One threshold step: the color values at or above `value` take. The base step +/// carries `value: null` β€” Grafana's "everything below the first threshold". +#[derive(Clone, Serialize)] +pub struct ThresholdStep { + pub color: String, + pub value: Option, +} + +/// A threshold ladder, driving gauge/stat coloring. +#[derive(Clone, Serialize)] +pub struct Thresholds { + pub mode: ThresholdMode, + pub steps: Vec, +} + +impl Thresholds { + /// Start a ladder with the color used below every threshold. + pub fn base(color: impl Into) -> Self { + Self { + mode: ThresholdMode::Absolute, + steps: vec![ThresholdStep { + color: color.into(), + value: None, + }], + } + } + + /// Add a step: values at or above `value` render in `color`. + #[must_use] + pub fn step(mut self, value: f64, color: impl Into) -> Self { + self.steps.push(ThresholdStep { + color: color.into(), + value: Some(value), + }); + self + } +} + +// Optional timeseries styling vocabularies. Each derives `PartialEq` so the +// public setters can panic when handed the Grafana default (see `styling`). + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum LineInterpolation { + Linear, + Smooth, + StepBefore, + StepAfter, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum ShowPoints { + Auto, + Never, + Always, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum GradientMode { + None, + Opacity, + Hue, + Scheme, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum StackingMode { + None, + Normal, + Percent, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum AxisPlacement { + Auto, + Left, + Right, + Hidden, +} + +#[derive(Serialize)] +pub struct Stacking { + pub mode: StackingMode, + pub group: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Custom { + pub draw_style: DrawStyle, + pub line_width: u32, + pub fill_opacity: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub span_nulls: Option, + // Optional styling β€” omitted (left at Grafana's default) unless a setter + // fills it in. See `styling`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub line_interpolation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub show_points: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gradient_mode: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stacking: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub axis_placement: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub axis_label: Option, +} + +#[derive(Clone, Serialize)] +pub struct Datasource { + #[serde(rename = "type")] + pub kind: DatasourceKind, + pub uid: String, +} + +impl Datasource { + pub fn prometheus() -> Self { + Self { + kind: DatasourceKind::Prometheus, + uid: DATASOURCE_UID.to_owned(), + } + } +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "kebab-case")] +#[serde(tag = "mode")] +pub enum Color { + Fixed { + #[serde(rename = "fixedColor")] + fixed_color: String, + }, + PaletteClassic, +} + +impl Color { + pub fn fixed(color: impl Into) -> Self { + Self::Fixed { + fixed_color: color.into(), + } + } + + #[must_use] + pub const fn palette_classic() -> Self { + Self::PaletteClassic + } +} + +#[derive(Serialize)] +pub struct LineStyle { + pub dash: [u32; 2], + pub fill: Fill, +} + +#[derive(Serialize)] +#[serde(untagged)] +pub enum PropertyValue { + Color(Color), + LineStyle(LineStyle), +} + +#[derive(Serialize)] +pub struct OverrideProperty { + pub id: PropertyId, + pub value: PropertyValue, +} + +#[derive(Serialize)] +pub struct Matcher { + pub id: MatcherKind, + pub options: String, +} + +#[derive(Serialize)] +pub struct Defaults { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub color: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub custom: Option, + pub unit: Unit, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub decimals: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thresholds: Option, +} + +#[derive(Serialize)] +pub struct FieldConfig { + pub defaults: Defaults, + pub overrides: Vec, +} + +#[derive(Serialize)] +pub struct ReduceOptions { + pub calcs: Vec, + // Empty string means "all fields"; genuinely free-form, not a vocabulary. + pub fields: String, + pub values: bool, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct StatOptions { + pub color_mode: StatColorMode, + pub graph_mode: GraphMode, + pub reduce_options: ReduceOptions, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Legend { + pub display_mode: LegendDisplay, + pub placement: Placement, + pub calcs: Vec, +} + +#[derive(Serialize)] +pub struct Tooltip { + pub mode: TooltipMode, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sort: Option, +} + +#[derive(Serialize)] +pub struct TimeSeriesOptions { + pub legend: Legend, + pub tooltip: Tooltip, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GaugeOptions { + pub reduce_options: ReduceOptions, + pub show_threshold_labels: bool, + pub show_threshold_markers: bool, +} + +#[derive(Serialize)] +#[serde(untagged)] +pub enum Options { + Stat(StatOptions), + TimeSeries(TimeSeriesOptions), + Gauge(GaugeOptions), +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum VariableKind { + /// A fixed list of choices, spelled out in the dashboard itself. + Custom, +} + +/// One choice in a [`Variable`] dropdown: `text` is displayed, `value` is what +/// `$name` interpolates to in a query. +#[derive(Clone, Serialize)] +pub struct VariableOption { + pub selected: bool, + pub text: String, + pub value: String, +} + +/// A dashboard-level dropdown, rendered in the top bar. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Variable { + pub current: VariableOption, + pub include_all: bool, + pub label: String, + pub multi: bool, + pub name: String, + pub options: Vec, + /// Grafana's own encoding of `options`, as `text : value` pairs. + pub query: String, + #[serde(rename = "type")] + pub kind: VariableKind, +} + +#[derive(Serialize, Default)] +pub struct Templating { + pub list: Vec, +} + +#[derive(Clone, Copy, Serialize)] +pub struct GridPos { + pub h: u32, + pub w: u32, + pub x: u32, + pub y: u32, +} + +/// A fully positioned panel, ready to serialize. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PanelModel { + pub datasource: Datasource, + pub field_config: FieldConfig, + pub grid_pos: GridPos, + pub id: u32, + pub options: Options, + pub targets: Vec, + pub title: String, + #[serde(rename = "type")] + pub panel_type: PanelType, +} + +#[expect( + clippy::trailing_empty_array, + reason = "Grafana expects `list: []` for the blocks we don't populate" +)] +#[derive(Serialize, Default)] +pub struct EmptyList { + pub list: [u8; 0], +} + +#[derive(Serialize)] +pub struct TimeRange { + // Free-form Grafana time expressions, not a closed vocabulary. + pub from: String, + pub to: String, +} diff --git a/tools/dashboard_gen/src/styling.rs b/tools/dashboard_gen/src/styling.rs new file mode 100644 index 00000000..36b0cc38 --- /dev/null +++ b/tools/dashboard_gen/src/styling.rs @@ -0,0 +1,124 @@ +//! Optional timeseries styling setters. +//! +//! These are real setters β€” each fills in a field of the panel's `custom` block. +//! Every one documents Grafana's default and **panics if handed that default**: +//! passing the default is always redundant (Grafana emits it anyway), and the +//! generator only serializes fields that differ from the default. So if a call +//! wouldn't change the rendered panel, it's a mistake worth catching loudly at +//! generation time rather than shipping a no-op. +//! +//! These affect timeseries panels only; on a stat panel the `custom` block is +//! not emitted, so the value is silently dropped. + +use crate::{ + DEFAULT_FILL_OPACITY, Panel, + schema::{AxisPlacement, GradientMode, LineInterpolation, ShowPoints, StackingMode}, +}; + +#[expect( + clippy::multiple_inherent_impl, + reason = "styling setters intentionally live in their own file, so `Panel` has a second inherent impl here" +)] +impl Panel { + /// Area fill under the line, as a percentage. Builder default: + /// [`DEFAULT_FILL_OPACITY`]. + /// + /// Panics if passed that default, or a value above 100. + #[must_use] + pub fn fill_opacity(mut self, opacity: u32) -> Self { + assert!( + opacity <= 100, + "fill_opacity({opacity}) is not a percentage" + ); + assert_ne!( + opacity, DEFAULT_FILL_OPACITY, + "fill_opacity({DEFAULT_FILL_OPACITY}) is redundant: it is the builder's default. Omit the call.", + ); + self.fill_opacity = Some(opacity); + self + } + + /// Interpolation between points. Grafana default: `Linear`. + /// + /// Panics if passed `Linear` β€” that's the default and would be redundant. + #[must_use] + pub fn line_interpolation(mut self, value: LineInterpolation) -> Self { + assert_ne!( + value, + LineInterpolation::Linear, + "line_interpolation(Linear) is redundant: `linear` is Grafana's default. Omit the call.", + ); + self.line_interpolation = Some(value); + self + } + + /// Whether/when to draw point markers. Grafana default: `Auto`. + /// + /// Panics if passed `Auto` β€” that's the default and would be redundant. + #[must_use] + pub fn show_points(mut self, value: ShowPoints) -> Self { + assert_ne!( + value, + ShowPoints::Auto, + "show_points(Auto) is redundant: `auto` is Grafana's default. Omit the call.", + ); + self.show_points = Some(value); + self + } + + /// Area fill gradient. Grafana default: `None`. + /// + /// Panics if passed `None` β€” that's the default and would be redundant. + #[must_use] + pub fn gradient_mode(mut self, value: GradientMode) -> Self { + assert_ne!( + value, + GradientMode::None, + "gradient_mode(None) is redundant: `none` is Grafana's default. Omit the call.", + ); + self.gradient_mode = Some(value); + self + } + + /// Series stacking. Grafana default: `None`. + /// + /// Panics if passed `None` β€” that's the default and would be redundant. + #[must_use] + pub fn stacking(mut self, mode: StackingMode) -> Self { + assert_ne!( + mode, + StackingMode::None, + "stacking(None) is redundant: `none` is Grafana's default. Omit the call.", + ); + self.stacking = Some(mode); + self + } + + /// Y-axis placement. Grafana default: `Auto`. + /// + /// Panics if passed `Auto` β€” that's the default and would be redundant. + #[must_use] + pub fn axis_placement(mut self, value: AxisPlacement) -> Self { + assert_ne!( + value, + AxisPlacement::Auto, + "axis_placement(Auto) is redundant: `auto` is Grafana's default. Omit the call.", + ); + self.axis_placement = Some(value); + self + } + + /// Y-axis label. Grafana default: `""` (no label). + /// + /// Panics if passed an empty string β€” that's the default and would be redundant. + #[must_use] + pub fn axis_label(mut self, label: impl Into) -> Self { + let label = label.into(); + assert_ne!( + label, "", + "axis_label(\"\") is redundant: no label is Grafana's default. Omit the call.", + ); + self.axis_label = Some(label); + self + } +} diff --git a/tools/dashboard_gen/src/unit.rs b/tools/dashboard_gen/src/unit.rs new file mode 100644 index 00000000..a91f302b --- /dev/null +++ b/tools/dashboard_gen/src/unit.rs @@ -0,0 +1,69 @@ +//! Panel value units. +//! +//! Grafana identifies a field's unit by a short id string (`"s"`, `"bytes"`, +//! `"reqps"`, …). We model the handful we actually use as named variants and +//! fall back to [`Unit::custom`] for anything else, so the value always +//! serializes to the exact id Grafana expects. + +use serde::{Serialize, Serializer}; + +/// A panel value unit. Serializes to Grafana's unit id string. +/// +/// Only the most common units are named; [`Unit::custom`] carries any other +/// Grafana unit id verbatim. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Unit { + /// Plain number, SI-abbreviated (`short`). Grafana's default. + Short, + /// Percentage on a 0–100 scale (`percent`). + Percent, + /// Percentage on a 0.0–1.0 scale (`percentunit`). + PercentUnit, + /// Seconds (`s`). + Seconds, + /// Milliseconds (`ms`). + Milliseconds, + /// Nanoseconds (`ns`). + Nanoseconds, + /// Bytes, IEC/binary (`bytes`). + Bytes, + /// Bytes per second, SI (`Bps`). + BytesPerSec, + /// Requests per second (`reqps`). + RequestsPerSec, + /// Operations per second (`ops`). + OpsPerSec, + /// Any other Grafana unit id, kept verbatim. + Custom(String), +} + +impl Unit { + /// Wrap an arbitrary Grafana unit id (e.g. `"dtdurationms"`, `"celsius"`). + #[must_use] + pub fn custom(id: impl Into) -> Self { + Self::Custom(id.into()) + } + + /// The Grafana unit id this value serializes to. + fn as_id(&self) -> &str { + match self { + Self::Short => "short", + Self::Percent => "percent", + Self::PercentUnit => "percentunit", + Self::Seconds => "s", + Self::Milliseconds => "ms", + Self::Nanoseconds => "ns", + Self::Bytes => "bytes", + Self::BytesPerSec => "Bps", + Self::RequestsPerSec => "reqps", + Self::OpsPerSec => "ops", + Self::Custom(id) => id, + } + } +} + +impl Serialize for Unit { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_id()) + } +}