Merge pull request #651 from logos-blockchain/arjentix/metrics

feat(metrics): add sequencer metrics & grafana dashboard generation tool
This commit is contained in:
Daniil Polyakov 2026-08-04 00:57:09 +03:00 committed by GitHub
commit cfac4d5b0b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
43 changed files with 2878 additions and 79 deletions

View File

@ -353,3 +353,34 @@ jobs:
exit 1
fi
echo "✅ Artifacts are up to date"
# The dashboard generator is plain Rust (no Docker), so this runs directly in
# the CI image container rather than through `run-in-ci-image`.
dashboards:
needs: ci-image
runs-on: ubuntu-latest
container:
image: ${{ needs.ci-image.outputs.image }}
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
timeout-minutes: 30
name: dashboards
steps:
- uses: actions/checkout@v5
with:
ref: ${{ github.event.pull_request.head.sha || github.head_ref }}
- name: Regenerate dashboards
run: just regenerate-dashboards
- name: Check if dashboards match repository
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
if ! git diff --exit-code monitoring/grafana/dashboards/; then
echo "❌ Dashboards in the repository are out of date!"
echo "Please run 'just regenerate-dashboards' and commit the changes."
exit 1
fi
echo "✅ Dashboards are up to date"

1
.gitignore vendored
View File

@ -11,6 +11,7 @@ data/
rocksdb*
sequencer/service/data/
storage.json
statistics.json
result

View File

@ -6,6 +6,10 @@ This document describes the guidelines for contributing to the project. We will
If you have any questions, come say hi to our [Discord](https://discord.gg/tGJwgGrSPN)!
## Metrics
We have guidelines about metrics, for more information refer to [metrics](docs/metrics/metrics.md).
## Commit title format
We use [Conventional Commits](https://www.conventionalcommits.org/).
@ -59,7 +63,7 @@ Could be squashed to an empty commit if they belong to the same PR.
## Default branch
By default all PRs must be directed into the `dev` branch. This helps us to keep releases stable.
By default all PRs must be directed into the `dev` branch. This helps us to keep releases stable.
## Branch workflow

293
Cargo.lock generated
View File

@ -794,6 +794,29 @@ version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "aws-lc-rs"
version = "1.17.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1"
dependencies = [
"aws-lc-sys",
"zeroize",
]
[[package]]
name = "aws-lc-sys"
version = "0.43.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c"
dependencies = [
"cc",
"cmake",
"dunce",
"fs_extra",
"pkg-config",
]
[[package]]
name = "axum"
version = "0.7.9"
@ -1589,6 +1612,15 @@ dependencies = [
"lee_core",
]
[[package]]
name = "cmake"
version = "0.1.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
dependencies = [
"cc",
]
[[package]]
name = "cmov"
version = "0.5.4"
@ -2259,6 +2291,18 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "dashboard_gen"
version = "0.1.0"
dependencies = [
"clap",
"json-pretty-compact",
"sequencer_core_metrics",
"sequencer_service_metrics",
"serde",
"serde_json",
]
[[package]]
name = "data-encoding"
version = "2.11.0"
@ -2554,6 +2598,12 @@ version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590"
[[package]]
name = "dunce"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "dyn-clone"
version = "1.0.20"
@ -2858,6 +2908,17 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "evmap"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8"
dependencies = [
"hashbag",
"left-right",
"smallvec",
]
[[package]]
name = "example_program_deployment_methods"
version = "0.1.0"
@ -3036,6 +3097,12 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "foldhash"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "foreign-types"
version = "0.5.0"
@ -3072,6 +3139,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fs_extra"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "funty"
version = "2.0.0"
@ -3231,6 +3304,21 @@ dependencies = [
"num-traits",
]
[[package]]
name = "generator"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae"
dependencies = [
"cc",
"cfg-if",
"libc",
"log",
"rustversion",
"windows-link",
"windows-result",
]
[[package]]
name = "generic-array"
version = "0.14.7"
@ -3436,6 +3524,12 @@ dependencies = [
"byteorder",
]
[[package]]
name = "hashbag"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064"
[[package]]
name = "hashbrown"
version = "0.12.3"
@ -3459,7 +3553,16 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
"foldhash 0.1.5",
]
[[package]]
name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"foldhash 0.2.0",
]
[[package]]
@ -3817,6 +3920,7 @@ dependencies = [
"hyper-util",
"log",
"rustls",
"rustls-native-certs",
"tokio",
"tokio-rustls",
"tower-service",
@ -4503,6 +4607,16 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "json-pretty-compact"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62a9c1f06b173b0da0ccc8cae00599d7b3ceda6e76d68be9b6bc2c941adafe0e"
dependencies = [
"serde_json",
"thiserror 1.0.69",
]
[[package]]
name = "jsonrpsee"
version = "0.26.0"
@ -4890,6 +5004,17 @@ dependencies = [
"thiserror 2.0.18",
]
[[package]]
name = "left-right"
version = "0.11.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bc015ded5d9b3054dbbdb63332cdd6ee42352ccef19e911e25117490e2f48ee"
dependencies = [
"crossbeam-utils",
"loom",
"slab",
]
[[package]]
name = "leptos"
version = "0.8.19"
@ -5896,7 +6021,7 @@ dependencies = [
"multiaddr",
"num-bigint 0.4.6",
"serde",
"strum",
"strum 0.27.2",
"thiserror 2.0.18",
"time",
"tracing",
@ -6435,6 +6560,19 @@ dependencies = [
"prost-types 0.13.5",
]
[[package]]
name = "loom"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca"
dependencies = [
"cfg-if",
"generator",
"scoped-tls",
"tracing",
"tracing-subscriber 0.3.23",
]
[[package]]
name = "lru"
version = "0.12.5"
@ -6613,6 +6751,7 @@ dependencies = [
name = "mempool"
version = "0.1.0"
dependencies = [
"futures",
"tokio",
]
@ -6643,6 +6782,56 @@ dependencies = [
"paste",
]
[[package]]
name = "metrics"
version = "0.24.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2"
dependencies = [
"portable-atomic",
"rapidhash",
]
[[package]]
name = "metrics-exporter-prometheus"
version = "0.18.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108"
dependencies = [
"base64 0.22.1",
"evmap",
"http-body-util",
"hyper",
"hyper-rustls",
"hyper-util",
"indexmap 2.14.0",
"ipnet",
"metrics",
"metrics-util",
"quanta",
"rustls",
"thiserror 2.0.18",
"tokio",
"tracing",
]
[[package]]
name = "metrics-util"
version = "0.20.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
"hashbrown 0.16.1",
"metrics",
"quanta",
"rand 0.9.4",
"rand_xoshiro",
"rapidhash",
"sketches-ddsketch",
]
[[package]]
name = "mime"
version = "0.3.17"
@ -7965,6 +8154,21 @@ dependencies = [
"parking_lot",
]
[[package]]
name = "quanta"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
dependencies = [
"crossbeam-utils",
"libc",
"once_cell",
"raw-cpuid",
"wasi",
"web-sys",
"winapi",
]
[[package]]
name = "quick-protobuf"
version = "0.8.1"
@ -8177,6 +8381,33 @@ dependencies = [
"rand_core 0.9.5",
]
[[package]]
name = "rand_xoshiro"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41"
dependencies = [
"rand_core 0.9.5",
]
[[package]]
name = "rapidhash"
version = "4.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e"
dependencies = [
"rustversion",
]
[[package]]
name = "raw-cpuid"
version = "11.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
dependencies = [
"bitflags 2.12.1",
]
[[package]]
name = "rawpointer"
version = "0.2.1"
@ -8978,6 +9209,7 @@ version = "0.23.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
dependencies = [
"aws-lc-rs",
"log",
"once_cell",
"ring",
@ -9042,6 +9274,7 @@ version = "0.103.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
dependencies = [
"aws-lc-rs",
"ring",
"rustls-pki-types",
"untrusted",
@ -9091,7 +9324,7 @@ dependencies = [
"serde",
"serde_with",
"sha2 0.10.9",
"strum",
"strum 0.27.2",
"tempfile",
"thiserror 2.0.18",
"toml 0.8.23",
@ -9162,6 +9395,12 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "scoped-tls"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
[[package]]
name = "scopeguard"
version = "1.2.0"
@ -9271,6 +9510,7 @@ dependencies = [
"programs",
"rand 0.8.6",
"risc0-zkvm",
"sequencer_core_metrics",
"serde",
"serde_json",
"storage",
@ -9285,6 +9525,15 @@ dependencies = [
"vault_core",
]
[[package]]
name = "sequencer_core_metrics"
version = "0.1.0"
dependencies = [
"common",
"metrics",
"strum 0.28.0",
]
[[package]]
name = "sequencer_service"
version = "0.1.0"
@ -9301,14 +9550,23 @@ dependencies = [
"lee",
"log",
"mempool",
"metrics-exporter-prometheus",
"programs",
"sequencer_core",
"sequencer_service_metrics",
"sequencer_service_protocol",
"sequencer_service_rpc",
"tokio",
"tokio-util",
]
[[package]]
name = "sequencer_service_metrics"
version = "0.1.0"
dependencies = [
"metrics",
]
[[package]]
name = "sequencer_service_protocol"
version = "0.1.0"
@ -9724,6 +9982,12 @@ version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
[[package]]
name = "sketches-ddsketch"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
[[package]]
name = "slab"
version = "0.4.12"
@ -9923,7 +10187,16 @@ version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
dependencies = [
"strum_macros",
"strum_macros 0.27.2",
]
[[package]]
name = "strum"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd"
dependencies = [
"strum_macros 0.28.0",
]
[[package]]
@ -9938,6 +10211,18 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "strum_macros"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "subtle"
version = "2.6.1"

View File

@ -18,9 +18,11 @@ members = [
"lez/system_accounts",
"lez/chain_state",
"lez/sequencer/core",
"lez/sequencer/core/metrics",
"lez/sequencer/service",
"lez/sequencer/service/protocol",
"lez/sequencer/service/rpc",
"lez/sequencer/service/metrics",
"lez/indexer/core",
"lez/indexer/service",
"lez/indexer/service/protocol",
@ -67,6 +69,7 @@ members = [
"tools/crypto_primitives_bench",
"tools/integration_bench",
"tools/cross_zone_chat",
"tools/dashboard_gen",
]
[workspace.dependencies]
@ -78,8 +81,10 @@ mempool = { path = "lez/mempool" }
storage = { path = "lez/storage" }
key_protocol = { path = "lee/key_protocol" }
sequencer_core = { path = "lez/sequencer/core" }
sequencer_core_metrics = { path = "lez/sequencer/core/metrics" }
sequencer_service_protocol = { path = "lez/sequencer/service/protocol" }
sequencer_service_rpc = { path = "lez/sequencer/service/rpc" }
sequencer_service_metrics = { path = "lez/sequencer/service/metrics" }
sequencer_service = { path = "lez/sequencer/service" }
indexer_core = { path = "lez/indexer/core" }
indexer_service = { path = "lez/indexer/service" }
@ -142,6 +147,8 @@ actix-rt = "*"
lazy_static = "1.5.0"
env_logger = "0.11"
log = "0.4.28"
metrics = "0.24.6"
metrics-exporter-prometheus = "0.18.3"
lru = "0.16.3"
thiserror = "2.0"
sha2 = "0.10.8"
@ -173,6 +180,7 @@ url = { version = "2.5.4", features = ["serde"] }
tokio-retry = "0.3.0"
schemars = "1.2"
async-stream = "0.3.6"
strum = { version = "0.28.0", features = ["derive"] }
logos-blockchain-common-http-client = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" }
logos-blockchain-key-management-system-service = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" }

View File

@ -53,6 +53,13 @@ regenerate-test-fixture:
@echo "🧪 Regenerating test fixture"
RISC0_DEV_MODE=1 cargo run -p test_fixtures --bin regenerate_test_fixture
# Regenerate the committed Grafana dashboards from the Rust generator
# (tools/dashboard_gen) and commit the result. CI checks these are up to date.
regenerate-dashboards:
@echo "📊 Regenerating Grafana dashboards"
@cargo build -q -p dashboard_gen
@cargo run -q -p dashboard_gen -- sequencer > monitoring/grafana/dashboards/sequencer.json
# Run criterion benches: fast crypto primitives, then the slow PPE verify (real proving setup).
bench:
@echo "📊 Running criterion benches"
@ -65,19 +72,24 @@ run-bedrock:
@echo "⛓️ Running bedrock"
docker compose up
# Run Sequencer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration.
# Optional home/port let a second instance run off the same config, e.g.
# `just run-sequencer "" "$TMPDIR/lez-sequencer2" 3041` for the multi-sequencer demo.
# Run Prometheus + Grafana in docker. Grafana: http://localhost:3000 (anonymous
# admin), Prometheus: http://localhost:9090. Scrapes the sequencer's /metrics.
[working-directory: 'monitoring']
run-monitoring:
@echo "📊 Running Prometheus (http://localhost:9090) + Grafana (http://localhost:3000)"
docker compose up
# Run Sequencer. Extra args are forwarded to the binary. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration.
[working-directory: 'lez/sequencer/service']
run-sequencer standalone="" home="" port="3040":
run-sequencer *args:
@echo "🧠 Running sequencer"
@if [ "{{standalone}}" = "standalone" ]; then \
echo "🧪 Running in standalone mode"; \
RUST_LOG=info cargo run --features standalone --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \
else \
echo "🚀 Running in normal mode"; \
RUST_LOG=info cargo run --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \
fi
RUST_LOG=info cargo run --release -p sequencer_service -- configs/debug/sequencer_config.json {{args}}
# Run Sequencer with mocked Bedrock clients. Takes the same args as `run-sequencer`.
[working-directory: 'lez/sequencer/service']
run-sequencer-standalone *args:
@echo "🧪 Running sequencer in standalone mode"
RUST_LOG=info cargo run --features standalone --release -p sequencer_service -- configs/debug/sequencer_config.json {{args}}
# Run Indexer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration.
[working-directory: 'lez/indexer/service']
@ -103,6 +115,11 @@ run-wallet +args:
@echo "🔑 Running wallet"
LEE_WALLET_HOME_DIR=$(pwd)/configs/debug cargo run --release -p wallet -- {{args}}
# Query sequencer metrics in raw format. Useful for quick debugging. For a more detailed view, use `just run-monitoring`.
get-sequencer-metrics:
@echo "📊 Querying sequencer's metrics"
curl http://localhost:9000/metrics
# Import test accounts supplied in sequencer configuration.
wallet-import-test-accounts:
@echo "⚙️ Initializing accounts"
@ -143,4 +160,5 @@ clean:
rm -rf lez/wallet/configs/debug/storage.json
rm -rf lez/wallet/configs/debug/statistics.json
rm -rf rocksdb*
cd bedrock && docker compose down -v
cd bedrock && docker compose down -v && cd ..
cd monitoring && docker compose down -v && cd ..

View File

@ -11,3 +11,5 @@ include:
lez/indexer/service/docker-compose.yml
- path:
lez/explorer_service/docker-compose.yml
- path:
monitoring/docker-compose.yml

113
docs/metrics/metrics.md Normal file
View File

@ -0,0 +1,113 @@
# Metrics
Services expose Prometheus metrics; Grafana dashboards are generated from Rust so panel queries and metric names cannot drift apart.
## Metrics crates
Every crate that emits metrics gets a sibling `metrics` crate — `lez/sequencer/core/metrics``sequencer_core_metrics`. Each has two halves:
| Module | Gated by | Contents |
|---|---|---|
| `names` | always compiled | `pub const BLOCKS_PRODUCED_TOTAL: &str = "blocks_produced_total";` — one const per metric |
| `record` | `record` feature | `record_*` / `increment_*` functions, plus `init()` |
The emitting crate depends on it with `features = ["record"]`; consumers that only need the names (i.e. `dashboard_gen`) take the default features and pull in nothing. Dashboards reference the same consts the recording code does, so **renaming a metric is a compile error rather than a silently empty panel**.
## Naming
The recorder runs with `with_recommended_naming(true)`, which enforces Prometheus convention:
| Kind | Suffix | Example |
|---|---|---|
| Counter | `_total` | `blocks_produced_total`, `submitted_transactions_total` |
| Histogram | unit | `block_creation_time_seconds` |
| Gauge | none | `mempool_size` |
**Spell the suffix in the const.** The exporter appends a missing unit suffix to the *rendered* name, but bucket matchers (below) run against the registered name — a duration metric named without `_seconds` renders correctly yet silently gets the wrong buckets.
## Metric types
| Type | Use for | Example |
|---|---|---|
| Counter | monotonically increasing event counts | `mempool_failed_transactions_total` |
| Gauge | a value that moves both ways | `mempool_size`, `chain_height` (a reorg lowers it) |
| Histogram | distributions — latencies, sizes, per-batch counts | `mempool_transaction_application_time_seconds` |
Each metric gets a private constructor plus a public recording wrapper, so its description, unit and labels are declared once:
```rust
fn blocks_produced_total_counter() -> Counter {
counter!(
description: "Number of blocks produced by this sequencer and applied to the head",
unit: Unit::Count,
names::BLOCKS_PRODUCED_TOTAL
)
}
pub fn increment_blocks_produced_total() {
blocks_produced_total_counter().increment(1);
}
```
Labels are passed as `"origin" => <&'static str>::from(origin)`; keep them low-cardinality (enums, never IDs or hashes).
## `init()`
Each `record` module exposes `init()`, called once at startup after the recorder is installed. It publishes every metric at zero.
This is not cosmetic. A metric only materialises when first touched, and `rate()`/`increase()` need a sample from *before* an increment to see it — a series that springs into existence at `1` reads as `0` until the second event, so the first one is lost forever. Zero-publishing also means an idle service exports `0` instead of nothing at all.
For histograms, creating the handle publishes zeroed buckets without recording an observation (recording a fake `0` would skew the distribution). Label combinations must each be registered, so `init()` iterates the label enums via `strum::EnumIter`.
## Metrics in libraries
**Yes, record metrics from library crates.** The `metrics` facade is a no-op until a recorder is installed, so a library that records costs nothing to a consumer that never installs one — including tests. Libraries record; only the binary installs the exporter.
## Exporter setup
`sequencer_service`'s `main.rs` installs the Prometheus recorder on the config's `metrics_address` (default `0.0.0.0:9000`) with **explicit histogram buckets**. This matters: without buckets, `metrics-exporter-prometheus` renders histograms as rolling-window summaries whose quantiles **reset to `0`** once the window (default 60 s) drains — an idle period reads as "took 0 s" rather than "no data". With buckets you get real `_bucket`/`_sum`/`_count` counters that never decay, are aggregatable, and honour the dashboard's time range.
Ladders are matched by name suffix, so a new timing metric is covered automatically:
```rust
.set_buckets(COUNT_BUCKETS) // fallback
.set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), LATENCY_BUCKETS)
```
## `dashboard_gen`
`tools/dashboard_gen` is a small Grafana dashboard builder plus the dashboard definitions. It prints JSON to stdout; the result is committed under `monitoring/grafana/dashboards/` and CI fails if it is stale.
```
src/lib.rs, schema.rs, styling.rs, unit.rs the builder library
src/dashboards/<name>.rs one dashboard per module
src/main.rs CLI: `dashboard_gen sequencer`
```
Panels are built fluently, and every query is composed from the `names` consts:
```rust
Panel::timeseries("Block production rate")
.width(18)
.target(rate_per_min(sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL, "blocks/min"))
```
Query helpers: `rate_per_min` for counters, `avg` for histograms, and `selected_percentile` for percentile lines — the latter reads a `percentile` dashboard dropdown created by `percentile_variable`, so one panel serves p50/p90/p95/p99 instead of drawing all four. Rate windows use `$__rate_interval`, which tracks the panel's zoom.
**Extending it:**
| Goal | Change |
|---|---|
| New panel | Add a `Panel::…` to a row in the dashboard module |
| New dashboard | `src/dashboards/<name>.rs` with `pub fn dashboard()`, a `pub mod` line, a `DashboardKind` variant, and a `just regenerate-dashboards` line |
| Grafana option we don't model yet | Add the field to `schema.rs` and a setter on `Panel` (styling setters live in `styling.rs` and panic when handed a redundant default) |
The builder deliberately models only the subset of Grafana's schema we use.
## Justfile
| Recipe | Purpose |
|---|---|
| `just regenerate-dashboards` | Rebuild the committed dashboard JSON. Run after touching metric names or dashboard code — CI checks it is current. |
| `just run-monitoring` | Prometheus (`:9090`) + Grafana (`:3000`, anonymous admin) in docker, scraping the sequencer every 5 s |
| `just get-sequencer-metrics` | `curl` the raw `/metrics` endpoint — quickest way to confirm a metric name and value |

View File

@ -35,6 +35,15 @@ impl LeeTransaction {
})
}
#[must_use]
pub const fn kind(&self) -> TxKind {
match self {
Self::Public(_) => TxKind::Public,
Self::PrivacyPreserving(_) => TxKind::PrivacyPreserving,
Self::ProgramDeployment(_) => TxKind::ProgramDeployment,
}
}
#[must_use]
pub fn affected_public_account_ids(&self) -> Vec<AccountId> {
match self {

View File

@ -12,3 +12,4 @@ tokio = { workspace = true, features = ["sync"] }
[dev-dependencies]
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
futures.workspace = true

View File

@ -18,6 +18,19 @@ impl<T> MemPool<T> {
(mem_pool, sender)
}
/// Returns the total number of items in the mempool, including both the front buffer and the
/// channel.
#[must_use]
pub fn len(&self) -> usize {
self.front_buffer.len().saturating_add(self.receiver.len())
}
/// Returns true if the mempool is empty, false otherwise.
#[must_use]
pub fn is_empty(&self) -> bool {
self.front_buffer.is_empty() && self.receiver.is_empty()
}
/// Pop an item from the mempool first checking the front buffer (LIFO) then the channel (FIFO).
pub fn pop(&mut self) -> Option<T> {
use tokio::sync::mpsc::error::TryRecvError;
@ -74,6 +87,7 @@ impl<T> MemPoolHandle<T> {
#[cfg(test)]
mod tests {
use futures::FutureExt as _;
use tokio::test;
use super::*;
@ -82,6 +96,7 @@ mod tests {
async fn mempool_new() {
let (mut pool, _handle): (MemPool<u64>, _) = MemPool::new(10);
assert_eq!(pool.pop(), None);
assert_eq!(pool.len(), 0);
}
#[test]
@ -89,10 +104,12 @@ mod tests {
let (mut pool, handle) = MemPool::new(10);
handle.push(1).await.unwrap();
assert_eq!(pool.len(), 1);
let item = pool.pop();
assert_eq!(item, Some(1));
assert_eq!(pool.pop(), None);
assert_eq!(pool.len(), 0);
}
#[test]
@ -103,29 +120,23 @@ mod tests {
handle.push(2).await.unwrap();
handle.push(3).await.unwrap();
assert_eq!(pool.len(), 3);
assert_eq!(pool.pop(), Some(1));
assert_eq!(pool.pop(), Some(2));
assert_eq!(pool.pop(), Some(3));
assert_eq!(pool.pop(), None);
}
#[test]
async fn pop_empty() {
let (mut pool, _handle): (MemPool<u64>, _) = MemPool::new(10);
assert_eq!(pool.pop(), None);
assert_eq!(pool.len(), 0);
}
#[test]
async fn max_size() {
let (mut pool, handle) = MemPool::new(2);
let (_pool, handle) = MemPool::new(2);
handle.push(1).await.unwrap();
handle.push(2).await.unwrap();
// This should block if buffer is full, but we'll use try_send in a real scenario
// For now, just verify we can pop items
assert_eq!(pool.pop(), Some(1));
assert_eq!(pool.pop(), Some(2));
// This should block if buffer is full
assert_eq!(handle.push(3).now_or_never(), None);
}
#[test]

View File

@ -12,6 +12,7 @@ lee.workspace = true
lee_core.workspace = true
chain_state.workspace = true
common.workspace = true
sequencer_core_metrics = { workspace = true, features = ["record"] }
storage.workspace = true
mempool.workspace = true
logos-blockchain-zone-sdk.workspace = true

View File

@ -0,0 +1,19 @@
[package]
name = "sequencer_core_metrics"
version = "0.1.0"
edition = "2024"
license = { workspace = true }
[lints]
workspace = true
[features]
default = []
# Enable metrics record
record = ["dep:common", "dep:metrics", "dep:strum"]
[dependencies]
common = { workspace = true, optional = true }
metrics = { workspace = true, optional = true }
strum = { workspace = true, optional = true }

View File

@ -0,0 +1,9 @@
//! This crate provides all metrics exposed by the sequencer core crate.
#[cfg(feature = "record")]
pub use record::*;
pub mod names;
#[cfg(feature = "record")]
pub mod record;

View File

@ -0,0 +1,9 @@
pub const BLOCK_CREATION_TIME: &str = "block_creation_time_seconds";
pub const CHAIN_HEIGHT: &str = "chain_height";
pub const BLOCKS_PRODUCED_TOTAL: &str = "blocks_produced_total";
pub const MEMPOOL_SIZE: &str = "mempool_size";
pub const MEMPOOL_MAX_SIZE: &str = "mempool_max_size";
pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str =
"mempool_transaction_application_time_seconds";
pub const TRANSACTIONS_PER_BLOCK: &str = "transactions_per_block";
pub const MEMPOOL_FAILED_TRANSACTIONS_TOTAL: &str = "mempool_failed_transactions_total";

View File

@ -0,0 +1,167 @@
#![expect(
clippy::cast_precision_loss,
clippy::as_conversions,
reason = "It's okay for metrics"
)]
use std::time::Duration;
use metrics::{Counter, Histogram, Unit, counter, gauge, histogram};
use strum::IntoEnumIterator as _;
use crate::names;
#[derive(Debug, Clone, Copy, strum::IntoStaticStr, strum::EnumIter)]
#[strum(serialize_all = "snake_case")]
pub enum TransactionOrigin {
User,
Sequencer,
}
#[derive(Debug, Clone, Copy, strum::IntoStaticStr, strum::EnumIter)]
#[strum(serialize_all = "snake_case")]
pub enum TxKind {
Public,
PrivacyPreserving,
ProgramDeployment,
}
/// Whether applying a transaction to the block's working state succeeded.
#[derive(Debug, Clone, Copy, strum::IntoStaticStr, strum::EnumIter)]
#[strum(serialize_all = "snake_case")]
pub enum ApplyStatus {
Applied,
Failed,
}
impl From<common::transaction::TxKind> for TxKind {
fn from(kind: common::transaction::TxKind) -> Self {
match kind {
common::transaction::TxKind::Public => Self::Public,
common::transaction::TxKind::PrivacyPreserving => Self::PrivacyPreserving,
common::transaction::TxKind::ProgramDeployment => Self::ProgramDeployment,
}
}
}
/// Initialize metrics.
pub fn init() {
blocks_produced_total_counter().increment(0);
mempool_failed_transactions_total_counter().increment(0);
record_mempool_size(0);
record_chain_height(0);
drop(block_creation_time_histogram());
drop(transactions_per_block_histogram());
for origin in TransactionOrigin::iter() {
for kind in TxKind::iter() {
for status in ApplyStatus::iter() {
drop(mempool_transaction_application_time_histogram(
origin, kind, status,
));
}
}
}
}
fn block_creation_time_histogram() -> Histogram {
histogram!(
description: "Time taken to create a block",
unit: Unit::Seconds,
names::BLOCK_CREATION_TIME
)
}
pub fn record_block_creation_time(duration: Duration) {
block_creation_time_histogram().record(duration.as_secs_f64());
}
/// Height of the chain head, which moves backwards on a reorg, hence a gauge.
pub fn record_chain_height(height: u64) {
gauge!(
description: "Height of the chain head",
unit: Unit::Count,
names::CHAIN_HEIGHT
)
.set(height as f64);
}
fn blocks_produced_total_counter() -> Counter {
counter!(
description: "Number of blocks produced by this sequencer and applied to the head",
unit: Unit::Count,
names::BLOCKS_PRODUCED_TOTAL
)
}
pub fn increment_blocks_produced_total() {
blocks_produced_total_counter().increment(1);
}
pub fn record_mempool_size(size: usize) {
gauge!(
description: "Size of the mempool",
unit: Unit::Count,
names::MEMPOOL_SIZE
)
.set(u64::try_from(size).expect("Mempool size should fit into u64") as f64);
}
pub fn record_mempool_max_size(size: usize) {
gauge!(
description: "Configured maximum size of the mempool",
unit: Unit::Count,
names::MEMPOOL_MAX_SIZE
)
.set(u64::try_from(size).expect("Mempool max size should fit into u64") as f64);
}
fn mempool_transaction_application_time_histogram(
origin: TransactionOrigin,
kind: TxKind,
status: ApplyStatus,
) -> Histogram {
histogram!(
description: "Time taken to apply a mempool transaction",
unit: Unit::Seconds,
names::MEMPOOL_TRANSACTION_APPLICATION_TIME,
"origin" => <&'static str>::from(origin),
"kind" => <&'static str>::from(kind),
"status" => <&'static str>::from(status),
)
}
pub fn record_mempool_transaction_application_time(
origin: TransactionOrigin,
kind: TxKind,
status: ApplyStatus,
duration: Duration,
) {
mempool_transaction_application_time_histogram(origin, kind, status)
.record(duration.as_secs_f64());
}
fn transactions_per_block_histogram() -> Histogram {
histogram!(
description: "Number of transactions from mempool included in block",
unit: Unit::Count,
names::TRANSACTIONS_PER_BLOCK
)
}
pub fn record_transactions_per_block(count: usize) {
transactions_per_block_histogram()
.record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64);
}
fn mempool_failed_transactions_total_counter() -> Counter {
counter!(
description: "Number of transactions from mempool that failed to be included in blocks",
unit: Unit::Count,
names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL
)
}
pub fn increment_mempool_failed_transactions_total() {
mempool_failed_transactions_total_counter().increment(1);
}

View File

@ -1,6 +1,7 @@
use std::{
fs::File,
io::BufReader,
net::{IpAddr, Ipv4Addr, SocketAddr},
path::{Path, PathBuf},
time::Duration,
};
@ -63,6 +64,9 @@ pub struct SequencerConfig {
/// Cross-zone messaging configuration. `None` disables the watcher.
#[serde(default)]
pub cross_zone: Option<CrossZoneConfig>,
/// Address the Prometheus metrics exporter binds to.
#[serde(default = "default_metrics_address")]
pub metrics_address: Option<SocketAddr>,
}
#[derive(Clone, Serialize, Deserialize)]
@ -77,6 +81,10 @@ pub struct BedrockConfig {
}
impl SequencerConfig {
/// Address [`Self::metrics_address`] falls back to when the config omits it.
pub const DEFAULT_METRICS_ADDRESS: SocketAddr =
SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 9000);
pub fn from_path(config_home: &Path) -> Result<Self> {
let file = File::open(config_home)?;
let reader = BufReader::new(file);
@ -88,3 +96,8 @@ impl SequencerConfig {
const fn default_max_block_size() -> ByteSize {
ByteSize::mib(1)
}
#[expect(clippy::unnecessary_wraps, reason = "Required by serde")]
const fn default_metrics_address() -> Option<SocketAddr> {
Some(SequencerConfig::DEFAULT_METRICS_ADDRESS)
}

View File

@ -80,6 +80,15 @@ pub enum TransactionOrigin {
Sequencer,
}
impl From<TransactionOrigin> for sequencer_core_metrics::TransactionOrigin {
fn from(origin: TransactionOrigin) -> Self {
match origin {
TransactionOrigin::User => Self::User,
TransactionOrigin::Sequencer => Self::Sequencer,
}
}
}
#[derive(Clone, Debug, BorshDeserialize)]
struct DepositMetadata {
recipient_id: lee::AccountId,
@ -144,6 +153,9 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
)
.expect("Failed to create database with genesis block");
// Incrementing count for genesis.
sequencer_core_metrics::increment_blocks_produced_total();
(store, genesis_state)
}
}
@ -195,6 +207,8 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
pub async fn start_from_config(
config: SequencerConfig,
) -> (Self, MemPoolHandle<(TransactionOrigin, LeeTransaction)>) {
sequencer_core_metrics::init();
let bedrock_signing_key =
load_or_create_signing_key(&config.home.join("bedrock_signing_key"))
.expect("Failed to load or create bedrock signing key");
@ -215,6 +229,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
let is_fresh_start = initial_checkpoint.is_none();
let (mempool, mempool_handle) = MemPool::new(config.mempool_max_size);
sequencer_core_metrics::record_mempool_max_size(config.mempool_max_size);
let block_publisher = BP::new(
&config.bedrock_config,
@ -299,6 +314,8 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
watchers,
};
sequencer_core_metrics::record_chain_height(sequencer_core.chain_height());
(sequencer_core, mempool_handle)
}
@ -596,6 +613,9 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
chain.head_state(),
Some(&checkpoint_bytes),
)?;
sequencer_core_metrics::increment_blocks_produced_total();
sequencer_core_metrics::record_chain_height(block.header.block_id);
}
// Neither branch persists anything, checkpoint included: the
// inscription it holds as pending belongs to a block that is not
@ -802,6 +822,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
let clock_tx = clock_invocation(new_block_timestamp);
let clock_lee_tx = LeeTransaction::Public(clock_tx.clone());
sequencer_core_metrics::record_mempool_size(self.mempool.len());
// Everything drained from the store first, then user work. `from_store`
// is not the same as a `Sequencer` origin: it says the transaction has a
// record behind it and so needs no requeue, where the origin only says
@ -872,16 +893,31 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
break;
}
if Self::apply_mempool_transaction(
let before_tx_apply = Instant::now();
let applied = Self::apply_mempool_transaction(
&mut working_state,
origin,
&tx,
new_block_height,
new_block_timestamp,
&mut withdrawals,
) {
);
if applied {
sequencer_core_metrics::record_mempool_transaction_application_time(
origin.into(),
tx.kind().into(),
sequencer_core_metrics::ApplyStatus::Applied,
before_tx_apply.elapsed(),
);
valid_transactions.push(tx);
} else {
sequencer_core_metrics::increment_mempool_failed_transactions_total();
sequencer_core_metrics::record_mempool_transaction_application_time(
origin.into(),
tx.kind().into(),
sequencer_core_metrics::ApplyStatus::Failed,
before_tx_apply.elapsed(),
);
// A failed transaction is simply left out of the block, except a
// dispatch: that one is re-fed from the store every turn, so one
// that can never execute would fail on every block for ever.
@ -897,6 +933,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
.transition_from_public_transaction(&clock_tx, new_block_height, new_block_timestamp)
.context("Clock transaction failed. Aborting block production.")?;
valid_transactions.push(clock_lee_tx);
sequencer_core_metrics::record_transactions_per_block(valid_transactions.len());
let hashable_data = HashableBlockData {
block_id: new_block_height,
@ -915,6 +952,8 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
now.elapsed().as_secs()
);
sequencer_core_metrics::record_block_creation_time(now.elapsed());
Ok(BlockWithMeta { block, withdrawals })
}
@ -1159,7 +1198,7 @@ fn apply_follow_update(
// The lock is held across the persist below so disk writes land in apply
// order — the produce path persists under this same lock.
let (resubmit_txs, outcome) = {
let (resubmit_txs, outcome, head_height) = {
let mut chain = chain.lock().expect("chain state mutex poisoned");
// Outcomes align with `adopted`.
@ -1259,9 +1298,11 @@ fn apply_follow_update(
})
.unwrap_or_else(|err| panic!("Failed to persist follow update: {err:#}"));
(resubmit_txs, outcome)
(resubmit_txs, outcome, head_tip.map_or(0, |tip| tip.id))
};
sequencer_core_metrics::record_chain_height(head_height);
if outcome.accepted_deposits > 0 {
info!(
"Recorded {} Bedrock Deposit event(s); their mints are drained from the store on our next turn",

View File

@ -90,6 +90,7 @@ fn setup_sequencer_config() -> SequencerConfig {
retry_pending_blocks_timeout: Duration::from_mins(4),
genesis: vec![],
cross_zone: None,
metrics_address: None,
}
}

View File

@ -15,6 +15,7 @@ mempool.workspace = true
sequencer_core = { workspace = true, features = ["testnet"] }
sequencer_service_protocol.workspace = true
sequencer_service_rpc = { workspace = true, features = ["server"] }
sequencer_service_metrics = { workspace = true, features = ["record"] }
programs.workspace = true
clap = { workspace = true, features = ["derive", "env"] }
@ -22,6 +23,7 @@ anyhow.workspace = true
env_logger.workspace = true
hex.workspace = true
log.workspace = true
metrics-exporter-prometheus.workspace = true
tokio.workspace = true
tokio-util.workspace = true
jsonrpsee.workspace = true

View File

@ -63,8 +63,9 @@ COPY --from=builder --chown=sequencer_service_user:sequencer_service_user /usr/l
VOLUME /var/lib/sequencer_service
# Expose default port
# Expose default ports
EXPOSE 3040
EXPOSE 9000
# Health check (TODO #244: Replace when a real health endpoint is available)
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \

View File

@ -18,6 +18,7 @@ services:
container_name: sequencer_service
ports:
- "3040:3040"
- "9000:9000"
volumes:
# Mount configuration file
- ./configs/docker/sequencer_config.json:/etc/sequencer_service/sequencer_config.json

View File

@ -0,0 +1,16 @@
[package]
name = "sequencer_service_metrics"
version = "0.1.0"
edition = "2024"
license = { workspace = true }
[lints]
workspace = true
[features]
default = []
# Enable metrics record
record = ["dep:metrics"]
[dependencies]
metrics = { workspace = true, optional = true }

View File

@ -0,0 +1,9 @@
//! This crate provides all metrics exposed by the sequencer service crate.
#[cfg(feature = "record")]
pub use record::*;
pub mod names;
#[cfg(feature = "record")]
pub mod record;

View File

@ -0,0 +1,3 @@
pub const SUBMITTED_TRANSACTIONS_TOTAL: &str = "submitted_transactions_total";
pub const BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL: &str =
"before_mempool_failed_transactions_total";

View File

@ -0,0 +1,32 @@
use metrics::{Counter, Unit, counter};
use crate::names;
pub fn init() {
submitted_transactions_total_counter().increment(0);
before_mempool_failed_transactions_total_counter().increment(0);
}
fn submitted_transactions_total_counter() -> Counter {
counter!(
description: "Number of transactions submitted",
unit: Unit::Count,
names::SUBMITTED_TRANSACTIONS_TOTAL
)
}
pub fn increment_submitted_transactions_total() {
submitted_transactions_total_counter().increment(1);
}
fn before_mempool_failed_transactions_total_counter() -> Counter {
counter!(
description: "Number of transactions that failed before reaching the mempool",
unit: Unit::Count,
names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL
)
}
pub fn increment_before_mempool_failed_transactions_total() {
before_mempool_failed_transactions_total_counter().increment(1);
}

View File

@ -211,6 +211,8 @@ async fn wait_for_store_release(store: &StoreRelease) {
}
pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result<SequencerHandle> {
sequencer_service_metrics::init();
let block_timeout = config.block_create_timeout;
let max_block_size = config.max_block_size;

View File

@ -3,9 +3,10 @@ use std::{
path::PathBuf,
};
use anyhow::Result;
use anyhow::{Context as _, Result};
use clap::Parser;
use log::{error, info};
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
use tokio::signal::unix::{SignalKind, signal};
use tokio_util::sync::CancellationToken;
@ -24,6 +25,10 @@ struct Args {
/// so multiple instances can share one config file.
#[clap(long)]
home: Option<PathBuf>,
/// Override the config's `metrics_address`, so multiple instances can share
/// one config file without fighting over the exporter port.
#[clap(long)]
metrics_address: Option<SocketAddr>,
}
#[tokio::main]
@ -34,21 +39,18 @@ struct Args {
async fn main() -> Result<()> {
env_logger::init();
let Args {
config_path,
port,
listen_address,
home,
} = Args::parse();
let args = Args::parse();
let cancellation_token = listen_for_shutdown_signal();
let mut config = sequencer_service::SequencerConfig::from_path(&config_path)?;
if let Some(home) = home {
config.home = home;
let mut config = sequencer_service::SequencerConfig::from_path(&args.config_path)?;
apply_config_overrides(&args, &mut config);
if let Some(metrics_address) = config.metrics_address {
install_prometheus_recorder(metrics_address)?;
}
let mut sequencer_handle =
sequencer_service::run(config, SocketAddr::new(listen_address, port)).await?;
sequencer_service::run(config, SocketAddr::new(args.listen_address, args.port)).await?;
tokio::select! {
() = cancellation_token.cancelled() => {
@ -71,6 +73,45 @@ async fn main() -> Result<()> {
Ok(())
}
fn apply_config_overrides(args: &Args, config: &mut sequencer_service::SequencerConfig) {
let Args {
home,
metrics_address,
config_path: _,
port: _,
listen_address: _,
} = args;
if let Some(home) = home {
config.home.clone_from(home);
}
if let Some(metrics_address) = metrics_address {
config.metrics_address = Some(*metrics_address);
}
}
/// Installs the recorder on `metrics_address`.
fn install_prometheus_recorder(metrics_address: SocketAddr) -> Result<()> {
/// Ladder for `*_seconds` histograms, densest across the 1100 ms band where
/// block production and transaction application actually land.
const LATENCY_BUCKETS: &[f64] = &[
0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
];
/// Fallback ladder for histograms that count things rather than measure time.
const COUNT_BUCKETS: &[f64] = &[1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0];
PrometheusBuilder::new()
.with_http_listener(metrics_address)
.with_recommended_naming(true)
.set_buckets(COUNT_BUCKETS)
.context("Failed to set default histogram buckets")?
.set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), LATENCY_BUCKETS)
.context("Failed to set latency histogram buckets")?
.install()
.context("Failed to install Prometheus recorder")
}
/// Cancelled on Ctrl-C or `SIGTERM`.
///
/// `SIGTERM` is what a container runtime sends first, so without it every

View File

@ -6,7 +6,7 @@ use jsonrpsee::{
types::{ErrorCode, ErrorObjectOwned},
};
use lee;
use log::warn;
use log::{error, warn};
use mempool::MemPoolHandle;
use sequencer_core::{
DbError, SequencerCore, TransactionOrigin, block_publisher::BlockPublisherTrait,
@ -44,49 +44,62 @@ impl<BC: BlockPublisherTrait + Send + Sync + 'static> sequencer_service_rpc::Rpc
for SequencerService<BC>
{
async fn send_transaction(&self, tx: LeeTransaction) -> Result<HashType, ErrorObjectOwned> {
// Reserve ~200 bytes for block header overhead
const BLOCK_HEADER_OVERHEAD: u64 = 200;
sequencer_service_metrics::increment_submitted_transactions_total();
let tx_hash = tx.hash();
let encoded_tx =
borsh::to_vec(&tx).expect("Transaction borsh serialization should not fail");
let tx_size = u64::try_from(encoded_tx.len()).expect("Transaction size should fit in u64");
let res = async move {
// Reserve ~200 bytes for block header overhead
const BLOCK_HEADER_OVERHEAD: u64 = 200;
let max_tx_size = self.max_block_size.saturating_sub(BLOCK_HEADER_OVERHEAD);
let encoded_tx =
borsh::to_vec(&tx).expect("Transaction borsh serialization should not fail");
let tx_size =
u64::try_from(encoded_tx.len()).expect("Transaction size should fit in u64");
if tx_size > max_tx_size {
return Err(ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
format!("Transaction too large: size {tx_size}, max {max_tx_size}"),
None::<()>,
));
}
let max_tx_size = self.max_block_size.saturating_sub(BLOCK_HEADER_OVERHEAD);
let authenticated_tx = tx
.transaction_stateless_check()
.inspect_err(|err| warn!("Error at pre_check {err:#?}"))
.map_err(|err| {
ErrorObjectOwned::owned(
if tx_size > max_tx_size {
return Err(ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
format!("{err:?}"),
format!("Transaction too large: size {tx_size}, max {max_tx_size}"),
None::<()>,
)
})?;
));
}
// Sequencer-only programs (the cross-zone inbox) are injected by the
// watcher; a user must not invoke them top-level, or anyone could forge
// an inbound cross-zone delivery. Chained user calls are already rejected
// by the inbox guest's caller-is-none assertion.
if let LeeTransaction::Public(public_tx) = &authenticated_tx
&& sequencer_core::is_sequencer_only_program(public_tx.message().program_id)
{
return Err(ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
"Program is sequencer-only and cannot be invoked by a user transaction".to_owned(),
None::<()>,
));
}
let authenticated_tx = tx
.transaction_stateless_check()
.inspect_err(|err| warn!("Error at pre_check {err:#?}"))
.map_err(|err| {
ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
format!("{err:?}"),
None::<()>,
)
})?;
// Sequencer-only programs (the cross-zone inbox) are injected by the
// watcher; a user must not invoke them top-level, or anyone could forge
// an inbound cross-zone delivery. Chained user calls are already rejected
// by the inbox guest's caller-is-none assertion.
if let LeeTransaction::Public(public_tx) = &authenticated_tx
&& sequencer_core::is_sequencer_only_program(public_tx.message().program_id)
{
return Err(ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
"Program is sequencer-only and cannot be invoked by a user transaction"
.to_owned(),
None::<()>,
));
}
Ok(authenticated_tx)
};
let authenticated_tx = res.await.inspect_err(|err| {
sequencer_service_metrics::increment_before_mempool_failed_transactions_total();
error!("Transaction failed before reaching mempool: {err:#?}");
})?;
self.mempool_handle
.push((TransactionOrigin::User, authenticated_tx))

View File

@ -0,0 +1,36 @@
# Prometheus + Grafana monitoring stack.
services:
prometheus:
image: prom/prometheus:v3.13.1
container_name: prometheus
command:
- --config.file=/etc/prometheus/prometheus.yml
ports:
- "9090:9090"
volumes:
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus_data:/prometheus
# Lets Prometheus reach services running natively on the host.
extra_hosts:
- "host.docker.internal:host-gateway"
grafana:
image: grafana/grafana:13.1.1
container_name: grafana
ports:
- "3000:3000"
environment:
# Dev-only: open Grafana with no login, full access.
- GF_AUTH_ANONYMOUS_ENABLED=true
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
- GF_AUTH_DISABLE_LOGIN_FORM=true
volumes:
- ./grafana/provisioning:/etc/grafana/provisioning:ro
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
depends_on:
- prometheus
volumes:
prometheus_data:
grafana_data:

View File

@ -0,0 +1,377 @@
{
"annotations": { "list": [ ] },
"editable": true,
"graphTooltip": 1,
"panels": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "color": { "mode": "fixed", "fixedColor": "blue" }, "unit": "short", "decimals": 0 },
"overrides": [ ]
},
"gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 },
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "chain_height",
"legendFormat": "height",
"refId": "A"
}
],
"title": "Chain height",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "color": { "mode": "fixed", "fixedColor": "green" }, "unit": "short", "decimals": 0 },
"overrides": [ ]
},
"gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 },
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "blocks_produced_total",
"legendFormat": "produced",
"refId": "A"
}
],
"title": "Blocks produced by this sequencer since startup",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" },
"overrides": [
{
"matcher": { "id": "byName", "options": "produced · blocks/min" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ]
}
]
},
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 },
"id": 3,
"options": {
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "single" }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(blocks_produced_total[$__rate_interval]) * 60",
"legendFormat": "produced · blocks/min",
"refId": "A"
}
],
"title": "Block production rate",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "s" },
"overrides": [
{
"matcher": { "id": "byName", "options": "avg" },
"properties": [
{ "id": "custom.lineStyle", "value": { "dash": [ 8, 4 ], "fill": "dash" } },
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } }
]
}
]
},
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 7 },
"id": 4,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(${percentile}, sum by (le) (rate(block_creation_time_seconds_bucket[$__rate_interval])))",
"legendFormat": "${percentile:text}",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(block_creation_time_seconds_sum[$__rate_interval]) / rate(block_creation_time_seconds_count[$__rate_interval])",
"legendFormat": "avg",
"refId": "B"
}
],
"title": "Block creation time",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "s" },
"overrides": [ ]
},
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 16 },
"id": 5,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(${percentile}, sum by (le, kind, origin, status) (rate(mempool_transaction_application_time_seconds_bucket[$__rate_interval])))",
"legendFormat": "{{kind}} · {{origin}} · {{status}}",
"refId": "A"
}
],
"title": "Transaction application time",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" },
"overrides": [
{
"matcher": { "id": "byName", "options": "avg" },
"properties": [
{ "id": "custom.lineStyle", "value": { "dash": [ 8, 4 ], "fill": "dash" } },
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } }
]
}
]
},
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 },
"id": 6,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(${percentile}, sum by (le) (rate(transactions_per_block_bucket[$__rate_interval])))",
"legendFormat": "${percentile:text}",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(transactions_per_block_sum[$__rate_interval]) / rate(transactions_per_block_count[$__rate_interval])",
"legendFormat": "avg",
"refId": "B"
}
],
"title": "Transactions per block",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"unit": "percent",
"decimals": 1,
"min": 0.0,
"max": 100.0,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "orange", "value": 70.0 },
{ "color": "red", "value": 90.0 }
]
}
},
"overrides": [ ]
},
"gridPos": { "h": 8, "w": 6, "x": 0, "y": 25 },
"id": 7,
"options": {
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "100 * mempool_size / mempool_max_size",
"legendFormat": "utilization",
"refId": "A"
}
],
"title": "Mempool utilization",
"type": "gauge"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "spanNulls": true },
"unit": "short",
"min": 0.0
},
"overrides": [
{
"matcher": { "id": "byName", "options": "capacity" },
"properties": [
{ "id": "custom.lineStyle", "value": { "dash": [ 8, 4 ], "fill": "dash" } },
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }
]
},
{
"matcher": { "id": "byName", "options": "queued" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ]
}
]
},
"gridPos": { "h": 8, "w": 18, "x": 6, "y": 25 },
"id": 8,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "mempool_size",
"legendFormat": "queued",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "mempool_max_size",
"legendFormat": "capacity",
"refId": "B"
}
],
"title": "Mempool size vs capacity",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"unit": "percent",
"decimals": 2,
"min": 0.0,
"max": 100.0,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "orange", "value": 1.0 },
{ "color": "red", "value": 5.0 }
]
}
},
"overrides": [ ]
},
"gridPos": { "h": 8, "w": 6, "x": 0, "y": 33 },
"id": 9,
"options": {
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "100 * (increase(before_mempool_failed_transactions_total[$__range]) + increase(mempool_failed_transactions_total[$__range])) / clamp_min(increase(submitted_transactions_total[$__range]), 1)",
"legendFormat": "failed",
"refId": "A"
}
],
"title": "Failed transactions share",
"type": "gauge"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 35, "gradientMode": "opacity" },
"unit": "short",
"min": 0.0
},
"overrides": [
{
"matcher": { "id": "byName", "options": "submitted" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ]
},
{
"matcher": { "id": "byName", "options": "failed · before mempool" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ]
},
{
"matcher": { "id": "byName", "options": "failed · in mempool" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ]
}
]
},
"gridPos": { "h": 8, "w": 18, "x": 6, "y": 33 },
"id": 10,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(submitted_transactions_total[$__rate_interval]) * 60",
"legendFormat": "submitted",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(before_mempool_failed_transactions_total[$__rate_interval]) * 60",
"legendFormat": "failed · before mempool",
"refId": "B"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(mempool_failed_transactions_total[$__rate_interval]) * 60",
"legendFormat": "failed · in mempool",
"refId": "C"
}
],
"title": "Submitted vs failed transactions (per minute)",
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 39,
"tags": [ "sequencer" ],
"templating": {
"list": [
{
"current": { "selected": true, "text": "p95", "value": "0.95" },
"includeAll": false,
"label": "Percentile",
"multi": false,
"name": "percentile",
"options": [
{ "selected": false, "text": "p50", "value": "0.5" },
{ "selected": false, "text": "p90", "value": "0.9" },
{ "selected": true, "text": "p95", "value": "0.95" },
{ "selected": false, "text": "p99", "value": "0.99" }
],
"query": "p50 : 0.5, p90 : 0.9, p95 : 0.95, p99 : 0.99",
"type": "custom"
}
]
},
"time": { "from": "now-15m", "to": "now" },
"timezone": "",
"title": "Sequencer",
"uid": "sequencer"
}

View File

@ -0,0 +1,9 @@
apiVersion: 1
providers:
- name: sequencer
type: file
allowUiUpdates: true
options:
path: /var/lib/grafana/dashboards
foldersFromFilesStructure: false

View File

@ -0,0 +1,13 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: true
jsonData:
# Must match prometheus.yml's `scrape_interval`
timeInterval: 5s

View File

@ -0,0 +1,10 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
scrape_configs:
- job_name: sequencer
metrics_path: /metrics
static_configs:
- targets:
- host.docker.internal:9000

View File

@ -110,6 +110,7 @@ pub fn sequencer_config(
auth: None,
},
cross_zone,
metrics_address: Some(SequencerConfig::DEFAULT_METRICS_ADDRESS),
})
}

View File

@ -0,0 +1,17 @@
[package]
name = "dashboard_gen"
version = "0.1.0"
edition = "2024"
license = { workspace = true }
[lints]
workspace = true
[dependencies]
sequencer_core_metrics.workspace = true
sequencer_service_metrics.workspace = true
clap = { workspace = true, features = ["derive"] }
serde = { workspace = true, features = ["derive", "alloc"] }
serde_json.workspace = true
json-pretty-compact = "0.1.2"

View File

@ -0,0 +1,3 @@
//! One module per dashboard, each exposing a `dashboard()` builder.
pub mod sequencer;

View File

@ -0,0 +1,195 @@
//! The sequencer dashboard: chain progress, block timings, mempool and
//! transaction outcomes.
#![expect(
clippy::non_ascii_literal,
reason = "legend separators use `·` intentionally, matching the rendered Grafana labels"
)]
use dashboard_gen::{
Color, Dashboard, FieldOverride, GradientMode, Panel, Target, Thresholds, Unit, avg,
percentile_legend, percentile_variable, rate_per_min, selected_percentile,
};
const PERCENTILES: &[u32] = &[50, 90, 95, 99];
const DEFAULT_PERCENTILE: u32 = 95;
pub fn dashboard() -> Dashboard {
Dashboard::new("Sequencer", "sequencer")
.tag("sequencer")
.variable(percentile_variable(PERCENTILES, DEFAULT_PERCENTILE))
.row(
7,
[
Panel::stat("Chain height")
.width(6)
.unit(Unit::Short)
.decimals(0)
.color(Color::fixed("blue"))
.target(
Target::new(sequencer_core_metrics::names::CHAIN_HEIGHT).legend("height"),
),
Panel::stat("Blocks produced by this sequencer since startup")
.width(6)
.unit(Unit::Short)
.decimals(0)
.color(Color::fixed("green"))
.target(
Target::new(sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL)
.legend("produced"),
),
Panel::timeseries("Block production rate")
.width(12)
.unit(Unit::Short)
.target(rate_per_min(
sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL,
"produced · blocks/min",
))
.with_override(
FieldOverride::by_name("produced · blocks/min").color(Color::fixed("green")),
),
],
)
.row(
9,
[Panel::timeseries("Block creation time")
.width(24)
.unit(Unit::Seconds)
.target(selected_percentile(
sequencer_core_metrics::names::BLOCK_CREATION_TIME,
&[],
&percentile_legend(),
))
.target(avg(sequencer_core_metrics::names::BLOCK_CREATION_TIME))
.with_override(
FieldOverride::by_name("avg")
.dashed_line()
.color(Color::fixed("text")),
)],
)
.row(
9,
[
Panel::timeseries("Transaction application time")
.width(12)
.unit(Unit::Seconds)
.target(selected_percentile(
sequencer_core_metrics::names::MEMPOOL_TRANSACTION_APPLICATION_TIME,
&["kind", "origin", "status"],
"{{kind}} · {{origin}} · {{status}}",
)),
Panel::timeseries("Transactions per block")
.width(12)
.unit(Unit::Short)
.target(selected_percentile(
sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK,
&[],
&percentile_legend(),
))
.target(avg(sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK))
.with_override(
FieldOverride::by_name("avg")
.dashed_line()
.color(Color::fixed("text")),
),
],
)
.row(
8,
[
Panel::gauge("Mempool utilization")
.width(6)
.unit(Unit::Percent)
.decimals(1)
.min(0.0)
.max(100.0)
.thresholds(
Thresholds::base("green")
.step(70.0, "orange")
.step(90.0, "red"),
)
.target(
Target::new(format!(
"100 * {size} / {max_size}",
size = sequencer_core_metrics::names::MEMPOOL_SIZE,
max_size = sequencer_core_metrics::names::MEMPOOL_MAX_SIZE,
))
.legend("utilization"),
),
Panel::timeseries("Mempool size vs capacity")
.width(18)
.unit(Unit::Short)
.span_nulls()
.min(0.0)
.target(
Target::new(sequencer_core_metrics::names::MEMPOOL_SIZE).legend("queued"),
)
.target(
Target::new(sequencer_core_metrics::names::MEMPOOL_MAX_SIZE)
.legend("capacity"),
)
.with_override(
FieldOverride::by_name("capacity")
.dashed_line()
.color(Color::fixed("red")),
)
.with_override(FieldOverride::by_name("queued").color(Color::fixed("blue"))),
],
)
.row(
8,
[
Panel::gauge("Failed transactions share")
.width(6)
.unit(Unit::Percent)
.decimals(2)
.min(0.0)
.max(100.0)
.thresholds(
Thresholds::base("green")
.step(1.0, "orange")
.step(5.0, "red"),
)
.target(
Target::new(format!(
// Both failure stages against the same submission base;
// `clamp_min` keeps an idle window (nothing submitted)
// reading as 0% instead of a division by zero.
"100 * (increase({before_mempool}[$__range]) + increase({in_mempool}[$__range])) / clamp_min(increase({submitted}[$__range]), 1)",
before_mempool = sequencer_service_metrics::names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL,
in_mempool = sequencer_core_metrics::names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL,
submitted = sequencer_service_metrics::names::SUBMITTED_TRANSACTIONS_TOTAL,
))
.legend("failed"),
),
Panel::timeseries("Submitted vs failed transactions (per minute)")
.width(18)
.unit(Unit::Short)
.min(0.0)
.fill_opacity(35)
.gradient_mode(GradientMode::Opacity)
.target(rate_per_min(
sequencer_service_metrics::names::SUBMITTED_TRANSACTIONS_TOTAL,
"submitted",
))
.target(rate_per_min(
sequencer_service_metrics::names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL,
"failed · before mempool",
))
.target(rate_per_min(
sequencer_core_metrics::names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL,
"failed · in mempool",
))
.with_override(
FieldOverride::by_name("submitted").color(Color::fixed("green")),
)
.with_override(
FieldOverride::by_name("failed · before mempool")
.color(Color::fixed("orange")),
)
.with_override(
FieldOverride::by_name("failed · in mempool").color(Color::fixed("red")),
),
],
)
}

View File

@ -0,0 +1,603 @@
//! A tiny, hand-rolled Grafana dashboard builder.
//!
//! This is a deliberately small subset of what the (Rust-less) Grafana
//! Foundation SDK does: model only the panel types and options we actually use,
//! expose a fluent builder, and render to the same dashboard JSON Grafana
//! provisions. The payoff over hand-written JSON:
//!
//! * metric names live in Rust `const`s, so a rename is a compile error here;
//! * repetitive structure (percentile targets, grid layout, legend/tooltip defaults) collapses into
//! a single call instead of copy-pasted JSON.
//!
//! Build a [`Dashboard`] and serialize it directly.
// Styling vocabularies passed to the optional `styling` setters are part of the
// public API (and are used by this module's `Panel` fields and `finalize`).
pub use schema::{
AxisPlacement, Color, GradientMode, LineInterpolation, ShowPoints, StackingMode, Thresholds,
Variable,
};
use schema::{
Calc, Custom, Datasource, Defaults, DrawStyle, EmptyList, FieldConfig, Fill, GaugeOptions,
GraphMode, GridPos, Legend, LegendDisplay, LineStyle, Matcher, MatcherKind, Options,
OverrideProperty, PanelModel, PanelType, Placement, PropertyId, PropertyValue, ReduceOptions,
SortOrder, Stacking, StatColorMode, StatOptions, Templating, TimeRange, TimeSeriesOptions,
Tooltip, TooltipMode, VariableKind, VariableOption,
};
use serde::Serialize;
pub use unit::Unit;
mod schema;
mod styling;
mod unit;
/// Datasource uid every panel/target points at. Dashboards stay portable across
/// environments because they reference the datasource by this stable uid rather
/// than by a per-environment URL.
pub const DATASOURCE_UID: &str = "prometheus";
/// Window every rate query — counter and histogram alike — rates over.
/// `$__rate_interval` tracks the panel's zoom, so the window always covers the
/// step Grafana samples at: a fixed one shorter than the step leaves gaps the
/// query never looks at, dropping events from the graph entirely.
const RATE_WINDOW: &str = "$__rate_interval";
/// Dashboard variable holding the quantile every percentile query reads.
const PERCENTILE_VAR: &str = "percentile";
/// Area fill under a timeseries line, as a percentage. Enough to read a series'
/// shape at a glance without drowning the ones stacked behind it.
pub(crate) const DEFAULT_FILL_OPACITY: u32 = 10;
/// A single Prometheus query within a panel.
#[derive(Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Target {
datasource: Datasource,
expr: String,
#[serde(rename = "legendFormat")]
legend: String,
ref_id: String,
}
impl Target {
pub fn new(expr: impl Into<String>) -> Self {
Self {
datasource: Datasource::prometheus(),
expr: expr.into(),
legend: String::new(),
ref_id: ref_letter(0),
}
}
#[must_use]
pub fn legend(mut self, legend: impl Into<String>) -> Self {
self.legend = legend.into();
self
}
}
/// A per-series style override, matched by series name.
#[derive(Serialize)]
pub struct FieldOverride {
matcher: Matcher,
properties: Vec<OverrideProperty>,
}
impl FieldOverride {
pub fn by_name(name: impl Into<String>) -> Self {
Self {
matcher: Matcher {
id: MatcherKind::ByName,
options: name.into(),
},
properties: Vec::new(),
}
}
#[must_use]
pub fn color(mut self, color: Color) -> Self {
self.properties.push(OverrideProperty {
id: PropertyId::Color,
value: PropertyValue::Color(color),
});
self
}
#[must_use]
pub fn dashed_line(mut self) -> Self {
self.properties.push(OverrideProperty {
id: PropertyId::LineStyle,
value: PropertyValue::LineStyle(LineStyle {
dash: [8, 4],
fill: Fill::Dash,
}),
});
self
}
}
#[derive(Clone, Copy)]
enum Kind {
Stat,
TimeSeries,
Gauge,
}
/// A dashboard panel builder. Grid position and panel id are assigned by
/// [`Dashboard::row`]; everything else is set here.
pub struct Panel {
title: String,
kind: Kind,
targets: Vec<Target>,
width: u32,
unit: Option<Unit>,
decimals: Option<u32>,
color: Option<Color>,
min: Option<f64>,
max: Option<f64>,
thresholds: Option<Thresholds>,
span_nulls: bool,
overrides: Vec<FieldOverride>,
// Optional timeseries styling, set via the `styling` setters.
fill_opacity: Option<u32>,
line_interpolation: Option<LineInterpolation>,
show_points: Option<ShowPoints>,
gradient_mode: Option<GradientMode>,
stacking: Option<StackingMode>,
axis_placement: Option<AxisPlacement>,
axis_label: Option<String>,
}
impl Panel {
fn new(title: impl Into<String>, kind: Kind) -> Self {
Self {
title: title.into(),
kind,
targets: Vec::new(),
width: 0,
unit: None,
decimals: None,
color: None,
min: None,
max: None,
thresholds: None,
span_nulls: false,
overrides: Vec::new(),
fill_opacity: None,
line_interpolation: None,
show_points: None,
gradient_mode: None,
stacking: None,
axis_placement: None,
axis_label: None,
}
}
/// A single big-number panel.
pub fn stat(title: impl Into<String>) -> Self {
Self::new(title, Kind::Stat)
}
/// A time-series line panel.
pub fn timeseries(title: impl Into<String>) -> Self {
Self::new(title, Kind::TimeSeries)
}
/// A radial gauge panel. Pair it with [`Panel::min`]/[`Panel::max`] — the
/// dial needs a range to fill — and [`Panel::thresholds`] for its coloring.
pub fn gauge(title: impl Into<String>) -> Self {
Self::new(title, Kind::Gauge)
}
/// Grid width in Grafana's 24-column units. Unset panels split the row's
/// remaining width evenly.
#[must_use]
pub const fn width(mut self, width: u32) -> Self {
self.width = width;
self
}
#[must_use]
pub fn unit(mut self, unit: Unit) -> Self {
self.unit = Some(unit);
self
}
#[must_use]
pub const fn decimals(mut self, decimals: u32) -> Self {
self.decimals = Some(decimals);
self
}
#[must_use]
pub fn color(mut self, color: Color) -> Self {
self.color = Some(color);
self
}
/// Lower bound of the value scale.
#[must_use]
pub const fn min(mut self, min: f64) -> Self {
self.min = Some(min);
self
}
/// Upper bound of the value scale.
#[must_use]
pub const fn max(mut self, max: f64) -> Self {
self.max = Some(max);
self
}
#[must_use]
pub fn thresholds(mut self, thresholds: Thresholds) -> Self {
self.thresholds = Some(thresholds);
self
}
#[must_use]
pub const fn span_nulls(mut self) -> Self {
self.span_nulls = true;
self
}
#[must_use]
pub fn target(mut self, target: Target) -> Self {
self.targets.push(target);
self
}
#[must_use]
pub fn targets(mut self, targets: impl IntoIterator<Item = Target>) -> Self {
self.targets.extend(targets);
self
}
#[must_use]
pub fn with_override(mut self, over: FieldOverride) -> Self {
self.overrides.push(over);
self
}
fn finalize(self, id: u32, grid_pos: GridPos) -> PanelModel {
let targets: Vec<Target> = self
.targets
.into_iter()
.enumerate()
.map(|(i, mut t)| {
t.ref_id = ref_letter(i);
t
})
.collect();
let unit = self.unit.unwrap_or_else(default_unit);
let (defaults, options, panel_type) = match self.kind {
Kind::Stat => {
let defaults = Defaults {
color: self.color,
custom: None,
unit,
decimals: self.decimals,
min: self.min,
max: self.max,
thresholds: self.thresholds,
};
let options = Options::Stat(StatOptions {
color_mode: StatColorMode::Value,
graph_mode: GraphMode::Area,
reduce_options: ReduceOptions {
calcs: vec![Calc::LastNotNull],
fields: String::new(),
values: false,
},
});
(defaults, options, PanelType::Stat)
}
Kind::Gauge => {
let defaults = Defaults {
color: self.color,
custom: None,
unit,
decimals: self.decimals,
min: self.min,
max: self.max,
thresholds: self.thresholds,
};
let options = Options::Gauge(GaugeOptions {
reduce_options: ReduceOptions {
calcs: vec![Calc::LastNotNull],
fields: String::new(),
values: false,
},
show_threshold_labels: false,
show_threshold_markers: true,
});
(defaults, options, PanelType::Gauge)
}
Kind::TimeSeries => {
let defaults = Defaults {
color: None,
custom: Some(Custom {
draw_style: DrawStyle::Line,
line_width: 1,
fill_opacity: self.fill_opacity.unwrap_or(DEFAULT_FILL_OPACITY),
span_nulls: self.span_nulls.then_some(true),
line_interpolation: self.line_interpolation,
show_points: self.show_points,
gradient_mode: self.gradient_mode,
stacking: self.stacking.map(|mode| Stacking {
mode,
group: "A".to_owned(),
}),
axis_placement: self.axis_placement,
axis_label: self.axis_label,
}),
unit,
decimals: None,
min: self.min,
max: self.max,
thresholds: self.thresholds,
};
// Panels with several series read better as a sortable table with
// a multi-series tooltip; single-series panels stay compact. A
// `{{label}}` legend fans one target out into a series per label
// value, so it counts as several too.
let multi =
targets.len() > 1 || targets.iter().any(|target| target.legend.contains("{{"));
let options = Options::TimeSeries(TimeSeriesOptions {
legend: Legend {
display_mode: if multi {
LegendDisplay::Table
} else {
LegendDisplay::List
},
placement: Placement::Bottom,
calcs: vec![Calc::Last, Calc::Max],
},
tooltip: if multi {
Tooltip {
mode: TooltipMode::Multi,
sort: Some(SortOrder::Desc),
}
} else {
Tooltip {
mode: TooltipMode::Single,
sort: None,
}
},
});
(defaults, options, PanelType::Timeseries)
}
};
PanelModel {
datasource: Datasource::prometheus(),
field_config: FieldConfig {
defaults,
overrides: self.overrides,
},
grid_pos,
id,
options,
targets,
title: self.title,
panel_type,
}
}
}
/// A dashboard, built row by row. Serialize it directly to get the JSON.
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Dashboard {
annotations: EmptyList,
editable: bool,
graph_tooltip: u32,
panels: Vec<PanelModel>,
refresh: String,
schema_version: u32,
tags: Vec<String>,
templating: Templating,
time: TimeRange,
timezone: String,
title: String,
uid: String,
// Layout cursor — not part of the dashboard schema.
#[serde(skip)]
next_id: u32,
#[serde(skip)]
cursor_y: u32,
}
impl Dashboard {
pub fn new(title: impl Into<String>, uid: impl Into<String>) -> Self {
Self {
annotations: EmptyList::default(),
editable: true,
graph_tooltip: 1,
panels: Vec::new(),
refresh: "5s".to_owned(),
schema_version: 39,
tags: Vec::new(),
templating: Templating::default(),
time: TimeRange {
from: "now-15m".to_owned(),
to: "now".to_owned(),
},
timezone: String::new(),
title: title.into(),
uid: uid.into(),
next_id: 1,
cursor_y: 0,
}
}
#[must_use]
pub fn tag(mut self, tag: impl Into<String>) -> Self {
self.tags.push(tag.into());
self
}
#[must_use]
pub fn refresh(mut self, refresh: impl Into<String>) -> Self {
self.refresh = refresh.into();
self
}
/// Add a dropdown to the dashboard's top bar, e.g. [`percentile_variable`].
#[must_use]
pub fn variable(mut self, variable: Variable) -> Self {
self.templating.list.push(variable);
self
}
/// Place a horizontal row of panels at the current vertical cursor. Panel
/// ids, x offsets and y are assigned here; unset widths split the remaining
/// 24 columns evenly.
#[must_use]
pub fn row(mut self, height: u32, panels: impl IntoIterator<Item = Panel>) -> Self {
let panels: Vec<Panel> = panels.into_iter().collect();
let specified: u32 = panels.iter().map(|p| p.width).sum();
let auto_count = u32::try_from(panels.iter().filter(|p| p.width == 0).count()).unwrap_or(0);
// `checked_div` yields `None` when there are no auto-width panels; the
// fallback width is unused in that case.
let auto_width = 24_u32
.saturating_sub(specified)
.checked_div(auto_count)
.unwrap_or(0);
let mut x = 0;
for panel in panels {
let w = if panel.width == 0 {
auto_width
} else {
panel.width
};
let grid_pos = GridPos {
h: height,
w,
x,
y: self.cursor_y,
};
let id = self.next_id;
self.next_id = self.next_id.saturating_add(1);
x = x.saturating_add(w);
self.panels.push(panel.finalize(id, grid_pos));
}
self.cursor_y = self.cursor_y.saturating_add(height);
self
}
}
/// The dropdown driving every [`selected_percentile`] query, offering
/// `percentiles` (e.g. `[50, 90, 95, 99]`) with `default` pre-selected.
///
/// Panics if `default` is not one of `percentiles`, or if any of them falls
/// outside `p1..=p99`.
#[must_use]
pub fn percentile_variable(percentiles: &[u32], default: u32) -> Variable {
assert!(
percentiles.contains(&default),
"default p{default} is not one of the offered percentiles {percentiles:?}",
);
let options: Vec<VariableOption> = percentiles
.iter()
.map(|&p| VariableOption {
selected: p == default,
text: format!("p{p}"),
value: quantile(p),
})
.collect();
let current = options
.iter()
.find(|option| option.selected)
.cloned()
.expect("`default` is one of `percentiles`, asserted above");
let query = options
.iter()
.map(|option| format!("{} : {}", option.text, option.value))
.collect::<Vec<_>>()
.join(", ");
Variable {
current,
include_all: false,
label: "Percentile".to_owned(),
// A quantile is a scalar argument to `histogram_quantile`, so exactly
// one may be selected.
multi: false,
name: PERCENTILE_VAR.to_owned(),
options,
query,
kind: VariableKind::Custom,
}
}
/// The legend fragment that renders the dropdown's current choice, e.g. `p95`.
#[must_use]
pub fn percentile_legend() -> String {
format!("${{{PERCENTILE_VAR}:text}}")
}
/// A [`histogram_quantile`] line over `metric`'s buckets, at whatever quantile
/// [`percentile_variable`] currently holds. `labels` stay split out into their
/// own series; every other label is summed away.
///
/// [`histogram_quantile`]: https://prometheus.io/docs/prometheus/latest/querying/functions/#histogram_quantile
#[must_use]
pub fn selected_percentile(metric: &str, labels: &[&str], legend: &str) -> Target {
// `le` carries the bucket boundary, so it must survive the aggregation.
let grouping = std::iter::once("le")
.chain(labels.iter().copied())
.collect::<Vec<_>>()
.join(", ");
Target::new(format!(
"histogram_quantile(${{{PERCENTILE_VAR}}}, sum by ({grouping}) (rate({metric}_bucket[{RATE_WINDOW}])))"
))
.legend(legend)
}
/// A percentile as its `histogram_quantile` argument, derived without float
/// math: zero-pad to two digits then drop trailing zeros (50 → `0.5`).
///
/// Panics outside `1..=99`, the only range two digits render: p100 would come
/// out as `0.1` and p0 as `0.`, neither of them loudly.
fn quantile(percentile: u32) -> String {
assert!(
(1..=99).contains(&percentile),
"p{percentile} is outside the supported range p1..=p99",
);
let quantile = format!("0.{percentile:02}");
quantile.trim_end_matches('0').to_owned()
}
/// An `avg` target for a histogram metric: `rate(sum) / rate(count)`.
#[must_use]
pub fn avg(metric: &str) -> Target {
Target::new(format!(
"rate({metric}_sum[{RATE_WINDOW}]) / rate({metric}_count[{RATE_WINDOW}])"
))
.legend("avg")
}
/// A per-minute rate target for a counter metric.
#[must_use]
pub fn rate_per_min(metric: &str, legend: &str) -> Target {
//`rate` is per-second whatever the window, so `* 60` is per-minute regardless of the panel's
//`rate` zoom.
Target::new(format!("rate({metric}[{RATE_WINDOW}]) * 60")).legend(legend)
}
const fn default_unit() -> Unit {
Unit::Short
}
fn ref_letter(index: usize) -> String {
let offset = u8::try_from(index).unwrap_or(0);
char::from(b'A'.saturating_add(offset)).to_string()
}

View File

@ -0,0 +1,45 @@
//! Builds one of the Grafana dashboards and prints its JSON to stdout.
#![expect(
clippy::print_stdout,
reason = "CLI tool: emitting the dashboard JSON on stdout is the deliverable"
)]
use clap::{Parser, ValueEnum};
use dashboard_gen::Dashboard;
use json_pretty_compact::PrettyCompactFormatter;
use serde::Serialize as _;
mod dashboards;
#[derive(Debug, Parser)]
#[clap(version)]
struct Args {
/// Which dashboard to build.
dashboard: DashboardKind,
}
#[derive(Debug, Clone, Copy, ValueEnum)]
enum DashboardKind {
Sequencer,
}
impl DashboardKind {
fn build(self) -> Dashboard {
match self {
Self::Sequencer => dashboards::sequencer::dashboard(),
}
}
}
fn main() {
let Args { dashboard } = Args::parse();
let formatter = PrettyCompactFormatter::new();
let mut output = Vec::new();
let mut ser = serde_json::Serializer::with_formatter(&mut output, formatter);
dashboard.build().serialize(&mut ser).unwrap();
let json = String::from_utf8(output).unwrap();
println!("{json}");
}

View File

@ -0,0 +1,434 @@
//! The serializable Grafana dashboard schema — the internal data model the
//! public builders assemble into.
//!
//! Every type here is `Serialize`-only: the model is sized for what we emit.
use serde::Serialize;
use crate::{DATASOURCE_UID, FieldOverride, Target, unit::Unit};
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum DatasourceKind {
Prometheus,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Fill {
Dash,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum DrawStyle {
Line,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum MatcherKind {
ByName,
}
#[derive(Clone, Copy, Serialize)]
pub enum PropertyId {
#[serde(rename = "color")]
Color,
#[serde(rename = "custom.lineStyle")]
LineStyle,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum Calc {
LastNotNull,
Last,
Max,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum StatColorMode {
Value,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum GraphMode {
Area,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum LegendDisplay {
Table,
List,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Placement {
Bottom,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum TooltipMode {
Single,
Multi,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum SortOrder {
Desc,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum PanelType {
Stat,
Timeseries,
Gauge,
}
#[derive(Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum ThresholdMode {
Absolute,
/// Steps expressed as a percentage of the minmax range. The builder never
/// emits this; it exists so Grafana exports using it still parse.
Percentage,
}
/// One threshold step: the color values at or above `value` take. The base step
/// carries `value: null` — Grafana's "everything below the first threshold".
#[derive(Clone, Serialize)]
pub struct ThresholdStep {
pub color: String,
pub value: Option<f64>,
}
/// A threshold ladder, driving gauge/stat coloring.
#[derive(Clone, Serialize)]
pub struct Thresholds {
pub mode: ThresholdMode,
pub steps: Vec<ThresholdStep>,
}
impl Thresholds {
/// Start a ladder with the color used below every threshold.
pub fn base(color: impl Into<String>) -> Self {
Self {
mode: ThresholdMode::Absolute,
steps: vec![ThresholdStep {
color: color.into(),
value: None,
}],
}
}
/// Add a step: values at or above `value` render in `color`.
#[must_use]
pub fn step(mut self, value: f64, color: impl Into<String>) -> Self {
self.steps.push(ThresholdStep {
color: color.into(),
value: Some(value),
});
self
}
}
// Optional timeseries styling vocabularies. Each derives `PartialEq` so the
// public setters can panic when handed the Grafana default (see `styling`).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum LineInterpolation {
Linear,
Smooth,
StepBefore,
StepAfter,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum ShowPoints {
Auto,
Never,
Always,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum GradientMode {
None,
Opacity,
Hue,
Scheme,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum StackingMode {
None,
Normal,
Percent,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum AxisPlacement {
Auto,
Left,
Right,
Hidden,
}
#[derive(Serialize)]
pub struct Stacking {
pub mode: StackingMode,
pub group: String,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Custom {
pub draw_style: DrawStyle,
pub line_width: u32,
pub fill_opacity: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub span_nulls: Option<bool>,
// Optional styling — omitted (left at Grafana's default) unless a setter
// fills it in. See `styling`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub line_interpolation: Option<LineInterpolation>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub show_points: Option<ShowPoints>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gradient_mode: Option<GradientMode>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub stacking: Option<Stacking>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub axis_placement: Option<AxisPlacement>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub axis_label: Option<String>,
}
#[derive(Clone, Serialize)]
pub struct Datasource {
#[serde(rename = "type")]
pub kind: DatasourceKind,
pub uid: String,
}
impl Datasource {
pub fn prometheus() -> Self {
Self {
kind: DatasourceKind::Prometheus,
uid: DATASOURCE_UID.to_owned(),
}
}
}
#[derive(Clone, Serialize)]
#[serde(rename_all = "kebab-case")]
#[serde(tag = "mode")]
pub enum Color {
Fixed {
#[serde(rename = "fixedColor")]
fixed_color: String,
},
PaletteClassic,
}
impl Color {
pub fn fixed(color: impl Into<String>) -> Self {
Self::Fixed {
fixed_color: color.into(),
}
}
#[must_use]
pub const fn palette_classic() -> Self {
Self::PaletteClassic
}
}
#[derive(Serialize)]
pub struct LineStyle {
pub dash: [u32; 2],
pub fill: Fill,
}
#[derive(Serialize)]
#[serde(untagged)]
pub enum PropertyValue {
Color(Color),
LineStyle(LineStyle),
}
#[derive(Serialize)]
pub struct OverrideProperty {
pub id: PropertyId,
pub value: PropertyValue,
}
#[derive(Serialize)]
pub struct Matcher {
pub id: MatcherKind,
pub options: String,
}
#[derive(Serialize)]
pub struct Defaults {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub color: Option<Color>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub custom: Option<Custom>,
pub unit: Unit,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub decimals: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub min: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub thresholds: Option<Thresholds>,
}
#[derive(Serialize)]
pub struct FieldConfig {
pub defaults: Defaults,
pub overrides: Vec<FieldOverride>,
}
#[derive(Serialize)]
pub struct ReduceOptions {
pub calcs: Vec<Calc>,
// Empty string means "all fields"; genuinely free-form, not a vocabulary.
pub fields: String,
pub values: bool,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct StatOptions {
pub color_mode: StatColorMode,
pub graph_mode: GraphMode,
pub reduce_options: ReduceOptions,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Legend {
pub display_mode: LegendDisplay,
pub placement: Placement,
pub calcs: Vec<Calc>,
}
#[derive(Serialize)]
pub struct Tooltip {
pub mode: TooltipMode,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sort: Option<SortOrder>,
}
#[derive(Serialize)]
pub struct TimeSeriesOptions {
pub legend: Legend,
pub tooltip: Tooltip,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct GaugeOptions {
pub reduce_options: ReduceOptions,
pub show_threshold_labels: bool,
pub show_threshold_markers: bool,
}
#[derive(Serialize)]
#[serde(untagged)]
pub enum Options {
Stat(StatOptions),
TimeSeries(TimeSeriesOptions),
Gauge(GaugeOptions),
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum VariableKind {
/// A fixed list of choices, spelled out in the dashboard itself.
Custom,
}
/// One choice in a [`Variable`] dropdown: `text` is displayed, `value` is what
/// `$name` interpolates to in a query.
#[derive(Clone, Serialize)]
pub struct VariableOption {
pub selected: bool,
pub text: String,
pub value: String,
}
/// A dashboard-level dropdown, rendered in the top bar.
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Variable {
pub current: VariableOption,
pub include_all: bool,
pub label: String,
pub multi: bool,
pub name: String,
pub options: Vec<VariableOption>,
/// Grafana's own encoding of `options`, as `text : value` pairs.
pub query: String,
#[serde(rename = "type")]
pub kind: VariableKind,
}
#[derive(Serialize, Default)]
pub struct Templating {
pub list: Vec<Variable>,
}
#[derive(Clone, Copy, Serialize)]
pub struct GridPos {
pub h: u32,
pub w: u32,
pub x: u32,
pub y: u32,
}
/// A fully positioned panel, ready to serialize.
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct PanelModel {
pub datasource: Datasource,
pub field_config: FieldConfig,
pub grid_pos: GridPos,
pub id: u32,
pub options: Options,
pub targets: Vec<Target>,
pub title: String,
#[serde(rename = "type")]
pub panel_type: PanelType,
}
#[expect(
clippy::trailing_empty_array,
reason = "Grafana expects `list: []` for the blocks we don't populate"
)]
#[derive(Serialize, Default)]
pub struct EmptyList {
pub list: [u8; 0],
}
#[derive(Serialize)]
pub struct TimeRange {
// Free-form Grafana time expressions, not a closed vocabulary.
pub from: String,
pub to: String,
}

View File

@ -0,0 +1,124 @@
//! Optional timeseries styling setters.
//!
//! These are real setters — each fills in a field of the panel's `custom` block.
//! Every one documents Grafana's default and **panics if handed that default**:
//! passing the default is always redundant (Grafana emits it anyway), and the
//! generator only serializes fields that differ from the default. So if a call
//! wouldn't change the rendered panel, it's a mistake worth catching loudly at
//! generation time rather than shipping a no-op.
//!
//! These affect timeseries panels only; on a stat panel the `custom` block is
//! not emitted, so the value is silently dropped.
use crate::{
DEFAULT_FILL_OPACITY, Panel,
schema::{AxisPlacement, GradientMode, LineInterpolation, ShowPoints, StackingMode},
};
#[expect(
clippy::multiple_inherent_impl,
reason = "styling setters intentionally live in their own file, so `Panel` has a second inherent impl here"
)]
impl Panel {
/// Area fill under the line, as a percentage. Builder default:
/// [`DEFAULT_FILL_OPACITY`].
///
/// Panics if passed that default, or a value above 100.
#[must_use]
pub fn fill_opacity(mut self, opacity: u32) -> Self {
assert!(
opacity <= 100,
"fill_opacity({opacity}) is not a percentage"
);
assert_ne!(
opacity, DEFAULT_FILL_OPACITY,
"fill_opacity({DEFAULT_FILL_OPACITY}) is redundant: it is the builder's default. Omit the call.",
);
self.fill_opacity = Some(opacity);
self
}
/// Interpolation between points. Grafana default: `Linear`.
///
/// Panics if passed `Linear` — that's the default and would be redundant.
#[must_use]
pub fn line_interpolation(mut self, value: LineInterpolation) -> Self {
assert_ne!(
value,
LineInterpolation::Linear,
"line_interpolation(Linear) is redundant: `linear` is Grafana's default. Omit the call.",
);
self.line_interpolation = Some(value);
self
}
/// Whether/when to draw point markers. Grafana default: `Auto`.
///
/// Panics if passed `Auto` — that's the default and would be redundant.
#[must_use]
pub fn show_points(mut self, value: ShowPoints) -> Self {
assert_ne!(
value,
ShowPoints::Auto,
"show_points(Auto) is redundant: `auto` is Grafana's default. Omit the call.",
);
self.show_points = Some(value);
self
}
/// Area fill gradient. Grafana default: `None`.
///
/// Panics if passed `None` — that's the default and would be redundant.
#[must_use]
pub fn gradient_mode(mut self, value: GradientMode) -> Self {
assert_ne!(
value,
GradientMode::None,
"gradient_mode(None) is redundant: `none` is Grafana's default. Omit the call.",
);
self.gradient_mode = Some(value);
self
}
/// Series stacking. Grafana default: `None`.
///
/// Panics if passed `None` — that's the default and would be redundant.
#[must_use]
pub fn stacking(mut self, mode: StackingMode) -> Self {
assert_ne!(
mode,
StackingMode::None,
"stacking(None) is redundant: `none` is Grafana's default. Omit the call.",
);
self.stacking = Some(mode);
self
}
/// Y-axis placement. Grafana default: `Auto`.
///
/// Panics if passed `Auto` — that's the default and would be redundant.
#[must_use]
pub fn axis_placement(mut self, value: AxisPlacement) -> Self {
assert_ne!(
value,
AxisPlacement::Auto,
"axis_placement(Auto) is redundant: `auto` is Grafana's default. Omit the call.",
);
self.axis_placement = Some(value);
self
}
/// Y-axis label. Grafana default: `""` (no label).
///
/// Panics if passed an empty string — that's the default and would be redundant.
#[must_use]
pub fn axis_label(mut self, label: impl Into<String>) -> Self {
let label = label.into();
assert_ne!(
label, "",
"axis_label(\"\") is redundant: no label is Grafana's default. Omit the call.",
);
self.axis_label = Some(label);
self
}
}

View File

@ -0,0 +1,69 @@
//! Panel value units.
//!
//! Grafana identifies a field's unit by a short id string (`"s"`, `"bytes"`,
//! `"reqps"`, …). We model the handful we actually use as named variants and
//! fall back to [`Unit::custom`] for anything else, so the value always
//! serializes to the exact id Grafana expects.
use serde::{Serialize, Serializer};
/// A panel value unit. Serializes to Grafana's unit id string.
///
/// Only the most common units are named; [`Unit::custom`] carries any other
/// Grafana unit id verbatim.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Unit {
/// Plain number, SI-abbreviated (`short`). Grafana's default.
Short,
/// Percentage on a 0100 scale (`percent`).
Percent,
/// Percentage on a 0.01.0 scale (`percentunit`).
PercentUnit,
/// Seconds (`s`).
Seconds,
/// Milliseconds (`ms`).
Milliseconds,
/// Nanoseconds (`ns`).
Nanoseconds,
/// Bytes, IEC/binary (`bytes`).
Bytes,
/// Bytes per second, SI (`Bps`).
BytesPerSec,
/// Requests per second (`reqps`).
RequestsPerSec,
/// Operations per second (`ops`).
OpsPerSec,
/// Any other Grafana unit id, kept verbatim.
Custom(String),
}
impl Unit {
/// Wrap an arbitrary Grafana unit id (e.g. `"dtdurationms"`, `"celsius"`).
#[must_use]
pub fn custom(id: impl Into<String>) -> Self {
Self::Custom(id.into())
}
/// The Grafana unit id this value serializes to.
fn as_id(&self) -> &str {
match self {
Self::Short => "short",
Self::Percent => "percent",
Self::PercentUnit => "percentunit",
Self::Seconds => "s",
Self::Milliseconds => "ms",
Self::Nanoseconds => "ns",
Self::Bytes => "bytes",
Self::BytesPerSec => "Bps",
Self::RequestsPerSec => "reqps",
Self::OpsPerSec => "ops",
Self::Custom(id) => id,
}
}
}
impl Serialize for Unit {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
serializer.serialize_str(self.as_id())
}
}