From 97071b575a1efdd5ebfa99d2850b35e920461779 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Thu, 23 Jul 2026 20:00:57 +0300 Subject: [PATCH 01/21] feat(sequencer): add initial metrics --- Cargo.lock | 257 ++++++++++++++++++++++++++- Cargo.toml | 3 + Justfile | 4 + lez/common/Cargo.toml | 1 + lez/common/src/transaction.rs | 20 ++- lez/mempool/Cargo.toml | 1 + lez/mempool/src/lib.rs | 33 ++-- lez/sequencer/core/Cargo.toml | 2 + lez/sequencer/core/src/lib.rs | 23 ++- lez/sequencer/core/src/metrics.rs | 90 ++++++++++ lez/sequencer/service/Cargo.toml | 2 + lez/sequencer/service/src/lib.rs | 1 + lez/sequencer/service/src/main.rs | 7 +- lez/sequencer/service/src/metrics.rs | 16 ++ lez/sequencer/service/src/service.rs | 2 + 15 files changed, 442 insertions(+), 20 deletions(-) create mode 100644 lez/sequencer/core/src/metrics.rs create mode 100644 lez/sequencer/service/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index e4eab65f..eec70062 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -794,6 +794,29 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-lc-rs" +version = "1.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "axum" version = "0.7.9" @@ -1589,6 +1612,15 @@ dependencies = [ "lee_core", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cmov" version = "0.5.4" @@ -1655,6 +1687,7 @@ dependencies = [ "serde", "serde_with", "sha2 0.10.9", + "strum 0.28.0", "system_accounts", "thiserror 2.0.18", ] @@ -2554,6 +2587,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -2858,6 +2897,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "evmap" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8" +dependencies = [ + "hashbag", + "left-right", + "smallvec", +] + [[package]] name = "example_program_deployment_methods" version = "0.1.0" @@ -3036,6 +3086,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.5.0" @@ -3072,6 +3128,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "2.0.0" @@ -3231,6 +3293,21 @@ dependencies = [ "num-traits", ] +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -3436,6 +3513,12 @@ dependencies = [ "byteorder", ] +[[package]] +name = "hashbag" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" + [[package]] name = "hashbrown" version = "0.12.3" @@ -3459,7 +3542,16 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", ] [[package]] @@ -3817,6 +3909,7 @@ dependencies = [ "hyper-util", "log", "rustls", + "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", @@ -4890,6 +4983,17 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "left-right" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc015ded5d9b3054dbbdb63332cdd6ee42352ccef19e911e25117490e2f48ee" +dependencies = [ + "crossbeam-utils", + "loom", + "slab", +] + [[package]] name = "leptos" version = "0.8.19" @@ -5896,7 +6000,7 @@ dependencies = [ "multiaddr", "num-bigint 0.4.6", "serde", - "strum", + "strum 0.27.2", "thiserror 2.0.18", "time", "tracing", @@ -6435,6 +6539,19 @@ dependencies = [ "prost-types 0.13.5", ] +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber 0.3.23", +] + [[package]] name = "lru" version = "0.12.5" @@ -6613,6 +6730,7 @@ dependencies = [ name = "mempool" version = "0.1.0" dependencies = [ + "futures", "tokio", ] @@ -6643,6 +6761,56 @@ dependencies = [ "paste", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" +dependencies = [ + "base64 0.22.1", + "evmap", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "indexmap 2.14.0", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "rustls", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.16.1", + "metrics", + "quanta", + "rand 0.9.4", + "rand_xoshiro", + "rapidhash", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -7965,6 +8133,21 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quick-protobuf" version = "0.8.1" @@ -8177,6 +8360,33 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.12.1", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -8978,6 +9188,7 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -9042,6 +9253,7 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -9091,7 +9303,7 @@ dependencies = [ "serde", "serde_with", "sha2 0.10.9", - "strum", + "strum 0.27.2", "tempfile", "thiserror 2.0.18", "toml 0.8.23", @@ -9162,6 +9374,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -9266,6 +9484,7 @@ dependencies = [ "logos-blockchain-key-management-system-service", "logos-blockchain-zone-sdk", "mempool", + "metrics", "num-bigint 0.4.6", "ping_core", "programs", @@ -9274,6 +9493,7 @@ dependencies = [ "serde", "serde_json", "storage", + "strum 0.28.0", "system_accounts", "tempfile", "test_programs", @@ -9301,6 +9521,8 @@ dependencies = [ "lee", "log", "mempool", + "metrics", + "metrics-exporter-prometheus", "programs", "sequencer_core", "sequencer_service_protocol", @@ -9724,6 +9946,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -9923,7 +10151,16 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros", + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", ] [[package]] @@ -9938,6 +10175,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "subtle" version = "2.6.1" diff --git a/Cargo.toml b/Cargo.toml index 397eb965..534b4009 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,6 +142,8 @@ actix-rt = "*" lazy_static = "1.5.0" env_logger = "0.11" log = "0.4.28" +metrics = "0.24.6" +metrics-exporter-prometheus = "0.18.3" lru = "0.16.3" thiserror = "2.0" sha2 = "0.10.8" @@ -173,6 +175,7 @@ url = { version = "2.5.4", features = ["serde"] } tokio-retry = "0.3.0" schemars = "1.2" async-stream = "0.3.6" +strum = { version = "0.28.0", features = ["derive"] } logos-blockchain-common-http-client = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" } logos-blockchain-key-management-system-service = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" } diff --git a/Justfile b/Justfile index d436689d..ec22afb9 100644 --- a/Justfile +++ b/Justfile @@ -103,6 +103,10 @@ run-wallet +args: @echo "🔑 Running wallet" LEE_WALLET_HOME_DIR=$(pwd)/configs/debug cargo run --release -p wallet -- {{args}} +get-sequencer-metrics: + @echo "📊 Querying sequencer metrics" + curl http://localhost:9000/metrics + # Import test accounts supplied in sequencer configuration. wallet-import-test-accounts: @echo "⚙️ Initializing accounts" diff --git a/lez/common/Cargo.toml b/lez/common/Cargo.toml index 7582e885..d03dcbbf 100644 --- a/lez/common/Cargo.toml +++ b/lez/common/Cargo.toml @@ -25,6 +25,7 @@ log.workspace = true hex.workspace = true borsh.workspace = true logos-blockchain-common-http-client.workspace = true +strum.workspace = true [dev-dependencies] lee = { workspace = true, features = ["test-utils"] } diff --git a/lez/common/src/transaction.rs b/lez/common/src/transaction.rs index 13b2ada5..3c4df83c 100644 --- a/lez/common/src/transaction.rs +++ b/lez/common/src/transaction.rs @@ -35,6 +35,15 @@ impl LeeTransaction { }) } + #[must_use] + pub const fn kind(&self) -> TxKind { + match self { + Self::Public(_) => TxKind::Public, + Self::PrivacyPreserving(_) => TxKind::PrivacyPreserving, + Self::ProgramDeployment(_) => TxKind::ProgramDeployment, + } + } + #[must_use] pub fn affected_public_account_ids(&self) -> Vec { match self { @@ -200,7 +209,16 @@ impl From for LeeTransaction { } #[derive( - Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize, + Debug, + Clone, + Copy, + PartialEq, + Eq, + Serialize, + Deserialize, + BorshSerialize, + BorshDeserialize, + strum::IntoStaticStr, )] pub enum TxKind { Public, diff --git a/lez/mempool/Cargo.toml b/lez/mempool/Cargo.toml index a2f51bc0..47550fc9 100644 --- a/lez/mempool/Cargo.toml +++ b/lez/mempool/Cargo.toml @@ -12,3 +12,4 @@ tokio = { workspace = true, features = ["sync"] } [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +futures.workspace = true diff --git a/lez/mempool/src/lib.rs b/lez/mempool/src/lib.rs index 0006f2c3..c081e9e7 100644 --- a/lez/mempool/src/lib.rs +++ b/lez/mempool/src/lib.rs @@ -18,6 +18,19 @@ impl MemPool { (mem_pool, sender) } + /// Returns the total number of items in the mempool, including both the front buffer and the + /// channel. + #[must_use] + pub fn len(&self) -> usize { + self.front_buffer.len().saturating_add(self.receiver.len()) + } + + /// Returns true if the mempool is empty, false otherwise. + #[must_use] + pub fn is_empty(&self) -> bool { + self.front_buffer.is_empty() && self.receiver.is_empty() + } + /// Pop an item from the mempool first checking the front buffer (LIFO) then the channel (FIFO). pub fn pop(&mut self) -> Option { use tokio::sync::mpsc::error::TryRecvError; @@ -74,6 +87,7 @@ impl MemPoolHandle { #[cfg(test)] mod tests { + use futures::FutureExt as _; use tokio::test; use super::*; @@ -82,6 +96,7 @@ mod tests { async fn mempool_new() { let (mut pool, _handle): (MemPool, _) = MemPool::new(10); assert_eq!(pool.pop(), None); + assert_eq!(pool.len(), 0); } #[test] @@ -89,10 +104,12 @@ mod tests { let (mut pool, handle) = MemPool::new(10); handle.push(1).await.unwrap(); + assert_eq!(pool.len(), 1); let item = pool.pop(); assert_eq!(item, Some(1)); assert_eq!(pool.pop(), None); + assert_eq!(pool.len(), 0); } #[test] @@ -103,29 +120,23 @@ mod tests { handle.push(2).await.unwrap(); handle.push(3).await.unwrap(); + assert_eq!(pool.len(), 3); assert_eq!(pool.pop(), Some(1)); assert_eq!(pool.pop(), Some(2)); assert_eq!(pool.pop(), Some(3)); assert_eq!(pool.pop(), None); - } - - #[test] - async fn pop_empty() { - let (mut pool, _handle): (MemPool, _) = MemPool::new(10); - assert_eq!(pool.pop(), None); + assert_eq!(pool.len(), 0); } #[test] async fn max_size() { - let (mut pool, handle) = MemPool::new(2); + let (_pool, handle) = MemPool::new(2); handle.push(1).await.unwrap(); handle.push(2).await.unwrap(); - // This should block if buffer is full, but we'll use try_send in a real scenario - // For now, just verify we can pop items - assert_eq!(pool.pop(), Some(1)); - assert_eq!(pool.pop(), Some(2)); + // This should block if buffer is full + assert_eq!(handle.push(3).now_or_never(), None); } #[test] diff --git a/lez/sequencer/core/Cargo.toml b/lez/sequencer/core/Cargo.toml index 64b154f4..41b70fca 100644 --- a/lez/sequencer/core/Cargo.toml +++ b/lez/sequencer/core/Cargo.toml @@ -34,6 +34,7 @@ humantime-serde.workspace = true tempfile.workspace = true chrono.workspace = true log.workspace = true +metrics.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } tokio-util.workspace = true rand.workspace = true @@ -45,6 +46,7 @@ num-bigint.workspace = true risc0-zkvm.workspace = true futures.workspace = true itertools.workspace = true +strum.workspace = true [features] default = [] diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 34ca24a5..1dd55c5b 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -50,6 +50,7 @@ pub mod block_publisher; pub mod block_store; pub mod config; pub mod cross_zone_watcher; +mod metrics; #[cfg(feature = "mock")] pub mod mock; @@ -72,7 +73,7 @@ const RETIRE_DISPATCH_AFTER_FAILURES: u32 = 3; const MAX_DISPATCHES_PER_BLOCK: usize = 16; /// The origin of a transaction. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, strum::IntoStaticStr)] pub enum TransactionOrigin { /// Basic transactions submitted by users via RPC. User, @@ -299,6 +300,8 @@ impl SequencerCore { watchers, }; + crate::metrics::set_block_count(sequencer_core.chain_height()); + (sequencer_core, mempool_handle) } @@ -802,6 +805,7 @@ impl SequencerCore { let clock_tx = clock_invocation(new_block_timestamp); let clock_lee_tx = LeeTransaction::Public(clock_tx.clone()); + crate::metrics::record_mempool_size(self.mempool.len()); // Everything drained from the store first, then user work. `from_store` // is not the same as a `Sequencer` origin: it says the transaction has a // record behind it and so needs no requeue, where the origin only says @@ -872,16 +876,25 @@ impl SequencerCore { break; } - if Self::apply_mempool_transaction( + let before_tx_apply = Instant::now(); + let applied = Self::apply_mempool_transaction( &mut working_state, origin, &tx, new_block_height, new_block_timestamp, &mut withdrawals, - ) { + ); + crate::metrics::record_mempool_transaction_application_time( + origin, + tx.kind(), + before_tx_apply.elapsed(), + ); + + if applied { valid_transactions.push(tx); } else { + crate::metrics::increment_failed_transaction_count(); // A failed transaction is simply left out of the block, except a // dispatch: that one is re-fed from the store every turn, so one // that can never execute would fail on every block for ever. @@ -897,6 +910,7 @@ impl SequencerCore { .transition_from_public_transaction(&clock_tx, new_block_height, new_block_timestamp) .context("Clock transaction failed. Aborting block production.")?; valid_transactions.push(clock_lee_tx); + crate::metrics::set_block_transaction_count(valid_transactions.len()); let hashable_data = HashableBlockData { block_id: new_block_height, @@ -915,6 +929,9 @@ impl SequencerCore { now.elapsed().as_secs() ); + crate::metrics::record_block_creation_time(now.elapsed()); + crate::metrics::increment_block_count(); + Ok(BlockWithMeta { block, withdrawals }) } diff --git a/lez/sequencer/core/src/metrics.rs b/lez/sequencer/core/src/metrics.rs new file mode 100644 index 00000000..2486b3aa --- /dev/null +++ b/lez/sequencer/core/src/metrics.rs @@ -0,0 +1,90 @@ +//! This module provides all metrics exposed by the sequencer core crate. + +#![expect( + clippy::cast_precision_loss, + clippy::as_conversions, + reason = "It's okay for metrics" +)] + +use std::time::Duration; + +use common::transaction::TxKind; +use metrics::{Counter, Unit, counter, gauge, histogram}; + +use crate::TransactionOrigin; + +mod names { + pub const BLOCK_CREATION_TIME: &str = "block_creation_time"; + pub const BLOCK_COUNT: &str = "block_count"; + pub const MEMPOOL_SIZE: &str = "mempool_size"; + pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = "mempool_transaction_application_time"; + pub const BLOCK_TRANSACTION_COUNT: &str = "block_transaction_count"; + pub const FAILED_TRANSACTION_COUNT: &str = "failed_transaction_count"; +} + +pub fn record_block_creation_time(duration: Duration) { + histogram!( + description: "Time taken to create a block", + unit: Unit::Seconds, + names::BLOCK_CREATION_TIME + ) + .record(duration.as_secs_f64()); +} + +fn block_count_counter() -> Counter { + counter!( + description: "Number of blocks in chain", + unit: Unit::Count, + names::BLOCK_COUNT + ) +} + +pub fn set_block_count(value: u64) { + block_count_counter().absolute(value); +} + +pub fn increment_block_count() { + block_count_counter().increment(1); +} + +pub fn record_mempool_size(size: usize) { + gauge!( + description: "Size of the mempool", + unit: Unit::Count, + names::MEMPOOL_SIZE + ) + .set(u64::try_from(size).expect("Mempool size should fit into u64") as f64); +} + +pub fn record_mempool_transaction_application_time( + origin: TransactionOrigin, + kind: TxKind, + duration: Duration, +) { + histogram!( + description: "Time taken to apply a mempool transaction", + unit: Unit::Seconds, + names::MEMPOOL_TRANSACTION_APPLICATION_TIME, + "origin" => <&'static str>::from(origin), + "kind" => <&'static str>::from(kind), + ) + .record(duration.as_secs_f64()); +} + +pub fn set_block_transaction_count(count: usize) { + histogram!( + description: "Number of transactions included in block", + unit: Unit::Count, + names::BLOCK_TRANSACTION_COUNT + ) + .record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64); +} + +pub fn increment_failed_transaction_count() { + counter!( + description: "Number of transactions that failed to be included in blocks", + unit: Unit::Count, + names::FAILED_TRANSACTION_COUNT + ) + .increment(1); +} diff --git a/lez/sequencer/service/Cargo.toml b/lez/sequencer/service/Cargo.toml index 338aac0d..03400935 100644 --- a/lez/sequencer/service/Cargo.toml +++ b/lez/sequencer/service/Cargo.toml @@ -22,6 +22,8 @@ anyhow.workspace = true env_logger.workspace = true hex.workspace = true log.workspace = true +metrics.workspace = true +metrics-exporter-prometheus.workspace = true tokio.workspace = true tokio-util.workspace = true jsonrpsee.workspace = true diff --git a/lez/sequencer/service/src/lib.rs b/lez/sequencer/service/src/lib.rs index ba5b68ec..5624cc91 100644 --- a/lez/sequencer/service/src/lib.rs +++ b/lez/sequencer/service/src/lib.rs @@ -21,6 +21,7 @@ use sequencer_service_rpc::RpcServer as _; use tokio::{sync::Mutex, task::JoinHandle}; use tokio_util::sync::CancellationToken; +mod metrics; pub mod service; const REQUEST_BODY_MAX_SIZE: ByteSize = ByteSize::mib(10); diff --git a/lez/sequencer/service/src/main.rs b/lez/sequencer/service/src/main.rs index 95d02e16..95facfff 100644 --- a/lez/sequencer/service/src/main.rs +++ b/lez/sequencer/service/src/main.rs @@ -3,7 +3,7 @@ use std::{ path::PathBuf, }; -use anyhow::Result; +use anyhow::{Context as _, Result}; use clap::Parser; use log::{error, info}; use tokio::signal::unix::{SignalKind, signal}; @@ -34,6 +34,11 @@ struct Args { async fn main() -> Result<()> { env_logger::init(); + metrics_exporter_prometheus::PrometheusBuilder::new() + .with_recommended_naming(true) + .install() + .context("Failed to install Prometheus recorder")?; + let Args { config_path, port, diff --git a/lez/sequencer/service/src/metrics.rs b/lez/sequencer/service/src/metrics.rs new file mode 100644 index 00000000..27cd47d1 --- /dev/null +++ b/lez/sequencer/service/src/metrics.rs @@ -0,0 +1,16 @@ +//! This module provides all metrics exposed by the sequencer service crate. + +use metrics::{Unit, counter}; + +mod names { + pub const SUBMITTED_TRANSACTION_COUNT: &str = "submitted_transaction_count"; +} + +pub fn increment_submitted_transaction_count() { + counter!( + description: "Number of transactions submitted", + unit: Unit::Count, + names::SUBMITTED_TRANSACTION_COUNT + ) + .increment(1); +} diff --git a/lez/sequencer/service/src/service.rs b/lez/sequencer/service/src/service.rs index 7ab9ed3c..d9eaf97f 100644 --- a/lez/sequencer/service/src/service.rs +++ b/lez/sequencer/service/src/service.rs @@ -93,6 +93,8 @@ impl sequencer_service_rpc::Rpc .await .expect("Mempool is closed, this is a bug"); + crate::metrics::increment_submitted_transaction_count(); + Ok(tx_hash) } From 8afd2cea66da0e81a42d43168b4ec1fea11ebf3e Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Fri, 24 Jul 2026 01:37:21 +0300 Subject: [PATCH 02/21] feat(docker): introduce monitoring --- Justfile | 13 +- docker-compose.yml | 2 + lez/sequencer/core/src/lib.rs | 2 +- lez/sequencer/core/src/metrics.rs | 6 +- monitoring/docker-compose.yml | 36 +++ monitoring/grafana/dashboards/sequencer.json | 296 ++++++++++++++++++ .../provisioning/dashboards/dashboards.yml | 9 + .../provisioning/datasources/prometheus.yml | 10 + monitoring/prometheus/prometheus.yml | 15 + 9 files changed, 383 insertions(+), 6 deletions(-) create mode 100644 monitoring/docker-compose.yml create mode 100644 monitoring/grafana/dashboards/sequencer.json create mode 100644 monitoring/grafana/provisioning/dashboards/dashboards.yml create mode 100644 monitoring/grafana/provisioning/datasources/prometheus.yml create mode 100644 monitoring/prometheus/prometheus.yml diff --git a/Justfile b/Justfile index ec22afb9..ee122105 100644 --- a/Justfile +++ b/Justfile @@ -65,6 +65,13 @@ run-bedrock: @echo "⛓️ Running bedrock" docker compose up +# Run Prometheus + Grafana in docker. Grafana: http://localhost:3000 (anonymous +# admin), Prometheus: http://localhost:9090. Scrapes the sequencer's /metrics. +[working-directory: 'monitoring'] +run-monitoring: + @echo "📊 Running Prometheus + Grafana" + docker compose up + # Run Sequencer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. # Optional home/port let a second instance run off the same config, e.g. # `just run-sequencer "" "$TMPDIR/lez-sequencer2" 3041` for the multi-sequencer demo. @@ -103,8 +110,9 @@ run-wallet +args: @echo "🔑 Running wallet" LEE_WALLET_HOME_DIR=$(pwd)/configs/debug cargo run --release -p wallet -- {{args}} +# Query sequencer metrics in raw format. Useful for quick debugging. For a more detailed view, use `just run-monitoring`. get-sequencer-metrics: - @echo "📊 Querying sequencer metrics" + @echo "📊 Querying sequencer's metrics" curl http://localhost:9000/metrics # Import test accounts supplied in sequencer configuration. @@ -147,4 +155,5 @@ clean: rm -rf lez/wallet/configs/debug/storage.json rm -rf lez/wallet/configs/debug/statistics.json rm -rf rocksdb* - cd bedrock && docker compose down -v + cd bedrock && docker compose down -v && cd .. + cd monitoring && docker compose down -v && cd .. diff --git a/docker-compose.yml b/docker-compose.yml index 3644b2aa..4da0d04c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,3 +11,5 @@ include: lez/indexer/service/docker-compose.yml - path: lez/explorer_service/docker-compose.yml + - path: + monitoring/docker-compose.yml diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 1dd55c5b..a56b3c88 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -910,7 +910,7 @@ impl SequencerCore { .transition_from_public_transaction(&clock_tx, new_block_height, new_block_timestamp) .context("Clock transaction failed. Aborting block production.")?; valid_transactions.push(clock_lee_tx); - crate::metrics::set_block_transaction_count(valid_transactions.len()); + crate::metrics::record_transactions_per_block(valid_transactions.len()); let hashable_data = HashableBlockData { block_id: new_block_height, diff --git a/lez/sequencer/core/src/metrics.rs b/lez/sequencer/core/src/metrics.rs index 2486b3aa..a6c79454 100644 --- a/lez/sequencer/core/src/metrics.rs +++ b/lez/sequencer/core/src/metrics.rs @@ -18,7 +18,7 @@ mod names { pub const BLOCK_COUNT: &str = "block_count"; pub const MEMPOOL_SIZE: &str = "mempool_size"; pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = "mempool_transaction_application_time"; - pub const BLOCK_TRANSACTION_COUNT: &str = "block_transaction_count"; + pub const TRANSACTIONS_PER_BLOCK: &str = "transactions_per_block"; pub const FAILED_TRANSACTION_COUNT: &str = "failed_transaction_count"; } @@ -71,11 +71,11 @@ pub fn record_mempool_transaction_application_time( .record(duration.as_secs_f64()); } -pub fn set_block_transaction_count(count: usize) { +pub fn record_transactions_per_block(count: usize) { histogram!( description: "Number of transactions included in block", unit: Unit::Count, - names::BLOCK_TRANSACTION_COUNT + names::TRANSACTIONS_PER_BLOCK ) .record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64); } diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml new file mode 100644 index 00000000..71953636 --- /dev/null +++ b/monitoring/docker-compose.yml @@ -0,0 +1,36 @@ +# Prometheus + Grafana monitoring stack. +services: + prometheus: + image: prom/prometheus:v3.1.0 + container_name: prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + # Lets Prometheus reach services running natively on the host. + extra_hosts: + - "host.docker.internal:host-gateway" + + grafana: + image: grafana/grafana:11.4.0 + container_name: grafana + ports: + - "3000:3000" + environment: + # Dev-only: open Grafana with no login, full access. + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + depends_on: + - prometheus + +volumes: + prometheus_data: + grafana_data: diff --git a/monitoring/grafana/dashboards/sequencer.json b/monitoring/grafana/dashboards/sequencer.json new file mode 100644 index 00000000..39824dd2 --- /dev/null +++ b/monitoring/grafana/dashboards/sequencer.json @@ -0,0 +1,296 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "graphTooltip": 1, + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "blue" }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "block_count_total", + "legendFormat": "height", + "refId": "A" + } + ], + "title": "Chain height", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 18, "x": 6, "y": 0 }, + "id": 2, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "single" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(block_count_total[1m]) * 60", + "legendFormat": "blocks/min", + "refId": "A" + } + ], + "title": "Block production rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "avg" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "dash": [8, 4], "fill": "dash" } }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } } + ] + } + ] + }, + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 7 }, + "id": 3, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "block_creation_time_seconds{quantile=\"0.5\"}", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "block_creation_time_seconds{quantile=\"0.9\"}", + "legendFormat": "p90", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "block_creation_time_seconds{quantile=\"0.95\"}", + "legendFormat": "p95", + "refId": "C" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "block_creation_time_seconds{quantile=\"0.99\"}", + "legendFormat": "p99", + "refId": "D" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(block_creation_time_seconds_sum[1m]) / rate(block_creation_time_seconds_count[1m])", + "legendFormat": "avg", + "refId": "E" + } + ], + "title": "Block creation time", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 16 }, + "id": 4, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_transaction_application_time_seconds{quantile=\"0.5\"}", + "legendFormat": "p50 · {{kind}} · {{origin}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_transaction_application_time_seconds{quantile=\"0.9\"}", + "legendFormat": "p90 · {{kind}} · {{origin}}", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_transaction_application_time_seconds{quantile=\"0.95\"}", + "legendFormat": "p95 · {{kind}} · {{origin}}", + "refId": "C" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_transaction_application_time_seconds{quantile=\"0.99\"}", + "legendFormat": "p99 · {{kind}} · {{origin}}", + "refId": "D" + } + ], + "title": "Transaction application time", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "spanNulls": true }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 }, + "id": 5, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "single" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_size", + "legendFormat": "mempool size", + "refId": "A" + } + ], + "title": "Mempool size", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "avg" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "dash": [8, 4], "fill": "dash" } }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } } + ] + } + ] + }, + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 25 }, + "id": 6, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "transactions_per_block{quantile=\"0.5\"}", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "transactions_per_block{quantile=\"0.9\"}", + "legendFormat": "p90", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "transactions_per_block{quantile=\"0.95\"}", + "legendFormat": "p95", + "refId": "C" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "transactions_per_block{quantile=\"0.99\"}", + "legendFormat": "p99", + "refId": "D" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(transactions_per_block_sum[1m]) / rate(transactions_per_block_count[1m])", + "legendFormat": "avg", + "refId": "E" + } + ], + "title": "Transactions per block", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "failed" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] + }, + { + "matcher": { "id": "byName", "options": "submitted" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] + } + ] + }, + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 25 }, + "id": 7, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(submitted_transaction_count_total[1m]) * 60", + "legendFormat": "submitted", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(failed_transaction_count_total[1m]) * 60", + "legendFormat": "failed", + "refId": "B" + } + ], + "title": "Transaction throughput (per minute)", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": ["sequencer"], + "templating": { "list": [] }, + "time": { "from": "now-15m", "to": "now" }, + "timezone": "", + "title": "Sequencer", + "uid": "sequencer" +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 00000000..e26d8f8c --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: sequencer + type: file + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..0b304bc9 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 00000000..aaae81b0 --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: sequencer + metrics_path: /metrics + static_configs: + # `sequencer_service:9000` resolves when monitoring runs inside the + # all-in-one compose network. `host.docker.internal:9000` resolves when + # the sequencer runs natively on the host and only monitoring runs in Docker. + # Whichever is unreachable stays `down`. + - targets: + - sequencer_service:9000 + - host.docker.internal:9000 From 008eb247e9191caad402722c1212e33feb676dcb Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Fri, 24 Jul 2026 18:22:49 +0300 Subject: [PATCH 03/21] feat(tools): add dashboard generator --- .github/workflows/ci.yml | 31 ++ Cargo.lock | 19 + Cargo.toml | 1 + Justfile | 7 + monitoring/grafana/dashboards/sequencer.json | 67 ++-- tools/dashboard_gen/Cargo.toml | 13 + tools/dashboard_gen/src/lib.rs | 397 +++++++++++++++++++ tools/dashboard_gen/src/main.rs | 111 ++++++ tools/dashboard_gen/src/schema.rs | 260 ++++++++++++ 9 files changed, 863 insertions(+), 43 deletions(-) create mode 100644 tools/dashboard_gen/Cargo.toml create mode 100644 tools/dashboard_gen/src/lib.rs create mode 100644 tools/dashboard_gen/src/main.rs create mode 100644 tools/dashboard_gen/src/schema.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88be5dab..1a075d77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -353,3 +353,34 @@ jobs: exit 1 fi echo "✅ Artifacts are up to date" + + # The dashboard generator is plain Rust (no Docker), so this runs directly in + # the CI image container rather than through `run-in-ci-image`. + dashboards: + needs: ci-image + runs-on: ubuntu-latest + container: + image: ${{ needs.ci-image.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + timeout-minutes: 30 + + name: dashboards + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.head.sha || github.head_ref }} + + - name: Regenerate dashboards + run: just regenerate-dashboards + + - name: Check if dashboards match repository + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + if ! git diff --exit-code monitoring/grafana/dashboards/; then + echo "❌ Dashboards in the repository are out of date!" + echo "Please run 'just regenerate-dashboards' and commit the changes." + exit 1 + fi + echo "✅ Dashboards are up to date" diff --git a/Cargo.lock b/Cargo.lock index eec70062..35eb6480 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2292,6 +2292,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dashboard_gen" +version = "0.1.0" +dependencies = [ + "json-pretty-compact", + "serde", + "serde_json", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -4596,6 +4605,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-pretty-compact" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62a9c1f06b173b0da0ccc8cae00599d7b3ceda6e76d68be9b6bc2c941adafe0e" +dependencies = [ + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "jsonrpsee" version = "0.26.0" diff --git a/Cargo.toml b/Cargo.toml index 534b4009..467dd9dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ members = [ "tools/crypto_primitives_bench", "tools/integration_bench", "tools/cross_zone_chat", + "tools/dashboard_gen", ] [workspace.dependencies] diff --git a/Justfile b/Justfile index ee122105..51899221 100644 --- a/Justfile +++ b/Justfile @@ -53,6 +53,13 @@ regenerate-test-fixture: @echo "🧪 Regenerating test fixture" RISC0_DEV_MODE=1 cargo run -p test_fixtures --bin regenerate_test_fixture +# Regenerate the committed Grafana dashboards from the Rust generator +# (tools/dashboard_gen) and commit the result. CI checks these are up to date. +regenerate-dashboards: + @echo "📊 Regenerating Grafana dashboards" + @cargo build -q -p dashboard_gen + @cargo run -q -p dashboard_gen > monitoring/grafana/dashboards/sequencer.json + # Run criterion benches: fast crypto primitives, then the slow PPE verify (real proving setup). bench: @echo "📊 Running criterion benches" diff --git a/monitoring/grafana/dashboards/sequencer.json b/monitoring/grafana/dashboards/sequencer.json index 39824dd2..3cd0fbcc 100644 --- a/monitoring/grafana/dashboards/sequencer.json +++ b/monitoring/grafana/dashboards/sequencer.json @@ -1,24 +1,20 @@ { - "annotations": { "list": [] }, + "annotations": { "list": [ ] }, "editable": true, "graphTooltip": 1, "panels": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { - "defaults": { - "color": { "mode": "fixed", "fixedColor": "blue" }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] + "defaults": { "color": { "mode": "fixed", "fixedColor": "blue" }, "unit": "short", "decimals": 0 }, + "overrides": [ ] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, "id": 1, "options": { "colorMode": "value", "graphMode": "area", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { @@ -34,16 +30,13 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { - "defaults": { - "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, - "unit": "short" - }, - "overrides": [] + "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" }, + "overrides": [ ] }, "gridPos": { "h": 7, "w": 18, "x": 6, "y": 0 }, "id": 2, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["last", "max"] }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "single" } }, "targets": [ @@ -60,15 +53,12 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { - "defaults": { - "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, - "unit": "s" - }, + "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "s" }, "overrides": [ { "matcher": { "id": "byName", "options": "avg" }, "properties": [ - { "id": "custom.lineStyle", "value": { "dash": [8, 4], "fill": "dash" } }, + { "id": "custom.lineStyle", "value": { "dash": [ 8, 4 ], "fill": "dash" } }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } } ] } @@ -77,7 +67,7 @@ "gridPos": { "h": 9, "w": 24, "x": 0, "y": 7 }, "id": 3, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ @@ -118,16 +108,13 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { - "defaults": { - "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, - "unit": "s" - }, - "overrides": [] + "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "s" }, + "overrides": [ ] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 16 }, "id": 4, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ @@ -166,12 +153,12 @@ "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "spanNulls": true }, "unit": "short" }, - "overrides": [] + "overrides": [ ] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 }, "id": 5, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["last", "max"] }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "single" } }, "targets": [ @@ -188,15 +175,12 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { - "defaults": { - "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, - "unit": "short" - }, + "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "avg" }, "properties": [ - { "id": "custom.lineStyle", "value": { "dash": [8, 4], "fill": "dash" } }, + { "id": "custom.lineStyle", "value": { "dash": [ 8, 4 ], "fill": "dash" } }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } } ] } @@ -205,7 +189,7 @@ "gridPos": { "h": 9, "w": 12, "x": 0, "y": 25 }, "id": 6, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ @@ -246,25 +230,22 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { - "defaults": { - "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, - "unit": "short" - }, + "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "failed" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }, { "matcher": { "id": "byName", "options": "submitted" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] } ] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 25 }, "id": 7, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ @@ -287,8 +268,8 @@ ], "refresh": "5s", "schemaVersion": 39, - "tags": ["sequencer"], - "templating": { "list": [] }, + "tags": [ "sequencer" ], + "templating": { "list": [ ] }, "time": { "from": "now-15m", "to": "now" }, "timezone": "", "title": "Sequencer", diff --git a/tools/dashboard_gen/Cargo.toml b/tools/dashboard_gen/Cargo.toml new file mode 100644 index 00000000..d05ac951 --- /dev/null +++ b/tools/dashboard_gen/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "dashboard_gen" +version = "0.1.0" +edition = "2024" +license = { workspace = true } + +# NOTE: experiment — intentionally NOT inheriting `[lints] workspace = true`. +# If we keep this crate, turn workspace lints on and fix the fallout. + +[dependencies] +serde = { workspace = true, features = ["derive", "alloc"] } +serde_json.workspace = true +json-pretty-compact = "0.1.2" diff --git a/tools/dashboard_gen/src/lib.rs b/tools/dashboard_gen/src/lib.rs new file mode 100644 index 00000000..fad718e8 --- /dev/null +++ b/tools/dashboard_gen/src/lib.rs @@ -0,0 +1,397 @@ +//! A tiny, hand-rolled Grafana dashboard builder. +//! +//! This is a deliberately small subset of what the (Rust-less) Grafana +//! Foundation SDK does: model only the panel types and options we actually use, +//! expose a fluent builder, and render to the same dashboard JSON Grafana +//! provisions. The payoff over hand-written JSON: +//! +//! * metric names live in Rust `const`s, so a rename is a compile error here; +//! * repetitive structure (percentile targets, grid layout, legend/tooltip defaults) collapses into +//! a single call instead of copy-pasted JSON. +//! +//! Build a [`Dashboard`] and serialize it directly. + +use schema::{ + Calc, Color, Custom, Datasource, Defaults, DrawStyle, EmptyList, FieldConfig, Fill, GraphMode, + GridPos, Legend, LegendDisplay, LineStyle, Matcher, MatcherKind, Options, OverrideProperty, + PanelModel, PanelType, Placement, PropertyId, PropertyValue, ReduceOptions, SortOrder, + StatColorMode, StatOptions, TimeRange, TimeSeriesOptions, Tooltip, TooltipMode, +}; +use serde::Serialize; + +mod schema; + +/// Datasource uid every panel/target points at. Dashboards stay portable across +/// environments because they reference the datasource by this stable uid rather +/// than by a per-environment URL. +pub const DATASOURCE_UID: &str = "prometheus"; + +fn default_unit() -> String { + "short".to_owned() +} + +fn ref_letter(index: usize) -> String { + ((b'A' + index as u8) as char).to_string() +} + +/// A single Prometheus query within a panel. +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Target { + datasource: Datasource, + expr: String, + #[serde(rename = "legendFormat")] + legend: String, + ref_id: String, +} + +impl Target { + pub fn new(expr: impl Into) -> Self { + Self { + datasource: Datasource::prometheus(), + expr: expr.into(), + legend: String::new(), + ref_id: ref_letter(0), + } + } + + pub fn legend(mut self, legend: impl Into) -> Self { + self.legend = legend.into(); + self + } +} + +/// A per-series style override, matched by series name. +#[derive(Serialize)] +pub struct FieldOverride { + matcher: Matcher, + properties: Vec, +} + +impl FieldOverride { + pub fn by_name(name: impl Into) -> Self { + Self { + matcher: Matcher { + id: MatcherKind::ByName, + options: name.into(), + }, + properties: Vec::new(), + } + } + + pub fn fixed_color(mut self, color: impl Into) -> Self { + self.properties.push(OverrideProperty { + id: PropertyId::Color, + value: PropertyValue::Color(Color::fixed(color.into())), + }); + self + } + + pub fn dashed_line(mut self) -> Self { + self.properties.push(OverrideProperty { + id: PropertyId::LineStyle, + value: PropertyValue::LineStyle(LineStyle { + dash: [8, 4], + fill: Fill::Dash, + }), + }); + self + } +} + +#[derive(Clone, Copy)] +enum Kind { + Stat, + TimeSeries, +} + +/// A dashboard panel builder. Grid position and panel id are assigned by +/// [`Dashboard::row`]; everything else is set here. +pub struct Panel { + title: String, + kind: Kind, + targets: Vec, + width: u32, + unit: Option, + decimals: Option, + fixed_color: Option, + span_nulls: bool, + overrides: Vec, +} + +impl Panel { + fn new(title: impl Into, kind: Kind) -> Self { + Self { + title: title.into(), + kind, + targets: Vec::new(), + width: 0, + unit: None, + decimals: None, + fixed_color: None, + span_nulls: false, + overrides: Vec::new(), + } + } + + /// A single big-number panel. + pub fn stat(title: impl Into) -> Self { + Self::new(title, Kind::Stat) + } + + /// A time-series line panel. + pub fn timeseries(title: impl Into) -> Self { + Self::new(title, Kind::TimeSeries) + } + + /// Grid width in Grafana's 24-column units. Unset panels split the row's + /// remaining width evenly. + pub fn width(mut self, width: u32) -> Self { + self.width = width; + self + } + + pub fn unit(mut self, unit: impl Into) -> Self { + self.unit = Some(unit.into()); + self + } + + pub fn decimals(mut self, decimals: u32) -> Self { + self.decimals = Some(decimals); + self + } + + pub fn fixed_color(mut self, color: impl Into) -> Self { + self.fixed_color = Some(color.into()); + self + } + + pub fn span_nulls(mut self) -> Self { + self.span_nulls = true; + self + } + + pub fn target(mut self, target: Target) -> Self { + self.targets.push(target); + self + } + + pub fn targets(mut self, targets: impl IntoIterator) -> Self { + self.targets.extend(targets); + self + } + + pub fn with_override(mut self, over: FieldOverride) -> Self { + self.overrides.push(over); + self + } + + fn finalize(self, id: u32, grid_pos: GridPos) -> PanelModel { + let targets: Vec = self + .targets + .into_iter() + .enumerate() + .map(|(i, mut t)| { + t.ref_id = ref_letter(i); + t + }) + .collect(); + + let unit = self.unit.unwrap_or_else(default_unit); + let (defaults, options, panel_type) = match self.kind { + Kind::Stat => { + let defaults = Defaults { + color: self.fixed_color.map(Color::fixed), + custom: None, + unit, + decimals: self.decimals, + }; + let options = Options::Stat(StatOptions { + color_mode: StatColorMode::Value, + graph_mode: GraphMode::Area, + reduce_options: ReduceOptions { + calcs: vec![Calc::LastNotNull], + fields: "", + values: false, + }, + }); + (defaults, options, PanelType::Stat) + } + Kind::TimeSeries => { + let defaults = Defaults { + color: None, + custom: Some(Custom { + draw_style: DrawStyle::Line, + line_width: 1, + fill_opacity: 10, + span_nulls: self.span_nulls.then_some(true), + }), + unit, + decimals: None, + }; + // Panels with several series read better as a sortable table with + // a multi-series tooltip; single-series panels stay compact. + let multi = targets.len() > 1; + let options = Options::TimeSeries(TimeSeriesOptions { + legend: Legend { + display_mode: if multi { + LegendDisplay::Table + } else { + LegendDisplay::List + }, + placement: Placement::Bottom, + calcs: vec![Calc::Last, Calc::Max], + }, + tooltip: if multi { + Tooltip { + mode: TooltipMode::Multi, + sort: Some(SortOrder::Desc), + } + } else { + Tooltip { + mode: TooltipMode::Single, + sort: None, + } + }, + }); + (defaults, options, PanelType::Timeseries) + } + }; + + PanelModel { + datasource: Datasource::prometheus(), + field_config: FieldConfig { + defaults, + overrides: self.overrides, + }, + grid_pos, + id, + options, + targets, + title: self.title, + panel_type, + } + } +} + +/// A dashboard, built row by row. Serialize it directly to get the JSON. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Dashboard { + annotations: EmptyList, + editable: bool, + graph_tooltip: u32, + panels: Vec, + refresh: String, + schema_version: u32, + tags: Vec, + templating: EmptyList, + time: TimeRange, + timezone: String, + title: String, + uid: String, + + // Layout cursor — not part of the dashboard schema. + #[serde(skip)] + next_id: u32, + #[serde(skip)] + cursor_y: u32, +} + +impl Dashboard { + pub fn new(title: impl Into, uid: impl Into) -> Self { + Self { + annotations: EmptyList::default(), + editable: true, + graph_tooltip: 1, + panels: Vec::new(), + refresh: "5s".to_owned(), + schema_version: 39, + tags: Vec::new(), + templating: EmptyList::default(), + time: TimeRange { + from: "now-15m", + to: "now", + }, + timezone: String::new(), + title: title.into(), + uid: uid.into(), + next_id: 1, + cursor_y: 0, + } + } + + pub fn tag(mut self, tag: impl Into) -> Self { + self.tags.push(tag.into()); + self + } + + pub fn refresh(mut self, refresh: impl Into) -> Self { + self.refresh = refresh.into(); + self + } + + /// Place a horizontal row of panels at the current vertical cursor. Panel + /// ids, x offsets and y are assigned here; unset widths split the remaining + /// 24 columns evenly. + pub fn row(mut self, height: u32, panels: impl IntoIterator) -> Self { + let panels: Vec = panels.into_iter().collect(); + let specified: u32 = panels.iter().map(|p| p.width).sum(); + let auto_count = panels.iter().filter(|p| p.width == 0).count() as u32; + let auto_width = if auto_count > 0 { + 24u32.saturating_sub(specified) / auto_count + } else { + 0 + }; + + let mut x = 0; + for panel in panels { + let w = if panel.width == 0 { + auto_width + } else { + panel.width + }; + let grid_pos = GridPos { + h: height, + w, + x, + y: self.cursor_y, + }; + let id = self.next_id; + self.next_id += 1; + x += w; + self.panels.push(panel.finalize(id, grid_pos)); + } + self.cursor_y += height; + self + } +} + +/// Percentile line targets for a summary metric: `p50`, `p90`, … each querying +/// the matching `quantile="0.x"` series. +pub fn percentiles(metric: &str, percentiles: &[u32]) -> Vec { + percentiles_labeled(metric, percentiles, "") +} + +/// Like [`percentiles`], but appends `legend_suffix` to every legend — handy +/// when the metric carries labels (e.g. ` · {{kind}} · {{origin}}`). +pub fn percentiles_labeled(metric: &str, percentiles: &[u32], legend_suffix: &str) -> Vec { + percentiles + .iter() + .map(|&p| { + let quantile = f64::from(p) / 100.0; + Target::new(format!("{metric}{{quantile=\"{quantile}\"}}")) + .legend(format!("p{p}{legend_suffix}")) + }) + .collect() +} + +/// An `avg` target for a summary metric: `rate(sum) / rate(count)` over 1m. +pub fn avg(metric: &str) -> Target { + Target::new(format!("rate({metric}_sum[1m]) / rate({metric}_count[1m])")).legend("avg") +} + +/// A per-minute rate target for a counter metric. +pub fn rate_per_min(metric: &str, legend: &str) -> Target { + Target::new(format!("rate({metric}[1m]) * 60")).legend(legend) +} diff --git a/tools/dashboard_gen/src/main.rs b/tools/dashboard_gen/src/main.rs new file mode 100644 index 00000000..f6d177bc --- /dev/null +++ b/tools/dashboard_gen/src/main.rs @@ -0,0 +1,111 @@ +//! Generates the sequencer dashboard and prints it to stdout. +//! +//! Compare against the committed file: +//! cargo run -p dashboard_gen | diff - monitoring/grafana/dashboards/sequencer.json +//! Regenerate it: +//! cargo run -p dashboard_gen > monitoring/grafana/dashboards/sequencer.json + +use dashboard_gen::{ + Dashboard, FieldOverride, Panel, Target, avg, percentiles, percentiles_labeled, rate_per_min, +}; +use json_pretty_compact::PrettyCompactFormatter; +use serde::Serialize as _; + +// Rendered Prometheus metric names. In a real setup these would be re-exported +// from `sequencer_core::metrics` (base name + unit/counter suffix rule) so that +// a rename in the recorder is a compile error here — that is the whole point of +// generating dashboards from the same codebase that emits the metrics. +const BLOCK_COUNT: &str = "block_count_total"; +const BLOCK_CREATION_TIME: &str = "block_creation_time_seconds"; +const TX_APPLY_TIME: &str = "mempool_transaction_application_time_seconds"; +const MEMPOOL_SIZE: &str = "mempool_size"; +const TX_PER_BLOCK: &str = "transactions_per_block"; +const SUBMITTED_TX: &str = "submitted_transaction_count_total"; +const FAILED_TX: &str = "failed_transaction_count_total"; + +const PERCENTILES: &[u32] = &[50, 90, 95, 99]; + +fn sequencer_dashboard() -> Dashboard { + Dashboard::new("Sequencer", "sequencer") + .tag("sequencer") + .row( + 7, + [ + Panel::stat("Chain height") + .width(6) + .unit("short") + .decimals(0) + .fixed_color("blue") + .target(Target::new(BLOCK_COUNT).legend("height")), + Panel::timeseries("Block production rate") + .width(18) + .unit("short") + .target(rate_per_min(BLOCK_COUNT, "blocks/min")), + ], + ) + .row( + 9, + [Panel::timeseries("Block creation time") + .width(24) + .unit("s") + .targets(percentiles(BLOCK_CREATION_TIME, PERCENTILES)) + .target(avg(BLOCK_CREATION_TIME)) + .with_override( + FieldOverride::by_name("avg") + .dashed_line() + .fixed_color("text"), + )], + ) + .row( + 9, + [ + Panel::timeseries("Transaction application time") + .width(12) + .unit("s") + .targets(percentiles_labeled( + TX_APPLY_TIME, + PERCENTILES, + " · {{kind}} · {{origin}}", + )), + Panel::timeseries("Mempool size") + .width(12) + .unit("short") + .span_nulls() + .target(Target::new(MEMPOOL_SIZE).legend("mempool size")), + ], + ) + .row( + 9, + [ + Panel::timeseries("Transactions per block") + .width(12) + .unit("short") + .targets(percentiles(TX_PER_BLOCK, PERCENTILES)) + .target(avg(TX_PER_BLOCK)) + .with_override( + FieldOverride::by_name("avg") + .dashed_line() + .fixed_color("text"), + ), + Panel::timeseries("Transaction throughput (per minute)") + .width(12) + .unit("short") + .target(rate_per_min(SUBMITTED_TX, "submitted")) + .target(rate_per_min(FAILED_TX, "failed")) + .with_override(FieldOverride::by_name("failed").fixed_color("red")) + .with_override(FieldOverride::by_name("submitted").fixed_color("green")), + ], + ) +} + +fn main() { + let dashboard = sequencer_dashboard(); + + let formatter = PrettyCompactFormatter::new(); + let mut output = Vec::new(); + let mut ser = serde_json::Serializer::with_formatter(&mut output, formatter); + dashboard.serialize(&mut ser).unwrap(); + + let json = String::from_utf8(output).unwrap(); + println!("{json}"); +} diff --git a/tools/dashboard_gen/src/schema.rs b/tools/dashboard_gen/src/schema.rs new file mode 100644 index 00000000..3a925742 --- /dev/null +++ b/tools/dashboard_gen/src/schema.rs @@ -0,0 +1,260 @@ +//! The serializable Grafana dashboard schema — the internal data model the +//! public builders assemble into. + +use serde::Serialize; + +use crate::{DATASOURCE_UID, FieldOverride, Target}; + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum DatasourceKind { + Prometheus, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum ColorMode { + Fixed, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Fill { + Dash, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum DrawStyle { + Line, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum MatcherKind { + ByName, +} + +#[derive(Clone, Copy, Serialize)] +pub enum PropertyId { + #[serde(rename = "color")] + Color, + #[serde(rename = "custom.lineStyle")] + LineStyle, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum Calc { + LastNotNull, + Last, + Max, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum StatColorMode { + Value, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum GraphMode { + Area, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum LegendDisplay { + Table, + List, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Placement { + Bottom, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum TooltipMode { + Single, + Multi, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum SortOrder { + Desc, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum PanelType { + Stat, + Timeseries, +} + +#[derive(Clone, Serialize)] +pub struct Datasource { + #[serde(rename = "type")] + pub kind: DatasourceKind, + pub uid: &'static str, +} + +impl Datasource { + pub fn prometheus() -> Self { + Self { + kind: DatasourceKind::Prometheus, + uid: DATASOURCE_UID, + } + } +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Color { + pub mode: ColorMode, + pub fixed_color: String, +} + +impl Color { + pub fn fixed(color: String) -> Self { + Self { + mode: ColorMode::Fixed, + fixed_color: color, + } + } +} + +#[derive(Serialize)] +pub struct LineStyle { + pub dash: [u32; 2], + pub fill: Fill, +} + +#[derive(Serialize)] +#[serde(untagged)] +pub enum PropertyValue { + Color(Color), + LineStyle(LineStyle), +} + +#[derive(Serialize)] +pub struct OverrideProperty { + pub id: PropertyId, + pub value: PropertyValue, +} + +#[derive(Serialize)] +pub struct Matcher { + pub id: MatcherKind, + pub options: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Custom { + pub draw_style: DrawStyle, + pub line_width: u32, + pub fill_opacity: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub span_nulls: Option, +} + +#[derive(Serialize)] +pub struct Defaults { + #[serde(skip_serializing_if = "Option::is_none")] + pub color: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub custom: Option, + pub unit: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub decimals: Option, +} + +#[derive(Serialize)] +pub struct FieldConfig { + pub defaults: Defaults, + pub overrides: Vec, +} + +#[derive(Serialize)] +pub struct ReduceOptions { + pub calcs: Vec, + // Empty string means "all fields"; genuinely free-form, not a vocabulary. + pub fields: &'static str, + pub values: bool, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct StatOptions { + pub color_mode: StatColorMode, + pub graph_mode: GraphMode, + pub reduce_options: ReduceOptions, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Legend { + pub display_mode: LegendDisplay, + pub placement: Placement, + pub calcs: Vec, +} + +#[derive(Serialize)] +pub struct Tooltip { + pub mode: TooltipMode, + #[serde(skip_serializing_if = "Option::is_none")] + pub sort: Option, +} + +#[derive(Serialize)] +pub struct TimeSeriesOptions { + pub legend: Legend, + pub tooltip: Tooltip, +} + +#[derive(Serialize)] +#[serde(untagged)] +pub enum Options { + Stat(StatOptions), + TimeSeries(TimeSeriesOptions), +} + +#[derive(Clone, Copy, Serialize)] +pub struct GridPos { + pub h: u32, + pub w: u32, + pub x: u32, + pub y: u32, +} + +/// A fully positioned panel, ready to serialize. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PanelModel { + pub datasource: Datasource, + pub field_config: FieldConfig, + pub grid_pos: GridPos, + pub id: u32, + pub options: Options, + pub targets: Vec, + pub title: String, + #[serde(rename = "type")] + pub panel_type: PanelType, +} + +#[derive(Serialize, Default)] +pub struct EmptyList { + pub list: [u8; 0], +} + +#[derive(Serialize)] +pub struct TimeRange { + // Free-form Grafana time expressions, not a closed vocabulary. + pub from: &'static str, + pub to: &'static str, +} From 462de8f11de816dbe6c427b91b8d792f94774eb8 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 01:09:55 +0300 Subject: [PATCH 04/21] feat(tools): add grafana panel json to rust generator --- Justfile | 10 +- tools/dashboard_gen/Cargo.toml | 4 +- .../gen_sequencer_dashboard.rs} | 29 +-- .../src/bin/panel_json_to_rust.rs | 50 +++++ tools/dashboard_gen/src/codegen.rs | 185 ++++++++++++++++++ tools/dashboard_gen/src/input.rs | 128 ++++++++++++ tools/dashboard_gen/src/lib.rs | 118 +++++++---- tools/dashboard_gen/src/schema.rs | 152 ++++++++++---- tools/dashboard_gen/src/styling.rs | 106 ++++++++++ 9 files changed, 698 insertions(+), 84 deletions(-) rename tools/dashboard_gen/src/{main.rs => bin/gen_sequencer_dashboard.rs} (83%) create mode 100644 tools/dashboard_gen/src/bin/panel_json_to_rust.rs create mode 100644 tools/dashboard_gen/src/codegen.rs create mode 100644 tools/dashboard_gen/src/input.rs create mode 100644 tools/dashboard_gen/src/styling.rs diff --git a/Justfile b/Justfile index 51899221..1a51980e 100644 --- a/Justfile +++ b/Justfile @@ -57,8 +57,14 @@ regenerate-test-fixture: # (tools/dashboard_gen) and commit the result. CI checks these are up to date. regenerate-dashboards: @echo "📊 Regenerating Grafana dashboards" - @cargo build -q -p dashboard_gen - @cargo run -q -p dashboard_gen > monitoring/grafana/dashboards/sequencer.json + @cargo build -q -p dashboard_gen --bin gen_sequencer_dashboard + @cargo run -q -p dashboard_gen --bin gen_sequencer_dashboard > monitoring/grafana/dashboards/sequencer.json + +# Transpile a single Grafana panel JSON (stdin) — Inspect → Panel JSON — into a +# Rust builder expression (stdout), omitting Grafana defaults. Paste into a row. +# Usage: `just panel-to-rust < panel.json`. +panel-to-rust: + @cargo run -q -p dashboard_gen --bin panel_json_to_rust # Run criterion benches: fast crypto primitives, then the slow PPE verify (real proving setup). bench: diff --git a/tools/dashboard_gen/Cargo.toml b/tools/dashboard_gen/Cargo.toml index d05ac951..a21916e1 100644 --- a/tools/dashboard_gen/Cargo.toml +++ b/tools/dashboard_gen/Cargo.toml @@ -4,8 +4,8 @@ version = "0.1.0" edition = "2024" license = { workspace = true } -# NOTE: experiment — intentionally NOT inheriting `[lints] workspace = true`. -# If we keep this crate, turn workspace lints on and fix the fallout. +[lints] +workspace = true [dependencies] serde = { workspace = true, features = ["derive", "alloc"] } diff --git a/tools/dashboard_gen/src/main.rs b/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs similarity index 83% rename from tools/dashboard_gen/src/main.rs rename to tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs index f6d177bc..22a4b8ee 100644 --- a/tools/dashboard_gen/src/main.rs +++ b/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs @@ -1,12 +1,17 @@ //! Generates the sequencer dashboard and prints it to stdout. -//! -//! Compare against the committed file: -//! cargo run -p dashboard_gen | diff - monitoring/grafana/dashboards/sequencer.json -//! Regenerate it: -//! cargo run -p dashboard_gen > monitoring/grafana/dashboards/sequencer.json + +#![expect( + clippy::print_stdout, + reason = "CLI tool: emitting the dashboard JSON on stdout is the deliverable" +)] +#![expect( + clippy::non_ascii_literal, + reason = "legend separators use `·` intentionally, matching the rendered Grafana labels" +)] use dashboard_gen::{ - Dashboard, FieldOverride, Panel, Target, avg, percentiles, percentiles_labeled, rate_per_min, + Color, Dashboard, FieldOverride, Panel, Target, avg, percentiles, percentiles_labeled, + rate_per_min, }; use json_pretty_compact::PrettyCompactFormatter; use serde::Serialize as _; @@ -35,7 +40,7 @@ fn sequencer_dashboard() -> Dashboard { .width(6) .unit("short") .decimals(0) - .fixed_color("blue") + .color(Color::fixed("blue")) .target(Target::new(BLOCK_COUNT).legend("height")), Panel::timeseries("Block production rate") .width(18) @@ -53,7 +58,7 @@ fn sequencer_dashboard() -> Dashboard { .with_override( FieldOverride::by_name("avg") .dashed_line() - .fixed_color("text"), + .color(Color::fixed("text")), )], ) .row( @@ -85,15 +90,17 @@ fn sequencer_dashboard() -> Dashboard { .with_override( FieldOverride::by_name("avg") .dashed_line() - .fixed_color("text"), + .color(Color::fixed("text")), ), Panel::timeseries("Transaction throughput (per minute)") .width(12) .unit("short") .target(rate_per_min(SUBMITTED_TX, "submitted")) .target(rate_per_min(FAILED_TX, "failed")) - .with_override(FieldOverride::by_name("failed").fixed_color("red")) - .with_override(FieldOverride::by_name("submitted").fixed_color("green")), + .with_override(FieldOverride::by_name("failed").color(Color::fixed("red"))) + .with_override( + FieldOverride::by_name("submitted").color(Color::fixed("green")), + ), ], ) } diff --git a/tools/dashboard_gen/src/bin/panel_json_to_rust.rs b/tools/dashboard_gen/src/bin/panel_json_to_rust.rs new file mode 100644 index 00000000..9e10f046 --- /dev/null +++ b/tools/dashboard_gen/src/bin/panel_json_to_rust.rs @@ -0,0 +1,50 @@ +//! Reads a single Grafana panel JSON on stdin (Grafana → panel menu → Inspect → +//! Panel JSON) and prints the Rust `Panel::…` builder expression that rebuilds +//! it through `dashboard_gen`, omitting values Grafana supplies by default. +//! +//! Paste the result into a dashboard's `.row(…)` and run `cargo fmt`. + +#![expect( + clippy::print_stderr, + reason = "CLI tool: diagnostics on stderr are the deliverable" +)] +#![expect( + clippy::non_ascii_literal, + reason = "help text mirrors Grafana's `Inspect → Panel JSON` menu path" +)] + +use std::{ + io::{self, Read as _, Write as _}, + process::ExitCode, +}; + +fn main() -> ExitCode { + let mut input = String::new(); + if let Err(err) = io::stdin().read_to_string(&mut input) { + eprintln!("error: failed to read panel JSON from stdin: {err}"); + return ExitCode::FAILURE; + } + + match dashboard_gen::panel_to_rust_source(&input) { + Ok(source) => { + if let Err(err) = io::stdout().write_all(source.as_bytes()) { + eprintln!("error: failed to write generated source to stdout: {err}"); + return ExitCode::FAILURE; + } + ExitCode::SUCCESS + } + Err(err) => { + eprintln!("error: could not parse panel JSON: {err}"); + eprintln!( + "hint: paste one panel (Inspect → Panel JSON);\ + only stat and timeseries are supported." + ); + eprintln!( + "hint: this tool supports only the subset of Grafana's panel JSON, you might need \ + to manually implement support for new fields." + ); + + ExitCode::FAILURE + } + } +} diff --git a/tools/dashboard_gen/src/codegen.rs b/tools/dashboard_gen/src/codegen.rs new file mode 100644 index 00000000..efa8d8c3 --- /dev/null +++ b/tools/dashboard_gen/src/codegen.rs @@ -0,0 +1,185 @@ +//! Reverse of the builder: turn a single parsed panel back into the Rust +//! `Panel::…` builder expression, omitting values Grafana supplies by default. +//! Backs the `panel_json_to_rust` binary. +//! +//! Input is parsed leniently (see [`crate::input`]) so a raw Grafana panel +//! export — full of fields and vocabularies we don't model — still works; +//! anything unrecognized is dropped. The emitted expression is valid but only +//! lightly formatted, so `cargo fmt` re-indents it once it lands in a file. + +use std::fmt::Write as _; + +use crate::{ + input::PanelInput, + schema::{ + AxisPlacement, Color, GradientMode, LineInterpolation, PanelType, ShowPoints, StackingMode, + }, +}; + +/// Parse one Grafana panel JSON (Inspect → Panel JSON) and emit the Rust +/// builder expression that reproduces it (minus Grafana defaults). +pub fn panel_to_rust_source(json: &str) -> serde_json::Result { + let panel: PanelInput = serde_json::from_str(json)?; + Ok(format!("{}\n", panel_expr(&panel))) +} + +/// A `Panel::…()` expression with one method call per line. +fn panel_expr(panel: &PanelInput) -> String { + panel_expr_inner(panel).expect("writing to a String never fails") +} + +fn panel_expr_inner(panel: &PanelInput) -> Result { + let mut expr = match panel.panel_type { + PanelType::Stat => format!("Panel::stat({:?})", panel.title), + PanelType::Timeseries => format!("Panel::timeseries({:?})", panel.title), + }; + write!(expr, "\n .width({})", panel.grid_pos.w)?; + + let defaults = &panel.field_config.defaults; + // `short` is the builder's own default unit, so it round-trips without a call. + if let Some(unit) = defaults.unit.as_deref().filter(|u| *u != "short") { + write!(expr, "\n .unit({unit:?})")?; + } + if let Some(decimals) = defaults.decimals { + write!(expr, "\n .decimals({decimals})")?; + } + // Only a fixed color is worth emitting; `palette-classic` is Grafana's default. + if let Some(Color::Fixed { fixed_color }) = &defaults.color { + write!(expr, "\n .color(Color::fixed({fixed_color:?}))")?; + } + + if let Some(custom) = &defaults.custom { + if custom.span_nulls == Some(true) { + expr.push_str("\n .span_nulls()"); + } + // Each optional styling field is emitted only when it differs from the + // Grafana default (matching the setters' panic-on-default contract). + if let Some(value) = custom + .line_interpolation + .filter(|&v| v != LineInterpolation::Linear) + { + write!( + expr, + "\n .line_interpolation(LineInterpolation::{})", + line_interp(value) + )?; + } + if let Some(value) = custom.show_points.filter(|&v| v != ShowPoints::Auto) { + write!( + expr, + "\n .show_points(ShowPoints::{})", + show_points(value) + )?; + } + if let Some(value) = custom.gradient_mode.filter(|&v| v != GradientMode::None) { + write!( + expr, + "\n .gradient_mode(GradientMode::{})", + gradient_mode(value) + )?; + } + let stacking = custom.stacking.as_ref().and_then(|s| s.mode); + if let Some(mode) = stacking.filter(|&m| m != StackingMode::None) { + write!( + expr, + "\n .stacking(StackingMode::{})", + stacking_mode(mode) + )?; + } + if let Some(value) = custom.axis_placement.filter(|&v| v != AxisPlacement::Auto) { + write!( + expr, + "\n .axis_placement(AxisPlacement::{})", + axis_placement(value) + )?; + } + if let Some(label) = custom.axis_label.as_deref().filter(|l| !l.is_empty()) { + write!(expr, "\n .axis_label({label:?})")?; + } + } + + for over in &panel.field_config.overrides { + // Only `byName` matchers map to the builder; skip anything else. + let Some(name) = over.matcher.by_name() else { + continue; + }; + let mut calls = String::new(); + for property in &over.properties { + match property.id.as_str() { + "color" => { + if let Ok(Color::Fixed { fixed_color }) = + serde_json::from_value::(property.value.clone()) + { + write!(calls, ".color(Color::fixed({fixed_color:?}))")?; + } + } + "custom.lineStyle" => calls.push_str(".dashed_line()"), + _ => {} // property kind the builder can't express — drop it + } + } + // An override with nothing representable adds no information. + if !calls.is_empty() { + write!( + expr, + "\n .with_override(FieldOverride::by_name({name:?}){calls})" + )?; + } + } + + for target in &panel.targets { + if target.expr.is_empty() { + continue; + } + write!(expr, "\n .target(Target::new({:?})", target.expr)?; + // `__auto` is Grafana's "no explicit legend" sentinel, i.e. the default. + if !target.legend.is_empty() && target.legend != "__auto" { + write!(expr, ".legend({:?})", target.legend)?; + } + expr.push(')'); + } + + Ok(expr) +} + +const fn line_interp(value: LineInterpolation) -> &'static str { + match value { + LineInterpolation::Linear => "Linear", + LineInterpolation::Smooth => "Smooth", + LineInterpolation::StepBefore => "StepBefore", + LineInterpolation::StepAfter => "StepAfter", + } +} + +const fn show_points(value: ShowPoints) -> &'static str { + match value { + ShowPoints::Auto => "Auto", + ShowPoints::Never => "Never", + ShowPoints::Always => "Always", + } +} + +const fn gradient_mode(value: GradientMode) -> &'static str { + match value { + GradientMode::None => "None", + GradientMode::Opacity => "Opacity", + GradientMode::Hue => "Hue", + GradientMode::Scheme => "Scheme", + } +} + +const fn stacking_mode(value: StackingMode) -> &'static str { + match value { + StackingMode::None => "None", + StackingMode::Normal => "Normal", + StackingMode::Percent => "Percent", + } +} + +const fn axis_placement(value: AxisPlacement) -> &'static str { + match value { + AxisPlacement::Auto => "Auto", + AxisPlacement::Left => "Left", + AxisPlacement::Right => "Right", + AxisPlacement::Hidden => "Hidden", + } +} diff --git a/tools/dashboard_gen/src/input.rs b/tools/dashboard_gen/src/input.rs new file mode 100644 index 00000000..e35a3ca3 --- /dev/null +++ b/tools/dashboard_gen/src/input.rs @@ -0,0 +1,128 @@ +//! Lenient, deserialize-only model of a Grafana panel, for the panel→Rust +//! transpiler (`codegen`). +//! +//! A real Grafana export is far wider than what we emit: `unit` may be absent, +//! `options.tooltip.sort` may be `"none"`, overrides carry property types we +//! don't model, and there are dozens of fields we ignore. So this model is +//! deliberately separate from the strict `schema` (sized for *output*): it +//! captures only what codegen reads, makes every field optional, and lets serde +//! drop everything else — including the whole `options` block, which codegen +//! reconstructs from the panel type rather than reading. + +use serde::Deserialize; +use serde_json::Value; + +use crate::schema::{ + AxisPlacement, Color, GradientMode, LineInterpolation, PanelType, ShowPoints, StackingMode, +}; + +#[derive(Deserialize)] +pub struct PanelInput { + #[serde(rename = "type")] + pub panel_type: PanelType, + #[serde(default)] + pub title: String, + #[serde(rename = "gridPos", default)] + pub grid_pos: GridPos, + #[serde(rename = "fieldConfig", default)] + pub field_config: FieldConfig, + #[serde(default)] + pub targets: Vec, +} + +#[derive(Deserialize, Default)] +pub struct GridPos { + #[serde(default)] + pub w: u32, +} + +#[derive(Deserialize, Default)] +pub struct FieldConfig { + #[serde(default)] + pub defaults: Defaults, + #[serde(default)] + pub overrides: Vec, +} + +#[derive(Deserialize, Default)] +pub struct Defaults { + #[serde(default)] + pub unit: Option, + #[serde(default)] + pub decimals: Option, + #[serde(default)] + pub color: Option, + #[serde(default)] + pub custom: Option, +} + +#[derive(Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct Custom { + #[serde(default)] + pub span_nulls: Option, + #[serde(default)] + pub line_interpolation: Option, + #[serde(default)] + pub show_points: Option, + #[serde(default)] + pub gradient_mode: Option, + #[serde(default)] + pub stacking: Option, + #[serde(default)] + pub axis_placement: Option, + #[serde(default)] + pub axis_label: Option, +} + +#[derive(Deserialize, Default)] +pub struct Stacking { + #[serde(default)] + pub mode: Option, +} + +#[derive(Deserialize)] +pub struct Override { + #[serde(default)] + pub matcher: Matcher, + #[serde(default)] + pub properties: Vec, +} + +#[derive(Deserialize, Default)] +pub struct Matcher { + #[serde(default)] + pub id: String, + #[serde(default)] + pub options: Value, +} + +impl Matcher { + /// The series name for a `byName` matcher; `None` for matcher kinds the + /// builder can't express (which the caller skips). + pub fn by_name(&self) -> Option<&str> { + if self.id == "byName" { + self.options.as_str() + } else { + None + } + } +} + +/// One override property. `id`/`value` are kept raw so unknown kinds are simply +/// ignored by codegen rather than failing the whole parse. +#[derive(Deserialize)] +pub struct Property { + #[serde(default)] + pub id: String, + #[serde(default)] + pub value: Value, +} + +#[derive(Deserialize, Default)] +pub struct Target { + #[serde(default)] + pub expr: String, + #[serde(rename = "legendFormat", default)] + pub legend: String, +} diff --git a/tools/dashboard_gen/src/lib.rs b/tools/dashboard_gen/src/lib.rs index fad718e8..243b22cb 100644 --- a/tools/dashboard_gen/src/lib.rs +++ b/tools/dashboard_gen/src/lib.rs @@ -11,29 +11,28 @@ //! //! Build a [`Dashboard`] and serialize it directly. +// Styling vocabularies passed to the optional `styling` setters are part of the +// public API (and are used by this module's `Panel` fields and `finalize`). +pub use codegen::panel_to_rust_source; +pub use schema::{AxisPlacement, Color, GradientMode, LineInterpolation, ShowPoints, StackingMode}; use schema::{ - Calc, Color, Custom, Datasource, Defaults, DrawStyle, EmptyList, FieldConfig, Fill, GraphMode, + Calc, Custom, Datasource, Defaults, DrawStyle, EmptyList, FieldConfig, Fill, GraphMode, GridPos, Legend, LegendDisplay, LineStyle, Matcher, MatcherKind, Options, OverrideProperty, PanelModel, PanelType, Placement, PropertyId, PropertyValue, ReduceOptions, SortOrder, - StatColorMode, StatOptions, TimeRange, TimeSeriesOptions, Tooltip, TooltipMode, + Stacking, StatColorMode, StatOptions, TimeRange, TimeSeriesOptions, Tooltip, TooltipMode, }; use serde::Serialize; +mod codegen; +mod input; mod schema; +mod styling; /// Datasource uid every panel/target points at. Dashboards stay portable across /// environments because they reference the datasource by this stable uid rather /// than by a per-environment URL. pub const DATASOURCE_UID: &str = "prometheus"; -fn default_unit() -> String { - "short".to_owned() -} - -fn ref_letter(index: usize) -> String { - ((b'A' + index as u8) as char).to_string() -} - /// A single Prometheus query within a panel. #[derive(Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -55,6 +54,7 @@ impl Target { } } + #[must_use] pub fn legend(mut self, legend: impl Into) -> Self { self.legend = legend.into(); self @@ -79,14 +79,16 @@ impl FieldOverride { } } - pub fn fixed_color(mut self, color: impl Into) -> Self { + #[must_use] + pub fn color(mut self, color: Color) -> Self { self.properties.push(OverrideProperty { id: PropertyId::Color, - value: PropertyValue::Color(Color::fixed(color.into())), + value: PropertyValue::Color(color), }); self } + #[must_use] pub fn dashed_line(mut self) -> Self { self.properties.push(OverrideProperty { id: PropertyId::LineStyle, @@ -114,9 +116,16 @@ pub struct Panel { width: u32, unit: Option, decimals: Option, - fixed_color: Option, + color: Option, span_nulls: bool, overrides: Vec, + // Optional timeseries styling, set via the `styling` setters. + line_interpolation: Option, + show_points: Option, + gradient_mode: Option, + stacking: Option, + axis_placement: Option, + axis_label: Option, } impl Panel { @@ -128,9 +137,15 @@ impl Panel { width: 0, unit: None, decimals: None, - fixed_color: None, + color: None, span_nulls: false, overrides: Vec::new(), + line_interpolation: None, + show_points: None, + gradient_mode: None, + stacking: None, + axis_placement: None, + axis_label: None, } } @@ -146,41 +161,49 @@ impl Panel { /// Grid width in Grafana's 24-column units. Unset panels split the row's /// remaining width evenly. - pub fn width(mut self, width: u32) -> Self { + #[must_use] + pub const fn width(mut self, width: u32) -> Self { self.width = width; self } + #[must_use] pub fn unit(mut self, unit: impl Into) -> Self { self.unit = Some(unit.into()); self } - pub fn decimals(mut self, decimals: u32) -> Self { + #[must_use] + pub const fn decimals(mut self, decimals: u32) -> Self { self.decimals = Some(decimals); self } - pub fn fixed_color(mut self, color: impl Into) -> Self { - self.fixed_color = Some(color.into()); + #[must_use] + pub fn color(mut self, color: Color) -> Self { + self.color = Some(color); self } - pub fn span_nulls(mut self) -> Self { + #[must_use] + pub const fn span_nulls(mut self) -> Self { self.span_nulls = true; self } + #[must_use] pub fn target(mut self, target: Target) -> Self { self.targets.push(target); self } + #[must_use] pub fn targets(mut self, targets: impl IntoIterator) -> Self { self.targets.extend(targets); self } + #[must_use] pub fn with_override(mut self, over: FieldOverride) -> Self { self.overrides.push(over); self @@ -201,7 +224,7 @@ impl Panel { let (defaults, options, panel_type) = match self.kind { Kind::Stat => { let defaults = Defaults { - color: self.fixed_color.map(Color::fixed), + color: self.color, custom: None, unit, decimals: self.decimals, @@ -211,7 +234,7 @@ impl Panel { graph_mode: GraphMode::Area, reduce_options: ReduceOptions { calcs: vec![Calc::LastNotNull], - fields: "", + fields: String::new(), values: false, }, }); @@ -225,6 +248,15 @@ impl Panel { line_width: 1, fill_opacity: 10, span_nulls: self.span_nulls.then_some(true), + line_interpolation: self.line_interpolation, + show_points: self.show_points, + gradient_mode: self.gradient_mode, + stacking: self.stacking.map(|mode| Stacking { + mode, + group: "A".to_owned(), + }), + axis_placement: self.axis_placement, + axis_label: self.axis_label, }), unit, decimals: None, @@ -310,8 +342,8 @@ impl Dashboard { tags: Vec::new(), templating: EmptyList::default(), time: TimeRange { - from: "now-15m", - to: "now", + from: "now-15m".to_owned(), + to: "now".to_owned(), }, timezone: String::new(), title: title.into(), @@ -321,11 +353,13 @@ impl Dashboard { } } + #[must_use] pub fn tag(mut self, tag: impl Into) -> Self { self.tags.push(tag.into()); self } + #[must_use] pub fn refresh(mut self, refresh: impl Into) -> Self { self.refresh = refresh.into(); self @@ -334,15 +368,17 @@ impl Dashboard { /// Place a horizontal row of panels at the current vertical cursor. Panel /// ids, x offsets and y are assigned here; unset widths split the remaining /// 24 columns evenly. + #[must_use] pub fn row(mut self, height: u32, panels: impl IntoIterator) -> Self { let panels: Vec = panels.into_iter().collect(); let specified: u32 = panels.iter().map(|p| p.width).sum(); - let auto_count = panels.iter().filter(|p| p.width == 0).count() as u32; - let auto_width = if auto_count > 0 { - 24u32.saturating_sub(specified) / auto_count - } else { - 0 - }; + let auto_count = u32::try_from(panels.iter().filter(|p| p.width == 0).count()).unwrap_or(0); + // `checked_div` yields `None` when there are no auto-width panels; the + // fallback width is unused in that case. + let auto_width = 24_u32 + .saturating_sub(specified) + .checked_div(auto_count) + .unwrap_or(0); let mut x = 0; for panel in panels { @@ -358,28 +394,33 @@ impl Dashboard { y: self.cursor_y, }; let id = self.next_id; - self.next_id += 1; - x += w; + self.next_id = self.next_id.saturating_add(1); + x = x.saturating_add(w); self.panels.push(panel.finalize(id, grid_pos)); } - self.cursor_y += height; + self.cursor_y = self.cursor_y.saturating_add(height); self } } /// Percentile line targets for a summary metric: `p50`, `p90`, … each querying /// the matching `quantile="0.x"` series. +#[must_use] pub fn percentiles(metric: &str, percentiles: &[u32]) -> Vec { percentiles_labeled(metric, percentiles, "") } /// Like [`percentiles`], but appends `legend_suffix` to every legend — handy /// when the metric carries labels (e.g. ` · {{kind}} · {{origin}}`). +#[must_use] pub fn percentiles_labeled(metric: &str, percentiles: &[u32], legend_suffix: &str) -> Vec { percentiles .iter() .map(|&p| { - let quantile = f64::from(p) / 100.0; + // `quantile="0.x"` label, derived without float math: zero-pad to two + // digits then drop trailing zeros (50 → "0.5", 95 → "0.95"). + let quantile = format!("0.{p:02}"); + let quantile = quantile.trim_end_matches('0'); Target::new(format!("{metric}{{quantile=\"{quantile}\"}}")) .legend(format!("p{p}{legend_suffix}")) }) @@ -387,11 +428,22 @@ pub fn percentiles_labeled(metric: &str, percentiles: &[u32], legend_suffix: &st } /// An `avg` target for a summary metric: `rate(sum) / rate(count)` over 1m. +#[must_use] pub fn avg(metric: &str) -> Target { Target::new(format!("rate({metric}_sum[1m]) / rate({metric}_count[1m])")).legend("avg") } /// A per-minute rate target for a counter metric. +#[must_use] pub fn rate_per_min(metric: &str, legend: &str) -> Target { Target::new(format!("rate({metric}[1m]) * 60")).legend(legend) } + +fn default_unit() -> String { + "short".to_owned() +} + +fn ref_letter(index: usize) -> String { + let offset = u8::try_from(index).unwrap_or(0); + char::from(b'A'.saturating_add(offset)).to_string() +} diff --git a/tools/dashboard_gen/src/schema.rs b/tools/dashboard_gen/src/schema.rs index 3a925742..7b6138d8 100644 --- a/tools/dashboard_gen/src/schema.rs +++ b/tools/dashboard_gen/src/schema.rs @@ -1,7 +1,12 @@ //! The serializable Grafana dashboard schema — the internal data model the //! public builders assemble into. +//! +//! Most types are `Serialize`-only, sized for what we *emit*. A handful of +//! vocabularies (`PanelType`, `Color`, and the styling enums) additionally +//! derive `Deserialize` because the lenient `input` model — which backs the +//! panel→Rust transpiler — reuses them. -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::{DATASOURCE_UID, FieldOverride, Target}; @@ -11,12 +16,6 @@ pub enum DatasourceKind { Prometheus, } -#[derive(Clone, Copy, Serialize)] -#[serde(rename_all = "lowercase")] -pub enum ColorMode { - Fixed, -} - #[derive(Clone, Copy, Serialize)] #[serde(rename_all = "lowercase")] pub enum Fill { @@ -89,43 +88,130 @@ pub enum SortOrder { Desc, } -#[derive(Clone, Copy, Serialize)] +// Reused by the `input` model, hence `Deserialize`. +#[derive(Clone, Copy, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum PanelType { Stat, Timeseries, } +// Optional timeseries styling vocabularies. Each derives `PartialEq` so the +// public setters can panic when handed the Grafana default (see `styling`), and +// `Deserialize` because the `input` model reuses them. + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum LineInterpolation { + Linear, + Smooth, + StepBefore, + StepAfter, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ShowPoints { + Auto, + Never, + Always, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum GradientMode { + None, + Opacity, + Hue, + Scheme, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum StackingMode { + None, + Normal, + Percent, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AxisPlacement { + Auto, + Left, + Right, + Hidden, +} + +#[derive(Serialize)] +pub struct Stacking { + pub mode: StackingMode, + pub group: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Custom { + pub draw_style: DrawStyle, + pub line_width: u32, + pub fill_opacity: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub span_nulls: Option, + // Optional styling — omitted (left at Grafana's default) unless a setter + // fills it in. See `styling`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub line_interpolation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub show_points: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gradient_mode: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stacking: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub axis_placement: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub axis_label: Option, +} + #[derive(Clone, Serialize)] pub struct Datasource { #[serde(rename = "type")] pub kind: DatasourceKind, - pub uid: &'static str, + pub uid: String, } impl Datasource { pub fn prometheus() -> Self { Self { kind: DatasourceKind::Prometheus, - uid: DATASOURCE_UID, + uid: DATASOURCE_UID.to_owned(), } } } -#[derive(Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct Color { - pub mode: ColorMode, - pub fixed_color: String, +// Reused by the `input` model, hence `Deserialize`. +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +#[serde(tag = "mode")] +pub enum Color { + Fixed { + #[serde(rename = "fixedColor")] + fixed_color: String, + }, + PaletteClassic, } impl Color { - pub fn fixed(color: String) -> Self { - Self { - mode: ColorMode::Fixed, - fixed_color: color, + pub fn fixed(color: impl Into) -> Self { + Self::Fixed { + fixed_color: color.into(), } } + + #[must_use] + pub const fn palette_classic() -> Self { + Self::PaletteClassic + } } #[derive(Serialize)] @@ -153,24 +239,14 @@ pub struct Matcher { pub options: String, } -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct Custom { - pub draw_style: DrawStyle, - pub line_width: u32, - pub fill_opacity: u32, - #[serde(skip_serializing_if = "Option::is_none")] - pub span_nulls: Option, -} - #[derive(Serialize)] pub struct Defaults { - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "Option::is_none")] pub color: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "Option::is_none")] pub custom: Option, pub unit: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "Option::is_none")] pub decimals: Option, } @@ -184,7 +260,7 @@ pub struct FieldConfig { pub struct ReduceOptions { pub calcs: Vec, // Empty string means "all fields"; genuinely free-form, not a vocabulary. - pub fields: &'static str, + pub fields: String, pub values: bool, } @@ -207,7 +283,7 @@ pub struct Legend { #[derive(Serialize)] pub struct Tooltip { pub mode: TooltipMode, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "Option::is_none")] pub sort: Option, } @@ -247,6 +323,10 @@ pub struct PanelModel { pub panel_type: PanelType, } +#[expect( + clippy::trailing_empty_array, + reason = "Grafana expects `list: []` for the blocks we don't populate" +)] #[derive(Serialize, Default)] pub struct EmptyList { pub list: [u8; 0], @@ -255,6 +335,6 @@ pub struct EmptyList { #[derive(Serialize)] pub struct TimeRange { // Free-form Grafana time expressions, not a closed vocabulary. - pub from: &'static str, - pub to: &'static str, + pub from: String, + pub to: String, } diff --git a/tools/dashboard_gen/src/styling.rs b/tools/dashboard_gen/src/styling.rs new file mode 100644 index 00000000..5414ec73 --- /dev/null +++ b/tools/dashboard_gen/src/styling.rs @@ -0,0 +1,106 @@ +//! Optional timeseries styling setters. +//! +//! These are real setters — each fills in a field of the panel's `custom` block. +//! Every one documents Grafana's default and **panics if handed that default**: +//! passing the default is always redundant (Grafana emits it anyway), and the +//! generator only serializes fields that differ from the default. So if a call +//! wouldn't change the rendered panel, it's a mistake worth catching loudly at +//! generation time rather than shipping a no-op. +//! +//! These affect timeseries panels only; on a stat panel the `custom` block is +//! not emitted, so the value is silently dropped. + +use crate::{ + Panel, + schema::{AxisPlacement, GradientMode, LineInterpolation, ShowPoints, StackingMode}, +}; + +#[expect( + clippy::multiple_inherent_impl, + reason = "styling setters intentionally live in their own file, so `Panel` has a second inherent impl here" +)] +impl Panel { + /// Interpolation between points. Grafana default: `Linear`. + /// + /// Panics if passed `Linear` — that's the default and would be redundant. + #[must_use] + pub fn line_interpolation(mut self, value: LineInterpolation) -> Self { + assert_ne!( + value, + LineInterpolation::Linear, + "line_interpolation(Linear) is redundant: `linear` is Grafana's default. Omit the call.", + ); + self.line_interpolation = Some(value); + self + } + + /// Whether/when to draw point markers. Grafana default: `Auto`. + /// + /// Panics if passed `Auto` — that's the default and would be redundant. + #[must_use] + pub fn show_points(mut self, value: ShowPoints) -> Self { + assert_ne!( + value, + ShowPoints::Auto, + "show_points(Auto) is redundant: `auto` is Grafana's default. Omit the call.", + ); + self.show_points = Some(value); + self + } + + /// Area fill gradient. Grafana default: `None`. + /// + /// Panics if passed `None` — that's the default and would be redundant. + #[must_use] + pub fn gradient_mode(mut self, value: GradientMode) -> Self { + assert_ne!( + value, + GradientMode::None, + "gradient_mode(None) is redundant: `none` is Grafana's default. Omit the call.", + ); + self.gradient_mode = Some(value); + self + } + + /// Series stacking. Grafana default: `None`. + /// + /// Panics if passed `None` — that's the default and would be redundant. + #[must_use] + pub fn stacking(mut self, mode: StackingMode) -> Self { + assert_ne!( + mode, + StackingMode::None, + "stacking(None) is redundant: `none` is Grafana's default. Omit the call.", + ); + self.stacking = Some(mode); + self + } + + /// Y-axis placement. Grafana default: `Auto`. + /// + /// Panics if passed `Auto` — that's the default and would be redundant. + #[must_use] + pub fn axis_placement(mut self, value: AxisPlacement) -> Self { + assert_ne!( + value, + AxisPlacement::Auto, + "axis_placement(Auto) is redundant: `auto` is Grafana's default. Omit the call.", + ); + self.axis_placement = Some(value); + self + } + + /// Y-axis label. Grafana default: `""` (no label). + /// + /// Panics if passed an empty string — that's the default and would be redundant. + #[must_use] + pub fn axis_label(mut self, label: impl Into) -> Self { + let label = label.into(); + assert_ne!( + label, "", + "axis_label(\"\") is redundant: no label is Grafana's default. Omit the call.", + ); + self.axis_label = Some(label); + self + } +} From ea36fa2e93a6f0a24dcec76ec1606204bb8bed2a Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 02:00:11 +0300 Subject: [PATCH 05/21] refactor(metrics): move metrics into a separate crates and reuse names in dashboard generation --- Cargo.lock | 23 ++++++-- Cargo.toml | 4 ++ lez/sequencer/core/Cargo.toml | 3 +- lez/sequencer/core/metrics/Cargo.toml | 19 +++++++ lez/sequencer/core/metrics/src/lib.rs | 9 ++++ lez/sequencer/core/metrics/src/names.rs | 6 +++ .../metrics.rs => metrics/src/writing.rs} | 15 ++---- lez/sequencer/core/src/lib.rs | 28 ++++++---- lez/sequencer/service/Cargo.toml | 2 +- lez/sequencer/service/metrics/Cargo.toml | 16 ++++++ lez/sequencer/service/metrics/src/lib.rs | 9 ++++ lez/sequencer/service/metrics/src/names.rs | 1 + .../metrics.rs => metrics/src/writing.rs} | 6 +-- lez/sequencer/service/src/lib.rs | 1 - lez/sequencer/service/src/service.rs | 2 +- tools/dashboard_gen/Cargo.toml | 3 ++ .../src/bin/gen_sequencer_dashboard.rs | 52 +++++++++++-------- 17 files changed, 144 insertions(+), 55 deletions(-) create mode 100644 lez/sequencer/core/metrics/Cargo.toml create mode 100644 lez/sequencer/core/metrics/src/lib.rs create mode 100644 lez/sequencer/core/metrics/src/names.rs rename lez/sequencer/core/{src/metrics.rs => metrics/src/writing.rs} (79%) create mode 100644 lez/sequencer/service/metrics/Cargo.toml create mode 100644 lez/sequencer/service/metrics/src/lib.rs create mode 100644 lez/sequencer/service/metrics/src/names.rs rename lez/sequencer/service/{src/metrics.rs => metrics/src/writing.rs} (59%) diff --git a/Cargo.lock b/Cargo.lock index 35eb6480..6eb08255 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2297,6 +2297,8 @@ name = "dashboard_gen" version = "0.1.0" dependencies = [ "json-pretty-compact", + "sequencer_core_metrics", + "sequencer_service_metrics", "serde", "serde_json", ] @@ -9503,16 +9505,15 @@ dependencies = [ "logos-blockchain-key-management-system-service", "logos-blockchain-zone-sdk", "mempool", - "metrics", "num-bigint 0.4.6", "ping_core", "programs", "rand 0.8.6", "risc0-zkvm", + "sequencer_core_metrics", "serde", "serde_json", "storage", - "strum 0.28.0", "system_accounts", "tempfile", "test_programs", @@ -9524,6 +9525,15 @@ dependencies = [ "vault_core", ] +[[package]] +name = "sequencer_core_metrics" +version = "0.1.0" +dependencies = [ + "common", + "metrics", + "strum 0.28.0", +] + [[package]] name = "sequencer_service" version = "0.1.0" @@ -9540,16 +9550,23 @@ dependencies = [ "lee", "log", "mempool", - "metrics", "metrics-exporter-prometheus", "programs", "sequencer_core", + "sequencer_service_metrics", "sequencer_service_protocol", "sequencer_service_rpc", "tokio", "tokio-util", ] +[[package]] +name = "sequencer_service_metrics" +version = "0.1.0" +dependencies = [ + "metrics", +] + [[package]] name = "sequencer_service_protocol" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 467dd9dd..d5ebbe48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,9 +18,11 @@ members = [ "lez/system_accounts", "lez/chain_state", "lez/sequencer/core", + "lez/sequencer/core/metrics", "lez/sequencer/service", "lez/sequencer/service/protocol", "lez/sequencer/service/rpc", + "lez/sequencer/service/metrics", "lez/indexer/core", "lez/indexer/service", "lez/indexer/service/protocol", @@ -79,8 +81,10 @@ mempool = { path = "lez/mempool" } storage = { path = "lez/storage" } key_protocol = { path = "lee/key_protocol" } sequencer_core = { path = "lez/sequencer/core" } +sequencer_core_metrics = { path = "lez/sequencer/core/metrics" } sequencer_service_protocol = { path = "lez/sequencer/service/protocol" } sequencer_service_rpc = { path = "lez/sequencer/service/rpc" } +sequencer_service_metrics = { path = "lez/sequencer/service/metrics" } sequencer_service = { path = "lez/sequencer/service" } indexer_core = { path = "lez/indexer/core" } indexer_service = { path = "lez/indexer/service" } diff --git a/lez/sequencer/core/Cargo.toml b/lez/sequencer/core/Cargo.toml index 41b70fca..08f359f7 100644 --- a/lez/sequencer/core/Cargo.toml +++ b/lez/sequencer/core/Cargo.toml @@ -12,6 +12,7 @@ lee.workspace = true lee_core.workspace = true chain_state.workspace = true common.workspace = true +sequencer_core_metrics = { workspace = true, features = ["writing"] } storage.workspace = true mempool.workspace = true logos-blockchain-zone-sdk.workspace = true @@ -34,7 +35,6 @@ humantime-serde.workspace = true tempfile.workspace = true chrono.workspace = true log.workspace = true -metrics.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } tokio-util.workspace = true rand.workspace = true @@ -46,7 +46,6 @@ num-bigint.workspace = true risc0-zkvm.workspace = true futures.workspace = true itertools.workspace = true -strum.workspace = true [features] default = [] diff --git a/lez/sequencer/core/metrics/Cargo.toml b/lez/sequencer/core/metrics/Cargo.toml new file mode 100644 index 00000000..541102b4 --- /dev/null +++ b/lez/sequencer/core/metrics/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "sequencer_core_metrics" +version = "0.1.0" +edition = "2024" +license = { workspace = true } + +[lints] +workspace = true + +[features] +default = [] +# Enable metrics writing +writing = ["dep:common", "dep:metrics", "dep:strum"] + +[dependencies] +common = { workspace = true, optional = true } + +metrics = { workspace = true, optional = true } +strum = { workspace = true, optional = true } diff --git a/lez/sequencer/core/metrics/src/lib.rs b/lez/sequencer/core/metrics/src/lib.rs new file mode 100644 index 00000000..b633f2d2 --- /dev/null +++ b/lez/sequencer/core/metrics/src/lib.rs @@ -0,0 +1,9 @@ +//! This crate provides all metrics exposed by the sequencer core crate. + +#[cfg(feature = "writing")] +pub use writing::*; + +pub mod names; + +#[cfg(feature = "writing")] +pub mod writing; diff --git a/lez/sequencer/core/metrics/src/names.rs b/lez/sequencer/core/metrics/src/names.rs new file mode 100644 index 00000000..1ea87c26 --- /dev/null +++ b/lez/sequencer/core/metrics/src/names.rs @@ -0,0 +1,6 @@ +pub const BLOCK_CREATION_TIME: &str = "block_creation_time"; +pub const BLOCK_COUNT: &str = "block_count"; +pub const MEMPOOL_SIZE: &str = "mempool_size"; +pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = "mempool_transaction_application_time"; +pub const TRANSACTIONS_PER_BLOCK: &str = "transactions_per_block"; +pub const FAILED_TRANSACTION_COUNT: &str = "failed_transaction_count"; diff --git a/lez/sequencer/core/src/metrics.rs b/lez/sequencer/core/metrics/src/writing.rs similarity index 79% rename from lez/sequencer/core/src/metrics.rs rename to lez/sequencer/core/metrics/src/writing.rs index a6c79454..2a392b84 100644 --- a/lez/sequencer/core/src/metrics.rs +++ b/lez/sequencer/core/metrics/src/writing.rs @@ -1,5 +1,3 @@ -//! This module provides all metrics exposed by the sequencer core crate. - #![expect( clippy::cast_precision_loss, clippy::as_conversions, @@ -11,15 +9,12 @@ use std::time::Duration; use common::transaction::TxKind; use metrics::{Counter, Unit, counter, gauge, histogram}; -use crate::TransactionOrigin; +use crate::names; -mod names { - pub const BLOCK_CREATION_TIME: &str = "block_creation_time"; - pub const BLOCK_COUNT: &str = "block_count"; - pub const MEMPOOL_SIZE: &str = "mempool_size"; - pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = "mempool_transaction_application_time"; - pub const TRANSACTIONS_PER_BLOCK: &str = "transactions_per_block"; - pub const FAILED_TRANSACTION_COUNT: &str = "failed_transaction_count"; +#[derive(Clone, Copy, strum::IntoStaticStr)] +pub enum TransactionOrigin { + User, + Sequencer, } pub fn record_block_creation_time(duration: Duration) { diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index a56b3c88..7cbe8fa7 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -50,7 +50,6 @@ pub mod block_publisher; pub mod block_store; pub mod config; pub mod cross_zone_watcher; -mod metrics; #[cfg(feature = "mock")] pub mod mock; @@ -73,7 +72,7 @@ const RETIRE_DISPATCH_AFTER_FAILURES: u32 = 3; const MAX_DISPATCHES_PER_BLOCK: usize = 16; /// The origin of a transaction. -#[derive(Clone, Copy, strum::IntoStaticStr)] +#[derive(Clone, Copy)] pub enum TransactionOrigin { /// Basic transactions submitted by users via RPC. User, @@ -81,6 +80,15 @@ pub enum TransactionOrigin { Sequencer, } +impl From for sequencer_core_metrics::TransactionOrigin { + fn from(origin: TransactionOrigin) -> Self { + match origin { + TransactionOrigin::User => Self::User, + TransactionOrigin::Sequencer => Self::Sequencer, + } + } +} + #[derive(Clone, Debug, BorshDeserialize)] struct DepositMetadata { recipient_id: lee::AccountId, @@ -300,7 +308,7 @@ impl SequencerCore { watchers, }; - crate::metrics::set_block_count(sequencer_core.chain_height()); + sequencer_core_metrics::set_block_count(sequencer_core.chain_height()); (sequencer_core, mempool_handle) } @@ -805,7 +813,7 @@ impl SequencerCore { let clock_tx = clock_invocation(new_block_timestamp); let clock_lee_tx = LeeTransaction::Public(clock_tx.clone()); - crate::metrics::record_mempool_size(self.mempool.len()); + sequencer_core_metrics::record_mempool_size(self.mempool.len()); // Everything drained from the store first, then user work. `from_store` // is not the same as a `Sequencer` origin: it says the transaction has a // record behind it and so needs no requeue, where the origin only says @@ -885,8 +893,8 @@ impl SequencerCore { new_block_timestamp, &mut withdrawals, ); - crate::metrics::record_mempool_transaction_application_time( - origin, + sequencer_core_metrics::record_mempool_transaction_application_time( + origin.into(), tx.kind(), before_tx_apply.elapsed(), ); @@ -894,7 +902,7 @@ impl SequencerCore { if applied { valid_transactions.push(tx); } else { - crate::metrics::increment_failed_transaction_count(); + sequencer_core_metrics::increment_failed_transaction_count(); // A failed transaction is simply left out of the block, except a // dispatch: that one is re-fed from the store every turn, so one // that can never execute would fail on every block for ever. @@ -910,7 +918,7 @@ impl SequencerCore { .transition_from_public_transaction(&clock_tx, new_block_height, new_block_timestamp) .context("Clock transaction failed. Aborting block production.")?; valid_transactions.push(clock_lee_tx); - crate::metrics::record_transactions_per_block(valid_transactions.len()); + sequencer_core_metrics::record_transactions_per_block(valid_transactions.len()); let hashable_data = HashableBlockData { block_id: new_block_height, @@ -929,8 +937,8 @@ impl SequencerCore { now.elapsed().as_secs() ); - crate::metrics::record_block_creation_time(now.elapsed()); - crate::metrics::increment_block_count(); + sequencer_core_metrics::record_block_creation_time(now.elapsed()); + sequencer_core_metrics::increment_block_count(); Ok(BlockWithMeta { block, withdrawals }) } diff --git a/lez/sequencer/service/Cargo.toml b/lez/sequencer/service/Cargo.toml index 03400935..0b936e8b 100644 --- a/lez/sequencer/service/Cargo.toml +++ b/lez/sequencer/service/Cargo.toml @@ -15,6 +15,7 @@ mempool.workspace = true sequencer_core = { workspace = true, features = ["testnet"] } sequencer_service_protocol.workspace = true sequencer_service_rpc = { workspace = true, features = ["server"] } +sequencer_service_metrics = { workspace = true, features = ["writing"] } programs.workspace = true clap = { workspace = true, features = ["derive", "env"] } @@ -22,7 +23,6 @@ anyhow.workspace = true env_logger.workspace = true hex.workspace = true log.workspace = true -metrics.workspace = true metrics-exporter-prometheus.workspace = true tokio.workspace = true tokio-util.workspace = true diff --git a/lez/sequencer/service/metrics/Cargo.toml b/lez/sequencer/service/metrics/Cargo.toml new file mode 100644 index 00000000..4e242db4 --- /dev/null +++ b/lez/sequencer/service/metrics/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "sequencer_service_metrics" +version = "0.1.0" +edition = "2024" +license = { workspace = true } + +[lints] +workspace = true + +[features] +default = [] +# Enable metrics writing +writing = ["dep:metrics"] + +[dependencies] +metrics = { workspace = true, optional = true } diff --git a/lez/sequencer/service/metrics/src/lib.rs b/lez/sequencer/service/metrics/src/lib.rs new file mode 100644 index 00000000..b633f2d2 --- /dev/null +++ b/lez/sequencer/service/metrics/src/lib.rs @@ -0,0 +1,9 @@ +//! This crate provides all metrics exposed by the sequencer core crate. + +#[cfg(feature = "writing")] +pub use writing::*; + +pub mod names; + +#[cfg(feature = "writing")] +pub mod writing; diff --git a/lez/sequencer/service/metrics/src/names.rs b/lez/sequencer/service/metrics/src/names.rs new file mode 100644 index 00000000..8bf61acb --- /dev/null +++ b/lez/sequencer/service/metrics/src/names.rs @@ -0,0 +1 @@ +pub const SUBMITTED_TRANSACTION_COUNT: &str = "submitted_transaction_count"; diff --git a/lez/sequencer/service/src/metrics.rs b/lez/sequencer/service/metrics/src/writing.rs similarity index 59% rename from lez/sequencer/service/src/metrics.rs rename to lez/sequencer/service/metrics/src/writing.rs index 27cd47d1..3b625006 100644 --- a/lez/sequencer/service/src/metrics.rs +++ b/lez/sequencer/service/metrics/src/writing.rs @@ -1,10 +1,6 @@ -//! This module provides all metrics exposed by the sequencer service crate. - use metrics::{Unit, counter}; -mod names { - pub const SUBMITTED_TRANSACTION_COUNT: &str = "submitted_transaction_count"; -} +use crate::names; pub fn increment_submitted_transaction_count() { counter!( diff --git a/lez/sequencer/service/src/lib.rs b/lez/sequencer/service/src/lib.rs index 5624cc91..ba5b68ec 100644 --- a/lez/sequencer/service/src/lib.rs +++ b/lez/sequencer/service/src/lib.rs @@ -21,7 +21,6 @@ use sequencer_service_rpc::RpcServer as _; use tokio::{sync::Mutex, task::JoinHandle}; use tokio_util::sync::CancellationToken; -mod metrics; pub mod service; const REQUEST_BODY_MAX_SIZE: ByteSize = ByteSize::mib(10); diff --git a/lez/sequencer/service/src/service.rs b/lez/sequencer/service/src/service.rs index d9eaf97f..309c9949 100644 --- a/lez/sequencer/service/src/service.rs +++ b/lez/sequencer/service/src/service.rs @@ -93,7 +93,7 @@ impl sequencer_service_rpc::Rpc .await .expect("Mempool is closed, this is a bug"); - crate::metrics::increment_submitted_transaction_count(); + sequencer_service_metrics::increment_submitted_transaction_count(); Ok(tx_hash) } diff --git a/tools/dashboard_gen/Cargo.toml b/tools/dashboard_gen/Cargo.toml index a21916e1..1a167db6 100644 --- a/tools/dashboard_gen/Cargo.toml +++ b/tools/dashboard_gen/Cargo.toml @@ -8,6 +8,9 @@ license = { workspace = true } workspace = true [dependencies] +sequencer_core_metrics.workspace = true +sequencer_service_metrics.workspace = true + serde = { workspace = true, features = ["derive", "alloc"] } serde_json.workspace = true json-pretty-compact = "0.1.2" diff --git a/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs b/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs index 22a4b8ee..8c1a9321 100644 --- a/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs +++ b/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs @@ -16,18 +16,6 @@ use dashboard_gen::{ use json_pretty_compact::PrettyCompactFormatter; use serde::Serialize as _; -// Rendered Prometheus metric names. In a real setup these would be re-exported -// from `sequencer_core::metrics` (base name + unit/counter suffix rule) so that -// a rename in the recorder is a compile error here — that is the whole point of -// generating dashboards from the same codebase that emits the metrics. -const BLOCK_COUNT: &str = "block_count_total"; -const BLOCK_CREATION_TIME: &str = "block_creation_time_seconds"; -const TX_APPLY_TIME: &str = "mempool_transaction_application_time_seconds"; -const MEMPOOL_SIZE: &str = "mempool_size"; -const TX_PER_BLOCK: &str = "transactions_per_block"; -const SUBMITTED_TX: &str = "submitted_transaction_count_total"; -const FAILED_TX: &str = "failed_transaction_count_total"; - const PERCENTILES: &[u32] = &[50, 90, 95, 99]; fn sequencer_dashboard() -> Dashboard { @@ -41,11 +29,16 @@ fn sequencer_dashboard() -> Dashboard { .unit("short") .decimals(0) .color(Color::fixed("blue")) - .target(Target::new(BLOCK_COUNT).legend("height")), + .target( + Target::new(sequencer_core_metrics::names::BLOCK_COUNT).legend("height"), + ), Panel::timeseries("Block production rate") .width(18) .unit("short") - .target(rate_per_min(BLOCK_COUNT, "blocks/min")), + .target(rate_per_min( + sequencer_core_metrics::names::BLOCK_COUNT, + "blocks/min", + )), ], ) .row( @@ -53,8 +46,11 @@ fn sequencer_dashboard() -> Dashboard { [Panel::timeseries("Block creation time") .width(24) .unit("s") - .targets(percentiles(BLOCK_CREATION_TIME, PERCENTILES)) - .target(avg(BLOCK_CREATION_TIME)) + .targets(percentiles( + sequencer_core_metrics::names::BLOCK_CREATION_TIME, + PERCENTILES, + )) + .target(avg(sequencer_core_metrics::names::BLOCK_CREATION_TIME)) .with_override( FieldOverride::by_name("avg") .dashed_line() @@ -68,7 +64,7 @@ fn sequencer_dashboard() -> Dashboard { .width(12) .unit("s") .targets(percentiles_labeled( - TX_APPLY_TIME, + sequencer_core_metrics::names::MEMPOOL_TRANSACTION_APPLICATION_TIME, PERCENTILES, " · {{kind}} · {{origin}}", )), @@ -76,7 +72,10 @@ fn sequencer_dashboard() -> Dashboard { .width(12) .unit("short") .span_nulls() - .target(Target::new(MEMPOOL_SIZE).legend("mempool size")), + .target( + Target::new(sequencer_core_metrics::names::MEMPOOL_SIZE) + .legend("mempool size"), + ), ], ) .row( @@ -85,8 +84,11 @@ fn sequencer_dashboard() -> Dashboard { Panel::timeseries("Transactions per block") .width(12) .unit("short") - .targets(percentiles(TX_PER_BLOCK, PERCENTILES)) - .target(avg(TX_PER_BLOCK)) + .targets(percentiles( + sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK, + PERCENTILES, + )) + .target(avg(sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK)) .with_override( FieldOverride::by_name("avg") .dashed_line() @@ -95,8 +97,14 @@ fn sequencer_dashboard() -> Dashboard { Panel::timeseries("Transaction throughput (per minute)") .width(12) .unit("short") - .target(rate_per_min(SUBMITTED_TX, "submitted")) - .target(rate_per_min(FAILED_TX, "failed")) + .target(rate_per_min( + sequencer_service_metrics::names::SUBMITTED_TRANSACTION_COUNT, + "submitted", + )) + .target(rate_per_min( + sequencer_core_metrics::names::FAILED_TRANSACTION_COUNT, + "failed", + )) .with_override(FieldOverride::by_name("failed").color(Color::fixed("red"))) .with_override( FieldOverride::by_name("submitted").color(Color::fixed("green")), From ec53979442fae50c3b75b5cae892e19d0e05bd45 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 02:15:43 +0300 Subject: [PATCH 06/21] feat(dashboard_gen): make unit an enum --- .../src/bin/gen_sequencer_dashboard.rs | 16 +-- tools/dashboard_gen/src/codegen.rs | 7 +- tools/dashboard_gen/src/lib.rs | 12 +- tools/dashboard_gen/src/schema.rs | 4 +- tools/dashboard_gen/src/unit.rs | 107 ++++++++++++++++++ 5 files changed, 130 insertions(+), 16 deletions(-) create mode 100644 tools/dashboard_gen/src/unit.rs diff --git a/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs b/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs index 8c1a9321..4e0716ab 100644 --- a/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs +++ b/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs @@ -10,7 +10,7 @@ )] use dashboard_gen::{ - Color, Dashboard, FieldOverride, Panel, Target, avg, percentiles, percentiles_labeled, + Color, Dashboard, FieldOverride, Panel, Target, Unit, avg, percentiles, percentiles_labeled, rate_per_min, }; use json_pretty_compact::PrettyCompactFormatter; @@ -26,7 +26,7 @@ fn sequencer_dashboard() -> Dashboard { [ Panel::stat("Chain height") .width(6) - .unit("short") + .unit(Unit::Short) .decimals(0) .color(Color::fixed("blue")) .target( @@ -34,7 +34,7 @@ fn sequencer_dashboard() -> Dashboard { ), Panel::timeseries("Block production rate") .width(18) - .unit("short") + .unit(Unit::Short) .target(rate_per_min( sequencer_core_metrics::names::BLOCK_COUNT, "blocks/min", @@ -45,7 +45,7 @@ fn sequencer_dashboard() -> Dashboard { 9, [Panel::timeseries("Block creation time") .width(24) - .unit("s") + .unit(Unit::Seconds) .targets(percentiles( sequencer_core_metrics::names::BLOCK_CREATION_TIME, PERCENTILES, @@ -62,7 +62,7 @@ fn sequencer_dashboard() -> Dashboard { [ Panel::timeseries("Transaction application time") .width(12) - .unit("s") + .unit(Unit::Seconds) .targets(percentiles_labeled( sequencer_core_metrics::names::MEMPOOL_TRANSACTION_APPLICATION_TIME, PERCENTILES, @@ -70,7 +70,7 @@ fn sequencer_dashboard() -> Dashboard { )), Panel::timeseries("Mempool size") .width(12) - .unit("short") + .unit(Unit::Short) .span_nulls() .target( Target::new(sequencer_core_metrics::names::MEMPOOL_SIZE) @@ -83,7 +83,7 @@ fn sequencer_dashboard() -> Dashboard { [ Panel::timeseries("Transactions per block") .width(12) - .unit("short") + .unit(Unit::Short) .targets(percentiles( sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK, PERCENTILES, @@ -96,7 +96,7 @@ fn sequencer_dashboard() -> Dashboard { ), Panel::timeseries("Transaction throughput (per minute)") .width(12) - .unit("short") + .unit(Unit::Short) .target(rate_per_min( sequencer_service_metrics::names::SUBMITTED_TRANSACTION_COUNT, "submitted", diff --git a/tools/dashboard_gen/src/codegen.rs b/tools/dashboard_gen/src/codegen.rs index efa8d8c3..15f79cb4 100644 --- a/tools/dashboard_gen/src/codegen.rs +++ b/tools/dashboard_gen/src/codegen.rs @@ -10,6 +10,7 @@ use std::fmt::Write as _; use crate::{ + Unit, input::PanelInput, schema::{ AxisPlacement, Color, GradientMode, LineInterpolation, PanelType, ShowPoints, StackingMode, @@ -38,7 +39,11 @@ fn panel_expr_inner(panel: &PanelInput) -> Result { let defaults = &panel.field_config.defaults; // `short` is the builder's own default unit, so it round-trips without a call. if let Some(unit) = defaults.unit.as_deref().filter(|u| *u != "short") { - write!(expr, "\n .unit({unit:?})")?; + write!( + expr, + "\n .unit({})", + Unit::from_id(unit).to_rust_source() + )?; } if let Some(decimals) = defaults.decimals { write!(expr, "\n .decimals({decimals})")?; diff --git a/tools/dashboard_gen/src/lib.rs b/tools/dashboard_gen/src/lib.rs index 243b22cb..918061a0 100644 --- a/tools/dashboard_gen/src/lib.rs +++ b/tools/dashboard_gen/src/lib.rs @@ -22,11 +22,13 @@ use schema::{ Stacking, StatColorMode, StatOptions, TimeRange, TimeSeriesOptions, Tooltip, TooltipMode, }; use serde::Serialize; +pub use unit::Unit; mod codegen; mod input; mod schema; mod styling; +mod unit; /// Datasource uid every panel/target points at. Dashboards stay portable across /// environments because they reference the datasource by this stable uid rather @@ -114,7 +116,7 @@ pub struct Panel { kind: Kind, targets: Vec, width: u32, - unit: Option, + unit: Option, decimals: Option, color: Option, span_nulls: bool, @@ -168,8 +170,8 @@ impl Panel { } #[must_use] - pub fn unit(mut self, unit: impl Into) -> Self { - self.unit = Some(unit.into()); + pub fn unit(mut self, unit: Unit) -> Self { + self.unit = Some(unit); self } @@ -439,8 +441,8 @@ pub fn rate_per_min(metric: &str, legend: &str) -> Target { Target::new(format!("rate({metric}[1m]) * 60")).legend(legend) } -fn default_unit() -> String { - "short".to_owned() +const fn default_unit() -> Unit { + Unit::Short } fn ref_letter(index: usize) -> String { diff --git a/tools/dashboard_gen/src/schema.rs b/tools/dashboard_gen/src/schema.rs index 7b6138d8..88d24bd8 100644 --- a/tools/dashboard_gen/src/schema.rs +++ b/tools/dashboard_gen/src/schema.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; -use crate::{DATASOURCE_UID, FieldOverride, Target}; +use crate::{DATASOURCE_UID, FieldOverride, Target, unit::Unit}; #[derive(Clone, Copy, Serialize)] #[serde(rename_all = "lowercase")] @@ -245,7 +245,7 @@ pub struct Defaults { pub color: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub custom: Option, - pub unit: String, + pub unit: Unit, #[serde(default, skip_serializing_if = "Option::is_none")] pub decimals: Option, } diff --git a/tools/dashboard_gen/src/unit.rs b/tools/dashboard_gen/src/unit.rs new file mode 100644 index 00000000..adeebf3b --- /dev/null +++ b/tools/dashboard_gen/src/unit.rs @@ -0,0 +1,107 @@ +//! Panel value units. +//! +//! Grafana identifies a field's unit by a short id string (`"s"`, `"bytes"`, +//! `"reqps"`, …). We model the handful we actually use as named variants and +//! fall back to [`Unit::custom`] for anything else, so the value always +//! serializes to the exact id Grafana expects. + +use serde::{Serialize, Serializer}; + +/// A panel value unit. Serializes to Grafana's unit id string. +/// +/// Only the most common units are named; [`Unit::custom`] carries any other +/// Grafana unit id verbatim. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Unit { + /// Plain number, SI-abbreviated (`short`). Grafana's default. + Short, + /// Percentage on a 0–100 scale (`percent`). + Percent, + /// Percentage on a 0.0–1.0 scale (`percentunit`). + PercentUnit, + /// Seconds (`s`). + Seconds, + /// Milliseconds (`ms`). + Milliseconds, + /// Nanoseconds (`ns`). + Nanoseconds, + /// Bytes, IEC/binary (`bytes`). + Bytes, + /// Bytes per second, SI (`Bps`). + BytesPerSec, + /// Requests per second (`reqps`). + RequestsPerSec, + /// Operations per second (`ops`). + OpsPerSec, + /// Any other Grafana unit id, kept verbatim. + Custom(String), +} + +impl Unit { + /// Wrap an arbitrary Grafana unit id (e.g. `"dtdurationms"`, `"celsius"`). + #[must_use] + pub fn custom(id: impl Into) -> Self { + Self::Custom(id.into()) + } + + /// The Grafana unit id this value serializes to. + fn as_id(&self) -> &str { + match self { + Self::Short => "short", + Self::Percent => "percent", + Self::PercentUnit => "percentunit", + Self::Seconds => "s", + Self::Milliseconds => "ms", + Self::Nanoseconds => "ns", + Self::Bytes => "bytes", + Self::BytesPerSec => "Bps", + Self::RequestsPerSec => "reqps", + Self::OpsPerSec => "ops", + Self::Custom(id) => id, + } + } + + /// Reverse of [`Self::as_id`]: map a Grafana unit id back to a `Unit`, + /// falling back to [`Self::Custom`] for ids we don't name. Used by the + /// panel→Rust transpiler. + #[must_use] + pub(crate) fn from_id(id: &str) -> Self { + match id { + "short" => Self::Short, + "percent" => Self::Percent, + "percentunit" => Self::PercentUnit, + "s" => Self::Seconds, + "ms" => Self::Milliseconds, + "ns" => Self::Nanoseconds, + "bytes" => Self::Bytes, + "Bps" => Self::BytesPerSec, + "reqps" => Self::RequestsPerSec, + "ops" => Self::OpsPerSec, + other => Self::custom(other), + } + } + + /// The Rust builder expression that reconstructs this unit, for codegen. + #[must_use] + pub(crate) fn to_rust_source(&self) -> String { + match self { + Self::Custom(id) => format!("Unit::custom({id:?})"), + named @ (Self::Short + | Self::Percent + | Self::PercentUnit + | Self::Seconds + | Self::Milliseconds + | Self::Nanoseconds + | Self::Bytes + | Self::BytesPerSec + | Self::RequestsPerSec + | Self::OpsPerSec) => format!("Unit::{named:?}"), + } + } +} + +impl Serialize for Unit { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_id()) + } +} From ab12fb104c30b088b877085c1cd3d93cd64dce46 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 02:31:48 +0300 Subject: [PATCH 07/21] chore(monitoring): bump prometheus & grafana versions --- Justfile | 2 +- monitoring/docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Justfile b/Justfile index 1a51980e..021f6774 100644 --- a/Justfile +++ b/Justfile @@ -82,7 +82,7 @@ run-bedrock: # admin), Prometheus: http://localhost:9090. Scrapes the sequencer's /metrics. [working-directory: 'monitoring'] run-monitoring: - @echo "📊 Running Prometheus + Grafana" + @echo "📊 Running Prometheus (http://localhost:9090) + Grafana (http://localhost:3000)" docker compose up # Run Sequencer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml index 71953636..5d6ab935 100644 --- a/monitoring/docker-compose.yml +++ b/monitoring/docker-compose.yml @@ -1,7 +1,7 @@ # Prometheus + Grafana monitoring stack. services: prometheus: - image: prom/prometheus:v3.1.0 + image: prom/prometheus:v3.13.1 container_name: prometheus command: - --config.file=/etc/prometheus/prometheus.yml @@ -15,7 +15,7 @@ services: - "host.docker.internal:host-gateway" grafana: - image: grafana/grafana:11.4.0 + image: grafana/grafana:13.1.1 container_name: grafana ports: - "3000:3000" From 849bb6bbe1f040f0908b2c0ddaa053eddb58549c Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 17:06:18 +0300 Subject: [PATCH 08/21] feat(metrics): improve mempool and failed transaction metrics & panels --- lez/sequencer/core/metrics/src/names.rs | 1 + lez/sequencer/core/metrics/src/writing.rs | 9 + lez/sequencer/core/src/lib.rs | 1 + monitoring/grafana/dashboards/sequencer.json | 187 +++++++++++++----- .../src/bin/gen_sequencer_dashboard.rs | 90 +++++++-- tools/dashboard_gen/src/codegen.rs | 79 ++++++-- tools/dashboard_gen/src/input.rs | 7 + tools/dashboard_gen/src/lib.rs | 73 ++++++- tools/dashboard_gen/src/schema.rs | 63 ++++++ 9 files changed, 427 insertions(+), 83 deletions(-) diff --git a/lez/sequencer/core/metrics/src/names.rs b/lez/sequencer/core/metrics/src/names.rs index 1ea87c26..a8aae270 100644 --- a/lez/sequencer/core/metrics/src/names.rs +++ b/lez/sequencer/core/metrics/src/names.rs @@ -1,6 +1,7 @@ pub const BLOCK_CREATION_TIME: &str = "block_creation_time"; pub const BLOCK_COUNT: &str = "block_count"; pub const MEMPOOL_SIZE: &str = "mempool_size"; +pub const MEMPOOL_MAX_SIZE: &str = "mempool_max_size"; pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = "mempool_transaction_application_time"; pub const TRANSACTIONS_PER_BLOCK: &str = "transactions_per_block"; pub const FAILED_TRANSACTION_COUNT: &str = "failed_transaction_count"; diff --git a/lez/sequencer/core/metrics/src/writing.rs b/lez/sequencer/core/metrics/src/writing.rs index 2a392b84..320e01fb 100644 --- a/lez/sequencer/core/metrics/src/writing.rs +++ b/lez/sequencer/core/metrics/src/writing.rs @@ -51,6 +51,15 @@ pub fn record_mempool_size(size: usize) { .set(u64::try_from(size).expect("Mempool size should fit into u64") as f64); } +pub fn record_mempool_max_size(size: usize) { + gauge!( + description: "Configured maximum size of the mempool", + unit: Unit::Count, + names::MEMPOOL_MAX_SIZE + ) + .set(u64::try_from(size).expect("Mempool max size should fit into u64") as f64); +} + pub fn record_mempool_transaction_application_time( origin: TransactionOrigin, kind: TxKind, diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 7cbe8fa7..d20c46cc 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -224,6 +224,7 @@ impl SequencerCore { let is_fresh_start = initial_checkpoint.is_none(); let (mempool, mempool_handle) = MemPool::new(config.mempool_max_size); + sequencer_core_metrics::record_mempool_max_size(config.mempool_max_size); let block_publisher = BP::new( &config.bedrock_config, diff --git a/monitoring/grafana/dashboards/sequencer.json b/monitoring/grafana/dashboards/sequencer.json index 3cd0fbcc..c2711d73 100644 --- a/monitoring/grafana/dashboards/sequencer.json +++ b/monitoring/grafana/dashboards/sequencer.json @@ -19,7 +19,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_count_total", + "expr": "block_count", "legendFormat": "height", "refId": "A" } @@ -42,7 +42,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(block_count_total[1m]) * 60", + "expr": "rate(block_count[1m]) * 60", "legendFormat": "blocks/min", "refId": "A" } @@ -73,31 +73,31 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_creation_time_seconds{quantile=\"0.5\"}", + "expr": "block_creation_time{quantile=\"0.5\"}", "legendFormat": "p50", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_creation_time_seconds{quantile=\"0.9\"}", + "expr": "block_creation_time{quantile=\"0.9\"}", "legendFormat": "p90", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_creation_time_seconds{quantile=\"0.95\"}", + "expr": "block_creation_time{quantile=\"0.95\"}", "legendFormat": "p95", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_creation_time_seconds{quantile=\"0.99\"}", + "expr": "block_creation_time{quantile=\"0.99\"}", "legendFormat": "p99", "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(block_creation_time_seconds_sum[1m]) / rate(block_creation_time_seconds_count[1m])", + "expr": "rate(block_creation_time_sum[1m]) / rate(block_creation_time_count[1m])", "legendFormat": "avg", "refId": "E" } @@ -120,25 +120,25 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_transaction_application_time_seconds{quantile=\"0.5\"}", + "expr": "mempool_transaction_application_time{quantile=\"0.5\"}", "legendFormat": "p50 · {{kind}} · {{origin}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_transaction_application_time_seconds{quantile=\"0.9\"}", + "expr": "mempool_transaction_application_time{quantile=\"0.9\"}", "legendFormat": "p90 · {{kind}} · {{origin}}", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_transaction_application_time_seconds{quantile=\"0.95\"}", + "expr": "mempool_transaction_application_time{quantile=\"0.95\"}", "legendFormat": "p95 · {{kind}} · {{origin}}", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_transaction_application_time_seconds{quantile=\"0.99\"}", + "expr": "mempool_transaction_application_time{quantile=\"0.99\"}", "legendFormat": "p99 · {{kind}} · {{origin}}", "refId": "D" } @@ -146,32 +146,6 @@ "title": "Transaction application time", "type": "timeseries" }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "spanNulls": true }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 }, - "id": 5, - "options": { - "legend": { "displayMode": "list", "placement": "bottom", "calcs": [ "last", "max" ] }, - "tooltip": { "mode": "single" } - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_size", - "legendFormat": "mempool size", - "refId": "A" - } - ], - "title": "Mempool size", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { @@ -186,8 +160,8 @@ } ] }, - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 25 }, - "id": 6, + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 }, + "id": 5, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } @@ -230,7 +204,130 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { - "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" }, + "defaults": { + "unit": "percent", + "decimals": 1, + "min": 0.0, + "max": 100.0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 70.0 }, + { "color": "red", "value": 90.0 } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 25 }, + "id": 6, + "options": { + "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "100 * mempool_size / mempool_max_size", + "legendFormat": "utilization", + "refId": "A" + } + ], + "title": "Mempool utilization", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "spanNulls": true }, + "unit": "short", + "min": 0.0 + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "capacity" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "dash": [ 8, 4 ], "fill": "dash" } }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } + ] + }, + { + "matcher": { "id": "byName", "options": "queued" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] + } + ] + }, + "gridPos": { "h": 8, "w": 18, "x": 6, "y": 25 }, + "id": 7, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_size", + "legendFormat": "queued", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_max_size", + "legendFormat": "capacity", + "refId": "B" + } + ], + "title": "Mempool size vs capacity", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 2, + "min": 0.0, + "max": 100.0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1.0 }, + { "color": "red", "value": 5.0 } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 33 }, + "id": 8, + "options": { + "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "100 * increase(failed_transaction_count[$__range]) / clamp_min(increase(submitted_transaction_count[$__range]), 1)", + "legendFormat": "failed", + "refId": "A" + } + ], + "title": "Failed transactions share", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "unit": "short", + "min": 0.0 + }, "overrides": [ { "matcher": { "id": "byName", "options": "failed" }, @@ -242,8 +339,8 @@ } ] }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 25 }, - "id": 7, + "gridPos": { "h": 8, "w": 18, "x": 6, "y": 33 }, + "id": 9, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } @@ -251,18 +348,18 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(submitted_transaction_count_total[1m]) * 60", + "expr": "rate(submitted_transaction_count[1m]) * 60", "legendFormat": "submitted", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(failed_transaction_count_total[1m]) * 60", + "expr": "rate(failed_transaction_count[1m]) * 60", "legendFormat": "failed", "refId": "B" } ], - "title": "Transaction throughput (per minute)", + "title": "Submitted vs failed transactions (per minute)", "type": "timeseries" } ], diff --git a/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs b/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs index 4e0716ab..da44f508 100644 --- a/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs +++ b/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs @@ -10,8 +10,8 @@ )] use dashboard_gen::{ - Color, Dashboard, FieldOverride, Panel, Target, Unit, avg, percentiles, percentiles_labeled, - rate_per_min, + Color, Dashboard, FieldOverride, Panel, Target, Thresholds, Unit, avg, percentiles, + percentiles_labeled, rate_per_min, }; use json_pretty_compact::PrettyCompactFormatter; use serde::Serialize as _; @@ -68,19 +68,6 @@ fn sequencer_dashboard() -> Dashboard { PERCENTILES, " · {{kind}} · {{origin}}", )), - Panel::timeseries("Mempool size") - .width(12) - .unit(Unit::Short) - .span_nulls() - .target( - Target::new(sequencer_core_metrics::names::MEMPOOL_SIZE) - .legend("mempool size"), - ), - ], - ) - .row( - 9, - [ Panel::timeseries("Transactions per block") .width(12) .unit(Unit::Short) @@ -94,9 +81,78 @@ fn sequencer_dashboard() -> Dashboard { .dashed_line() .color(Color::fixed("text")), ), - Panel::timeseries("Transaction throughput (per minute)") - .width(12) + ], + ) + .row( + 8, + [ + Panel::gauge("Mempool utilization") + .width(6) + .unit(Unit::Percent) + .decimals(1) + .min(0.0) + .max(100.0) + .thresholds( + Thresholds::base("green") + .step(70.0, "orange") + .step(90.0, "red"), + ) + .target( + Target::new(format!( + "100 * {size} / {max_size}", + size = sequencer_core_metrics::names::MEMPOOL_SIZE, + max_size = sequencer_core_metrics::names::MEMPOOL_MAX_SIZE, + )) + .legend("utilization"), + ), + Panel::timeseries("Mempool size vs capacity") + .width(18) .unit(Unit::Short) + .span_nulls() + .min(0.0) + .target( + Target::new(sequencer_core_metrics::names::MEMPOOL_SIZE).legend("queued"), + ) + .target( + Target::new(sequencer_core_metrics::names::MEMPOOL_MAX_SIZE) + .legend("capacity"), + ) + .with_override( + FieldOverride::by_name("capacity") + .dashed_line() + .color(Color::fixed("red")), + ) + .with_override(FieldOverride::by_name("queued").color(Color::fixed("blue"))), + ], + ) + .row( + 8, + [ + Panel::gauge("Failed transactions share") + .width(6) + .unit(Unit::Percent) + .decimals(2) + .min(0.0) + .max(100.0) + .thresholds( + Thresholds::base("green") + .step(1.0, "orange") + .step(5.0, "red"), + ) + .target( + Target::new(format!( + // `clamp_min` keeps an idle window (nothing submitted) + // reading as 0% instead of a division by zero. + "100 * increase({failed}[$__range]) / clamp_min(increase({submitted}[$__range]), 1)", + failed = sequencer_core_metrics::names::FAILED_TRANSACTION_COUNT, + submitted = sequencer_service_metrics::names::SUBMITTED_TRANSACTION_COUNT, + )) + .legend("failed"), + ), + Panel::timeseries("Submitted vs failed transactions (per minute)") + .width(18) + .unit(Unit::Short) + .min(0.0) .target(rate_per_min( sequencer_service_metrics::names::SUBMITTED_TRANSACTION_COUNT, "submitted", diff --git a/tools/dashboard_gen/src/codegen.rs b/tools/dashboard_gen/src/codegen.rs index 15f79cb4..b554ffd6 100644 --- a/tools/dashboard_gen/src/codegen.rs +++ b/tools/dashboard_gen/src/codegen.rs @@ -11,9 +11,10 @@ use std::fmt::Write as _; use crate::{ Unit, - input::PanelInput, + input::{Defaults, PanelInput}, schema::{ AxisPlacement, Color, GradientMode, LineInterpolation, PanelType, ShowPoints, StackingMode, + ThresholdMode, Thresholds, }, }; @@ -33,25 +34,12 @@ fn panel_expr_inner(panel: &PanelInput) -> Result { let mut expr = match panel.panel_type { PanelType::Stat => format!("Panel::stat({:?})", panel.title), PanelType::Timeseries => format!("Panel::timeseries({:?})", panel.title), + PanelType::Gauge => format!("Panel::gauge({:?})", panel.title), }; write!(expr, "\n .width({})", panel.grid_pos.w)?; let defaults = &panel.field_config.defaults; - // `short` is the builder's own default unit, so it round-trips without a call. - if let Some(unit) = defaults.unit.as_deref().filter(|u| *u != "short") { - write!( - expr, - "\n .unit({})", - Unit::from_id(unit).to_rust_source() - )?; - } - if let Some(decimals) = defaults.decimals { - write!(expr, "\n .decimals({decimals})")?; - } - // Only a fixed color is worth emitting; `palette-classic` is Grafana's default. - if let Some(Color::Fixed { fixed_color }) = &defaults.color { - write!(expr, "\n .color(Color::fixed({fixed_color:?}))")?; - } + write_defaults(&mut expr, defaults)?; if let Some(custom) = &defaults.custom { if custom.span_nulls == Some(true) { @@ -146,6 +134,65 @@ fn panel_expr_inner(panel: &PanelInput) -> Result { Ok(expr) } +/// The field-level setters — everything outside the `custom` styling block. +fn write_defaults(expr: &mut String, defaults: &Defaults) -> Result<(), std::fmt::Error> { + // `short` is the builder's own default unit, so it round-trips without a call. + if let Some(unit) = defaults.unit.as_deref().filter(|u| *u != "short") { + write!( + expr, + "\n .unit({})", + Unit::from_id(unit).to_rust_source() + )?; + } + if let Some(decimals) = defaults.decimals { + write!(expr, "\n .decimals({decimals})")?; + } + // Only a fixed color is worth emitting; `palette-classic` is Grafana's default. + if let Some(Color::Fixed { fixed_color }) = &defaults.color { + write!(expr, "\n .color(Color::fixed({fixed_color:?}))")?; + } + if let Some(min) = defaults.min { + write!(expr, "\n .min({min:?})")?; + } + if let Some(max) = defaults.max { + write!(expr, "\n .max({max:?})")?; + } + + let ladder = defaults + .thresholds + .as_ref() + .filter(|thresholds| is_expressible_ladder(thresholds)) + .and_then(|thresholds| thresholds.steps.split_first()); + if let Some((base, steps)) = ladder { + write!(expr, "\n .thresholds(Thresholds::base({:?})", base.color)?; + for step in steps { + // A non-base step without a value is nonsense Grafana wouldn't render. + if let Some(value) = step.value { + write!(expr, ".step({value:?}, {:?})", step.color)?; + } + } + expr.push(')'); + } + + Ok(()) +} + +/// Whether a ladder is worth emitting: `percentage` mode is beyond the builder, +/// and green/red-at-80 is the ladder Grafana attaches to every panel by default. +fn is_expressible_ladder(thresholds: &Thresholds) -> bool { + if thresholds.mode != ThresholdMode::Absolute { + return false; + } + !matches!( + thresholds.steps.as_slice(), + [base, red] + if base.color == "green" + && base.value.is_none() + && red.color == "red" + && red.value == Some(80.0) + ) +} + const fn line_interp(value: LineInterpolation) -> &'static str { match value { LineInterpolation::Linear => "Linear", diff --git a/tools/dashboard_gen/src/input.rs b/tools/dashboard_gen/src/input.rs index e35a3ca3..ea3c4f78 100644 --- a/tools/dashboard_gen/src/input.rs +++ b/tools/dashboard_gen/src/input.rs @@ -14,6 +14,7 @@ use serde_json::Value; use crate::schema::{ AxisPlacement, Color, GradientMode, LineInterpolation, PanelType, ShowPoints, StackingMode, + Thresholds, }; #[derive(Deserialize)] @@ -54,6 +55,12 @@ pub struct Defaults { pub color: Option, #[serde(default)] pub custom: Option, + #[serde(default)] + pub min: Option, + #[serde(default)] + pub max: Option, + #[serde(default)] + pub thresholds: Option, } #[derive(Deserialize, Default)] diff --git a/tools/dashboard_gen/src/lib.rs b/tools/dashboard_gen/src/lib.rs index 918061a0..d1fb38c9 100644 --- a/tools/dashboard_gen/src/lib.rs +++ b/tools/dashboard_gen/src/lib.rs @@ -14,12 +14,15 @@ // Styling vocabularies passed to the optional `styling` setters are part of the // public API (and are used by this module's `Panel` fields and `finalize`). pub use codegen::panel_to_rust_source; -pub use schema::{AxisPlacement, Color, GradientMode, LineInterpolation, ShowPoints, StackingMode}; +pub use schema::{ + AxisPlacement, Color, GradientMode, LineInterpolation, ShowPoints, StackingMode, Thresholds, +}; use schema::{ - Calc, Custom, Datasource, Defaults, DrawStyle, EmptyList, FieldConfig, Fill, GraphMode, - GridPos, Legend, LegendDisplay, LineStyle, Matcher, MatcherKind, Options, OverrideProperty, - PanelModel, PanelType, Placement, PropertyId, PropertyValue, ReduceOptions, SortOrder, - Stacking, StatColorMode, StatOptions, TimeRange, TimeSeriesOptions, Tooltip, TooltipMode, + Calc, Custom, Datasource, Defaults, DrawStyle, EmptyList, FieldConfig, Fill, GaugeOptions, + GraphMode, GridPos, Legend, LegendDisplay, LineStyle, Matcher, MatcherKind, Options, + OverrideProperty, PanelModel, PanelType, Placement, PropertyId, PropertyValue, ReduceOptions, + SortOrder, Stacking, StatColorMode, StatOptions, TimeRange, TimeSeriesOptions, Tooltip, + TooltipMode, }; use serde::Serialize; pub use unit::Unit; @@ -107,6 +110,7 @@ impl FieldOverride { enum Kind { Stat, TimeSeries, + Gauge, } /// A dashboard panel builder. Grid position and panel id are assigned by @@ -119,6 +123,9 @@ pub struct Panel { unit: Option, decimals: Option, color: Option, + min: Option, + max: Option, + thresholds: Option, span_nulls: bool, overrides: Vec, // Optional timeseries styling, set via the `styling` setters. @@ -140,6 +147,9 @@ impl Panel { unit: None, decimals: None, color: None, + min: None, + max: None, + thresholds: None, span_nulls: false, overrides: Vec::new(), line_interpolation: None, @@ -161,6 +171,12 @@ impl Panel { Self::new(title, Kind::TimeSeries) } + /// A radial gauge panel. Pair it with [`Panel::min`]/[`Panel::max`] — the + /// dial needs a range to fill — and [`Panel::thresholds`] for its coloring. + pub fn gauge(title: impl Into) -> Self { + Self::new(title, Kind::Gauge) + } + /// Grid width in Grafana's 24-column units. Unset panels split the row's /// remaining width evenly. #[must_use] @@ -187,6 +203,26 @@ impl Panel { self } + /// Lower bound of the value scale. + #[must_use] + pub const fn min(mut self, min: f64) -> Self { + self.min = Some(min); + self + } + + /// Upper bound of the value scale. + #[must_use] + pub const fn max(mut self, max: f64) -> Self { + self.max = Some(max); + self + } + + #[must_use] + pub fn thresholds(mut self, thresholds: Thresholds) -> Self { + self.thresholds = Some(thresholds); + self + } + #[must_use] pub const fn span_nulls(mut self) -> Self { self.span_nulls = true; @@ -230,6 +266,9 @@ impl Panel { custom: None, unit, decimals: self.decimals, + min: self.min, + max: self.max, + thresholds: self.thresholds, }; let options = Options::Stat(StatOptions { color_mode: StatColorMode::Value, @@ -242,6 +281,27 @@ impl Panel { }); (defaults, options, PanelType::Stat) } + Kind::Gauge => { + let defaults = Defaults { + color: self.color, + custom: None, + unit, + decimals: self.decimals, + min: self.min, + max: self.max, + thresholds: self.thresholds, + }; + let options = Options::Gauge(GaugeOptions { + reduce_options: ReduceOptions { + calcs: vec![Calc::LastNotNull], + fields: String::new(), + values: false, + }, + show_threshold_labels: false, + show_threshold_markers: true, + }); + (defaults, options, PanelType::Gauge) + } Kind::TimeSeries => { let defaults = Defaults { color: None, @@ -262,6 +322,9 @@ impl Panel { }), unit, decimals: None, + min: self.min, + max: self.max, + thresholds: self.thresholds, }; // Panels with several series read better as a sortable table with // a multi-series tooltip; single-series panels stay compact. diff --git a/tools/dashboard_gen/src/schema.rs b/tools/dashboard_gen/src/schema.rs index 88d24bd8..009c7c40 100644 --- a/tools/dashboard_gen/src/schema.rs +++ b/tools/dashboard_gen/src/schema.rs @@ -94,6 +94,54 @@ pub enum SortOrder { pub enum PanelType { Stat, Timeseries, + Gauge, +} + +#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ThresholdMode { + Absolute, + /// Steps expressed as a percentage of the min–max range. The builder never + /// emits this; it exists so Grafana exports using it still parse. + Percentage, +} + +/// One threshold step: the color values at or above `value` take. The base step +/// carries `value: null` — Grafana's "everything below the first threshold". +#[derive(Clone, Serialize, Deserialize)] +pub struct ThresholdStep { + pub color: String, + pub value: Option, +} + +/// A threshold ladder, driving gauge/stat coloring. +#[derive(Clone, Serialize, Deserialize)] +pub struct Thresholds { + pub mode: ThresholdMode, + pub steps: Vec, +} + +impl Thresholds { + /// Start a ladder with the color used below every threshold. + pub fn base(color: impl Into) -> Self { + Self { + mode: ThresholdMode::Absolute, + steps: vec![ThresholdStep { + color: color.into(), + value: None, + }], + } + } + + /// Add a step: values at or above `value` render in `color`. + #[must_use] + pub fn step(mut self, value: f64, color: impl Into) -> Self { + self.steps.push(ThresholdStep { + color: color.into(), + value: Some(value), + }); + self + } } // Optional timeseries styling vocabularies. Each derives `PartialEq` so the @@ -248,6 +296,12 @@ pub struct Defaults { pub unit: Unit, #[serde(default, skip_serializing_if = "Option::is_none")] pub decimals: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thresholds: Option, } #[derive(Serialize)] @@ -293,11 +347,20 @@ pub struct TimeSeriesOptions { pub tooltip: Tooltip, } +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GaugeOptions { + pub reduce_options: ReduceOptions, + pub show_threshold_labels: bool, + pub show_threshold_markers: bool, +} + #[derive(Serialize)] #[serde(untagged)] pub enum Options { Stat(StatOptions), TimeSeries(TimeSeriesOptions), + Gauge(GaugeOptions), } #[derive(Clone, Copy, Serialize)] From aa6969386ec9aba8bfe0c05ca2c0688ce260c497 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 17:04:44 +0300 Subject: [PATCH 09/21] refactor(metrics): improve metrics setup --- .gitignore | 1 + lez/common/src/transaction.rs | 1 + lez/sequencer/core/metrics/src/names.rs | 9 +- lez/sequencer/core/metrics/src/writing.rs | 80 +++++++---- lez/sequencer/core/src/lib.rs | 8 +- lez/sequencer/service/metrics/src/names.rs | 4 +- lez/sequencer/service/metrics/src/writing.rs | 28 +++- lez/sequencer/service/src/lib.rs | 2 + lez/sequencer/service/src/main.rs | 36 ++++- lez/sequencer/service/src/service.rs | 85 ++++++----- monitoring/grafana/dashboards/sequencer.json | 126 +++++++--------- .../src/bin/gen_sequencer_dashboard.rs | 54 ++++--- tools/dashboard_gen/src/codegen.rs | 7 +- tools/dashboard_gen/src/input.rs | 2 + tools/dashboard_gen/src/lib.rs | 134 ++++++++++++++---- tools/dashboard_gen/src/schema.rs | 37 +++++ tools/dashboard_gen/src/styling.rs | 20 ++- 17 files changed, 434 insertions(+), 200 deletions(-) diff --git a/.gitignore b/.gitignore index f32b258c..3befdb9c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ data/ rocksdb* sequencer/service/data/ storage.json +statistics.json result diff --git a/lez/common/src/transaction.rs b/lez/common/src/transaction.rs index 3c4df83c..22d4b70e 100644 --- a/lez/common/src/transaction.rs +++ b/lez/common/src/transaction.rs @@ -219,6 +219,7 @@ impl From for LeeTransaction { BorshSerialize, BorshDeserialize, strum::IntoStaticStr, + strum::EnumIter, )] pub enum TxKind { Public, diff --git a/lez/sequencer/core/metrics/src/names.rs b/lez/sequencer/core/metrics/src/names.rs index a8aae270..98034cd6 100644 --- a/lez/sequencer/core/metrics/src/names.rs +++ b/lez/sequencer/core/metrics/src/names.rs @@ -1,7 +1,8 @@ -pub const BLOCK_CREATION_TIME: &str = "block_creation_time"; -pub const BLOCK_COUNT: &str = "block_count"; +pub const BLOCK_CREATION_TIME: &str = "block_creation_time_seconds"; +pub const BLOCKS_TOTAL: &str = "blocks_total"; pub const MEMPOOL_SIZE: &str = "mempool_size"; pub const MEMPOOL_MAX_SIZE: &str = "mempool_max_size"; -pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = "mempool_transaction_application_time"; +pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = + "mempool_transaction_application_time_seconds"; pub const TRANSACTIONS_PER_BLOCK: &str = "transactions_per_block"; -pub const FAILED_TRANSACTION_COUNT: &str = "failed_transaction_count"; +pub const MEMPOOL_FAILED_TRANSACTIONS_TOTAL: &str = "mempool_failed_transactions_total"; diff --git a/lez/sequencer/core/metrics/src/writing.rs b/lez/sequencer/core/metrics/src/writing.rs index 320e01fb..1f6131f6 100644 --- a/lez/sequencer/core/metrics/src/writing.rs +++ b/lez/sequencer/core/metrics/src/writing.rs @@ -7,39 +7,58 @@ use std::time::Duration; use common::transaction::TxKind; -use metrics::{Counter, Unit, counter, gauge, histogram}; +use metrics::{Counter, Histogram, Unit, counter, gauge, histogram}; +use strum::IntoEnumIterator as _; use crate::names; -#[derive(Clone, Copy, strum::IntoStaticStr)] +#[derive(Clone, Copy, strum::IntoStaticStr, strum::EnumIter)] pub enum TransactionOrigin { User, Sequencer, } -pub fn record_block_creation_time(duration: Duration) { +/// Initialize metrics. +pub fn init() { + blocks_total_counter().increment(0); + mempool_failed_transactions_total_counter().increment(0); + record_mempool_size(0); + + drop(block_creation_time_histogram()); + drop(transactions_per_block_histogram()); + for origin in TransactionOrigin::iter() { + for kind in TxKind::iter() { + drop(mempool_transaction_application_time_histogram(origin, kind)); + } + } +} + +fn block_creation_time_histogram() -> Histogram { histogram!( description: "Time taken to create a block", unit: Unit::Seconds, names::BLOCK_CREATION_TIME ) - .record(duration.as_secs_f64()); } -fn block_count_counter() -> Counter { +pub fn record_block_creation_time(duration: Duration) { + block_creation_time_histogram().record(duration.as_secs_f64()); +} + +fn blocks_total_counter() -> Counter { counter!( description: "Number of blocks in chain", unit: Unit::Count, - names::BLOCK_COUNT + names::BLOCKS_TOTAL ) } -pub fn set_block_count(value: u64) { - block_count_counter().absolute(value); +pub fn set_blocks_total(value: u64) { + blocks_total_counter().absolute(value); } -pub fn increment_block_count() { - block_count_counter().increment(1); +pub fn increment_blocks_total() { + blocks_total_counter().increment(1); } pub fn record_mempool_size(size: usize) { @@ -60,11 +79,10 @@ pub fn record_mempool_max_size(size: usize) { .set(u64::try_from(size).expect("Mempool max size should fit into u64") as f64); } -pub fn record_mempool_transaction_application_time( +fn mempool_transaction_application_time_histogram( origin: TransactionOrigin, kind: TxKind, - duration: Duration, -) { +) -> Histogram { histogram!( description: "Time taken to apply a mempool transaction", unit: Unit::Seconds, @@ -72,23 +90,37 @@ pub fn record_mempool_transaction_application_time( "origin" => <&'static str>::from(origin), "kind" => <&'static str>::from(kind), ) - .record(duration.as_secs_f64()); } -pub fn record_transactions_per_block(count: usize) { +pub fn record_mempool_transaction_application_time( + origin: TransactionOrigin, + kind: TxKind, + duration: Duration, +) { + mempool_transaction_application_time_histogram(origin, kind).record(duration.as_secs_f64()); +} + +fn transactions_per_block_histogram() -> Histogram { histogram!( - description: "Number of transactions included in block", + description: "Number of transactions from mempool included in block", unit: Unit::Count, names::TRANSACTIONS_PER_BLOCK ) - .record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64); } -pub fn increment_failed_transaction_count() { - counter!( - description: "Number of transactions that failed to be included in blocks", - unit: Unit::Count, - names::FAILED_TRANSACTION_COUNT - ) - .increment(1); +pub fn record_transactions_per_block(count: usize) { + transactions_per_block_histogram() + .record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64); +} + +fn mempool_failed_transactions_total_counter() -> Counter { + counter!( + description: "Number of transactions from mempool that failed to be included in blocks", + unit: Unit::Count, + names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL + ) +} + +pub fn increment_mempool_failed_transactions_total() { + mempool_failed_transactions_total_counter().increment(1); } diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index d20c46cc..ca079ab7 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -204,6 +204,8 @@ impl SequencerCore { pub async fn start_from_config( config: SequencerConfig, ) -> (Self, MemPoolHandle<(TransactionOrigin, LeeTransaction)>) { + sequencer_core_metrics::init(); + let bedrock_signing_key = load_or_create_signing_key(&config.home.join("bedrock_signing_key")) .expect("Failed to load or create bedrock signing key"); @@ -309,7 +311,7 @@ impl SequencerCore { watchers, }; - sequencer_core_metrics::set_block_count(sequencer_core.chain_height()); + sequencer_core_metrics::set_blocks_total(sequencer_core.chain_height()); (sequencer_core, mempool_handle) } @@ -903,7 +905,7 @@ impl SequencerCore { if applied { valid_transactions.push(tx); } else { - sequencer_core_metrics::increment_failed_transaction_count(); + sequencer_core_metrics::increment_mempool_failed_transactions_total(); // A failed transaction is simply left out of the block, except a // dispatch: that one is re-fed from the store every turn, so one // that can never execute would fail on every block for ever. @@ -939,7 +941,7 @@ impl SequencerCore { ); sequencer_core_metrics::record_block_creation_time(now.elapsed()); - sequencer_core_metrics::increment_block_count(); + sequencer_core_metrics::increment_blocks_total(); Ok(BlockWithMeta { block, withdrawals }) } diff --git a/lez/sequencer/service/metrics/src/names.rs b/lez/sequencer/service/metrics/src/names.rs index 8bf61acb..1ac19948 100644 --- a/lez/sequencer/service/metrics/src/names.rs +++ b/lez/sequencer/service/metrics/src/names.rs @@ -1 +1,3 @@ -pub const SUBMITTED_TRANSACTION_COUNT: &str = "submitted_transaction_count"; +pub const SUBMITTED_TRANSACTIONS_TOTAL: &str = "submitted_transactions_total"; +pub const BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL: &str = + "before_mempool_failed_transactions_total"; diff --git a/lez/sequencer/service/metrics/src/writing.rs b/lez/sequencer/service/metrics/src/writing.rs index 3b625006..05eaac1a 100644 --- a/lez/sequencer/service/metrics/src/writing.rs +++ b/lez/sequencer/service/metrics/src/writing.rs @@ -1,12 +1,32 @@ -use metrics::{Unit, counter}; +use metrics::{Counter, Unit, counter}; use crate::names; -pub fn increment_submitted_transaction_count() { +pub fn init() { + submitted_transactions_total_counter().increment(0); + before_mempool_failed_transactions_total_counter().increment(0); +} + +fn submitted_transactions_total_counter() -> Counter { counter!( description: "Number of transactions submitted", unit: Unit::Count, - names::SUBMITTED_TRANSACTION_COUNT + names::SUBMITTED_TRANSACTIONS_TOTAL ) - .increment(1); +} + +pub fn increment_submitted_transactions_total() { + submitted_transactions_total_counter().increment(1); +} + +fn before_mempool_failed_transactions_total_counter() -> Counter { + counter!( + description: "Number of transactions that failed before reaching the mempool", + unit: Unit::Count, + names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL + ) +} + +pub fn increment_before_mempool_failed_transactions_total() { + before_mempool_failed_transactions_total_counter().increment(1); } diff --git a/lez/sequencer/service/src/lib.rs b/lez/sequencer/service/src/lib.rs index ba5b68ec..3073823a 100644 --- a/lez/sequencer/service/src/lib.rs +++ b/lez/sequencer/service/src/lib.rs @@ -211,6 +211,8 @@ async fn wait_for_store_release(store: &StoreRelease) { } pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result { + sequencer_service_metrics::init(); + let block_timeout = config.block_create_timeout; let max_block_size = config.max_block_size; diff --git a/lez/sequencer/service/src/main.rs b/lez/sequencer/service/src/main.rs index 95facfff..e648bd70 100644 --- a/lez/sequencer/service/src/main.rs +++ b/lez/sequencer/service/src/main.rs @@ -6,6 +6,7 @@ use std::{ use anyhow::{Context as _, Result}; use clap::Parser; use log::{error, info}; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder}; use tokio::signal::unix::{SignalKind, signal}; use tokio_util::sync::CancellationToken; @@ -34,10 +35,7 @@ struct Args { async fn main() -> Result<()> { env_logger::init(); - metrics_exporter_prometheus::PrometheusBuilder::new() - .with_recommended_naming(true) - .install() - .context("Failed to install Prometheus recorder")?; + install_prometheus_recorder()?; let Args { config_path, @@ -76,6 +74,36 @@ async fn main() -> Result<()> { Ok(()) } +/// Installs the recorder with explicit buckets, which makes every histogram +/// export as a Prometheus histogram (`_bucket`/`_sum`/`_count`) rather than the +/// default rolling-window summary. Summary quantiles reset to zero once their +/// window drains, so an idle period reads as "took 0s" instead of "no data". +/// +/// Ladders are picked by name suffix, so a new timing metric is covered without +/// touching this function. The matcher sees the name as registered, *before* +/// [`PrometheusBuilder::with_recommended_naming`] appends a unit suffix of its +/// own — a duration metric whose name omits `_seconds` silently falls through to +/// [`COUNT_BUCKETS`]. +fn install_prometheus_recorder() -> Result<()> { + /// Ladder for `*_seconds` histograms, densest across the 1–100 ms band where + /// block production and transaction application actually land. + const LATENCY_BUCKETS: &[f64] = &[ + 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ]; + + /// Fallback ladder for histograms that count things rather than measure time. + const COUNT_BUCKETS: &[f64] = &[1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0]; + + PrometheusBuilder::new() + .with_recommended_naming(true) + .set_buckets(COUNT_BUCKETS) + .context("Failed to set default histogram buckets")? + .set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), LATENCY_BUCKETS) + .context("Failed to set latency histogram buckets")? + .install() + .context("Failed to install Prometheus recorder") +} + /// Cancelled on Ctrl-C or `SIGTERM`. /// /// `SIGTERM` is what a container runtime sends first, so without it every diff --git a/lez/sequencer/service/src/service.rs b/lez/sequencer/service/src/service.rs index 309c9949..e55735c0 100644 --- a/lez/sequencer/service/src/service.rs +++ b/lez/sequencer/service/src/service.rs @@ -6,7 +6,7 @@ use jsonrpsee::{ types::{ErrorCode, ErrorObjectOwned}, }; use lee; -use log::warn; +use log::{error, warn}; use mempool::MemPoolHandle; use sequencer_core::{ DbError, SequencerCore, TransactionOrigin, block_publisher::BlockPublisherTrait, @@ -44,57 +44,68 @@ impl sequencer_service_rpc::Rpc for SequencerService { async fn send_transaction(&self, tx: LeeTransaction) -> Result { - // Reserve ~200 bytes for block header overhead - const BLOCK_HEADER_OVERHEAD: u64 = 200; + sequencer_service_metrics::increment_submitted_transactions_total(); let tx_hash = tx.hash(); - let encoded_tx = - borsh::to_vec(&tx).expect("Transaction borsh serialization should not fail"); - let tx_size = u64::try_from(encoded_tx.len()).expect("Transaction size should fit in u64"); + let res = async move { + // Reserve ~200 bytes for block header overhead + const BLOCK_HEADER_OVERHEAD: u64 = 200; - let max_tx_size = self.max_block_size.saturating_sub(BLOCK_HEADER_OVERHEAD); + let encoded_tx = + borsh::to_vec(&tx).expect("Transaction borsh serialization should not fail"); + let tx_size = + u64::try_from(encoded_tx.len()).expect("Transaction size should fit in u64"); - if tx_size > max_tx_size { - return Err(ErrorObjectOwned::owned( - ErrorCode::InvalidParams.code(), - format!("Transaction too large: size {tx_size}, max {max_tx_size}"), - None::<()>, - )); - } + let max_tx_size = self.max_block_size.saturating_sub(BLOCK_HEADER_OVERHEAD); - let authenticated_tx = tx - .transaction_stateless_check() - .inspect_err(|err| warn!("Error at pre_check {err:#?}")) - .map_err(|err| { - ErrorObjectOwned::owned( + if tx_size > max_tx_size { + return Err(ErrorObjectOwned::owned( ErrorCode::InvalidParams.code(), - format!("{err:?}"), + format!("Transaction too large: size {tx_size}, max {max_tx_size}"), None::<()>, - ) - })?; + )); + } - // Sequencer-only programs (the cross-zone inbox) are injected by the - // watcher; a user must not invoke them top-level, or anyone could forge - // an inbound cross-zone delivery. Chained user calls are already rejected - // by the inbox guest's caller-is-none assertion. - if let LeeTransaction::Public(public_tx) = &authenticated_tx - && sequencer_core::is_sequencer_only_program(public_tx.message().program_id) - { - return Err(ErrorObjectOwned::owned( - ErrorCode::InvalidParams.code(), - "Program is sequencer-only and cannot be invoked by a user transaction".to_owned(), - None::<()>, - )); - } + let authenticated_tx = tx + .transaction_stateless_check() + .inspect_err(|err| warn!("Error at pre_check {err:#?}")) + .map_err(|err| { + ErrorObjectOwned::owned( + ErrorCode::InvalidParams.code(), + format!("{err:?}"), + None::<()>, + ) + })?; + + // Sequencer-only programs (the cross-zone inbox) are injected by the + // watcher; a user must not invoke them top-level, or anyone could forge + // an inbound cross-zone delivery. Chained user calls are already rejected + // by the inbox guest's caller-is-none assertion. + if let LeeTransaction::Public(public_tx) = &authenticated_tx + && sequencer_core::is_sequencer_only_program(public_tx.message().program_id) + { + return Err(ErrorObjectOwned::owned( + ErrorCode::InvalidParams.code(), + "Program is sequencer-only and cannot be invoked by a user transaction" + .to_owned(), + None::<()>, + )); + } + + Ok(authenticated_tx) + }; + + let authenticated_tx = res.await.inspect_err(|err| { + sequencer_service_metrics::increment_before_mempool_failed_transactions_total(); + error!("Transaction failed before reaching mempool: {err:#?}"); + })?; self.mempool_handle .push((TransactionOrigin::User, authenticated_tx)) .await .expect("Mempool is closed, this is a bug"); - sequencer_service_metrics::increment_submitted_transaction_count(); - Ok(tx_hash) } diff --git a/monitoring/grafana/dashboards/sequencer.json b/monitoring/grafana/dashboards/sequencer.json index c2711d73..20627045 100644 --- a/monitoring/grafana/dashboards/sequencer.json +++ b/monitoring/grafana/dashboards/sequencer.json @@ -19,7 +19,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_count", + "expr": "blocks_total", "legendFormat": "height", "refId": "A" } @@ -42,7 +42,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(block_count[1m]) * 60", + "expr": "rate(blocks_total[1m]) * 60", "legendFormat": "blocks/min", "refId": "A" } @@ -73,33 +73,15 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_creation_time{quantile=\"0.5\"}", - "legendFormat": "p50", + "expr": "histogram_quantile(${percentile}, sum by (le) (rate(block_creation_time_seconds_bucket[$__rate_interval])))", + "legendFormat": "${percentile:text}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_creation_time{quantile=\"0.9\"}", - "legendFormat": "p90", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_creation_time{quantile=\"0.95\"}", - "legendFormat": "p95", - "refId": "C" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_creation_time{quantile=\"0.99\"}", - "legendFormat": "p99", - "refId": "D" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(block_creation_time_sum[1m]) / rate(block_creation_time_count[1m])", + "expr": "rate(block_creation_time_seconds_sum[$__rate_interval]) / rate(block_creation_time_seconds_count[$__rate_interval])", "legendFormat": "avg", - "refId": "E" + "refId": "B" } ], "title": "Block creation time", @@ -120,27 +102,9 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_transaction_application_time{quantile=\"0.5\"}", - "legendFormat": "p50 · {{kind}} · {{origin}}", + "expr": "histogram_quantile(${percentile}, sum by (le, kind, origin) (rate(mempool_transaction_application_time_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{kind}} · {{origin}}", "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_transaction_application_time{quantile=\"0.9\"}", - "legendFormat": "p90 · {{kind}} · {{origin}}", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_transaction_application_time{quantile=\"0.95\"}", - "legendFormat": "p95 · {{kind}} · {{origin}}", - "refId": "C" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_transaction_application_time{quantile=\"0.99\"}", - "legendFormat": "p99 · {{kind}} · {{origin}}", - "refId": "D" } ], "title": "Transaction application time", @@ -169,33 +133,15 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transactions_per_block{quantile=\"0.5\"}", - "legendFormat": "p50", + "expr": "histogram_quantile(${percentile}, sum by (le) (rate(transactions_per_block_bucket[$__rate_interval])))", + "legendFormat": "${percentile:text}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transactions_per_block{quantile=\"0.9\"}", - "legendFormat": "p90", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transactions_per_block{quantile=\"0.95\"}", - "legendFormat": "p95", - "refId": "C" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transactions_per_block{quantile=\"0.99\"}", - "legendFormat": "p99", - "refId": "D" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(transactions_per_block_sum[1m]) / rate(transactions_per_block_count[1m])", + "expr": "rate(transactions_per_block_sum[$__rate_interval]) / rate(transactions_per_block_count[$__rate_interval])", "legendFormat": "avg", - "refId": "E" + "refId": "B" } ], "title": "Transactions per block", @@ -312,7 +258,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "100 * increase(failed_transaction_count[$__range]) / clamp_min(increase(submitted_transaction_count[$__range]), 1)", + "expr": "100 * (increase(before_mempool_failed_transactions_total[$__range]) + increase(mempool_failed_transactions_total[$__range])) / clamp_min(increase(submitted_transactions_total[$__range]), 1)", "legendFormat": "failed", "refId": "A" } @@ -324,18 +270,22 @@ "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { - "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 35, "gradientMode": "opacity" }, "unit": "short", "min": 0.0 }, "overrides": [ - { - "matcher": { "id": "byName", "options": "failed" }, - "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] - }, { "matcher": { "id": "byName", "options": "submitted" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] + }, + { + "matcher": { "id": "byName", "options": "failed · before mempool" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] + }, + { + "matcher": { "id": "byName", "options": "failed · in mempool" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] } ] }, @@ -348,15 +298,21 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(submitted_transaction_count[1m]) * 60", + "expr": "rate(submitted_transactions_total[1m]) * 60", "legendFormat": "submitted", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(failed_transaction_count[1m]) * 60", - "legendFormat": "failed", + "expr": "rate(before_mempool_failed_transactions_total[1m]) * 60", + "legendFormat": "failed · before mempool", "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(mempool_failed_transactions_total[1m]) * 60", + "legendFormat": "failed · in mempool", + "refId": "C" } ], "title": "Submitted vs failed transactions (per minute)", @@ -366,7 +322,25 @@ "refresh": "5s", "schemaVersion": 39, "tags": [ "sequencer" ], - "templating": { "list": [ ] }, + "templating": { + "list": [ + { + "current": { "selected": true, "text": "p95", "value": "0.95" }, + "includeAll": false, + "label": "Percentile", + "multi": false, + "name": "percentile", + "options": [ + { "selected": false, "text": "p50", "value": "0.5" }, + { "selected": false, "text": "p90", "value": "0.9" }, + { "selected": true, "text": "p95", "value": "0.95" }, + { "selected": false, "text": "p99", "value": "0.99" } + ], + "query": "p50 : 0.5, p90 : 0.9, p95 : 0.95, p99 : 0.99", + "type": "custom" + } + ] + }, "time": { "from": "now-15m", "to": "now" }, "timezone": "", "title": "Sequencer", diff --git a/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs b/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs index da44f508..e2f7df78 100644 --- a/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs +++ b/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs @@ -10,17 +10,19 @@ )] use dashboard_gen::{ - Color, Dashboard, FieldOverride, Panel, Target, Thresholds, Unit, avg, percentiles, - percentiles_labeled, rate_per_min, + Color, Dashboard, FieldOverride, GradientMode, Panel, Target, Thresholds, Unit, avg, + percentile_legend, percentile_variable, rate_per_min, selected_percentile, }; use json_pretty_compact::PrettyCompactFormatter; use serde::Serialize as _; const PERCENTILES: &[u32] = &[50, 90, 95, 99]; +const DEFAULT_PERCENTILE: u32 = 95; fn sequencer_dashboard() -> Dashboard { Dashboard::new("Sequencer", "sequencer") .tag("sequencer") + .variable(percentile_variable(PERCENTILES, DEFAULT_PERCENTILE)) .row( 7, [ @@ -30,13 +32,13 @@ fn sequencer_dashboard() -> Dashboard { .decimals(0) .color(Color::fixed("blue")) .target( - Target::new(sequencer_core_metrics::names::BLOCK_COUNT).legend("height"), + Target::new(sequencer_core_metrics::names::BLOCKS_TOTAL).legend("height"), ), Panel::timeseries("Block production rate") .width(18) .unit(Unit::Short) .target(rate_per_min( - sequencer_core_metrics::names::BLOCK_COUNT, + sequencer_core_metrics::names::BLOCKS_TOTAL, "blocks/min", )), ], @@ -46,9 +48,10 @@ fn sequencer_dashboard() -> Dashboard { [Panel::timeseries("Block creation time") .width(24) .unit(Unit::Seconds) - .targets(percentiles( + .target(selected_percentile( sequencer_core_metrics::names::BLOCK_CREATION_TIME, - PERCENTILES, + &[], + &percentile_legend(), )) .target(avg(sequencer_core_metrics::names::BLOCK_CREATION_TIME)) .with_override( @@ -63,17 +66,18 @@ fn sequencer_dashboard() -> Dashboard { Panel::timeseries("Transaction application time") .width(12) .unit(Unit::Seconds) - .targets(percentiles_labeled( + .target(selected_percentile( sequencer_core_metrics::names::MEMPOOL_TRANSACTION_APPLICATION_TIME, - PERCENTILES, - " · {{kind}} · {{origin}}", + &["kind", "origin"], + "{{kind}} · {{origin}}", )), Panel::timeseries("Transactions per block") .width(12) .unit(Unit::Short) - .targets(percentiles( + .target(selected_percentile( sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK, - PERCENTILES, + &[], + &percentile_legend(), )) .target(avg(sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK)) .with_override( @@ -141,11 +145,13 @@ fn sequencer_dashboard() -> Dashboard { ) .target( Target::new(format!( + // Both failure stages against the same submission base; // `clamp_min` keeps an idle window (nothing submitted) // reading as 0% instead of a division by zero. - "100 * increase({failed}[$__range]) / clamp_min(increase({submitted}[$__range]), 1)", - failed = sequencer_core_metrics::names::FAILED_TRANSACTION_COUNT, - submitted = sequencer_service_metrics::names::SUBMITTED_TRANSACTION_COUNT, + "100 * (increase({before_mempool}[$__range]) + increase({in_mempool}[$__range])) / clamp_min(increase({submitted}[$__range]), 1)", + before_mempool = sequencer_service_metrics::names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL, + in_mempool = sequencer_core_metrics::names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL, + submitted = sequencer_service_metrics::names::SUBMITTED_TRANSACTIONS_TOTAL, )) .legend("failed"), ), @@ -153,17 +159,29 @@ fn sequencer_dashboard() -> Dashboard { .width(18) .unit(Unit::Short) .min(0.0) + .fill_opacity(35) + .gradient_mode(GradientMode::Opacity) .target(rate_per_min( - sequencer_service_metrics::names::SUBMITTED_TRANSACTION_COUNT, + sequencer_service_metrics::names::SUBMITTED_TRANSACTIONS_TOTAL, "submitted", )) .target(rate_per_min( - sequencer_core_metrics::names::FAILED_TRANSACTION_COUNT, - "failed", + sequencer_service_metrics::names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL, + "failed · before mempool", + )) + .target(rate_per_min( + sequencer_core_metrics::names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL, + "failed · in mempool", )) - .with_override(FieldOverride::by_name("failed").color(Color::fixed("red"))) .with_override( FieldOverride::by_name("submitted").color(Color::fixed("green")), + ) + .with_override( + FieldOverride::by_name("failed · before mempool") + .color(Color::fixed("orange")), + ) + .with_override( + FieldOverride::by_name("failed · in mempool").color(Color::fixed("red")), ), ], ) diff --git a/tools/dashboard_gen/src/codegen.rs b/tools/dashboard_gen/src/codegen.rs index b554ffd6..4874f053 100644 --- a/tools/dashboard_gen/src/codegen.rs +++ b/tools/dashboard_gen/src/codegen.rs @@ -10,7 +10,7 @@ use std::fmt::Write as _; use crate::{ - Unit, + DEFAULT_FILL_OPACITY, Unit, input::{Defaults, PanelInput}, schema::{ AxisPlacement, Color, GradientMode, LineInterpolation, PanelType, ShowPoints, StackingMode, @@ -42,6 +42,11 @@ fn panel_expr_inner(panel: &PanelInput) -> Result { write_defaults(&mut expr, defaults)?; if let Some(custom) = &defaults.custom { + // Emitted against the builder's default, not Grafana's (which is 0), so + // a genuinely unfilled panel still round-trips. + if let Some(opacity) = custom.fill_opacity.filter(|&o| o != DEFAULT_FILL_OPACITY) { + write!(expr, "\n .fill_opacity({opacity})")?; + } if custom.span_nulls == Some(true) { expr.push_str("\n .span_nulls()"); } diff --git a/tools/dashboard_gen/src/input.rs b/tools/dashboard_gen/src/input.rs index ea3c4f78..9d69002c 100644 --- a/tools/dashboard_gen/src/input.rs +++ b/tools/dashboard_gen/src/input.rs @@ -66,6 +66,8 @@ pub struct Defaults { #[derive(Deserialize, Default)] #[serde(rename_all = "camelCase")] pub struct Custom { + #[serde(default)] + pub fill_opacity: Option, #[serde(default)] pub span_nulls: Option, #[serde(default)] diff --git a/tools/dashboard_gen/src/lib.rs b/tools/dashboard_gen/src/lib.rs index d1fb38c9..5bbaa672 100644 --- a/tools/dashboard_gen/src/lib.rs +++ b/tools/dashboard_gen/src/lib.rs @@ -16,13 +16,14 @@ pub use codegen::panel_to_rust_source; pub use schema::{ AxisPlacement, Color, GradientMode, LineInterpolation, ShowPoints, StackingMode, Thresholds, + Variable, }; use schema::{ Calc, Custom, Datasource, Defaults, DrawStyle, EmptyList, FieldConfig, Fill, GaugeOptions, GraphMode, GridPos, Legend, LegendDisplay, LineStyle, Matcher, MatcherKind, Options, OverrideProperty, PanelModel, PanelType, Placement, PropertyId, PropertyValue, ReduceOptions, - SortOrder, Stacking, StatColorMode, StatOptions, TimeRange, TimeSeriesOptions, Tooltip, - TooltipMode, + SortOrder, Stacking, StatColorMode, StatOptions, Templating, TimeRange, TimeSeriesOptions, + Tooltip, TooltipMode, VariableKind, VariableOption, }; use serde::Serialize; pub use unit::Unit; @@ -38,6 +39,18 @@ mod unit; /// than by a per-environment URL. pub const DATASOURCE_UID: &str = "prometheus"; +/// Window every histogram query rates over. `$__rate_interval` tracks the +/// panel's zoom, so a percentile covers the range you are actually looking at, +/// and an idle window yields no value rather than a zero. +const RATE_WINDOW: &str = "$__rate_interval"; + +/// Dashboard variable holding the quantile every percentile query reads. +const PERCENTILE_VAR: &str = "percentile"; + +/// Area fill under a timeseries line, as a percentage. Enough to read a series' +/// shape at a glance without drowning the ones stacked behind it. +pub(crate) const DEFAULT_FILL_OPACITY: u32 = 10; + /// A single Prometheus query within a panel. #[derive(Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -129,6 +142,7 @@ pub struct Panel { span_nulls: bool, overrides: Vec, // Optional timeseries styling, set via the `styling` setters. + fill_opacity: Option, line_interpolation: Option, show_points: Option, gradient_mode: Option, @@ -152,6 +166,7 @@ impl Panel { thresholds: None, span_nulls: false, overrides: Vec::new(), + fill_opacity: None, line_interpolation: None, show_points: None, gradient_mode: None, @@ -308,7 +323,7 @@ impl Panel { custom: Some(Custom { draw_style: DrawStyle::Line, line_width: 1, - fill_opacity: 10, + fill_opacity: self.fill_opacity.unwrap_or(DEFAULT_FILL_OPACITY), span_nulls: self.span_nulls.then_some(true), line_interpolation: self.line_interpolation, show_points: self.show_points, @@ -327,8 +342,11 @@ impl Panel { thresholds: self.thresholds, }; // Panels with several series read better as a sortable table with - // a multi-series tooltip; single-series panels stay compact. - let multi = targets.len() > 1; + // a multi-series tooltip; single-series panels stay compact. A + // `{{label}}` legend fans one target out into a series per label + // value, so it counts as several too. + let multi = + targets.len() > 1 || targets.iter().any(|target| target.legend.contains("{{")); let options = Options::TimeSeries(TimeSeriesOptions { legend: Legend { display_mode: if multi { @@ -382,7 +400,7 @@ pub struct Dashboard { refresh: String, schema_version: u32, tags: Vec, - templating: EmptyList, + templating: Templating, time: TimeRange, timezone: String, title: String, @@ -405,7 +423,7 @@ impl Dashboard { refresh: "5s".to_owned(), schema_version: 39, tags: Vec::new(), - templating: EmptyList::default(), + templating: Templating::default(), time: TimeRange { from: "now-15m".to_owned(), to: "now".to_owned(), @@ -430,6 +448,13 @@ impl Dashboard { self } + /// Add a dropdown to the dashboard's top bar, e.g. [`percentile_variable`]. + #[must_use] + pub fn variable(mut self, variable: Variable) -> Self { + self.templating.list.push(variable); + self + } + /// Place a horizontal row of panels at the current vertical cursor. Panel /// ids, x offsets and y are assigned here; unset widths split the remaining /// 24 columns evenly. @@ -468,34 +493,89 @@ impl Dashboard { } } -/// Percentile line targets for a summary metric: `p50`, `p90`, … each querying -/// the matching `quantile="0.x"` series. +/// The dropdown driving every [`selected_percentile`] query, offering +/// `percentiles` (e.g. `[50, 90, 95, 99]`) with `default` pre-selected. +/// +/// Panics if `default` is not one of `percentiles`. #[must_use] -pub fn percentiles(metric: &str, percentiles: &[u32]) -> Vec { - percentiles_labeled(metric, percentiles, "") -} +pub fn percentile_variable(percentiles: &[u32], default: u32) -> Variable { + assert!( + percentiles.contains(&default), + "default p{default} is not one of the offered percentiles {percentiles:?}", + ); -/// Like [`percentiles`], but appends `legend_suffix` to every legend — handy -/// when the metric carries labels (e.g. ` · {{kind}} · {{origin}}`). -#[must_use] -pub fn percentiles_labeled(metric: &str, percentiles: &[u32], legend_suffix: &str) -> Vec { - percentiles + let options: Vec = percentiles .iter() - .map(|&p| { - // `quantile="0.x"` label, derived without float math: zero-pad to two - // digits then drop trailing zeros (50 → "0.5", 95 → "0.95"). - let quantile = format!("0.{p:02}"); - let quantile = quantile.trim_end_matches('0'); - Target::new(format!("{metric}{{quantile=\"{quantile}\"}}")) - .legend(format!("p{p}{legend_suffix}")) + .map(|&p| VariableOption { + selected: p == default, + text: format!("p{p}"), + value: quantile(p), }) - .collect() + .collect(); + let current = options + .iter() + .find(|option| option.selected) + .cloned() + .expect("`default` is one of `percentiles`, asserted above"); + let query = options + .iter() + .map(|option| format!("{} : {}", option.text, option.value)) + .collect::>() + .join(", "); + + Variable { + current, + include_all: false, + label: "Percentile".to_owned(), + // A quantile is a scalar argument to `histogram_quantile`, so exactly + // one may be selected. + multi: false, + name: PERCENTILE_VAR.to_owned(), + options, + query, + kind: VariableKind::Custom, + } } -/// An `avg` target for a summary metric: `rate(sum) / rate(count)` over 1m. +/// The legend fragment that renders the dropdown's current choice, e.g. `p95`. +#[must_use] +pub fn percentile_legend() -> String { + format!("${{{PERCENTILE_VAR}:text}}") +} + +/// A [`histogram_quantile`] line over `metric`'s buckets, at whatever quantile +/// [`percentile_variable`] currently holds. `labels` stay split out into their +/// own series; every other label is summed away. +/// +/// [`histogram_quantile`]: https://prometheus.io/docs/prometheus/latest/querying/functions/#histogram_quantile +#[must_use] +pub fn selected_percentile(metric: &str, labels: &[&str], legend: &str) -> Target { + // `le` carries the bucket boundary, so it must survive the aggregation. + let grouping = std::iter::once("le") + .chain(labels.iter().copied()) + .collect::>() + .join(", "); + + Target::new(format!( + "histogram_quantile(${{{PERCENTILE_VAR}}}, sum by ({grouping}) (rate({metric}_bucket[{RATE_WINDOW}])))" + )) + .legend(legend) +} + +/// A percentile as its `histogram_quantile` argument, derived without float +/// math: zero-pad to two digits then drop trailing zeros (50 → `0.5`). +fn quantile(percentile: u32) -> String { + let quantile = format!("0.{percentile:02}"); + quantile.trim_end_matches('0').to_owned() +} + +/// An `avg` target for a histogram metric: `rate(sum) / rate(count)`. #[must_use] pub fn avg(metric: &str) -> Target { - Target::new(format!("rate({metric}_sum[1m]) / rate({metric}_count[1m])")).legend("avg") + Target::new(format!( + "rate({metric}_sum[{RATE_WINDOW}]) / rate({metric}_count[{RATE_WINDOW}])" + )) + .legend("avg") } /// A per-minute rate target for a counter metric. diff --git a/tools/dashboard_gen/src/schema.rs b/tools/dashboard_gen/src/schema.rs index 009c7c40..190c4606 100644 --- a/tools/dashboard_gen/src/schema.rs +++ b/tools/dashboard_gen/src/schema.rs @@ -363,6 +363,43 @@ pub enum Options { Gauge(GaugeOptions), } +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum VariableKind { + /// A fixed list of choices, spelled out in the dashboard itself. + Custom, +} + +/// One choice in a [`Variable`] dropdown: `text` is displayed, `value` is what +/// `$name` interpolates to in a query. +#[derive(Clone, Serialize)] +pub struct VariableOption { + pub selected: bool, + pub text: String, + pub value: String, +} + +/// A dashboard-level dropdown, rendered in the top bar. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Variable { + pub current: VariableOption, + pub include_all: bool, + pub label: String, + pub multi: bool, + pub name: String, + pub options: Vec, + /// Grafana's own encoding of `options`, as `text : value` pairs. + pub query: String, + #[serde(rename = "type")] + pub kind: VariableKind, +} + +#[derive(Serialize, Default)] +pub struct Templating { + pub list: Vec, +} + #[derive(Clone, Copy, Serialize)] pub struct GridPos { pub h: u32, diff --git a/tools/dashboard_gen/src/styling.rs b/tools/dashboard_gen/src/styling.rs index 5414ec73..36b0cc38 100644 --- a/tools/dashboard_gen/src/styling.rs +++ b/tools/dashboard_gen/src/styling.rs @@ -11,7 +11,7 @@ //! not emitted, so the value is silently dropped. use crate::{ - Panel, + DEFAULT_FILL_OPACITY, Panel, schema::{AxisPlacement, GradientMode, LineInterpolation, ShowPoints, StackingMode}, }; @@ -20,6 +20,24 @@ use crate::{ reason = "styling setters intentionally live in their own file, so `Panel` has a second inherent impl here" )] impl Panel { + /// Area fill under the line, as a percentage. Builder default: + /// [`DEFAULT_FILL_OPACITY`]. + /// + /// Panics if passed that default, or a value above 100. + #[must_use] + pub fn fill_opacity(mut self, opacity: u32) -> Self { + assert!( + opacity <= 100, + "fill_opacity({opacity}) is not a percentage" + ); + assert_ne!( + opacity, DEFAULT_FILL_OPACITY, + "fill_opacity({DEFAULT_FILL_OPACITY}) is redundant: it is the builder's default. Omit the call.", + ); + self.fill_opacity = Some(opacity); + self + } + /// Interpolation between points. Grafana default: `Linear`. /// /// Panics if passed `Linear` — that's the default and would be redundant. From 343ba0b97cc2af21ec1e9348a1e05f2dc6e607fe Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 17:11:49 +0300 Subject: [PATCH 10/21] feat(dashboard_gen): remove panel json to rust generator --- Cargo.lock | 1 + Justfile | 10 +- tools/dashboard_gen/Cargo.toml | 1 + .../src/bin/panel_json_to_rust.rs | 50 ---- tools/dashboard_gen/src/codegen.rs | 242 ------------------ tools/dashboard_gen/src/dashboards.rs | 3 + .../sequencer.rs} | 23 +- tools/dashboard_gen/src/input.rs | 137 ---------- tools/dashboard_gen/src/lib.rs | 3 - tools/dashboard_gen/src/main.rs | 45 ++++ tools/dashboard_gen/src/schema.rs | 32 +-- tools/dashboard_gen/src/unit.rs | 38 --- 12 files changed, 68 insertions(+), 517 deletions(-) delete mode 100644 tools/dashboard_gen/src/bin/panel_json_to_rust.rs delete mode 100644 tools/dashboard_gen/src/codegen.rs create mode 100644 tools/dashboard_gen/src/dashboards.rs rename tools/dashboard_gen/src/{bin/gen_sequencer_dashboard.rs => dashboards/sequencer.rs} (92%) delete mode 100644 tools/dashboard_gen/src/input.rs create mode 100644 tools/dashboard_gen/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 6eb08255..86d8102a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2296,6 +2296,7 @@ dependencies = [ name = "dashboard_gen" version = "0.1.0" dependencies = [ + "clap", "json-pretty-compact", "sequencer_core_metrics", "sequencer_service_metrics", diff --git a/Justfile b/Justfile index 021f6774..ed124de1 100644 --- a/Justfile +++ b/Justfile @@ -57,14 +57,8 @@ regenerate-test-fixture: # (tools/dashboard_gen) and commit the result. CI checks these are up to date. regenerate-dashboards: @echo "📊 Regenerating Grafana dashboards" - @cargo build -q -p dashboard_gen --bin gen_sequencer_dashboard - @cargo run -q -p dashboard_gen --bin gen_sequencer_dashboard > monitoring/grafana/dashboards/sequencer.json - -# Transpile a single Grafana panel JSON (stdin) — Inspect → Panel JSON — into a -# Rust builder expression (stdout), omitting Grafana defaults. Paste into a row. -# Usage: `just panel-to-rust < panel.json`. -panel-to-rust: - @cargo run -q -p dashboard_gen --bin panel_json_to_rust + @cargo build -q -p dashboard_gen + @cargo run -q -p dashboard_gen -- sequencer > monitoring/grafana/dashboards/sequencer.json # Run criterion benches: fast crypto primitives, then the slow PPE verify (real proving setup). bench: diff --git a/tools/dashboard_gen/Cargo.toml b/tools/dashboard_gen/Cargo.toml index 1a167db6..4bc73e86 100644 --- a/tools/dashboard_gen/Cargo.toml +++ b/tools/dashboard_gen/Cargo.toml @@ -11,6 +11,7 @@ workspace = true sequencer_core_metrics.workspace = true sequencer_service_metrics.workspace = true +clap = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive", "alloc"] } serde_json.workspace = true json-pretty-compact = "0.1.2" diff --git a/tools/dashboard_gen/src/bin/panel_json_to_rust.rs b/tools/dashboard_gen/src/bin/panel_json_to_rust.rs deleted file mode 100644 index 9e10f046..00000000 --- a/tools/dashboard_gen/src/bin/panel_json_to_rust.rs +++ /dev/null @@ -1,50 +0,0 @@ -//! Reads a single Grafana panel JSON on stdin (Grafana → panel menu → Inspect → -//! Panel JSON) and prints the Rust `Panel::…` builder expression that rebuilds -//! it through `dashboard_gen`, omitting values Grafana supplies by default. -//! -//! Paste the result into a dashboard's `.row(…)` and run `cargo fmt`. - -#![expect( - clippy::print_stderr, - reason = "CLI tool: diagnostics on stderr are the deliverable" -)] -#![expect( - clippy::non_ascii_literal, - reason = "help text mirrors Grafana's `Inspect → Panel JSON` menu path" -)] - -use std::{ - io::{self, Read as _, Write as _}, - process::ExitCode, -}; - -fn main() -> ExitCode { - let mut input = String::new(); - if let Err(err) = io::stdin().read_to_string(&mut input) { - eprintln!("error: failed to read panel JSON from stdin: {err}"); - return ExitCode::FAILURE; - } - - match dashboard_gen::panel_to_rust_source(&input) { - Ok(source) => { - if let Err(err) = io::stdout().write_all(source.as_bytes()) { - eprintln!("error: failed to write generated source to stdout: {err}"); - return ExitCode::FAILURE; - } - ExitCode::SUCCESS - } - Err(err) => { - eprintln!("error: could not parse panel JSON: {err}"); - eprintln!( - "hint: paste one panel (Inspect → Panel JSON);\ - only stat and timeseries are supported." - ); - eprintln!( - "hint: this tool supports only the subset of Grafana's panel JSON, you might need \ - to manually implement support for new fields." - ); - - ExitCode::FAILURE - } - } -} diff --git a/tools/dashboard_gen/src/codegen.rs b/tools/dashboard_gen/src/codegen.rs deleted file mode 100644 index 4874f053..00000000 --- a/tools/dashboard_gen/src/codegen.rs +++ /dev/null @@ -1,242 +0,0 @@ -//! Reverse of the builder: turn a single parsed panel back into the Rust -//! `Panel::…` builder expression, omitting values Grafana supplies by default. -//! Backs the `panel_json_to_rust` binary. -//! -//! Input is parsed leniently (see [`crate::input`]) so a raw Grafana panel -//! export — full of fields and vocabularies we don't model — still works; -//! anything unrecognized is dropped. The emitted expression is valid but only -//! lightly formatted, so `cargo fmt` re-indents it once it lands in a file. - -use std::fmt::Write as _; - -use crate::{ - DEFAULT_FILL_OPACITY, Unit, - input::{Defaults, PanelInput}, - schema::{ - AxisPlacement, Color, GradientMode, LineInterpolation, PanelType, ShowPoints, StackingMode, - ThresholdMode, Thresholds, - }, -}; - -/// Parse one Grafana panel JSON (Inspect → Panel JSON) and emit the Rust -/// builder expression that reproduces it (minus Grafana defaults). -pub fn panel_to_rust_source(json: &str) -> serde_json::Result { - let panel: PanelInput = serde_json::from_str(json)?; - Ok(format!("{}\n", panel_expr(&panel))) -} - -/// A `Panel::…()` expression with one method call per line. -fn panel_expr(panel: &PanelInput) -> String { - panel_expr_inner(panel).expect("writing to a String never fails") -} - -fn panel_expr_inner(panel: &PanelInput) -> Result { - let mut expr = match panel.panel_type { - PanelType::Stat => format!("Panel::stat({:?})", panel.title), - PanelType::Timeseries => format!("Panel::timeseries({:?})", panel.title), - PanelType::Gauge => format!("Panel::gauge({:?})", panel.title), - }; - write!(expr, "\n .width({})", panel.grid_pos.w)?; - - let defaults = &panel.field_config.defaults; - write_defaults(&mut expr, defaults)?; - - if let Some(custom) = &defaults.custom { - // Emitted against the builder's default, not Grafana's (which is 0), so - // a genuinely unfilled panel still round-trips. - if let Some(opacity) = custom.fill_opacity.filter(|&o| o != DEFAULT_FILL_OPACITY) { - write!(expr, "\n .fill_opacity({opacity})")?; - } - if custom.span_nulls == Some(true) { - expr.push_str("\n .span_nulls()"); - } - // Each optional styling field is emitted only when it differs from the - // Grafana default (matching the setters' panic-on-default contract). - if let Some(value) = custom - .line_interpolation - .filter(|&v| v != LineInterpolation::Linear) - { - write!( - expr, - "\n .line_interpolation(LineInterpolation::{})", - line_interp(value) - )?; - } - if let Some(value) = custom.show_points.filter(|&v| v != ShowPoints::Auto) { - write!( - expr, - "\n .show_points(ShowPoints::{})", - show_points(value) - )?; - } - if let Some(value) = custom.gradient_mode.filter(|&v| v != GradientMode::None) { - write!( - expr, - "\n .gradient_mode(GradientMode::{})", - gradient_mode(value) - )?; - } - let stacking = custom.stacking.as_ref().and_then(|s| s.mode); - if let Some(mode) = stacking.filter(|&m| m != StackingMode::None) { - write!( - expr, - "\n .stacking(StackingMode::{})", - stacking_mode(mode) - )?; - } - if let Some(value) = custom.axis_placement.filter(|&v| v != AxisPlacement::Auto) { - write!( - expr, - "\n .axis_placement(AxisPlacement::{})", - axis_placement(value) - )?; - } - if let Some(label) = custom.axis_label.as_deref().filter(|l| !l.is_empty()) { - write!(expr, "\n .axis_label({label:?})")?; - } - } - - for over in &panel.field_config.overrides { - // Only `byName` matchers map to the builder; skip anything else. - let Some(name) = over.matcher.by_name() else { - continue; - }; - let mut calls = String::new(); - for property in &over.properties { - match property.id.as_str() { - "color" => { - if let Ok(Color::Fixed { fixed_color }) = - serde_json::from_value::(property.value.clone()) - { - write!(calls, ".color(Color::fixed({fixed_color:?}))")?; - } - } - "custom.lineStyle" => calls.push_str(".dashed_line()"), - _ => {} // property kind the builder can't express — drop it - } - } - // An override with nothing representable adds no information. - if !calls.is_empty() { - write!( - expr, - "\n .with_override(FieldOverride::by_name({name:?}){calls})" - )?; - } - } - - for target in &panel.targets { - if target.expr.is_empty() { - continue; - } - write!(expr, "\n .target(Target::new({:?})", target.expr)?; - // `__auto` is Grafana's "no explicit legend" sentinel, i.e. the default. - if !target.legend.is_empty() && target.legend != "__auto" { - write!(expr, ".legend({:?})", target.legend)?; - } - expr.push(')'); - } - - Ok(expr) -} - -/// The field-level setters — everything outside the `custom` styling block. -fn write_defaults(expr: &mut String, defaults: &Defaults) -> Result<(), std::fmt::Error> { - // `short` is the builder's own default unit, so it round-trips without a call. - if let Some(unit) = defaults.unit.as_deref().filter(|u| *u != "short") { - write!( - expr, - "\n .unit({})", - Unit::from_id(unit).to_rust_source() - )?; - } - if let Some(decimals) = defaults.decimals { - write!(expr, "\n .decimals({decimals})")?; - } - // Only a fixed color is worth emitting; `palette-classic` is Grafana's default. - if let Some(Color::Fixed { fixed_color }) = &defaults.color { - write!(expr, "\n .color(Color::fixed({fixed_color:?}))")?; - } - if let Some(min) = defaults.min { - write!(expr, "\n .min({min:?})")?; - } - if let Some(max) = defaults.max { - write!(expr, "\n .max({max:?})")?; - } - - let ladder = defaults - .thresholds - .as_ref() - .filter(|thresholds| is_expressible_ladder(thresholds)) - .and_then(|thresholds| thresholds.steps.split_first()); - if let Some((base, steps)) = ladder { - write!(expr, "\n .thresholds(Thresholds::base({:?})", base.color)?; - for step in steps { - // A non-base step without a value is nonsense Grafana wouldn't render. - if let Some(value) = step.value { - write!(expr, ".step({value:?}, {:?})", step.color)?; - } - } - expr.push(')'); - } - - Ok(()) -} - -/// Whether a ladder is worth emitting: `percentage` mode is beyond the builder, -/// and green/red-at-80 is the ladder Grafana attaches to every panel by default. -fn is_expressible_ladder(thresholds: &Thresholds) -> bool { - if thresholds.mode != ThresholdMode::Absolute { - return false; - } - !matches!( - thresholds.steps.as_slice(), - [base, red] - if base.color == "green" - && base.value.is_none() - && red.color == "red" - && red.value == Some(80.0) - ) -} - -const fn line_interp(value: LineInterpolation) -> &'static str { - match value { - LineInterpolation::Linear => "Linear", - LineInterpolation::Smooth => "Smooth", - LineInterpolation::StepBefore => "StepBefore", - LineInterpolation::StepAfter => "StepAfter", - } -} - -const fn show_points(value: ShowPoints) -> &'static str { - match value { - ShowPoints::Auto => "Auto", - ShowPoints::Never => "Never", - ShowPoints::Always => "Always", - } -} - -const fn gradient_mode(value: GradientMode) -> &'static str { - match value { - GradientMode::None => "None", - GradientMode::Opacity => "Opacity", - GradientMode::Hue => "Hue", - GradientMode::Scheme => "Scheme", - } -} - -const fn stacking_mode(value: StackingMode) -> &'static str { - match value { - StackingMode::None => "None", - StackingMode::Normal => "Normal", - StackingMode::Percent => "Percent", - } -} - -const fn axis_placement(value: AxisPlacement) -> &'static str { - match value { - AxisPlacement::Auto => "Auto", - AxisPlacement::Left => "Left", - AxisPlacement::Right => "Right", - AxisPlacement::Hidden => "Hidden", - } -} diff --git a/tools/dashboard_gen/src/dashboards.rs b/tools/dashboard_gen/src/dashboards.rs new file mode 100644 index 00000000..39ed51ab --- /dev/null +++ b/tools/dashboard_gen/src/dashboards.rs @@ -0,0 +1,3 @@ +//! One module per dashboard, each exposing a `dashboard()` builder. + +pub mod sequencer; diff --git a/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs b/tools/dashboard_gen/src/dashboards/sequencer.rs similarity index 92% rename from tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs rename to tools/dashboard_gen/src/dashboards/sequencer.rs index e2f7df78..3c609841 100644 --- a/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs +++ b/tools/dashboard_gen/src/dashboards/sequencer.rs @@ -1,9 +1,6 @@ -//! Generates the sequencer dashboard and prints it to stdout. +//! The sequencer dashboard: chain progress, block timings, mempool and +//! transaction outcomes. -#![expect( - clippy::print_stdout, - reason = "CLI tool: emitting the dashboard JSON on stdout is the deliverable" -)] #![expect( clippy::non_ascii_literal, reason = "legend separators use `·` intentionally, matching the rendered Grafana labels" @@ -13,13 +10,11 @@ use dashboard_gen::{ Color, Dashboard, FieldOverride, GradientMode, Panel, Target, Thresholds, Unit, avg, percentile_legend, percentile_variable, rate_per_min, selected_percentile, }; -use json_pretty_compact::PrettyCompactFormatter; -use serde::Serialize as _; const PERCENTILES: &[u32] = &[50, 90, 95, 99]; const DEFAULT_PERCENTILE: u32 = 95; -fn sequencer_dashboard() -> Dashboard { +pub fn dashboard() -> Dashboard { Dashboard::new("Sequencer", "sequencer") .tag("sequencer") .variable(percentile_variable(PERCENTILES, DEFAULT_PERCENTILE)) @@ -186,15 +181,3 @@ fn sequencer_dashboard() -> Dashboard { ], ) } - -fn main() { - let dashboard = sequencer_dashboard(); - - let formatter = PrettyCompactFormatter::new(); - let mut output = Vec::new(); - let mut ser = serde_json::Serializer::with_formatter(&mut output, formatter); - dashboard.serialize(&mut ser).unwrap(); - - let json = String::from_utf8(output).unwrap(); - println!("{json}"); -} diff --git a/tools/dashboard_gen/src/input.rs b/tools/dashboard_gen/src/input.rs deleted file mode 100644 index 9d69002c..00000000 --- a/tools/dashboard_gen/src/input.rs +++ /dev/null @@ -1,137 +0,0 @@ -//! Lenient, deserialize-only model of a Grafana panel, for the panel→Rust -//! transpiler (`codegen`). -//! -//! A real Grafana export is far wider than what we emit: `unit` may be absent, -//! `options.tooltip.sort` may be `"none"`, overrides carry property types we -//! don't model, and there are dozens of fields we ignore. So this model is -//! deliberately separate from the strict `schema` (sized for *output*): it -//! captures only what codegen reads, makes every field optional, and lets serde -//! drop everything else — including the whole `options` block, which codegen -//! reconstructs from the panel type rather than reading. - -use serde::Deserialize; -use serde_json::Value; - -use crate::schema::{ - AxisPlacement, Color, GradientMode, LineInterpolation, PanelType, ShowPoints, StackingMode, - Thresholds, -}; - -#[derive(Deserialize)] -pub struct PanelInput { - #[serde(rename = "type")] - pub panel_type: PanelType, - #[serde(default)] - pub title: String, - #[serde(rename = "gridPos", default)] - pub grid_pos: GridPos, - #[serde(rename = "fieldConfig", default)] - pub field_config: FieldConfig, - #[serde(default)] - pub targets: Vec, -} - -#[derive(Deserialize, Default)] -pub struct GridPos { - #[serde(default)] - pub w: u32, -} - -#[derive(Deserialize, Default)] -pub struct FieldConfig { - #[serde(default)] - pub defaults: Defaults, - #[serde(default)] - pub overrides: Vec, -} - -#[derive(Deserialize, Default)] -pub struct Defaults { - #[serde(default)] - pub unit: Option, - #[serde(default)] - pub decimals: Option, - #[serde(default)] - pub color: Option, - #[serde(default)] - pub custom: Option, - #[serde(default)] - pub min: Option, - #[serde(default)] - pub max: Option, - #[serde(default)] - pub thresholds: Option, -} - -#[derive(Deserialize, Default)] -#[serde(rename_all = "camelCase")] -pub struct Custom { - #[serde(default)] - pub fill_opacity: Option, - #[serde(default)] - pub span_nulls: Option, - #[serde(default)] - pub line_interpolation: Option, - #[serde(default)] - pub show_points: Option, - #[serde(default)] - pub gradient_mode: Option, - #[serde(default)] - pub stacking: Option, - #[serde(default)] - pub axis_placement: Option, - #[serde(default)] - pub axis_label: Option, -} - -#[derive(Deserialize, Default)] -pub struct Stacking { - #[serde(default)] - pub mode: Option, -} - -#[derive(Deserialize)] -pub struct Override { - #[serde(default)] - pub matcher: Matcher, - #[serde(default)] - pub properties: Vec, -} - -#[derive(Deserialize, Default)] -pub struct Matcher { - #[serde(default)] - pub id: String, - #[serde(default)] - pub options: Value, -} - -impl Matcher { - /// The series name for a `byName` matcher; `None` for matcher kinds the - /// builder can't express (which the caller skips). - pub fn by_name(&self) -> Option<&str> { - if self.id == "byName" { - self.options.as_str() - } else { - None - } - } -} - -/// One override property. `id`/`value` are kept raw so unknown kinds are simply -/// ignored by codegen rather than failing the whole parse. -#[derive(Deserialize)] -pub struct Property { - #[serde(default)] - pub id: String, - #[serde(default)] - pub value: Value, -} - -#[derive(Deserialize, Default)] -pub struct Target { - #[serde(default)] - pub expr: String, - #[serde(rename = "legendFormat", default)] - pub legend: String, -} diff --git a/tools/dashboard_gen/src/lib.rs b/tools/dashboard_gen/src/lib.rs index 5bbaa672..350188a2 100644 --- a/tools/dashboard_gen/src/lib.rs +++ b/tools/dashboard_gen/src/lib.rs @@ -13,7 +13,6 @@ // Styling vocabularies passed to the optional `styling` setters are part of the // public API (and are used by this module's `Panel` fields and `finalize`). -pub use codegen::panel_to_rust_source; pub use schema::{ AxisPlacement, Color, GradientMode, LineInterpolation, ShowPoints, StackingMode, Thresholds, Variable, @@ -28,8 +27,6 @@ use schema::{ use serde::Serialize; pub use unit::Unit; -mod codegen; -mod input; mod schema; mod styling; mod unit; diff --git a/tools/dashboard_gen/src/main.rs b/tools/dashboard_gen/src/main.rs new file mode 100644 index 00000000..b428817d --- /dev/null +++ b/tools/dashboard_gen/src/main.rs @@ -0,0 +1,45 @@ +//! Builds one of the Grafana dashboards and prints its JSON to stdout. + +#![expect( + clippy::print_stdout, + reason = "CLI tool: emitting the dashboard JSON on stdout is the deliverable" +)] + +use clap::{Parser, ValueEnum}; +use dashboard_gen::Dashboard; +use json_pretty_compact::PrettyCompactFormatter; +use serde::Serialize as _; + +mod dashboards; + +#[derive(Debug, Parser)] +#[clap(version)] +struct Args { + /// Which dashboard to build. + dashboard: DashboardKind, +} + +#[derive(Debug, Clone, Copy, ValueEnum)] +enum DashboardKind { + Sequencer, +} + +impl DashboardKind { + fn build(self) -> Dashboard { + match self { + Self::Sequencer => dashboards::sequencer::dashboard(), + } + } +} + +fn main() { + let Args { dashboard } = Args::parse(); + + let formatter = PrettyCompactFormatter::new(); + let mut output = Vec::new(); + let mut ser = serde_json::Serializer::with_formatter(&mut output, formatter); + dashboard.build().serialize(&mut ser).unwrap(); + + let json = String::from_utf8(output).unwrap(); + println!("{json}"); +} diff --git a/tools/dashboard_gen/src/schema.rs b/tools/dashboard_gen/src/schema.rs index 190c4606..d04b6bad 100644 --- a/tools/dashboard_gen/src/schema.rs +++ b/tools/dashboard_gen/src/schema.rs @@ -1,12 +1,9 @@ //! The serializable Grafana dashboard schema — the internal data model the //! public builders assemble into. //! -//! Most types are `Serialize`-only, sized for what we *emit*. A handful of -//! vocabularies (`PanelType`, `Color`, and the styling enums) additionally -//! derive `Deserialize` because the lenient `input` model — which backs the -//! panel→Rust transpiler — reuses them. +//! Every type here is `Serialize`-only: the model is sized for what we emit. -use serde::{Deserialize, Serialize}; +use serde::Serialize; use crate::{DATASOURCE_UID, FieldOverride, Target, unit::Unit}; @@ -88,8 +85,7 @@ pub enum SortOrder { Desc, } -// Reused by the `input` model, hence `Deserialize`. -#[derive(Clone, Copy, Serialize, Deserialize)] +#[derive(Clone, Copy, Serialize)] #[serde(rename_all = "lowercase")] pub enum PanelType { Stat, @@ -97,7 +93,7 @@ pub enum PanelType { Gauge, } -#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "lowercase")] pub enum ThresholdMode { Absolute, @@ -108,14 +104,14 @@ pub enum ThresholdMode { /// One threshold step: the color values at or above `value` take. The base step /// carries `value: null` — Grafana's "everything below the first threshold". -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize)] pub struct ThresholdStep { pub color: String, pub value: Option, } /// A threshold ladder, driving gauge/stat coloring. -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize)] pub struct Thresholds { pub mode: ThresholdMode, pub steps: Vec, @@ -145,10 +141,9 @@ impl Thresholds { } // Optional timeseries styling vocabularies. Each derives `PartialEq` so the -// public setters can panic when handed the Grafana default (see `styling`), and -// `Deserialize` because the `input` model reuses them. +// public setters can panic when handed the Grafana default (see `styling`). -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub enum LineInterpolation { Linear, @@ -157,7 +152,7 @@ pub enum LineInterpolation { StepAfter, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] #[serde(rename_all = "lowercase")] pub enum ShowPoints { Auto, @@ -165,7 +160,7 @@ pub enum ShowPoints { Always, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] #[serde(rename_all = "lowercase")] pub enum GradientMode { None, @@ -174,7 +169,7 @@ pub enum GradientMode { Scheme, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] #[serde(rename_all = "lowercase")] pub enum StackingMode { None, @@ -182,7 +177,7 @@ pub enum StackingMode { Percent, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] #[serde(rename_all = "lowercase")] pub enum AxisPlacement { Auto, @@ -237,8 +232,7 @@ impl Datasource { } } -// Reused by the `input` model, hence `Deserialize`. -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize)] #[serde(rename_all = "kebab-case")] #[serde(tag = "mode")] pub enum Color { diff --git a/tools/dashboard_gen/src/unit.rs b/tools/dashboard_gen/src/unit.rs index adeebf3b..a91f302b 100644 --- a/tools/dashboard_gen/src/unit.rs +++ b/tools/dashboard_gen/src/unit.rs @@ -60,44 +60,6 @@ impl Unit { Self::Custom(id) => id, } } - - /// Reverse of [`Self::as_id`]: map a Grafana unit id back to a `Unit`, - /// falling back to [`Self::Custom`] for ids we don't name. Used by the - /// panel→Rust transpiler. - #[must_use] - pub(crate) fn from_id(id: &str) -> Self { - match id { - "short" => Self::Short, - "percent" => Self::Percent, - "percentunit" => Self::PercentUnit, - "s" => Self::Seconds, - "ms" => Self::Milliseconds, - "ns" => Self::Nanoseconds, - "bytes" => Self::Bytes, - "Bps" => Self::BytesPerSec, - "reqps" => Self::RequestsPerSec, - "ops" => Self::OpsPerSec, - other => Self::custom(other), - } - } - - /// The Rust builder expression that reconstructs this unit, for codegen. - #[must_use] - pub(crate) fn to_rust_source(&self) -> String { - match self { - Self::Custom(id) => format!("Unit::custom({id:?})"), - named @ (Self::Short - | Self::Percent - | Self::PercentUnit - | Self::Seconds - | Self::Milliseconds - | Self::Nanoseconds - | Self::Bytes - | Self::BytesPerSec - | Self::RequestsPerSec - | Self::OpsPerSec) => format!("Unit::{named:?}"), - } - } } impl Serialize for Unit { From d0ae9c27685baca457e7c98595d2c4424f95c12f Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 17:31:12 +0300 Subject: [PATCH 11/21] chore(metrics): rename `writing` feature to `record` --- lez/sequencer/core/Cargo.toml | 2 +- lez/sequencer/core/metrics/Cargo.toml | 4 ++-- lez/sequencer/core/metrics/src/lib.rs | 8 ++++---- lez/sequencer/core/metrics/src/{writing.rs => record.rs} | 0 lez/sequencer/service/Cargo.toml | 2 +- lez/sequencer/service/metrics/Cargo.toml | 4 ++-- lez/sequencer/service/metrics/src/lib.rs | 8 ++++---- .../service/metrics/src/{writing.rs => record.rs} | 0 8 files changed, 14 insertions(+), 14 deletions(-) rename lez/sequencer/core/metrics/src/{writing.rs => record.rs} (100%) rename lez/sequencer/service/metrics/src/{writing.rs => record.rs} (100%) diff --git a/lez/sequencer/core/Cargo.toml b/lez/sequencer/core/Cargo.toml index 08f359f7..a2d8a21c 100644 --- a/lez/sequencer/core/Cargo.toml +++ b/lez/sequencer/core/Cargo.toml @@ -12,7 +12,7 @@ lee.workspace = true lee_core.workspace = true chain_state.workspace = true common.workspace = true -sequencer_core_metrics = { workspace = true, features = ["writing"] } +sequencer_core_metrics = { workspace = true, features = ["record"] } storage.workspace = true mempool.workspace = true logos-blockchain-zone-sdk.workspace = true diff --git a/lez/sequencer/core/metrics/Cargo.toml b/lez/sequencer/core/metrics/Cargo.toml index 541102b4..c67496d3 100644 --- a/lez/sequencer/core/metrics/Cargo.toml +++ b/lez/sequencer/core/metrics/Cargo.toml @@ -9,8 +9,8 @@ workspace = true [features] default = [] -# Enable metrics writing -writing = ["dep:common", "dep:metrics", "dep:strum"] +# Enable metrics record +record = ["dep:common", "dep:metrics", "dep:strum"] [dependencies] common = { workspace = true, optional = true } diff --git a/lez/sequencer/core/metrics/src/lib.rs b/lez/sequencer/core/metrics/src/lib.rs index b633f2d2..b8884fa2 100644 --- a/lez/sequencer/core/metrics/src/lib.rs +++ b/lez/sequencer/core/metrics/src/lib.rs @@ -1,9 +1,9 @@ //! This crate provides all metrics exposed by the sequencer core crate. -#[cfg(feature = "writing")] -pub use writing::*; +#[cfg(feature = "record")] +pub use record::*; pub mod names; -#[cfg(feature = "writing")] -pub mod writing; +#[cfg(feature = "record")] +pub mod record; diff --git a/lez/sequencer/core/metrics/src/writing.rs b/lez/sequencer/core/metrics/src/record.rs similarity index 100% rename from lez/sequencer/core/metrics/src/writing.rs rename to lez/sequencer/core/metrics/src/record.rs diff --git a/lez/sequencer/service/Cargo.toml b/lez/sequencer/service/Cargo.toml index 0b936e8b..396fbe6b 100644 --- a/lez/sequencer/service/Cargo.toml +++ b/lez/sequencer/service/Cargo.toml @@ -15,7 +15,7 @@ mempool.workspace = true sequencer_core = { workspace = true, features = ["testnet"] } sequencer_service_protocol.workspace = true sequencer_service_rpc = { workspace = true, features = ["server"] } -sequencer_service_metrics = { workspace = true, features = ["writing"] } +sequencer_service_metrics = { workspace = true, features = ["record"] } programs.workspace = true clap = { workspace = true, features = ["derive", "env"] } diff --git a/lez/sequencer/service/metrics/Cargo.toml b/lez/sequencer/service/metrics/Cargo.toml index 4e242db4..46dd2d5a 100644 --- a/lez/sequencer/service/metrics/Cargo.toml +++ b/lez/sequencer/service/metrics/Cargo.toml @@ -9,8 +9,8 @@ workspace = true [features] default = [] -# Enable metrics writing -writing = ["dep:metrics"] +# Enable metrics record +record = ["dep:metrics"] [dependencies] metrics = { workspace = true, optional = true } diff --git a/lez/sequencer/service/metrics/src/lib.rs b/lez/sequencer/service/metrics/src/lib.rs index b633f2d2..b8884fa2 100644 --- a/lez/sequencer/service/metrics/src/lib.rs +++ b/lez/sequencer/service/metrics/src/lib.rs @@ -1,9 +1,9 @@ //! This crate provides all metrics exposed by the sequencer core crate. -#[cfg(feature = "writing")] -pub use writing::*; +#[cfg(feature = "record")] +pub use record::*; pub mod names; -#[cfg(feature = "writing")] -pub mod writing; +#[cfg(feature = "record")] +pub mod record; diff --git a/lez/sequencer/service/metrics/src/writing.rs b/lez/sequencer/service/metrics/src/record.rs similarity index 100% rename from lez/sequencer/service/metrics/src/writing.rs rename to lez/sequencer/service/metrics/src/record.rs From dc5804b7c2ca086c2c5c707c1082e3ecaae18ba7 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 17:35:06 +0300 Subject: [PATCH 12/21] docs(metrics): add metrics.md --- CONTRIBUTING.md | 6 ++- docs/metrics/metrics.md | 113 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 docs/metrics/metrics.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b15693b8..68843859 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,6 +6,10 @@ This document describes the guidelines for contributing to the project. We will If you have any questions, come say hi to our [Discord](https://discord.gg/tGJwgGrSPN)! +## Metrics + +We have guidelines about metrics, for more information refer to [metrics](docs/metrics/metrics.md). + ## Commit title format We use [Conventional Commits](https://www.conventionalcommits.org/). @@ -59,7 +63,7 @@ Could be squashed to an empty commit if they belong to the same PR. ## Default branch -By default all PRs must be directed into the `dev` branch. This helps us to keep releases stable. +By default all PRs must be directed into the `dev` branch. This helps us to keep releases stable. ## Branch workflow diff --git a/docs/metrics/metrics.md b/docs/metrics/metrics.md new file mode 100644 index 00000000..3794a846 --- /dev/null +++ b/docs/metrics/metrics.md @@ -0,0 +1,113 @@ +# Metrics + +Services expose Prometheus metrics; Grafana dashboards are generated from Rust so panel queries and metric names cannot drift apart. + +## Metrics crates + +Every crate that emits metrics gets a sibling `metrics` crate — `lez/sequencer/core/metrics` → `sequencer_core_metrics`. Each has two halves: + +| Module | Gated by | Contents | +|---|---|---| +| `names` | always compiled | `pub const BLOCKS_TOTAL: &str = "blocks_total";` — one const per metric | +| `record` | `record` feature | `record_*` / `increment_*` functions, plus `init()` | + +The emitting crate depends on it with `features = ["record"]`; consumers that only need the names (i.e. `dashboard_gen`) take the default features and pull in nothing. Dashboards reference the same consts the recording code does, so **renaming a metric is a compile error rather than a silently empty panel**. + +## Naming + +The recorder runs with `with_recommended_naming(true)`, which enforces Prometheus convention: + +| Kind | Suffix | Example | +|---|---|---| +| Counter | `_total` | `blocks_total`, `submitted_transactions_total` | +| Histogram | unit | `block_creation_time_seconds` | +| Gauge | none | `mempool_size` | + +**Spell the suffix in the const.** The exporter appends a missing unit suffix to the *rendered* name, but bucket matchers (below) run against the registered name — a duration metric named without `_seconds` renders correctly yet silently gets the wrong buckets. + +## Metric types + +| Type | Use for | Example | +|---|---|---| +| Counter | monotonically increasing event counts | `mempool_failed_transactions_total` | +| Gauge | a value that moves both ways | `mempool_size` | +| Histogram | distributions — latencies, sizes, per-batch counts | `mempool_transaction_application_time_seconds` | + +Each metric gets a private constructor plus a public recording wrapper, so its description, unit and labels are declared once: + +```rust +fn blocks_total_counter() -> Counter { + counter!( + description: "Number of blocks in chain", + unit: Unit::Count, + names::BLOCKS_TOTAL + ) +} + +pub fn increment_blocks_total() { + blocks_total_counter().increment(1); +} +``` + +Labels are passed as `"origin" => <&'static str>::from(origin)`; keep them low-cardinality (enums, never IDs or hashes). + +## `init()` + +Each `record` module exposes `init()`, called once at startup after the recorder is installed. It publishes every metric at zero. + +This is not cosmetic. A metric only materialises when first touched, and `rate()`/`increase()` need a sample from *before* an increment to see it — a series that springs into existence at `1` reads as `0` until the second event, so the first one is lost forever. Zero-publishing also means an idle service exports `0` instead of nothing at all. + +For histograms, creating the handle publishes zeroed buckets without recording an observation (recording a fake `0` would skew the distribution). Label combinations must each be registered, so `init()` iterates the label enums via `strum::EnumIter`. + +## Metrics in libraries + +**Yes, record metrics from library crates.** The `metrics` facade is a no-op until a recorder is installed, so a library that records costs nothing to a consumer that never installs one — including tests. Libraries record; only the binary installs the exporter. + +## Exporter setup + +`sequencer_service`'s `main.rs` installs the Prometheus recorder on `:9000` with **explicit histogram buckets**. This matters: without buckets, `metrics-exporter-prometheus` renders histograms as rolling-window summaries whose quantiles **reset to `0`** once the window (default 60 s) drains — an idle period reads as "took 0 s" rather than "no data". With buckets you get real `_bucket`/`_sum`/`_count` counters that never decay, are aggregatable, and honour the dashboard's time range. + +Ladders are matched by name suffix, so a new timing metric is covered automatically: + +```rust +.set_buckets(COUNT_BUCKETS) // fallback +.set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), LATENCY_BUCKETS) +``` + +## `dashboard_gen` + +`tools/dashboard_gen` is a small Grafana dashboard builder plus the dashboard definitions. It prints JSON to stdout; the result is committed under `monitoring/grafana/dashboards/` and CI fails if it is stale. + +``` +src/lib.rs, schema.rs, styling.rs, unit.rs the builder library +src/dashboards/.rs one dashboard per module +src/main.rs CLI: `dashboard_gen sequencer` +``` + +Panels are built fluently, and every query is composed from the `names` consts: + +```rust +Panel::timeseries("Block production rate") + .width(18) + .target(rate_per_min(sequencer_core_metrics::names::BLOCKS_TOTAL, "blocks/min")) +``` + +Query helpers: `rate_per_min` for counters, `avg` for histograms, and `selected_percentile` for percentile lines — the latter reads a `percentile` dashboard dropdown created by `percentile_variable`, so one panel serves p50/p90/p95/p99 instead of drawing all four. Rate windows use `$__rate_interval`, which tracks the panel's zoom. + +**Extending it:** + +| Goal | Change | +|---|---| +| New panel | Add a `Panel::…` to a row in the dashboard module | +| New dashboard | `src/dashboards/.rs` with `pub fn dashboard()`, a `pub mod` line, a `DashboardKind` variant, and a `just regenerate-dashboards` line | +| Grafana option we don't model yet | Add the field to `schema.rs` and a setter on `Panel` (styling setters live in `styling.rs` and panic when handed a redundant default) | + +The builder deliberately models only the subset of Grafana's schema we use. + +## Justfile + +| Recipe | Purpose | +|---|---| +| `just regenerate-dashboards` | Rebuild the committed dashboard JSON. Run after touching metric names or dashboard code — CI checks it is current. | +| `just run-monitoring` | Prometheus (`:9090`) + Grafana (`:3000`, anonymous admin) in docker, scraping the sequencer every 5 s | +| `just get-sequencer-metrics` | `curl` the raw `/metrics` endpoint — quickest way to confirm a metric name and value | From f2e92a9c1f6e7b84234008adc0c5f0fc191c0383 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 23:43:22 +0300 Subject: [PATCH 13/21] feat(sequencer): add `metrics_address` config field --- Justfile | 20 +++++++++----------- docs/metrics/metrics.md | 2 +- lez/sequencer/core/src/config.rs | 12 ++++++++++++ lez/sequencer/core/src/tests.rs | 1 + lez/sequencer/service/src/main.rs | 26 +++++++++++++------------- test_fixtures/src/config.rs | 1 + 6 files changed, 37 insertions(+), 25 deletions(-) diff --git a/Justfile b/Justfile index ed124de1..4741f6c8 100644 --- a/Justfile +++ b/Justfile @@ -79,19 +79,17 @@ run-monitoring: @echo "📊 Running Prometheus (http://localhost:9090) + Grafana (http://localhost:3000)" docker compose up -# Run Sequencer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. -# Optional home/port let a second instance run off the same config, e.g. -# `just run-sequencer "" "$TMPDIR/lez-sequencer2" 3041` for the multi-sequencer demo. +# Run Sequencer. Extra args are forwarded to the binary. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. [working-directory: 'lez/sequencer/service'] -run-sequencer standalone="" home="" port="3040": +run-sequencer *args: @echo "🧠 Running sequencer" - @if [ "{{standalone}}" = "standalone" ]; then \ - echo "🧪 Running in standalone mode"; \ - RUST_LOG=info cargo run --features standalone --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \ - else \ - echo "🚀 Running in normal mode"; \ - RUST_LOG=info cargo run --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \ - fi + RUST_LOG=info cargo run --release -p sequencer_service -- configs/debug/sequencer_config.json {{args}} + +# Run Sequencer with mocked Bedrock clients. Takes the same args as `run-sequencer`. +[working-directory: 'lez/sequencer/service'] +run-sequencer-standalone *args: + @echo "🧪 Running sequencer in standalone mode" + RUST_LOG=info cargo run --features standalone --release -p sequencer_service -- configs/debug/sequencer_config.json {{args}} # Run Indexer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. [working-directory: 'lez/indexer/service'] diff --git a/docs/metrics/metrics.md b/docs/metrics/metrics.md index 3794a846..10298bdb 100644 --- a/docs/metrics/metrics.md +++ b/docs/metrics/metrics.md @@ -65,7 +65,7 @@ For histograms, creating the handle publishes zeroed buckets without recording a ## Exporter setup -`sequencer_service`'s `main.rs` installs the Prometheus recorder on `:9000` with **explicit histogram buckets**. This matters: without buckets, `metrics-exporter-prometheus` renders histograms as rolling-window summaries whose quantiles **reset to `0`** once the window (default 60 s) drains — an idle period reads as "took 0 s" rather than "no data". With buckets you get real `_bucket`/`_sum`/`_count` counters that never decay, are aggregatable, and honour the dashboard's time range. +`sequencer_service`'s `main.rs` installs the Prometheus recorder on the config's `metrics_address` (default `0.0.0.0:9000`) with **explicit histogram buckets**. This matters: without buckets, `metrics-exporter-prometheus` renders histograms as rolling-window summaries whose quantiles **reset to `0`** once the window (default 60 s) drains — an idle period reads as "took 0 s" rather than "no data". With buckets you get real `_bucket`/`_sum`/`_count` counters that never decay, are aggregatable, and honour the dashboard's time range. Ladders are matched by name suffix, so a new timing metric is covered automatically: diff --git a/lez/sequencer/core/src/config.rs b/lez/sequencer/core/src/config.rs index 60bd8502..42cd4945 100644 --- a/lez/sequencer/core/src/config.rs +++ b/lez/sequencer/core/src/config.rs @@ -1,6 +1,7 @@ use std::{ fs::File, io::BufReader, + net::{IpAddr, Ipv4Addr, SocketAddr}, path::{Path, PathBuf}, time::Duration, }; @@ -63,6 +64,9 @@ pub struct SequencerConfig { /// Cross-zone messaging configuration. `None` disables the watcher. #[serde(default)] pub cross_zone: Option, + /// Address the Prometheus metrics exporter binds to. + #[serde(default = "default_metrics_address")] + pub metrics_address: SocketAddr, } #[derive(Clone, Serialize, Deserialize)] @@ -77,6 +81,10 @@ pub struct BedrockConfig { } impl SequencerConfig { + /// Address [`Self::metrics_address`] falls back to when the config omits it. + pub const DEFAULT_METRICS_ADDRESS: SocketAddr = + SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 9000); + pub fn from_path(config_home: &Path) -> Result { let file = File::open(config_home)?; let reader = BufReader::new(file); @@ -88,3 +96,7 @@ impl SequencerConfig { const fn default_max_block_size() -> ByteSize { ByteSize::mib(1) } + +const fn default_metrics_address() -> SocketAddr { + SequencerConfig::DEFAULT_METRICS_ADDRESS +} diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 863bb99e..10b398e0 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -90,6 +90,7 @@ fn setup_sequencer_config() -> SequencerConfig { retry_pending_blocks_timeout: Duration::from_mins(4), genesis: vec![], cross_zone: None, + metrics_address: SequencerConfig::DEFAULT_METRICS_ADDRESS, } } diff --git a/lez/sequencer/service/src/main.rs b/lez/sequencer/service/src/main.rs index e648bd70..09df20f9 100644 --- a/lez/sequencer/service/src/main.rs +++ b/lez/sequencer/service/src/main.rs @@ -25,6 +25,10 @@ struct Args { /// so multiple instances can share one config file. #[clap(long)] home: Option, + /// Override the config's `metrics_address`, so multiple instances can share + /// one config file without fighting over the exporter port. + #[clap(long)] + metrics_address: Option, } #[tokio::main] @@ -35,13 +39,12 @@ struct Args { async fn main() -> Result<()> { env_logger::init(); - install_prometheus_recorder()?; - let Args { config_path, port, listen_address, home, + metrics_address, } = Args::parse(); let cancellation_token = listen_for_shutdown_signal(); @@ -50,6 +53,11 @@ async fn main() -> Result<()> { if let Some(home) = home { config.home = home; } + if let Some(metrics_address) = metrics_address { + config.metrics_address = metrics_address; + } + + install_prometheus_recorder(config.metrics_address)?; let mut sequencer_handle = sequencer_service::run(config, SocketAddr::new(listen_address, port)).await?; @@ -74,17 +82,8 @@ async fn main() -> Result<()> { Ok(()) } -/// Installs the recorder with explicit buckets, which makes every histogram -/// export as a Prometheus histogram (`_bucket`/`_sum`/`_count`) rather than the -/// default rolling-window summary. Summary quantiles reset to zero once their -/// window drains, so an idle period reads as "took 0s" instead of "no data". -/// -/// Ladders are picked by name suffix, so a new timing metric is covered without -/// touching this function. The matcher sees the name as registered, *before* -/// [`PrometheusBuilder::with_recommended_naming`] appends a unit suffix of its -/// own — a duration metric whose name omits `_seconds` silently falls through to -/// [`COUNT_BUCKETS`]. -fn install_prometheus_recorder() -> Result<()> { +/// Installs the recorder on `metrics_address`. +fn install_prometheus_recorder(metrics_address: SocketAddr) -> Result<()> { /// Ladder for `*_seconds` histograms, densest across the 1–100 ms band where /// block production and transaction application actually land. const LATENCY_BUCKETS: &[f64] = &[ @@ -95,6 +94,7 @@ fn install_prometheus_recorder() -> Result<()> { const COUNT_BUCKETS: &[f64] = &[1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0]; PrometheusBuilder::new() + .with_http_listener(metrics_address) .with_recommended_naming(true) .set_buckets(COUNT_BUCKETS) .context("Failed to set default histogram buckets")? diff --git a/test_fixtures/src/config.rs b/test_fixtures/src/config.rs index 8df57fb3..4de7f511 100644 --- a/test_fixtures/src/config.rs +++ b/test_fixtures/src/config.rs @@ -110,6 +110,7 @@ pub fn sequencer_config( auth: None, }, cross_zone, + metrics_address: SequencerConfig::DEFAULT_METRICS_ADDRESS, }) } From 97152b8df678462052eb3ac381580b4b4938c241 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 23:51:41 +0300 Subject: [PATCH 14/21] chore(sequencer): apply snake_case convention to labels --- lez/sequencer/core/metrics/src/record.rs | 22 ++++++++++++++++++++-- lez/sequencer/core/src/lib.rs | 2 +- lez/sequencer/service/metrics/src/lib.rs | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/lez/sequencer/core/metrics/src/record.rs b/lez/sequencer/core/metrics/src/record.rs index 1f6131f6..59c9087a 100644 --- a/lez/sequencer/core/metrics/src/record.rs +++ b/lez/sequencer/core/metrics/src/record.rs @@ -6,18 +6,36 @@ use std::time::Duration; -use common::transaction::TxKind; use metrics::{Counter, Histogram, Unit, counter, gauge, histogram}; use strum::IntoEnumIterator as _; use crate::names; -#[derive(Clone, Copy, strum::IntoStaticStr, strum::EnumIter)] +#[derive(Debug, Clone, Copy, strum::IntoStaticStr, strum::EnumIter)] +#[strum(serialize_all = "snake_case")] pub enum TransactionOrigin { User, Sequencer, } +#[derive(Debug, Clone, Copy, strum::IntoStaticStr, strum::EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum TxKind { + Public, + PrivacyPreserving, + ProgramDeployment, +} + +impl From for TxKind { + fn from(kind: common::transaction::TxKind) -> Self { + match kind { + common::transaction::TxKind::Public => Self::Public, + common::transaction::TxKind::PrivacyPreserving => Self::PrivacyPreserving, + common::transaction::TxKind::ProgramDeployment => Self::ProgramDeployment, + } + } +} + /// Initialize metrics. pub fn init() { blocks_total_counter().increment(0); diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index ca079ab7..1769a8a7 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -898,7 +898,7 @@ impl SequencerCore { ); sequencer_core_metrics::record_mempool_transaction_application_time( origin.into(), - tx.kind(), + tx.kind().into(), before_tx_apply.elapsed(), ); diff --git a/lez/sequencer/service/metrics/src/lib.rs b/lez/sequencer/service/metrics/src/lib.rs index b8884fa2..f375ff1b 100644 --- a/lez/sequencer/service/metrics/src/lib.rs +++ b/lez/sequencer/service/metrics/src/lib.rs @@ -1,4 +1,4 @@ -//! This crate provides all metrics exposed by the sequencer core crate. +//! This crate provides all metrics exposed by the sequencer service crate. #[cfg(feature = "record")] pub use record::*; From f4f78ddc28d65a05e4031f14142588502bb2fcc3 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 23:54:55 +0300 Subject: [PATCH 15/21] fix(sequencer): expose 9000 metrics port from docker --- lez/sequencer/service/Dockerfile | 3 ++- lez/sequencer/service/docker-compose.yml | 1 + monitoring/prometheus/prometheus.yml | 5 ----- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lez/sequencer/service/Dockerfile b/lez/sequencer/service/Dockerfile index 1919f775..2e0b89a9 100644 --- a/lez/sequencer/service/Dockerfile +++ b/lez/sequencer/service/Dockerfile @@ -63,8 +63,9 @@ COPY --from=builder --chown=sequencer_service_user:sequencer_service_user /usr/l VOLUME /var/lib/sequencer_service -# Expose default port +# Expose default ports EXPOSE 3040 +EXPOSE 9000 # Health check (TODO #244: Replace when a real health endpoint is available) HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ diff --git a/lez/sequencer/service/docker-compose.yml b/lez/sequencer/service/docker-compose.yml index 477072ad..d9c573d3 100644 --- a/lez/sequencer/service/docker-compose.yml +++ b/lez/sequencer/service/docker-compose.yml @@ -18,6 +18,7 @@ services: container_name: sequencer_service ports: - "3040:3040" + - "9000:9000" volumes: # Mount configuration file - ./configs/docker/sequencer_config.json:/etc/sequencer_service/sequencer_config.json diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index aaae81b0..a4793aba 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -6,10 +6,5 @@ scrape_configs: - job_name: sequencer metrics_path: /metrics static_configs: - # `sequencer_service:9000` resolves when monitoring runs inside the - # all-in-one compose network. `host.docker.internal:9000` resolves when - # the sequencer runs natively on the host and only monitoring runs in Docker. - # Whichever is unreachable stays `down`. - targets: - - sequencer_service:9000 - host.docker.internal:9000 From 037901fe941e66ebb73f5c83f4a7bacdcafaee36 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Mon, 3 Aug 2026 19:08:30 +0300 Subject: [PATCH 16/21] fix(sequencer): differentiate between height and blocks produced by this sequencer --- docs/metrics/metrics.md | 18 +++---- lez/sequencer/core/metrics/src/names.rs | 3 +- lez/sequencer/core/metrics/src/record.rs | 29 ++++++---- lez/sequencer/core/src/lib.rs | 15 ++++-- monitoring/grafana/dashboards/sequencer.json | 53 ++++++++++++++----- .../dashboard_gen/src/dashboards/sequencer.rs | 22 ++++++-- 6 files changed, 98 insertions(+), 42 deletions(-) diff --git a/docs/metrics/metrics.md b/docs/metrics/metrics.md index 10298bdb..4d758157 100644 --- a/docs/metrics/metrics.md +++ b/docs/metrics/metrics.md @@ -8,7 +8,7 @@ Every crate that emits metrics gets a sibling `metrics` crate — `lez/sequencer | Module | Gated by | Contents | |---|---|---| -| `names` | always compiled | `pub const BLOCKS_TOTAL: &str = "blocks_total";` — one const per metric | +| `names` | always compiled | `pub const BLOCKS_PRODUCED_TOTAL: &str = "blocks_produced_total";` — one const per metric | | `record` | `record` feature | `record_*` / `increment_*` functions, plus `init()` | The emitting crate depends on it with `features = ["record"]`; consumers that only need the names (i.e. `dashboard_gen`) take the default features and pull in nothing. Dashboards reference the same consts the recording code does, so **renaming a metric is a compile error rather than a silently empty panel**. @@ -19,7 +19,7 @@ The recorder runs with `with_recommended_naming(true)`, which enforces Prometheu | Kind | Suffix | Example | |---|---|---| -| Counter | `_total` | `blocks_total`, `submitted_transactions_total` | +| Counter | `_total` | `blocks_produced_total`, `submitted_transactions_total` | | Histogram | unit | `block_creation_time_seconds` | | Gauge | none | `mempool_size` | @@ -30,22 +30,22 @@ The recorder runs with `with_recommended_naming(true)`, which enforces Prometheu | Type | Use for | Example | |---|---|---| | Counter | monotonically increasing event counts | `mempool_failed_transactions_total` | -| Gauge | a value that moves both ways | `mempool_size` | +| Gauge | a value that moves both ways | `mempool_size`, `chain_height` (a reorg lowers it) | | Histogram | distributions — latencies, sizes, per-batch counts | `mempool_transaction_application_time_seconds` | Each metric gets a private constructor plus a public recording wrapper, so its description, unit and labels are declared once: ```rust -fn blocks_total_counter() -> Counter { +fn blocks_produced_total_counter() -> Counter { counter!( - description: "Number of blocks in chain", + description: "Number of blocks produced by this sequencer and applied to the head", unit: Unit::Count, - names::BLOCKS_TOTAL + names::BLOCKS_PRODUCED_TOTAL ) } -pub fn increment_blocks_total() { - blocks_total_counter().increment(1); +pub fn increment_blocks_produced_total() { + blocks_produced_total_counter().increment(1); } ``` @@ -89,7 +89,7 @@ Panels are built fluently, and every query is composed from the `names` consts: ```rust Panel::timeseries("Block production rate") .width(18) - .target(rate_per_min(sequencer_core_metrics::names::BLOCKS_TOTAL, "blocks/min")) + .target(rate_per_min(sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL, "blocks/min")) ``` Query helpers: `rate_per_min` for counters, `avg` for histograms, and `selected_percentile` for percentile lines — the latter reads a `percentile` dashboard dropdown created by `percentile_variable`, so one panel serves p50/p90/p95/p99 instead of drawing all four. Rate windows use `$__rate_interval`, which tracks the panel's zoom. diff --git a/lez/sequencer/core/metrics/src/names.rs b/lez/sequencer/core/metrics/src/names.rs index 98034cd6..92b1595e 100644 --- a/lez/sequencer/core/metrics/src/names.rs +++ b/lez/sequencer/core/metrics/src/names.rs @@ -1,5 +1,6 @@ pub const BLOCK_CREATION_TIME: &str = "block_creation_time_seconds"; -pub const BLOCKS_TOTAL: &str = "blocks_total"; +pub const CHAIN_HEIGHT: &str = "chain_height"; +pub const BLOCKS_PRODUCED_TOTAL: &str = "blocks_produced_total"; pub const MEMPOOL_SIZE: &str = "mempool_size"; pub const MEMPOOL_MAX_SIZE: &str = "mempool_max_size"; pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = diff --git a/lez/sequencer/core/metrics/src/record.rs b/lez/sequencer/core/metrics/src/record.rs index 59c9087a..9a9b3f42 100644 --- a/lez/sequencer/core/metrics/src/record.rs +++ b/lez/sequencer/core/metrics/src/record.rs @@ -38,9 +38,10 @@ impl From for TxKind { /// Initialize metrics. pub fn init() { - blocks_total_counter().increment(0); + blocks_produced_total_counter().increment(0); mempool_failed_transactions_total_counter().increment(0); record_mempool_size(0); + record_chain_height(0); drop(block_creation_time_histogram()); drop(transactions_per_block_histogram()); @@ -63,20 +64,26 @@ pub fn record_block_creation_time(duration: Duration) { block_creation_time_histogram().record(duration.as_secs_f64()); } -fn blocks_total_counter() -> Counter { - counter!( - description: "Number of blocks in chain", +/// Height of the chain head, which moves backwards on a reorg, hence a gauge. +pub fn record_chain_height(height: u64) { + gauge!( + description: "Height of the chain head", unit: Unit::Count, - names::BLOCKS_TOTAL + names::CHAIN_HEIGHT + ) + .set(height as f64); +} + +fn blocks_produced_total_counter() -> Counter { + counter!( + description: "Number of blocks produced by this sequencer and applied to the head", + unit: Unit::Count, + names::BLOCKS_PRODUCED_TOTAL ) } -pub fn set_blocks_total(value: u64) { - blocks_total_counter().absolute(value); -} - -pub fn increment_blocks_total() { - blocks_total_counter().increment(1); +pub fn increment_blocks_produced_total() { + blocks_produced_total_counter().increment(1); } pub fn record_mempool_size(size: usize) { diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 1769a8a7..77888b35 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -153,6 +153,9 @@ impl SequencerCore { ) .expect("Failed to create database with genesis block"); + // Incrementing count for genesis. + sequencer_core_metrics::increment_blocks_produced_total(); + (store, genesis_state) } } @@ -311,7 +314,7 @@ impl SequencerCore { watchers, }; - sequencer_core_metrics::set_blocks_total(sequencer_core.chain_height()); + sequencer_core_metrics::record_chain_height(sequencer_core.chain_height()); (sequencer_core, mempool_handle) } @@ -610,6 +613,9 @@ impl SequencerCore { chain.head_state(), Some(&checkpoint_bytes), )?; + + sequencer_core_metrics::increment_blocks_produced_total(); + sequencer_core_metrics::record_chain_height(block.header.block_id); } // Neither branch persists anything, checkpoint included: the // inscription it holds as pending belongs to a block that is not @@ -941,7 +947,6 @@ impl SequencerCore { ); sequencer_core_metrics::record_block_creation_time(now.elapsed()); - sequencer_core_metrics::increment_blocks_total(); Ok(BlockWithMeta { block, withdrawals }) } @@ -1187,7 +1192,7 @@ fn apply_follow_update( // The lock is held across the persist below so disk writes land in apply // order — the produce path persists under this same lock. - let (resubmit_txs, outcome) = { + let (resubmit_txs, outcome, head_height) = { let mut chain = chain.lock().expect("chain state mutex poisoned"); // Outcomes align with `adopted`. @@ -1287,9 +1292,11 @@ fn apply_follow_update( }) .unwrap_or_else(|err| panic!("Failed to persist follow update: {err:#}")); - (resubmit_txs, outcome) + (resubmit_txs, outcome, head_tip.map_or(0, |tip| tip.id)) }; + sequencer_core_metrics::record_chain_height(head_height); + if outcome.accepted_deposits > 0 { info!( "Recorded {} Bedrock Deposit event(s); their mints are drained from the store on our next turn", diff --git a/monitoring/grafana/dashboards/sequencer.json b/monitoring/grafana/dashboards/sequencer.json index 20627045..85ee0b28 100644 --- a/monitoring/grafana/dashboards/sequencer.json +++ b/monitoring/grafana/dashboards/sequencer.json @@ -19,7 +19,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "blocks_total", + "expr": "chain_height", "legendFormat": "height", "refId": "A" } @@ -30,11 +30,40 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { - "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" }, + "defaults": { "color": { "mode": "fixed", "fixedColor": "green" }, "unit": "short", "decimals": 0 }, "overrides": [ ] }, - "gridPos": { "h": 7, "w": 18, "x": 6, "y": 0 }, + "gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 }, "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "blocks_produced_total", + "legendFormat": "produced", + "refId": "A" + } + ], + "title": "Blocks produced by this sequencer since startup", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "produced · blocks/min" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] + } + ] + }, + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, + "id": 3, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "single" } @@ -42,8 +71,8 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(blocks_total[1m]) * 60", - "legendFormat": "blocks/min", + "expr": "rate(blocks_produced_total[1m]) * 60", + "legendFormat": "produced · blocks/min", "refId": "A" } ], @@ -65,7 +94,7 @@ ] }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 7 }, - "id": 3, + "id": 4, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } @@ -94,7 +123,7 @@ "overrides": [ ] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 16 }, - "id": 4, + "id": 5, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } @@ -125,7 +154,7 @@ ] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 }, - "id": 5, + "id": 6, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } @@ -167,7 +196,7 @@ "overrides": [ ] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 25 }, - "id": 6, + "id": 7, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, @@ -207,7 +236,7 @@ ] }, "gridPos": { "h": 8, "w": 18, "x": 6, "y": 25 }, - "id": 7, + "id": 8, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } @@ -249,7 +278,7 @@ "overrides": [ ] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 33 }, - "id": 8, + "id": 9, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, @@ -290,7 +319,7 @@ ] }, "gridPos": { "h": 8, "w": 18, "x": 6, "y": 33 }, - "id": 9, + "id": 10, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } diff --git a/tools/dashboard_gen/src/dashboards/sequencer.rs b/tools/dashboard_gen/src/dashboards/sequencer.rs index 3c609841..2956edfa 100644 --- a/tools/dashboard_gen/src/dashboards/sequencer.rs +++ b/tools/dashboard_gen/src/dashboards/sequencer.rs @@ -27,15 +27,27 @@ pub fn dashboard() -> Dashboard { .decimals(0) .color(Color::fixed("blue")) .target( - Target::new(sequencer_core_metrics::names::BLOCKS_TOTAL).legend("height"), + Target::new(sequencer_core_metrics::names::CHAIN_HEIGHT).legend("height"), + ), + Panel::stat("Blocks produced by this sequencer since startup") + .width(6) + .unit(Unit::Short) + .decimals(0) + .color(Color::fixed("green")) + .target( + Target::new(sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL) + .legend("produced"), ), Panel::timeseries("Block production rate") - .width(18) + .width(12) .unit(Unit::Short) .target(rate_per_min( - sequencer_core_metrics::names::BLOCKS_TOTAL, - "blocks/min", - )), + sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL, + "produced · blocks/min", + )) + .with_override( + FieldOverride::by_name("produced · blocks/min").color(Color::fixed("green")), + ), ], ) .row( From acb73dce0cfa88a797fa96fed815d1075fad85fc Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Mon, 3 Aug 2026 19:29:16 +0300 Subject: [PATCH 17/21] fix(dashboard_gen): use RATE_WINDOW in rate_per_min calculation --- monitoring/grafana/dashboards/sequencer.json | 8 ++++---- .../grafana/provisioning/datasources/prometheus.yml | 3 +++ tools/dashboard_gen/src/lib.rs | 11 +++++++---- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/monitoring/grafana/dashboards/sequencer.json b/monitoring/grafana/dashboards/sequencer.json index 85ee0b28..f9297c80 100644 --- a/monitoring/grafana/dashboards/sequencer.json +++ b/monitoring/grafana/dashboards/sequencer.json @@ -71,7 +71,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(blocks_produced_total[1m]) * 60", + "expr": "rate(blocks_produced_total[$__rate_interval]) * 60", "legendFormat": "produced · blocks/min", "refId": "A" } @@ -327,19 +327,19 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(submitted_transactions_total[1m]) * 60", + "expr": "rate(submitted_transactions_total[$__rate_interval]) * 60", "legendFormat": "submitted", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(before_mempool_failed_transactions_total[1m]) * 60", + "expr": "rate(before_mempool_failed_transactions_total[$__rate_interval]) * 60", "legendFormat": "failed · before mempool", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(mempool_failed_transactions_total[1m]) * 60", + "expr": "rate(mempool_failed_transactions_total[$__rate_interval]) * 60", "legendFormat": "failed · in mempool", "refId": "C" } diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml index 0b304bc9..9bf28c50 100644 --- a/monitoring/grafana/provisioning/datasources/prometheus.yml +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -8,3 +8,6 @@ datasources: url: http://prometheus:9090 isDefault: true editable: true + jsonData: + # Must match prometheus.yml's `scrape_interval` + timeInterval: 5s diff --git a/tools/dashboard_gen/src/lib.rs b/tools/dashboard_gen/src/lib.rs index 350188a2..06104582 100644 --- a/tools/dashboard_gen/src/lib.rs +++ b/tools/dashboard_gen/src/lib.rs @@ -36,9 +36,10 @@ mod unit; /// than by a per-environment URL. pub const DATASOURCE_UID: &str = "prometheus"; -/// Window every histogram query rates over. `$__rate_interval` tracks the -/// panel's zoom, so a percentile covers the range you are actually looking at, -/// and an idle window yields no value rather than a zero. +/// Window every rate query — counter and histogram alike — rates over. +/// `$__rate_interval` tracks the panel's zoom, so the window always covers the +/// step Grafana samples at: a fixed one shorter than the step leaves gaps the +/// query never looks at, dropping events from the graph entirely. const RATE_WINDOW: &str = "$__rate_interval"; /// Dashboard variable holding the quantile every percentile query reads. @@ -578,7 +579,9 @@ pub fn avg(metric: &str) -> Target { /// A per-minute rate target for a counter metric. #[must_use] pub fn rate_per_min(metric: &str, legend: &str) -> Target { - Target::new(format!("rate({metric}[1m]) * 60")).legend(legend) + //`rate` is per-second whatever the window, so `* 60` is per-minute regardless of the panel's + //`rate` zoom. + Target::new(format!("rate({metric}[{RATE_WINDOW}]) * 60")).legend(legend) } const fn default_unit() -> Unit { From 69ecc116a84d51b09ed787c9e824de4aeb5f1ca4 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Mon, 3 Aug 2026 19:30:42 +0300 Subject: [PATCH 18/21] fix(common): drop strum --- Cargo.lock | 1 - lez/common/Cargo.toml | 1 - lez/common/src/transaction.rs | 12 +----------- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 86d8102a..634cd99e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1687,7 +1687,6 @@ dependencies = [ "serde", "serde_with", "sha2 0.10.9", - "strum 0.28.0", "system_accounts", "thiserror 2.0.18", ] diff --git a/lez/common/Cargo.toml b/lez/common/Cargo.toml index d03dcbbf..7582e885 100644 --- a/lez/common/Cargo.toml +++ b/lez/common/Cargo.toml @@ -25,7 +25,6 @@ log.workspace = true hex.workspace = true borsh.workspace = true logos-blockchain-common-http-client.workspace = true -strum.workspace = true [dev-dependencies] lee = { workspace = true, features = ["test-utils"] } diff --git a/lez/common/src/transaction.rs b/lez/common/src/transaction.rs index 22d4b70e..9970bf7d 100644 --- a/lez/common/src/transaction.rs +++ b/lez/common/src/transaction.rs @@ -209,17 +209,7 @@ impl From for LeeTransaction { } #[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - Serialize, - Deserialize, - BorshSerialize, - BorshDeserialize, - strum::IntoStaticStr, - strum::EnumIter, + Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize, )] pub enum TxKind { Public, From 218955dd19c46f7e251e7eac6914edebe2284885 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Mon, 3 Aug 2026 19:40:14 +0300 Subject: [PATCH 19/21] feat(sequencer): add status label to transaction application time --- lez/sequencer/core/metrics/src/record.rs | 20 +++++++++++++++++-- lez/sequencer/core/src/lib.rs | 18 +++++++++++------ monitoring/grafana/dashboards/sequencer.json | 4 ++-- .../dashboard_gen/src/dashboards/sequencer.rs | 4 ++-- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/lez/sequencer/core/metrics/src/record.rs b/lez/sequencer/core/metrics/src/record.rs index 9a9b3f42..96b3304e 100644 --- a/lez/sequencer/core/metrics/src/record.rs +++ b/lez/sequencer/core/metrics/src/record.rs @@ -26,6 +26,14 @@ pub enum TxKind { ProgramDeployment, } +/// Whether applying a transaction to the block's working state succeeded. +#[derive(Debug, Clone, Copy, strum::IntoStaticStr, strum::EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum ApplyStatus { + Applied, + Failed, +} + impl From for TxKind { fn from(kind: common::transaction::TxKind) -> Self { match kind { @@ -47,7 +55,11 @@ pub fn init() { drop(transactions_per_block_histogram()); for origin in TransactionOrigin::iter() { for kind in TxKind::iter() { - drop(mempool_transaction_application_time_histogram(origin, kind)); + for status in ApplyStatus::iter() { + drop(mempool_transaction_application_time_histogram( + origin, kind, status, + )); + } } } } @@ -107,6 +119,7 @@ pub fn record_mempool_max_size(size: usize) { fn mempool_transaction_application_time_histogram( origin: TransactionOrigin, kind: TxKind, + status: ApplyStatus, ) -> Histogram { histogram!( description: "Time taken to apply a mempool transaction", @@ -114,15 +127,18 @@ fn mempool_transaction_application_time_histogram( names::MEMPOOL_TRANSACTION_APPLICATION_TIME, "origin" => <&'static str>::from(origin), "kind" => <&'static str>::from(kind), + "status" => <&'static str>::from(status), ) } pub fn record_mempool_transaction_application_time( origin: TransactionOrigin, kind: TxKind, + status: ApplyStatus, duration: Duration, ) { - mempool_transaction_application_time_histogram(origin, kind).record(duration.as_secs_f64()); + mempool_transaction_application_time_histogram(origin, kind, status) + .record(duration.as_secs_f64()); } fn transactions_per_block_histogram() -> Histogram { diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 77888b35..e38a19ca 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -902,16 +902,22 @@ impl SequencerCore { new_block_timestamp, &mut withdrawals, ); - sequencer_core_metrics::record_mempool_transaction_application_time( - origin.into(), - tx.kind().into(), - before_tx_apply.elapsed(), - ); - if applied { + sequencer_core_metrics::record_mempool_transaction_application_time( + origin.into(), + tx.kind().into(), + sequencer_core_metrics::ApplyStatus::Applied, + before_tx_apply.elapsed(), + ); valid_transactions.push(tx); } else { sequencer_core_metrics::increment_mempool_failed_transactions_total(); + sequencer_core_metrics::record_mempool_transaction_application_time( + origin.into(), + tx.kind().into(), + sequencer_core_metrics::ApplyStatus::Failed, + before_tx_apply.elapsed(), + ); // A failed transaction is simply left out of the block, except a // dispatch: that one is re-fed from the store every turn, so one // that can never execute would fail on every block for ever. diff --git a/monitoring/grafana/dashboards/sequencer.json b/monitoring/grafana/dashboards/sequencer.json index f9297c80..ede0ae4b 100644 --- a/monitoring/grafana/dashboards/sequencer.json +++ b/monitoring/grafana/dashboards/sequencer.json @@ -131,8 +131,8 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(${percentile}, sum by (le, kind, origin) (rate(mempool_transaction_application_time_seconds_bucket[$__rate_interval])))", - "legendFormat": "{{kind}} · {{origin}}", + "expr": "histogram_quantile(${percentile}, sum by (le, kind, origin, status) (rate(mempool_transaction_application_time_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{kind}} · {{origin}} · {{status}}", "refId": "A" } ], diff --git a/tools/dashboard_gen/src/dashboards/sequencer.rs b/tools/dashboard_gen/src/dashboards/sequencer.rs index 2956edfa..0520251c 100644 --- a/tools/dashboard_gen/src/dashboards/sequencer.rs +++ b/tools/dashboard_gen/src/dashboards/sequencer.rs @@ -75,8 +75,8 @@ pub fn dashboard() -> Dashboard { .unit(Unit::Seconds) .target(selected_percentile( sequencer_core_metrics::names::MEMPOOL_TRANSACTION_APPLICATION_TIME, - &["kind", "origin"], - "{{kind}} · {{origin}}", + &["kind", "origin", "status"], + "{{kind}} · {{origin}} · {{status}}", )), Panel::timeseries("Transactions per block") .width(12) From 8bbce62290e8dacc55ca565ec9c2aa9a3d4a5460 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Mon, 3 Aug 2026 19:44:55 +0300 Subject: [PATCH 20/21] chore(dashboard_gen): add quantile range check --- tools/dashboard_gen/src/lib.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/dashboard_gen/src/lib.rs b/tools/dashboard_gen/src/lib.rs index 06104582..113592c0 100644 --- a/tools/dashboard_gen/src/lib.rs +++ b/tools/dashboard_gen/src/lib.rs @@ -494,7 +494,8 @@ impl Dashboard { /// The dropdown driving every [`selected_percentile`] query, offering /// `percentiles` (e.g. `[50, 90, 95, 99]`) with `default` pre-selected. /// -/// Panics if `default` is not one of `percentiles`. +/// Panics if `default` is not one of `percentiles`, or if any of them falls +/// outside `p1..=p99`. #[must_use] pub fn percentile_variable(percentiles: &[u32], default: u32) -> Variable { assert!( @@ -562,7 +563,15 @@ pub fn selected_percentile(metric: &str, labels: &[&str], legend: &str) -> Targe /// A percentile as its `histogram_quantile` argument, derived without float /// math: zero-pad to two digits then drop trailing zeros (50 → `0.5`). +/// +/// Panics outside `1..=99`, the only range two digits render: p100 would come +/// out as `0.1` and p0 as `0.`, neither of them loudly. fn quantile(percentile: u32) -> String { + assert!( + (1..=99).contains(&percentile), + "p{percentile} is outside the supported range p1..=p99", + ); + let quantile = format!("0.{percentile:02}"); quantile.trim_end_matches('0').to_owned() } From cd835bd0c917cb5e6aeb7408e03fbd6c63091998 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Mon, 3 Aug 2026 23:43:54 +0300 Subject: [PATCH 21/21] feat(sequencer): add possibility to disable metrics --- lez/sequencer/core/src/config.rs | 7 +++--- lez/sequencer/core/src/tests.rs | 2 +- lez/sequencer/service/src/main.rs | 40 ++++++++++++++++++------------- test_fixtures/src/config.rs | 2 +- 4 files changed, 30 insertions(+), 21 deletions(-) diff --git a/lez/sequencer/core/src/config.rs b/lez/sequencer/core/src/config.rs index 42cd4945..35f790dc 100644 --- a/lez/sequencer/core/src/config.rs +++ b/lez/sequencer/core/src/config.rs @@ -66,7 +66,7 @@ pub struct SequencerConfig { pub cross_zone: Option, /// Address the Prometheus metrics exporter binds to. #[serde(default = "default_metrics_address")] - pub metrics_address: SocketAddr, + pub metrics_address: Option, } #[derive(Clone, Serialize, Deserialize)] @@ -97,6 +97,7 @@ const fn default_max_block_size() -> ByteSize { ByteSize::mib(1) } -const fn default_metrics_address() -> SocketAddr { - SequencerConfig::DEFAULT_METRICS_ADDRESS +#[expect(clippy::unnecessary_wraps, reason = "Required by serde")] +const fn default_metrics_address() -> Option { + Some(SequencerConfig::DEFAULT_METRICS_ADDRESS) } diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 10b398e0..00f78c4d 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -90,7 +90,7 @@ fn setup_sequencer_config() -> SequencerConfig { retry_pending_blocks_timeout: Duration::from_mins(4), genesis: vec![], cross_zone: None, - metrics_address: SequencerConfig::DEFAULT_METRICS_ADDRESS, + metrics_address: None, } } diff --git a/lez/sequencer/service/src/main.rs b/lez/sequencer/service/src/main.rs index 09df20f9..b3d5bf71 100644 --- a/lez/sequencer/service/src/main.rs +++ b/lez/sequencer/service/src/main.rs @@ -39,27 +39,18 @@ struct Args { async fn main() -> Result<()> { env_logger::init(); - let Args { - config_path, - port, - listen_address, - home, - metrics_address, - } = Args::parse(); + let args = Args::parse(); let cancellation_token = listen_for_shutdown_signal(); - let mut config = sequencer_service::SequencerConfig::from_path(&config_path)?; - if let Some(home) = home { - config.home = home; - } - if let Some(metrics_address) = metrics_address { - config.metrics_address = metrics_address; - } + let mut config = sequencer_service::SequencerConfig::from_path(&args.config_path)?; + apply_config_overrides(&args, &mut config); - install_prometheus_recorder(config.metrics_address)?; + if let Some(metrics_address) = config.metrics_address { + install_prometheus_recorder(metrics_address)?; + } let mut sequencer_handle = - sequencer_service::run(config, SocketAddr::new(listen_address, port)).await?; + sequencer_service::run(config, SocketAddr::new(args.listen_address, args.port)).await?; tokio::select! { () = cancellation_token.cancelled() => { @@ -82,6 +73,23 @@ async fn main() -> Result<()> { Ok(()) } +fn apply_config_overrides(args: &Args, config: &mut sequencer_service::SequencerConfig) { + let Args { + home, + metrics_address, + config_path: _, + port: _, + listen_address: _, + } = args; + + if let Some(home) = home { + config.home.clone_from(home); + } + if let Some(metrics_address) = metrics_address { + config.metrics_address = Some(*metrics_address); + } +} + /// Installs the recorder on `metrics_address`. fn install_prometheus_recorder(metrics_address: SocketAddr) -> Result<()> { /// Ladder for `*_seconds` histograms, densest across the 1–100 ms band where diff --git a/test_fixtures/src/config.rs b/test_fixtures/src/config.rs index 4de7f511..9a8bf53f 100644 --- a/test_fixtures/src/config.rs +++ b/test_fixtures/src/config.rs @@ -110,7 +110,7 @@ pub fn sequencer_config( auth: None, }, cross_zone, - metrics_address: SequencerConfig::DEFAULT_METRICS_ADDRESS, + metrics_address: Some(SequencerConfig::DEFAULT_METRICS_ADDRESS), }) }