From aa6969386ec9aba8bfe0c05ca2c0688ce260c497 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Tue, 28 Jul 2026 17:04:44 +0300 Subject: [PATCH] refactor(metrics): improve metrics setup --- .gitignore | 1 + lez/common/src/transaction.rs | 1 + lez/sequencer/core/metrics/src/names.rs | 9 +- lez/sequencer/core/metrics/src/writing.rs | 80 +++++++---- lez/sequencer/core/src/lib.rs | 8 +- lez/sequencer/service/metrics/src/names.rs | 4 +- lez/sequencer/service/metrics/src/writing.rs | 28 +++- lez/sequencer/service/src/lib.rs | 2 + lez/sequencer/service/src/main.rs | 36 ++++- lez/sequencer/service/src/service.rs | 85 ++++++----- monitoring/grafana/dashboards/sequencer.json | 126 +++++++--------- .../src/bin/gen_sequencer_dashboard.rs | 54 ++++--- tools/dashboard_gen/src/codegen.rs | 7 +- tools/dashboard_gen/src/input.rs | 2 + tools/dashboard_gen/src/lib.rs | 134 ++++++++++++++---- tools/dashboard_gen/src/schema.rs | 37 +++++ tools/dashboard_gen/src/styling.rs | 20 ++- 17 files changed, 434 insertions(+), 200 deletions(-) diff --git a/.gitignore b/.gitignore index f32b258c..3befdb9c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ data/ rocksdb* sequencer/service/data/ storage.json +statistics.json result diff --git a/lez/common/src/transaction.rs b/lez/common/src/transaction.rs index 3c4df83c..22d4b70e 100644 --- a/lez/common/src/transaction.rs +++ b/lez/common/src/transaction.rs @@ -219,6 +219,7 @@ impl From for LeeTransaction { BorshSerialize, BorshDeserialize, strum::IntoStaticStr, + strum::EnumIter, )] pub enum TxKind { Public, diff --git a/lez/sequencer/core/metrics/src/names.rs b/lez/sequencer/core/metrics/src/names.rs index a8aae270..98034cd6 100644 --- a/lez/sequencer/core/metrics/src/names.rs +++ b/lez/sequencer/core/metrics/src/names.rs @@ -1,7 +1,8 @@ -pub const BLOCK_CREATION_TIME: &str = "block_creation_time"; -pub const BLOCK_COUNT: &str = "block_count"; +pub const BLOCK_CREATION_TIME: &str = "block_creation_time_seconds"; +pub const BLOCKS_TOTAL: &str = "blocks_total"; pub const MEMPOOL_SIZE: &str = "mempool_size"; pub const MEMPOOL_MAX_SIZE: &str = "mempool_max_size"; -pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = "mempool_transaction_application_time"; +pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = + "mempool_transaction_application_time_seconds"; pub const TRANSACTIONS_PER_BLOCK: &str = "transactions_per_block"; -pub const FAILED_TRANSACTION_COUNT: &str = "failed_transaction_count"; +pub const MEMPOOL_FAILED_TRANSACTIONS_TOTAL: &str = "mempool_failed_transactions_total"; diff --git a/lez/sequencer/core/metrics/src/writing.rs b/lez/sequencer/core/metrics/src/writing.rs index 320e01fb..1f6131f6 100644 --- a/lez/sequencer/core/metrics/src/writing.rs +++ b/lez/sequencer/core/metrics/src/writing.rs @@ -7,39 +7,58 @@ use std::time::Duration; use common::transaction::TxKind; -use metrics::{Counter, Unit, counter, gauge, histogram}; +use metrics::{Counter, Histogram, Unit, counter, gauge, histogram}; +use strum::IntoEnumIterator as _; use crate::names; -#[derive(Clone, Copy, strum::IntoStaticStr)] +#[derive(Clone, Copy, strum::IntoStaticStr, strum::EnumIter)] pub enum TransactionOrigin { User, Sequencer, } -pub fn record_block_creation_time(duration: Duration) { +/// Initialize metrics. +pub fn init() { + blocks_total_counter().increment(0); + mempool_failed_transactions_total_counter().increment(0); + record_mempool_size(0); + + drop(block_creation_time_histogram()); + drop(transactions_per_block_histogram()); + for origin in TransactionOrigin::iter() { + for kind in TxKind::iter() { + drop(mempool_transaction_application_time_histogram(origin, kind)); + } + } +} + +fn block_creation_time_histogram() -> Histogram { histogram!( description: "Time taken to create a block", unit: Unit::Seconds, names::BLOCK_CREATION_TIME ) - .record(duration.as_secs_f64()); } -fn block_count_counter() -> Counter { +pub fn record_block_creation_time(duration: Duration) { + block_creation_time_histogram().record(duration.as_secs_f64()); +} + +fn blocks_total_counter() -> Counter { counter!( description: "Number of blocks in chain", unit: Unit::Count, - names::BLOCK_COUNT + names::BLOCKS_TOTAL ) } -pub fn set_block_count(value: u64) { - block_count_counter().absolute(value); +pub fn set_blocks_total(value: u64) { + blocks_total_counter().absolute(value); } -pub fn increment_block_count() { - block_count_counter().increment(1); +pub fn increment_blocks_total() { + blocks_total_counter().increment(1); } pub fn record_mempool_size(size: usize) { @@ -60,11 +79,10 @@ pub fn record_mempool_max_size(size: usize) { .set(u64::try_from(size).expect("Mempool max size should fit into u64") as f64); } -pub fn record_mempool_transaction_application_time( +fn mempool_transaction_application_time_histogram( origin: TransactionOrigin, kind: TxKind, - duration: Duration, -) { +) -> Histogram { histogram!( description: "Time taken to apply a mempool transaction", unit: Unit::Seconds, @@ -72,23 +90,37 @@ pub fn record_mempool_transaction_application_time( "origin" => <&'static str>::from(origin), "kind" => <&'static str>::from(kind), ) - .record(duration.as_secs_f64()); } -pub fn record_transactions_per_block(count: usize) { +pub fn record_mempool_transaction_application_time( + origin: TransactionOrigin, + kind: TxKind, + duration: Duration, +) { + mempool_transaction_application_time_histogram(origin, kind).record(duration.as_secs_f64()); +} + +fn transactions_per_block_histogram() -> Histogram { histogram!( - description: "Number of transactions included in block", + description: "Number of transactions from mempool included in block", unit: Unit::Count, names::TRANSACTIONS_PER_BLOCK ) - .record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64); } -pub fn increment_failed_transaction_count() { - counter!( - description: "Number of transactions that failed to be included in blocks", - unit: Unit::Count, - names::FAILED_TRANSACTION_COUNT - ) - .increment(1); +pub fn record_transactions_per_block(count: usize) { + transactions_per_block_histogram() + .record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64); +} + +fn mempool_failed_transactions_total_counter() -> Counter { + counter!( + description: "Number of transactions from mempool that failed to be included in blocks", + unit: Unit::Count, + names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL + ) +} + +pub fn increment_mempool_failed_transactions_total() { + mempool_failed_transactions_total_counter().increment(1); } diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index d20c46cc..ca079ab7 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -204,6 +204,8 @@ impl SequencerCore { pub async fn start_from_config( config: SequencerConfig, ) -> (Self, MemPoolHandle<(TransactionOrigin, LeeTransaction)>) { + sequencer_core_metrics::init(); + let bedrock_signing_key = load_or_create_signing_key(&config.home.join("bedrock_signing_key")) .expect("Failed to load or create bedrock signing key"); @@ -309,7 +311,7 @@ impl SequencerCore { watchers, }; - sequencer_core_metrics::set_block_count(sequencer_core.chain_height()); + sequencer_core_metrics::set_blocks_total(sequencer_core.chain_height()); (sequencer_core, mempool_handle) } @@ -903,7 +905,7 @@ impl SequencerCore { if applied { valid_transactions.push(tx); } else { - sequencer_core_metrics::increment_failed_transaction_count(); + sequencer_core_metrics::increment_mempool_failed_transactions_total(); // A failed transaction is simply left out of the block, except a // dispatch: that one is re-fed from the store every turn, so one // that can never execute would fail on every block for ever. @@ -939,7 +941,7 @@ impl SequencerCore { ); sequencer_core_metrics::record_block_creation_time(now.elapsed()); - sequencer_core_metrics::increment_block_count(); + sequencer_core_metrics::increment_blocks_total(); Ok(BlockWithMeta { block, withdrawals }) } diff --git a/lez/sequencer/service/metrics/src/names.rs b/lez/sequencer/service/metrics/src/names.rs index 8bf61acb..1ac19948 100644 --- a/lez/sequencer/service/metrics/src/names.rs +++ b/lez/sequencer/service/metrics/src/names.rs @@ -1 +1,3 @@ -pub const SUBMITTED_TRANSACTION_COUNT: &str = "submitted_transaction_count"; +pub const SUBMITTED_TRANSACTIONS_TOTAL: &str = "submitted_transactions_total"; +pub const BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL: &str = + "before_mempool_failed_transactions_total"; diff --git a/lez/sequencer/service/metrics/src/writing.rs b/lez/sequencer/service/metrics/src/writing.rs index 3b625006..05eaac1a 100644 --- a/lez/sequencer/service/metrics/src/writing.rs +++ b/lez/sequencer/service/metrics/src/writing.rs @@ -1,12 +1,32 @@ -use metrics::{Unit, counter}; +use metrics::{Counter, Unit, counter}; use crate::names; -pub fn increment_submitted_transaction_count() { +pub fn init() { + submitted_transactions_total_counter().increment(0); + before_mempool_failed_transactions_total_counter().increment(0); +} + +fn submitted_transactions_total_counter() -> Counter { counter!( description: "Number of transactions submitted", unit: Unit::Count, - names::SUBMITTED_TRANSACTION_COUNT + names::SUBMITTED_TRANSACTIONS_TOTAL ) - .increment(1); +} + +pub fn increment_submitted_transactions_total() { + submitted_transactions_total_counter().increment(1); +} + +fn before_mempool_failed_transactions_total_counter() -> Counter { + counter!( + description: "Number of transactions that failed before reaching the mempool", + unit: Unit::Count, + names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL + ) +} + +pub fn increment_before_mempool_failed_transactions_total() { + before_mempool_failed_transactions_total_counter().increment(1); } diff --git a/lez/sequencer/service/src/lib.rs b/lez/sequencer/service/src/lib.rs index ba5b68ec..3073823a 100644 --- a/lez/sequencer/service/src/lib.rs +++ b/lez/sequencer/service/src/lib.rs @@ -211,6 +211,8 @@ async fn wait_for_store_release(store: &StoreRelease) { } pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result { + sequencer_service_metrics::init(); + let block_timeout = config.block_create_timeout; let max_block_size = config.max_block_size; diff --git a/lez/sequencer/service/src/main.rs b/lez/sequencer/service/src/main.rs index 95facfff..e648bd70 100644 --- a/lez/sequencer/service/src/main.rs +++ b/lez/sequencer/service/src/main.rs @@ -6,6 +6,7 @@ use std::{ use anyhow::{Context as _, Result}; use clap::Parser; use log::{error, info}; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder}; use tokio::signal::unix::{SignalKind, signal}; use tokio_util::sync::CancellationToken; @@ -34,10 +35,7 @@ struct Args { async fn main() -> Result<()> { env_logger::init(); - metrics_exporter_prometheus::PrometheusBuilder::new() - .with_recommended_naming(true) - .install() - .context("Failed to install Prometheus recorder")?; + install_prometheus_recorder()?; let Args { config_path, @@ -76,6 +74,36 @@ async fn main() -> Result<()> { Ok(()) } +/// Installs the recorder with explicit buckets, which makes every histogram +/// export as a Prometheus histogram (`_bucket`/`_sum`/`_count`) rather than the +/// default rolling-window summary. Summary quantiles reset to zero once their +/// window drains, so an idle period reads as "took 0s" instead of "no data". +/// +/// Ladders are picked by name suffix, so a new timing metric is covered without +/// touching this function. The matcher sees the name as registered, *before* +/// [`PrometheusBuilder::with_recommended_naming`] appends a unit suffix of its +/// own — a duration metric whose name omits `_seconds` silently falls through to +/// [`COUNT_BUCKETS`]. +fn install_prometheus_recorder() -> Result<()> { + /// Ladder for `*_seconds` histograms, densest across the 1–100 ms band where + /// block production and transaction application actually land. + const LATENCY_BUCKETS: &[f64] = &[ + 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ]; + + /// Fallback ladder for histograms that count things rather than measure time. + const COUNT_BUCKETS: &[f64] = &[1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0]; + + PrometheusBuilder::new() + .with_recommended_naming(true) + .set_buckets(COUNT_BUCKETS) + .context("Failed to set default histogram buckets")? + .set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), LATENCY_BUCKETS) + .context("Failed to set latency histogram buckets")? + .install() + .context("Failed to install Prometheus recorder") +} + /// Cancelled on Ctrl-C or `SIGTERM`. /// /// `SIGTERM` is what a container runtime sends first, so without it every diff --git a/lez/sequencer/service/src/service.rs b/lez/sequencer/service/src/service.rs index 309c9949..e55735c0 100644 --- a/lez/sequencer/service/src/service.rs +++ b/lez/sequencer/service/src/service.rs @@ -6,7 +6,7 @@ use jsonrpsee::{ types::{ErrorCode, ErrorObjectOwned}, }; use lee; -use log::warn; +use log::{error, warn}; use mempool::MemPoolHandle; use sequencer_core::{ DbError, SequencerCore, TransactionOrigin, block_publisher::BlockPublisherTrait, @@ -44,57 +44,68 @@ impl sequencer_service_rpc::Rpc for SequencerService { async fn send_transaction(&self, tx: LeeTransaction) -> Result { - // Reserve ~200 bytes for block header overhead - const BLOCK_HEADER_OVERHEAD: u64 = 200; + sequencer_service_metrics::increment_submitted_transactions_total(); let tx_hash = tx.hash(); - let encoded_tx = - borsh::to_vec(&tx).expect("Transaction borsh serialization should not fail"); - let tx_size = u64::try_from(encoded_tx.len()).expect("Transaction size should fit in u64"); + let res = async move { + // Reserve ~200 bytes for block header overhead + const BLOCK_HEADER_OVERHEAD: u64 = 200; - let max_tx_size = self.max_block_size.saturating_sub(BLOCK_HEADER_OVERHEAD); + let encoded_tx = + borsh::to_vec(&tx).expect("Transaction borsh serialization should not fail"); + let tx_size = + u64::try_from(encoded_tx.len()).expect("Transaction size should fit in u64"); - if tx_size > max_tx_size { - return Err(ErrorObjectOwned::owned( - ErrorCode::InvalidParams.code(), - format!("Transaction too large: size {tx_size}, max {max_tx_size}"), - None::<()>, - )); - } + let max_tx_size = self.max_block_size.saturating_sub(BLOCK_HEADER_OVERHEAD); - let authenticated_tx = tx - .transaction_stateless_check() - .inspect_err(|err| warn!("Error at pre_check {err:#?}")) - .map_err(|err| { - ErrorObjectOwned::owned( + if tx_size > max_tx_size { + return Err(ErrorObjectOwned::owned( ErrorCode::InvalidParams.code(), - format!("{err:?}"), + format!("Transaction too large: size {tx_size}, max {max_tx_size}"), None::<()>, - ) - })?; + )); + } - // Sequencer-only programs (the cross-zone inbox) are injected by the - // watcher; a user must not invoke them top-level, or anyone could forge - // an inbound cross-zone delivery. Chained user calls are already rejected - // by the inbox guest's caller-is-none assertion. - if let LeeTransaction::Public(public_tx) = &authenticated_tx - && sequencer_core::is_sequencer_only_program(public_tx.message().program_id) - { - return Err(ErrorObjectOwned::owned( - ErrorCode::InvalidParams.code(), - "Program is sequencer-only and cannot be invoked by a user transaction".to_owned(), - None::<()>, - )); - } + let authenticated_tx = tx + .transaction_stateless_check() + .inspect_err(|err| warn!("Error at pre_check {err:#?}")) + .map_err(|err| { + ErrorObjectOwned::owned( + ErrorCode::InvalidParams.code(), + format!("{err:?}"), + None::<()>, + ) + })?; + + // Sequencer-only programs (the cross-zone inbox) are injected by the + // watcher; a user must not invoke them top-level, or anyone could forge + // an inbound cross-zone delivery. Chained user calls are already rejected + // by the inbox guest's caller-is-none assertion. + if let LeeTransaction::Public(public_tx) = &authenticated_tx + && sequencer_core::is_sequencer_only_program(public_tx.message().program_id) + { + return Err(ErrorObjectOwned::owned( + ErrorCode::InvalidParams.code(), + "Program is sequencer-only and cannot be invoked by a user transaction" + .to_owned(), + None::<()>, + )); + } + + Ok(authenticated_tx) + }; + + let authenticated_tx = res.await.inspect_err(|err| { + sequencer_service_metrics::increment_before_mempool_failed_transactions_total(); + error!("Transaction failed before reaching mempool: {err:#?}"); + })?; self.mempool_handle .push((TransactionOrigin::User, authenticated_tx)) .await .expect("Mempool is closed, this is a bug"); - sequencer_service_metrics::increment_submitted_transaction_count(); - Ok(tx_hash) } diff --git a/monitoring/grafana/dashboards/sequencer.json b/monitoring/grafana/dashboards/sequencer.json index c2711d73..20627045 100644 --- a/monitoring/grafana/dashboards/sequencer.json +++ b/monitoring/grafana/dashboards/sequencer.json @@ -19,7 +19,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_count", + "expr": "blocks_total", "legendFormat": "height", "refId": "A" } @@ -42,7 +42,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(block_count[1m]) * 60", + "expr": "rate(blocks_total[1m]) * 60", "legendFormat": "blocks/min", "refId": "A" } @@ -73,33 +73,15 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_creation_time{quantile=\"0.5\"}", - "legendFormat": "p50", + "expr": "histogram_quantile(${percentile}, sum by (le) (rate(block_creation_time_seconds_bucket[$__rate_interval])))", + "legendFormat": "${percentile:text}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_creation_time{quantile=\"0.9\"}", - "legendFormat": "p90", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_creation_time{quantile=\"0.95\"}", - "legendFormat": "p95", - "refId": "C" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "block_creation_time{quantile=\"0.99\"}", - "legendFormat": "p99", - "refId": "D" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(block_creation_time_sum[1m]) / rate(block_creation_time_count[1m])", + "expr": "rate(block_creation_time_seconds_sum[$__rate_interval]) / rate(block_creation_time_seconds_count[$__rate_interval])", "legendFormat": "avg", - "refId": "E" + "refId": "B" } ], "title": "Block creation time", @@ -120,27 +102,9 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_transaction_application_time{quantile=\"0.5\"}", - "legendFormat": "p50 · {{kind}} · {{origin}}", + "expr": "histogram_quantile(${percentile}, sum by (le, kind, origin) (rate(mempool_transaction_application_time_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{kind}} · {{origin}}", "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_transaction_application_time{quantile=\"0.9\"}", - "legendFormat": "p90 · {{kind}} · {{origin}}", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_transaction_application_time{quantile=\"0.95\"}", - "legendFormat": "p95 · {{kind}} · {{origin}}", - "refId": "C" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "mempool_transaction_application_time{quantile=\"0.99\"}", - "legendFormat": "p99 · {{kind}} · {{origin}}", - "refId": "D" } ], "title": "Transaction application time", @@ -169,33 +133,15 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transactions_per_block{quantile=\"0.5\"}", - "legendFormat": "p50", + "expr": "histogram_quantile(${percentile}, sum by (le) (rate(transactions_per_block_bucket[$__rate_interval])))", + "legendFormat": "${percentile:text}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transactions_per_block{quantile=\"0.9\"}", - "legendFormat": "p90", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transactions_per_block{quantile=\"0.95\"}", - "legendFormat": "p95", - "refId": "C" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transactions_per_block{quantile=\"0.99\"}", - "legendFormat": "p99", - "refId": "D" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(transactions_per_block_sum[1m]) / rate(transactions_per_block_count[1m])", + "expr": "rate(transactions_per_block_sum[$__rate_interval]) / rate(transactions_per_block_count[$__rate_interval])", "legendFormat": "avg", - "refId": "E" + "refId": "B" } ], "title": "Transactions per block", @@ -312,7 +258,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "100 * increase(failed_transaction_count[$__range]) / clamp_min(increase(submitted_transaction_count[$__range]), 1)", + "expr": "100 * (increase(before_mempool_failed_transactions_total[$__range]) + increase(mempool_failed_transactions_total[$__range])) / clamp_min(increase(submitted_transactions_total[$__range]), 1)", "legendFormat": "failed", "refId": "A" } @@ -324,18 +270,22 @@ "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { - "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 35, "gradientMode": "opacity" }, "unit": "short", "min": 0.0 }, "overrides": [ - { - "matcher": { "id": "byName", "options": "failed" }, - "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] - }, { "matcher": { "id": "byName", "options": "submitted" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] + }, + { + "matcher": { "id": "byName", "options": "failed · before mempool" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] + }, + { + "matcher": { "id": "byName", "options": "failed · in mempool" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] } ] }, @@ -348,15 +298,21 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(submitted_transaction_count[1m]) * 60", + "expr": "rate(submitted_transactions_total[1m]) * 60", "legendFormat": "submitted", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(failed_transaction_count[1m]) * 60", - "legendFormat": "failed", + "expr": "rate(before_mempool_failed_transactions_total[1m]) * 60", + "legendFormat": "failed · before mempool", "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(mempool_failed_transactions_total[1m]) * 60", + "legendFormat": "failed · in mempool", + "refId": "C" } ], "title": "Submitted vs failed transactions (per minute)", @@ -366,7 +322,25 @@ "refresh": "5s", "schemaVersion": 39, "tags": [ "sequencer" ], - "templating": { "list": [ ] }, + "templating": { + "list": [ + { + "current": { "selected": true, "text": "p95", "value": "0.95" }, + "includeAll": false, + "label": "Percentile", + "multi": false, + "name": "percentile", + "options": [ + { "selected": false, "text": "p50", "value": "0.5" }, + { "selected": false, "text": "p90", "value": "0.9" }, + { "selected": true, "text": "p95", "value": "0.95" }, + { "selected": false, "text": "p99", "value": "0.99" } + ], + "query": "p50 : 0.5, p90 : 0.9, p95 : 0.95, p99 : 0.99", + "type": "custom" + } + ] + }, "time": { "from": "now-15m", "to": "now" }, "timezone": "", "title": "Sequencer", diff --git a/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs b/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs index da44f508..e2f7df78 100644 --- a/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs +++ b/tools/dashboard_gen/src/bin/gen_sequencer_dashboard.rs @@ -10,17 +10,19 @@ )] use dashboard_gen::{ - Color, Dashboard, FieldOverride, Panel, Target, Thresholds, Unit, avg, percentiles, - percentiles_labeled, rate_per_min, + Color, Dashboard, FieldOverride, GradientMode, Panel, Target, Thresholds, Unit, avg, + percentile_legend, percentile_variable, rate_per_min, selected_percentile, }; use json_pretty_compact::PrettyCompactFormatter; use serde::Serialize as _; const PERCENTILES: &[u32] = &[50, 90, 95, 99]; +const DEFAULT_PERCENTILE: u32 = 95; fn sequencer_dashboard() -> Dashboard { Dashboard::new("Sequencer", "sequencer") .tag("sequencer") + .variable(percentile_variable(PERCENTILES, DEFAULT_PERCENTILE)) .row( 7, [ @@ -30,13 +32,13 @@ fn sequencer_dashboard() -> Dashboard { .decimals(0) .color(Color::fixed("blue")) .target( - Target::new(sequencer_core_metrics::names::BLOCK_COUNT).legend("height"), + Target::new(sequencer_core_metrics::names::BLOCKS_TOTAL).legend("height"), ), Panel::timeseries("Block production rate") .width(18) .unit(Unit::Short) .target(rate_per_min( - sequencer_core_metrics::names::BLOCK_COUNT, + sequencer_core_metrics::names::BLOCKS_TOTAL, "blocks/min", )), ], @@ -46,9 +48,10 @@ fn sequencer_dashboard() -> Dashboard { [Panel::timeseries("Block creation time") .width(24) .unit(Unit::Seconds) - .targets(percentiles( + .target(selected_percentile( sequencer_core_metrics::names::BLOCK_CREATION_TIME, - PERCENTILES, + &[], + &percentile_legend(), )) .target(avg(sequencer_core_metrics::names::BLOCK_CREATION_TIME)) .with_override( @@ -63,17 +66,18 @@ fn sequencer_dashboard() -> Dashboard { Panel::timeseries("Transaction application time") .width(12) .unit(Unit::Seconds) - .targets(percentiles_labeled( + .target(selected_percentile( sequencer_core_metrics::names::MEMPOOL_TRANSACTION_APPLICATION_TIME, - PERCENTILES, - " · {{kind}} · {{origin}}", + &["kind", "origin"], + "{{kind}} · {{origin}}", )), Panel::timeseries("Transactions per block") .width(12) .unit(Unit::Short) - .targets(percentiles( + .target(selected_percentile( sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK, - PERCENTILES, + &[], + &percentile_legend(), )) .target(avg(sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK)) .with_override( @@ -141,11 +145,13 @@ fn sequencer_dashboard() -> Dashboard { ) .target( Target::new(format!( + // Both failure stages against the same submission base; // `clamp_min` keeps an idle window (nothing submitted) // reading as 0% instead of a division by zero. - "100 * increase({failed}[$__range]) / clamp_min(increase({submitted}[$__range]), 1)", - failed = sequencer_core_metrics::names::FAILED_TRANSACTION_COUNT, - submitted = sequencer_service_metrics::names::SUBMITTED_TRANSACTION_COUNT, + "100 * (increase({before_mempool}[$__range]) + increase({in_mempool}[$__range])) / clamp_min(increase({submitted}[$__range]), 1)", + before_mempool = sequencer_service_metrics::names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL, + in_mempool = sequencer_core_metrics::names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL, + submitted = sequencer_service_metrics::names::SUBMITTED_TRANSACTIONS_TOTAL, )) .legend("failed"), ), @@ -153,17 +159,29 @@ fn sequencer_dashboard() -> Dashboard { .width(18) .unit(Unit::Short) .min(0.0) + .fill_opacity(35) + .gradient_mode(GradientMode::Opacity) .target(rate_per_min( - sequencer_service_metrics::names::SUBMITTED_TRANSACTION_COUNT, + sequencer_service_metrics::names::SUBMITTED_TRANSACTIONS_TOTAL, "submitted", )) .target(rate_per_min( - sequencer_core_metrics::names::FAILED_TRANSACTION_COUNT, - "failed", + sequencer_service_metrics::names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL, + "failed · before mempool", + )) + .target(rate_per_min( + sequencer_core_metrics::names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL, + "failed · in mempool", )) - .with_override(FieldOverride::by_name("failed").color(Color::fixed("red"))) .with_override( FieldOverride::by_name("submitted").color(Color::fixed("green")), + ) + .with_override( + FieldOverride::by_name("failed · before mempool") + .color(Color::fixed("orange")), + ) + .with_override( + FieldOverride::by_name("failed · in mempool").color(Color::fixed("red")), ), ], ) diff --git a/tools/dashboard_gen/src/codegen.rs b/tools/dashboard_gen/src/codegen.rs index b554ffd6..4874f053 100644 --- a/tools/dashboard_gen/src/codegen.rs +++ b/tools/dashboard_gen/src/codegen.rs @@ -10,7 +10,7 @@ use std::fmt::Write as _; use crate::{ - Unit, + DEFAULT_FILL_OPACITY, Unit, input::{Defaults, PanelInput}, schema::{ AxisPlacement, Color, GradientMode, LineInterpolation, PanelType, ShowPoints, StackingMode, @@ -42,6 +42,11 @@ fn panel_expr_inner(panel: &PanelInput) -> Result { write_defaults(&mut expr, defaults)?; if let Some(custom) = &defaults.custom { + // Emitted against the builder's default, not Grafana's (which is 0), so + // a genuinely unfilled panel still round-trips. + if let Some(opacity) = custom.fill_opacity.filter(|&o| o != DEFAULT_FILL_OPACITY) { + write!(expr, "\n .fill_opacity({opacity})")?; + } if custom.span_nulls == Some(true) { expr.push_str("\n .span_nulls()"); } diff --git a/tools/dashboard_gen/src/input.rs b/tools/dashboard_gen/src/input.rs index ea3c4f78..9d69002c 100644 --- a/tools/dashboard_gen/src/input.rs +++ b/tools/dashboard_gen/src/input.rs @@ -66,6 +66,8 @@ pub struct Defaults { #[derive(Deserialize, Default)] #[serde(rename_all = "camelCase")] pub struct Custom { + #[serde(default)] + pub fill_opacity: Option, #[serde(default)] pub span_nulls: Option, #[serde(default)] diff --git a/tools/dashboard_gen/src/lib.rs b/tools/dashboard_gen/src/lib.rs index d1fb38c9..5bbaa672 100644 --- a/tools/dashboard_gen/src/lib.rs +++ b/tools/dashboard_gen/src/lib.rs @@ -16,13 +16,14 @@ pub use codegen::panel_to_rust_source; pub use schema::{ AxisPlacement, Color, GradientMode, LineInterpolation, ShowPoints, StackingMode, Thresholds, + Variable, }; use schema::{ Calc, Custom, Datasource, Defaults, DrawStyle, EmptyList, FieldConfig, Fill, GaugeOptions, GraphMode, GridPos, Legend, LegendDisplay, LineStyle, Matcher, MatcherKind, Options, OverrideProperty, PanelModel, PanelType, Placement, PropertyId, PropertyValue, ReduceOptions, - SortOrder, Stacking, StatColorMode, StatOptions, TimeRange, TimeSeriesOptions, Tooltip, - TooltipMode, + SortOrder, Stacking, StatColorMode, StatOptions, Templating, TimeRange, TimeSeriesOptions, + Tooltip, TooltipMode, VariableKind, VariableOption, }; use serde::Serialize; pub use unit::Unit; @@ -38,6 +39,18 @@ mod unit; /// than by a per-environment URL. pub const DATASOURCE_UID: &str = "prometheus"; +/// Window every histogram query rates over. `$__rate_interval` tracks the +/// panel's zoom, so a percentile covers the range you are actually looking at, +/// and an idle window yields no value rather than a zero. +const RATE_WINDOW: &str = "$__rate_interval"; + +/// Dashboard variable holding the quantile every percentile query reads. +const PERCENTILE_VAR: &str = "percentile"; + +/// Area fill under a timeseries line, as a percentage. Enough to read a series' +/// shape at a glance without drowning the ones stacked behind it. +pub(crate) const DEFAULT_FILL_OPACITY: u32 = 10; + /// A single Prometheus query within a panel. #[derive(Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -129,6 +142,7 @@ pub struct Panel { span_nulls: bool, overrides: Vec, // Optional timeseries styling, set via the `styling` setters. + fill_opacity: Option, line_interpolation: Option, show_points: Option, gradient_mode: Option, @@ -152,6 +166,7 @@ impl Panel { thresholds: None, span_nulls: false, overrides: Vec::new(), + fill_opacity: None, line_interpolation: None, show_points: None, gradient_mode: None, @@ -308,7 +323,7 @@ impl Panel { custom: Some(Custom { draw_style: DrawStyle::Line, line_width: 1, - fill_opacity: 10, + fill_opacity: self.fill_opacity.unwrap_or(DEFAULT_FILL_OPACITY), span_nulls: self.span_nulls.then_some(true), line_interpolation: self.line_interpolation, show_points: self.show_points, @@ -327,8 +342,11 @@ impl Panel { thresholds: self.thresholds, }; // Panels with several series read better as a sortable table with - // a multi-series tooltip; single-series panels stay compact. - let multi = targets.len() > 1; + // a multi-series tooltip; single-series panels stay compact. A + // `{{label}}` legend fans one target out into a series per label + // value, so it counts as several too. + let multi = + targets.len() > 1 || targets.iter().any(|target| target.legend.contains("{{")); let options = Options::TimeSeries(TimeSeriesOptions { legend: Legend { display_mode: if multi { @@ -382,7 +400,7 @@ pub struct Dashboard { refresh: String, schema_version: u32, tags: Vec, - templating: EmptyList, + templating: Templating, time: TimeRange, timezone: String, title: String, @@ -405,7 +423,7 @@ impl Dashboard { refresh: "5s".to_owned(), schema_version: 39, tags: Vec::new(), - templating: EmptyList::default(), + templating: Templating::default(), time: TimeRange { from: "now-15m".to_owned(), to: "now".to_owned(), @@ -430,6 +448,13 @@ impl Dashboard { self } + /// Add a dropdown to the dashboard's top bar, e.g. [`percentile_variable`]. + #[must_use] + pub fn variable(mut self, variable: Variable) -> Self { + self.templating.list.push(variable); + self + } + /// Place a horizontal row of panels at the current vertical cursor. Panel /// ids, x offsets and y are assigned here; unset widths split the remaining /// 24 columns evenly. @@ -468,34 +493,89 @@ impl Dashboard { } } -/// Percentile line targets for a summary metric: `p50`, `p90`, … each querying -/// the matching `quantile="0.x"` series. +/// The dropdown driving every [`selected_percentile`] query, offering +/// `percentiles` (e.g. `[50, 90, 95, 99]`) with `default` pre-selected. +/// +/// Panics if `default` is not one of `percentiles`. #[must_use] -pub fn percentiles(metric: &str, percentiles: &[u32]) -> Vec { - percentiles_labeled(metric, percentiles, "") -} +pub fn percentile_variable(percentiles: &[u32], default: u32) -> Variable { + assert!( + percentiles.contains(&default), + "default p{default} is not one of the offered percentiles {percentiles:?}", + ); -/// Like [`percentiles`], but appends `legend_suffix` to every legend — handy -/// when the metric carries labels (e.g. ` · {{kind}} · {{origin}}`). -#[must_use] -pub fn percentiles_labeled(metric: &str, percentiles: &[u32], legend_suffix: &str) -> Vec { - percentiles + let options: Vec = percentiles .iter() - .map(|&p| { - // `quantile="0.x"` label, derived without float math: zero-pad to two - // digits then drop trailing zeros (50 → "0.5", 95 → "0.95"). - let quantile = format!("0.{p:02}"); - let quantile = quantile.trim_end_matches('0'); - Target::new(format!("{metric}{{quantile=\"{quantile}\"}}")) - .legend(format!("p{p}{legend_suffix}")) + .map(|&p| VariableOption { + selected: p == default, + text: format!("p{p}"), + value: quantile(p), }) - .collect() + .collect(); + let current = options + .iter() + .find(|option| option.selected) + .cloned() + .expect("`default` is one of `percentiles`, asserted above"); + let query = options + .iter() + .map(|option| format!("{} : {}", option.text, option.value)) + .collect::>() + .join(", "); + + Variable { + current, + include_all: false, + label: "Percentile".to_owned(), + // A quantile is a scalar argument to `histogram_quantile`, so exactly + // one may be selected. + multi: false, + name: PERCENTILE_VAR.to_owned(), + options, + query, + kind: VariableKind::Custom, + } } -/// An `avg` target for a summary metric: `rate(sum) / rate(count)` over 1m. +/// The legend fragment that renders the dropdown's current choice, e.g. `p95`. +#[must_use] +pub fn percentile_legend() -> String { + format!("${{{PERCENTILE_VAR}:text}}") +} + +/// A [`histogram_quantile`] line over `metric`'s buckets, at whatever quantile +/// [`percentile_variable`] currently holds. `labels` stay split out into their +/// own series; every other label is summed away. +/// +/// [`histogram_quantile`]: https://prometheus.io/docs/prometheus/latest/querying/functions/#histogram_quantile +#[must_use] +pub fn selected_percentile(metric: &str, labels: &[&str], legend: &str) -> Target { + // `le` carries the bucket boundary, so it must survive the aggregation. + let grouping = std::iter::once("le") + .chain(labels.iter().copied()) + .collect::>() + .join(", "); + + Target::new(format!( + "histogram_quantile(${{{PERCENTILE_VAR}}}, sum by ({grouping}) (rate({metric}_bucket[{RATE_WINDOW}])))" + )) + .legend(legend) +} + +/// A percentile as its `histogram_quantile` argument, derived without float +/// math: zero-pad to two digits then drop trailing zeros (50 → `0.5`). +fn quantile(percentile: u32) -> String { + let quantile = format!("0.{percentile:02}"); + quantile.trim_end_matches('0').to_owned() +} + +/// An `avg` target for a histogram metric: `rate(sum) / rate(count)`. #[must_use] pub fn avg(metric: &str) -> Target { - Target::new(format!("rate({metric}_sum[1m]) / rate({metric}_count[1m])")).legend("avg") + Target::new(format!( + "rate({metric}_sum[{RATE_WINDOW}]) / rate({metric}_count[{RATE_WINDOW}])" + )) + .legend("avg") } /// A per-minute rate target for a counter metric. diff --git a/tools/dashboard_gen/src/schema.rs b/tools/dashboard_gen/src/schema.rs index 009c7c40..190c4606 100644 --- a/tools/dashboard_gen/src/schema.rs +++ b/tools/dashboard_gen/src/schema.rs @@ -363,6 +363,43 @@ pub enum Options { Gauge(GaugeOptions), } +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum VariableKind { + /// A fixed list of choices, spelled out in the dashboard itself. + Custom, +} + +/// One choice in a [`Variable`] dropdown: `text` is displayed, `value` is what +/// `$name` interpolates to in a query. +#[derive(Clone, Serialize)] +pub struct VariableOption { + pub selected: bool, + pub text: String, + pub value: String, +} + +/// A dashboard-level dropdown, rendered in the top bar. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Variable { + pub current: VariableOption, + pub include_all: bool, + pub label: String, + pub multi: bool, + pub name: String, + pub options: Vec, + /// Grafana's own encoding of `options`, as `text : value` pairs. + pub query: String, + #[serde(rename = "type")] + pub kind: VariableKind, +} + +#[derive(Serialize, Default)] +pub struct Templating { + pub list: Vec, +} + #[derive(Clone, Copy, Serialize)] pub struct GridPos { pub h: u32, diff --git a/tools/dashboard_gen/src/styling.rs b/tools/dashboard_gen/src/styling.rs index 5414ec73..36b0cc38 100644 --- a/tools/dashboard_gen/src/styling.rs +++ b/tools/dashboard_gen/src/styling.rs @@ -11,7 +11,7 @@ //! not emitted, so the value is silently dropped. use crate::{ - Panel, + DEFAULT_FILL_OPACITY, Panel, schema::{AxisPlacement, GradientMode, LineInterpolation, ShowPoints, StackingMode}, }; @@ -20,6 +20,24 @@ use crate::{ reason = "styling setters intentionally live in their own file, so `Panel` has a second inherent impl here" )] impl Panel { + /// Area fill under the line, as a percentage. Builder default: + /// [`DEFAULT_FILL_OPACITY`]. + /// + /// Panics if passed that default, or a value above 100. + #[must_use] + pub fn fill_opacity(mut self, opacity: u32) -> Self { + assert!( + opacity <= 100, + "fill_opacity({opacity}) is not a percentage" + ); + assert_ne!( + opacity, DEFAULT_FILL_OPACITY, + "fill_opacity({DEFAULT_FILL_OPACITY}) is redundant: it is the builder's default. Omit the call.", + ); + self.fill_opacity = Some(opacity); + self + } + /// Interpolation between points. Grafana default: `Linear`. /// /// Panics if passed `Linear` — that's the default and would be redundant.