refactor(metrics): improve metrics setup

This commit is contained in:
Daniil Polyakov 2026-07-28 17:04:44 +03:00
parent 849bb6bbe1
commit aa6969386e
17 changed files with 434 additions and 200 deletions

1
.gitignore vendored
View File

@ -11,6 +11,7 @@ data/
rocksdb*
sequencer/service/data/
storage.json
statistics.json
result

View File

@ -219,6 +219,7 @@ impl From<lee::ProgramDeploymentTransaction> for LeeTransaction {
BorshSerialize,
BorshDeserialize,
strum::IntoStaticStr,
strum::EnumIter,
)]
pub enum TxKind {
Public,

View File

@ -1,7 +1,8 @@
pub const BLOCK_CREATION_TIME: &str = "block_creation_time";
pub const BLOCK_COUNT: &str = "block_count";
pub const BLOCK_CREATION_TIME: &str = "block_creation_time_seconds";
pub const BLOCKS_TOTAL: &str = "blocks_total";
pub const MEMPOOL_SIZE: &str = "mempool_size";
pub const MEMPOOL_MAX_SIZE: &str = "mempool_max_size";
pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = "mempool_transaction_application_time";
pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str =
"mempool_transaction_application_time_seconds";
pub const TRANSACTIONS_PER_BLOCK: &str = "transactions_per_block";
pub const FAILED_TRANSACTION_COUNT: &str = "failed_transaction_count";
pub const MEMPOOL_FAILED_TRANSACTIONS_TOTAL: &str = "mempool_failed_transactions_total";

View File

@ -7,39 +7,58 @@
use std::time::Duration;
use common::transaction::TxKind;
use metrics::{Counter, Unit, counter, gauge, histogram};
use metrics::{Counter, Histogram, Unit, counter, gauge, histogram};
use strum::IntoEnumIterator as _;
use crate::names;
#[derive(Clone, Copy, strum::IntoStaticStr)]
#[derive(Clone, Copy, strum::IntoStaticStr, strum::EnumIter)]
pub enum TransactionOrigin {
User,
Sequencer,
}
pub fn record_block_creation_time(duration: Duration) {
/// Initialize metrics.
pub fn init() {
blocks_total_counter().increment(0);
mempool_failed_transactions_total_counter().increment(0);
record_mempool_size(0);
drop(block_creation_time_histogram());
drop(transactions_per_block_histogram());
for origin in TransactionOrigin::iter() {
for kind in TxKind::iter() {
drop(mempool_transaction_application_time_histogram(origin, kind));
}
}
}
fn block_creation_time_histogram() -> Histogram {
histogram!(
description: "Time taken to create a block",
unit: Unit::Seconds,
names::BLOCK_CREATION_TIME
)
.record(duration.as_secs_f64());
}
fn block_count_counter() -> Counter {
pub fn record_block_creation_time(duration: Duration) {
block_creation_time_histogram().record(duration.as_secs_f64());
}
fn blocks_total_counter() -> Counter {
counter!(
description: "Number of blocks in chain",
unit: Unit::Count,
names::BLOCK_COUNT
names::BLOCKS_TOTAL
)
}
pub fn set_block_count(value: u64) {
block_count_counter().absolute(value);
pub fn set_blocks_total(value: u64) {
blocks_total_counter().absolute(value);
}
pub fn increment_block_count() {
block_count_counter().increment(1);
pub fn increment_blocks_total() {
blocks_total_counter().increment(1);
}
pub fn record_mempool_size(size: usize) {
@ -60,11 +79,10 @@ pub fn record_mempool_max_size(size: usize) {
.set(u64::try_from(size).expect("Mempool max size should fit into u64") as f64);
}
pub fn record_mempool_transaction_application_time(
fn mempool_transaction_application_time_histogram(
origin: TransactionOrigin,
kind: TxKind,
duration: Duration,
) {
) -> Histogram {
histogram!(
description: "Time taken to apply a mempool transaction",
unit: Unit::Seconds,
@ -72,23 +90,37 @@ pub fn record_mempool_transaction_application_time(
"origin" => <&'static str>::from(origin),
"kind" => <&'static str>::from(kind),
)
.record(duration.as_secs_f64());
}
pub fn record_transactions_per_block(count: usize) {
pub fn record_mempool_transaction_application_time(
origin: TransactionOrigin,
kind: TxKind,
duration: Duration,
) {
mempool_transaction_application_time_histogram(origin, kind).record(duration.as_secs_f64());
}
fn transactions_per_block_histogram() -> Histogram {
histogram!(
description: "Number of transactions included in block",
description: "Number of transactions from mempool included in block",
unit: Unit::Count,
names::TRANSACTIONS_PER_BLOCK
)
.record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64);
}
pub fn increment_failed_transaction_count() {
counter!(
description: "Number of transactions that failed to be included in blocks",
unit: Unit::Count,
names::FAILED_TRANSACTION_COUNT
)
.increment(1);
pub fn record_transactions_per_block(count: usize) {
transactions_per_block_histogram()
.record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64);
}
fn mempool_failed_transactions_total_counter() -> Counter {
counter!(
description: "Number of transactions from mempool that failed to be included in blocks",
unit: Unit::Count,
names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL
)
}
pub fn increment_mempool_failed_transactions_total() {
mempool_failed_transactions_total_counter().increment(1);
}

View File

@ -204,6 +204,8 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
pub async fn start_from_config(
config: SequencerConfig,
) -> (Self, MemPoolHandle<(TransactionOrigin, LeeTransaction)>) {
sequencer_core_metrics::init();
let bedrock_signing_key =
load_or_create_signing_key(&config.home.join("bedrock_signing_key"))
.expect("Failed to load or create bedrock signing key");
@ -309,7 +311,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
watchers,
};
sequencer_core_metrics::set_block_count(sequencer_core.chain_height());
sequencer_core_metrics::set_blocks_total(sequencer_core.chain_height());
(sequencer_core, mempool_handle)
}
@ -903,7 +905,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
if applied {
valid_transactions.push(tx);
} else {
sequencer_core_metrics::increment_failed_transaction_count();
sequencer_core_metrics::increment_mempool_failed_transactions_total();
// A failed transaction is simply left out of the block, except a
// dispatch: that one is re-fed from the store every turn, so one
// that can never execute would fail on every block for ever.
@ -939,7 +941,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
);
sequencer_core_metrics::record_block_creation_time(now.elapsed());
sequencer_core_metrics::increment_block_count();
sequencer_core_metrics::increment_blocks_total();
Ok(BlockWithMeta { block, withdrawals })
}

View File

@ -1 +1,3 @@
pub const SUBMITTED_TRANSACTION_COUNT: &str = "submitted_transaction_count";
pub const SUBMITTED_TRANSACTIONS_TOTAL: &str = "submitted_transactions_total";
pub const BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL: &str =
"before_mempool_failed_transactions_total";

View File

@ -1,12 +1,32 @@
use metrics::{Unit, counter};
use metrics::{Counter, Unit, counter};
use crate::names;
pub fn increment_submitted_transaction_count() {
pub fn init() {
submitted_transactions_total_counter().increment(0);
before_mempool_failed_transactions_total_counter().increment(0);
}
fn submitted_transactions_total_counter() -> Counter {
counter!(
description: "Number of transactions submitted",
unit: Unit::Count,
names::SUBMITTED_TRANSACTION_COUNT
names::SUBMITTED_TRANSACTIONS_TOTAL
)
.increment(1);
}
pub fn increment_submitted_transactions_total() {
submitted_transactions_total_counter().increment(1);
}
fn before_mempool_failed_transactions_total_counter() -> Counter {
counter!(
description: "Number of transactions that failed before reaching the mempool",
unit: Unit::Count,
names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL
)
}
pub fn increment_before_mempool_failed_transactions_total() {
before_mempool_failed_transactions_total_counter().increment(1);
}

View File

@ -211,6 +211,8 @@ async fn wait_for_store_release(store: &StoreRelease) {
}
pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result<SequencerHandle> {
sequencer_service_metrics::init();
let block_timeout = config.block_create_timeout;
let max_block_size = config.max_block_size;

View File

@ -6,6 +6,7 @@ use std::{
use anyhow::{Context as _, Result};
use clap::Parser;
use log::{error, info};
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
use tokio::signal::unix::{SignalKind, signal};
use tokio_util::sync::CancellationToken;
@ -34,10 +35,7 @@ struct Args {
async fn main() -> Result<()> {
env_logger::init();
metrics_exporter_prometheus::PrometheusBuilder::new()
.with_recommended_naming(true)
.install()
.context("Failed to install Prometheus recorder")?;
install_prometheus_recorder()?;
let Args {
config_path,
@ -76,6 +74,36 @@ async fn main() -> Result<()> {
Ok(())
}
/// Installs the recorder with explicit buckets, which makes every histogram
/// export as a Prometheus histogram (`_bucket`/`_sum`/`_count`) rather than the
/// default rolling-window summary. Summary quantiles reset to zero once their
/// window drains, so an idle period reads as "took 0s" instead of "no data".
///
/// Ladders are picked by name suffix, so a new timing metric is covered without
/// touching this function. The matcher sees the name as registered, *before*
/// [`PrometheusBuilder::with_recommended_naming`] appends a unit suffix of its
/// own — a duration metric whose name omits `_seconds` silently falls through to
/// [`COUNT_BUCKETS`].
fn install_prometheus_recorder() -> Result<()> {
/// Ladder for `*_seconds` histograms, densest across the 1100 ms band where
/// block production and transaction application actually land.
const LATENCY_BUCKETS: &[f64] = &[
0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
];
/// Fallback ladder for histograms that count things rather than measure time.
const COUNT_BUCKETS: &[f64] = &[1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0];
PrometheusBuilder::new()
.with_recommended_naming(true)
.set_buckets(COUNT_BUCKETS)
.context("Failed to set default histogram buckets")?
.set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), LATENCY_BUCKETS)
.context("Failed to set latency histogram buckets")?
.install()
.context("Failed to install Prometheus recorder")
}
/// Cancelled on Ctrl-C or `SIGTERM`.
///
/// `SIGTERM` is what a container runtime sends first, so without it every

View File

@ -6,7 +6,7 @@ use jsonrpsee::{
types::{ErrorCode, ErrorObjectOwned},
};
use lee;
use log::warn;
use log::{error, warn};
use mempool::MemPoolHandle;
use sequencer_core::{
DbError, SequencerCore, TransactionOrigin, block_publisher::BlockPublisherTrait,
@ -44,57 +44,68 @@ impl<BC: BlockPublisherTrait + Send + Sync + 'static> sequencer_service_rpc::Rpc
for SequencerService<BC>
{
async fn send_transaction(&self, tx: LeeTransaction) -> Result<HashType, ErrorObjectOwned> {
// Reserve ~200 bytes for block header overhead
const BLOCK_HEADER_OVERHEAD: u64 = 200;
sequencer_service_metrics::increment_submitted_transactions_total();
let tx_hash = tx.hash();
let encoded_tx =
borsh::to_vec(&tx).expect("Transaction borsh serialization should not fail");
let tx_size = u64::try_from(encoded_tx.len()).expect("Transaction size should fit in u64");
let res = async move {
// Reserve ~200 bytes for block header overhead
const BLOCK_HEADER_OVERHEAD: u64 = 200;
let max_tx_size = self.max_block_size.saturating_sub(BLOCK_HEADER_OVERHEAD);
let encoded_tx =
borsh::to_vec(&tx).expect("Transaction borsh serialization should not fail");
let tx_size =
u64::try_from(encoded_tx.len()).expect("Transaction size should fit in u64");
if tx_size > max_tx_size {
return Err(ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
format!("Transaction too large: size {tx_size}, max {max_tx_size}"),
None::<()>,
));
}
let max_tx_size = self.max_block_size.saturating_sub(BLOCK_HEADER_OVERHEAD);
let authenticated_tx = tx
.transaction_stateless_check()
.inspect_err(|err| warn!("Error at pre_check {err:#?}"))
.map_err(|err| {
ErrorObjectOwned::owned(
if tx_size > max_tx_size {
return Err(ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
format!("{err:?}"),
format!("Transaction too large: size {tx_size}, max {max_tx_size}"),
None::<()>,
)
})?;
));
}
// Sequencer-only programs (the cross-zone inbox) are injected by the
// watcher; a user must not invoke them top-level, or anyone could forge
// an inbound cross-zone delivery. Chained user calls are already rejected
// by the inbox guest's caller-is-none assertion.
if let LeeTransaction::Public(public_tx) = &authenticated_tx
&& sequencer_core::is_sequencer_only_program(public_tx.message().program_id)
{
return Err(ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
"Program is sequencer-only and cannot be invoked by a user transaction".to_owned(),
None::<()>,
));
}
let authenticated_tx = tx
.transaction_stateless_check()
.inspect_err(|err| warn!("Error at pre_check {err:#?}"))
.map_err(|err| {
ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
format!("{err:?}"),
None::<()>,
)
})?;
// Sequencer-only programs (the cross-zone inbox) are injected by the
// watcher; a user must not invoke them top-level, or anyone could forge
// an inbound cross-zone delivery. Chained user calls are already rejected
// by the inbox guest's caller-is-none assertion.
if let LeeTransaction::Public(public_tx) = &authenticated_tx
&& sequencer_core::is_sequencer_only_program(public_tx.message().program_id)
{
return Err(ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
"Program is sequencer-only and cannot be invoked by a user transaction"
.to_owned(),
None::<()>,
));
}
Ok(authenticated_tx)
};
let authenticated_tx = res.await.inspect_err(|err| {
sequencer_service_metrics::increment_before_mempool_failed_transactions_total();
error!("Transaction failed before reaching mempool: {err:#?}");
})?;
self.mempool_handle
.push((TransactionOrigin::User, authenticated_tx))
.await
.expect("Mempool is closed, this is a bug");
sequencer_service_metrics::increment_submitted_transaction_count();
Ok(tx_hash)
}

View File

@ -19,7 +19,7 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "block_count",
"expr": "blocks_total",
"legendFormat": "height",
"refId": "A"
}
@ -42,7 +42,7 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(block_count[1m]) * 60",
"expr": "rate(blocks_total[1m]) * 60",
"legendFormat": "blocks/min",
"refId": "A"
}
@ -73,33 +73,15 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "block_creation_time{quantile=\"0.5\"}",
"legendFormat": "p50",
"expr": "histogram_quantile(${percentile}, sum by (le) (rate(block_creation_time_seconds_bucket[$__rate_interval])))",
"legendFormat": "${percentile:text}",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "block_creation_time{quantile=\"0.9\"}",
"legendFormat": "p90",
"refId": "B"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "block_creation_time{quantile=\"0.95\"}",
"legendFormat": "p95",
"refId": "C"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "block_creation_time{quantile=\"0.99\"}",
"legendFormat": "p99",
"refId": "D"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(block_creation_time_sum[1m]) / rate(block_creation_time_count[1m])",
"expr": "rate(block_creation_time_seconds_sum[$__rate_interval]) / rate(block_creation_time_seconds_count[$__rate_interval])",
"legendFormat": "avg",
"refId": "E"
"refId": "B"
}
],
"title": "Block creation time",
@ -120,27 +102,9 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "mempool_transaction_application_time{quantile=\"0.5\"}",
"legendFormat": "p50 · {{kind}} · {{origin}}",
"expr": "histogram_quantile(${percentile}, sum by (le, kind, origin) (rate(mempool_transaction_application_time_seconds_bucket[$__rate_interval])))",
"legendFormat": "{{kind}} · {{origin}}",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "mempool_transaction_application_time{quantile=\"0.9\"}",
"legendFormat": "p90 · {{kind}} · {{origin}}",
"refId": "B"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "mempool_transaction_application_time{quantile=\"0.95\"}",
"legendFormat": "p95 · {{kind}} · {{origin}}",
"refId": "C"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "mempool_transaction_application_time{quantile=\"0.99\"}",
"legendFormat": "p99 · {{kind}} · {{origin}}",
"refId": "D"
}
],
"title": "Transaction application time",
@ -169,33 +133,15 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "transactions_per_block{quantile=\"0.5\"}",
"legendFormat": "p50",
"expr": "histogram_quantile(${percentile}, sum by (le) (rate(transactions_per_block_bucket[$__rate_interval])))",
"legendFormat": "${percentile:text}",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "transactions_per_block{quantile=\"0.9\"}",
"legendFormat": "p90",
"refId": "B"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "transactions_per_block{quantile=\"0.95\"}",
"legendFormat": "p95",
"refId": "C"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "transactions_per_block{quantile=\"0.99\"}",
"legendFormat": "p99",
"refId": "D"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(transactions_per_block_sum[1m]) / rate(transactions_per_block_count[1m])",
"expr": "rate(transactions_per_block_sum[$__rate_interval]) / rate(transactions_per_block_count[$__rate_interval])",
"legendFormat": "avg",
"refId": "E"
"refId": "B"
}
],
"title": "Transactions per block",
@ -312,7 +258,7 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "100 * increase(failed_transaction_count[$__range]) / clamp_min(increase(submitted_transaction_count[$__range]), 1)",
"expr": "100 * (increase(before_mempool_failed_transactions_total[$__range]) + increase(mempool_failed_transactions_total[$__range])) / clamp_min(increase(submitted_transactions_total[$__range]), 1)",
"legendFormat": "failed",
"refId": "A"
}
@ -324,18 +270,22 @@
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 },
"custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 35, "gradientMode": "opacity" },
"unit": "short",
"min": 0.0
},
"overrides": [
{
"matcher": { "id": "byName", "options": "failed" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ]
},
{
"matcher": { "id": "byName", "options": "submitted" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ]
},
{
"matcher": { "id": "byName", "options": "failed · before mempool" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ]
},
{
"matcher": { "id": "byName", "options": "failed · in mempool" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ]
}
]
},
@ -348,15 +298,21 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(submitted_transaction_count[1m]) * 60",
"expr": "rate(submitted_transactions_total[1m]) * 60",
"legendFormat": "submitted",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(failed_transaction_count[1m]) * 60",
"legendFormat": "failed",
"expr": "rate(before_mempool_failed_transactions_total[1m]) * 60",
"legendFormat": "failed · before mempool",
"refId": "B"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(mempool_failed_transactions_total[1m]) * 60",
"legendFormat": "failed · in mempool",
"refId": "C"
}
],
"title": "Submitted vs failed transactions (per minute)",
@ -366,7 +322,25 @@
"refresh": "5s",
"schemaVersion": 39,
"tags": [ "sequencer" ],
"templating": { "list": [ ] },
"templating": {
"list": [
{
"current": { "selected": true, "text": "p95", "value": "0.95" },
"includeAll": false,
"label": "Percentile",
"multi": false,
"name": "percentile",
"options": [
{ "selected": false, "text": "p50", "value": "0.5" },
{ "selected": false, "text": "p90", "value": "0.9" },
{ "selected": true, "text": "p95", "value": "0.95" },
{ "selected": false, "text": "p99", "value": "0.99" }
],
"query": "p50 : 0.5, p90 : 0.9, p95 : 0.95, p99 : 0.99",
"type": "custom"
}
]
},
"time": { "from": "now-15m", "to": "now" },
"timezone": "",
"title": "Sequencer",

View File

@ -10,17 +10,19 @@
)]
use dashboard_gen::{
Color, Dashboard, FieldOverride, Panel, Target, Thresholds, Unit, avg, percentiles,
percentiles_labeled, rate_per_min,
Color, Dashboard, FieldOverride, GradientMode, Panel, Target, Thresholds, Unit, avg,
percentile_legend, percentile_variable, rate_per_min, selected_percentile,
};
use json_pretty_compact::PrettyCompactFormatter;
use serde::Serialize as _;
const PERCENTILES: &[u32] = &[50, 90, 95, 99];
const DEFAULT_PERCENTILE: u32 = 95;
fn sequencer_dashboard() -> Dashboard {
Dashboard::new("Sequencer", "sequencer")
.tag("sequencer")
.variable(percentile_variable(PERCENTILES, DEFAULT_PERCENTILE))
.row(
7,
[
@ -30,13 +32,13 @@ fn sequencer_dashboard() -> Dashboard {
.decimals(0)
.color(Color::fixed("blue"))
.target(
Target::new(sequencer_core_metrics::names::BLOCK_COUNT).legend("height"),
Target::new(sequencer_core_metrics::names::BLOCKS_TOTAL).legend("height"),
),
Panel::timeseries("Block production rate")
.width(18)
.unit(Unit::Short)
.target(rate_per_min(
sequencer_core_metrics::names::BLOCK_COUNT,
sequencer_core_metrics::names::BLOCKS_TOTAL,
"blocks/min",
)),
],
@ -46,9 +48,10 @@ fn sequencer_dashboard() -> Dashboard {
[Panel::timeseries("Block creation time")
.width(24)
.unit(Unit::Seconds)
.targets(percentiles(
.target(selected_percentile(
sequencer_core_metrics::names::BLOCK_CREATION_TIME,
PERCENTILES,
&[],
&percentile_legend(),
))
.target(avg(sequencer_core_metrics::names::BLOCK_CREATION_TIME))
.with_override(
@ -63,17 +66,18 @@ fn sequencer_dashboard() -> Dashboard {
Panel::timeseries("Transaction application time")
.width(12)
.unit(Unit::Seconds)
.targets(percentiles_labeled(
.target(selected_percentile(
sequencer_core_metrics::names::MEMPOOL_TRANSACTION_APPLICATION_TIME,
PERCENTILES,
" · {{kind}} · {{origin}}",
&["kind", "origin"],
"{{kind}} · {{origin}}",
)),
Panel::timeseries("Transactions per block")
.width(12)
.unit(Unit::Short)
.targets(percentiles(
.target(selected_percentile(
sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK,
PERCENTILES,
&[],
&percentile_legend(),
))
.target(avg(sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK))
.with_override(
@ -141,11 +145,13 @@ fn sequencer_dashboard() -> Dashboard {
)
.target(
Target::new(format!(
// Both failure stages against the same submission base;
// `clamp_min` keeps an idle window (nothing submitted)
// reading as 0% instead of a division by zero.
"100 * increase({failed}[$__range]) / clamp_min(increase({submitted}[$__range]), 1)",
failed = sequencer_core_metrics::names::FAILED_TRANSACTION_COUNT,
submitted = sequencer_service_metrics::names::SUBMITTED_TRANSACTION_COUNT,
"100 * (increase({before_mempool}[$__range]) + increase({in_mempool}[$__range])) / clamp_min(increase({submitted}[$__range]), 1)",
before_mempool = sequencer_service_metrics::names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL,
in_mempool = sequencer_core_metrics::names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL,
submitted = sequencer_service_metrics::names::SUBMITTED_TRANSACTIONS_TOTAL,
))
.legend("failed"),
),
@ -153,17 +159,29 @@ fn sequencer_dashboard() -> Dashboard {
.width(18)
.unit(Unit::Short)
.min(0.0)
.fill_opacity(35)
.gradient_mode(GradientMode::Opacity)
.target(rate_per_min(
sequencer_service_metrics::names::SUBMITTED_TRANSACTION_COUNT,
sequencer_service_metrics::names::SUBMITTED_TRANSACTIONS_TOTAL,
"submitted",
))
.target(rate_per_min(
sequencer_core_metrics::names::FAILED_TRANSACTION_COUNT,
"failed",
sequencer_service_metrics::names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL,
"failed · before mempool",
))
.target(rate_per_min(
sequencer_core_metrics::names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL,
"failed · in mempool",
))
.with_override(FieldOverride::by_name("failed").color(Color::fixed("red")))
.with_override(
FieldOverride::by_name("submitted").color(Color::fixed("green")),
)
.with_override(
FieldOverride::by_name("failed · before mempool")
.color(Color::fixed("orange")),
)
.with_override(
FieldOverride::by_name("failed · in mempool").color(Color::fixed("red")),
),
],
)

View File

@ -10,7 +10,7 @@
use std::fmt::Write as _;
use crate::{
Unit,
DEFAULT_FILL_OPACITY, Unit,
input::{Defaults, PanelInput},
schema::{
AxisPlacement, Color, GradientMode, LineInterpolation, PanelType, ShowPoints, StackingMode,
@ -42,6 +42,11 @@ fn panel_expr_inner(panel: &PanelInput) -> Result<String, std::fmt::Error> {
write_defaults(&mut expr, defaults)?;
if let Some(custom) = &defaults.custom {
// Emitted against the builder's default, not Grafana's (which is 0), so
// a genuinely unfilled panel still round-trips.
if let Some(opacity) = custom.fill_opacity.filter(|&o| o != DEFAULT_FILL_OPACITY) {
write!(expr, "\n .fill_opacity({opacity})")?;
}
if custom.span_nulls == Some(true) {
expr.push_str("\n .span_nulls()");
}

View File

@ -66,6 +66,8 @@ pub struct Defaults {
#[derive(Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct Custom {
#[serde(default)]
pub fill_opacity: Option<u32>,
#[serde(default)]
pub span_nulls: Option<bool>,
#[serde(default)]

View File

@ -16,13 +16,14 @@
pub use codegen::panel_to_rust_source;
pub use schema::{
AxisPlacement, Color, GradientMode, LineInterpolation, ShowPoints, StackingMode, Thresholds,
Variable,
};
use schema::{
Calc, Custom, Datasource, Defaults, DrawStyle, EmptyList, FieldConfig, Fill, GaugeOptions,
GraphMode, GridPos, Legend, LegendDisplay, LineStyle, Matcher, MatcherKind, Options,
OverrideProperty, PanelModel, PanelType, Placement, PropertyId, PropertyValue, ReduceOptions,
SortOrder, Stacking, StatColorMode, StatOptions, TimeRange, TimeSeriesOptions, Tooltip,
TooltipMode,
SortOrder, Stacking, StatColorMode, StatOptions, Templating, TimeRange, TimeSeriesOptions,
Tooltip, TooltipMode, VariableKind, VariableOption,
};
use serde::Serialize;
pub use unit::Unit;
@ -38,6 +39,18 @@ mod unit;
/// than by a per-environment URL.
pub const DATASOURCE_UID: &str = "prometheus";
/// Window every histogram query rates over. `$__rate_interval` tracks the
/// panel's zoom, so a percentile covers the range you are actually looking at,
/// and an idle window yields no value rather than a zero.
const RATE_WINDOW: &str = "$__rate_interval";
/// Dashboard variable holding the quantile every percentile query reads.
const PERCENTILE_VAR: &str = "percentile";
/// Area fill under a timeseries line, as a percentage. Enough to read a series'
/// shape at a glance without drowning the ones stacked behind it.
pub(crate) const DEFAULT_FILL_OPACITY: u32 = 10;
/// A single Prometheus query within a panel.
#[derive(Clone, Serialize)]
#[serde(rename_all = "camelCase")]
@ -129,6 +142,7 @@ pub struct Panel {
span_nulls: bool,
overrides: Vec<FieldOverride>,
// Optional timeseries styling, set via the `styling` setters.
fill_opacity: Option<u32>,
line_interpolation: Option<LineInterpolation>,
show_points: Option<ShowPoints>,
gradient_mode: Option<GradientMode>,
@ -152,6 +166,7 @@ impl Panel {
thresholds: None,
span_nulls: false,
overrides: Vec::new(),
fill_opacity: None,
line_interpolation: None,
show_points: None,
gradient_mode: None,
@ -308,7 +323,7 @@ impl Panel {
custom: Some(Custom {
draw_style: DrawStyle::Line,
line_width: 1,
fill_opacity: 10,
fill_opacity: self.fill_opacity.unwrap_or(DEFAULT_FILL_OPACITY),
span_nulls: self.span_nulls.then_some(true),
line_interpolation: self.line_interpolation,
show_points: self.show_points,
@ -327,8 +342,11 @@ impl Panel {
thresholds: self.thresholds,
};
// Panels with several series read better as a sortable table with
// a multi-series tooltip; single-series panels stay compact.
let multi = targets.len() > 1;
// a multi-series tooltip; single-series panels stay compact. A
// `{{label}}` legend fans one target out into a series per label
// value, so it counts as several too.
let multi =
targets.len() > 1 || targets.iter().any(|target| target.legend.contains("{{"));
let options = Options::TimeSeries(TimeSeriesOptions {
legend: Legend {
display_mode: if multi {
@ -382,7 +400,7 @@ pub struct Dashboard {
refresh: String,
schema_version: u32,
tags: Vec<String>,
templating: EmptyList,
templating: Templating,
time: TimeRange,
timezone: String,
title: String,
@ -405,7 +423,7 @@ impl Dashboard {
refresh: "5s".to_owned(),
schema_version: 39,
tags: Vec::new(),
templating: EmptyList::default(),
templating: Templating::default(),
time: TimeRange {
from: "now-15m".to_owned(),
to: "now".to_owned(),
@ -430,6 +448,13 @@ impl Dashboard {
self
}
/// Add a dropdown to the dashboard's top bar, e.g. [`percentile_variable`].
#[must_use]
pub fn variable(mut self, variable: Variable) -> Self {
self.templating.list.push(variable);
self
}
/// Place a horizontal row of panels at the current vertical cursor. Panel
/// ids, x offsets and y are assigned here; unset widths split the remaining
/// 24 columns evenly.
@ -468,34 +493,89 @@ impl Dashboard {
}
}
/// Percentile line targets for a summary metric: `p50`, `p90`, … each querying
/// the matching `quantile="0.x"` series.
/// The dropdown driving every [`selected_percentile`] query, offering
/// `percentiles` (e.g. `[50, 90, 95, 99]`) with `default` pre-selected.
///
/// Panics if `default` is not one of `percentiles`.
#[must_use]
pub fn percentiles(metric: &str, percentiles: &[u32]) -> Vec<Target> {
percentiles_labeled(metric, percentiles, "")
}
pub fn percentile_variable(percentiles: &[u32], default: u32) -> Variable {
assert!(
percentiles.contains(&default),
"default p{default} is not one of the offered percentiles {percentiles:?}",
);
/// Like [`percentiles`], but appends `legend_suffix` to every legend — handy
/// when the metric carries labels (e.g. ` · {{kind}} · {{origin}}`).
#[must_use]
pub fn percentiles_labeled(metric: &str, percentiles: &[u32], legend_suffix: &str) -> Vec<Target> {
percentiles
let options: Vec<VariableOption> = percentiles
.iter()
.map(|&p| {
// `quantile="0.x"` label, derived without float math: zero-pad to two
// digits then drop trailing zeros (50 → "0.5", 95 → "0.95").
let quantile = format!("0.{p:02}");
let quantile = quantile.trim_end_matches('0');
Target::new(format!("{metric}{{quantile=\"{quantile}\"}}"))
.legend(format!("p{p}{legend_suffix}"))
.map(|&p| VariableOption {
selected: p == default,
text: format!("p{p}"),
value: quantile(p),
})
.collect()
.collect();
let current = options
.iter()
.find(|option| option.selected)
.cloned()
.expect("`default` is one of `percentiles`, asserted above");
let query = options
.iter()
.map(|option| format!("{} : {}", option.text, option.value))
.collect::<Vec<_>>()
.join(", ");
Variable {
current,
include_all: false,
label: "Percentile".to_owned(),
// A quantile is a scalar argument to `histogram_quantile`, so exactly
// one may be selected.
multi: false,
name: PERCENTILE_VAR.to_owned(),
options,
query,
kind: VariableKind::Custom,
}
}
/// An `avg` target for a summary metric: `rate(sum) / rate(count)` over 1m.
/// The legend fragment that renders the dropdown's current choice, e.g. `p95`.
#[must_use]
pub fn percentile_legend() -> String {
format!("${{{PERCENTILE_VAR}:text}}")
}
/// A [`histogram_quantile`] line over `metric`'s buckets, at whatever quantile
/// [`percentile_variable`] currently holds. `labels` stay split out into their
/// own series; every other label is summed away.
///
/// [`histogram_quantile`]: https://prometheus.io/docs/prometheus/latest/querying/functions/#histogram_quantile
#[must_use]
pub fn selected_percentile(metric: &str, labels: &[&str], legend: &str) -> Target {
// `le` carries the bucket boundary, so it must survive the aggregation.
let grouping = std::iter::once("le")
.chain(labels.iter().copied())
.collect::<Vec<_>>()
.join(", ");
Target::new(format!(
"histogram_quantile(${{{PERCENTILE_VAR}}}, sum by ({grouping}) (rate({metric}_bucket[{RATE_WINDOW}])))"
))
.legend(legend)
}
/// A percentile as its `histogram_quantile` argument, derived without float
/// math: zero-pad to two digits then drop trailing zeros (50 → `0.5`).
fn quantile(percentile: u32) -> String {
let quantile = format!("0.{percentile:02}");
quantile.trim_end_matches('0').to_owned()
}
/// An `avg` target for a histogram metric: `rate(sum) / rate(count)`.
#[must_use]
pub fn avg(metric: &str) -> Target {
Target::new(format!("rate({metric}_sum[1m]) / rate({metric}_count[1m])")).legend("avg")
Target::new(format!(
"rate({metric}_sum[{RATE_WINDOW}]) / rate({metric}_count[{RATE_WINDOW}])"
))
.legend("avg")
}
/// A per-minute rate target for a counter metric.

View File

@ -363,6 +363,43 @@ pub enum Options {
Gauge(GaugeOptions),
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum VariableKind {
/// A fixed list of choices, spelled out in the dashboard itself.
Custom,
}
/// One choice in a [`Variable`] dropdown: `text` is displayed, `value` is what
/// `$name` interpolates to in a query.
#[derive(Clone, Serialize)]
pub struct VariableOption {
pub selected: bool,
pub text: String,
pub value: String,
}
/// A dashboard-level dropdown, rendered in the top bar.
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Variable {
pub current: VariableOption,
pub include_all: bool,
pub label: String,
pub multi: bool,
pub name: String,
pub options: Vec<VariableOption>,
/// Grafana's own encoding of `options`, as `text : value` pairs.
pub query: String,
#[serde(rename = "type")]
pub kind: VariableKind,
}
#[derive(Serialize, Default)]
pub struct Templating {
pub list: Vec<Variable>,
}
#[derive(Clone, Copy, Serialize)]
pub struct GridPos {
pub h: u32,

View File

@ -11,7 +11,7 @@
//! not emitted, so the value is silently dropped.
use crate::{
Panel,
DEFAULT_FILL_OPACITY, Panel,
schema::{AxisPlacement, GradientMode, LineInterpolation, ShowPoints, StackingMode},
};
@ -20,6 +20,24 @@ use crate::{
reason = "styling setters intentionally live in their own file, so `Panel` has a second inherent impl here"
)]
impl Panel {
/// Area fill under the line, as a percentage. Builder default:
/// [`DEFAULT_FILL_OPACITY`].
///
/// Panics if passed that default, or a value above 100.
#[must_use]
pub fn fill_opacity(mut self, opacity: u32) -> Self {
assert!(
opacity <= 100,
"fill_opacity({opacity}) is not a percentage"
);
assert_ne!(
opacity, DEFAULT_FILL_OPACITY,
"fill_opacity({DEFAULT_FILL_OPACITY}) is redundant: it is the builder's default. Omit the call.",
);
self.fill_opacity = Some(opacity);
self
}
/// Interpolation between points. Grafana default: `Linear`.
///
/// Panics if passed `Linear` — that's the default and would be redundant.