From 037901fe941e66ebb73f5c83f4a7bacdcafaee36 Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Mon, 3 Aug 2026 19:08:30 +0300 Subject: [PATCH] fix(sequencer): differentiate between height and blocks produced by this sequencer --- docs/metrics/metrics.md | 18 +++---- lez/sequencer/core/metrics/src/names.rs | 3 +- lez/sequencer/core/metrics/src/record.rs | 29 ++++++---- lez/sequencer/core/src/lib.rs | 15 ++++-- monitoring/grafana/dashboards/sequencer.json | 53 ++++++++++++++----- .../dashboard_gen/src/dashboards/sequencer.rs | 22 ++++++-- 6 files changed, 98 insertions(+), 42 deletions(-) diff --git a/docs/metrics/metrics.md b/docs/metrics/metrics.md index 10298bdb..4d758157 100644 --- a/docs/metrics/metrics.md +++ b/docs/metrics/metrics.md @@ -8,7 +8,7 @@ Every crate that emits metrics gets a sibling `metrics` crate — `lez/sequencer | Module | Gated by | Contents | |---|---|---| -| `names` | always compiled | `pub const BLOCKS_TOTAL: &str = "blocks_total";` — one const per metric | +| `names` | always compiled | `pub const BLOCKS_PRODUCED_TOTAL: &str = "blocks_produced_total";` — one const per metric | | `record` | `record` feature | `record_*` / `increment_*` functions, plus `init()` | The emitting crate depends on it with `features = ["record"]`; consumers that only need the names (i.e. `dashboard_gen`) take the default features and pull in nothing. Dashboards reference the same consts the recording code does, so **renaming a metric is a compile error rather than a silently empty panel**. @@ -19,7 +19,7 @@ The recorder runs with `with_recommended_naming(true)`, which enforces Prometheu | Kind | Suffix | Example | |---|---|---| -| Counter | `_total` | `blocks_total`, `submitted_transactions_total` | +| Counter | `_total` | `blocks_produced_total`, `submitted_transactions_total` | | Histogram | unit | `block_creation_time_seconds` | | Gauge | none | `mempool_size` | @@ -30,22 +30,22 @@ The recorder runs with `with_recommended_naming(true)`, which enforces Prometheu | Type | Use for | Example | |---|---|---| | Counter | monotonically increasing event counts | `mempool_failed_transactions_total` | -| Gauge | a value that moves both ways | `mempool_size` | +| Gauge | a value that moves both ways | `mempool_size`, `chain_height` (a reorg lowers it) | | Histogram | distributions — latencies, sizes, per-batch counts | `mempool_transaction_application_time_seconds` | Each metric gets a private constructor plus a public recording wrapper, so its description, unit and labels are declared once: ```rust -fn blocks_total_counter() -> Counter { +fn blocks_produced_total_counter() -> Counter { counter!( - description: "Number of blocks in chain", + description: "Number of blocks produced by this sequencer and applied to the head", unit: Unit::Count, - names::BLOCKS_TOTAL + names::BLOCKS_PRODUCED_TOTAL ) } -pub fn increment_blocks_total() { - blocks_total_counter().increment(1); +pub fn increment_blocks_produced_total() { + blocks_produced_total_counter().increment(1); } ``` @@ -89,7 +89,7 @@ Panels are built fluently, and every query is composed from the `names` consts: ```rust Panel::timeseries("Block production rate") .width(18) - .target(rate_per_min(sequencer_core_metrics::names::BLOCKS_TOTAL, "blocks/min")) + .target(rate_per_min(sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL, "blocks/min")) ``` Query helpers: `rate_per_min` for counters, `avg` for histograms, and `selected_percentile` for percentile lines — the latter reads a `percentile` dashboard dropdown created by `percentile_variable`, so one panel serves p50/p90/p95/p99 instead of drawing all four. Rate windows use `$__rate_interval`, which tracks the panel's zoom. diff --git a/lez/sequencer/core/metrics/src/names.rs b/lez/sequencer/core/metrics/src/names.rs index 98034cd6..92b1595e 100644 --- a/lez/sequencer/core/metrics/src/names.rs +++ b/lez/sequencer/core/metrics/src/names.rs @@ -1,5 +1,6 @@ pub const BLOCK_CREATION_TIME: &str = "block_creation_time_seconds"; -pub const BLOCKS_TOTAL: &str = "blocks_total"; +pub const CHAIN_HEIGHT: &str = "chain_height"; +pub const BLOCKS_PRODUCED_TOTAL: &str = "blocks_produced_total"; pub const MEMPOOL_SIZE: &str = "mempool_size"; pub const MEMPOOL_MAX_SIZE: &str = "mempool_max_size"; pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = diff --git a/lez/sequencer/core/metrics/src/record.rs b/lez/sequencer/core/metrics/src/record.rs index 59c9087a..9a9b3f42 100644 --- a/lez/sequencer/core/metrics/src/record.rs +++ b/lez/sequencer/core/metrics/src/record.rs @@ -38,9 +38,10 @@ impl From for TxKind { /// Initialize metrics. pub fn init() { - blocks_total_counter().increment(0); + blocks_produced_total_counter().increment(0); mempool_failed_transactions_total_counter().increment(0); record_mempool_size(0); + record_chain_height(0); drop(block_creation_time_histogram()); drop(transactions_per_block_histogram()); @@ -63,20 +64,26 @@ pub fn record_block_creation_time(duration: Duration) { block_creation_time_histogram().record(duration.as_secs_f64()); } -fn blocks_total_counter() -> Counter { - counter!( - description: "Number of blocks in chain", +/// Height of the chain head, which moves backwards on a reorg, hence a gauge. +pub fn record_chain_height(height: u64) { + gauge!( + description: "Height of the chain head", unit: Unit::Count, - names::BLOCKS_TOTAL + names::CHAIN_HEIGHT + ) + .set(height as f64); +} + +fn blocks_produced_total_counter() -> Counter { + counter!( + description: "Number of blocks produced by this sequencer and applied to the head", + unit: Unit::Count, + names::BLOCKS_PRODUCED_TOTAL ) } -pub fn set_blocks_total(value: u64) { - blocks_total_counter().absolute(value); -} - -pub fn increment_blocks_total() { - blocks_total_counter().increment(1); +pub fn increment_blocks_produced_total() { + blocks_produced_total_counter().increment(1); } pub fn record_mempool_size(size: usize) { diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 1769a8a7..77888b35 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -153,6 +153,9 @@ impl SequencerCore { ) .expect("Failed to create database with genesis block"); + // Incrementing count for genesis. + sequencer_core_metrics::increment_blocks_produced_total(); + (store, genesis_state) } } @@ -311,7 +314,7 @@ impl SequencerCore { watchers, }; - sequencer_core_metrics::set_blocks_total(sequencer_core.chain_height()); + sequencer_core_metrics::record_chain_height(sequencer_core.chain_height()); (sequencer_core, mempool_handle) } @@ -610,6 +613,9 @@ impl SequencerCore { chain.head_state(), Some(&checkpoint_bytes), )?; + + sequencer_core_metrics::increment_blocks_produced_total(); + sequencer_core_metrics::record_chain_height(block.header.block_id); } // Neither branch persists anything, checkpoint included: the // inscription it holds as pending belongs to a block that is not @@ -941,7 +947,6 @@ impl SequencerCore { ); sequencer_core_metrics::record_block_creation_time(now.elapsed()); - sequencer_core_metrics::increment_blocks_total(); Ok(BlockWithMeta { block, withdrawals }) } @@ -1187,7 +1192,7 @@ fn apply_follow_update( // The lock is held across the persist below so disk writes land in apply // order — the produce path persists under this same lock. - let (resubmit_txs, outcome) = { + let (resubmit_txs, outcome, head_height) = { let mut chain = chain.lock().expect("chain state mutex poisoned"); // Outcomes align with `adopted`. @@ -1287,9 +1292,11 @@ fn apply_follow_update( }) .unwrap_or_else(|err| panic!("Failed to persist follow update: {err:#}")); - (resubmit_txs, outcome) + (resubmit_txs, outcome, head_tip.map_or(0, |tip| tip.id)) }; + sequencer_core_metrics::record_chain_height(head_height); + if outcome.accepted_deposits > 0 { info!( "Recorded {} Bedrock Deposit event(s); their mints are drained from the store on our next turn", diff --git a/monitoring/grafana/dashboards/sequencer.json b/monitoring/grafana/dashboards/sequencer.json index 20627045..85ee0b28 100644 --- a/monitoring/grafana/dashboards/sequencer.json +++ b/monitoring/grafana/dashboards/sequencer.json @@ -19,7 +19,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "blocks_total", + "expr": "chain_height", "legendFormat": "height", "refId": "A" } @@ -30,11 +30,40 @@ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { - "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" }, + "defaults": { "color": { "mode": "fixed", "fixedColor": "green" }, "unit": "short", "decimals": 0 }, "overrides": [ ] }, - "gridPos": { "h": 7, "w": 18, "x": 6, "y": 0 }, + "gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 }, "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "blocks_produced_total", + "legendFormat": "produced", + "refId": "A" + } + ], + "title": "Blocks produced by this sequencer since startup", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "produced · blocks/min" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] + } + ] + }, + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, + "id": 3, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "single" } @@ -42,8 +71,8 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(blocks_total[1m]) * 60", - "legendFormat": "blocks/min", + "expr": "rate(blocks_produced_total[1m]) * 60", + "legendFormat": "produced · blocks/min", "refId": "A" } ], @@ -65,7 +94,7 @@ ] }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 7 }, - "id": 3, + "id": 4, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } @@ -94,7 +123,7 @@ "overrides": [ ] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 16 }, - "id": 4, + "id": 5, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } @@ -125,7 +154,7 @@ ] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 }, - "id": 5, + "id": 6, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } @@ -167,7 +196,7 @@ "overrides": [ ] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 25 }, - "id": 6, + "id": 7, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, @@ -207,7 +236,7 @@ ] }, "gridPos": { "h": 8, "w": 18, "x": 6, "y": 25 }, - "id": 7, + "id": 8, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } @@ -249,7 +278,7 @@ "overrides": [ ] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 33 }, - "id": 8, + "id": 9, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, @@ -290,7 +319,7 @@ ] }, "gridPos": { "h": 8, "w": 18, "x": 6, "y": 33 }, - "id": 9, + "id": 10, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } diff --git a/tools/dashboard_gen/src/dashboards/sequencer.rs b/tools/dashboard_gen/src/dashboards/sequencer.rs index 3c609841..2956edfa 100644 --- a/tools/dashboard_gen/src/dashboards/sequencer.rs +++ b/tools/dashboard_gen/src/dashboards/sequencer.rs @@ -27,15 +27,27 @@ pub fn dashboard() -> Dashboard { .decimals(0) .color(Color::fixed("blue")) .target( - Target::new(sequencer_core_metrics::names::BLOCKS_TOTAL).legend("height"), + Target::new(sequencer_core_metrics::names::CHAIN_HEIGHT).legend("height"), + ), + Panel::stat("Blocks produced by this sequencer since startup") + .width(6) + .unit(Unit::Short) + .decimals(0) + .color(Color::fixed("green")) + .target( + Target::new(sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL) + .legend("produced"), ), Panel::timeseries("Block production rate") - .width(18) + .width(12) .unit(Unit::Short) .target(rate_per_min( - sequencer_core_metrics::names::BLOCKS_TOTAL, - "blocks/min", - )), + sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL, + "produced · blocks/min", + )) + .with_override( + FieldOverride::by_name("produced · blocks/min").color(Color::fixed("green")), + ), ], ) .row(