logos-blockchain/logos-execution-zone
1
0
Fork 0
mirror of https://github.com/logos-blockchain/logos-execution-zone.git synced 2026-08-05 04:23:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(sequencer): differentiate between height and blocks produced by this sequencer

Browse Source
This commit is contained in:
Daniil Polyakov 2026-08-03 19:08:30 +03:00
parent f4f78ddc28
commit 037901fe94
6 changed files with 98 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
docs/metrics/metrics.md
View File

@ -8,7 +8,7 @@ Every crate that emits metrics gets a sibling `metrics` crate — `lez/sequencer
| Module | Gated by | Contents |
|---|---|---|
| `names` | always compiled | `pub const BLOCKS_TOTAL: &str = "blocks_total";` — one const per metric |
| `names` | always compiled | `pub const BLOCKS_PRODUCED_TOTAL: &str = "blocks_produced_total";` — one const per metric |
| `record` | `record` feature | `record_*` / `increment_*` functions, plus `init()` |
The emitting crate depends on it with `features = ["record"]`; consumers that only need the names (i.e. `dashboard_gen`) take the default features and pull in nothing. Dashboards reference the same consts the recording code does, so **renaming a metric is a compile error rather than a silently empty panel**.
@ -19,7 +19,7 @@ The recorder runs with `with_recommended_naming(true)`, which enforces Prometheu
| Kind | Suffix | Example |
|---|---|---|
| Counter | `_total` | `blocks_total`, `submitted_transactions_total` |
| Counter | `_total` | `blocks_produced_total`, `submitted_transactions_total` |
| Histogram | unit | `block_creation_time_seconds` |
| Gauge | none | `mempool_size` |
@ -30,22 +30,22 @@ The recorder runs with `with_recommended_naming(true)`, which enforces Prometheu
| Type | Use for | Example |
|---|---|---|
| Counter | monotonically increasing event counts | `mempool_failed_transactions_total` |
| Gauge | a value that moves both ways | `mempool_size` |
| Gauge | a value that moves both ways | `mempool_size`, `chain_height` (a reorg lowers it) |
| Histogram | distributions — latencies, sizes, per-batch counts | `mempool_transaction_application_time_seconds` |
Each metric gets a private constructor plus a public recording wrapper, so its description, unit and labels are declared once:
```rust
fn blocks_total_counter() -> Counter {
fn blocks_produced_total_counter() -> Counter {
counter!(
description: "Number of blocks in chain",
description: "Number of blocks produced by this sequencer and applied to the head",
unit: Unit::Count,
names::BLOCKS_TOTAL
names::BLOCKS_PRODUCED_TOTAL
)
}
pub fn increment_blocks_total() {
blocks_total_counter().increment(1);
pub fn increment_blocks_produced_total() {
blocks_produced_total_counter().increment(1);
}
```
@ -89,7 +89,7 @@ Panels are built fluently, and every query is composed from the `names` consts:
```rust
Panel::timeseries("Block production rate")
.width(18)
.target(rate_per_min(sequencer_core_metrics::names::BLOCKS_TOTAL, "blocks/min"))
.target(rate_per_min(sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL, "blocks/min"))
```
Query helpers: `rate_per_min` for counters, `avg` for histograms, and `selected_percentile` for percentile lines — the latter reads a `percentile` dashboard dropdown created by `percentile_variable`, so one panel serves p50/p90/p95/p99 instead of drawing all four. Rate windows use `$__rate_interval`, which tracks the panel's zoom.

3
lez/sequencer/core/metrics/src/names.rs
View File

@ -1,5 +1,6 @@
pub const BLOCK_CREATION_TIME: &str = "block_creation_time_seconds";
pub const BLOCKS_TOTAL: &str = "blocks_total";
pub const CHAIN_HEIGHT: &str = "chain_height";
pub const BLOCKS_PRODUCED_TOTAL: &str = "blocks_produced_total";
pub const MEMPOOL_SIZE: &str = "mempool_size";
pub const MEMPOOL_MAX_SIZE: &str = "mempool_max_size";
pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str =

29
lez/sequencer/core/metrics/src/record.rs
View File

@ -38,9 +38,10 @@ impl From<common::transaction::TxKind> for TxKind {
/// Initialize metrics.
pub fn init() {
blocks_total_counter().increment(0);
blocks_produced_total_counter().increment(0);
mempool_failed_transactions_total_counter().increment(0);
record_mempool_size(0);
record_chain_height(0);
drop(block_creation_time_histogram());
drop(transactions_per_block_histogram());
@ -63,20 +64,26 @@ pub fn record_block_creation_time(duration: Duration) {
block_creation_time_histogram().record(duration.as_secs_f64());
}
fn blocks_total_counter() -> Counter {
counter!(
description: "Number of blocks in chain",
/// Height of the chain head, which moves backwards on a reorg, hence a gauge.
pub fn record_chain_height(height: u64) {
gauge!(
description: "Height of the chain head",
unit: Unit::Count,
names::BLOCKS_TOTAL
names::CHAIN_HEIGHT
)
.set(height as f64);
}
fn blocks_produced_total_counter() -> Counter {
counter!(
description: "Number of blocks produced by this sequencer and applied to the head",
unit: Unit::Count,
names::BLOCKS_PRODUCED_TOTAL
)
}
pub fn set_blocks_total(value: u64) {
blocks_total_counter().absolute(value);
}
pub fn increment_blocks_total() {
blocks_total_counter().increment(1);
pub fn increment_blocks_produced_total() {
blocks_produced_total_counter().increment(1);
}
pub fn record_mempool_size(size: usize) {

15
lez/sequencer/core/src/lib.rs
View File

@ -153,6 +153,9 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
)
.expect("Failed to create database with genesis block");
// Incrementing count for genesis.
sequencer_core_metrics::increment_blocks_produced_total();
(store, genesis_state)
}
}
@ -311,7 +314,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
watchers,
};
sequencer_core_metrics::set_blocks_total(sequencer_core.chain_height());
sequencer_core_metrics::record_chain_height(sequencer_core.chain_height());
(sequencer_core, mempool_handle)
}
@ -610,6 +613,9 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
chain.head_state(),
Some(&checkpoint_bytes),
)?;
sequencer_core_metrics::increment_blocks_produced_total();
sequencer_core_metrics::record_chain_height(block.header.block_id);
}
// Neither branch persists anything, checkpoint included: the
// inscription it holds as pending belongs to a block that is not
@ -941,7 +947,6 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
);
sequencer_core_metrics::record_block_creation_time(now.elapsed());
sequencer_core_metrics::increment_blocks_total();
Ok(BlockWithMeta { block, withdrawals })
}
@ -1187,7 +1192,7 @@ fn apply_follow_update(
// The lock is held across the persist below so disk writes land in apply
// order — the produce path persists under this same lock.
let (resubmit_txs, outcome) = {
let (resubmit_txs, outcome, head_height) = {
let mut chain = chain.lock().expect("chain state mutex poisoned");
// Outcomes align with `adopted`.
@ -1287,9 +1292,11 @@ fn apply_follow_update(
})
.unwrap_or_else(|err| panic!("Failed to persist follow update: {err:#}"));
(resubmit_txs, outcome)
(resubmit_txs, outcome, head_tip.map_or(0, |tip| tip.id))
};
sequencer_core_metrics::record_chain_height(head_height);
if outcome.accepted_deposits > 0 {
info!(
"Recorded {} Bedrock Deposit event(s); their mints are drained from the store on our next turn",

53
monitoring/grafana/dashboards/sequencer.json
View File

@ -19,7 +19,7 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "blocks_total",
"expr": "chain_height",
"legendFormat": "height",
"refId": "A"
}
@ -30,11 +30,40 @@
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" },
"defaults": { "color": { "mode": "fixed", "fixedColor": "green" }, "unit": "short", "decimals": 0 },
"overrides": [ ]
},
"gridPos": { "h": 7, "w": 18, "x": 6, "y": 0 },
"gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 },
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "blocks_produced_total",
"legendFormat": "produced",
"refId": "A"
}
],
"title": "Blocks produced by this sequencer since startup",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" },
"overrides": [
{
"matcher": { "id": "byName", "options": "produced · blocks/min" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ]
}
]
},
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 },
"id": 3,
"options": {
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "single" }
@ -42,8 +71,8 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(blocks_total[1m]) * 60",
"legendFormat": "blocks/min",
"expr": "rate(blocks_produced_total[1m]) * 60",
"legendFormat": "produced · blocks/min",
"refId": "A"
}
],
@ -65,7 +94,7 @@
]
},
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 7 },
"id": 3,
"id": 4,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
@ -94,7 +123,7 @@
"overrides": [ ]
},
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 16 },
"id": 4,
"id": 5,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
@ -125,7 +154,7 @@
]
},
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 },
"id": 5,
"id": 6,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
@ -167,7 +196,7 @@
"overrides": [ ]
},
"gridPos": { "h": 8, "w": 6, "x": 0, "y": 25 },
"id": 6,
"id": 7,
"options": {
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
"showThresholdLabels": false,
@ -207,7 +236,7 @@
]
},
"gridPos": { "h": 8, "w": 18, "x": 6, "y": 25 },
"id": 7,
"id": 8,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
@ -249,7 +278,7 @@
"overrides": [ ]
},
"gridPos": { "h": 8, "w": 6, "x": 0, "y": 33 },
"id": 8,
"id": 9,
"options": {
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
"showThresholdLabels": false,
@ -290,7 +319,7 @@
]
},
"gridPos": { "h": 8, "w": 18, "x": 6, "y": 33 },
"id": 9,
"id": 10,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }

22
tools/dashboard_gen/src/dashboards/sequencer.rs
View File

@ -27,15 +27,27 @@ pub fn dashboard() -> Dashboard {
.decimals(0)
.color(Color::fixed("blue"))
.target(
Target::new(sequencer_core_metrics::names::BLOCKS_TOTAL).legend("height"),
Target::new(sequencer_core_metrics::names::CHAIN_HEIGHT).legend("height"),
),
Panel::stat("Blocks produced by this sequencer since startup")
.width(6)
.unit(Unit::Short)
.decimals(0)
.color(Color::fixed("green"))
.target(
Target::new(sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL)
.legend("produced"),
),
Panel::timeseries("Block production rate")
.width(18)
.width(12)
.unit(Unit::Short)
.target(rate_per_min(
sequencer_core_metrics::names::BLOCKS_TOTAL,
"blocks/min",
)),
sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL,
"produced · blocks/min",
))
.with_override(
FieldOverride::by_name("produced · blocks/min").color(Color::fixed("green")),
),
],
)
.row(