From 8afd2cea66da0e81a42d43168b4ec1fea11ebf3e Mon Sep 17 00:00:00 2001 From: Daniil Polyakov Date: Fri, 24 Jul 2026 01:37:21 +0300 Subject: [PATCH] feat(docker): introduce monitoring --- Justfile | 13 +- docker-compose.yml | 2 + lez/sequencer/core/src/lib.rs | 2 +- lez/sequencer/core/src/metrics.rs | 6 +- monitoring/docker-compose.yml | 36 +++ monitoring/grafana/dashboards/sequencer.json | 296 ++++++++++++++++++ .../provisioning/dashboards/dashboards.yml | 9 + .../provisioning/datasources/prometheus.yml | 10 + monitoring/prometheus/prometheus.yml | 15 + 9 files changed, 383 insertions(+), 6 deletions(-) create mode 100644 monitoring/docker-compose.yml create mode 100644 monitoring/grafana/dashboards/sequencer.json create mode 100644 monitoring/grafana/provisioning/dashboards/dashboards.yml create mode 100644 monitoring/grafana/provisioning/datasources/prometheus.yml create mode 100644 monitoring/prometheus/prometheus.yml diff --git a/Justfile b/Justfile index ec22afb9..ee122105 100644 --- a/Justfile +++ b/Justfile @@ -65,6 +65,13 @@ run-bedrock: @echo "鉀擄笍 Running bedrock" docker compose up +# Run Prometheus + Grafana in docker. Grafana: http://localhost:3000 (anonymous +# admin), Prometheus: http://localhost:9090. Scrapes the sequencer's /metrics. +[working-directory: 'monitoring'] +run-monitoring: + @echo "馃搳 Running Prometheus + Grafana" + docker compose up + # Run Sequencer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. # Optional home/port let a second instance run off the same config, e.g. # `just run-sequencer "" "$TMPDIR/lez-sequencer2" 3041` for the multi-sequencer demo. @@ -103,8 +110,9 @@ run-wallet +args: @echo "馃攽 Running wallet" LEE_WALLET_HOME_DIR=$(pwd)/configs/debug cargo run --release -p wallet -- {{args}} +# Query sequencer metrics in raw format. Useful for quick debugging. For a more detailed view, use `just run-monitoring`. get-sequencer-metrics: - @echo "馃搳 Querying sequencer metrics" + @echo "馃搳 Querying sequencer's metrics" curl http://localhost:9000/metrics # Import test accounts supplied in sequencer configuration. @@ -147,4 +155,5 @@ clean: rm -rf lez/wallet/configs/debug/storage.json rm -rf lez/wallet/configs/debug/statistics.json rm -rf rocksdb* - cd bedrock && docker compose down -v + cd bedrock && docker compose down -v && cd .. + cd monitoring && docker compose down -v && cd .. diff --git a/docker-compose.yml b/docker-compose.yml index 3644b2aa..4da0d04c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,3 +11,5 @@ include: lez/indexer/service/docker-compose.yml - path: lez/explorer_service/docker-compose.yml + - path: + monitoring/docker-compose.yml diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 1dd55c5b..a56b3c88 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -910,7 +910,7 @@ impl SequencerCore { .transition_from_public_transaction(&clock_tx, new_block_height, new_block_timestamp) .context("Clock transaction failed. Aborting block production.")?; valid_transactions.push(clock_lee_tx); - crate::metrics::set_block_transaction_count(valid_transactions.len()); + crate::metrics::record_transactions_per_block(valid_transactions.len()); let hashable_data = HashableBlockData { block_id: new_block_height, diff --git a/lez/sequencer/core/src/metrics.rs b/lez/sequencer/core/src/metrics.rs index 2486b3aa..a6c79454 100644 --- a/lez/sequencer/core/src/metrics.rs +++ b/lez/sequencer/core/src/metrics.rs @@ -18,7 +18,7 @@ mod names { pub const BLOCK_COUNT: &str = "block_count"; pub const MEMPOOL_SIZE: &str = "mempool_size"; pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = "mempool_transaction_application_time"; - pub const BLOCK_TRANSACTION_COUNT: &str = "block_transaction_count"; + pub const TRANSACTIONS_PER_BLOCK: &str = "transactions_per_block"; pub const FAILED_TRANSACTION_COUNT: &str = "failed_transaction_count"; } @@ -71,11 +71,11 @@ pub fn record_mempool_transaction_application_time( .record(duration.as_secs_f64()); } -pub fn set_block_transaction_count(count: usize) { +pub fn record_transactions_per_block(count: usize) { histogram!( description: "Number of transactions included in block", unit: Unit::Count, - names::BLOCK_TRANSACTION_COUNT + names::TRANSACTIONS_PER_BLOCK ) .record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64); } diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml new file mode 100644 index 00000000..71953636 --- /dev/null +++ b/monitoring/docker-compose.yml @@ -0,0 +1,36 @@ +# Prometheus + Grafana monitoring stack. +services: + prometheus: + image: prom/prometheus:v3.1.0 + container_name: prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + # Lets Prometheus reach services running natively on the host. + extra_hosts: + - "host.docker.internal:host-gateway" + + grafana: + image: grafana/grafana:11.4.0 + container_name: grafana + ports: + - "3000:3000" + environment: + # Dev-only: open Grafana with no login, full access. + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + depends_on: + - prometheus + +volumes: + prometheus_data: + grafana_data: diff --git a/monitoring/grafana/dashboards/sequencer.json b/monitoring/grafana/dashboards/sequencer.json new file mode 100644 index 00000000..39824dd2 --- /dev/null +++ b/monitoring/grafana/dashboards/sequencer.json @@ -0,0 +1,296 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "graphTooltip": 1, + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "blue" }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "block_count_total", + "legendFormat": "height", + "refId": "A" + } + ], + "title": "Chain height", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 18, "x": 6, "y": 0 }, + "id": 2, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "single" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(block_count_total[1m]) * 60", + "legendFormat": "blocks/min", + "refId": "A" + } + ], + "title": "Block production rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "avg" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "dash": [8, 4], "fill": "dash" } }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } } + ] + } + ] + }, + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 7 }, + "id": 3, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "block_creation_time_seconds{quantile=\"0.5\"}", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "block_creation_time_seconds{quantile=\"0.9\"}", + "legendFormat": "p90", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "block_creation_time_seconds{quantile=\"0.95\"}", + "legendFormat": "p95", + "refId": "C" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "block_creation_time_seconds{quantile=\"0.99\"}", + "legendFormat": "p99", + "refId": "D" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(block_creation_time_seconds_sum[1m]) / rate(block_creation_time_seconds_count[1m])", + "legendFormat": "avg", + "refId": "E" + } + ], + "title": "Block creation time", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 16 }, + "id": 4, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_transaction_application_time_seconds{quantile=\"0.5\"}", + "legendFormat": "p50 路 {{kind}} 路 {{origin}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_transaction_application_time_seconds{quantile=\"0.9\"}", + "legendFormat": "p90 路 {{kind}} 路 {{origin}}", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_transaction_application_time_seconds{quantile=\"0.95\"}", + "legendFormat": "p95 路 {{kind}} 路 {{origin}}", + "refId": "C" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_transaction_application_time_seconds{quantile=\"0.99\"}", + "legendFormat": "p99 路 {{kind}} 路 {{origin}}", + "refId": "D" + } + ], + "title": "Transaction application time", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "spanNulls": true }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 }, + "id": 5, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "single" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "mempool_size", + "legendFormat": "mempool size", + "refId": "A" + } + ], + "title": "Mempool size", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "avg" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "dash": [8, 4], "fill": "dash" } }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } } + ] + } + ] + }, + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 25 }, + "id": 6, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "transactions_per_block{quantile=\"0.5\"}", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "transactions_per_block{quantile=\"0.9\"}", + "legendFormat": "p90", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "transactions_per_block{quantile=\"0.95\"}", + "legendFormat": "p95", + "refId": "C" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "transactions_per_block{quantile=\"0.99\"}", + "legendFormat": "p99", + "refId": "D" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(transactions_per_block_sum[1m]) / rate(transactions_per_block_count[1m])", + "legendFormat": "avg", + "refId": "E" + } + ], + "title": "Transactions per block", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "failed" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] + }, + { + "matcher": { "id": "byName", "options": "submitted" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] + } + ] + }, + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 25 }, + "id": 7, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(submitted_transaction_count_total[1m]) * 60", + "legendFormat": "submitted", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(failed_transaction_count_total[1m]) * 60", + "legendFormat": "failed", + "refId": "B" + } + ], + "title": "Transaction throughput (per minute)", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": ["sequencer"], + "templating": { "list": [] }, + "time": { "from": "now-15m", "to": "now" }, + "timezone": "", + "title": "Sequencer", + "uid": "sequencer" +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 00000000..e26d8f8c --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: sequencer + type: file + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..0b304bc9 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 00000000..aaae81b0 --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: sequencer + metrics_path: /metrics + static_configs: + # `sequencer_service:9000` resolves when monitoring runs inside the + # all-in-one compose network. `host.docker.internal:9000` resolves when + # the sequencer runs natively on the host and only monitoring runs in Docker. + # Whichever is unreachable stays `down`. + - targets: + - sequencer_service:9000 + - host.docker.internal:9000