feat(docker): introduce monitoring

Justfile

@@ -65,6 +65,13 @@ run-bedrock:
     @echo "⛓️ Running bedrock"
     docker compose up
 
+# Run Prometheus + Grafana in docker. Grafana: http://localhost:3000 (anonymous
+# admin), Prometheus: http://localhost:9090. Scrapes the sequencer's /metrics.
+[working-directory: 'monitoring']
+run-monitoring:
+    @echo "📊 Running Prometheus + Grafana"
+    docker compose up
+
 # Run Sequencer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration.
 # Optional home/port let a second instance run off the same config, e.g.
 # `just run-sequencer "" "$TMPDIR/lez-sequencer2" 3041` for the multi-sequencer demo.
@@ -103,8 +110,9 @@ run-wallet +args:
     @echo "🔑 Running wallet"
     LEE_WALLET_HOME_DIR=$(pwd)/configs/debug cargo run --release -p wallet -- {{args}}
 
+# Query sequencer metrics in raw format. Useful for quick debugging. For a more detailed view, use `just run-monitoring`.
 get-sequencer-metrics:
-    @echo "📊 Querying sequencer metrics"
+    @echo "📊 Querying sequencer's metrics"
     curl http://localhost:9000/metrics
 
 # Import test accounts supplied in sequencer configuration.
@@ -147,4 +155,5 @@ clean:
     rm -rf lez/wallet/configs/debug/storage.json
     rm -rf lez/wallet/configs/debug/statistics.json
     rm -rf rocksdb*
-    cd bedrock && docker compose down -v
+    cd bedrock && docker compose down -v && cd ..
+    cd monitoring && docker compose down -v && cd ..

docker-compose.yml

@@ -11,3 +11,5 @@ include:
       lez/indexer/service/docker-compose.yml
   - path:
       lez/explorer_service/docker-compose.yml
+  - path:
+      monitoring/docker-compose.yml

lez/sequencer/core/src/lib.rs

@@ -910,7 +910,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
             .transition_from_public_transaction(&clock_tx, new_block_height, new_block_timestamp)
             .context("Clock transaction failed. Aborting block production.")?;
         valid_transactions.push(clock_lee_tx);
-        crate::metrics::set_block_transaction_count(valid_transactions.len());
+        crate::metrics::record_transactions_per_block(valid_transactions.len());
 
         let hashable_data = HashableBlockData {
             block_id: new_block_height,

lez/sequencer/core/src/metrics.rs

@@ -18,7 +18,7 @@ mod names {
     pub const BLOCK_COUNT: &str = "block_count";
     pub const MEMPOOL_SIZE: &str = "mempool_size";
     pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str = "mempool_transaction_application_time";
-    pub const BLOCK_TRANSACTION_COUNT: &str = "block_transaction_count";
+    pub const TRANSACTIONS_PER_BLOCK: &str = "transactions_per_block";
     pub const FAILED_TRANSACTION_COUNT: &str = "failed_transaction_count";
 }
 
@@ -71,11 +71,11 @@ pub fn record_mempool_transaction_application_time(
     .record(duration.as_secs_f64());
 }
 
-pub fn set_block_transaction_count(count: usize) {
+pub fn record_transactions_per_block(count: usize) {
     histogram!(
         description: "Number of transactions included in block",
         unit: Unit::Count,
-        names::BLOCK_TRANSACTION_COUNT
+        names::TRANSACTIONS_PER_BLOCK
     )
     .record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64);
 }

monitoring/docker-compose.yml

@@ -0,0 +1,36 @@
+# Prometheus + Grafana monitoring stack.
+services:
+  prometheus:
+    image: prom/prometheus:v3.1.0
+    container_name: prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus_data:/prometheus
+    # Lets Prometheus reach services running natively on the host.
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  grafana:
+    image: grafana/grafana:11.4.0
+    container_name: grafana
+    ports:
+      - "3000:3000"
+    environment:
+      # Dev-only: open Grafana with no login, full access.
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - grafana_data:/var/lib/grafana
+    depends_on:
+      - prometheus
+
+volumes:
+  prometheus_data:
+  grafana_data:

monitoring/grafana/dashboards/sequencer.json

@@ -0,0 +1,296 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "graphTooltip": 1,
+  "panels": [
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "fixed", "fixedColor": "blue" },
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "block_count_total",
+          "legendFormat": "height",
+          "refId": "A"
+        }
+      ],
+      "title": "Chain height",
+      "type": "stat"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 7, "w": 18, "x": 6, "y": 0 },
+      "id": 2,
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["last", "max"] },
+        "tooltip": { "mode": "single" }
+      },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "rate(block_count_total[1m]) * 60",
+          "legendFormat": "blocks/min",
+          "refId": "A"
+        }
+      ],
+      "title": "Block production rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 },
+          "unit": "s"
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "avg" },
+            "properties": [
+              { "id": "custom.lineStyle", "value": { "dash": [8, 4], "fill": "dash" } },
+              { "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } }
+            ]
+          }
+        ]
+      },
+      "gridPos": { "h": 9, "w": 24, "x": 0, "y": 7 },
+      "id": 3,
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "block_creation_time_seconds{quantile=\"0.5\"}",
+          "legendFormat": "p50",
+          "refId": "A"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "block_creation_time_seconds{quantile=\"0.9\"}",
+          "legendFormat": "p90",
+          "refId": "B"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "block_creation_time_seconds{quantile=\"0.95\"}",
+          "legendFormat": "p95",
+          "refId": "C"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "block_creation_time_seconds{quantile=\"0.99\"}",
+          "legendFormat": "p99",
+          "refId": "D"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "rate(block_creation_time_seconds_sum[1m]) / rate(block_creation_time_seconds_count[1m])",
+          "legendFormat": "avg",
+          "refId": "E"
+        }
+      ],
+      "title": "Block creation time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 16 },
+      "id": 4,
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "mempool_transaction_application_time_seconds{quantile=\"0.5\"}",
+          "legendFormat": "p50 · {{kind}} · {{origin}}",
+          "refId": "A"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "mempool_transaction_application_time_seconds{quantile=\"0.9\"}",
+          "legendFormat": "p90 · {{kind}} · {{origin}}",
+          "refId": "B"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "mempool_transaction_application_time_seconds{quantile=\"0.95\"}",
+          "legendFormat": "p95 · {{kind}} · {{origin}}",
+          "refId": "C"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "mempool_transaction_application_time_seconds{quantile=\"0.99\"}",
+          "legendFormat": "p99 · {{kind}} · {{origin}}",
+          "refId": "D"
+        }
+      ],
+      "title": "Transaction application time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "spanNulls": true },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 },
+      "id": 5,
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["last", "max"] },
+        "tooltip": { "mode": "single" }
+      },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "mempool_size",
+          "legendFormat": "mempool size",
+          "refId": "A"
+        }
+      ],
+      "title": "Mempool size",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "avg" },
+            "properties": [
+              { "id": "custom.lineStyle", "value": { "dash": [8, 4], "fill": "dash" } },
+              { "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } }
+            ]
+          }
+        ]
+      },
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 25 },
+      "id": 6,
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "transactions_per_block{quantile=\"0.5\"}",
+          "legendFormat": "p50",
+          "refId": "A"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "transactions_per_block{quantile=\"0.9\"}",
+          "legendFormat": "p90",
+          "refId": "B"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "transactions_per_block{quantile=\"0.95\"}",
+          "legendFormat": "p95",
+          "refId": "C"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "transactions_per_block{quantile=\"0.99\"}",
+          "legendFormat": "p99",
+          "refId": "D"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "rate(transactions_per_block_sum[1m]) / rate(transactions_per_block_count[1m])",
+          "legendFormat": "avg",
+          "refId": "E"
+        }
+      ],
+      "title": "Transactions per block",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "failed" },
+            "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }]
+          },
+          {
+            "matcher": { "id": "byName", "options": "submitted" },
+            "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }]
+          }
+        ]
+      },
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 25 },
+      "id": 7,
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "rate(submitted_transaction_count_total[1m]) * 60",
+          "legendFormat": "submitted",
+          "refId": "A"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "rate(failed_transaction_count_total[1m]) * 60",
+          "legendFormat": "failed",
+          "refId": "B"
+        }
+      ],
+      "title": "Transaction throughput (per minute)",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 39,
+  "tags": ["sequencer"],
+  "templating": { "list": [] },
+  "time": { "from": "now-15m", "to": "now" },
+  "timezone": "",
+  "title": "Sequencer",
+  "uid": "sequencer"
+}

monitoring/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+providers:
+  - name: sequencer
+    type: file
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: false

monitoring/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    uid: prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true

monitoring/prometheus/prometheus.yml

@@ -0,0 +1,15 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+  - job_name: sequencer
+    metrics_path: /metrics
+    static_configs:
+      # `sequencer_service:9000` resolves when monitoring runs inside the
+      # all-in-one compose network. `host.docker.internal:9000` resolves when
+      # the sequencer runs natively on the host and only monitoring runs in Docker.
+      # Whichever is unreachable stays `down`.
+      - targets:
+          - sequencer_service:9000
+          - host.docker.internal:9000