diff --git a/Justfile b/Justfile index ed124de1..4741f6c8 100644 --- a/Justfile +++ b/Justfile @@ -79,19 +79,17 @@ run-monitoring: @echo "๐Ÿ“Š Running Prometheus (http://localhost:9090) + Grafana (http://localhost:3000)" docker compose up -# Run Sequencer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. -# Optional home/port let a second instance run off the same config, e.g. -# `just run-sequencer "" "$TMPDIR/lez-sequencer2" 3041` for the multi-sequencer demo. +# Run Sequencer. Extra args are forwarded to the binary. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. [working-directory: 'lez/sequencer/service'] -run-sequencer standalone="" home="" port="3040": +run-sequencer *args: @echo "๐Ÿง  Running sequencer" - @if [ "{{standalone}}" = "standalone" ]; then \ - echo "๐Ÿงช Running in standalone mode"; \ - RUST_LOG=info cargo run --features standalone --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \ - else \ - echo "๐Ÿš€ Running in normal mode"; \ - RUST_LOG=info cargo run --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \ - fi + RUST_LOG=info cargo run --release -p sequencer_service -- configs/debug/sequencer_config.json {{args}} + +# Run Sequencer with mocked Bedrock clients. Takes the same args as `run-sequencer`. +[working-directory: 'lez/sequencer/service'] +run-sequencer-standalone *args: + @echo "๐Ÿงช Running sequencer in standalone mode" + RUST_LOG=info cargo run --features standalone --release -p sequencer_service -- configs/debug/sequencer_config.json {{args}} # Run Indexer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. [working-directory: 'lez/indexer/service'] diff --git a/docs/metrics/metrics.md b/docs/metrics/metrics.md index 3794a846..10298bdb 100644 --- a/docs/metrics/metrics.md +++ b/docs/metrics/metrics.md @@ -65,7 +65,7 @@ For histograms, creating the handle publishes zeroed buckets without recording a ## Exporter setup -`sequencer_service`'s `main.rs` installs the Prometheus recorder on `:9000` with **explicit histogram buckets**. This matters: without buckets, `metrics-exporter-prometheus` renders histograms as rolling-window summaries whose quantiles **reset to `0`** once the window (default 60 s) drains โ€” an idle period reads as "took 0 s" rather than "no data". With buckets you get real `_bucket`/`_sum`/`_count` counters that never decay, are aggregatable, and honour the dashboard's time range. +`sequencer_service`'s `main.rs` installs the Prometheus recorder on the config's `metrics_address` (default `0.0.0.0:9000`) with **explicit histogram buckets**. This matters: without buckets, `metrics-exporter-prometheus` renders histograms as rolling-window summaries whose quantiles **reset to `0`** once the window (default 60 s) drains โ€” an idle period reads as "took 0 s" rather than "no data". With buckets you get real `_bucket`/`_sum`/`_count` counters that never decay, are aggregatable, and honour the dashboard's time range. Ladders are matched by name suffix, so a new timing metric is covered automatically: diff --git a/lez/sequencer/core/src/config.rs b/lez/sequencer/core/src/config.rs index 60bd8502..42cd4945 100644 --- a/lez/sequencer/core/src/config.rs +++ b/lez/sequencer/core/src/config.rs @@ -1,6 +1,7 @@ use std::{ fs::File, io::BufReader, + net::{IpAddr, Ipv4Addr, SocketAddr}, path::{Path, PathBuf}, time::Duration, }; @@ -63,6 +64,9 @@ pub struct SequencerConfig { /// Cross-zone messaging configuration. `None` disables the watcher. #[serde(default)] pub cross_zone: Option, + /// Address the Prometheus metrics exporter binds to. + #[serde(default = "default_metrics_address")] + pub metrics_address: SocketAddr, } #[derive(Clone, Serialize, Deserialize)] @@ -77,6 +81,10 @@ pub struct BedrockConfig { } impl SequencerConfig { + /// Address [`Self::metrics_address`] falls back to when the config omits it. + pub const DEFAULT_METRICS_ADDRESS: SocketAddr = + SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 9000); + pub fn from_path(config_home: &Path) -> Result { let file = File::open(config_home)?; let reader = BufReader::new(file); @@ -88,3 +96,7 @@ impl SequencerConfig { const fn default_max_block_size() -> ByteSize { ByteSize::mib(1) } + +const fn default_metrics_address() -> SocketAddr { + SequencerConfig::DEFAULT_METRICS_ADDRESS +} diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 863bb99e..10b398e0 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -90,6 +90,7 @@ fn setup_sequencer_config() -> SequencerConfig { retry_pending_blocks_timeout: Duration::from_mins(4), genesis: vec![], cross_zone: None, + metrics_address: SequencerConfig::DEFAULT_METRICS_ADDRESS, } } diff --git a/lez/sequencer/service/src/main.rs b/lez/sequencer/service/src/main.rs index e648bd70..09df20f9 100644 --- a/lez/sequencer/service/src/main.rs +++ b/lez/sequencer/service/src/main.rs @@ -25,6 +25,10 @@ struct Args { /// so multiple instances can share one config file. #[clap(long)] home: Option, + /// Override the config's `metrics_address`, so multiple instances can share + /// one config file without fighting over the exporter port. + #[clap(long)] + metrics_address: Option, } #[tokio::main] @@ -35,13 +39,12 @@ struct Args { async fn main() -> Result<()> { env_logger::init(); - install_prometheus_recorder()?; - let Args { config_path, port, listen_address, home, + metrics_address, } = Args::parse(); let cancellation_token = listen_for_shutdown_signal(); @@ -50,6 +53,11 @@ async fn main() -> Result<()> { if let Some(home) = home { config.home = home; } + if let Some(metrics_address) = metrics_address { + config.metrics_address = metrics_address; + } + + install_prometheus_recorder(config.metrics_address)?; let mut sequencer_handle = sequencer_service::run(config, SocketAddr::new(listen_address, port)).await?; @@ -74,17 +82,8 @@ async fn main() -> Result<()> { Ok(()) } -/// Installs the recorder with explicit buckets, which makes every histogram -/// export as a Prometheus histogram (`_bucket`/`_sum`/`_count`) rather than the -/// default rolling-window summary. Summary quantiles reset to zero once their -/// window drains, so an idle period reads as "took 0s" instead of "no data". -/// -/// Ladders are picked by name suffix, so a new timing metric is covered without -/// touching this function. The matcher sees the name as registered, *before* -/// [`PrometheusBuilder::with_recommended_naming`] appends a unit suffix of its -/// own โ€” a duration metric whose name omits `_seconds` silently falls through to -/// [`COUNT_BUCKETS`]. -fn install_prometheus_recorder() -> Result<()> { +/// Installs the recorder on `metrics_address`. +fn install_prometheus_recorder(metrics_address: SocketAddr) -> Result<()> { /// Ladder for `*_seconds` histograms, densest across the 1โ€“100 ms band where /// block production and transaction application actually land. const LATENCY_BUCKETS: &[f64] = &[ @@ -95,6 +94,7 @@ fn install_prometheus_recorder() -> Result<()> { const COUNT_BUCKETS: &[f64] = &[1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0]; PrometheusBuilder::new() + .with_http_listener(metrics_address) .with_recommended_naming(true) .set_buckets(COUNT_BUCKETS) .context("Failed to set default histogram buckets")? diff --git a/test_fixtures/src/config.rs b/test_fixtures/src/config.rs index 8df57fb3..4de7f511 100644 --- a/test_fixtures/src/config.rs +++ b/test_fixtures/src/config.rs @@ -110,6 +110,7 @@ pub fn sequencer_config( auth: None, }, cross_zone, + metrics_address: SequencerConfig::DEFAULT_METRICS_ADDRESS, }) }