refactor(lez): sequencer retries on startup when bedrock is unreachable

Cargo.lock

@@ -990,7 +990,7 @@ dependencies = [
  "bitflags 2.12.1",
  "cexpr",
  "clang-sys",
- "itertools 0.11.0",
+ "itertools 0.13.0",
  "proc-macro2",
  "quote",
  "regex",
@@ -8095,7 +8095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
 dependencies = [
  "anyhow",
- "itertools 0.11.0",
+ "itertools 0.14.0",
  "proc-macro2",
  "quote",
  "syn 2.0.117",
@@ -8108,7 +8108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
 dependencies = [
  "anyhow",
- "itertools 0.11.0",
+ "itertools 0.14.0",
  "proc-macro2",
  "quote",
  "syn 2.0.117",
@@ -9516,6 +9516,7 @@ dependencies = [
  "testnet_initial_state",
  "token_core",
  "tokio",
+ "tokio-retry",
  "tokio-util",
  "url",
  "vault_core",
@@ -10793,6 +10794,17 @@ dependencies = [
  "syn 2.0.117",
 ]
 
+[[package]]
+name = "tokio-retry"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470"
+dependencies = [
+ "pin-project-lite",
+ "rand 0.10.1",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.26.4"

lez/sequencer/core/Cargo.toml

@@ -39,6 +39,7 @@ chrono.workspace = true
 log.workspace = true
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 tokio-util.workspace = true
+tokio-retry.workspace = true
 rand.workspace = true
 borsh.workspace = true
 bytesize.workspace = true

lez/sequencer/core/src/lib.rs

@@ -43,6 +43,7 @@ use storage::sequencer::{
         WithdrawalReconciliationKey, ZoneAnchorRecord,
     },
 };
+use tokio_retry::{Retry, strategy::FixedInterval};
 
 use crate::{
     block_publisher::{BlockPublisherTrait, MsgId, NoteId, ZoneSdkPublisher},
@@ -140,6 +141,9 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
     /// `DEFAULT_SEQUENCER_POSTING_TIMEOUT` slots) plus normal confirmation lag.
     const COMMITTEE_SUBMISSION_COOLDOWN: Duration = Duration::from_secs(20);
 
+    const CHANNEL_PROBE_RETRIES: usize = 29;
+    const CHANNEL_PROBE_RETRY_DELAY: Duration = Duration::from_secs(2);
+
     /// Starts the sequencer using the provided configuration.
     /// If an existing database is found, the sequencer state is loaded from it and
     /// assumed to represent the correct latest state consistent with Bedrock-finalized data.
@@ -264,9 +268,15 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
         // Only seed our own key into genesis as the bootstrap sequencer if the
         // channel doesn't exist yet. Otherwise it's someone else's channel and
         // we join later, the normal self-join way.
-        let channel_already_exists = BP::channel_exists(&config.bedrock_config)
-            .await
-            .expect("Failed to probe Bedrock channel");
+        let channel_probe_retry_strategy =
+            FixedInterval::new(Self::CHANNEL_PROBE_RETRY_DELAY).take(Self::CHANNEL_PROBE_RETRIES);
+        let channel_already_exists = Retry::start(channel_probe_retry_strategy, || async {
+            BP::channel_exists(&config.bedrock_config)
+                .await
+                .inspect_err(|err| warn!("Failed to probe Bedrock channel: {err:#}"))
+        })
+        .await
+        .expect("Failed to probe Bedrock channel");
         if channel_already_exists {
             info!("Channel already exists; joining as a non channel creator");
         } else {

lez/system_accounts/src/lib.rs

@@ -9,8 +9,6 @@ use lee_core::account::{Account, AccountId, Nonce};
 /// Minimum summed stake for a Bedrock sequencer key to be a committee candidate.
 pub const DEFAULT_MINIMUM_SEQUENCER_STAKE: u128 = 149;
 
-pub type Slots = u32;
-
 /// Channel administration defaults.
 ///
 /// Slots, not seconds (1 slot = 1s on the current devnet): 20-slot turns,
@@ -22,6 +20,8 @@ pub const DEFAULT_SEQUENCER_POSTING_TIMEOUT: Slots = 10;
 pub const DEFAULT_SEQUENCER_CONFIGURATION_THRESHOLD: u16 = 1;
 pub const DEFAULT_SEQUENCER_WITHDRAW_THRESHOLD: u16 = 1;
 
+pub type Slots = u32;
+
 #[must_use]
 pub fn pinata_account_id() -> AccountId {
     // TODO: Use derivation from a public key?