From ab0fd355ea074f7566101e8530070cc4822e043c Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 13:36:10 +0300 Subject: [PATCH 01/32] refactor(indexer): rename execute_on_state and drop indexer-side tx checks test with: RISC0_DEV_MODE=1 RISC0_SKIP_BUILD=1 cargo test -p common -p storage -p indexer_core --- lez/common/src/transaction.rs | 19 +++++++------------ lez/indexer/core/src/block_store.rs | 6 +----- lez/storage/src/indexer/mod.rs | 15 +-------------- 3 files changed, 9 insertions(+), 31 deletions(-) diff --git a/lez/common/src/transaction.rs b/lez/common/src/transaction.rs index b5aee648..b5f0bc87 100644 --- a/lez/common/src/transaction.rs +++ b/lez/common/src/transaction.rs @@ -92,9 +92,8 @@ impl LeeTransaction { Ok(diff) } - /// Computes the validated state diff without enforcing the system-account - /// restriction. Shared by [`Self::validate_on_state`] and - /// [`Self::execute_without_system_accounts_check_on_state`]. + /// Computes the validated state diff. Shared by [`Self::validate_on_state`] + /// (which adds the system-account guards) and [`Self::execute_on_state`]. fn compute_state_diff( &self, state: &V03State, @@ -129,16 +128,12 @@ impl LeeTransaction { Ok(self) } - /// Similar to [`Self::execute_check_on_state`], but skips the system-account guard. + /// Executes the transaction against the current state and applies the resulting diff, + /// without the system-account guards enforced by [`Self::execute_check_on_state`]. /// - /// FIXME: HOT FIX (testnet v0.2): the indexer replays blocks the sequencer already - /// accepted, including sequencer-generated deposit transactions that - /// legitimately modify the bridge account. The `TransactionOrigin::Sequencer` - /// tag that lets the sequencer bypass the guard is not carried in the block, - /// so the indexer cannot yet distinguish deposit txs from user txs. - /// - /// REMOVE ME when the indexer can authenticate deposit transactions. - pub fn execute_without_system_accounts_check_on_state( + /// The indexer replays blocks the sequencer already validated and inscribed on Bedrock, + /// so it trusts those inscriptions and re-derives state without re-validating them. + pub fn execute_on_state( self, state: &mut V03State, block_id: BlockId, diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 4d583686..d0065206 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -175,11 +175,7 @@ impl IndexerStore { } else { transaction .clone() - .transaction_stateless_check()? - // FIXME: HOT FIX (testnet v0.2): does not check for system account updates due to - // sequencer-generated deposit tx'es; - // CHANGE ME back to `execute_check_on_state` when the indexer can authenticate deposit transactions - .execute_without_system_accounts_check_on_state( + .execute_on_state( &mut state_guard, block.header.block_id, block.header.timestamp, diff --git a/lez/storage/src/indexer/mod.rs b/lez/storage/src/indexer/mod.rs index 87d586fb..a0ee1e82 100644 --- a/lez/storage/src/indexer/mod.rs +++ b/lez/storage/src/indexer/mod.rs @@ -211,20 +211,7 @@ fn apply_block_transactions(mut block: Block, state: &mut V03State) -> DbResult< })?; } else { transaction - .transaction_stateless_check() - .map_err(|err| { - DbError::db_interaction_error(format!( - "transaction pre check failed with err {err:?}" - )) - })? - // FIXME: HOT FIX (testnet v0.2): does not check for system account updates due to - // sequencer-generated deposit tx'es; - // CHANGE ME back to `execute_check_on_state` when the indexer can authenticate deposit transactions - .execute_without_system_accounts_check_on_state( - state, - block.header.block_id, - block.header.timestamp, - ) + .execute_on_state(state, block.header.block_id, block.header.timestamp) .map_err(|err| { DbError::db_interaction_error(format!( "transaction execution failed with err {err:?}" From 308d75ab3791cbda262ec8a564e19cdbc5e2d185 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 13:49:56 +0300 Subject: [PATCH 02/32] feat(common): add block hash recomputation helper test: `RISC0_DEV_MODE=1 RISC0_SKIP_BUILD=1 cargo test -p common --lib block::tests` --- lez/common/src/block.rs | 56 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/lez/common/src/block.rs b/lez/common/src/block.rs index 6e956f9f..029bcfba 100644 --- a/lez/common/src/block.rs +++ b/lez/common/src/block.rs @@ -55,6 +55,21 @@ pub struct Block { pub bedrock_status: BedrockStatus, } +impl Block { + /// Recomputes the hash from this block's contents, for integrity verification + /// against the value stored in `header.hash`. + #[must_use] + pub fn recompute_hash(&self) -> BlockHash { + HashableBlockData { + block_id: self.header.block_id, + prev_block_hash: self.header.prev_block_hash, + timestamp: self.header.timestamp, + transactions: self.body.transactions.clone(), + } + .compute_hash() + } +} + impl Serialize for Block { fn serialize(&self, serializer: S) -> Result { crate::borsh_base64::serialize(self, serializer) @@ -76,11 +91,13 @@ pub struct HashableBlockData { } impl HashableBlockData { + /// Domain-separated hash of the block contents: `SHA256(PREFIX || borsh(self))`. + /// The single source of truth for both producing and verifying a block hash. #[must_use] - pub fn into_pending_block(self, signing_key: &lee::PrivateKey) -> Block { + pub fn compute_hash(&self) -> BlockHash { const PREFIX: &[u8; 32] = b"/LEE/v0.3/Message/Block/\x00\x00\x00\x00\x00\x00\x00\x00"; - let data_bytes = borsh::to_vec(&self).unwrap(); + let data_bytes = borsh::to_vec(self).unwrap(); let mut bytes = Vec::with_capacity( PREFIX .len() @@ -89,8 +106,12 @@ impl HashableBlockData { ); bytes.extend_from_slice(PREFIX); bytes.extend_from_slice(&data_bytes); + OwnHasher::hash(&bytes) + } - let hash = OwnHasher::hash(&bytes); + #[must_use] + pub fn into_pending_block(self, signing_key: &lee::PrivateKey) -> Block { + let hash = self.compute_hash(); let signature = lee::Signature::new(signing_key, &hash.0); Block { header: BlockHeader { @@ -132,4 +153,33 @@ mod tests { let block_from_bytes = borsh::from_slice::(&bytes).unwrap(); assert_eq!(hashable, block_from_bytes); } + + #[test] + fn recompute_hash_matches_header_for_well_formed_block() { + let key = lee::PrivateKey::try_new([7_u8; 32]).expect("valid key"); + let block = HashableBlockData { + block_id: 5, + prev_block_hash: HashType([9_u8; 32]), + timestamp: 42, + transactions: vec![test_utils::produce_dummy_empty_transaction()], + } + .into_pending_block(&key); + assert_eq!(block.recompute_hash(), block.header.hash); + } + + #[test] + fn recompute_hash_detects_tampering() { + let key = lee::PrivateKey::try_new([7_u8; 32]).expect("valid key"); + let block = HashableBlockData { + block_id: 5, + prev_block_hash: HashType([9_u8; 32]), + timestamp: 42, + transactions: vec![test_utils::produce_dummy_empty_transaction()], + } + .into_pending_block(&key); + + let mut tampered = block.clone(); + tampered.header.timestamp = 99; // header changed; stale hash no longer matches + assert_ne!(tampered.recompute_hash(), tampered.header.hash); + } } From 8394597fd568ab097d50c96a08f264e545b506f2 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 14:00:44 +0300 Subject: [PATCH 03/32] feat(indexer): add `BlockIngestError` enum, fix linting --- Cargo.lock | 1 + lez/common/src/block.rs | 2 +- lez/indexer/core/Cargo.toml | 1 + lez/indexer/core/src/block_store.rs | 12 +++--- lez/indexer/core/src/ingest_error.rs | 59 ++++++++++++++++++++++++++++ lez/indexer/core/src/lib.rs | 3 +- 6 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 lez/indexer/core/src/ingest_error.rs diff --git a/Cargo.lock b/Cargo.lock index 84b5e1d6..efb8cd2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3865,6 +3865,7 @@ dependencies = [ "storage", "tempfile", "testnet_initial_state", + "thiserror 2.0.18", "tokio", "url", ] diff --git a/lez/common/src/block.rs b/lez/common/src/block.rs index 029bcfba..a8149e7c 100644 --- a/lez/common/src/block.rs +++ b/lez/common/src/block.rs @@ -178,7 +178,7 @@ mod tests { } .into_pending_block(&key); - let mut tampered = block.clone(); + let mut tampered = block; tampered.header.timestamp = 99; // header changed; stale hash no longer matches assert_ne!(tampered.recompute_hash(), tampered.header.hash); } diff --git a/lez/indexer/core/Cargo.toml b/lez/indexer/core/Cargo.toml index 2ae2d9c0..b3877639 100644 --- a/lez/indexer/core/Cargo.toml +++ b/lez/indexer/core/Cargo.toml @@ -29,6 +29,7 @@ futures.workspace = true url.workspace = true logos-blockchain-core.workspace = true serde_json.workspace = true +thiserror.workspace = true async-stream.workspace = true tokio.workspace = true diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index d0065206..6292447c 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -173,13 +173,11 @@ impl IndexerStore { ) .context("Failed to execute genesis public transaction")?; } else { - transaction - .clone() - .execute_on_state( - &mut state_guard, - block.header.block_id, - block.header.timestamp, - )?; + transaction.clone().execute_on_state( + &mut state_guard, + block.header.block_id, + block.header.timestamp, + )?; } } diff --git a/lez/indexer/core/src/ingest_error.rs b/lez/indexer/core/src/ingest_error.rs new file mode 100644 index 00000000..4240a84c --- /dev/null +++ b/lez/indexer/core/src/ingest_error.rs @@ -0,0 +1,59 @@ +use common::HashType; +use serde::{Deserialize, Serialize}; + +/// Why the indexer could not apply an L2 block from the channel. Stored inside a +/// [`crate::chain_breaker::ChainBreaker`] and surfaced on the status snapshot. +#[derive(Debug, Clone, Serialize, Deserialize, thiserror::Error)] +#[serde(rename_all = "camelCase")] +pub enum BlockIngestError { + #[error("failed to deserialize L2 block: {0}")] + Deserialize(String), + #[error("unexpected block id: expected {expected}, got {got}")] + UnexpectedBlockId { expected: u64, got: u64 }, + #[error("broken chain link: expected prev {expected_prev}, got {got_prev}")] + BrokenChainLink { + expected_prev: HashType, + got_prev: HashType, + }, + #[error("block hash mismatch: computed {computed}, header {header}")] + HashMismatch { + computed: HashType, + header: HashType, + }, + #[error("state transition failed: {0}")] + StateTransition(String), + #[error("storage error: {0}")] + Storage(String), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn serializes_and_round_trips_externally_tagged() { + let err = BlockIngestError::UnexpectedBlockId { + expected: 5, + got: 7, + }; + let value = serde_json::to_value(&err).expect("serialize"); + assert_eq!( + value, + serde_json::json!({ "unexpectedBlockId": { "expected": 5, "got": 7 } }) + ); + let back: BlockIngestError = serde_json::from_value(value).expect("deserialize"); + assert!(matches!( + back, + BlockIngestError::UnexpectedBlockId { + expected: 5, + got: 7 + } + )); + } + + #[test] + fn display_is_human_readable() { + let err = BlockIngestError::StateTransition("nonce too low".to_owned()); + assert_eq!(err.to_string(), "state transition failed: nonce too low"); + } +} diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 0d595fc0..35ff3c6b 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -5,6 +5,7 @@ use arc_swap::ArcSwap; use common::block::Block; // ToDo: Remove after testnet use futures::StreamExt as _; +pub use ingest_error::BlockIngestError; use log::{error, info, warn}; use logos_blockchain_core::header::HeaderId; use logos_blockchain_zone_sdk::{ @@ -16,9 +17,9 @@ use crate::{ config::IndexerConfig, status::{IndexerStatus, IndexerSyncStatus}, }; - pub mod block_store; pub mod config; +pub mod ingest_error; pub mod status; #[derive(Clone)] From 0899eb35475256d27a17db7e4c84260d5f1694c4 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 14:25:02 +0300 Subject: [PATCH 04/32] feat(indexer): persist ChainBreaker in RocksDB meta test `RISC0_DEV_MODE=1 RISC0_SKIP_BUILD=1 cargo test -p storage -p indexer_core --lib chain_breaker` --- lez/indexer/core/src/block_store.rs | 54 +++++++++++++++++++++ lez/indexer/core/src/chain_breaker.rs | 23 +++++++++ lez/indexer/core/src/lib.rs | 2 + lez/storage/src/indexer/indexer_cells.rs | 41 ++++++++++++++-- lez/storage/src/indexer/mod.rs | 2 + lez/storage/src/indexer/read_once.rs | 11 ++++- lez/storage/src/indexer/write_non_atomic.rs | 6 ++- 7 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 lez/indexer/core/src/chain_breaker.rs diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 6292447c..7c89df3f 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -13,6 +13,8 @@ use logos_blockchain_zone_sdk::Slot; use storage::indexer::RocksDBIO; use tokio::sync::RwLock; +use crate::chain_breaker::ChainBreaker; + #[derive(Clone)] pub struct IndexerStore { dbio: Arc, @@ -118,6 +120,21 @@ impl IndexerStore { Ok(()) } + pub fn get_chain_breaker(&self) -> Result> { + let Some(bytes) = self.dbio.get_chain_breaker_bytes()? else { + return Ok(None); + }; + let breaker: Option = + serde_json::from_slice(&bytes).context("Failed to deserialize stored chain breaker")?; + Ok(breaker) + } + + pub fn set_chain_breaker(&self, breaker: &Option) -> Result<()> { + let bytes = serde_json::to_vec(breaker).context("Failed to serialize chain breaker")?; + self.dbio.put_chain_breaker_bytes(&bytes)?; + Ok(()) + } + /// Recalculation of final state directly from DB. /// /// Used for indexer healthcheck. @@ -202,6 +219,43 @@ impl IndexerStore { } } +#[cfg(test)] +mod chain_breaker_tests { + use common::HashType; + + use super::*; + use crate::{chain_breaker::ChainBreaker, ingest_error::BlockIngestError}; + + #[tokio::test] + async fn chain_breaker_roundtrips_and_clears() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path()).expect("open store"); + + assert!(store.get_chain_breaker().expect("get").is_none()); + + let breaker = ChainBreaker { + block_id: Some(7), + block_hash: Some(HashType([1_u8; 32])), + prev_block_hash: Some(HashType([2_u8; 32])), + l1_slot: serde_json::Value::Null, + error: BlockIngestError::StateTransition("boom".to_owned()), + first_seen: Some(99), + orphans_since: 3, + }; + store + .set_chain_breaker(&Some(breaker)) + .expect("set breaker"); + + let got = store.get_chain_breaker().expect("get").expect("present"); + assert_eq!(got.block_id, Some(7)); + assert_eq!(got.orphans_since, 3); + assert!(matches!(got.error, BlockIngestError::StateTransition(_))); + + store.set_chain_breaker(&None).expect("clear"); + assert!(store.get_chain_breaker().expect("get").is_none()); + } +} + #[cfg(test)] mod tests { use common::{HashType, block::HashableBlockData}; diff --git a/lez/indexer/core/src/chain_breaker.rs b/lez/indexer/core/src/chain_breaker.rs new file mode 100644 index 00000000..7aacbd79 --- /dev/null +++ b/lez/indexer/core/src/chain_breaker.rs @@ -0,0 +1,23 @@ +use common::HashType; +use serde::{Deserialize, Serialize}; + +use crate::ingest_error::BlockIngestError; + +/// Diagnostic record of the first block that broke the L2 chain. +/// +/// Later non-chaining blocks (orphans, since the tip is frozen) only bump `orphans_since`. +/// +/// The block-derived fields are `None` for a deserialize break (no header was +/// ever parsed). `l1_slot` is the zone-sdk cursor position captured as JSON. +/// `first_seen` is the breaking block's L2 timestamp (`None` for a deserialize break). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChainBreaker { + pub block_id: Option, + pub block_hash: Option, + pub prev_block_hash: Option, + pub l1_slot: serde_json::Value, + pub error: BlockIngestError, + pub first_seen: Option, + pub orphans_since: u64, +} diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 35ff3c6b..a48712fd 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -2,6 +2,7 @@ use std::{path::Path, sync::Arc}; use anyhow::Result; use arc_swap::ArcSwap; +pub use chain_breaker::ChainBreaker; use common::block::Block; // ToDo: Remove after testnet use futures::StreamExt as _; @@ -18,6 +19,7 @@ use crate::{ status::{IndexerStatus, IndexerSyncStatus}, }; pub mod block_store; +pub mod chain_breaker; pub mod config; pub mod ingest_error; pub mod status; diff --git a/lez/storage/src/indexer/indexer_cells.rs b/lez/storage/src/indexer/indexer_cells.rs index b19a5510..84379a1d 100644 --- a/lez/storage/src/indexer/indexer_cells.rs +++ b/lez/storage/src/indexer/indexer_cells.rs @@ -7,9 +7,9 @@ use crate::{ error::DbError, indexer::{ ACC_NUM_CELL_NAME, BLOCK_HASH_CELL_NAME, BREAKPOINT_CELL_NAME, CF_ACC_META, - CF_BREAKPOINT_NAME, CF_HASH_TO_ID, CF_TX_TO_ID, DB_META_LAST_BREAKPOINT_ID, - DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DB_META_ZONE_SDK_INDEXER_CURSOR_KEY, - TX_HASH_CELL_NAME, + CF_BREAKPOINT_NAME, CF_HASH_TO_ID, CF_TX_TO_ID, DB_META_CHAIN_BREAKER_KEY, + DB_META_LAST_BREAKPOINT_ID, DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, + DB_META_ZONE_SDK_INDEXER_CURSOR_KEY, TX_HASH_CELL_NAME, }, }; @@ -247,6 +247,41 @@ impl SimpleWritableCell for ZoneSdkIndexerCursorCellRef<'_> { } } +/// Opaque JSON bytes for the indexer's persisted `Option`. +/// Serialized via `serde_json` by the caller (mirrors the zone-sdk cursor cell). +#[derive(BorshDeserialize)] +pub struct ChainBreakerCellOwned(pub Vec); + +impl SimpleStorableCell for ChainBreakerCellOwned { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_META_CHAIN_BREAKER_KEY; + const CF_NAME: &'static str = CF_META_NAME; +} + +impl SimpleReadableCell for ChainBreakerCellOwned {} + +#[derive(BorshSerialize)] +pub struct ChainBreakerCellRef<'bytes>(pub &'bytes [u8]); + +impl SimpleStorableCell for ChainBreakerCellRef<'_> { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_META_CHAIN_BREAKER_KEY; + const CF_NAME: &'static str = CF_META_NAME; +} + +impl SimpleWritableCell for ChainBreakerCellRef<'_> { + fn value_constructor(&self) -> DbResult> { + borsh::to_vec(&self).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize chain breaker cell".to_owned()), + ) + }) + } +} + #[cfg(test)] mod uniform_tests { use crate::{ diff --git a/lez/storage/src/indexer/mod.rs b/lez/storage/src/indexer/mod.rs index a0ee1e82..77482312 100644 --- a/lez/storage/src/indexer/mod.rs +++ b/lez/storage/src/indexer/mod.rs @@ -24,6 +24,8 @@ pub const DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY: &str = pub const DB_META_LAST_BREAKPOINT_ID: &str = "last_breakpoint_id"; /// Key base for storing the zone-sdk indexer cursor (opaque bytes). pub const DB_META_ZONE_SDK_INDEXER_CURSOR_KEY: &str = "zone_sdk_indexer_cursor"; +/// Key base for storing the persisted `Option` diagnostic record (opaque JSON bytes). +pub const DB_META_CHAIN_BREAKER_KEY: &str = "chain_breaker"; /// Cell name for a breakpoint. pub const BREAKPOINT_CELL_NAME: &str = "breakpoint"; diff --git a/lez/storage/src/indexer/read_once.rs b/lez/storage/src/indexer/read_once.rs index 6e79adc4..777bbf58 100644 --- a/lez/storage/src/indexer/read_once.rs +++ b/lez/storage/src/indexer/read_once.rs @@ -3,8 +3,9 @@ use crate::{ DBIO as _, cells::shared_cells::{BlockCell, FirstBlockCell, FirstBlockSetCell, LastBlockCell}, indexer::indexer_cells::{ - AccNumTxCell, BlockHashToBlockIdMapCell, BreakpointCellOwned, LastBreakpointIdCell, - LastObservedL1LibHeaderCell, TxHashToBlockIdMapCell, ZoneSdkIndexerCursorCellOwned, + AccNumTxCell, BlockHashToBlockIdMapCell, BreakpointCellOwned, ChainBreakerCellOwned, + LastBreakpointIdCell, LastObservedL1LibHeaderCell, TxHashToBlockIdMapCell, + ZoneSdkIndexerCursorCellOwned, }, }; @@ -73,4 +74,10 @@ impl RocksDBIO { .get_opt::(())? .map(|cell| cell.0)) } + + pub fn get_chain_breaker_bytes(&self) -> DbResult>> { + Ok(self + .get_opt::(())? + .map(|cell| cell.0)) + } } diff --git a/lez/storage/src/indexer/write_non_atomic.rs b/lez/storage/src/indexer/write_non_atomic.rs index 7ddab1dd..555c5efb 100644 --- a/lez/storage/src/indexer/write_non_atomic.rs +++ b/lez/storage/src/indexer/write_non_atomic.rs @@ -3,7 +3,7 @@ use crate::{ DBIO as _, cells::shared_cells::{FirstBlockSetCell, LastBlockCell}, indexer::indexer_cells::{ - BreakpointCellRef, LastBreakpointIdCell, LastObservedL1LibHeaderCell, + BreakpointCellRef, ChainBreakerCellRef, LastBreakpointIdCell, LastObservedL1LibHeaderCell, ZoneSdkIndexerCursorCellRef, }, }; @@ -35,6 +35,10 @@ impl RocksDBIO { self.put(&ZoneSdkIndexerCursorCellRef(bytes), ()) } + pub fn put_chain_breaker_bytes(&self, bytes: &[u8]) -> DbResult<()> { + self.put(&ChainBreakerCellRef(bytes), ()) + } + // State pub fn put_breakpoint(&self, br_id: u64, breakpoint: &V03State) -> DbResult<()> { From 7916efb5c6cd2c47e1ad12236431a7d1e63dcbdf Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 14:50:26 +0300 Subject: [PATCH 05/32] feat(indexer): add accept_block with chain-linkage check and atomic apply test `RISC0_DEV_MODE=1 RISC0_SKIP_BUILD=1 cargo test -p indexer_core --lib accept_tests` --- lez/indexer/core/src/block_store.rs | 298 +++++++++++++++++++++++++++- 1 file changed, 295 insertions(+), 3 deletions(-) diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 7c89df3f..1475104f 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -2,10 +2,11 @@ use std::{path::Path, sync::Arc}; use anyhow::{Context as _, Result}; use common::{ - block::{BedrockStatus, Block}, + HashType, + block::{BedrockStatus, Block, BlockHeader}, transaction::{LeeTransaction, clock_invocation}, }; -use lee::{Account, AccountId, V03State}; +use lee::{Account, AccountId, GENESIS_BLOCK_ID, V03State}; use lee_core::BlockId; use log::info; use logos_blockchain_core::header::HeaderId; @@ -13,7 +14,20 @@ use logos_blockchain_zone_sdk::Slot; use storage::indexer::RocksDBIO; use tokio::sync::RwLock; -use crate::chain_breaker::ChainBreaker; +use crate::{chain_breaker::ChainBreaker, ingest_error::BlockIngestError}; + +struct Tip { + block_id: u64, + hash: HashType, +} + +/// Outcome of feeding a parsed L2 block to the validated tip. +pub enum AcceptOutcome { + /// Chained and applied; tip and L1 read cursor both advance. + Applied, + /// Did not chain or failed to apply; tip stays frozen, breaker recorded. + Parked(BlockIngestError), +} #[derive(Clone)] pub struct IndexerStore { @@ -156,6 +170,129 @@ impl IndexerStore { .get_account_by_id(*account_id)) } + /// The last successfully applied block as `{block_id, hash}`, or `None` before + /// any block is stored (cold start). Read fresh from the store each call. + fn validated_tip(&self) -> Result> { + let Some(block_id) = self.dbio.get_meta_last_block_id_in_db()? else { + return Ok(None); + }; + let Some(block) = self.dbio.get_block(block_id)? else { + return Ok(None); + }; + Ok(Some(Tip { + block_id, + hash: block.header.hash, + })) + } + + /// Returns `Some(err)` if `block` is not the valid continuation of the tip: + /// hash integrity, then block-id continuity, then `prev_block_hash` linkage. + fn acceptance_error(&self, block: &Block) -> Result> { + let computed = block.recompute_hash(); + if computed != block.header.hash { + return Ok(Some(BlockIngestError::HashMismatch { + computed, + header: block.header.hash, + })); + } + + match self.validated_tip()? { + None => { + if block.header.block_id != GENESIS_BLOCK_ID { + return Ok(Some(BlockIngestError::UnexpectedBlockId { + expected: GENESIS_BLOCK_ID, + got: block.header.block_id, + })); + } + } + Some(tip) => { + let expected = tip.block_id.checked_add(1).expect("block id overflow"); + if block.header.block_id != expected { + return Ok(Some(BlockIngestError::UnexpectedBlockId { + expected, + got: block.header.block_id, + })); + } + if block.header.prev_block_hash != tip.hash { + return Ok(Some(BlockIngestError::BrokenChainLink { + expected_prev: tip.hash, + got_prev: block.header.prev_block_hash, + })); + } + } + } + Ok(None) + } + + /// Records the chain breaker: the first break is stored verbatim; subsequent + /// breaks only bump `orphans_since`, preserving the original cause. + fn record_break( + &self, + header: Option<&BlockHeader>, + l1_slot: serde_json::Value, + error: BlockIngestError, + ) -> Result<()> { + let breaker = match self.get_chain_breaker()? { + Some(mut existing) => { + existing.orphans_since = existing.orphans_since.saturating_add(1); + existing + } + None => ChainBreaker { + block_id: header.map(|h| h.block_id), + block_hash: header.map(|h| h.hash), + prev_block_hash: header.map(|h| h.prev_block_hash), + first_seen: header.map(|h| h.timestamp), + l1_slot, + error, + orphans_since: 0, + }, + }; + self.set_chain_breaker(&Some(breaker)) + } + + /// Records a breaker for an inscription that could not even be parsed. + pub fn record_deserialize_break( + &self, + l1_slot: serde_json::Value, + error: String, + ) -> Result<()> { + self.record_break(None, l1_slot, BlockIngestError::Deserialize(error)) + } + + /// Validates `block` against the tip and, if it chains, applies it atomically + /// (scratch clone, commit only on full success) and advances the tip. On any + /// failure records the breaker and returns `Parked` without touching state. + pub async fn accept_block( + &self, + block: &Block, + l1_slot: serde_json::Value, + ) -> Result { + if let Some(err) = self.acceptance_error(block)? { + self.record_break(Some(&block.header), l1_slot, err.clone())?; + return Ok(AcceptOutcome::Parked(err)); + } + + // TODO: we use scratch state to be atomic, but need to revisit how expensive a clone is + let mut scratch = self.current_state.read().await.clone(); + if let Err(err) = apply_block_to_scratch(block, &mut scratch) { + self.record_break(Some(&block.header), l1_slot, err.clone())?; + return Ok(AcceptOutcome::Parked(err)); + } + + let mut stored = block.clone(); + stored.bedrock_status = BedrockStatus::Finalized; + if let Err(err) = self.dbio.put_block(&stored, [0_u8; 32]) { + let ingest_err = BlockIngestError::Storage(err.to_string()); + self.record_break(Some(&block.header), l1_slot, ingest_err.clone())?; + return Ok(AcceptOutcome::Parked(ingest_err)); + } + + // Commit in-memory state (infallible) only after the DB write succeeded. + *self.current_state.write().await = scratch; + self.set_chain_breaker(&None)?; + Ok(AcceptOutcome::Applied) + } + pub async fn put_block(&self, mut block: Block, l1_header: HeaderId) -> Result<()> { info!("Applying block {}", block.header.block_id); { @@ -219,6 +356,61 @@ impl IndexerStore { } } +/// Applies a block's transactions to `state`, mapping every failure to a +/// [`BlockIngestError`] so the caller can park rather than crash. Operates on a +/// scratch state; the caller commits only on `Ok`. +fn apply_block_to_scratch(block: &Block, state: &mut V03State) -> Result<(), BlockIngestError> { + let (clock_tx, user_txs) = + block.body.transactions.split_last().ok_or_else(|| { + BlockIngestError::StateTransition("block has no transactions".to_owned()) + })?; + + let expected_clock = LeeTransaction::Public(clock_invocation(block.header.timestamp)); + if *clock_tx != expected_clock { + return Err(BlockIngestError::StateTransition( + "last transaction must be the clock invocation for the block timestamp".to_owned(), + )); + } + + let is_genesis = block.header.block_id == GENESIS_BLOCK_ID; + for transaction in user_txs { + if is_genesis { + let LeeTransaction::Public(public_tx) = transaction else { + return Err(BlockIngestError::StateTransition( + "genesis block should contain only public transactions".to_owned(), + )); + }; + state + .transition_from_public_transaction( + public_tx, + block.header.block_id, + block.header.timestamp, + ) + .map_err(|err| BlockIngestError::StateTransition(format!("{err:?}")))?; + } else { + transaction + .clone() + .execute_on_state(state, block.header.block_id, block.header.timestamp) + .map_err(|err| BlockIngestError::StateTransition(format!("{err:?}")))?; + } + } + + let LeeTransaction::Public(clock_public_tx) = clock_tx else { + return Err(BlockIngestError::StateTransition( + "clock invocation must be a public transaction".to_owned(), + )); + }; + state + .transition_from_public_transaction( + clock_public_tx, + block.header.block_id, + block.header.timestamp, + ) + .map_err(|err| BlockIngestError::StateTransition(format!("{err:?}")))?; + + Ok(()) +} + #[cfg(test)] mod chain_breaker_tests { use common::HashType; @@ -402,3 +594,103 @@ mod tests { assert_eq!(acc2_at_9.balance, 20090); } } + +#[cfg(test)] +mod accept_tests { + use common::{HashType, block::HashableBlockData}; + + use super::*; + use crate::ingest_error::BlockIngestError; + + fn signing_key() -> lee::PrivateKey { + lee::PrivateKey::try_new([7_u8; 32]).expect("valid key") + } + + // A block with a correct hash but empty body — enough to exercise the + // acceptance checks (id/link/hash), which run before any state application. + fn valid_hash_block(block_id: u64, prev: HashType) -> common::block::Block { + HashableBlockData { + block_id, + prev_block_hash: prev, + timestamp: 0, + transactions: vec![], + } + .into_pending_block(&signing_key()) + } + + #[tokio::test] + async fn non_genesis_first_block_parks_with_unexpected_id() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path()).expect("open store"); + + let block = valid_hash_block(2, HashType([0_u8; 32])); + let outcome = store + .accept_block(&block, serde_json::Value::Null) + .await + .expect("accept"); + + assert!(matches!( + outcome, + AcceptOutcome::Parked(BlockIngestError::UnexpectedBlockId { + expected: 1, + got: 2 + }) + )); + let breaker = store.get_chain_breaker().expect("get").expect("present"); + assert_eq!(breaker.block_id, Some(2)); + assert_eq!(breaker.orphans_since, 0); + } + + #[tokio::test] + async fn hash_mismatch_parks() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path()).expect("open store"); + + let mut block = valid_hash_block(1, HashType([0_u8; 32])); + block.header.timestamp = 999; // invalidates the stored hash + + let outcome = store + .accept_block(&block, serde_json::Value::Null) + .await + .expect("accept"); + assert!(matches!( + outcome, + AcceptOutcome::Parked(BlockIngestError::HashMismatch { .. }) + )); + } + + #[tokio::test] + async fn second_break_bumps_orphan_count_and_keeps_first() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path()).expect("open store"); + + let first = valid_hash_block(2, HashType([0_u8; 32])); + store + .accept_block(&first, serde_json::Value::Null) + .await + .expect("accept"); + let second = valid_hash_block(3, HashType([0_u8; 32])); + store + .accept_block(&second, serde_json::Value::Null) + .await + .expect("accept"); + + let breaker = store.get_chain_breaker().expect("get").expect("present"); + assert_eq!(breaker.block_id, Some(2), "first breaker preserved"); + assert_eq!(breaker.orphans_since, 1, "second break counted as orphan"); + } + + #[tokio::test] + async fn deserialize_break_records_breaker_without_header() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path()).expect("open store"); + + store + .record_deserialize_break(serde_json::Value::Null, "bad bytes".to_owned()) + .expect("record"); + + let breaker = store.get_chain_breaker().expect("get").expect("present"); + assert_eq!(breaker.block_id, None); + assert!(matches!(breaker.error, BlockIngestError::Deserialize(_))); + } +} From 46732698b7bd6ba534ebd63de3f987f722064571 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 15:07:45 +0300 Subject: [PATCH 06/32] feat(indexer): add Stalled status and chain breaker snapshot test `RISC0_DEV_MODE=1 RISC0_SKIP_BUILD=1 cargo test -p indexer_core --lib status` --- lez/indexer/core/src/lib.rs | 2 ++ lez/indexer/core/src/status.rs | 41 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index a48712fd..b2cf077a 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -62,9 +62,11 @@ impl IndexerCore { pub fn status(&self) -> IndexerStatus { let sync = IndexerSyncStatus::clone(&self.status.load()); let indexed_block_id = self.store.get_last_block_id().ok().flatten(); + let chain_breaker = self.store.get_chain_breaker().ok().flatten(); IndexerStatus { sync, indexed_block_id, + chain_breaker, } } diff --git a/lez/indexer/core/src/status.rs b/lez/indexer/core/src/status.rs index 1193e124..954cfa50 100644 --- a/lez/indexer/core/src/status.rs +++ b/lez/indexer/core/src/status.rs @@ -1,5 +1,7 @@ use serde::Serialize; +use crate::chain_breaker::ChainBreaker; + /// Coarse lifecycle state of the indexer's ingestion loop, so a client can tell /// "still catching up" apart from "something went wrong". #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] @@ -13,6 +15,9 @@ pub enum IndexerSyncState { CaughtUp, /// The last cycle failed (e.g. the L1 node is unreachable). See `last_error`. Error, + /// Parked on a chain breaker: the validated tip is frozen awaiting a valid + /// continuation. See `last_error` and the snapshot's `chain_breaker`. + Stalled, } /// Live ingestion status owned by the ingest loop: the coarse `state` plus the @@ -56,6 +61,15 @@ impl IndexerSyncStatus { last_error: Some(reason), } } + + /// Parked on a chain breaker; `reason` mirrors the breaker's error message. + /// The full breaker is attached to the [`IndexerStatus`] snapshot. + pub(crate) const fn stalled(reason: String) -> Self { + Self { + state: IndexerSyncState::Stalled, + last_error: Some(reason), + } + } } /// Full status snapshot returned to callers (FFI/RPC): the live [`IndexerSyncStatus`] @@ -69,6 +83,7 @@ pub struct IndexerStatus { #[serde(flatten)] pub sync: IndexerSyncStatus, pub indexed_block_id: Option, + pub chain_breaker: Option, } #[cfg(test)] @@ -80,6 +95,7 @@ mod tests { let status = IndexerStatus { sync: IndexerSyncStatus::error("boom".to_owned()), indexed_block_id: Some(7), + chain_breaker: None, }; let value = serde_json::to_value(&status).expect("serialize"); assert_eq!( @@ -88,6 +104,7 @@ mod tests { "state": "error", "lastError": "boom", "indexedBlockId": 7, + "chainBreaker": null, }) ); } @@ -100,4 +117,28 @@ mod tests { serde_json::json!({ "state": "caught_up", "lastError": null }) ); } + + #[test] + fn stalled_status_serializes_with_breaker() { + use crate::{chain_breaker::ChainBreaker, ingest_error::BlockIngestError}; + + let status = IndexerStatus { + sync: IndexerSyncStatus::stalled("broken chain link".to_owned()), + indexed_block_id: Some(41), + chain_breaker: Some(ChainBreaker { + block_id: Some(42), + block_hash: None, + prev_block_hash: None, + l1_slot: serde_json::Value::Null, + error: BlockIngestError::StateTransition("boom".to_owned()), + first_seen: None, + orphans_since: 2, + }), + }; + let value = serde_json::to_value(&status).expect("serialize"); + assert_eq!(value["state"], serde_json::json!("stalled")); + assert_eq!(value["lastError"], serde_json::json!("broken chain link")); + assert_eq!(value["indexedBlockId"], serde_json::json!(41)); + assert_eq!(value["chainBreaker"]["orphansSince"], serde_json::json!(2)); + } } From 30aa1686517e9ac20c19d309559134a9c404c1da Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 15:34:17 +0300 Subject: [PATCH 07/32] refactor(indexer): use "stall reason" instead of "chain breaker" --- lez/indexer/core/src/block_store.rs | 88 +++++++++---------- lez/indexer/core/src/ingest_error.rs | 2 +- lez/indexer/core/src/lib.rs | 8 +- .../src/{chain_breaker.rs => stall_reason.rs} | 7 +- lez/indexer/core/src/status.rs | 24 ++--- lez/storage/src/indexer/indexer_cells.rs | 24 ++--- lez/storage/src/indexer/mod.rs | 4 +- lez/storage/src/indexer/read_once.rs | 10 +-- lez/storage/src/indexer/write_non_atomic.rs | 6 +- 9 files changed, 85 insertions(+), 88 deletions(-) rename lez/indexer/core/src/{chain_breaker.rs => stall_reason.rs} (79%) diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 1475104f..aca21178 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -14,7 +14,7 @@ use logos_blockchain_zone_sdk::Slot; use storage::indexer::RocksDBIO; use tokio::sync::RwLock; -use crate::{chain_breaker::ChainBreaker, ingest_error::BlockIngestError}; +use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; struct Tip { block_id: u64, @@ -25,7 +25,7 @@ struct Tip { pub enum AcceptOutcome { /// Chained and applied; tip and L1 read cursor both advance. Applied, - /// Did not chain or failed to apply; tip stays frozen, breaker recorded. + /// Did not chain or failed to apply; tip stays frozen, stall recorded. Parked(BlockIngestError), } @@ -134,18 +134,18 @@ impl IndexerStore { Ok(()) } - pub fn get_chain_breaker(&self) -> Result> { - let Some(bytes) = self.dbio.get_chain_breaker_bytes()? else { + pub fn get_stall_reason(&self) -> Result> { + let Some(bytes) = self.dbio.get_stall_reason_bytes()? else { return Ok(None); }; - let breaker: Option = - serde_json::from_slice(&bytes).context("Failed to deserialize stored chain breaker")?; - Ok(breaker) + let stall: Option = + serde_json::from_slice(&bytes).context("Failed to deserialize stored stall reason")?; + Ok(stall) } - pub fn set_chain_breaker(&self, breaker: &Option) -> Result<()> { - let bytes = serde_json::to_vec(breaker).context("Failed to serialize chain breaker")?; - self.dbio.put_chain_breaker_bytes(&bytes)?; + pub fn set_stall_reason(&self, stall: &Option) -> Result<()> { + let bytes = serde_json::to_vec(stall).context("Failed to serialize stall reason")?; + self.dbio.put_stall_reason_bytes(&bytes)?; Ok(()) } @@ -224,20 +224,20 @@ impl IndexerStore { Ok(None) } - /// Records the chain breaker: the first break is stored verbatim; subsequent + /// Records the stall reason: the first break is stored verbatim; subsequent /// breaks only bump `orphans_since`, preserving the original cause. - fn record_break( + fn record_stall( &self, header: Option<&BlockHeader>, l1_slot: serde_json::Value, error: BlockIngestError, ) -> Result<()> { - let breaker = match self.get_chain_breaker()? { + let stall = match self.get_stall_reason()? { Some(mut existing) => { existing.orphans_since = existing.orphans_since.saturating_add(1); existing } - None => ChainBreaker { + None => StallReason { block_id: header.map(|h| h.block_id), block_hash: header.map(|h| h.hash), prev_block_hash: header.map(|h| h.prev_block_hash), @@ -247,35 +247,35 @@ impl IndexerStore { orphans_since: 0, }, }; - self.set_chain_breaker(&Some(breaker)) + self.set_stall_reason(&Some(stall)) } - /// Records a breaker for an inscription that could not even be parsed. - pub fn record_deserialize_break( + /// Records a stall for an inscription that could not even be parsed. + pub fn record_deserialize_stall( &self, l1_slot: serde_json::Value, error: String, ) -> Result<()> { - self.record_break(None, l1_slot, BlockIngestError::Deserialize(error)) + self.record_stall(None, l1_slot, BlockIngestError::Deserialize(error)) } /// Validates `block` against the tip and, if it chains, applies it atomically /// (scratch clone, commit only on full success) and advances the tip. On any - /// failure records the breaker and returns `Parked` without touching state. + /// failure records the stall and returns `Parked` without touching state. pub async fn accept_block( &self, block: &Block, l1_slot: serde_json::Value, ) -> Result { if let Some(err) = self.acceptance_error(block)? { - self.record_break(Some(&block.header), l1_slot, err.clone())?; + self.record_stall(Some(&block.header), l1_slot, err.clone())?; return Ok(AcceptOutcome::Parked(err)); } // TODO: we use scratch state to be atomic, but need to revisit how expensive a clone is let mut scratch = self.current_state.read().await.clone(); if let Err(err) = apply_block_to_scratch(block, &mut scratch) { - self.record_break(Some(&block.header), l1_slot, err.clone())?; + self.record_stall(Some(&block.header), l1_slot, err.clone())?; return Ok(AcceptOutcome::Parked(err)); } @@ -283,13 +283,13 @@ impl IndexerStore { stored.bedrock_status = BedrockStatus::Finalized; if let Err(err) = self.dbio.put_block(&stored, [0_u8; 32]) { let ingest_err = BlockIngestError::Storage(err.to_string()); - self.record_break(Some(&block.header), l1_slot, ingest_err.clone())?; + self.record_stall(Some(&block.header), l1_slot, ingest_err.clone())?; return Ok(AcceptOutcome::Parked(ingest_err)); } // Commit in-memory state (infallible) only after the DB write succeeded. *self.current_state.write().await = scratch; - self.set_chain_breaker(&None)?; + self.set_stall_reason(&None)?; Ok(AcceptOutcome::Applied) } @@ -412,20 +412,20 @@ fn apply_block_to_scratch(block: &Block, state: &mut V03State) -> Result<(), Blo } #[cfg(test)] -mod chain_breaker_tests { +mod stall_reason_tests { use common::HashType; use super::*; - use crate::{chain_breaker::ChainBreaker, ingest_error::BlockIngestError}; + use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; #[tokio::test] - async fn chain_breaker_roundtrips_and_clears() { + async fn stall_reason_roundtrips_and_clears() { let dir = tempfile::tempdir().expect("tempdir"); let store = IndexerStore::open_db(dir.path()).expect("open store"); - assert!(store.get_chain_breaker().expect("get").is_none()); + assert!(store.get_stall_reason().expect("get").is_none()); - let breaker = ChainBreaker { + let stall = StallReason { block_id: Some(7), block_hash: Some(HashType([1_u8; 32])), prev_block_hash: Some(HashType([2_u8; 32])), @@ -434,17 +434,15 @@ mod chain_breaker_tests { first_seen: Some(99), orphans_since: 3, }; - store - .set_chain_breaker(&Some(breaker)) - .expect("set breaker"); + store.set_stall_reason(&Some(stall)).expect("set stall"); - let got = store.get_chain_breaker().expect("get").expect("present"); + let got = store.get_stall_reason().expect("get").expect("present"); assert_eq!(got.block_id, Some(7)); assert_eq!(got.orphans_since, 3); assert!(matches!(got.error, BlockIngestError::StateTransition(_))); - store.set_chain_breaker(&None).expect("clear"); - assert!(store.get_chain_breaker().expect("get").is_none()); + store.set_stall_reason(&None).expect("clear"); + assert!(store.get_stall_reason().expect("get").is_none()); } } @@ -636,9 +634,9 @@ mod accept_tests { got: 2 }) )); - let breaker = store.get_chain_breaker().expect("get").expect("present"); - assert_eq!(breaker.block_id, Some(2)); - assert_eq!(breaker.orphans_since, 0); + let stall = store.get_stall_reason().expect("get").expect("present"); + assert_eq!(stall.block_id, Some(2)); + assert_eq!(stall.orphans_since, 0); } #[tokio::test] @@ -675,22 +673,22 @@ mod accept_tests { .await .expect("accept"); - let breaker = store.get_chain_breaker().expect("get").expect("present"); - assert_eq!(breaker.block_id, Some(2), "first breaker preserved"); - assert_eq!(breaker.orphans_since, 1, "second break counted as orphan"); + let stall = store.get_stall_reason().expect("get").expect("present"); + assert_eq!(stall.block_id, Some(2), "first stall preserved"); + assert_eq!(stall.orphans_since, 1, "second break counted as orphan"); } #[tokio::test] - async fn deserialize_break_records_breaker_without_header() { + async fn deserialize_break_records_stall_without_header() { let dir = tempfile::tempdir().expect("tempdir"); let store = IndexerStore::open_db(dir.path()).expect("open store"); store - .record_deserialize_break(serde_json::Value::Null, "bad bytes".to_owned()) + .record_deserialize_stall(serde_json::Value::Null, "bad bytes".to_owned()) .expect("record"); - let breaker = store.get_chain_breaker().expect("get").expect("present"); - assert_eq!(breaker.block_id, None); - assert!(matches!(breaker.error, BlockIngestError::Deserialize(_))); + let stall = store.get_stall_reason().expect("get").expect("present"); + assert_eq!(stall.block_id, None); + assert!(matches!(stall.error, BlockIngestError::Deserialize(_))); } } diff --git a/lez/indexer/core/src/ingest_error.rs b/lez/indexer/core/src/ingest_error.rs index 4240a84c..76693a8e 100644 --- a/lez/indexer/core/src/ingest_error.rs +++ b/lez/indexer/core/src/ingest_error.rs @@ -2,7 +2,7 @@ use common::HashType; use serde::{Deserialize, Serialize}; /// Why the indexer could not apply an L2 block from the channel. Stored inside a -/// [`crate::chain_breaker::ChainBreaker`] and surfaced on the status snapshot. +/// [`crate::stall_reason::StallReason`] and surfaced on the status snapshot. #[derive(Debug, Clone, Serialize, Deserialize, thiserror::Error)] #[serde(rename_all = "camelCase")] pub enum BlockIngestError { diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index b2cf077a..88f9feae 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -2,7 +2,6 @@ use std::{path::Path, sync::Arc}; use anyhow::Result; use arc_swap::ArcSwap; -pub use chain_breaker::ChainBreaker; use common::block::Block; // ToDo: Remove after testnet use futures::StreamExt as _; @@ -12,6 +11,7 @@ use logos_blockchain_core::header::HeaderId; use logos_blockchain_zone_sdk::{ CommonHttpClient, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer, }; +pub use stall_reason::StallReason; use crate::{ block_store::IndexerStore, @@ -19,9 +19,9 @@ use crate::{ status::{IndexerStatus, IndexerSyncStatus}, }; pub mod block_store; -pub mod chain_breaker; pub mod config; pub mod ingest_error; +pub mod stall_reason; pub mod status; #[derive(Clone)] @@ -62,11 +62,11 @@ impl IndexerCore { pub fn status(&self) -> IndexerStatus { let sync = IndexerSyncStatus::clone(&self.status.load()); let indexed_block_id = self.store.get_last_block_id().ok().flatten(); - let chain_breaker = self.store.get_chain_breaker().ok().flatten(); + let stall_reason = self.store.get_stall_reason().ok().flatten(); IndexerStatus { sync, indexed_block_id, - chain_breaker, + stall_reason, } } diff --git a/lez/indexer/core/src/chain_breaker.rs b/lez/indexer/core/src/stall_reason.rs similarity index 79% rename from lez/indexer/core/src/chain_breaker.rs rename to lez/indexer/core/src/stall_reason.rs index 7aacbd79..273093d8 100644 --- a/lez/indexer/core/src/chain_breaker.rs +++ b/lez/indexer/core/src/stall_reason.rs @@ -5,19 +5,20 @@ use crate::ingest_error::BlockIngestError; /// Diagnostic record of the first block that broke the L2 chain. /// -/// Later non-chaining blocks (orphans, since the tip is frozen) only bump `orphans_since`. -/// /// The block-derived fields are `None` for a deserialize break (no header was /// ever parsed). `l1_slot` is the zone-sdk cursor position captured as JSON. /// `first_seen` is the breaking block's L2 timestamp (`None` for a deserialize break). #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct ChainBreaker { +pub struct StallReason { pub block_id: Option, pub block_hash: Option, pub prev_block_hash: Option, pub l1_slot: serde_json::Value, pub error: BlockIngestError, pub first_seen: Option, + /// Number of later non-chaining blocks (orphans, since the tip is frozen). + /// + /// TODO: We could store a different "branch" of blocks following this sta pub orphans_since: u64, } diff --git a/lez/indexer/core/src/status.rs b/lez/indexer/core/src/status.rs index 954cfa50..d4f6d918 100644 --- a/lez/indexer/core/src/status.rs +++ b/lez/indexer/core/src/status.rs @@ -1,6 +1,6 @@ use serde::Serialize; -use crate::chain_breaker::ChainBreaker; +use crate::stall_reason::StallReason; /// Coarse lifecycle state of the indexer's ingestion loop, so a client can tell /// "still catching up" apart from "something went wrong". @@ -15,8 +15,8 @@ pub enum IndexerSyncState { CaughtUp, /// The last cycle failed (e.g. the L1 node is unreachable). See `last_error`. Error, - /// Parked on a chain breaker: the validated tip is frozen awaiting a valid - /// continuation. See `last_error` and the snapshot's `chain_breaker`. + /// Parked on a stall reason: the validated tip is frozen awaiting a valid + /// continuation. See `last_error` and the snapshot's `stall_reason`. Stalled, } @@ -62,8 +62,8 @@ impl IndexerSyncStatus { } } - /// Parked on a chain breaker; `reason` mirrors the breaker's error message. - /// The full breaker is attached to the [`IndexerStatus`] snapshot. + /// Parked on a stall reason; `reason` mirrors the stall's error message. + /// The full stall is attached to the [`IndexerStatus`] snapshot. pub(crate) const fn stalled(reason: String) -> Self { Self { state: IndexerSyncState::Stalled, @@ -83,7 +83,7 @@ pub struct IndexerStatus { #[serde(flatten)] pub sync: IndexerSyncStatus, pub indexed_block_id: Option, - pub chain_breaker: Option, + pub stall_reason: Option, } #[cfg(test)] @@ -95,7 +95,7 @@ mod tests { let status = IndexerStatus { sync: IndexerSyncStatus::error("boom".to_owned()), indexed_block_id: Some(7), - chain_breaker: None, + stall_reason: None, }; let value = serde_json::to_value(&status).expect("serialize"); assert_eq!( @@ -104,7 +104,7 @@ mod tests { "state": "error", "lastError": "boom", "indexedBlockId": 7, - "chainBreaker": null, + "stallReason": null, }) ); } @@ -119,13 +119,13 @@ mod tests { } #[test] - fn stalled_status_serializes_with_breaker() { - use crate::{chain_breaker::ChainBreaker, ingest_error::BlockIngestError}; + fn stalled_status_serializes_with_stall_reason() { + use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; let status = IndexerStatus { sync: IndexerSyncStatus::stalled("broken chain link".to_owned()), indexed_block_id: Some(41), - chain_breaker: Some(ChainBreaker { + stall_reason: Some(StallReason { block_id: Some(42), block_hash: None, prev_block_hash: None, @@ -139,6 +139,6 @@ mod tests { assert_eq!(value["state"], serde_json::json!("stalled")); assert_eq!(value["lastError"], serde_json::json!("broken chain link")); assert_eq!(value["indexedBlockId"], serde_json::json!(41)); - assert_eq!(value["chainBreaker"]["orphansSince"], serde_json::json!(2)); + assert_eq!(value["stallReason"]["orphansSince"], serde_json::json!(2)); } } diff --git a/lez/storage/src/indexer/indexer_cells.rs b/lez/storage/src/indexer/indexer_cells.rs index 84379a1d..89e01a43 100644 --- a/lez/storage/src/indexer/indexer_cells.rs +++ b/lez/storage/src/indexer/indexer_cells.rs @@ -7,8 +7,8 @@ use crate::{ error::DbError, indexer::{ ACC_NUM_CELL_NAME, BLOCK_HASH_CELL_NAME, BREAKPOINT_CELL_NAME, CF_ACC_META, - CF_BREAKPOINT_NAME, CF_HASH_TO_ID, CF_TX_TO_ID, DB_META_CHAIN_BREAKER_KEY, - DB_META_LAST_BREAKPOINT_ID, DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, + CF_BREAKPOINT_NAME, CF_HASH_TO_ID, CF_TX_TO_ID, DB_META_LAST_BREAKPOINT_ID, + DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DB_META_STALL_REASON_KEY, DB_META_ZONE_SDK_INDEXER_CURSOR_KEY, TX_HASH_CELL_NAME, }, }; @@ -247,36 +247,36 @@ impl SimpleWritableCell for ZoneSdkIndexerCursorCellRef<'_> { } } -/// Opaque JSON bytes for the indexer's persisted `Option`. +/// Opaque JSON bytes for the indexer's persisted `Option`. /// Serialized via `serde_json` by the caller (mirrors the zone-sdk cursor cell). #[derive(BorshDeserialize)] -pub struct ChainBreakerCellOwned(pub Vec); +pub struct StallReasonCellOwned(pub Vec); -impl SimpleStorableCell for ChainBreakerCellOwned { +impl SimpleStorableCell for StallReasonCellOwned { type KeyParams = (); - const CELL_NAME: &'static str = DB_META_CHAIN_BREAKER_KEY; + const CELL_NAME: &'static str = DB_META_STALL_REASON_KEY; const CF_NAME: &'static str = CF_META_NAME; } -impl SimpleReadableCell for ChainBreakerCellOwned {} +impl SimpleReadableCell for StallReasonCellOwned {} #[derive(BorshSerialize)] -pub struct ChainBreakerCellRef<'bytes>(pub &'bytes [u8]); +pub struct StallReasonCellRef<'bytes>(pub &'bytes [u8]); -impl SimpleStorableCell for ChainBreakerCellRef<'_> { +impl SimpleStorableCell for StallReasonCellRef<'_> { type KeyParams = (); - const CELL_NAME: &'static str = DB_META_CHAIN_BREAKER_KEY; + const CELL_NAME: &'static str = DB_META_STALL_REASON_KEY; const CF_NAME: &'static str = CF_META_NAME; } -impl SimpleWritableCell for ChainBreakerCellRef<'_> { +impl SimpleWritableCell for StallReasonCellRef<'_> { fn value_constructor(&self) -> DbResult> { borsh::to_vec(&self).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to serialize chain breaker cell".to_owned()), + Some("Failed to serialize stall reason cell".to_owned()), ) }) } diff --git a/lez/storage/src/indexer/mod.rs b/lez/storage/src/indexer/mod.rs index 77482312..10c80e63 100644 --- a/lez/storage/src/indexer/mod.rs +++ b/lez/storage/src/indexer/mod.rs @@ -24,8 +24,8 @@ pub const DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY: &str = pub const DB_META_LAST_BREAKPOINT_ID: &str = "last_breakpoint_id"; /// Key base for storing the zone-sdk indexer cursor (opaque bytes). pub const DB_META_ZONE_SDK_INDEXER_CURSOR_KEY: &str = "zone_sdk_indexer_cursor"; -/// Key base for storing the persisted `Option` diagnostic record (opaque JSON bytes). -pub const DB_META_CHAIN_BREAKER_KEY: &str = "chain_breaker"; +/// Key base for storing the persisted `Option` diagnostic record (opaque JSON bytes). +pub const DB_META_STALL_REASON_KEY: &str = "stall_reason"; /// Cell name for a breakpoint. pub const BREAKPOINT_CELL_NAME: &str = "breakpoint"; diff --git a/lez/storage/src/indexer/read_once.rs b/lez/storage/src/indexer/read_once.rs index 777bbf58..591f8405 100644 --- a/lez/storage/src/indexer/read_once.rs +++ b/lez/storage/src/indexer/read_once.rs @@ -3,8 +3,8 @@ use crate::{ DBIO as _, cells::shared_cells::{BlockCell, FirstBlockCell, FirstBlockSetCell, LastBlockCell}, indexer::indexer_cells::{ - AccNumTxCell, BlockHashToBlockIdMapCell, BreakpointCellOwned, ChainBreakerCellOwned, - LastBreakpointIdCell, LastObservedL1LibHeaderCell, TxHashToBlockIdMapCell, + AccNumTxCell, BlockHashToBlockIdMapCell, BreakpointCellOwned, LastBreakpointIdCell, + LastObservedL1LibHeaderCell, StallReasonCellOwned, TxHashToBlockIdMapCell, ZoneSdkIndexerCursorCellOwned, }, }; @@ -75,9 +75,7 @@ impl RocksDBIO { .map(|cell| cell.0)) } - pub fn get_chain_breaker_bytes(&self) -> DbResult>> { - Ok(self - .get_opt::(())? - .map(|cell| cell.0)) + pub fn get_stall_reason_bytes(&self) -> DbResult>> { + Ok(self.get_opt::(())?.map(|cell| cell.0)) } } diff --git a/lez/storage/src/indexer/write_non_atomic.rs b/lez/storage/src/indexer/write_non_atomic.rs index 555c5efb..215250b7 100644 --- a/lez/storage/src/indexer/write_non_atomic.rs +++ b/lez/storage/src/indexer/write_non_atomic.rs @@ -3,7 +3,7 @@ use crate::{ DBIO as _, cells::shared_cells::{FirstBlockSetCell, LastBlockCell}, indexer::indexer_cells::{ - BreakpointCellRef, ChainBreakerCellRef, LastBreakpointIdCell, LastObservedL1LibHeaderCell, + BreakpointCellRef, LastBreakpointIdCell, LastObservedL1LibHeaderCell, StallReasonCellRef, ZoneSdkIndexerCursorCellRef, }, }; @@ -35,8 +35,8 @@ impl RocksDBIO { self.put(&ZoneSdkIndexerCursorCellRef(bytes), ()) } - pub fn put_chain_breaker_bytes(&self, bytes: &[u8]) -> DbResult<()> { - self.put(&ChainBreakerCellRef(bytes), ()) + pub fn put_stall_reason_bytes(&self, bytes: &[u8]) -> DbResult<()> { + self.put(&StallReasonCellRef(bytes), ()) } // State From 2b3fd9b5da650a3b30b34198e5d3e979eda4a1da Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 16:39:37 +0300 Subject: [PATCH 08/32] feat(indexer): park ingest loop on bad blocks instead of skipping `RISC0_DEV_MODE=1 RISC0_SKIP_BUILD=1 cargo test -p indexer_core` --- lez/indexer/core/src/block_store.rs | 254 +++++++++------------------- lez/indexer/core/src/lib.rs | 88 ++++++---- 2 files changed, 138 insertions(+), 204 deletions(-) diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index aca21178..8f983ee6 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -8,7 +8,6 @@ use common::{ }; use lee::{Account, AccountId, GENESIS_BLOCK_ID, V03State}; use lee_core::BlockId; -use log::info; use logos_blockchain_core::header::HeaderId; use logos_blockchain_zone_sdk::Slot; use storage::indexer::RocksDBIO; @@ -292,68 +291,6 @@ impl IndexerStore { self.set_stall_reason(&None)?; Ok(AcceptOutcome::Applied) } - - pub async fn put_block(&self, mut block: Block, l1_header: HeaderId) -> Result<()> { - info!("Applying block {}", block.header.block_id); - { - let mut state_guard = self.current_state.write().await; - - let (clock_tx, user_txs) = block - .body - .transactions - .split_last() - .ok_or_else(|| anyhow::anyhow!("Block has no transactions"))?; - - anyhow::ensure!( - *clock_tx == LeeTransaction::Public(clock_invocation(block.header.timestamp)), - "Last transaction in block must be the clock invocation for the block timestamp" - ); - - let is_genesis = block.header.block_id == 1; - for transaction in user_txs { - if is_genesis { - let genesis_tx = match transaction { - LeeTransaction::Public(public_tx) => public_tx, - LeeTransaction::PrivacyPreserving(_) - | LeeTransaction::ProgramDeployment(_) => { - anyhow::bail!("Genesis block should contain only public transactions") - } - }; - state_guard - .transition_from_public_transaction( - genesis_tx, - block.header.block_id, - block.header.timestamp, - ) - .context("Failed to execute genesis public transaction")?; - } else { - transaction.clone().execute_on_state( - &mut state_guard, - block.header.block_id, - block.header.timestamp, - )?; - } - } - - // Apply the clock invocation directly (it is expected to modify clock accounts). - let LeeTransaction::Public(clock_public_tx) = clock_tx else { - anyhow::bail!("Clock invocation must be a public transaction"); - }; - state_guard.transition_from_public_transaction( - clock_public_tx, - block.header.block_id, - block.header.timestamp, - )?; - } - - // ToDo: Currently we are fetching only finalized blocks - // if it changes, the following lines need to be updated - // to represent correct block finality - block.bedrock_status = BedrockStatus::Finalized; - - info!("Putting block {} into DB", block.header.block_id); - Ok(self.dbio.put_block(&block, l1_header.into())?) - } } /// Applies a block's transactions to `state`, mapping every failure to a @@ -448,66 +385,12 @@ mod stall_reason_tests { #[cfg(test)] mod tests { - use common::{HashType, block::HashableBlockData}; + use common::test_utils::{create_transaction_native_token_transfer, produce_dummy_block}; use tempfile::tempdir; use testnet_initial_state::initial_pub_accounts_private_keys; use super::*; - struct TestFixture { - storage: IndexerStore, - from: AccountId, - to: AccountId, - _home: tempfile::TempDir, - } - - #[expect( - clippy::arithmetic_side_effects, - reason = "test helper with bounded inputs" - )] - async fn store_with_transfer_blocks( - block_count: u64, - prev_hash: Option, - ) -> TestFixture { - let home = tempdir().unwrap(); - let storage = IndexerStore::open_db(home.path()).unwrap(); - - let initial_accounts = initial_pub_accounts_private_keys(); - let from = initial_accounts[0].account_id; - let to = initial_accounts[1].account_id; - let sign_key = initial_accounts[0].pub_sign_key.clone(); - - let mut prev_hash = prev_hash; - for i in 0..block_count { - let tx = common::test_utils::create_transaction_native_token_transfer( - from, - u128::from(i), - to, - 10, - &sign_key, - ); - let block_id = i + 1; - - let next_block = common::test_utils::produce_dummy_block(block_id, prev_hash, vec![tx]); - prev_hash = Some(next_block.header.hash); - - storage - .put_block( - next_block, - HeaderId::from([u8::try_from(i + 1).unwrap(); 32]), - ) - .await - .unwrap(); - } - - TestFixture { - storage, - from, - to, - _home: home, - } - } - #[test] fn correct_startup() { let home = tempdir().unwrap(); @@ -520,76 +403,99 @@ mod tests { } #[tokio::test] - async fn state_transition() { + async fn accept_block_applies_transfers_and_advances_tip() { let home = tempdir().unwrap(); - let storage = IndexerStore::open_db(home.as_ref()).unwrap(); + let store = IndexerStore::open_db(home.as_ref()).unwrap(); let initial_accounts = initial_pub_accounts_private_keys(); let from = initial_accounts[0].account_id; let to = initial_accounts[1].account_id; let sign_key = initial_accounts[0].pub_sign_key.clone(); - let clock_tx = LeeTransaction::Public(clock_invocation(0)); - let genesis_block_data = HashableBlockData { - block_id: 1, - prev_block_hash: HashType::default(), - timestamp: 0, - transactions: vec![clock_tx], - }; - let genesis_block = genesis_block_data - .into_pending_block(&common::test_utils::sequencer_sign_key_for_testing()); - let mut prev_hash = Some(genesis_block.header.hash); - storage - .put_block(genesis_block, HeaderId::from([0_u8; 32])) + // Genesis (block 1): clock-only. + let genesis = produce_dummy_block(1, None, vec![]); + let mut prev_hash = genesis.header.hash; + assert!(matches!( + store + .accept_block(&genesis, serde_json::Value::Null) + .await + .unwrap(), + AcceptOutcome::Applied + )); + + // Blocks 2..=11: one native transfer of 10 each (nonces 0..=9). + for i in 0..10_u64 { + let tx = create_transaction_native_token_transfer(from, i as u128, to, 10, &sign_key); + let block = produce_dummy_block(i + 2, Some(prev_hash), vec![tx]); + prev_hash = block.header.hash; + assert!(matches!( + store + .accept_block(&block, serde_json::Value::Null) + .await + .unwrap(), + AcceptOutcome::Applied + )); + } + + assert_eq!( + store.account_current_state(&from).await.unwrap().balance, + 9900 + ); + assert_eq!( + store.account_current_state(&to).await.unwrap().balance, + 20100 + ); + // Tip advanced to the last applied block; a clean run leaves no stall. + assert_eq!(store.get_last_block_id().unwrap(), Some(11)); + assert!(store.get_stall_reason().unwrap().is_none()); + } + + #[tokio::test] + async fn account_state_at_block_reflects_history() { + let home = tempdir().unwrap(); + let store = IndexerStore::open_db(home.as_ref()).unwrap(); + + let initial_accounts = initial_pub_accounts_private_keys(); + let from = initial_accounts[0].account_id; + let to = initial_accounts[1].account_id; + let sign_key = initial_accounts[0].pub_sign_key.clone(); + + let genesis = produce_dummy_block(1, None, vec![]); + let mut prev_hash = genesis.header.hash; + store + .accept_block(&genesis, serde_json::Value::Null) .await .unwrap(); - for i in 0..10_u128 { - let tx = common::test_utils::create_transaction_native_token_transfer( - from, i, to, 10, &sign_key, - ); - let block_id = u64::try_from(i + 1).unwrap(); - let next_block = common::test_utils::produce_dummy_block(block_id, prev_hash, vec![tx]); - prev_hash = Some(next_block.header.hash); - storage - .put_block( - next_block, - HeaderId::from([u8::try_from(i + 1).unwrap(); 32]), - ) + for i in 0..10_u64 { + let tx = create_transaction_native_token_transfer(from, i as u128, to, 10, &sign_key); + let block = produce_dummy_block(i + 2, Some(prev_hash), vec![tx]); + prev_hash = block.header.hash; + store + .accept_block(&block, serde_json::Value::Null) .await .unwrap(); } - let acc1_val = storage.account_current_state(&from).await.unwrap(); - let acc2_val = storage.account_current_state(&to).await.unwrap(); - - assert_eq!(acc1_val.balance, 9900); - assert_eq!(acc2_val.balance, 20100); - } - - #[tokio::test] - async fn account_state_at_block() { - let TestFixture { - storage, - from, - to, - _home, - } = store_with_transfer_blocks(10, None).await; - - let acc1_at_1 = storage.account_state_at_block(&from, 1).unwrap(); - let acc2_at_1 = storage.account_state_at_block(&to, 1).unwrap(); - assert_eq!(acc1_at_1.balance, 9990); - assert_eq!(acc2_at_1.balance, 20010); - - let acc1_at_5 = storage.account_state_at_block(&from, 5).unwrap(); - let acc2_at_5 = storage.account_state_at_block(&to, 5).unwrap(); - assert_eq!(acc1_at_5.balance, 9950); - assert_eq!(acc2_at_5.balance, 20050); - - let acc1_at_9 = storage.account_state_at_block(&from, 9).unwrap(); - let acc2_at_9 = storage.account_state_at_block(&to, 9).unwrap(); - assert_eq!(acc1_at_9.balance, 9910); - assert_eq!(acc2_at_9.balance, 20090); + // State at block N is inclusive of block N. + // Block 1 (genesis, clock-only): no transfers yet. + assert_eq!( + store.account_state_at_block(&from, 1).unwrap().balance, + 10000 + ); + assert_eq!(store.account_state_at_block(&to, 1).unwrap().balance, 20000); + // Through block 5: 4 transfers applied (blocks 2..=5). + assert_eq!( + store.account_state_at_block(&from, 5).unwrap().balance, + 9960 + ); + assert_eq!(store.account_state_at_block(&to, 5).unwrap().balance, 20040); + // Through block 9: 8 transfers applied (blocks 2..=9). + assert_eq!( + store.account_state_at_block(&from, 9).unwrap().balance, + 9920 + ); + assert_eq!(store.account_state_at_block(&to, 9).unwrap().balance, 20080); } } diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 88f9feae..669b389c 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -7,14 +7,13 @@ use common::block::Block; use futures::StreamExt as _; pub use ingest_error::BlockIngestError; use log::{error, info, warn}; -use logos_blockchain_core::header::HeaderId; use logos_blockchain_zone_sdk::{ CommonHttpClient, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer, }; pub use stall_reason::StallReason; use crate::{ - block_store::IndexerStore, + block_store::{AcceptOutcome, IndexerStore}, config::IndexerConfig, status::{IndexerStatus, IndexerSyncStatus}, }; @@ -95,8 +94,6 @@ impl IndexerCore { let stream = match self.zone_indexer.next_messages(cursor).await { Ok(s) => s, Err(err) => { - // `next_messages` reads L1 consensus info internally, so - // this also covers an unreachable/misconfigured L1 node. error!("Failed to start zone-sdk next_messages stream: {err}"); self.set_status(IndexerSyncStatus::error(format!( "cannot reach L1 / read channel: {err}" @@ -107,11 +104,8 @@ impl IndexerCore { }; let mut stream = std::pin::pin!(stream); - // Flip to Syncing on the first message of this cycle (not merely on - // a successful poll) so the steady-state CaughtUp status doesn't - // flicker. Until then the state stays Starting (cold-start scan of - // empty L1 history) or CaughtUp (idle). let mut announced_syncing = false; + let mut had_cycle_error = false; while let Some((msg, slot)) = stream.next().await { if !announced_syncing { @@ -121,17 +115,25 @@ impl IndexerCore { let zone_block = match msg { ZoneMessage::Block(b) => b, - // Non-block messages don't carry a cursor position; the - // next ZoneBlock advances past them implicitly. ZoneMessage::Deposit(_) | ZoneMessage::Withdraw(_) => continue, }; + let l1_slot = serde_json::to_value(&slot).unwrap_or(serde_json::Value::Null); + let block: Block = match borsh::from_slice(&zone_block.data) { Ok(b) => b, Err(e) => { error!("Failed to deserialize L2 block from zone-sdk: {e}"); - // Advance past the broken inscription so we don't - // re-process it on restart. + if let Err(err) = + self.store.record_deserialize_stall(l1_slot, e.to_string()) + { + warn!("Failed to record stall reason: {err:#}"); + } + self.set_status(IndexerSyncStatus::stalled(format!( + "failed to deserialize L2 block: {e}" + ))); + // Advance the L1 read cursor past the broken inscription; + // the validated tip stays frozen. cursor = Some(slot); if let Err(err) = self.store.set_zone_cursor(&slot) { warn!("Failed to persist indexer cursor: {err:#}"); @@ -140,27 +142,53 @@ impl IndexerCore { } }; - info!("Indexed L2 block {}", block.header.block_id); - - // TODO: Remove l1_header placeholder once storage layer - // no longer requires it. Zone-sdk handles L1 tracking internally. - let placeholder_l1_header = HeaderId::from([0_u8; 32]); - if let Err(err) = self.store.put_block(block.clone(), placeholder_l1_header).await { - error!("Failed to store block {}: {err:#}", block.header.block_id); + match self.store.accept_block(&block, l1_slot).await { + Ok(AcceptOutcome::Applied) => { + info!("Indexed L2 block {}", block.header.block_id); + self.set_status(IndexerSyncStatus::syncing()); + cursor = Some(slot); + if let Err(err) = self.store.set_zone_cursor(&slot) { + warn!("Failed to persist indexer cursor: {err:#}"); + } + yield Ok(block); + } + Ok(AcceptOutcome::Parked(ingest_err)) => { + error!( + "Parked at block {}: {ingest_err}", + block.header.block_id + ); + self.set_status(IndexerSyncStatus::stalled(ingest_err.to_string())); + // Advance the L1 read cursor; tip stays frozen, no yield. + cursor = Some(slot); + if let Err(err) = self.store.set_zone_cursor(&slot) { + warn!("Failed to persist indexer cursor: {err:#}"); + } + } + Err(err) => { + // Infrastructure error (DB read/write), not a bad block. + // Keep the cursor put; re-poll the same position next cycle. + error!( + "Store error applying block {}: {err:#}", + block.header.block_id + ); + self.set_status(IndexerSyncStatus::error(format!( + "store error: {err:#}" + ))); + had_cycle_error = true; + break; + } } - - cursor = Some(slot); - if let Err(err) = self.store.set_zone_cursor(&slot) { - warn!("Failed to persist indexer cursor: {err:#}"); - } - yield Ok(block); } - // Stream drained: caught up to LIB as of this cycle. Clears any - // prior error (e.g. a transient L1 disconnect that left no - // backlog, so the `Syncing` branch above never ran). Sleep then - // poll again. - self.set_status(IndexerSyncStatus::caught_up()); + if had_cycle_error { + tokio::time::sleep(poll_interval).await; + continue; + } + + // Stream drained. Stay Stalled if parked; otherwise we are caught up. + if self.store.get_stall_reason().ok().flatten().is_none() { + self.set_status(IndexerSyncStatus::caught_up()); + } tokio::time::sleep(poll_interval).await; } } From 5f6f26d0126f45a7bdbad9978bd78bfa8f3584de Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 16:41:39 +0300 Subject: [PATCH 09/32] chore: fix linter --- lez/indexer/core/src/block_store.rs | 4 ++-- lez/indexer/core/src/lib.rs | 2 +- lez/indexer/core/src/stall_reason.rs | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 8f983ee6..08b5391d 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -425,7 +425,7 @@ mod tests { // Blocks 2..=11: one native transfer of 10 each (nonces 0..=9). for i in 0..10_u64 { - let tx = create_transaction_native_token_transfer(from, i as u128, to, 10, &sign_key); + let tx = create_transaction_native_token_transfer(from, i.into(), to, 10, &sign_key); let block = produce_dummy_block(i + 2, Some(prev_hash), vec![tx]); prev_hash = block.header.hash; assert!(matches!( @@ -468,7 +468,7 @@ mod tests { .unwrap(); for i in 0..10_u64 { - let tx = create_transaction_native_token_transfer(from, i as u128, to, 10, &sign_key); + let tx = create_transaction_native_token_transfer(from, i.into(), to, 10, &sign_key); let block = produce_dummy_block(i + 2, Some(prev_hash), vec![tx]); prev_hash = block.header.hash; store diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 669b389c..74fa6582 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -118,7 +118,7 @@ impl IndexerCore { ZoneMessage::Deposit(_) | ZoneMessage::Withdraw(_) => continue, }; - let l1_slot = serde_json::to_value(&slot).unwrap_or(serde_json::Value::Null); + let l1_slot = serde_json::to_value(slot).unwrap_or(serde_json::Value::Null); let block: Block = match borsh::from_slice(&zone_block.data) { Ok(b) => b, diff --git a/lez/indexer/core/src/stall_reason.rs b/lez/indexer/core/src/stall_reason.rs index 273093d8..fdad9861 100644 --- a/lez/indexer/core/src/stall_reason.rs +++ b/lez/indexer/core/src/stall_reason.rs @@ -19,6 +19,7 @@ pub struct StallReason { pub first_seen: Option, /// Number of later non-chaining blocks (orphans, since the tip is frozen). /// - /// TODO: We could store a different "branch" of blocks following this sta + /// TODO: We could store a different "branch" of blocks following this break, but for now we + /// just count them. pub orphans_since: u64, } From 206f3de5247bffcee0fc736417f6a7b1b89222f4 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 17:44:00 +0300 Subject: [PATCH 10/32] feat(indexer): expose status over RPC and add integration coverage test `RUST_LOG=info RISC0_DEV_MODE=1 cargo test -p integration_tests --test bridge --indexer_status_rpc_reports_caught_up_with_no_stall --exact --nocapture --include-ignored` --- integration_tests/tests/bridge.rs | 32 +++++++++++++++++++++++++ lez/indexer/service/rpc/src/lib.rs | 3 +++ lez/indexer/service/src/mock_service.rs | 18 ++++++++++++++ lez/indexer/service/src/service.rs | 5 ++++ 4 files changed, 58 insertions(+) diff --git a/integration_tests/tests/bridge.rs b/integration_tests/tests/bridge.rs index 9c241ebb..9a76bad6 100644 --- a/integration_tests/tests/bridge.rs +++ b/integration_tests/tests/bridge.rs @@ -557,6 +557,38 @@ fn create_zone_indexer_observer( )) } +/// Test that the indexer status RPC reports caught-up with no stall after a clean run. +/// +/// TODO: Integration-level park testing (publishing a bad block to force a stall) is a follow-up +/// needing fault injection support in the test harness. +#[ignore = "requires the full local stack (bedrock + sequencer + indexer)"] +#[test] +async fn indexer_status_rpc_reports_caught_up_with_no_stall() -> anyhow::Result<()> { + use indexer_service_rpc::RpcClient as _; + + let ctx = TestContext::new().await?; + + let indexer_tip = wait_for_indexer_to_catch_up(&ctx).await?; + + let status = ctx.indexer_client().get_status().await?; + assert_eq!( + status["state"], + serde_json::json!("caught_up"), + "indexer should be caught up, got {status}" + ); + assert!( + status["stallReason"].is_null(), + "indexer should have no stall reason after a clean run, got {status}" + ); + assert_eq!( + status["indexedBlockId"], + serde_json::json!(indexer_tip), + "status indexedBlockId should equal the caught-up tip" + ); + + Ok(()) +} + async fn wait_for_finalized_withdraw_op( observer: &ZoneIndexer, expected_amount: u64, diff --git a/lez/indexer/service/rpc/src/lib.rs b/lez/indexer/service/rpc/src/lib.rs index 5763fe82..108ee6c2 100644 --- a/lez/indexer/service/rpc/src/lib.rs +++ b/lez/indexer/service/rpc/src/lib.rs @@ -69,6 +69,9 @@ pub trait Rpc { limit: u64, ) -> Result, ErrorObjectOwned>; + #[method(name = "getStatus")] + async fn get_status(&self) -> Result; + // ToDo: expand healthcheck response into some kind of report #[method(name = "checkHealth")] async fn healthcheck(&self) -> Result<(), ErrorObjectOwned>; diff --git a/lez/indexer/service/src/mock_service.rs b/lez/indexer/service/src/mock_service.rs index 7bf6c528..c92ebb0f 100644 --- a/lez/indexer/service/src/mock_service.rs +++ b/lez/indexer/service/src/mock_service.rs @@ -325,6 +325,24 @@ impl indexer_service_rpc::RpcServer for MockIndexerService { .collect()) } + async fn get_status(&self) -> Result { + let indexed_block_id = self + .state + .read() + .await + .blocks + .iter() + .rev() + .find(|block| block.bedrock_status == BedrockStatus::Finalized) + .map(|block| block.header.block_id); + Ok(serde_json::json!({ + "state": "caught_up", + "lastError": null, + "indexedBlockId": indexed_block_id, + "stallReason": null, + })) + } + async fn healthcheck(&self) -> Result<(), ErrorObjectOwned> { Ok(()) } diff --git a/lez/indexer/service/src/service.rs b/lez/indexer/service/src/service.rs index 7a8ed90f..7abab18f 100644 --- a/lez/indexer/service/src/service.rs +++ b/lez/indexer/service/src/service.rs @@ -149,6 +149,11 @@ impl indexer_service_rpc::RpcServer for IndexerService { Ok(tx_res) } + async fn get_status(&self) -> Result { + Ok(serde_json::to_value(self.indexer.status()) + .expect("IndexerStatus serialization should not fail")) + } + async fn healthcheck(&self) -> Result<(), ErrorObjectOwned> { // Checking, that indexer can calculate last state let _ = self From 02eb338a6ab65ea3dc7908cac182bca207d5bd77 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 18:00:49 +0300 Subject: [PATCH 11/32] test(indexer): cover stall recovery and full StallReason roundtrip test `RISC0_DEV_MODE=1 RISC0_SKIP_BUILD=1 cargo test -p indexer_core` --- lez/indexer/core/src/block_store.rs | 63 ++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 08b5391d..0a32e763 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -377,6 +377,10 @@ mod stall_reason_tests { assert_eq!(got.block_id, Some(7)); assert_eq!(got.orphans_since, 3); assert!(matches!(got.error, BlockIngestError::StateTransition(_))); + assert_eq!(got.block_hash, Some(HashType([1_u8; 32]))); + assert_eq!(got.prev_block_hash, Some(HashType([2_u8; 32]))); + assert_eq!(got.l1_slot, serde_json::Value::Null); + assert_eq!(got.first_seen, Some(99)); store.set_stall_reason(&None).expect("clear"); assert!(store.get_stall_reason().expect("get").is_none()); @@ -501,7 +505,7 @@ mod tests { #[cfg(test)] mod accept_tests { - use common::{HashType, block::HashableBlockData}; + use common::{HashType, block::HashableBlockData, test_utils::produce_dummy_block}; use super::*; use crate::ingest_error::BlockIngestError; @@ -597,4 +601,61 @@ mod accept_tests { assert_eq!(stall.block_id, None); assert!(matches!(stall.error, BlockIngestError::Deserialize(_))); } + + #[tokio::test] + async fn parks_then_recovers_on_valid_continuation() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path()).expect("open store"); + + // Genesis (block 1, clock-only) applies and advances the tip. + let genesis = produce_dummy_block(1, None, vec![]); + assert!(matches!( + store + .accept_block(&genesis, serde_json::Value::Null) + .await + .unwrap(), + AcceptOutcome::Applied + )); + + // A block that skips ahead (id 3 while the tip is 1) parks the indexer. + let bad = produce_dummy_block(3, Some(genesis.header.hash), vec![]); + assert!(matches!( + store + .accept_block(&bad, serde_json::Value::Null) + .await + .unwrap(), + AcceptOutcome::Parked(BlockIngestError::UnexpectedBlockId { + expected: 2, + got: 3 + }) + )); + assert!( + store.get_stall_reason().unwrap().is_some(), + "indexer should be parked after the bad block" + ); + assert_eq!( + store.get_last_block_id().unwrap(), + Some(1), + "validated tip must stay frozen at genesis while parked" + ); + + // The valid continuation (block 2 chaining on genesis) recovers the chain. + let next = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + assert!(matches!( + store + .accept_block(&next, serde_json::Value::Null) + .await + .unwrap(), + AcceptOutcome::Applied + )); + assert!( + store.get_stall_reason().unwrap().is_none(), + "stall reason must clear on recovery" + ); + assert_eq!( + store.get_last_block_id().unwrap(), + Some(2), + "tip must advance to the recovered block" + ); + } } From 406b6ab011208c4f7597d658800bbbd947cb97b3 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 20:37:35 +0300 Subject: [PATCH 12/32] feat(indexer): add startup genesis-consistency check test `RISC0_DEV_MODE=1 RISC0_SKIP_BUILD=1 cargo test -p indexer_core` --- lez/indexer/core/src/config.rs | 35 +++++++++ lez/indexer/core/src/lib.rs | 139 ++++++++++++++++++++++++++++++++- test_fixtures/src/config.rs | 1 + 3 files changed, 173 insertions(+), 2 deletions(-) diff --git a/lez/indexer/core/src/config.rs b/lez/indexer/core/src/config.rs index cb7f3dfe..580b9fbd 100644 --- a/lez/indexer/core/src/config.rs +++ b/lez/indexer/core/src/config.rs @@ -20,6 +20,12 @@ pub struct IndexerConfig { pub consensus_info_polling_interval: Duration, pub bedrock_config: ClientConfig, pub channel_id: ChannelId, + /// Whether to wipe the indexer store and re-index from scratch when a genesis mismatch occurs + /// (i.e. the L1/sequencer was reset but the old store was reused). + /// + /// Defaults to `false`: on mismatch the indexer refuses to start. + #[serde(default)] + pub allow_chain_reset: bool, } impl IndexerConfig { @@ -37,3 +43,32 @@ impl IndexerConfig { }) } } + +#[cfg(test)] +mod tests { + use super::IndexerConfig; + + const MINIMAL_CONFIG: &str = r#"{ + "consensus_info_polling_interval": "1s", + "bedrock_config": { "addr": "http://localhost:18080" }, + "channel_id": "0101010101010101010101010101010101010101010101010101010101010101" + }"#; + + #[test] + fn allow_chain_reset_defaults_to_false() { + let config: IndexerConfig = serde_json::from_str(MINIMAL_CONFIG).unwrap(); + assert!(!config.allow_chain_reset); + } + + #[test] + fn allow_chain_reset_parses_true() { + let json = r#"{ + "consensus_info_polling_interval": "1s", + "bedrock_config": { "addr": "http://localhost:18080" }, + "channel_id": "0101010101010101010101010101010101010101010101010101010101010101", + "allow_chain_reset": true + }"#; + let config: IndexerConfig = serde_json::from_str(json).unwrap(); + assert!(config.allow_chain_reset); + } +} diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 74fa6582..1f915ba4 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -1,11 +1,12 @@ use std::{path::Path, sync::Arc}; -use anyhow::Result; +use anyhow::{Context as _, Result}; use arc_swap::ArcSwap; -use common::block::Block; +use common::{HashType, block::Block}; // ToDo: Remove after testnet use futures::StreamExt as _; pub use ingest_error::BlockIngestError; +use lee::GENESIS_BLOCK_ID; use log::{error, info, warn}; use logos_blockchain_zone_sdk::{ CommonHttpClient, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer, @@ -23,6 +24,14 @@ pub mod ingest_error; pub mod stall_reason; pub mod status; +/// Result of comparing the indexer's stored genesis against the channel's. +enum GenesisOutcome { + /// Match, or nothing to compare (fresh store / empty channel / L1 unreachable) — proceed. + Consistent, + /// The store holds a different chain than the channel now serves. + Mismatch { stored: HashType, current: HashType }, +} + #[derive(Clone)] pub struct IndexerCore { pub zone_indexer: Arc>, @@ -53,6 +62,73 @@ impl IndexerCore { }) } + /// Builds the core, then verifies the stored genesis matches the channel's. + /// On mismatch: refuse (error) unless `allow_reset`, in which case wipe the store + /// and re-index from scratch. Used at service/FFI startup in place of `new`. + pub async fn new_with_genesis_check( + config: IndexerConfig, + storage_dir: &Path, + allow_reset: bool, + ) -> Result { + let home = storage_dir.join(format!("rocksdb-{}", config.channel_id)); + let core = Self::new(config.clone(), storage_dir)?; + match core.genesis_outcome().await? { + GenesisOutcome::Consistent => Ok(core), + GenesisOutcome::Mismatch { stored, current } if allow_reset => { + warn!( + "Chain reset detected: stored genesis {stored} != channel genesis {current}. \ + Wiping indexer store at {} and re-indexing.", + home.display() + ); + drop(core); // sole owner before the ingest task is spawned → closes the DB + storage::indexer::RocksDBIO::destroy(&home)?; + Self::new(config, storage_dir) + } + GenesisOutcome::Mismatch { stored, current } => Err(anyhow::anyhow!( + "Indexer store at {} holds a different chain than the channel now serves \ + (stored genesis {stored} != channel genesis {current}). Run `just clean`, \ + point at a fresh storage dir, or set `allow_chain_reset` in the indexer config.", + home.display() + )), + } + } + + /// Compares the stored genesis (block 1) against the channel's current genesis. + async fn genesis_outcome(&self) -> Result { + let stored = self.store.get_block_at_id(GENESIS_BLOCK_ID)?; + if stored.is_none() { + return Ok(GenesisOutcome::Consistent); // fresh store: skip the L1 read entirely + } + let current = self.fetch_channel_genesis().await?; + Ok(compare_genesis(stored.as_ref(), current.as_ref())) + } + + /// Reads the channel's genesis (first `Block`) from the start of the channel. + /// Returns `None` if the channel has no block yet or L1 can't be reached within + /// the timeout — the check is best-effort and must never block or fail startup. + async fn fetch_channel_genesis(&self) -> Result> { + const GENESIS_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + let fetch = async { + let stream = self.zone_indexer.next_messages(None).await?; + let mut stream = std::pin::pin!(stream); + while let Some((msg, _slot)) = stream.next().await { + if let ZoneMessage::Block(zone_block) = msg { + let block: Block = borsh::from_slice(&zone_block.data) + .context("Failed to deserialize channel genesis block")?; + return Ok::, anyhow::Error>(Some(block)); + } + } + Ok(None) + }; + match tokio::time::timeout(GENESIS_FETCH_TIMEOUT, fetch).await { + Ok(res) => res, + Err(_elapsed) => { + warn!("Timed out reading channel genesis for the consistency check; proceeding"); + Ok(None) + } + } + } + /// Snapshot of the current ingestion status (sync state + indexed tip). /// /// Combines the ingest loop's live status with the L2 tip read fresh from the @@ -194,3 +270,62 @@ impl IndexerCore { } } } + +/// Pure comparison: a mismatch requires BOTH a stored and a channel genesis whose +/// hashes differ; any missing side is treated as Consistent (nothing to act on). +fn compare_genesis(stored: Option<&Block>, current: Option<&Block>) -> GenesisOutcome { + match (stored, current) { + (Some(s), Some(c)) if s.header.hash != c.header.hash => GenesisOutcome::Mismatch { + stored: s.header.hash, + current: c.header.hash, + }, + _ => GenesisOutcome::Consistent, + } +} + +#[cfg(test)] +mod genesis_check_tests { + use common::{HashType, block::Block, test_utils::produce_dummy_block}; + + use super::{GenesisOutcome, compare_genesis}; + + fn genesis_with_prev(prev_seed: u8) -> Block { + produce_dummy_block(1, Some(HashType([prev_seed; 32])), vec![]) + } + + #[test] + fn matching_genesis_is_consistent() { + let g = genesis_with_prev(1); + assert!(matches!( + compare_genesis(Some(&g), Some(&g)), + GenesisOutcome::Consistent + )); + } + + #[test] + fn differing_genesis_is_mismatch() { + let stored = genesis_with_prev(1); + let current = genesis_with_prev(2); + assert!(matches!( + compare_genesis(Some(&stored), Some(¤t)), + GenesisOutcome::Mismatch { .. } + )); + } + + #[test] + fn missing_either_side_is_consistent() { + let g = genesis_with_prev(1); + assert!(matches!( + compare_genesis(None, Some(&g)), + GenesisOutcome::Consistent + )); + assert!(matches!( + compare_genesis(Some(&g), None), + GenesisOutcome::Consistent + )); + assert!(matches!( + compare_genesis(None, None), + GenesisOutcome::Consistent + )); + } +} diff --git a/test_fixtures/src/config.rs b/test_fixtures/src/config.rs index 8684dd4d..ebba9783 100644 --- a/test_fixtures/src/config.rs +++ b/test_fixtures/src/config.rs @@ -172,6 +172,7 @@ pub fn indexer_config(bedrock_addr: SocketAddr) -> Result { auth: None, }, channel_id: bedrock_channel_id(), + allow_chain_reset: false, }) } From 8d0c2f44226eddcabb4a886f0a6d1007507dc429 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 21:08:06 +0300 Subject: [PATCH 13/32] feat(indexer): run genesis-consistency check at service and FFI startup --- lez/indexer/ffi/src/api/lifecycle.rs | 15 +++++++++++---- .../service/configs/debug/indexer_config.json | 3 ++- lez/indexer/service/src/lib.rs | 1 + lez/indexer/service/src/service.rs | 5 +++-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/lez/indexer/ffi/src/api/lifecycle.rs b/lez/indexer/ffi/src/api/lifecycle.rs index f668f3ee..b9d5873b 100644 --- a/lez/indexer/ffi/src/api/lifecycle.rs +++ b/lez/indexer/ffi/src/api/lifecycle.rs @@ -112,10 +112,17 @@ unsafe fn setup_indexer( unsafe { Runtime::from_borrowed(caller.as_ref()) } }; - let core = IndexerCore::new(config, &storage_dir).map_err(|e| { - log::error!("Could not initialize indexer core: {e}"); - OperationStatus::InitializationError - })?; + let allow_reset = config.allow_chain_reset; + let core = runtime + .block_on(IndexerCore::new_with_genesis_check( + config, + &storage_dir, + allow_reset, + )) + .map_err(|e| { + log::error!("Could not initialize indexer core: {e}"); + OperationStatus::InitializationError + })?; // The block stream writes each parsed block into the store as a side effect // of being polled, so we spawn a task that simply drains it. There are no diff --git a/lez/indexer/service/configs/debug/indexer_config.json b/lez/indexer/service/configs/debug/indexer_config.json index 85227700..a87421da 100644 --- a/lez/indexer/service/configs/debug/indexer_config.json +++ b/lez/indexer/service/configs/debug/indexer_config.json @@ -3,5 +3,6 @@ "bedrock_config": { "addr": "http://localhost:18080" }, - "channel_id": "0101010101010101010101010101010101010101010101010101010101010101" + "channel_id": "0101010101010101010101010101010101010101010101010101010101010101", + "allow_chain_reset": false } diff --git a/lez/indexer/service/src/lib.rs b/lez/indexer/service/src/lib.rs index b1c57163..355d7801 100644 --- a/lez/indexer/service/src/lib.rs +++ b/lez/indexer/service/src/lib.rs @@ -87,6 +87,7 @@ pub async fn run_server( #[cfg(not(feature = "mock-responses"))] let handle = { let service = service::IndexerService::new(config, storage_dir) + .await .context("Failed to initialize indexer service")?; server.start(service.into_rpc()) }; diff --git a/lez/indexer/service/src/service.rs b/lez/indexer/service/src/service.rs index 7abab18f..7a0b3a87 100644 --- a/lez/indexer/service/src/service.rs +++ b/lez/indexer/service/src/service.rs @@ -19,8 +19,9 @@ pub struct IndexerService { } impl IndexerService { - pub fn new(config: IndexerConfig, storage_dir: &Path) -> Result { - let indexer = IndexerCore::new(config, storage_dir)?; + pub async fn new(config: IndexerConfig, storage_dir: &Path) -> Result { + let allow_reset = config.allow_chain_reset; + let indexer = IndexerCore::new_with_genesis_check(config, storage_dir, allow_reset).await?; let subscription_service = SubscriptionService::spawn_new(indexer.clone()); Ok(Self { From 50bdeefc8477791677184d9a85a19479464b4a4f Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 26 Jun 2026 21:55:54 +0300 Subject: [PATCH 14/32] chore: greatly increase timeout --- lez/indexer/core/src/lib.rs | 19 +++++++++++++++---- .../service/configs/debug/indexer_config.json | 12 ++++++------ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 1f915ba4..2ba9186d 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -104,10 +104,13 @@ impl IndexerCore { } /// Reads the channel's genesis (first `Block`) from the start of the channel. - /// Returns `None` if the channel has no block yet or L1 can't be reached within - /// the timeout — the check is best-effort and must never block or fail startup. + /// + /// Bedrock can be slow to serve the channel right after boot, so we allow a + /// generous timeout. Returns `None` if the channel has no block yet, the read + /// errors, or the timeout elapses — the check is best-effort and must never + /// fail startup; a genuine reset is still caught by the per-block park logic. async fn fetch_channel_genesis(&self) -> Result> { - const GENESIS_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + const GENESIS_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(180); let fetch = async { let stream = self.zone_indexer.next_messages(None).await?; let mut stream = std::pin::pin!(stream); @@ -121,7 +124,15 @@ impl IndexerCore { Ok(None) }; match tokio::time::timeout(GENESIS_FETCH_TIMEOUT, fetch).await { - Ok(res) => res, + Ok(Ok(maybe_block)) => Ok(maybe_block), + // A read error (e.g. bedrock briefly unreachable at boot) must not refuse + // startup — skip the check and proceed, consistent with the ingest loop. + Ok(Err(err)) => { + warn!( + "Failed to read channel genesis for the consistency check; proceeding: {err:#}" + ); + Ok(None) + } Err(_elapsed) => { warn!("Timed out reading channel genesis for the consistency check; proceeding"); Ok(None) diff --git a/lez/indexer/service/configs/debug/indexer_config.json b/lez/indexer/service/configs/debug/indexer_config.json index a87421da..be5cf353 100644 --- a/lez/indexer/service/configs/debug/indexer_config.json +++ b/lez/indexer/service/configs/debug/indexer_config.json @@ -1,8 +1,8 @@ { - "consensus_info_polling_interval": "1s", - "bedrock_config": { - "addr": "http://localhost:18080" - }, - "channel_id": "0101010101010101010101010101010101010101010101010101010101010101", - "allow_chain_reset": false + "consensus_info_polling_interval": "1s", + "bedrock_config": { + "addr": "http://localhost:18080" + }, + "channel_id": "0101010101010101010101010101010101010101010101010101010101010101", + "allow_chain_reset": true } From 64497526fd9576402b64e760b712a9d8e6438cbd Mon Sep 17 00:00:00 2001 From: erhant Date: Sat, 27 Jun 2026 17:50:08 +0300 Subject: [PATCH 15/32] fix(indexer): detect chain reset via anchor block, **not** deterministic genesis test `RISC0_DEV_MODE=1 RISC0_SKIP_BUILD=1 cargo test -p indexer_core` --- lez/indexer/core/src/lib.rs | 200 ++++++++++-------- lez/indexer/ffi/src/api/lifecycle.rs | 2 +- .../configs/docker/indexer_config.json | 3 +- lez/indexer/service/src/service.rs | 2 +- 4 files changed, 119 insertions(+), 88 deletions(-) diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 2ba9186d..73bb0505 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -2,8 +2,8 @@ use std::{path::Path, sync::Arc}; use anyhow::{Context as _, Result}; use arc_swap::ArcSwap; -use common::{HashType, block::Block}; -// ToDo: Remove after testnet +use common::block::Block; +// TODO: Remove after testnet use futures::StreamExt as _; pub use ingest_error::BlockIngestError; use lee::GENESIS_BLOCK_ID; @@ -24,12 +24,40 @@ pub mod ingest_error; pub mod stall_reason; pub mod status; -/// Result of comparing the indexer's stored genesis against the channel's. -enum GenesisOutcome { - /// Match, or nothing to compare (fresh store / empty channel / L1 unreachable) — proceed. +/// First post-genesis L2 block. +/// +/// We use this to differentiate between a local rocksdb chain +/// and the connected chain, to see if we should reset (mostly dev purposes). +/// While such a discrepancy will not happen during live-chain indexing, this +/// saves us some trouble during development (especially when used within UI +/// with the indexer module). +/// +/// Genesis is deterministic so it is byte-identical across chains, so instead +/// we use the second block to differentiate between chains. +const ANCHOR_BLOCK_ID: u64 = GENESIS_BLOCK_ID + 1; + +/// Result of comparing the indexer's stored anchor block against the channel's. +enum ChainIdentityOutcome { + /// One of the following possibilities: + /// - Anchors match + /// - Nothing to compare (store has only genesis) + /// - L1 was unreadable (proceed from the persisted cursor; fail-over) Consistent, - /// The store holds a different chain than the channel now serves. - Mismatch { stored: HashType, current: HashType }, + /// The store holds a different chain than the channel now serves; `detail` + /// describes how (differing anchor, or the channel lacks the anchor entirely). + Mismatch { detail: String }, +} + +/// Outcome of reading the channel's anchor block at startup. +enum AnchorRead { + /// The channel's anchor block. + Found(Block), + /// The channel definitively has no anchor (drained up to LIB without it). + /// Since our stored anchor was finalized, a finalized channel that lacks it is + /// a different chain. + Absent, + /// Could not read the channel in time (slow/unreachable L1) — best-effort skip. + Unreadable, } #[derive(Clone)] @@ -62,80 +90,96 @@ impl IndexerCore { }) } - /// Builds the core, then verifies the stored genesis matches the channel's. - /// On mismatch: refuse (error) unless `allow_reset`, in which case wipe the store - /// and re-index from scratch. Used at service/FFI startup in place of `new`. - pub async fn new_with_genesis_check( + /// Builds the core, then verifies the stored chain matches the channel's by + /// comparing the anchor block (id 2 — genesis is identical across chains, so + /// it can't detect a reset). On mismatch: refuse (error) unless `allow_reset`, + /// in which case wipe the store and re-index from scratch. Used at service/FFI + /// startup in place of `new`. + pub async fn new_with_chain_check( config: IndexerConfig, storage_dir: &Path, allow_reset: bool, ) -> Result { let home = storage_dir.join(format!("rocksdb-{}", config.channel_id)); let core = Self::new(config.clone(), storage_dir)?; - match core.genesis_outcome().await? { - GenesisOutcome::Consistent => Ok(core), - GenesisOutcome::Mismatch { stored, current } if allow_reset => { + match core.chain_identity_outcome().await? { + ChainIdentityOutcome::Consistent => Ok(core), + ChainIdentityOutcome::Mismatch { detail } if allow_reset => { warn!( - "Chain reset detected: stored genesis {stored} != channel genesis {current}. \ - Wiping indexer store at {} and re-indexing.", + "Chain reset detected ({detail}). Wiping indexer store at {} and re-indexing.", home.display() ); drop(core); // sole owner before the ingest task is spawned → closes the DB storage::indexer::RocksDBIO::destroy(&home)?; Self::new(config, storage_dir) } - GenesisOutcome::Mismatch { stored, current } => Err(anyhow::anyhow!( + ChainIdentityOutcome::Mismatch { detail } => Err(anyhow::anyhow!( "Indexer store at {} holds a different chain than the channel now serves \ - (stored genesis {stored} != channel genesis {current}). Run `just clean`, \ - point at a fresh storage dir, or set `allow_chain_reset` in the indexer config.", + ({detail}). Run `just clean`, point at a fresh storage dir, or set \ + `allow_chain_reset` in the indexer config.", home.display() )), } } - /// Compares the stored genesis (block 1) against the channel's current genesis. - async fn genesis_outcome(&self) -> Result { - let stored = self.store.get_block_at_id(GENESIS_BLOCK_ID)?; - if stored.is_none() { - return Ok(GenesisOutcome::Consistent); // fresh store: skip the L1 read entirely - } - let current = self.fetch_channel_genesis().await?; - Ok(compare_genesis(stored.as_ref(), current.as_ref())) + /// Compares the stored anchor block (id 2) against the channel's current one. + /// An absent channel anchor means a different (shorter) chain, since our stored + /// anchor was finalized and finalized history does not shrink on the same chain. + async fn chain_identity_outcome(&self) -> Result { + let Some(stored) = self.store.get_block_at_id(ANCHOR_BLOCK_ID)? else { + // Store has at most genesis: nothing post-genesis to compare against. + return Ok(ChainIdentityOutcome::Consistent); + }; + Ok(match self.fetch_channel_anchor().await? { + AnchorRead::Found(current) => compare_anchor(&stored, ¤t), + AnchorRead::Absent => ChainIdentityOutcome::Mismatch { + detail: format!( + "channel serves no block {ANCHOR_BLOCK_ID}, but the store holds anchor {}", + stored.header.hash + ), + }, + AnchorRead::Unreadable => ChainIdentityOutcome::Consistent, + }) } - /// Reads the channel's genesis (first `Block`) from the start of the channel. + /// Reads the channel's anchor block (first `Block` with id [`ANCHOR_BLOCK_ID`]) + /// from the start of the channel. /// /// Bedrock can be slow to serve the channel right after boot, so we allow a - /// generous timeout. Returns `None` if the channel has no block yet, the read - /// errors, or the timeout elapses — the check is best-effort and must never - /// fail startup; a genuine reset is still caught by the per-block park logic. - async fn fetch_channel_genesis(&self) -> Result> { - const GENESIS_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(180); + /// generous timeout. `Unreadable` on error/timeout keeps startup best-effort + /// (never refuse on a transient L1 hiccup); `Absent` means the channel has no + /// anchor in its finalized history, which the caller treats as a reset. + async fn fetch_channel_anchor(&self) -> Result { + const ANCHOR_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(180); let fetch = async { let stream = self.zone_indexer.next_messages(None).await?; let mut stream = std::pin::pin!(stream); while let Some((msg, _slot)) = stream.next().await { - if let ZoneMessage::Block(zone_block) = msg { - let block: Block = borsh::from_slice(&zone_block.data) - .context("Failed to deserialize channel genesis block")?; - return Ok::, anyhow::Error>(Some(block)); + let ZoneMessage::Block(zone_block) = msg else { + continue; + }; + let block: Block = borsh::from_slice(&zone_block.data) + .context("Failed to deserialize channel block")?; + if block.header.block_id == ANCHOR_BLOCK_ID { + return Ok::(AnchorRead::Found(block)); + } + if block.header.block_id > ANCHOR_BLOCK_ID { + break; // blocks arrive in order: we passed the anchor without seeing it } } - Ok(None) + Ok(AnchorRead::Absent) }; - match tokio::time::timeout(GENESIS_FETCH_TIMEOUT, fetch).await { - Ok(Ok(maybe_block)) => Ok(maybe_block), - // A read error (e.g. bedrock briefly unreachable at boot) must not refuse - // startup — skip the check and proceed, consistent with the ingest loop. + match tokio::time::timeout(ANCHOR_FETCH_TIMEOUT, fetch).await { + Ok(Ok(read)) => Ok(read), Ok(Err(err)) => { warn!( - "Failed to read channel genesis for the consistency check; proceeding: {err:#}" + "Failed to read channel anchor for the consistency check; proceeding: {err:#}" ); - Ok(None) + Ok(AnchorRead::Unreadable) } Err(_elapsed) => { - warn!("Timed out reading channel genesis for the consistency check; proceeding"); - Ok(None) + warn!("Timed out reading channel anchor for the consistency check; proceeding"); + Ok(AnchorRead::Unreadable) } } } @@ -282,61 +326,47 @@ impl IndexerCore { } } -/// Pure comparison: a mismatch requires BOTH a stored and a channel genesis whose -/// hashes differ; any missing side is treated as Consistent (nothing to act on). -fn compare_genesis(stored: Option<&Block>, current: Option<&Block>) -> GenesisOutcome { - match (stored, current) { - (Some(s), Some(c)) if s.header.hash != c.header.hash => GenesisOutcome::Mismatch { - stored: s.header.hash, - current: c.header.hash, - }, - _ => GenesisOutcome::Consistent, +/// Pure comparison of two anchor blocks: a mismatch is differing hashes. The +/// missing-side cases are handled upstream (`Absent`/`Unreadable`). +fn compare_anchor(stored: &Block, current: &Block) -> ChainIdentityOutcome { + if stored.header.hash == current.header.hash { + ChainIdentityOutcome::Consistent + } else { + ChainIdentityOutcome::Mismatch { + detail: format!( + "stored anchor {} != channel anchor {}", + stored.header.hash, current.header.hash + ), + } } } #[cfg(test)] -mod genesis_check_tests { +mod chain_identity_tests { use common::{HashType, block::Block, test_utils::produce_dummy_block}; - use super::{GenesisOutcome, compare_genesis}; + use super::{ANCHOR_BLOCK_ID, ChainIdentityOutcome, compare_anchor}; - fn genesis_with_prev(prev_seed: u8) -> Block { - produce_dummy_block(1, Some(HashType([prev_seed; 32])), vec![]) + fn anchor_with_prev(prev_seed: u8) -> Block { + produce_dummy_block(ANCHOR_BLOCK_ID, Some(HashType([prev_seed; 32])), vec![]) } #[test] - fn matching_genesis_is_consistent() { - let g = genesis_with_prev(1); + fn matching_anchor_is_consistent() { + let a = anchor_with_prev(1); assert!(matches!( - compare_genesis(Some(&g), Some(&g)), - GenesisOutcome::Consistent + compare_anchor(&a, &a), + ChainIdentityOutcome::Consistent )); } #[test] - fn differing_genesis_is_mismatch() { - let stored = genesis_with_prev(1); - let current = genesis_with_prev(2); + fn differing_anchor_is_mismatch() { + let stored = anchor_with_prev(1); + let current = anchor_with_prev(2); assert!(matches!( - compare_genesis(Some(&stored), Some(¤t)), - GenesisOutcome::Mismatch { .. } - )); - } - - #[test] - fn missing_either_side_is_consistent() { - let g = genesis_with_prev(1); - assert!(matches!( - compare_genesis(None, Some(&g)), - GenesisOutcome::Consistent - )); - assert!(matches!( - compare_genesis(Some(&g), None), - GenesisOutcome::Consistent - )); - assert!(matches!( - compare_genesis(None, None), - GenesisOutcome::Consistent + compare_anchor(&stored, ¤t), + ChainIdentityOutcome::Mismatch { .. } )); } } diff --git a/lez/indexer/ffi/src/api/lifecycle.rs b/lez/indexer/ffi/src/api/lifecycle.rs index b9d5873b..1a71718c 100644 --- a/lez/indexer/ffi/src/api/lifecycle.rs +++ b/lez/indexer/ffi/src/api/lifecycle.rs @@ -114,7 +114,7 @@ unsafe fn setup_indexer( let allow_reset = config.allow_chain_reset; let core = runtime - .block_on(IndexerCore::new_with_genesis_check( + .block_on(IndexerCore::new_with_chain_check( config, &storage_dir, allow_reset, diff --git a/lez/indexer/service/configs/docker/indexer_config.json b/lez/indexer/service/configs/docker/indexer_config.json index f083ca27..ce28af0b 100644 --- a/lez/indexer/service/configs/docker/indexer_config.json +++ b/lez/indexer/service/configs/docker/indexer_config.json @@ -3,5 +3,6 @@ "bedrock_config": { "addr": "http://host.docker.internal:18080" }, - "channel_id": "0101010101010101010101010101010101010101010101010101010101010101" + "channel_id": "0101010101010101010101010101010101010101010101010101010101010101", + "allow_chain_reset": true } diff --git a/lez/indexer/service/src/service.rs b/lez/indexer/service/src/service.rs index 7a0b3a87..ca42fc06 100644 --- a/lez/indexer/service/src/service.rs +++ b/lez/indexer/service/src/service.rs @@ -21,7 +21,7 @@ pub struct IndexerService { impl IndexerService { pub async fn new(config: IndexerConfig, storage_dir: &Path) -> Result { let allow_reset = config.allow_chain_reset; - let indexer = IndexerCore::new_with_genesis_check(config, storage_dir, allow_reset).await?; + let indexer = IndexerCore::new_with_chain_check(config, storage_dir, allow_reset).await?; let subscription_service = SubscriptionService::spawn_new(indexer.clone()); Ok(Self { From 2cdda18b8819afd7f47beb9bf57c1a83f9270eac Mon Sep 17 00:00:00 2001 From: erhant Date: Sat, 27 Jun 2026 19:22:54 +0300 Subject: [PATCH 16/32] fix(indexer): run the chain-identity check even when the store is parked --- lez/indexer/core/src/lib.rs | 190 ++++++++++++++++++------------------ 1 file changed, 94 insertions(+), 96 deletions(-) diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 73bb0505..18c09cfc 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -6,10 +6,9 @@ use common::block::Block; // TODO: Remove after testnet use futures::StreamExt as _; pub use ingest_error::BlockIngestError; -use lee::GENESIS_BLOCK_ID; use log::{error, info, warn}; use logos_blockchain_zone_sdk::{ - CommonHttpClient, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer, + CommonHttpClient, Slot, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer, }; pub use stall_reason::StallReason; @@ -24,42 +23,17 @@ pub mod ingest_error; pub mod stall_reason; pub mod status; -/// First post-genesis L2 block. -/// -/// We use this to differentiate between a local rocksdb chain -/// and the connected chain, to see if we should reset (mostly dev purposes). -/// While such a discrepancy will not happen during live-chain indexing, this -/// saves us some trouble during development (especially when used within UI -/// with the indexer module). -/// -/// Genesis is deterministic so it is byte-identical across chains, so instead -/// we use the second block to differentiate between chains. -const ANCHOR_BLOCK_ID: u64 = GENESIS_BLOCK_ID + 1; - -/// Result of comparing the indexer's stored anchor block against the channel's. +/// Result of comparing the indexer's stored tip against the channel. enum ChainIdentityOutcome { - /// One of the following possibilities: - /// - Anchors match - /// - Nothing to compare (store has only genesis) - /// - L1 was unreadable (proceed from the persisted cursor; fail-over) + /// Proceed from the persisted cursor. Either the channel still serves our tip + /// (same chain), there is nothing to compare (empty store / parked / no cursor), + /// or the check was inconclusive (L1 unreadable, or bedrock's LIB still behind + /// our tip slot) — none of which prove a reset. Consistent, - /// The store holds a different chain than the channel now serves; `detail` - /// describes how (differing anchor, or the channel lacks the anchor entirely). + /// The channel serves a *different* block at the tip's id — a chain reset. Mismatch { detail: String }, } -/// Outcome of reading the channel's anchor block at startup. -enum AnchorRead { - /// The channel's anchor block. - Found(Block), - /// The channel definitively has no anchor (drained up to LIB without it). - /// Since our stored anchor was finalized, a finalized channel that lacks it is - /// a different chain. - Absent, - /// Could not read the channel in time (slow/unreachable L1) — best-effort skip. - Unreadable, -} - #[derive(Clone)] pub struct IndexerCore { pub zone_indexer: Arc>, @@ -91,10 +65,9 @@ impl IndexerCore { } /// Builds the core, then verifies the stored chain matches the channel's by - /// comparing the anchor block (id 2 — genesis is identical across chains, so - /// it can't detect a reset). On mismatch: refuse (error) unless `allow_reset`, - /// in which case wipe the store and re-index from scratch. Used at service/FFI - /// startup in place of `new`. + /// re-reading the channel at the stored tip's position. On mismatch: refuse + /// (error) unless `allow_reset`, in which case wipe the store and re-index from + /// scratch. Used at service/FFI startup in place of `new`. pub async fn new_with_chain_check( config: IndexerConfig, storage_dir: &Path, @@ -122,64 +95,86 @@ impl IndexerCore { } } - /// Compares the stored anchor block (id 2) against the channel's current one. - /// An absent channel anchor means a different (shorter) chain, since our stored - /// anchor was finalized and finalized history does not shrink on the same chain. + /// Verifies the channel still serves our chain at the tip's L1 slot (the persisted + /// cursor), by comparing the first block it serves there against our stored block + /// of the *same id*. + /// + /// This is mostly a development convenience: it lets the indexer notice that its + /// local state belongs to a different chain than the one it is now connected to + /// (e.g. a wiped/restarted bedrock) and reset instead of silently diverging. It + /// will not trigger during normal live indexing. Reading at the cursor — which is + /// recent — keeps this to ~one batch rather than a scan from genesis (L1 slots + /// are wall-clock-derived, so genesis is millions of empty slots away). async fn chain_identity_outcome(&self) -> Result { - let Some(stored) = self.store.get_block_at_id(ANCHOR_BLOCK_ID)? else { - // Store has at most genesis: nothing post-genesis to compare against. + // We deliberately do NOT skip parked stores: a parked store must still be able + // to detect a reset, or it stays parked across reboots forever (its persisted + // stall would short-circuit the check every boot). A same-chain park is still + // safe — the parked block sits at an id we never applied, so the lookup below + // misses and we stay Consistent. + let Some(cursor) = self.store.get_zone_cursor()? else { + return Ok(ChainIdentityOutcome::Consistent); // empty / cold store: nothing to verify + }; + + // Compare the first block the channel serves at/after the cursor against our + // stored block of the same id. On the same chain that is our tip and matches; + // a reset serves a different block here — crucially including a freshly + // restarted, *shorter* chain whose low-id block at this slot differs from ours + // (the old "look for our tip id" approach missed this: a short chain has no + // block at our tip id). + // + // Anything inconclusive stays Consistent (proceed, let ingest park if truly + // divergent) rather than wiping a valid store: an empty/unreadable read (most + // importantly bedrock's LIB still behind our tip slot), or a channel block at + // an id we don't hold. Blind spot: a store holding only genesis can't be + // distinguished (genesis is deterministic across chains), but that window is + // transient. + let Some(channel_block) = self.fetch_channel_block_from(cursor).await? else { return Ok(ChainIdentityOutcome::Consistent); }; - Ok(match self.fetch_channel_anchor().await? { - AnchorRead::Found(current) => compare_anchor(&stored, ¤t), - AnchorRead::Absent => ChainIdentityOutcome::Mismatch { - detail: format!( - "channel serves no block {ANCHOR_BLOCK_ID}, but the store holds anchor {}", - stored.header.hash - ), - }, - AnchorRead::Unreadable => ChainIdentityOutcome::Consistent, - }) + let Some(ours) = self.store.get_block_at_id(channel_block.header.block_id)? else { + return Ok(ChainIdentityOutcome::Consistent); + }; + Ok(compare_block(&ours, &channel_block)) } - /// Reads the channel's anchor block (first `Block` with id [`ANCHOR_BLOCK_ID`]) - /// from the start of the channel. + /// Reads the channel starting at the tip's L1 slot (the `cursor`) and returns the + /// first block it serves there. `next_messages` is exclusive of its argument, so + /// `cursor - 1` is passed to include the tip's own slot. /// - /// Bedrock can be slow to serve the channel right after boot, so we allow a - /// generous timeout. `Unreadable` on error/timeout keeps startup best-effort - /// (never refuse on a transient L1 hiccup); `Absent` means the channel has no - /// anchor in its finalized history, which the caller treats as a reset. - async fn fetch_channel_anchor(&self) -> Result { - const ANCHOR_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(180); + /// Cheap: the cursor is recent, so this reads roughly one batch. `None` covers the + /// inconclusive cases — a slow/unreachable L1 (timeout/error) or bedrock's LIB + /// still behind our tip slot (empty stream) — neither of which proves a reset. + async fn fetch_channel_block_from(&self, cursor: Slot) -> Result> { + const TIP_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + // A slot-0 cursor is degenerate (real inscriptions live at wall-clock slots); + // bail rather than let `next_messages(None)` fall back to a from-genesis scan. + let Some(from_slot) = cursor.into_inner().checked_sub(1) else { + return Ok(None); + }; let fetch = async { - let stream = self.zone_indexer.next_messages(None).await?; + let stream = self + .zone_indexer + .next_messages(Some(Slot::from(from_slot))) + .await?; let mut stream = std::pin::pin!(stream); while let Some((msg, _slot)) = stream.next().await { - let ZoneMessage::Block(zone_block) = msg else { - continue; - }; - let block: Block = borsh::from_slice(&zone_block.data) - .context("Failed to deserialize channel block")?; - if block.header.block_id == ANCHOR_BLOCK_ID { - return Ok::(AnchorRead::Found(block)); - } - if block.header.block_id > ANCHOR_BLOCK_ID { - break; // blocks arrive in order: we passed the anchor without seeing it + if let ZoneMessage::Block(zone_block) = msg { + let block: Block = borsh::from_slice(&zone_block.data) + .context("Failed to deserialize channel block")?; + return Ok::, anyhow::Error>(Some(block)); } } - Ok(AnchorRead::Absent) + Ok(None) }; - match tokio::time::timeout(ANCHOR_FETCH_TIMEOUT, fetch).await { - Ok(Ok(read)) => Ok(read), + match tokio::time::timeout(TIP_FETCH_TIMEOUT, fetch).await { + Ok(Ok(found)) => Ok(found), Ok(Err(err)) => { - warn!( - "Failed to read channel anchor for the consistency check; proceeding: {err:#}" - ); - Ok(AnchorRead::Unreadable) + warn!("Failed to read channel tip for the consistency check; proceeding: {err:#}"); + Ok(None) } Err(_elapsed) => { - warn!("Timed out reading channel anchor for the consistency check; proceeding"); - Ok(AnchorRead::Unreadable) + warn!("Timed out reading channel tip for the consistency check; proceeding"); + Ok(None) } } } @@ -326,16 +321,19 @@ impl IndexerCore { } } -/// Pure comparison of two anchor blocks: a mismatch is differing hashes. The -/// missing-side cases are handled upstream (`Absent`/`Unreadable`). -fn compare_anchor(stored: &Block, current: &Block) -> ChainIdentityOutcome { - if stored.header.hash == current.header.hash { +/// Pure comparison of our stored block against the channel's block at the same id: +/// a mismatch is differing hashes. Missing/unreadable cases are handled upstream. +fn compare_block(ours: &Block, channel: &Block) -> ChainIdentityOutcome { + if ours.header.hash == channel.header.hash { ChainIdentityOutcome::Consistent } else { ChainIdentityOutcome::Mismatch { detail: format!( - "stored anchor {} != channel anchor {}", - stored.header.hash, current.header.hash + "stored block {} {} != channel block {} {}", + ours.header.block_id, + ours.header.hash, + channel.header.block_id, + channel.header.hash ), } } @@ -345,27 +343,27 @@ fn compare_anchor(stored: &Block, current: &Block) -> ChainIdentityOutcome { mod chain_identity_tests { use common::{HashType, block::Block, test_utils::produce_dummy_block}; - use super::{ANCHOR_BLOCK_ID, ChainIdentityOutcome, compare_anchor}; + use super::{ChainIdentityOutcome, compare_block}; - fn anchor_with_prev(prev_seed: u8) -> Block { - produce_dummy_block(ANCHOR_BLOCK_ID, Some(HashType([prev_seed; 32])), vec![]) + fn block_with_prev(prev_seed: u8) -> Block { + produce_dummy_block(5, Some(HashType([prev_seed; 32])), vec![]) } #[test] - fn matching_anchor_is_consistent() { - let a = anchor_with_prev(1); + fn matching_block_is_consistent() { + let b = block_with_prev(1); assert!(matches!( - compare_anchor(&a, &a), + compare_block(&b, &b), ChainIdentityOutcome::Consistent )); } #[test] - fn differing_anchor_is_mismatch() { - let stored = anchor_with_prev(1); - let current = anchor_with_prev(2); + fn differing_block_is_mismatch() { + let stored = block_with_prev(1); + let current = block_with_prev(2); assert!(matches!( - compare_anchor(&stored, ¤t), + compare_block(&stored, ¤t), ChainIdentityOutcome::Mismatch { .. } )); } From 375e76df48bc0a5fb4603551852347cccaa80c99 Mon Sep 17 00:00:00 2001 From: erhant Date: Sat, 27 Jun 2026 19:30:53 +0300 Subject: [PATCH 17/32] fix(indexer): run the chain-identity check even when the store is parked --- lez/indexer/core/src/lib.rs | 67 ++++++++++++++----------------------- 1 file changed, 25 insertions(+), 42 deletions(-) diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 18c09cfc..dfe3eca6 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -25,12 +25,10 @@ pub mod status; /// Result of comparing the indexer's stored tip against the channel. enum ChainIdentityOutcome { - /// Proceed from the persisted cursor. Either the channel still serves our tip - /// (same chain), there is nothing to compare (empty store / parked / no cursor), - /// or the check was inconclusive (L1 unreadable, or bedrock's LIB still behind - /// our tip slot) — none of which prove a reset. + /// Proceed from the cursor: channel still serves our chain, nothing to compare, + /// or the check was inconclusive — none of which prove a reset. Consistent, - /// The channel serves a *different* block at the tip's id — a chain reset. + /// The channel serves a different block at one of our ids — a chain reset. Mismatch { detail: String }, } @@ -95,39 +93,28 @@ impl IndexerCore { } } - /// Verifies the channel still serves our chain at the tip's L1 slot (the persisted - /// cursor), by comparing the first block it serves there against our stored block - /// of the *same id*. + /// Detects when the local store belongs to a different chain than the connected L1 + /// (e.g. a wiped/restarted bedrock) so startup can reset instead of silently + /// diverging. Mostly a dev convenience; won't trigger during normal live indexing. /// - /// This is mostly a development convenience: it lets the indexer notice that its - /// local state belongs to a different chain than the one it is now connected to - /// (e.g. a wiped/restarted bedrock) and reset instead of silently diverging. It - /// will not trigger during normal live indexing. Reading at the cursor — which is - /// recent — keeps this to ~one batch rather than a scan from genesis (L1 slots - /// are wall-clock-derived, so genesis is millions of empty slots away). + /// Compares the first block the channel serves at/after the cursor (the tip's + /// recent L1 slot, so ~one batch — not a from-genesis scan) against our stored + /// block of the *same id*. Comparing by the channel's id, not our tip id, is what + /// catches a *shorter* reset chain: it has no block at our tip id, but its low-id + /// block here differs from ours. async fn chain_identity_outcome(&self) -> Result { - // We deliberately do NOT skip parked stores: a parked store must still be able - // to detect a reset, or it stays parked across reboots forever (its persisted - // stall would short-circuit the check every boot). A same-chain park is still - // safe — the parked block sits at an id we never applied, so the lookup below - // misses and we stay Consistent. + // Don't skip parked stores: the stall is persisted and only clears on a + // successful apply, so skipping would re-park forever and never catch a reset. + // Safe anyway — a same-chain park sits at an id we never applied, so the lookup + // below misses → Consistent. let Some(cursor) = self.store.get_zone_cursor()? else { - return Ok(ChainIdentityOutcome::Consistent); // empty / cold store: nothing to verify + return Ok(ChainIdentityOutcome::Consistent); // empty / cold store }; - // Compare the first block the channel serves at/after the cursor against our - // stored block of the same id. On the same chain that is our tip and matches; - // a reset serves a different block here — crucially including a freshly - // restarted, *shorter* chain whose low-id block at this slot differs from ours - // (the old "look for our tip id" approach missed this: a short chain has no - // block at our tip id). - // - // Anything inconclusive stays Consistent (proceed, let ingest park if truly - // divergent) rather than wiping a valid store: an empty/unreadable read (most - // importantly bedrock's LIB still behind our tip slot), or a channel block at - // an id we don't hold. Blind spot: a store holding only genesis can't be - // distinguished (genesis is deterministic across chains), but that window is - // transient. + // Inconclusive cases stay Consistent (ingest parks if truly divergent) rather + // than wipe a valid store: empty/unreadable read (notably bedrock's LIB behind + // our tip), or an id we don't hold. Blind spot: a genesis-only store can't be + // told apart (genesis is deterministic), but that window is transient. let Some(channel_block) = self.fetch_channel_block_from(cursor).await? else { return Ok(ChainIdentityOutcome::Consistent); }; @@ -137,17 +124,13 @@ impl IndexerCore { Ok(compare_block(&ours, &channel_block)) } - /// Reads the channel starting at the tip's L1 slot (the `cursor`) and returns the - /// first block it serves there. `next_messages` is exclusive of its argument, so - /// `cursor - 1` is passed to include the tip's own slot. - /// - /// Cheap: the cursor is recent, so this reads roughly one batch. `None` covers the - /// inconclusive cases — a slow/unreachable L1 (timeout/error) or bedrock's LIB - /// still behind our tip slot (empty stream) — neither of which proves a reset. + /// Reads the first block the channel serves at/after the tip's slot. `next_messages` + /// is exclusive, so `cursor - 1` includes the tip's own slot. `None` = inconclusive + /// (timeout/error, or bedrock's LIB behind our tip → empty stream). async fn fetch_channel_block_from(&self, cursor: Slot) -> Result> { const TIP_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); - // A slot-0 cursor is degenerate (real inscriptions live at wall-clock slots); - // bail rather than let `next_messages(None)` fall back to a from-genesis scan. + // Slot-0 cursor is degenerate (inscriptions live at wall-clock slots); bail + // rather than let `next_messages(None)` do a from-genesis scan. let Some(from_slot) = cursor.into_inner().checked_sub(1) else { return Ok(None); }; From 022a456bc55e13c8372d45689d7bac4faa592031 Mon Sep 17 00:00:00 2001 From: erhant Date: Sat, 27 Jun 2026 23:09:48 +0300 Subject: [PATCH 18/32] chore: rm redundant unit test [skip ci] --- lez/indexer/core/src/config.rs | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/lez/indexer/core/src/config.rs b/lez/indexer/core/src/config.rs index 580b9fbd..b5f3e810 100644 --- a/lez/indexer/core/src/config.rs +++ b/lez/indexer/core/src/config.rs @@ -43,32 +43,3 @@ impl IndexerConfig { }) } } - -#[cfg(test)] -mod tests { - use super::IndexerConfig; - - const MINIMAL_CONFIG: &str = r#"{ - "consensus_info_polling_interval": "1s", - "bedrock_config": { "addr": "http://localhost:18080" }, - "channel_id": "0101010101010101010101010101010101010101010101010101010101010101" - }"#; - - #[test] - fn allow_chain_reset_defaults_to_false() { - let config: IndexerConfig = serde_json::from_str(MINIMAL_CONFIG).unwrap(); - assert!(!config.allow_chain_reset); - } - - #[test] - fn allow_chain_reset_parses_true() { - let json = r#"{ - "consensus_info_polling_interval": "1s", - "bedrock_config": { "addr": "http://localhost:18080" }, - "channel_id": "0101010101010101010101010101010101010101010101010101010101010101", - "allow_chain_reset": true - }"#; - let config: IndexerConfig = serde_json::from_str(json).unwrap(); - assert!(config.allow_chain_reset); - } -} From d24e25fea3b74fc164fd1ddccef3653f637825b4 Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 30 Jun 2026 13:58:37 +0300 Subject: [PATCH 19/32] fix(indexer): attend to copilot comments, rm Store error from a park case --- lez/indexer/core/src/block_store.rs | 8 +++----- lez/indexer/core/src/config.rs | 5 +++-- lez/indexer/core/src/ingest_error.rs | 2 -- lez/indexer/service/src/service.rs | 9 +++++++-- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 0a32e763..f0f09ec8 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -280,11 +280,9 @@ impl IndexerStore { let mut stored = block.clone(); stored.bedrock_status = BedrockStatus::Finalized; - if let Err(err) = self.dbio.put_block(&stored, [0_u8; 32]) { - let ingest_err = BlockIngestError::Storage(err.to_string()); - self.record_stall(Some(&block.header), l1_slot, ingest_err.clone())?; - return Ok(AcceptOutcome::Parked(ingest_err)); - } + self.dbio + .put_block(&stored, [0_u8; 32]) + .context("Failed to persist accepted block")?; // Commit in-memory state (infallible) only after the DB write succeeded. *self.current_state.write().await = scratch; diff --git a/lez/indexer/core/src/config.rs b/lez/indexer/core/src/config.rs index b5f3e810..c069d5cf 100644 --- a/lez/indexer/core/src/config.rs +++ b/lez/indexer/core/src/config.rs @@ -20,8 +20,9 @@ pub struct IndexerConfig { pub consensus_info_polling_interval: Duration, pub bedrock_config: ClientConfig, pub channel_id: ChannelId, - /// Whether to wipe the indexer store and re-index from scratch when a genesis mismatch occurs - /// (i.e. the L1/sequencer was reset but the old store was reused). + /// Whether to wipe the indexer store and re-index from scratch when the startup + /// chain-identity check finds the channel serving a different block than the one + /// stored at the same id. /// /// Defaults to `false`: on mismatch the indexer refuses to start. #[serde(default)] diff --git a/lez/indexer/core/src/ingest_error.rs b/lez/indexer/core/src/ingest_error.rs index 76693a8e..dce82b9c 100644 --- a/lez/indexer/core/src/ingest_error.rs +++ b/lez/indexer/core/src/ingest_error.rs @@ -22,8 +22,6 @@ pub enum BlockIngestError { }, #[error("state transition failed: {0}")] StateTransition(String), - #[error("storage error: {0}")] - Storage(String), } #[cfg(test)] diff --git a/lez/indexer/service/src/service.rs b/lez/indexer/service/src/service.rs index ca42fc06..0d1d8692 100644 --- a/lez/indexer/service/src/service.rs +++ b/lez/indexer/service/src/service.rs @@ -151,8 +151,13 @@ impl indexer_service_rpc::RpcServer for IndexerService { } async fn get_status(&self) -> Result { - Ok(serde_json::to_value(self.indexer.status()) - .expect("IndexerStatus serialization should not fail")) + serde_json::to_value(self.indexer.status()).map_err(|err| { + ErrorObjectOwned::owned( + ErrorCode::InternalError.code(), + "failed to serialize indexer status".to_owned(), + Some(format!("{err:#}")), + ) + }) } async fn healthcheck(&self) -> Result<(), ErrorObjectOwned> { From 15ebccb11eb073a9d70c0608626ed5c1cc3b9feb Mon Sep 17 00:00:00 2001 From: erhant Date: Wed, 1 Jul 2026 20:04:32 +0300 Subject: [PATCH 20/32] fix(indexer): catch the edge case of re-using the last valid tip on restart --- lez/indexer/core/src/block_store.rs | 74 ++++++++++++++++++++++++++++- lez/indexer/core/src/lib.rs | 10 ++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index f0f09ec8..f5fd08b4 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -24,6 +24,8 @@ struct Tip { pub enum AcceptOutcome { /// Chained and applied; tip and L1 read cursor both advance. Applied, + /// A duplicate re-delivery of the current tip. Just L2 advances. + AlreadyApplied, /// Did not chain or failed to apply; tip stays frozen, stall recorded. Parked(BlockIngestError), } @@ -266,6 +268,15 @@ impl IndexerStore { block: &Block, l1_slot: serde_json::Value, ) -> Result { + // idempotent edge case: re-delivery of the current tip + // (e.g. after a crash that left the L1 cursor behind the applied tip) + if let Some(tip) = self.validated_tip()? + && block.header.block_id == tip.block_id + && block.header.hash == tip.hash + { + return Ok(AcceptOutcome::AlreadyApplied); + } + if let Some(err) = self.acceptance_error(block)? { self.record_stall(Some(&block.header), l1_slot, err.clone())?; return Ok(AcceptOutcome::Parked(err)); @@ -286,7 +297,12 @@ impl IndexerStore { // Commit in-memory state (infallible) only after the DB write succeeded. *self.current_state.write().await = scratch; - self.set_stall_reason(&None)?; + // Clear a recorded stall now that a block has chained and applied. Only write + // when one is actually present, so the steady state does no extra per-block + // write (and can't turn a transient clear-write error into a spurious park). + if self.get_stall_reason()?.is_some() { + self.set_stall_reason(&None)?; + } Ok(AcceptOutcome::Applied) } } @@ -656,4 +672,60 @@ mod accept_tests { "tip must advance to the recovered block" ); } + + #[tokio::test] + async fn redelivered_tip_block_is_idempotent_not_parked() { + use testnet_initial_state::initial_pub_accounts_private_keys; + + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path()).expect("open store"); + + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let genesis = produce_dummy_block(1, None, vec![]); + store + .accept_block(&genesis, serde_json::Value::Null) + .await + .expect("accept genesis"); + + // Block 2: a single transfer of 10. + let tx = common::test_utils::create_transaction_native_token_transfer( + from, 0, to, 10, &sign_key, + ); + let block = produce_dummy_block(2, Some(genesis.header.hash), vec![tx]); + assert!(matches!( + store + .accept_block(&block, serde_json::Value::Null) + .await + .unwrap(), + AcceptOutcome::Applied + )); + let balance_after = store.account_current_state(&from).await.unwrap().balance; + + // Re-deliver the exact same block: idempotent skip, no state change, no park. + assert!(matches!( + store + .accept_block(&block, serde_json::Value::Null) + .await + .unwrap(), + AcceptOutcome::AlreadyApplied + )); + assert_eq!( + store.account_current_state(&from).await.unwrap().balance, + balance_after, + "re-delivered block must not be applied twice" + ); + assert_eq!( + store.get_last_block_id().unwrap(), + Some(2), + "tip must stay at the already-applied block" + ); + assert!( + store.get_stall_reason().unwrap().is_none(), + "a benign duplicate must not park the indexer" + ); + } } diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index dfe3eca6..ba0e7ace 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -261,6 +261,16 @@ impl IndexerCore { } yield Ok(block); } + Ok(AcceptOutcome::AlreadyApplied) => { + info!( + "Skipping already-applied block {}", + block.header.block_id + ); + cursor = Some(slot); + if let Err(err) = self.store.set_zone_cursor(&slot) { + warn!("Failed to persist indexer cursor: {err:#}"); + } + } Ok(AcceptOutcome::Parked(ingest_err)) => { error!( "Parked at block {}: {ingest_err}", From 357c92f947472eed06339843f2ffbe7f2f0fcbe4 Mon Sep 17 00:00:00 2001 From: erhant Date: Wed, 1 Jul 2026 20:50:01 +0300 Subject: [PATCH 21/32] chore: very small comment about future fix [skip ci] --- lez/indexer/core/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index ba0e7ace..d1feca64 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -224,6 +224,7 @@ impl IndexerCore { let zone_block = match msg { ZoneMessage::Block(b) => b, + // FIXME: will be handled in prep of decentralized sequencers ZoneMessage::Deposit(_) | ZoneMessage::Withdraw(_) => continue, }; From 1547a51fa66ace494af72337a0c0f015ddb6fb65 Mon Sep 17 00:00:00 2001 From: erhant Date: Thu, 2 Jul 2026 21:39:30 +0300 Subject: [PATCH 22/32] fix(indexer)!: address several reviews, use better return types with conversions, some docfixes BREAKING CHANGE: dropping the serde rename attrs changes the FFI `query_status` JSON: `state` values are now variant-cased (`caught_up` -> `CaughtUp`) and fields snake_case (`indexedBlockId` -> `indexed_block_id`, `lastError` -> `last_error`). Consumers (lez-indexer-module, lez-explorer-ui) must update their parsing. --- Cargo.lock | 1 + integration_tests/tests/bridge.rs | 16 +- lez/indexer/core/src/block_store.rs | 164 ++++++++------------ lez/indexer/core/src/ingest_error.rs | 3 +- lez/indexer/core/src/lib.rs | 126 +++++++-------- lez/indexer/core/src/stall_reason.rs | 6 +- lez/indexer/core/src/status.rs | 25 ++- lez/indexer/ffi/indexer_ffi.h | 6 +- lez/indexer/ffi/src/api/lifecycle.rs | 7 +- lez/indexer/ffi/src/api/query.rs | 6 +- lez/indexer/service/protocol/Cargo.toml | 3 +- lez/indexer/service/protocol/src/convert.rs | 93 ++++++++++- lez/indexer/service/protocol/src/lib.rs | 63 ++++++++ lez/indexer/service/rpc/src/lib.rs | 6 +- lez/indexer/service/src/mock_service.rs | 22 +-- lez/indexer/service/src/service.rs | 17 +- 16 files changed, 337 insertions(+), 227 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index efb8cd2a..2d4d3dbc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3913,6 +3913,7 @@ dependencies = [ "base64", "common", "hex", + "indexer_core", "lee", "lee_core", "schemars 1.2.1", diff --git a/integration_tests/tests/bridge.rs b/integration_tests/tests/bridge.rs index 9a76bad6..a8fff35e 100644 --- a/integration_tests/tests/bridge.rs +++ b/integration_tests/tests/bridge.rs @@ -572,18 +572,18 @@ async fn indexer_status_rpc_reports_caught_up_with_no_stall() -> anyhow::Result< let status = ctx.indexer_client().get_status().await?; assert_eq!( - status["state"], - serde_json::json!("caught_up"), - "indexer should be caught up, got {status}" + status.state, + indexer_service_protocol::IndexerSyncState::CaughtUp, + "indexer should be caught up, got {status:?}" ); assert!( - status["stallReason"].is_null(), - "indexer should have no stall reason after a clean run, got {status}" + status.stall_reason.is_none(), + "indexer should have no stall reason after a clean run, got {status:?}" ); assert_eq!( - status["indexedBlockId"], - serde_json::json!(indexer_tip), - "status indexedBlockId should equal the caught-up tip" + status.indexed_block_id, + Some(indexer_tip), + "status indexed_block_id should equal the caught-up tip" ); Ok(()) diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index f5fd08b4..3625396c 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -171,8 +171,8 @@ impl IndexerStore { .get_account_by_id(*account_id)) } - /// The last successfully applied block as `{block_id, hash}`, or `None` before - /// any block is stored (cold start). Read fresh from the store each call. + /// The last successfully applied block, or `None` on a cold store. + /// Read fresh from the store each call. fn validated_tip(&self) -> Result> { let Some(block_id) = self.dbio.get_meta_last_block_id_in_db()? else { return Ok(None); @@ -186,51 +186,12 @@ impl IndexerStore { })) } - /// Returns `Some(err)` if `block` is not the valid continuation of the tip: - /// hash integrity, then block-id continuity, then `prev_block_hash` linkage. - fn acceptance_error(&self, block: &Block) -> Result> { - let computed = block.recompute_hash(); - if computed != block.header.hash { - return Ok(Some(BlockIngestError::HashMismatch { - computed, - header: block.header.hash, - })); - } - - match self.validated_tip()? { - None => { - if block.header.block_id != GENESIS_BLOCK_ID { - return Ok(Some(BlockIngestError::UnexpectedBlockId { - expected: GENESIS_BLOCK_ID, - got: block.header.block_id, - })); - } - } - Some(tip) => { - let expected = tip.block_id.checked_add(1).expect("block id overflow"); - if block.header.block_id != expected { - return Ok(Some(BlockIngestError::UnexpectedBlockId { - expected, - got: block.header.block_id, - })); - } - if block.header.prev_block_hash != tip.hash { - return Ok(Some(BlockIngestError::BrokenChainLink { - expected_prev: tip.hash, - got_prev: block.header.prev_block_hash, - })); - } - } - } - Ok(None) - } - /// Records the stall reason: the first break is stored verbatim; subsequent /// breaks only bump `orphans_since`, preserving the original cause. fn record_stall( &self, header: Option<&BlockHeader>, - l1_slot: serde_json::Value, + l1_slot: Slot, error: BlockIngestError, ) -> Result<()> { let stall = match self.get_stall_reason()? { @@ -252,32 +213,26 @@ impl IndexerStore { } /// Records a stall for an inscription that could not even be parsed. - pub fn record_deserialize_stall( - &self, - l1_slot: serde_json::Value, - error: String, - ) -> Result<()> { + pub fn record_deserialize_stall(&self, l1_slot: Slot, error: String) -> Result<()> { self.record_stall(None, l1_slot, BlockIngestError::Deserialize(error)) } /// Validates `block` against the tip and, if it chains, applies it atomically /// (scratch clone, commit only on full success) and advances the tip. On any /// failure records the stall and returns `Parked` without touching state. - pub async fn accept_block( - &self, - block: &Block, - l1_slot: serde_json::Value, - ) -> Result { + pub async fn accept_block(&self, block: &Block, l1_slot: Slot) -> Result { + let tip = self.validated_tip()?; + // idempotent edge case: re-delivery of the current tip // (e.g. after a crash that left the L1 cursor behind the applied tip) - if let Some(tip) = self.validated_tip()? + if let Some(tip) = &tip && block.header.block_id == tip.block_id && block.header.hash == tip.hash { return Ok(AcceptOutcome::AlreadyApplied); } - if let Some(err) = self.acceptance_error(block)? { + if let Err(err) = validate_against_tip(tip.as_ref(), block) { self.record_stall(Some(&block.header), l1_slot, err.clone())?; return Ok(AcceptOutcome::Parked(err)); } @@ -307,6 +262,46 @@ impl IndexerStore { } } +/// Checks that `block` is the valid continuation of `tip`: hash integrity, +/// then block-id continuity, then `prev_block_hash` linkage. A `None` tip +/// (cold store) expects the genesis block. +fn validate_against_tip(tip: Option<&Tip>, block: &Block) -> Result<(), BlockIngestError> { + let computed = block.recompute_hash(); + if computed != block.header.hash { + return Err(BlockIngestError::HashMismatch { + computed, + header: block.header.hash, + }); + } + + match tip { + None => { + if block.header.block_id != GENESIS_BLOCK_ID { + return Err(BlockIngestError::UnexpectedBlockId { + expected: GENESIS_BLOCK_ID, + got: block.header.block_id, + }); + } + } + Some(tip) => { + let expected = tip.block_id.checked_add(1).expect("block id overflow"); + if block.header.block_id != expected { + return Err(BlockIngestError::UnexpectedBlockId { + expected, + got: block.header.block_id, + }); + } + if block.header.prev_block_hash != tip.hash { + return Err(BlockIngestError::BrokenChainLink { + expected_prev: tip.hash, + got_prev: block.header.prev_block_hash, + }); + } + } + } + Ok(()) +} + /// Applies a block's transactions to `state`, mapping every failure to a /// [`BlockIngestError`] so the caller can park rather than crash. Operates on a /// scratch state; the caller commits only on `Ok`. @@ -380,7 +375,7 @@ mod stall_reason_tests { block_id: Some(7), block_hash: Some(HashType([1_u8; 32])), prev_block_hash: Some(HashType([2_u8; 32])), - l1_slot: serde_json::Value::Null, + l1_slot: Slot::from(42), error: BlockIngestError::StateTransition("boom".to_owned()), first_seen: Some(99), orphans_since: 3, @@ -393,7 +388,7 @@ mod stall_reason_tests { assert!(matches!(got.error, BlockIngestError::StateTransition(_))); assert_eq!(got.block_hash, Some(HashType([1_u8; 32]))); assert_eq!(got.prev_block_hash, Some(HashType([2_u8; 32]))); - assert_eq!(got.l1_slot, serde_json::Value::Null); + assert_eq!(got.l1_slot, Slot::from(42)); assert_eq!(got.first_seen, Some(99)); store.set_stall_reason(&None).expect("clear"); @@ -434,10 +429,7 @@ mod tests { let genesis = produce_dummy_block(1, None, vec![]); let mut prev_hash = genesis.header.hash; assert!(matches!( - store - .accept_block(&genesis, serde_json::Value::Null) - .await - .unwrap(), + store.accept_block(&genesis, Slot::from(0)).await.unwrap(), AcceptOutcome::Applied )); @@ -447,10 +439,7 @@ mod tests { let block = produce_dummy_block(i + 2, Some(prev_hash), vec![tx]); prev_hash = block.header.hash; assert!(matches!( - store - .accept_block(&block, serde_json::Value::Null) - .await - .unwrap(), + store.accept_block(&block, Slot::from(0)).await.unwrap(), AcceptOutcome::Applied )); } @@ -480,19 +469,13 @@ mod tests { let genesis = produce_dummy_block(1, None, vec![]); let mut prev_hash = genesis.header.hash; - store - .accept_block(&genesis, serde_json::Value::Null) - .await - .unwrap(); + store.accept_block(&genesis, Slot::from(0)).await.unwrap(); for i in 0..10_u64 { let tx = create_transaction_native_token_transfer(from, i.into(), to, 10, &sign_key); let block = produce_dummy_block(i + 2, Some(prev_hash), vec![tx]); prev_hash = block.header.hash; - store - .accept_block(&block, serde_json::Value::Null) - .await - .unwrap(); + store.accept_block(&block, Slot::from(0)).await.unwrap(); } // State at block N is inclusive of block N. @@ -547,7 +530,7 @@ mod accept_tests { let block = valid_hash_block(2, HashType([0_u8; 32])); let outcome = store - .accept_block(&block, serde_json::Value::Null) + .accept_block(&block, Slot::from(0)) .await .expect("accept"); @@ -572,7 +555,7 @@ mod accept_tests { block.header.timestamp = 999; // invalidates the stored hash let outcome = store - .accept_block(&block, serde_json::Value::Null) + .accept_block(&block, Slot::from(0)) .await .expect("accept"); assert!(matches!( @@ -588,12 +571,12 @@ mod accept_tests { let first = valid_hash_block(2, HashType([0_u8; 32])); store - .accept_block(&first, serde_json::Value::Null) + .accept_block(&first, Slot::from(0)) .await .expect("accept"); let second = valid_hash_block(3, HashType([0_u8; 32])); store - .accept_block(&second, serde_json::Value::Null) + .accept_block(&second, Slot::from(0)) .await .expect("accept"); @@ -608,7 +591,7 @@ mod accept_tests { let store = IndexerStore::open_db(dir.path()).expect("open store"); store - .record_deserialize_stall(serde_json::Value::Null, "bad bytes".to_owned()) + .record_deserialize_stall(Slot::from(0), "bad bytes".to_owned()) .expect("record"); let stall = store.get_stall_reason().expect("get").expect("present"); @@ -624,20 +607,14 @@ mod accept_tests { // Genesis (block 1, clock-only) applies and advances the tip. let genesis = produce_dummy_block(1, None, vec![]); assert!(matches!( - store - .accept_block(&genesis, serde_json::Value::Null) - .await - .unwrap(), + store.accept_block(&genesis, Slot::from(0)).await.unwrap(), AcceptOutcome::Applied )); // A block that skips ahead (id 3 while the tip is 1) parks the indexer. let bad = produce_dummy_block(3, Some(genesis.header.hash), vec![]); assert!(matches!( - store - .accept_block(&bad, serde_json::Value::Null) - .await - .unwrap(), + store.accept_block(&bad, Slot::from(0)).await.unwrap(), AcceptOutcome::Parked(BlockIngestError::UnexpectedBlockId { expected: 2, got: 3 @@ -656,10 +633,7 @@ mod accept_tests { // The valid continuation (block 2 chaining on genesis) recovers the chain. let next = produce_dummy_block(2, Some(genesis.header.hash), vec![]); assert!(matches!( - store - .accept_block(&next, serde_json::Value::Null) - .await - .unwrap(), + store.accept_block(&next, Slot::from(0)).await.unwrap(), AcceptOutcome::Applied )); assert!( @@ -687,7 +661,7 @@ mod accept_tests { let genesis = produce_dummy_block(1, None, vec![]); store - .accept_block(&genesis, serde_json::Value::Null) + .accept_block(&genesis, Slot::from(0)) .await .expect("accept genesis"); @@ -697,20 +671,14 @@ mod accept_tests { ); let block = produce_dummy_block(2, Some(genesis.header.hash), vec![tx]); assert!(matches!( - store - .accept_block(&block, serde_json::Value::Null) - .await - .unwrap(), + store.accept_block(&block, Slot::from(0)).await.unwrap(), AcceptOutcome::Applied )); let balance_after = store.account_current_state(&from).await.unwrap().balance; // Re-deliver the exact same block: idempotent skip, no state change, no park. assert!(matches!( - store - .accept_block(&block, serde_json::Value::Null) - .await - .unwrap(), + store.accept_block(&block, Slot::from(0)).await.unwrap(), AcceptOutcome::AlreadyApplied )); assert_eq!( diff --git a/lez/indexer/core/src/ingest_error.rs b/lez/indexer/core/src/ingest_error.rs index dce82b9c..1235120a 100644 --- a/lez/indexer/core/src/ingest_error.rs +++ b/lez/indexer/core/src/ingest_error.rs @@ -4,7 +4,6 @@ use serde::{Deserialize, Serialize}; /// Why the indexer could not apply an L2 block from the channel. Stored inside a /// [`crate::stall_reason::StallReason`] and surfaced on the status snapshot. #[derive(Debug, Clone, Serialize, Deserialize, thiserror::Error)] -#[serde(rename_all = "camelCase")] pub enum BlockIngestError { #[error("failed to deserialize L2 block: {0}")] Deserialize(String), @@ -37,7 +36,7 @@ mod tests { let value = serde_json::to_value(&err).expect("serialize"); assert_eq!( value, - serde_json::json!({ "unexpectedBlockId": { "expected": 5, "got": 7 } }) + serde_json::json!({ "UnexpectedBlockId": { "expected": 5, "got": 7 } }) ); let back: BlockIngestError = serde_json::from_value(value).expect("deserialize"); assert!(matches!( diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index d1feca64..d24488ee 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -42,7 +42,35 @@ pub struct IndexerCore { } impl IndexerCore { - pub fn new(config: IndexerConfig, storage_dir: &Path) -> Result { + /// Builds the core, then verifies the stored chain matches the channel's by + /// re-reading the channel at the stored tip's position. On mismatch: refuse + /// (error) unless `config.allow_chain_reset` is set, in which case wipe the + /// store and re-index from scratch. + pub async fn new(config: IndexerConfig, storage_dir: &Path) -> Result { + let home = storage_dir.join(format!("rocksdb-{}", config.channel_id)); + let core = Self::open(config.clone(), storage_dir)?; + match core.chain_identity_outcome().await? { + ChainIdentityOutcome::Consistent => Ok(core), + ChainIdentityOutcome::Mismatch { detail } if config.allow_chain_reset => { + warn!( + "Chain reset detected ({detail}). Wiping indexer store at {} and re-indexing.", + home.display() + ); + drop(core); // sole owner before the ingest task is spawned → closes the DB + storage::indexer::RocksDBIO::destroy(&home)?; + Self::open(config, storage_dir) + } + ChainIdentityOutcome::Mismatch { detail } => Err(anyhow::anyhow!( + "Indexer store at {} holds a different chain than the channel now serves \ + ({detail}). Delete the indexer storage directory, point at a fresh one, or \ + set `allow_chain_reset` in the indexer config.", + home.display() + )), + } + } + + /// Opens the store and builds the core without the chain-identity check. + fn open(config: IndexerConfig, storage_dir: &Path) -> Result { // Namespace the DB by channel so indexers on different channels can // share a storage dir without their RocksDB state colliding. let home = storage_dir.join(format!("rocksdb-{}", config.channel_id)); @@ -62,37 +90,6 @@ impl IndexerCore { }) } - /// Builds the core, then verifies the stored chain matches the channel's by - /// re-reading the channel at the stored tip's position. On mismatch: refuse - /// (error) unless `allow_reset`, in which case wipe the store and re-index from - /// scratch. Used at service/FFI startup in place of `new`. - pub async fn new_with_chain_check( - config: IndexerConfig, - storage_dir: &Path, - allow_reset: bool, - ) -> Result { - let home = storage_dir.join(format!("rocksdb-{}", config.channel_id)); - let core = Self::new(config.clone(), storage_dir)?; - match core.chain_identity_outcome().await? { - ChainIdentityOutcome::Consistent => Ok(core), - ChainIdentityOutcome::Mismatch { detail } if allow_reset => { - warn!( - "Chain reset detected ({detail}). Wiping indexer store at {} and re-indexing.", - home.display() - ); - drop(core); // sole owner before the ingest task is spawned → closes the DB - storage::indexer::RocksDBIO::destroy(&home)?; - Self::new(config, storage_dir) - } - ChainIdentityOutcome::Mismatch { detail } => Err(anyhow::anyhow!( - "Indexer store at {} holds a different chain than the channel now serves \ - ({detail}). Run `just clean`, point at a fresh storage dir, or set \ - `allow_chain_reset` in the indexer config.", - home.display() - )), - } - } - /// Detects when the local store belongs to a different chain than the connected L1 /// (e.g. a wiped/restarted bedrock) so startup can reset instead of silently /// diverging. Mostly a dev convenience; won't trigger during normal live indexing. @@ -183,6 +180,28 @@ impl IndexerCore { self.status.store(Arc::new(status)); } + /// Advances the in-memory L1 read cursor past `slot` and persists it. + /// A persist failure is only logged: the worst case is re-reading a batch + /// after a restart, which ingestion handles idempotently. + fn advance_cursor(&self, cursor: &mut Option, slot: Slot) { + *cursor = Some(slot); + if let Err(err) = self.store.set_zone_cursor(&slot) { + warn!("Failed to persist indexer cursor: {err:#}"); + } + } + + /// Parks on an inscription that could not be parsed as an L2 block: + /// records the stall and flips the status. The validated tip stays frozen. + fn park_undeserializable(&self, slot: Slot, error: &impl std::fmt::Display) { + error!("Failed to deserialize L2 block from zone-sdk: {error}"); + if let Err(err) = self.store.record_deserialize_stall(slot, error.to_string()) { + warn!("Failed to record stall reason: {err:#}"); + } + self.set_status(IndexerSyncStatus::stalled(format!( + "failed to deserialize L2 block: {error}" + ))); + } + pub fn subscribe_parse_block_stream(&self) -> impl futures::Stream> + '_ { let poll_interval = self.config.consensus_info_polling_interval; let initial_cursor = self @@ -228,38 +247,29 @@ impl IndexerCore { ZoneMessage::Deposit(_) | ZoneMessage::Withdraw(_) => continue, }; - let l1_slot = serde_json::to_value(slot).unwrap_or(serde_json::Value::Null); - let block: Block = match borsh::from_slice(&zone_block.data) { Ok(b) => b, - Err(e) => { - error!("Failed to deserialize L2 block from zone-sdk: {e}"); - if let Err(err) = - self.store.record_deserialize_stall(l1_slot, e.to_string()) - { + Err(error) => { + error!("Failed to deserialize L2 block from zone-sdk: {error}"); + if let Err(err) = self.store.record_deserialize_stall(slot, error.to_string()) { warn!("Failed to record stall reason: {err:#}"); } self.set_status(IndexerSyncStatus::stalled(format!( - "failed to deserialize L2 block: {e}" + "failed to deserialize L2 block: {error}" ))); - // Advance the L1 read cursor past the broken inscription; - // the validated tip stays frozen. - cursor = Some(slot); - if let Err(err) = self.store.set_zone_cursor(&slot) { - warn!("Failed to persist indexer cursor: {err:#}"); - } + + + // L1 proceeds regardless + self.advance_cursor(&mut cursor, slot); continue; } }; - match self.store.accept_block(&block, l1_slot).await { + match self.store.accept_block(&block, slot).await { Ok(AcceptOutcome::Applied) => { info!("Indexed L2 block {}", block.header.block_id); self.set_status(IndexerSyncStatus::syncing()); - cursor = Some(slot); - if let Err(err) = self.store.set_zone_cursor(&slot) { - warn!("Failed to persist indexer cursor: {err:#}"); - } + self.advance_cursor(&mut cursor, slot); yield Ok(block); } Ok(AcceptOutcome::AlreadyApplied) => { @@ -267,10 +277,7 @@ impl IndexerCore { "Skipping already-applied block {}", block.header.block_id ); - cursor = Some(slot); - if let Err(err) = self.store.set_zone_cursor(&slot) { - warn!("Failed to persist indexer cursor: {err:#}"); - } + self.advance_cursor(&mut cursor, slot); } Ok(AcceptOutcome::Parked(ingest_err)) => { error!( @@ -278,15 +285,12 @@ impl IndexerCore { block.header.block_id ); self.set_status(IndexerSyncStatus::stalled(ingest_err.to_string())); - // Advance the L1 read cursor; tip stays frozen, no yield. - cursor = Some(slot); - if let Err(err) = self.store.set_zone_cursor(&slot) { - warn!("Failed to persist indexer cursor: {err:#}"); - } + // L1 proceeds regardless + self.advance_cursor(&mut cursor, slot); } Err(err) => { // Infrastructure error (DB read/write), not a bad block. - // Keep the cursor put; re-poll the same position next cycle. + // will re-poll from the same cursor next cycle. error!( "Store error applying block {}: {err:#}", block.header.block_id diff --git a/lez/indexer/core/src/stall_reason.rs b/lez/indexer/core/src/stall_reason.rs index fdad9861..c5118fcc 100644 --- a/lez/indexer/core/src/stall_reason.rs +++ b/lez/indexer/core/src/stall_reason.rs @@ -1,4 +1,5 @@ use common::HashType; +use logos_blockchain_zone_sdk::Slot; use serde::{Deserialize, Serialize}; use crate::ingest_error::BlockIngestError; @@ -6,15 +7,14 @@ use crate::ingest_error::BlockIngestError; /// Diagnostic record of the first block that broke the L2 chain. /// /// The block-derived fields are `None` for a deserialize break (no header was -/// ever parsed). `l1_slot` is the zone-sdk cursor position captured as JSON. +/// ever parsed). `l1_slot` is the L1 slot the breaking inscription was read at. /// `first_seen` is the breaking block's L2 timestamp (`None` for a deserialize break). #[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] pub struct StallReason { pub block_id: Option, pub block_hash: Option, pub prev_block_hash: Option, - pub l1_slot: serde_json::Value, + pub l1_slot: Slot, pub error: BlockIngestError, pub first_seen: Option, /// Number of later non-chaining blocks (orphans, since the tip is frozen). diff --git a/lez/indexer/core/src/status.rs b/lez/indexer/core/src/status.rs index d4f6d918..939abec5 100644 --- a/lez/indexer/core/src/status.rs +++ b/lez/indexer/core/src/status.rs @@ -5,7 +5,6 @@ use crate::stall_reason::StallReason; /// Coarse lifecycle state of the indexer's ingestion loop, so a client can tell /// "still catching up" apart from "something went wrong". #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] pub enum IndexerSyncState { /// Booted; no ingestion cycle has run yet. Starting, @@ -23,7 +22,6 @@ pub enum IndexerSyncState { /// Live ingestion status owned by the ingest loop: the coarse `state` plus the /// reason when it is `Error`. #[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] pub struct IndexerSyncStatus { pub state: IndexerSyncState, pub last_error: Option, @@ -78,7 +76,6 @@ impl IndexerSyncStatus { /// The tip is tracked by the store, not the ingest loop, so it lives here on the /// returned snapshot rather than inside the shared [`IndexerSyncStatus`]. #[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] pub struct IndexerStatus { #[serde(flatten)] pub sync: IndexerSyncStatus, @@ -101,10 +98,10 @@ mod tests { assert_eq!( value, serde_json::json!({ - "state": "error", - "lastError": "boom", - "indexedBlockId": 7, - "stallReason": null, + "state": "Error", + "last_error": "boom", + "indexed_block_id": 7, + "stall_reason": null, }) ); } @@ -114,12 +111,14 @@ mod tests { let value = serde_json::to_value(IndexerSyncStatus::caught_up()).expect("serialize"); assert_eq!( value, - serde_json::json!({ "state": "caught_up", "lastError": null }) + serde_json::json!({ "state": "CaughtUp", "last_error": null }) ); } #[test] fn stalled_status_serializes_with_stall_reason() { + use logos_blockchain_zone_sdk::Slot; + use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; let status = IndexerStatus { @@ -129,16 +128,16 @@ mod tests { block_id: Some(42), block_hash: None, prev_block_hash: None, - l1_slot: serde_json::Value::Null, + l1_slot: Slot::from(0), error: BlockIngestError::StateTransition("boom".to_owned()), first_seen: None, orphans_since: 2, }), }; let value = serde_json::to_value(&status).expect("serialize"); - assert_eq!(value["state"], serde_json::json!("stalled")); - assert_eq!(value["lastError"], serde_json::json!("broken chain link")); - assert_eq!(value["indexedBlockId"], serde_json::json!(41)); - assert_eq!(value["stallReason"]["orphansSince"], serde_json::json!(2)); + assert_eq!(value["state"], serde_json::json!("Stalled")); + assert_eq!(value["last_error"], serde_json::json!("broken chain link")); + assert_eq!(value["indexed_block_id"], serde_json::json!(41)); + assert_eq!(value["stall_reason"]["orphans_since"], serde_json::json!(2)); } } diff --git a/lez/indexer/ffi/indexer_ffi.h b/lez/indexer/ffi/indexer_ffi.h index 8347ad3c..26857af7 100644 --- a/lez/indexer/ffi/indexer_ffi.h +++ b/lez/indexer/ffi/indexer_ffi.h @@ -503,9 +503,9 @@ struct LastBlockIdResult query_last_block(const struct IndexerServiceFFI *indexe * Query the indexer's current sync status as a JSON C-string. * * The JSON schema is owned by `indexer_core` (`IndexerStatus`): an object with - * `state` (`starting`/`syncing`/`caught_up`/`error`), `indexedBlockId`, and - * `lastError`. Lets a client distinguish "still catching up" from "something - * went wrong". + * `state` (`Starting`/`Syncing`/`CaughtUp`/`Error`/`Stalled`), + * `indexed_block_id`, `last_error`, and `stall_reason`. Lets a client + * distinguish "still catching up" from "something went wrong". * * # Arguments * diff --git a/lez/indexer/ffi/src/api/lifecycle.rs b/lez/indexer/ffi/src/api/lifecycle.rs index 1a71718c..8a1eafaf 100644 --- a/lez/indexer/ffi/src/api/lifecycle.rs +++ b/lez/indexer/ffi/src/api/lifecycle.rs @@ -112,13 +112,8 @@ unsafe fn setup_indexer( unsafe { Runtime::from_borrowed(caller.as_ref()) } }; - let allow_reset = config.allow_chain_reset; let core = runtime - .block_on(IndexerCore::new_with_chain_check( - config, - &storage_dir, - allow_reset, - )) + .block_on(IndexerCore::new(config, &storage_dir)) .map_err(|e| { log::error!("Could not initialize indexer core: {e}"); OperationStatus::InitializationError diff --git a/lez/indexer/ffi/src/api/query.rs b/lez/indexer/ffi/src/api/query.rs index 1943f6d4..dff027fa 100644 --- a/lez/indexer/ffi/src/api/query.rs +++ b/lez/indexer/ffi/src/api/query.rs @@ -91,9 +91,9 @@ pub unsafe extern "C" fn query_last_block(indexer: *const IndexerServiceFFI) -> /// Query the indexer's current sync status as a JSON C-string. /// /// The JSON schema is owned by `indexer_core` (`IndexerStatus`): an object with -/// `state` (`starting`/`syncing`/`caught_up`/`error`), `indexedBlockId`, and -/// `lastError`. Lets a client distinguish "still catching up" from "something -/// went wrong". +/// `state` (`Starting`/`Syncing`/`CaughtUp`/`Error`/`Stalled`), +/// `indexed_block_id`, `last_error`, and `stall_reason`. Lets a client +/// distinguish "still catching up" from "something went wrong". /// /// # Arguments /// diff --git a/lez/indexer/service/protocol/Cargo.toml b/lez/indexer/service/protocol/Cargo.toml index 5a4176f5..b3e8f65c 100644 --- a/lez/indexer/service/protocol/Cargo.toml +++ b/lez/indexer/service/protocol/Cargo.toml @@ -11,6 +11,7 @@ workspace = true lee_core = { workspace = true, optional = true, features = ["host"] } lee = { workspace = true, optional = true } common = { workspace = true, optional = true } +indexer_core = { workspace = true, optional = true } serde = { workspace = true, features = ["derive"] } serde_with.workspace = true @@ -22,4 +23,4 @@ anyhow.workspace = true [features] # Enable conversion to/from LEE core types -convert = ["dep:lee_core", "dep:lee", "dep:common"] +convert = ["dep:lee_core", "dep:lee", "dep:common", "dep:indexer_core"] diff --git a/lez/indexer/service/protocol/src/convert.rs b/lez/indexer/service/protocol/src/convert.rs index cd0bff7e..f497574d 100644 --- a/lez/indexer/service/protocol/src/convert.rs +++ b/lez/indexer/service/protocol/src/convert.rs @@ -3,11 +3,12 @@ use lee_core::account::Nonce; use crate::{ - Account, AccountId, BedrockStatus, Block, BlockBody, BlockHeader, Ciphertext, Commitment, - CommitmentSetDigest, Data, EncryptedAccountData, EphemeralPublicKey, HashType, Nullifier, - PrivacyPreservingMessage, PrivacyPreservingTransaction, ProgramDeploymentMessage, - ProgramDeploymentTransaction, ProgramId, Proof, PublicKey, PublicMessage, PublicTransaction, - Signature, Transaction, ValidityWindow, WitnessSet, + Account, AccountId, BedrockStatus, Block, BlockBody, BlockHeader, BlockIngestError, Ciphertext, + Commitment, CommitmentSetDigest, Data, EncryptedAccountData, EphemeralPublicKey, HashType, + IndexerStatus, IndexerSyncState, Nullifier, PrivacyPreservingMessage, + PrivacyPreservingTransaction, ProgramDeploymentMessage, ProgramDeploymentTransaction, + ProgramId, Proof, PublicKey, PublicMessage, PublicTransaction, Signature, StallReason, + Transaction, ValidityWindow, WitnessSet, }; // ============================================================================ @@ -707,3 +708,85 @@ impl TryFrom for lee_core::program::ValidityWindow { value.0.try_into() } } + +// ============================================================================ +// Indexer status conversions +// ============================================================================ + +impl From for IndexerSyncState { + fn from(value: indexer_core::status::IndexerSyncState) -> Self { + match value { + indexer_core::status::IndexerSyncState::Starting => Self::Starting, + indexer_core::status::IndexerSyncState::Syncing => Self::Syncing, + indexer_core::status::IndexerSyncState::CaughtUp => Self::CaughtUp, + indexer_core::status::IndexerSyncState::Error => Self::Error, + indexer_core::status::IndexerSyncState::Stalled => Self::Stalled, + } + } +} + +impl From for BlockIngestError { + fn from(value: indexer_core::BlockIngestError) -> Self { + match value { + indexer_core::BlockIngestError::Deserialize(msg) => Self::Deserialize(msg), + indexer_core::BlockIngestError::UnexpectedBlockId { expected, got } => { + Self::UnexpectedBlockId { expected, got } + } + indexer_core::BlockIngestError::BrokenChainLink { + expected_prev, + got_prev, + } => Self::BrokenChainLink { + expected_prev: expected_prev.into(), + got_prev: got_prev.into(), + }, + indexer_core::BlockIngestError::HashMismatch { computed, header } => { + Self::HashMismatch { + computed: computed.into(), + header: header.into(), + } + } + indexer_core::BlockIngestError::StateTransition(msg) => Self::StateTransition(msg), + } + } +} + +impl From for StallReason { + fn from(value: indexer_core::StallReason) -> Self { + let indexer_core::StallReason { + block_id, + block_hash, + prev_block_hash, + l1_slot, + error, + first_seen, + orphans_since, + } = value; + + Self { + block_id, + block_hash: block_hash.map(Into::into), + prev_block_hash: prev_block_hash.map(Into::into), + l1_slot: l1_slot.into_inner(), + error: error.into(), + first_seen, + orphans_since, + } + } +} + +impl From for IndexerStatus { + fn from(value: indexer_core::status::IndexerStatus) -> Self { + let indexer_core::status::IndexerStatus { + sync, + indexed_block_id, + stall_reason, + } = value; + + Self { + state: sync.state.into(), + last_error: sync.last_error, + indexed_block_id, + stall_reason: stall_reason.map(Into::into), + } + } +} diff --git a/lez/indexer/service/protocol/src/lib.rs b/lez/indexer/service/protocol/src/lib.rs index a670dee6..6ea06a81 100644 --- a/lez/indexer/service/protocol/src/lib.rs +++ b/lez/indexer/service/protocol/src/lib.rs @@ -363,3 +363,66 @@ pub enum BedrockStatus { Safe, Finalized, } + +/// Coarse lifecycle state of the indexer's ingestion loop, so a client can tell +/// "still catching up" apart from "something went wrong". +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +pub enum IndexerSyncState { + /// Booted; no ingestion cycle has run yet. + Starting, + /// Streaming finalized messages toward the L1 frontier. + Syncing, + /// Drained the stream up to the last finalized block; idle until new blocks finalize. + CaughtUp, + /// The last cycle failed (e.g. the L1 node is unreachable). See `last_error`. + Error, + /// Parked on a stall reason: the validated tip is frozen awaiting a valid + /// continuation. See `last_error` and `stall_reason`. + Stalled, +} + +/// Why the indexer could not apply an L2 block from the channel. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +pub enum BlockIngestError { + Deserialize(String), + UnexpectedBlockId { + expected: u64, + got: u64, + }, + BrokenChainLink { + expected_prev: HashType, + got_prev: HashType, + }, + HashMismatch { + computed: HashType, + header: HashType, + }, + StateTransition(String), +} + +/// Diagnostic record of the first block that broke the L2 chain. +/// +/// The block-derived fields are `None` for a deserialize break (no header was +/// ever parsed). `l1_slot` is the L1 slot the breaking inscription was read at. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +pub struct StallReason { + pub block_id: Option, + pub block_hash: Option, + pub prev_block_hash: Option, + pub l1_slot: u64, + pub error: BlockIngestError, + /// The breaking block's L2 timestamp (`None` for a deserialize break). + pub first_seen: Option, + /// Number of later non-chaining blocks seen while the tip is frozen. + pub orphans_since: u64, +} + +/// Status snapshot returned by `getStatus`: the ingestion state plus the +/// indexed L2 tip and, when stalled, the stall diagnostics. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +pub struct IndexerStatus { + pub state: IndexerSyncState, + pub last_error: Option, + pub indexed_block_id: Option, + pub stall_reason: Option, +} diff --git a/lez/indexer/service/rpc/src/lib.rs b/lez/indexer/service/rpc/src/lib.rs index 108ee6c2..8ea807eb 100644 --- a/lez/indexer/service/rpc/src/lib.rs +++ b/lez/indexer/service/rpc/src/lib.rs @@ -1,4 +1,6 @@ -use indexer_service_protocol::{Account, AccountId, Block, BlockId, HashType, Transaction}; +use indexer_service_protocol::{ + Account, AccountId, Block, BlockId, HashType, IndexerStatus, Transaction, +}; use jsonrpsee::proc_macros::rpc; #[cfg(feature = "server")] use jsonrpsee::{core::SubscriptionResult, types::ErrorObjectOwned}; @@ -70,7 +72,7 @@ pub trait Rpc { ) -> Result, ErrorObjectOwned>; #[method(name = "getStatus")] - async fn get_status(&self) -> Result; + async fn get_status(&self) -> Result; // ToDo: expand healthcheck response into some kind of report #[method(name = "checkHealth")] diff --git a/lez/indexer/service/src/mock_service.rs b/lez/indexer/service/src/mock_service.rs index c92ebb0f..70af6239 100644 --- a/lez/indexer/service/src/mock_service.rs +++ b/lez/indexer/service/src/mock_service.rs @@ -10,10 +10,10 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; use indexer_service_protocol::{ Account, AccountId, BedrockStatus, Block, BlockBody, BlockHeader, BlockId, Commitment, - CommitmentSetDigest, Data, EncryptedAccountData, HashType, PrivacyPreservingMessage, - PrivacyPreservingTransaction, ProgramDeploymentMessage, ProgramDeploymentTransaction, - ProgramId, PublicMessage, PublicTransaction, Signature, Transaction, ValidityWindow, - WitnessSet, + CommitmentSetDigest, Data, EncryptedAccountData, HashType, IndexerStatus, IndexerSyncState, + PrivacyPreservingMessage, PrivacyPreservingTransaction, ProgramDeploymentMessage, + ProgramDeploymentTransaction, ProgramId, PublicMessage, PublicTransaction, Signature, + Transaction, ValidityWindow, WitnessSet, }; use jsonrpsee::{ core::{SubscriptionResult, async_trait}, @@ -325,7 +325,7 @@ impl indexer_service_rpc::RpcServer for MockIndexerService { .collect()) } - async fn get_status(&self) -> Result { + async fn get_status(&self) -> Result { let indexed_block_id = self .state .read() @@ -335,12 +335,12 @@ impl indexer_service_rpc::RpcServer for MockIndexerService { .rev() .find(|block| block.bedrock_status == BedrockStatus::Finalized) .map(|block| block.header.block_id); - Ok(serde_json::json!({ - "state": "caught_up", - "lastError": null, - "indexedBlockId": indexed_block_id, - "stallReason": null, - })) + Ok(IndexerStatus { + state: IndexerSyncState::CaughtUp, + last_error: None, + indexed_block_id, + stall_reason: None, + }) } async fn healthcheck(&self) -> Result<(), ErrorObjectOwned> { diff --git a/lez/indexer/service/src/service.rs b/lez/indexer/service/src/service.rs index 0d1d8692..767560b1 100644 --- a/lez/indexer/service/src/service.rs +++ b/lez/indexer/service/src/service.rs @@ -4,7 +4,9 @@ use anyhow::{Context as _, Result, bail}; use arc_swap::ArcSwap; use futures::{StreamExt as _, never::Never}; use indexer_core::{IndexerCore, config::IndexerConfig}; -use indexer_service_protocol::{Account, AccountId, Block, BlockId, HashType, Transaction}; +use indexer_service_protocol::{ + Account, AccountId, Block, BlockId, HashType, IndexerStatus, Transaction, +}; use jsonrpsee::{ SubscriptionSink, core::{Serialize, SubscriptionResult, async_trait}, @@ -20,8 +22,7 @@ pub struct IndexerService { impl IndexerService { pub async fn new(config: IndexerConfig, storage_dir: &Path) -> Result { - let allow_reset = config.allow_chain_reset; - let indexer = IndexerCore::new_with_chain_check(config, storage_dir, allow_reset).await?; + let indexer = IndexerCore::new(config, storage_dir).await?; let subscription_service = SubscriptionService::spawn_new(indexer.clone()); Ok(Self { @@ -150,14 +151,8 @@ impl indexer_service_rpc::RpcServer for IndexerService { Ok(tx_res) } - async fn get_status(&self) -> Result { - serde_json::to_value(self.indexer.status()).map_err(|err| { - ErrorObjectOwned::owned( - ErrorCode::InternalError.code(), - "failed to serialize indexer status".to_owned(), - Some(format!("{err:#}")), - ) - }) + async fn get_status(&self) -> Result { + Ok(self.indexer.status().into()) } async fn healthcheck(&self) -> Result<(), ErrorObjectOwned> { From 39c19912c5c6206d8852a482698c31e44cc6b20d Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 3 Jul 2026 22:36:42 +0300 Subject: [PATCH 23/32] fix(indexer): use better error types, tend to few @Arjentix PR review --- .../docker-all-in-one/indexer_config.json | 3 +- lez/indexer/core/src/block_store.rs | 66 ++++++++------- lez/indexer/core/src/ingest_error.rs | 39 +++++---- lez/indexer/core/src/lib.rs | 83 +++++++++++-------- lez/indexer/core/src/status.rs | 5 +- lez/indexer/service/protocol/src/convert.rs | 11 ++- lez/indexer/service/protocol/src/lib.rs | 9 +- 7 files changed, 136 insertions(+), 80 deletions(-) diff --git a/lez/configs/docker-all-in-one/indexer_config.json b/lez/configs/docker-all-in-one/indexer_config.json index c1ff65b0..5791f64a 100644 --- a/lez/configs/docker-all-in-one/indexer_config.json +++ b/lez/configs/docker-all-in-one/indexer_config.json @@ -3,5 +3,6 @@ "bedrock_config": { "addr": "http://logos-blockchain-node-0:18080" }, - "channel_id": "0101010101010101010101010101010101010101010101010101010101010101" + "channel_id": "0101010101010101010101010101010101010101010101010101010101010101", + "allow_chain_reset": true } diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 3625396c..7269f687 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -186,9 +186,11 @@ impl IndexerStore { })) } - /// Records the stall reason: the first break is stored verbatim; subsequent - /// breaks only bump `orphans_since`, preserving the original cause. - fn record_stall( + /// Record the stall reason. + /// + /// - First stall is stored verbatim + /// - Subsequent stalls only bump `orphans_since`, preserving the original cause. + pub fn record_stall( &self, header: Option<&BlockHeader>, l1_slot: Slot, @@ -200,6 +202,7 @@ impl IndexerStore { existing } None => StallReason { + // need to map out of `header` because they are not ser/de block_id: header.map(|h| h.block_id), block_hash: header.map(|h| h.hash), prev_block_hash: header.map(|h| h.prev_block_hash), @@ -212,11 +215,6 @@ impl IndexerStore { self.set_stall_reason(&Some(stall)) } - /// Records a stall for an inscription that could not even be parsed. - pub fn record_deserialize_stall(&self, l1_slot: Slot, error: String) -> Result<()> { - self.record_stall(None, l1_slot, BlockIngestError::Deserialize(error)) - } - /// Validates `block` against the tip and, if it chains, applies it atomically /// (scratch clone, commit only on full success) and advances the tip. On any /// failure records the stall and returns `Parked` without touching state. @@ -306,25 +304,26 @@ fn validate_against_tip(tip: Option<&Tip>, block: &Block) -> Result<(), BlockIng /// [`BlockIngestError`] so the caller can park rather than crash. Operates on a /// scratch state; the caller commits only on `Ok`. fn apply_block_to_scratch(block: &Block, state: &mut V03State) -> Result<(), BlockIngestError> { - let (clock_tx, user_txs) = - block.body.transactions.split_last().ok_or_else(|| { - BlockIngestError::StateTransition("block has no transactions".to_owned()) - })?; + let (clock_tx, user_txs) = block + .body + .transactions + .split_last() + .ok_or(BlockIngestError::EmptyBlock)?; let expected_clock = LeeTransaction::Public(clock_invocation(block.header.timestamp)); if *clock_tx != expected_clock { - return Err(BlockIngestError::StateTransition( - "last transaction must be the clock invocation for the block timestamp".to_owned(), - )); + return Err(BlockIngestError::InvalidClockTransaction); } let is_genesis = block.header.block_id == GENESIS_BLOCK_ID; - for transaction in user_txs { + for (tx_index, transaction) in user_txs.iter().enumerate() { + let state_transition = |err: anyhow::Error| BlockIngestError::StateTransition { + tx_index: tx_index.try_into().expect("tx index fits in u64"), + reason: format!("{err:#}"), + }; if is_genesis { let LeeTransaction::Public(public_tx) = transaction else { - return Err(BlockIngestError::StateTransition( - "genesis block should contain only public transactions".to_owned(), - )); + return Err(BlockIngestError::NonPublicGenesisTransaction); }; state .transition_from_public_transaction( @@ -332,19 +331,17 @@ fn apply_block_to_scratch(block: &Block, state: &mut V03State) -> Result<(), Blo block.header.block_id, block.header.timestamp, ) - .map_err(|err| BlockIngestError::StateTransition(format!("{err:?}")))?; + .map_err(|err| state_transition(err.into()))?; } else { transaction .clone() .execute_on_state(state, block.header.block_id, block.header.timestamp) - .map_err(|err| BlockIngestError::StateTransition(format!("{err:?}")))?; + .map_err(|err| state_transition(err.into()))?; } } let LeeTransaction::Public(clock_public_tx) = clock_tx else { - return Err(BlockIngestError::StateTransition( - "clock invocation must be a public transaction".to_owned(), - )); + return Err(BlockIngestError::InvalidClockTransaction); }; state .transition_from_public_transaction( @@ -352,7 +349,10 @@ fn apply_block_to_scratch(block: &Block, state: &mut V03State) -> Result<(), Blo block.header.block_id, block.header.timestamp, ) - .map_err(|err| BlockIngestError::StateTransition(format!("{err:?}")))?; + .map_err(|err| BlockIngestError::StateTransition { + tx_index: user_txs.len().try_into().expect("tx index fits in u64"), + reason: format!("{:#}", anyhow::Error::from(err)), + })?; Ok(()) } @@ -376,7 +376,10 @@ mod stall_reason_tests { block_hash: Some(HashType([1_u8; 32])), prev_block_hash: Some(HashType([2_u8; 32])), l1_slot: Slot::from(42), - error: BlockIngestError::StateTransition("boom".to_owned()), + error: BlockIngestError::StateTransition { + tx_index: 0, + reason: "boom".to_owned(), + }, first_seen: Some(99), orphans_since: 3, }; @@ -385,7 +388,10 @@ mod stall_reason_tests { let got = store.get_stall_reason().expect("get").expect("present"); assert_eq!(got.block_id, Some(7)); assert_eq!(got.orphans_since, 3); - assert!(matches!(got.error, BlockIngestError::StateTransition(_))); + assert!(matches!( + got.error, + BlockIngestError::StateTransition { .. } + )); assert_eq!(got.block_hash, Some(HashType([1_u8; 32]))); assert_eq!(got.prev_block_hash, Some(HashType([2_u8; 32]))); assert_eq!(got.l1_slot, Slot::from(42)); @@ -591,7 +597,11 @@ mod accept_tests { let store = IndexerStore::open_db(dir.path()).expect("open store"); store - .record_deserialize_stall(Slot::from(0), "bad bytes".to_owned()) + .record_stall( + None, + Slot::from(0), + BlockIngestError::Deserialize("bad bytes".to_owned()), + ) .expect("record"); let stall = store.get_stall_reason().expect("get").expect("present"); diff --git a/lez/indexer/core/src/ingest_error.rs b/lez/indexer/core/src/ingest_error.rs index 1235120a..bda53497 100644 --- a/lez/indexer/core/src/ingest_error.rs +++ b/lez/indexer/core/src/ingest_error.rs @@ -1,26 +1,43 @@ use common::HashType; use serde::{Deserialize, Serialize}; -/// Why the indexer could not apply an L2 block from the channel. Stored inside a -/// [`crate::stall_reason::StallReason`] and surfaced on the status snapshot. +/// Why the indexer could not apply an L2 block from the channel. +/// +/// Persisted in `RocksDB`, so every variant must have the following +/// traits: `Clone + Serialize + Deserialize`. #[derive(Debug, Clone, Serialize, Deserialize, thiserror::Error)] pub enum BlockIngestError { - #[error("failed to deserialize L2 block: {0}")] + #[error("Failed to deserialize L2 block: {0}")] + /// Here we store the error string that is derived from [`borsh::from_slice`]'s [`Err`]. Deserialize(String), - #[error("unexpected block id: expected {expected}, got {got}")] + #[error("Unexpected block id: expected {expected}, got {got}")] UnexpectedBlockId { expected: u64, got: u64 }, - #[error("broken chain link: expected prev {expected_prev}, got {got_prev}")] + #[error("Broken chain link: expected prev {expected_prev}, got {got_prev}")] BrokenChainLink { expected_prev: HashType, got_prev: HashType, }, - #[error("block hash mismatch: computed {computed}, header {header}")] + #[error("Block hash mismatch: computed {computed}, header {header}")] HashMismatch { computed: HashType, header: HashType, }, - #[error("state transition failed: {0}")] - StateTransition(String), + #[error("Block has no transactions")] + EmptyBlock, + #[error("Last transaction must be the public clock invocation for the block timestamp")] + InvalidClockTransaction, + #[error("Genesis block must contain only public transactions")] + NonPublicGenesisTransaction, + #[error("State transition failed at transaction {tx_index}: {reason}")] + StateTransition { + /// Index of the failing transaction within the block body. + tx_index: u64, + /// Reason string from `lee::Error` to `anyhow::Error` to `{:#}`. + /// + /// This is required because `lee::Error` is not `Clone + Serialize + Deserialize`, so we + /// cannot store it directly. + reason: String, + }, } #[cfg(test)] @@ -47,10 +64,4 @@ mod tests { } )); } - - #[test] - fn display_is_human_readable() { - let err = BlockIngestError::StateTransition("nonce too low".to_owned()); - assert_eq!(err.to_string(), "state transition failed: nonce too low"); - } } diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index d24488ee..7f56ced0 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -2,7 +2,7 @@ use std::{path::Path, sync::Arc}; use anyhow::{Context as _, Result}; use arc_swap::ArcSwap; -use common::block::Block; +use common::{HashType, block::Block}; // TODO: Remove after testnet use futures::StreamExt as _; pub use ingest_error::BlockIngestError; @@ -29,7 +29,25 @@ enum ChainIdentityOutcome { /// or the check was inconclusive — none of which prove a reset. Consistent, /// The channel serves a different block at one of our ids — a chain reset. - Mismatch { detail: String }, + Mismatch(ChainMismatch), +} + +/// The differing pair behind a [`ChainIdentityOutcome::Mismatch`]: our stored +/// block vs. the block the channel serves at the same id, as `(block_id, hash)`. +struct ChainMismatch { + ours: (u64, HashType), + channel: (u64, HashType), +} + +impl std::fmt::Display for ChainMismatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Self { ours, channel } = self; + write!( + f, + "stored block {} {} != channel block {} {}", + ours.0, ours.1, channel.0, channel.1 + ) + } } #[derive(Clone)] @@ -51,18 +69,19 @@ impl IndexerCore { let core = Self::open(config.clone(), storage_dir)?; match core.chain_identity_outcome().await? { ChainIdentityOutcome::Consistent => Ok(core), - ChainIdentityOutcome::Mismatch { detail } if config.allow_chain_reset => { + ChainIdentityOutcome::Mismatch(mismatch) if config.allow_chain_reset => { warn!( - "Chain reset detected ({detail}). Wiping indexer store at {} and re-indexing.", + "Chain reset detected ({mismatch}). Wiping indexer store at {} and \ + re-indexing.", home.display() ); drop(core); // sole owner before the ingest task is spawned → closes the DB storage::indexer::RocksDBIO::destroy(&home)?; Self::open(config, storage_dir) } - ChainIdentityOutcome::Mismatch { detail } => Err(anyhow::anyhow!( + ChainIdentityOutcome::Mismatch(mismatch) => Err(anyhow::anyhow!( "Indexer store at {} holds a different chain than the channel now serves \ - ({detail}). Delete the indexer storage directory, point at a fresh one, or \ + ({mismatch}). Delete the indexer storage directory, point at a fresh one, or \ set `allow_chain_reset` in the indexer config.", home.display() )), @@ -121,14 +140,16 @@ impl IndexerCore { Ok(compare_block(&ours, &channel_block)) } - /// Reads the first block the channel serves at/after the tip's slot. `next_messages` - /// is exclusive, so `cursor - 1` includes the tip's own slot. `None` = inconclusive - /// (timeout/error, or bedrock's LIB behind our tip → empty stream). + /// Reads the first block the channel serves at/after the tip's slot. + /// + /// `None` = inconclusive (timeout/error, or bedrock's LIB behind our tip → empty stream). async fn fetch_channel_block_from(&self, cursor: Slot) -> Result> { const TIP_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); - // Slot-0 cursor is degenerate (inscriptions live at wall-clock slots); bail - // rather than let `next_messages(None)` do a from-genesis scan. + + // `next_messages` is exclusive, so `cursor - 1` includes the tip's own slot. let Some(from_slot) = cursor.into_inner().checked_sub(1) else { + // Slot-0 cursor is degenerate (inscriptions live at wall-clock slots); + // so we bail rather than let `next_messages(None)` do a from-genesis scan. return Ok(None); }; let fetch = async { @@ -192,13 +213,20 @@ impl IndexerCore { /// Parks on an inscription that could not be parsed as an L2 block: /// records the stall and flips the status. The validated tip stays frozen. - fn park_undeserializable(&self, slot: Slot, error: &impl std::fmt::Display) { - error!("Failed to deserialize L2 block from zone-sdk: {error}"); - if let Err(err) = self.store.record_deserialize_stall(slot, error.to_string()) { + fn park_undeserializable(&self, slot: Slot, error: std::io::Error) { + let error = anyhow::Error::new(error); + + // use `:#` to get the entire error chain + let reason = format!("{error:#}"); + error!("Failed to deserialize L2 block from zone-sdk: {reason}"); + if let Err(err) = + self.store + .record_stall(None, slot, BlockIngestError::Deserialize(reason.clone())) + { warn!("Failed to record stall reason: {err:#}"); } self.set_status(IndexerSyncStatus::stalled(format!( - "failed to deserialize L2 block: {error}" + "failed to deserialize L2 block: {reason}" ))); } @@ -250,15 +278,7 @@ impl IndexerCore { let block: Block = match borsh::from_slice(&zone_block.data) { Ok(b) => b, Err(error) => { - error!("Failed to deserialize L2 block from zone-sdk: {error}"); - if let Err(err) = self.store.record_deserialize_stall(slot, error.to_string()) { - warn!("Failed to record stall reason: {err:#}"); - } - self.set_status(IndexerSyncStatus::stalled(format!( - "failed to deserialize L2 block: {error}" - ))); - - + self.park_undeserializable(slot, error); // L1 proceeds regardless self.advance_cursor(&mut cursor, slot); continue; @@ -325,15 +345,10 @@ fn compare_block(ours: &Block, channel: &Block) -> ChainIdentityOutcome { if ours.header.hash == channel.header.hash { ChainIdentityOutcome::Consistent } else { - ChainIdentityOutcome::Mismatch { - detail: format!( - "stored block {} {} != channel block {} {}", - ours.header.block_id, - ours.header.hash, - channel.header.block_id, - channel.header.hash - ), - } + ChainIdentityOutcome::Mismatch(ChainMismatch { + ours: (ours.header.block_id, ours.header.hash), + channel: (channel.header.block_id, channel.header.hash), + }) } } @@ -362,7 +377,7 @@ mod chain_identity_tests { let current = block_with_prev(2); assert!(matches!( compare_block(&stored, ¤t), - ChainIdentityOutcome::Mismatch { .. } + ChainIdentityOutcome::Mismatch(_) )); } } diff --git a/lez/indexer/core/src/status.rs b/lez/indexer/core/src/status.rs index 939abec5..e229d4d8 100644 --- a/lez/indexer/core/src/status.rs +++ b/lez/indexer/core/src/status.rs @@ -129,7 +129,10 @@ mod tests { block_hash: None, prev_block_hash: None, l1_slot: Slot::from(0), - error: BlockIngestError::StateTransition("boom".to_owned()), + error: BlockIngestError::StateTransition { + tx_index: 0, + reason: Default::default(), + }, first_seen: None, orphans_since: 2, }), diff --git a/lez/indexer/service/protocol/src/convert.rs b/lez/indexer/service/protocol/src/convert.rs index f497574d..55c4dc6c 100644 --- a/lez/indexer/service/protocol/src/convert.rs +++ b/lez/indexer/service/protocol/src/convert.rs @@ -745,7 +745,16 @@ impl From for BlockIngestError { header: header.into(), } } - indexer_core::BlockIngestError::StateTransition(msg) => Self::StateTransition(msg), + indexer_core::BlockIngestError::EmptyBlock => Self::EmptyBlock, + indexer_core::BlockIngestError::InvalidClockTransaction => { + Self::InvalidClockTransaction + } + indexer_core::BlockIngestError::NonPublicGenesisTransaction => { + Self::NonPublicGenesisTransaction + } + indexer_core::BlockIngestError::StateTransition { tx_index, reason } => { + Self::StateTransition { tx_index, reason } + } } } } diff --git a/lez/indexer/service/protocol/src/lib.rs b/lez/indexer/service/protocol/src/lib.rs index 6ea06a81..e17d539b 100644 --- a/lez/indexer/service/protocol/src/lib.rs +++ b/lez/indexer/service/protocol/src/lib.rs @@ -397,7 +397,14 @@ pub enum BlockIngestError { computed: HashType, header: HashType, }, - StateTransition(String), + EmptyBlock, + InvalidClockTransaction, + NonPublicGenesisTransaction, + StateTransition { + /// Index of the failing transaction within the block body. + tx_index: u64, + reason: String, + }, } /// Diagnostic record of the first block that broke the L2 chain. From 416eddc73335d6091773d8da86345893ee2cc3c3 Mon Sep 17 00:00:00 2001 From: erhant Date: Mon, 6 Jul 2026 11:23:17 +0300 Subject: [PATCH 24/32] refactor(indexer): tend to several reviews by @schouhy --- lez/indexer/core/src/lib.rs | 179 +++++++++++++---------- lez/indexer/core/src/status.rs | 2 +- lez/storage/src/indexer/indexer_cells.rs | 1 - 3 files changed, 106 insertions(+), 76 deletions(-) diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 7f56ced0..37d7bc7d 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -24,22 +24,32 @@ pub mod stall_reason; pub mod status; /// Result of comparing the indexer's stored tip against the channel. -enum ChainIdentityOutcome { - /// Proceed from the cursor: channel still serves our chain, nothing to compare, - /// or the check was inconclusive — none of which prove a reset. +enum ChainConsistenty { + /// Channel serves the same block at our tip's id. Consistent, - /// The channel serves a different block at one of our ids — a chain reset. - Mismatch(ChainMismatch), + /// We could not determine the outcome due to one of: + /// + /// - cold store + /// - the channel served no comparable block, we hold no block at that id + /// - or the channel read was inconclusive (timeout / error / empty stream) + /// + /// NOTE: None of these prove a reset, so the caller proceeds. + /// A genuine divergence is still caught later when the ingest loop tries to apply and parks. + Inconclusive, + /// The channel serves a different block at one of our ids. + /// + /// Details in [`BlockMismatch`], and impl's Display trait. + Inconsistent(BlockMismatch), } -/// The differing pair behind a [`ChainIdentityOutcome::Mismatch`]: our stored +/// The differing pair behind a [`ChainConsistenty::Inconsistent`]: our stored /// block vs. the block the channel serves at the same id, as `(block_id, hash)`. -struct ChainMismatch { +struct BlockMismatch { ours: (u64, HashType), channel: (u64, HashType), } -impl std::fmt::Display for ChainMismatch { +impl std::fmt::Display for BlockMismatch { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let Self { ours, channel } = self; write!( @@ -61,15 +71,21 @@ pub struct IndexerCore { impl IndexerCore { /// Builds the core, then verifies the stored chain matches the channel's by - /// re-reading the channel at the stored tip's position. On mismatch: refuse - /// (error) unless `config.allow_chain_reset` is set, in which case wipe the + /// re-reading the channel at the stored tip's position. + /// + /// On mismatch: refuse (error) unless `config.allow_chain_reset` is set, in which case wipe the /// store and re-index from scratch. pub async fn new(config: IndexerConfig, storage_dir: &Path) -> Result { let home = storage_dir.join(format!("rocksdb-{}", config.channel_id)); let core = Self::open(config.clone(), storage_dir)?; - match core.chain_identity_outcome().await? { - ChainIdentityOutcome::Consistent => Ok(core), - ChainIdentityOutcome::Mismatch(mismatch) if config.allow_chain_reset => { + match core.verify_chain_consistency().await? { + // `Inconclusive` is deliberately treated the same as `Consistent`. + // + // We could not prove a reset, so proceed from the cursor without wiping + // a possibly-valid store. A genuinely divergent chain is still caught + // later when the ingest loop tries to apply and parks. + ChainConsistenty::Consistent | ChainConsistenty::Inconclusive => Ok(core), + ChainConsistenty::Inconsistent(mismatch) if config.allow_chain_reset => { warn!( "Chain reset detected ({mismatch}). Wiping indexer store at {} and \ re-indexing.", @@ -79,7 +95,7 @@ impl IndexerCore { storage::indexer::RocksDBIO::destroy(&home)?; Self::open(config, storage_dir) } - ChainIdentityOutcome::Mismatch(mismatch) => Err(anyhow::anyhow!( + ChainConsistenty::Inconsistent(mismatch) => Err(anyhow::anyhow!( "Indexer store at {} holds a different chain than the channel now serves \ ({mismatch}). Delete the indexer storage directory, point at a fresh one, or \ set `allow_chain_reset` in the indexer config.", @@ -113,38 +129,45 @@ impl IndexerCore { /// (e.g. a wiped/restarted bedrock) so startup can reset instead of silently /// diverging. Mostly a dev convenience; won't trigger during normal live indexing. /// - /// Compares the first block the channel serves at/after the cursor (the tip's - /// recent L1 slot, so ~one batch — not a from-genesis scan) against our stored - /// block of the *same id*. Comparing by the channel's id, not our tip id, is what - /// catches a *shorter* reset chain: it has no block at our tip id, but its low-id - /// block here differs from ours. - async fn chain_identity_outcome(&self) -> Result { - // Don't skip parked stores: the stall is persisted and only clears on a - // successful apply, so skipping would re-park forever and never catch a reset. - // Safe anyway — a same-chain park sits at an id we never applied, so the lookup - // below misses → Consistent. + /// Compares the **first** block the channel serves at/after the cursor (the tip's + /// recent L1 slot) against our stored block of the *same id*. + /// Note that we have to respect the channel's tip slot, because if our tip + /// is ahead of the channel even in the same chain, it may looks like we are + /// ahead of the channel, but we are actually on a different chain. + async fn verify_chain_consistency(&self) -> Result { let Some(cursor) = self.store.get_zone_cursor()? else { - return Ok(ChainIdentityOutcome::Consistent); // empty / cold store + // if we have no cursor, the store is empty or cold: nothing to compare + return Ok(ChainConsistenty::Inconclusive); }; - // Inconclusive cases stay Consistent (ingest parks if truly divergent) rather - // than wipe a valid store: empty/unreadable read (notably bedrock's LIB behind - // our tip), or an id we don't hold. Blind spot: a genesis-only store can't be - // told apart (genesis is deterministic), but that window is transient. - let Some(channel_block) = self.fetch_channel_block_from(cursor).await? else { - return Ok(ChainIdentityOutcome::Consistent); + let Some(channel) = self.fetch_channel_block_from(cursor).await? else { + // if the channel is empty, we have nothing to compare + return Ok(ChainConsistenty::Inconclusive); }; - let Some(ours) = self.store.get_block_at_id(channel_block.header.block_id)? else { - return Ok(ChainIdentityOutcome::Consistent); + + let Some(ours) = self.store.get_block_at_id(channel.header.block_id)? else { + // if we do not have the block the channel serves at that id, we cannot compare + return Ok(ChainConsistenty::Inconclusive); }; - Ok(compare_block(&ours, &channel_block)) + + // copare the block w.r.t hashes + let comparison_result = if ours.header.hash == channel.header.hash { + ChainConsistenty::Consistent + } else { + ChainConsistenty::Inconsistent(BlockMismatch { + ours: (ours.header.block_id, ours.header.hash), + channel: (channel.header.block_id, channel.header.hash), + }) + }; + + Ok(comparison_result) } /// Reads the first block the channel serves at/after the tip's slot. /// - /// `None` = inconclusive (timeout/error, or bedrock's LIB behind our tip → empty stream). + /// `None` means we could not read a comparable block. async fn fetch_channel_block_from(&self, cursor: Slot) -> Result> { - const TIP_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + const TIP_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); // `next_messages` is exclusive, so `cursor - 1` includes the tip's own slot. let Some(from_slot) = cursor.into_inner().checked_sub(1) else { @@ -167,6 +190,7 @@ impl IndexerCore { } Ok(None) }; + match tokio::time::timeout(TIP_FETCH_TIMEOUT, fetch).await { Ok(Ok(found)) => Ok(found), Ok(Err(err)) => { @@ -187,8 +211,23 @@ impl IndexerCore { #[must_use] pub fn status(&self) -> IndexerStatus { let sync = IndexerSyncStatus::clone(&self.status.load()); - let indexed_block_id = self.store.get_last_block_id().ok().flatten(); - let stall_reason = self.store.get_stall_reason().ok().flatten(); + // Log-and-fall-back rather than collapsing a store error into the same + // `None` as "legitimately absent": a DB read failure must not silently + // masquerade as "no tip yet" / "no stall recorded" in the snapshot. + let indexed_block_id = match self.store.get_last_block_id() { + Ok(id) => id, + Err(err) => { + warn!("Failed to read last indexed block id for status: {err:#}"); + None + } + }; + let stall_reason = match self.store.get_stall_reason() { + Ok(reason) => reason, + Err(err) => { + warn!("Failed to read stall reason for status: {err:#}"); + None + } + }; IndexerStatus { sync, indexed_block_id, @@ -330,8 +369,14 @@ impl IndexerCore { } // Stream drained. Stay Stalled if parked; otherwise we are caught up. - if self.store.get_stall_reason().ok().flatten().is_none() { - self.set_status(IndexerSyncStatus::caught_up()); + // A store error here must not be collapsed to "no stall recorded": + // that would wrongly flip us to caught-up, so we log and hold state. + match self.store.get_stall_reason() { + Ok(None) => self.set_status(IndexerSyncStatus::caught_up()), + Ok(Some(_)) => {} + Err(err) => { + warn!("Failed to read stall reason after draining stream; not marking caught up: {err:#}"); + } } tokio::time::sleep(poll_interval).await; } @@ -339,45 +384,31 @@ impl IndexerCore { } } -/// Pure comparison of our stored block against the channel's block at the same id: -/// a mismatch is differing hashes. Missing/unreadable cases are handled upstream. -fn compare_block(ours: &Block, channel: &Block) -> ChainIdentityOutcome { - if ours.header.hash == channel.header.hash { - ChainIdentityOutcome::Consistent - } else { - ChainIdentityOutcome::Mismatch(ChainMismatch { - ours: (ours.header.block_id, ours.header.hash), - channel: (channel.header.block_id, channel.header.hash), - }) - } -} - #[cfg(test)] mod chain_identity_tests { - use common::{HashType, block::Block, test_utils::produce_dummy_block}; + use std::time::Duration; - use super::{ChainIdentityOutcome, compare_block}; + use super::{ChainConsistenty, IndexerCore}; + use crate::config::{ChannelId, ClientConfig, IndexerConfig}; - fn block_with_prev(prev_seed: u8) -> Block { - produce_dummy_block(5, Some(HashType([prev_seed; 32])), vec![]) - } - - #[test] - fn matching_block_is_consistent() { - let b = block_with_prev(1); + #[tokio::test] + async fn cold_store_is_inconclusive() { + // An empty store has no cursor, so there is nothing to compare: the check + // must be Inconclusive (not Consistent), and it returns before any L1 read. + let dir = tempfile::tempdir().expect("tempdir"); + let config = IndexerConfig { + consensus_info_polling_interval: Duration::from_secs(1), + bedrock_config: ClientConfig { + addr: "http://localhost:1".parse().expect("url"), + auth: None, + }, + channel_id: ChannelId::from([1; 32]), + allow_chain_reset: false, + }; + let core = IndexerCore::open(config, dir.path()).expect("open core"); assert!(matches!( - compare_block(&b, &b), - ChainIdentityOutcome::Consistent - )); - } - - #[test] - fn differing_block_is_mismatch() { - let stored = block_with_prev(1); - let current = block_with_prev(2); - assert!(matches!( - compare_block(&stored, ¤t), - ChainIdentityOutcome::Mismatch(_) + core.verify_chain_consistency().await.expect("verify"), + ChainConsistenty::Inconclusive )); } } diff --git a/lez/indexer/core/src/status.rs b/lez/indexer/core/src/status.rs index e229d4d8..a483fde6 100644 --- a/lez/indexer/core/src/status.rs +++ b/lez/indexer/core/src/status.rs @@ -131,7 +131,7 @@ mod tests { l1_slot: Slot::from(0), error: BlockIngestError::StateTransition { tx_index: 0, - reason: Default::default(), + reason: String::default(), }, first_seen: None, orphans_since: 2, diff --git a/lez/storage/src/indexer/indexer_cells.rs b/lez/storage/src/indexer/indexer_cells.rs index 89e01a43..d71a2ed6 100644 --- a/lez/storage/src/indexer/indexer_cells.rs +++ b/lez/storage/src/indexer/indexer_cells.rs @@ -248,7 +248,6 @@ impl SimpleWritableCell for ZoneSdkIndexerCursorCellRef<'_> { } /// Opaque JSON bytes for the indexer's persisted `Option`. -/// Serialized via `serde_json` by the caller (mirrors the zone-sdk cursor cell). #[derive(BorshDeserialize)] pub struct StallReasonCellOwned(pub Vec); From 2fa67933490e24724cae4ad00419ba28b1567603 Mon Sep 17 00:00:00 2001 From: erhant Date: Mon, 6 Jul 2026 11:32:26 +0300 Subject: [PATCH 25/32] chore: typo fix `ChainConsistency` [skip ci] --- lez/indexer/core/src/lib.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 37d7bc7d..e793c8a1 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -24,7 +24,7 @@ pub mod stall_reason; pub mod status; /// Result of comparing the indexer's stored tip against the channel. -enum ChainConsistenty { +enum ChainConsistency { /// Channel serves the same block at our tip's id. Consistent, /// We could not determine the outcome due to one of: @@ -84,8 +84,8 @@ impl IndexerCore { // We could not prove a reset, so proceed from the cursor without wiping // a possibly-valid store. A genuinely divergent chain is still caught // later when the ingest loop tries to apply and parks. - ChainConsistenty::Consistent | ChainConsistenty::Inconclusive => Ok(core), - ChainConsistenty::Inconsistent(mismatch) if config.allow_chain_reset => { + ChainConsistency::Consistent | ChainConsistency::Inconclusive => Ok(core), + ChainConsistency::Inconsistent(mismatch) if config.allow_chain_reset => { warn!( "Chain reset detected ({mismatch}). Wiping indexer store at {} and \ re-indexing.", @@ -95,7 +95,7 @@ impl IndexerCore { storage::indexer::RocksDBIO::destroy(&home)?; Self::open(config, storage_dir) } - ChainConsistenty::Inconsistent(mismatch) => Err(anyhow::anyhow!( + ChainConsistency::Inconsistent(mismatch) => Err(anyhow::anyhow!( "Indexer store at {} holds a different chain than the channel now serves \ ({mismatch}). Delete the indexer storage directory, point at a fresh one, or \ set `allow_chain_reset` in the indexer config.", @@ -134,27 +134,27 @@ impl IndexerCore { /// Note that we have to respect the channel's tip slot, because if our tip /// is ahead of the channel even in the same chain, it may looks like we are /// ahead of the channel, but we are actually on a different chain. - async fn verify_chain_consistency(&self) -> Result { + async fn verify_chain_consistency(&self) -> Result { let Some(cursor) = self.store.get_zone_cursor()? else { // if we have no cursor, the store is empty or cold: nothing to compare - return Ok(ChainConsistenty::Inconclusive); + return Ok(ChainConsistency::Inconclusive); }; let Some(channel) = self.fetch_channel_block_from(cursor).await? else { // if the channel is empty, we have nothing to compare - return Ok(ChainConsistenty::Inconclusive); + return Ok(ChainConsistency::Inconclusive); }; let Some(ours) = self.store.get_block_at_id(channel.header.block_id)? else { // if we do not have the block the channel serves at that id, we cannot compare - return Ok(ChainConsistenty::Inconclusive); + return Ok(ChainConsistency::Inconclusive); }; // copare the block w.r.t hashes let comparison_result = if ours.header.hash == channel.header.hash { - ChainConsistenty::Consistent + ChainConsistency::Consistent } else { - ChainConsistenty::Inconsistent(BlockMismatch { + ChainConsistency::Inconsistent(BlockMismatch { ours: (ours.header.block_id, ours.header.hash), channel: (channel.header.block_id, channel.header.hash), }) @@ -388,7 +388,7 @@ impl IndexerCore { mod chain_identity_tests { use std::time::Duration; - use super::{ChainConsistenty, IndexerCore}; + use super::{ChainConsistency, IndexerCore}; use crate::config::{ChannelId, ClientConfig, IndexerConfig}; #[tokio::test] @@ -408,7 +408,7 @@ mod chain_identity_tests { let core = IndexerCore::open(config, dir.path()).expect("open core"); assert!(matches!( core.verify_chain_consistency().await.expect("verify"), - ChainConsistenty::Inconclusive + ChainConsistency::Inconclusive )); } } From b2f7fae9720205eb2ea88df403bc804affb0ad5a Mon Sep 17 00:00:00 2001 From: erhant Date: Mon, 6 Jul 2026 14:45:15 +0300 Subject: [PATCH 26/32] fix(indexer): tend to @schouhy reviews, allow re-applied old blocks --- lez/indexer/core/src/block_store.rs | 91 +++++++++++++++++++++++++---- 1 file changed, 81 insertions(+), 10 deletions(-) diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 7269f687..0fd34c36 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -8,6 +8,7 @@ use common::{ }; use lee::{Account, AccountId, GENESIS_BLOCK_ID, V03State}; use lee_core::BlockId; +use log::warn; use logos_blockchain_core::header::HeaderId; use logos_blockchain_zone_sdk::Slot; use storage::indexer::RocksDBIO; @@ -150,6 +151,14 @@ impl IndexerStore { Ok(()) } + /// Clears a recorded stall marker if one is present, skipping the write otherwise. + fn clear_stall_if_present(&self) -> Result<()> { + if self.get_stall_reason()?.is_some() { + self.set_stall_reason(&None)?; + } + Ok(()) + } + /// Recalculation of final state directly from DB. /// /// Used for indexer healthcheck. @@ -221,11 +230,11 @@ impl IndexerStore { pub async fn accept_block(&self, block: &Block, l1_slot: Slot) -> Result { let tip = self.validated_tip()?; - // idempotent edge case: re-delivery of the current tip - // (e.g. after a crash that left the L1 cursor behind the applied tip) + // Re-delivery of an already-applied block is idempotent, not a divergence if let Some(tip) = &tip - && block.header.block_id == tip.block_id - && block.header.hash == tip.hash + && block.header.block_id <= tip.block_id + && let Some(stored) = self.get_block_at_id(block.header.block_id)? + && stored.header.hash == block.header.hash { return Ok(AcceptOutcome::AlreadyApplied); } @@ -250,11 +259,10 @@ impl IndexerStore { // Commit in-memory state (infallible) only after the DB write succeeded. *self.current_state.write().await = scratch; - // Clear a recorded stall now that a block has chained and applied. Only write - // when one is actually present, so the steady state does no extra per-block - // write (and can't turn a transient clear-write error into a spurious park). - if self.get_stall_reason()?.is_some() { - self.set_stall_reason(&None)?; + // Best-effort: the block is durably applied, so a failed stall clear must not + // fail the apply. It self-heals on the next clear. + if let Err(err) = self.clear_stall_if_present() { + warn!("Failed to clear stall marker after applying block: {err:#}"); } Ok(AcceptOutcome::Applied) } @@ -282,7 +290,10 @@ fn validate_against_tip(tip: Option<&Tip>, block: &Block) -> Result<(), BlockIng } } Some(tip) => { - let expected = tip.block_id.checked_add(1).expect("block id overflow"); + let expected = tip + .block_id + .checked_add(1) + .expect("block id should not overflow"); if block.header.block_id != expected { return Err(BlockIngestError::UnexpectedBlockId { expected, @@ -706,4 +717,64 @@ mod accept_tests { "a benign duplicate must not park the indexer" ); } + + #[tokio::test] + async fn redelivered_block_below_tip_is_idempotent_not_parked() { + use testnet_initial_state::initial_pub_accounts_private_keys; + + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path()).expect("open store"); + + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + // Build a short chain: genesis (1) -> block 2 -> block 3, so the tip is 3. + let genesis = produce_dummy_block(1, None, vec![]); + store + .accept_block(&genesis, Slot::from(0)) + .await + .expect("accept genesis"); + + let tx2 = common::test_utils::create_transaction_native_token_transfer( + from, 0, to, 10, &sign_key, + ); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2]); + assert!(matches!( + store.accept_block(&block2, Slot::from(0)).await.unwrap(), + AcceptOutcome::Applied + )); + + let tx3 = common::test_utils::create_transaction_native_token_transfer( + from, 1, to, 10, &sign_key, + ); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![tx3]); + assert!(matches!( + store.accept_block(&block3, Slot::from(0)).await.unwrap(), + AcceptOutcome::Applied + )); + + let balance_after = store.account_current_state(&from).await.unwrap().balance; + + // Re-deliver block 2 (id below the tip): a re-delivery, not a divergence. + assert!(matches!( + store.accept_block(&block2, Slot::from(0)).await.unwrap(), + AcceptOutcome::AlreadyApplied + )); + assert_eq!( + store.account_current_state(&from).await.unwrap().balance, + balance_after, + "re-delivered block below the tip must not be applied again" + ); + assert_eq!( + store.get_last_block_id().unwrap(), + Some(3), + "tip must stay at the current head" + ); + assert!( + store.get_stall_reason().unwrap().is_none(), + "a benign re-delivery must not park the indexer" + ); + } } From 4287e64673d2bfb21fd2964d8880a997021abcd4 Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 7 Jul 2026 10:49:01 +0300 Subject: [PATCH 27/32] chore: fix typos, enable ignored test --- integration_tests/tests/bridge.rs | 1 - lez/indexer/core/src/lib.rs | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/integration_tests/tests/bridge.rs b/integration_tests/tests/bridge.rs index a8fff35e..67c75abc 100644 --- a/integration_tests/tests/bridge.rs +++ b/integration_tests/tests/bridge.rs @@ -561,7 +561,6 @@ fn create_zone_indexer_observer( /// /// TODO: Integration-level park testing (publishing a bad block to force a stall) is a follow-up /// needing fault injection support in the test harness. -#[ignore = "requires the full local stack (bedrock + sequencer + indexer)"] #[test] async fn indexer_status_rpc_reports_caught_up_with_no_stall() -> anyhow::Result<()> { use indexer_service_rpc::RpcClient as _; diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index e793c8a1..029de321 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -42,7 +42,7 @@ enum ChainConsistency { Inconsistent(BlockMismatch), } -/// The differing pair behind a [`ChainConsistenty::Inconsistent`]: our stored +/// The differing pair behind a [`ChainConsistency::Inconsistent`]: our stored /// block vs. the block the channel serves at the same id, as `(block_id, hash)`. struct BlockMismatch { ours: (u64, HashType), @@ -150,7 +150,7 @@ impl IndexerCore { return Ok(ChainConsistency::Inconclusive); }; - // copare the block w.r.t hashes + // compare the block w.r.t hashes let comparison_result = if ours.header.hash == channel.header.hash { ChainConsistency::Consistent } else { From 98af8e33f7780774478929050beb99e27dc610ee Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 7 Jul 2026 11:17:27 +0300 Subject: [PATCH 28/32] chore: docfix w.r.t Copilot --- lez/indexer/core/src/lib.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 029de321..78abf34d 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -125,15 +125,14 @@ impl IndexerCore { }) } - /// Detects when the local store belongs to a different chain than the connected L1 - /// (e.g. a wiped/restarted bedrock) so startup can reset instead of silently - /// diverging. Mostly a dev convenience; won't trigger during normal live indexing. + /// Detects when the local store belongs to a different chain than the connected + /// L1 (e.g. a wiped/restarted bedrock) so startup can reset instead of silently + /// diverging. Best-effort dev convenience; won't trigger during normal live indexing. /// - /// Compares the **first** block the channel serves at/after the cursor (the tip's - /// recent L1 slot) against our stored block of the *same id*. - /// Note that we have to respect the channel's tip slot, because if our tip - /// is ahead of the channel even in the same chain, it may looks like we are - /// ahead of the channel, but we are actually on a different chain. + /// Compares the first block the channel serves at/after the read cursor against our + /// stored block of the same id. Conclusive only while caught up (cursor at the tip); + /// once parked the cursor runs ahead, so the compared id is one we never stored and + /// the result is `Inconclusive` — a genuine reset is then caught by the ingest loop. async fn verify_chain_consistency(&self) -> Result { let Some(cursor) = self.store.get_zone_cursor()? else { // if we have no cursor, the store is empty or cold: nothing to compare From 97720fc35d42885c4e0860f3003f5d19382cd9a8 Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 7 Jul 2026 14:30:10 +0300 Subject: [PATCH 29/32] fix(indexer): allow recovery from parked-and-stalled store (w.r.t Copilot review) refactor(indexer): move chain-consistency code within a dedicated file --- lez/indexer/core/src/chain_consistency.rs | 500 ++++++++++++++++++++++ lez/indexer/core/src/lib.rs | 155 +------ 2 files changed, 508 insertions(+), 147 deletions(-) create mode 100644 lez/indexer/core/src/chain_consistency.rs diff --git a/lez/indexer/core/src/chain_consistency.rs b/lez/indexer/core/src/chain_consistency.rs new file mode 100644 index 00000000..4cfab7f2 --- /dev/null +++ b/lez/indexer/core/src/chain_consistency.rs @@ -0,0 +1,500 @@ +//! Startup check that the local store still belongs to the chain the +//! connected channel serves. + +use anyhow::Result; +use common::{HashType, block::Block}; +use futures::StreamExt as _; +use log::warn; +use logos_blockchain_zone_sdk::{Slot, ZoneMessage, adapter::Node as _}; + +use crate::IndexerCore; + +/// Upper bound on the channel reads of the startup consistency check. +const CHANNEL_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); + +/// Result of comparing the indexer's stored chain against the channel. +pub enum ChainConsistency { + /// Channel still serves our anchor block (the stored tip position, or the + /// parked block while stalled). + Consistent, + /// We could not determine the outcome due to one of: + /// + /// - cold store (no anchor to compare at) + /// - the channel served only blocks newer than the anchor + /// - or the channel read was inconclusive (timeout / error / empty stream) + /// + /// NOTE: None of these prove a reset, so the caller proceeds. + /// A genuine divergence is still caught later when the ingest loop tries to apply and parks. + Inconclusive, + /// Positive evidence that the channel is a different chain than the store. + /// + /// Details in [`ChainMismatch`], and impl's Display trait. + Inconsistent(ChainMismatch), +} + +/// The evidence behind a [`ChainConsistency::Inconsistent`]. +pub enum ChainMismatch { + /// The channel serves a different block at the anchor's id. + Block { + ours: (u64, HashType), + channel: (u64, HashType), + }, + /// The channel serves a block at/below the anchor's id past the anchor + /// slot; on the same chain those ids live at earlier slots. + ReinscribedBlock { + channel: (u64, HashType), + slot: Slot, + anchor_slot: Slot, + }, + /// The channel has content past the anchor slot but no longer the + /// inscription we anchored on. + AnchorSlotChanged { anchor_slot: Slot }, + /// The channel does not exist on the connected chain. + ChannelMissing, + /// The channel's history ends before the anchor slot. + ChannelBehindAnchor { tip_slot: Slot, anchor_slot: Slot }, +} + +impl std::fmt::Display for ChainMismatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Block { ours, channel } => write!( + f, + "stored block {} {} != channel block {} {}", + ours.0, ours.1, channel.0, channel.1 + ), + Self::ReinscribedBlock { + channel, + slot, + anchor_slot, + } => write!( + f, + "channel re-serves block {} {} at slot {} past our anchor slot {}", + channel.0, + channel.1, + slot.into_inner(), + anchor_slot.into_inner() + ), + Self::AnchorSlotChanged { anchor_slot } => write!( + f, + "channel content at slot {} no longer includes the inscription we parked on", + anchor_slot.into_inner() + ), + Self::ChannelMissing => write!(f, "channel does not exist on the connected chain"), + Self::ChannelBehindAnchor { + tip_slot, + anchor_slot, + } => write!( + f, + "channel tip slot {} is behind our anchor slot {}", + tip_slot.into_inner(), + anchor_slot.into_inner() + ), + } + } +} + +/// A block that must still be inscribed at `slot` if the channel is the chain +/// the store was built from: the tip at the read cursor, or the recorded +/// parked block while stalled. +struct Anchor { + slot: Slot, + /// The anchor block's `(id, hash)` + /// + /// `None` when parked on an undeserializable inscription (no header was recorded). + block: Option<(u64, HashType)>, +} + +impl Anchor { + /// Probes a channel message read at/after the anchor slot. + /// See [`IndexerCore::verify_chain_at_anchor`]. + pub fn probe_anchor_slot(&self, msg: &ZoneMessage, slot: Slot) -> AnchorProbe { + if slot < self.slot { + return AnchorProbe::KeepLooking; + } + let Some((anchor_id, anchor_hash)) = self.block else { + // Anchored on an undeserializable inscription: any message still + // present at that slot means the history is intact. + return if slot == self.slot { + AnchorProbe::SameChain + } else { + AnchorProbe::Mismatch(ChainMismatch::AnchorSlotChanged { + anchor_slot: self.slot, + }) + }; + }; + let ZoneMessage::Block(zone_block) = msg else { + return AnchorProbe::KeepLooking; + }; + let Ok(block) = borsh::from_slice::(&zone_block.data) else { + return AnchorProbe::KeepLooking; + }; + let (id, hash) = (block.header.block_id, block.header.hash); + if id == anchor_id { + return if hash == anchor_hash { + AnchorProbe::SameChain + } else { + AnchorProbe::Mismatch(ChainMismatch::Block { + ours: (anchor_id, anchor_hash), + channel: (id, hash), + }) + }; + } + if id > anchor_id { + return AnchorProbe::Bail; + } + if slot == self.slot { + // Older ids can share the anchor's slot on the same chain. + return AnchorProbe::KeepLooking; + } + // An id below the anchor served past the anchor slot is impossible on the + // same chain, even if the content is identical (deterministic genesis). + AnchorProbe::Mismatch(ChainMismatch::ReinscribedBlock { + channel: (id, hash), + slot, + anchor_slot: self.slot, + }) + } +} + +/// What a single channel message tells the anchored consistency check. +enum AnchorProbe { + /// The anchor is still in place: same chain. + SameChain, + Mismatch(ChainMismatch), + /// Only newer ids past the anchor: plausible on the same chain, so stop + /// scanning without a verdict. + Bail, + KeepLooking, +} + +#[expect( + clippy::multiple_inherent_impl, + reason = "split for clarity & isolation of relevant code" +)] +impl IndexerCore { + /// Detects when the local store belongs to a different chain than the connected + /// L1 (e.g. a wiped/restarted bedrock) so startup can reset instead of silently + /// diverging. Best-effort dev convenience; won't trigger during normal live indexing. + /// + /// Anchors on a block the same chain must still serve at a known slot: the + /// recorded parked block while stalled, otherwise the tip at the read + /// cursor (only blocks advance the cursor, so when not parked it is the + /// tip's inscription slot). See [`Self::verify_chain_at_anchor`]. + pub(crate) async fn verify_chain_consistency(&self) -> Result { + let anchor = if let Some(stall) = self.store.get_stall_reason()? { + Anchor { + slot: stall.l1_slot, + block: stall.block_id.zip(stall.block_hash), + } + } else { + let Some(cursor) = self.store.get_zone_cursor()? else { + // if we have no cursor, the store is empty or cold: nothing to compare + return Ok(ChainConsistency::Inconclusive); + }; + let Some(tip_id) = self.store.get_last_block_id()? else { + return Ok(ChainConsistency::Inconclusive); + }; + let Some(tip) = self.store.get_block_at_id(tip_id)? else { + return Ok(ChainConsistency::Inconclusive); + }; + Anchor { + slot: cursor, + block: Some((tip_id, tip.header.hash)), + } + }; + + self.verify_chain_at_anchor(&anchor).await + } + + /// Verifies the channel still carries the anchor block at its slot. + /// + /// The anchor was finalized at `anchor.slot`, so the same chain must still + /// serve it there, while a reset chain re-inscribes its content only at + /// later wall-clock slots. Only positive evidence of a different chain + /// yields `Inconsistent`; absence of data stays `Inconclusive`. + async fn verify_chain_at_anchor(&self, anchor: &Anchor) -> Result { + match self.node.channel_state(self.config.channel_id).await { + Ok(state) => { + if let Some(mismatch) = frontier_verdict(anchor.slot, state.map(|s| s.tip_slot)) { + return Ok(ChainConsistency::Inconsistent(mismatch)); + } + } + Err(err) => { + warn!("Failed to read channel state for the consistency check: {err:#}"); + } + } + + // `next_messages` is exclusive, so `slot - 1` includes the anchor slot. + let Some(from_slot) = anchor.slot.into_inner().checked_sub(1) else { + return Ok(ChainConsistency::Inconclusive); + }; + + let scan = async { + let stream = self + .zone_indexer + .next_messages(Some(Slot::from(from_slot))) + .await?; + let mut stream = std::pin::pin!(stream); + + while let Some((msg, slot)) = stream.next().await { + match anchor.probe_anchor_slot(&msg, slot) { + AnchorProbe::SameChain => return Ok(Some(ChainConsistency::Consistent)), + AnchorProbe::Mismatch(mismatch) => { + return Ok(Some(ChainConsistency::Inconsistent(mismatch))); + } + AnchorProbe::Bail => return Ok(Some(ChainConsistency::Inconclusive)), + AnchorProbe::KeepLooking => { /* dont do anything */ } + } + } + Ok::<_, anyhow::Error>(None) + }; + + match tokio::time::timeout(CHANNEL_READ_TIMEOUT, scan).await { + Ok(Ok(Some(outcome))) => Ok(outcome), + Ok(Ok(None)) => Ok(ChainConsistency::Inconclusive), + Ok(Err(err)) => { + warn!( + "Failed to read the anchor slot for the consistency check; proceeding: {err:#}" + ); + Ok(ChainConsistency::Inconclusive) + } + Err(_elapsed) => { + warn!("Timed out reading the anchor slot for the consistency check; proceeding"); + Ok(ChainConsistency::Inconclusive) + } + } + } +} + +/// Checks the channel frontier against the anchor slot. +/// +/// The anchor block was finalized at `anchor_slot`, so on the same chain the +/// channel tip can never be behind it, and the channel must exist. +fn frontier_verdict(anchor_slot: Slot, channel_tip_slot: Option) -> Option { + match channel_tip_slot { + None => Some(ChainMismatch::ChannelMissing), + Some(tip_slot) if tip_slot < anchor_slot => Some(ChainMismatch::ChannelBehindAnchor { + tip_slot, + anchor_slot, + }), + Some(_) => None, + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use common::block::HashableBlockData; + use logos_blockchain_core::mantle::ops::channel::{MsgId, inscribe::Inscription}; + use logos_blockchain_zone_sdk::ZoneBlock; + + use super::*; + use crate::{ + BlockIngestError, + block_store::AcceptOutcome, + config::{ChannelId, ClientConfig, IndexerConfig}, + }; + + fn unreachable_core(dir: &std::path::Path) -> IndexerCore { + let config = IndexerConfig { + consensus_info_polling_interval: Duration::from_secs(1), + bedrock_config: ClientConfig { + addr: "http://localhost:1".parse().expect("url"), + auth: None, + }, + channel_id: ChannelId::from([1; 32]), + allow_chain_reset: false, + }; + IndexerCore::open(config, dir).expect("open core") + } + + fn test_block(block_id: u64, timestamp: u64) -> Block { + HashableBlockData { + block_id, + prev_block_hash: HashType([0; 32]), + timestamp, + transactions: vec![], + } + .into_pending_block(&lee::PrivateKey::try_new([7; 32]).expect("valid key")) + } + + fn block_msg(block: &Block) -> ZoneMessage { + let bytes = borsh::to_vec(block).expect("serialize"); + ZoneMessage::Block(ZoneBlock { + id: MsgId::from([0_u8; 32]), + data: Inscription::try_from(bytes.as_slice()).expect("inscription"), + }) + } + + fn anchor_for(block: &Block, slot: Slot) -> Anchor { + Anchor { + slot, + block: Some((block.header.block_id, block.header.hash)), + } + } + + #[tokio::test] + async fn cold_store_is_inconclusive() { + // An empty store has no cursor, so there is nothing to compare: the check + // must be Inconclusive (not Consistent), and it returns before any L1 read. + let dir = tempfile::tempdir().expect("tempdir"); + let core = unreachable_core(dir.path()); + assert!(matches!( + core.verify_chain_consistency().await.expect("verify"), + ChainConsistency::Inconclusive + )); + } + + #[tokio::test] + async fn parked_store_with_unreachable_node_is_inconclusive() { + // Network failure is not evidence of a reset: a parked store must stay + // parked (Inconclusive), not error out or trip the wipe path. + let dir = tempfile::tempdir().expect("tempdir"); + let core = unreachable_core(dir.path()); + let parked = test_block(5, 42); + core.store + .record_stall( + Some(&parked.header), + Slot::from(1_000), + BlockIngestError::EmptyBlock, + ) + .expect("record stall"); + assert!(matches!( + core.verify_chain_consistency().await.expect("verify"), + ChainConsistency::Inconclusive + )); + } + + #[tokio::test] + async fn caught_up_store_with_unreachable_node_is_inconclusive() { + let dir = tempfile::tempdir().expect("tempdir"); + let core = unreachable_core(dir.path()); + let genesis = common::test_utils::produce_dummy_block(1, None, vec![]); + assert!(matches!( + core.store + .accept_block(&genesis, Slot::from(1_000)) + .await + .expect("accept"), + AcceptOutcome::Applied + )); + core.store + .set_zone_cursor(&Slot::from(1_000)) + .expect("set cursor"); + assert!(matches!( + core.verify_chain_consistency().await.expect("verify"), + ChainConsistency::Inconclusive + )); + } + + #[test] + fn probe_finds_anchor_block_at_slot() { + let tip = test_block(5, 42); + let anchor = anchor_for(&tip, Slot::from(1_000)); + assert!(matches!( + anchor.probe_anchor_slot(&block_msg(&tip), Slot::from(1_000)), + AnchorProbe::SameChain + )); + } + + #[test] + fn probe_flags_different_block_at_anchor_id() { + let tip = test_block(5, 42); + let anchor = anchor_for(&tip, Slot::from(1_000)); + // Same id, different content (timestamp) => different hash. + let other = test_block(5, 43); + assert!(matches!( + anchor.probe_anchor_slot(&block_msg(&other), Slot::from(1_000)), + AnchorProbe::Mismatch(ChainMismatch::Block { .. }) + )); + } + + #[test] + fn probe_flags_old_id_reinscribed_past_the_anchor_slot() { + // A reset chain re-inscribes from genesis at later slots. Even if the + // content is byte-identical (deterministic genesis), an id at/below + // the anchor past the anchor slot is impossible on the same chain. + let tip = test_block(5, 42); + let anchor = anchor_for(&tip, Slot::from(1_000)); + let genesis = test_block(1, 0); + assert!(matches!( + anchor.probe_anchor_slot(&block_msg(&genesis), Slot::from(1_001)), + AnchorProbe::Mismatch(ChainMismatch::ReinscribedBlock { .. }) + )); + } + + #[test] + fn probe_skips_older_blocks_sharing_the_anchor_slot() { + let tip = test_block(5, 42); + let anchor = anchor_for(&tip, Slot::from(1_000)); + let earlier = test_block(4, 41); + assert!(matches!( + anchor.probe_anchor_slot(&block_msg(&earlier), Slot::from(1_000)), + AnchorProbe::KeepLooking + )); + } + + #[test] + fn probe_bails_on_newer_ids_past_the_anchor() { + // Blocks newer than the anchor are plausible on the same chain (e.g. + // published while we were down), so they must never count as evidence. + let tip = test_block(5, 42); + let anchor = anchor_for(&tip, Slot::from(1_000)); + let newer = test_block(6, 43); + assert!(matches!( + anchor.probe_anchor_slot(&block_msg(&newer), Slot::from(1_001)), + AnchorProbe::Bail + )); + } + + #[test] + fn probe_skips_undeserializable_inscriptions() { + let tip = test_block(5, 42); + let anchor = anchor_for(&tip, Slot::from(1_000)); + let garbage = ZoneMessage::Block(ZoneBlock { + id: MsgId::from([0_u8; 32]), + data: Inscription::try_from(&[1_u8, 2, 3][..]).expect("inscription"), + }); + assert!(matches!( + anchor.probe_anchor_slot(&garbage, Slot::from(1_000)), + AnchorProbe::KeepLooking + )); + } + + #[test] + fn probe_accepts_any_message_for_a_headerless_anchor() { + // A deserialize park records no header: any message still present at + // the anchor slot means the history is intact. + let anchor = Anchor { + slot: Slot::from(1_000), + block: None, + }; + let garbage = ZoneMessage::Block(ZoneBlock { + id: MsgId::from([0_u8; 32]), + data: Inscription::try_from(&[1_u8, 2, 3][..]).expect("inscription"), + }); + assert!(matches!( + anchor.probe_anchor_slot(&garbage, Slot::from(1_000)), + AnchorProbe::SameChain + )); + assert!(matches!( + anchor.probe_anchor_slot(&garbage, Slot::from(1_001)), + AnchorProbe::Mismatch(ChainMismatch::AnchorSlotChanged { .. }) + )); + } + + #[test] + fn frontier_flags_missing_channel_and_short_history() { + assert!(matches!( + frontier_verdict(Slot::from(1_000), None), + Some(ChainMismatch::ChannelMissing) + )); + assert!(matches!( + frontier_verdict(Slot::from(1_000), Some(Slot::from(999))), + Some(ChainMismatch::ChannelBehindAnchor { .. }) + )); + assert!(frontier_verdict(Slot::from(1_000), Some(Slot::from(1_000))).is_none()); + assert!(frontier_verdict(Slot::from(1_000), Some(Slot::from(2_000))).is_none()); + } +} diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 78abf34d..77839296 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -1,8 +1,8 @@ use std::{path::Path, sync::Arc}; -use anyhow::{Context as _, Result}; +use anyhow::Result; use arc_swap::ArcSwap; -use common::{HashType, block::Block}; +use common::block::Block; // TODO: Remove after testnet use futures::StreamExt as _; pub use ingest_error::BlockIngestError; @@ -14,55 +14,22 @@ pub use stall_reason::StallReason; use crate::{ block_store::{AcceptOutcome, IndexerStore}, + chain_consistency::ChainConsistency, config::IndexerConfig, status::{IndexerStatus, IndexerSyncStatus}, }; pub mod block_store; +pub mod chain_consistency; pub mod config; pub mod ingest_error; pub mod stall_reason; pub mod status; -/// Result of comparing the indexer's stored tip against the channel. -enum ChainConsistency { - /// Channel serves the same block at our tip's id. - Consistent, - /// We could not determine the outcome due to one of: - /// - /// - cold store - /// - the channel served no comparable block, we hold no block at that id - /// - or the channel read was inconclusive (timeout / error / empty stream) - /// - /// NOTE: None of these prove a reset, so the caller proceeds. - /// A genuine divergence is still caught later when the ingest loop tries to apply and parks. - Inconclusive, - /// The channel serves a different block at one of our ids. - /// - /// Details in [`BlockMismatch`], and impl's Display trait. - Inconsistent(BlockMismatch), -} - -/// The differing pair behind a [`ChainConsistency::Inconsistent`]: our stored -/// block vs. the block the channel serves at the same id, as `(block_id, hash)`. -struct BlockMismatch { - ours: (u64, HashType), - channel: (u64, HashType), -} - -impl std::fmt::Display for BlockMismatch { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let Self { ours, channel } = self; - write!( - f, - "stored block {} {} != channel block {} {}", - ours.0, ours.1, channel.0, channel.1 - ) - } -} - #[derive(Clone)] pub struct IndexerCore { pub zone_indexer: Arc>, + /// Direct node handle for queries outside `ZoneIndexer`'s streaming API. + pub node: NodeHttpClient, pub config: IndexerConfig, pub store: IndexerStore, /// Live ingestion status; updated by the ingest stream, read by `status`. @@ -115,94 +82,17 @@ impl IndexerCore { CommonHttpClient::new(basic_auth), config.bedrock_config.addr.clone(), ); - let zone_indexer = ZoneIndexer::new(config.channel_id, node); + let zone_indexer = ZoneIndexer::new(config.channel_id, node.clone()); Ok(Self { zone_indexer: Arc::new(zone_indexer), + node, config, store: IndexerStore::open_db(&home)?, status: Arc::new(ArcSwap::from_pointee(IndexerSyncStatus::starting())), }) } - /// Detects when the local store belongs to a different chain than the connected - /// L1 (e.g. a wiped/restarted bedrock) so startup can reset instead of silently - /// diverging. Best-effort dev convenience; won't trigger during normal live indexing. - /// - /// Compares the first block the channel serves at/after the read cursor against our - /// stored block of the same id. Conclusive only while caught up (cursor at the tip); - /// once parked the cursor runs ahead, so the compared id is one we never stored and - /// the result is `Inconclusive` — a genuine reset is then caught by the ingest loop. - async fn verify_chain_consistency(&self) -> Result { - let Some(cursor) = self.store.get_zone_cursor()? else { - // if we have no cursor, the store is empty or cold: nothing to compare - return Ok(ChainConsistency::Inconclusive); - }; - - let Some(channel) = self.fetch_channel_block_from(cursor).await? else { - // if the channel is empty, we have nothing to compare - return Ok(ChainConsistency::Inconclusive); - }; - - let Some(ours) = self.store.get_block_at_id(channel.header.block_id)? else { - // if we do not have the block the channel serves at that id, we cannot compare - return Ok(ChainConsistency::Inconclusive); - }; - - // compare the block w.r.t hashes - let comparison_result = if ours.header.hash == channel.header.hash { - ChainConsistency::Consistent - } else { - ChainConsistency::Inconsistent(BlockMismatch { - ours: (ours.header.block_id, ours.header.hash), - channel: (channel.header.block_id, channel.header.hash), - }) - }; - - Ok(comparison_result) - } - - /// Reads the first block the channel serves at/after the tip's slot. - /// - /// `None` means we could not read a comparable block. - async fn fetch_channel_block_from(&self, cursor: Slot) -> Result> { - const TIP_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); - - // `next_messages` is exclusive, so `cursor - 1` includes the tip's own slot. - let Some(from_slot) = cursor.into_inner().checked_sub(1) else { - // Slot-0 cursor is degenerate (inscriptions live at wall-clock slots); - // so we bail rather than let `next_messages(None)` do a from-genesis scan. - return Ok(None); - }; - let fetch = async { - let stream = self - .zone_indexer - .next_messages(Some(Slot::from(from_slot))) - .await?; - let mut stream = std::pin::pin!(stream); - while let Some((msg, _slot)) = stream.next().await { - if let ZoneMessage::Block(zone_block) = msg { - let block: Block = borsh::from_slice(&zone_block.data) - .context("Failed to deserialize channel block")?; - return Ok::, anyhow::Error>(Some(block)); - } - } - Ok(None) - }; - - match tokio::time::timeout(TIP_FETCH_TIMEOUT, fetch).await { - Ok(Ok(found)) => Ok(found), - Ok(Err(err)) => { - warn!("Failed to read channel tip for the consistency check; proceeding: {err:#}"); - Ok(None) - } - Err(_elapsed) => { - warn!("Timed out reading channel tip for the consistency check; proceeding"); - Ok(None) - } - } - } - /// Snapshot of the current ingestion status (sync state + indexed tip). /// /// Combines the ingest loop's live status with the L2 tip read fresh from the @@ -382,32 +272,3 @@ impl IndexerCore { } } } - -#[cfg(test)] -mod chain_identity_tests { - use std::time::Duration; - - use super::{ChainConsistency, IndexerCore}; - use crate::config::{ChannelId, ClientConfig, IndexerConfig}; - - #[tokio::test] - async fn cold_store_is_inconclusive() { - // An empty store has no cursor, so there is nothing to compare: the check - // must be Inconclusive (not Consistent), and it returns before any L1 read. - let dir = tempfile::tempdir().expect("tempdir"); - let config = IndexerConfig { - consensus_info_polling_interval: Duration::from_secs(1), - bedrock_config: ClientConfig { - addr: "http://localhost:1".parse().expect("url"), - auth: None, - }, - channel_id: ChannelId::from([1; 32]), - allow_chain_reset: false, - }; - let core = IndexerCore::open(config, dir.path()).expect("open core"); - assert!(matches!( - core.verify_chain_consistency().await.expect("verify"), - ChainConsistency::Inconclusive - )); - } -} From e87bad71abeb4c1e0950eb3fa54aaabeda29fb22 Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 7 Jul 2026 17:08:02 +0300 Subject: [PATCH 30/32] chore: clippy docfix + bump crossbeam-epoch (RUSTSEC-2026-0204) --- Cargo.lock | 4 ++-- lez/indexer/core/src/chain_consistency.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d4d3dbc..0bd194bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1854,9 +1854,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] diff --git a/lez/indexer/core/src/chain_consistency.rs b/lez/indexer/core/src/chain_consistency.rs index 4cfab7f2..4ffa244c 100644 --- a/lez/indexer/core/src/chain_consistency.rs +++ b/lez/indexer/core/src/chain_consistency.rs @@ -99,7 +99,7 @@ impl std::fmt::Display for ChainMismatch { /// parked block while stalled. struct Anchor { slot: Slot, - /// The anchor block's `(id, hash)` + /// The anchor block's `(id, hash)`. /// /// `None` when parked on an undeserializable inscription (no header was recorded). block: Option<(u64, HashType)>, From 4d60a1c59c4c9a4186f154fc6b04a166b09ee9ac Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 7 Jul 2026 21:56:00 +0300 Subject: [PATCH 31/32] test(indexer): rename test, move to more suitable file --- integration_tests/tests/bridge.rs | 31 -------------- integration_tests/tests/indexer_stall.rs | 54 ++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 31 deletions(-) create mode 100644 integration_tests/tests/indexer_stall.rs diff --git a/integration_tests/tests/bridge.rs b/integration_tests/tests/bridge.rs index 67c75abc..9c241ebb 100644 --- a/integration_tests/tests/bridge.rs +++ b/integration_tests/tests/bridge.rs @@ -557,37 +557,6 @@ fn create_zone_indexer_observer( )) } -/// Test that the indexer status RPC reports caught-up with no stall after a clean run. -/// -/// TODO: Integration-level park testing (publishing a bad block to force a stall) is a follow-up -/// needing fault injection support in the test harness. -#[test] -async fn indexer_status_rpc_reports_caught_up_with_no_stall() -> anyhow::Result<()> { - use indexer_service_rpc::RpcClient as _; - - let ctx = TestContext::new().await?; - - let indexer_tip = wait_for_indexer_to_catch_up(&ctx).await?; - - let status = ctx.indexer_client().get_status().await?; - assert_eq!( - status.state, - indexer_service_protocol::IndexerSyncState::CaughtUp, - "indexer should be caught up, got {status:?}" - ); - assert!( - status.stall_reason.is_none(), - "indexer should have no stall reason after a clean run, got {status:?}" - ); - assert_eq!( - status.indexed_block_id, - Some(indexer_tip), - "status indexed_block_id should equal the caught-up tip" - ); - - Ok(()) -} - async fn wait_for_finalized_withdraw_op( observer: &ZoneIndexer, expected_amount: u64, diff --git a/integration_tests/tests/indexer_stall.rs b/integration_tests/tests/indexer_stall.rs new file mode 100644 index 00000000..ae1b8b6a --- /dev/null +++ b/integration_tests/tests/indexer_stall.rs @@ -0,0 +1,54 @@ +#![expect( + clippy::tests_outside_test_module, + reason = "We don't care about these in tests" +)] + +use std::time::Duration; + +use anyhow::{Context as _, Result}; +use indexer_service_protocol::IndexerSyncState; +use indexer_service_rpc::RpcClient as _; +use integration_tests::{TestContext, wait_for_indexer_to_catch_up}; +use log::info; + +const CAUGHT_UP_STATUS_TIMEOUT: Duration = Duration::from_secs(60); + +/// Test that the indexer status RPC reports caught-up with no stall after a clean run. +/// +/// The sequencer keeps producing blocks while we assert, so the status is polled until a +/// `CaughtUp` snapshot is observed and the indexed tip is checked as a lower bound. +/// +/// TODO: Integration-level park testing (publishing a bad block to force a stall) is a follow-up +/// needing fault injection support in the test harness. +#[tokio::test] +async fn indexer_status_rpc_reports_caught_up_with_no_stall() -> Result<()> { + let ctx = TestContext::new().await?; + + let indexer_tip = wait_for_indexer_to_catch_up(&ctx).await?; + + let status = tokio::time::timeout(CAUGHT_UP_STATUS_TIMEOUT, async { + loop { + let status = ctx.indexer_client().get_status().await?; + if status.state == IndexerSyncState::CaughtUp { + return anyhow::Ok(status); + } + info!("Waiting for caught-up indexer status, got {status:?}"); + tokio::time::sleep(Duration::from_millis(500)).await; + } + }) + .await + .context("Timed out waiting for indexer status to report caught-up")??; + + assert!( + status.stall_reason.is_none(), + "indexer should have no stall reason after a clean run, got {status:?}" + ); + // test for >= here because the sequencer keeps producing blocks while we assert, + // so the indexed tip may be ahead of the tip we observed when we waited for caught-up. + assert!( + status.indexed_block_id >= Some(indexer_tip), + "status indexed_block_id should be at least the caught-up tip {indexer_tip}, got {status:?}" + ); + + Ok(()) +} From 9b2b4a3d8ca714fa089df634a230d1e0764dcdd7 Mon Sep 17 00:00:00 2001 From: erhant Date: Wed, 8 Jul 2026 14:29:57 +0300 Subject: [PATCH 32/32] fix(indexer): keep track of L1 slot of last inscription for correct tracking --- lez/indexer/core/src/block_store.rs | 49 +++++++++- lez/indexer/core/src/chain_consistency.rs | 106 ++++++++++++++++------ lez/storage/src/indexer/indexer_cells.rs | 23 ++++- lez/storage/src/indexer/mod.rs | 2 + lez/storage/src/indexer/read_once.rs | 7 +- lez/storage/src/indexer/tests.rs | 50 +++++++--- lez/storage/src/indexer/write_atomic.rs | 16 +++- 7 files changed, 202 insertions(+), 51 deletions(-) diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 0fd34c36..2a2bfbce 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -136,6 +136,13 @@ impl IndexerStore { Ok(()) } + /// The L1 inscription slot of the validated tip, written atomically with it + /// by [`Self::accept_block`]. `None` on a cold store or one written before + /// the slot was recorded. + pub fn get_tip_slot(&self) -> Result> { + Ok(self.dbio.get_meta_tip_slot_in_db()?.map(Slot::from)) + } + pub fn get_stall_reason(&self) -> Result> { let Some(bytes) = self.dbio.get_stall_reason_bytes()? else { return Ok(None); @@ -254,7 +261,7 @@ impl IndexerStore { let mut stored = block.clone(); stored.bedrock_status = BedrockStatus::Finalized; self.dbio - .put_block(&stored, [0_u8; 32]) + .put_block(&stored, [0_u8; 32], l1_slot.into_inner()) .context("Failed to persist accepted block")?; // Commit in-memory state (infallible) only after the DB write succeeded. @@ -668,6 +675,46 @@ mod accept_tests { ); } + #[tokio::test] + async fn accept_block_records_tip_inscription_slot() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path()).expect("open store"); + + assert_eq!(store.get_tip_slot().expect("get"), None); + + let genesis = produce_dummy_block(1, None, vec![]); + store + .accept_block(&genesis, Slot::from(1_000)) + .await + .expect("accept"); + assert_eq!(store.get_tip_slot().expect("get"), Some(Slot::from(1_000))); + + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + store + .accept_block(&block2, Slot::from(1_005)) + .await + .expect("accept"); + assert_eq!(store.get_tip_slot().expect("get"), Some(Slot::from(1_005))); + + // A parked block freezes the tip, so its slot must not advance either. + let bad = produce_dummy_block(4, Some(block2.header.hash), vec![]); + assert!(matches!( + store.accept_block(&bad, Slot::from(1_010)).await.unwrap(), + AcceptOutcome::Parked(_) + )); + assert_eq!(store.get_tip_slot().expect("get"), Some(Slot::from(1_005))); + + // Neither must a re-delivered old block move it. + assert!(matches!( + store + .accept_block(&genesis, Slot::from(1_015)) + .await + .unwrap(), + AcceptOutcome::AlreadyApplied + )); + assert_eq!(store.get_tip_slot().expect("get"), Some(Slot::from(1_005))); + } + #[tokio::test] async fn redelivered_tip_block_is_idempotent_not_parked() { use testnet_initial_state::initial_pub_accounts_private_keys; diff --git a/lez/indexer/core/src/chain_consistency.rs b/lez/indexer/core/src/chain_consistency.rs index 4ffa244c..00f28822 100644 --- a/lez/indexer/core/src/chain_consistency.rs +++ b/lez/indexer/core/src/chain_consistency.rs @@ -173,46 +173,59 @@ enum AnchorProbe { reason = "split for clarity & isolation of relevant code" )] impl IndexerCore { - /// Detects when the local store belongs to a different chain than the connected - /// L1 (e.g. a wiped/restarted bedrock) so startup can reset instead of silently - /// diverging. Best-effort dev convenience; won't trigger during normal live indexing. + /// Verifies whether the channel still serves the same chain the store was built from. + /// This may change frequently during development where we reset the chain from time to + /// time in devnet/testnet, but we do not expect [`ChainConsistency::Inconsistent`] in + /// production. /// - /// Anchors on a block the same chain must still serve at a known slot: the - /// recorded parked block while stalled, otherwise the tip at the read - /// cursor (only blocks advance the cursor, so when not parked it is the - /// tip's inscription slot). See [`Self::verify_chain_at_anchor`]. + /// To compare the chains, we use an [`Anchor`] block that is either the parked L2 block + /// while stalled, or the tip L2 block at its own inscription L1 slot. pub(crate) async fn verify_chain_consistency(&self) -> Result { - let anchor = if let Some(stall) = self.store.get_stall_reason()? { - Anchor { - slot: stall.l1_slot, - block: stall.block_id.zip(stall.block_hash), - } - } else { - let Some(cursor) = self.store.get_zone_cursor()? else { - // if we have no cursor, the store is empty or cold: nothing to compare - return Ok(ChainConsistency::Inconclusive); - }; - let Some(tip_id) = self.store.get_last_block_id()? else { - return Ok(ChainConsistency::Inconclusive); - }; - let Some(tip) = self.store.get_block_at_id(tip_id)? else { - return Ok(ChainConsistency::Inconclusive); - }; - Anchor { - slot: cursor, - block: Some((tip_id, tip.header.hash)), - } + let Some(anchor) = self.get_startup_anchor()? else { + // empty or cold store: nothing to compare + return Ok(ChainConsistency::Inconclusive); }; self.verify_chain_at_anchor(&anchor).await } + /// Builds the anchor for the startup check. + /// + /// - If stalled, returns the recorded _parked_ block + /// - If not stalled, returns the validated tip at its _own_ inscription slot. + /// - If the store is empty, returns `None`. + fn get_startup_anchor(&self) -> Result> { + if let Some(stall) = self.store.get_stall_reason()? { + return Ok(Some(Anchor { + slot: stall.l1_slot, + block: stall.block_id.zip(stall.block_hash), + })); + } + + // not stalled, so anchor on the tip at its own inscription slot + let Some(slot) = self.store.get_tip_slot()?.or(self.store.get_zone_cursor()?) else { + return Ok(None); + }; + let Some(tip_id) = self.store.get_last_block_id()? else { + return Ok(None); + }; + let Some(tip) = self.store.get_block_at_id(tip_id)? else { + return Ok(None); + }; + Ok(Some(Anchor { + slot, + block: Some((tip_id, tip.header.hash)), + })) + } + /// Verifies the channel still carries the anchor block at its slot. /// /// The anchor was finalized at `anchor.slot`, so the same chain must still /// serve it there, while a reset chain re-inscribes its content only at - /// later wall-clock slots. Only positive evidence of a different chain - /// yields `Inconsistent`; absence of data stays `Inconclusive`. + /// later wall-clock slots. + /// + /// Only positive evidence of a different chain yields `Inconsistent`. + /// Absence of data stays `Inconclusive`. async fn verify_chain_at_anchor(&self, anchor: &Anchor) -> Result { match self.node.channel_state(self.config.channel_id).await { Ok(state) => { @@ -388,6 +401,41 @@ mod tests { )); } + #[tokio::test] + async fn startup_anchor_prefers_tip_slot_over_lagging_cursor() { + // Cursor persist failures are warn-only, so the read cursor can lag the + // tip by several blocks. The anchor must pair the tip with its own + // inscription slot; pairing it with the stale cursor would make the scan + // misread the chain's intermediate blocks as re-inscriptions. + let dir = tempfile::tempdir().expect("tempdir"); + let core = unreachable_core(dir.path()); + + let genesis = common::test_utils::produce_dummy_block(1, None, vec![]); + core.store + .accept_block(&genesis, Slot::from(1_000)) + .await + .expect("accept"); + let block2 = common::test_utils::produce_dummy_block(2, Some(genesis.header.hash), vec![]); + core.store + .accept_block(&block2, Slot::from(1_005)) + .await + .expect("accept"); + let block3 = common::test_utils::produce_dummy_block(3, Some(block2.header.hash), vec![]); + core.store + .accept_block(&block3, Slot::from(1_010)) + .await + .expect("accept"); + + // Cursor last persisted at the genesis slot: two blocks behind the tip. + core.store + .set_zone_cursor(&Slot::from(1_000)) + .expect("set cursor"); + + let anchor = core.get_startup_anchor().expect("anchor").expect("present"); + assert_eq!(anchor.slot, Slot::from(1_010)); + assert_eq!(anchor.block, Some((3, block3.header.hash))); + } + #[test] fn probe_finds_anchor_block_at_slot() { let tip = test_block(5, 42); diff --git a/lez/storage/src/indexer/indexer_cells.rs b/lez/storage/src/indexer/indexer_cells.rs index d71a2ed6..e00e6faf 100644 --- a/lez/storage/src/indexer/indexer_cells.rs +++ b/lez/storage/src/indexer/indexer_cells.rs @@ -9,7 +9,7 @@ use crate::{ ACC_NUM_CELL_NAME, BLOCK_HASH_CELL_NAME, BREAKPOINT_CELL_NAME, CF_ACC_META, CF_BREAKPOINT_NAME, CF_HASH_TO_ID, CF_TX_TO_ID, DB_META_LAST_BREAKPOINT_ID, DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DB_META_STALL_REASON_KEY, - DB_META_ZONE_SDK_INDEXER_CURSOR_KEY, TX_HASH_CELL_NAME, + DB_META_TIP_SLOT_KEY, DB_META_ZONE_SDK_INDEXER_CURSOR_KEY, TX_HASH_CELL_NAME, }, }; @@ -212,6 +212,27 @@ impl SimpleWritableCell for AccNumTxCell { } } +/// The L1 inscription slot of the tip block, written atomically with the tip. +#[derive(Debug, BorshSerialize, BorshDeserialize)] +pub struct TipSlotCell(pub u64); + +impl SimpleStorableCell for TipSlotCell { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_META_TIP_SLOT_KEY; + const CF_NAME: &'static str = CF_META_NAME; +} + +impl SimpleReadableCell for TipSlotCell {} + +impl SimpleWritableCell for TipSlotCell { + fn value_constructor(&self) -> DbResult> { + borsh::to_vec(&self).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize tip slot".to_owned())) + }) + } +} + /// Opaque bytes for the zone-sdk indexer cursor `Option<(MsgId, Slot)>`. /// The caller serializes via `serde_json` (neither type derives borsh). #[derive(BorshDeserialize)] diff --git a/lez/storage/src/indexer/mod.rs b/lez/storage/src/indexer/mod.rs index 10c80e63..0226bd48 100644 --- a/lez/storage/src/indexer/mod.rs +++ b/lez/storage/src/indexer/mod.rs @@ -26,6 +26,8 @@ pub const DB_META_LAST_BREAKPOINT_ID: &str = "last_breakpoint_id"; pub const DB_META_ZONE_SDK_INDEXER_CURSOR_KEY: &str = "zone_sdk_indexer_cursor"; /// Key base for storing the persisted `Option` diagnostic record (opaque JSON bytes). pub const DB_META_STALL_REASON_KEY: &str = "stall_reason"; +/// Key base for storing the L1 inscription slot of the tip block. +pub const DB_META_TIP_SLOT_KEY: &str = "tip_slot"; /// Cell name for a breakpoint. pub const BREAKPOINT_CELL_NAME: &str = "breakpoint"; diff --git a/lez/storage/src/indexer/read_once.rs b/lez/storage/src/indexer/read_once.rs index 591f8405..83f58d58 100644 --- a/lez/storage/src/indexer/read_once.rs +++ b/lez/storage/src/indexer/read_once.rs @@ -4,7 +4,7 @@ use crate::{ cells::shared_cells::{BlockCell, FirstBlockCell, FirstBlockSetCell, LastBlockCell}, indexer::indexer_cells::{ AccNumTxCell, BlockHashToBlockIdMapCell, BreakpointCellOwned, LastBreakpointIdCell, - LastObservedL1LibHeaderCell, StallReasonCellOwned, TxHashToBlockIdMapCell, + LastObservedL1LibHeaderCell, StallReasonCellOwned, TipSlotCell, TxHashToBlockIdMapCell, ZoneSdkIndexerCursorCellOwned, }, }; @@ -37,6 +37,11 @@ impl RocksDBIO { .map(|opt| opt.map(|cell| cell.0)) } + pub fn get_meta_tip_slot_in_db(&self) -> DbResult> { + self.get_opt::(()) + .map(|opt| opt.map(|cell| cell.0)) + } + // Block pub fn get_block(&self, block_id: u64) -> DbResult> { diff --git a/lez/storage/src/indexer/tests.rs b/lez/storage/src/indexer/tests.rs index 44df23d5..5cd0607b 100644 --- a/lez/storage/src/indexer/tests.rs +++ b/lez/storage/src/indexer/tests.rs @@ -87,7 +87,7 @@ fn one_block_insertion() { let dbio = RocksDBIO::open_or_create(temdir_path, &initial_state()).unwrap(); let genesis_block = genesis_block(); - dbio.put_block(&genesis_block, [0; 32]).unwrap(); + dbio.put_block(&genesis_block, [0; 32], 0).unwrap(); let prev_hash = genesis_block.header.hash; let from = acc1(); @@ -98,7 +98,7 @@ fn one_block_insertion() { common::test_utils::create_transaction_native_token_transfer(from, 0, to, 1, &sign_key); let block = produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(&block, [1; 32]).unwrap(); + dbio.put_block(&block, [1; 32], 0).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let first_id = dbio.get_meta_first_block_id_in_db().unwrap(); @@ -130,6 +130,26 @@ fn one_block_insertion() { ); } +#[test] +fn put_block_records_tip_inscription_slot() { + let temp_dir = tempdir().unwrap(); + let dbio = RocksDBIO::open_or_create(temp_dir.path(), &initial_state()).unwrap(); + + assert_eq!(dbio.get_meta_tip_slot_in_db().unwrap(), None); + + let genesis_block = genesis_block(); + dbio.put_block(&genesis_block, [0; 32], 1_000).unwrap(); + assert_eq!(dbio.get_meta_tip_slot_in_db().unwrap(), Some(1_000)); + + let block = produce_dummy_block(2, Some(genesis_block.header.hash), vec![]); + dbio.put_block(&block, [1; 32], 1_005).unwrap(); + assert_eq!(dbio.get_meta_tip_slot_in_db().unwrap(), Some(1_005)); + + // Re-inserting a block at/below the tip must not move the tip slot. + dbio.put_block(&genesis_block, [0; 32], 1_010).unwrap(); + assert_eq!(dbio.get_meta_tip_slot_in_db().unwrap(), Some(1_005)); +} + #[test] fn new_breakpoint() { let temp_dir = tempdir().unwrap(); @@ -155,7 +175,7 @@ fn new_breakpoint() { &sign_key, ); let block = produce_dummy_block(i.into(), prev_hash, vec![transfer_tx]); - dbio.put_block(&block, [i; 32]).unwrap(); + dbio.put_block(&block, [i; 32], 0).unwrap(); } let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); @@ -211,7 +231,7 @@ fn simple_maps() { let control_hash1 = block.header.hash; - dbio.put_block(&block, [1; 32]).unwrap(); + dbio.put_block(&block, [1; 32], 0).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -223,7 +243,7 @@ fn simple_maps() { let control_hash2 = block.header.hash; - dbio.put_block(&block, [2; 32]).unwrap(); + dbio.put_block(&block, [2; 32], 0).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -235,7 +255,7 @@ fn simple_maps() { let control_tx_hash1 = transfer_tx.hash(); let block = produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(&block, [3; 32]).unwrap(); + dbio.put_block(&block, [3; 32], 0).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -247,7 +267,7 @@ fn simple_maps() { let control_tx_hash2 = transfer_tx.hash(); let block = produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(&block, [4; 32]).unwrap(); + dbio.put_block(&block, [4; 32], 0).unwrap(); let control_block_id1 = dbio.get_block_id_by_hash(control_hash1.0).unwrap().unwrap(); let control_block_id2 = dbio.get_block_id_by_hash(control_hash2.0).unwrap().unwrap(); @@ -284,7 +304,7 @@ fn block_batch() { let block = produce_dummy_block(1, None, vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(&block, [1; 32]).unwrap(); + dbio.put_block(&block, [1; 32], 0).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -295,7 +315,7 @@ fn block_batch() { let block = produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(&block, [2; 32]).unwrap(); + dbio.put_block(&block, [2; 32], 0).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -306,7 +326,7 @@ fn block_batch() { let block = produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(&block, [3; 32]).unwrap(); + dbio.put_block(&block, [3; 32], 0).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -317,7 +337,7 @@ fn block_batch() { let block = produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(&block, [4; 32]).unwrap(); + dbio.put_block(&block, [4; 32], 0).unwrap(); let block_hashes_mem: Vec<[u8; 32]> = block_res.into_iter().map(|bl| bl.header.hash.0).collect(); @@ -376,7 +396,7 @@ fn account_map() { let block = produce_dummy_block(1, None, vec![transfer_tx1, transfer_tx2]); - dbio.put_block(&block, [1; 32]).unwrap(); + dbio.put_block(&block, [1; 32], 0).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -391,7 +411,7 @@ fn account_map() { let block = produce_dummy_block(2, Some(prev_hash), vec![transfer_tx1, transfer_tx2]); - dbio.put_block(&block, [2; 32]).unwrap(); + dbio.put_block(&block, [2; 32], 0).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -406,7 +426,7 @@ fn account_map() { let block = produce_dummy_block(3, Some(prev_hash), vec![transfer_tx1, transfer_tx2]); - dbio.put_block(&block, [3; 32]).unwrap(); + dbio.put_block(&block, [3; 32], 0).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -418,7 +438,7 @@ fn account_map() { let block = produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(&block, [4; 32]).unwrap(); + dbio.put_block(&block, [4; 32], 0).unwrap(); let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 7).unwrap(); let acc1_tx_hashes: Vec<[u8; 32]> = acc1_tx.into_iter().map(|tx| tx.hash().0).collect(); diff --git a/lez/storage/src/indexer/write_atomic.rs b/lez/storage/src/indexer/write_atomic.rs index ec28f8ec..bed1c53b 100644 --- a/lez/storage/src/indexer/write_atomic.rs +++ b/lez/storage/src/indexer/write_atomic.rs @@ -8,7 +8,7 @@ use crate::{ cells::shared_cells::{FirstBlockCell, FirstBlockSetCell, LastBlockCell}, indexer::indexer_cells::{ AccNumTxCell, BlockHashToBlockIdMapCell, LastBreakpointIdCell, LastObservedL1LibHeaderCell, - TxHashToBlockIdMapCell, + TipSlotCell, TxHashToBlockIdMapCell, }, }; @@ -139,13 +139,20 @@ impl RocksDBIO { self.put_batch(&LastBreakpointIdCell(br_id), (), write_batch) } + pub fn put_meta_tip_slot_in_db_batch( + &self, + l1_slot: u64, + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + self.put_batch(&TipSlotCell(l1_slot), (), write_batch) + } + pub fn put_meta_is_first_block_set_batch(&self, write_batch: &mut WriteBatch) -> DbResult<()> { self.put_batch(&FirstBlockSetCell(true), (), write_batch) } - // Block - - pub fn put_block(&self, block: &Block, l1_lib_header: [u8; 32]) -> DbResult<()> { + /// Put a block atomically (via [`WriteBatch`]) along with its L1 header and `Slot`. + pub fn put_block(&self, block: &Block, l1_lib_header: [u8; 32], l1_slot: u64) -> DbResult<()> { let cf_block = self.block_column(); let last_curr_block = self.get_meta_last_block_id_in_db()?.unwrap_or(0); let mut write_batch = WriteBatch::default(); @@ -163,6 +170,7 @@ impl RocksDBIO { if block.header.block_id > last_curr_block { self.put_meta_last_block_in_db_batch(block.header.block_id, &mut write_batch)?; self.put_meta_last_observed_l1_lib_header_in_db_batch(l1_lib_header, &mut write_batch)?; + self.put_meta_tip_slot_in_db_batch(l1_slot, &mut write_batch)?; } if last_curr_block == 0 { self.put_meta_first_block_in_db_batch(block, &mut write_batch)?;