From 8402afd3410bf553a04b7bd24e19224eb9b90434 Mon Sep 17 00:00:00 2001 From: erhant Date: Wed, 8 Jul 2026 16:33:58 +0300 Subject: [PATCH 01/32] feat(chain_state): initial design doc --- lez/chain_state/DESIGN.md | 248 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 lez/chain_state/DESIGN.md diff --git a/lez/chain_state/DESIGN.md b/lez/chain_state/DESIGN.md new file mode 100644 index 00000000..3bb993e3 --- /dev/null +++ b/lez/chain_state/DESIGN.md @@ -0,0 +1,248 @@ +# `lez/chain_state` — Two-Tier Chain State + +Design doc for the shared block-apply engine and two-tier chain state that +backs decentralized sequencing. Status: **interface freeze** — the +`apply_block` signature and the `ChainState` tip/state shape below are the +contract the produce-on-turn and follow-blocks tracks build against. Changing +them after the tracks split forces rework in both. + +Branch: `erhant/lez-two-tip-chain-state` (off `erhant/indexer-recoverable-invalid-blocks`). + +--- + +## 1. Motivation + +Decentralized sequencing requires every honest node — sequencer or indexer — to +converge on the same chain and the same state by running one deterministic +_validate-then-apply_ path over blocks pulled from the channel. That path today +lives only inside the indexer (`lez/indexer/core/src/block_store.rs`), where the +recoverability work built a park-and-skip ingest: `accept_block` validates a +block against the current tip, applies it to a scratch clone of state atomically, +and on any failure records a `StallReason`, freezes the tip, and marks the bad +block _processed_ without applying it. The sequencer has no equivalent — it only +produces blocks and reads peer inscriptions for finalization; it never executes +peer blocks into its own state. + +This crate lifts that logic into a shared home and generalizes it into a +**two-tier** state machine so the sequencer can produce on the head while both +sequencer and indexer expose their exact current state. + +## 2. Crate placement & layering + +``` +lee_core ← lee (owns V03State) ← common (owns Block, BedrockStatus, clock_invocation, recompute_hash) ← lez/chain_state ← { indexer/core, sequencer/core } +``` + +- **Not** `lee_core`: that is the RISC0 guest crate; a stateful host machine has + no place in the zkVM circuit and does not know what a `Block` is. +- **Not** `lee`: the apply logic needs `Block`/`BedrockStatus`/`clock_invocation` + from `common`, and `common` depends on `lee` — putting it in `lee` inverts the + layer. +- `lez/chain_state` sits above `common`, depends on `common` + `lee`, and is + consumed by both `indexer/core` and `sequencer/core`. + +**Persistence boundary (decision: A).** `chain_state` holds the in-memory state +machine and the pure logic; it performs **no I/O**. Each consumer keeps its own +`RocksDBIO` and drives the `scratch → put_block → commit` ordering, exactly as +`accept_block` does today. This keeps the crate fully unit-testable without a DB. +A storage-trait abstraction (option B) is a possible later follow-up, not part of +this task. + +## 3. The `apply_block` entry point + +A single pure function, called identically whether the block was produced by us, +adopted from a peer, or read finalized from the channel: + +```rust +/// Validate `block` against `tip`, then apply it to `state`. Pure: no I/O. +/// Mutates `state` only on success; on failure `state` is untouched and the +/// caller parks. +fn apply_block( + tip: Option<&Tip>, + block: &Block, + state: &mut V03State, +) -> Result<(), BlockIngestError>; +``` + +Validation order (unchanged from the indexer): hash integrity +(`recompute_hash`) → block-id continuity → `prev_block_hash` linkage, with a +`None` tip expecting the genesis block. Application splits off the mandatory +trailing clock tx, executes user txs (genesis = public-only), applies the clock +last. + +Shared types moved into the crate: `AcceptOutcome`, `BlockIngestError`, +`StallReason`, and `Tip`. + +```rust +struct Tip { block_id: u64, hash: HashType, l1_slot: Slot } + +enum AcceptOutcome { Applied, AlreadyApplied, Parked(BlockIngestError) } +``` + +`Tip` carries `l1_slot` (recorded atomically with the tip) because the anchor / +chain-consistency logic keys on the inscription slot, not just `(id, hash)`. + +## 4. The two-tier `ChainState` + +```rust +struct ChainState { + final_state: V03State, // driven by finalized channel ops + final_tip: Option, + head_state: V03State, // final_state + applied head blocks + head_blocks: Vec, // ordered, above final_tip + stall: Option, +} + +struct HeadEntry { this_msg: MsgId, block: Block } +``` + +The **head** tier is a MsgId-keyed chain (adopted/orphaned reference +`this_msg`/`parent_msg`); the **final** tier is block-id-keyed. `apply_block` +validation stays LEZ-level (`block_id` + `prev_block_hash`) — the two chains run +in parallel and must agree, so we validate via `apply_block` _and_ track `MsgId` +for revert correlation. + +Operations: + +- `apply_adopted(inscription) -> AcceptOutcome` — dedup by `this_msg` against our + outbox, else `apply_block` on the head tip; on success push `HeadEntry`; on + failure record stall + park (head tip frozen). +- `apply_channel_update(orphaned, adopted)` — revert every `orphaned` by + `this_msg`, re-derive `head_state` (clone `final_state`, replay survivors), + then apply every `adopted` in order. Atomic per event. +- `finalize_up_to(block_id)` — move `head_blocks` up to `block_id` into + `final_state` (already validated; a move, not a re-apply). +- `apply_finalized(inscription)` — steady state: if present in head by + `this_msg`, `finalize_up_to`; cold-start backfill (not in head): + `apply_block` directly to `final_state`, mirror into head. +- `rollback_orphan(this_msg)` — drop from that entry forward, re-derive head. +- `status() -> { final_height, head_height, stall }` — for RPC/UI. + +For the **indexer** (finalized-only `next_messages` stream), `head_blocks` stays +empty and `head == final`; it exercises only `apply_finalized`. The **sequencer** +uses both tiers from day one. + +## 5. Event → tier mapping + +`Event::BlocksProcessed { checkpoint, channel_update: { orphaned, adopted }, finalized }`: + +| Input | Source | Effect | +| --------------------- | ---------------------------------------------- | --------------------------------------------------- | +| adopted inscription | `channel_update.adopted` | validate + apply to **head** | +| orphaned inscription | `channel_update.orphaned` | revert from **head**, return txs to mempool | +| finalized inscription | `finalized[].ops` (`FinalizedOp::Inscription`) | move head→**final**, or apply directly on backfill | +| own publish | publish-return | optimistically apply to **head**, record `this_msg` | + +**Golden rules:** (1) validation is deterministic, so every honest node makes the +same accept/park decision. (2) An invalid block is _processed but discarded_ — +never applied, never halts the node. (3) Finalized is never reverted. (4) We +rebuild orphaned blocks ourselves; we do not trust the SDK's republish (it keeps +stale LEZ contents — prev-hash, tx selection, and resulting state were all +computed against the old parent). + +--- + +## 6. Scenarios + +### Processing one `BlocksProcessed` event + +```mermaid +flowchart TD + EV["Event::BlocksProcessed"] --> ORPH{"orphaned
non-empty?"} + ORPH -->|yes| REV["For each orphaned by this_msg:
drop from head_blocks,
return its txs to mempool"] + REV --> RED["Re-derive head_state:
clone final_state, replay survivors"] + ORPH -->|no| ADO + RED --> ADO{"adopted
non-empty?"} + ADO -->|"yes, in order"| DEDUP{"this_msg in
our outbox?"} + ADO -->|no| FIN + DEDUP -->|"yes (our own)"| SKIP["skip: already applied optimistically"] + DEDUP -->|no| VAL["apply_block on head tip"] + VAL --> OUT{"AcceptOutcome"} + OUT -->|Applied| APP["append this_msg+block to head,
advance head tip, clear stall"] + OUT -->|AlreadyApplied| SKIP + OUT -->|"Parked(err)"| PARK["record StallReason,
freeze head tip,
mark processed — do NOT apply"] + SKIP --> FIN + APP --> FIN + PARK --> FIN + FIN{"finalized
inscriptions?"} + FIN -->|"already in head (steady state)"| MOVE["finalize_up_to:
move head→final, trim head_blocks"] + FIN -->|"not in head (cold-start backfill)"| DIRECT["apply_block directly to final,
mirror into head"] + FIN -->|none| CP + MOVE --> CP + DIRECT --> CP + CP["persist checkpoint atomically"] +``` + +### Park / recovery status + +```mermaid +stateDiagram-v2 + [*] --> Syncing + Syncing --> CaughtUp: stream drained, no stall + CaughtUp --> Syncing: new adopted / finalized arrives + Syncing --> Parked: apply_block returns Parked(err) + CaughtUp --> Parked: invalid block adopted + Parked --> Parked: further non-chaining blocks (orphans_since++) + Parked --> Syncing: valid successor chains on frozen tip → stall cleared + note right of Parked + head tip frozen at last valid block. + Producer (on our turn) builds on this + frozen tip, same as honest peers, + so the invalid block is skipped. + end note +``` + +### Scenario table + +**Normal flow** + +| # | Scenario | Handling | Expected | +| --- | ---------------------------------------- | ------------------------------------------------------------- | ----------------------------------- | +| 1 | Adopted block chains cleanly on head tip | `apply_block` → `Applied`; append `{this_msg, block}` to head | head advances; converges with peers | +| 2 | Our own block comes back in `adopted` | dedup by `this_msg` against outbox → skip | no double-apply | +| 3 | Adopted block later finalizes | `finalize_up_to` moves head→final, trims `head_blocks` | final advances; no re-apply | +| 4 | Re-delivery of an already-applied block | id ≤ tip & stored hash matches → `AlreadyApplied` | idempotent, no state change | + +**Reorg / orphan** + +| # | Scenario | Handling | Expected | +| --- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------- | ----------------------------------------------- | +| 5 | Our block orphaned at turn handoff (stale-parent race) | revert by `this_msg`, return txs to mempool, **rebuild** on new head tip (not SDK republish) | our txs re-queued; next block on correct parent | +| 6 | Batch reorg: some `orphaned` + some `adopted` in one event | revert all orphaned, re-derive head, then apply all adopted in order | deterministic convergence | +| 7 | Orphan chain (parent transitively off canonical) | SDK surfaces all affected as `orphaned`; revert each, replay survivors | head_state matches new canonical branch | + +**Invalid / bad block** + +| # | Scenario | Handling | Expected | +| --- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | ----------------------------------------------- | +| 8 | Authorized sequencer posts a block with an invalid state transition | `apply_block` → `Parked(StateTransition)`; freeze head tip, record stall, mark processed | park-and-skip; no apply, no halt | +| 9 | Broken chain link / hash mismatch / unexpected id in adopted | `Parked(BrokenChainLink / HashMismatch / UnexpectedBlockId)`; same park | frozen tip; peers park identically | +| 10 | Undeserializable inscription payload | park with `Deserialize` (no header); processing advances | recover when a valid block chains on frozen tip | +| 11 | Valid successor after a park (recovery) | block chaining on frozen tip → `Applied` → clear stall | head resumes automatically; no divergence | +| 12 | Further non-chaining blocks while parked | keep first `StallReason`, bump `orphans_since` | original cause preserved; still parked | + +**Producing while parked** + +| # | Scenario | Handling | Expected | +| --- | ----------------------------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| 13 | It's our turn but head is parked on a bad block | producer builds on the **frozen valid tip** (head tip = last valid), skipping the invalid block | we emit the next valid block on the same parent honest peers use — chain moves on our turn | + +**Startup / backfill** + +| # | Scenario | Handling | Expected | +| --- | --------------------------------------------------- | -------------------------------------------------------------------------------------------------- | ------------------------------------- | +| 14 | Cold start / reconnect backfill | history via `finalized`, empty `channel_update`; apply directly to final + mirror head | head == final until live deltas start | +| 15 | Local store belongs to a different chain (L1 reset) | anchor-based `chain_consistency` check at startup: wipe+reindex if `allow_chain_reset`, else error | no silent divergence | + +## 7. Invariants + +Should-never-happen conditions — assert/log, don't silently absorb: + +- An `orphaned` entry never references a block at or below the **final** tip — + finalized is irreversible. If seen, it is a bug. +- `head` tip ≥ `final` tip at all times; `head_blocks` holds exactly the blocks + between them. +- After processing any event, `head_state == final_state` replayed through + `head_blocks` (the re-derivation is the source of truth). +- A parked node's frozen tip is identical across all honest nodes for the same + invalid block (deterministic validation). From 0838a150bf2418478eba2e25cd49319beb6333bd Mon Sep 17 00:00:00 2001 From: erhant Date: Wed, 8 Jul 2026 17:00:06 +0300 Subject: [PATCH 02/32] chore(chain_state): update DESIGN.md --- lez/chain_state/DESIGN.md | 103 ++++++++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 27 deletions(-) diff --git a/lez/chain_state/DESIGN.md b/lez/chain_state/DESIGN.md index 3bb993e3..3e0bc384 100644 --- a/lez/chain_state/DESIGN.md +++ b/lez/chain_state/DESIGN.md @@ -33,20 +33,16 @@ sequencer and indexer expose their exact current state. lee_core ← lee (owns V03State) ← common (owns Block, BedrockStatus, clock_invocation, recompute_hash) ← lez/chain_state ← { indexer/core, sequencer/core } ``` -- **Not** `lee_core`: that is the RISC0 guest crate; a stateful host machine has - no place in the zkVM circuit and does not know what a `Block` is. - **Not** `lee`: the apply logic needs `Block`/`BedrockStatus`/`clock_invocation` from `common`, and `common` depends on `lee` — putting it in `lee` inverts the layer. - `lez/chain_state` sits above `common`, depends on `common` + `lee`, and is consumed by both `indexer/core` and `sequencer/core`. -**Persistence boundary (decision: A).** `chain_state` holds the in-memory state -machine and the pure logic; it performs **no I/O**. Each consumer keeps its own -`RocksDBIO` and drives the `scratch → put_block → commit` ordering, exactly as -`accept_block` does today. This keeps the crate fully unit-testable without a DB. -A storage-trait abstraction (option B) is a possible later follow-up, not part of -this task. +**Persistence boundary.** `chain_state` holds the in-memory state machine and the +pure logic; it performs **no I/O**. Each consumer keeps its own `RocksDBIO` and +drives the `scratch → put_block → commit` ordering, exactly as `accept_block` does +today. This keeps the crate fully unit-testable without a DB. ## 3. The `apply_block` entry point @@ -90,7 +86,8 @@ struct ChainState { final_tip: Option, head_state: V03State, // final_state + applied head blocks head_blocks: Vec, // ordered, above final_tip - stall: Option, + final_stall: Option, // persisted to RocksDB — see §4a + head_stall: Option, // in-memory only — recomputed from the stream on restart } struct HeadEntry { this_msg: MsgId, block: Block } @@ -105,8 +102,9 @@ for revert correlation. Operations: - `apply_adopted(inscription) -> AcceptOutcome` — dedup by `this_msg` against our - outbox, else `apply_block` on the head tip; on success push `HeadEntry`; on - failure record stall + park (head tip frozen). + outbox, else `apply_block` on the head tip; on success push `HeadEntry`. On + failure, set **`head_stall`** (in-memory `StallReason`) and freeze the head tip + at the last valid block — do **not** persist. - `apply_channel_update(orphaned, adopted)` — revert every `orphaned` by `this_msg`, re-derive `head_state` (clone `final_state`, replay survivors), then apply every `adopted` in order. Atomic per event. @@ -114,14 +112,59 @@ Operations: `final_state` (already validated; a move, not a re-apply). - `apply_finalized(inscription)` — steady state: if present in head by `this_msg`, `finalize_up_to`; cold-start backfill (not in head): - `apply_block` directly to `final_state`, mirror into head. -- `rollback_orphan(this_msg)` — drop from that entry forward, re-derive head. -- `status() -> { final_height, head_height, stall }` — for RPC/UI. + `apply_block` directly to `final_state`, mirror into head. If a finalized block + fails to apply, set **`final_stall`** and persist it — this is the **only** + `StallReason` written to disk (see §4a). +- `rollback_orphan(this_msg)` — drop from that entry forward, re-derive head; + clears `head_stall` if the re-derived head is clean. +- `status() -> { final_height, head_height, head_stall, final_stall }` — for RPC/UI. For the **indexer** (finalized-only `next_messages` stream), `head_blocks` stays empty and `head == final`; it exercises only `apply_finalized`. The **sequencer** uses both tiers from day one. +### 4a. Two stalls — persisted vs in-memory + +Both tiers carry a `StallReason` (`final_stall`, `head_stall`); they are equally +informative. The difference is **persistence**, which follows from durability. L1 +finality is about inscription **canonicality**, not LEZ-block **content** +validity, so an authorized sequencer can get a content-invalid block finalized — +which is why both tiers can meet an invalid block in the first place. + +- **`head_stall` — in-memory only.** The head is reorg-able and re-derived from + every `channel_update`. An invalid adopted block is transient: it is either + orphaned (a competing valid block at the same height wins) or it finalizes. We + set `head_stall` for observability (RPC/UI can show "head blocked at `N`: + StateTransition at tx 3") and freeze the head tip at the last valid block, but + we do **not** write it to disk — on restart the head is rebuilt from the stream, + so a persisted head stall would be redundant and could go stale. Because the + next adopted block chains on the bad one, it fails validation too, so the head + stays frozen until a valid block (competing or post-reorg) is adopted. +- **`final_stall` — persisted.** The final tier is irreversible. If an invalid + block *finalizes*, the node is durably stuck until a valid successor (built by + honest sequencers on the last valid parent) finalizes. This must survive + restart: the startup chain-consistency / anchor check reads it, it is what we + surface as `Stalled`, and it is the signal the committee acts on to evict a bad + sequencer. + +An invalid block **migrates the problem head→final** when it finalizes: `head_stall` +is set at `N` first; once `N+1(bad)` finalizes, `apply_finalized` fails and records +the persisted `final_stall`. Both tiers end up stuck at `N` consistently, and both +recover when `N+1′` finalizes. The indexer (final tier only) never sets +`head_stall`, so it behaves exactly as it does today. + +### 4b. Producer contract — write on turn, build on last valid + +The sequencer publishes **only on its own turn** (the SDK queues out-of-turn +publishes). When it is our turn we build the next block on the **current head +tip**, which is by construction the last validly-applied block. So if the head is +frozen (`head_stall` set) on a peer's bad block, we build on that frozen valid +tip — the same parent every honest sequencer chooses — and thereby skip the bad +block rather than extend it. A parked node keeps following peers' valid blocks as +they arrive; the moment it also gets a turn, it produces the next valid block on +its last valid tip. Net: parking never stops us from producing correctly on our +turn. + ## 5. Event → tier mapping `Event::BlocksProcessed { checkpoint, channel_update: { orphaned, adopted }, finalized }`: @@ -160,16 +203,20 @@ flowchart TD VAL --> OUT{"AcceptOutcome"} OUT -->|Applied| APP["append this_msg+block to head,
advance head tip, clear stall"] OUT -->|AlreadyApplied| SKIP - OUT -->|"Parked(err)"| PARK["record StallReason,
freeze head tip,
mark processed — do NOT apply"] + OUT -->|"Parked(err)"| PARK["set head_stall (in-memory),
freeze head tip — do NOT apply.
Not persisted"] SKIP --> FIN APP --> FIN PARK --> FIN FIN{"finalized
inscriptions?"} FIN -->|"already in head (steady state)"| MOVE["finalize_up_to:
move head→final, trim head_blocks"] - FIN -->|"not in head (cold-start backfill)"| DIRECT["apply_block directly to final,
mirror into head"] + FIN -->|"not in head (cold-start backfill)"| DIRECT["apply_block directly to final"] + DIRECT --> DOK{"applied?"} + DOK -->|yes| MIRROR["mirror into head"] + DOK -->|"no (invalid finalized)"| FSTALL["record StallReason on FINAL,
freeze final tip"] FIN -->|none| CP MOVE --> CP - DIRECT --> CP + MIRROR --> CP + FSTALL --> CP CP["persist checkpoint atomically"] ``` @@ -180,15 +227,15 @@ stateDiagram-v2 [*] --> Syncing Syncing --> CaughtUp: stream drained, no stall CaughtUp --> Syncing: new adopted / finalized arrives - Syncing --> Parked: apply_block returns Parked(err) - CaughtUp --> Parked: invalid block adopted - Parked --> Parked: further non-chaining blocks (orphans_since++) - Parked --> Syncing: valid successor chains on frozen tip → stall cleared + Syncing --> Parked: invalid block FINALIZED (apply_finalized fails) + Parked --> Parked: further non-chaining finalized blocks (orphans_since++) + Parked --> Syncing: valid successor finalizes on frozen final tip → stall cleared note right of Parked - head tip frozen at last valid block. - Producer (on our turn) builds on this - frozen tip, same as honest peers, - so the invalid block is skipped. + final_stall — persisted, survives restart. + Head-tier bad blocks do NOT enter this state: + they set head_stall (in-memory) and self-heal + via reorg/finalization. Producer (on our turn) + builds on the last valid tip either way. end note ``` @@ -211,11 +258,13 @@ stateDiagram-v2 | 6 | Batch reorg: some `orphaned` + some `adopted` in one event | revert all orphaned, re-derive head, then apply all adopted in order | deterministic convergence | | 7 | Orphan chain (parent transitively off canonical) | SDK surfaces all affected as `orphaned`; revert each, replay survivors | head_state matches new canonical branch | -**Invalid / bad block** +**Invalid / bad block** — "stall" below means the **persisted `final_stall`** +(§4a). A bad block seen only in `adopted` sets the in-memory `head_stall` (not +persisted); it becomes a persisted `final_stall` only if it finalizes. | # | Scenario | Handling | Expected | | --- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | ----------------------------------------------- | -| 8 | Authorized sequencer posts a block with an invalid state transition | `apply_block` → `Parked(StateTransition)`; freeze head tip, record stall, mark processed | park-and-skip; no apply, no halt | +| 8 | Authorized sequencer posts a block with an invalid state transition | head: `apply_block` → `Parked`, set `head_stall` (in-memory), no persist. If it finalizes: persisted `final_stall` | park-and-skip; no apply, no halt | | 9 | Broken chain link / hash mismatch / unexpected id in adopted | `Parked(BrokenChainLink / HashMismatch / UnexpectedBlockId)`; same park | frozen tip; peers park identically | | 10 | Undeserializable inscription payload | park with `Deserialize` (no header); processing advances | recover when a valid block chains on frozen tip | | 11 | Valid successor after a park (recovery) | block chaining on frozen tip → `Applied` → clear stall | head resumes automatically; no divergence | From 41eeef1c3bd7333159301f947e33a2182acff40c Mon Sep 17 00:00:00 2001 From: erhant Date: Wed, 8 Jul 2026 17:40:20 +0300 Subject: [PATCH 03/32] feat(chain_state): initial preps, one more design fix --- Cargo.lock | 14 ++ Cargo.toml | 2 + lez/chain_state/Cargo.toml | 21 +++ lez/chain_state/DESIGN.md | 99 +++++----- lez/chain_state/src/apply.rs | 280 ++++++++++++++++++++++++++++ lez/chain_state/src/ingest_error.rs | 67 +++++++ lez/chain_state/src/lib.rs | 17 ++ lez/chain_state/src/stall_reason.rs | 25 +++ 8 files changed, 480 insertions(+), 45 deletions(-) create mode 100644 lez/chain_state/Cargo.toml create mode 100644 lez/chain_state/src/apply.rs create mode 100644 lez/chain_state/src/ingest_error.rs create mode 100644 lez/chain_state/src/lib.rs create mode 100644 lez/chain_state/src/stall_reason.rs diff --git a/Cargo.lock b/Cargo.lock index d6afde36..7f9b2a5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1368,6 +1368,20 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chain_state" +version = "0.1.0" +dependencies = [ + "anyhow", + "common", + "lee", + "logos-blockchain-zone-sdk", + "serde", + "serde_json", + "testnet_initial_state", + "thiserror 2.0.18", +] + [[package]] name = "chkstk_stub" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 164b64cc..11956920 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "lez", "lez/system_accounts", + "lez/chain_state", "lez/sequencer/core", "lez/sequencer/service", "lez/sequencer/service/protocol", @@ -72,6 +73,7 @@ members = [ lee = { path = "lee/state_machine" } lee_core = { path = "lee/state_machine/core" } common = { path = "lez/common" } +chain_state = { path = "lez/chain_state" } mempool = { path = "lez/mempool" } storage = { path = "lez/storage" } key_protocol = { path = "lee/key_protocol" } diff --git a/lez/chain_state/Cargo.toml b/lez/chain_state/Cargo.toml new file mode 100644 index 00000000..337367cc --- /dev/null +++ b/lez/chain_state/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "chain_state" +version = "0.1.0" +edition = "2024" +license = { workspace = true } + +[lints] +workspace = true + +[dependencies] +common.workspace = true +lee.workspace = true +logos-blockchain-zone-sdk.workspace = true + +anyhow.workspace = true +serde.workspace = true +thiserror.workspace = true + +[dev-dependencies] +testnet_initial_state.workspace = true +serde_json.workspace = true diff --git a/lez/chain_state/DESIGN.md b/lez/chain_state/DESIGN.md index 3e0bc384..600fb23f 100644 --- a/lez/chain_state/DESIGN.md +++ b/lez/chain_state/DESIGN.md @@ -86,8 +86,7 @@ struct ChainState { final_tip: Option, head_state: V03State, // final_state + applied head blocks head_blocks: Vec, // ordered, above final_tip - final_stall: Option, // persisted to RocksDB — see §4a - head_stall: Option, // in-memory only — recomputed from the stream on restart + final_stall: Option, // the one stall — persisted to RocksDB. See §4a } struct HeadEntry { this_msg: MsgId, block: Block } @@ -103,8 +102,8 @@ Operations: - `apply_adopted(inscription) -> AcceptOutcome` — dedup by `this_msg` against our outbox, else `apply_block` on the head tip; on success push `HeadEntry`. On - failure, set **`head_stall`** (in-memory `StallReason`) and freeze the head tip - at the last valid block — do **not** persist. + failure, **do nothing durable**: the head tip simply stays at the last valid + block. No stall is recorded (see §4a) — the head self-heals from the stream. - `apply_channel_update(orphaned, adopted)` — revert every `orphaned` by `this_msg`, re-derive `head_state` (clone `final_state`, replay survivors), then apply every `adopted` in order. Atomic per event. @@ -113,52 +112,62 @@ Operations: - `apply_finalized(inscription)` — steady state: if present in head by `this_msg`, `finalize_up_to`; cold-start backfill (not in head): `apply_block` directly to `final_state`, mirror into head. If a finalized block - fails to apply, set **`final_stall`** and persist it — this is the **only** - `StallReason` written to disk (see §4a). -- `rollback_orphan(this_msg)` — drop from that entry forward, re-derive head; - clears `head_stall` if the re-derived head is clean. -- `status() -> { final_height, head_height, head_stall, final_stall }` — for RPC/UI. + fails to apply, set **`final_stall`** and persist it — this is the **only** stall + (see §4a). +- `rollback_orphan(this_msg)` — drop from that entry forward, re-derive head. +- `status() -> { final_height, head_height, final_stall }` — for RPC/UI. A derived + "head blocked" indicator can be computed on demand (see §4a) without persisting. For the **indexer** (finalized-only `next_messages` stream), `head_blocks` stays empty and `head == final`; it exercises only `apply_finalized`. The **sequencer** uses both tiers from day one. -### 4a. Two stalls — persisted vs in-memory +### 4a. One stall — `final_stall`, persisted -Both tiers carry a `StallReason` (`final_stall`, `head_stall`); they are equally -informative. The difference is **persistence**, which follows from durability. L1 -finality is about inscription **canonicality**, not LEZ-block **content** -validity, so an authorized sequencer can get a content-invalid block finalized — -which is why both tiers can meet an invalid block in the first place. +There is a single stall, `final_stall`, on the final tier. The head tier does +**not** carry its own stall, and this is deliberate. -- **`head_stall` — in-memory only.** The head is reorg-able and re-derived from - every `channel_update`. An invalid adopted block is transient: it is either - orphaned (a competing valid block at the same height wins) or it finalizes. We - set `head_stall` for observability (RPC/UI can show "head blocked at `N`: - StateTransition at tx 3") and freeze the head tip at the last valid block, but - we do **not** write it to disk — on restart the head is rebuilt from the stream, - so a persisted head stall would be redundant and could go stale. Because the - next adopted block chains on the bad one, it fails validation too, so the head - stays frozen until a valid block (competing or post-reorg) is adopted. -- **`final_stall` — persisted.** The final tier is irreversible. If an invalid - block *finalizes*, the node is durably stuck until a valid successor (built by - honest sequencers on the last valid parent) finalizes. This must survive - restart: the startup chain-consistency / anchor check reads it, it is what we - surface as `Stalled`, and it is the signal the committee acts on to evict a bad - sequencer. +The head and the final tier never represent two independent problems: a block +always reaches the head first (as `adopted`) and only later the final tier (as +`finalized`), so a would-be "head stall" is just the earlier, provisional sighting +of the exact block that `final_stall` records durably if it finalizes — the same +event modeled twice. -An invalid block **migrates the problem head→final** when it finalizes: `head_stall` -is set at `N` first; once `N+1(bad)` finalizes, `apply_finalized` fails and records -the persisted `final_stall`. Both tiers end up stuck at `N` consistently, and both -recover when `N+1′` finalizes. The indexer (final tier only) never sets -`head_stall`, so it behaves exactly as it does today. +And the head does not need a recorded reason to freeze. **The tip-freeze is +intrinsic**: not applying a bad block is what freezes the tip; no marker is +required. The head's freeze is also **transient and self-healing** — the bad block +is either orphaned (a competing valid block at the same height wins and applies on +its own) or it finalizes. Subsequent adopted blocks that chain on the bad one fail +validation by themselves, and the producer builds on the head tip regardless of +any marker. So a persisted head stall would be redundant (re-derived from the +stream on restart) and buys no behavior. + +`final_stall` is the stall that does real work: + +- The **indexer already requires it** and ships it today. The indexer has only a + final tier (finalized-only stream, no head); its startup chain-consistency / + anchor check reads the persisted stall to know where it is parked. The shared + `final_stall` serves that unchanged. +- It **survives restart** and is what we surface as `Stalled`. +- "A bad block **finalized**" is the only irreversible, actionable condition — the + signal the committee acts on to evict a bad sequencer. A provisional head block + that may vanish on the next reorg is not something to evict over. + +The one thing we forgo is an *early warning* that a sequencer is posting garbage +before it finalizes. That condition frequently self-heals via reorg, so alarming on +it is mostly noise; if wanted, it is a **derived, non-persisted** indicator (e.g. +"k adopted inscriptions above the head tip remain unapplied"), computed on demand — +not a second `StallReason` in the struct. + +So the sequencer and the indexer share exactly one stall concept, keeping the two +consumers uniform. ### 4b. Producer contract — write on turn, build on last valid The sequencer publishes **only on its own turn** (the SDK queues out-of-turn publishes). When it is our turn we build the next block on the **current head tip**, which is by construction the last validly-applied block. So if the head is -frozen (`head_stall` set) on a peer's bad block, we build on that frozen valid +frozen on a peer's bad block, we build on that frozen valid tip — the same parent every honest sequencer chooses — and thereby skip the bad block rather than extend it. A parked node keeps following peers' valid blocks as they arrive; the moment it also gets a turn, it produces the next valid block on @@ -203,7 +212,7 @@ flowchart TD VAL --> OUT{"AcceptOutcome"} OUT -->|Applied| APP["append this_msg+block to head,
advance head tip, clear stall"] OUT -->|AlreadyApplied| SKIP - OUT -->|"Parked(err)"| PARK["set head_stall (in-memory),
freeze head tip — do NOT apply.
Not persisted"] + OUT -->|"Parked(err)"| PARK["freeze head tip — do NOT apply.
No stall recorded (self-heals
via reorg/finalization)"] SKIP --> FIN APP --> FIN PARK --> FIN @@ -231,11 +240,11 @@ stateDiagram-v2 Parked --> Parked: further non-chaining finalized blocks (orphans_since++) Parked --> Syncing: valid successor finalizes on frozen final tip → stall cleared note right of Parked - final_stall — persisted, survives restart. + final_stall — the one stall. Persisted, survives restart. Head-tier bad blocks do NOT enter this state: - they set head_stall (in-memory) and self-heal - via reorg/finalization. Producer (on our turn) - builds on the last valid tip either way. + the head tip freezes intrinsically (no stall) and + self-heals via reorg/finalization. Producer (on our + turn) builds on the last valid tip either way. end note ``` @@ -258,13 +267,13 @@ stateDiagram-v2 | 6 | Batch reorg: some `orphaned` + some `adopted` in one event | revert all orphaned, re-derive head, then apply all adopted in order | deterministic convergence | | 7 | Orphan chain (parent transitively off canonical) | SDK surfaces all affected as `orphaned`; revert each, replay survivors | head_state matches new canonical branch | -**Invalid / bad block** — "stall" below means the **persisted `final_stall`** -(§4a). A bad block seen only in `adopted` sets the in-memory `head_stall` (not -persisted); it becomes a persisted `final_stall` only if it finalizes. +**Invalid / bad block** — "stall" below means the one **persisted `final_stall`** +(§4a). A bad block seen only in `adopted` records **no** stall — the head tip just +freezes and self-heals; it becomes a `final_stall` only if it finalizes. | # | Scenario | Handling | Expected | | --- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | ----------------------------------------------- | -| 8 | Authorized sequencer posts a block with an invalid state transition | head: `apply_block` → `Parked`, set `head_stall` (in-memory), no persist. If it finalizes: persisted `final_stall` | park-and-skip; no apply, no halt | +| 8 | Authorized sequencer posts a block with an invalid state transition | head: `apply_block` → `Parked`, freeze head tip, no stall recorded. If it finalizes: persisted `final_stall` | park-and-skip; no apply, no halt | | 9 | Broken chain link / hash mismatch / unexpected id in adopted | `Parked(BrokenChainLink / HashMismatch / UnexpectedBlockId)`; same park | frozen tip; peers park identically | | 10 | Undeserializable inscription payload | park with `Deserialize` (no header); processing advances | recover when a valid block chains on frozen tip | | 11 | Valid successor after a park (recovery) | block chaining on frozen tip → `Applied` → clear stall | head resumes automatically; no divergence | diff --git a/lez/chain_state/src/apply.rs b/lez/chain_state/src/apply.rs new file mode 100644 index 00000000..38e49b53 --- /dev/null +++ b/lez/chain_state/src/apply.rs @@ -0,0 +1,280 @@ +//! The single validate-then-apply entry point shared by the sequencer and the +//! indexer. Pure and storage-free: callers apply on a scratch clone of state and +//! commit only on `Ok`. + +use common::{ + HashType, + block::Block, + transaction::{LeeTransaction, clock_invocation}, +}; +use lee::{GENESIS_BLOCK_ID, V03State}; + +use crate::ingest_error::BlockIngestError; + +/// The last successfully applied block: the parent the next block must chain on. +/// +/// Only what validation needs today (`block_id` + `hash`). The two-tier +/// `ChainState`'s `final_tip` will extend this with the inscription `l1_slot` +/// when the anchor layer lands; the slot is currently tracked separately. +#[derive(Debug, Clone)] +pub struct Tip { + pub block_id: u64, + pub hash: HashType, +} + +/// Outcome of feeding a parsed L2 block to a validated tip. +pub enum AcceptOutcome { + /// Chained and applied; the tip advances. + Applied, + /// A duplicate re-delivery of an already-applied block. No state change. + AlreadyApplied, + /// Did not chain or failed to apply; the tip stays frozen. + Parked(BlockIngestError), +} + +/// Validates `block` against `tip`, then applies it to `state`. +/// +/// Validation runs first and touches nothing. Application then mutates `state` +/// in place and can fail partway, so callers pass a scratch clone and commit it +/// only when this returns `Ok`. +pub fn apply_block( + tip: Option<&Tip>, + block: &Block, + state: &mut V03State, +) -> Result<(), BlockIngestError> { + validate_against_tip(tip, block)?; + apply_block_to_state(block, state)?; + Ok(()) +} + +/// Checks that `block` is the valid continuation of `tip`: hash integrity, +/// then block-id continuity, then `prev_block_hash` linkage. A `None` tip +/// (cold state) expects the genesis block. +fn validate_against_tip(tip: Option<&Tip>, block: &Block) -> Result<(), BlockIngestError> { + let computed = block.recompute_hash(); + if computed != block.header.hash { + return Err(BlockIngestError::HashMismatch { + computed, + header: block.header.hash, + }); + } + + match tip { + None => { + if block.header.block_id != GENESIS_BLOCK_ID { + return Err(BlockIngestError::UnexpectedBlockId { + expected: GENESIS_BLOCK_ID, + got: block.header.block_id, + }); + } + } + Some(tip) => { + let expected = tip + .block_id + .checked_add(1) + .expect("block id should not overflow"); + if block.header.block_id != expected { + return Err(BlockIngestError::UnexpectedBlockId { + expected, + got: block.header.block_id, + }); + } + if block.header.prev_block_hash != tip.hash { + return Err(BlockIngestError::BrokenChainLink { + expected_prev: tip.hash, + got_prev: block.header.prev_block_hash, + }); + } + } + } + Ok(()) +} + +/// Applies a block's transactions to `state`, mapping every failure to a +/// [`BlockIngestError`] so the caller can park rather than crash. Operates in +/// place; the caller commits only on `Ok`. +fn apply_block_to_state(block: &Block, state: &mut V03State) -> Result<(), BlockIngestError> { + let (clock_tx, user_txs) = block + .body + .transactions + .split_last() + .ok_or(BlockIngestError::EmptyBlock)?; + + let expected_clock = LeeTransaction::Public(clock_invocation(block.header.timestamp)); + if *clock_tx != expected_clock { + return Err(BlockIngestError::InvalidClockTransaction); + } + + let is_genesis = block.header.block_id == GENESIS_BLOCK_ID; + for (tx_index, transaction) in user_txs.iter().enumerate() { + let state_transition = |err: anyhow::Error| BlockIngestError::StateTransition { + tx_index: tx_index.try_into().expect("tx index fits in u64"), + reason: format!("{err:#}"), + }; + if is_genesis { + let LeeTransaction::Public(public_tx) = transaction else { + return Err(BlockIngestError::NonPublicGenesisTransaction); + }; + state + .transition_from_public_transaction( + public_tx, + block.header.block_id, + block.header.timestamp, + ) + .map_err(|err| state_transition(err.into()))?; + } else { + transaction + .clone() + .execute_on_state(state, block.header.block_id, block.header.timestamp) + .map_err(|err| state_transition(err.into()))?; + } + } + + let LeeTransaction::Public(clock_public_tx) = clock_tx else { + return Err(BlockIngestError::InvalidClockTransaction); + }; + state + .transition_from_public_transaction( + clock_public_tx, + block.header.block_id, + block.header.timestamp, + ) + .map_err(|err| BlockIngestError::StateTransition { + tx_index: user_txs.len().try_into().expect("tx index fits in u64"), + reason: format!("{:#}", anyhow::Error::from(err)), + })?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use common::{ + block::HashableBlockData, + test_utils::{ + create_transaction_native_token_transfer, produce_dummy_block, + produce_dummy_empty_transaction, sequencer_sign_key_for_testing, + }, + }; + use testnet_initial_state::{initial_pub_accounts_private_keys, initial_state}; + + use super::*; + + fn tip_of(block: &Block) -> Tip { + Tip { + block_id: block.header.block_id, + hash: block.header.hash, + } + } + + #[test] + fn genesis_applies_on_empty_tip() { + let mut state = initial_state(); + let genesis = produce_dummy_block(1, None, vec![]); + apply_block(None, &genesis, &mut state).expect("genesis applies"); + } + + #[test] + fn non_genesis_first_block_is_unexpected_id() { + let mut state = initial_state(); + let block = produce_dummy_block(2, None, vec![]); + let err = apply_block(None, &block, &mut state).expect_err("should reject"); + assert!(matches!( + err, + BlockIngestError::UnexpectedBlockId { expected: 1, got: 2 } + )); + } + + #[test] + fn skip_ahead_block_is_unexpected_id() { + let mut state = initial_state(); + let genesis = produce_dummy_block(1, None, vec![]); + apply_block(None, &genesis, &mut state).expect("genesis applies"); + + // Tip is at 1; a block with id 3 skips ahead. + let bad = produce_dummy_block(3, Some(genesis.header.hash), vec![]); + let err = apply_block(Some(&tip_of(&genesis)), &bad, &mut state).expect_err("should reject"); + assert!(matches!( + err, + BlockIngestError::UnexpectedBlockId { expected: 2, got: 3 } + )); + } + + #[test] + fn broken_chain_link_detected() { + let mut state = initial_state(); + let genesis = produce_dummy_block(1, None, vec![]); + apply_block(None, &genesis, &mut state).expect("genesis applies"); + + // Correct id (2), wrong parent hash. + let block2 = produce_dummy_block(2, Some(HashType([9_u8; 32])), vec![]); + let err = + apply_block(Some(&tip_of(&genesis)), &block2, &mut state).expect_err("should reject"); + assert!(matches!(err, BlockIngestError::BrokenChainLink { .. })); + } + + #[test] + fn hash_mismatch_detected() { + let mut state = initial_state(); + let mut genesis = produce_dummy_block(1, None, vec![]); + // Tampering with the header invalidates the stored hash. + genesis.header.timestamp = 999; + let err = apply_block(None, &genesis, &mut state).expect_err("should reject"); + assert!(matches!(err, BlockIngestError::HashMismatch { .. })); + } + + #[test] + fn empty_block_rejected() { + let mut state = initial_state(); + // A block with no transactions at all (not even the mandatory clock tx). + let block = HashableBlockData { + block_id: 1, + prev_block_hash: HashType([0_u8; 32]), + timestamp: 0, + transactions: vec![], + } + .into_pending_block(&sequencer_sign_key_for_testing()); + let err = apply_block(None, &block, &mut state).expect_err("should reject"); + assert!(matches!(err, BlockIngestError::EmptyBlock)); + } + + #[test] + fn missing_clock_tail_is_invalid_clock() { + let mut state = initial_state(); + // Last tx is not the expected clock invocation for the timestamp. + let block = HashableBlockData { + block_id: 1, + prev_block_hash: HashType([0_u8; 32]), + timestamp: 50, + transactions: vec![produce_dummy_empty_transaction()], + } + .into_pending_block(&sequencer_sign_key_for_testing()); + let err = apply_block(None, &block, &mut state).expect_err("should reject"); + assert!(matches!(err, BlockIngestError::InvalidClockTransaction)); + } + + #[test] + fn applies_transfers_and_advances_state() { + let mut state = initial_state(); + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + // Genesis (block 1): clock-only. + let genesis = produce_dummy_block(1, None, vec![]); + apply_block(None, &genesis, &mut state).expect("genesis applies"); + let mut tip = tip_of(&genesis); + + // Blocks 2..=11: one native transfer of 10 each (nonces 0..=9). + for i in 0..10_u64 { + let tx = create_transaction_native_token_transfer(from, i.into(), to, 10, &sign_key); + let block = produce_dummy_block(i + 2, Some(tip.hash), vec![tx]); + apply_block(Some(&tip), &block, &mut state).expect("transfer applies"); + tip = tip_of(&block); + } + + assert_eq!(state.get_account_by_id(from).balance, 9900); + assert_eq!(state.get_account_by_id(to).balance, 20100); + } +} diff --git a/lez/chain_state/src/ingest_error.rs b/lez/chain_state/src/ingest_error.rs new file mode 100644 index 00000000..bd377a0e --- /dev/null +++ b/lez/chain_state/src/ingest_error.rs @@ -0,0 +1,67 @@ +use common::HashType; +use serde::{Deserialize, Serialize}; + +/// Why an L2 block from the channel could not be applied. +/// +/// Persisted in `RocksDB` (as part of [`crate::StallReason`]), so every variant +/// must be `Clone + Serialize + Deserialize`. +#[derive(Debug, Clone, Serialize, Deserialize, thiserror::Error)] +pub enum BlockIngestError { + #[error("Failed to deserialize L2 block: {0}")] + /// Here we store the error string that is derived from [`borsh::from_slice`]'s [`Err`]. + Deserialize(String), + #[error("Unexpected block id: expected {expected}, got {got}")] + UnexpectedBlockId { expected: u64, got: u64 }, + #[error("Broken chain link: expected prev {expected_prev}, got {got_prev}")] + BrokenChainLink { + expected_prev: HashType, + got_prev: HashType, + }, + #[error("Block hash mismatch: computed {computed}, header {header}")] + HashMismatch { + computed: HashType, + header: HashType, + }, + #[error("Block has no transactions")] + EmptyBlock, + #[error("Last transaction must be the public clock invocation for the block timestamp")] + InvalidClockTransaction, + #[error("Genesis block must contain only public transactions")] + NonPublicGenesisTransaction, + #[error("State transition failed at transaction {tx_index}: {reason}")] + StateTransition { + /// Index of the failing transaction within the block body. + tx_index: u64, + /// Reason string from `lee::Error` to `anyhow::Error` to `{:#}`. + /// + /// This is required because `lee::Error` is not `Clone + Serialize + Deserialize`, so we + /// cannot store it directly. + reason: String, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn serializes_and_round_trips_externally_tagged() { + let err = BlockIngestError::UnexpectedBlockId { + expected: 5, + got: 7, + }; + let value = serde_json::to_value(&err).expect("serialize"); + assert_eq!( + value, + serde_json::json!({ "UnexpectedBlockId": { "expected": 5, "got": 7 } }) + ); + let back: BlockIngestError = serde_json::from_value(value).expect("deserialize"); + assert!(matches!( + back, + BlockIngestError::UnexpectedBlockId { + expected: 5, + got: 7 + } + )); + } +} diff --git a/lez/chain_state/src/lib.rs b/lez/chain_state/src/lib.rs new file mode 100644 index 00000000..67efa146 --- /dev/null +++ b/lez/chain_state/src/lib.rs @@ -0,0 +1,17 @@ +//! Shared, storage-free chain-state core for the LEZ sequencer and indexer. +//! +//! Hosts the single validate-then-apply entry point ([`apply_block`]) plus the +//! shared types ([`BlockIngestError`], [`StallReason`], [`Tip`], +//! [`AcceptOutcome`]) that both the sequencer and the indexer build on. The +//! crate performs no I/O: callers own their storage and drive the +//! `scratch → persist → commit` ordering around these primitives. +//! +//! See `DESIGN.md` in this crate for the two-tier chain-state model this backs. + +pub mod apply; +pub mod ingest_error; +pub mod stall_reason; + +pub use apply::{AcceptOutcome, Tip, apply_block}; +pub use ingest_error::BlockIngestError; +pub use stall_reason::StallReason; diff --git a/lez/chain_state/src/stall_reason.rs b/lez/chain_state/src/stall_reason.rs new file mode 100644 index 00000000..c5118fcc --- /dev/null +++ b/lez/chain_state/src/stall_reason.rs @@ -0,0 +1,25 @@ +use common::HashType; +use logos_blockchain_zone_sdk::Slot; +use serde::{Deserialize, Serialize}; + +use crate::ingest_error::BlockIngestError; + +/// Diagnostic record of the first block that broke the L2 chain. +/// +/// The block-derived fields are `None` for a deserialize break (no header was +/// ever parsed). `l1_slot` is the L1 slot the breaking inscription was read at. +/// `first_seen` is the breaking block's L2 timestamp (`None` for a deserialize break). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StallReason { + pub block_id: Option, + pub block_hash: Option, + pub prev_block_hash: Option, + pub l1_slot: Slot, + pub error: BlockIngestError, + pub first_seen: Option, + /// Number of later non-chaining blocks (orphans, since the tip is frozen). + /// + /// TODO: We could store a different "branch" of blocks following this break, but for now we + /// just count them. + pub orphans_since: u64, +} From 480b720293f50d8f1dd33241d999b2631e5a31e0 Mon Sep 17 00:00:00 2001 From: erhant Date: Wed, 8 Jul 2026 17:49:21 +0300 Subject: [PATCH 04/32] chore(chain_state): doc fix --- lez/chain_state/src/apply.rs | 24 +++++++++++++----------- lez/chain_state/src/lib.rs | 12 +++--------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/lez/chain_state/src/apply.rs b/lez/chain_state/src/apply.rs index 38e49b53..8a791a78 100644 --- a/lez/chain_state/src/apply.rs +++ b/lez/chain_state/src/apply.rs @@ -11,11 +11,8 @@ use lee::{GENESIS_BLOCK_ID, V03State}; use crate::ingest_error::BlockIngestError; -/// The last successfully applied block: the parent the next block must chain on. -/// -/// Only what validation needs today (`block_id` + `hash`). The two-tier -/// `ChainState`'s `final_tip` will extend this with the inscription `l1_slot` -/// when the anchor layer lands; the slot is currently tracked separately. +/// The parent the next block must chain on. +// `l1_slot` will be added here when the `ChainState` anchor layer lands. #[derive(Debug, Clone)] pub struct Tip { pub block_id: u64, @@ -34,9 +31,7 @@ pub enum AcceptOutcome { /// Validates `block` against `tip`, then applies it to `state`. /// -/// Validation runs first and touches nothing. Application then mutates `state` -/// in place and can fail partway, so callers pass a scratch clone and commit it -/// only when this returns `Ok`. +/// Mutates `state` in place, so callers pass a scratch clone and commit on `Ok`. pub fn apply_block( tip: Option<&Tip>, block: &Block, @@ -181,7 +176,10 @@ mod tests { let err = apply_block(None, &block, &mut state).expect_err("should reject"); assert!(matches!( err, - BlockIngestError::UnexpectedBlockId { expected: 1, got: 2 } + BlockIngestError::UnexpectedBlockId { + expected: 1, + got: 2 + } )); } @@ -193,10 +191,14 @@ mod tests { // Tip is at 1; a block with id 3 skips ahead. let bad = produce_dummy_block(3, Some(genesis.header.hash), vec![]); - let err = apply_block(Some(&tip_of(&genesis)), &bad, &mut state).expect_err("should reject"); + let err = + apply_block(Some(&tip_of(&genesis)), &bad, &mut state).expect_err("should reject"); assert!(matches!( err, - BlockIngestError::UnexpectedBlockId { expected: 2, got: 3 } + BlockIngestError::UnexpectedBlockId { + expected: 2, + got: 3 + } )); } diff --git a/lez/chain_state/src/lib.rs b/lez/chain_state/src/lib.rs index 67efa146..065f643d 100644 --- a/lez/chain_state/src/lib.rs +++ b/lez/chain_state/src/lib.rs @@ -1,12 +1,6 @@ -//! Shared, storage-free chain-state core for the LEZ sequencer and indexer. -//! -//! Hosts the single validate-then-apply entry point ([`apply_block`]) plus the -//! shared types ([`BlockIngestError`], [`StallReason`], [`Tip`], -//! [`AcceptOutcome`]) that both the sequencer and the indexer build on. The -//! crate performs no I/O: callers own their storage and drive the -//! `scratch → persist → commit` ordering around these primitives. -//! -//! See `DESIGN.md` in this crate for the two-tier chain-state model this backs. +//! Storage-free chain-state core shared by the LEZ sequencer and indexer: +//! the [`apply_block`] entry point plus [`BlockIngestError`], [`StallReason`], +//! [`Tip`], and [`AcceptOutcome`]. See `DESIGN.md` for the two-tier model. pub mod apply; pub mod ingest_error; From d89150b00ffb561a35e2f0a9e663cb359608824f Mon Sep 17 00:00:00 2001 From: erhant Date: Wed, 8 Jul 2026 18:35:10 +0300 Subject: [PATCH 05/32] refactor!(indexer): migr to `chain_state` --- Cargo.lock | 2 +- lez/chain_state/src/apply.rs | 4 + lez/chain_state/src/ingest_error.rs | 13 ++ lez/indexer/core/Cargo.toml | 2 +- lez/indexer/core/src/block_store.rs | 137 +--------------------- lez/indexer/core/src/chain_consistency.rs | 3 +- lez/indexer/core/src/ingest_error.rs | 80 ------------- lez/indexer/core/src/lib.rs | 7 +- lez/indexer/core/src/stall_reason.rs | 25 ---- lez/indexer/core/src/status.rs | 6 +- 10 files changed, 28 insertions(+), 251 deletions(-) delete mode 100644 lez/indexer/core/src/ingest_error.rs delete mode 100644 lez/indexer/core/src/stall_reason.rs diff --git a/Cargo.lock b/Cargo.lock index 7f9b2a5c..ab52ad0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3953,6 +3953,7 @@ dependencies = [ "arc-swap", "async-stream", "borsh", + "chain_state", "common", "cross_zone", "cross_zone_inbox_core", @@ -3972,7 +3973,6 @@ dependencies = [ "storage", "tempfile", "testnet_initial_state", - "thiserror 2.0.18", "tokio", "url", ] diff --git a/lez/chain_state/src/apply.rs b/lez/chain_state/src/apply.rs index 8a791a78..42a77b35 100644 --- a/lez/chain_state/src/apply.rs +++ b/lez/chain_state/src/apply.rs @@ -27,6 +27,10 @@ pub enum AcceptOutcome { AlreadyApplied, /// Did not chain or failed to apply; the tip stays frozen. Parked(BlockIngestError), + /// Chained but failed to apply, possibly transiently + /// ([`BlockIngestError::is_retryable`]); nothing recorded, tip and state + /// untouched. The caller retries and parks once it gives up. + RetryableFailure(BlockIngestError), } /// Validates `block` against `tip`, then applies it to `state`. diff --git a/lez/chain_state/src/ingest_error.rs b/lez/chain_state/src/ingest_error.rs index bd377a0e..d259b5f7 100644 --- a/lez/chain_state/src/ingest_error.rs +++ b/lez/chain_state/src/ingest_error.rs @@ -40,6 +40,19 @@ pub enum BlockIngestError { }, } +impl BlockIngestError { + /// Whether the failure may be transient rather than a property of the block. + /// + /// FIXME: `StateTransition` is too coarse — its `reason` string mixes genuine + /// state-transition rejections with infra failures (risc0 executor teardown, + /// storage errors). Once it carries a structured cause, narrow this so only + /// infra failures retry. + #[must_use] + pub const fn is_retryable(&self) -> bool { + matches!(self, Self::StateTransition { .. }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/lez/indexer/core/Cargo.toml b/lez/indexer/core/Cargo.toml index 14941773..afba6f29 100644 --- a/lez/indexer/core/Cargo.toml +++ b/lez/indexer/core/Cargo.toml @@ -12,6 +12,7 @@ default = [] testnet = [] [dependencies] +chain_state.workspace = true common.workspace = true logos-blockchain-zone-sdk.workspace = true lee.workspace = true @@ -32,7 +33,6 @@ futures.workspace = true url.workspace = true logos-blockchain-core.workspace = true serde_json.workspace = true -thiserror.workspace = true async-stream.workspace = true tokio.workspace = true risc0-zkvm.workspace = true diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 8c974399..7a9f410d 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -1,12 +1,12 @@ use std::{path::Path, sync::Arc}; use anyhow::{Context as _, Result}; +use chain_state::{AcceptOutcome, BlockIngestError, StallReason, Tip, apply_block}; use common::{ - HashType, block::{BedrockStatus, Block, BlockHeader}, - transaction::{LeeTransaction, clock_invocation}, + transaction::LeeTransaction, }; -use lee::{Account, AccountId, GENESIS_BLOCK_ID, V03State}; +use lee::{Account, AccountId, V03State}; use lee_core::BlockId; use log::warn; use logos_blockchain_core::header::HeaderId; @@ -14,28 +14,6 @@ use logos_blockchain_zone_sdk::Slot; use storage::indexer::RocksDBIO; use tokio::sync::RwLock; -use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; - -struct Tip { - block_id: u64, - hash: HashType, -} - -/// Outcome of feeding a parsed L2 block to the validated tip. -pub enum AcceptOutcome { - /// Chained and applied; tip and L1 read cursor both advance. - Applied, - /// A duplicate re-delivery of the current tip. Just L2 advances. - AlreadyApplied, - /// Did not chain or failed to apply; tip stays frozen, stall recorded. - Parked(BlockIngestError), - /// Chained but failed to apply, possibly transiently - /// ([`BlockIngestError::is_retryable`]); nothing recorded, tip and state - /// untouched. The caller retries and parks via - /// [`IndexerStore::record_stall`] once it gives up. - RetryableFailure(BlockIngestError), -} - #[derive(Clone)] pub struct IndexerStore { dbio: Arc, @@ -258,14 +236,9 @@ impl IndexerStore { return Ok(AcceptOutcome::AlreadyApplied); } - if let Err(err) = validate_against_tip(tip.as_ref(), block) { - self.record_stall(Some(&block.header), l1_slot, err.clone())?; - return Ok(AcceptOutcome::Parked(err)); - } - // TODO: we use scratch state to be atomic, but need to revisit how expensive a clone is let mut scratch = self.current_state.read().await.clone(); - if let Err(err) = apply_block_to_scratch(block, &mut scratch) { + if let Err(err) = apply_block(tip.as_ref(), block, &mut scratch) { if err.is_retryable() { return Ok(AcceptOutcome::RetryableFailure(err)); } @@ -290,112 +263,11 @@ impl IndexerStore { } } -/// Checks that `block` is the valid continuation of `tip`: hash integrity, -/// then block-id continuity, then `prev_block_hash` linkage. A `None` tip -/// (cold store) expects the genesis block. -fn validate_against_tip(tip: Option<&Tip>, block: &Block) -> Result<(), BlockIngestError> { - let computed = block.recompute_hash(); - if computed != block.header.hash { - return Err(BlockIngestError::HashMismatch { - computed, - header: block.header.hash, - }); - } - - match tip { - None => { - if block.header.block_id != GENESIS_BLOCK_ID { - return Err(BlockIngestError::UnexpectedBlockId { - expected: GENESIS_BLOCK_ID, - got: block.header.block_id, - }); - } - } - Some(tip) => { - let expected = tip - .block_id - .checked_add(1) - .expect("block id should not overflow"); - if block.header.block_id != expected { - return Err(BlockIngestError::UnexpectedBlockId { - expected, - got: block.header.block_id, - }); - } - if block.header.prev_block_hash != tip.hash { - return Err(BlockIngestError::BrokenChainLink { - expected_prev: tip.hash, - got_prev: block.header.prev_block_hash, - }); - } - } - } - Ok(()) -} - -/// Applies a block's transactions to `state`, mapping every failure to a -/// [`BlockIngestError`] so the caller can park rather than crash. Operates on a -/// scratch state; the caller commits only on `Ok`. -fn apply_block_to_scratch(block: &Block, state: &mut V03State) -> Result<(), BlockIngestError> { - let (clock_tx, user_txs) = block - .body - .transactions - .split_last() - .ok_or(BlockIngestError::EmptyBlock)?; - - let expected_clock = LeeTransaction::Public(clock_invocation(block.header.timestamp)); - if *clock_tx != expected_clock { - return Err(BlockIngestError::InvalidClockTransaction); - } - - let is_genesis = block.header.block_id == GENESIS_BLOCK_ID; - for (tx_index, transaction) in user_txs.iter().enumerate() { - let state_transition = |err: anyhow::Error| BlockIngestError::StateTransition { - tx_index: tx_index.try_into().expect("tx index fits in u64"), - reason: format!("{err:#}"), - }; - if is_genesis { - let LeeTransaction::Public(public_tx) = transaction else { - return Err(BlockIngestError::NonPublicGenesisTransaction); - }; - state - .transition_from_public_transaction( - public_tx, - block.header.block_id, - block.header.timestamp, - ) - .map_err(|err| state_transition(err.into()))?; - } else { - transaction - .clone() - .execute_on_state(state, block.header.block_id, block.header.timestamp) - .map_err(|err| state_transition(err.into()))?; - } - } - - let LeeTransaction::Public(clock_public_tx) = clock_tx else { - return Err(BlockIngestError::InvalidClockTransaction); - }; - state - .transition_from_public_transaction( - clock_public_tx, - block.header.block_id, - block.header.timestamp, - ) - .map_err(|err| BlockIngestError::StateTransition { - tx_index: user_txs.len().try_into().expect("tx index fits in u64"), - reason: format!("{:#}", anyhow::Error::from(err)), - })?; - - Ok(()) -} - #[cfg(test)] mod stall_reason_tests { use common::HashType; use super::*; - use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; #[tokio::test] async fn stall_reason_roundtrips_and_clears() { @@ -544,7 +416,6 @@ mod accept_tests { use common::{HashType, block::HashableBlockData, test_utils::produce_dummy_block}; use super::*; - use crate::ingest_error::BlockIngestError; fn signing_key() -> lee::PrivateKey { lee::PrivateKey::try_new([7_u8; 32]).expect("valid key") diff --git a/lez/indexer/core/src/chain_consistency.rs b/lez/indexer/core/src/chain_consistency.rs index df4b5ba6..a2c4bb41 100644 --- a/lez/indexer/core/src/chain_consistency.rs +++ b/lez/indexer/core/src/chain_consistency.rs @@ -305,8 +305,7 @@ mod tests { use super::*; use crate::{ - BlockIngestError, - block_store::AcceptOutcome, + AcceptOutcome, BlockIngestError, config::{ChannelId, ClientConfig, IndexerConfig}, }; diff --git a/lez/indexer/core/src/ingest_error.rs b/lez/indexer/core/src/ingest_error.rs deleted file mode 100644 index 299013d7..00000000 --- a/lez/indexer/core/src/ingest_error.rs +++ /dev/null @@ -1,80 +0,0 @@ -use common::HashType; -use serde::{Deserialize, Serialize}; - -/// Why the indexer could not apply an L2 block from the channel. -/// -/// Persisted in `RocksDB`, so every variant must have the following -/// traits: `Clone + Serialize + Deserialize`. -#[derive(Debug, Clone, Serialize, Deserialize, thiserror::Error)] -pub enum BlockIngestError { - #[error("Failed to deserialize L2 block: {0}")] - /// Here we store the error string that is derived from [`borsh::from_slice`]'s [`Err`]. - Deserialize(String), - #[error("Unexpected block id: expected {expected}, got {got}")] - UnexpectedBlockId { expected: u64, got: u64 }, - #[error("Broken chain link: expected prev {expected_prev}, got {got_prev}")] - BrokenChainLink { - expected_prev: HashType, - got_prev: HashType, - }, - #[error("Block hash mismatch: computed {computed}, header {header}")] - HashMismatch { - computed: HashType, - header: HashType, - }, - #[error("Block has no transactions")] - EmptyBlock, - #[error("Last transaction must be the public clock invocation for the block timestamp")] - InvalidClockTransaction, - #[error("Genesis block must contain only public transactions")] - NonPublicGenesisTransaction, - #[error("State transition failed at transaction {tx_index}: {reason}")] - StateTransition { - /// Index of the failing transaction within the block body. - tx_index: u64, - /// Reason string from `lee::Error` to `anyhow::Error` to `{:#}`. - /// - /// This is required because `lee::Error` is not `Clone + Serialize + Deserialize`, so we - /// cannot store it directly. - reason: String, - }, -} - -impl BlockIngestError { - /// Whether the failure may be transient rather than a property of the block. - /// - /// FIXME: `StateTransition` is too coarse — its `reason` string mixes genuine - /// state-transition rejections with infra failures (risc0 executor teardown, - /// storage errors). Once it carries a structured cause, narrow this so only - /// infra failures retry. - #[must_use] - pub const fn is_retryable(&self) -> bool { - matches!(self, Self::StateTransition { .. }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn serializes_and_round_trips_externally_tagged() { - let err = BlockIngestError::UnexpectedBlockId { - expected: 5, - got: 7, - }; - let value = serde_json::to_value(&err).expect("serialize"); - assert_eq!( - value, - serde_json::json!({ "UnexpectedBlockId": { "expected": 5, "got": 7 } }) - ); - let back: BlockIngestError = serde_json::from_value(value).expect("deserialize"); - assert!(matches!( - back, - BlockIngestError::UnexpectedBlockId { - expected: 5, - got: 7 - } - )); - } -} diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 7bee6387..ba772783 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -2,19 +2,18 @@ use std::{path::Path, sync::Arc}; use anyhow::Result; use arc_swap::ArcSwap; +pub use chain_state::{AcceptOutcome, BlockIngestError, StallReason}; use common::block::Block; // TODO: Remove after testnet use futures::StreamExt as _; -pub use ingest_error::BlockIngestError; use log::{error, info, warn}; use logos_blockchain_zone_sdk::{ CommonHttpClient, Slot, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer, }; use retry::ApplyRetryGate; -pub use stall_reason::StallReason; use crate::{ - block_store::{AcceptOutcome, IndexerStore}, + block_store::IndexerStore, chain_consistency::ChainConsistency, config::IndexerConfig, cross_zone_verifier::CrossZoneVerifier, @@ -24,9 +23,7 @@ pub mod block_store; pub mod chain_consistency; pub mod config; pub mod cross_zone_verifier; -pub mod ingest_error; mod retry; -pub mod stall_reason; pub mod status; /// Consecutive failed apply attempts of the same block before parking. diff --git a/lez/indexer/core/src/stall_reason.rs b/lez/indexer/core/src/stall_reason.rs deleted file mode 100644 index c5118fcc..00000000 --- a/lez/indexer/core/src/stall_reason.rs +++ /dev/null @@ -1,25 +0,0 @@ -use common::HashType; -use logos_blockchain_zone_sdk::Slot; -use serde::{Deserialize, Serialize}; - -use crate::ingest_error::BlockIngestError; - -/// Diagnostic record of the first block that broke the L2 chain. -/// -/// The block-derived fields are `None` for a deserialize break (no header was -/// ever parsed). `l1_slot` is the L1 slot the breaking inscription was read at. -/// `first_seen` is the breaking block's L2 timestamp (`None` for a deserialize break). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StallReason { - pub block_id: Option, - pub block_hash: Option, - pub prev_block_hash: Option, - pub l1_slot: Slot, - pub error: BlockIngestError, - pub first_seen: Option, - /// Number of later non-chaining blocks (orphans, since the tip is frozen). - /// - /// TODO: We could store a different "branch" of blocks following this break, but for now we - /// just count them. - pub orphans_since: u64, -} diff --git a/lez/indexer/core/src/status.rs b/lez/indexer/core/src/status.rs index a483fde6..aa182c23 100644 --- a/lez/indexer/core/src/status.rs +++ b/lez/indexer/core/src/status.rs @@ -1,7 +1,6 @@ +use chain_state::StallReason; use serde::Serialize; -use crate::stall_reason::StallReason; - /// Coarse lifecycle state of the indexer's ingestion loop, so a client can tell /// "still catching up" apart from "something went wrong". #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] @@ -117,10 +116,9 @@ mod tests { #[test] fn stalled_status_serializes_with_stall_reason() { + use chain_state::{BlockIngestError, StallReason}; use logos_blockchain_zone_sdk::Slot; - use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; - let status = IndexerStatus { sync: IndexerSyncStatus::stalled("broken chain link".to_owned()), indexed_block_id: Some(41), From 3ed67126e86fe4a9a43d4c38d36d686c4ef91f4d Mon Sep 17 00:00:00 2001 From: erhant Date: Thu, 9 Jul 2026 17:47:48 +0300 Subject: [PATCH 06/32] feat(chain_state): implement two-tip `ChainState` struct --- Cargo.lock | 1 + lez/chain_state/Cargo.toml | 1 + lez/chain_state/src/chain.rs | 417 +++++++++++++++++++++++++++++++++++ lez/chain_state/src/lib.rs | 10 +- 4 files changed, 425 insertions(+), 4 deletions(-) create mode 100644 lez/chain_state/src/chain.rs diff --git a/Cargo.lock b/Cargo.lock index ab52ad0c..14ad615e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1375,6 +1375,7 @@ dependencies = [ "anyhow", "common", "lee", + "logos-blockchain-core", "logos-blockchain-zone-sdk", "serde", "serde_json", diff --git a/lez/chain_state/Cargo.toml b/lez/chain_state/Cargo.toml index 337367cc..1fcc052e 100644 --- a/lez/chain_state/Cargo.toml +++ b/lez/chain_state/Cargo.toml @@ -10,6 +10,7 @@ workspace = true [dependencies] common.workspace = true lee.workspace = true +logos-blockchain-core.workspace = true logos-blockchain-zone-sdk.workspace = true anyhow.workspace = true diff --git a/lez/chain_state/src/chain.rs b/lez/chain_state/src/chain.rs new file mode 100644 index 00000000..f0b4eddb --- /dev/null +++ b/lez/chain_state/src/chain.rs @@ -0,0 +1,417 @@ +//! Two-tier chain state: a reorg-able `head` the sequencer builds on, plus an +//! irreversible `final` tier. See `DESIGN.md` for the model and rationale. + +use common::block::Block; +use lee::V03State; +use logos_blockchain_core::mantle::ops::channel::MsgId; +use logos_blockchain_zone_sdk::Slot; + +use crate::{ + AcceptOutcome, BlockIngestError, StallReason, + apply::{Tip, apply_block}, +}; + +/// A head block plus the channel message that carried it. +pub struct HeadEntry { + pub this_msg: MsgId, + pub block: Block, +} + +/// The head tier (reorg-able, from `adopted`/`orphaned`) over the final tier +/// (irreversible, from `finalized`). `head_state == final_state` replayed +/// through `head_blocks`. +pub struct ChainState { + final_state: V03State, + final_tip: Option, + head_state: V03State, + head_blocks: Vec, + final_stall: Option, +} + +impl ChainState { + /// Fresh state anchored at the genesis/initial state, no blocks applied. + #[must_use] + pub fn new(initial_state: V03State) -> Self { + Self::from_final(initial_state, None) + } + + /// State restored from a persisted final tier; head mirrors final. + #[must_use] + pub fn from_final(final_state: V03State, final_tip: Option) -> Self { + Self { + head_state: final_state.clone(), + final_state, + final_tip, + head_blocks: Vec::new(), + final_stall: None, + } + } + + /// State the sequencer builds its next block on. + #[must_use] + pub const fn head_state(&self) -> &V03State { + &self.head_state + } + + #[must_use] + pub const fn final_state(&self) -> &V03State { + &self.final_state + } + + /// Parent the next produced block must chain on. + #[must_use] + pub fn head_tip(&self) -> Option { + self.head_blocks + .last() + .map(|entry| tip_of(&entry.block)) + .or_else(|| self.final_tip.clone()) + } + + #[must_use] + pub fn final_tip(&self) -> Option { + self.final_tip.clone() + } + + #[must_use] + pub const fn final_stall(&self) -> Option<&StallReason> { + self.final_stall.as_ref() + } + + /// Applies an adopted head block. On failure the head tip stays at the last + /// valid block and no stall is recorded (§4a); the head self-heals. + pub fn apply_adopted(&mut self, this_msg: MsgId, block: &Block) -> AcceptOutcome { + let tip = self.head_tip(); + if self + .head_blocks + .iter() + .any(|entry| entry.this_msg == this_msg) + || tip + .as_ref() + .is_some_and(|current| block.header.block_id <= current.block_id) + { + return AcceptOutcome::AlreadyApplied; + } + + let mut scratch = self.head_state.clone(); + match apply_block(tip.as_ref(), block, &mut scratch) { + Ok(()) => { + self.head_state = scratch; + self.head_blocks.push(HeadEntry { + this_msg, + block: block.clone(), + }); + AcceptOutcome::Applied + } + Err(err) => AcceptOutcome::Parked(err), + } + } + + /// Reverts an orphaned head block and everything after it, then re-derives head. + pub fn revert_orphan(&mut self, this_msg: MsgId) { + if let Some(idx) = self + .head_blocks + .iter() + .position(|entry| entry.this_msg == this_msg) + { + self.head_blocks.truncate(idx); + self.rederive_head(); + } + } + + /// One channel update: revert every `orphaned`, then apply every `adopted`. + pub fn apply_channel_update( + &mut self, + orphaned: &[MsgId], + adopted: &[(MsgId, Block)], + ) -> Vec { + let earliest = orphaned + .iter() + .filter_map(|msg| { + self.head_blocks + .iter() + .position(|entry| entry.this_msg == *msg) + }) + .min(); + if let Some(idx) = earliest { + self.head_blocks.truncate(idx); + self.rederive_head(); + } + adopted + .iter() + .map(|(msg, block)| self.apply_adopted(*msg, block)) + .collect() + } + + /// A finalized inscription. In steady state the block is already in head and is + /// moved into `final`; on backfill (not in head) it is applied directly and may + /// set `final_stall`. + pub fn apply_finalized( + &mut self, + this_msg: MsgId, + block: &Block, + l1_slot: Slot, + ) -> AcceptOutcome { + // Match by MsgId, or by block identity to handle a re-inscribed but + // identical block arriving under a fresh MsgId. + let in_head = self.head_blocks.iter().position(|entry| { + entry.this_msg == this_msg || entry.block.header.hash == block.header.hash + }); + match in_head { + Some(idx) => { + self.finalize_through(idx); + AcceptOutcome::Applied + } + None => self.apply_finalized_direct(block, l1_slot), + } + } + + /// Moves `head_blocks[0..=idx]` into the final tier (already validated in head). + fn finalize_through(&mut self, idx: usize) { + let finalized: Vec = self.head_blocks.drain(0..=idx).collect(); + for entry in finalized { + apply_block(self.final_tip.as_ref(), &entry.block, &mut self.final_state) + .expect("a validated head block must apply to the final tier"); + self.final_tip = Some(tip_of(&entry.block)); + } + self.final_stall = None; + } + + /// Applies a finalized block straight to the final tier. On success the + /// finalized chain is authoritative, so head rebases onto it. + fn apply_finalized_direct(&mut self, block: &Block, l1_slot: Slot) -> AcceptOutcome { + let mut scratch = self.final_state.clone(); + match apply_block(self.final_tip.as_ref(), block, &mut scratch) { + Ok(()) => { + self.final_state = scratch; + self.final_tip = Some(tip_of(block)); + self.final_stall = None; + self.head_blocks.clear(); + self.head_state = self.final_state.clone(); + AcceptOutcome::Applied + } + Err(err) => { + self.record_final_stall(block, l1_slot, err.clone()); + AcceptOutcome::Parked(err) + } + } + } + + /// Rebuilds `head_state` from the final tier plus the current `head_blocks`. + fn rederive_head(&mut self) { + let mut state = self.final_state.clone(); + let mut tip = self.final_tip.clone(); + for entry in &self.head_blocks { + apply_block(tip.as_ref(), &entry.block, &mut state) + .expect("validated head blocks must replay"); + tip = Some(tip_of(&entry.block)); + } + self.head_state = state; + } + + /// First stall is stored verbatim; later ones only bump `orphans_since`. + fn record_final_stall(&mut self, block: &Block, l1_slot: Slot, error: BlockIngestError) { + self.final_stall = Some(match self.final_stall.take() { + Some(mut existing) => { + existing.orphans_since = existing.orphans_since.saturating_add(1); + existing + } + None => stall_for(block, l1_slot, error), + }); + } +} + +const fn tip_of(block: &Block) -> Tip { + Tip { + block_id: block.header.block_id, + hash: block.header.hash, + } +} + +const fn stall_for(block: &Block, l1_slot: Slot, error: BlockIngestError) -> StallReason { + StallReason { + block_id: Some(block.header.block_id), + block_hash: Some(block.header.hash), + prev_block_hash: Some(block.header.prev_block_hash), + l1_slot, + error, + first_seen: Some(block.header.timestamp), + orphans_since: 0, + } +} + +#[cfg(test)] +mod tests { + use common::test_utils::{create_transaction_native_token_transfer, produce_dummy_block}; + use testnet_initial_state::{initial_pub_accounts_private_keys, initial_state}; + + use super::*; + + fn msg(n: u8) -> MsgId { + MsgId::from([n; 32]) + } + + fn slot(n: u64) -> Slot { + Slot::from(n) + } + + #[test] + fn adopted_blocks_advance_head() { + let mut chain = ChainState::new(initial_state()); + + let genesis = produce_dummy_block(1, None, vec![]); + assert!(matches!( + chain.apply_adopted(msg(1), &genesis), + AcceptOutcome::Applied + )); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + assert!(matches!( + chain.apply_adopted(msg(2), &block2), + AcceptOutcome::Applied + )); + + assert_eq!(chain.head_tip().expect("head tip").block_id, 2); + // Nothing finalized yet. + assert!(chain.final_tip().is_none()); + } + + #[test] + fn adopted_bad_block_freezes_head_without_stall() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + + // Skips ahead (id 3 while head tip is 1). + let bad = produce_dummy_block(3, Some(genesis.header.hash), vec![]); + assert!(matches!( + chain.apply_adopted(msg(3), &bad), + AcceptOutcome::Parked(BlockIngestError::UnexpectedBlockId { + expected: 2, + got: 3 + }) + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 1); + assert!( + chain.final_stall().is_none(), + "head freeze records no stall" + ); + } + + #[test] + fn adopted_is_idempotent() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + + assert!(matches!( + chain.apply_adopted(msg(1), &genesis), + AcceptOutcome::AlreadyApplied + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 1); + } + + #[test] + fn orphan_reverts_head() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + chain.apply_adopted(msg(3), &block3); + + chain.revert_orphan(msg(3)); + assert_eq!(chain.head_tip().expect("head tip").block_id, 2); + + // A competing block 3 now applies cleanly on block 2. + let block3_prime = produce_dummy_block(3, Some(block2.header.hash), vec![]); + assert!(matches!( + chain.apply_adopted(msg(13), &block3_prime), + AcceptOutcome::Applied + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + } + + #[test] + fn channel_update_reverts_then_applies() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + chain.apply_adopted(msg(3), &block3); + + let block3_prime = produce_dummy_block(3, Some(block2.header.hash), vec![]); + let outcomes = chain.apply_channel_update(&[msg(3)], &[(msg(13), block3_prime)]); + assert!(matches!(outcomes.as_slice(), [AcceptOutcome::Applied])); + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + } + + #[test] + fn finalize_moves_head_into_final() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + chain.apply_adopted(msg(3), &block3); + + // Finalize through block 2. + assert!(matches!( + chain.apply_finalized(msg(2), &block2, slot(100)), + AcceptOutcome::Applied + )); + assert_eq!(chain.final_tip().expect("final tip").block_id, 2); + // Head tip unchanged; head still ends at 3. + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + } + + #[test] + fn backfill_applies_directly_to_final() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + assert!(matches!( + chain.apply_finalized(msg(1), &genesis, slot(10)), + AcceptOutcome::Applied + )); + assert_eq!(chain.final_tip().expect("final tip").block_id, 1); + // Head mirrors final during backfill. + assert_eq!(chain.head_tip().expect("head tip").block_id, 1); + } + + #[test] + fn invalid_finalized_block_sets_final_stall() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_finalized(msg(1), &genesis, slot(10)); + + // Skip-ahead finalized block, not in head: parks the final tier. + let bad = produce_dummy_block(3, Some(genesis.header.hash), vec![]); + assert!(matches!( + chain.apply_finalized(msg(3), &bad, slot(20)), + AcceptOutcome::Parked(_) + )); + let stall = chain.final_stall().expect("final stall recorded"); + assert_eq!(stall.block_id, Some(3)); + } + + #[test] + fn head_state_reflects_applied_transfers() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + + let tx = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx]); + chain.apply_adopted(msg(2), &block2); + + assert_eq!(chain.head_state().get_account_by_id(from).balance, 9990); + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20010); + } +} diff --git a/lez/chain_state/src/lib.rs b/lez/chain_state/src/lib.rs index 065f643d..c4725ec3 100644 --- a/lez/chain_state/src/lib.rs +++ b/lez/chain_state/src/lib.rs @@ -2,10 +2,12 @@ //! the [`apply_block`] entry point plus [`BlockIngestError`], [`StallReason`], //! [`Tip`], and [`AcceptOutcome`]. See `DESIGN.md` for the two-tier model. -pub mod apply; -pub mod ingest_error; -pub mod stall_reason; - pub use apply::{AcceptOutcome, Tip, apply_block}; +pub use chain::{ChainState, HeadEntry}; pub use ingest_error::BlockIngestError; pub use stall_reason::StallReason; + +pub mod apply; +pub mod chain; +pub mod ingest_error; +pub mod stall_reason; From d4d28d1134e71b9ebb2e7e72654590ad79e83d39 Mon Sep 17 00:00:00 2001 From: erhant Date: Thu, 9 Jul 2026 18:06:08 +0300 Subject: [PATCH 07/32] feat!(sequencer): add `is_our_turn` --- lez/sequencer/core/src/block_publisher.rs | 19 +++++++++++++++++-- lez/sequencer/core/src/lib.rs | 6 ++++++ lez/sequencer/core/src/mock.rs | 4 ++++ lez/sequencer/service/src/lib.rs | 15 +++++++-------- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/lez/sequencer/core/src/block_publisher.rs b/lez/sequencer/core/src/block_publisher.rs index 21551131..f8a64c5d 100644 --- a/lez/sequencer/core/src/block_publisher.rs +++ b/lez/sequencer/core/src/block_publisher.rs @@ -12,10 +12,14 @@ use logos_blockchain_zone_sdk::{ adapter::NodeHttpClient, sequencer::{ DepositInfo, Event, FinalizedOp, InscriptionInfo, - SequencerConfig as ZoneSdkSequencerConfig, WithdrawArg, WithdrawInfo, ZoneSequencer, + SequencerConfig as ZoneSdkSequencerConfig, TurnNotification, WithdrawArg, WithdrawInfo, + ZoneSequencer, }, }; -use tokio::{sync::mpsc, task::JoinHandle}; +use tokio::{ + sync::{mpsc, watch}, + task::JoinHandle, +}; use crate::config::BedrockConfig; @@ -63,6 +67,9 @@ pub trait BlockPublisherTrait: Clone { async fn publish_block(&self, block: &Block, withdrawals: Vec) -> Result<()>; fn channel_id(&self) -> ChannelId; + + /// Whether this sequencer is currently authorized to write to the channel. + fn is_our_turn(&self) -> bool; } /// Real block publisher backed by zone-sdk's `ZoneSequencer`. @@ -70,6 +77,7 @@ pub trait BlockPublisherTrait: Clone { pub struct ZoneSdkPublisher { channel_id: ChannelId, publish_tx: mpsc::Sender<(Inscription, Vec)>, + turn_rx: watch::Receiver, // Aborts the drive task when the last clone is dropped. _drive_task: Arc, } @@ -112,6 +120,8 @@ impl BlockPublisherTrait for ZoneSdkPublisher { // Grab readiness receiver before moving the sequencer into the drive // task so we can await cold-start completion below. let mut ready_rx = sequencer.subscribe_ready(); + // Grab the turn watch before the move; the sdk actor keeps it current. + let turn_rx = sequencer.subscribe_turn_to_write(); let (publish_tx, mut publish_rx) = mpsc::channel::<(Inscription, Vec)>(PUBLISH_INBOX_CAPACITY); @@ -197,6 +207,7 @@ impl BlockPublisherTrait for ZoneSdkPublisher { Ok(Self { channel_id: config.channel_id, publish_tx, + turn_rx, _drive_task: Arc::new(DriveTaskGuard(drive_task)), }) } @@ -218,6 +229,10 @@ impl BlockPublisherTrait for ZoneSdkPublisher { fn channel_id(&self) -> ChannelId { self.channel_id } + + fn is_our_turn(&self) -> bool { + self.turn_rx.borrow().our_turn_to_write + } } /// Deserialize inscription payload as a `Block` and return it's`block_id`. diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 3a96e6d5..b92e5ba3 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -565,6 +565,12 @@ impl SequencerCore { self.block_publisher.clone() } + /// Whether this sequencer is currently authorized to write to the channel. + #[must_use] + pub fn is_our_turn(&self) -> bool { + self.block_publisher.is_our_turn() + } + fn next_block_id(&self) -> u64 { self.chain_height .checked_add(1) diff --git a/lez/sequencer/core/src/mock.rs b/lez/sequencer/core/src/mock.rs index 39f635f9..81508f7c 100644 --- a/lez/sequencer/core/src/mock.rs +++ b/lez/sequencer/core/src/mock.rs @@ -48,4 +48,8 @@ impl BlockPublisherTrait for MockBlockPublisher { fn channel_id(&self) -> ChannelId { self.channel_id } + + fn is_our_turn(&self) -> bool { + true + } } diff --git a/lez/sequencer/service/src/lib.rs b/lez/sequencer/service/src/lib.rs index 687ee424..d6779102 100644 --- a/lez/sequencer/service/src/lib.rs +++ b/lez/sequencer/service/src/lib.rs @@ -172,16 +172,15 @@ async fn main_loop(seq_core: Arc>, block_timeout: Duration) loop { tokio::time::sleep(block_timeout).await; - info!("Collecting transactions from mempool, block creation"); + let mut state = seq_core.lock().await; - let id = { - let mut state = seq_core.lock().await; - - state.produce_new_block().await? - }; + // Only produce on our turn. + if !state.is_our_turn() { + continue; + } + info!("Our turn: collecting transactions from mempool, creating block"); + let id = state.produce_new_block().await?; info!("Block with id {id} created"); - - info!("Waiting for new transactions"); } } From d0be0fe46c29d837ad01df62bdd23eb6b213d06a Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 10 Jul 2026 11:08:26 +0300 Subject: [PATCH 08/32] feat(sequencer): read `channel_update` and respect `adopted/orphaned` blocks --- Cargo.lock | 1 + lez/sequencer/core/Cargo.toml | 1 + lez/sequencer/core/src/block_publisher.rs | 63 +++++++++++++++++++---- lez/sequencer/core/src/lib.rs | 53 ++++++++++++++++++- lez/sequencer/core/src/mock.rs | 3 +- 5 files changed, 109 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 14ad615e..ca7cea65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9038,6 +9038,7 @@ dependencies = [ "borsh", "bridge_core", "bytesize", + "chain_state", "chrono", "common", "cross_zone", diff --git a/lez/sequencer/core/Cargo.toml b/lez/sequencer/core/Cargo.toml index cd89f223..2217abcd 100644 --- a/lez/sequencer/core/Cargo.toml +++ b/lez/sequencer/core/Cargo.toml @@ -10,6 +10,7 @@ workspace = true [dependencies] lee.workspace = true lee_core.workspace = true +chain_state.workspace = true common.workspace = true storage.workspace = true mempool.workspace = true diff --git a/lez/sequencer/core/src/block_publisher.rs b/lez/sequencer/core/src/block_publisher.rs index f8a64c5d..5654dd34 100644 --- a/lez/sequencer/core/src/block_publisher.rs +++ b/lez/sequencer/core/src/block_publisher.rs @@ -11,7 +11,7 @@ use logos_blockchain_zone_sdk::{ CommonHttpClient, adapter::NodeHttpClient, sequencer::{ - DepositInfo, Event, FinalizedOp, InscriptionInfo, + DepositInfo, Event, FinalizedOp, InscriptionInfo, OrphanedTx, SequencerConfig as ZoneSdkSequencerConfig, TurnNotification, WithdrawArg, WithdrawInfo, ZoneSequencer, }, @@ -45,6 +45,19 @@ pub type OnDepositEventSink = pub type OnWithdrawEventSink = Box Pin + Send>> + Send + 'static>; +/// The channel delta the follow path consumes from one `Event::BlocksProcessed`, +/// with inscription payloads decoded into `(MsgId, Block)` pairs. +pub struct FollowUpdate { + pub adopted: Vec<(MsgId, Block)>, + pub orphaned: Vec<(MsgId, Block)>, + pub finalized: Vec<(MsgId, Block)>, +} + +/// Sink for the follow path: apply adopted/finalized blocks to chain state and +/// revert orphaned ones. +pub type OnFollowSink = + Box Pin + Send>> + Send + 'static>; + #[expect(async_fn_in_trait, reason = "We don't care about Send/Sync here")] pub trait BlockPublisherTrait: Clone { #[expect( @@ -60,6 +73,7 @@ pub trait BlockPublisherTrait: Clone { on_finalized_block: FinalizedBlockSink, on_deposit_event: OnDepositEventSink, on_withdraw_event: OnWithdrawEventSink, + on_follow: OnFollowSink, ) -> Result; /// Fire-and-forget publish. Zone-sdk drives the actual submission and @@ -100,6 +114,7 @@ impl BlockPublisherTrait for ZoneSdkPublisher { on_finalized_block: FinalizedBlockSink, on_deposit_event: OnDepositEventSink, on_withdraw_event: OnWithdrawEventSink, + on_follow: OnFollowSink, ) -> Result { let basic_auth = config.auth.clone().map(Into::into); let node = NodeHttpClient::new(CommonHttpClient::new(basic_auth), config.node_url.clone()); @@ -167,17 +182,32 @@ impl BlockPublisherTrait for ZoneSdkPublisher { match event { Event::BlocksProcessed { checkpoint, - channel_update: _, + channel_update, finalized, } => { on_checkpoint(checkpoint); + + let adopted = channel_update + .adopted + .iter() + .filter_map(block_from_inscription) + .collect(); + let orphaned = channel_update + .orphaned + .iter() + .map(orphan_inscription) + .filter_map(block_from_inscription) + .collect(); + + let mut finalized_blocks = Vec::new(); for op in finalized.into_iter().flat_map(|item| item.ops) { match op { FinalizedOp::Inscription(inscription) => { - if let Some(block_id) = - block_id_from_inscription(&inscription) + if let Some((msg, block)) = + block_from_inscription(&inscription) { - on_finalized_block(block_id); + on_finalized_block(block.header.block_id); + finalized_blocks.push((msg, block)); } } FinalizedOp::Deposit(deposit) => { @@ -188,6 +218,13 @@ impl BlockPublisherTrait for ZoneSdkPublisher { } } } + + on_follow(FollowUpdate { + adopted, + orphaned, + finalized: finalized_blocks, + }) + .await; } Event::Ready | Event::TurnNotification { .. } => {} } @@ -235,13 +272,21 @@ impl BlockPublisherTrait for ZoneSdkPublisher { } } -/// Deserialize inscription payload as a `Block` and return it's`block_id`. -/// Bad payloads are logged and skipped. -fn block_id_from_inscription(inscription: &InscriptionInfo) -> Option { +/// Deserialize an inscription payload into `(this_msg, Block)`. Bad payloads are +/// logged and skipped. +fn block_from_inscription(inscription: &InscriptionInfo) -> Option<(MsgId, Block)> { borsh::from_slice::(&inscription.payload) .inspect_err(|err| { warn!("Failed to deserialize block from inscription: {err:?}"); }) .ok() - .map(|block| block.header.block_id) + .map(|block| (inscription.this_msg, block)) +} + +/// The inscription carried by an orphaned tx (plain or atomic-withdraw bundle). +const fn orphan_inscription(orphan: &OrphanedTx) -> &InscriptionInfo { + match orphan { + OrphanedTx::Inscription(info) => info, + OrphanedTx::AtomicWithdraw(bundle) => &bundle.inscription, + } } diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index b92e5ba3..44da46f3 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -1,7 +1,12 @@ -use std::{path::Path, sync::Arc, time::Instant}; +use std::{ + path::Path, + sync::{Arc, Mutex}, + time::Instant, +}; use anyhow::{Context as _, Result, anyhow}; use borsh::BorshDeserialize; +use chain_state::{ChainState, Tip}; use common::{ HashType, block::{BedrockStatus, Block, HashableBlockData}, @@ -13,7 +18,10 @@ use lee::{AccountId, PublicTransaction, public_transaction::Message}; use lee_core::GENESIS_BLOCK_ID; use log::{error, info, warn}; use logos_blockchain_key_management_system_service::keys::{ED25519_SECRET_KEY_SIZE, Ed25519Key}; -use logos_blockchain_zone_sdk::sequencer::{DepositInfo, WithdrawArg}; +use logos_blockchain_zone_sdk::{ + Slot, + sequencer::{DepositInfo, WithdrawArg}, +}; use mempool::{MemPool, MemPoolHandle}; #[cfg(feature = "mock")] pub use mock::SequencerCoreWithMockClients; @@ -59,6 +67,9 @@ impl DepositMetadata { pub struct SequencerCore { state: lee::V03State, + /// Two-tier follow state fed by the publisher's `on_follow` sink. Shared with + /// the drive task. Passive today; production moves onto its head next. + chain: Arc>, store: SequencerStore, mempool: MemPool<(TransactionOrigin, LeeTransaction)>, sequencer_config: SequencerConfig, @@ -132,6 +143,14 @@ impl SequencerCore { .latest_block_meta() .expect("Failed to read latest block meta from store"); + let chain = Arc::new(Mutex::new(ChainState::from_final( + state.clone(), + Some(Tip { + block_id: latest_block_meta.id, + hash: latest_block_meta.hash, + }), + ))); + let initial_checkpoint = store .get_zone_checkpoint() .expect("Failed to load zone-sdk checkpoint"); @@ -149,6 +168,7 @@ impl SequencerCore { Self::on_finalized_block(store.dbio()), Self::on_deposit_event(store.dbio(), mempool_handle.clone()), Self::on_withdraw_event(store.dbio()), + Self::on_follow(Arc::clone(&chain)), ) .await .expect("Failed to initialize Block Publisher"); @@ -195,6 +215,7 @@ impl SequencerCore { let sequencer_core = Self { state, + chain, store, mempool, chain_height: latest_block_meta.id, @@ -335,6 +356,28 @@ impl SequencerCore { }) } + /// Feed one channel delta into the follow state: revert orphaned, apply + /// adopted, then finalize. Passive today — production still uses `self.state`. + fn on_follow(chain: Arc>) -> block_publisher::OnFollowSink { + Box::new(move |update: block_publisher::FollowUpdate| { + let chain = Arc::clone(&chain); + Box::pin(async move { + let mut chain = chain.lock().expect("chain state mutex poisoned"); + for (this_msg, _) in &update.orphaned { + chain.revert_orphan(*this_msg); + } + for (this_msg, block) in &update.adopted { + chain.apply_adopted(*this_msg, block); + } + for (this_msg, block) in &update.finalized { + // TODO: thread the finalized inscription's L1 slot once the sdk + // surfaces it; only used for the rare invalid-finalized stall. + chain.apply_finalized(*this_msg, block, Slot::from(0)); + } + }) + }) + } + /// Produces a new block from mempool transactions and publishes it via zone-sdk. pub async fn produce_new_block(&mut self) -> Result { let BlockWithMeta { @@ -571,6 +614,12 @@ impl SequencerCore { self.block_publisher.is_our_turn() } + /// Shared handle to the two-tier follow state. + #[must_use] + pub fn chain(&self) -> Arc> { + Arc::clone(&self.chain) + } + fn next_block_id(&self) -> u64 { self.chain_height .checked_add(1) diff --git a/lez/sequencer/core/src/mock.rs b/lez/sequencer/core/src/mock.rs index 81508f7c..a3922c22 100644 --- a/lez/sequencer/core/src/mock.rs +++ b/lez/sequencer/core/src/mock.rs @@ -8,7 +8,7 @@ use logos_blockchain_zone_sdk::sequencer::WithdrawArg; use crate::{ block_publisher::{ - BlockPublisherTrait, CheckpointSink, FinalizedBlockSink, OnDepositEventSink, + BlockPublisherTrait, CheckpointSink, FinalizedBlockSink, OnDepositEventSink, OnFollowSink, OnWithdrawEventSink, SequencerCheckpoint, }, config::BedrockConfig, @@ -31,6 +31,7 @@ impl BlockPublisherTrait for MockBlockPublisher { _on_finalized_block: FinalizedBlockSink, _on_deposit_event: OnDepositEventSink, _on_withdraw_event: OnWithdrawEventSink, + _on_follow: OnFollowSink, ) -> Result { Ok(Self { channel_id: config.channel_id, From c0d4aa4fac7f9a1ede4c5361df47fb271f643007 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 10 Jul 2026 12:10:47 +0300 Subject: [PATCH 09/32] fix(sequencer): apply the own-published block immediately to the head & dedup it later if received --- lez/sequencer/core/src/block_publisher.rs | 72 ++++++++++++++--------- lez/sequencer/core/src/lib.rs | 10 +++- lez/sequencer/core/src/mock.rs | 11 ++-- 3 files changed, 60 insertions(+), 33 deletions(-) diff --git a/lez/sequencer/core/src/block_publisher.rs b/lez/sequencer/core/src/block_publisher.rs index 5654dd34..9cd21b45 100644 --- a/lez/sequencer/core/src/block_publisher.rs +++ b/lez/sequencer/core/src/block_publisher.rs @@ -17,7 +17,7 @@ use logos_blockchain_zone_sdk::{ }, }; use tokio::{ - sync::{mpsc, watch}, + sync::{mpsc, oneshot, watch}, task::JoinHandle, }; @@ -58,6 +58,14 @@ pub struct FollowUpdate { pub type OnFollowSink = Box Pin + Send>> + Send + 'static>; +/// Publish request channel: the inscription, its bridge withdrawals, and a +/// oneshot for the `MsgId` zone-sdk assigns. +type PublishSender = mpsc::Sender<( + Inscription, + Vec, + oneshot::Sender>, +)>; + #[expect(async_fn_in_trait, reason = "We don't care about Send/Sync here")] pub trait BlockPublisherTrait: Clone { #[expect( @@ -76,9 +84,9 @@ pub trait BlockPublisherTrait: Clone { on_follow: OnFollowSink, ) -> Result; - /// Fire-and-forget publish. Zone-sdk drives the actual submission and - /// retries internally; this just hands the payload off. - async fn publish_block(&self, block: &Block, withdrawals: Vec) -> Result<()>; + /// Publish a block and return the `MsgId` zone-sdk assigned its inscription. + /// Zone-sdk drives the actual submission and retries internally. + async fn publish_block(&self, block: &Block, withdrawals: Vec) -> Result; fn channel_id(&self) -> ChannelId; @@ -90,7 +98,7 @@ pub trait BlockPublisherTrait: Clone { #[derive(Clone)] pub struct ZoneSdkPublisher { channel_id: ChannelId, - publish_tx: mpsc::Sender<(Inscription, Vec)>, + publish_tx: PublishSender, turn_rx: watch::Receiver, // Aborts the drive task when the last clone is dropped. _drive_task: Arc, @@ -138,8 +146,8 @@ impl BlockPublisherTrait for ZoneSdkPublisher { // Grab the turn watch before the move; the sdk actor keeps it current. let turn_rx = sequencer.subscribe_turn_to_write(); - let (publish_tx, mut publish_rx) = - mpsc::channel::<(Inscription, Vec)>(PUBLISH_INBOX_CAPACITY); + let (publish_tx, mut publish_rx): (PublishSender, _) = + mpsc::channel(PUBLISH_INBOX_CAPACITY); let drive_task = tokio::spawn(async move { loop { @@ -152,28 +160,33 @@ impl BlockPublisherTrait for ZoneSdkPublisher { // Drain external publish requests by calling the // borrowing handle — `&mut sequencer` is only // available here. - Some((data_bounded, withdrawals)) = publish_rx.recv() => { + Some((data_bounded, withdrawals, resp_tx)) = publish_rx.recv() => { let data_byte_size = data_bounded.len(); - if withdrawals.is_empty() { - if let Err(e) = sequencer.handle() - .publish(data_bounded) - .context("Failed to publish block") { - warn!("zone-sdk publish failed: {e:?}"); - } - - info!("Published block with the size of {data_byte_size} bytes"); + let withdraw_count = withdrawals.len(); + let published = if withdrawals.is_empty() { + sequencer.handle() + .publish(data_bounded) + .context("Failed to publish block") } else { - let withdraw_count = withdrawals.len(); - if let Err(e) = sequencer.handle() - .publish_atomic_withdraw(data_bounded, withdrawals) - .context("Failed to publish block with withdrawals") { - warn!("zone-sdk publish failed: {e:?}"); - } + sequencer.handle() + .publish_atomic_withdraw(data_bounded, withdrawals) + .context("Failed to publish block with withdrawals") + }; - info!( - "Published block with the size of {data_byte_size} bytes and {withdraw_count} bridge withdrawals", - ); + let msg_result = published + .map(|(result, _checkpoint)| result.tx.inscription().this_msg); + match &msg_result { + Ok(_) if withdraw_count == 0 => { + info!("Published block with the size of {data_byte_size} bytes"); + } + Ok(_) => { + info!( + "Published block with the size of {data_byte_size} bytes and {withdraw_count} bridge withdrawals", + ); + } + Err(e) => warn!("zone-sdk publish failed: {e:?}"), } + let _dontcare = resp_tx.send(msg_result); } event = sequencer.next_event() => { let Some(event) = event else { @@ -249,18 +262,21 @@ impl BlockPublisherTrait for ZoneSdkPublisher { }) } - async fn publish_block(&self, block: &Block, withdrawals: Vec) -> Result<()> { + async fn publish_block(&self, block: &Block, withdrawals: Vec) -> Result { let data = borsh::to_vec(block).context("Failed to serialize block")?; let data_bounded: Inscription = data .try_into() .context("Block data exceeds maximum allowed size")?; + let (resp_tx, resp_rx) = oneshot::channel(); self.publish_tx - .send((data_bounded, withdrawals)) + .send((data_bounded, withdrawals, resp_tx)) .await .map_err(|_closed| anyhow!("Drive task is no longer running"))?; - Ok(()) + resp_rx + .await + .map_err(|_closed| anyhow!("Drive task dropped the publish response"))? } fn channel_id(&self) -> ChannelId { diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 44da46f3..5efb8348 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -394,11 +394,19 @@ impl SequencerCore { .collect::>() .context("Failed to build reconciliation keys for block withdrawals")?; - self.block_publisher + let this_msg = self + .block_publisher .publish_block(&block, withdrawals) .await .context("Failed to publish block to Bedrock")?; + // Apply our own block to the head with the MsgId the publish assigned it, + // so the head advances and the later adopted redelivery dedups. + self.chain + .lock() + .expect("chain state mutex poisoned") + .apply_adopted(this_msg, &block); + self.store.update( &block, &deposit_event_ids, diff --git a/lez/sequencer/core/src/mock.rs b/lez/sequencer/core/src/mock.rs index a3922c22..37c43254 100644 --- a/lez/sequencer/core/src/mock.rs +++ b/lez/sequencer/core/src/mock.rs @@ -2,7 +2,7 @@ use std::time::Duration; use anyhow::Result; use common::block::Block; -use logos_blockchain_core::mantle::ops::channel::ChannelId; +use logos_blockchain_core::mantle::ops::channel::{ChannelId, MsgId}; use logos_blockchain_key_management_system_service::keys::Ed25519Key; use logos_blockchain_zone_sdk::sequencer::WithdrawArg; @@ -40,10 +40,13 @@ impl BlockPublisherTrait for MockBlockPublisher { async fn publish_block( &self, - _block: &Block, + block: &Block, _bridge_withdrawals: Vec, - ) -> Result<()> { - Ok(()) + ) -> Result { + // Deterministic per-block id so head dedup behaves in tests. + // + // TODO: should we allow more "mockability" here? + Ok(MsgId::from(block.header.hash.0)) } fn channel_id(&self) -> ChannelId { From 530f7d77c44f193d116d5da33cd0d1084eceeff8 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 10 Jul 2026 13:31:12 +0300 Subject: [PATCH 10/32] feat!(sequencer): derive state from `ChainState` via `state()` method, remove the `state` attribute --- lez/chain_state/src/chain.rs | 7 +++ lez/sequencer/core/src/lib.rs | 97 ++++++++++++++++++--------------- lez/sequencer/core/src/tests.rs | 81 +++++++++++++++++---------- 3 files changed, 113 insertions(+), 72 deletions(-) diff --git a/lez/chain_state/src/chain.rs b/lez/chain_state/src/chain.rs index f0b4eddb..fe4e36c5 100644 --- a/lez/chain_state/src/chain.rs +++ b/lez/chain_state/src/chain.rs @@ -53,6 +53,13 @@ impl ChainState { &self.head_state } + /// Mutable access to the head state. Bypasses the `head_blocks` invariant, so + /// it is meant for tests and low-level callers. + #[must_use] + pub const fn head_state_mut(&mut self) -> &mut V03State { + &mut self.head_state + } + #[must_use] pub const fn final_state(&self) -> &V03State { &self.final_state diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 5efb8348..28808bc6 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -66,14 +66,12 @@ impl DepositMetadata { } pub struct SequencerCore { - state: lee::V03State, - /// Two-tier follow state fed by the publisher's `on_follow` sink. Shared with - /// the drive task. Passive today; production moves onto its head next. + /// Two-tier chain state: production builds on its head; the publisher's + /// `on_follow` sink feeds adopted/orphaned/finalized peer blocks into it. chain: Arc>, store: SequencerStore, mempool: MemPool<(TransactionOrigin, LeeTransaction)>, sequencer_config: SequencerConfig, - chain_height: u64, block_publisher: BP, } @@ -144,7 +142,7 @@ impl SequencerCore { .expect("Failed to read latest block meta from store"); let chain = Arc::new(Mutex::new(ChainState::from_final( - state.clone(), + state, Some(Tip { block_id: latest_block_meta.id, hash: latest_block_meta.hash, @@ -214,11 +212,9 @@ impl SequencerCore { } let sequencer_core = Self { - state, chain, store, mempool, - chain_height: latest_block_meta.id, sequencer_config: config, block_publisher, }; @@ -357,7 +353,10 @@ impl SequencerCore { } /// Feed one channel delta into the follow state: revert orphaned, apply - /// adopted, then finalize. Passive today — production still uses `self.state`. + /// adopted, then finalize. Production builds on this same head. + /// + /// TODO: persist adopted/finalized peer blocks to the store here so restart and RPC see the full canonical chain, not just our own blocks. + /// Open question: will peers communicate in another way? fn on_follow(chain: Arc>) -> block_publisher::OnFollowSink { Box::new(move |update: block_publisher::FollowUpdate| { let chain = Arc::clone(&chain); @@ -402,26 +401,27 @@ impl SequencerCore { // Apply our own block to the head with the MsgId the publish assigned it, // so the head advances and the later adopted redelivery dedups. - self.chain - .lock() - .expect("chain state mutex poisoned") - .apply_adopted(this_msg, &block); + let head_state = { + let mut chain = self.chain.lock().expect("chain state mutex poisoned"); + chain.apply_adopted(this_msg, &block); + chain.head_state().clone() + }; self.store.update( &block, &deposit_event_ids, withdrawal_reconciliation_keys, - &self.state, + &head_state, )?; - Ok(self.chain_height) + Ok(block.header.block_id) } /// Validates and applies a single mempool transaction to the current state. /// Returns `Ok(true)` if the transaction was valid and applied, `Ok(false)` if /// it was skipped due to validation failure. fn apply_mempool_transaction( - &mut self, + state: &mut lee::V03State, origin: TransactionOrigin, tx: &LeeTransaction, block_height: u64, @@ -432,11 +432,7 @@ impl SequencerCore { let tx_hash = tx.hash(); match origin { TransactionOrigin::User => { - let validated_diff = match tx.validate_on_state( - &self.state, - block_height, - timestamp, - ) { + let validated_diff = match tx.validate_on_state(state, block_height, timestamp) { Ok(diff) => diff, Err(err) => { error!( @@ -450,7 +446,7 @@ impl SequencerCore { withdrawals.push(withdraw_data); } - self.state.apply_state_diff(validated_diff); + state.apply_state_diff(validated_diff); } TransactionOrigin::Sequencer => { let LeeTransaction::Public(public_tx) = tx else { @@ -461,7 +457,7 @@ impl SequencerCore { deposit_event_ids.push(deposit_op_id); } - self.state + state .transition_from_public_transaction(public_tx, block_height, timestamp) .context("Failed to execute sequencer-generated transaction")?; } @@ -474,7 +470,18 @@ impl SequencerCore { fn build_block_from_mempool(&mut self) -> Result { let now = Instant::now(); - let new_block_height = self.next_block_id(); + // Build on the head: its tip is the parent, its state the validation base. + let (prev_block_hash, new_block_height, mut working_state) = { + let chain = self.chain.lock().expect("chain state mutex poisoned"); + let tip = chain.head_tip(); + let height = tip.as_ref().map_or(GENESIS_BLOCK_ID, |head| { + head.block_id + .checked_add(1) + .expect("block id should not overflow") + }); + let prev = tip.map_or(HashType([0; 32]), |head| head.hash); + (prev, height, chain.head_state().clone()) + }; let mut valid_transactions = Vec::new(); let mut deposit_event_ids = Vec::new(); @@ -483,11 +490,6 @@ impl SequencerCore { let max_block_size = usize::try_from(self.sequencer_config.max_block_size.as_u64()) .expect("`max_block_size` should fit into usize"); - let latest_block_meta = self - .store - .latest_block_meta() - .context("Failed to get latest block meta from store")?; - let new_block_timestamp = u64::try_from(chrono::Utc::now().timestamp_millis()) .expect("Timestamp must be positive"); @@ -506,7 +508,7 @@ impl SequencerCore { let temp_hashable_data = HashableBlockData { block_id: new_block_height, transactions: temp_valid_transactions, - prev_block_hash: latest_block_meta.hash, + prev_block_hash, timestamp: new_block_timestamp, }; @@ -523,7 +525,8 @@ impl SequencerCore { break; } - if self.apply_mempool_transaction( + if Self::apply_mempool_transaction( + &mut working_state, origin, &tx, new_block_height, @@ -539,7 +542,7 @@ impl SequencerCore { } } - self.state + working_state .transition_from_public_transaction(&clock_tx, new_block_height, new_block_timestamp) .context("Clock transaction failed. Aborting block production.")?; valid_transactions.push(clock_lee_tx); @@ -547,7 +550,7 @@ impl SequencerCore { let hashable_data = HashableBlockData { block_id: new_block_height, transactions: valid_transactions, - prev_block_hash: latest_block_meta.hash, + prev_block_hash, timestamp: new_block_timestamp, }; @@ -555,8 +558,6 @@ impl SequencerCore { .clone() .into_pending_block(self.store.signing_key()); - self.chain_height = new_block_height; - log::info!( "Created block with {} transactions in {} seconds", hashable_data.transactions.len(), @@ -570,16 +571,30 @@ impl SequencerCore { }) } - pub const fn state(&self) -> &lee::V03State { - &self.state + /// A clone of the current head state. + /// + /// TODO: cloning the whole state per call is wasteful; add targeted + /// account/nonce/proof accessors so RPC reads don't clone the whole state. + #[must_use] + pub fn state(&self) -> lee::V03State { + self.chain + .lock() + .expect("chain state mutex poisoned") + .head_state() + .clone() } pub const fn block_store(&self) -> &SequencerStore { &self.store } - pub const fn chain_height(&self) -> u64 { - self.chain_height + #[must_use] + pub fn chain_height(&self) -> u64 { + self.chain + .lock() + .expect("chain state mutex poisoned") + .head_tip() + .map_or(0, |tip| tip.block_id) } pub const fn sequencer_config(&self) -> &SequencerConfig { @@ -627,12 +642,6 @@ impl SequencerCore { pub fn chain(&self) -> Arc> { Arc::clone(&self.chain) } - - fn next_block_id(&self) -> u64 { - self.chain_height - .checked_add(1) - .unwrap_or_else(|| panic!("Max block height reached: {}", self.chain_height)) - } } struct BlockWithMeta { diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 05e2ec4c..816c6ce6 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -146,14 +146,14 @@ async fn start_from_config() { let (sequencer, _mempool_handle) = SequencerCoreWithMockClients::start_from_config(config.clone()).await; - assert_eq!(sequencer.chain_height, 1); + assert_eq!(sequencer.chain_height(), 1); assert_eq!(sequencer.sequencer_config.max_num_tx_in_block, 10); let acc1_account_id = initial_public_user_accounts()[0].account_id; let acc2_account_id = initial_public_user_accounts()[1].account_id; - let balance_acc_1 = sequencer.state.get_account_by_id(acc1_account_id).balance; - let balance_acc_2 = sequencer.state.get_account_by_id(acc2_account_id).balance; + let balance_acc_1 = sequencer.state().get_account_by_id(acc1_account_id).balance; + let balance_acc_2 = sequencer.state().get_account_by_id(acc2_account_id).balance; assert_eq!(10000, balance_acc_1); assert_eq!(20000, balance_acc_2); @@ -186,7 +186,7 @@ async fn start_from_config_opens_existing_db_if_it_exists() { let (sequencer, _mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; - assert_eq!(sequencer.chain_height, 1); + assert_eq!(sequencer.chain_height(), 1); assert!(sequencer.store.latest_block_meta().is_ok()); } @@ -295,7 +295,7 @@ async fn transaction_pre_check_native_transfer_valid() { #[tokio::test] async fn transaction_pre_check_native_transfer_other_signature() { - let (mut sequencer, _mempool_handle) = common_setup().await; + let (sequencer, _mempool_handle) = common_setup().await; let acc1 = initial_public_user_accounts()[0].account_id; let acc2 = initial_public_user_accounts()[1].account_id; @@ -309,7 +309,15 @@ async fn transaction_pre_check_native_transfer_other_signature() { let tx = tx.transaction_stateless_check().unwrap(); // Signature is not from sender. Execution fails - let result = tx.execute_check_on_state(&mut sequencer.state, 0, 0); + let result = tx.execute_check_on_state( + sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .head_state_mut(), + 0, + 0, + ); assert!(matches!( result, @@ -319,7 +327,7 @@ async fn transaction_pre_check_native_transfer_other_signature() { #[tokio::test] async fn transaction_pre_check_native_transfer_sent_too_much() { - let (mut sequencer, _mempool_handle) = common_setup().await; + let (sequencer, _mempool_handle) = common_setup().await; let acc1 = initial_public_user_accounts()[0].account_id; let acc2 = initial_public_user_accounts()[1].account_id; @@ -335,9 +343,15 @@ async fn transaction_pre_check_native_transfer_sent_too_much() { // Passed pre-check assert!(result.is_ok()); - let result = result - .unwrap() - .execute_check_on_state(&mut sequencer.state, 0, 0); + let result = result.unwrap().execute_check_on_state( + sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .head_state_mut(), + 0, + 0, + ); let is_failed_at_balance_mismatch = matches!( result.err().unwrap(), lee::error::LeeError::ProgramExecutionFailed(_) @@ -348,7 +362,7 @@ async fn transaction_pre_check_native_transfer_sent_too_much() { #[tokio::test] async fn transaction_execute_native_transfer() { - let (mut sequencer, _mempool_handle) = common_setup().await; + let (sequencer, _mempool_handle) = common_setup().await; let acc1 = initial_public_user_accounts()[0].account_id; let acc2 = initial_public_user_accounts()[1].account_id; @@ -359,11 +373,19 @@ async fn transaction_execute_native_transfer() { acc1, 0, acc2, 100, &sign_key1, ); - tx.execute_check_on_state(&mut sequencer.state, 0, 0) - .unwrap(); + tx.execute_check_on_state( + sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .head_state_mut(), + 0, + 0, + ) + .unwrap(); - let bal_from = sequencer.state.get_account_by_id(acc1).balance; - let bal_to = sequencer.state.get_account_by_id(acc2).balance; + let bal_from = sequencer.state().get_account_by_id(acc1).balance; + let bal_to = sequencer.state().get_account_by_id(acc2).balance; assert_eq!(bal_from, 9900); assert_eq!(bal_to, 20100); @@ -400,7 +422,7 @@ async fn push_tx_into_mempool_blocks_until_mempool_is_full() { #[tokio::test] async fn build_block_from_mempool() { let (mut sequencer, mempool_handle) = common_setup().await; - let genesis_height = sequencer.chain_height; + let genesis_height = sequencer.chain_height(); let tx = common::test_utils::produce_dummy_empty_transaction(); mempool_handle @@ -410,7 +432,7 @@ async fn build_block_from_mempool() { let result = sequencer.build_block_from_mempool(); assert!(result.is_ok()); - assert_eq!(sequencer.chain_height, genesis_height + 1); + assert_eq!(sequencer.chain_height(), genesis_height + 1); } #[tokio::test] @@ -442,7 +464,7 @@ async fn replay_transactions_are_rejected_in_the_same_block() { sequencer.produce_new_block().await.unwrap(); let block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); @@ -477,7 +499,7 @@ async fn replay_transactions_are_rejected_in_different_blocks() { sequencer.produce_new_block().await.unwrap(); let block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); assert_eq!( @@ -496,7 +518,7 @@ async fn replay_transactions_are_rejected_in_different_blocks() { sequencer.produce_new_block().await.unwrap(); let block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); // The replay is rejected, so only the clock tx is in the block. @@ -538,7 +560,7 @@ async fn restart_from_storage() { sequencer.produce_new_block().await.unwrap(); let block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); assert_eq!( @@ -554,8 +576,8 @@ async fn restart_from_storage() { // with the above transaction and update the state to reflect that. let (sequencer, _mempool_handle) = SequencerCoreWithMockClients::start_from_config(config.clone()).await; - let balance_acc_1 = sequencer.state.get_account_by_id(acc1_account_id).balance; - let balance_acc_2 = sequencer.state.get_account_by_id(acc2_account_id).balance; + let balance_acc_1 = sequencer.state().get_account_by_id(acc1_account_id).balance; + let balance_acc_2 = sequencer.state().get_account_by_id(acc2_account_id).balance; // Balances should be consistent with the stored block assert_eq!( @@ -653,7 +675,7 @@ async fn produce_block_with_correct_prev_meta_after_restart() { // Step 5: Verify the new block has correct previous block metadata let new_block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); @@ -705,7 +727,7 @@ async fn transactions_touching_clock_account_are_dropped_from_block() { let block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); @@ -760,7 +782,7 @@ async fn user_tx_that_chain_calls_clock_is_dropped() { let block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); @@ -779,10 +801,13 @@ async fn block_production_aborts_when_clock_account_data_is_corrupted() { // Corrupt the clock 01 account data so the clock program panics on deserialization. let clock_account_id = system_accounts::clock_account_ids()[0]; - let mut corrupted = sequencer.state.get_account_by_id(clock_account_id); + let mut corrupted = sequencer.state().get_account_by_id(clock_account_id); corrupted.data = vec![0xff; 3].try_into().unwrap(); sequencer - .state + .chain() + .lock() + .expect("chain mutex poisoned") + .head_state_mut() .force_insert_account(clock_account_id, corrupted); // Push a dummy transaction so the mempool is non-empty. From 82bb45461510494685b722d6112163b22af80c94 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 10 Jul 2026 13:32:07 +0300 Subject: [PATCH 11/32] chore: formatting --- lez/sequencer/core/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 28808bc6..0fa93471 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -355,8 +355,9 @@ impl SequencerCore { /// Feed one channel delta into the follow state: revert orphaned, apply /// adopted, then finalize. Production builds on this same head. /// - /// TODO: persist adopted/finalized peer blocks to the store here so restart and RPC see the full canonical chain, not just our own blocks. - /// Open question: will peers communicate in another way? + /// TODO: persist adopted/finalized peer blocks to the store here so restart and RPC see the + /// full canonical chain, not just our own blocks. Open question: will peers communicate in + /// another way? fn on_follow(chain: Arc>) -> block_publisher::OnFollowSink { Box::new(move |update: block_publisher::FollowUpdate| { let chain = Arc::clone(&chain); From cb23561b3a65524cd623cec1bc6f4a94c29fcad4 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 10 Jul 2026 14:46:34 +0300 Subject: [PATCH 12/32] feat(sequencer): store adopted and finalized blocks on disk within `on_follow` --- lez/sequencer/core/src/lib.rs | 74 ++++++++++++++++++++++++-------- lez/storage/src/sequencer/mod.rs | 28 ++++++++++++ 2 files changed, 83 insertions(+), 19 deletions(-) diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 0fa93471..3e8d9b78 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -6,7 +6,7 @@ use std::{ use anyhow::{Context as _, Result, anyhow}; use borsh::BorshDeserialize; -use chain_state::{ChainState, Tip}; +use chain_state::{AcceptOutcome, ChainState, Tip}; use common::{ HashType, block::{BedrockStatus, Block, HashableBlockData}, @@ -166,7 +166,7 @@ impl SequencerCore { Self::on_finalized_block(store.dbio()), Self::on_deposit_event(store.dbio(), mempool_handle.clone()), Self::on_withdraw_event(store.dbio()), - Self::on_follow(Arc::clone(&chain)), + Self::on_follow(store.dbio(), Arc::clone(&chain)), ) .await .expect("Failed to initialize Block Publisher"); @@ -352,27 +352,63 @@ impl SequencerCore { }) } - /// Feed one channel delta into the follow state: revert orphaned, apply - /// adopted, then finalize. Production builds on this same head. - /// - /// TODO: persist adopted/finalized peer blocks to the store here so restart and RPC see the - /// full canonical chain, not just our own blocks. Open question: will peers communicate in - /// another way? - fn on_follow(chain: Arc>) -> block_publisher::OnFollowSink { + /// Feed one channel delta into the follow state and mirror it to the store: + /// revert orphaned, then apply and persist adopted and finalized blocks. + /// Production builds on this same head. + fn on_follow( + dbio: Arc, + chain: Arc>, + ) -> block_publisher::OnFollowSink { Box::new(move |update: block_publisher::FollowUpdate| { let chain = Arc::clone(&chain); + let dbio = Arc::clone(&dbio); Box::pin(async move { - let mut chain = chain.lock().expect("chain state mutex poisoned"); - for (this_msg, _) in &update.orphaned { - chain.revert_orphan(*this_msg); + // Apply under the lock and collect what to persist; take a single + // head snapshot. Release the lock before touching disk so the + // producer is never blocked on the follow path's I/O. + let (adopted, finalized, head_snapshot) = { + let mut chain = chain.lock().expect("chain state mutex poisoned"); + for (this_msg, _) in &update.orphaned { + chain.revert_orphan(*this_msg); + } + let mut adopted = Vec::new(); + for (this_msg, block) in &update.adopted { + if matches!( + chain.apply_adopted(*this_msg, block), + AcceptOutcome::Applied + ) { + adopted.push(block); + } + } + let mut finalized = Vec::new(); + for (this_msg, block) in &update.finalized { + // TODO: thread the finalized inscription's L1 slot once the + // sdk surfaces it; only used for the invalid-finalized stall. + if matches!( + chain.apply_finalized(*this_msg, block, Slot::from(0)), + AcceptOutcome::Applied + ) { + finalized.push(block); + } + } + (adopted, finalized, chain.head_state().clone()) + }; + + for block in adopted { + if let Err(err) = dbio.store_followed_block(block, &head_snapshot, false) { + error!( + "Failed to persist adopted block {}: {err:#}", + block.header.block_id + ); + } } - for (this_msg, block) in &update.adopted { - chain.apply_adopted(*this_msg, block); - } - for (this_msg, block) in &update.finalized { - // TODO: thread the finalized inscription's L1 slot once the sdk - // surfaces it; only used for the rare invalid-finalized stall. - chain.apply_finalized(*this_msg, block, Slot::from(0)); + for block in finalized { + if let Err(err) = dbio.store_followed_block(block, &head_snapshot, true) { + error!( + "Failed to persist finalized block {}: {err:#}", + block.header.block_id + ); + } } }) }) diff --git a/lez/storage/src/sequencer/mod.rs b/lez/storage/src/sequencer/mod.rs index 1a3a1886..1e2feca2 100644 --- a/lez/storage/src/sequencer/mod.rs +++ b/lez/storage/src/sequencer/mod.rs @@ -612,6 +612,34 @@ impl RocksDBIO { Ok(()) } + /// Persists a followed (peer) block so the store mirrors the canonical chain. + /// + /// Skips the write when the store already holds this block (matched by id and + /// hash) — covering our own blocks and re-deliveries — but still marks it + /// finalized when `finalized` is set. + /// + /// When `finalized`, the block is marked so backfilled blocks (which arrive + /// without a prior pending write) are not left pending. + pub fn store_followed_block( + &self, + block: &Block, + state: &V03State, + finalized: bool, + ) -> DbResult<()> { + let block_id = block.header.block_id; + let already_stored = self + .get_block(block_id)? + .is_some_and(|stored| stored.header.hash == block.header.hash); + + if !already_stored { + self.atomic_update(block, &[], vec![], state)?; + } + if finalized { + self.mark_block_as_finalized(block_id)?; + } + Ok(()) + } + pub fn get_all_blocks(&self) -> impl Iterator> { let cf_block = self.block_column(); self.db From 0046a1c822791d379620fd9603c5bf89c9908570 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 10 Jul 2026 15:54:37 +0300 Subject: [PATCH 13/32] feat(sequencer): now keep `orphaned` in mempool + use `with_state` to avoid clone for method calls to `V03State` --- lez/sequencer/core/src/lib.rs | 48 ++++++++++++++++++++-------- lez/sequencer/core/src/tests.rs | 14 ++++---- lez/sequencer/service/src/service.rs | 29 +++++++++-------- 3 files changed, 58 insertions(+), 33 deletions(-) diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 3e8d9b78..8ea7ffe0 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -166,7 +166,7 @@ impl SequencerCore { Self::on_finalized_block(store.dbio()), Self::on_deposit_event(store.dbio(), mempool_handle.clone()), Self::on_withdraw_event(store.dbio()), - Self::on_follow(store.dbio(), Arc::clone(&chain)), + Self::on_follow(store.dbio(), Arc::clone(&chain), mempool_handle.clone()), ) .await .expect("Failed to initialize Block Publisher"); @@ -358,18 +358,22 @@ impl SequencerCore { fn on_follow( dbio: Arc, chain: Arc>, + mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>, ) -> block_publisher::OnFollowSink { Box::new(move |update: block_publisher::FollowUpdate| { let chain = Arc::clone(&chain); let dbio = Arc::clone(&dbio); + let mempool_handle = mempool_handle.clone(); Box::pin(async move { // Apply under the lock and collect what to persist; take a single // head snapshot. Release the lock before touching disk so the // producer is never blocked on the follow path's I/O. - let (adopted, finalized, head_snapshot) = { + let (adopted, finalized, resubmit_txs, head_snapshot) = { let mut chain = chain.lock().expect("chain state mutex poisoned"); - for (this_msg, _) in &update.orphaned { + let mut resubmit_txs = Vec::new(); + for (this_msg, block) in &update.orphaned { chain.revert_orphan(*this_msg); + resubmit_txs.extend(resubmittable_txs(block)); } let mut adopted = Vec::new(); for (this_msg, block) in &update.adopted { @@ -391,7 +395,7 @@ impl SequencerCore { finalized.push(block); } } - (adopted, finalized, chain.head_state().clone()) + (adopted, finalized, resubmit_txs, chain.head_state().clone()) }; for block in adopted { @@ -410,6 +414,14 @@ impl SequencerCore { ); } } + + // Rebuild orphaned work: return its user txs to the mempool so the + // next on-turn production re-includes them on the new head. + for tx in resubmit_txs { + if let Err(err) = mempool_handle.push((TransactionOrigin::User, tx)).await { + error!("Failed to resubmit orphaned transaction: {err:#}"); + } + } }) }) } @@ -608,17 +620,14 @@ impl SequencerCore { }) } - /// A clone of the current head state. - /// - /// TODO: cloning the whole state per call is wasteful; add targeted - /// account/nonce/proof accessors so RPC reads don't clone the whole state. - #[must_use] - pub fn state(&self) -> lee::V03State { - self.chain + /// Reads the current head state under the lock without cloning it, so callers + /// reuse `V03State`'s own API (accounts, nonces, proofs) with no whole-state copy. + pub fn with_state(&self, f: impl FnOnce(&lee::V03State) -> R) -> R { + f(self + .chain .lock() .expect("chain state mutex poisoned") - .head_state() - .clone() + .head_state()) } pub const fn block_store(&self) -> &SequencerStore { @@ -880,6 +889,19 @@ fn build_bridge_deposit_tx_from_event(event: &PendingDepositEventRecord) -> Resu ))) } +/// User transactions of an orphaned block to return to the mempool: everything +/// except the trailing clock tx and sequencer-generated bridge deposits (those are +/// replayed from their own bedrock events, not the mempool). +fn resubmittable_txs(block: &Block) -> Vec { + let Some((_clock, rest)) = block.body.transactions.split_last() else { + return Vec::new(); + }; + rest.iter() + .filter(|tx| extract_bridge_deposit_id(tx).is_none()) + .cloned() + .collect() +} + #[must_use] fn extract_bridge_deposit_id(tx: &LeeTransaction) -> Option { let LeeTransaction::Public(tx) = tx else { diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 816c6ce6..542f1bfb 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -152,8 +152,8 @@ async fn start_from_config() { let acc1_account_id = initial_public_user_accounts()[0].account_id; let acc2_account_id = initial_public_user_accounts()[1].account_id; - let balance_acc_1 = sequencer.state().get_account_by_id(acc1_account_id).balance; - let balance_acc_2 = sequencer.state().get_account_by_id(acc2_account_id).balance; + let balance_acc_1 = sequencer.with_state(|s| s.get_account_by_id(acc1_account_id).balance); + let balance_acc_2 = sequencer.with_state(|s| s.get_account_by_id(acc2_account_id).balance); assert_eq!(10000, balance_acc_1); assert_eq!(20000, balance_acc_2); @@ -384,8 +384,8 @@ async fn transaction_execute_native_transfer() { ) .unwrap(); - let bal_from = sequencer.state().get_account_by_id(acc1).balance; - let bal_to = sequencer.state().get_account_by_id(acc2).balance; + let bal_from = sequencer.with_state(|s| s.get_account_by_id(acc1).balance); + let bal_to = sequencer.with_state(|s| s.get_account_by_id(acc2).balance); assert_eq!(bal_from, 9900); assert_eq!(bal_to, 20100); @@ -576,8 +576,8 @@ async fn restart_from_storage() { // with the above transaction and update the state to reflect that. let (sequencer, _mempool_handle) = SequencerCoreWithMockClients::start_from_config(config.clone()).await; - let balance_acc_1 = sequencer.state().get_account_by_id(acc1_account_id).balance; - let balance_acc_2 = sequencer.state().get_account_by_id(acc2_account_id).balance; + let balance_acc_1 = sequencer.with_state(|s| s.get_account_by_id(acc1_account_id).balance); + let balance_acc_2 = sequencer.with_state(|s| s.get_account_by_id(acc2_account_id).balance); // Balances should be consistent with the stored block assert_eq!( @@ -801,7 +801,7 @@ async fn block_production_aborts_when_clock_account_data_is_corrupted() { // Corrupt the clock 01 account data so the clock program panics on deserialization. let clock_account_id = system_accounts::clock_account_ids()[0]; - let mut corrupted = sequencer.state().get_account_by_id(clock_account_id); + let mut corrupted = sequencer.with_state(|s| s.get_account_by_id(clock_account_id)); corrupted.data = vec![0xff; 3].try_into().unwrap(); sequencer .chain() diff --git a/lez/sequencer/service/src/service.rs b/lez/sequencer/service/src/service.rs index d0850837..f4b09b43 100644 --- a/lez/sequencer/service/src/service.rs +++ b/lez/sequencer/service/src/service.rs @@ -138,8 +138,8 @@ impl sequencer_service_rpc::RpcServer async fn get_account_balance(&self, account_id: AccountId) -> Result { let sequencer = self.sequencer.lock().await; - let account = sequencer.state().get_account_by_id(account_id); - Ok(account.balance) + let balance = sequencer.with_state(|state| state.get_account_by_id(account_id).balance); + Ok(balance) } async fn get_transaction( @@ -155,10 +155,12 @@ impl sequencer_service_rpc::RpcServer account_ids: Vec, ) -> Result, ErrorObjectOwned> { let sequencer = self.sequencer.lock().await; - let nonces = account_ids - .into_iter() - .map(|account_id| sequencer.state().get_account_by_id(account_id).nonce) - .collect(); + let nonces = sequencer.with_state(|state| { + account_ids + .into_iter() + .map(|account_id| state.get_account_by_id(account_id).nonce) + .collect() + }); Ok(nonces) } @@ -167,17 +169,18 @@ impl sequencer_service_rpc::RpcServer commitments: Vec, ) -> Result<(Vec>, CommitmentSetDigest), ErrorObjectOwned> { let sequencer = self.sequencer.lock().await; - let state = sequencer.state(); - let proofs = commitments - .iter() - .map(|commitment| state.get_proof_for_commitment(commitment)) - .collect(); - Ok((proofs, state.commitment_root())) + Ok(sequencer.with_state(|state| { + let proofs = commitments + .iter() + .map(|commitment| state.get_proof_for_commitment(commitment)) + .collect(); + (proofs, state.commitment_root()) + })) } async fn get_account(&self, account_id: AccountId) -> Result { let sequencer = self.sequencer.lock().await; - Ok(sequencer.state().get_account_by_id(account_id)) + Ok(sequencer.with_state(|state| state.get_account_by_id(account_id))) } async fn get_program_ids(&self) -> Result, ErrorObjectOwned> { From a0f7957468b27be66837b3637b9876207f57f547 Mon Sep 17 00:00:00 2001 From: erhant Date: Fri, 10 Jul 2026 22:43:41 +0300 Subject: [PATCH 14/32] chore: add TODO doc about transient apply failures [skip ci] --- lez/chain_state/DESIGN.md | 4 ---- lez/chain_state/src/apply.rs | 3 +++ lez/sequencer/core/src/lib.rs | 6 ++++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lez/chain_state/DESIGN.md b/lez/chain_state/DESIGN.md index 600fb23f..35e9a2b9 100644 --- a/lez/chain_state/DESIGN.md +++ b/lez/chain_state/DESIGN.md @@ -6,10 +6,6 @@ backs decentralized sequencing. Status: **interface freeze** — the contract the produce-on-turn and follow-blocks tracks build against. Changing them after the tracks split forces rework in both. -Branch: `erhant/lez-two-tip-chain-state` (off `erhant/indexer-recoverable-invalid-blocks`). - ---- - ## 1. Motivation Decentralized sequencing requires every honest node — sequencer or indexer — to diff --git a/lez/chain_state/src/apply.rs b/lez/chain_state/src/apply.rs index 42a77b35..5e2ff6f1 100644 --- a/lez/chain_state/src/apply.rs +++ b/lez/chain_state/src/apply.rs @@ -30,6 +30,9 @@ pub enum AcceptOutcome { /// Chained but failed to apply, possibly transiently /// ([`BlockIngestError::is_retryable`]); nothing recorded, tip and state /// untouched. The caller retries and parks once it gives up. + /// + /// TODO: Only the indexer's `accept_block` emits this today; the sequencer's + /// `ChainState` parks on all failures without retrying (see `on_follow`). RetryableFailure(BlockIngestError), } diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 8ea7ffe0..36689627 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -355,6 +355,12 @@ impl SequencerCore { /// Feed one channel delta into the follow state and mirror it to the store: /// revert orphaned, then apply and persist adopted and finalized blocks. /// Production builds on this same head. + /// + /// TODO: unlike the indexer's ingest loop, this path does not retry + /// `is_retryable` (transient) apply failures — a failed block just parks and + /// relies on a valid successor or a restart. `ChainState` never emits + /// `AcceptOutcome::RetryableFailure` yet; adding retry parity here is a + /// follow-up. fn on_follow( dbio: Arc, chain: Arc>, From 0b6a9477138475f59e514562724011477c436fc8 Mon Sep 17 00:00:00 2001 From: erhant Date: Mon, 13 Jul 2026 11:09:29 +0300 Subject: [PATCH 15/32] fix(sequencer): correct test assertion --- lez/sequencer/core/src/tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 542f1bfb..158e5b17 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -432,7 +432,8 @@ async fn build_block_from_mempool() { let result = sequencer.build_block_from_mempool(); assert!(result.is_ok()); - assert_eq!(sequencer.chain_height(), genesis_height + 1); + // Building itself does not advance the head; only apply-after-publish does. + assert_eq!(sequencer.chain_height(), genesis_height); } #[tokio::test] From 0612132bed03ec8acd900076a162558865e79a6e Mon Sep 17 00:00:00 2001 From: erhant Date: Mon, 13 Jul 2026 13:22:17 +0300 Subject: [PATCH 16/32] test(chain_state,sequencer): add unit tests for `chain_state` --- Cargo.lock | 1 + lez/chain_state/Cargo.toml | 3 + lez/chain_state/src/chain.rs | 260 ++++++++++++++++++++++++++- lez/sequencer/core/src/lib.rs | 153 ++++++++-------- lez/sequencer/core/src/tests.rs | 280 ++++++++++++++++++++++++++++- lez/storage/src/sequencer/mod.rs | 3 + lez/storage/src/sequencer/tests.rs | 108 +++++++++++ 7 files changed, 729 insertions(+), 79 deletions(-) create mode 100644 lez/storage/src/sequencer/tests.rs diff --git a/Cargo.lock b/Cargo.lock index ca7cea65..fcdcec14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1373,6 +1373,7 @@ name = "chain_state" version = "0.1.0" dependencies = [ "anyhow", + "borsh", "common", "lee", "logos-blockchain-core", diff --git a/lez/chain_state/Cargo.toml b/lez/chain_state/Cargo.toml index 1fcc052e..ca9876ab 100644 --- a/lez/chain_state/Cargo.toml +++ b/lez/chain_state/Cargo.toml @@ -19,4 +19,7 @@ thiserror.workspace = true [dev-dependencies] testnet_initial_state.workspace = true +# we use borsh to compare byte-to-byte matching within tests +# (it sorts hashmap's before serialization, so we can compare two states for equality) +borsh.workspace = true serde_json.workspace = true diff --git a/lez/chain_state/src/chain.rs b/lez/chain_state/src/chain.rs index fe4e36c5..07c9890b 100644 --- a/lez/chain_state/src/chain.rs +++ b/lez/chain_state/src/chain.rs @@ -18,14 +18,16 @@ pub struct HeadEntry { } /// The head tier (reorg-able, from `adopted`/`orphaned`) over the final tier -/// (irreversible, from `finalized`). `head_state == final_state` replayed -/// through `head_blocks`. +/// (irreversible, from `finalized`). +/// +/// `head_state` is given by `final_state` replayed through `head_blocks`. pub struct ChainState { final_state: V03State, final_tip: Option, + final_stall: Option, + head_state: V03State, head_blocks: Vec, - final_stall: Option, } impl ChainState { @@ -248,7 +250,10 @@ const fn stall_for(block: &Block, l1_slot: Slot, error: BlockIngestError) -> Sta #[cfg(test)] mod tests { - use common::test_utils::{create_transaction_native_token_transfer, produce_dummy_block}; + use common::{ + HashType, + test_utils::{create_transaction_native_token_transfer, produce_dummy_block}, + }; use testnet_initial_state::{initial_pub_accounts_private_keys, initial_state}; use super::*; @@ -261,6 +266,21 @@ mod tests { Slot::from(n) } + /// `head_state` equals `final_state` replayed through `head_blocks`. + fn assert_head_matches_replay(chain: &ChainState) { + let mut state = chain.final_state.clone(); + let mut tip = chain.final_tip.clone(); + for entry in &chain.head_blocks { + apply_block(tip.as_ref(), &entry.block, &mut state).expect("head blocks must replay"); + tip = Some(tip_of(&entry.block)); + } + assert_eq!( + borsh::to_vec(&state).expect("state serializes"), + borsh::to_vec(chain.head_state()).expect("state serializes"), + "head_state must equal final_state replayed through head_blocks" + ); + } + #[test] fn adopted_blocks_advance_head() { let mut chain = ChainState::new(initial_state()); @@ -403,6 +423,238 @@ mod tests { assert_eq!(stall.block_id, Some(3)); } + #[test] + fn orphaning_a_suffix_rederives_head_state() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + + let tx2 = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2]); + chain.apply_adopted(msg(2), &block2); + let tx3 = create_transaction_native_token_transfer(from, 1, to, 10, &sign_key); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![tx3]); + chain.apply_adopted(msg(3), &block3); + let tx4 = create_transaction_native_token_transfer(from, 2, to, 10, &sign_key); + let block4 = produce_dummy_block(4, Some(block3.header.hash), vec![tx4]); + chain.apply_adopted(msg(4), &block4); + + // Orphaning block 3 drops the whole suffix (3 and 4). + chain.revert_orphan(msg(3)); + + assert_eq!(chain.head_tip().expect("head tip").block_id, 2); + assert_eq!(chain.head_state().get_account_by_id(from).balance, 9990); + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20010); + assert_head_matches_replay(&chain); + } + + #[test] + fn channel_update_replaces_multi_block_suffix() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + + let tx2 = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2]); + chain.apply_adopted(msg(2), &block2); + let tx3 = create_transaction_native_token_transfer(from, 1, to, 10, &sign_key); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![tx3]); + chain.apply_adopted(msg(3), &block3); + let tx4 = create_transaction_native_token_transfer(from, 2, to, 10, &sign_key); + let block4 = produce_dummy_block(4, Some(block3.header.hash), vec![tx4]); + chain.apply_adopted(msg(4), &block4); + + // A competing branch replaces blocks 3 and 4; orphans arrive unordered. + let tx3_prime = create_transaction_native_token_transfer(from, 1, to, 20, &sign_key); + let block3_prime = produce_dummy_block(3, Some(block2.header.hash), vec![tx3_prime]); + let tx4_prime = create_transaction_native_token_transfer(from, 2, to, 30, &sign_key); + let block4_prime = produce_dummy_block(4, Some(block3_prime.header.hash), vec![tx4_prime]); + + let outcomes = chain.apply_channel_update( + &[msg(4), msg(3)], + &[(msg(13), block3_prime), (msg(14), block4_prime)], + ); + + assert!(matches!( + outcomes.as_slice(), + [AcceptOutcome::Applied, AcceptOutcome::Applied] + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 4); + assert_eq!(chain.head_state().get_account_by_id(from).balance, 9940); + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20060); + assert_head_matches_replay(&chain); + } + + #[test] + fn channel_update_ignores_unknown_orphan() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + let outcomes = chain.apply_channel_update(&[msg(99)], &[(msg(3), block3)]); + + assert!(matches!(outcomes.as_slice(), [AcceptOutcome::Applied])); + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + assert_head_matches_replay(&chain); + } + + #[test] + fn finalized_reinscription_matches_by_block_hash() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + chain.apply_adopted(msg(3), &block3); + + // Block 2 finalizes re-inscribed under a fresh MsgId: matched by hash, + // finalized through, and the head above it survives. + assert!(matches!( + chain.apply_finalized(msg(42), &block2, slot(5)), + AcceptOutcome::Applied + )); + assert_eq!(chain.final_tip().expect("final tip").block_id, 2); + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + assert!(chain.final_stall().is_none()); + assert_head_matches_replay(&chain); + } + + #[test] + fn finalize_through_preserves_head_state_and_advances_final_state() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + let tx2 = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2]); + chain.apply_adopted(msg(2), &block2); + let tx3 = create_transaction_native_token_transfer(from, 1, to, 10, &sign_key); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![tx3]); + chain.apply_adopted(msg(3), &block3); + + chain.apply_finalized(msg(2), &block2, slot(10)); + + // Head still reflects both transfers + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20020); + // ...while final reflects only the finalized prefix. + assert_eq!(chain.final_state().get_account_by_id(to).balance, 20010); + assert_head_matches_replay(&chain); + } + + #[test] + fn head_self_heals_with_valid_competitor_after_park() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + + // Correct id, wrong parent: parked, head frozen at 1, no stall. + let bad = produce_dummy_block(2, Some(HashType([9; 32])), vec![]); + assert!(matches!( + chain.apply_adopted(msg(2), &bad), + AcceptOutcome::Parked(BlockIngestError::BrokenChainLink { .. }) + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 1); + assert!(chain.final_stall().is_none()); + + // A valid competitor at the same height applies without any reorg event. + let good = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + assert!(matches!( + chain.apply_adopted(msg(12), &good), + AcceptOutcome::Applied + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 2); + assert_head_matches_replay(&chain); + } + + #[test] + fn repeated_invalid_finalized_bumps_orphans_since() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_finalized(msg(1), &genesis, slot(10)); + + let bad3 = produce_dummy_block(3, Some(genesis.header.hash), vec![]); + chain.apply_finalized(msg(3), &bad3, slot(20)); + let bad5 = produce_dummy_block(5, Some(bad3.header.hash), vec![]); + assert!(matches!( + chain.apply_finalized(msg(5), &bad5, slot(30)), + AcceptOutcome::Parked(_) + )); + + let stall = chain.final_stall().expect("final stall recorded"); + assert_eq!(stall.block_id, Some(3), "first stall reason is preserved"); + assert_eq!(stall.orphans_since, 1); + } + + #[test] + fn valid_finalized_successor_clears_final_stall() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_finalized(msg(1), &genesis, slot(10)); + + let bad = produce_dummy_block(3, Some(genesis.header.hash), vec![]); + chain.apply_finalized(msg(3), &bad, slot(20)); + assert!(chain.final_stall().is_some()); + + // The valid successor of the frozen final tip finalizes: stall clears. + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + assert!(matches!( + chain.apply_finalized(msg(2), &block2, slot(30)), + AcceptOutcome::Applied + )); + assert!(chain.final_stall().is_none()); + assert_eq!(chain.final_tip().expect("final tip").block_id, 2); + assert_head_matches_replay(&chain); + } + + #[test] + fn finalized_unknown_block_rebases_head() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_finalized(msg(1), &genesis, slot(10)); + + // Head advances on a competing branch… + let block2a = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + chain.apply_adopted(msg(2), &block2a); + + // …but a different block 2 finalizes. The finalized chain is + // authoritative, so head rebases onto it. + let tx = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2b = produce_dummy_block(2, Some(genesis.header.hash), vec![tx]); + assert!(matches!( + chain.apply_finalized(msg(22), &block2b, slot(20)), + AcceptOutcome::Applied + )); + + assert_eq!(chain.final_tip().expect("final tip").block_id, 2); + assert_eq!(chain.head_tip().expect("head tip").block_id, 2); + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20010); + assert_head_matches_replay(&chain); + } + #[test] fn head_state_reflects_applied_transfers() { let accounts = initial_pub_accounts_private_keys(); diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 36689627..1cc1ed4b 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -352,83 +352,19 @@ impl SequencerCore { }) } - /// Feed one channel delta into the follow state and mirror it to the store: - /// revert orphaned, then apply and persist adopted and finalized blocks. - /// Production builds on this same head. - /// - /// TODO: unlike the indexer's ingest loop, this path does not retry - /// `is_retryable` (transient) apply failures — a failed block just parks and - /// relies on a valid successor or a restart. `ChainState` never emits - /// `AcceptOutcome::RetryableFailure` yet; adding retry parity here is a - /// follow-up. + /// Publisher sink adapter over [`apply_follow_update`]. fn on_follow( dbio: Arc, chain: Arc>, mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>, ) -> block_publisher::OnFollowSink { Box::new(move |update: block_publisher::FollowUpdate| { - let chain = Arc::clone(&chain); - let dbio = Arc::clone(&dbio); - let mempool_handle = mempool_handle.clone(); - Box::pin(async move { - // Apply under the lock and collect what to persist; take a single - // head snapshot. Release the lock before touching disk so the - // producer is never blocked on the follow path's I/O. - let (adopted, finalized, resubmit_txs, head_snapshot) = { - let mut chain = chain.lock().expect("chain state mutex poisoned"); - let mut resubmit_txs = Vec::new(); - for (this_msg, block) in &update.orphaned { - chain.revert_orphan(*this_msg); - resubmit_txs.extend(resubmittable_txs(block)); - } - let mut adopted = Vec::new(); - for (this_msg, block) in &update.adopted { - if matches!( - chain.apply_adopted(*this_msg, block), - AcceptOutcome::Applied - ) { - adopted.push(block); - } - } - let mut finalized = Vec::new(); - for (this_msg, block) in &update.finalized { - // TODO: thread the finalized inscription's L1 slot once the - // sdk surfaces it; only used for the invalid-finalized stall. - if matches!( - chain.apply_finalized(*this_msg, block, Slot::from(0)), - AcceptOutcome::Applied - ) { - finalized.push(block); - } - } - (adopted, finalized, resubmit_txs, chain.head_state().clone()) - }; - - for block in adopted { - if let Err(err) = dbio.store_followed_block(block, &head_snapshot, false) { - error!( - "Failed to persist adopted block {}: {err:#}", - block.header.block_id - ); - } - } - for block in finalized { - if let Err(err) = dbio.store_followed_block(block, &head_snapshot, true) { - error!( - "Failed to persist finalized block {}: {err:#}", - block.header.block_id - ); - } - } - - // Rebuild orphaned work: return its user txs to the mempool so the - // next on-turn production re-includes them on the new head. - for tx in resubmit_txs { - if let Err(err) = mempool_handle.push((TransactionOrigin::User, tx)).await { - error!("Failed to resubmit orphaned transaction: {err:#}"); - } - } - }) + Box::pin(apply_follow_update( + Arc::clone(&dbio), + Arc::clone(&chain), + mempool_handle.clone(), + update, + )) }) } @@ -702,6 +638,81 @@ struct BlockWithMeta { withdrawals: Vec, } +/// Feed one channel delta into the follow state and mirror it to the store: +/// revert orphaned, then apply and persist adopted and finalized blocks. +/// Production builds on this same head. Wired to the publisher via +/// [`SequencerCore::on_follow`]; a free function so tests can drive it directly. +/// +/// TODO: unlike the indexer's ingest loop, this path does not retry +/// `is_retryable` (transient) apply failures — a failed block just parks and +/// relies on a valid successor or a restart. `ChainState` never emits +/// `AcceptOutcome::RetryableFailure` yet; adding retry parity here is a +/// follow-up. +async fn apply_follow_update( + dbio: Arc, + chain: Arc>, + mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>, + update: block_publisher::FollowUpdate, +) { + // Apply under the lock and collect what to persist; take a single + // head snapshot. Release the lock before touching disk so the + // producer is never blocked on the follow path's I/O. + let (adopted, finalized, resubmit_txs, head_snapshot) = { + let mut chain = chain.lock().expect("chain state mutex poisoned"); + let mut resubmit_txs = Vec::new(); + for (this_msg, block) in &update.orphaned { + chain.revert_orphan(*this_msg); + resubmit_txs.extend(resubmittable_txs(block)); + } + let mut adopted = Vec::new(); + for (this_msg, block) in &update.adopted { + if matches!( + chain.apply_adopted(*this_msg, block), + AcceptOutcome::Applied + ) { + adopted.push(block); + } + } + let mut finalized = Vec::new(); + for (this_msg, block) in &update.finalized { + // TODO: thread the finalized inscription's L1 slot once the + // sdk surfaces it; only used for the invalid-finalized stall. + if matches!( + chain.apply_finalized(*this_msg, block, Slot::from(0)), + AcceptOutcome::Applied + ) { + finalized.push(block); + } + } + (adopted, finalized, resubmit_txs, chain.head_state().clone()) + }; + + for block in adopted { + if let Err(err) = dbio.store_followed_block(block, &head_snapshot, false) { + error!( + "Failed to persist adopted block {}: {err:#}", + block.header.block_id + ); + } + } + for block in finalized { + if let Err(err) = dbio.store_followed_block(block, &head_snapshot, true) { + error!( + "Failed to persist finalized block {}: {err:#}", + block.header.block_id + ); + } + } + + // Rebuild orphaned work: return its user txs to the mempool so the + // next on-turn production re-includes them on the new head. + for tx in resubmit_txs { + if let Err(err) = mempool_handle.push((TransactionOrigin::User, tx)).await { + error!("Failed to resubmit orphaned transaction: {err:#}"); + } + } +} + /// Checks the database for any pending deposit events that have not yet been marked as submitted in /// a block, and re-queues them in the mempool in a separate async task for inclusion in the next /// block. diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 158e5b17..f78e8ef4 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -4,7 +4,7 @@ use std::{pin::pin, time::Duration}; use common::{ HashType, - block::HashableBlockData, + block::{BedrockStatus, HashableBlockData}, test_utils::sequencer_sign_key_for_testing, transaction::{LeeTransaction, clock_invocation}, }; @@ -22,19 +22,21 @@ use lee_core::{ account::{AccountWithMetadata, Nonce}, program::PdaSeed, }; -use logos_blockchain_core::mantle::ops::channel::ChannelId; +use logos_blockchain_core::mantle::ops::channel::{ChannelId, MsgId}; use mempool::MemPoolHandle; use storage::sequencer::sequencer_cells::PendingDepositEventRecord; use tempfile::tempdir; use testnet_initial_state::{initial_pub_accounts_private_keys, initial_public_user_accounts}; use crate::{ - TransactionOrigin, + TransactionOrigin, apply_follow_update, + block_publisher::FollowUpdate, block_store::SequencerStore, - build_genesis_state, + build_bridge_deposit_tx_from_event, build_genesis_state, config::{BedrockConfig, SequencerConfig}, is_sequencer_only_program, mock::SequencerCoreWithMockClients, + resubmittable_txs, }; #[derive(borsh::BorshSerialize)] @@ -1271,3 +1273,273 @@ fn pda_mechanism_with_pinata_token_program() { expected_winner_token_holding_post ); } + +#[test] +fn resubmittable_txs_drops_clock_and_bridge_deposits() { + let user_tx = common::test_utils::produce_dummy_empty_transaction(); + let deposit_tx = build_bridge_deposit_tx_from_event(&PendingDepositEventRecord { + deposit_op_id: HashType([13; 32]), + source_tx_hash: HashType([7; 32]), + amount: 1, + metadata: borsh::to_vec(&DepositMetadataForEncoding { + recipient_id: initial_public_user_accounts()[0].account_id, + }) + .unwrap(), + submitted_in_block_id: None, + }) + .unwrap(); + let withdraw_tx = { + let message = lee::public_transaction::Message::try_new( + programs::bridge().id(), + vec![system_accounts::bridge_account_id()], + vec![], + bridge_core::Instruction::Withdraw { + amount: 1, + bedrock_account_pk: [0; 32], + }, + ) + .unwrap(); + LeeTransaction::Public(PublicTransaction::new( + message, + lee::public_transaction::WitnessSet::from_raw_parts(vec![]), + )) + }; + + let block = common::test_utils::produce_dummy_block( + 2, + Some(HashType([1; 32])), + vec![user_tx.clone(), deposit_tx, withdraw_tx.clone()], + ); + + // The trailing clock tx and the sequencer-generated deposit are dropped; + // user txs (withdrawals included) are returned. + assert_eq!(resubmittable_txs(&block), vec![user_tx, withdraw_tx]); +} + +#[test] +fn resubmittable_txs_of_blocks_without_user_txs_is_empty() { + // No transactions at all (not even the mandatory clock tx). + let empty = HashableBlockData { + block_id: 1, + prev_block_hash: HashType([0; 32]), + timestamp: 0, + transactions: vec![], + } + .into_pending_block(&sequencer_sign_key_for_testing()); + assert!(resubmittable_txs(&empty).is_empty()); + + let clock_only = common::test_utils::produce_dummy_block(1, None, vec![]); + assert!(resubmittable_txs(&clock_only).is_empty()); +} + +#[tokio::test] +async fn follow_adopted_peer_block_applies_and_persists() { + let config = setup_sequencer_config(); + let (sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; + let genesis_meta = sequencer.store.latest_block_meta().unwrap(); + + let acc1 = initial_public_user_accounts()[0].account_id; + let acc2 = initial_public_user_accounts()[1].account_id; + let tx = common::test_utils::create_transaction_native_token_transfer( + acc1, + 0, + acc2, + 10, + &create_signing_key_for_account1(), + ); + let peer_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![tx]); + + apply_follow_update( + sequencer.store.dbio(), + sequencer.chain(), + mempool_handle.clone(), + FollowUpdate { + adopted: vec![(MsgId::from([1; 32]), peer_block.clone())], + orphaned: vec![], + finalized: vec![], + }, + ) + .await; + + assert_eq!(sequencer.chain_height(), 2); + let stored = sequencer + .store + .get_block_at_id(2) + .unwrap() + .expect("adopted peer block should be persisted"); + assert_eq!(stored.header.hash, peer_block.header.hash); + assert_eq!( + sequencer.with_state(|s| s.get_account_by_id(acc2).balance), + 20010 + ); +} + +#[tokio::test] +async fn follow_redelivery_of_own_block_is_deduped() { + let config = setup_sequencer_config(); + let (mut sequencer, mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config).await; + + let acc1 = initial_public_user_accounts()[0].account_id; + let acc2 = initial_public_user_accounts()[1].account_id; + let tx = common::test_utils::create_transaction_native_token_transfer( + acc1, + 0, + acc2, + 10, + &create_signing_key_for_account1(), + ); + mempool_handle + .push((TransactionOrigin::User, tx)) + .await + .unwrap(); + sequencer.produce_new_block().await.unwrap(); + let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + + // The channel redelivers our own block under the MsgId the mock publisher + // assigned at publish time. + apply_follow_update( + sequencer.store.dbio(), + sequencer.chain(), + mempool_handle.clone(), + FollowUpdate { + adopted: vec![(MsgId::from(block2.header.hash.0), block2.clone())], + orphaned: vec![], + finalized: vec![], + }, + ) + .await; + + assert_eq!(sequencer.chain_height(), 2); + assert_eq!( + sequencer.with_state(|s| s.get_account_by_id(acc2).balance), + 20010, + "the transfer must not be double-applied" + ); +} + +#[tokio::test] +async fn follow_orphan_reverts_head_and_requeues_user_txs() { + let config = setup_sequencer_config(); + let (mut sequencer, mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config).await; + + let acc1 = initial_public_user_accounts()[0].account_id; + let acc2 = initial_public_user_accounts()[1].account_id; + let tx = common::test_utils::create_transaction_native_token_transfer( + acc1, + 0, + acc2, + 10, + &create_signing_key_for_account1(), + ); + mempool_handle + .push((TransactionOrigin::User, tx.clone())) + .await + .unwrap(); + sequencer.produce_new_block().await.unwrap(); + let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + + apply_follow_update( + sequencer.store.dbio(), + sequencer.chain(), + mempool_handle.clone(), + FollowUpdate { + adopted: vec![], + orphaned: vec![(MsgId::from(block2.header.hash.0), block2.clone())], + finalized: vec![], + }, + ) + .await; + + assert_eq!(sequencer.chain_height(), 1); + assert_eq!( + sequencer.with_state(|s| s.get_account_by_id(acc1).balance), + 10000, + "the orphaned transfer must be reverted from the head" + ); + let (origin, requeued) = sequencer + .mempool + .pop() + .expect("orphaned user tx should be requeued"); + assert!(matches!(origin, TransactionOrigin::User)); + assert_eq!(requeued, tx); + assert!( + sequencer.mempool.pop().is_none(), + "the clock tx must not be requeued" + ); +} + +#[tokio::test] +async fn follow_finalized_own_block_moves_final_tier_and_marks_store() { + let config = setup_sequencer_config(); + let (mut sequencer, mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config).await; + + let tx = common::test_utils::produce_dummy_empty_transaction(); + mempool_handle + .push((TransactionOrigin::User, tx)) + .await + .unwrap(); + sequencer.produce_new_block().await.unwrap(); + let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + + apply_follow_update( + sequencer.store.dbio(), + sequencer.chain(), + mempool_handle.clone(), + FollowUpdate { + adopted: vec![], + orphaned: vec![], + finalized: vec![(MsgId::from(block2.header.hash.0), block2.clone())], + }, + ) + .await; + + let final_tip = sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .final_tip() + .expect("final tip set"); + assert_eq!(final_tip.block_id, 2); + assert_eq!(sequencer.chain_height(), 2, "head is unchanged"); + let stored = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + assert!(matches!(stored.bedrock_status, BedrockStatus::Finalized)); +} + +#[tokio::test] +async fn follow_finalized_backfill_block_is_applied_and_marked_finalized() { + let config = setup_sequencer_config(); + let (sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; + let genesis_meta = sequencer.store.latest_block_meta().unwrap(); + + // A peer block we never saw as adopted arrives straight from the + // finalized (backfill) stream. + let peer_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![]); + + apply_follow_update( + sequencer.store.dbio(), + sequencer.chain(), + mempool_handle.clone(), + FollowUpdate { + adopted: vec![], + orphaned: vec![], + finalized: vec![(MsgId::from([2; 32]), peer_block.clone())], + }, + ) + .await; + + assert_eq!( + sequencer.chain_height(), + 2, + "head mirrors final on backfill" + ); + let stored = sequencer + .store + .get_block_at_id(2) + .unwrap() + .expect("backfilled block should be persisted"); + assert_eq!(stored.header.hash, peer_block.header.hash); + assert!(matches!(stored.bedrock_status, BedrockStatus::Finalized)); +} diff --git a/lez/storage/src/sequencer/mod.rs b/lez/storage/src/sequencer/mod.rs index 1e2feca2..cb986e26 100644 --- a/lez/storage/src/sequencer/mod.rs +++ b/lez/storage/src/sequencer/mod.rs @@ -689,3 +689,6 @@ impl RocksDBIO { }) } } + +#[cfg(test)] +mod tests; diff --git a/lez/storage/src/sequencer/tests.rs b/lez/storage/src/sequencer/tests.rs new file mode 100644 index 00000000..1473d27e --- /dev/null +++ b/lez/storage/src/sequencer/tests.rs @@ -0,0 +1,108 @@ +use common::test_utils::produce_dummy_block; +use lee::{Account, AccountId}; +use tempfile::tempdir; + +use super::*; + +fn marker_id() -> AccountId { + AccountId::new([1; 32]) +} + +/// A state distinguishable by the marker account's balance, so tests can tell +/// which snapshot a write persisted. +/// +/// TODO: is this a bit too much of a hot-fix for test snapshot? +fn state_with_balance(balance: u128) -> V03State { + V03State::new().with_public_accounts([( + marker_id(), + Account { + balance, + ..Account::default() + }, + )]) +} + +fn dbio_with_genesis(path: &Path) -> (RocksDBIO, Block) { + let genesis = produce_dummy_block(1, None, vec![]); + let dbio = RocksDBIO::create(path, &genesis, &state_with_balance(100)).unwrap(); + (dbio, genesis) +} + +fn stored_balance(dbio: &RocksDBIO) -> u128 { + dbio.get_lee_state() + .unwrap() + .get_account_by_id(marker_id()) + .balance +} + +#[test] +fn store_followed_block_persists_new_block_and_state() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2, &state_with_balance(200), false) + .unwrap(); + + let stored = dbio.get_block(2).unwrap().expect("block 2 is stored"); + assert_eq!(stored.header.hash, block2.header.hash); + assert!(matches!(stored.bedrock_status, BedrockStatus::Pending)); + assert_eq!(dbio.latest_block_meta().unwrap().id, 2); + assert_eq!(stored_balance(&dbio), 200); +} + +#[test] +fn store_followed_block_finalized_marks_block() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2, &state_with_balance(200), true) + .unwrap(); + + let stored = dbio.get_block(2).unwrap().expect("block 2 is stored"); + assert!(matches!(stored.bedrock_status, BedrockStatus::Finalized)); +} + +#[test] +fn store_followed_block_redelivery_is_a_noop_and_keeps_finalized() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2, &state_with_balance(200), true) + .unwrap(); + dbio.store_followed_block(&block2, &state_with_balance(300), false) + .unwrap(); + + let stored = dbio.get_block(2).unwrap().expect("block 2 is stored"); + assert!( + matches!(stored.bedrock_status, BedrockStatus::Finalized), + "re-delivery must not demote a finalized block" + ); + assert_eq!( + stored_balance(&dbio), + 200, + "re-delivery must not overwrite the persisted state" + ); +} + +#[test] +fn store_followed_block_overwrites_competing_block_at_same_id() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + let block2a = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2a, &state_with_balance(200), false) + .unwrap(); + + // A reorg replaces block 2: the competing block wins the slot. + let block2b = produce_dummy_block(2, Some(HashType([9; 32])), vec![]); + dbio.store_followed_block(&block2b, &state_with_balance(300), false) + .unwrap(); + + let stored = dbio.get_block(2).unwrap().expect("block 2 is stored"); + assert_eq!(stored.header.hash, block2b.header.hash); + assert!(matches!(stored.bedrock_status, BedrockStatus::Pending)); + assert_eq!(stored_balance(&dbio), 300); +} From f761d66dbae050918135ca581b6ae9f32a8bed81 Mon Sep 17 00:00:00 2001 From: erhant Date: Mon, 13 Jul 2026 23:25:38 +0300 Subject: [PATCH 17/32] feat(sequencer): channel roster configuration through the block publisher --- lez/sequencer/core/src/block_publisher.rs | 169 ++++++++++++++++------ lez/sequencer/core/src/lib.rs | 23 ++- lez/sequencer/core/src/mock.rs | 52 ++++++- lez/sequencer/core/src/tests.rs | 21 +++ 4 files changed, 219 insertions(+), 46 deletions(-) diff --git a/lez/sequencer/core/src/block_publisher.rs b/lez/sequencer/core/src/block_publisher.rs index 9cd21b45..ecd1ffc8 100644 --- a/lez/sequencer/core/src/block_publisher.rs +++ b/lez/sequencer/core/src/block_publisher.rs @@ -3,8 +3,11 @@ use std::{pin::Pin, sync::Arc, time::Duration}; use anyhow::{Context as _, Result, anyhow}; use common::block::Block; use log::{info, warn}; -pub use logos_blockchain_core::mantle::ops::channel::MsgId; -use logos_blockchain_core::mantle::ops::channel::{ChannelId, inscribe::Inscription}; +pub use logos_blockchain_core::mantle::ops::channel::{Ed25519PublicKey, MsgId}; +use logos_blockchain_core::mantle::{ + channel::{SlotTimeframe, SlotTimeout}, + ops::channel::{ChannelId, config::Keys, inscribe::Inscription}, +}; pub use logos_blockchain_key_management_system_service::keys::{Ed25519Key, ZkKey}; pub use logos_blockchain_zone_sdk::sequencer::SequencerCheckpoint; use logos_blockchain_zone_sdk::{ @@ -58,13 +61,26 @@ pub struct FollowUpdate { pub type OnFollowSink = Box Pin + Send>> + Send + 'static>; -/// Publish request channel: the inscription, its bridge withdrawals, and a -/// oneshot for the `MsgId` zone-sdk assigns. -type PublishSender = mpsc::Sender<( - Inscription, - Vec, - oneshot::Sender>, -)>; +/// Commands the drive task executes with `&mut sequencer`. +enum Command { + /// Publish an inscription (+ atomic withdrawals); responds with the assigned `MsgId`. + Publish { + inscription: Inscription, + withdrawals: Vec, + resp: oneshot::Sender>, + }, + /// Post a `ChannelConfig` op replacing the accredited keys / rotation params. + ConfigureChannel { + keys: Keys, + posting_timeframe: SlotTimeframe, + posting_timeout: SlotTimeout, + configuration_threshold: u16, + withdraw_threshold: u16, + resp: oneshot::Sender>, + }, +} + +type CommandSender = mpsc::Sender; #[expect(async_fn_in_trait, reason = "We don't care about Send/Sync here")] pub trait BlockPublisherTrait: Clone { @@ -88,6 +104,19 @@ pub trait BlockPublisherTrait: Clone { /// Zone-sdk drives the actual submission and retries internally. async fn publish_block(&self, block: &Block, withdrawals: Vec) -> Result; + /// Update the channel's accredited key set and rotation parameters via a + /// `ChannelConfig` op. The sequencer's bedrock key must be the channel + /// admin (`keys[0]`); the L1 rejects non-admin signers, so this is not + /// re-validated here. + async fn configure_channel( + &self, + keys: Vec, + posting_timeframe: u32, + posting_timeout: u32, + configuration_threshold: u16, + withdraw_threshold: u16, + ) -> Result<()>; + fn channel_id(&self) -> ChannelId; /// Whether this sequencer is currently authorized to write to the channel. @@ -98,7 +127,7 @@ pub trait BlockPublisherTrait: Clone { #[derive(Clone)] pub struct ZoneSdkPublisher { channel_id: ChannelId, - publish_tx: PublishSender, + command_tx: CommandSender, turn_rx: watch::Receiver, // Aborts the drive task when the last clone is dropped. _drive_task: Arc, @@ -146,7 +175,7 @@ impl BlockPublisherTrait for ZoneSdkPublisher { // Grab the turn watch before the move; the sdk actor keeps it current. let turn_rx = sequencer.subscribe_turn_to_write(); - let (publish_tx, mut publish_rx): (PublishSender, _) = + let (command_tx, mut command_rx): (CommandSender, _) = mpsc::channel(PUBLISH_INBOX_CAPACITY); let drive_task = tokio::spawn(async move { @@ -157,37 +186,62 @@ impl BlockPublisherTrait for ZoneSdkPublisher { )] { tokio::select! { - // Drain external publish requests by calling the - // borrowing handle — `&mut sequencer` is only - // available here. - Some((data_bounded, withdrawals, resp_tx)) = publish_rx.recv() => { - let data_byte_size = data_bounded.len(); - let withdraw_count = withdrawals.len(); - let published = if withdrawals.is_empty() { - sequencer.handle() - .publish(data_bounded) - .context("Failed to publish block") - } else { - sequencer.handle() - .publish_atomic_withdraw(data_bounded, withdrawals) - .context("Failed to publish block with withdrawals") - }; + // Drain external commands by calling the borrowing + // handle — `&mut sequencer` is only available here. + Some(command) = command_rx.recv() => match command { + Command::Publish { inscription: data_bounded, withdrawals, resp: resp_tx } => { + let data_byte_size = data_bounded.len(); + let withdraw_count = withdrawals.len(); + let published = if withdrawals.is_empty() { + sequencer.handle() + .publish(data_bounded) + .context("Failed to publish block") + } else { + sequencer.handle() + .publish_atomic_withdraw(data_bounded, withdrawals) + .context("Failed to publish block with withdrawals") + }; - let msg_result = published - .map(|(result, _checkpoint)| result.tx.inscription().this_msg); - match &msg_result { - Ok(_) if withdraw_count == 0 => { - info!("Published block with the size of {data_byte_size} bytes"); + let msg_result = published + .map(|(result, _checkpoint)| result.tx.inscription().this_msg); + match &msg_result { + Ok(_) if withdraw_count == 0 => { + info!("Published block with the size of {data_byte_size} bytes"); + } + Ok(_) => { + info!( + "Published block with the size of {data_byte_size} bytes and {withdraw_count} bridge withdrawals", + ); + } + Err(e) => warn!("zone-sdk publish failed: {e:?}"), } - Ok(_) => { - info!( - "Published block with the size of {data_byte_size} bytes and {withdraw_count} bridge withdrawals", - ); - } - Err(e) => warn!("zone-sdk publish failed: {e:?}"), + let _dontcare = resp_tx.send(msg_result); } - let _dontcare = resp_tx.send(msg_result); - } + Command::ConfigureChannel { + keys, + posting_timeframe, + posting_timeout, + configuration_threshold, + withdraw_threshold, + resp, + } => { + let result = sequencer + .handle() + .channel_config( + keys, + posting_timeframe, + posting_timeout, + configuration_threshold, + withdraw_threshold, + ) + .map(|_queued| ()) + .context("Failed to post channel config"); + if let Err(err) = &result { + warn!("zone-sdk channel config failed: {err:?}"); + } + let _dontcare = resp.send(result); + } + }, event = sequencer.next_event() => { let Some(event) = event else { continue; @@ -256,7 +310,7 @@ impl BlockPublisherTrait for ZoneSdkPublisher { Ok(Self { channel_id: config.channel_id, - publish_tx, + command_tx, turn_rx, _drive_task: Arc::new(DriveTaskGuard(drive_task)), }) @@ -269,8 +323,12 @@ impl BlockPublisherTrait for ZoneSdkPublisher { .context("Block data exceeds maximum allowed size")?; let (resp_tx, resp_rx) = oneshot::channel(); - self.publish_tx - .send((data_bounded, withdrawals, resp_tx)) + self.command_tx + .send(Command::Publish { + inscription: data_bounded, + withdrawals, + resp: resp_tx, + }) .await .map_err(|_closed| anyhow!("Drive task is no longer running"))?; @@ -279,6 +337,33 @@ impl BlockPublisherTrait for ZoneSdkPublisher { .map_err(|_closed| anyhow!("Drive task dropped the publish response"))? } + async fn configure_channel( + &self, + keys: Vec, + posting_timeframe: u32, + posting_timeout: u32, + configuration_threshold: u16, + withdraw_threshold: u16, + ) -> Result<()> { + let keys = Keys::try_from(keys) + .map_err(|_err| anyhow!("Channel key list must be non-empty and within bounds"))?; + let (resp_tx, resp_rx) = oneshot::channel(); + self.command_tx + .send(Command::ConfigureChannel { + keys, + posting_timeframe: posting_timeframe.into(), + posting_timeout: posting_timeout.into(), + configuration_threshold, + withdraw_threshold, + resp: resp_tx, + }) + .await + .map_err(|_closed| anyhow!("Drive task is no longer running"))?; + resp_rx + .await + .map_err(|_closed| anyhow!("Drive task dropped the config response"))? + } + fn channel_id(&self) -> ChannelId { self.channel_id } diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 1cc1ed4b..c0f09135 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -33,7 +33,7 @@ use storage::sequencer::{ }; use crate::{ - block_publisher::{BlockPublisherTrait, ZoneSdkPublisher}, + block_publisher::{BlockPublisherTrait, Ed25519PublicKey, ZoneSdkPublisher}, block_store::SequencerStore, }; @@ -625,6 +625,27 @@ impl SequencerCore { self.block_publisher.is_our_turn() } + /// Update the channel's accredited key set and rotation parameters. + /// This sequencer's bedrock key must be the channel admin (`keys[0]`). + pub async fn configure_channel( + &self, + keys: Vec, + posting_timeframe: u32, + posting_timeout: u32, + configuration_threshold: u16, + withdraw_threshold: u16, + ) -> Result<()> { + self.block_publisher + .configure_channel( + keys, + posting_timeframe, + posting_timeout, + configuration_threshold, + withdraw_threshold, + ) + .await + } + /// Shared handle to the two-tier follow state. #[must_use] pub fn chain(&self) -> Arc> { diff --git a/lez/sequencer/core/src/mock.rs b/lez/sequencer/core/src/mock.rs index 37c43254..5cc46900 100644 --- a/lez/sequencer/core/src/mock.rs +++ b/lez/sequencer/core/src/mock.rs @@ -1,4 +1,7 @@ -use std::time::Duration; +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; use anyhow::Result; use common::block::Block; @@ -8,17 +11,38 @@ use logos_blockchain_zone_sdk::sequencer::WithdrawArg; use crate::{ block_publisher::{ - BlockPublisherTrait, CheckpointSink, FinalizedBlockSink, OnDepositEventSink, OnFollowSink, - OnWithdrawEventSink, SequencerCheckpoint, + BlockPublisherTrait, CheckpointSink, Ed25519PublicKey, FinalizedBlockSink, + OnDepositEventSink, OnFollowSink, OnWithdrawEventSink, SequencerCheckpoint, }, config::BedrockConfig, }; pub type SequencerCoreWithMockClients = crate::SequencerCore; +/// One recorded `configure_channel` invocation. +#[derive(Clone)] +pub struct ConfigureChannelCall { + pub keys: Vec, + pub posting_timeframe: u32, + pub posting_timeout: u32, + pub configuration_threshold: u16, + pub withdraw_threshold: u16, +} + #[derive(Clone)] pub struct MockBlockPublisher { channel_id: ChannelId, + configure_channel_calls: Arc>>, +} + +impl MockBlockPublisher { + #[must_use] + pub fn configure_channel_calls(&self) -> Vec { + self.configure_channel_calls + .lock() + .expect("mock mutex poisoned") + .clone() + } } impl BlockPublisherTrait for MockBlockPublisher { @@ -35,6 +59,7 @@ impl BlockPublisherTrait for MockBlockPublisher { ) -> Result { Ok(Self { channel_id: config.channel_id, + configure_channel_calls: Arc::default(), }) } @@ -49,6 +74,27 @@ impl BlockPublisherTrait for MockBlockPublisher { Ok(MsgId::from(block.header.hash.0)) } + async fn configure_channel( + &self, + keys: Vec, + posting_timeframe: u32, + posting_timeout: u32, + configuration_threshold: u16, + withdraw_threshold: u16, + ) -> Result<()> { + self.configure_channel_calls + .lock() + .expect("mock mutex poisoned") + .push(ConfigureChannelCall { + keys, + posting_timeframe, + posting_timeout, + configuration_threshold, + withdraw_threshold, + }); + Ok(()) + } + fn channel_id(&self) -> ChannelId { self.channel_id } diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index f78e8ef4..177b2783 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -23,6 +23,7 @@ use lee_core::{ program::PdaSeed, }; use logos_blockchain_core::mantle::ops::channel::{ChannelId, MsgId}; +use logos_blockchain_key_management_system_service::keys::{ED25519_SECRET_KEY_SIZE, Ed25519Key}; use mempool::MemPoolHandle; use storage::sequencer::sequencer_cells::PendingDepositEventRecord; use tempfile::tempdir; @@ -1543,3 +1544,23 @@ async fn follow_finalized_backfill_block_is_applied_and_marked_finalized() { assert_eq!(stored.header.hash, peer_block.header.hash); assert!(matches!(stored.bedrock_status, BedrockStatus::Finalized)); } + +#[tokio::test] +async fn configure_channel_delegates_to_publisher() { + let config = setup_sequencer_config(); + let (sequencer, _mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config).await; + + let admin = Ed25519Key::from_bytes(&[0xA1; ED25519_SECRET_KEY_SIZE]).public_key(); + let peer = Ed25519Key::from_bytes(&[0xB2; ED25519_SECRET_KEY_SIZE]).public_key(); + sequencer + .configure_channel(vec![admin, peer], 20, 30, 1, 1) + .await + .unwrap(); + + let calls = sequencer.block_publisher().configure_channel_calls(); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].keys.len(), 2); + assert_eq!(calls[0].posting_timeframe, 20); + assert_eq!(calls[0].posting_timeout, 30); +} From e802a2a36420801ada617f92f30f9330c16c23b2 Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 14 Jul 2026 13:47:52 +0300 Subject: [PATCH 18/32] feat(sequencer): adminConfigureChannel RPC for channel roster changes --- Cargo.lock | 2 + lez/sequencer/core/src/block_publisher.rs | 7 ++- lez/sequencer/service/Cargo.toml | 1 + lez/sequencer/service/protocol/Cargo.toml | 1 + lez/sequencer/service/protocol/src/lib.rs | 14 +++++ lez/sequencer/service/rpc/src/lib.rs | 12 +++- lez/sequencer/service/src/service.rs | 67 +++++++++++++++++++++-- 7 files changed, 96 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fcdcec14..93d5f16b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9085,6 +9085,7 @@ dependencies = [ "common", "env_logger", "futures", + "hex", "jsonrpsee", "lee", "log", @@ -9105,6 +9106,7 @@ dependencies = [ "hex", "lee", "lee_core", + "serde", "serde_with", ] diff --git a/lez/sequencer/core/src/block_publisher.rs b/lez/sequencer/core/src/block_publisher.rs index ecd1ffc8..6230d32c 100644 --- a/lez/sequencer/core/src/block_publisher.rs +++ b/lez/sequencer/core/src/block_publisher.rs @@ -108,14 +108,17 @@ pub trait BlockPublisherTrait: Clone { /// `ChannelConfig` op. The sequencer's bedrock key must be the channel /// admin (`keys[0]`); the L1 rejects non-admin signers, so this is not /// re-validated here. - async fn configure_channel( + /// + /// Desugared (not `async fn`) so the returned future is provably `Send` — + /// generic callers awaiting it inside jsonrpsee handlers require that. + fn configure_channel( &self, keys: Vec, posting_timeframe: u32, posting_timeout: u32, configuration_threshold: u16, withdraw_threshold: u16, - ) -> Result<()>; + ) -> impl Future> + Send; fn channel_id(&self) -> ChannelId; diff --git a/lez/sequencer/service/Cargo.toml b/lez/sequencer/service/Cargo.toml index 3427dc22..94617bd9 100644 --- a/lez/sequencer/service/Cargo.toml +++ b/lez/sequencer/service/Cargo.toml @@ -19,6 +19,7 @@ programs.workspace = true clap = { workspace = true, features = ["derive", "env"] } anyhow.workspace = true env_logger.workspace = true +hex.workspace = true log.workspace = true tokio.workspace = true tokio-util.workspace = true diff --git a/lez/sequencer/service/protocol/Cargo.toml b/lez/sequencer/service/protocol/Cargo.toml index ced19e75..1eb413d0 100644 --- a/lez/sequencer/service/protocol/Cargo.toml +++ b/lez/sequencer/service/protocol/Cargo.toml @@ -13,4 +13,5 @@ lee.workspace = true lee_core.workspace = true hex.workspace = true +serde.workspace = true serde_with.workspace = true diff --git a/lez/sequencer/service/protocol/src/lib.rs b/lez/sequencer/service/protocol/src/lib.rs index ce669d31..9de04c9a 100644 --- a/lez/sequencer/service/protocol/src/lib.rs +++ b/lez/sequencer/service/protocol/src/lib.rs @@ -26,3 +26,17 @@ impl FromStr for ChannelId { Ok(Self(bytes)) } } + +/// Request for `adminConfigureChannel`: replaces the channel's accredited key +/// set and rotation parameters. +/// +/// `keys` are hex-encoded 32-byte Ed25519 public keys; `keys[0]` must be this +/// sequencer's (admin) key. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ConfigureChannelRequest { + pub keys: Vec, + pub posting_timeframe: u32, + pub posting_timeout: u32, + pub configuration_threshold: u16, + pub withdraw_threshold: u16, +} diff --git a/lez/sequencer/service/rpc/src/lib.rs b/lez/sequencer/service/rpc/src/lib.rs index f32f6bd2..887041e4 100644 --- a/lez/sequencer/service/rpc/src/lib.rs +++ b/lez/sequencer/service/rpc/src/lib.rs @@ -6,8 +6,8 @@ use jsonrpsee::types::ErrorObjectOwned; #[cfg(feature = "client")] pub use jsonrpsee::{core::ClientError, http_client::HttpClientBuilder as SequencerClientBuilder}; use sequencer_service_protocol::{ - Account, AccountId, Block, BlockId, ChannelId, Commitment, CommitmentSetDigest, HashType, - LeeTransaction, MembershipProof, Nonce, ProgramId, + Account, AccountId, Block, BlockId, ChannelId, Commitment, CommitmentSetDigest, + ConfigureChannelRequest, HashType, LeeTransaction, MembershipProof, Nonce, ProgramId, }; #[cfg(all(not(feature = "server"), not(feature = "client")))] @@ -92,4 +92,12 @@ pub trait Rpc { async fn get_channel_id(&self) -> Result; // ============================================================================================= + + /// Admin-only in effect: the L1 rejects the config op unless this + /// sequencer's key is the channel admin (`keys[0]` of the current roster). + #[method(name = "adminConfigureChannel")] + async fn admin_configure_channel( + &self, + request: ConfigureChannelRequest, + ) -> Result<(), ErrorObjectOwned>; } diff --git a/lez/sequencer/service/src/service.rs b/lez/sequencer/service/src/service.rs index f4b09b43..93679b64 100644 --- a/lez/sequencer/service/src/service.rs +++ b/lez/sequencer/service/src/service.rs @@ -9,11 +9,12 @@ use lee; use log::warn; use mempool::MemPoolHandle; use sequencer_core::{ - DbError, SequencerCore, TransactionOrigin, block_publisher::BlockPublisherTrait, + DbError, SequencerCore, TransactionOrigin, + block_publisher::{BlockPublisherTrait, Ed25519PublicKey}, }; use sequencer_service_protocol::{ - Account, AccountId, Block, BlockId, ChannelId, Commitment, CommitmentSetDigest, HashType, - MembershipProof, Nonce, ProgramId, + Account, AccountId, Block, BlockId, ChannelId, Commitment, CommitmentSetDigest, + ConfigureChannelRequest, HashType, MembershipProof, Nonce, ProgramId, }; use tokio::sync::Mutex; @@ -40,7 +41,7 @@ impl SequencerService { } #[async_trait] -impl sequencer_service_rpc::RpcServer +impl sequencer_service_rpc::RpcServer for SequencerService { async fn send_transaction(&self, tx: LeeTransaction) -> Result { @@ -204,8 +205,66 @@ impl sequencer_service_rpc::RpcServer let channel_id = self.sequencer.lock().await.block_publisher().channel_id(); Ok(ChannelId(*channel_id.as_ref())) } + + async fn admin_configure_channel( + &self, + request: ConfigureChannelRequest, + ) -> Result<(), ErrorObjectOwned> { + let keys = request + .keys + .iter() + .map(|hex_key| parse_channel_key(hex_key)) + .collect::, _>>()?; + + let sequencer = self.sequencer.lock().await; + sequencer + .configure_channel( + keys, + request.posting_timeframe, + request.posting_timeout, + request.configuration_threshold, + request.withdraw_threshold, + ) + .await + .map_err(|err| { + ErrorObjectOwned::owned( + ErrorCode::InternalError.code(), + format!("{err:#}"), + None::<()>, + ) + }) + } } fn internal_error(err: &DbError) -> ErrorObjectOwned { ErrorObjectOwned::owned(ErrorCode::InternalError.code(), err.to_string(), None::<()>) } + +/// Parses one hex-encoded 32-byte Ed25519 public key from an RPC request. +fn parse_channel_key(hex_key: &str) -> Result { + let invalid = |detail: String| { + ErrorObjectOwned::owned(ErrorCode::InvalidParams.code(), detail, None::<()>) + }; + let mut bytes = [0_u8; 32]; + hex::decode_to_slice(hex_key, &mut bytes) + .map_err(|err| invalid(format!("Invalid hex-encoded key: {err}")))?; + Ed25519PublicKey::from_bytes(&bytes) + .map_err(|err| invalid(format!("Invalid Ed25519 public key: {err}"))) +} + +#[cfg(test)] +mod tests { + use sequencer_core::block_publisher::Ed25519Key; + + use super::*; + + #[test] + fn parse_channel_key_roundtrips_and_rejects_garbage() { + let key = Ed25519Key::from_bytes(&[7; 32]).public_key(); + let parsed = parse_channel_key(&hex::encode(key.to_bytes())).unwrap(); + assert_eq!(parsed.to_bytes(), key.to_bytes()); + + assert!(parse_channel_key("not-hex").is_err()); + assert!(parse_channel_key("abcd").is_err()); + } +} From 213dc94c930a42993ef0985e0fa934b521038de6 Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 14 Jul 2026 13:48:00 +0300 Subject: [PATCH 19/32] test(fixtures): allow injecting a pre-generated bedrock signing key --- test_fixtures/src/setup.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test_fixtures/src/setup.rs b/test_fixtures/src/setup.rs index 325e1628..cdf87c9c 100644 --- a/test_fixtures/src/setup.rs +++ b/test_fixtures/src/setup.rs @@ -152,6 +152,7 @@ pub async fn setup_sequencer( SequencerInit::Genesis(genesis_transactions), channel_id, cross_zone, + None, ) .await } @@ -169,6 +170,30 @@ pub async fn setup_sequencer_from_prebuilt( SequencerInit::Prebuilt(&dump), config::bedrock_channel_id(), None, + None, + ) + .await +} + +/// Like [`setup_sequencer`], but with a pre-generated bedrock (Ed25519, 32-byte +/// seed) signing key written into the home so tests know the sequencer's +/// public key before it boots — required to accredit a committee member that +/// has not started yet. +pub async fn setup_sequencer_with_bedrock_key( + partial: config::SequencerPartialConfig, + bedrock_addr: SocketAddr, + genesis_transactions: Vec, + channel_id: ChannelId, + cross_zone: Option, + bedrock_signing_key: [u8; 32], +) -> Result<(SequencerHandle, TempDir)> { + setup_sequencer_inner( + partial, + bedrock_addr, + SequencerInit::Genesis(genesis_transactions), + channel_id, + cross_zone, + Some(bedrock_signing_key), ) .await } @@ -179,6 +204,7 @@ async fn setup_sequencer_inner( init: SequencerInit<'_>, channel_id: ChannelId, cross_zone: Option, + bedrock_signing_key: Option<[u8; 32]>, ) -> Result<(SequencerHandle, TempDir)> { let temp_sequencer_dir = tempfile::tempdir().context("Failed to create temp dir for sequencer home")?; @@ -188,6 +214,14 @@ async fn setup_sequencer_inner( temp_sequencer_dir.path().display() ); + if let Some(key_bytes) = bedrock_signing_key { + std::fs::write( + temp_sequencer_dir.path().join("bedrock_signing_key"), + key_bytes, + ) + .context("Failed to write pre-generated bedrock signing key")?; + } + let genesis_transactions = match init { SequencerInit::Genesis(genesis) => genesis, SequencerInit::Prebuilt(dump) => { From e2283e8f47d1cb599035f4b8524d0fc5dcb6d24b Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 14 Jul 2026 14:53:34 +0300 Subject: [PATCH 20/32] test(integration): multi-sequencer committee convergence --- Cargo.lock | 2 + integration_tests/Cargo.toml | 2 + integration_tests/tests/multi_sequencer.rs | 251 +++++++++++++++++++++ test_fixtures/src/setup.rs | 7 +- 4 files changed, 259 insertions(+), 3 deletions(-) create mode 100644 integration_tests/tests/multi_sequencer.rs diff --git a/Cargo.lock b/Cargo.lock index 93d5f16b..648f7925 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4148,12 +4148,14 @@ dependencies = [ "reqwest", "risc0-zkvm", "sequencer_core", + "sequencer_service_protocol", "sequencer_service_rpc", "serde_json", "system_accounts", "tempfile", "test_fixtures", "test_programs", + "testnet_initial_state", "token_core", "tokio", "vault_core", diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index a225d657..a2ab08b7 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -29,6 +29,7 @@ bridge_lock_core.workspace = true wrapped_token_core.workspace = true risc0-zkvm.workspace = true indexer_service_rpc = { workspace = true, features = ["client"] } +sequencer_service_protocol.workspace = true sequencer_service_rpc = { workspace = true, features = ["client"] } wallet-ffi.workspace = true indexer_ffi.workspace = true @@ -36,6 +37,7 @@ indexer_service_protocol.workspace = true system_accounts.workspace = true programs.workspace = true test_programs.workspace = true +testnet_initial_state.workspace = true logos-blockchain-http-api-common.workspace = true logos-blockchain-core.workspace = true diff --git a/integration_tests/tests/multi_sequencer.rs b/integration_tests/tests/multi_sequencer.rs new file mode 100644 index 00000000..0e4055ea --- /dev/null +++ b/integration_tests/tests/multi_sequencer.rs @@ -0,0 +1,251 @@ +#![expect( + clippy::tests_outside_test_module, + reason = "top-level test functions are conventional for integration tests" +)] + +//! Two sequencers share one channel: A starts solo as channel admin, live- +//! accredits `[A, B]` with round-robin rotation, B joins and syncs, both +//! produce on their turns, and A, B and an indexer converge on the same chain. + +use std::{net::SocketAddr, time::Duration}; + +use anyhow::{Context as _, Result, ensure}; +use indexer_service_rpc::RpcClient as _; +use integration_tests::{ + config::{self, SequencerPartialConfig}, + indexer_client::IndexerClient, + setup::{setup_bedrock_node, setup_indexer, setup_sequencer_with_bedrock_key}, +}; +use logos_blockchain_key_management_system_service::keys::Ed25519Key; +use sequencer_service_protocol::ConfigureChannelRequest; +use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuilder}; +use testnet_initial_state::{initial_pub_accounts_private_keys, initial_public_user_accounts}; +use tokio::test; + +/// 1 s bedrock slots: rotate the turn every ~20 s of tenure; steal a stalled +/// turn after ~30 s (bounds the stall while B is accredited but not started). +const POSTING_TIMEFRAME_SLOTS: u32 = 20; +const POSTING_TIMEOUT_SLOTS: u32 = 30; +const PHASE_TIMEOUT: Duration = Duration::from_secs(360); +const POLL_INTERVAL: Duration = Duration::from_secs(2); +const TRANSFER_AMOUNT: u128 = 10; +/// ≈4 turn windows past B's join (5 s blocks, ~20 s turns → ~4 blocks/window). +const ROTATION_BLOCKS: u64 = 8; + +#[test] +async fn multi_sequencer_committee_converges() -> Result<()> { + let (_bedrock, bedrock_addr) = setup_bedrock_node() + .await + .context("Failed to set up Bedrock node")?; + + // Fixed seeds so A can accredit B's public key before B exists. + let key_a = [0xA1_u8; 32]; + let key_b = [0xB2_u8; 32]; + let pub_a = Ed25519Key::from_bytes(&key_a).public_key(); + let pub_b = Ed25519Key::from_bytes(&key_b).public_key(); + + let partial = SequencerPartialConfig { + block_create_timeout: Duration::from_secs(5), + ..SequencerPartialConfig::default() + }; + + // Phase 1: A solo (its first inscription creates the channel), plus an indexer. + let (seq_a, _a_home) = setup_sequencer_with_bedrock_key( + partial, + bedrock_addr, + vec![], + config::bedrock_channel_id(), + None, + key_a, + ) + .await + .context("Failed to set up sequencer A")?; + let a = sequencer_client(seq_a.addr())?; + let (idx, _idx_home) = setup_indexer(bedrock_addr, config::bedrock_channel_id(), None) + .await + .context("Failed to set up indexer")?; + let indexer = indexer_client(idx.addr()).await?; + + wait_for_height(&a, 2, "sequencer A to produce past genesis").await?; + + // Phase 2: live roster change to [A, B] with rotation enabled. + a.admin_configure_channel(ConfigureChannelRequest { + keys: vec![hex::encode(pub_a.to_bytes()), hex::encode(pub_b.to_bytes())], + posting_timeframe: POSTING_TIMEFRAME_SLOTS, + posting_timeout: POSTING_TIMEOUT_SLOTS, + configuration_threshold: 1, + withdraw_threshold: 1, + }) + .await + .context("Failed to configure the channel committee")?; + + let height_at_config = a.get_last_block_id().await?; + wait_for_height( + &a, + height_at_config + 1, + "A to produce after the roster change", + ) + .await?; + + // Phase 3: B joins live and syncs the existing chain. + let (seq_b, _b_home) = setup_sequencer_with_bedrock_key( + partial, + bedrock_addr, + vec![], + config::bedrock_channel_id(), + None, + key_b, + ) + .await + .context("Failed to set up sequencer B")?; + let b = sequencer_client(seq_b.addr())?; + + let join_height = a.get_last_block_id().await?; + wait_for_height(&b, join_height, "B to sync to A's height at join").await?; + + // Phase 4: rotation + convergence over ≈4 turn windows. + let rotation_target = join_height + ROTATION_BLOCKS; + wait_for_height( + &a, + rotation_target, + "the chain to advance across turn windows", + ) + .await?; + wait_for_height(&b, rotation_target, "B to follow across turn windows").await?; + assert_same_chain(&a, &b).await?; + + // Phase 5: a tx submitted only to B is included by B and visible on A. + let accounts = initial_public_user_accounts(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = initial_pub_accounts_private_keys()[0].pub_sign_key.clone(); + + let to_balance_before = a.get_account_balance(to).await?; + let nonce = b.get_accounts_nonces(vec![from]).await?[0]; + let tx = common::test_utils::create_transaction_native_token_transfer( + from, + nonce.0, + to, + TRANSFER_AMOUNT, + &sign_key, + ); + b.send_transaction(tx) + .await + .context("Failed to submit the transfer to B")?; + + wait_for_balance(&a, to, to_balance_before + TRANSFER_AMOUNT).await?; + + // Phase 6: the indexer finalizes the same chain, with no stall. + wait_for_finalized(&indexer, join_height).await?; + let finalized = indexer.get_last_finalized_block_id().await?.unwrap_or(0); + for id in 1..=finalized { + let block_i = indexer + .get_block_by_id(id) + .await? + .with_context(|| format!("Indexer is missing finalized block {id}"))?; + let block_a = a + .get_block(id) + .await? + .with_context(|| format!("A is missing block {id}"))?; + ensure!( + block_i.header.hash == indexer_service_protocol::HashType::from(block_a.header.hash), + "Indexer diverges from A at block {id}" + ); + } + let status = indexer.get_status().await?; + ensure!( + status.stall_reason.is_none(), + "Indexer is stalled: {:?}", + status.stall_reason + ); + + Ok(()) +} + +fn sequencer_client(addr: SocketAddr) -> Result { + let url = config::addr_to_url(config::UrlProtocol::Http, addr) + .context("Failed to build sequencer URL")?; + SequencerClientBuilder::default() + .build(url) + .context("Failed to build sequencer client") +} + +async fn indexer_client(addr: SocketAddr) -> Result { + let url = config::addr_to_url(config::UrlProtocol::Ws, addr) + .context("Failed to build indexer URL")?; + IndexerClient::new(&url).await +} + +/// Polls the sequencer until its chain height reaches `target`. +async fn wait_for_height(client: &SequencerClient, target: u64, what: &str) -> Result<()> { + let wait = async { + loop { + if client.get_last_block_id().await? >= target { + return Ok::<(), anyhow::Error>(()); + } + tokio::time::sleep(POLL_INTERVAL).await; + } + }; + tokio::time::timeout(PHASE_TIMEOUT, wait) + .await + .with_context(|| format!("Timed out waiting for {what} (target height {target})"))? +} + +/// Polls the sequencer until `account`'s balance reaches `expected`. +async fn wait_for_balance( + client: &SequencerClient, + account: lee::AccountId, + expected: u128, +) -> Result<()> { + let wait = async { + loop { + if client.get_account_balance(account).await? == expected { + return Ok::<(), anyhow::Error>(()); + } + tokio::time::sleep(POLL_INTERVAL).await; + } + }; + tokio::time::timeout(PHASE_TIMEOUT, wait) + .await + .context("Timed out waiting for the cross-sequencer transfer to reach A")? +} + +/// Polls the indexer until its finalized height reaches `target`. +async fn wait_for_finalized(indexer: &IndexerClient, target: u64) -> Result<()> { + let wait = async { + loop { + if indexer.get_last_finalized_block_id().await?.unwrap_or(0) >= target { + return Ok::<(), anyhow::Error>(()); + } + tokio::time::sleep(POLL_INTERVAL).await; + } + }; + tokio::time::timeout(PHASE_TIMEOUT, wait) + .await + .context("Timed out waiting for the indexer to finalize")? +} + +/// Asserts A and B hold byte-identical block hashes over their common prefix. +async fn assert_same_chain(a: &SequencerClient, b: &SequencerClient) -> Result<()> { + let common = a + .get_last_block_id() + .await? + .min(b.get_last_block_id().await?); + for id in 1..=common { + let block_a = a + .get_block(id) + .await? + .with_context(|| format!("A is missing block {id}"))?; + let block_b = b + .get_block(id) + .await? + .with_context(|| format!("B is missing block {id}"))?; + ensure!( + block_a.header.hash == block_b.header.hash, + "Chain divergence at block {id}: A {:?} vs B {:?}", + block_a.header.hash, + block_b.header.hash + ); + } + Ok(()) +} diff --git a/test_fixtures/src/setup.rs b/test_fixtures/src/setup.rs index cdf87c9c..fe41ee99 100644 --- a/test_fixtures/src/setup.rs +++ b/test_fixtures/src/setup.rs @@ -176,9 +176,10 @@ pub async fn setup_sequencer_from_prebuilt( } /// Like [`setup_sequencer`], but with a pre-generated bedrock (Ed25519, 32-byte -/// seed) signing key written into the home so tests know the sequencer's -/// public key before it boots — required to accredit a committee member that -/// has not started yet. +/// seed) signing key. +/// +/// This allows the tests to know the sequencer's public key before it boots which +/// is required to accredit a committee member that has not started yet. pub async fn setup_sequencer_with_bedrock_key( partial: config::SequencerPartialConfig, bedrock_addr: SocketAddr, From 2c5bac506b332b336f333e7c6dbbb79db1cbe28a Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 14 Jul 2026 16:21:23 +0300 Subject: [PATCH 21/32] docs(sequencer): clarify queued semantics of adminConfigureChannel --- lez/sequencer/core/src/block_publisher.rs | 3 +++ lez/sequencer/core/src/tests.rs | 3 +++ lez/sequencer/service/rpc/src/lib.rs | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/lez/sequencer/core/src/block_publisher.rs b/lez/sequencer/core/src/block_publisher.rs index 6230d32c..f429e780 100644 --- a/lez/sequencer/core/src/block_publisher.rs +++ b/lez/sequencer/core/src/block_publisher.rs @@ -109,6 +109,9 @@ pub trait BlockPublisherTrait: Clone { /// admin (`keys[0]`); the L1 rejects non-admin signers, so this is not /// re-validated here. /// + /// `Ok(())` only means the signed op was queued locally, not that the + /// L1 accepted it — acceptance is asynchronous. + /// /// Desugared (not `async fn`) so the returned future is provably `Send` — /// generic callers awaiting it inside jsonrpsee handlers require that. fn configure_channel( diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 177b2783..a5dd971b 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -1561,6 +1561,9 @@ async fn configure_channel_delegates_to_publisher() { let calls = sequencer.block_publisher().configure_channel_calls(); assert_eq!(calls.len(), 1); assert_eq!(calls[0].keys.len(), 2); + assert_eq!(calls[0].keys[0], admin); assert_eq!(calls[0].posting_timeframe, 20); assert_eq!(calls[0].posting_timeout, 30); + assert_eq!(calls[0].configuration_threshold, 1); + assert_eq!(calls[0].withdraw_threshold, 1); } diff --git a/lez/sequencer/service/rpc/src/lib.rs b/lez/sequencer/service/rpc/src/lib.rs index 887041e4..fc3a94d6 100644 --- a/lez/sequencer/service/rpc/src/lib.rs +++ b/lez/sequencer/service/rpc/src/lib.rs @@ -95,6 +95,11 @@ pub trait Rpc { /// Admin-only in effect: the L1 rejects the config op unless this /// sequencer's key is the channel admin (`keys[0]` of the current roster). + /// + /// `Ok(())` only means the signed config op was queued locally (like + /// block publishing), not that L1 accepted it: acceptance is asynchronous, + /// and a rejection (e.g. non-admin signer) is not reported here — it only + /// shows up in node logs and on-chain behavior. #[method(name = "adminConfigureChannel")] async fn admin_configure_channel( &self, From abd70d699c55bd3855ccf78690e84712192dcd2e Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 14 Jul 2026 17:25:17 +0300 Subject: [PATCH 22/32] fix(sequencer): use `ED25519_SECRET_KEY_SIZE` and retain `BoundedError` message --- integration_tests/tests/multi_sequencer.rs | 6 +++--- lez/sequencer/core/src/block_publisher.rs | 8 +++++--- lez/sequencer/service/src/service.rs | 4 ++-- test_fixtures/src/setup.rs | 9 ++++++--- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/integration_tests/tests/multi_sequencer.rs b/integration_tests/tests/multi_sequencer.rs index 0e4055ea..c0c4a265 100644 --- a/integration_tests/tests/multi_sequencer.rs +++ b/integration_tests/tests/multi_sequencer.rs @@ -16,7 +16,7 @@ use integration_tests::{ indexer_client::IndexerClient, setup::{setup_bedrock_node, setup_indexer, setup_sequencer_with_bedrock_key}, }; -use logos_blockchain_key_management_system_service::keys::Ed25519Key; +use logos_blockchain_key_management_system_service::keys::{ED25519_SECRET_KEY_SIZE, Ed25519Key}; use sequencer_service_protocol::ConfigureChannelRequest; use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuilder}; use testnet_initial_state::{initial_pub_accounts_private_keys, initial_public_user_accounts}; @@ -39,8 +39,8 @@ async fn multi_sequencer_committee_converges() -> Result<()> { .context("Failed to set up Bedrock node")?; // Fixed seeds so A can accredit B's public key before B exists. - let key_a = [0xA1_u8; 32]; - let key_b = [0xB2_u8; 32]; + let key_a = [0xA1_u8; ED25519_SECRET_KEY_SIZE]; + let key_b = [0xB2_u8; ED25519_SECRET_KEY_SIZE]; let pub_a = Ed25519Key::from_bytes(&key_a).public_key(); let pub_b = Ed25519Key::from_bytes(&key_b).public_key(); diff --git a/lez/sequencer/core/src/block_publisher.rs b/lez/sequencer/core/src/block_publisher.rs index f429e780..105e0c77 100644 --- a/lez/sequencer/core/src/block_publisher.rs +++ b/lez/sequencer/core/src/block_publisher.rs @@ -8,7 +8,9 @@ use logos_blockchain_core::mantle::{ channel::{SlotTimeframe, SlotTimeout}, ops::channel::{ChannelId, config::Keys, inscribe::Inscription}, }; -pub use logos_blockchain_key_management_system_service::keys::{Ed25519Key, ZkKey}; +pub use logos_blockchain_key_management_system_service::keys::{ + ED25519_SECRET_KEY_SIZE, Ed25519Key, ZkKey, +}; pub use logos_blockchain_zone_sdk::sequencer::SequencerCheckpoint; use logos_blockchain_zone_sdk::{ CommonHttpClient, @@ -351,8 +353,8 @@ impl BlockPublisherTrait for ZoneSdkPublisher { configuration_threshold: u16, withdraw_threshold: u16, ) -> Result<()> { - let keys = Keys::try_from(keys) - .map_err(|_err| anyhow!("Channel key list must be non-empty and within bounds"))?; + let keys = + Keys::try_from(keys).map_err(|err| anyhow!("Invalid channel key list: {err}"))?; let (resp_tx, resp_rx) = oneshot::channel(); self.command_tx .send(Command::ConfigureChannel { diff --git a/lez/sequencer/service/src/service.rs b/lez/sequencer/service/src/service.rs index 93679b64..eccdbcf2 100644 --- a/lez/sequencer/service/src/service.rs +++ b/lez/sequencer/service/src/service.rs @@ -254,13 +254,13 @@ fn parse_channel_key(hex_key: &str) -> Result, channel_id: ChannelId, cross_zone: Option, - bedrock_signing_key: [u8; 32], + bedrock_signing_key: [u8; ED25519_SECRET_KEY_SIZE], ) -> Result<(SequencerHandle, TempDir)> { setup_sequencer_inner( partial, @@ -205,7 +208,7 @@ async fn setup_sequencer_inner( init: SequencerInit<'_>, channel_id: ChannelId, cross_zone: Option, - bedrock_signing_key: Option<[u8; 32]>, + bedrock_signing_key: Option<[u8; ED25519_SECRET_KEY_SIZE]>, ) -> Result<(SequencerHandle, TempDir)> { let temp_sequencer_dir = tempfile::tempdir().context("Failed to create temp dir for sequencer home")?; From 14ce6ad871f2ada976da4ce7749115655c118f5d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:58:43 +0000 Subject: [PATCH 23/32] fix: update spin v0.9.8 -> v0.9.9 (yanked crate fix for cargo-deny) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 648f7925..06a3e08e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9540,9 +9540,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] From e34acc35f98c4c8fc36b509220c89c6936270896 Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 14 Jul 2026 18:10:34 +0300 Subject: [PATCH 24/32] fix(sequencer): validate `adminConfigureChannel` requests, per Copilot review --- lez/sequencer/service/protocol/src/lib.rs | 82 ++++++++++++++++++++++- lez/sequencer/service/src/service.rs | 12 ++-- 2 files changed, 87 insertions(+), 7 deletions(-) diff --git a/lez/sequencer/service/protocol/src/lib.rs b/lez/sequencer/service/protocol/src/lib.rs index 9de04c9a..d621e561 100644 --- a/lez/sequencer/service/protocol/src/lib.rs +++ b/lez/sequencer/service/protocol/src/lib.rs @@ -30,8 +30,8 @@ impl FromStr for ChannelId { /// Request for `adminConfigureChannel`: replaces the channel's accredited key /// set and rotation parameters. /// -/// `keys` are hex-encoded 32-byte Ed25519 public keys; `keys[0]` must be this -/// sequencer's (admin) key. +/// - `keys` are hex-encoded 32-byte Ed25519 public keys +/// - `keys[0]` must be this sequencer's (admin) key. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct ConfigureChannelRequest { pub keys: Vec, @@ -40,3 +40,81 @@ pub struct ConfigureChannelRequest { pub configuration_threshold: u16, pub withdraw_threshold: u16, } + +impl ConfigureChannelRequest { + /// Structural sanity checks for the request. + /// + /// The L1 validates this too, but its async so we don't immediately + /// know about them when we submit. Checking this here instead gives + /// immediate feedback to the caller. + /// + /// We don't need a particular error type here, it's going to be logged only. + pub fn validate(&self) -> Result<(), String> { + let key_count = self.keys.len(); + if key_count == 0 { + return Err("Channel key list must not be empty".to_owned()); + } + for (name, threshold) in [ + ("configuration_threshold", self.configuration_threshold), + ("withdraw_threshold", self.withdraw_threshold), + ] { + if threshold == 0 || usize::from(threshold) > key_count { + return Err(format!( + "{name} must be between 1 and the key count ({key_count}), got {threshold}" + )); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn configure_channel_request_validate_rejects_static_garbage() { + // `validate` is structural: key contents are not parsed here. + let base = ConfigureChannelRequest { + keys: vec!["unparsed".to_owned(), "unparsed".to_owned()], + posting_timeframe: 20, + posting_timeout: 30, + configuration_threshold: 1, + withdraw_threshold: 2, + }; + + assert!(base.validate().is_ok()); + assert!( + ConfigureChannelRequest { + keys: vec![], + ..base + } + .validate() + .is_err() + ); + assert!( + ConfigureChannelRequest { + configuration_threshold: 0, + ..base.clone() + } + .validate() + .is_err() + ); + assert!( + ConfigureChannelRequest { + configuration_threshold: 3, + ..base.clone() + } + .validate() + .is_err() + ); + assert!( + ConfigureChannelRequest { + withdraw_threshold: 3, + ..base + } + .validate() + .is_err() + ); + } +} diff --git a/lez/sequencer/service/src/service.rs b/lez/sequencer/service/src/service.rs index eccdbcf2..63eac921 100644 --- a/lez/sequencer/service/src/service.rs +++ b/lez/sequencer/service/src/service.rs @@ -210,6 +210,7 @@ impl sequencer_service_rpc::Rpc &self, request: ConfigureChannelRequest, ) -> Result<(), ErrorObjectOwned> { + request.validate().map_err(invalid_params)?; let keys = request .keys .iter() @@ -240,16 +241,17 @@ fn internal_error(err: &DbError) -> ErrorObjectOwned { ErrorObjectOwned::owned(ErrorCode::InternalError.code(), err.to_string(), None::<()>) } +fn invalid_params(detail: String) -> ErrorObjectOwned { + ErrorObjectOwned::owned(ErrorCode::InvalidParams.code(), detail, None::<()>) +} + /// Parses one hex-encoded 32-byte Ed25519 public key from an RPC request. fn parse_channel_key(hex_key: &str) -> Result { - let invalid = |detail: String| { - ErrorObjectOwned::owned(ErrorCode::InvalidParams.code(), detail, None::<()>) - }; let mut bytes = [0_u8; 32]; hex::decode_to_slice(hex_key, &mut bytes) - .map_err(|err| invalid(format!("Invalid hex-encoded key: {err}")))?; + .map_err(|err| invalid_params(format!("Invalid hex-encoded key: {err}")))?; Ed25519PublicKey::from_bytes(&bytes) - .map_err(|err| invalid(format!("Invalid Ed25519 public key: {err}"))) + .map_err(|err| invalid_params(format!("Invalid Ed25519 public key: {err}"))) } #[cfg(test)] From 66fb83276f5d5c6884dbd3f39f571d84e1783d80 Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 14 Jul 2026 18:35:56 +0300 Subject: [PATCH 25/32] fix(sequencer): fix copilot reviews --- lez/chain_state/DESIGN.md | 14 ++++++++++---- lez/storage/src/sequencer/mod.rs | 4 +++- lez/storage/src/sequencer/tests.rs | 6 ++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/lez/chain_state/DESIGN.md b/lez/chain_state/DESIGN.md index 35e9a2b9..cb8595aa 100644 --- a/lez/chain_state/DESIGN.md +++ b/lez/chain_state/DESIGN.md @@ -66,13 +66,19 @@ Shared types moved into the crate: `AcceptOutcome`, `BlockIngestError`, `StallReason`, and `Tip`. ```rust -struct Tip { block_id: u64, hash: HashType, l1_slot: Slot } +struct Tip { block_id: u64, hash: HashType } -enum AcceptOutcome { Applied, AlreadyApplied, Parked(BlockIngestError) } +enum AcceptOutcome { + Applied, + AlreadyApplied, + Parked(BlockIngestError), + RetryableFailure(BlockIngestError), +} ``` -`Tip` carries `l1_slot` (recorded atomically with the tip) because the anchor / -chain-consistency logic keys on the inscription slot, not just `(id, hash)`. +`Tip` will additionally carry `l1_slot` once the anchor layer lands (recorded +atomically with the tip), because the anchor / chain-consistency logic keys on +the inscription slot, not just `(id, hash)`. ## 4. The two-tier `ChainState` diff --git a/lez/storage/src/sequencer/mod.rs b/lez/storage/src/sequencer/mod.rs index cb986e26..bd2364e9 100644 --- a/lez/storage/src/sequencer/mod.rs +++ b/lez/storage/src/sequencer/mod.rs @@ -487,7 +487,9 @@ impl RocksDBIO { if !first { let last_curr_block = self.get_meta_last_block_in_db()?; - if block.header.block_id > last_curr_block { + // `>=` so a same-height overwrite (a reorg replacing the tip block) + // also refreshes the tip hash in `latest_block_meta`. + if block.header.block_id >= last_curr_block { self.put_meta_last_block_in_db_batch(block.header.block_id, batch)?; self.put_meta_latest_block_meta_batch( &BlockMeta { diff --git a/lez/storage/src/sequencer/tests.rs b/lez/storage/src/sequencer/tests.rs index 1473d27e..ea9dc96a 100644 --- a/lez/storage/src/sequencer/tests.rs +++ b/lez/storage/src/sequencer/tests.rs @@ -105,4 +105,10 @@ fn store_followed_block_overwrites_competing_block_at_same_id() { assert_eq!(stored.header.hash, block2b.header.hash); assert!(matches!(stored.bedrock_status, BedrockStatus::Pending)); assert_eq!(stored_balance(&dbio), 300); + + // The tip meta must follow the reorg winner, or a restart seeds the chain + // from the orphaned block's hash. + let meta = dbio.latest_block_meta().unwrap(); + assert_eq!(meta.id, 2); + assert_eq!(meta.hash, block2b.header.hash); } From 9d1d33363ab5098087c9ecf50abb8ef2a7f819e7 Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 14 Jul 2026 20:00:13 +0300 Subject: [PATCH 26/32] fix(sequencer): skip persistence when a produced block loses the competing-write race --- lez/sequencer/core/src/lib.rs | 63 +++++++++++++++++++++++++++------ lez/sequencer/core/src/tests.rs | 54 ++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 11 deletions(-) diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index c0f09135..cd693ee1 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -33,7 +33,7 @@ use storage::sequencer::{ }; use crate::{ - block_publisher::{BlockPublisherTrait, Ed25519PublicKey, ZoneSdkPublisher}, + block_publisher::{BlockPublisherTrait, Ed25519PublicKey, MsgId, ZoneSdkPublisher}, block_store::SequencerStore, }; @@ -390,24 +390,65 @@ impl SequencerCore { .await .context("Failed to publish block to Bedrock")?; - // Apply our own block to the head with the MsgId the publish assigned it, - // so the head advances and the later adopted redelivery dedups. - let head_state = { - let mut chain = self.chain.lock().expect("chain state mutex poisoned"); - chain.apply_adopted(this_msg, &block); - chain.head_state().clone() - }; - - self.store.update( + self.record_produced_block( + this_msg, &block, &deposit_event_ids, withdrawal_reconciliation_keys, - &head_state, )?; Ok(block.header.block_id) } + /// Applies our own freshly-published block to the head with the [`MsgId`] the + /// publish assigned it, so the head advances and the later adopted + /// redelivery dedups, then persists it. + /// + /// Persistence is gated on the block actually becoming the head: if a peer + /// block won this height while we were publishing (`AlreadyApplied`, or + /// `Parked` when the head reorged to a different parent), the canonical + /// block is persisted by the follow path instead, and our invalidated + /// inscription comes back via `orphaned`. + fn record_produced_block( + &mut self, + this_msg: MsgId, + block: &Block, + deposit_event_ids: &[HashType], + withdrawal_reconciliation_keys: Vec, + ) -> Result<()> { + let head_state = { + let mut chain = self.chain.lock().expect("chain state mutex poisoned"); + match chain.apply_adopted(this_msg, block) { + AcceptOutcome::Applied => Some(chain.head_state().clone()), + AcceptOutcome::AlreadyApplied => { + warn!( + "Produced block {} lost a competing-write race, skipping persistence", + block.header.block_id + ); + None + } + AcceptOutcome::Parked(err) | AcceptOutcome::RetryableFailure(err) => { + warn!( + "Produced block {} no longer chains on the head, skipping persistence: {err}", + block.header.block_id + ); + None + } + } + }; + + if let Some(head_state) = head_state { + self.store.update( + block, + deposit_event_ids, + withdrawal_reconciliation_keys, + &head_state, + )?; + } + + Ok(()) + } + /// Validates and applies a single mempool transaction to the current state. /// Returns `Ok(true)` if the transaction was valid and applied, `Ok(false)` if /// it was skipped due to validation failure. diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index a5dd971b..5c3630ce 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -1567,3 +1567,57 @@ async fn configure_channel_delegates_to_publisher() { assert_eq!(calls[0].configuration_threshold, 1); assert_eq!(calls[0].withdraw_threshold, 1); } + +#[tokio::test] +async fn record_produced_block_skips_persistence_on_lost_race() { + let config = setup_sequencer_config(); + let (mut sequencer, _mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config).await; + let genesis_meta = sequencer.store.latest_block_meta().unwrap(); + + // A peer block wins height 2 while "our" block is in flight. + let peer_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![]); + sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .apply_adopted(MsgId::from([9; 32]), &peer_block); + + // Our competing block at the same height: same parent, different content. + let tx = common::test_utils::produce_dummy_empty_transaction(); + let our_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![tx]); + sequencer + .record_produced_block( + MsgId::from(our_block.header.hash.0), + &our_block, + &[], + vec![], + ) + .unwrap(); + + // The lost-race block must not reach the store; the head keeps the peer block. + assert!(sequencer.store.get_block_at_id(2).unwrap().is_none()); + let head_tip = sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .head_tip() + .expect("head tip"); + assert_eq!(head_tip.hash, peer_block.header.hash); +} + +#[tokio::test] +async fn record_produced_block_skips_persistence_when_block_no_longer_chains() { + let config = setup_sequencer_config(); + let (mut sequencer, _mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config).await; + + // The head reorged under us: our block's parent is no longer the tip. + let stale = common::test_utils::produce_dummy_block(2, Some(HashType([9; 32])), vec![]); + sequencer + .record_produced_block(MsgId::from(stale.header.hash.0), &stale, &[], vec![]) + .unwrap(); + + assert!(sequencer.store.get_block_at_id(2).unwrap().is_none()); + assert_eq!(sequencer.chain_height(), 1, "head is unchanged"); +} From ed8c9c253d3e3b0634563a39d3c6cefb0cf27f5d Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 14 Jul 2026 21:30:42 +0300 Subject: [PATCH 27/32] fix(sequencer): persist follow updates atomically in one write batch --- lez/sequencer/core/src/lib.rs | 37 +++++------ lez/sequencer/core/src/tests.rs | 50 +++++++++++++++ lez/storage/src/sequencer/mod.rs | 98 ++++++++++++++++++++++++------ lez/storage/src/sequencer/tests.rs | 30 +++++++++ 4 files changed, 176 insertions(+), 39 deletions(-) diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index cd693ee1..19bc7acf 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -719,23 +719,22 @@ async fn apply_follow_update( // Apply under the lock and collect what to persist; take a single // head snapshot. Release the lock before touching disk so the // producer is never blocked on the follow path's I/O. - let (adopted, finalized, resubmit_txs, head_snapshot) = { + let (to_persist, resubmit_txs, head_snapshot) = { let mut chain = chain.lock().expect("chain state mutex poisoned"); let mut resubmit_txs = Vec::new(); for (this_msg, block) in &update.orphaned { chain.revert_orphan(*this_msg); resubmit_txs.extend(resubmittable_txs(block)); } - let mut adopted = Vec::new(); + let mut to_persist = Vec::new(); for (this_msg, block) in &update.adopted { if matches!( chain.apply_adopted(*this_msg, block), AcceptOutcome::Applied ) { - adopted.push(block); + to_persist.push((block, false)); } } - let mut finalized = Vec::new(); for (this_msg, block) in &update.finalized { // TODO: thread the finalized inscription's L1 slot once the // sdk surfaces it; only used for the invalid-finalized stall. @@ -743,27 +742,23 @@ async fn apply_follow_update( chain.apply_finalized(*this_msg, block, Slot::from(0)), AcceptOutcome::Applied ) { - finalized.push(block); + to_persist.push((block, true)); } } - (adopted, finalized, resubmit_txs, chain.head_state().clone()) + (to_persist, resubmit_txs, chain.head_state().clone()) }; - for block in adopted { - if let Err(err) = dbio.store_followed_block(block, &head_snapshot, false) { - error!( - "Failed to persist adopted block {}: {err:#}", - block.header.block_id - ); - } - } - for block in finalized { - if let Err(err) = dbio.store_followed_block(block, &head_snapshot, true) { - error!( - "Failed to persist finalized block {}: {err:#}", - block.header.block_id - ); - } + // One atomic write for the whole update: blocks, tip meta and the state + // after the last block land together, so a crash can never leave the + // stored state ahead of the stored blocks. + // + // TODO: the zone-sdk checkpoint is persisted by `on_checkpoint` before + // this write; a crash in between resumes past these blocks without them + // ever landing in the store. Full `BlocksProcessed` atomicity (checkpoint + // + blocks + state in one batch, per the sdk's event contract) is a + // follow-up. + if let Err(err) = dbio.store_followed_blocks(&to_persist, &head_snapshot) { + error!("Failed to persist followed blocks: {err:#}"); } // Rebuild orphaned work: return its user txs to the mempool so the diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 5c3630ce..965136ba 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -1621,3 +1621,53 @@ async fn record_produced_block_skips_persistence_when_block_no_longer_chains() { assert!(sequencer.store.get_block_at_id(2).unwrap().is_none()); assert_eq!(sequencer.chain_height(), 1, "head is unchanged"); } + +#[tokio::test] +async fn follow_update_persists_blocks_meta_and_state_atomically() { + let config = setup_sequencer_config(); + let (sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; + let genesis_meta = sequencer.store.latest_block_meta().unwrap(); + + let acc1 = initial_public_user_accounts()[0].account_id; + let acc2 = initial_public_user_accounts()[1].account_id; + let tx = common::test_utils::create_transaction_native_token_transfer( + acc1, + 0, + acc2, + 10, + &create_signing_key_for_account1(), + ); + let block2 = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![tx]); + let block3 = common::test_utils::produce_dummy_block(3, Some(block2.header.hash), vec![]); + + // One update carrying several blocks: both adopted, block 2 also finalized. + apply_follow_update( + sequencer.store.dbio(), + sequencer.chain(), + mempool_handle.clone(), + FollowUpdate { + adopted: vec![ + (MsgId::from([2; 32]), block2.clone()), + (MsgId::from([3; 32]), block3.clone()), + ], + orphaned: vec![], + finalized: vec![(MsgId::from([2; 32]), block2.clone())], + }, + ) + .await; + + // Blocks, tip meta and state all reflect the end of the batch: a late + // finalized entry for an earlier block must not drag the tip meta back. + let meta = sequencer.store.latest_block_meta().unwrap(); + assert_eq!(meta.id, 3); + assert_eq!(meta.hash, block3.header.hash); + let stored2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + assert!(matches!(stored2.bedrock_status, BedrockStatus::Finalized)); + let stored_balance = sequencer + .store + .get_lee_state() + .unwrap() + .get_account_by_id(acc2) + .balance; + assert_eq!(stored_balance, 20010); +} diff --git a/lez/storage/src/sequencer/mod.rs b/lez/storage/src/sequencer/mod.rs index bd2364e9..3a7ab858 100644 --- a/lez/storage/src/sequencer/mod.rs +++ b/lez/storage/src/sequencer/mod.rs @@ -482,8 +482,6 @@ impl RocksDBIO { } pub fn put_block(&self, block: &Block, first: bool, batch: &mut WriteBatch) -> DbResult<()> { - let cf_block = self.block_column(); - if !first { let last_curr_block = self.get_meta_last_block_in_db()?; @@ -501,6 +499,12 @@ impl RocksDBIO { } } + self.put_block_payload(block, batch) + } + + /// Stages just the block payload into `batch`, without touching the tip meta. + fn put_block_payload(&self, block: &Block, batch: &mut WriteBatch) -> DbResult<()> { + let cf_block = self.block_column(); batch.put_cf( &cf_block, borsh::to_vec(&block.header.block_id).map_err(|err| { @@ -615,31 +619,89 @@ impl RocksDBIO { } /// Persists a followed (peer) block so the store mirrors the canonical chain. - /// - /// Skips the write when the store already holds this block (matched by id and - /// hash) — covering our own blocks and re-deliveries — but still marks it - /// finalized when `finalized` is set. - /// - /// When `finalized`, the block is marked so backfilled blocks (which arrive - /// without a prior pending write) are not left pending. + /// One-block form of [`Self::store_followed_blocks`]. pub fn store_followed_block( &self, block: &Block, state: &V03State, finalized: bool, ) -> DbResult<()> { - let block_id = block.header.block_id; - let already_stored = self - .get_block(block_id)? - .is_some_and(|stored| stored.header.hash == block.header.hash); + self.store_followed_blocks(&[(block, finalized)], state) + } - if !already_stored { - self.atomic_update(block, &[], vec![], state)?; + /// Persists a **batch** of followed (peer) blocks and the state after the last + /// of them in one atomic write, so a crash can never leave the stored state + /// ahead of the stored blocks and tip meta. + /// + /// Per block, it: + /// - skips the payload write when the store already holds it (matched by id + /// and hash) thus covering our own blocks and re-deliveries + /// - stores it as finalized when its `finalized` flag is set + /// + /// When nothing needs writing, the state is left untouched too + /// + /// TODO: the zone-sdk checkpoint is persisted by `on_checkpoint` *before* + /// this write; a crash in between resumes past these blocks without them + /// ever landing in the store. + /// Full `BlocksProcessed` atomicity by writing the checkpoint, blocks and + /// state in one batch, per the sdk's event contract is a follow-up. + pub fn store_followed_blocks( + &self, + blocks: &[(&Block, bool)], + state: &V03State, + ) -> DbResult<()> { + let last_block_in_db = self.get_meta_last_block_in_db()?; + let mut batch = WriteBatch::default(); + let mut batch_tip: Option = None; + + for (block, finalized) in blocks { + let stored = self.get_block(block.header.block_id)?; + let already_stored = stored + .as_ref() + .is_some_and(|stored| stored.header.hash == block.header.hash); + + if already_stored && !finalized { + continue; + } + + let mut to_write = if already_stored { + stored.expect("`already_stored` implies a stored block") + } else { + (*block).clone() + }; + if *finalized { + to_write.bedrock_status = BedrockStatus::Finalized; + } + self.put_block_payload(&to_write, &mut batch)?; + + let id = to_write.header.block_id; + if batch_tip.as_ref().is_none_or(|tip| id >= tip.id) { + batch_tip = Some(BlockMeta { + id, + hash: to_write.header.hash, + }); + } } - if finalized { - self.mark_block_as_finalized(block_id)?; + + if batch.is_empty() { + return Ok(()); } - Ok(()) + // Meta is staged once for the batch's highest block. + // (`>=` so a same-height overwrite refreshes the tip hash, matching `put_block`) + if let Some(tip) = batch_tip + && tip.id >= last_block_in_db + { + self.put_meta_last_block_in_db_batch(tip.id, &mut batch)?; + self.put_meta_latest_block_meta_batch(&tip, &mut batch)?; + } + self.put_lee_state_in_db_batch(state, &mut batch)?; + + self.db.write(batch).map_err(|rerr| { + DbError::rocksdb_cast_message( + rerr, + Some("Failed to write followed blocks batch".to_owned()), + ) + }) } pub fn get_all_blocks(&self) -> impl Iterator> { diff --git a/lez/storage/src/sequencer/tests.rs b/lez/storage/src/sequencer/tests.rs index ea9dc96a..2d86c494 100644 --- a/lez/storage/src/sequencer/tests.rs +++ b/lez/storage/src/sequencer/tests.rs @@ -87,6 +87,36 @@ fn store_followed_block_redelivery_is_a_noop_and_keeps_finalized() { ); } +#[test] +fn store_followed_blocks_batch_lands_meta_and_state_on_last_block() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + // Block 2 is already stored (own production); one update then finalizes it + // and adopts block 3. + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2, &state_with_balance(200), false) + .unwrap(); + + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + dbio.store_followed_blocks( + &[(&block2, true), (&block3, false)], + &state_with_balance(300), + ) + .unwrap(); + + let stored2 = dbio.get_block(2).unwrap().expect("block 2 is stored"); + assert!(matches!(stored2.bedrock_status, BedrockStatus::Finalized)); + let stored3 = dbio.get_block(3).unwrap().expect("block 3 is stored"); + assert!(matches!(stored3.bedrock_status, BedrockStatus::Pending)); + + // Meta and state land together on the last block of the batch. + let meta = dbio.latest_block_meta().unwrap(); + assert_eq!(meta.id, 3); + assert_eq!(meta.hash, block3.header.hash); + assert_eq!(stored_balance(&dbio), 300); +} + #[test] fn store_followed_block_overwrites_competing_block_at_same_id() { let temp_dir = tempdir().unwrap(); From 1a5da3d9258810ea8acd8cc5dfcf6f07d2c6977d Mon Sep 17 00:00:00 2001 From: erhant Date: Tue, 14 Jul 2026 21:43:03 +0300 Subject: [PATCH 28/32] chore: fix lint --- lez/storage/src/sequencer/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lez/storage/src/sequencer/mod.rs b/lez/storage/src/sequencer/mod.rs index 3a7ab858..d7775e20 100644 --- a/lez/storage/src/sequencer/mod.rs +++ b/lez/storage/src/sequencer/mod.rs @@ -634,11 +634,11 @@ impl RocksDBIO { /// ahead of the stored blocks and tip meta. /// /// Per block, it: - /// - skips the payload write when the store already holds it (matched by id - /// and hash) thus covering our own blocks and re-deliveries + /// - skips the payload write when the store already holds it (matched by id and hash) thus + /// covering our own blocks and re-deliveries /// - stores it as finalized when its `finalized` flag is set /// - /// When nothing needs writing, the state is left untouched too + /// When nothing needs writing, the state is left untouched too. /// /// TODO: the zone-sdk checkpoint is persisted by `on_checkpoint` *before* /// this write; a crash in between resumes past these blocks without them From c9e3bf2a055d8b56e70ea1738a32d26a72738911 Mon Sep 17 00:00:00 2001 From: erhant Date: Wed, 15 Jul 2026 13:00:46 +0300 Subject: [PATCH 29/32] feat(sequencer): multi-sequencer demo ergonomics (`--home` override, pubkey helper, demo recipes) --- Justfile | 8 +++-- lez/sequencer/core/src/block_publisher.rs | 10 +++++- lez/sequencer/core/src/lib.rs | 6 +++- lez/sequencer/service/Cargo.toml | 1 + .../service/src/bin/bedrock_pubkey.rs | 35 +++++++++++++++++++ lez/sequencer/service/src/main.rs | 15 ++++++-- 6 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 lez/sequencer/service/src/bin/bedrock_pubkey.rs diff --git a/Justfile b/Justfile index cf7c833e..93c2f8ab 100644 --- a/Justfile +++ b/Justfile @@ -58,15 +58,17 @@ run-bedrock: docker compose up # Run Sequencer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. +# Optional home/port let a second instance run off the same config, e.g. +# `just run-sequencer "" "$TMPDIR/lez-sequencer2" 3041` for the multi-sequencer demo. [working-directory: 'lez/sequencer/service'] -run-sequencer standalone="": +run-sequencer standalone="" home="" port="3040": @echo "🧠 Running sequencer" @if [ "{{standalone}}" = "standalone" ]; then \ echo "🧪 Running in standalone mode"; \ - RUST_LOG=info cargo run --features standalone --release -p sequencer_service configs/debug/sequencer_config.json; \ + RUST_LOG=info cargo run --features standalone --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \ else \ echo "🚀 Running in normal mode"; \ - RUST_LOG=info cargo run --release -p sequencer_service configs/debug/sequencer_config.json; \ + RUST_LOG=info cargo run --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \ fi # Run Indexer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. diff --git a/lez/sequencer/core/src/block_publisher.rs b/lez/sequencer/core/src/block_publisher.rs index 105e0c77..0bf32719 100644 --- a/lez/sequencer/core/src/block_publisher.rs +++ b/lez/sequencer/core/src/block_publisher.rs @@ -301,7 +301,15 @@ impl BlockPublisherTrait for ZoneSdkPublisher { }) .await; } - Event::Ready | Event::TurnNotification { .. } => {} + Event::Ready => {} + Event::TurnNotification { notification } => { + info!( + "Turn update: our_turn={}, starting_slot={:?}, ends_at_slot={:?}", + notification.our_turn_to_write, + notification.starting_slot, + notification.ends_at_slot + ); + } } } } diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 19bc7acf..3ebc8abf 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -134,6 +134,10 @@ impl SequencerCore { let bedrock_signing_key = load_or_create_signing_key(&config.home.join("bedrock_signing_key")) .expect("Failed to load or create bedrock signing key"); + info!( + "Bedrock signing public key: {}", + hex::encode(bedrock_signing_key.public_key().to_bytes()) + ); let (store, state, _genesis_block) = Self::open_or_create_store(&config); @@ -1065,7 +1069,7 @@ fn withdraw_event_reconciliation_key( } /// Load signing key from file or generate a new one if it doesn't exist. -fn load_or_create_signing_key(path: &Path) -> Result { +pub fn load_or_create_signing_key(path: &Path) -> Result { if path.exists() { let key_bytes = std::fs::read(path)?; diff --git a/lez/sequencer/service/Cargo.toml b/lez/sequencer/service/Cargo.toml index 94617bd9..338aac0d 100644 --- a/lez/sequencer/service/Cargo.toml +++ b/lez/sequencer/service/Cargo.toml @@ -1,6 +1,7 @@ [package] name = "sequencer_service" version = "0.1.0" +default-run = "sequencer_service" edition = "2024" license = { workspace = true } diff --git a/lez/sequencer/service/src/bin/bedrock_pubkey.rs b/lez/sequencer/service/src/bin/bedrock_pubkey.rs new file mode 100644 index 00000000..ad602be6 --- /dev/null +++ b/lez/sequencer/service/src/bin/bedrock_pubkey.rs @@ -0,0 +1,35 @@ +//! Prints the sequencer's bedrock signing public key (hex) without booting it. +//! +//! Loads `/bedrock_signing_key` from the given sequencer config, creating +//! the key if it doesn't exist yet, so that a node can be accredited into the +//! channel committee before its first boot. + +use std::path::PathBuf; + +use anyhow::Result; +use clap::Parser; + +#[derive(Debug, Parser)] +#[clap(version)] +struct Args { + #[clap(name = "config")] + config_path: PathBuf, + /// Override the config's home directory, matching the sequencer's --home. + #[clap(long)] + home: Option, +} + +#[expect( + clippy::print_stdout, + reason = "the hex pubkey on stdout is this binary's output" +)] +fn main() -> Result<()> { + let Args { config_path, home } = Args::parse(); + + let config = sequencer_service::SequencerConfig::from_path(&config_path)?; + let home = home.unwrap_or(config.home); + let key = sequencer_core::load_or_create_signing_key(&home.join("bedrock_signing_key"))?; + println!("{}", hex::encode(key.public_key().to_bytes())); + + Ok(()) +} diff --git a/lez/sequencer/service/src/main.rs b/lez/sequencer/service/src/main.rs index 8b577bb8..4dede6d2 100644 --- a/lez/sequencer/service/src/main.rs +++ b/lez/sequencer/service/src/main.rs @@ -12,6 +12,10 @@ struct Args { config_path: PathBuf, #[clap(short, long, default_value = "3040")] port: u16, + /// Override the config's home directory (`RocksDB` + bedrock signing key), + /// so multiple instances can share one config file. + #[clap(long)] + home: Option, } #[tokio::main] @@ -22,13 +26,20 @@ struct Args { async fn main() -> Result<()> { env_logger::init(); - let Args { config_path, port } = Args::parse(); + let Args { + config_path, + port, + home, + } = Args::parse(); // TODO: handle this cancellation token more gracefully within Sequencer service // similar to how we do in Indexer let cancellation_token = listen_for_shutdown_signal(); - let config = sequencer_service::SequencerConfig::from_path(&config_path)?; + let mut config = sequencer_service::SequencerConfig::from_path(&config_path)?; + if let Some(home) = home { + config.home = home; + } let sequencer_handle = sequencer_service::run(config, port).await?; tokio::select! { From f8bc1581ceb5fd9070458d1d659697235a05c6df Mon Sep 17 00:00:00 2001 From: erhant Date: Thu, 16 Jul 2026 14:10:54 +0300 Subject: [PATCH 30/32] fix(chain_state, sequencer, indexer): finalized re-delivery dedup, non-blocking orphan resubmit, fail-fast validation --- lez/chain_state/src/apply.rs | 4 +- lez/chain_state/src/chain.rs | 63 +++++++++++++++++++++++++++-- lez/chain_state/src/lib.rs | 2 +- lez/indexer/core/src/block_store.rs | 14 ++++++- lez/mempool/src/lib.rs | 18 +++++++++ lez/sequencer/core/src/lib.rs | 39 ++++++++++-------- lez/sequencer/core/src/tests.rs | 62 +++++++++++++--------------- 7 files changed, 144 insertions(+), 58 deletions(-) diff --git a/lez/chain_state/src/apply.rs b/lez/chain_state/src/apply.rs index 5e2ff6f1..98c91a19 100644 --- a/lez/chain_state/src/apply.rs +++ b/lez/chain_state/src/apply.rs @@ -52,7 +52,7 @@ pub fn apply_block( /// Checks that `block` is the valid continuation of `tip`: hash integrity, /// then block-id continuity, then `prev_block_hash` linkage. A `None` tip /// (cold state) expects the genesis block. -fn validate_against_tip(tip: Option<&Tip>, block: &Block) -> Result<(), BlockIngestError> { +pub fn validate_against_tip(tip: Option<&Tip>, block: &Block) -> Result<(), BlockIngestError> { let computed = block.recompute_hash(); if computed != block.header.hash { return Err(BlockIngestError::HashMismatch { @@ -95,7 +95,7 @@ fn validate_against_tip(tip: Option<&Tip>, block: &Block) -> Result<(), BlockIng /// Applies a block's transactions to `state`, mapping every failure to a /// [`BlockIngestError`] so the caller can park rather than crash. Operates in /// place; the caller commits only on `Ok`. -fn apply_block_to_state(block: &Block, state: &mut V03State) -> Result<(), BlockIngestError> { +pub fn apply_block_to_state(block: &Block, state: &mut V03State) -> Result<(), BlockIngestError> { let (clock_tx, user_txs) = block .body .transactions diff --git a/lez/chain_state/src/chain.rs b/lez/chain_state/src/chain.rs index 07c9890b..c3a810c0 100644 --- a/lez/chain_state/src/chain.rs +++ b/lez/chain_state/src/chain.rs @@ -160,8 +160,8 @@ impl ChainState { block: &Block, l1_slot: Slot, ) -> AcceptOutcome { - // Match by MsgId, or by block identity to handle a re-inscribed but - // identical block arriving under a fresh MsgId. + // Match by [`MsgId`] or by block identity to handle a re-inscribed but + // identical block arriving under a fresh [`MsgId`]. let in_head = self.head_blocks.iter().position(|entry| { entry.this_msg == this_msg || entry.block.header.hash == block.header.hash }); @@ -179,7 +179,7 @@ impl ChainState { let finalized: Vec = self.head_blocks.drain(0..=idx).collect(); for entry in finalized { apply_block(self.final_tip.as_ref(), &entry.block, &mut self.final_state) - .expect("a validated head block must apply to the final tier"); + .expect("validated head block must apply to the final tier"); self.final_tip = Some(tip_of(&entry.block)); } self.final_stall = None; @@ -188,6 +188,19 @@ impl ChainState { /// Applies a finalized block straight to the final tier. On success the /// finalized chain is authoritative, so head rebases onto it. fn apply_finalized_direct(&mut self, block: &Block, l1_slot: Slot) -> AcceptOutcome { + // A finalized block at or below the final tip is a re-delivery — e.g. + // the finalization of blocks restored from the store at boot, which + // are the final tier from the start and so never sit in `head_blocks`. + // Idempotent, no stall. A *different* block at the tip height falls + // through to validation and parks: finalized is irreversible, so a + // conflicting finalized block at final height is a genuine stall. + if let Some(tip) = &self.final_tip + && (block.header.block_id < tip.block_id + || (block.header.block_id == tip.block_id && block.header.hash == tip.hash)) + { + return AcceptOutcome::AlreadyApplied; + } + let mut scratch = self.final_state.clone(); match apply_block(self.final_tip.as_ref(), block, &mut scratch) { Ok(()) => { @@ -624,6 +637,50 @@ mod tests { assert_head_matches_replay(&chain); } + #[test] + fn finalized_redelivery_at_or_below_final_tip_is_already_applied() { + // Restart shape: the store's tip (incl. not-yet-finalized blocks) is + // restored as the final tier, so their later finalization arrives for + // blocks that were never in `head_blocks`. + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + chain.apply_finalized(msg(1), &genesis, slot(10)); + chain.apply_finalized(msg(2), &block2, slot(20)); + + // Below the tip, and at the tip with a matching hash: idempotent. + assert!(matches!( + chain.apply_finalized(msg(41), &genesis, slot(30)), + AcceptOutcome::AlreadyApplied + )); + assert!(matches!( + chain.apply_finalized(msg(42), &block2, slot(30)), + AcceptOutcome::AlreadyApplied + )); + assert!(chain.final_stall().is_none()); + assert_eq!(chain.final_tip().expect("final tip").block_id, 2); + assert_head_matches_replay(&chain); + } + + #[test] + fn conflicting_finalized_at_final_tip_parks() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + chain.apply_finalized(msg(1), &genesis, slot(10)); + chain.apply_finalized(msg(2), &block2, slot(20)); + + // A different finalized block at the final height: finalized is + // irreversible, so this is a genuine stall, not a re-delivery. + let block2_prime = produce_dummy_block(2, Some(HashType([9; 32])), vec![]); + assert!(matches!( + chain.apply_finalized(msg(22), &block2_prime, slot(30)), + AcceptOutcome::Parked(_) + )); + assert!(chain.final_stall().is_some()); + assert_eq!(chain.final_tip().expect("final tip").block_id, 2); + } + #[test] fn finalized_unknown_block_rebases_head() { let accounts = initial_pub_accounts_private_keys(); diff --git a/lez/chain_state/src/lib.rs b/lez/chain_state/src/lib.rs index c4725ec3..6398ce77 100644 --- a/lez/chain_state/src/lib.rs +++ b/lez/chain_state/src/lib.rs @@ -2,7 +2,7 @@ //! the [`apply_block`] entry point plus [`BlockIngestError`], [`StallReason`], //! [`Tip`], and [`AcceptOutcome`]. See `DESIGN.md` for the two-tier model. -pub use apply::{AcceptOutcome, Tip, apply_block}; +pub use apply::{AcceptOutcome, Tip, apply_block, apply_block_to_state, validate_against_tip}; pub use chain::{ChainState, HeadEntry}; pub use ingest_error::BlockIngestError; pub use stall_reason::StallReason; diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 7a9f410d..0a6efe28 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -1,7 +1,9 @@ use std::{path::Path, sync::Arc}; use anyhow::{Context as _, Result}; -use chain_state::{AcceptOutcome, BlockIngestError, StallReason, Tip, apply_block}; +use chain_state::{ + AcceptOutcome, BlockIngestError, StallReason, Tip, apply_block_to_state, validate_against_tip, +}; use common::{ block::{BedrockStatus, Block, BlockHeader}, transaction::LeeTransaction, @@ -236,9 +238,17 @@ impl IndexerStore { return Ok(AcceptOutcome::AlreadyApplied); } + // Fail fast on validation before paying for the scratch clone — while + // parked, every re-delivered non-chaining block takes this path. + // Validation failures are never retryable, so parking here is exact. + if let Err(err) = validate_against_tip(tip.as_ref(), block) { + self.record_stall(Some(&block.header), l1_slot, err.clone())?; + return Ok(AcceptOutcome::Parked(err)); + } + // TODO: we use scratch state to be atomic, but need to revisit how expensive a clone is let mut scratch = self.current_state.read().await.clone(); - if let Err(err) = apply_block(tip.as_ref(), block, &mut scratch) { + if let Err(err) = apply_block_to_state(block, &mut scratch) { if err.is_retryable() { return Ok(AcceptOutcome::RetryableFailure(err)); } diff --git a/lez/mempool/src/lib.rs b/lez/mempool/src/lib.rs index 1b36eaf7..0006f2c3 100644 --- a/lez/mempool/src/lib.rs +++ b/lez/mempool/src/lib.rs @@ -65,6 +65,11 @@ impl MemPoolHandle { pub async fn push(&self, item: T) -> Result<(), tokio::sync::mpsc::error::SendError> { self.sender.send(item).await } + + /// Send an item to the mempool, failing _immediately_ if it is full. + pub fn try_push(&self, item: T) -> Result<(), tokio::sync::mpsc::error::TrySendError> { + self.sender.try_send(item) + } } #[cfg(test)] @@ -123,6 +128,19 @@ mod tests { assert_eq!(pool.pop(), Some(2)); } + #[test] + async fn try_push_fails_when_full_without_blocking() { + let (mut pool, handle) = MemPool::new(1); + + handle.try_push(1).unwrap(); + assert!(handle.try_push(2).is_err(), "full mempool must not accept"); + + // Popping frees capacity again. + assert_eq!(pool.pop(), Some(1)); + handle.try_push(2).unwrap(); + assert_eq!(pool.pop(), Some(2)); + } + #[test] async fn push_front() { let (mut pool, handle) = MemPool::new(10); diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 3ebc8abf..1fb52b1a 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -363,12 +363,8 @@ impl SequencerCore { mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>, ) -> block_publisher::OnFollowSink { Box::new(move |update: block_publisher::FollowUpdate| { - Box::pin(apply_follow_update( - Arc::clone(&dbio), - Arc::clone(&chain), - mempool_handle.clone(), - update, - )) + apply_follow_update(&dbio, &chain, &mempool_handle, update); + Box::pin(std::future::ready(())) }) } @@ -714,24 +710,30 @@ struct BlockWithMeta { /// relies on a valid successor or a restart. `ChainState` never emits /// `AcceptOutcome::RetryableFailure` yet; adding retry parity here is a /// follow-up. -async fn apply_follow_update( - dbio: Arc, - chain: Arc>, - mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>, +fn apply_follow_update( + dbio: &RocksDBIO, + chain: &Mutex, + mempool_handle: &MemPoolHandle<(TransactionOrigin, LeeTransaction)>, update: block_publisher::FollowUpdate, ) { + let block_publisher::FollowUpdate { + adopted, + orphaned, + finalized, + } = update; + // Apply under the lock and collect what to persist; take a single // head snapshot. Release the lock before touching disk so the // producer is never blocked on the follow path's I/O. let (to_persist, resubmit_txs, head_snapshot) = { let mut chain = chain.lock().expect("chain state mutex poisoned"); let mut resubmit_txs = Vec::new(); - for (this_msg, block) in &update.orphaned { + for (this_msg, block) in &orphaned { chain.revert_orphan(*this_msg); resubmit_txs.extend(resubmittable_txs(block)); } let mut to_persist = Vec::new(); - for (this_msg, block) in &update.adopted { + for (this_msg, block) in &adopted { if matches!( chain.apply_adopted(*this_msg, block), AcceptOutcome::Applied @@ -739,8 +741,8 @@ async fn apply_follow_update( to_persist.push((block, false)); } } - for (this_msg, block) in &update.finalized { - // TODO: thread the finalized inscription's L1 slot once the + for (this_msg, block) in &finalized { + // FIXME: thread the finalized inscription's L1 slot once the // sdk surfaces it; only used for the invalid-finalized stall. if matches!( chain.apply_finalized(*this_msg, block, Slot::from(0)), @@ -767,9 +769,14 @@ async fn apply_follow_update( // Rebuild orphaned work: return its user txs to the mempool so the // next on-turn production re-includes them on the new head. + // + // We use [`try_push`] here because this is called from the + // publisher's drive task, and only the block production drains the mempool. + // A blocking push on a full mempool would deadlock here. for tx in resubmit_txs { - if let Err(err) = mempool_handle.push((TransactionOrigin::User, tx)).await { - error!("Failed to resubmit orphaned transaction: {err:#}"); + let tx_hash = tx.hash(); + if let Err(err) = mempool_handle.try_push((TransactionOrigin::User, tx)) { + warn!("Dropping orphaned transaction {tx_hash} on resubmit: {err}"); } } } diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 965136ba..46af7019 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -1351,16 +1351,15 @@ async fn follow_adopted_peer_block_applies_and_persists() { let peer_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![tx]); apply_follow_update( - sequencer.store.dbio(), - sequencer.chain(), - mempool_handle.clone(), + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, FollowUpdate { adopted: vec![(MsgId::from([1; 32]), peer_block.clone())], orphaned: vec![], finalized: vec![], }, - ) - .await; + ); assert_eq!(sequencer.chain_height(), 2); let stored = sequencer @@ -1400,16 +1399,15 @@ async fn follow_redelivery_of_own_block_is_deduped() { // The channel redelivers our own block under the MsgId the mock publisher // assigned at publish time. apply_follow_update( - sequencer.store.dbio(), - sequencer.chain(), - mempool_handle.clone(), + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, FollowUpdate { - adopted: vec![(MsgId::from(block2.header.hash.0), block2.clone())], + adopted: vec![(MsgId::from(block2.header.hash.0), block2)], orphaned: vec![], finalized: vec![], }, - ) - .await; + ); assert_eq!(sequencer.chain_height(), 2); assert_eq!( @@ -1442,16 +1440,15 @@ async fn follow_orphan_reverts_head_and_requeues_user_txs() { let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); apply_follow_update( - sequencer.store.dbio(), - sequencer.chain(), - mempool_handle.clone(), + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, FollowUpdate { adopted: vec![], - orphaned: vec![(MsgId::from(block2.header.hash.0), block2.clone())], + orphaned: vec![(MsgId::from(block2.header.hash.0), block2)], finalized: vec![], }, - ) - .await; + ); assert_eq!(sequencer.chain_height(), 1); assert_eq!( @@ -1486,16 +1483,15 @@ async fn follow_finalized_own_block_moves_final_tier_and_marks_store() { let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); apply_follow_update( - sequencer.store.dbio(), - sequencer.chain(), - mempool_handle.clone(), + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, FollowUpdate { adopted: vec![], orphaned: vec![], - finalized: vec![(MsgId::from(block2.header.hash.0), block2.clone())], + finalized: vec![(MsgId::from(block2.header.hash.0), block2)], }, - ) - .await; + ); let final_tip = sequencer .chain() @@ -1520,16 +1516,15 @@ async fn follow_finalized_backfill_block_is_applied_and_marked_finalized() { let peer_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![]); apply_follow_update( - sequencer.store.dbio(), - sequencer.chain(), - mempool_handle.clone(), + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, FollowUpdate { adopted: vec![], orphaned: vec![], finalized: vec![(MsgId::from([2; 32]), peer_block.clone())], }, - ) - .await; + ); assert_eq!( sequencer.chain_height(), @@ -1642,19 +1637,18 @@ async fn follow_update_persists_blocks_meta_and_state_atomically() { // One update carrying several blocks: both adopted, block 2 also finalized. apply_follow_update( - sequencer.store.dbio(), - sequencer.chain(), - mempool_handle.clone(), + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, FollowUpdate { adopted: vec![ (MsgId::from([2; 32]), block2.clone()), (MsgId::from([3; 32]), block3.clone()), ], orphaned: vec![], - finalized: vec![(MsgId::from([2; 32]), block2.clone())], + finalized: vec![(MsgId::from([2; 32]), block2)], }, - ) - .await; + ); // Blocks, tip meta and state all reflect the end of the batch: a late // finalized entry for an earlier block must not drag the tip meta back. From 013504971001b8c8eb971075ce2e9e1021c405b3 Mon Sep 17 00:00:00 2001 From: erhant Date: Thu, 16 Jul 2026 18:11:44 +0300 Subject: [PATCH 31/32] fix(chain_state, sequencer, storage): rebuild the two-tier state on restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - persist a final-tier snapshot (state + meta) atomically with the follow update that advanced it; on boot, anchor `final` on it and replay stored blocks above it as `head`, so a post-restart orphan of a not-yet-finalized block still reverts instead of silently diverging - correlate orphan/finalize events by block hash: restored head entries carry hash-derived sentinel MsgIds (the real ones are not persisted) - a finalized block chaining on an unfinalized head entry finalizes that prefix (finality is prefix-monotone) - warn on (and keep ignoring) a same-height adopted competitor that arrives without its orphan sibling — an SDK-contract breach worth surfacing - follow path reverts orphans via one batched `apply_channel_update` (single truncate + head re-derivation) instead of a re-derivation per orphan - remove DESIGN.md, move to task docs instead - trim very long comments / docstrings --- Cargo.lock | 1 + lez/chain_state/Cargo.toml | 1 + lez/chain_state/DESIGN.md | 308 ------------------- lez/chain_state/src/chain.rs | 228 +++++++++++--- lez/chain_state/src/lib.rs | 2 +- lez/indexer/core/src/block_store.rs | 5 +- lez/sequencer/core/src/lib.rs | 206 +++++++++---- lez/sequencer/core/src/tests.rs | 110 +++++++ lez/storage/src/sequencer/mod.rs | 65 ++-- lez/storage/src/sequencer/sequencer_cells.rs | 73 ++++- lez/storage/src/sequencer/tests.rs | 33 ++ 11 files changed, 592 insertions(+), 440 deletions(-) delete mode 100644 lez/chain_state/DESIGN.md diff --git a/Cargo.lock b/Cargo.lock index 06a3e08e..792f3856 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1376,6 +1376,7 @@ dependencies = [ "borsh", "common", "lee", + "log", "logos-blockchain-core", "logos-blockchain-zone-sdk", "serde", diff --git a/lez/chain_state/Cargo.toml b/lez/chain_state/Cargo.toml index ca9876ab..f788f55a 100644 --- a/lez/chain_state/Cargo.toml +++ b/lez/chain_state/Cargo.toml @@ -14,6 +14,7 @@ logos-blockchain-core.workspace = true logos-blockchain-zone-sdk.workspace = true anyhow.workspace = true +log.workspace = true serde.workspace = true thiserror.workspace = true diff --git a/lez/chain_state/DESIGN.md b/lez/chain_state/DESIGN.md deleted file mode 100644 index cb8595aa..00000000 --- a/lez/chain_state/DESIGN.md +++ /dev/null @@ -1,308 +0,0 @@ -# `lez/chain_state` — Two-Tier Chain State - -Design doc for the shared block-apply engine and two-tier chain state that -backs decentralized sequencing. Status: **interface freeze** — the -`apply_block` signature and the `ChainState` tip/state shape below are the -contract the produce-on-turn and follow-blocks tracks build against. Changing -them after the tracks split forces rework in both. - -## 1. Motivation - -Decentralized sequencing requires every honest node — sequencer or indexer — to -converge on the same chain and the same state by running one deterministic -_validate-then-apply_ path over blocks pulled from the channel. That path today -lives only inside the indexer (`lez/indexer/core/src/block_store.rs`), where the -recoverability work built a park-and-skip ingest: `accept_block` validates a -block against the current tip, applies it to a scratch clone of state atomically, -and on any failure records a `StallReason`, freezes the tip, and marks the bad -block _processed_ without applying it. The sequencer has no equivalent — it only -produces blocks and reads peer inscriptions for finalization; it never executes -peer blocks into its own state. - -This crate lifts that logic into a shared home and generalizes it into a -**two-tier** state machine so the sequencer can produce on the head while both -sequencer and indexer expose their exact current state. - -## 2. Crate placement & layering - -``` -lee_core ← lee (owns V03State) ← common (owns Block, BedrockStatus, clock_invocation, recompute_hash) ← lez/chain_state ← { indexer/core, sequencer/core } -``` - -- **Not** `lee`: the apply logic needs `Block`/`BedrockStatus`/`clock_invocation` - from `common`, and `common` depends on `lee` — putting it in `lee` inverts the - layer. -- `lez/chain_state` sits above `common`, depends on `common` + `lee`, and is - consumed by both `indexer/core` and `sequencer/core`. - -**Persistence boundary.** `chain_state` holds the in-memory state machine and the -pure logic; it performs **no I/O**. Each consumer keeps its own `RocksDBIO` and -drives the `scratch → put_block → commit` ordering, exactly as `accept_block` does -today. This keeps the crate fully unit-testable without a DB. - -## 3. The `apply_block` entry point - -A single pure function, called identically whether the block was produced by us, -adopted from a peer, or read finalized from the channel: - -```rust -/// Validate `block` against `tip`, then apply it to `state`. Pure: no I/O. -/// Mutates `state` only on success; on failure `state` is untouched and the -/// caller parks. -fn apply_block( - tip: Option<&Tip>, - block: &Block, - state: &mut V03State, -) -> Result<(), BlockIngestError>; -``` - -Validation order (unchanged from the indexer): hash integrity -(`recompute_hash`) → block-id continuity → `prev_block_hash` linkage, with a -`None` tip expecting the genesis block. Application splits off the mandatory -trailing clock tx, executes user txs (genesis = public-only), applies the clock -last. - -Shared types moved into the crate: `AcceptOutcome`, `BlockIngestError`, -`StallReason`, and `Tip`. - -```rust -struct Tip { block_id: u64, hash: HashType } - -enum AcceptOutcome { - Applied, - AlreadyApplied, - Parked(BlockIngestError), - RetryableFailure(BlockIngestError), -} -``` - -`Tip` will additionally carry `l1_slot` once the anchor layer lands (recorded -atomically with the tip), because the anchor / chain-consistency logic keys on -the inscription slot, not just `(id, hash)`. - -## 4. The two-tier `ChainState` - -```rust -struct ChainState { - final_state: V03State, // driven by finalized channel ops - final_tip: Option, - head_state: V03State, // final_state + applied head blocks - head_blocks: Vec, // ordered, above final_tip - final_stall: Option, // the one stall — persisted to RocksDB. See §4a -} - -struct HeadEntry { this_msg: MsgId, block: Block } -``` - -The **head** tier is a MsgId-keyed chain (adopted/orphaned reference -`this_msg`/`parent_msg`); the **final** tier is block-id-keyed. `apply_block` -validation stays LEZ-level (`block_id` + `prev_block_hash`) — the two chains run -in parallel and must agree, so we validate via `apply_block` _and_ track `MsgId` -for revert correlation. - -Operations: - -- `apply_adopted(inscription) -> AcceptOutcome` — dedup by `this_msg` against our - outbox, else `apply_block` on the head tip; on success push `HeadEntry`. On - failure, **do nothing durable**: the head tip simply stays at the last valid - block. No stall is recorded (see §4a) — the head self-heals from the stream. -- `apply_channel_update(orphaned, adopted)` — revert every `orphaned` by - `this_msg`, re-derive `head_state` (clone `final_state`, replay survivors), - then apply every `adopted` in order. Atomic per event. -- `finalize_up_to(block_id)` — move `head_blocks` up to `block_id` into - `final_state` (already validated; a move, not a re-apply). -- `apply_finalized(inscription)` — steady state: if present in head by - `this_msg`, `finalize_up_to`; cold-start backfill (not in head): - `apply_block` directly to `final_state`, mirror into head. If a finalized block - fails to apply, set **`final_stall`** and persist it — this is the **only** stall - (see §4a). -- `rollback_orphan(this_msg)` — drop from that entry forward, re-derive head. -- `status() -> { final_height, head_height, final_stall }` — for RPC/UI. A derived - "head blocked" indicator can be computed on demand (see §4a) without persisting. - -For the **indexer** (finalized-only `next_messages` stream), `head_blocks` stays -empty and `head == final`; it exercises only `apply_finalized`. The **sequencer** -uses both tiers from day one. - -### 4a. One stall — `final_stall`, persisted - -There is a single stall, `final_stall`, on the final tier. The head tier does -**not** carry its own stall, and this is deliberate. - -The head and the final tier never represent two independent problems: a block -always reaches the head first (as `adopted`) and only later the final tier (as -`finalized`), so a would-be "head stall" is just the earlier, provisional sighting -of the exact block that `final_stall` records durably if it finalizes — the same -event modeled twice. - -And the head does not need a recorded reason to freeze. **The tip-freeze is -intrinsic**: not applying a bad block is what freezes the tip; no marker is -required. The head's freeze is also **transient and self-healing** — the bad block -is either orphaned (a competing valid block at the same height wins and applies on -its own) or it finalizes. Subsequent adopted blocks that chain on the bad one fail -validation by themselves, and the producer builds on the head tip regardless of -any marker. So a persisted head stall would be redundant (re-derived from the -stream on restart) and buys no behavior. - -`final_stall` is the stall that does real work: - -- The **indexer already requires it** and ships it today. The indexer has only a - final tier (finalized-only stream, no head); its startup chain-consistency / - anchor check reads the persisted stall to know where it is parked. The shared - `final_stall` serves that unchanged. -- It **survives restart** and is what we surface as `Stalled`. -- "A bad block **finalized**" is the only irreversible, actionable condition — the - signal the committee acts on to evict a bad sequencer. A provisional head block - that may vanish on the next reorg is not something to evict over. - -The one thing we forgo is an *early warning* that a sequencer is posting garbage -before it finalizes. That condition frequently self-heals via reorg, so alarming on -it is mostly noise; if wanted, it is a **derived, non-persisted** indicator (e.g. -"k adopted inscriptions above the head tip remain unapplied"), computed on demand — -not a second `StallReason` in the struct. - -So the sequencer and the indexer share exactly one stall concept, keeping the two -consumers uniform. - -### 4b. Producer contract — write on turn, build on last valid - -The sequencer publishes **only on its own turn** (the SDK queues out-of-turn -publishes). When it is our turn we build the next block on the **current head -tip**, which is by construction the last validly-applied block. So if the head is -frozen on a peer's bad block, we build on that frozen valid -tip — the same parent every honest sequencer chooses — and thereby skip the bad -block rather than extend it. A parked node keeps following peers' valid blocks as -they arrive; the moment it also gets a turn, it produces the next valid block on -its last valid tip. Net: parking never stops us from producing correctly on our -turn. - -## 5. Event → tier mapping - -`Event::BlocksProcessed { checkpoint, channel_update: { orphaned, adopted }, finalized }`: - -| Input | Source | Effect | -| --------------------- | ---------------------------------------------- | --------------------------------------------------- | -| adopted inscription | `channel_update.adopted` | validate + apply to **head** | -| orphaned inscription | `channel_update.orphaned` | revert from **head**, return txs to mempool | -| finalized inscription | `finalized[].ops` (`FinalizedOp::Inscription`) | move head→**final**, or apply directly on backfill | -| own publish | publish-return | optimistically apply to **head**, record `this_msg` | - -**Golden rules:** (1) validation is deterministic, so every honest node makes the -same accept/park decision. (2) An invalid block is _processed but discarded_ — -never applied, never halts the node. (3) Finalized is never reverted. (4) We -rebuild orphaned blocks ourselves; we do not trust the SDK's republish (it keeps -stale LEZ contents — prev-hash, tx selection, and resulting state were all -computed against the old parent). - ---- - -## 6. Scenarios - -### Processing one `BlocksProcessed` event - -```mermaid -flowchart TD - EV["Event::BlocksProcessed"] --> ORPH{"orphaned
non-empty?"} - ORPH -->|yes| REV["For each orphaned by this_msg:
drop from head_blocks,
return its txs to mempool"] - REV --> RED["Re-derive head_state:
clone final_state, replay survivors"] - ORPH -->|no| ADO - RED --> ADO{"adopted
non-empty?"} - ADO -->|"yes, in order"| DEDUP{"this_msg in
our outbox?"} - ADO -->|no| FIN - DEDUP -->|"yes (our own)"| SKIP["skip: already applied optimistically"] - DEDUP -->|no| VAL["apply_block on head tip"] - VAL --> OUT{"AcceptOutcome"} - OUT -->|Applied| APP["append this_msg+block to head,
advance head tip, clear stall"] - OUT -->|AlreadyApplied| SKIP - OUT -->|"Parked(err)"| PARK["freeze head tip — do NOT apply.
No stall recorded (self-heals
via reorg/finalization)"] - SKIP --> FIN - APP --> FIN - PARK --> FIN - FIN{"finalized
inscriptions?"} - FIN -->|"already in head (steady state)"| MOVE["finalize_up_to:
move head→final, trim head_blocks"] - FIN -->|"not in head (cold-start backfill)"| DIRECT["apply_block directly to final"] - DIRECT --> DOK{"applied?"} - DOK -->|yes| MIRROR["mirror into head"] - DOK -->|"no (invalid finalized)"| FSTALL["record StallReason on FINAL,
freeze final tip"] - FIN -->|none| CP - MOVE --> CP - MIRROR --> CP - FSTALL --> CP - CP["persist checkpoint atomically"] -``` - -### Park / recovery status - -```mermaid -stateDiagram-v2 - [*] --> Syncing - Syncing --> CaughtUp: stream drained, no stall - CaughtUp --> Syncing: new adopted / finalized arrives - Syncing --> Parked: invalid block FINALIZED (apply_finalized fails) - Parked --> Parked: further non-chaining finalized blocks (orphans_since++) - Parked --> Syncing: valid successor finalizes on frozen final tip → stall cleared - note right of Parked - final_stall — the one stall. Persisted, survives restart. - Head-tier bad blocks do NOT enter this state: - the head tip freezes intrinsically (no stall) and - self-heals via reorg/finalization. Producer (on our - turn) builds on the last valid tip either way. - end note -``` - -### Scenario table - -**Normal flow** - -| # | Scenario | Handling | Expected | -| --- | ---------------------------------------- | ------------------------------------------------------------- | ----------------------------------- | -| 1 | Adopted block chains cleanly on head tip | `apply_block` → `Applied`; append `{this_msg, block}` to head | head advances; converges with peers | -| 2 | Our own block comes back in `adopted` | dedup by `this_msg` against outbox → skip | no double-apply | -| 3 | Adopted block later finalizes | `finalize_up_to` moves head→final, trims `head_blocks` | final advances; no re-apply | -| 4 | Re-delivery of an already-applied block | id ≤ tip & stored hash matches → `AlreadyApplied` | idempotent, no state change | - -**Reorg / orphan** - -| # | Scenario | Handling | Expected | -| --- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------- | ----------------------------------------------- | -| 5 | Our block orphaned at turn handoff (stale-parent race) | revert by `this_msg`, return txs to mempool, **rebuild** on new head tip (not SDK republish) | our txs re-queued; next block on correct parent | -| 6 | Batch reorg: some `orphaned` + some `adopted` in one event | revert all orphaned, re-derive head, then apply all adopted in order | deterministic convergence | -| 7 | Orphan chain (parent transitively off canonical) | SDK surfaces all affected as `orphaned`; revert each, replay survivors | head_state matches new canonical branch | - -**Invalid / bad block** — "stall" below means the one **persisted `final_stall`** -(§4a). A bad block seen only in `adopted` records **no** stall — the head tip just -freezes and self-heals; it becomes a `final_stall` only if it finalizes. - -| # | Scenario | Handling | Expected | -| --- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | ----------------------------------------------- | -| 8 | Authorized sequencer posts a block with an invalid state transition | head: `apply_block` → `Parked`, freeze head tip, no stall recorded. If it finalizes: persisted `final_stall` | park-and-skip; no apply, no halt | -| 9 | Broken chain link / hash mismatch / unexpected id in adopted | `Parked(BrokenChainLink / HashMismatch / UnexpectedBlockId)`; same park | frozen tip; peers park identically | -| 10 | Undeserializable inscription payload | park with `Deserialize` (no header); processing advances | recover when a valid block chains on frozen tip | -| 11 | Valid successor after a park (recovery) | block chaining on frozen tip → `Applied` → clear stall | head resumes automatically; no divergence | -| 12 | Further non-chaining blocks while parked | keep first `StallReason`, bump `orphans_since` | original cause preserved; still parked | - -**Producing while parked** - -| # | Scenario | Handling | Expected | -| --- | ----------------------------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | -| 13 | It's our turn but head is parked on a bad block | producer builds on the **frozen valid tip** (head tip = last valid), skipping the invalid block | we emit the next valid block on the same parent honest peers use — chain moves on our turn | - -**Startup / backfill** - -| # | Scenario | Handling | Expected | -| --- | --------------------------------------------------- | -------------------------------------------------------------------------------------------------- | ------------------------------------- | -| 14 | Cold start / reconnect backfill | history via `finalized`, empty `channel_update`; apply directly to final + mirror head | head == final until live deltas start | -| 15 | Local store belongs to a different chain (L1 reset) | anchor-based `chain_consistency` check at startup: wipe+reindex if `allow_chain_reset`, else error | no silent divergence | - -## 7. Invariants - -Should-never-happen conditions — assert/log, don't silently absorb: - -- An `orphaned` entry never references a block at or below the **final** tip — - finalized is irreversible. If seen, it is a bug. -- `head` tip ≥ `final` tip at all times; `head_blocks` holds exactly the blocks - between them. -- After processing any event, `head_state == final_state` replayed through - `head_blocks` (the re-derivation is the source of truth). -- A parked node's frozen tip is identical across all honest nodes for the same - invalid block (deterministic validation). diff --git a/lez/chain_state/src/chain.rs b/lez/chain_state/src/chain.rs index c3a810c0..35dc458b 100644 --- a/lez/chain_state/src/chain.rs +++ b/lez/chain_state/src/chain.rs @@ -1,8 +1,9 @@ //! Two-tier chain state: a reorg-able `head` the sequencer builds on, plus an -//! irreversible `final` tier. See `DESIGN.md` for the model and rationale. +//! irreversible `final` tier. -use common::block::Block; +use common::{HashType, block::Block}; use lee::V03State; +use log::warn; use logos_blockchain_core::mantle::ops::channel::MsgId; use logos_blockchain_zone_sdk::Slot; @@ -21,6 +22,9 @@ pub struct HeadEntry { /// (irreversible, from `finalized`). /// /// `head_state` is given by `final_state` replayed through `head_blocks`. +/// +/// Only the final tier stalls: an invalid `adopted` block just freezes the +/// head tip and self-heals via reorg or finalization. pub struct ChainState { final_state: V03State, final_tip: Option, @@ -86,18 +90,50 @@ impl ChainState { self.final_stall.as_ref() } + /// Position of a head entry, matched by `MsgId` or block hash (restored + /// entries carry sentinel `MsgId`s; re-inscriptions arrive under fresh ones). + fn head_position_of(&self, this_msg: MsgId, block_hash: HashType) -> Option { + self.head_blocks + .iter() + .position(|entry| entry.this_msg == this_msg || entry.block.header.hash == block_hash) + } + + /// Hash of the block we hold at `block_id` (final tip or head entry), if any. + fn hash_at(&self, block_id: u64) -> Option { + if let Some(tip) = &self.final_tip + && tip.block_id == block_id + { + return Some(tip.hash); + } + + self.head_blocks + .iter() + .find(|entry| entry.block.header.block_id == block_id) + .map(|entry| entry.block.header.hash) + } + /// Applies an adopted head block. On failure the head tip stays at the last - /// valid block and no stall is recorded (§4a); the head self-heals. + /// valid block and no stall is recorded; the head self-heals. pub fn apply_adopted(&mut self, this_msg: MsgId, block: &Block) -> AcceptOutcome { let tip = self.head_tip(); - if self - .head_blocks - .iter() - .any(|entry| entry.this_msg == this_msg) - || tip - .as_ref() - .is_some_and(|current| block.header.block_id <= current.block_id) + if self.head_position_of(this_msg, block.header.hash).is_some() { + return AcceptOutcome::AlreadyApplied; + } + if tip + .as_ref() + .is_some_and(|current| block.header.block_id <= current.block_id) { + // Re-delivery of a block we already hold; a different hash means an + // adopted competitor arrived without its orphan sibling — warn, keep ours. + if let Some(known) = self.hash_at(block.header.block_id) + && known != block.header.hash + { + warn!( + "Ignoring adopted block {} with hash {} — a different block ({}) \ + holds this height and no orphan event preceded it", + block.header.block_id, block.header.hash, known + ); + } return AcceptOutcome::AlreadyApplied; } @@ -116,30 +152,23 @@ impl ChainState { } /// Reverts an orphaned head block and everything after it, then re-derives head. - pub fn revert_orphan(&mut self, this_msg: MsgId) { - if let Some(idx) = self - .head_blocks - .iter() - .position(|entry| entry.this_msg == this_msg) - { + pub fn revert_orphan(&mut self, this_msg: MsgId, block: &Block) { + if let Some(idx) = self.head_position_of(this_msg, block.header.hash) { self.head_blocks.truncate(idx); self.rederive_head(); } } - /// One channel update: revert every `orphaned`, then apply every `adopted`. + /// One channel update: revert every `orphaned` (one truncate + re-derive), + /// then apply every `adopted` in order. Outcomes align with `adopted`. pub fn apply_channel_update( &mut self, - orphaned: &[MsgId], + orphaned: &[(MsgId, Block)], adopted: &[(MsgId, Block)], ) -> Vec { let earliest = orphaned .iter() - .filter_map(|msg| { - self.head_blocks - .iter() - .position(|entry| entry.this_msg == *msg) - }) + .filter_map(|(msg, block)| self.head_position_of(*msg, block.header.hash)) .min(); if let Some(idx) = earliest { self.head_blocks.truncate(idx); @@ -151,6 +180,20 @@ impl ChainState { .collect() } + /// Rebuilds one head entry from a persisted block, applying it in place (the + /// caller treats `Err` as fatal). + /// + /// Pass a hash-derived sentinel for `this_msg`; orphan/finalize events then correlate by block hash. + pub fn restore_head_block( + &mut self, + this_msg: MsgId, + block: Block, + ) -> Result<(), BlockIngestError> { + apply_block(self.head_tip().as_ref(), &block, &mut self.head_state)?; + self.head_blocks.push(HeadEntry { this_msg, block }); + Ok(()) + } + /// A finalized inscription. In steady state the block is already in head and is /// moved into `final`; on backfill (not in head) it is applied directly and may /// set `final_stall`. @@ -160,18 +203,22 @@ impl ChainState { block: &Block, l1_slot: Slot, ) -> AcceptOutcome { - // Match by [`MsgId`] or by block identity to handle a re-inscribed but - // identical block arriving under a fresh [`MsgId`]. - let in_head = self.head_blocks.iter().position(|entry| { - entry.this_msg == this_msg || entry.block.header.hash == block.header.hash - }); - match in_head { - Some(idx) => { - self.finalize_through(idx); - AcceptOutcome::Applied - } - None => self.apply_finalized_direct(block, l1_slot), + // Match by `MsgId` or block hash (re-inscriptions, restored entries). + if let Some(idx) = self.head_position_of(this_msg, block.header.hash) { + self.finalize_through(idx); + return AcceptOutcome::Applied; } + + // Finality is prefix-monotone: a finalized block chaining on an + // unfinalized head entry finalizes that prefix too. + if let Some(idx) = self + .head_blocks + .iter() + .position(|entry| entry.block.header.hash == block.header.prev_block_hash) + { + self.finalize_through(idx); + } + self.apply_finalized_direct(block, l1_slot) } /// Moves `head_blocks[0..=idx]` into the final tier (already validated in head). @@ -188,12 +235,9 @@ impl ChainState { /// Applies a finalized block straight to the final tier. On success the /// finalized chain is authoritative, so head rebases onto it. fn apply_finalized_direct(&mut self, block: &Block, l1_slot: Slot) -> AcceptOutcome { - // A finalized block at or below the final tip is a re-delivery — e.g. - // the finalization of blocks restored from the store at boot, which - // are the final tier from the start and so never sit in `head_blocks`. - // Idempotent, no stall. A *different* block at the tip height falls - // through to validation and parks: finalized is irreversible, so a - // conflicting finalized block at final height is a genuine stall. + // A finalized block at or below the final tip is a re-delivery: + // idempotent. A *different* block at the tip height falls through + // to validation and parks. if let Some(tip) = &self.final_tip && (block.header.block_id < tip.block_id || (block.header.block_id == tip.block_id && block.header.hash == tip.hash)) @@ -359,7 +403,7 @@ mod tests { chain.apply_adopted(msg(2), &block2); chain.apply_adopted(msg(3), &block3); - chain.revert_orphan(msg(3)); + chain.revert_orphan(msg(3), &block3); assert_eq!(chain.head_tip().expect("head tip").block_id, 2); // A competing block 3 now applies cleanly on block 2. @@ -382,7 +426,7 @@ mod tests { chain.apply_adopted(msg(3), &block3); let block3_prime = produce_dummy_block(3, Some(block2.header.hash), vec![]); - let outcomes = chain.apply_channel_update(&[msg(3)], &[(msg(13), block3_prime)]); + let outcomes = chain.apply_channel_update(&[(msg(3), block3)], &[(msg(13), block3_prime)]); assert!(matches!(outcomes.as_slice(), [AcceptOutcome::Applied])); assert_eq!(chain.head_tip().expect("head tip").block_id, 3); } @@ -458,7 +502,7 @@ mod tests { chain.apply_adopted(msg(4), &block4); // Orphaning block 3 drops the whole suffix (3 and 4). - chain.revert_orphan(msg(3)); + chain.revert_orphan(msg(3), &block3); assert_eq!(chain.head_tip().expect("head tip").block_id, 2); assert_eq!(chain.head_state().get_account_by_id(from).balance, 9990); @@ -494,7 +538,7 @@ mod tests { let block4_prime = produce_dummy_block(4, Some(block3_prime.header.hash), vec![tx4_prime]); let outcomes = chain.apply_channel_update( - &[msg(4), msg(3)], + &[(msg(4), block4), (msg(3), block3)], &[(msg(13), block3_prime), (msg(14), block4_prime)], ); @@ -517,13 +561,84 @@ mod tests { chain.apply_adopted(msg(2), &block2); let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); - let outcomes = chain.apply_channel_update(&[msg(99)], &[(msg(3), block3)]); + let unknown = produce_dummy_block(9, Some(HashType([7; 32])), vec![]); + let outcomes = chain.apply_channel_update(&[(msg(99), unknown)], &[(msg(3), block3)]); assert!(matches!(outcomes.as_slice(), [AcceptOutcome::Applied])); assert_eq!(chain.head_tip().expect("head tip").block_id, 3); assert_head_matches_replay(&chain); } + #[test] + fn same_height_adopted_competitor_without_orphan_is_ignored() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + + // Same height, different hash, no preceding orphan event: an SDK + // contract breach. We keep the block we already applied (and warn). + let block2_prime = produce_dummy_block(2, Some(HashType([9; 32])), vec![]); + assert!(matches!( + chain.apply_adopted(msg(12), &block2_prime), + AcceptOutcome::AlreadyApplied + )); + assert_eq!(chain.head_tip().expect("head tip").hash, block2.header.hash); + assert_head_matches_replay(&chain); + } + + #[test] + fn restore_head_block_rebuilds_head_and_correlates_by_hash() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + // Restart shape: final tier from a persisted snapshot, head rebuilt from + // stored blocks under hash-derived sentinel MsgIds. + let mut state = initial_state(); + let genesis = produce_dummy_block(1, None, vec![]); + apply_block(None, &genesis, &mut state).expect("genesis applies"); + let mut chain = ChainState::from_final(state, Some(tip_of(&genesis))); + + let tx2 = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2]); + let tx3 = create_transaction_native_token_transfer(from, 1, to, 10, &sign_key); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![tx3]); + for block in [&block2, &block3] { + chain + .restore_head_block(MsgId::from(block.header.hash.0), block.clone()) + .expect("stored blocks must replay"); + } + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + assert_head_matches_replay(&chain); + + // The L1 orphans restored block 3 under its real (unknown-to-us) MsgId: + // correlated by hash, the revert works and a competitor applies. + chain.revert_orphan(msg(33), &block3); + assert_eq!(chain.head_tip().expect("head tip").block_id, 2); + + let block3_prime = produce_dummy_block(3, Some(block2.header.hash), vec![]); + assert!(matches!( + chain.apply_adopted(msg(13), &block3_prime), + AcceptOutcome::Applied + )); + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20010); + assert_head_matches_replay(&chain); + } + + #[test] + fn restore_head_block_rejects_non_chaining_block() { + let mut chain = ChainState::new(initial_state()); + let skipped = produce_dummy_block(3, Some(HashType([9; 32])), vec![]); + assert!( + chain + .restore_head_block(MsgId::from(skipped.header.hash.0), skipped) + .is_err() + ); + } + #[test] fn finalized_reinscription_matches_by_block_hash() { let mut chain = ChainState::new(initial_state()); @@ -637,6 +752,29 @@ mod tests { assert_head_matches_replay(&chain); } + #[test] + fn finalized_successor_of_head_entry_finalizes_the_prefix() { + // Head holds unfinalized blocks 1..=2 (e.g. restored after a restart); + // a peer block 3 we never saw adopted arrives finalized. Its ancestry + // finalizes our prefix implicitly, then 3 applies to final directly. + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + assert!(chain.final_tip().is_none()); + + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + assert!(matches!( + chain.apply_finalized(msg(3), &block3, slot(10)), + AcceptOutcome::Applied + )); + assert_eq!(chain.final_tip().expect("final tip").block_id, 3); + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + assert!(chain.final_stall().is_none()); + assert_head_matches_replay(&chain); + } + #[test] fn finalized_redelivery_at_or_below_final_tip_is_already_applied() { // Restart shape: the store's tip (incl. not-yet-finalized blocks) is diff --git a/lez/chain_state/src/lib.rs b/lez/chain_state/src/lib.rs index 6398ce77..5a9a026b 100644 --- a/lez/chain_state/src/lib.rs +++ b/lez/chain_state/src/lib.rs @@ -1,6 +1,6 @@ //! Storage-free chain-state core shared by the LEZ sequencer and indexer: //! the [`apply_block`] entry point plus [`BlockIngestError`], [`StallReason`], -//! [`Tip`], and [`AcceptOutcome`]. See `DESIGN.md` for the two-tier model. +//! [`Tip`], and [`AcceptOutcome`]. See [`ChainState`] for the two-tier model. pub use apply::{AcceptOutcome, Tip, apply_block, apply_block_to_state, validate_against_tip}; pub use chain::{ChainState, HeadEntry}; diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 0a6efe28..4722bf26 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -238,9 +238,8 @@ impl IndexerStore { return Ok(AcceptOutcome::AlreadyApplied); } - // Fail fast on validation before paying for the scratch clone — while - // parked, every re-delivered non-chaining block takes this path. - // Validation failures are never retryable, so parking here is exact. + // Validate before paying for the scratch clone; validation failures + // are never retryable, so parking immediately is exact. if let Err(err) = validate_against_tip(tip.as_ref(), block) { self.record_stall(Some(&block.header), l1_slot, err.clone())?; return Ok(AcceptOutcome::Parked(err)); diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 1fb52b1a..580edc44 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -9,7 +9,7 @@ use borsh::BorshDeserialize; use chain_state::{AcceptOutcome, ChainState, Tip}; use common::{ HashType, - block::{BedrockStatus, Block, HashableBlockData}, + block::{BedrockStatus, Block, BlockMeta, HashableBlockData}, transaction::{LeeTransaction, clock_invocation}, }; use config::{GenesisAction, SequencerConfig}; @@ -128,6 +128,67 @@ impl SequencerCore { } } + /// Rebuilds the two-tier [`ChainState`]: the final tier from the persisted + /// final snapshot (pre-genesis state when absent), the head tier by replaying + /// every stored block above it, so a post-restart orphan can still revert. + fn restore_chain_state( + config: &SequencerConfig, + store: &SequencerStore, + stored_head_state: &lee::V03State, + ) -> ChainState { + let final_snapshot = store + .dbio() + .get_final_snapshot() + .expect("Failed to read final snapshot from store"); + let (final_state, final_tip) = match final_snapshot { + Some((state, meta)) => ( + state, + Some(Tip { + block_id: meta.id, + hash: meta.hash, + }), + ), + // Nothing finalized yet: replay the whole stored chain. + None => (build_initial_state(config), None), + }; + let boundary = final_tip.as_ref().map_or(0, |tip| tip.block_id); + + let mut head_blocks = store + .get_all_blocks() + .filter_ok(|block| block.header.block_id > boundary) + .collect::, _>>() + .expect("Failed to read blocks from store while restoring chain state"); + head_blocks.sort_unstable_by_key(|block| block.header.block_id); + + let mut chain = ChainState::from_final(final_state, final_tip); + for block in head_blocks { + let block_id = block.header.block_id; + // NOTE: sentinel for the real (unpersisted) `MsgId`; never leaves the + // process, events correlate by hash. Persisting the real one needs a + // sidecar cell (block_id -> MsgId), since growing the shared `Block` + // type would ripple through every consumer's serialization. + let sentinel = MsgId::from(block.header.hash.0); + chain + .restore_head_block(sentinel, block) + .unwrap_or_else(|err| { + panic!( + "Stored block {block_id} does not replay while restoring chain state: {err}" + ) + }); + } + + // The replayed head must reproduce the persisted state byte-for-byte, + // else store and config disagree (e.g. edited genesis actions). + let replayed = borsh::to_vec(chain.head_state()).expect("state serializes"); + let stored = borsh::to_vec(stored_head_state).expect("state serializes"); + assert_eq!( + replayed, stored, + "Persisted state does not match the replayed chain; reset the store or restore the original config" + ); + + chain + } + pub async fn start_from_config( config: SequencerConfig, ) -> (Self, MemPoolHandle<(TransactionOrigin, LeeTransaction)>) { @@ -141,16 +202,8 @@ impl SequencerCore { let (store, state, _genesis_block) = Self::open_or_create_store(&config); - let latest_block_meta = store - .latest_block_meta() - .expect("Failed to read latest block meta from store"); - - let chain = Arc::new(Mutex::new(ChainState::from_final( - state, - Some(Tip { - block_id: latest_block_meta.id, - hash: latest_block_meta.hash, - }), + let chain = Arc::new(Mutex::new(Self::restore_chain_state( + &config, &store, &state, ))); let initial_checkpoint = store @@ -722,25 +775,27 @@ fn apply_follow_update( finalized, } = update; - // Apply under the lock and collect what to persist; take a single - // head snapshot. Release the lock before touching disk so the - // producer is never blocked on the follow path's I/O. - let (to_persist, resubmit_txs, head_snapshot) = { + // Apply under the lock and collect what to persist; release it before + // touching disk so the producer is never blocked on follow I/O. + let (to_persist, resubmit_txs, head_snapshot, final_snapshot) = { let mut chain = chain.lock().expect("chain state mutex poisoned"); - let mut resubmit_txs = Vec::new(); - for (this_msg, block) in &orphaned { - chain.revert_orphan(*this_msg); - resubmit_txs.extend(resubmittable_txs(block)); - } - let mut to_persist = Vec::new(); - for (this_msg, block) in &adopted { - if matches!( - chain.apply_adopted(*this_msg, block), - AcceptOutcome::Applied - ) { - to_persist.push((block, false)); - } - } + + // User txs of orphaned blocks, returned to the mempool below. + let resubmit_txs: Vec = orphaned + .iter() + .flat_map(|(_, block)| resubmittable_txs(block)) + .collect(); + + // Outcomes align with `adopted`. + let outcomes = chain.apply_channel_update(&orphaned, &adopted); + let mut to_persist: Vec<(&Block, bool)> = adopted + .iter() + .zip(&outcomes) + .filter(|(_, outcome)| matches!(outcome, AcceptOutcome::Applied)) + .map(|((_, block), _)| (block, false)) + .collect(); + + let mut final_advanced = false; for (this_msg, block) in &finalized { // FIXME: thread the finalized inscription's L1 slot once the // sdk surfaces it; only used for the invalid-finalized stall. @@ -749,9 +804,27 @@ fn apply_follow_update( AcceptOutcome::Applied ) { to_persist.push((block, true)); + final_advanced = true; } } - (to_persist, resubmit_txs, chain.head_state().clone()) + // Snapshot the advanced final tier so a restart re-anchors on it. + let final_snapshot = final_advanced.then(|| { + let tip = chain.final_tip().expect("advanced final tier has a tip"); + ( + chain.final_state().clone(), + BlockMeta { + id: tip.block_id, + hash: tip.hash, + }, + ) + }); + + ( + to_persist, + resubmit_txs, + chain.head_state().clone(), + final_snapshot, + ) }; // One atomic write for the whole update: blocks, tip meta and the state @@ -763,7 +836,11 @@ fn apply_follow_update( // ever landing in the store. Full `BlocksProcessed` atomicity (checkpoint // + blocks + state in one batch, per the sdk's event contract) is a // follow-up. - if let Err(err) = dbio.store_followed_blocks(&to_persist, &head_snapshot) { + if let Err(err) = dbio.store_followed_blocks( + &to_persist, + &head_snapshot, + final_snapshot.as_ref().map(|(state, meta)| (state, meta)), + ) { error!("Failed to persist followed blocks: {err:#}"); } @@ -830,41 +907,15 @@ fn replay_unfulfilled_deposit_events( }); } -/// Builds the initial genesis state from `testnet_initial_state` plus configured genesis -/// transactions. Returns the final state and the list of [`LeeTransaction`]s that should be -/// committed to the genesis block so external observers can replay them. -fn build_genesis_state(config: &SequencerConfig) -> (lee::V03State, Vec) { +/// The pre-genesis state: `testnet_initial_state` plus accounts seeded outside +/// any transaction (bridge-lock holdings, the cross-zone inbox config). +fn build_initial_state(config: &SequencerConfig) -> lee::V03State { #[cfg(not(feature = "testnet"))] let mut state = testnet_initial_state::initial_state(); #[cfg(feature = "testnet")] let mut state = testnet_initial_state::initial_state_testnet(); - let genesis_txs = config - .genesis - .iter() - .filter_map(|genesis_tx| match genesis_tx { - GenesisAction::SupplyAccount { - account_id, - balance, - } => Some(build_supply_account_genesis_transaction( - account_id, *balance, - )), - GenesisAction::SupplyBridgeAccount { balance } => { - Some(build_supply_bridge_account_genesis_transaction(*balance)) - } - // Force-inserted below: bridge_lock has no mint transaction. - GenesisAction::SupplyBridgeLockHolding { .. } => None, - }) - .chain(std::iter::once(clock_invocation(0))) - .inspect(|tx| { - state - .transition_from_public_transaction(tx, GENESIS_BLOCK_ID, 0) - .expect("Failed to execute genesis transaction"); - }) - .map(LeeTransaction::Public) - .collect(); - // Seed bridge-lock holder balances directly: they are not produced by any tx. for action in &config.genesis { if let GenesisAction::SupplyBridgeLockHolding { holder, amount } = action { @@ -882,6 +933,41 @@ fn build_genesis_state(config: &SequencerConfig) -> (lee::V03State, Vec (lee::V03State, Vec) { + let mut state = build_initial_state(config); + + let genesis_txs = config + .genesis + .iter() + .filter_map(|genesis_tx| match genesis_tx { + GenesisAction::SupplyAccount { + account_id, + balance, + } => Some(build_supply_account_genesis_transaction( + account_id, *balance, + )), + GenesisAction::SupplyBridgeAccount { balance } => { + Some(build_supply_bridge_account_genesis_transaction(*balance)) + } + // Force-inserted in `build_initial_state`: bridge_lock has no mint transaction. + GenesisAction::SupplyBridgeLockHolding { .. } => None, + }) + .chain(std::iter::once(clock_invocation(0))) + .inspect(|tx| { + state + .transition_from_public_transaction(tx, GENESIS_BLOCK_ID, 0) + .expect("Failed to execute genesis transaction"); + }) + .map(LeeTransaction::Public) + .collect(); + (state, genesis_txs) } diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 46af7019..ca35fe7f 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -1540,6 +1540,116 @@ async fn follow_finalized_backfill_block_is_applied_and_marked_finalized() { assert!(matches!(stored.bedrock_status, BedrockStatus::Finalized)); } +#[tokio::test] +async fn restart_restores_head_tier_and_recovers_from_orphan() { + let config = setup_sequencer_config(); + let acc1 = initial_public_user_accounts()[0].account_id; + let acc2 = initial_public_user_accounts()[1].account_id; + + // Produce block 2 (a user transfer), then "crash" before it finalizes. + let (tx, block2) = { + let (mut sequencer, mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let tx = common::test_utils::create_transaction_native_token_transfer( + acc1, + 0, + acc2, + 10, + &create_signing_key_for_account1(), + ); + mempool_handle + .push((TransactionOrigin::User, tx.clone())) + .await + .unwrap(); + sequencer.produce_new_block().await.unwrap(); + (tx, sequencer.store.get_block_at_id(2).unwrap().unwrap()) + }; + + // Restart: nothing is finalized, so block 2 must come back as *head*, not + // final — the L1 can still orphan it. + let (mut sequencer, mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config.clone()).await; + assert_eq!(sequencer.chain_height(), 2); + + // The L1 orphans block 2 under its real MsgId (which we never persisted) + // and adopts a competing empty block 2'. + let genesis = sequencer.store.get_block_at_id(1).unwrap().unwrap(); + let block2_prime = + common::test_utils::produce_dummy_block(2, Some(genesis.header.hash), vec![]); + apply_follow_update( + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, + FollowUpdate { + adopted: vec![(MsgId::from([21; 32]), block2_prime.clone())], + orphaned: vec![(MsgId::from([20; 32]), block2)], + finalized: vec![], + }, + ); + + // The head reorged onto 2': transfer reverted, store overwritten, and the + // orphaned user tx returned to the mempool. + assert_eq!(sequencer.chain_height(), 2); + let head_tip = sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .head_tip() + .expect("head tip set"); + assert_eq!(head_tip.hash, block2_prime.header.hash); + assert_eq!( + sequencer.with_state(|s| s.get_account_by_id(acc1).balance), + 10000, + "the orphaned transfer must be reverted" + ); + let stored = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + assert_eq!(stored.header.hash, block2_prime.header.hash); + let (origin, requeued) = sequencer + .mempool + .pop() + .expect("orphaned user tx should be requeued"); + assert!(matches!(origin, TransactionOrigin::User)); + assert_eq!(requeued, tx); +} + +#[tokio::test] +async fn restart_reanchors_on_the_persisted_final_snapshot() { + let config = setup_sequencer_config(); + + // Produce block 2 and follow its finalization, which persists the final + // snapshot; then "crash". + { + let (mut sequencer, mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let tx = common::test_utils::produce_dummy_empty_transaction(); + mempool_handle + .push((TransactionOrigin::User, tx)) + .await + .unwrap(); + sequencer.produce_new_block().await.unwrap(); + let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + apply_follow_update( + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, + FollowUpdate { + adopted: vec![], + orphaned: vec![], + finalized: vec![(MsgId::from(block2.header.hash.0), block2)], + }, + ); + } + + // Restart: the final tier re-anchors on the snapshot instead of treating + // the whole stored chain as final. + let (sequencer, _mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let chain = sequencer.chain(); + let chain = chain.lock().expect("chain mutex poisoned"); + assert_eq!(chain.final_tip().expect("final tip set").block_id, 2); + assert_eq!(chain.head_tip().expect("head tip set").block_id, 2); +} + #[tokio::test] async fn configure_channel_delegates_to_publisher() { let config = setup_sequencer_config(); diff --git a/lez/storage/src/sequencer/mod.rs b/lez/storage/src/sequencer/mod.rs index d7775e20..8d3a467d 100644 --- a/lez/storage/src/sequencer/mod.rs +++ b/lez/storage/src/sequencer/mod.rs @@ -19,10 +19,11 @@ use crate::{ }, error::DbError, sequencer::sequencer_cells::{ - LEEStateCellOwned, LEEStateCellRef, LastFinalizedBlockIdCell, LatestBlockMetaCellOwned, - LatestBlockMetaCellRef, PendingDepositEventRecord, PendingDepositEventsCellOwned, - PendingDepositEventsCellRef, UnseenWithdrawCountCell, WithdrawalReconciliationKey, - ZoneSdkCheckpointCellOwned, ZoneSdkCheckpointCellRef, + FinalBlockMetaCellOwned, FinalBlockMetaCellRef, FinalLeeStateCellOwned, + FinalLeeStateCellRef, LEEStateCellOwned, LEEStateCellRef, LastFinalizedBlockIdCell, + LatestBlockMetaCellOwned, LatestBlockMetaCellRef, PendingDepositEventRecord, + PendingDepositEventsCellOwned, PendingDepositEventsCellRef, UnseenWithdrawCountCell, + WithdrawalReconciliationKey, ZoneSdkCheckpointCellOwned, ZoneSdkCheckpointCellRef, }, }; @@ -42,6 +43,10 @@ pub const DB_META_UNSEEN_WITHDRAW_COUNT_KEY: &str = "unseen_withdraw_count"; /// Key base for storing the LEE state. pub const DB_LEE_STATE_KEY: &str = "lee_state"; +/// Key base for storing the LEE state at the last L1-finalized block. +pub const DB_FINAL_LEE_STATE_KEY: &str = "final_lee_state"; +/// Key base for storing `(id, hash)` of the last L1-finalized block. +pub const DB_FINAL_BLOCK_META_KEY: &str = "final_block_meta"; /// Name of state column family. pub const CF_LEE_STATE_NAME: &str = "cf_lee_state"; @@ -522,6 +527,26 @@ impl RocksDBIO { .map(|opt| opt.map(|val| val.0)) } + /// `(state, meta)` at the last L1-finalized block; `None` until the first + /// finalization is observed. + pub fn get_final_snapshot(&self) -> DbResult> { + let Some(meta) = self.get_opt::(())? else { + return Ok(None); + }; + let state = self.get::(())?; + Ok(Some((state.0, meta.0))) + } + + fn put_final_snapshot_batch( + &self, + state: &V03State, + meta: &BlockMeta, + batch: &mut WriteBatch, + ) -> DbResult<()> { + self.put_batch(&FinalLeeStateCellRef(state), (), batch)?; + self.put_batch(&FinalBlockMetaCellRef(meta), (), batch) + } + pub fn get_lee_state(&self) -> DbResult { self.get::(()).map(|val| val.0) } @@ -619,36 +644,33 @@ impl RocksDBIO { } /// Persists a followed (peer) block so the store mirrors the canonical chain. - /// One-block form of [`Self::store_followed_blocks`]. + /// One-block form of [`Self::store_followed_blocks`], without a final snapshot. pub fn store_followed_block( &self, block: &Block, state: &V03State, finalized: bool, ) -> DbResult<()> { - self.store_followed_blocks(&[(block, finalized)], state) + self.store_followed_blocks(&[(block, finalized)], state, None) } - /// Persists a **batch** of followed (peer) blocks and the state after the last - /// of them in one atomic write, so a crash can never leave the stored state - /// ahead of the stored blocks and tip meta. + /// Persists a batch of followed blocks, the caller's head-tip `state`, and + /// the optional final-tier snapshot in one atomic write. /// - /// Per block, it: - /// - skips the payload write when the store already holds it (matched by id and hash) thus - /// covering our own blocks and re-deliveries - /// - stores it as finalized when its `finalized` flag is set - /// - /// When nothing needs writing, the state is left untouched too. + /// Per block: skips the payload write when the store already holds it (by + /// id and hash), unless `finalized` is set, which rewrites it with the + /// finalized status. When nothing needs writing, nothing is written — + /// an orphan-only update keeps the orphaned block and pre-revert state on + /// disk until the replacement overwrites them. /// /// TODO: the zone-sdk checkpoint is persisted by `on_checkpoint` *before* - /// this write; a crash in between resumes past these blocks without them - /// ever landing in the store. - /// Full `BlocksProcessed` atomicity by writing the checkpoint, blocks and - /// state in one batch, per the sdk's event contract is a follow-up. + /// this write. Full `BlocksProcessed` atomicity (checkpoint, blocks, state + /// and orphan reverts in one batch) is a follow-up. pub fn store_followed_blocks( &self, blocks: &[(&Block, bool)], state: &V03State, + final_snapshot: Option<(&V03State, &BlockMeta)>, ) -> DbResult<()> { let last_block_in_db = self.get_meta_last_block_in_db()?; let mut batch = WriteBatch::default(); @@ -683,7 +705,7 @@ impl RocksDBIO { } } - if batch.is_empty() { + if batch.is_empty() && final_snapshot.is_none() { return Ok(()); } // Meta is staged once for the batch's highest block. @@ -695,6 +717,9 @@ impl RocksDBIO { self.put_meta_latest_block_meta_batch(&tip, &mut batch)?; } self.put_lee_state_in_db_batch(state, &mut batch)?; + if let Some((final_state, final_meta)) = final_snapshot { + self.put_final_snapshot_batch(final_state, final_meta, &mut batch)?; + } self.db.write(batch).map_err(|rerr| { DbError::rocksdb_cast_message( diff --git a/lez/storage/src/sequencer/sequencer_cells.rs b/lez/storage/src/sequencer/sequencer_cells.rs index 7672e271..1d22e0f5 100644 --- a/lez/storage/src/sequencer/sequencer_cells.rs +++ b/lez/storage/src/sequencer/sequencer_cells.rs @@ -7,9 +7,10 @@ use crate::{ cells::{SimpleReadableCell, SimpleStorableCell, SimpleWritableCell}, error::DbError, sequencer::{ - CF_LEE_STATE_NAME, DB_LEE_STATE_KEY, DB_META_LAST_FINALIZED_BLOCK_ID, - DB_META_LATEST_BLOCK_META_KEY, DB_META_PENDING_DEPOSIT_EVENTS_KEY, - DB_META_UNSEEN_WITHDRAW_COUNT_KEY, DB_META_ZONE_SDK_CHECKPOINT_KEY, + CF_LEE_STATE_NAME, DB_FINAL_BLOCK_META_KEY, DB_FINAL_LEE_STATE_KEY, DB_LEE_STATE_KEY, + DB_META_LAST_FINALIZED_BLOCK_ID, DB_META_LATEST_BLOCK_META_KEY, + DB_META_PENDING_DEPOSIT_EVENTS_KEY, DB_META_UNSEEN_WITHDRAW_COUNT_KEY, + DB_META_ZONE_SDK_CHECKPOINT_KEY, }, }; @@ -43,6 +44,72 @@ impl SimpleWritableCell for LEEStateCellRef<'_> { } } +/// State at the last L1-finalized block, written atomically with +/// [`FinalBlockMetaCellRef`]. +#[derive(BorshDeserialize)] +pub struct FinalLeeStateCellOwned(pub V03State); + +impl SimpleStorableCell for FinalLeeStateCellOwned { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_FINAL_LEE_STATE_KEY; + const CF_NAME: &'static str = CF_LEE_STATE_NAME; +} + +impl SimpleReadableCell for FinalLeeStateCellOwned {} + +#[derive(BorshSerialize)] +pub struct FinalLeeStateCellRef<'state>(pub &'state V03State); + +impl SimpleStorableCell for FinalLeeStateCellRef<'_> { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_FINAL_LEE_STATE_KEY; + const CF_NAME: &'static str = CF_LEE_STATE_NAME; +} + +impl SimpleWritableCell for FinalLeeStateCellRef<'_> { + fn value_constructor(&self) -> DbResult> { + borsh::to_vec(&self).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize final state".to_owned())) + }) + } +} + +/// `(id, hash)` of the last L1-finalized block, paired with [`FinalLeeStateCellRef`]. +#[derive(BorshDeserialize)] +pub struct FinalBlockMetaCellOwned(pub BlockMeta); + +impl SimpleStorableCell for FinalBlockMetaCellOwned { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_FINAL_BLOCK_META_KEY; + const CF_NAME: &'static str = CF_META_NAME; +} + +impl SimpleReadableCell for FinalBlockMetaCellOwned {} + +#[derive(BorshSerialize)] +pub struct FinalBlockMetaCellRef<'blockmeta>(pub &'blockmeta BlockMeta); + +impl SimpleStorableCell for FinalBlockMetaCellRef<'_> { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_FINAL_BLOCK_META_KEY; + const CF_NAME: &'static str = CF_META_NAME; +} + +impl SimpleWritableCell for FinalBlockMetaCellRef<'_> { + fn value_constructor(&self) -> DbResult> { + borsh::to_vec(&self).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize final block meta".to_owned()), + ) + }) + } +} + #[derive(Debug, BorshSerialize, BorshDeserialize)] pub struct LastFinalizedBlockIdCell(pub Option); diff --git a/lez/storage/src/sequencer/tests.rs b/lez/storage/src/sequencer/tests.rs index 2d86c494..1938684e 100644 --- a/lez/storage/src/sequencer/tests.rs +++ b/lez/storage/src/sequencer/tests.rs @@ -102,6 +102,7 @@ fn store_followed_blocks_batch_lands_meta_and_state_on_last_block() { dbio.store_followed_blocks( &[(&block2, true), (&block3, false)], &state_with_balance(300), + None, ) .unwrap(); @@ -117,6 +118,38 @@ fn store_followed_blocks_batch_lands_meta_and_state_on_last_block() { assert_eq!(stored_balance(&dbio), 300); } +#[test] +fn final_snapshot_round_trips_and_is_absent_on_fresh_store() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + // Fresh store: no finalization observed yet. + assert!(dbio.get_final_snapshot().unwrap().is_none()); + + // A follow update that finalizes block 2 lands the snapshot in the same batch. + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + let final_meta = BlockMeta { + id: 2, + hash: block2.header.hash, + }; + dbio.store_followed_blocks( + &[(&block2, true)], + &state_with_balance(300), + Some((&state_with_balance(200), &final_meta)), + ) + .unwrap(); + + let (final_state, meta) = dbio + .get_final_snapshot() + .unwrap() + .expect("final snapshot is stored"); + assert_eq!(meta.id, 2); + assert_eq!(meta.hash, block2.header.hash); + assert_eq!(final_state.get_account_by_id(marker_id()).balance, 200); + // The head state is stored independently of the final snapshot. + assert_eq!(stored_balance(&dbio), 300); +} + #[test] fn store_followed_block_overwrites_competing_block_at_same_id() { let temp_dir = tempdir().unwrap(); From a8797d8b373d2dd8a65a170cf6dcdf0db2d2ba23 Mon Sep 17 00:00:00 2001 From: erhant Date: Thu, 16 Jul 2026 18:12:07 +0300 Subject: [PATCH 32/32] chore: fmt --- lez/chain_state/src/chain.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lez/chain_state/src/chain.rs b/lez/chain_state/src/chain.rs index 35dc458b..b65ece7b 100644 --- a/lez/chain_state/src/chain.rs +++ b/lez/chain_state/src/chain.rs @@ -183,7 +183,8 @@ impl ChainState { /// Rebuilds one head entry from a persisted block, applying it in place (the /// caller treats `Err` as fatal). /// - /// Pass a hash-derived sentinel for `this_msg`; orphan/finalize events then correlate by block hash. + /// Pass a hash-derived sentinel for `this_msg`; orphan/finalize events then correlate by block + /// hash. pub fn restore_head_block( &mut self, this_msg: MsgId,