From f80bbe62007d22524034b4cb48a0ea2b4e26fcc9 Mon Sep 17 00:00:00 2001 From: erhant Date: Wed, 22 Jul 2026 19:33:13 +0300 Subject: [PATCH] feat(sequencer): two-tier chain state and multi-sequencer support Decentralized-sequencing foundation: a shared chain_state crate (two-tier head/final ChainState, apply_block, AcceptOutcome, StallReason, and the absorbed channel-consistency machinery), turn-gated block production, the publisher follow path for adopted/orphaned/finalized peer blocks, and persistence that keeps disk order equal to apply order under the chain lock. Rebased onto dev after #600/#606: chain_consistency is absorbed into chain_state, the sequencer bootstrap's verify_and_reconstruct is re-wired onto the two-tier ChainState (reconstruction applies channel history through the final tier and persists via the follow-path primitives), and test fixtures adopt the SequencerSetup builder extended with with_bedrock_signing_key. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 11 +- Cargo.toml | 4 +- Justfile | 8 +- integration_tests/Cargo.toml | 2 + integration_tests/tests/cross_zone_bridge.rs | 18 +- .../tests/cross_zone_ingress_guard.rs | 14 +- integration_tests/tests/cross_zone_ping.rs | 14 +- .../tests/cross_zone_verified.rs | 18 +- integration_tests/tests/multi_sequencer.rs | 236 +++++ .../tests/sequencer_bootstrap.rs | 13 +- lee/state_machine/src/merkle_tree/mod.rs | 4 +- lee/state_machine/src/state/mod.rs | 12 +- lez/chain_consistency/src/apply.rs | 196 ---- lez/chain_consistency/src/lib.rs | 9 - .../Cargo.toml | 9 +- lez/chain_state/src/apply.rs | 308 ++++++ lez/chain_state/src/chain.rs | 952 ++++++++++++++++++ .../src/consistency.rs | 0 lez/chain_state/src/ingest_error.rs | 80 ++ lez/chain_state/src/lib.rs | 17 + lez/chain_state/src/stall_reason.rs | 50 + lez/common/src/block.rs | 9 + lez/indexer/core/Cargo.toml | 2 +- lez/indexer/core/src/block_store.rs | 53 +- lez/indexer/core/src/lib.rs | 10 +- lez/indexer/core/src/status.rs | 27 +- lez/mempool/src/lib.rs | 18 + lez/sequencer/core/Cargo.toml | 3 +- lez/sequencer/core/src/block_publisher.rs | 297 +++++- lez/sequencer/core/src/block_store.rs | 10 + lez/sequencer/core/src/lib.rs | 495 ++++++--- lez/sequencer/core/src/mock.rs | 29 +- lez/sequencer/core/src/tests.rs | 590 ++++++++++- .../core/src/tests/reconstruction.rs | 137 ++- lez/sequencer/service/Cargo.toml | 2 + lez/sequencer/service/rpc/src/lib.rs | 2 - .../service/src/bin/bedrock_pubkey.rs | 35 + .../service/src/bin/configure_channel.rs | 71 ++ lez/sequencer/service/src/lib.rs | 51 +- lez/sequencer/service/src/main.rs | 28 +- lez/sequencer/service/src/service.rs | 31 +- lez/storage/src/sequencer/mod.rs | 179 +++- lez/storage/src/sequencer/sequencer_cells.rs | 74 +- lez/storage/src/sequencer/tests.rs | 283 ++++++ test_fixtures/src/lib.rs | 11 +- test_fixtures/src/setup.rs | 44 +- 46 files changed, 3768 insertions(+), 698 deletions(-) create mode 100644 integration_tests/tests/multi_sequencer.rs delete mode 100644 lez/chain_consistency/src/apply.rs delete mode 100644 lez/chain_consistency/src/lib.rs rename lez/{chain_consistency => chain_state}/Cargo.toml (83%) create mode 100644 lez/chain_state/src/apply.rs create mode 100644 lez/chain_state/src/chain.rs rename lez/{chain_consistency => chain_state}/src/consistency.rs (100%) create mode 100644 lez/chain_state/src/ingest_error.rs create mode 100644 lez/chain_state/src/lib.rs create mode 100644 lez/chain_state/src/stall_reason.rs create mode 100644 lez/sequencer/service/src/bin/bedrock_pubkey.rs create mode 100644 lez/sequencer/service/src/bin/configure_channel.rs create mode 100644 lez/storage/src/sequencer/tests.rs diff --git a/Cargo.lock b/Cargo.lock index e7c9979a..e3439891 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1369,7 +1369,7 @@ dependencies = [ ] [[package]] -name = "chain_consistency" +name = "chain_state" version = "0.1.0" dependencies = [ "anyhow", @@ -1383,6 +1383,7 @@ dependencies = [ "logos-blockchain-zone-sdk", "serde", "serde_json", + "testnet_initial_state", "thiserror 2.0.18", "tokio", ] @@ -3958,7 +3959,7 @@ dependencies = [ "arc-swap", "async-stream", "borsh", - "chain_consistency", + "chain_state", "common", "cross_zone", "cross_zone_inbox_core", @@ -4151,12 +4152,14 @@ dependencies = [ "reqwest", "risc0-zkvm", "sequencer_core", + "sequencer_service_protocol", "sequencer_service_rpc", "serde_json", "system_accounts", "tempfile", "test_fixtures", "test_programs", + "testnet_initial_state", "token_core", "tokio", "vault_core", @@ -9042,7 +9045,7 @@ dependencies = [ "borsh", "bridge_core", "bytesize", - "chain_consistency", + "chain_state", "chrono", "common", "cross_zone", @@ -9073,6 +9076,7 @@ dependencies = [ "testnet_initial_state", "token_core", "tokio", + "tokio-util", "url", "vault_core", ] @@ -9088,6 +9092,7 @@ dependencies = [ "common", "env_logger", "futures", + "hex", "jsonrpsee", "lee", "log", diff --git a/Cargo.toml b/Cargo.toml index bac0ea4f..11956920 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "lez", "lez/system_accounts", + "lez/chain_state", "lez/sequencer/core", "lez/sequencer/service", "lez/sequencer/service/protocol", @@ -32,7 +33,6 @@ members = [ "lez/wallet", "lez/wallet-ffi", "lez/common", - "lez/chain_consistency", "lez/programs", "lez/programs/amm", "lez/programs/associated_token_account", @@ -73,7 +73,7 @@ members = [ lee = { path = "lee/state_machine" } lee_core = { path = "lee/state_machine/core" } common = { path = "lez/common" } -chain_consistency = { path = "lez/chain_consistency" } +chain_state = { path = "lez/chain_state" } mempool = { path = "lez/mempool" } storage = { path = "lez/storage" } key_protocol = { path = "lee/key_protocol" } diff --git a/Justfile b/Justfile index b838f4fc..b65c7ff3 100644 --- a/Justfile +++ b/Justfile @@ -64,15 +64,17 @@ run-bedrock: docker compose up # Run Sequencer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. +# Optional home/port let a second instance run off the same config, e.g. +# `just run-sequencer "" "$TMPDIR/lez-sequencer2" 3041` for the multi-sequencer demo. [working-directory: 'lez/sequencer/service'] -run-sequencer standalone="": +run-sequencer standalone="" home="" port="3040": @echo "๐Ÿง  Running sequencer" @if [ "{{standalone}}" = "standalone" ]; then \ echo "๐Ÿงช Running in standalone mode"; \ - RUST_LOG=info cargo run --features standalone --release -p sequencer_service configs/debug/sequencer_config.json; \ + RUST_LOG=info cargo run --features standalone --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \ else \ echo "๐Ÿš€ Running in normal mode"; \ - RUST_LOG=info cargo run --release -p sequencer_service configs/debug/sequencer_config.json; \ + RUST_LOG=info cargo run --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \ fi # Run Indexer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration. diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index a225d657..a2ab08b7 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -29,6 +29,7 @@ bridge_lock_core.workspace = true wrapped_token_core.workspace = true risc0-zkvm.workspace = true indexer_service_rpc = { workspace = true, features = ["client"] } +sequencer_service_protocol.workspace = true sequencer_service_rpc = { workspace = true, features = ["client"] } wallet-ffi.workspace = true indexer_ffi.workspace = true @@ -36,6 +37,7 @@ indexer_service_protocol.workspace = true system_accounts.workspace = true programs.workspace = true test_programs.workspace = true +testnet_initial_state.workspace = true logos-blockchain-http-api-common.workspace = true logos-blockchain-core.workspace = true diff --git a/integration_tests/tests/cross_zone_bridge.rs b/integration_tests/tests/cross_zone_bridge.rs index f835602a..9107314a 100644 --- a/integration_tests/tests/cross_zone_bridge.rs +++ b/integration_tests/tests/cross_zone_bridge.rs @@ -9,7 +9,7 @@ //! wrapped token is minted to the recipient. Reuses the M3/M4 spine unchanged; //! only the source caller (`bridge_lock`) and target (`wrapped_token`) are new. -use std::{net::SocketAddr, time::Duration}; +use std::time::Duration; use anyhow::{Context as _, Result}; use common::transaction::LeeTransaction; @@ -17,14 +17,14 @@ use cross_zone_outbox_core::outbox_pda; use integration_tests::{ config::{self, SequencerPartialConfig}, indexer_client::IndexerClient, - setup::{SequencerSetup, setup_bedrock_node, setup_indexer}, + setup::{SequencerSetup, indexer_client, sequencer_client, setup_bedrock_node, setup_indexer}, }; use lee::{ AccountId, PrivateKey, PublicKey, PublicTransaction, public_transaction::{Message, WitnessSet}, }; use sequencer_core::config::{CrossZoneConfig, CrossZonePeer, GenesisAction}; -use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuilder}; +use sequencer_service_rpc::RpcClient as _; use tokio::test; const DELIVERY_TIMEOUT: Duration = Duration::from_secs(600); @@ -88,9 +88,7 @@ async fn lock_on_zone_a_mints_wrapped_token_on_zone_b() -> Result<()> { // Wait until zone B's indexer reflects the verified mint. let holding_id = wrapped_token_core::holding_account_id(wrapped_token_id, &RECIPIENT); - let indexer_url = config::addr_to_url(config::UrlProtocol::Ws, idx_b.addr()) - .context("Failed to build indexer URL")?; - let indexer = IndexerClient::new(&indexer_url) + let indexer = indexer_client(idx_b.addr()) .await .context("Failed to build indexer client")?; @@ -168,14 +166,6 @@ fn build_lock_tx( LeeTransaction::Public(PublicTransaction::new(message, witness)) } -fn sequencer_client(addr: SocketAddr) -> Result { - let url = config::addr_to_url(config::UrlProtocol::Http, addr) - .context("Failed to build sequencer URL")?; - SequencerClientBuilder::default() - .build(url) - .context("Failed to build sequencer client") -} - /// Polls zone B's indexer until the recipient's wrapped holding is non-zero. async fn wait_for_mint(indexer: &IndexerClient, holding_id: AccountId) -> Result { let account_id = indexer_service_protocol::AccountId { diff --git a/integration_tests/tests/cross_zone_ingress_guard.rs b/integration_tests/tests/cross_zone_ingress_guard.rs index fe5fe7fc..86731477 100644 --- a/integration_tests/tests/cross_zone_ingress_guard.rs +++ b/integration_tests/tests/cross_zone_ingress_guard.rs @@ -9,8 +9,6 @@ //! inbox guest's caller-is-none assertion passes for a top-level user tx, so the //! sequencer ingress guard is the only thing that stops this. -use std::net::SocketAddr; - use anyhow::{Context as _, Result}; use common::transaction::LeeTransaction; use cross_zone_inbox_core::{ @@ -18,13 +16,13 @@ use cross_zone_inbox_core::{ }; use integration_tests::{ config::{self, SequencerPartialConfig}, - setup::{SequencerSetup, setup_bedrock_node}, + setup::{SequencerSetup, sequencer_client, setup_bedrock_node}, }; use lee::{ PublicTransaction, public_transaction::{Message, WitnessSet}, }; -use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuilder}; +use sequencer_service_rpc::RpcClient as _; use tokio::test; #[test] @@ -73,11 +71,3 @@ async fn user_origin_inbox_call_rejected() -> Result<()> { ); Ok(()) } - -fn sequencer_client(addr: SocketAddr) -> Result { - let url = config::addr_to_url(config::UrlProtocol::Http, addr) - .context("Failed to build sequencer URL")?; - SequencerClientBuilder::default() - .build(url) - .context("Failed to build sequencer client") -} diff --git a/integration_tests/tests/cross_zone_ping.rs b/integration_tests/tests/cross_zone_ping.rs index a3ed0c0c..f106bda1 100644 --- a/integration_tests/tests/cross_zone_ping.rs +++ b/integration_tests/tests/cross_zone_ping.rs @@ -11,20 +11,20 @@ //! inbox dispatch, and zone B's sequencer delivers it. This is the M3 milestone, //! sequencer-trusted, with no indexer re-derivation (that is M4). -use std::{net::SocketAddr, time::Duration}; +use std::time::Duration; use anyhow::{Context as _, Result}; use common::transaction::LeeTransaction; use cross_zone_outbox_core::outbox_pda; use integration_tests::{ config::{self, SequencerPartialConfig}, - setup::{SequencerSetup, setup_bedrock_node}, + setup::{SequencerSetup, sequencer_client, setup_bedrock_node}, }; use lee::{AccountId, PublicTransaction, public_transaction::Message}; use lee_core::program::ProgramId; use ping_core::{ReceiverInstruction, SenderInstruction, ping_record_pda}; use sequencer_core::config::{CrossZoneConfig, CrossZonePeer}; -use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuilder}; +use sequencer_service_rpc::{RpcClient as _, SequencerClient}; use tokio::test; const DELIVERY_TIMEOUT: Duration = Duration::from_secs(480); @@ -123,14 +123,6 @@ fn build_ping_tx(target_zone: [u8; 32], receiver_id: ProgramId) -> LeeTransactio )) } -fn sequencer_client(addr: SocketAddr) -> Result { - let url = config::addr_to_url(config::UrlProtocol::Http, addr) - .context("Failed to build sequencer URL")?; - SequencerClientBuilder::default() - .build(url) - .context("Failed to build sequencer client") -} - /// Polls zone B's sequencer until the ping record PDA holds a payload. async fn wait_for_delivery(client: SequencerClient, record_id: AccountId) -> Result> { let wait = async { diff --git a/integration_tests/tests/cross_zone_verified.rs b/integration_tests/tests/cross_zone_verified.rs index aa2ac06a..cc21c42a 100644 --- a/integration_tests/tests/cross_zone_verified.rs +++ b/integration_tests/tests/cross_zone_verified.rs @@ -9,7 +9,7 @@ //! payload landing in the indexer's state proves verification passed; a forgery //! would have halted the indexer instead. -use std::{net::SocketAddr, time::Duration}; +use std::time::Duration; use anyhow::{Context as _, Result}; use common::transaction::LeeTransaction; @@ -17,13 +17,13 @@ use cross_zone_outbox_core::outbox_pda; use integration_tests::{ config::{self, SequencerPartialConfig}, indexer_client::IndexerClient, - setup::{SequencerSetup, setup_bedrock_node, setup_indexer}, + setup::{SequencerSetup, indexer_client, sequencer_client, setup_bedrock_node, setup_indexer}, }; use lee::{AccountId, PublicTransaction, public_transaction::Message}; use lee_core::program::ProgramId; use ping_core::{ReceiverInstruction, SenderInstruction, ping_record_pda}; use sequencer_core::config::{CrossZoneConfig, CrossZonePeer}; -use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuilder}; +use sequencer_service_rpc::RpcClient as _; use tokio::test; const DELIVERY_TIMEOUT: Duration = Duration::from_secs(600); @@ -83,9 +83,7 @@ async fn indexer_verifies_and_delivers_cross_zone_ping() -> Result<()> { // Wait until zone B's indexer records the delivered payload. The indexer only // applies the dispatch after re-deriving and verifying it. let record_id = ping_record_pda(receiver_id); - let indexer_url = config::addr_to_url(config::UrlProtocol::Ws, idx_b.addr()) - .context("Failed to build indexer URL")?; - let indexer = IndexerClient::new(&indexer_url) + let indexer = indexer_client(idx_b.addr()) .await .context("Failed to build indexer client")?; @@ -130,14 +128,6 @@ fn build_ping_tx(target_zone: [u8; 32], receiver_id: ProgramId) -> LeeTransactio )) } -fn sequencer_client(addr: SocketAddr) -> Result { - let url = config::addr_to_url(config::UrlProtocol::Http, addr) - .context("Failed to build sequencer URL")?; - SequencerClientBuilder::default() - .build(url) - .context("Failed to build sequencer client") -} - /// Polls zone B's indexer until the ping record PDA holds a payload. async fn wait_for_indexer_delivery( indexer: &IndexerClient, diff --git a/integration_tests/tests/multi_sequencer.rs b/integration_tests/tests/multi_sequencer.rs new file mode 100644 index 00000000..dfca2cf2 --- /dev/null +++ b/integration_tests/tests/multi_sequencer.rs @@ -0,0 +1,236 @@ +#![expect( + clippy::tests_outside_test_module, + reason = "top-level test functions are conventional for integration tests" +)] + +//! Two sequencers share one channel: A starts solo as channel admin, live- +//! accredits `[A, B]` with round-robin rotation, B joins and syncs, both +//! produce on their turns, and A, B and an indexer converge on the same chain. + +use std::time::Duration; + +use anyhow::{Context as _, Result, ensure}; +use indexer_service_rpc::RpcClient as _; +use integration_tests::{ + config::{self, SequencerPartialConfig}, + indexer_client::IndexerClient, + setup::{SequencerSetup, indexer_client, sequencer_client, setup_bedrock_node, setup_indexer}, +}; +use logos_blockchain_key_management_system_service::keys::{ED25519_SECRET_KEY_SIZE, Ed25519Key}; +use sequencer_core::{block_publisher::post_channel_config, config::BedrockConfig}; +use sequencer_service_rpc::{RpcClient as _, SequencerClient}; +use testnet_initial_state::{initial_pub_accounts_private_keys, initial_public_user_accounts}; +use tokio::test; + +/// 1 s bedrock slots: rotate the turn every ~20 s of tenure; steal a stalled +/// turn after ~30 s (bounds the stall while B is accredited but not started). +const POSTING_TIMEFRAME_SLOTS: u32 = 20; +const POSTING_TIMEOUT_SLOTS: u32 = 30; +const PHASE_TIMEOUT: Duration = Duration::from_secs(360); +const POLL_INTERVAL: Duration = Duration::from_secs(2); +const TRANSFER_AMOUNT: u128 = 10; +/// โ‰ˆ4 turn windows past B's join (5 s blocks, ~20 s turns โ†’ ~4 blocks/window). +const ROTATION_BLOCKS: u64 = 8; + +#[test] +async fn multi_sequencer_committee_converges() -> Result<()> { + let (_bedrock, bedrock_addr) = setup_bedrock_node() + .await + .context("Failed to set up Bedrock node")?; + + // Fixed seeds so A can accredit B's public key before B exists. + let key_a = [0xA1_u8; ED25519_SECRET_KEY_SIZE]; + let key_b = [0xB2_u8; ED25519_SECRET_KEY_SIZE]; + let pub_a = Ed25519Key::from_bytes(&key_a).public_key(); + let pub_b = Ed25519Key::from_bytes(&key_b).public_key(); + + let partial = SequencerPartialConfig { + block_create_timeout: Duration::from_secs(5), + ..SequencerPartialConfig::default() + }; + + // Phase 1: A solo (its first inscription creates the channel), plus an indexer. + let (seq_a, _a_home) = SequencerSetup::new(partial, bedrock_addr) + .with_genesis(vec![]) + .with_bedrock_signing_key(key_a) + .setup() + .await + .context("Failed to set up sequencer A")?; + let a = sequencer_client(seq_a.addr())?; + let (idx, _idx_home) = setup_indexer(bedrock_addr, config::bedrock_channel_id(), None) + .await + .context("Failed to set up indexer")?; + let indexer = indexer_client(idx.addr()).await?; + + wait_for_height(&a, 2, "sequencer A to produce past genesis").await?; + + // Phase 2: live roster change to [A, B] with rotation enabled, posted + // straight to bedrock with A's admin key (the operator one-shot path). + post_channel_config( + &BedrockConfig { + channel_id: config::bedrock_channel_id(), + node_url: config::addr_to_url(config::UrlProtocol::Http, bedrock_addr)?, + auth: None, + }, + &Ed25519Key::from_bytes(&key_a), + vec![pub_a, pub_b], + POSTING_TIMEFRAME_SLOTS, + POSTING_TIMEOUT_SLOTS, + 1, + 1, + ) + .await + .context("Failed to configure the channel committee")?; + + let height_at_config = a.get_last_block_id().await?; + wait_for_height( + &a, + height_at_config + 1, + "A to produce after the roster change", + ) + .await?; + + // Phase 3: B joins live and syncs the existing chain. + let (seq_b, _b_home) = SequencerSetup::new(partial, bedrock_addr) + .with_genesis(vec![]) + .with_bedrock_signing_key(key_b) + .setup() + .await + .context("Failed to set up sequencer B")?; + let b = sequencer_client(seq_b.addr())?; + + let join_height = a.get_last_block_id().await?; + wait_for_height(&b, join_height, "B to sync to A's height at join").await?; + + // Phase 4: rotation + convergence over โ‰ˆ4 turn windows. + let rotation_target = join_height + ROTATION_BLOCKS; + wait_for_height( + &a, + rotation_target, + "the chain to advance across turn windows", + ) + .await?; + wait_for_height(&b, rotation_target, "B to follow across turn windows").await?; + assert_same_chain(&a, &b).await?; + + // Phase 5: a tx submitted only to B is included by B and visible on A. + let accounts = initial_public_user_accounts(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = initial_pub_accounts_private_keys()[0].pub_sign_key.clone(); + + let to_balance_before = a.get_account_balance(to).await?; + let nonce = b.get_accounts_nonces(vec![from]).await?[0]; + let tx = common::test_utils::create_transaction_native_token_transfer( + from, + nonce.0, + to, + TRANSFER_AMOUNT, + &sign_key, + ); + b.send_transaction(tx) + .await + .context("Failed to submit the transfer to B")?; + + wait_for_balance(&a, to, to_balance_before + TRANSFER_AMOUNT).await?; + + // Phase 6: the indexer finalizes the same chain, with no stall. + wait_for_finalized(&indexer, join_height).await?; + let finalized = indexer.get_last_finalized_block_id().await?.unwrap_or(0); + for id in 1..=finalized { + let block_i = indexer + .get_block_by_id(id) + .await? + .with_context(|| format!("Indexer is missing finalized block {id}"))?; + let block_a = a + .get_block(id) + .await? + .with_context(|| format!("A is missing block {id}"))?; + ensure!( + block_i.header.hash == indexer_service_protocol::HashType::from(block_a.header.hash), + "Indexer diverges from A at block {id}" + ); + } + let status = indexer.get_status().await?; + ensure!( + status.stall_reason.is_none(), + "Indexer is stalled: {:?}", + status.stall_reason + ); + + Ok(()) +} + +/// Polls the sequencer until its chain height reaches `target`. +async fn wait_for_height(client: &SequencerClient, target: u64, what: &str) -> Result<()> { + let wait = async { + loop { + if client.get_last_block_id().await? >= target { + return Ok::<(), anyhow::Error>(()); + } + tokio::time::sleep(POLL_INTERVAL).await; + } + }; + tokio::time::timeout(PHASE_TIMEOUT, wait) + .await + .with_context(|| format!("Timed out waiting for {what} (target height {target})"))? +} + +/// Polls the sequencer until `account`'s balance reaches `expected`. +async fn wait_for_balance( + client: &SequencerClient, + account: lee::AccountId, + expected: u128, +) -> Result<()> { + let wait = async { + loop { + if client.get_account_balance(account).await? == expected { + return Ok::<(), anyhow::Error>(()); + } + tokio::time::sleep(POLL_INTERVAL).await; + } + }; + tokio::time::timeout(PHASE_TIMEOUT, wait) + .await + .context("Timed out waiting for the cross-sequencer transfer to reach A")? +} + +/// Polls the indexer until its finalized height reaches `target`. +async fn wait_for_finalized(indexer: &IndexerClient, target: u64) -> Result<()> { + let wait = async { + loop { + if indexer.get_last_finalized_block_id().await?.unwrap_or(0) >= target { + return Ok::<(), anyhow::Error>(()); + } + tokio::time::sleep(POLL_INTERVAL).await; + } + }; + tokio::time::timeout(PHASE_TIMEOUT, wait) + .await + .context("Timed out waiting for the indexer to finalize")? +} + +/// Asserts A and B hold byte-identical block hashes over their common prefix. +async fn assert_same_chain(a: &SequencerClient, b: &SequencerClient) -> Result<()> { + let common = a + .get_last_block_id() + .await? + .min(b.get_last_block_id().await?); + for id in 1..=common { + let block_a = a + .get_block(id) + .await? + .with_context(|| format!("A is missing block {id}"))?; + let block_b = b + .get_block(id) + .await? + .with_context(|| format!("B is missing block {id}"))?; + ensure!( + block_a.header.hash == block_b.header.hash, + "Chain divergence at block {id}: A {:?} vs B {:?}", + block_a.header.hash, + block_b.header.hash + ); + } + Ok(()) +} diff --git a/integration_tests/tests/sequencer_bootstrap.rs b/integration_tests/tests/sequencer_bootstrap.rs index 5261c412..9dd2b586 100644 --- a/integration_tests/tests/sequencer_bootstrap.rs +++ b/integration_tests/tests/sequencer_bootstrap.rs @@ -8,7 +8,7 @@ reason = "Integration tests live at crate root and don't care about these lints" )] -use std::{net::SocketAddr, path::Path, time::Duration}; +use std::{path::Path, time::Duration}; use anyhow::{Context as _, Result, bail}; use indexer_service_rpc::RpcClient as _; @@ -16,11 +16,11 @@ use integration_tests::L2_TO_L1_TIMEOUT; use lee::{AccountId, PrivateKey, PublicKey}; use logos_blockchain_core::mantle::ops::channel::ChannelId; use sequencer_core::config::GenesisAction; -use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuilder}; +use sequencer_service_rpc::{RpcClient as _, SequencerClient}; use test_fixtures::{ config::{SequencerPartialConfig, UrlProtocol, addr_to_url}, indexer_client::IndexerClient, - setup::{SequencerSetup, setup_bedrock_node, setup_indexer}, + setup::{SequencerSetup, sequencer_client, setup_bedrock_node, setup_indexer}, }; use tokio::test; @@ -41,13 +41,6 @@ fn slow_blocks() -> SequencerPartialConfig { } } -fn sequencer_client(addr: SocketAddr) -> Result { - let url = addr_to_url(UrlProtocol::Http, addr).context("Failed to build sequencer URL")?; - SequencerClientBuilder::default() - .build(url) - .context("Failed to build sequencer client") -} - /// Polls the indexer's last finalized block id until it reaches `target`. The /// indexer reads finalized channel history, so this is our oracle for "block /// `target` is finalized on Bedrock" โ€” exactly what a reconstructing sequencer diff --git a/lee/state_machine/src/merkle_tree/mod.rs b/lee/state_machine/src/merkle_tree/mod.rs index ee8106c5..37e706f7 100644 --- a/lee/state_machine/src/merkle_tree/mod.rs +++ b/lee/state_machine/src/merkle_tree/mod.rs @@ -8,8 +8,8 @@ mod default_values; type Value = [u8; 32]; type Node = [u8; 32]; -#[cfg_attr(test, derive(Debug, PartialEq, Eq))] -#[derive(Clone, BorshSerialize, BorshDeserialize)] +#[cfg_attr(test, derive(Debug))] +#[derive(Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub struct MerkleTree { nodes: Vec, capacity: usize, diff --git a/lee/state_machine/src/state/mod.rs b/lee/state_machine/src/state/mod.rs index 94d919ca..6a59cadb 100644 --- a/lee/state_machine/src/state/mod.rs +++ b/lee/state_machine/src/state/mod.rs @@ -20,8 +20,8 @@ use crate::{ pub const MAX_NUMBER_CHAINED_CALLS: usize = 10; -#[derive(Clone, BorshSerialize, BorshDeserialize)] -#[cfg_attr(test, derive(Debug, PartialEq, Eq))] +#[derive(Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +#[cfg_attr(test, derive(Debug))] pub struct CommitmentSet { merkle_tree: MerkleTree, commitments: HashMap, @@ -67,8 +67,8 @@ impl CommitmentSet { } } -#[cfg_attr(test, derive(Debug, PartialEq, Eq))] -#[derive(Clone)] +#[cfg_attr(test, derive(Debug))] +#[derive(Clone, PartialEq, Eq)] struct NullifierSet(BTreeSet); impl NullifierSet { @@ -109,8 +109,8 @@ impl BorshDeserialize for NullifierSet { } } -#[derive(Clone, BorshSerialize, BorshDeserialize)] -#[cfg_attr(test, derive(Debug, PartialEq, Eq))] +#[derive(Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +#[cfg_attr(test, derive(Debug))] pub struct V03State { public_state: HashMap, private_state: (CommitmentSet, NullifierSet), diff --git a/lez/chain_consistency/src/apply.rs b/lez/chain_consistency/src/apply.rs deleted file mode 100644 index 28961502..00000000 --- a/lez/chain_consistency/src/apply.rs +++ /dev/null @@ -1,196 +0,0 @@ -//! Validating and applying channel L2 blocks to a `V03State`. -//! -//! These primitives are shared by the consumers that reconstruct L2 state from -//! the Bedrock channel. - -use common::{ - HashType, - block::Block, - transaction::{LeeTransaction, clock_invocation}, -}; -use lee::{GENESIS_BLOCK_ID, V03State}; -use lee_core::BlockId; -use serde::{Deserialize, Serialize}; - -/// Why a channel L2 block could not be validated or applied. -/// -/// Persisted in `RocksDB` (via the Indexer's stall reason), so every variant -/// must be `Clone + Serialize + Deserialize`. -#[derive(Debug, Clone, Serialize, Deserialize, thiserror::Error)] -pub enum BlockIngestError { - #[error("Failed to deserialize L2 block: {0}")] - /// Here we store the error string that is derived from [`borsh::from_slice`]'s [`Err`]. - Deserialize(String), - #[error("Unexpected block id: expected {expected}, got {got}")] - UnexpectedBlockId { expected: BlockId, got: BlockId }, - #[error("Broken chain link: expected prev {expected_prev}, got {got_prev}")] - BrokenChainLink { - expected_prev: HashType, - got_prev: HashType, - }, - #[error("Block hash mismatch: computed {computed}, header {header}")] - HashMismatch { - computed: HashType, - header: HashType, - }, - #[error("Block has no transactions")] - EmptyBlock, - #[error("Last transaction must be the public clock invocation for the block timestamp")] - InvalidClockTransaction, - #[error("Genesis block must contain only public transactions")] - NonPublicGenesisTransaction, - #[error("State transition failed at transaction {tx_index}: {reason}")] - StateTransition { - /// Index of the failing transaction within the block body. - tx_index: u64, - /// Reason string from `lee::Error` to `anyhow::Error` to `{:#}`. - /// - /// This is required because `lee::Error` is not `Clone + Serialize + Deserialize`, so we - /// cannot store it directly. - reason: String, - }, -} - -impl BlockIngestError { - /// Whether the failure may be transient rather than a property of the block. - /// - /// FIXME: `StateTransition` is too coarse โ€” its `reason` string mixes genuine - /// state-transition rejections with infra failures (risc0 executor teardown, - /// storage errors). Once it carries a structured cause, narrow this so only - /// infra failures retry. - #[must_use] - pub const fn is_retryable(&self) -> bool { - matches!(self, Self::StateTransition { .. }) - } -} - -/// The last successfully applied block a candidate must extend. -#[derive(Debug, Copy, Clone)] -pub struct Tip { - pub block_id: BlockId, - pub hash: HashType, -} - -/// Checks that `block` is the valid continuation of `tip`: hash integrity, -/// then block-id continuity, then `prev_block_hash` linkage. A `None` tip -/// (cold store) expects the genesis block. -pub fn validate_against_tip(tip: Option, block: &Block) -> Result<(), BlockIngestError> { - let computed = block.recompute_hash(); - if computed != block.header.hash { - return Err(BlockIngestError::HashMismatch { - computed, - header: block.header.hash, - }); - } - - match tip { - None => { - if block.header.block_id != GENESIS_BLOCK_ID { - return Err(BlockIngestError::UnexpectedBlockId { - expected: GENESIS_BLOCK_ID, - got: block.header.block_id, - }); - } - } - Some(tip) => { - let expected = tip - .block_id - .checked_add(1) - .expect("block id should not overflow"); - if block.header.block_id != expected { - return Err(BlockIngestError::UnexpectedBlockId { - expected, - got: block.header.block_id, - }); - } - if block.header.prev_block_hash != tip.hash { - return Err(BlockIngestError::BrokenChainLink { - expected_prev: tip.hash, - got_prev: block.header.prev_block_hash, - }); - } - } - } - Ok(()) -} - -/// Applies a block's transactions to `state`. -pub fn apply_block(block: &Block, state: &mut V03State) -> Result<(), BlockIngestError> { - let (clock_tx, user_txs) = block - .body - .transactions - .split_last() - .ok_or(BlockIngestError::EmptyBlock)?; - - let expected_clock = LeeTransaction::Public(clock_invocation(block.header.timestamp)); - if *clock_tx != expected_clock { - return Err(BlockIngestError::InvalidClockTransaction); - } - - let is_genesis = block.header.block_id == GENESIS_BLOCK_ID; - for (tx_index, transaction) in user_txs.iter().enumerate() { - let state_transition = |err: anyhow::Error| BlockIngestError::StateTransition { - tx_index: tx_index.try_into().expect("tx index fits in u64"), - reason: format!("{err:#}"), - }; - if is_genesis { - let LeeTransaction::Public(public_tx) = transaction else { - return Err(BlockIngestError::NonPublicGenesisTransaction); - }; - state - .transition_from_public_transaction( - public_tx, - block.header.block_id, - block.header.timestamp, - ) - .map_err(|err| state_transition(err.into()))?; - } else { - transaction - .clone() - .execute_on_state(state, block.header.block_id, block.header.timestamp) - .map_err(|err| state_transition(err.into()))?; - } - } - - let LeeTransaction::Public(clock_public_tx) = clock_tx else { - return Err(BlockIngestError::InvalidClockTransaction); - }; - state - .transition_from_public_transaction( - clock_public_tx, - block.header.block_id, - block.header.timestamp, - ) - .map_err(|err| BlockIngestError::StateTransition { - tx_index: user_txs.len().try_into().expect("tx index fits in u64"), - reason: format!("{:#}", anyhow::Error::from(err)), - })?; - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn serializes_and_round_trips_externally_tagged() { - let err = BlockIngestError::UnexpectedBlockId { - expected: 5, - got: 7, - }; - let value = serde_json::to_value(&err).expect("serialize"); - assert_eq!( - value, - serde_json::json!({ "UnexpectedBlockId": { "expected": 5, "got": 7 } }) - ); - let back: BlockIngestError = serde_json::from_value(value).expect("deserialize"); - assert!(matches!( - back, - BlockIngestError::UnexpectedBlockId { - expected: 5, - got: 7 - } - )); - } -} diff --git a/lez/chain_consistency/src/lib.rs b/lez/chain_consistency/src/lib.rs deleted file mode 100644 index d2b45ca9..00000000 --- a/lez/chain_consistency/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Reconstructing and verifying L2 chain state from a Bedrock (L1) channel. - -pub use apply::{BlockIngestError, Tip, apply_block, validate_against_tip}; -pub use consistency::{ - Anchor, AnchorConsistencyCheck, ChainConsistency, ChainMismatch, verify_chain_consistency, -}; - -pub mod apply; -pub mod consistency; diff --git a/lez/chain_consistency/Cargo.toml b/lez/chain_state/Cargo.toml similarity index 83% rename from lez/chain_consistency/Cargo.toml rename to lez/chain_state/Cargo.toml index 02bf0ea0..0a0610a5 100644 --- a/lez/chain_consistency/Cargo.toml +++ b/lez/chain_state/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "chain_consistency" +name = "chain_state" version = "0.1.0" edition = "2024" -license.workspace = true +license = { workspace = true } [lints] workspace = true @@ -11,9 +11,9 @@ workspace = true common.workspace = true lee.workspace = true lee_core.workspace = true - -logos-blockchain-zone-sdk.workspace = true logos-blockchain-core.workspace = true +logos-blockchain-zone-sdk.workspace = true + anyhow.workspace = true borsh.workspace = true futures.workspace = true @@ -23,4 +23,5 @@ thiserror.workspace = true tokio.workspace = true [dev-dependencies] +testnet_initial_state.workspace = true serde_json.workspace = true diff --git a/lez/chain_state/src/apply.rs b/lez/chain_state/src/apply.rs new file mode 100644 index 00000000..05134b88 --- /dev/null +++ b/lez/chain_state/src/apply.rs @@ -0,0 +1,308 @@ +//! The single validate-then-apply entry point shared by the sequencer and the +//! indexer. Pure and storage-free: callers apply on a scratch clone of state and +//! commit only on `Ok`. + +use common::{ + HashType, + block::{Block, BlockMeta}, + transaction::{LeeTransaction, clock_invocation}, +}; +use lee::{GENESIS_BLOCK_ID, V03State}; + +use crate::ingest_error::BlockIngestError; + +/// The parent the next block must chain on. +// `l1_slot` will be added here when the `ChainState` anchor layer lands. +#[derive(Debug, Clone)] +pub struct Tip { + pub block_id: u64, + pub hash: HashType, +} + +impl From<&Block> for Tip { + fn from(block: &Block) -> Self { + Self { + block_id: block.header.block_id, + hash: block.header.hash, + } + } +} + +impl From for Tip { + fn from(meta: BlockMeta) -> Self { + Self { + block_id: meta.id, + hash: meta.hash, + } + } +} + +impl From<&Tip> for BlockMeta { + fn from(tip: &Tip) -> Self { + Self { + id: tip.block_id, + hash: tip.hash, + } + } +} + +/// Outcome of feeding a parsed L2 block to a validated tip. +pub enum AcceptOutcome { + /// Chained and applied; the tip advances. + Applied, + /// A duplicate re-delivery of an already-applied block. No state change. + AlreadyApplied, + /// Did not chain or failed to apply; the tip stays frozen. + Parked(BlockIngestError), + /// Chained but failed to apply, possibly transiently + /// ([`BlockIngestError::is_retryable`]); nothing recorded, tip and state + /// untouched. The caller retries and parks once it gives up. + /// + /// TODO: Only the indexer's `accept_block` emits this today; the sequencer's + /// `ChainState` parks on all failures without retrying (see `on_follow`). + RetryableFailure(BlockIngestError), +} + +/// Validates `block` against `tip`, then applies it to `state`. +/// +/// Mutates `state` in place, so callers pass a scratch clone and commit on `Ok`. +pub fn apply_block( + tip: Option<&Tip>, + block: &Block, + state: &mut V03State, +) -> Result<(), BlockIngestError> { + validate_against_tip(tip, block)?; + apply_block_to_state(block, state)?; + Ok(()) +} + +/// Checks that `block` is the valid continuation of `tip`: hash integrity, +/// then block-id continuity, then `prev_block_hash` linkage. A `None` tip +/// (cold state) expects the genesis block. +pub fn validate_against_tip(tip: Option<&Tip>, block: &Block) -> Result<(), BlockIngestError> { + let computed = block.recompute_hash(); + if computed != block.header.hash { + return Err(BlockIngestError::HashMismatch { + computed, + header: block.header.hash, + }); + } + + match tip { + None => { + if block.header.block_id != GENESIS_BLOCK_ID { + return Err(BlockIngestError::UnexpectedBlockId { + expected: GENESIS_BLOCK_ID, + got: block.header.block_id, + }); + } + } + Some(tip) => { + let expected = tip + .block_id + .checked_add(1) + .expect("block id should not overflow"); + if block.header.block_id != expected { + return Err(BlockIngestError::UnexpectedBlockId { + expected, + got: block.header.block_id, + }); + } + if block.header.prev_block_hash != tip.hash { + return Err(BlockIngestError::BrokenChainLink { + expected_prev: tip.hash, + got_prev: block.header.prev_block_hash, + }); + } + } + } + Ok(()) +} + +/// Applies a block's transactions to `state`, mapping every failure to a +/// [`BlockIngestError`] so the caller can park rather than crash. Operates in +/// place; the caller commits only on `Ok`. +pub fn apply_block_to_state(block: &Block, state: &mut V03State) -> Result<(), BlockIngestError> { + let (clock_tx, user_txs) = block + .body + .transactions + .split_last() + .ok_or(BlockIngestError::EmptyBlock)?; + + let LeeTransaction::Public(clock_tx) = clock_tx else { + return Err(BlockIngestError::InvalidClockTransaction); + }; + if *clock_tx != clock_invocation(block.header.timestamp) { + return Err(BlockIngestError::InvalidClockTransaction); + } + + let is_genesis = block.header.block_id == GENESIS_BLOCK_ID; + for (tx_index, transaction) in user_txs.iter().enumerate() { + let state_transition = |err: anyhow::Error| BlockIngestError::StateTransition { + tx_index: tx_index.try_into().expect("tx index fits in u64"), + reason: format!("{err:#}"), + }; + if is_genesis { + let LeeTransaction::Public(public_tx) = transaction else { + return Err(BlockIngestError::NonPublicGenesisTransaction); + }; + state + .transition_from_public_transaction( + public_tx, + block.header.block_id, + block.header.timestamp, + ) + .map_err(|err| state_transition(err.into()))?; + } else { + transaction + .clone() + .execute_on_state(state, block.header.block_id, block.header.timestamp) + .map_err(|err| state_transition(err.into()))?; + } + } + + state + .transition_from_public_transaction(clock_tx, block.header.block_id, block.header.timestamp) + .map_err(|err| BlockIngestError::StateTransition { + tx_index: user_txs.len().try_into().expect("tx index fits in u64"), + reason: format!("{:#}", anyhow::Error::from(err)), + })?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use common::{ + block::HashableBlockData, + test_utils::{ + create_transaction_native_token_transfer, produce_dummy_block, + produce_dummy_empty_transaction, sequencer_sign_key_for_testing, + }, + }; + use testnet_initial_state::{initial_pub_accounts_private_keys, initial_state}; + + use super::*; + + fn tip_of(block: &Block) -> Tip { + Tip::from(block) + } + + #[test] + fn genesis_applies_on_empty_tip() { + let mut state = initial_state(); + let genesis = produce_dummy_block(1, None, vec![]); + apply_block(None, &genesis, &mut state).expect("genesis applies"); + } + + #[test] + fn non_genesis_first_block_is_unexpected_id() { + let mut state = initial_state(); + let block = produce_dummy_block(2, None, vec![]); + let err = apply_block(None, &block, &mut state).expect_err("should reject"); + assert!(matches!( + err, + BlockIngestError::UnexpectedBlockId { + expected: 1, + got: 2 + } + )); + } + + #[test] + fn skip_ahead_block_is_unexpected_id() { + let mut state = initial_state(); + let genesis = produce_dummy_block(1, None, vec![]); + apply_block(None, &genesis, &mut state).expect("genesis applies"); + + // Tip is at 1; a block with id 3 skips ahead. + let bad = produce_dummy_block(3, Some(genesis.header.hash), vec![]); + let err = + apply_block(Some(&tip_of(&genesis)), &bad, &mut state).expect_err("should reject"); + assert!(matches!( + err, + BlockIngestError::UnexpectedBlockId { + expected: 2, + got: 3 + } + )); + } + + #[test] + fn broken_chain_link_detected() { + let mut state = initial_state(); + let genesis = produce_dummy_block(1, None, vec![]); + apply_block(None, &genesis, &mut state).expect("genesis applies"); + + // Correct id (2), wrong parent hash. + let block2 = produce_dummy_block(2, Some(HashType([9_u8; 32])), vec![]); + let err = + apply_block(Some(&tip_of(&genesis)), &block2, &mut state).expect_err("should reject"); + assert!(matches!(err, BlockIngestError::BrokenChainLink { .. })); + } + + #[test] + fn hash_mismatch_detected() { + let mut state = initial_state(); + let mut genesis = produce_dummy_block(1, None, vec![]); + // Tampering with the header invalidates the stored hash. + genesis.header.timestamp = 999; + let err = apply_block(None, &genesis, &mut state).expect_err("should reject"); + assert!(matches!(err, BlockIngestError::HashMismatch { .. })); + } + + #[test] + fn empty_block_rejected() { + let mut state = initial_state(); + // A block with no transactions at all (not even the mandatory clock tx). + let block = HashableBlockData { + block_id: 1, + prev_block_hash: HashType([0_u8; 32]), + timestamp: 0, + transactions: vec![], + } + .into_pending_block(&sequencer_sign_key_for_testing()); + let err = apply_block(None, &block, &mut state).expect_err("should reject"); + assert!(matches!(err, BlockIngestError::EmptyBlock)); + } + + #[test] + fn missing_clock_tail_is_invalid_clock() { + let mut state = initial_state(); + // Last tx is not the expected clock invocation for the timestamp. + let block = HashableBlockData { + block_id: 1, + prev_block_hash: HashType([0_u8; 32]), + timestamp: 50, + transactions: vec![produce_dummy_empty_transaction()], + } + .into_pending_block(&sequencer_sign_key_for_testing()); + let err = apply_block(None, &block, &mut state).expect_err("should reject"); + assert!(matches!(err, BlockIngestError::InvalidClockTransaction)); + } + + #[test] + fn applies_transfers_and_advances_state() { + let mut state = initial_state(); + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + // Genesis (block 1): clock-only. + let genesis = produce_dummy_block(1, None, vec![]); + apply_block(None, &genesis, &mut state).expect("genesis applies"); + let mut tip = tip_of(&genesis); + + // Blocks 2..=11: one native transfer of 10 each (nonces 0..=9). + for i in 0..10_u64 { + let tx = create_transaction_native_token_transfer(from, i.into(), to, 10, &sign_key); + let block = produce_dummy_block(i + 2, Some(tip.hash), vec![tx]); + apply_block(Some(&tip), &block, &mut state).expect("transfer applies"); + tip = tip_of(&block); + } + + assert_eq!(state.get_account_by_id(from).balance, 9900); + assert_eq!(state.get_account_by_id(to).balance, 20100); + } +} diff --git a/lez/chain_state/src/chain.rs b/lez/chain_state/src/chain.rs new file mode 100644 index 00000000..2d2bebd3 --- /dev/null +++ b/lez/chain_state/src/chain.rs @@ -0,0 +1,952 @@ +//! Two-tier chain state: a reorg-able `head` the sequencer builds on, plus an +//! irreversible `final` tier. + +use common::{HashType, block::Block}; +use lee::V03State; +use log::warn; +use logos_blockchain_core::mantle::ops::channel::MsgId; +use logos_blockchain_zone_sdk::Slot; + +use crate::{ + AcceptOutcome, BlockIngestError, StallReason, + apply::{Tip, apply_block}, +}; + +/// A head block plus the channel message that carried it. +pub struct HeadEntry { + pub this_msg: MsgId, + pub block: Block, +} + +/// The head tier (reorg-able, from `adopted`/`orphaned`) over the final tier +/// (irreversible, from `finalized`). +/// +/// `head_state` is given by `final_state` replayed through `head_blocks`. +/// +/// Only the final tier stalls: an invalid `adopted` block just freezes the +/// head tip and self-heals via reorg or finalization. +pub struct ChainState { + final_state: V03State, + final_tip: Option, + final_stall: Option, + + head_state: V03State, + head_blocks: Vec, +} + +impl ChainState { + /// Fresh state anchored at the genesis/initial state, no blocks applied. + #[must_use] + pub fn new(initial_state: V03State) -> Self { + Self::from_final(initial_state, None) + } + + /// State restored from a persisted final tier; head mirrors final. + #[must_use] + pub fn from_final(final_state: V03State, final_tip: Option) -> Self { + Self { + head_state: final_state.clone(), + final_state, + final_tip, + head_blocks: Vec::new(), + final_stall: None, + } + } + + /// State the sequencer builds its next block on. + #[must_use] + pub const fn head_state(&self) -> &V03State { + &self.head_state + } + + /// Mutable access to the head state. Bypasses the `head_blocks` invariant, so + /// it is meant for tests and low-level callers. + #[must_use] + pub const fn head_state_mut(&mut self) -> &mut V03State { + &mut self.head_state + } + + #[must_use] + pub const fn final_state(&self) -> &V03State { + &self.final_state + } + + /// Parent the next produced block must chain on. + #[must_use] + pub fn head_tip(&self) -> Option { + self.head_blocks + .last() + .map(|entry| Tip::from(&entry.block)) + .or_else(|| self.final_tip.clone()) + } + + #[must_use] + pub fn final_tip(&self) -> Option { + self.final_tip.clone() + } + + #[must_use] + pub const fn final_stall(&self) -> Option<&StallReason> { + self.final_stall.as_ref() + } + + /// Position of a head entry, matched by `MsgId` or block hash (restored + /// entries carry sentinel `MsgId`s; re-inscriptions arrive under fresh ones). + fn head_position_of(&self, this_msg: MsgId, block_hash: HashType) -> Option { + self.head_blocks + .iter() + .position(|entry| entry.this_msg == this_msg || entry.block.header.hash == block_hash) + } + + /// Applies an adopted head block. + /// + /// The adopted stream is authoritative: a competitor at a height + /// the head already holds reorgs the head back to that height with + /// no orphan event required. + /// + /// On failure the head stays unchanged and no stall is recorded. + pub fn apply_adopted(&mut self, this_msg: MsgId, block: &Block) -> AcceptOutcome { + if self.head_position_of(this_msg, block.header.hash).is_some() { + return AcceptOutcome::AlreadyApplied; + } + + // If we receive a pre-final adoption, its an SDK fault; just log and ignore it + if let Some(final_tip) = &self.final_tip + && block.header.block_id <= final_tip.block_id + { + // The final tier is irreversible: a matching block here is a stale + // re-delivery, a conflicting one an SDK contract breach. + if block.header.block_id == final_tip.block_id && block.header.hash != final_tip.hash { + warn!( + "Ignoring adopted block {} with hash {} conflicting with the \ + finalized block ({}) at this height", + block.header.block_id, block.header.hash, final_tip.hash + ); + } + return AcceptOutcome::AlreadyApplied; + } + + // A tip extension applies on the current head state, a lower-or-equal + // id rebuilds the state at the competitor's parent instead. + // + // If adoptions are over the current head, `reorg_at` is None. + let reorg_at = self + .head_blocks + .iter() + .position(|entry| entry.block.header.block_id >= block.header.block_id); + let (mut scratch, tip) = match reorg_at { + // continue from the tip + None => (self.head_state.clone(), self.head_tip()), + // reorg upto `idx` + Some(idx) => self.replay_head_prefix(idx), + }; + + match apply_block(tip.as_ref(), block, &mut scratch) { + Ok(()) => { + // now that `apply_block` succeeded, actually reorg the head + if let Some(idx) = reorg_at { + self.head_blocks.truncate(idx); + } + self.head_state = scratch; + self.head_blocks.push(HeadEntry { + this_msg, + block: block.to_owned(), + }); + AcceptOutcome::Applied + } + Err(err) => AcceptOutcome::Parked(err), + } + } + + /// Reverts an orphaned head block and everything after it, then re-derives head. + pub fn revert_orphan(&mut self, this_msg: MsgId, block: &Block) { + if let Some(idx) = self.head_position_of(this_msg, block.header.hash) { + self.head_blocks.truncate(idx); + self.rederive_head(); + } + } + + /// One channel update: revert every `orphaned` (one truncate + re-derive), + /// then apply every `adopted` in order. Outcomes align with `adopted`. + pub fn apply_channel_update( + &mut self, + orphaned: &[(MsgId, Block)], + adopted: &[(MsgId, Block)], + ) -> Vec { + let earliest = orphaned + .iter() + .filter_map(|(msg, block)| self.head_position_of(*msg, block.header.hash)) + .min(); + if let Some(idx) = earliest { + self.head_blocks.truncate(idx); + self.rederive_head(); + } + adopted + .iter() + .map(|(msg, block)| self.apply_adopted(*msg, block)) + .collect() + } + + /// Rebuilds one head entry from a persisted block, applying it in place (the + /// caller treats `Err` as fatal). + /// + /// The entry gets a hash-derived sentinel `MsgId` (the real one is not + /// persisted โ€” that would need a sidecar `block_id -> MsgId` cell); later + /// orphan/finalize events correlate by block hash. + pub fn restore_head_block(&mut self, block: Block) -> Result<(), BlockIngestError> { + apply_block(self.head_tip().as_ref(), &block, &mut self.head_state)?; + let this_msg = MsgId::from(block.header.hash.0); + self.head_blocks.push(HeadEntry { this_msg, block }); + Ok(()) + } + + /// A finalized inscription. In steady state the block is already in head and is + /// moved into `final`; on backfill (not in head) it is applied directly and may + /// set `final_stall`. + pub fn apply_finalized( + &mut self, + this_msg: MsgId, + block: &Block, + l1_slot: Slot, + ) -> AcceptOutcome { + // Match by `MsgId` or block hash (re-inscriptions, restored entries). + if let Some(idx) = self.head_position_of(this_msg, block.header.hash) { + self.finalize_through(idx); + return AcceptOutcome::Applied; + } + + // Finality is prefix-monotone: a finalized block chaining on an + // unfinalized head entry finalizes that prefix too. + if let Some(idx) = self + .head_blocks + .iter() + .position(|entry| entry.block.header.hash == block.header.prev_block_hash) + { + self.finalize_through(idx); + } + self.apply_finalized_direct(block, l1_slot) + } + + /// Moves `head_blocks[0..=idx]` into the final tier (already validated in head). + fn finalize_through(&mut self, idx: usize) { + let finalized: Vec = self.head_blocks.drain(0..=idx).collect(); + for entry in finalized { + apply_block(self.final_tip.as_ref(), &entry.block, &mut self.final_state) + .expect("validated head block must apply to the final tier"); + self.final_tip = Some(Tip::from(&entry.block)); + } + self.final_stall = None; + } + + /// Applies a finalized block straight to the final tier. On success the + /// finalized chain is authoritative, so head rebases onto it. + fn apply_finalized_direct(&mut self, block: &Block, l1_slot: Slot) -> AcceptOutcome { + // A finalized block at or below the final tip is a re-delivery: + // idempotent. A *different* block at the tip height falls through + // to validation and parks. + if let Some(tip) = &self.final_tip + && (block.header.block_id < tip.block_id + || (block.header.block_id == tip.block_id && block.header.hash == tip.hash)) + { + return AcceptOutcome::AlreadyApplied; + } + + let mut scratch = self.final_state.clone(); + match apply_block(self.final_tip.as_ref(), block, &mut scratch) { + Ok(()) => { + self.final_state = scratch; + self.final_tip = Some(Tip::from(block)); + self.final_stall = None; + // Any head suffix dropped here was already reverted as + // `orphaned` earlier in the same channel update (the sdk + // orders orphans before their finalized replacement), so its + // txs are back in the caller's mempool. + self.head_blocks.clear(); + self.head_state = self.final_state.clone(); + AcceptOutcome::Applied + } + Err(err) => { + self.record_final_stall(block, l1_slot, err.clone()); + AcceptOutcome::Parked(err) + } + } + } + + /// Rebuilds `head_state` from the final tier plus the current `head_blocks`. + fn rederive_head(&mut self) { + self.head_state = self.replay_head_prefix(self.head_blocks.len()).0; + } + + /// State and tip after replaying `head_blocks[..count]` on the final tier. + fn replay_head_prefix(&self, count: usize) -> (V03State, Option) { + let mut state = self.final_state.clone(); + let mut tip = self.final_tip.clone(); + for entry in &self.head_blocks[..count] { + apply_block(tip.as_ref(), &entry.block, &mut state) + .expect("validated head blocks must replay"); + tip = Some(Tip::from(&entry.block)); + } + (state, tip) + } + + /// First stall is stored verbatim; later ones only bump `orphans_since`. + fn record_final_stall(&mut self, block: &Block, l1_slot: Slot, error: BlockIngestError) { + self.final_stall = Some(self.final_stall.take().map_or_else( + || StallReason::new(Some(&block.header), l1_slot, error), + StallReason::escalate, + )); + } +} + +#[cfg(test)] +mod tests { + use common::{ + HashType, + test_utils::{create_transaction_native_token_transfer, produce_dummy_block}, + }; + use testnet_initial_state::{initial_pub_accounts_private_keys, initial_state}; + + use super::*; + + fn msg(n: u8) -> MsgId { + MsgId::from([n; 32]) + } + + fn slot(n: u64) -> Slot { + Slot::from(n) + } + + /// `head_state` equals `final_state` replayed through `head_blocks`. + fn assert_head_matches_replay(chain: &ChainState) { + let mut state = chain.final_state.clone(); + let mut tip = chain.final_tip.clone(); + for entry in &chain.head_blocks { + apply_block(tip.as_ref(), &entry.block, &mut state).expect("head blocks must replay"); + tip = Some(Tip::from(&entry.block)); + } + assert_eq!( + borsh::to_vec(&state).expect("state serializes"), + borsh::to_vec(chain.head_state()).expect("state serializes"), + "head_state must equal final_state replayed through head_blocks" + ); + } + + #[test] + fn adopted_blocks_advance_head() { + let mut chain = ChainState::new(initial_state()); + + let genesis = produce_dummy_block(1, None, vec![]); + assert!(matches!( + chain.apply_adopted(msg(1), &genesis), + AcceptOutcome::Applied + )); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + assert!(matches!( + chain.apply_adopted(msg(2), &block2), + AcceptOutcome::Applied + )); + + assert_eq!(chain.head_tip().expect("head tip").block_id, 2); + // Nothing finalized yet. + assert!(chain.final_tip().is_none()); + } + + #[test] + fn adopted_bad_block_freezes_head_without_stall() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + + // Skips ahead (id 3 while head tip is 1). + let bad = produce_dummy_block(3, Some(genesis.header.hash), vec![]); + assert!(matches!( + chain.apply_adopted(msg(3), &bad), + AcceptOutcome::Parked(BlockIngestError::UnexpectedBlockId { + expected: 2, + got: 3 + }) + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 1); + assert!( + chain.final_stall().is_none(), + "head freeze records no stall" + ); + } + + #[test] + fn adopted_is_idempotent() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + + assert!(matches!( + chain.apply_adopted(msg(1), &genesis), + AcceptOutcome::AlreadyApplied + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 1); + } + + #[test] + fn orphan_reverts_head() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + chain.apply_adopted(msg(3), &block3); + + chain.revert_orphan(msg(3), &block3); + assert_eq!(chain.head_tip().expect("head tip").block_id, 2); + + // A competing block 3 now applies cleanly on block 2. + let block3_prime = produce_dummy_block(3, Some(block2.header.hash), vec![]); + assert!(matches!( + chain.apply_adopted(msg(13), &block3_prime), + AcceptOutcome::Applied + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + } + + #[test] + fn channel_update_reverts_then_applies() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + chain.apply_adopted(msg(3), &block3); + + let block3_prime = produce_dummy_block(3, Some(block2.header.hash), vec![]); + let outcomes = chain.apply_channel_update(&[(msg(3), block3)], &[(msg(13), block3_prime)]); + assert!(matches!(outcomes.as_slice(), [AcceptOutcome::Applied])); + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + } + + #[test] + fn finalize_moves_head_into_final() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + chain.apply_adopted(msg(3), &block3); + + // Finalize through block 2. + assert!(matches!( + chain.apply_finalized(msg(2), &block2, slot(100)), + AcceptOutcome::Applied + )); + assert_eq!(chain.final_tip().expect("final tip").block_id, 2); + // Head tip unchanged; head still ends at 3. + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + } + + #[test] + fn backfill_applies_directly_to_final() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + assert!(matches!( + chain.apply_finalized(msg(1), &genesis, slot(10)), + AcceptOutcome::Applied + )); + assert_eq!(chain.final_tip().expect("final tip").block_id, 1); + // Head mirrors final during backfill. + assert_eq!(chain.head_tip().expect("head tip").block_id, 1); + } + + #[test] + fn invalid_finalized_block_sets_final_stall() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_finalized(msg(1), &genesis, slot(10)); + + // Skip-ahead finalized block, not in head: parks the final tier. + let bad = produce_dummy_block(3, Some(genesis.header.hash), vec![]); + assert!(matches!( + chain.apply_finalized(msg(3), &bad, slot(20)), + AcceptOutcome::Parked(_) + )); + let stall = chain.final_stall().expect("final stall recorded"); + assert_eq!(stall.block_id, Some(3)); + } + + #[test] + fn orphaning_a_suffix_rederives_head_state() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + + let tx2 = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2]); + chain.apply_adopted(msg(2), &block2); + let tx3 = create_transaction_native_token_transfer(from, 1, to, 10, &sign_key); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![tx3]); + chain.apply_adopted(msg(3), &block3); + let tx4 = create_transaction_native_token_transfer(from, 2, to, 10, &sign_key); + let block4 = produce_dummy_block(4, Some(block3.header.hash), vec![tx4]); + chain.apply_adopted(msg(4), &block4); + + // Orphaning block 3 drops the whole suffix (3 and 4). + chain.revert_orphan(msg(3), &block3); + + assert_eq!(chain.head_tip().expect("head tip").block_id, 2); + assert_eq!(chain.head_state().get_account_by_id(from).balance, 9990); + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20010); + assert_head_matches_replay(&chain); + } + + #[test] + fn channel_update_replaces_multi_block_suffix() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + + let tx2 = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2]); + chain.apply_adopted(msg(2), &block2); + let tx3 = create_transaction_native_token_transfer(from, 1, to, 10, &sign_key); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![tx3]); + chain.apply_adopted(msg(3), &block3); + let tx4 = create_transaction_native_token_transfer(from, 2, to, 10, &sign_key); + let block4 = produce_dummy_block(4, Some(block3.header.hash), vec![tx4]); + chain.apply_adopted(msg(4), &block4); + + // A competing branch replaces blocks 3 and 4; orphans arrive unordered. + let tx3_prime = create_transaction_native_token_transfer(from, 1, to, 20, &sign_key); + let block3_prime = produce_dummy_block(3, Some(block2.header.hash), vec![tx3_prime]); + let tx4_prime = create_transaction_native_token_transfer(from, 2, to, 30, &sign_key); + let block4_prime = produce_dummy_block(4, Some(block3_prime.header.hash), vec![tx4_prime]); + + let outcomes = chain.apply_channel_update( + &[(msg(4), block4), (msg(3), block3)], + &[(msg(13), block3_prime), (msg(14), block4_prime)], + ); + + assert!(matches!( + outcomes.as_slice(), + [AcceptOutcome::Applied, AcceptOutcome::Applied] + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 4); + assert_eq!(chain.head_state().get_account_by_id(from).balance, 9940); + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20060); + assert_head_matches_replay(&chain); + } + + #[test] + fn adopted_only_channel_update_replaces_suffix() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + let tx2 = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2]); + chain.apply_adopted(msg(2), &block2); + let tx3 = create_transaction_native_token_transfer(from, 1, to, 10, &sign_key); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![tx3]); + chain.apply_adopted(msg(3), &block3); + + // The replacement branch arrives with no orphan events: the adopted + // list alone reorgs the head. + let tx2_prime = create_transaction_native_token_transfer(from, 0, to, 20, &sign_key); + let block2_prime = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2_prime]); + let tx3_prime = create_transaction_native_token_transfer(from, 1, to, 30, &sign_key); + let block3_prime = produce_dummy_block(3, Some(block2_prime.header.hash), vec![tx3_prime]); + + let outcomes = + chain.apply_channel_update(&[], &[(msg(12), block2_prime), (msg(13), block3_prime)]); + + assert!(matches!( + outcomes.as_slice(), + [AcceptOutcome::Applied, AcceptOutcome::Applied] + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20050); + assert_head_matches_replay(&chain); + } + + #[test] + fn channel_update_ignores_unknown_orphan() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + let unknown = produce_dummy_block(9, Some(HashType([7; 32])), vec![]); + let outcomes = chain.apply_channel_update(&[(msg(99), unknown)], &[(msg(3), block3)]); + + assert!(matches!(outcomes.as_slice(), [AcceptOutcome::Applied])); + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + assert_head_matches_replay(&chain); + } + + #[test] + fn adopted_competitor_reorgs_head_without_orphan_event() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + let tx2 = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2]); + chain.apply_adopted(msg(2), &block2); + let tx3 = create_transaction_native_token_transfer(from, 1, to, 10, &sign_key); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![tx3]); + chain.apply_adopted(msg(3), &block3); + + // A valid competitor at height 2, no orphan events: the head reorgs + // back onto it, dropping the old 2..=3 suffix and its transfers. + let block2_prime = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + assert!(matches!( + chain.apply_adopted(msg(12), &block2_prime), + AcceptOutcome::Applied + )); + let tip = chain.head_tip().expect("head tip"); + assert_eq!(tip.block_id, 2); + assert_eq!(tip.hash, block2_prime.header.hash); + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20000); + assert_head_matches_replay(&chain); + } + + #[test] + fn invalid_adopted_competitor_leaves_head_intact() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + chain.apply_adopted(msg(3), &block3); + + // A competitor at height 2 with a bogus parent parks; the truncation + // is not committed, so the 2..=3 suffix survives. + let bad = produce_dummy_block(2, Some(HashType([9; 32])), vec![]); + assert!(matches!( + chain.apply_adopted(msg(12), &bad), + AcceptOutcome::Parked(BlockIngestError::BrokenChainLink { .. }) + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + assert_head_matches_replay(&chain); + } + + #[test] + fn adopted_conflicting_with_final_tip_is_ignored() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + chain.apply_finalized(msg(1), &genesis, slot(10)); + chain.apply_finalized(msg(2), &block2, slot(20)); + + // Finalized is irreversible: an adopted competitor at (or below) the + // final tip is ignored, not reorged onto. + let block2_prime = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + assert!(matches!( + chain.apply_adopted(msg(12), &block2_prime), + AcceptOutcome::AlreadyApplied + )); + assert_eq!( + chain.final_tip().expect("final tip").hash, + block2.header.hash + ); + assert_eq!(chain.head_tip().expect("head tip").hash, block2.header.hash); + assert_head_matches_replay(&chain); + } + + #[test] + fn restore_head_block_rebuilds_head_and_correlates_by_hash() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + // Restart shape: final tier from a persisted snapshot, head rebuilt from + // stored blocks under hash-derived sentinel MsgIds. + let mut state = initial_state(); + let genesis = produce_dummy_block(1, None, vec![]); + apply_block(None, &genesis, &mut state).expect("genesis applies"); + let mut chain = ChainState::from_final(state, Some(Tip::from(&genesis))); + + let tx2 = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2]); + let tx3 = create_transaction_native_token_transfer(from, 1, to, 10, &sign_key); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![tx3]); + for block in [&block2, &block3] { + chain + .restore_head_block(block.clone()) + .expect("stored blocks must replay"); + } + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + assert_head_matches_replay(&chain); + + // The L1 orphans restored block 3 under its real (unknown-to-us) MsgId: + // correlated by hash, the revert works and a competitor applies. + chain.revert_orphan(msg(33), &block3); + assert_eq!(chain.head_tip().expect("head tip").block_id, 2); + + let block3_prime = produce_dummy_block(3, Some(block2.header.hash), vec![]); + assert!(matches!( + chain.apply_adopted(msg(13), &block3_prime), + AcceptOutcome::Applied + )); + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20010); + assert_head_matches_replay(&chain); + } + + #[test] + fn restore_head_block_rejects_non_chaining_block() { + let mut chain = ChainState::new(initial_state()); + let skipped = produce_dummy_block(3, Some(HashType([9; 32])), vec![]); + assert!(chain.restore_head_block(skipped).is_err()); + } + + #[test] + fn finalized_reinscription_matches_by_block_hash() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + chain.apply_adopted(msg(3), &block3); + + // Block 2 finalizes re-inscribed under a fresh MsgId: matched by hash, + // finalized through, and the head above it survives. + assert!(matches!( + chain.apply_finalized(msg(42), &block2, slot(5)), + AcceptOutcome::Applied + )); + assert_eq!(chain.final_tip().expect("final tip").block_id, 2); + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + assert!(chain.final_stall().is_none()); + assert_head_matches_replay(&chain); + } + + #[test] + fn finalize_through_preserves_head_state_and_advances_final_state() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + let tx2 = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2]); + chain.apply_adopted(msg(2), &block2); + let tx3 = create_transaction_native_token_transfer(from, 1, to, 10, &sign_key); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![tx3]); + chain.apply_adopted(msg(3), &block3); + + chain.apply_finalized(msg(2), &block2, slot(10)); + + // Head still reflects both transfers + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20020); + // ...while final reflects only the finalized prefix. + assert_eq!(chain.final_state().get_account_by_id(to).balance, 20010); + assert_head_matches_replay(&chain); + } + + #[test] + fn head_self_heals_with_valid_competitor_after_park() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + + // Correct id, wrong parent: parked, head frozen at 1, no stall. + let bad = produce_dummy_block(2, Some(HashType([9; 32])), vec![]); + assert!(matches!( + chain.apply_adopted(msg(2), &bad), + AcceptOutcome::Parked(BlockIngestError::BrokenChainLink { .. }) + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 1); + assert!(chain.final_stall().is_none()); + + // A valid competitor at the same height applies without any reorg event. + let good = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + assert!(matches!( + chain.apply_adopted(msg(12), &good), + AcceptOutcome::Applied + )); + assert_eq!(chain.head_tip().expect("head tip").block_id, 2); + assert_head_matches_replay(&chain); + } + + #[test] + fn repeated_invalid_finalized_bumps_orphans_since() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_finalized(msg(1), &genesis, slot(10)); + + let bad3 = produce_dummy_block(3, Some(genesis.header.hash), vec![]); + chain.apply_finalized(msg(3), &bad3, slot(20)); + let bad5 = produce_dummy_block(5, Some(bad3.header.hash), vec![]); + assert!(matches!( + chain.apply_finalized(msg(5), &bad5, slot(30)), + AcceptOutcome::Parked(_) + )); + + let stall = chain.final_stall().expect("final stall recorded"); + assert_eq!(stall.block_id, Some(3), "first stall reason is preserved"); + assert_eq!(stall.orphans_since, 1); + } + + #[test] + fn valid_finalized_successor_clears_final_stall() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_finalized(msg(1), &genesis, slot(10)); + + let bad = produce_dummy_block(3, Some(genesis.header.hash), vec![]); + chain.apply_finalized(msg(3), &bad, slot(20)); + assert!(chain.final_stall().is_some()); + + // The valid successor of the frozen final tip finalizes: stall clears. + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + assert!(matches!( + chain.apply_finalized(msg(2), &block2, slot(30)), + AcceptOutcome::Applied + )); + assert!(chain.final_stall().is_none()); + assert_eq!(chain.final_tip().expect("final tip").block_id, 2); + assert_head_matches_replay(&chain); + } + + #[test] + fn finalized_successor_of_head_entry_finalizes_the_prefix() { + // Head holds unfinalized blocks 1..=2 (e.g. restored after a restart); + // a peer block 3 we never saw adopted arrives finalized. Its ancestry + // finalizes our prefix implicitly, then 3 applies to final directly. + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_adopted(msg(2), &block2); + assert!(chain.final_tip().is_none()); + + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + assert!(matches!( + chain.apply_finalized(msg(3), &block3, slot(10)), + AcceptOutcome::Applied + )); + assert_eq!(chain.final_tip().expect("final tip").block_id, 3); + assert_eq!(chain.head_tip().expect("head tip").block_id, 3); + assert!(chain.final_stall().is_none()); + assert_head_matches_replay(&chain); + } + + #[test] + fn finalized_redelivery_at_or_below_final_tip_is_already_applied() { + // Restart shape: the store's tip (incl. not-yet-finalized blocks) is + // restored as the final tier, so their later finalization arrives for + // blocks that were never in `head_blocks`. + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + chain.apply_finalized(msg(1), &genesis, slot(10)); + chain.apply_finalized(msg(2), &block2, slot(20)); + + // Below the tip, and at the tip with a matching hash: idempotent. + assert!(matches!( + chain.apply_finalized(msg(41), &genesis, slot(30)), + AcceptOutcome::AlreadyApplied + )); + assert!(matches!( + chain.apply_finalized(msg(42), &block2, slot(30)), + AcceptOutcome::AlreadyApplied + )); + assert!(chain.final_stall().is_none()); + assert_eq!(chain.final_tip().expect("final tip").block_id, 2); + assert_head_matches_replay(&chain); + } + + #[test] + fn conflicting_finalized_at_final_tip_parks() { + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + chain.apply_finalized(msg(1), &genesis, slot(10)); + chain.apply_finalized(msg(2), &block2, slot(20)); + + // A different finalized block at the final height: finalized is + // irreversible, so this is a genuine stall, not a re-delivery. + let block2_prime = produce_dummy_block(2, Some(HashType([9; 32])), vec![]); + assert!(matches!( + chain.apply_finalized(msg(22), &block2_prime, slot(30)), + AcceptOutcome::Parked(_) + )); + assert!(chain.final_stall().is_some()); + assert_eq!(chain.final_tip().expect("final tip").block_id, 2); + } + + #[test] + fn finalized_unknown_block_rebases_head() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + chain.apply_finalized(msg(1), &genesis, slot(10)); + + // Head advances on a competing branchโ€ฆ + let block2a = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + chain.apply_adopted(msg(2), &block2a); + + // โ€ฆbut a different block 2 finalizes. The finalized chain is + // authoritative, so head rebases onto it. + let tx = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2b = produce_dummy_block(2, Some(genesis.header.hash), vec![tx]); + assert!(matches!( + chain.apply_finalized(msg(22), &block2b, slot(20)), + AcceptOutcome::Applied + )); + + assert_eq!(chain.final_tip().expect("final tip").block_id, 2); + assert_eq!(chain.head_tip().expect("head tip").block_id, 2); + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20010); + assert_head_matches_replay(&chain); + } + + #[test] + fn head_state_reflects_applied_transfers() { + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let mut chain = ChainState::new(initial_state()); + let genesis = produce_dummy_block(1, None, vec![]); + chain.apply_adopted(msg(1), &genesis); + + let tx = create_transaction_native_token_transfer(from, 0, to, 10, &sign_key); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx]); + chain.apply_adopted(msg(2), &block2); + + assert_eq!(chain.head_state().get_account_by_id(from).balance, 9990); + assert_eq!(chain.head_state().get_account_by_id(to).balance, 20010); + } +} diff --git a/lez/chain_consistency/src/consistency.rs b/lez/chain_state/src/consistency.rs similarity index 100% rename from lez/chain_consistency/src/consistency.rs rename to lez/chain_state/src/consistency.rs diff --git a/lez/chain_state/src/ingest_error.rs b/lez/chain_state/src/ingest_error.rs new file mode 100644 index 00000000..d259b5f7 --- /dev/null +++ b/lez/chain_state/src/ingest_error.rs @@ -0,0 +1,80 @@ +use common::HashType; +use serde::{Deserialize, Serialize}; + +/// Why an L2 block from the channel could not be applied. +/// +/// Persisted in `RocksDB` (as part of [`crate::StallReason`]), so every variant +/// must be `Clone + Serialize + Deserialize`. +#[derive(Debug, Clone, Serialize, Deserialize, thiserror::Error)] +pub enum BlockIngestError { + #[error("Failed to deserialize L2 block: {0}")] + /// Here we store the error string that is derived from [`borsh::from_slice`]'s [`Err`]. + Deserialize(String), + #[error("Unexpected block id: expected {expected}, got {got}")] + UnexpectedBlockId { expected: u64, got: u64 }, + #[error("Broken chain link: expected prev {expected_prev}, got {got_prev}")] + BrokenChainLink { + expected_prev: HashType, + got_prev: HashType, + }, + #[error("Block hash mismatch: computed {computed}, header {header}")] + HashMismatch { + computed: HashType, + header: HashType, + }, + #[error("Block has no transactions")] + EmptyBlock, + #[error("Last transaction must be the public clock invocation for the block timestamp")] + InvalidClockTransaction, + #[error("Genesis block must contain only public transactions")] + NonPublicGenesisTransaction, + #[error("State transition failed at transaction {tx_index}: {reason}")] + StateTransition { + /// Index of the failing transaction within the block body. + tx_index: u64, + /// Reason string from `lee::Error` to `anyhow::Error` to `{:#}`. + /// + /// This is required because `lee::Error` is not `Clone + Serialize + Deserialize`, so we + /// cannot store it directly. + reason: String, + }, +} + +impl BlockIngestError { + /// Whether the failure may be transient rather than a property of the block. + /// + /// FIXME: `StateTransition` is too coarse โ€” its `reason` string mixes genuine + /// state-transition rejections with infra failures (risc0 executor teardown, + /// storage errors). Once it carries a structured cause, narrow this so only + /// infra failures retry. + #[must_use] + pub const fn is_retryable(&self) -> bool { + matches!(self, Self::StateTransition { .. }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn serializes_and_round_trips_externally_tagged() { + let err = BlockIngestError::UnexpectedBlockId { + expected: 5, + got: 7, + }; + let value = serde_json::to_value(&err).expect("serialize"); + assert_eq!( + value, + serde_json::json!({ "UnexpectedBlockId": { "expected": 5, "got": 7 } }) + ); + let back: BlockIngestError = serde_json::from_value(value).expect("deserialize"); + assert!(matches!( + back, + BlockIngestError::UnexpectedBlockId { + expected: 5, + got: 7 + } + )); + } +} diff --git a/lez/chain_state/src/lib.rs b/lez/chain_state/src/lib.rs new file mode 100644 index 00000000..89357af0 --- /dev/null +++ b/lez/chain_state/src/lib.rs @@ -0,0 +1,17 @@ +//! Storage-free chain-state core shared by the LEZ sequencer and indexer: +//! the [`apply_block`] entry point plus [`BlockIngestError`], [`StallReason`], +//! [`Tip`], and [`AcceptOutcome`]. See [`ChainState`] for the two-tier model. + +pub use apply::{AcceptOutcome, Tip, apply_block, apply_block_to_state, validate_against_tip}; +pub use chain::{ChainState, HeadEntry}; +pub use consistency::{ + Anchor, AnchorConsistencyCheck, ChainConsistency, ChainMismatch, verify_chain_consistency, +}; +pub use ingest_error::BlockIngestError; +pub use stall_reason::StallReason; + +pub mod apply; +pub mod chain; +pub mod consistency; +pub mod ingest_error; +pub mod stall_reason; diff --git a/lez/chain_state/src/stall_reason.rs b/lez/chain_state/src/stall_reason.rs new file mode 100644 index 00000000..e1d9b3ac --- /dev/null +++ b/lez/chain_state/src/stall_reason.rs @@ -0,0 +1,50 @@ +use common::{HashType, block::BlockHeader}; +use logos_blockchain_zone_sdk::Slot; +use serde::{Deserialize, Serialize}; + +use crate::ingest_error::BlockIngestError; + +/// Diagnostic record of the first block that broke the L2 chain. +/// +/// The block-derived fields are `None` for a deserialize break (no header was +/// ever parsed). `l1_slot` is the L1 slot the breaking inscription was read at. +/// `first_seen` is the breaking block's L2 timestamp (`None` for a deserialize break). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StallReason { + pub block_id: Option, + pub block_hash: Option, + pub prev_block_hash: Option, + pub l1_slot: Slot, + pub error: BlockIngestError, + pub first_seen: Option, + /// Number of later non-chaining blocks (orphans, since the tip is frozen). + /// + /// TODO: We could store a different "branch" of blocks following this break, but for now we + /// just count them. + pub orphans_since: u64, +} + +impl StallReason { + /// First stall for a break, built from the breaking block's header + /// (`None` for a deserialize break). + #[must_use] + pub fn new(header: Option<&BlockHeader>, l1_slot: Slot, error: BlockIngestError) -> Self { + Self { + block_id: header.map(|header| header.block_id), + block_hash: header.map(|header| header.hash), + prev_block_hash: header.map(|header| header.prev_block_hash), + first_seen: header.map(|header| header.timestamp), + l1_slot, + error, + orphans_since: 0, + } + } + + /// A later stall on the same break: bumps `orphans_since`, preserving the + /// original cause. + #[must_use] + pub const fn escalate(mut self) -> Self { + self.orphans_since = self.orphans_since.saturating_add(1); + self + } +} diff --git a/lez/common/src/block.rs b/lez/common/src/block.rs index b22eec9f..53a2f033 100644 --- a/lez/common/src/block.rs +++ b/lez/common/src/block.rs @@ -13,6 +13,15 @@ pub struct BlockMeta { pub hash: BlockHash, } +impl From<&Block> for BlockMeta { + fn from(block: &Block) -> Self { + Self { + id: block.header.block_id, + hash: block.header.hash, + } + } +} + #[derive(Debug, Clone)] /// Our own hasher. /// Currently it is SHA256 hasher wrapper. May change in a future. diff --git a/lez/indexer/core/Cargo.toml b/lez/indexer/core/Cargo.toml index cbe0b7e8..afba6f29 100644 --- a/lez/indexer/core/Cargo.toml +++ b/lez/indexer/core/Cargo.toml @@ -12,8 +12,8 @@ default = [] testnet = [] [dependencies] +chain_state.workspace = true common.workspace = true -chain_consistency.workspace = true logos-blockchain-zone-sdk.workspace = true lee.workspace = true lee_core.workspace = true diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 3d840f90..a5d66a23 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -1,7 +1,9 @@ use std::{path::Path, sync::Arc}; use anyhow::{Context as _, Result}; -use chain_consistency::{BlockIngestError, Tip}; +use chain_state::{ + AcceptOutcome, BlockIngestError, StallReason, Tip, apply_block_to_state, validate_against_tip, +}; use common::{ block::{BedrockStatus, Block, BlockHeader}, transaction::LeeTransaction, @@ -14,23 +16,6 @@ use logos_blockchain_zone_sdk::Slot; use storage::indexer::RocksDBIO; use tokio::sync::RwLock; -use crate::status::StallReason; - -/// Outcome of feeding a parsed L2 block to the validated tip. -pub enum AcceptOutcome { - /// Chained and applied; tip and L1 read cursor both advance. - Applied, - /// A duplicate re-delivery of the current tip. Just L2 advances. - AlreadyApplied, - /// Did not chain or failed to apply; tip stays frozen, stall recorded. - Parked(BlockIngestError), - /// Chained but failed to apply, possibly transiently - /// ([`BlockIngestError::is_retryable`]); nothing recorded, tip and state - /// untouched. The caller retries and parks via - /// [`IndexerStore::record_stall`] once it gives up. - RetryableFailure(BlockIngestError), -} - #[derive(Clone)] pub struct IndexerStore { dbio: Arc, @@ -202,10 +187,7 @@ impl IndexerStore { let Some(block) = self.dbio.get_block(block_id)? else { return Ok(None); }; - Ok(Some(Tip { - block_id, - hash: block.header.hash, - })) + Ok(Some(Tip::from(&block))) } /// Record the stall reason. @@ -218,22 +200,10 @@ impl IndexerStore { l1_slot: Slot, error: BlockIngestError, ) -> Result<()> { - let stall = match self.get_stall_reason()? { - Some(mut existing) => { - existing.orphans_since = existing.orphans_since.saturating_add(1); - existing - } - None => StallReason { - // need to map out of `header` because they are not ser/de - block_id: header.map(|h| h.block_id), - block_hash: header.map(|h| h.hash), - prev_block_hash: header.map(|h| h.prev_block_hash), - first_seen: header.map(|h| h.timestamp), - l1_slot, - error, - orphans_since: 0, - }, - }; + let stall = self.get_stall_reason()?.map_or_else( + || StallReason::new(header, l1_slot, error), + StallReason::escalate, + ); self.set_stall_reason(&Some(stall)) } @@ -253,14 +223,16 @@ impl IndexerStore { return Ok(AcceptOutcome::AlreadyApplied); } - if let Err(err) = chain_consistency::validate_against_tip(tip, block) { + // Validate before paying for the scratch clone; validation failures + // are never retryable, so parking immediately is exact. + if let Err(err) = validate_against_tip(tip.as_ref(), block) { self.record_stall(Some(&block.header), l1_slot, err.clone())?; return Ok(AcceptOutcome::Parked(err)); } // TODO: we use scratch state to be atomic, but need to revisit how expensive a clone is let mut scratch = self.current_state.read().await.clone(); - if let Err(err) = chain_consistency::apply_block(block, &mut scratch) { + if let Err(err) = apply_block_to_state(block, &mut scratch) { if err.is_retryable() { return Ok(AcceptOutcome::RetryableFailure(err)); } @@ -435,7 +407,6 @@ mod tests { #[cfg(test)] mod accept_tests { - use chain_consistency::BlockIngestError; use common::{HashType, block::HashableBlockData, test_utils::produce_dummy_block}; use super::*; diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index dd58eaae..5a4909f4 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -2,8 +2,8 @@ use std::{path::Path, sync::Arc}; use anyhow::Result; use arc_swap::ArcSwap; -pub use chain_consistency::BlockIngestError; -use chain_consistency::{Anchor, ChainConsistency}; +pub use chain_state::{AcceptOutcome, BlockIngestError, StallReason}; +use chain_state::{Anchor, ChainConsistency}; use common::block::Block; // TODO: Remove after testnet use futures::StreamExt as _; @@ -12,10 +12,9 @@ use logos_blockchain_zone_sdk::{ CommonHttpClient, Slot, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer, }; use retry::ApplyRetryGate; -pub use status::StallReason; use crate::{ - block_store::{AcceptOutcome, IndexerStore}, + block_store::IndexerStore, config::IndexerConfig, cross_zone_verifier::CrossZoneVerifier, status::{IndexerStatus, IndexerSyncStatus}, @@ -136,8 +135,7 @@ impl IndexerCore { return Ok(ChainConsistency::Inconclusive); }; - chain_consistency::verify_chain_consistency(&self.node, self.config.channel_id, &anchor) - .await + chain_state::verify_chain_consistency(&self.node, self.config.channel_id, &anchor).await } /// Builds the anchor for the startup check. diff --git a/lez/indexer/core/src/status.rs b/lez/indexer/core/src/status.rs index 1307238b..aa182c23 100644 --- a/lez/indexer/core/src/status.rs +++ b/lez/indexer/core/src/status.rs @@ -1,27 +1,5 @@ -use chain_consistency::BlockIngestError; -use common::HashType; -use logos_blockchain_zone_sdk::Slot; -use serde::{Deserialize, Serialize}; - -/// Diagnostic record of the first block that broke the L2 chain. -/// -/// The block-derived fields are `None` for a deserialize break (no header was -/// ever parsed). `l1_slot` is the L1 slot the breaking inscription was read at. -/// `first_seen` is the breaking block's L2 timestamp (`None` for a deserialize break). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StallReason { - pub block_id: Option, - pub block_hash: Option, - pub prev_block_hash: Option, - pub l1_slot: Slot, - pub error: BlockIngestError, - pub first_seen: Option, - /// Number of later non-chaining blocks (orphans, since the tip is frozen). - /// - /// TODO: We could store a different "branch" of blocks following this break, but for now we - /// just count them. - pub orphans_since: u64, -} +use chain_state::StallReason; +use serde::Serialize; /// Coarse lifecycle state of the indexer's ingestion loop, so a client can tell /// "still catching up" apart from "something went wrong". @@ -138,6 +116,7 @@ mod tests { #[test] fn stalled_status_serializes_with_stall_reason() { + use chain_state::{BlockIngestError, StallReason}; use logos_blockchain_zone_sdk::Slot; let status = IndexerStatus { diff --git a/lez/mempool/src/lib.rs b/lez/mempool/src/lib.rs index 1b36eaf7..0006f2c3 100644 --- a/lez/mempool/src/lib.rs +++ b/lez/mempool/src/lib.rs @@ -65,6 +65,11 @@ impl MemPoolHandle { pub async fn push(&self, item: T) -> Result<(), tokio::sync::mpsc::error::SendError> { self.sender.send(item).await } + + /// Send an item to the mempool, failing _immediately_ if it is full. + pub fn try_push(&self, item: T) -> Result<(), tokio::sync::mpsc::error::TrySendError> { + self.sender.try_send(item) + } } #[cfg(test)] @@ -123,6 +128,19 @@ mod tests { assert_eq!(pool.pop(), Some(2)); } + #[test] + async fn try_push_fails_when_full_without_blocking() { + let (mut pool, handle) = MemPool::new(1); + + handle.try_push(1).unwrap(); + assert!(handle.try_push(2).is_err(), "full mempool must not accept"); + + // Popping frees capacity again. + assert_eq!(pool.pop(), Some(1)); + handle.try_push(2).unwrap(); + assert_eq!(pool.pop(), Some(2)); + } + #[test] async fn push_front() { let (mut pool, handle) = MemPool::new(10); diff --git a/lez/sequencer/core/Cargo.toml b/lez/sequencer/core/Cargo.toml index 1fa7494d..c8590c2a 100644 --- a/lez/sequencer/core/Cargo.toml +++ b/lez/sequencer/core/Cargo.toml @@ -10,8 +10,8 @@ workspace = true [dependencies] lee.workspace = true lee_core.workspace = true +chain_state.workspace = true common.workspace = true -chain_consistency.workspace = true storage.workspace = true mempool.workspace = true logos-blockchain-zone-sdk.workspace = true @@ -34,6 +34,7 @@ tempfile.workspace = true chrono.workspace = true log.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +tokio-util.workspace = true rand.workspace = true borsh.workspace = true bytesize.workspace = true diff --git a/lez/sequencer/core/src/block_publisher.rs b/lez/sequencer/core/src/block_publisher.rs index 3f2b2a62..8f2f2031 100644 --- a/lez/sequencer/core/src/block_publisher.rs +++ b/lez/sequencer/core/src/block_publisher.rs @@ -1,23 +1,44 @@ use std::{pin::Pin, sync::Arc, time::Duration}; -use anyhow::{Context as _, Result, anyhow}; +use anyhow::{Context as _, Result, anyhow, ensure}; use common::block::Block; use futures::Stream; use log::{info, warn}; -pub use logos_blockchain_core::mantle::ops::channel::MsgId; -use logos_blockchain_core::mantle::ops::channel::{ChannelId, inscribe::Inscription}; -pub use logos_blockchain_key_management_system_service::keys::{Ed25519Key, ZkKey}; +pub use logos_blockchain_core::mantle::ops::channel::{Ed25519PublicKey, MsgId}; +use logos_blockchain_core::{ + mantle::{ + MantleTx, SignedMantleTx, Transaction as _, + channel::{SlotTimeframe, SlotTimeout}, + ops::{ + Op, OpProof, + channel::{ + ChannelId, + config::{ChannelConfigOp, Keys}, + inscribe::Inscription, + }, + }, + }, + proofs::channel_multi_sig_proof::{ChannelMultiSigProof, IndexedSignature}, +}; +pub use logos_blockchain_key_management_system_service::keys::{ + ED25519_SECRET_KEY_SIZE, Ed25519Key, ZkKey, +}; pub use logos_blockchain_zone_sdk::sequencer::SequencerCheckpoint; use logos_blockchain_zone_sdk::{ CommonHttpClient, Slot, ZoneMessage, adapter::{Node as _, NodeHttpClient}, indexer::ZoneIndexer, sequencer::{ - DepositInfo, Event, FinalizedOp, InscriptionInfo, - SequencerConfig as ZoneSdkSequencerConfig, WithdrawArg, WithdrawInfo, ZoneSequencer, + DepositInfo, Event, FinalizedOp, InscriptionInfo, OrphanedTx, + SequencerConfig as ZoneSdkSequencerConfig, TurnNotification, WithdrawArg, WithdrawInfo, + ZoneSequencer, }, }; -use tokio::{sync::mpsc, task::JoinHandle}; +use tokio::{ + sync::{mpsc, oneshot, watch}, + task::JoinHandle, +}; +use tokio_util::sync::CancellationToken; use crate::config::BedrockConfig; @@ -43,6 +64,36 @@ pub type OnDepositEventSink = pub type OnWithdrawEventSink = Box Pin + Send>> + Send + 'static>; +/// The channel delta the follow path consumes from one `Event::BlocksProcessed`, +/// with inscription payloads decoded into `(MsgId, Block)` pairs. +pub struct FollowUpdate { + /// Inscriptions newly on the followed L1 branch, in channel order: they + /// extend (or, after a reorg, replace part of) the `head` tier. + pub adopted: Vec<(MsgId, Block)>, + /// Inscriptions dropped from the branch by an L1 reorg: their blocks are + /// reverted from the `head` and their user txs resubmitted to the mempool. + pub orphaned: Vec<(MsgId, Block)>, + /// Inscriptions whose containing L1 block reached finality: their blocks + /// move into the irreversible `final` tier. + pub finalized: Vec<(MsgId, Block)>, +} + +/// Sink for the follow path: apply adopted/finalized blocks to chain state and +/// revert orphaned ones. +pub type OnFollowSink = Box; + +/// Commands the drive task executes with `&mut sequencer`. +enum Command { + /// Publish an inscription (+ atomic withdrawals); responds with the assigned `MsgId`. + Publish { + inscription: Inscription, + withdrawals: Vec, + resp: oneshot::Sender>, + }, +} + +type CommandSender = mpsc::Sender; + #[expect(async_fn_in_trait, reason = "We don't care about Send/Sync here")] pub trait BlockPublisherTrait: Sized { #[expect( @@ -58,14 +109,23 @@ pub trait BlockPublisherTrait: Sized { on_finalized_block: FinalizedBlockSink, on_deposit_event: OnDepositEventSink, on_withdraw_event: OnWithdrawEventSink, + on_follow: OnFollowSink, ) -> Result; - /// Fire-and-forget publish. Zone-sdk drives the actual submission and - /// retries internally; this just hands the payload off. - async fn publish_block(&self, block: &Block, withdrawals: Vec) -> Result<()>; + /// Publish a block and return the `MsgId` zone-sdk assigned its inscription. + /// Zone-sdk drives the actual submission and retries internally. + async fn publish_block(&self, block: &Block, withdrawals: Vec) -> Result; fn channel_id(&self) -> ChannelId; + /// Whether this sequencer is currently authorized to write to the channel. + fn is_our_turn(&self) -> bool; + + /// A [`CancellationToken`] cancelled when the publisher's background driver + /// terminates (a panicked sink, an ended event stream). No channel events + /// are processed past that point, so the node must halt. + fn driver_cancellation(&self) -> CancellationToken; + /// Current channel frontier slot on the connected chain, or `None` if the /// channel does not exist there. Drives the startup frontier check. async fn channel_tip_slot(&self) -> Result>; @@ -85,7 +145,10 @@ pub struct ZoneSdkPublisher { /// Direct node handle retained for channel reads (startup consistency check /// and reconstruction); the sequencer itself lives in the drive task. node: NodeHttpClient, - publish_tx: mpsc::Sender<(Inscription, Vec)>, + command_tx: CommandSender, + turn_rx: watch::Receiver, + // Cancelled when the drive task ends for any reason, including a panic. + driver_cancellation: CancellationToken, // Aborts the drive task when the last clone is dropped. _drive_task: Arc, indexer: ZoneIndexer, @@ -109,6 +172,7 @@ impl BlockPublisherTrait for ZoneSdkPublisher { on_finalized_block: FinalizedBlockSink, on_deposit_event: OnDepositEventSink, on_withdraw_event: OnWithdrawEventSink, + on_follow: OnFollowSink, ) -> Result { let basic_auth = config.auth.clone().map(Into::into); let node = NodeHttpClient::new(CommonHttpClient::new(basic_auth), config.node_url.clone()); @@ -129,11 +193,18 @@ impl BlockPublisherTrait for ZoneSdkPublisher { // Grab readiness receiver before moving the sequencer into the drive // task so we can await cold-start completion below. let mut ready_rx = sequencer.subscribe_ready(); + // Grab the turn watch before the move; the sdk actor keeps it current. + let turn_rx = sequencer.subscribe_turn_to_write(); - let (publish_tx, mut publish_rx) = - mpsc::channel::<(Inscription, Vec)>(PUBLISH_INBOX_CAPACITY); + let (command_tx, mut command_rx): (CommandSender, _) = + mpsc::channel(PUBLISH_INBOX_CAPACITY); + let driver_cancellation = CancellationToken::new(); + let driver_guard = driver_cancellation.clone().drop_guard(); let drive_task = tokio::spawn(async move { + // Dropped when this task ends (including panics in the sinks), + // cancelling every `driver_cancellation`. + let _driver_guard = driver_guard; loop { #[expect( clippy::integer_division_remainder_used, @@ -141,32 +212,38 @@ impl BlockPublisherTrait for ZoneSdkPublisher { )] { tokio::select! { - // Drain external publish requests by calling the - // borrowing handle โ€” `&mut sequencer` is only - // available here. - Some((data_bounded, withdrawals)) = publish_rx.recv() => { - let data_byte_size = data_bounded.len(); - if withdrawals.is_empty() { - if let Err(e) = sequencer.handle() - .publish(data_bounded) - .context("Failed to publish block") { - warn!("zone-sdk publish failed: {e:?}"); - } - - info!("Published block with the size of {data_byte_size} bytes"); - } else { + // Drain external commands by calling the borrowing + // handle โ€” `&mut sequencer` is only available here. + Some(command) = command_rx.recv() => match command { + Command::Publish { inscription: data_bounded, withdrawals, resp: resp_tx } => { + let data_byte_size = data_bounded.len(); let withdraw_count = withdrawals.len(); - if let Err(e) = sequencer.handle() - .publish_atomic_withdraw(data_bounded, withdrawals) - .context("Failed to publish block with withdrawals") { - warn!("zone-sdk publish failed: {e:?}"); - } + let published = if withdrawals.is_empty() { + sequencer.handle() + .publish(data_bounded) + .context("Failed to publish block") + } else { + sequencer.handle() + .publish_atomic_withdraw(data_bounded, withdrawals) + .context("Failed to publish block with withdrawals") + }; - info!( - "Published block with the size of {data_byte_size} bytes and {withdraw_count} bridge withdrawals", - ); + let msg_result = published + .map(|(result, _checkpoint)| result.tx.inscription().this_msg); + match &msg_result { + Ok(_) if withdraw_count == 0 => { + info!("Published block with the size of {data_byte_size} bytes"); + } + Ok(_) => { + info!( + "Published block with the size of {data_byte_size} bytes and {withdraw_count} bridge withdrawals", + ); + } + Err(e) => warn!("zone-sdk publish failed: {e:?}"), + } + let _dontcare = resp_tx.send(msg_result); } - } + }, event = sequencer.next_event() => { let Some(event) = event else { continue; @@ -174,17 +251,32 @@ impl BlockPublisherTrait for ZoneSdkPublisher { match event { Event::BlocksProcessed { checkpoint, - channel_update: _, + channel_update, finalized, } => { on_checkpoint(checkpoint); + + let adopted = channel_update + .adopted + .iter() + .filter_map(block_from_inscription) + .collect(); + let orphaned = channel_update + .orphaned + .iter() + .map(orphan_inscription) + .filter_map(block_from_inscription) + .collect(); + + let mut finalized_blocks = Vec::new(); for op in finalized.into_iter().flat_map(|item| item.ops) { match op { FinalizedOp::Inscription(inscription) => { - if let Some(block_id) = - block_id_from_inscription(&inscription) + if let Some((msg, block)) = + block_from_inscription(&inscription) { - on_finalized_block(block_id); + on_finalized_block(block.header.block_id); + finalized_blocks.push((msg, block)); } } FinalizedOp::Deposit(deposit) => { @@ -195,8 +287,22 @@ impl BlockPublisherTrait for ZoneSdkPublisher { } } } + + on_follow(FollowUpdate { + adopted, + orphaned, + finalized: finalized_blocks, + }); + } + Event::Ready => {} + Event::TurnNotification { notification } => { + info!( + "Turn update: our_turn={}, starting_slot={:?}, ends_at_slot={:?}", + notification.our_turn_to_write, + notification.starting_slot, + notification.ends_at_slot + ); } - Event::Ready | Event::TurnNotification { .. } => {} } } } @@ -215,29 +321,46 @@ impl BlockPublisherTrait for ZoneSdkPublisher { channel_id: config.channel_id, indexer: ZoneIndexer::new(config.channel_id, node.clone()), node, - publish_tx, + command_tx, + turn_rx, + driver_cancellation, _drive_task: Arc::new(DriveTaskGuard(drive_task)), }) } - async fn publish_block(&self, block: &Block, withdrawals: Vec) -> Result<()> { + async fn publish_block(&self, block: &Block, withdrawals: Vec) -> Result { let data = borsh::to_vec(block).context("Failed to serialize block")?; let data_bounded: Inscription = data .try_into() .context("Block data exceeds maximum allowed size")?; - self.publish_tx - .send((data_bounded, withdrawals)) + let (resp_tx, resp_rx) = oneshot::channel(); + self.command_tx + .send(Command::Publish { + inscription: data_bounded, + withdrawals, + resp: resp_tx, + }) .await .map_err(|_closed| anyhow!("Drive task is no longer running"))?; - Ok(()) + resp_rx + .await + .map_err(|_closed| anyhow!("Drive task dropped the publish response"))? } fn channel_id(&self) -> ChannelId { self.channel_id } + fn is_our_turn(&self) -> bool { + self.turn_rx.borrow().our_turn_to_write + } + + fn driver_cancellation(&self) -> CancellationToken { + self.driver_cancellation.clone() + } + async fn channel_tip_slot(&self) -> Result> { Ok(self .node @@ -260,13 +383,87 @@ impl BlockPublisherTrait for ZoneSdkPublisher { } } -/// Deserialize inscription payload as a `Block` and return it's`block_id`. -/// Bad payloads are logged and skipped. -fn block_id_from_inscription(inscription: &InscriptionInfo) -> Option { +/// Deserialize an inscription payload into `(this_msg, Block)`. Bad payloads are +/// logged and skipped. +fn block_from_inscription(inscription: &InscriptionInfo) -> Option<(MsgId, Block)> { borsh::from_slice::(&inscription.payload) .inspect_err(|err| { warn!("Failed to deserialize block from inscription: {err:?}"); }) .ok() - .map(|block| block.header.block_id) + .map(|block| (inscription.this_msg, block)) +} + +/// The inscription carried by an orphaned tx (plain or atomic-withdraw bundle). +const fn orphan_inscription(orphan: &OrphanedTx) -> &InscriptionInfo { + match orphan { + OrphanedTx::Inscription(info) => info, + OrphanedTx::AtomicWithdraw(bundle) => &bundle.inscription, + } +} + +/// Signs a `ChannelConfig` op (accredited keys + rotation params) with +/// `signing_key` and posts it straight to the bedrock node. +/// +/// A standalone one-shot โ€” no running sequencer involved, so authorization is +/// holding the admin key: the L1 rejects non-admin signers. `Ok(())` means the +/// node accepted the transaction; channel acceptance is asynchronous and a +/// rejection only shows up in node logs and on-chain behavior. +pub async fn post_channel_config( + config: &BedrockConfig, + signing_key: &Ed25519Key, + keys: Vec, + posting_timeframe: u32, + posting_timeout: u32, + configuration_threshold: u16, + withdraw_threshold: u16, +) -> Result<()> { + ensure!(!keys.is_empty(), "Channel key list must not be empty"); + for (name, threshold) in [ + ("configuration_threshold", configuration_threshold), + ("withdraw_threshold", withdraw_threshold), + ] { + ensure!( + threshold >= 1 && usize::from(threshold) <= keys.len(), + "{name} must be between 1 and the key count ({}), got {threshold}", + keys.len() + ); + } + ensure!( + posting_timeframe > 0 && posting_timeout >= posting_timeframe, + "posting_timeframe must be nonzero and posting_timeout at least as long, \ + got {posting_timeframe} and {posting_timeout}" + ); + + let keys = Keys::try_from(keys).map_err(|err| anyhow!("Invalid channel key list: {err}"))?; + let config_op = ChannelConfigOp { + channel: config.channel_id, + keys, + posting_timeframe: SlotTimeframe::from(posting_timeframe), + posting_timeout: SlotTimeout::from(posting_timeout), + configuration_threshold, + withdraw_threshold, + }; + + let mantle_tx = MantleTx([Op::ChannelConfig(config_op)].into()); + let tx_hash = mantle_tx.hash(); + // The admin key is `keys[0]`, hence signature index 0. + let signature = IndexedSignature::new( + 0, + signing_key.sign_payload(tx_hash.as_signing_bytes().as_ref()), + ); + let proof = ChannelMultiSigProof::new(vec![signature]) + .map_err(|err| anyhow!("Failed to assemble channel multi-sig proof: {err:?}"))?; + let signed_tx = SignedMantleTx { + ops_proofs: vec![OpProof::ChannelMultiSigProof(proof)], + mantle_tx, + }; + + let node = NodeHttpClient::new( + CommonHttpClient::new(config.auth.clone().map(Into::into)), + config.node_url.clone(), + ); + node.post_transaction(signed_tx) + .await + .context("Failed to post channel config transaction") } diff --git a/lez/sequencer/core/src/block_store.rs b/lez/sequencer/core/src/block_store.rs index 3a2de375..37394352 100644 --- a/lez/sequencer/core/src/block_store.rs +++ b/lez/sequencer/core/src/block_store.rs @@ -218,6 +218,16 @@ impl SequencerStore { pub fn is_deposit_event_submitted(&self, deposit_op_id: HashType) -> DbResult { self.dbio.is_deposit_event_submitted(deposit_op_id) } + + /// Marks the given deposit events submitted in `block_id`, in one write. + pub fn mark_deposit_events_submitted( + &self, + deposit_op_ids: &[HashType], + submitted_block_id: u64, + ) -> DbResult<()> { + self.dbio + .mark_deposit_events_submitted(deposit_op_ids, submitted_block_id) + } } pub(crate) fn block_to_transactions_map(block: &Block) -> HashMap { diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 9d1329a9..9ceff359 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -1,17 +1,23 @@ -use std::{path::Path, sync::Arc, time::Instant}; +use std::{ + path::Path, + sync::{Arc, Mutex}, + time::Instant, +}; use anyhow::{Context as _, Result, anyhow}; use borsh::BorshDeserialize; -use chain_consistency::{Anchor, AnchorConsistencyCheck, ChainConsistency, ChainMismatch, Tip}; +use chain_state::{ + AcceptOutcome, Anchor, AnchorConsistencyCheck, ChainConsistency, ChainMismatch, ChainState, Tip, +}; use common::{ HashType, - block::{BedrockStatus, Block, HashableBlockData}, + block::{BedrockStatus, Block, BlockMeta, HashableBlockData}, transaction::{LeeTransaction, clock_invocation}, }; use config::{GenesisAction, SequencerConfig}; use futures::StreamExt as _; use itertools::Itertools as _; -use lee::{AccountId, PublicTransaction, V03State, public_transaction::Message}; +use lee::{AccountId, PublicTransaction, public_transaction::Message}; use lee_core::GENESIS_BLOCK_ID; use log::{error, info, warn}; use logos_blockchain_key_management_system_service::keys::{ED25519_SECRET_KEY_SIZE, Ed25519Key}; @@ -30,7 +36,7 @@ use storage::sequencer::{ }; use crate::{ - block_publisher::{BlockPublisherTrait, ZoneSdkPublisher}, + block_publisher::{BlockPublisherTrait, MsgId, ZoneSdkPublisher}, block_store::SequencerStore, }; @@ -56,18 +62,13 @@ struct DepositMetadata { recipient_id: lee::AccountId, } -impl DepositMetadata { - fn decode(bytes: &[u8]) -> Result { - Self::try_from_slice(bytes) - } -} - pub struct SequencerCore { - state: lee::V03State, + /// Two-tier chain state: production builds on its head; the publisher's + /// `on_follow` sink feeds adopted/orphaned/finalized peer blocks into it. + chain: Arc>, store: SequencerStore, mempool: MemPool<(TransactionOrigin, LeeTransaction)>, sequencer_config: SequencerConfig, - chain_height: u64, block_publisher: BP, } @@ -120,14 +121,66 @@ impl SequencerCore { } } + /// Rebuilds the two-tier [`ChainState`]: the final tier from the persisted + /// final snapshot (pre-genesis state when absent), the head tier by replaying + /// every stored block above it, so a post-restart orphan can still revert. + fn restore_chain_state( + config: &SequencerConfig, + store: &SequencerStore, + stored_head_state: &lee::V03State, + ) -> ChainState { + let final_snapshot = store + .dbio() + .get_final_snapshot() + .expect("Failed to read final snapshot from store"); + let (final_state, final_tip) = match final_snapshot { + Some((state, meta)) => (state, Some(Tip::from(meta))), + // Nothing finalized yet: replay the whole stored chain. + None => (build_initial_state(config), None), + }; + let boundary = final_tip.as_ref().map_or(0, |tip| tip.block_id); + + let mut head_blocks = store + .get_all_blocks() + .filter_ok(|block| block.header.block_id > boundary) + .collect::, _>>() + .expect("Failed to read blocks from store while restoring chain state"); + head_blocks.sort_unstable_by_key(|block| block.header.block_id); + + let mut chain = ChainState::from_final(final_state, final_tip); + for block in head_blocks { + let block_id = block.header.block_id; + chain.restore_head_block(block).unwrap_or_else(|err| { + panic!("Stored block {block_id} does not replay while restoring chain state: {err}") + }); + } + + // The replayed head must reproduce the persisted state, else store + // and config disagree (e.g. edited genesis actions). + assert!( + chain.head_state() == stored_head_state, + "Persisted state does not match the replayed chain; reset the store or restore the original config" + ); + + chain + } + pub async fn start_from_config( config: SequencerConfig, ) -> (Self, MemPoolHandle<(TransactionOrigin, LeeTransaction)>) { let bedrock_signing_key = load_or_create_signing_key(&config.home.join("bedrock_signing_key")) .expect("Failed to load or create bedrock signing key"); + info!( + "Bedrock signing public key: {}", + hex::encode(bedrock_signing_key.public_key().to_bytes()) + ); - let (mut store, mut state) = Self::open_or_create_store(&config); + let (store, state) = Self::open_or_create_store(&config); + + let chain = Arc::new(Mutex::new(Self::restore_chain_state( + &config, &store, &state, + ))); let initial_checkpoint = store .get_zone_checkpoint() @@ -146,6 +199,7 @@ impl SequencerCore { Self::on_finalized_block(store.dbio()), Self::on_deposit_event(store.dbio(), mempool_handle.clone()), Self::on_withdraw_event(store.dbio()), + Self::on_follow(store.dbio(), Arc::clone(&chain), mempool_handle.clone()), ) .await .expect("Failed to initialize Block Publisher"); @@ -163,15 +217,15 @@ impl SequencerCore { // Before producing, verify our local state still belongs to the chain // the channel serves and replay any channel blocks we are missing // (e.g. from other sequencers). - let chanel_was_empty = - Self::verify_and_reconstruct(&block_publisher, &mut store, &mut state, is_fresh_start) + let channel_was_empty = + Self::verify_and_reconstruct(&block_publisher, &store, &chain, is_fresh_start) .await .expect("Failed to verify/reconstruct sequencer state from Bedrock"); // Publish our blocks only when bootstrapping an empty channel. If the // channel already has blocks (another sequencer bootstrapped it), we // adopted them during reconstruction instead. - if is_fresh_start && chanel_was_empty { + if is_fresh_start && channel_was_empty { let mut pending_blocks = store .get_all_blocks() .filter_ok(|block| matches!(block.bedrock_status, BedrockStatus::Pending)) @@ -199,16 +253,10 @@ impl SequencerCore { } } - let latest_block_meta = store - .latest_block_meta() - .expect("Failed to read latest block meta from store") - .expect("Sequencer store should have at least the genesis block after reconstruction"); - let sequencer_core = Self { - state, + chain, store, mempool, - chain_height: latest_block_meta.id, sequencer_config: config, block_publisher, }; @@ -224,8 +272,8 @@ impl SequencerCore { /// Returns whatever channel was empty or not. async fn verify_and_reconstruct( publisher: &BP, - store: &mut SequencerStore, - state: &mut V03State, + store: &SequencerStore, + chain: &Mutex, is_fresh_start: bool, ) -> Result { let anchor_record = store @@ -322,7 +370,11 @@ impl SequencerCore { ) })?; channel_is_empty = false; - Self::apply_reconstructed_block(store, state, &block, slot)?; + // Locked per message (not across the stream `await`): concurrent + // follow events interleave safely โ€” both paths apply idempotently + // and persist under this same lock. + let mut chain = chain.lock().expect("chain state mutex poisoned"); + Self::apply_reconstructed_block(store, &mut chain, &block, slot)?; } Ok(channel_is_empty) @@ -332,8 +384,8 @@ impl SequencerCore { /// blocks we already hold (verifying their hash), a validated continuation /// for new ones. Advances the persisted anchor to the block's slot. fn apply_reconstructed_block( - store: &mut SequencerStore, - state: &mut V03State, + store: &SequencerStore, + chain: &mut ChainState, block: &Block, slot: Slot, ) -> Result<()> { @@ -379,21 +431,31 @@ impl SequencerCore { } } - // New continuation: validate it chains onto the tip, then apply. - let cc_tip = tip.as_ref().map(|tip| Tip { - block_id: tip.id, - hash: tip.hash, - }); - chain_consistency::validate_against_tip(cc_tip, block).map_err(|err| { - anyhow!( - "Channel block {block_id} does not extend local tip {:?}: {err}", - tip.map(|tip| tip.id) - ) - })?; + // New continuation: channel history is finalized, so it goes through + // the final tier โ€” validation happens inside `apply_finalized`. + match chain.apply_finalized(MsgId::from(block.header.hash.0), block, slot) { + AcceptOutcome::Applied | AcceptOutcome::AlreadyApplied => {} + AcceptOutcome::Parked(err) | AcceptOutcome::RetryableFailure(err) => { + return Err(anyhow!( + "Channel block {block_id} does not extend local tip {:?}: {err}", + tip.map(|tip| tip.id) + )); + } + } - let mut scratch = state.clone(); - chain_consistency::apply_block(block, &mut scratch) - .map_err(|err| anyhow!("Failed to apply channel block {block_id}: {err}"))?; + // Persist like the follow path: the tip meta stays pinned to the head + // tip even when the reconstructed block lands below it. + let head_tip = chain.head_tip().map(|head| BlockMeta::from(&head)); + let final_meta = chain.final_tip().map(|meta| BlockMeta::from(&meta)); + store + .dbio() + .store_followed_blocks( + &[(block, true)], + head_tip.as_ref(), + chain.head_state(), + final_meta.as_ref().map(|meta| (chain.final_state(), meta)), + ) + .context("Failed to persist reconstructed block")?; // Mark the deposits' pending records submitted so the production-time // guard drops the mints cold-start backfill re-queued for them. Withdraw @@ -406,11 +468,10 @@ impl SequencerCore { .iter() .filter_map(extract_bridge_deposit_id) .collect(); - store - .update(block, &deposit_event_ids, Vec::new(), &scratch) - .context("Failed to persist reconstructed block")?; - *state = scratch; + .mark_deposit_events_submitted(&deposit_event_ids, block_id) + .context("Failed to mark reconstructed deposits submitted")?; + store .set_zone_anchor(&record) .context("Failed to persist zone anchor")?; @@ -547,6 +608,17 @@ impl SequencerCore { }) } + /// Publisher sink adapter over [`apply_follow_update`]. + fn on_follow( + dbio: Arc, + chain: Arc>, + mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>, + ) -> block_publisher::OnFollowSink { + Box::new(move |update: block_publisher::FollowUpdate| { + apply_follow_update(&dbio, &chain, &mempool_handle, update); + }) + } + /// Produces a new block from mempool transactions and publishes it via zone-sdk. pub async fn produce_new_block(&mut self) -> Result { let BlockWithMeta { @@ -563,26 +635,77 @@ impl SequencerCore { .collect::>() .context("Failed to build reconciliation keys for block withdrawals")?; - self.block_publisher + let this_msg = self + .block_publisher .publish_block(&block, withdrawals) .await .context("Failed to publish block to Bedrock")?; - self.store.update( + self.record_produced_block( + this_msg, &block, &deposit_event_ids, withdrawal_reconciliation_keys, - &self.state, )?; - Ok(self.chain_height) + Ok(block.header.block_id) + } + + /// Applies our own freshly-published block to the head with the [`MsgId`] the + /// publish assigned it, so the head advances and the later adopted + /// redelivery dedups, then persists it. + /// + /// Persistence is gated on the block actually becoming the head: if a peer + /// block won this height while we were publishing (`AlreadyApplied`, or + /// `Parked` when the head reorged to a different parent), the canonical + /// block is persisted by the follow path instead, and our invalidated + /// inscription comes back via `orphaned`. + fn record_produced_block( + &mut self, + this_msg: MsgId, + block: &Block, + deposit_event_ids: &[HashType], + withdrawal_reconciliation_keys: Vec, + ) -> Result<()> { + let mut chain = self.chain.lock().expect("chain state mutex poisoned"); + match chain.apply_adopted(this_msg, block) { + AcceptOutcome::Applied => { + // Persisted under the lock so disk writes land in apply order + // with the follow path. + self.store.update( + block, + deposit_event_ids, + withdrawal_reconciliation_keys, + chain.head_state(), + )?; + } + AcceptOutcome::AlreadyApplied => { + warn!( + "Produced block {} lost a competing-write race, skipping persistence", + block.header.block_id + ); + } + AcceptOutcome::Parked(err) | AcceptOutcome::RetryableFailure(err) => { + warn!( + "Produced block {} no longer chains on the head, skipping persistence: {err}", + block.header.block_id + ); + } + } + + Ok(()) } /// Validates and applies a single mempool transaction to the current state. /// Returns `Ok(true)` if the transaction was valid and applied, `Ok(false)` if /// it was skipped due to validation failure. + #[expect( + clippy::too_many_arguments, + reason = "splitting the produce-path accumulators into a struct buys nothing" + )] fn apply_mempool_transaction( - &mut self, + store: &SequencerStore, + state: &mut lee::V03State, origin: TransactionOrigin, tx: &LeeTransaction, block_height: u64, @@ -593,11 +716,7 @@ impl SequencerCore { let tx_hash = tx.hash(); match origin { TransactionOrigin::User => { - let validated_diff = match tx.validate_on_state( - &self.state, - block_height, - timestamp, - ) { + let validated_diff = match tx.validate_on_state(state, block_height, timestamp) { Ok(diff) => diff, Err(err) => { error!( @@ -611,7 +730,7 @@ impl SequencerCore { withdrawals.push(withdraw_data); } - self.state.apply_state_diff(validated_diff); + state.apply_state_diff(validated_diff); } TransactionOrigin::Sequencer => { let LeeTransaction::Public(public_tx) = tx else { @@ -619,8 +738,7 @@ impl SequencerCore { }; if let Some(deposit_op_id) = extract_bridge_deposit_id(tx) { - if self - .store + if store .is_deposit_event_submitted(deposit_op_id) .context("Failed to check whether deposit was already submitted")? { @@ -630,7 +748,7 @@ impl SequencerCore { deposit_event_ids.push(deposit_op_id); } - self.state + state .transition_from_public_transaction(public_tx, block_height, timestamp) .context("Failed to execute sequencer-generated transaction")?; } @@ -643,7 +761,18 @@ impl SequencerCore { fn build_block_from_mempool(&mut self) -> Result { let now = Instant::now(); - let new_block_height = self.next_block_id(); + // Build on the head: its tip is the parent, its state the validation base. + let (prev_block_hash, new_block_height, mut working_state) = { + let chain = self.chain.lock().expect("chain state mutex poisoned"); + let tip = chain.head_tip(); + let height = tip.as_ref().map_or(GENESIS_BLOCK_ID, |head| { + head.block_id + .checked_add(1) + .expect("block id should not overflow") + }); + let prev = tip.map_or(HashType([0; 32]), |head| head.hash); + (prev, height, chain.head_state().clone()) + }; let mut valid_transactions = Vec::new(); let mut deposit_event_ids = Vec::new(); @@ -652,12 +781,6 @@ impl SequencerCore { let max_block_size = usize::try_from(self.sequencer_config.max_block_size.as_u64()) .expect("`max_block_size` should fit into usize"); - let latest_block_meta = self - .store - .latest_block_meta() - .context("Failed to get latest block meta from store")? - .context("Sequencer store should have at least the genesis block")?; - let new_block_timestamp = u64::try_from(chrono::Utc::now().timestamp_millis()) .expect("Timestamp must be positive"); @@ -676,7 +799,7 @@ impl SequencerCore { let temp_hashable_data = HashableBlockData { block_id: new_block_height, transactions: temp_valid_transactions, - prev_block_hash: latest_block_meta.hash, + prev_block_hash, timestamp: new_block_timestamp, }; @@ -693,7 +816,9 @@ impl SequencerCore { break; } - if self.apply_mempool_transaction( + if Self::apply_mempool_transaction( + &self.store, + &mut working_state, origin, &tx, new_block_height, @@ -709,7 +834,7 @@ impl SequencerCore { } } - self.state + working_state .transition_from_public_transaction(&clock_tx, new_block_height, new_block_timestamp) .context("Clock transaction failed. Aborting block production.")?; valid_transactions.push(clock_lee_tx); @@ -717,7 +842,7 @@ impl SequencerCore { let hashable_data = HashableBlockData { block_id: new_block_height, transactions: valid_transactions, - prev_block_hash: latest_block_meta.hash, + prev_block_hash, timestamp: new_block_timestamp, }; @@ -725,8 +850,6 @@ impl SequencerCore { .clone() .into_pending_block(self.store.signing_key()); - self.chain_height = new_block_height; - log::info!( "Created block with {} transactions in {} seconds", hashable_data.transactions.len(), @@ -740,16 +863,27 @@ impl SequencerCore { }) } - pub const fn state(&self) -> &lee::V03State { - &self.state + /// Reads the current head state under the lock without cloning it, so callers + /// reuse `V03State`'s own API (accounts, nonces, proofs) with no whole-state copy. + pub fn with_state(&self, f: impl FnOnce(&lee::V03State) -> R) -> R { + f(self + .chain + .lock() + .expect("chain state mutex poisoned") + .head_state()) } pub const fn block_store(&self) -> &SequencerStore { &self.store } - pub const fn chain_height(&self) -> u64 { - self.chain_height + #[must_use] + pub fn chain_height(&self) -> u64 { + self.chain + .lock() + .expect("chain state mutex poisoned") + .head_tip() + .map_or(0, |tip| tip.block_id) } pub const fn sequencer_config(&self) -> &SequencerConfig { @@ -786,10 +920,17 @@ impl SequencerCore { &self.block_publisher } - fn next_block_id(&self) -> u64 { - self.chain_height - .checked_add(1) - .unwrap_or_else(|| panic!("Max block height reached: {}", self.chain_height)) + /// Whether this sequencer is currently authorized to write to the channel. + #[must_use] + pub fn is_our_turn(&self) -> bool { + self.block_publisher.is_our_turn() + } + + /// Shared handle to the two-tier follow state, for tests to drive the + /// follow path directly. + #[cfg(all(test, feature = "mock"))] + fn chain(&self) -> Arc> { + Arc::clone(&self.chain) } } @@ -799,6 +940,105 @@ struct BlockWithMeta { withdrawals: Vec, } +/// Feed one channel delta into the follow state and mirror it to the store: +/// revert orphaned, then apply and persist adopted and finalized blocks. +/// Production builds on this same head. Wired to the publisher via +/// [`SequencerCore::on_follow`]; a free function so tests can drive it directly. +/// +/// TODO: unlike the indexer's ingest loop, this path does not retry +/// `is_retryable` (transient) apply failures โ€” a failed block just parks and +/// relies on a valid successor or a restart. `ChainState` never emits +/// `AcceptOutcome::RetryableFailure` yet; adding retry parity here is a +/// follow-up. +fn apply_follow_update( + dbio: &RocksDBIO, + chain: &Mutex, + mempool_handle: &MemPoolHandle<(TransactionOrigin, LeeTransaction)>, + update: block_publisher::FollowUpdate, +) { + let block_publisher::FollowUpdate { + adopted, + orphaned, + finalized, + } = update; + + // The lock is held across the persist below so disk writes land in apply + // order โ€” the produce path persists under this same lock. + let resubmit_txs = { + let mut chain = chain.lock().expect("chain state mutex poisoned"); + + // User txs of orphaned blocks, returned to the mempool below. + let resubmit_txs: Vec = orphaned + .iter() + .flat_map(|(_, block)| resubmittable_txs(block)) + .collect(); + + // Outcomes align with `adopted`. + let outcomes = chain.apply_channel_update(&orphaned, &adopted); + let mut to_persist: Vec<(&Block, bool)> = adopted + .iter() + .zip(&outcomes) + .filter(|(_, outcome)| matches!(outcome, AcceptOutcome::Applied)) + .map(|((_, block), _)| (block, false)) + .collect(); + + let mut final_advanced = false; + for (this_msg, block) in &finalized { + // FIXME: thread the finalized inscription's L1 slot once the + // sdk surfaces it; only used for the invalid-finalized stall. + if matches!( + chain.apply_finalized(*this_msg, block, Slot::from(0)), + AcceptOutcome::Applied + ) { + to_persist.push((block, true)); + final_advanced = true; + } + } + // Snapshot the advanced final tier so a restart re-anchors on it. + let final_meta = final_advanced.then(|| { + let tip = chain.final_tip().expect("advanced final tier has a tip"); + BlockMeta::from(&tip) + }); + let head_tip = chain.head_tip().map(|tip| BlockMeta::from(&tip)); + + // One atomic write for the whole update: blocks, tip meta and the + // state after the last block land together, so a crash can never + // leave the stored state ahead of the stored blocks. A persist + // failure is fatal: the in-memory chain has already advanced, and + // continuing would leave a permanent gap in the store. + // + // The `panic!` ends the drive task, whose cancellation halts the node. + // + // TODO: the zone-sdk checkpoint is persisted by `on_checkpoint` + // before this write; a crash in between resumes past these blocks + // without them ever landing in the store. Full `BlocksProcessed` + // atomicity (checkpoint + blocks + state in one batch, per the sdk's + // event contract) is a follow-up. + dbio.store_followed_blocks( + &to_persist, + head_tip.as_ref(), + chain.head_state(), + final_meta.as_ref().map(|meta| (chain.final_state(), meta)), + ) + .unwrap_or_else(|err| panic!("Failed to persist followed blocks: {err:#}")); + + resubmit_txs + }; + + // Rebuild orphaned work: return its user txs to the mempool so the + // next on-turn production re-includes them on the new head. + // + // We use [`try_push`] here because this is called from the + // publisher's drive task, and only the block production drains the mempool. + // A blocking push on a full mempool would deadlock here. + for tx in resubmit_txs { + let tx_hash = tx.hash(); + if let Err(err) = mempool_handle.try_push((TransactionOrigin::User, tx)) { + warn!("Dropping orphaned transaction {tx_hash} on resubmit: {err}"); + } + } +} + /// Checks the database for any pending deposit events that have not yet been marked as submitted in /// a block, and re-queues them in the mempool in a separate async task for inclusion in the next /// block. @@ -848,41 +1088,15 @@ fn replay_unfulfilled_deposit_events( }); } -/// Builds the initial genesis state from `testnet_initial_state` plus configured genesis -/// transactions. Returns the final state and the list of [`LeeTransaction`]s that should be -/// committed to the genesis block so external observers can replay them. -fn build_genesis_state(config: &SequencerConfig) -> (lee::V03State, Vec) { +/// The pre-genesis state: `testnet_initial_state` plus accounts seeded outside +/// any transaction (bridge-lock holdings, the cross-zone inbox config). +fn build_initial_state(config: &SequencerConfig) -> lee::V03State { #[cfg(not(feature = "testnet"))] let mut state = testnet_initial_state::initial_state(); #[cfg(feature = "testnet")] let mut state = testnet_initial_state::initial_state_testnet(); - let genesis_txs = config - .genesis - .iter() - .filter_map(|genesis_tx| match genesis_tx { - GenesisAction::SupplyAccount { - account_id, - balance, - } => Some(build_supply_account_genesis_transaction( - account_id, *balance, - )), - GenesisAction::SupplyBridgeAccount { balance } => { - Some(build_supply_bridge_account_genesis_transaction(*balance)) - } - // Force-inserted below: bridge_lock has no mint transaction. - GenesisAction::SupplyBridgeLockHolding { .. } => None, - }) - .chain(std::iter::once(clock_invocation(0))) - .inspect(|tx| { - state - .transition_from_public_transaction(tx, GENESIS_BLOCK_ID, 0) - .expect("Failed to execute genesis transaction"); - }) - .map(LeeTransaction::Public) - .collect(); - // Seed bridge-lock holder balances directly: they are not produced by any tx. for action in &config.genesis { if let GenesisAction::SupplyBridgeLockHolding { holder, amount } = action { @@ -900,6 +1114,41 @@ fn build_genesis_state(config: &SequencerConfig) -> (lee::V03State, Vec (lee::V03State, Vec) { + let mut state = build_initial_state(config); + + let genesis_txs = config + .genesis + .iter() + .filter_map(|genesis_tx| match genesis_tx { + GenesisAction::SupplyAccount { + account_id, + balance, + } => Some(build_supply_account_genesis_transaction( + account_id, *balance, + )), + GenesisAction::SupplyBridgeAccount { balance } => { + Some(build_supply_bridge_account_genesis_transaction(*balance)) + } + // Force-inserted in `build_initial_state`: bridge_lock has no mint transaction. + GenesisAction::SupplyBridgeLockHolding { .. } => None, + }) + .chain(std::iter::once(clock_invocation(0))) + .inspect(|tx| { + state + .transition_from_public_transaction(tx, GENESIS_BLOCK_ID, 0) + .expect("Failed to execute genesis transaction"); + }) + .map(LeeTransaction::Public) + .collect(); + (state, genesis_txs) } @@ -964,7 +1213,7 @@ fn pending_deposit_event_record(deposit: &DepositInfo) -> PendingDepositEventRec } fn build_bridge_deposit_tx_from_event(event: &PendingDepositEventRecord) -> Result { - let metadata = DepositMetadata::decode(&event.metadata) + let metadata = DepositMetadata::try_from_slice(&event.metadata) .context("Failed to decode finalized Bedrock deposit metadata")?; let bridge_program_id = programs::bridge().id(); @@ -992,6 +1241,26 @@ fn build_bridge_deposit_tx_from_event(event: &PendingDepositEventRecord) -> Resu ))) } +/// User transactions of an orphaned block to return to the mempool: everything +/// except the trailing clock tx, sequencer-generated bridge deposits (replayed +/// from their own bedrock events) and sequencer-only cross-zone txs (replayed +/// by the watcher; the ingress guard rejects them as `User`). +fn resubmittable_txs(block: &Block) -> Vec { + let Some((_clock, rest)) = block.body.transactions.split_last() else { + return Vec::new(); + }; + rest.iter() + .filter(|tx| extract_bridge_deposit_id(tx).is_none() && !is_sequencer_only_tx(tx)) + .cloned() + .collect() +} + +#[must_use] +fn is_sequencer_only_tx(tx: &LeeTransaction) -> bool { + matches!(tx, LeeTransaction::Public(tx) + if is_sequencer_only_program(tx.message().program_id)) +} + #[must_use] fn extract_bridge_deposit_id(tx: &LeeTransaction) -> Option { let LeeTransaction::Public(tx) = tx else { @@ -1081,7 +1350,7 @@ fn withdraw_event_reconciliation_key( } /// Load signing key from file or generate a new one if it doesn't exist. -fn load_or_create_signing_key(path: &Path) -> Result { +pub fn load_or_create_signing_key(path: &Path) -> Result { if path.exists() { let key_bytes = std::fs::read(path)?; diff --git a/lez/sequencer/core/src/mock.rs b/lez/sequencer/core/src/mock.rs index f8349fac..4f2552b7 100644 --- a/lez/sequencer/core/src/mock.rs +++ b/lez/sequencer/core/src/mock.rs @@ -3,13 +3,14 @@ use std::time::Duration; use anyhow::Result; use common::block::Block; use futures::Stream; -use logos_blockchain_core::mantle::ops::channel::ChannelId; +use logos_blockchain_core::mantle::ops::channel::{ChannelId, MsgId}; use logos_blockchain_key_management_system_service::keys::Ed25519Key; use logos_blockchain_zone_sdk::{Slot, ZoneMessage, sequencer::WithdrawArg}; +use tokio_util::sync::CancellationToken; use crate::{ block_publisher::{ - BlockPublisherTrait, CheckpointSink, FinalizedBlockSink, OnDepositEventSink, + BlockPublisherTrait, CheckpointSink, FinalizedBlockSink, OnDepositEventSink, OnFollowSink, OnWithdrawEventSink, SequencerCheckpoint, }, config::BedrockConfig, @@ -20,6 +21,8 @@ pub type SequencerCoreWithMockClients = crate::SequencerCore #[derive(Clone)] pub struct MockBlockPublisher { channel_id: ChannelId, + // Never cancelled: the mock driver never dies. + driver_cancellation: CancellationToken, /// Canned channel frontier returned by [`Self::channel_tip_slot`]. tip_slot: Option, /// Canned finalized channel history returned by [`Self::read_channel_after`]. @@ -31,13 +34,14 @@ impl MockBlockPublisher { /// and consistency tests. The default (via [`BlockPublisherTrait::new`]) /// serves an empty channel. #[must_use] - pub const fn with_canned_channel( + pub fn with_canned_channel( channel_id: ChannelId, tip_slot: Option, messages: Vec<(ZoneMessage, Slot)>, ) -> Self { Self { channel_id, + driver_cancellation: CancellationToken::new(), tip_slot, messages, } @@ -54,9 +58,11 @@ impl BlockPublisherTrait for MockBlockPublisher { _on_finalized_block: FinalizedBlockSink, _on_deposit_event: OnDepositEventSink, _on_withdraw_event: OnWithdrawEventSink, + _on_follow: OnFollowSink, ) -> Result { Ok(Self { channel_id: config.channel_id, + driver_cancellation: CancellationToken::new(), tip_slot: None, messages: Vec::new(), }) @@ -64,16 +70,27 @@ impl BlockPublisherTrait for MockBlockPublisher { async fn publish_block( &self, - _block: &Block, + block: &Block, _bridge_withdrawals: Vec, - ) -> Result<()> { - Ok(()) + ) -> Result { + // Deterministic per-block id so head dedup behaves in tests. + // + // TODO: should we allow more "mockability" here? + Ok(MsgId::from(block.header.hash.0)) } fn channel_id(&self) -> ChannelId { self.channel_id } + fn is_our_turn(&self) -> bool { + true + } + + fn driver_cancellation(&self) -> CancellationToken { + self.driver_cancellation.clone() + } + async fn channel_tip_slot(&self) -> Result> { Ok(self.tip_slot) } diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index b398e160..11c54b50 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -4,7 +4,7 @@ use std::{pin::pin, time::Duration}; use common::{ HashType, - block::HashableBlockData, + block::{BedrockStatus, HashableBlockData}, test_utils::sequencer_sign_key_for_testing, transaction::{LeeTransaction, clock_invocation}, }; @@ -22,19 +22,21 @@ use lee_core::{ account::{AccountWithMetadata, Nonce}, program::PdaSeed, }; -use logos_blockchain_core::mantle::ops::channel::ChannelId; +use logos_blockchain_core::mantle::ops::channel::{ChannelId, MsgId}; use mempool::MemPoolHandle; use storage::sequencer::sequencer_cells::PendingDepositEventRecord; use tempfile::tempdir; use testnet_initial_state::{initial_pub_accounts_private_keys, initial_public_user_accounts}; use crate::{ - TransactionOrigin, + TransactionOrigin, apply_follow_update, + block_publisher::FollowUpdate, block_store::SequencerStore, - build_genesis_state, + build_bridge_deposit_tx_from_event, build_genesis_state, config::{BedrockConfig, SequencerConfig}, is_sequencer_only_program, mock::SequencerCoreWithMockClients, + resubmittable_txs, }; mod reconstruction; @@ -148,14 +150,14 @@ async fn start_from_config() { let (sequencer, _mempool_handle) = SequencerCoreWithMockClients::start_from_config(config.clone()).await; - assert_eq!(sequencer.chain_height, 1); + assert_eq!(sequencer.chain_height(), 1); assert_eq!(sequencer.sequencer_config.max_num_tx_in_block, 10); let acc1_account_id = initial_public_user_accounts()[0].account_id; let acc2_account_id = initial_public_user_accounts()[1].account_id; - let balance_acc_1 = sequencer.state.get_account_by_id(acc1_account_id).balance; - let balance_acc_2 = sequencer.state.get_account_by_id(acc2_account_id).balance; + let balance_acc_1 = sequencer.with_state(|s| s.get_account_by_id(acc1_account_id).balance); + let balance_acc_2 = sequencer.with_state(|s| s.get_account_by_id(acc2_account_id).balance); assert_eq!(10000, balance_acc_1); assert_eq!(20000, balance_acc_2); @@ -188,7 +190,7 @@ async fn start_from_config_opens_existing_db_if_it_exists() { let (sequencer, _mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; - assert_eq!(sequencer.chain_height, 1); + assert_eq!(sequencer.chain_height(), 1); assert!(sequencer.store.latest_block_meta().is_ok()); } @@ -297,7 +299,7 @@ async fn transaction_pre_check_native_transfer_valid() { #[tokio::test] async fn transaction_pre_check_native_transfer_other_signature() { - let (mut sequencer, _mempool_handle) = common_setup().await; + let (sequencer, _mempool_handle) = common_setup().await; let acc1 = initial_public_user_accounts()[0].account_id; let acc2 = initial_public_user_accounts()[1].account_id; @@ -311,7 +313,15 @@ async fn transaction_pre_check_native_transfer_other_signature() { let tx = tx.transaction_stateless_check().unwrap(); // Signature is not from sender. Execution fails - let result = tx.execute_check_on_state(&mut sequencer.state, 0, 0); + let result = tx.execute_check_on_state( + sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .head_state_mut(), + 0, + 0, + ); assert!(matches!( result, @@ -321,7 +331,7 @@ async fn transaction_pre_check_native_transfer_other_signature() { #[tokio::test] async fn transaction_pre_check_native_transfer_sent_too_much() { - let (mut sequencer, _mempool_handle) = common_setup().await; + let (sequencer, _mempool_handle) = common_setup().await; let acc1 = initial_public_user_accounts()[0].account_id; let acc2 = initial_public_user_accounts()[1].account_id; @@ -337,9 +347,15 @@ async fn transaction_pre_check_native_transfer_sent_too_much() { // Passed pre-check assert!(result.is_ok()); - let result = result - .unwrap() - .execute_check_on_state(&mut sequencer.state, 0, 0); + let result = result.unwrap().execute_check_on_state( + sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .head_state_mut(), + 0, + 0, + ); let is_failed_at_balance_mismatch = matches!( result.err().unwrap(), lee::error::LeeError::ProgramExecutionFailed(_) @@ -350,7 +366,7 @@ async fn transaction_pre_check_native_transfer_sent_too_much() { #[tokio::test] async fn transaction_execute_native_transfer() { - let (mut sequencer, _mempool_handle) = common_setup().await; + let (sequencer, _mempool_handle) = common_setup().await; let acc1 = initial_public_user_accounts()[0].account_id; let acc2 = initial_public_user_accounts()[1].account_id; @@ -361,11 +377,19 @@ async fn transaction_execute_native_transfer() { acc1, 0, acc2, 100, &sign_key1, ); - tx.execute_check_on_state(&mut sequencer.state, 0, 0) - .unwrap(); + tx.execute_check_on_state( + sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .head_state_mut(), + 0, + 0, + ) + .unwrap(); - let bal_from = sequencer.state.get_account_by_id(acc1).balance; - let bal_to = sequencer.state.get_account_by_id(acc2).balance; + let bal_from = sequencer.with_state(|s| s.get_account_by_id(acc1).balance); + let bal_to = sequencer.with_state(|s| s.get_account_by_id(acc2).balance); assert_eq!(bal_from, 9900); assert_eq!(bal_to, 20100); @@ -402,7 +426,7 @@ async fn push_tx_into_mempool_blocks_until_mempool_is_full() { #[tokio::test] async fn build_block_from_mempool() { let (mut sequencer, mempool_handle) = common_setup().await; - let genesis_height = sequencer.chain_height; + let genesis_height = sequencer.chain_height(); let tx = common::test_utils::produce_dummy_empty_transaction(); mempool_handle @@ -412,7 +436,8 @@ async fn build_block_from_mempool() { let result = sequencer.build_block_from_mempool(); assert!(result.is_ok()); - assert_eq!(sequencer.chain_height, genesis_height + 1); + // Building itself does not advance the head; only apply-after-publish does. + assert_eq!(sequencer.chain_height(), genesis_height); } #[tokio::test] @@ -444,7 +469,7 @@ async fn replay_transactions_are_rejected_in_the_same_block() { sequencer.produce_new_block().await.unwrap(); let block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); @@ -479,7 +504,7 @@ async fn replay_transactions_are_rejected_in_different_blocks() { sequencer.produce_new_block().await.unwrap(); let block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); assert_eq!( @@ -498,7 +523,7 @@ async fn replay_transactions_are_rejected_in_different_blocks() { sequencer.produce_new_block().await.unwrap(); let block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); // The replay is rejected, so only the clock tx is in the block. @@ -540,7 +565,7 @@ async fn restart_from_storage() { sequencer.produce_new_block().await.unwrap(); let block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); assert_eq!( @@ -556,8 +581,8 @@ async fn restart_from_storage() { // with the above transaction and update the state to reflect that. let (sequencer, _mempool_handle) = SequencerCoreWithMockClients::start_from_config(config.clone()).await; - let balance_acc_1 = sequencer.state.get_account_by_id(acc1_account_id).balance; - let balance_acc_2 = sequencer.state.get_account_by_id(acc2_account_id).balance; + let balance_acc_1 = sequencer.with_state(|s| s.get_account_by_id(acc1_account_id).balance); + let balance_acc_2 = sequencer.with_state(|s| s.get_account_by_id(acc2_account_id).balance); // Balances should be consistent with the stored block assert_eq!( @@ -655,7 +680,7 @@ async fn produce_block_with_correct_prev_meta_after_restart() { // Step 5: Verify the new block has correct previous block metadata let new_block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); @@ -707,7 +732,7 @@ async fn transactions_touching_clock_account_are_dropped_from_block() { let block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); @@ -762,7 +787,7 @@ async fn user_tx_that_chain_calls_clock_is_dropped() { let block = sequencer .store - .get_block_at_id(sequencer.chain_height) + .get_block_at_id(sequencer.chain_height()) .unwrap() .unwrap(); @@ -781,10 +806,13 @@ async fn block_production_aborts_when_clock_account_data_is_corrupted() { // Corrupt the clock 01 account data so the clock program panics on deserialization. let clock_account_id = system_accounts::clock_account_ids()[0]; - let mut corrupted = sequencer.state.get_account_by_id(clock_account_id); + let mut corrupted = sequencer.with_state(|s| s.get_account_by_id(clock_account_id)); corrupted.data = vec![0xff; 3].try_into().unwrap(); sequencer - .state + .chain() + .lock() + .expect("chain mutex poisoned") + .head_state_mut() .force_insert_account(clock_account_id, corrupted); // Push a dummy transaction so the mempool is non-empty. @@ -1247,3 +1275,501 @@ fn pda_mechanism_with_pinata_token_program() { expected_winner_token_holding_post ); } + +#[test] +fn resubmittable_txs_drops_clock_and_bridge_deposits() { + let user_tx = common::test_utils::produce_dummy_empty_transaction(); + let deposit_tx = build_bridge_deposit_tx_from_event(&PendingDepositEventRecord { + deposit_op_id: HashType([13; 32]), + source_tx_hash: HashType([7; 32]), + amount: 1, + metadata: borsh::to_vec(&DepositMetadataForEncoding { + recipient_id: initial_public_user_accounts()[0].account_id, + }) + .unwrap(), + submitted_in_block_id: None, + }) + .unwrap(); + let withdraw_tx = { + let message = lee::public_transaction::Message::try_new( + programs::bridge().id(), + vec![system_accounts::bridge_account_id()], + vec![], + bridge_core::Instruction::Withdraw { + amount: 1, + bedrock_account_pk: [0; 32], + }, + ) + .unwrap(); + LeeTransaction::Public(PublicTransaction::new( + message, + lee::public_transaction::WitnessSet::from_raw_parts(vec![]), + )) + }; + + let block = common::test_utils::produce_dummy_block( + 2, + Some(HashType([1; 32])), + vec![user_tx.clone(), deposit_tx, withdraw_tx.clone()], + ); + + // The trailing clock tx and the sequencer-generated deposit are dropped; + // user txs (withdrawals included) are returned. + assert_eq!(resubmittable_txs(&block), vec![user_tx, withdraw_tx]); +} + +#[test] +fn resubmittable_txs_of_blocks_without_user_txs_is_empty() { + // No transactions at all (not even the mandatory clock tx). + let empty = HashableBlockData { + block_id: 1, + prev_block_hash: HashType([0; 32]), + timestamp: 0, + transactions: vec![], + } + .into_pending_block(&sequencer_sign_key_for_testing()); + assert!(resubmittable_txs(&empty).is_empty()); + + let clock_only = common::test_utils::produce_dummy_block(1, None, vec![]); + assert!(resubmittable_txs(&clock_only).is_empty()); +} + +#[tokio::test] +async fn follow_adopted_peer_block_applies_and_persists() { + let config = setup_sequencer_config(); + let (sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; + let genesis_meta = sequencer + .store + .latest_block_meta() + .unwrap() + .expect("genesis meta is set"); + + let acc1 = initial_public_user_accounts()[0].account_id; + let acc2 = initial_public_user_accounts()[1].account_id; + let tx = common::test_utils::create_transaction_native_token_transfer( + acc1, + 0, + acc2, + 10, + &create_signing_key_for_account1(), + ); + let peer_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![tx]); + + apply_follow_update( + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, + FollowUpdate { + adopted: vec![(MsgId::from([1; 32]), peer_block.clone())], + orphaned: vec![], + finalized: vec![], + }, + ); + + assert_eq!(sequencer.chain_height(), 2); + let stored = sequencer + .store + .get_block_at_id(2) + .unwrap() + .expect("adopted peer block should be persisted"); + assert_eq!(stored.header.hash, peer_block.header.hash); + assert_eq!( + sequencer.with_state(|s| s.get_account_by_id(acc2).balance), + 20010 + ); +} + +#[tokio::test] +async fn follow_redelivery_of_own_block_is_deduped() { + let config = setup_sequencer_config(); + let (mut sequencer, mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config).await; + + let acc1 = initial_public_user_accounts()[0].account_id; + let acc2 = initial_public_user_accounts()[1].account_id; + let tx = common::test_utils::create_transaction_native_token_transfer( + acc1, + 0, + acc2, + 10, + &create_signing_key_for_account1(), + ); + mempool_handle + .push((TransactionOrigin::User, tx)) + .await + .unwrap(); + sequencer.produce_new_block().await.unwrap(); + let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + + // The channel redelivers our own block under the MsgId the mock publisher + // assigned at publish time. + apply_follow_update( + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, + FollowUpdate { + adopted: vec![(MsgId::from(block2.header.hash.0), block2)], + orphaned: vec![], + finalized: vec![], + }, + ); + + assert_eq!(sequencer.chain_height(), 2); + assert_eq!( + sequencer.with_state(|s| s.get_account_by_id(acc2).balance), + 20010, + "the transfer must not be double-applied" + ); +} + +#[tokio::test] +async fn follow_orphan_reverts_head_and_requeues_user_txs() { + let config = setup_sequencer_config(); + let (mut sequencer, mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config).await; + + let acc1 = initial_public_user_accounts()[0].account_id; + let acc2 = initial_public_user_accounts()[1].account_id; + let tx = common::test_utils::create_transaction_native_token_transfer( + acc1, + 0, + acc2, + 10, + &create_signing_key_for_account1(), + ); + mempool_handle + .push((TransactionOrigin::User, tx.clone())) + .await + .unwrap(); + sequencer.produce_new_block().await.unwrap(); + let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + + apply_follow_update( + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, + FollowUpdate { + adopted: vec![], + orphaned: vec![(MsgId::from(block2.header.hash.0), block2)], + finalized: vec![], + }, + ); + + assert_eq!(sequencer.chain_height(), 1); + assert_eq!( + sequencer.with_state(|s| s.get_account_by_id(acc1).balance), + 10000, + "the orphaned transfer must be reverted from the head" + ); + let (origin, requeued) = sequencer + .mempool + .pop() + .expect("orphaned user tx should be requeued"); + assert!(matches!(origin, TransactionOrigin::User)); + assert_eq!(requeued, tx); + assert!( + sequencer.mempool.pop().is_none(), + "the clock tx must not be requeued" + ); +} + +#[tokio::test] +async fn follow_finalized_own_block_moves_final_tier_and_marks_store() { + let config = setup_sequencer_config(); + let (mut sequencer, mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config).await; + + let tx = common::test_utils::produce_dummy_empty_transaction(); + mempool_handle + .push((TransactionOrigin::User, tx)) + .await + .unwrap(); + sequencer.produce_new_block().await.unwrap(); + let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + + apply_follow_update( + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, + FollowUpdate { + adopted: vec![], + orphaned: vec![], + finalized: vec![(MsgId::from(block2.header.hash.0), block2)], + }, + ); + + let final_tip = sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .final_tip() + .expect("final tip set"); + assert_eq!(final_tip.block_id, 2); + assert_eq!(sequencer.chain_height(), 2, "head is unchanged"); + let stored = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + assert!(matches!(stored.bedrock_status, BedrockStatus::Finalized)); +} + +#[tokio::test] +async fn follow_finalized_backfill_block_is_applied_and_marked_finalized() { + let config = setup_sequencer_config(); + let (sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; + let genesis_meta = sequencer + .store + .latest_block_meta() + .unwrap() + .expect("genesis meta is set"); + + // A peer block we never saw as adopted arrives straight from the + // finalized (backfill) stream. + let peer_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![]); + + apply_follow_update( + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, + FollowUpdate { + adopted: vec![], + orphaned: vec![], + finalized: vec![(MsgId::from([2; 32]), peer_block.clone())], + }, + ); + + assert_eq!( + sequencer.chain_height(), + 2, + "head mirrors final on backfill" + ); + let stored = sequencer + .store + .get_block_at_id(2) + .unwrap() + .expect("backfilled block should be persisted"); + assert_eq!(stored.header.hash, peer_block.header.hash); + assert!(matches!(stored.bedrock_status, BedrockStatus::Finalized)); +} + +#[tokio::test] +async fn restart_restores_head_tier_and_recovers_from_orphan() { + let config = setup_sequencer_config(); + let acc1 = initial_public_user_accounts()[0].account_id; + let acc2 = initial_public_user_accounts()[1].account_id; + + // Produce block 2 (a user transfer), then "crash" before it finalizes. + let (tx, block2) = { + let (mut sequencer, mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let tx = common::test_utils::create_transaction_native_token_transfer( + acc1, + 0, + acc2, + 10, + &create_signing_key_for_account1(), + ); + mempool_handle + .push((TransactionOrigin::User, tx.clone())) + .await + .unwrap(); + sequencer.produce_new_block().await.unwrap(); + (tx, sequencer.store.get_block_at_id(2).unwrap().unwrap()) + }; + + // Restart: nothing is finalized, so block 2 must come back as *head*, not + // final โ€” the L1 can still orphan it. + let (mut sequencer, mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config.clone()).await; + assert_eq!(sequencer.chain_height(), 2); + + // The L1 orphans block 2 under its real MsgId (which we never persisted) + // and adopts a competing empty block 2'. + let genesis = sequencer.store.get_block_at_id(1).unwrap().unwrap(); + let block2_prime = + common::test_utils::produce_dummy_block(2, Some(genesis.header.hash), vec![]); + apply_follow_update( + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, + FollowUpdate { + adopted: vec![(MsgId::from([21; 32]), block2_prime.clone())], + orphaned: vec![(MsgId::from([20; 32]), block2)], + finalized: vec![], + }, + ); + + // The head reorged onto 2': transfer reverted, store overwritten, and the + // orphaned user tx returned to the mempool. + assert_eq!(sequencer.chain_height(), 2); + let head_tip = sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .head_tip() + .expect("head tip set"); + assert_eq!(head_tip.hash, block2_prime.header.hash); + assert_eq!( + sequencer.with_state(|s| s.get_account_by_id(acc1).balance), + 10000, + "the orphaned transfer must be reverted" + ); + let stored = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + assert_eq!(stored.header.hash, block2_prime.header.hash); + let (origin, requeued) = sequencer + .mempool + .pop() + .expect("orphaned user tx should be requeued"); + assert!(matches!(origin, TransactionOrigin::User)); + assert_eq!(requeued, tx); +} + +#[tokio::test] +async fn restart_reanchors_on_the_persisted_final_snapshot() { + let config = setup_sequencer_config(); + + // Produce block 2 and follow its finalization, which persists the final + // snapshot; then "crash". + { + let (mut sequencer, mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let tx = common::test_utils::produce_dummy_empty_transaction(); + mempool_handle + .push((TransactionOrigin::User, tx)) + .await + .unwrap(); + sequencer.produce_new_block().await.unwrap(); + let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + apply_follow_update( + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, + FollowUpdate { + adopted: vec![], + orphaned: vec![], + finalized: vec![(MsgId::from(block2.header.hash.0), block2)], + }, + ); + } + + // Restart: the final tier re-anchors on the snapshot instead of treating + // the whole stored chain as final. + let (sequencer, _mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let chain = sequencer.chain(); + let chain = chain.lock().expect("chain mutex poisoned"); + assert_eq!(chain.final_tip().expect("final tip set").block_id, 2); + assert_eq!(chain.head_tip().expect("head tip set").block_id, 2); +} + +#[tokio::test] +async fn record_produced_block_skips_persistence_on_lost_race() { + let config = setup_sequencer_config(); + let (mut sequencer, _mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config).await; + let genesis_meta = sequencer + .store + .latest_block_meta() + .unwrap() + .expect("genesis meta is set"); + + // A peer block wins height 2 while "our" block is in flight. + let peer_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![]); + sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .apply_adopted(MsgId::from([9; 32]), &peer_block); + + // Our competing block at the same height: same parent, different content. + let tx = common::test_utils::produce_dummy_empty_transaction(); + let our_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![tx]); + sequencer + .record_produced_block( + MsgId::from(our_block.header.hash.0), + &our_block, + &[], + vec![], + ) + .unwrap(); + + // The lost-race block must not reach the store; the head keeps the peer block. + assert!(sequencer.store.get_block_at_id(2).unwrap().is_none()); + let head_tip = sequencer + .chain() + .lock() + .expect("chain mutex poisoned") + .head_tip() + .expect("head tip"); + assert_eq!(head_tip.hash, peer_block.header.hash); +} + +#[tokio::test] +async fn record_produced_block_skips_persistence_when_block_no_longer_chains() { + let config = setup_sequencer_config(); + let (mut sequencer, _mempool_handle) = + SequencerCoreWithMockClients::start_from_config(config).await; + + // The head reorged under us: our block's parent is no longer the tip. + let stale = common::test_utils::produce_dummy_block(2, Some(HashType([9; 32])), vec![]); + sequencer + .record_produced_block(MsgId::from(stale.header.hash.0), &stale, &[], vec![]) + .unwrap(); + + assert!(sequencer.store.get_block_at_id(2).unwrap().is_none()); + assert_eq!(sequencer.chain_height(), 1, "head is unchanged"); +} + +#[tokio::test] +async fn follow_update_persists_blocks_meta_and_state_atomically() { + let config = setup_sequencer_config(); + let (sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; + let genesis_meta = sequencer + .store + .latest_block_meta() + .unwrap() + .expect("genesis meta is set"); + + let acc1 = initial_public_user_accounts()[0].account_id; + let acc2 = initial_public_user_accounts()[1].account_id; + let tx = common::test_utils::create_transaction_native_token_transfer( + acc1, + 0, + acc2, + 10, + &create_signing_key_for_account1(), + ); + let block2 = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![tx]); + let block3 = common::test_utils::produce_dummy_block(3, Some(block2.header.hash), vec![]); + + // One update carrying several blocks: both adopted, block 2 also finalized. + apply_follow_update( + &sequencer.store.dbio(), + &sequencer.chain(), + &mempool_handle, + FollowUpdate { + adopted: vec![ + (MsgId::from([2; 32]), block2.clone()), + (MsgId::from([3; 32]), block3.clone()), + ], + orphaned: vec![], + finalized: vec![(MsgId::from([2; 32]), block2)], + }, + ); + + // Blocks, tip meta and state all reflect the end of the batch: a late + // finalized entry for an earlier block must not drag the tip meta back. + let meta = sequencer + .store + .latest_block_meta() + .unwrap() + .expect("meta is set"); + assert_eq!(meta.id, 3); + assert_eq!(meta.hash, block3.header.hash); + let stored2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + assert!(matches!(stored2.bedrock_status, BedrockStatus::Finalized)); + let stored_balance = sequencer + .store + .get_lee_state() + .unwrap() + .get_account_by_id(acc2) + .balance; + assert_eq!(stored_balance, 20010); +} diff --git a/lez/sequencer/core/src/tests/reconstruction.rs b/lez/sequencer/core/src/tests/reconstruction.rs index 40cbaa6c..e52c3364 100644 --- a/lez/sequencer/core/src/tests/reconstruction.rs +++ b/lez/sequencer/core/src/tests/reconstruction.rs @@ -4,6 +4,9 @@ reason = "We don't care about it in tests" )] +use std::sync::Mutex; + +use chain_state::ChainState; use common::block::Block; use logos_blockchain_core::mantle::ops::channel::{MsgId, inscribe::Inscription}; use logos_blockchain_zone_sdk::{Slot, ZoneBlock, ZoneMessage}; @@ -14,6 +17,16 @@ use crate::{ SequencerCore, block_store::SequencerStore, config::GenesisAction, mock::MockBlockPublisher, }; +/// Fresh `(store, chain)` pair for a reconstruction target, as +/// `start_from_config` would build them before the publisher starts. +fn fresh_store_and_chain(config: &SequencerConfig) -> (SequencerStore, Mutex) { + let (store, state) = SequencerCore::::open_or_create_store(config); + let chain = Mutex::new(SequencerCore::::restore_chain_state( + config, &store, &state, + )); + (store, chain) +} + fn block_to_channel_message(block: &Block, slot: u64) -> (ZoneMessage, Slot) { let bytes = borsh::to_vec(block).expect("serialize block"); let message = ZoneMessage::Block(ZoneBlock { @@ -53,15 +66,11 @@ async fn reconstructs_missing_channel_blocks_into_fresh_store() { // Sequencer B starts from a fresh store and reconstructs A's chain. let config_b = setup_sequencer_config(); - let (mut store_b, mut state_b) = - SequencerCore::::open_or_create_store(&config_b); + let (store_b, chain_b) = fresh_store_and_chain(&config_b); let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages); let channel_was_empty = SequencerCore::::verify_and_reconstruct( - &mock_b, - &mut store_b, - &mut state_b, - true, + &mock_b, &store_b, &chain_b, true, ) .await .expect("reconstruct"); @@ -72,10 +81,12 @@ async fn reconstructs_missing_channel_blocks_into_fresh_store() { assert_eq!(tip_b.hash, tip_a.hash); // State matches: initial account balances agree with sequencer A. + let state_b = chain_b.lock().unwrap().head_state().clone(); + let state_a = seq_a.chain().lock().unwrap().head_state().clone(); for account in initial_public_user_accounts() { assert_eq!( state_b.get_account_by_id(account.account_id).balance, - seq_a.state().get_account_by_id(account.account_id).balance, + state_a.get_account_by_id(account.account_id).balance, ); } @@ -85,10 +96,7 @@ async fn reconstructs_missing_channel_blocks_into_fresh_store() { // Re-running is idempotent: everything is already applied, no error. let channel_was_empty = SequencerCore::::verify_and_reconstruct( - &mock_b, - &mut store_b, - &mut state_b, - true, + &mock_b, &store_b, &chain_b, true, ) .await .expect("reconstruct idempotent"); @@ -99,7 +107,7 @@ async fn reconstructs_missing_channel_blocks_into_fresh_store() { #[tokio::test] async fn fails_when_channel_serves_a_divergent_block() { let config = setup_sequencer_config(); - let (mut store, mut state) = SequencerCore::::open_or_create_store(&config); + let (store, chain) = fresh_store_and_chain(&config); // Anchor on the local genesis at some slot. let genesis_id = store.genesis_id(); @@ -123,17 +131,16 @@ async fn fails_when_channel_serves_a_divergent_block() { messages, ); - let result = SequencerCore::::verify_and_reconstruct( - &mock, &mut store, &mut state, true, - ) - .await; + let result = + SequencerCore::::verify_and_reconstruct(&mock, &store, &chain, true) + .await; assert!(result.is_err(), "divergent channel must abort startup"); } #[tokio::test] async fn fails_when_channel_is_missing() { let config = setup_sequencer_config(); - let (mut store, mut state) = SequencerCore::::open_or_create_store(&config); + let (store, chain) = fresh_store_and_chain(&config); let genesis_id = store.genesis_id(); let genesis = store.get_block_at_id(genesis_id).unwrap().unwrap(); store @@ -147,10 +154,9 @@ async fn fails_when_channel_is_missing() { // Anchor present, but the channel does not exist on the connected chain. let mock = MockBlockPublisher::with_canned_channel(config.bedrock_config.channel_id, None, vec![]); - let result = SequencerCore::::verify_and_reconstruct( - &mock, &mut store, &mut state, true, - ) - .await; + let result = + SequencerCore::::verify_and_reconstruct(&mock, &store, &chain, true) + .await; assert!(result.is_err(), "missing channel must abort startup"); } @@ -161,7 +167,7 @@ async fn fails_when_channel_is_missing() { #[tokio::test] async fn fails_when_channel_reinscribes_genesis_with_a_different_hash() { let config = setup_sequencer_config(); - let (mut store, mut state) = SequencerCore::::open_or_create_store(&config); + let (store, chain) = fresh_store_and_chain(&config); // Fresh store, no anchor. The channel serves a genesis at the same id but a // different hash โ€” a foreign chain reinscribing genesis. @@ -174,10 +180,9 @@ async fn fails_when_channel_reinscribes_genesis_with_a_different_hash() { Some(Slot::from(10)), messages, ); - let result = SequencerCore::::verify_and_reconstruct( - &mock, &mut store, &mut state, true, - ) - .await; + let result = + SequencerCore::::verify_and_reconstruct(&mock, &store, &chain, true) + .await; assert!( result.is_err(), "a reinscribed genesis with a different hash must abort startup" @@ -209,10 +214,7 @@ async fn fails_when_a_stored_block_hash_diverges_from_the_channel() { messages, ); let result = SequencerCore::::verify_and_reconstruct( - &mock, - &mut seq.store, - &mut seq.state, - true, + &mock, &seq.store, &seq.chain, true, ) .await; assert!( @@ -224,7 +226,7 @@ async fn fails_when_a_stored_block_hash_diverges_from_the_channel() { #[tokio::test] async fn fails_when_a_channel_block_is_missing_locally() { let config = setup_sequencer_config(); - let (mut store, mut state) = SequencerCore::::open_or_create_store(&config); + let (store, chain) = fresh_store_and_chain(&config); // A block numbered below our genesis is at/below the local tip yet absent from // the store โ€” a foreign chain with a lower numbering. @@ -237,10 +239,9 @@ async fn fails_when_a_channel_block_is_missing_locally() { Some(Slot::from(10)), messages, ); - let result = SequencerCore::::verify_and_reconstruct( - &mock, &mut store, &mut state, true, - ) - .await; + let result = + SequencerCore::::verify_and_reconstruct(&mock, &store, &chain, true) + .await; assert!( result.is_err(), "a channel block below the local range must abort startup" @@ -250,7 +251,7 @@ async fn fails_when_a_channel_block_is_missing_locally() { #[tokio::test] async fn fails_when_a_channel_block_does_not_extend_the_tip() { let config = setup_sequencer_config(); - let (mut store, mut state) = SequencerCore::::open_or_create_store(&config); + let (store, chain) = fresh_store_and_chain(&config); // A block claiming an id far past genesis does not chain onto the local tip. let mut orphan = store.get_block_at_id(store.genesis_id()).unwrap().unwrap(); @@ -262,10 +263,9 @@ async fn fails_when_a_channel_block_does_not_extend_the_tip() { Some(Slot::from(10)), messages, ); - let result = SequencerCore::::verify_and_reconstruct( - &mock, &mut store, &mut state, true, - ) - .await; + let result = + SequencerCore::::verify_and_reconstruct(&mock, &store, &chain, true) + .await; assert!( result.is_err(), "a non-contiguous channel block must abort startup" @@ -386,13 +386,12 @@ async fn reconstructed_deposit_is_not_reminted_after_backfill_redelivery() { let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages); SequencerCore::::verify_and_reconstruct( &mock_b, - &mut seq_b.store, - &mut seq_b.state, + &seq_b.store, + &seq_b.chain, true, ) .await .expect("reconstruct"); - seq_b.chain_height = seq_b.store.latest_block_meta().unwrap().unwrap().id; let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap(); assert_eq!(tip_b.id, tip_a.id); @@ -402,15 +401,17 @@ async fn reconstructed_deposit_is_not_reminted_after_backfill_redelivery() { let vault_id = vault_core::compute_vault_account_id(programs::vault().id(), recipient); let bridge_id = system_accounts::bridge_account_id(); + let state_b = seq_b.chain().lock().unwrap().head_state().clone(); + let state_a = seq_a.chain().lock().unwrap().head_state().clone(); for account in [vault_id, bridge_id, recipient] { assert_eq!( - seq_b.state().get_account_by_id(account).balance, - seq_a.state().get_account_by_id(account).balance, + state_b.get_account_by_id(account).balance, + state_a.get_account_by_id(account).balance, "reconstructed balance mismatch for {account:?}", ); } assert_eq!( - seq_b.state().get_account_by_id(vault_id).balance, + state_b.get_account_by_id(vault_id).balance, u128::from(deposit_amount), "deposit must mint into the recipient vault exactly once, not twice" ); @@ -489,17 +490,11 @@ async fn reconstructed_withdraw_leaves_no_phantom_unseen_count() { // Sequencer B reconstructs A's chain from a fresh store. let config_b = bridge_funded_config(); - let (mut store_b, mut state_b) = - SequencerCore::::open_or_create_store(&config_b); + let (store_b, chain_b) = fresh_store_and_chain(&config_b); let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages); - SequencerCore::::verify_and_reconstruct( - &mock_b, - &mut store_b, - &mut state_b, - true, - ) - .await - .expect("reconstruct"); + SequencerCore::::verify_and_reconstruct(&mock_b, &store_b, &chain_b, true) + .await + .expect("reconstruct"); assert!( !store_b.dbio().consume_unseen_withdraw_count(key).unwrap(), @@ -540,8 +535,7 @@ async fn reconstruction_reconciles_already_finished_deposit() { // pre-seeded, as the cold-start backfill would when it re-observes this // already-finalized deposit. let config_b = bridge_funded_config(); - let (mut store_b, mut state_b) = - SequencerCore::::open_or_create_store(&config_b); + let (store_b, chain_b) = fresh_store_and_chain(&config_b); assert!( store_b .dbio() @@ -550,19 +544,19 @@ async fn reconstruction_reconciles_already_finished_deposit() { ); let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages); - SequencerCore::::verify_and_reconstruct( - &mock_b, - &mut store_b, - &mut state_b, - true, - ) - .await - .expect("reconstruct"); + SequencerCore::::verify_and_reconstruct(&mock_b, &store_b, &chain_b, true) + .await + .expect("reconstruct"); // The mint was applied exactly once. let vault_id = vault_core::compute_vault_account_id(programs::vault().id(), recipient); assert_eq!( - state_b.get_account_by_id(vault_id).balance, + chain_b + .lock() + .unwrap() + .head_state() + .get_account_by_id(vault_id) + .balance, u128::from(deposit_amount), "already-finished deposit must be applied exactly once" ); @@ -599,17 +593,16 @@ async fn committed_local_against_missing_channel_fails_without_anchor() { // Reopen: blocks beyond genesis, no anchor. `is_fresh_start = false` stands in // for a checkpoint persisted by a prior sync (the mock never emits one). - let (mut store, mut state) = SequencerCore::::open_or_create_store(&config); + let (store, chain) = fresh_store_and_chain(&config); assert!(store.get_zone_anchor().unwrap().is_none()); assert!(store.latest_block_meta().unwrap().unwrap().id > 1); // The channel is gone: no tip, no messages. let mock = MockBlockPublisher::with_canned_channel(config.bedrock_config.channel_id, None, vec![]); - let result = SequencerCore::::verify_and_reconstruct( - &mock, &mut store, &mut state, false, - ) - .await; + let result = + SequencerCore::::verify_and_reconstruct(&mock, &store, &chain, false) + .await; assert!( result.is_err(), "committed blocks against a missing channel must abort startup" diff --git a/lez/sequencer/service/Cargo.toml b/lez/sequencer/service/Cargo.toml index 3427dc22..338aac0d 100644 --- a/lez/sequencer/service/Cargo.toml +++ b/lez/sequencer/service/Cargo.toml @@ -1,6 +1,7 @@ [package] name = "sequencer_service" version = "0.1.0" +default-run = "sequencer_service" edition = "2024" license = { workspace = true } @@ -19,6 +20,7 @@ programs.workspace = true clap = { workspace = true, features = ["derive", "env"] } anyhow.workspace = true env_logger.workspace = true +hex.workspace = true log.workspace = true tokio.workspace = true tokio-util.workspace = true diff --git a/lez/sequencer/service/rpc/src/lib.rs b/lez/sequencer/service/rpc/src/lib.rs index f32f6bd2..a1d2acfb 100644 --- a/lez/sequencer/service/rpc/src/lib.rs +++ b/lez/sequencer/service/rpc/src/lib.rs @@ -90,6 +90,4 @@ pub trait Rpc { #[method(name = "getChannelId")] async fn get_channel_id(&self) -> Result; - - // ============================================================================================= } diff --git a/lez/sequencer/service/src/bin/bedrock_pubkey.rs b/lez/sequencer/service/src/bin/bedrock_pubkey.rs new file mode 100644 index 00000000..ad602be6 --- /dev/null +++ b/lez/sequencer/service/src/bin/bedrock_pubkey.rs @@ -0,0 +1,35 @@ +//! Prints the sequencer's bedrock signing public key (hex) without booting it. +//! +//! Loads `/bedrock_signing_key` from the given sequencer config, creating +//! the key if it doesn't exist yet, so that a node can be accredited into the +//! channel committee before its first boot. + +use std::path::PathBuf; + +use anyhow::Result; +use clap::Parser; + +#[derive(Debug, Parser)] +#[clap(version)] +struct Args { + #[clap(name = "config")] + config_path: PathBuf, + /// Override the config's home directory, matching the sequencer's --home. + #[clap(long)] + home: Option, +} + +#[expect( + clippy::print_stdout, + reason = "the hex pubkey on stdout is this binary's output" +)] +fn main() -> Result<()> { + let Args { config_path, home } = Args::parse(); + + let config = sequencer_service::SequencerConfig::from_path(&config_path)?; + let home = home.unwrap_or(config.home); + let key = sequencer_core::load_or_create_signing_key(&home.join("bedrock_signing_key"))?; + println!("{}", hex::encode(key.public_key().to_bytes())); + + Ok(()) +} diff --git a/lez/sequencer/service/src/bin/configure_channel.rs b/lez/sequencer/service/src/bin/configure_channel.rs new file mode 100644 index 00000000..023e32b0 --- /dev/null +++ b/lez/sequencer/service/src/bin/configure_channel.rs @@ -0,0 +1,71 @@ +//! Posts a `ChannelConfig` op (accredited keys + rotation params) to bedrock, +//! signed with `/bedrock_signing_key`, without booting the sequencer. +//! +//! Authorization is holding the admin key file โ€” the L1 rejects non-admin +//! signers. Acceptance is asynchronous: a rejection only shows up in node +//! logs and on-chain behavior. + +use std::path::PathBuf; + +use anyhow::{Context as _, Result, anyhow}; +use clap::Parser; +use sequencer_core::block_publisher::{Ed25519PublicKey, post_channel_config}; + +#[derive(Debug, Parser)] +#[clap(version)] +struct Args { + #[clap(name = "config")] + config_path: PathBuf, + /// Override the config's home directory, matching the sequencer's --home. + #[clap(long)] + home: Option, + /// Accredited ed25519 public keys (hex), admin (this node's key) first. + #[clap(long, required = true, value_delimiter = ',')] + keys: Vec, + /// Slots a sequencer's posting turn lasts. + #[clap(long)] + posting_timeframe: u32, + /// Slots after which a stalled turn can be taken over. + #[clap(long)] + posting_timeout: u32, + /// Signatures required for future config changes. + #[clap(long, default_value_t = 1)] + configuration_threshold: u16, + /// Signatures required for channel withdrawals. + #[clap(long, default_value_t = 1)] + withdraw_threshold: u16, +} + +#[tokio::main] +async fn main() -> Result<()> { + env_logger::init(); + let args = Args::parse(); + + let config = sequencer_service::SequencerConfig::from_path(&args.config_path)?; + let home = args.home.unwrap_or(config.home); + let signing_key = + sequencer_core::load_or_create_signing_key(&home.join("bedrock_signing_key"))?; + let keys = args + .keys + .iter() + .map(|key| parse_key(key)) + .collect::>>()?; + + post_channel_config( + &config.bedrock_config, + &signing_key, + keys, + args.posting_timeframe, + args.posting_timeout, + args.configuration_threshold, + args.withdraw_threshold, + ) + .await +} + +fn parse_key(hex_key: &str) -> Result { + let mut bytes = [0_u8; 32]; + hex::decode_to_slice(hex_key, &mut bytes) + .with_context(|| format!("Invalid hex-encoded key {hex_key}"))?; + Ed25519PublicKey::from_bytes(&bytes).map_err(|err| anyhow!("Invalid Ed25519 public key: {err}")) +} diff --git a/lez/sequencer/service/src/lib.rs b/lez/sequencer/service/src/lib.rs index 687ee424..a8aa856e 100644 --- a/lez/sequencer/service/src/lib.rs +++ b/lez/sequencer/service/src/lib.rs @@ -11,10 +11,11 @@ use mempool::MemPoolHandle; use sequencer_core::SequencerCore; #[cfg(feature = "standalone")] use sequencer_core::SequencerCoreWithMockClients as SequencerCore; -use sequencer_core::TransactionOrigin; pub use sequencer_core::config::*; +use sequencer_core::{TransactionOrigin, block_publisher::BlockPublisherTrait as _}; use sequencer_service_rpc::RpcServer as _; use tokio::{sync::Mutex, task::JoinHandle}; +use tokio_util::sync::CancellationToken; pub mod service; @@ -28,6 +29,9 @@ pub struct SequencerHandle { /// Option because of `Drop` which forbids to simply move out of `self` in `stopped()`. server_handle: Option, main_loop_handle: JoinHandle>, + /// Cancelled when the publisher's drive task terminates (e.g. a panicked + /// persist sink); no channel events are processed past that point. + driver_cancellation: CancellationToken, } impl SequencerHandle { @@ -35,11 +39,13 @@ impl SequencerHandle { addr: SocketAddr, server_handle: ServerHandle, main_loop_handle: JoinHandle>, + driver_cancellation: CancellationToken, ) -> Self { Self { addr, server_handle: Some(server_handle), main_loop_handle, + driver_cancellation, } } @@ -53,6 +59,7 @@ impl SequencerHandle { addr: _, server_handle, main_loop_handle, + driver_cancellation, } = &mut self; let server_handle = server_handle.take().expect("Server handle is set"); @@ -65,6 +72,9 @@ impl SequencerHandle { .context("Main loop task panicked")? .context("Main loop exited unexpectedly") } + () = driver_cancellation.cancelled() => { + Err(anyhow!("Publisher drive task terminated")) + } } } @@ -78,10 +88,12 @@ impl SequencerHandle { addr: _, server_handle, main_loop_handle, + driver_cancellation, } = self; let stopped = server_handle.as_ref().is_none_or(ServerHandle::is_stopped) - || main_loop_handle.is_finished(); + || main_loop_handle.is_finished() + || driver_cancellation.is_cancelled(); !stopped } @@ -97,6 +109,7 @@ impl Drop for SequencerHandle { addr: _, server_handle, main_loop_handle, + driver_cancellation: _, } = self; main_loop_handle.abort(); @@ -111,21 +124,23 @@ impl Drop for SequencerHandle { } } -pub async fn run(config: SequencerConfig, port: u16) -> Result { +pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result { let block_timeout = config.block_create_timeout; let max_block_size = config.max_block_size; - let (sequencer_core, mempool_handle) = SequencerCore::start_from_config(config).await; + let (sequencer_core, mempool_handle): (SequencerCore, _) = + SequencerCore::start_from_config(config).await; info!("Sequencer core set up"); + let driver_cancellation = sequencer_core.block_publisher().driver_cancellation(); let seq_core_wrapped = Arc::new(Mutex::new(sequencer_core)); let mempool_handle_for_server = mempool_handle.clone(); let (server_handle, addr) = run_server( Arc::clone(&seq_core_wrapped), mempool_handle_for_server, - port, + listen_addr, max_block_size.as_u64(), ) .await?; @@ -136,13 +151,18 @@ pub async fn run(config: SequencerConfig, port: u16) -> Result let _ = mempool_handle; - Ok(SequencerHandle::new(addr, server_handle, main_loop_handle)) + Ok(SequencerHandle::new( + addr, + server_handle, + main_loop_handle, + driver_cancellation, + )) } async fn run_server( sequencer: Arc>, mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>, - port: u16, + listen_addr: SocketAddr, max_block_size: u64, ) -> Result<(ServerHandle, SocketAddr)> { let server = jsonrpsee::server::ServerBuilder::with_config( @@ -153,7 +173,7 @@ async fn run_server( ) .build(), ) - .build(SocketAddr::from(([0, 0, 0, 0], port))) + .build(listen_addr) .await .context("Failed to build RPC server")?; @@ -172,16 +192,15 @@ async fn main_loop(seq_core: Arc>, block_timeout: Duration) loop { tokio::time::sleep(block_timeout).await; - info!("Collecting transactions from mempool, block creation"); + let mut state = seq_core.lock().await; - let id = { - let mut state = seq_core.lock().await; - - state.produce_new_block().await? - }; + // Only produce on our turn. + if !state.is_our_turn() { + continue; + } + info!("Our turn: collecting transactions from mempool, creating block"); + let id = state.produce_new_block().await?; info!("Block with id {id} created"); - - info!("Waiting for new transactions"); } } diff --git a/lez/sequencer/service/src/main.rs b/lez/sequencer/service/src/main.rs index 8b577bb8..7c50ece6 100644 --- a/lez/sequencer/service/src/main.rs +++ b/lez/sequencer/service/src/main.rs @@ -1,4 +1,7 @@ -use std::path::PathBuf; +use std::{ + net::{IpAddr, SocketAddr}, + path::PathBuf, +}; use anyhow::Result; use clap::Parser; @@ -12,6 +15,14 @@ struct Args { config_path: PathBuf, #[clap(short, long, default_value = "3040")] port: u16, + /// Interface the RPC server binds to. The RPC has no caller auth โ€” + /// bind loopback unless the port is firewalled. + #[clap(long, default_value = "0.0.0.0")] + listen_address: IpAddr, + /// Override the config's home directory (`RocksDB` + bedrock signing key), + /// so multiple instances can share one config file. + #[clap(long)] + home: Option, } #[tokio::main] @@ -22,14 +33,23 @@ struct Args { async fn main() -> Result<()> { env_logger::init(); - let Args { config_path, port } = Args::parse(); + let Args { + config_path, + port, + listen_address, + home, + } = Args::parse(); // TODO: handle this cancellation token more gracefully within Sequencer service // similar to how we do in Indexer let cancellation_token = listen_for_shutdown_signal(); - let config = sequencer_service::SequencerConfig::from_path(&config_path)?; - let sequencer_handle = sequencer_service::run(config, port).await?; + let mut config = sequencer_service::SequencerConfig::from_path(&config_path)?; + if let Some(home) = home { + config.home = home; + } + let sequencer_handle = + sequencer_service::run(config, SocketAddr::new(listen_address, port)).await?; tokio::select! { () = cancellation_token.cancelled() => { diff --git a/lez/sequencer/service/src/service.rs b/lez/sequencer/service/src/service.rs index d0850837..7ab9ed3c 100644 --- a/lez/sequencer/service/src/service.rs +++ b/lez/sequencer/service/src/service.rs @@ -40,7 +40,7 @@ impl SequencerService { } #[async_trait] -impl sequencer_service_rpc::RpcServer +impl sequencer_service_rpc::RpcServer for SequencerService { async fn send_transaction(&self, tx: LeeTransaction) -> Result { @@ -138,8 +138,8 @@ impl sequencer_service_rpc::RpcServer async fn get_account_balance(&self, account_id: AccountId) -> Result { let sequencer = self.sequencer.lock().await; - let account = sequencer.state().get_account_by_id(account_id); - Ok(account.balance) + let balance = sequencer.with_state(|state| state.get_account_by_id(account_id).balance); + Ok(balance) } async fn get_transaction( @@ -155,10 +155,12 @@ impl sequencer_service_rpc::RpcServer account_ids: Vec, ) -> Result, ErrorObjectOwned> { let sequencer = self.sequencer.lock().await; - let nonces = account_ids - .into_iter() - .map(|account_id| sequencer.state().get_account_by_id(account_id).nonce) - .collect(); + let nonces = sequencer.with_state(|state| { + account_ids + .into_iter() + .map(|account_id| state.get_account_by_id(account_id).nonce) + .collect() + }); Ok(nonces) } @@ -167,17 +169,18 @@ impl sequencer_service_rpc::RpcServer commitments: Vec, ) -> Result<(Vec>, CommitmentSetDigest), ErrorObjectOwned> { let sequencer = self.sequencer.lock().await; - let state = sequencer.state(); - let proofs = commitments - .iter() - .map(|commitment| state.get_proof_for_commitment(commitment)) - .collect(); - Ok((proofs, state.commitment_root())) + Ok(sequencer.with_state(|state| { + let proofs = commitments + .iter() + .map(|commitment| state.get_proof_for_commitment(commitment)) + .collect(); + (proofs, state.commitment_root()) + })) } async fn get_account(&self, account_id: AccountId) -> Result { let sequencer = self.sequencer.lock().await; - Ok(sequencer.state().get_account_by_id(account_id)) + Ok(sequencer.with_state(|state| state.get_account_by_id(account_id))) } async fn get_program_ids(&self) -> Result, ErrorObjectOwned> { diff --git a/lez/storage/src/sequencer/mod.rs b/lez/storage/src/sequencer/mod.rs index 47ebc41b..299cd27d 100644 --- a/lez/storage/src/sequencer/mod.rs +++ b/lez/storage/src/sequencer/mod.rs @@ -19,10 +19,12 @@ use crate::{ }, error::DbError, sequencer::sequencer_cells::{ - LEEStateCellOwned, LEEStateCellRef, LastFinalizedBlockIdCell, LatestBlockMetaCellOwned, - LatestBlockMetaCellRef, PendingDepositEventRecord, PendingDepositEventsCellOwned, - PendingDepositEventsCellRef, UnseenWithdrawCountCell, WithdrawalReconciliationKey, - ZoneAnchorCell, ZoneAnchorRecord, ZoneSdkCheckpointCellOwned, ZoneSdkCheckpointCellRef, + FinalBlockMetaCellOwned, FinalBlockMetaCellRef, FinalLeeStateCellOwned, + FinalLeeStateCellRef, LEEStateCellOwned, LEEStateCellRef, LastFinalizedBlockIdCell, + LatestBlockMetaCellOwned, LatestBlockMetaCellRef, PendingDepositEventRecord, + PendingDepositEventsCellOwned, PendingDepositEventsCellRef, UnseenWithdrawCountCell, + WithdrawalReconciliationKey, ZoneAnchorCell, ZoneAnchorRecord, ZoneSdkCheckpointCellOwned, + ZoneSdkCheckpointCellRef, }, }; @@ -46,6 +48,10 @@ pub const DB_META_UNSEEN_WITHDRAW_COUNT_KEY: &str = "unseen_withdraw_count"; /// Key base for storing the LEE state. pub const DB_LEE_STATE_KEY: &str = "lee_state"; +/// Key base for storing the LEE state at the last L1-finalized block. +pub const DB_FINAL_LEE_STATE_KEY: &str = "final_lee_state"; +/// Key base for storing `(id, hash)` of the last L1-finalized block. +pub const DB_FINAL_BLOCK_META_KEY: &str = "final_block_meta"; /// Name of state column family. pub const CF_LEE_STATE_NAME: &str = "cf_lee_state"; @@ -403,6 +409,25 @@ impl RocksDBIO { Ok(true) } + /// Marks the given deposit events submitted in `block_id`, in one write. + pub fn mark_deposit_events_submitted( + &self, + deposit_op_ids: &[HashType], + submitted_block_id: u64, + ) -> DbResult<()> { + if deposit_op_ids.is_empty() { + return Ok(()); + } + let mut batch = WriteBatch::default(); + self.mark_pending_deposit_events_submitted(deposit_op_ids, submitted_block_id, &mut batch)?; + self.db.write(batch).map_err(|rerr| { + DbError::rocksdb_cast_message( + rerr, + Some("Failed to mark deposit events submitted".to_owned()), + ) + }) + } + fn mark_pending_deposit_events_submitted( &self, deposit_op_ids: &[HashType], @@ -503,23 +528,36 @@ impl RocksDBIO { } pub fn put_block(&self, block: &Block, first: bool, batch: &mut WriteBatch) -> DbResult<()> { - let cf_block = self.block_column(); - if !first { + // A produced block is the new head tip by construction: pin the + // tip meta and drop any stale higher blocks a preceding reorg left + // behind (mirrors `store_followed_blocks`). let last_curr_block = self.get_meta_last_block_in_db()?; - - if block.header.block_id > last_curr_block { - self.put_meta_last_block_in_db_batch(block.header.block_id, batch)?; - self.put_meta_latest_block_meta_batch( - &BlockMeta { - id: block.header.block_id, - hash: block.header.hash, - }, - batch, - )?; + for stale_id in block.header.block_id.saturating_add(1)..=last_curr_block { + self.delete_block_payload(stale_id, batch)?; } + self.put_meta_last_block_in_db_batch(block.header.block_id, batch)?; + self.put_meta_latest_block_meta_batch(&BlockMeta::from(block), batch)?; } + self.put_block_payload(block, batch) + } + + /// Stages deletion of a block payload into `batch`. + fn delete_block_payload(&self, block_id: u64, batch: &mut WriteBatch) -> DbResult<()> { + let cf_block = self.block_column(); + batch.delete_cf( + &cf_block, + borsh::to_vec(&block_id).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize block id".to_owned())) + })?, + ); + Ok(()) + } + + /// Stages just the block payload into `batch`, without touching the tip meta. + fn put_block_payload(&self, block: &Block, batch: &mut WriteBatch) -> DbResult<()> { + let cf_block = self.block_column(); batch.put_cf( &cf_block, borsh::to_vec(&block.header.block_id).map_err(|err| { @@ -537,6 +575,26 @@ impl RocksDBIO { .map(|opt| opt.map(|val| val.0)) } + /// `(state, meta)` at the last L1-finalized block; `None` until the first + /// finalization is observed. + pub fn get_final_snapshot(&self) -> DbResult> { + let Some(meta) = self.get_opt::(())? else { + return Ok(None); + }; + let state = self.get::(())?; + Ok(Some((state.0, meta.0))) + } + + fn put_final_snapshot_batch( + &self, + state: &V03State, + meta: &BlockMeta, + batch: &mut WriteBatch, + ) -> DbResult<()> { + self.put_batch(&FinalLeeStateCellRef(state), (), batch)?; + self.put_batch(&FinalBlockMetaCellRef(meta), (), batch) + } + pub fn get_lee_state(&self) -> DbResult { self.get::(()).map(|val| val.0) } @@ -633,6 +691,92 @@ impl RocksDBIO { Ok(()) } + /// One-block form of [`Self::store_followed_blocks`], with the block as the + /// head tip and no final snapshot. Production always uses the batch form. + #[cfg(test)] + fn store_followed_block( + &self, + block: &Block, + state: &V03State, + finalized: bool, + ) -> DbResult<()> { + self.store_followed_blocks( + &[(block, finalized)], + Some(&BlockMeta::from(block)), + state, + None, + ) + } + + /// Persists a batch of followed blocks, the caller's head-tip `state`, and + /// the optional final-tier snapshot in one atomic write. + /// + /// The tip meta is pinned to `head_tip`, and blocks stored above it (left + /// behind by a net-shortening reorg) are deleted in the same write, so + /// restart replay never walks past the tip. + /// + /// Per block: skips the payload write when the store already holds it (by + /// id and hash), unless `finalized` is set, which rewrites it with the + /// finalized status. A no-op update (nothing to write, tip unchanged) + /// writes nothing. + /// + /// TODO: the zone-sdk checkpoint is persisted by `on_checkpoint` *before* + /// this write. Full `BlocksProcessed` atomicity (checkpoint, blocks, state + /// and orphan reverts in one batch) is a follow-up. + pub fn store_followed_blocks( + &self, + blocks: &[(&Block, bool)], + head_tip: Option<&BlockMeta>, + state: &V03State, + final_snapshot: Option<(&V03State, &BlockMeta)>, + ) -> DbResult<()> { + let last_block_in_db = self.get_meta_last_block_in_db()?; + let mut batch = WriteBatch::default(); + + for (block, finalized) in blocks { + let already_stored = self + .get_block(block.header.block_id)? + .filter(|stored| stored.header.hash == block.header.hash); + + let mut to_write = match already_stored { + Some(_) if !finalized => continue, + Some(stored) => stored, + None => (*block).clone(), + }; + if *finalized { + to_write.bedrock_status = BedrockStatus::Finalized; + } + self.put_block_payload(&to_write, &mut batch)?; + } + + // A shrink-only update (orphans without adopted replacements) has no + // payloads to write but must still rewind the tip meta, or the stored + // state tears against the stale disk head on the next produce. + let tip_rewound = head_tip.is_some_and(|tip| tip.id < last_block_in_db); + if batch.is_empty() && final_snapshot.is_none() && !tip_rewound { + return Ok(()); + } + + if let Some(tip) = head_tip { + for stale_id in tip.id.saturating_add(1)..=last_block_in_db { + self.delete_block_payload(stale_id, &mut batch)?; + } + self.put_meta_last_block_in_db_batch(tip.id, &mut batch)?; + self.put_meta_latest_block_meta_batch(tip, &mut batch)?; + } + self.put_lee_state_in_db_batch(state, &mut batch)?; + if let Some((final_state, final_meta)) = final_snapshot { + self.put_final_snapshot_batch(final_state, final_meta, &mut batch)?; + } + + self.db.write(batch).map_err(|rerr| { + DbError::rocksdb_cast_message( + rerr, + Some("Failed to write followed blocks batch".to_owned()), + ) + }) + } + pub fn get_all_blocks(&self) -> impl Iterator> { let cf_block = self.block_column(); self.db @@ -682,3 +826,6 @@ impl RocksDBIO { }) } } + +#[cfg(test)] +mod tests; diff --git a/lez/storage/src/sequencer/sequencer_cells.rs b/lez/storage/src/sequencer/sequencer_cells.rs index 368fb648..c10b3d30 100644 --- a/lez/storage/src/sequencer/sequencer_cells.rs +++ b/lez/storage/src/sequencer/sequencer_cells.rs @@ -7,10 +7,10 @@ use crate::{ cells::{SimpleReadableCell, SimpleStorableCell, SimpleWritableCell}, error::DbError, sequencer::{ - CF_LEE_STATE_NAME, DB_LEE_STATE_KEY, DB_META_LAST_FINALIZED_BLOCK_ID, - DB_META_LATEST_BLOCK_META_KEY, DB_META_PENDING_DEPOSIT_EVENTS_KEY, - DB_META_UNSEEN_WITHDRAW_COUNT_KEY, DB_META_ZONE_CURSOR_KEY, - DB_META_ZONE_SDK_CHECKPOINT_KEY, + CF_LEE_STATE_NAME, DB_FINAL_BLOCK_META_KEY, DB_FINAL_LEE_STATE_KEY, DB_LEE_STATE_KEY, + DB_META_LAST_FINALIZED_BLOCK_ID, DB_META_LATEST_BLOCK_META_KEY, + DB_META_PENDING_DEPOSIT_EVENTS_KEY, DB_META_UNSEEN_WITHDRAW_COUNT_KEY, + DB_META_ZONE_CURSOR_KEY, DB_META_ZONE_SDK_CHECKPOINT_KEY, }, }; @@ -44,6 +44,72 @@ impl SimpleWritableCell for LEEStateCellRef<'_> { } } +/// State at the last L1-finalized block, written atomically with +/// [`FinalBlockMetaCellRef`]. +#[derive(BorshDeserialize)] +pub struct FinalLeeStateCellOwned(pub V03State); + +impl SimpleStorableCell for FinalLeeStateCellOwned { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_FINAL_LEE_STATE_KEY; + const CF_NAME: &'static str = CF_LEE_STATE_NAME; +} + +impl SimpleReadableCell for FinalLeeStateCellOwned {} + +#[derive(BorshSerialize)] +pub struct FinalLeeStateCellRef<'state>(pub &'state V03State); + +impl SimpleStorableCell for FinalLeeStateCellRef<'_> { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_FINAL_LEE_STATE_KEY; + const CF_NAME: &'static str = CF_LEE_STATE_NAME; +} + +impl SimpleWritableCell for FinalLeeStateCellRef<'_> { + fn value_constructor(&self) -> DbResult> { + borsh::to_vec(&self).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize final state".to_owned())) + }) + } +} + +/// `(id, hash)` of the last L1-finalized block, paired with [`FinalLeeStateCellRef`]. +#[derive(BorshDeserialize)] +pub struct FinalBlockMetaCellOwned(pub BlockMeta); + +impl SimpleStorableCell for FinalBlockMetaCellOwned { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_FINAL_BLOCK_META_KEY; + const CF_NAME: &'static str = CF_META_NAME; +} + +impl SimpleReadableCell for FinalBlockMetaCellOwned {} + +#[derive(BorshSerialize)] +pub struct FinalBlockMetaCellRef<'blockmeta>(pub &'blockmeta BlockMeta); + +impl SimpleStorableCell for FinalBlockMetaCellRef<'_> { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_FINAL_BLOCK_META_KEY; + const CF_NAME: &'static str = CF_META_NAME; +} + +impl SimpleWritableCell for FinalBlockMetaCellRef<'_> { + fn value_constructor(&self) -> DbResult> { + borsh::to_vec(&self).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize final block meta".to_owned()), + ) + }) + } +} + #[derive(Debug, BorshSerialize, BorshDeserialize)] pub struct LastFinalizedBlockIdCell(pub Option); diff --git a/lez/storage/src/sequencer/tests.rs b/lez/storage/src/sequencer/tests.rs new file mode 100644 index 00000000..0b6ce862 --- /dev/null +++ b/lez/storage/src/sequencer/tests.rs @@ -0,0 +1,283 @@ +use common::test_utils::produce_dummy_block; +use lee::{Account, AccountId}; +use tempfile::tempdir; + +use super::*; + +fn marker_id() -> AccountId { + AccountId::new([1; 32]) +} + +/// A state distinguishable by the marker account's balance, so tests can tell +/// which snapshot a write persisted. +/// +/// TODO: is this a bit too much of a hot-fix for test snapshot? +fn state_with_balance(balance: u128) -> V03State { + V03State::new().with_public_accounts([( + marker_id(), + Account { + balance, + ..Account::default() + }, + )]) +} + +fn dbio_with_genesis(path: &Path) -> (RocksDBIO, Block) { + let genesis = produce_dummy_block(1, None, vec![]); + let dbio = RocksDBIO::create(path, &genesis, &state_with_balance(100)).unwrap(); + (dbio, genesis) +} + +fn stored_balance(dbio: &RocksDBIO) -> u128 { + dbio.get_lee_state() + .unwrap() + .get_account_by_id(marker_id()) + .balance +} + +#[test] +fn store_followed_block_persists_new_block_and_state() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2, &state_with_balance(200), false) + .unwrap(); + + let stored = dbio.get_block(2).unwrap().expect("block 2 is stored"); + assert_eq!(stored.header.hash, block2.header.hash); + assert!(matches!(stored.bedrock_status, BedrockStatus::Pending)); + assert_eq!( + dbio.latest_block_meta().unwrap().expect("meta is set").id, + 2 + ); + assert_eq!(stored_balance(&dbio), 200); +} + +#[test] +fn store_followed_block_finalized_marks_block() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2, &state_with_balance(200), true) + .unwrap(); + + let stored = dbio.get_block(2).unwrap().expect("block 2 is stored"); + assert!(matches!(stored.bedrock_status, BedrockStatus::Finalized)); +} + +#[test] +fn store_followed_block_redelivery_is_a_noop_and_keeps_finalized() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2, &state_with_balance(200), true) + .unwrap(); + dbio.store_followed_block(&block2, &state_with_balance(300), false) + .unwrap(); + + let stored = dbio.get_block(2).unwrap().expect("block 2 is stored"); + assert!( + matches!(stored.bedrock_status, BedrockStatus::Finalized), + "re-delivery must not demote a finalized block" + ); + assert_eq!( + stored_balance(&dbio), + 200, + "re-delivery must not overwrite the persisted state" + ); +} + +#[test] +fn store_followed_blocks_batch_lands_meta_and_state_on_last_block() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + // Block 2 is already stored (own production); one update then finalizes it + // and adopts block 3. + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2, &state_with_balance(200), false) + .unwrap(); + + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + let head_tip = BlockMeta { + id: 3, + hash: block3.header.hash, + }; + dbio.store_followed_blocks( + &[(&block2, true), (&block3, false)], + Some(&head_tip), + &state_with_balance(300), + None, + ) + .unwrap(); + + let stored2 = dbio.get_block(2).unwrap().expect("block 2 is stored"); + assert!(matches!(stored2.bedrock_status, BedrockStatus::Finalized)); + let stored3 = dbio.get_block(3).unwrap().expect("block 3 is stored"); + assert!(matches!(stored3.bedrock_status, BedrockStatus::Pending)); + + // Meta and state land together on the last block of the batch. + let meta = dbio.latest_block_meta().unwrap().expect("meta is set"); + assert_eq!(meta.id, 3); + assert_eq!(meta.hash, block3.header.hash); + assert_eq!(stored_balance(&dbio), 300); +} + +#[test] +fn final_snapshot_round_trips_and_is_absent_on_fresh_store() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + // Fresh store: no finalization observed yet. + assert!(dbio.get_final_snapshot().unwrap().is_none()); + + // A follow update that finalizes block 2 lands the snapshot in the same batch. + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + let final_meta = BlockMeta { + id: 2, + hash: block2.header.hash, + }; + dbio.store_followed_blocks( + &[(&block2, true)], + Some(&final_meta), + &state_with_balance(300), + Some((&state_with_balance(200), &final_meta)), + ) + .unwrap(); + + let (final_state, meta) = dbio + .get_final_snapshot() + .unwrap() + .expect("final snapshot is stored"); + assert_eq!(meta.id, 2); + assert_eq!(meta.hash, block2.header.hash); + assert_eq!(final_state.get_account_by_id(marker_id()).balance, 200); + // The head state is stored independently of the final snapshot. + assert_eq!(stored_balance(&dbio), 300); +} + +#[test] +fn store_followed_block_overwrites_competing_block_at_same_id() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + let block2a = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2a, &state_with_balance(200), false) + .unwrap(); + + // A reorg replaces block 2: the competing block wins the slot. + let block2b = produce_dummy_block(2, Some(HashType([9; 32])), vec![]); + dbio.store_followed_block(&block2b, &state_with_balance(300), false) + .unwrap(); + + let stored = dbio.get_block(2).unwrap().expect("block 2 is stored"); + assert_eq!(stored.header.hash, block2b.header.hash); + assert!(matches!(stored.bedrock_status, BedrockStatus::Pending)); + assert_eq!(stored_balance(&dbio), 300); + + // The tip meta must follow the reorg winner, or a restart seeds the chain + // from the orphaned block's hash. + let meta = dbio.latest_block_meta().unwrap().expect("meta is set"); + assert_eq!(meta.id, 2); + assert_eq!(meta.hash, block2b.header.hash); +} + +#[test] +fn net_shortening_reorg_drops_stale_blocks() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + let block2a = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2a, &state_with_balance(200), false) + .unwrap(); + let block3 = produce_dummy_block(3, Some(block2a.header.hash), vec![]); + dbio.store_followed_block(&block3, &state_with_balance(300), false) + .unwrap(); + + // A shorter competing chain wins: block 2 is replaced, block 3 gets no + // replacement. + let block2b = produce_dummy_block(2, Some(HashType([9; 32])), vec![]); + let head_tip = BlockMeta { + id: 2, + hash: block2b.header.hash, + }; + dbio.store_followed_blocks( + &[(&block2b, false)], + Some(&head_tip), + &state_with_balance(400), + None, + ) + .unwrap(); + + let stored2 = dbio.get_block(2).unwrap().expect("block 2 is stored"); + assert_eq!(stored2.header.hash, block2b.header.hash); + assert!( + dbio.get_block(3).unwrap().is_none(), + "stale block above the new head must be deleted, or restart replay panics on its broken link" + ); + let meta = dbio.latest_block_meta().unwrap().expect("meta is set"); + assert_eq!(meta.id, 2); + assert_eq!(meta.hash, block2b.header.hash); + assert_eq!(stored_balance(&dbio), 400); +} + +#[test] +fn shrink_only_reorg_rewinds_tip_meta() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2, &state_with_balance(200), false) + .unwrap(); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![]); + dbio.store_followed_block(&block3, &state_with_balance(300), false) + .unwrap(); + + // Orphan-only update: block 3 falls off the branch with no replacement. + let head_tip = BlockMeta { + id: 2, + hash: block2.header.hash, + }; + dbio.store_followed_blocks(&[], Some(&head_tip), &state_with_balance(200), None) + .unwrap(); + + assert!( + dbio.get_block(3).unwrap().is_none(), + "the orphaned block must not survive the tip rewind" + ); + let meta = dbio.latest_block_meta().unwrap().expect("meta is set"); + assert_eq!(meta.id, 2); + assert_eq!(meta.hash, block2.header.hash); + assert_eq!(stored_balance(&dbio), 200); +} + +#[test] +fn produced_block_below_disk_head_pins_meta_and_prunes() { + let temp_dir = tempdir().unwrap(); + let (dbio, genesis) = dbio_with_genesis(temp_dir.path()); + + let block2a = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.store_followed_block(&block2a, &state_with_balance(200), false) + .unwrap(); + let block3 = produce_dummy_block(3, Some(block2a.header.hash), vec![]); + dbio.store_followed_block(&block3, &state_with_balance(300), false) + .unwrap(); + + // Producing at height 2 while the disk head is still 3: the produce path + // pins the tip meta to the produced block and drops the stale suffix in + // the same write, mirroring the follow path. + let block2b = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.atomic_update(&block2b, &[], vec![], &state_with_balance(400)) + .unwrap(); + + let stored2 = dbio.get_block(2).unwrap().expect("block 2 is stored"); + assert_eq!(stored2.header.hash, block2b.header.hash); + assert!(dbio.get_block(3).unwrap().is_none()); + let meta = dbio.latest_block_meta().unwrap().expect("meta is set"); + assert_eq!(meta.id, 2); + assert_eq!(meta.hash, block2b.header.hash); + assert_eq!(stored_balance(&dbio), 400); +} diff --git a/test_fixtures/src/lib.rs b/test_fixtures/src/lib.rs index 75eaf941..596354ab 100644 --- a/test_fixtures/src/lib.rs +++ b/test_fixtures/src/lib.rs @@ -12,7 +12,7 @@ use lee_core::Commitment; use log::{debug, error}; use sequencer_core::config::GenesisAction; use sequencer_service::SequencerHandle; -use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuilder}; +use sequencer_service_rpc::{RpcClient as _, SequencerClient}; use serde::Serialize; use tempfile::TempDir; use testcontainers::compose::DockerCompose; @@ -340,9 +340,7 @@ impl TestContextBuilder { setup_indexer(bedrock_addr, config::bedrock_channel_id(), None) .await .context("Failed to setup Indexer")?; - let indexer_url = config::addr_to_url(config::UrlProtocol::Ws, indexer_handle.addr()) - .context("Failed to convert indexer addr to URL")?; - let indexer_client = IndexerClient::new(&indexer_url) + let indexer_client = setup::indexer_client(indexer_handle.addr()) .await .context("Failed to create indexer client")?; Some(IndexerComponents { @@ -404,10 +402,7 @@ impl TestContextBuilder { .context("Failed to initialize private accounts in wallet")?; } - let sequencer_url = config::addr_to_url(config::UrlProtocol::Http, sequencer_handle.addr()) - .context("Failed to convert sequencer addr to URL")?; - let sequencer_client = SequencerClientBuilder::default() - .build(sequencer_url) + let sequencer_client = setup::sequencer_client(sequencer_handle.addr()) .context("Failed to create sequencer client")?; Ok(TestContext { diff --git a/test_fixtures/src/setup.rs b/test_fixtures/src/setup.rs index ce816919..6c60a7e8 100644 --- a/test_fixtures/src/setup.rs +++ b/test_fixtures/src/setup.rs @@ -7,8 +7,12 @@ use anyhow::{Context as _, Result, bail}; use indexer_service::{ChannelId, IndexerHandle}; use lee::{AccountId, PrivateKey, PublicKey}; use log::{debug, warn}; -use sequencer_core::block_store::{DbDump, SequencerStore}; +use sequencer_core::{ + block_publisher::ED25519_SECRET_KEY_SIZE, + block_store::{DbDump, SequencerStore}, +}; use sequencer_service::{GenesisAction, SequencerHandle}; +use sequencer_service_rpc::{SequencerClient, SequencerClientBuilder}; use tempfile::TempDir; use testcontainers::compose::DockerCompose; use wallet::{ @@ -20,6 +24,7 @@ use wallet::{ use crate::{ BEDROCK_SERVICE_PORT, BEDROCK_SERVICE_WITH_OPEN_PORT, config::{self, InitialPrivateAccountForWallet}, + indexer_client::IndexerClient, private_mention, public_mention, }; @@ -29,6 +34,7 @@ pub struct SequencerSetup { channel_id: ChannelId, genesis_transactions: Option>, cross_zone: Option, + bedrock_signing_key: Option<[u8; ED25519_SECRET_KEY_SIZE]>, } impl SequencerSetup { @@ -40,6 +46,7 @@ impl SequencerSetup { channel_id: config::bedrock_channel_id(), genesis_transactions: None, cross_zone: None, + bedrock_signing_key: None, } } @@ -67,6 +74,15 @@ impl SequencerSetup { self } + /// Pre-write a bedrock (Ed25519, 32-byte seed) signing key into the home + /// before boot, so tests know the sequencer's public key in advance (e.g. + /// to accredit a committee member that has not started yet). + #[must_use] + pub const fn with_bedrock_signing_key(mut self, key: [u8; ED25519_SECRET_KEY_SIZE]) -> Self { + self.bedrock_signing_key = Some(key); + self + } + /// Set up the sequencer in a fresh temporary home directory, returning the /// owning [`TempDir`] alongside the handle. pub async fn setup(self) -> Result<(SequencerHandle, TempDir)> { @@ -88,10 +104,16 @@ impl SequencerSetup { channel_id, genesis_transactions, cross_zone, + bedrock_signing_key, } = self; debug!("Using sequencer home at {}", home.display()); + if let Some(key_bytes) = bedrock_signing_key { + std::fs::write(home.join("bedrock_signing_key"), key_bytes) + .context("Failed to write pre-generated bedrock signing key")?; + } + let genesis_transactions = if let Some(genesis) = genesis_transactions { genesis } else { @@ -119,7 +141,7 @@ impl SequencerSetup { ) .context("Failed to create Sequencer config")?; - sequencer_service::run(config, 0) + sequencer_service::run(config, SocketAddr::from(([127, 0, 0, 1], 0))) .await .context("Failed to run Sequencer Service") } @@ -139,6 +161,24 @@ fn load_prebuilt_dump() -> Result { DbDump::from_bytes(&bytes).context("Failed to deserialize prebuilt db dump") } +/// Builds an HTTP RPC client for the sequencer at `addr`. +pub fn sequencer_client(addr: SocketAddr) -> Result { + let url = config::addr_to_url(config::UrlProtocol::Http, addr) + .context("Failed to build sequencer URL")?; + SequencerClientBuilder::default() + .build(url) + .context("Failed to build sequencer client") +} + +/// Builds a WebSocket RPC client for the indexer at `addr`. +pub async fn indexer_client(addr: SocketAddr) -> Result { + let url = config::addr_to_url(config::UrlProtocol::Ws, addr) + .context("Failed to build indexer URL")?; + IndexerClient::new(&url) + .await + .context("Failed to build indexer client") +} + pub async fn setup_bedrock_node() -> Result<(DockerCompose, SocketAddr)> { let manifest_dir = env!("CARGO_MANIFEST_DIR"); let bedrock_compose_path = PathBuf::from(manifest_dir).join("../bedrock/docker-compose.yml");