diff --git a/.github/actions/run-in-ci-image/action.yml b/.github/actions/run-in-ci-image/action.yml index 3fafe38cc..5cef529d5 100644 --- a/.github/actions/run-in-ci-image/action.yml +++ b/.github/actions/run-in-ci-image/action.yml @@ -43,7 +43,11 @@ runs: # daemon resolves compose bind mounts against the host, and the test # binaries bake CARGO_MANIFEST_DIR in at compile time, so the path has # to mean the same thing inside and outside the container. + # + # Recipes branch on GITHUB_ACTIONS, which the runner sets but `docker + # run` does not inherit. Valueless `--env` forwards it from the host. docker run --rm \ + --env GITHUB_ACTIONS \ --network host \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$PWD:$PWD" \ diff --git a/Cargo.lock b/Cargo.lock index 26f1a2787..1a71d14de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1278,9 +1278,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" dependencies = [ "serde", ] @@ -2558,6 +2558,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + [[package]] name = "downcast-rs" version = "1.2.1" @@ -3149,6 +3155,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8878864ba14bb86e818a412bfd6f18f9eabd4ec0f008a28e8f7eb61db532fcf9" +dependencies = [ + "futures-core", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -6886,6 +6901,32 @@ dependencies = [ "sha3 0.11.0", ] +[[package]] +name = "mockall" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a6ceddfe3ce334925e96bf420fdb2dcee5bed6c632a168ece622676dadeaf8a" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cfe16fbe8a314aeec0b861ac24e60b1e123e97634bab045475b9d6a18416fd8" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "module-lattice" version = "0.2.3" @@ -7872,6 +7913,32 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -9503,7 +9570,7 @@ dependencies = [ "futures", "hex", "humantime-serde", - "itertools 0.14.0", + "kameo", "key_protocol", "lee", "lee_core", @@ -9521,6 +9588,7 @@ dependencies = [ "risc0-zkvm", "sequencer_core_metrics", "sequencer_stake_core", + "sequencer_storage_actor", "serde", "serde_json", "storage", @@ -9553,15 +9621,20 @@ dependencies = [ "bytesize", "common", "env_logger", + "futures", "hex", "kameo", "lee", "lee_core", "log", "mempool", + "mockall", "num-bigint 0.4.6", + "num-traits", "sequencer_core", - "storage", + "sequencer_stake_core", + "sequencer_storage_actor", + "system_accounts", "tempfile", "test_programs", "thiserror 2.0.18", @@ -9617,6 +9690,7 @@ dependencies = [ "sequencer_executor_actor", "sequencer_rpc_server_actor", "sequencer_stake_core", + "sequencer_storage_actor", "system_accounts", "tokio", "tokio-util", @@ -9661,6 +9735,23 @@ dependencies = [ "sequencer_stake_core", ] +[[package]] +name = "sequencer_storage_actor" +version = "0.1.0" +dependencies = [ + "common", + "itertools 0.14.0", + "kameo", + "lee", + "lee_core", + "log", + "mockall", + "storage", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + [[package]] name = "serde" version = "1.0.228" @@ -10482,6 +10573,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + [[package]] name = "test-case" version = "3.3.1" @@ -10528,6 +10625,7 @@ dependencies = [ "hex", "indexer_service", "jsonrpsee", + "kameo", "key_protocol", "lee", "lee_core", @@ -10539,6 +10637,7 @@ dependencies = [ "sequencer_service", "sequencer_service_rpc", "sequencer_stake_core", + "sequencer_storage_actor", "serde", "serde_json", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 1c294d202..ebc21a02a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "lez/sequencer/actors/executor", "lez/sequencer/actors/rpc_server", "lez/sequencer/actors/rpc_server/metrics", + "lez/sequencer/actors/storage", "lez/indexer/core", "lez/indexer/service", "lez/indexer/service/protocol", @@ -91,6 +92,7 @@ sequencer_service_rpc = { path = "lez/sequencer/service/rpc" } sequencer_executor_actor = { path = "lez/sequencer/actors/executor" } sequencer_rpc_server_actor = { path = "lez/sequencer/actors/rpc_server" } sequencer_rpc_server_actor_metrics = { path = "lez/sequencer/actors/rpc_server/metrics" } +sequencer_storage_actor = { path = "lez/sequencer/actors/storage" } sequencer_service = { path = "lez/sequencer/service" } indexer_core = { path = "lez/indexer/core" } indexer_service = { path = "lez/indexer/service" } @@ -202,6 +204,7 @@ tokio-retry = "0.3.0" schemars = "1.2" async-stream = "0.3.6" strum = { version = "0.28.0", features = ["derive"] } +mockall = "0.15.0" logos-blockchain-common-http-client = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "97d5e104fc20ce6e2fb404f8a9364afcb181a3e5" } logos-blockchain-key-management-system-service = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "97d5e104fc20ce6e2fb404f8a9364afcb181a3e5" } @@ -238,6 +241,7 @@ clap = { version = "4.5.42", features = ["derive", "env"] } reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"] } zeroize = "1" criterion = { version = "0.8", features = ["html_reports"] } +num-traits = "0.2.19" # Profile for leptos WASM release builds [profile.wasm-release] diff --git a/Justfile b/Justfile index 607b92817..9cfc8f119 100644 --- a/Justfile +++ b/Justfile @@ -51,7 +51,7 @@ test: # Regenerate the prebuilt sequencer db dump for fast TestContext::new() (needs Docker; commit the dump). regenerate-test-fixture: @echo "๐Ÿงช Regenerating test fixture" - RISC0_DEV_MODE=1 cargo run -p test_fixtures --bin regenerate_test_fixture + RISC0_DEV_MODE=1 RUST_LOG=info cargo run -p test_fixtures --bin regenerate_test_fixture # Regenerate the committed Grafana dashboards from the Rust generator # (tools/dashboard_gen) and commit the result. CI checks these are up to date. diff --git a/lez/chain_state/src/chain.rs b/lez/chain_state/src/chain.rs index 39e1efbaa..c96bacc26 100644 --- a/lez/chain_state/src/chain.rs +++ b/lez/chain_state/src/chain.rs @@ -1,6 +1,8 @@ //! Two-tier chain state: a reorg-able `head` the sequencer builds on, plus an //! irreversible `final` tier. +use std::sync::Arc; + use common::block::Block; use lee::V03State; use log::warn; @@ -26,11 +28,11 @@ pub struct HeadEntry { /// Only the final tier stalls: an invalid `adopted` block just freezes the /// head tip and self-heals via reorg or finalization. pub struct ChainState { - final_state: V03State, + final_state: Arc, final_tip: Option, final_stall: Option, - head_state: V03State, + head_state: Arc, head_blocks: Vec, } @@ -44,8 +46,9 @@ impl ChainState { /// State restored from a persisted final tier; head mirrors final. #[must_use] pub fn from_final(final_state: V03State, final_tip: Option) -> Self { + let final_state = Arc::new(final_state); Self { - head_state: final_state.clone(), + head_state: Arc::clone(&final_state), final_state, final_tip, head_blocks: Vec::new(), @@ -55,22 +58,36 @@ impl ChainState { /// State the sequencer builds its next block on. #[must_use] - pub const fn head_state(&self) -> &V03State { + pub fn head_state(&self) -> &V03State { &self.head_state } + /// A shared handle on the head state, for callers that need to own it. + #[must_use] + pub fn share_head_state(&self) -> Arc { + Arc::clone(&self.head_state) + } + /// Mutable access to the head state. Bypasses the `head_blocks` invariant, so /// it is meant for tests and low-level callers. + /// + /// Copies the state while a handle from [`Self::share_head_state`] is alive. #[must_use] - pub const fn head_state_mut(&mut self) -> &mut V03State { - &mut self.head_state + pub fn head_state_mut(&mut self) -> &mut V03State { + Arc::make_mut(&mut self.head_state) } #[must_use] - pub const fn final_state(&self) -> &V03State { + pub fn final_state(&self) -> &V03State { &self.final_state } + /// A shared handle on the final state, for callers that need to own it. + #[must_use] + pub fn share_final_state(&self) -> Arc { + Arc::clone(&self.final_state) + } + /// Parent the next produced block must chain on. #[must_use] pub fn head_tip(&self) -> Option { @@ -139,12 +156,12 @@ impl ChainState { .position(|entry| entry.block.header.block_id >= block.header.block_id); let (mut scratch, tip) = match reorg_at { // continue from the tip - None => (self.head_state.clone(), self.head_tip()), + None => (Arc::clone(&self.head_state), self.head_tip()), // reorg upto `idx` Some(idx) => self.replay_head_prefix(idx), }; - match apply_block(tip.as_ref(), block, &mut scratch) { + match apply_block(tip.as_ref(), block, Arc::make_mut(&mut scratch)) { Ok(()) => { // now that `apply_block` succeeded, actually reorg the head if let Some(idx) = reorg_at { @@ -213,7 +230,11 @@ impl ChainState { /// persisted โ€” that would need a sidecar `block_id -> MsgId` cell); later /// orphan/finalize events correlate by block hash. pub fn restore_head_block(&mut self, block: Block) -> Result<(), BlockIngestError> { - apply_block(self.head_tip().as_ref(), &block, &mut self.head_state)?; + apply_block( + self.head_tip().as_ref(), + &block, + Arc::make_mut(&mut self.head_state), + )?; let this_msg = MsgId::from(block.header.hash.0); self.head_blocks.push(HeadEntry { this_msg, block }); Ok(()) @@ -250,8 +271,12 @@ impl ChainState { fn finalize_through(&mut self, idx: usize) { let finalized: Vec = self.head_blocks.drain(0..=idx).collect(); for entry in finalized { - apply_block(self.final_tip.as_ref(), &entry.block, &mut self.final_state) - .expect("validated head block must apply to the final tier"); + apply_block( + self.final_tip.as_ref(), + &entry.block, + Arc::make_mut(&mut self.final_state), + ) + .expect("validated head block must apply to the final tier"); self.final_tip = Some(Tip::from(&entry.block)); } self.final_stall = None; @@ -270,8 +295,8 @@ impl ChainState { return AcceptOutcome::AlreadyApplied; } - let mut scratch = self.final_state.clone(); - match apply_block(self.final_tip.as_ref(), block, &mut scratch) { + let mut scratch = Arc::clone(&self.final_state); + match apply_block(self.final_tip.as_ref(), block, Arc::make_mut(&mut scratch)) { Ok(()) => { self.final_state = scratch; self.final_tip = Some(Tip::from(block)); @@ -281,7 +306,7 @@ impl ChainState { // orders orphans before their finalized replacement), so its // txs are back in the caller's mempool. self.head_blocks.clear(); - self.head_state = self.final_state.clone(); + self.head_state = Arc::clone(&self.final_state); AcceptOutcome::Applied } Err(err) => { @@ -297,11 +322,11 @@ impl ChainState { } /// State and tip after replaying `head_blocks[..count]` on the final tier. - fn replay_head_prefix(&self, count: usize) -> (V03State, Option) { - let mut state = self.final_state.clone(); + fn replay_head_prefix(&self, count: usize) -> (Arc, Option) { + let mut state = Arc::clone(&self.final_state); let mut tip = self.final_tip.clone(); for entry in &self.head_blocks[..count] { - apply_block(tip.as_ref(), &entry.block, &mut state) + apply_block(tip.as_ref(), &entry.block, Arc::make_mut(&mut state)) .expect("validated head blocks must replay"); tip = Some(Tip::from(&entry.block)); } @@ -337,14 +362,15 @@ mod tests { /// `head_state` equals `final_state` replayed through `head_blocks`. fn assert_head_matches_replay(chain: &ChainState) { - let mut state = chain.final_state.clone(); + let mut state = Arc::clone(&chain.final_state); let mut tip = chain.final_tip.clone(); for entry in &chain.head_blocks { - apply_block(tip.as_ref(), &entry.block, &mut state).expect("head blocks must replay"); + apply_block(tip.as_ref(), &entry.block, Arc::make_mut(&mut state)) + .expect("head blocks must replay"); tip = Some(Tip::from(&entry.block)); } assert_eq!( - borsh::to_vec(&state).expect("state serializes"), + borsh::to_vec(state.as_ref()).expect("state serializes"), borsh::to_vec(chain.head_state()).expect("state serializes"), "head_state must equal final_state replayed through head_blocks" ); diff --git a/lez/sequencer/actors/executor/Cargo.toml b/lez/sequencer/actors/executor/Cargo.toml index 969cefb99..d963c1b76 100644 --- a/lez/sequencer/actors/executor/Cargo.toml +++ b/lez/sequencer/actors/executor/Cargo.toml @@ -2,7 +2,7 @@ name = "sequencer_executor_actor" version = "0.1.0" edition = "2024" -license = { workspace = true } +license.workspace = true [lints] workspace = true @@ -12,22 +12,28 @@ sequencer_core.workspace = true common.workspace = true lee_core.workspace = true mempool.workspace = true -storage.workspace = true +sequencer_storage_actor.workspace = true kameo.workspace = true tokio.workspace = true tokio-util.workspace = true +futures.workspace = true log.workspace = true anyhow.workspace = true thiserror.workspace = true hex.workspace = true +num-traits.workspace = true [dev-dependencies] lee.workspace = true sequencer_core = { workspace = true, features = ["mock"] } +sequencer_storage_actor = { workspace = true, features = ["mock"] } +system_accounts.workspace = true +sequencer_stake_core.workspace = true test_programs.workspace = true env_logger.workspace = true +mockall.workspace = true tempfile.workspace = true bytesize.workspace = true num-bigint.workspace = true diff --git a/lez/sequencer/actors/executor/src/actor.rs b/lez/sequencer/actors/executor/src/actor.rs index 96dc9f039..64ba7663d 100644 --- a/lez/sequencer/actors/executor/src/actor.rs +++ b/lez/sequencer/actors/executor/src/actor.rs @@ -1,10 +1,12 @@ use common::{block::Block, transaction::LeeTransaction}; +use futures::{StreamExt as _, TryFutureExt as _, TryStreamExt as _, future::ready, stream}; use kameo::{ Actor, actor::{ActorRef, WeakActorRef}, error::ActorStopReason, mailbox::{MailboxReceiver, Signal}, message::{Context, Message}, + reply::DelegatedReply, }; use lee_core::{ BlockId, @@ -18,6 +20,7 @@ use sequencer_core::{ config::SequencerConfig, task_group::TaskGroup, }; +use sequencer_storage_actor::{StorageActor, StorageActorTrait}; use tokio::select; use tokio_util::sync::CancellationToken; @@ -31,10 +34,14 @@ use crate::{ }, }; +/// How many block lookups a single [`GetBlockRange`] keeps in flight. +const BLOCK_RANGE_CONCURRENCY: usize = 16; + // TODO: Remove `BP` once this part is moved to a separate actor -pub struct ExecutorActor { - sequencer: SequencerCore, +pub struct ExecutorActor { mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>, + sequencer: SequencerCore, + storage_ref: ActorRef, // --- TODO: Remove these fields below --- /// Cancelled when the publisher's drive task terminates (e.g. a panicked @@ -46,16 +53,20 @@ pub struct ExecutorActor { background_tasks: Vec, } -impl ExecutorActor { - pub async fn new(config: SequencerConfig) -> Self { - let (sequencer, mempool_handle) = SequencerCore::::start_from_config(config).await; +impl ExecutorActor { + pub async fn new(config: SequencerConfig, storage_ref: ActorRef) -> Self { + // TODO: Leave storage_ref as a top-level field only in `ExecutorActor`, + // while moving `SequencerCore` code into this actor. + let (sequencer, mempool_handle) = + SequencerCore::::start_from_config(config, storage_ref.clone()).await; let driver_cancellation = sequencer.block_publisher().driver_cancellation(); let background_tasks = sequencer.background_tasks(); Self { - sequencer, mempool_handle, + sequencer, + storage_ref, driver_cancellation, background_tasks, } @@ -69,7 +80,9 @@ impl ExecutorActor { } } -impl Actor for ExecutorActor { +impl Actor + for ExecutorActor +{ type Args = Self; type Error = Error; @@ -116,7 +129,9 @@ impl Actor for ExecutorActor { } } -impl Message for ExecutorActor { +impl Message + for ExecutorActor +{ type Reply = Result<()>; async fn handle( @@ -134,11 +149,11 @@ impl Message for Executo // channel would carry two chains from there and nothing resolves that. // The head rewinds under us when the sdk orphans our own unfinalized // blocks, and recovers once they finalize, so this is a wait. - if let Some(high_water) = self.sequencer.rewound_below_published() { + if let Some(high_water) = self.sequencer.rewound_below_published().await { warn!( "Skipping turn: head rewound to {} but block {high_water} is already inscribed; \ waiting for the channel to restore it", - self.sequencer.next_block_height().saturating_sub(1), + self.sequencer.next_block_height().await.saturating_sub(1), ); return Ok(()); } @@ -161,7 +176,9 @@ impl Message for Executo } } -impl Message for ExecutorActor { +impl Message + for ExecutorActor +{ type Reply = Result<()>; async fn handle( @@ -175,7 +192,9 @@ impl Message for Executor } } -impl Message for ExecutorActor { +impl Message + for ExecutorActor +{ type Reply = Result>; async fn handle( @@ -183,34 +202,45 @@ impl Message for ExecutorAct GetBlock { block_id }: GetBlock, _ctx: &mut Context, ) -> Self::Reply { - self.sequencer - .block_store() - .get_block_at_id(block_id) + self.storage_ref + .ask(sequencer_storage_actor::protocol::GetBlock { block_id }) + .await .map_err(Into::into) } } -impl Message for ExecutorActor { - type Reply = Result>; +impl Message + for ExecutorActor +{ + type Reply = DelegatedReply>>; async fn handle( &mut self, GetBlockRange { range }: GetBlockRange, - _ctx: &mut Context, + ctx: &mut Context, ) -> Self::Reply { - range - .map_while(|block_id| { - self.sequencer - .block_store() - .get_block_at_id(block_id) - .map_err(Into::into) - .transpose() - }) - .collect::>>() + let storage_ref = self.storage_ref.clone(); + + ctx.spawn(async move { + stream::iter(range.into_inner()) + .map(|block_id| { + storage_ref + .ask(sequencer_storage_actor::protocol::GetBlock { block_id }) + .into_future() + .map_err(Into::into) + }) + .buffered(BLOCK_RANGE_CONCURRENCY) + .try_take_while(|block_opt| ready(Ok(block_opt.is_some()))) + .try_filter_map(|block_opt| ready(Ok(block_opt))) + .try_collect() + .await + }) } } -impl Message for ExecutorActor { +impl Message + for ExecutorActor +{ type Reply = Result; async fn handle( @@ -218,11 +248,13 @@ impl Message for Execu GetLastBlockId: GetLastBlockId, _ctx: &mut Context, ) -> Self::Reply { - Ok(self.sequencer.chain_height()) + Ok(self.sequencer.chain_height().await) } } -impl Message for ExecutorActor { +impl + Message for ExecutorActor +{ type Reply = Balance; async fn handle( @@ -232,24 +264,30 @@ impl Message for Ex ) -> Self::Reply { self.sequencer .with_state(|state| state.get_account_by_id(account_id).balance) + .await } } -impl Message for ExecutorActor { - type Reply = Option<(LeeTransaction, BlockId)>; +impl Message + for ExecutorActor +{ + type Reply = Result>; async fn handle( &mut self, GetTransaction { tx_hash }: GetTransaction, _ctx: &mut Context, ) -> Self::Reply { - self.sequencer - .block_store() - .get_transaction_by_hash(tx_hash) + self.storage_ref + .ask(sequencer_storage_actor::protocol::GetTransactionByHash { hash: tx_hash }) + .await + .map_err(Into::into) } } -impl Message for ExecutorActor { +impl + Message for ExecutorActor +{ type Reply = Vec; async fn handle( @@ -257,16 +295,20 @@ impl Message for Exe GetAccountNonces { account_ids }: GetAccountNonces, _ctx: &mut Context, ) -> Self::Reply { - self.sequencer.with_state(|state| { - account_ids - .into_iter() - .map(|account_id| state.get_account_by_id(account_id).nonce) - .collect() - }) + self.sequencer + .with_state(|state| { + account_ids + .into_iter() + .map(|account_id| state.get_account_by_id(account_id).nonce) + .collect() + }) + .await } } -impl Message for ExecutorActor { +impl + Message for ExecutorActor +{ type Reply = ( Vec>, lee_core::CommitmentSetDigest, @@ -277,17 +319,21 @@ impl Message for Exe GetProofsAndRoot { commitments }: GetProofsAndRoot, _ctx: &mut Context, ) -> Self::Reply { - self.sequencer.with_state(|state| { - let proofs = commitments - .iter() - .map(|commitment| state.get_proof_for_commitment(commitment)) - .collect(); - (proofs, state.commitment_root()) - }) + self.sequencer + .with_state(|state| { + let proofs = commitments + .iter() + .map(|commitment| state.get_proof_for_commitment(commitment)) + .collect(); + (proofs, state.commitment_root()) + }) + .await } } -impl Message for ExecutorActor { +impl Message + for ExecutorActor +{ type Reply = GetAccountReply; async fn handle( @@ -298,12 +344,15 @@ impl Message for ExecutorA GetAccountReply { account: self .sequencer - .with_state(|state| state.get_account_by_id(account_id)), + .with_state(|state| state.get_account_by_id(account_id)) + .await, } } } -impl Message for ExecutorActor { +impl Message + for ExecutorActor +{ type Reply = GetChannelIdReply; async fn handle( @@ -317,8 +366,8 @@ impl Message for Executo } } -impl Message - for ExecutorActor +impl + Message for ExecutorActor { type Reply = Result; @@ -327,10 +376,14 @@ impl Message GetCrossZoneDeadLetters: GetCrossZoneDeadLetters, _ctx: &mut Context, ) -> Self::Reply { - let (total_retired, retained) = self.sequencer.cross_zone_dead_letters()?; + let (total_retired, retained) = self + .sequencer + .cross_zone_dead_letters() + .await + .map_err(Error::CrossZoneDeadLettersUnavailable)?; Ok(GetCrossZoneDeadLettersReply { total_retired, - retained, + retained: retained.into_iter().collect(), }) } } diff --git a/lez/sequencer/actors/executor/src/error.rs b/lez/sequencer/actors/executor/src/error.rs index e35bda128..9e5c5f5d4 100644 --- a/lez/sequencer/actors/executor/src/error.rs +++ b/lez/sequencer/actors/executor/src/error.rs @@ -9,9 +9,24 @@ pub enum Error { #[error("The mempool is full")] MempoolIsFull, - #[error("Storage error")] - StorageError(#[from] storage::error::DbError), + #[error("Storage request failed")] + StorageRequestFailed( + #[source] kameo::error::SendError, + ), - #[error(transparent)] - BlockProductionFailed(anyhow::Error), + #[error("Block production failed")] + BlockProductionFailed(#[source] anyhow::Error), + + #[error("Failed to read the cross-zone dead letter")] + CrossZoneDeadLettersUnavailable(#[source] anyhow::Error), +} + +/// A dummy struct replacing message type in [`kameo::error::SendError`] +/// as we don't want to expose the message type in the public API. +pub struct NoMatter; + +impl From> for Error { + fn from(err: kameo::error::SendError) -> Self { + Self::StorageRequestFailed(err.map_msg(|_| NoMatter)) + } } diff --git a/lez/sequencer/actors/executor/src/protocol.rs b/lez/sequencer/actors/executor/src/protocol.rs index e4dc73ee5..f5fef80bf 100644 --- a/lez/sequencer/actors/executor/src/protocol.rs +++ b/lez/sequencer/actors/executor/src/protocol.rs @@ -6,7 +6,9 @@ use lee_core::{ BlockId, Commitment, account::{Account, AccountId}, }; -use sequencer_core::DeadLetterDispatchRecord; + +/// The widest range a [`GetBlockRange`] may span. +pub const MAX_BLOCK_RANGE_LEN: usize = 1024; #[derive(Copy, Clone)] pub struct ProduceBlock; @@ -20,7 +22,46 @@ pub struct GetBlock { } pub struct GetBlockRange { - pub range: RangeInclusive, + pub range: BoundedRangeInclusive<{ MAX_BLOCK_RANGE_LEN }, BlockId>, +} + +pub struct BoundedRangeInclusive(RangeInclusive); + +#[derive(Debug, thiserror::Error)] +pub enum BoundedRangeInclusiveError { + #[error("Range goes backwards: start {start:?}, end {end:?}")] + RangeGoesBackwards { start: T, end: T }, + + #[error("Range is too large: max length is {N}")] + RangeTooLarge, +} + +impl BoundedRangeInclusive { + pub fn into_inner(self) -> RangeInclusive { + self.0 + } +} + +impl> TryFrom> + for BoundedRangeInclusive +{ + type Error = BoundedRangeInclusiveError; + + fn try_from(range: RangeInclusive) -> Result { + let (start, end) = range.into_inner(); + + let len = end + .checked_sub(&start) + .ok_or(BoundedRangeInclusiveError::RangeGoesBackwards { start, end })? + .try_into() + .map_err(|_err| BoundedRangeInclusiveError::RangeTooLarge)?; + + if len > N.saturating_sub(1) { + return Err(BoundedRangeInclusiveError::RangeTooLarge); + } + + Ok(Self(RangeInclusive::new(start, end))) + } } pub struct GetLastBlockId; @@ -62,5 +103,5 @@ pub struct GetCrossZoneDeadLetters; #[derive(Reply)] pub struct GetCrossZoneDeadLettersReply { pub total_retired: u64, - pub retained: Vec, + pub retained: Vec, } diff --git a/lez/sequencer/actors/executor/src/tests.rs b/lez/sequencer/actors/executor/src/tests.rs index 6995f3053..ec72c28aa 100644 --- a/lez/sequencer/actors/executor/src/tests.rs +++ b/lez/sequencer/actors/executor/src/tests.rs @@ -1,23 +1,31 @@ +use std::{collections::BTreeMap, time::Duration}; + use anyhow::Result; use bytesize::ByteSize; -use common::transaction::LeeTransaction; +use common::{ + HashType, + block::{BedrockStatus, Block, BlockBody, BlockHeader, BlockMeta}, + transaction::LeeTransaction, +}; use kameo::{actor::Spawn as _, error::SendError}; use lee::{ - AccountId, PrivateKey, PublicKey, PublicTransaction, + Account, AccountId, PrivateKey, PublicKey, PublicTransaction, Signature, V03State, public_transaction::{Message, WitnessSet}, }; +use mockall::predicate::{always, eq, function}; use num_bigint::BigUint; use sequencer_core::{ config::{BedrockConfig, SequencerConfig}, mock::MockBlockPublisher, }; -use tokio::test; +use sequencer_storage_actor::mock::MockStorageActor; +use tempfile::TempDir; +use tokio::{sync::mpsc, test, time::timeout}; use crate::{ExecutorActor, protocol}; -fn sequencer_config() -> (SequencerConfig, tempfile::TempDir) { - let home = tempfile::tempdir().expect("Failed to create tmp home dir"); - +fn sequencer_config() -> (SequencerConfig, TempDir) { + let home = TempDir::new().expect("Failed to create temporary home directory"); let config = SequencerConfig { home: home.path().to_path_buf(), max_num_tx_in_block: 10, @@ -62,13 +70,111 @@ fn test_transaction() -> LeeTransaction { PublicTransaction::new(message, witness_set).into() } +fn prepare_mock_storage_with_empty_genesis() -> MockStorageActor { + let genesis_block_meta = BlockMeta { + id: 1, + hash: HashType::default(), + }; + let genesis_block = Block { + header: BlockHeader { + block_id: genesis_block_meta.id, + prev_block_hash: HashType::default(), + hash: genesis_block_meta.hash, + timestamp: 0, + signature: Signature { value: [0; 64] }, + }, + body: BlockBody { + transactions: vec![], + }, + bedrock_status: BedrockStatus::Pending, + }; + let state = V03State::new().with_public_accounts([( + system_accounts::sequencer_stake_config_account_id(), + Account { + data: sequencer_stake_core::SequencerStakeConfig { + minimum_sequencer_stake: 0, + entries: BTreeMap::new(), + } + .to_bytes() + .try_into() + .expect("Sequencer stake config must fit into Data"), + ..Account::default() + }, + )]); + + let mut mock_storage = MockStorageActor::new(); + + mock_storage + .expect_handle_get_first_block_id() + .returning(|_, _| Ok(Some(1))); + + mock_storage + .expect_handle_get_last_block_id() + .returning(|_, _| Ok(Some(1))); + + let genesis_block_clone = genesis_block.clone(); + mock_storage + .expect_handle_get_block() + .with( + eq(sequencer_storage_actor::protocol::GetBlock { block_id: 1 }), + always(), + ) + .returning(move |_, _| Ok(Some(genesis_block_clone.clone()))); + + let state_clone = state.clone(); + mock_storage + .expect_handle_get_lee_state() + .returning(move |_, _| Ok(Some(state_clone.clone()))); + + let genesis_block_meta_clone = genesis_block_meta.clone(); + mock_storage + .expect_handle_get_final_snapshot() + .returning(move |_, _| Ok(Some((state.clone(), genesis_block_meta_clone.clone())))); + + mock_storage + .expect_handle_get_all_blocks() + .returning(move |_, _| Ok(vec![genesis_block.clone()])); + + mock_storage + .expect_handle_get_zone_checkpoint_bytes() + .returning(|_, _| Ok(None)); + + mock_storage + .expect_handle_get_zone_anchor() + .returning(|_, _| Ok(None)); + + mock_storage + .expect_handle_get_latest_block_meta() + .returning(move |_, _| Ok(Some(genesis_block_meta.clone()))); + + mock_storage + .expect_handle_raise_published_high_water() + .returning(|_, _| Ok(())); + + mock_storage + .expect_handle_get_dead_letter_dispatches() + .returning(|_, _| Ok(vec![])); + + mock_storage +} + #[test] async fn handle_transaction_fails_on_full_mempool() -> Result<()> { let _res = env_logger::try_init(); let (config, _home) = sequencer_config(); let mempool_max_size = config.mempool_max_size; - let executor = ExecutorActor::spawn(ExecutorActor::::new(config).await); + + let mock_storage = prepare_mock_storage_with_empty_genesis(); + let storage_ref = MockStorageActor::spawn(mock_storage); + + let executor = ExecutorActor::spawn( + ExecutorActor::::new(config, storage_ref.clone()).await, + ); + + storage_ref + .tell(sequencer_storage_actor::mock::Checkpoint) + .await?; // Fill mempool for _ in 0..mempool_max_size { @@ -90,3 +196,75 @@ async fn handle_transaction_fails_on_full_mempool() -> Result<()> { Ok(()) } + +#[test] +async fn get_block_range_keeps_executor_responsive() -> Result<()> { + /// Blocks the mock storage accepts but never answers. + const STALLED_FIRST: u64 = 100; + const STALLED_LAST: u64 = 105; + + let _res = env_logger::try_init(); + + let (config, _home) = sequencer_config(); + + let (stalled_tx, mut stalled_rx) = mpsc::unbounded_channel(); + #[expect( + clippy::collection_is_never_read, + reason = "Keeping the senders alive is what makes the asker wait forever" + )] + let mut held_replies = Vec::new(); + let mut mock_storage = prepare_mock_storage_with_empty_genesis(); + mock_storage + .expect_handle_get_block() + .with( + function(|msg: &sequencer_storage_actor::protocol::GetBlock| { + (STALLED_FIRST..=STALLED_LAST).contains(&msg.block_id) + }), + always(), + ) + .returning(move |_, ctx| { + // Holding the sender without ever sending leaves the asker waiting + // forever, while storage itself keeps draining its mailbox. + let (_delegated, reply_sender) = ctx.reply_sender(); + held_replies.extend(reply_sender); + stalled_tx.send(()).expect("Test must still be listening"); + Ok(None) + }); + + let storage_ref = MockStorageActor::spawn(mock_storage); + let executor = ExecutorActor::spawn( + ExecutorActor::::new(config, storage_ref.clone()).await, + ); + + let range = (STALLED_FIRST..=STALLED_LAST) + .try_into() + .expect("Range must be within the allowed length"); + let stalled_request = tokio::spawn({ + let executor = executor.clone(); + async move { executor.ask(protocol::GetBlockRange { range }).await } + }); + + stalled_rx + .recv() + .await + .expect("Executor must reach storage for the stalled range"); + + timeout( + Duration::from_secs(5), + executor.ask(protocol::GetLastBlockId), + ) + .await + .expect("Executor must answer while the stalled range is still in flight")?; + + assert!( + !stalled_request.is_finished(), + "The stalled range must still be waiting, otherwise nothing was proven" + ); + stalled_request.abort(); + + storage_ref + .tell(sequencer_storage_actor::mock::Checkpoint) + .await?; + + Ok(()) +} diff --git a/lez/sequencer/actors/rpc_server/Cargo.toml b/lez/sequencer/actors/rpc_server/Cargo.toml index 02bf9dfb3..6c6a84bad 100644 --- a/lez/sequencer/actors/rpc_server/Cargo.toml +++ b/lez/sequencer/actors/rpc_server/Cargo.toml @@ -2,7 +2,7 @@ name = "sequencer_rpc_server_actor" version = "0.1.0" edition = "2024" -license = { workspace = true } +license.workspace = true [lints] workspace = true diff --git a/lez/sequencer/actors/rpc_server/src/actor.rs b/lez/sequencer/actors/rpc_server/src/actor.rs index dfc8a551d..977dd0ffa 100644 --- a/lez/sequencer/actors/rpc_server/src/actor.rs +++ b/lez/sequencer/actors/rpc_server/src/actor.rs @@ -20,10 +20,10 @@ pub struct RpcServerActor { } impl RpcServerActor { - pub async fn new( - executor_ref: ActorRef>, + pub async fn new( listen_addr: SocketAddr, max_block_size: ByteSize, + executor_ref: ActorRef>, gossip_tx_publisher: Option, ) -> Result { let server = jsonrpsee::server::ServerBuilder::with_config( diff --git a/lez/sequencer/actors/rpc_server/src/actor/service.rs b/lez/sequencer/actors/rpc_server/src/actor/service.rs index f087c5523..d423d40ab 100644 --- a/lez/sequencer/actors/rpc_server/src/actor/service.rs +++ b/lez/sequencer/actors/rpc_server/src/actor/service.rs @@ -14,13 +14,13 @@ use sequencer_service_protocol::{ CrossZoneDeadLetter, CrossZoneDeadLetterReport, HashType, MembershipProof, Nonce, ProgramId, }; -pub struct Service { +pub struct Service { executor_ref: ActorRef>, max_block_size: ByteSize, gossip_tx_publisher: Option, } -impl Service { +impl Service { pub fn new( executor_ref: ActorRef>, max_block_size: ByteSize, @@ -37,7 +37,9 @@ impl Service { } #[async_trait] -impl sequencer_service_rpc::RpcServer for Service { +impl sequencer_service_rpc::RpcServer + for Service +{ async fn send_transaction(&self, tx: LeeTransaction) -> Result { sequencer_rpc_server_actor_metrics::increment_submitted_transactions_total(); @@ -132,10 +134,16 @@ impl sequencer_service_rpc::RpcServer start_block_id: BlockId, end_block_id: BlockId, ) -> Result, ErrorObjectOwned> { + let range = (start_block_id..=end_block_id).try_into().map_err(|err| { + ErrorObjectOwned::owned( + ErrorCode::InvalidParams.code(), + format!("Invalid block range: {err:#}"), + None::<()>, + ) + })?; + self.executor_ref - .ask(sequencer_executor_actor::protocol::GetBlockRange { - range: (start_block_id..=end_block_id), - }) + .ask(sequencer_executor_actor::protocol::GetBlockRange { range }) .await .map_err(internal_error) } diff --git a/lez/sequencer/actors/storage/Cargo.toml b/lez/sequencer/actors/storage/Cargo.toml new file mode 100644 index 000000000..4877a8e8e --- /dev/null +++ b/lez/sequencer/actors/storage/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "sequencer_storage_actor" +version = "0.1.0" +edition = "2024" +license.workspace = true + +[lints] +workspace = true + +[features] +mock = ["dep:mockall"] + +[dependencies] +lee_core.workspace = true +lee.workspace = true +common.workspace = true +storage.workspace = true + +kameo.workspace = true +thiserror.workspace = true +log.workspace = true +mockall = { workspace = true, optional = true } +itertools.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tempfile.workspace = true diff --git a/lez/sequencer/actors/storage/src/actor.rs b/lez/sequencer/actors/storage/src/actor.rs new file mode 100644 index 000000000..2edc7b917 --- /dev/null +++ b/lez/sequencer/actors/storage/src/actor.rs @@ -0,0 +1,687 @@ +use std::path::Path; + +use common::{ + block::{Block, BlockMeta, PeerChainTip}, + transaction::LeeTransaction, +}; +use itertools::Itertools as _; +use kameo::{ + Actor, + actor::{ActorRef, WeakActorRef}, + error::ActorStopReason, + message::{Context, Message}, +}; +use lee::V03State; +use lee_core::BlockId; +use log::debug; +use storage::sequencer::RocksDBIO; + +use crate::{ + Result, StorageActorTrait, + actor::tx_index::TransactionIndex, + error::Error, + protocol::{ + AddPendingCrossZoneDispatches, AddPendingDepositEvent, ApplyStoreUpdate, + CleanPendingBlocksUpTo, ConsumeUnseenWithdrawCount, DbDump, DeadLetterDispatchRecord, + DeleteBlock, DeleteCrossZonePeerFloor, DeleteZoneCheckpoint, DispatchFailure, + DropSettledCrossZoneDispatches, DumpDb, GetAllBlocks, GetBlock, GetCrossZonePeerFloorBytes, + GetCrossZonePeerTip, GetDeadLetterDispatchCount, GetDeadLetterDispatches, GetFinalSnapshot, + GetFirstBlockId, GetLastBlockId, GetLatestBlockMeta, GetLeeState, + GetPendingCrossZoneDispatches, GetPendingDepositEvents, GetPublishedHighWater, + GetTransactionByHash, GetZoneAnchor, GetZoneCheckpointBytes, MarkBlockAsFinalized, + PendingCrossZoneDispatchRecord, PendingDepositEventRecord, RaisePublishedHighWater, + RecordDispatchFailure, RecordNewBlock, ResetAllBlocksToPending, SetCrossZonePeerFloorBytes, + SetCrossZonePeerTip, SetZoneAnchor, SetZoneCheckpointBytes, StoreUpdateOutcome, + ZoneAnchorRecord, + }, +}; + +mod tx_index; + +pub struct StorageActor { + /// `None` after [`Actor::on_stop`] closed the database. + dbio: Option, + tx_index: TransactionIndex, +} + +impl StorageActor { + /// Creates a new `StorageActor` with a database at `location`. + /// If the database does not exist, it will be created. + pub fn new(location: &Path) -> Result { + let dbio = RocksDBIO::open_or_create(location)?; + Self::new_inner(dbio) + } + + /// Creates a fresh database at `location` from `dump`. + pub fn restore_from_dump(location: &Path, dump: &DbDump) -> Result { + let dbio = RocksDBIO::restore_from_dump(location, dump.as_db_dump())?; + Self::new_inner(dbio) + } + + fn new_inner(dbio: RocksDBIO) -> Result { + Ok(Self { + tx_index: Self::build_tx_index(&dbio)?, + dbio: Some(dbio), + }) + } + + /// The open database. + /// + /// # Panics + /// + /// If called after the actor has stopped, which can't happen for message handlers as kameo + /// stops delivering messages before [`Actor::on_stop`]. + const fn dbio(&self) -> &RocksDBIO { + self.dbio + .as_ref() + .expect("Database is closed, the actor has already stopped") + } + + fn build_tx_index(dbio: &RocksDBIO) -> Result { + debug!("Building the transaction index"); + + let index = + dbio.get_all_blocks() + .fold_ok(TransactionIndex::default(), |mut index, block| { + index.update_from_block(&block); + index + })?; + + debug!( + "Transaction index built, holding {} transactions", + index.transaction_count() + ); + + Ok(index) + } +} + +impl StorageActorTrait for StorageActor {} + +impl Actor for StorageActor { + type Args = Self; + type Error = Error; + + async fn on_start(args: Self::Args, _actor_ref: ActorRef) -> Result { + Ok(args) + } + + /// Closes the database, releasing the rocksdb lock on its directory. + async fn on_stop( + &mut self, + _actor_ref: WeakActorRef, + _reason: ActorStopReason, + ) -> Result<()> { + drop(self.dbio.take()); + Ok(()) + } +} + +impl Message for StorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + RecordNewBlock { + block, + withdrawals, + state, + checkpoint_bytes, + }: RecordNewBlock, + _ctx: &mut Context, + ) -> Self::Reply { + let withdrawals = withdrawals.into_iter().map(Into::into).collect::>(); + self.dbio() + .atomic_update(&block, &withdrawals, &state, checkpoint_bytes.as_deref())?; + + self.tx_index.update_from_block(&block); + Ok(()) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetBlock { block_id }: GetBlock, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio().get_block(block_id).map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetAllBlocks: GetAllBlocks, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .get_all_blocks() + .collect::>>() + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetTransactionByHash { hash }: GetTransactionByHash, + _ctx: &mut Context, + ) -> Self::Reply { + let Some(block_id) = self.tx_index.block_for_tx(&hash) else { + return Ok(None); + }; + let Some(block) = self.dbio().get_block(block_id)? else { + return Ok(None); + }; + + Ok(block + .body + .transactions + .into_iter() + .find(|transaction| transaction.hash() == hash) + .map(|transaction| (transaction, block_id))) + } +} + +impl Message for StorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + DeleteBlock { block_id }: DeleteBlock, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio().delete_block(block_id)?; + self.tx_index.delete_block(block_id); + Ok(()) + } +} + +impl Message for StorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + MarkBlockAsFinalized { block_id }: MarkBlockAsFinalized, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .mark_block_as_finalized(block_id) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + ResetAllBlocksToPending: ResetAllBlocksToPending, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .reset_all_blocks_to_pending() + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetFirstBlockId: GetFirstBlockId, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio().get_meta_first_block_in_db().map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetLastBlockId: GetLastBlockId, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio().get_meta_last_block_in_db().map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetLatestBlockMeta: GetLatestBlockMeta, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio().latest_block_meta().map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetLeeState: GetLeeState, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio().get_lee_state().map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>>; + + async fn handle( + &mut self, + GetZoneCheckpointBytes: GetZoneCheckpointBytes, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .get_zone_sdk_checkpoint_bytes() + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + SetZoneCheckpointBytes { bytes }: SetZoneCheckpointBytes, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .put_zone_sdk_checkpoint_bytes(&bytes) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + DeleteZoneCheckpoint: DeleteZoneCheckpoint, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .delete_zone_sdk_checkpoint_bytes() + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetZoneAnchor: GetZoneAnchor, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .get_zone_anchor() + .map(|anchor| anchor.map(Into::into)) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + SetZoneAnchor { anchor }: SetZoneAnchor, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .put_zone_anchor(&anchor.into()) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetPublishedHighWater: GetPublishedHighWater, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio().published_high_water().map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + RaisePublishedHighWater { block_id }: RaisePublishedHighWater, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .raise_published_high_water(block_id) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetPendingDepositEvents: GetPendingDepositEvents, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .get_pending_deposit_events() + .map(|events| events.into_iter().map(Into::into).collect()) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result; + + async fn handle( + &mut self, + DumpDb: DumpDb, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio().dump_all().map(Into::into).map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result; + + async fn handle( + &mut self, + msg: ApplyStoreUpdate, + _ctx: &mut Context, + ) -> Self::Reply { + let ApplyStoreUpdate { + checkpoint, + blocks, + head_tip, + head_state, + final_snapshot, + finalized_up_to, + new_deposit_events, + remove_deposit_records, + remove_dispatch_records, + consumed_withdrawals, + new_withdraw_intents, + zone_anchor, + } = msg; + + let blocks = blocks + .iter() + .map(|(block, finalized)| (block, *finalized)) + .collect::>(); + let zone_anchor = zone_anchor.map(Into::into); + let new_deposit_events = new_deposit_events + .into_iter() + .map(Into::into) + .collect::>(); + let consumed_withdrawals = consumed_withdrawals + .into_iter() + .map(Into::into) + .collect::>(); + let new_withdraw_intents = new_withdraw_intents + .into_iter() + .map(Into::into) + .collect::>(); + + let update = storage::sequencer::StoreUpdate { + checkpoint: checkpoint.as_deref(), + blocks: &blocks, + head_tip: head_tip.as_ref(), + head_state: &head_state, + final_snapshot: final_snapshot + .as_ref() + .map(|(state, meta)| (state.as_ref(), meta)), + finalized_up_to, + new_deposit_events: &new_deposit_events, + remove_deposit_records: &remove_deposit_records, + remove_dispatch_records: &remove_dispatch_records, + consumed_withdrawals: &consumed_withdrawals, + new_withdraw_intents: &new_withdraw_intents, + zone_anchor: zone_anchor.as_ref(), + }; + let outcome = self.dbio().store_update(&update)?; + + for (block, _) in blocks { + self.tx_index.update_from_block(block); + } + Ok(outcome.into()) + } +} + +impl Message for StorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + CleanPendingBlocksUpTo { last_finalized }: CleanPendingBlocksUpTo, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .clean_pending_blocks_up_to(last_finalized) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetFinalSnapshot: GetFinalSnapshot, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio().get_final_snapshot().map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result; + + async fn handle( + &mut self, + AddPendingDepositEvent { event }: AddPendingDepositEvent, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .add_pending_deposit_event(event.into()) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result; + + async fn handle( + &mut self, + ConsumeUnseenWithdrawCount { withdrawal }: ConsumeUnseenWithdrawCount, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .consume_unseen_withdraw_count(withdrawal.into()) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetPendingCrossZoneDispatches: GetPendingCrossZoneDispatches, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .get_pending_cross_zone_dispatches() + .map(|records| records.into_iter().map(Into::into).collect()) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result; + + async fn handle( + &mut self, + AddPendingCrossZoneDispatches { dispatches }: AddPendingCrossZoneDispatches, + _ctx: &mut Context, + ) -> Self::Reply { + let dispatches = dispatches.into_iter().map(Into::into).collect(); + self.dbio() + .add_pending_cross_zone_dispatches(dispatches) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result; + + async fn handle( + &mut self, + DropSettledCrossZoneDispatches { message_keys }: DropSettledCrossZoneDispatches, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .drop_settled_cross_zone_dispatches(&message_keys) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result; + + async fn handle( + &mut self, + RecordDispatchFailure { + message_key, + retire_at, + origin, + }: RecordDispatchFailure, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .record_dispatch_failure(message_key, retire_at, origin.into()) + .map(Into::into) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetDeadLetterDispatches: GetDeadLetterDispatches, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .get_dead_letter_cross_zone_dispatches() + .map(|records| records.into_iter().map(Into::into).collect()) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result; + + async fn handle( + &mut self, + GetDeadLetterDispatchCount: GetDeadLetterDispatchCount, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .get_dead_letter_cross_zone_dispatch_count() + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>>; + + async fn handle( + &mut self, + GetCrossZonePeerFloorBytes { peer_zone }: GetCrossZonePeerFloorBytes, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .get_cross_zone_peer_floor_bytes(peer_zone) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + SetCrossZonePeerFloorBytes { peer_zone, bytes }: SetCrossZonePeerFloorBytes, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .put_cross_zone_peer_floor_bytes(peer_zone, &bytes) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + DeleteCrossZonePeerFloor { peer_zone }: DeleteCrossZonePeerFloor, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .delete_cross_zone_peer_floor(peer_zone) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + GetCrossZonePeerTip { peer_zone }: GetCrossZonePeerTip, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .get_cross_zone_peer_tip(peer_zone) + .map_err(Into::into) + } +} + +impl Message for StorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + SetCrossZonePeerTip { peer_zone, tip }: SetCrossZonePeerTip, + _ctx: &mut Context, + ) -> Self::Reply { + self.dbio() + .put_cross_zone_peer_tip(peer_zone, tip) + .map_err(Into::into) + } +} diff --git a/lez/sequencer/actors/storage/src/actor/tx_index.rs b/lez/sequencer/actors/storage/src/actor/tx_index.rs new file mode 100644 index 000000000..9a396b09a --- /dev/null +++ b/lez/sequencer/actors/storage/src/actor/tx_index.rs @@ -0,0 +1,123 @@ +use std::collections::HashMap; + +use common::{HashType, block::Block}; +use lee_core::BlockId; + +#[derive(Default)] +pub struct TransactionIndex { + tx_to_block: HashMap, + block_to_txs: HashMap>, +} + +impl TransactionIndex { + /// Extend index from block removing old data if it existed for `block`. + pub fn update_from_block(&mut self, block: &Block) { + self.delete_block(block.header.block_id); + + for tx in &block.body.transactions { + let tx_hash = tx.hash(); + self.tx_to_block.insert(tx_hash, block.header.block_id); + self.block_to_txs + .entry(block.header.block_id) + .or_default() + .push(tx_hash); + } + } + + /// Delete all transactions associated with `block_id` from the index. + pub fn delete_block(&mut self, block_id: BlockId) { + if let Some(txs) = self.block_to_txs.remove(&block_id) { + for tx_hash in txs { + self.tx_to_block.remove(&tx_hash); + } + } + } + + /// Get the block id for a transaction hash if it exists in the index. + pub fn block_for_tx(&self, tx_hash: &HashType) -> Option { + self.tx_to_block.get(tx_hash).copied() + } + + /// Get the number of transactions in the index. + pub fn transaction_count(&self) -> usize { + self.tx_to_block.len() + } +} + +#[cfg(test)] +mod tests { + use common::test_utils::{produce_dummy_block, produce_dummy_empty_transaction}; + + use super::*; + + #[test] + fn update_from_block() { + let mut index = TransactionIndex::default(); + let block = create_test_block(1, 3); + + index.update_from_block(&block); + + assert_eq!(index.transaction_count(), 3); + } + + #[test] + fn update_from_block_replaces_existing() { + let mut index = TransactionIndex::default(); + let block1 = create_test_block(1, 2); + let block2 = create_test_block(1, 2); + + index.update_from_block(&block1); + assert_eq!(index.transaction_count(), 2); + + index.update_from_block(&block2); + assert_eq!(index.transaction_count(), 2); + } + + #[test] + fn block_for_tx() { + let mut index = TransactionIndex::default(); + let block = create_test_block(1, 3); + + index.update_from_block(&block); + + assert_eq!( + index.block_for_tx(&block.body.transactions[0].hash()), + Some(1) + ); + assert_eq!(index.block_for_tx(&HashType::default()), None); + } + + #[test] + fn delete_block() { + let mut index = TransactionIndex::default(); + let block = create_test_block(1, 3); + + index.update_from_block(&block); + assert_eq!(index.transaction_count(), 3); + + index.delete_block(1); + assert_eq!(index.transaction_count(), 0); + } + + #[test] + fn transaction_count() { + let mut index = TransactionIndex::default(); + assert_eq!(index.transaction_count(), 0); + + let block1 = create_test_block(1, 2); + index.update_from_block(&block1); + assert_eq!(index.transaction_count(), 2); + + let block2 = create_test_block(2, 3); + index.update_from_block(&block2); + assert_eq!(index.transaction_count(), 5); + } + + fn create_test_block(block_id: u64, tx_count: usize) -> Block { + // -1 for auto-inserted clock transaction + let transactions = std::iter::repeat_with(produce_dummy_empty_transaction) + .take(tx_count.saturating_sub(1)) + .collect(); + produce_dummy_block(block_id, None, transactions) + } +} diff --git a/lez/sequencer/actors/storage/src/error.rs b/lez/sequencer/actors/storage/src/error.rs new file mode 100644 index 000000000..904806ff0 --- /dev/null +++ b/lez/sequencer/actors/storage/src/error.rs @@ -0,0 +1,5 @@ +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Database error")] + DatabaseError(#[from] storage::error::DbError), +} diff --git a/lez/sequencer/actors/storage/src/lib.rs b/lez/sequencer/actors/storage/src/lib.rs new file mode 100644 index 000000000..6d86c9e02 --- /dev/null +++ b/lez/sequencer/actors/storage/src/lib.rs @@ -0,0 +1,15 @@ +//! Storage Actor is responsible for persisting data in local database. + +pub use actor::StorageActor; +pub use r#trait::StorageActorTrait; + +pub mod actor; +pub mod error; +#[cfg(feature = "mock")] +pub mod mock; +pub mod protocol; +#[cfg(test)] +mod tests; +pub mod r#trait; + +pub type Result = std::result::Result; diff --git a/lez/sequencer/actors/storage/src/mock.rs b/lez/sequencer/actors/storage/src/mock.rs new file mode 100644 index 000000000..18159c38d --- /dev/null +++ b/lez/sequencer/actors/storage/src/mock.rs @@ -0,0 +1,730 @@ +#![expect( + clippy::struct_field_names, + reason = "`handle*` prefix is used for convenience with `Message` trait" +)] + +use common::{ + block::{Block, BlockMeta, PeerChainTip}, + transaction::LeeTransaction, +}; +use kameo::{ + Actor, Reply, + actor::ActorRef, + message::{Context, Message}, +}; +use lee::V03State; +use lee_core::BlockId; + +use crate::{ + Result, StorageActorTrait, + error::Error, + protocol::{ + AddPendingCrossZoneDispatches, AddPendingDepositEvent, ApplyStoreUpdate, + CleanPendingBlocksUpTo, ConsumeUnseenWithdrawCount, DbDump, DeadLetterDispatchRecord, + DeleteBlock, DeleteCrossZonePeerFloor, DeleteZoneCheckpoint, DispatchFailure, + DropSettledCrossZoneDispatches, DumpDb, GetAllBlocks, GetBlock, GetCrossZonePeerFloorBytes, + GetCrossZonePeerTip, GetDeadLetterDispatchCount, GetDeadLetterDispatches, GetFinalSnapshot, + GetFirstBlockId, GetLastBlockId, GetLatestBlockMeta, GetLeeState, + GetPendingCrossZoneDispatches, GetPendingDepositEvents, GetPublishedHighWater, + GetTransactionByHash, GetZoneAnchor, GetZoneCheckpointBytes, MarkBlockAsFinalized, + PendingCrossZoneDispatchRecord, PendingDepositEventRecord, RaisePublishedHighWater, + RecordDispatchFailure, RecordNewBlock, ResetAllBlocksToPending, SetCrossZonePeerFloorBytes, + SetCrossZonePeerTip, SetZoneAnchor, SetZoneCheckpointBytes, StoreUpdateOutcome, + ZoneAnchorRecord, + }, +}; + +mockall::mock! { + pub StorageActor { + pub fn handle_record_new_block( + &mut self, + msg: RecordNewBlock, + ctx: &mut Context> + ) -> Result<()>; + + pub fn handle_get_block( + &mut self, + msg: GetBlock, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_get_all_blocks( + &mut self, + msg: GetAllBlocks, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_get_transaction_by_hash( + &mut self, + msg: GetTransactionByHash, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_delete_block( + &mut self, + msg: DeleteBlock, + ctx: &mut Context> + ) -> Result<()>; + + pub fn handle_mark_block_as_finalized( + &mut self, + msg: MarkBlockAsFinalized, + ctx: &mut Context> + ) -> Result<()>; + + pub fn handle_reset_all_blocks_to_pending( + &mut self, + msg: ResetAllBlocksToPending, + ctx: &mut Context> + ) -> Result<()>; + + pub fn handle_get_first_block_id( + &mut self, + msg: GetFirstBlockId, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_get_last_block_id( + &mut self, + msg: GetLastBlockId, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_get_latest_block_meta( + &mut self, + msg: GetLatestBlockMeta, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_get_lee_state( + &mut self, + msg: GetLeeState, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_get_zone_checkpoint_bytes( + &mut self, + msg: GetZoneCheckpointBytes, + ctx: &mut Context>>> + ) -> Result>>; + + pub fn handle_set_zone_checkpoint_bytes( + &mut self, + msg: SetZoneCheckpointBytes, + ctx: &mut Context> + ) -> Result<()>; + + pub fn handle_delete_zone_checkpoint( + &mut self, + msg: DeleteZoneCheckpoint, + ctx: &mut Context> + ) -> Result<()>; + + pub fn handle_get_zone_anchor( + &mut self, + msg: GetZoneAnchor, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_set_zone_anchor( + &mut self, + msg: SetZoneAnchor, + ctx: &mut Context> + ) -> Result<()>; + + pub fn handle_get_published_high_water( + &mut self, + msg: GetPublishedHighWater, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_raise_published_high_water( + &mut self, + msg: RaisePublishedHighWater, + ctx: &mut Context> + ) -> Result<()>; + + pub fn handle_get_pending_deposit_events( + &mut self, + msg: GetPendingDepositEvents, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_apply_store_update( + &mut self, + msg: ApplyStoreUpdate, + ctx: &mut Context> + ) -> Result; + + pub fn handle_clean_pending_blocks_up_to( + &mut self, + msg: CleanPendingBlocksUpTo, + ctx: &mut Context> + ) -> Result<()>; + + pub fn handle_get_final_snapshot( + &mut self, + msg: GetFinalSnapshot, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_add_pending_deposit_event( + &mut self, + msg: AddPendingDepositEvent, + ctx: &mut Context> + ) -> Result; + + pub fn handle_consume_unseen_withdraw_count( + &mut self, + msg: ConsumeUnseenWithdrawCount, + ctx: &mut Context> + ) -> Result; + + pub fn handle_get_pending_cross_zone_dispatches( + &mut self, + msg: GetPendingCrossZoneDispatches, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_add_pending_cross_zone_dispatches( + &mut self, + msg: AddPendingCrossZoneDispatches, + ctx: &mut Context> + ) -> Result; + + pub fn handle_drop_settled_cross_zone_dispatches( + &mut self, + msg: DropSettledCrossZoneDispatches, + ctx: &mut Context> + ) -> Result; + + pub fn handle_record_dispatch_failure( + &mut self, + msg: RecordDispatchFailure, + ctx: &mut Context> + ) -> Result; + + pub fn handle_get_dead_letter_dispatches( + &mut self, + msg: GetDeadLetterDispatches, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_get_dead_letter_dispatch_count( + &mut self, + msg: GetDeadLetterDispatchCount, + ctx: &mut Context> + ) -> Result; + + pub fn handle_get_cross_zone_peer_floor_bytes( + &mut self, + msg: GetCrossZonePeerFloorBytes, + ctx: &mut Context>>> + ) -> Result>>; + + pub fn handle_set_cross_zone_peer_floor_bytes( + &mut self, + msg: SetCrossZonePeerFloorBytes, + ctx: &mut Context> + ) -> Result<()>; + + pub fn handle_delete_cross_zone_peer_floor( + &mut self, + msg: DeleteCrossZonePeerFloor, + ctx: &mut Context> + ) -> Result<()>; + + pub fn handle_get_cross_zone_peer_tip( + &mut self, + msg: GetCrossZonePeerTip, + ctx: &mut Context>> + ) -> Result>; + + pub fn handle_set_cross_zone_peer_tip( + &mut self, + msg: SetCrossZonePeerTip, + ctx: &mut Context> + ) -> Result<()>; + + pub fn handle_dump_db( + &mut self, + msg: DumpDb, + ctx: &mut Context> + ) -> Result; + } +} + +impl StorageActorTrait for MockStorageActor {} + +impl Actor for MockStorageActor { + type Args = Self; + type Error = Error; + + async fn on_start(args: Self::Args, _actor_ref: ActorRef) -> Result { + Ok(args) + } +} + +/// Special message to trigger [`MockStorageActor::checkpoint()`]. +pub struct Checkpoint; + +impl Message for MockStorageActor { + type Reply = (); + + async fn handle( + &mut self, + Checkpoint: Checkpoint, + _ctx: &mut Context, + ) -> Self::Reply { + self.checkpoint(); + } +} + +/// Special message to [`std::mem::replace()`] the inner state of [`MockStorageActor`] with a new +/// one, returning old state. +/// This is useful for testing, to swap in a new mock with different expectations. +pub struct Replace { + pub mock: MockStorageActor, +} + +#[derive(Reply)] +pub struct ReplaceReply { + pub old_mock: MockStorageActor, +} + +impl Message for MockStorageActor { + type Reply = ReplaceReply; + + async fn handle( + &mut self, + Replace { mock }: Replace, + _ctx: &mut Context, + ) -> Self::Reply { + let old_mock = std::mem::replace(self, mock); + ReplaceReply { old_mock } + } +} + +impl Message for MockStorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + msg: RecordNewBlock, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_record_new_block(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle(&mut self, msg: GetBlock, ctx: &mut Context) -> Self::Reply { + self.handle_get_block(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetAllBlocks, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_all_blocks(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetTransactionByHash, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_transaction_by_hash(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + msg: DeleteBlock, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_delete_block(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + msg: MarkBlockAsFinalized, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_mark_block_as_finalized(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + msg: ResetAllBlocksToPending, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_reset_all_blocks_to_pending(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetFirstBlockId, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_first_block_id(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetLastBlockId, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_last_block_id(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetLatestBlockMeta, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_latest_block_meta(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetLeeState, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_lee_state(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>>; + + async fn handle( + &mut self, + msg: GetZoneCheckpointBytes, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_zone_checkpoint_bytes(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + msg: SetZoneCheckpointBytes, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_set_zone_checkpoint_bytes(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + msg: DeleteZoneCheckpoint, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_delete_zone_checkpoint(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetZoneAnchor, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_zone_anchor(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + msg: SetZoneAnchor, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_set_zone_anchor(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetPublishedHighWater, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_published_high_water(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + msg: RaisePublishedHighWater, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_raise_published_high_water(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetPendingDepositEvents, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_pending_deposit_events(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result; + + async fn handle(&mut self, msg: DumpDb, ctx: &mut Context) -> Self::Reply { + self.handle_dump_db(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result; + + async fn handle( + &mut self, + msg: ApplyStoreUpdate, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_apply_store_update(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + msg: CleanPendingBlocksUpTo, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_clean_pending_blocks_up_to(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetFinalSnapshot, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_final_snapshot(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result; + + async fn handle( + &mut self, + msg: AddPendingDepositEvent, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_add_pending_deposit_event(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result; + + async fn handle( + &mut self, + msg: ConsumeUnseenWithdrawCount, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_consume_unseen_withdraw_count(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetPendingCrossZoneDispatches, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_pending_cross_zone_dispatches(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result; + + async fn handle( + &mut self, + msg: AddPendingCrossZoneDispatches, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_add_pending_cross_zone_dispatches(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result; + + async fn handle( + &mut self, + msg: DropSettledCrossZoneDispatches, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_drop_settled_cross_zone_dispatches(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result; + + async fn handle( + &mut self, + msg: RecordDispatchFailure, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_record_dispatch_failure(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetDeadLetterDispatches, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_dead_letter_dispatches(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result; + + async fn handle( + &mut self, + msg: GetDeadLetterDispatchCount, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_dead_letter_dispatch_count(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>>; + + async fn handle( + &mut self, + msg: GetCrossZonePeerFloorBytes, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_cross_zone_peer_floor_bytes(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + msg: SetCrossZonePeerFloorBytes, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_set_cross_zone_peer_floor_bytes(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + msg: DeleteCrossZonePeerFloor, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_delete_cross_zone_peer_floor(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result>; + + async fn handle( + &mut self, + msg: GetCrossZonePeerTip, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_get_cross_zone_peer_tip(msg, ctx) + } +} + +impl Message for MockStorageActor { + type Reply = Result<()>; + + async fn handle( + &mut self, + msg: SetCrossZonePeerTip, + ctx: &mut Context, + ) -> Self::Reply { + self.handle_set_cross_zone_peer_tip(msg, ctx) + } +} diff --git a/lez/sequencer/actors/storage/src/protocol.rs b/lez/sequencer/actors/storage/src/protocol.rs new file mode 100644 index 000000000..e74819de4 --- /dev/null +++ b/lez/sequencer/actors/storage/src/protocol.rs @@ -0,0 +1,510 @@ +// TODO: Instead of `From` conversions from `protocol` types to `cell` types consider moving +// `storage` crate as sub-module in `actor` while removing `cell` types and implementing +// encoding/decoding for `protocol` structures. + +use std::sync::Arc; + +use common::{ + HashType, + block::{Block, BlockMeta, PeerChainTip}, +}; +use lee::V03State; +use lee_core::BlockId; +use storage::sequencer::{self as db, sequencer_cells as db_cells}; + +use crate::Result; + +/// Persists `block` with the effects it covers. +pub struct RecordNewBlock { + pub block: Block, + pub withdrawals: Vec, + pub state: Arc, + pub checkpoint_bytes: Option>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GetBlock { + pub block_id: BlockId, +} + +pub struct GetAllBlocks; + +pub struct GetTransactionByHash { + pub hash: HashType, +} + +pub struct DeleteBlock { + pub block_id: BlockId, +} + +pub struct MarkBlockAsFinalized { + pub block_id: BlockId, +} + +pub struct ResetAllBlocksToPending; + +pub struct GetFirstBlockId; + +pub struct GetLastBlockId; + +pub struct GetLatestBlockMeta; + +pub struct GetLeeState; + +pub struct GetZoneCheckpointBytes; + +pub struct SetZoneCheckpointBytes { + pub bytes: Vec, +} + +pub struct DeleteZoneCheckpoint; + +pub struct GetZoneAnchor; + +pub struct SetZoneAnchor { + pub anchor: ZoneAnchorRecord, +} + +pub struct GetPublishedHighWater; + +/// Raises the published high water mark to `block_id`, never lowering it. +pub struct RaisePublishedHighWater { + pub block_id: BlockId, +} + +pub struct GetPendingDepositEvents; + +pub struct AddPendingDepositEvent { + pub event: PendingDepositEventRecord, +} + +pub struct GetFinalSnapshot; + +/// Marks every stored pending block at or below `last_finalized` as finalized. +pub struct CleanPendingBlocksUpTo { + pub last_finalized: BlockId, +} + +pub struct ConsumeUnseenWithdrawCount { + pub withdrawal: WithdrawalReconciliationKey, +} + +pub struct GetPendingCrossZoneDispatches; + +pub struct AddPendingCrossZoneDispatches { + pub dispatches: Vec, +} + +pub struct DropSettledCrossZoneDispatches { + pub message_keys: Vec<[u8; 32]>, +} + +pub struct RecordDispatchFailure { + pub message_key: [u8; 32], + pub retire_at: u32, + pub origin: DispatchOrigin, +} + +pub struct GetDeadLetterDispatches; + +pub struct GetDeadLetterDispatchCount; + +pub struct GetCrossZonePeerFloorBytes { + pub peer_zone: PeerZoneKey, +} + +pub struct SetCrossZonePeerFloorBytes { + pub peer_zone: PeerZoneKey, + pub bytes: Vec, +} + +pub struct DeleteCrossZonePeerFloor { + pub peer_zone: PeerZoneKey, +} + +pub struct GetCrossZonePeerTip { + pub peer_zone: PeerZoneKey, +} + +pub struct SetCrossZonePeerTip { + pub peer_zone: PeerZoneKey, + pub tip: PeerChainTip, +} + +pub struct DumpDb; + +/// Everything one sequencer event writes. +// TODO: Consider removing [`SetZoneCheckpointBytes`] in favor of this. +// TODO: How does this stack with [`RecordNewBlock`] ? +// TODO: Reconsider options here. +pub struct ApplyStoreUpdate { + /// Serialized zone-sdk checkpoint for this event. + pub checkpoint: Option>, + + /// `(block, finalized)` payloads to write. + pub blocks: Vec<(Block, bool)>, + + /// Head tip to pin the stored chain to; `None` only for an empty chain. + pub head_tip: Option, + /// State after the last applied block. + pub head_state: Arc, + + /// `(state, meta)` of the final tier, when it advanced. + pub final_snapshot: Option<(Arc, BlockMeta)>, + + /// Highest block id this event made irreversible: stored blocks at or below + /// it become finalized. + pub finalized_up_to: Option, + + /// Deposit events observed on L1, recorded unless already pending. + pub new_deposit_events: Vec, + + /// Deposit op ids whose mint finalized: their pending records are dropped. + pub remove_deposit_records: Vec, + /// Message keys whose delivery finalized: their pending records are dropped. + pub remove_dispatch_records: Vec<[u8; 32]>, + /// L1 withdraw events to reconcile against the local unseen counters. + pub consumed_withdrawals: Vec, + /// L2 withdraw intents this update raises, awaiting their L1 event. + pub new_withdraw_intents: Vec, + + /// Advance the channel-read anchor. + pub zone_anchor: Option, +} + +/// Zone id of a cross-zone peer. +pub type PeerZoneKey = [u8; 32]; + +/// Identity of one withdrawal: the id of the channel note it releases. +/// +/// Shared by the intent recorded when the sequencer publishes a withdrawal and +/// the Bedrock event that later reports it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct WithdrawalReconciliationKey { + pub released_note_id: [u8; 32], +} + +impl From for db_cells::WithdrawalReconciliationKey { + fn from(WithdrawalReconciliationKey { released_note_id }: WithdrawalReconciliationKey) -> Self { + Self { released_note_id } + } +} + +impl From for WithdrawalReconciliationKey { + fn from( + db_cells::WithdrawalReconciliationKey { released_note_id }: db_cells::WithdrawalReconciliationKey, + ) -> Self { + Self { released_note_id } + } +} + +/// The last channel block read back and verified from Bedrock: the anchor for +/// the startup consistency check and the resume point for reconstruction. +/// +/// `slot` is the raw L1 inscription slot; the caller converts to and from the +/// zone-sdk `Slot`, which does not derive borsh and so cannot be stored. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ZoneAnchorRecord { + pub slot: u64, + pub block_id: u64, + pub hash: HashType, +} + +impl From for db_cells::ZoneAnchorRecord { + fn from( + ZoneAnchorRecord { + slot, + block_id, + hash, + }: ZoneAnchorRecord, + ) -> Self { + Self { + slot, + block_id, + hash, + } + } +} + +impl From for ZoneAnchorRecord { + fn from( + db_cells::ZoneAnchorRecord { + slot, + block_id, + hash, + }: db_cells::ZoneAnchorRecord, + ) -> Self { + Self { + slot, + block_id, + hash, + } + } +} + +/// An L1 deposit event observed but not yet seen finalized. +/// +/// Purely a liveness queue: whether to emit a mint is decided against chain +/// state, and the record is dropped once its mint finalizes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PendingDepositEventRecord { + pub deposit_op_id: HashType, + pub source_tx_hash: HashType, + pub amount: u64, + pub metadata: Vec, +} + +impl From for db_cells::PendingDepositEventRecord { + fn from( + PendingDepositEventRecord { + deposit_op_id, + source_tx_hash, + amount, + metadata, + }: PendingDepositEventRecord, + ) -> Self { + Self { + deposit_op_id, + source_tx_hash, + amount, + metadata, + } + } +} + +impl From for PendingDepositEventRecord { + fn from( + db_cells::PendingDepositEventRecord { + deposit_op_id, + source_tx_hash, + amount, + metadata, + }: db_cells::PendingDepositEventRecord, + ) -> Self { + Self { + deposit_op_id, + source_tx_hash, + amount, + metadata, + } + } +} + +/// A cross-zone delivery read off a peer block but not yet known to be +/// irreversibly delivered. +/// +/// The watcher's delivery floor is durable, so once it advances past a peer +/// block that block is never re-read; this record stands in its place. It +/// carries no "submitted" mark: it is dropped when the delivery finalizes, and +/// re-including one meanwhile is harmless because the inbox no-ops a replay on +/// chain. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PendingCrossZoneDispatchRecord { + /// Content-addressed replay key of the delivered message, and this record's + /// identity. + pub message_key: [u8; 32], + /// The borsh-encoded dispatch transaction, so production can re-feed it + /// without re-reading the peer channel. + pub transaction: Vec, + /// Production attempts that ended in an execution failure. Past a threshold + /// the record leaves this list for a [`DeadLetterDispatchRecord`]. + pub failed_attempts: u32, +} + +impl PendingCrossZoneDispatchRecord { + /// A delivery the watcher has just read: never attempted. + #[must_use] + pub const fn recorded(message_key: [u8; 32], transaction: Vec) -> Self { + Self { + message_key, + transaction, + failed_attempts: 0, + } + } +} + +impl From for db_cells::PendingCrossZoneDispatchRecord { + fn from( + PendingCrossZoneDispatchRecord { + message_key, + transaction, + failed_attempts, + }: PendingCrossZoneDispatchRecord, + ) -> Self { + Self { + message_key, + transaction, + failed_attempts, + } + } +} + +impl From for PendingCrossZoneDispatchRecord { + fn from( + db_cells::PendingCrossZoneDispatchRecord { + message_key, + transaction, + failed_attempts, + }: db_cells::PendingCrossZoneDispatchRecord, + ) -> Self { + Self { + message_key, + transaction, + failed_attempts, + } + } +} + +/// Which peer message a delivery carried, kept so a lost one can be traced back +/// to the peer block it was in. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DispatchOrigin { + pub src_zone: PeerZoneKey, + pub src_block_id: u64, + pub src_tx_index: u32, +} + +impl From for db_cells::DispatchOrigin { + fn from( + DispatchOrigin { + src_zone, + src_block_id, + src_tx_index, + }: DispatchOrigin, + ) -> Self { + Self { + src_zone, + src_block_id, + src_tx_index, + } + } +} + +impl From for DispatchOrigin { + fn from( + db_cells::DispatchOrigin { + src_zone, + src_block_id, + src_tx_index, + }: db_cells::DispatchOrigin, + ) -> Self { + Self { + src_zone, + src_block_id, + src_tx_index, + } + } +} + +/// A cross-zone delivery this node has given up on. +/// +/// A dispatch that fails execution is left out of the block, so nothing on +/// chain records that it was attempted; this is the only durable trace. It +/// identifies the message rather than carrying it: the peer block and index are +/// enough to read it back off the channel. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeadLetterDispatchRecord { + pub message_key: [u8; 32], + pub origin: DispatchOrigin, + /// Attempts made before giving up, so the record carries the policy that was + /// in force at the time. + pub failed_attempts: u32, + /// Size of the delivery transaction that would not execute, the diagnostic + /// for size-related failures. + pub transaction_bytes: u32, +} + +impl From for DeadLetterDispatchRecord { + fn from( + db_cells::DeadLetterDispatchRecord { + message_key, + origin, + failed_attempts, + transaction_bytes, + }: db_cells::DeadLetterDispatchRecord, + ) -> Self { + Self { + message_key, + origin: origin.into(), + failed_attempts, + transaction_bytes, + } + } +} + +/// What counting a failed production attempt did to a delivery's record, the +/// reply to [`RecordDispatchFailure`]. +/// +/// Three outcomes rather than a bool: only one means this node stopped trying, +/// and a settled delivery has no record, so it is [`Self::Absent`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DispatchFailure { + /// Counted; the delivery is still pending and will be attempted again. + Retried { failed_attempts: u32 }, + /// Given up on: moved out of the pending list and into the dead letter. + Retired(Box), + /// No pending record, so nothing was counted and nothing was given up on. + Absent, +} + +impl From for DispatchFailure { + fn from(value: db::DispatchFailure) -> Self { + match value { + db::DispatchFailure::Retried { failed_attempts } => Self::Retried { failed_attempts }, + db::DispatchFailure::Retired(record) => Self::Retired(Box::new((*record).into())), + db::DispatchFailure::Absent => Self::Absent, + } + } +} + +/// What [`ApplyStoreUpdate`] observed while staging, for the caller to act on +/// *after* the write committed. +#[derive(Debug, Default)] +pub struct StoreUpdateOutcome { + /// How many deposit events were newly recorded; the rest were already + /// pending, and so already owed. + pub accepted_deposits: usize, + /// Withdraw events with no matching local unseen counter, one entry per + /// unmatched occurrence. + pub unmatched_withdrawals: Vec, +} + +impl From for StoreUpdateOutcome { + fn from( + db::StoreUpdateOutcome { + accepted_deposits, + unmatched_withdrawals, + }: db::StoreUpdateOutcome, + ) -> Self { + Self { + accepted_deposits, + unmatched_withdrawals: unmatched_withdrawals.into_iter().map(Into::into).collect(), + } + } +} + +/// Schema-agnostic snapshot of a whole store, opaque by design. +pub struct DbDump(db::DbDump); + +impl DbDump { + /// Serializes the dump to a compressed blob. + pub fn to_bytes(&self) -> Result> { + self.0.to_bytes().map_err(Into::into) + } + + /// Reads back a dump produced by [`Self::to_bytes`]. + pub fn from_bytes(bytes: &[u8]) -> Result { + db::DbDump::from_bytes(bytes).map(Self).map_err(Into::into) + } + + /// The wrapped dump, for the restore path. + pub(crate) const fn as_db_dump(&self) -> &db::DbDump { + &self.0 + } +} + +impl From for DbDump { + fn from(value: db::DbDump) -> Self { + Self(value) + } +} diff --git a/lez/sequencer/actors/storage/src/tests.rs b/lez/sequencer/actors/storage/src/tests.rs new file mode 100644 index 000000000..e4da790eb --- /dev/null +++ b/lez/sequencer/actors/storage/src/tests.rs @@ -0,0 +1,174 @@ +use std::{path::Path, sync::Arc}; + +use common::{ + HashType, + block::{Block, BlockMeta}, + test_utils::{produce_dummy_block, produce_dummy_empty_transaction}, +}; +use kameo::actor::{ActorRef, Spawn as _}; +use lee::V03State; + +use crate::{ + StorageActor, + protocol::{ApplyStoreUpdate, GetTransactionByHash, RecordNewBlock}, +}; + +/// Spawns an actor on a database at `path` seeded with `blocks`. +async fn spawn_with_blocks(path: &Path, blocks: Vec) -> ActorRef { + let storage_ref = StorageActor::spawn(StorageActor::new(path).expect("Failed to open db")); + for block in blocks { + storage_ref + .ask(RecordNewBlock { + block, + withdrawals: vec![], + state: Arc::new(V03State::new()), + checkpoint_bytes: None, + }) + .await + .expect("Failed to record a block"); + } + storage_ref +} + +/// Holding the task's output keeps the stopped actor's state alive, which is exactly the +/// situation `on_stop` exists for: the lock has to be gone by the time the shutdown result +/// resolves, not by the time the state happens to be dropped. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stopped_actor_releases_the_database_lock() { + let dir = tempfile::tempdir().expect("Failed to create temp dir"); + let prepared = StorageActor::prepare(); + let actor_ref = prepared.actor_ref().clone(); + let join_handle = prepared.spawn(StorageActor::new(dir.path()).expect("Failed to open db")); + + actor_ref + .stop_gracefully() + .await + .expect("Failed to stop the actor"); + actor_ref.wait_for_shutdown_with_result(|_| ()).await; + + StorageActor::new(dir.path()).expect("Database lock must be released once the actor stops"); + + drop(join_handle); +} + +#[tokio::test] +async fn recorded_transaction_is_looked_up_by_hash() { + let dir = tempfile::tempdir().expect("Failed to create temp dir"); + let transaction = produce_dummy_empty_transaction(); + let block = produce_dummy_block(1, None, vec![transaction.clone()]); + let storage_ref = + spawn_with_blocks(dir.path(), vec![produce_dummy_block(0, None, vec![])]).await; + + assert_eq!( + storage_ref + .ask(GetTransactionByHash { + hash: transaction.hash() + }) + .await + .expect("Failed to look the transaction up"), + None, + "A transaction outside the chain has nowhere to be found" + ); + + storage_ref + .ask(RecordNewBlock { + block, + withdrawals: vec![], + state: Arc::new(V03State::new()), + checkpoint_bytes: None, + }) + .await + .expect("Failed to record the block"); + + assert_eq!( + storage_ref + .ask(GetTransactionByHash { + hash: transaction.hash() + }) + .await + .expect("Failed to look the transaction up"), + Some((transaction, 1)) + ); +} + +/// The index lives only in memory, so a fresh actor has to build it off the +/// stored blocks rather than off the writes it has seen. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn transaction_is_looked_up_on_a_reopened_database() { + let dir = tempfile::tempdir().expect("Failed to create temp dir"); + let transaction = produce_dummy_empty_transaction(); + + let storage_weak = spawn_with_blocks( + dir.path(), + vec![ + produce_dummy_block(0, None, vec![]), + produce_dummy_block(1, None, vec![transaction.clone()]), + ], + ) + .await + .downgrade(); + storage_weak.wait_for_shutdown_with_result(|_| ()).await; + + let storage_ref = spawn_with_blocks(dir.path(), vec![]).await; + assert_eq!( + storage_ref + .ask(GetTransactionByHash { + hash: transaction.hash() + }) + .await + .expect("Failed to look the transaction up"), + Some((transaction, 1)) + ); +} + +/// An update that replaces a stored block must not leave the transactions it +/// dropped reachable. +#[tokio::test] +async fn replaced_block_leaves_no_stale_index_entries() { + let dir = tempfile::tempdir().expect("Failed to create temp dir"); + let orphaned_transaction = produce_dummy_empty_transaction(); + let orphaned = produce_dummy_block(1, None, vec![orphaned_transaction.clone()]); + let adopted = produce_dummy_block(1, Some(HashType([1; 32])), vec![]); + + let storage_ref = spawn_with_blocks( + dir.path(), + vec![produce_dummy_block(0, None, vec![]), orphaned], + ) + .await; + // The index starts out holding the chain the adopted block replaces. + storage_ref + .ask(GetTransactionByHash { + hash: orphaned_transaction.hash(), + }) + .await + .expect("Failed to look the transaction up") + .expect("The orphaned block is the stored one so far"); + + storage_ref + .ask(ApplyStoreUpdate { + checkpoint: None, + blocks: vec![(adopted.clone(), false)], + head_tip: Some(BlockMeta::from(&adopted)), + head_state: Arc::new(V03State::new()), + final_snapshot: None, + finalized_up_to: None, + new_deposit_events: vec![], + remove_deposit_records: vec![], + remove_dispatch_records: vec![], + consumed_withdrawals: vec![], + new_withdraw_intents: vec![], + zone_anchor: None, + }) + .await + .expect("Failed to apply the update"); + + assert_eq!( + storage_ref + .ask(GetTransactionByHash { + hash: orphaned_transaction.hash() + }) + .await + .expect("Failed to look the transaction up"), + None + ); +} diff --git a/lez/sequencer/actors/storage/src/trait.rs b/lez/sequencer/actors/storage/src/trait.rs new file mode 100644 index 000000000..bae809f76 --- /dev/null +++ b/lez/sequencer/actors/storage/src/trait.rs @@ -0,0 +1,67 @@ +use common::{ + block::{Block, BlockMeta, PeerChainTip}, + transaction::LeeTransaction, +}; +use kameo::{Actor, message::Message}; +use lee::V03State; +use lee_core::BlockId; + +use crate::{ + Result, + error::Error, + protocol::{ + AddPendingCrossZoneDispatches, AddPendingDepositEvent, ApplyStoreUpdate, + CleanPendingBlocksUpTo, ConsumeUnseenWithdrawCount, DbDump, DeadLetterDispatchRecord, + DeleteBlock, DeleteCrossZonePeerFloor, DeleteZoneCheckpoint, DispatchFailure, + DropSettledCrossZoneDispatches, DumpDb, GetAllBlocks, GetBlock, GetCrossZonePeerFloorBytes, + GetCrossZonePeerTip, GetDeadLetterDispatchCount, GetDeadLetterDispatches, GetFinalSnapshot, + GetFirstBlockId, GetLastBlockId, GetLatestBlockMeta, GetLeeState, + GetPendingCrossZoneDispatches, GetPendingDepositEvents, GetPublishedHighWater, + GetTransactionByHash, GetZoneAnchor, GetZoneCheckpointBytes, MarkBlockAsFinalized, + PendingCrossZoneDispatchRecord, PendingDepositEventRecord, RaisePublishedHighWater, + RecordDispatchFailure, RecordNewBlock, ResetAllBlocksToPending, SetCrossZonePeerFloorBytes, + SetCrossZonePeerTip, SetZoneAnchor, SetZoneCheckpointBytes, StoreUpdateOutcome, + ZoneAnchorRecord, + }, +}; + +pub trait StorageActorTrait: + Actor + + Message> + + Message> + + Message>> + + Message>> + + Message>> + + Message> + + Message> + + Message> + + Message> + + Message>> + + Message>> + + Message>> + + Message>> + + Message>> + + Message>>> + + Message> + + Message> + + Message>> + + Message> + + Message>> + + Message> + + Message>> + + Message> + + Message> + + Message>> + + Message> + + Message> + + Message> + + Message>> + + Message> + + Message>>> + + Message> + + Message> + + Message>> + + Message> + + Message> +{ +} diff --git a/lez/sequencer/core/Cargo.toml b/lez/sequencer/core/Cargo.toml index 4854197d4..d3b3414fc 100644 --- a/lez/sequencer/core/Cargo.toml +++ b/lez/sequencer/core/Cargo.toml @@ -13,7 +13,7 @@ lee_core.workspace = true chain_state.workspace = true common.workspace = true sequencer_core_metrics = { workspace = true, features = ["record"] } -storage.workspace = true +sequencer_storage_actor.workspace = true mempool.workspace = true logos-blockchain-zone-sdk.workspace = true testnet_initial_state.workspace = true @@ -30,6 +30,7 @@ sequencer_stake_core.workspace = true logos-blockchain-key-management-system-service.workspace = true logos-blockchain-core.workspace = true logos-blockchain-http-api-common.workspace = true +kameo.workspace = true anyhow.workspace = true serde.workspace = true serde_json.workspace = true @@ -48,7 +49,6 @@ url.workspace = true num-bigint.workspace = true risc0-zkvm.workspace = true futures.workspace = true -itertools.workspace = true libp2p.workspace = true [features] @@ -61,6 +61,8 @@ mdns = [] [dev-dependencies] cross_zone = { workspace = true, features = ["test-utils"] } +sequencer_storage_actor = { workspace = true, features = ["mock"] } +storage.workspace = true futures.workspace = true test_programs.workspace = true lee = { workspace = true, features = ["test-utils"] } diff --git a/lez/sequencer/core/src/block_publisher.rs b/lez/sequencer/core/src/block_publisher.rs index 3462103b0..80e058716 100644 --- a/lez/sequencer/core/src/block_publisher.rs +++ b/lez/sequencer/core/src/block_publisher.rs @@ -2,7 +2,7 @@ use std::time::Duration; use anyhow::{Context as _, Result, anyhow, ensure}; use common::block::Block; -use futures::Stream; +use futures::{Stream, future::BoxFuture}; use log::{info, warn}; pub use logos_blockchain_core::mantle::{ ledger::NoteId, @@ -77,7 +77,7 @@ pub struct FollowUpdate { /// Sink for the follow path: apply the channel delta to chain state and /// persist the whole event in one write. -pub type OnFollowSink = Box; +pub type OnFollowSink = Box BoxFuture<'static, ()> + Send + Sync + 'static>; /// What one publish produced. pub struct PublishOutcome { @@ -417,10 +417,6 @@ impl BlockPublisherTrait for ZoneSdkPublisher { } } - // Nothing is awaited here: an await in this - // arm blocks the same task `publish_block` - // needs, and a non-turn sequencer never - // drains what it would be waiting on. on_follow(FollowUpdate { checkpoint, adopted, @@ -428,7 +424,7 @@ impl BlockPublisherTrait for ZoneSdkPublisher { finalized: finalized_blocks, deposits, withdrawals, - }); + }).await; } Event::Ready => {} Event::TurnNotification { notification } => { diff --git a/lez/sequencer/core/src/block_store.rs b/lez/sequencer/core/src/block_store.rs index 76e3fcfe0..85e16d524 100644 --- a/lez/sequencer/core/src/block_store.rs +++ b/lez/sequencer/core/src/block_store.rs @@ -1,142 +1,112 @@ -use std::{collections::HashMap, path::Path, sync::Arc}; +use std::sync::Arc; use anyhow::{Context as _, Result}; -use common::{ - HashType, - block::{Block, BlockMeta}, - transaction::LeeTransaction, -}; +use common::block::{Block, BlockMeta}; +use kameo::actor::ActorRef; use lee::V03State; use lee_core::BlockId; -use logos_blockchain_zone_sdk::{Slot, sequencer::SequencerCheckpoint}; -use storage::sequencer::{ - RocksDBIO, - sequencer_cells::{ - PeerZoneKey, PendingDepositEventRecord, WithdrawalReconciliationKey, ZoneAnchorRecord, +use logos_blockchain_zone_sdk::sequencer::SequencerCheckpoint; +use sequencer_storage_actor::{ + StorageActorTrait, + protocol::{ + CleanPendingBlocksUpTo, DeadLetterDispatchRecord, DeleteBlock, DeleteZoneCheckpoint, + DispatchFailure, DispatchOrigin, DropSettledCrossZoneDispatches, GetAllBlocks, GetBlock, + GetDeadLetterDispatchCount, GetDeadLetterDispatches, GetFinalSnapshot, GetFirstBlockId, + GetLastBlockId, GetLatestBlockMeta, GetLeeState, GetPendingCrossZoneDispatches, + GetPendingDepositEvents, GetPublishedHighWater, GetZoneAnchor, GetZoneCheckpointBytes, + MarkBlockAsFinalized, PendingCrossZoneDispatchRecord, PendingDepositEventRecord, + RaisePublishedHighWater, RecordDispatchFailure, RecordNewBlock, ResetAllBlocksToPending, + SetZoneAnchor, SetZoneCheckpointBytes, WithdrawalReconciliationKey, ZoneAnchorRecord, }, }; -pub use storage::{DbResult, sequencer::DbDump}; -pub struct SequencerStore { - dbio: Arc, - // TODO: Consider adding the hashmap to the database for faster recovery. - tx_hash_to_block_map: HashMap, +// TODO: Remove entirely, asking the storage actor directly and moving the +// remaining fields to [`crate::SequencerCore`]. +pub struct SequencerStore { + storage_ref: ActorRef, genesis_id: u64, signing_key: lee::PrivateKey, } -impl SequencerStore { - /// Open existing database at the given location. Fails if no database is found. - pub fn open_db(location: &Path, signing_key: lee::PrivateKey) -> DbResult { - let dbio = Arc::new(RocksDBIO::open(location)?); - Self::from_dbio_and_signing_key(dbio, signing_key) - } - - /// Create a fresh rocksdb at `location` from `dump`. - pub fn restore_db_from_dump( - location: &Path, - dump: &DbDump, - signing_key: lee::PrivateKey, - ) -> DbResult { - let dbio = Arc::new(RocksDBIO::restore_from_dump(location, dump)?); - Self::from_dbio_and_signing_key(dbio, signing_key) - } - - /// Starting database at the start of new chain. - /// Creates files if necessary. - /// - /// ATTENTION: Will overwrite genesis block. - pub fn create_db_with_genesis( - location: &Path, - genesis_block: &Block, - genesis_state: &V03State, - signing_key: lee::PrivateKey, - ) -> DbResult { - let dbio = Arc::new(RocksDBIO::create(location, genesis_block, genesis_state)?); - let genesis_id = dbio.get_meta_first_block_in_db()?; - let tx_hash_to_block_map = block_to_transactions_map(genesis_block); +impl SequencerStore { + pub async fn new(storage_ref: ActorRef, signing_key: lee::PrivateKey) -> Result { + let genesis_id = storage_ref + .ask(GetFirstBlockId) + .await? + .context("Store holds no chain; it must be seeded with a genesis block first")?; Ok(Self { - dbio, - tx_hash_to_block_map, + storage_ref, genesis_id, signing_key, }) } - fn from_dbio_and_signing_key( - dbio: Arc, - signing_key: lee::PrivateKey, - ) -> DbResult { - let genesis_id = dbio.get_meta_first_block_in_db()?; - let last_id = dbio.latest_block_meta()?.map(|meta| meta.id); - - let mut tx_hash_to_block_map = HashMap::new(); - - if let Some(last_id) = last_id { - log::info!("Preparing block cache"); - for i in genesis_id..=last_id { - let block = dbio - .get_block(i)? - .expect("Block should be present in the database"); - - tx_hash_to_block_map.extend(block_to_transactions_map(&block)); - } - log::info!( - "Block cache prepared. Total blocks in cache: {}", - tx_hash_to_block_map.len() - ); - } - - Ok(Self { - dbio, - tx_hash_to_block_map, - genesis_id, - signing_key, - }) + pub async fn block_at_id(&self, id: u64) -> Result> { + self.storage_ref + .ask(GetBlock { block_id: id }) + .await + .map_err(Into::into) } - /// Shared handle to the underlying rocksdb. Used to persist the zone-sdk - /// checkpoint from the sequencer's drive task without needing &mut to the - /// store. - #[must_use] - pub fn dbio(&self) -> Arc { - Arc::clone(&self.dbio) + pub async fn get_all_blocks(&self) -> Result> { + self.storage_ref.ask(GetAllBlocks).await.map_err(Into::into) } - pub fn get_block_at_id(&self, id: u64) -> DbResult> { - self.dbio.get_block(id) + pub async fn delete_block_at_id(&mut self, block_id: u64) -> Result<()> { + self.storage_ref + .ask(DeleteBlock { block_id }) + .await + .map_err(Into::into) } - pub fn delete_block_at_id(&mut self, block_id: u64) -> DbResult<()> { - self.dbio.delete_block(block_id) + pub async fn mark_block_as_finalized(&mut self, block_id: u64) -> Result<()> { + self.storage_ref + .ask(MarkBlockAsFinalized { block_id }) + .await + .map_err(Into::into) } - pub fn mark_block_as_finalized(&mut self, block_id: u64) -> DbResult<()> { - self.dbio.mark_block_as_finalized(block_id) + /// Reset every stored block to `Pending` so the next fresh start republishes the whole chain. + pub async fn reset_all_blocks_to_pending(&self) -> Result<()> { + self.storage_ref + .ask(ResetAllBlocksToPending) + .await + .map_err(Into::into) } - /// Returns the transaction corresponding to the given hash, if it exists in the blockchain. - #[must_use] - pub fn get_transaction_by_hash(&self, hash: HashType) -> Option<(LeeTransaction, BlockId)> { - let block_id = *self.tx_hash_to_block_map.get(&hash)?; - let block = self - .get_block_at_id(block_id) - .ok() - .flatten() - .expect("Block should be present since the hash is in the map"); - for transaction in block.body.transactions { - if transaction.hash() == hash { - return Some((transaction, block_id)); - } - } - panic!( - "Transaction hash was in the map but transaction was not found in the block. This should never happen." - ); + /// Persists `block` with the effects it covers. + pub async fn record_new_block( + &mut self, + block: Block, + withdrawals: Vec, + state: Arc, + checkpoint_bytes: Option>, + ) -> Result<()> { + self.storage_ref + .ask(RecordNewBlock { + block, + withdrawals, + state, + checkpoint_bytes, + }) + .await?; + Ok(()) } - pub fn latest_block_meta(&self) -> DbResult> { - self.dbio.latest_block_meta() + /// The id of the chain's last block, or `None` on a store holding no chain. + pub async fn last_block_id(&self) -> Result> { + self.storage_ref + .ask(GetLastBlockId) + .await + .map_err(Into::into) + } + + pub async fn latest_block_meta(&self) -> Result> { + self.storage_ref + .ask(GetLatestBlockMeta) + .await + .map_err(Into::into) } #[must_use] @@ -149,45 +119,22 @@ impl SequencerStore { &self.signing_key } - pub fn get_all_blocks(&self) -> impl Iterator> { - self.dbio.get_all_blocks() - } - - pub(crate) fn update( - &mut self, - block: &Block, - withdrawals: &[WithdrawalReconciliationKey], - state: &V03State, - checkpoint: Option<&[u8]>, - ) -> DbResult<()> { - let new_transactions_map = block_to_transactions_map(block); - self.dbio - .atomic_update(block, withdrawals, state, checkpoint)?; - self.tx_hash_to_block_map.extend(new_transactions_map); - Ok(()) - } - - pub fn get_lee_state(&self) -> DbResult { - self.dbio.get_lee_state() + /// The state after the last applied block, or `None` on a store holding no + /// chain. + pub async fn get_lee_state(&self) -> Result> { + self.storage_ref.ask(GetLeeState).await.map_err(Into::into) } /// Remove the persisted zone-sdk checkpoint so the next startup is treated as a fresh start. - pub fn delete_zone_checkpoint(&self) -> DbResult<()> { - self.dbio.delete_zone_sdk_checkpoint_bytes() + pub async fn delete_zone_checkpoint(&self) -> Result<()> { + self.storage_ref + .ask(DeleteZoneCheckpoint) + .await + .map_err(Into::into) } - /// Reset every stored block to `Pending` so the next fresh start republishes the whole chain. - pub fn reset_all_blocks_to_pending(&self) -> DbResult<()> { - self.dbio.reset_all_blocks_to_pending() - } - - /// Single-blob [`DbDump`] of the whole store; restore with [`Self::restore_db_from_dump`]. - pub fn dump(&self) -> DbResult { - self.dbio.dump_all() - } - - pub fn get_zone_checkpoint(&self) -> Result> { - let Some(bytes) = self.dbio.get_zone_sdk_checkpoint_bytes()? else { + pub async fn get_zone_checkpoint(&self) -> Result> { + let Some(bytes) = self.storage_ref.ask(GetZoneCheckpointBytes).await? else { return Ok(None); }; let checkpoint: SequencerCheckpoint = serde_json::from_slice(&bytes) @@ -197,36 +144,131 @@ impl SequencerStore { /// Persists `checkpoint` on its own. Only valid when the effects it covers /// are already durable โ€” otherwise it must ride in the same write as them, - /// via [`storage::sequencer::StoreUpdate`]. - pub fn set_zone_checkpoint(&self, checkpoint: &SequencerCheckpoint) -> Result<()> { - self.dbio - .put_zone_sdk_checkpoint_bytes(&checkpoint_bytes(checkpoint)?)?; + /// via [`Self::record_new_block`]. + pub async fn set_zone_checkpoint(&self, checkpoint: &SequencerCheckpoint) -> Result<()> { + self.storage_ref + .ask(SetZoneCheckpointBytes { + bytes: checkpoint_bytes(checkpoint)?, + }) + .await?; Ok(()) } /// The last channel block read back and verified from Bedrock (L1 slot + /// `id`/`hash`), or `None` before any block has been read from the channel. - pub fn get_zone_anchor(&self) -> DbResult> { - self.dbio.get_zone_anchor() + pub async fn get_zone_anchor(&self) -> Result> { + self.storage_ref + .ask(GetZoneAnchor) + .await + .map_err(Into::into) } - pub fn set_zone_anchor(&self, anchor: &ZoneAnchorRecord) -> DbResult<()> { - self.dbio.put_zone_anchor(anchor) + pub async fn set_zone_anchor(&self, anchor: ZoneAnchorRecord) -> Result<()> { + self.storage_ref + .ask(SetZoneAnchor { anchor }) + .await + .map_err(Into::into) } /// The highest block id ever inscribed on the channel by this sequencer, /// or `None` before it has published anything. - pub fn published_high_water(&self) -> DbResult> { - self.dbio.published_high_water() + pub async fn published_high_water(&self) -> Result> { + self.storage_ref + .ask(GetPublishedHighWater) + .await + .map_err(Into::into) } /// Raises the published high water mark to `block_id`, never lowering it. - pub fn raise_published_high_water(&self, block_id: u64) -> DbResult<()> { - self.dbio.raise_published_high_water(block_id) + pub async fn raise_published_high_water(&self, block_id: u64) -> Result<()> { + self.storage_ref + .ask(RaisePublishedHighWater { block_id }) + .await + .map_err(Into::into) } - pub fn get_pending_deposit_events(&self) -> DbResult> { - self.dbio.get_pending_deposit_events() + pub async fn get_pending_deposit_events(&self) -> Result> { + self.storage_ref + .ask(GetPendingDepositEvents) + .await + .map_err(Into::into) + } + + /// The persisted final-tier `(state, meta)`, or `None` before anything + /// finalized. + pub async fn get_final_snapshot(&self) -> Result> { + self.storage_ref + .ask(GetFinalSnapshot) + .await + .map_err(Into::into) + } + + /// Marks every stored pending block at or below `last_finalized` as + /// finalized. + pub async fn clean_pending_blocks_up_to(&self, last_finalized: BlockId) -> Result<()> { + self.storage_ref + .ask(CleanPendingBlocksUpTo { last_finalized }) + .await + .map_err(Into::into) + } + + pub async fn pending_cross_zone_dispatches( + &self, + ) -> Result> { + self.storage_ref + .ask(GetPendingCrossZoneDispatches) + .await + .map_err(Into::into) + } + + pub async fn drop_settled_cross_zone_dispatches( + &self, + message_keys: Vec<[u8; 32]>, + ) -> Result { + self.storage_ref + .ask(DropSettledCrossZoneDispatches { message_keys }) + .await + .map_err(Into::into) + } + + /// Counts one failed production attempt against `message_key`, giving up on + /// it once `retire_at` accumulate. + pub async fn record_dispatch_failure( + &self, + message_key: [u8; 32], + retire_at: u32, + origin: DispatchOrigin, + ) -> Result { + self.storage_ref + .ask(RecordDispatchFailure { + message_key, + retire_at, + origin, + }) + .await + .map_err(Into::into) + } + + pub async fn dead_letter_dispatches(&self) -> Result> { + self.storage_ref + .ask(GetDeadLetterDispatches) + .await + .map_err(Into::into) + } + + pub async fn dead_letter_dispatch_count(&self) -> Result { + self.storage_ref + .ask(GetDeadLetterDispatchCount) + .await + .map_err(Into::into) + } + + /// The handle to the actor behind this store, for the paths that hold no + /// store of their own: the publisher's follow sink and the cross-zone + /// watchers, each of which outlives any one caller. + #[must_use] + pub const fn storage_ref(&self) -> &ActorRef { + &self.storage_ref } } @@ -236,249 +278,123 @@ pub(crate) fn checkpoint_bytes(checkpoint: &SequencerCheckpoint) -> Result HashMap { - block - .body - .transactions - .iter() - .map(|transaction| (transaction.hash(), block.header.block_id)) - .collect() -} - -/// A cross-zone watcher's delivery floor on `peer_zone`'s channel. -/// -/// The highest slot every message of which was delivered, or `None` before it -/// has delivered anything from that peer. Stored as a little-endian `u64`. -/// -/// Free functions rather than only [`SequencerStore`] methods because each -/// watcher runs as its own spawned task and holds an `Arc`; -/// `SequencerStore` is not `Clone`. -pub fn get_cross_zone_peer_floor(dbio: &RocksDBIO, peer_zone: PeerZoneKey) -> Result> { - let Some(bytes) = dbio.get_cross_zone_peer_floor_bytes(peer_zone)? else { - return Ok(None); - }; - let bytes: [u8; 8] = bytes.as_slice().try_into().with_context(|| { - format!( - "Stored cross-zone peer floor is {} bytes, expected 8", - bytes.len() - ) - })?; - Ok(Some(Slot::new(u64::from_le_bytes(bytes)))) -} - -pub fn set_cross_zone_peer_floor( - dbio: &RocksDBIO, - peer_zone: PeerZoneKey, - floor: Slot, -) -> Result<()> { - dbio.put_cross_zone_peer_floor_bytes(peer_zone, &floor.to_le_bytes())?; - Ok(()) -} - -/// Drops the stored floor so the watcher reads `peer_zone`'s channel from the -/// peer's genesis again. -pub fn clear_cross_zone_peer_floor(dbio: &RocksDBIO, peer_zone: PeerZoneKey) -> Result<()> { - dbio.delete_cross_zone_peer_floor(peer_zone)?; - Ok(()) -} - #[cfg(test)] mod tests { - use common::{block::HashableBlockData, test_utils::sequencer_sign_key_for_testing}; + use std::path::Path; + + use common::{HashType, block::HashableBlockData, test_utils::sequencer_sign_key_for_testing}; + use kameo::actor::Spawn as _; + use sequencer_storage_actor::StorageActor; use tempfile::tempdir; use super::*; - #[test] - fn get_transaction_by_hash() { - let temp_dir = tempdir().unwrap(); - let path = temp_dir.path(); - - let signing_key = sequencer_sign_key_for_testing(); - - let genesis_block_hashable_data = HashableBlockData { + fn genesis_block(signing_key: &lee::PrivateKey) -> Block { + HashableBlockData { block_id: 0, prev_block_hash: HashType([0; 32]), timestamp: 0, transactions: vec![], - }; - - let genesis_block = genesis_block_hashable_data.into_pending_block(&signing_key); - // Start an empty node store - let mut node_store = SequencerStore::create_db_with_genesis( - path, - &genesis_block, - &testnet_initial_state::initial_state(), - signing_key, - ) - .unwrap(); - - let tx = common::test_utils::produce_dummy_empty_transaction(); - let block = common::test_utils::produce_dummy_block(1, None, vec![tx.clone()]); - - // Try retrieve a tx that's not in the chain yet. - let retrieved_tx = node_store.get_transaction_by_hash(tx.hash()); - assert_eq!(None, retrieved_tx); - // Add the block with the transaction - let dummy_state = V03State::new(); - node_store.update(&block, &[], &dummy_state, None).unwrap(); - // Try again - let output = node_store.get_transaction_by_hash(tx.hash()); - assert_eq!(Some((tx, 1)), output); + } + .into_pending_block(signing_key) } - #[test] - fn latest_block_meta_returns_genesis_meta_initially() { + /// Creates a fresh database at `path` seeded with `genesis` and opens a + /// store on the actor serving it. + async fn create_store( + path: &Path, + genesis: &Block, + signing_key: lee::PrivateKey, + ) -> SequencerStore { + let storage_ref = StorageActor::spawn(StorageActor::new(path).unwrap()); + storage_ref + .ask(RecordNewBlock { + block: genesis.clone(), + withdrawals: vec![], + state: Arc::new(testnet_initial_state::initial_state()), + checkpoint_bytes: None, + }) + .await + .unwrap(); + SequencerStore::new(storage_ref, signing_key).await.unwrap() + } + + #[tokio::test] + async fn latest_block_meta_returns_genesis_meta_initially() { let temp_dir = tempdir().unwrap(); - let path = temp_dir.path(); - let signing_key = sequencer_sign_key_for_testing(); + let genesis = genesis_block(&signing_key); + let genesis_hash = genesis.header.hash; - let genesis_block_hashable_data = HashableBlockData { - block_id: 0, - prev_block_hash: HashType([0; 32]), - timestamp: 0, - transactions: vec![], - }; - - let genesis_block = genesis_block_hashable_data.into_pending_block(&signing_key); - let genesis_hash = genesis_block.header.hash; - - let node_store = SequencerStore::create_db_with_genesis( - path, - &genesis_block, - &testnet_initial_state::initial_state(), - signing_key, - ) - .unwrap(); + let store = create_store(temp_dir.path(), &genesis, signing_key).await; // Verify that initially the latest block hash equals genesis hash - let latest_meta = node_store.latest_block_meta().unwrap().unwrap(); + let latest_meta = store.latest_block_meta().await.unwrap().unwrap(); assert_eq!(latest_meta.hash, genesis_hash); } - #[test] - fn latest_block_meta_updates_after_new_block() { + #[tokio::test] + async fn latest_block_meta_updates_after_new_block() { let temp_dir = tempdir().unwrap(); - let path = temp_dir.path(); - let signing_key = sequencer_sign_key_for_testing(); - - let genesis_block_hashable_data = HashableBlockData { - block_id: 0, - prev_block_hash: HashType([0; 32]), - timestamp: 0, - transactions: vec![], - }; - - let genesis_block = genesis_block_hashable_data.into_pending_block(&signing_key); - let mut node_store = SequencerStore::create_db_with_genesis( - path, - &genesis_block, - &testnet_initial_state::initial_state(), - signing_key, + let mut store = create_store( + temp_dir.path(), + &genesis_block(&signing_key), + signing_key.clone(), ) - .unwrap(); + .await; // Add a new block let tx = common::test_utils::produce_dummy_empty_transaction(); let block = common::test_utils::produce_dummy_block(1, None, vec![tx]); let block_hash = block.header.hash; - let dummy_state = V03State::new(); - node_store.update(&block, &[], &dummy_state, None).unwrap(); + store + .record_new_block(block.clone(), vec![], Arc::new(V03State::new()), None) + .await + .unwrap(); // Verify that the latest block meta now equals the new block's hash - let latest_meta = node_store.latest_block_meta().unwrap().unwrap(); + let latest_meta = store.latest_block_meta().await.unwrap().unwrap(); assert_eq!(latest_meta.hash, block_hash); } - #[test] - fn mark_block_finalized() { + #[tokio::test] + async fn mark_block_finalized() { let temp_dir = tempdir().unwrap(); - let path = temp_dir.path(); - let signing_key = sequencer_sign_key_for_testing(); - - let genesis_block_hashable_data = HashableBlockData { - block_id: 0, - prev_block_hash: HashType([0; 32]), - timestamp: 0, - transactions: vec![], - }; - - let genesis_block = genesis_block_hashable_data.into_pending_block(&signing_key); - let mut node_store = SequencerStore::create_db_with_genesis( - path, - &genesis_block, - &testnet_initial_state::initial_state(), - signing_key, + let mut store = create_store( + temp_dir.path(), + &genesis_block(&signing_key), + signing_key.clone(), ) - .unwrap(); + .await; // Add a new block with Pending status let tx = common::test_utils::produce_dummy_empty_transaction(); let block = common::test_utils::produce_dummy_block(1, None, vec![tx]); let block_id = block.header.block_id; - let dummy_state = V03State::new(); - node_store.update(&block, &[], &dummy_state, None).unwrap(); + store + .record_new_block(block.clone(), vec![], Arc::new(V03State::new()), None) + .await + .unwrap(); // Verify initial status is Pending - let retrieved_block = node_store.get_block_at_id(block_id).unwrap().unwrap(); + let retrieved_block = store.block_at_id(block_id).await.unwrap().unwrap(); assert!(matches!( retrieved_block.bedrock_status, common::block::BedrockStatus::Pending )); // Mark block as finalized - node_store.mark_block_as_finalized(block_id).unwrap(); + store.mark_block_as_finalized(block_id).await.unwrap(); // Verify status is now Finalized - let finalized_block = node_store.get_block_at_id(block_id).unwrap().unwrap(); + let finalized_block = store.block_at_id(block_id).await.unwrap().unwrap(); assert!(matches!( finalized_block.bedrock_status, common::block::BedrockStatus::Finalized )); } - - #[test] - fn open_existing_db_caches_transactions() { - let temp_dir = tempdir().unwrap(); - let path = temp_dir.path(); - - let signing_key = sequencer_sign_key_for_testing(); - - let genesis_block_hashable_data = HashableBlockData { - block_id: 0, - prev_block_hash: HashType([0; 32]), - timestamp: 0, - transactions: vec![], - }; - - let genesis_block = genesis_block_hashable_data.into_pending_block(&signing_key); - let tx = common::test_utils::produce_dummy_empty_transaction(); - { - // Create a scope to drop the first store after creating the db - let mut node_store = SequencerStore::create_db_with_genesis( - path, - &genesis_block, - &testnet_initial_state::initial_state(), - signing_key.clone(), - ) - .unwrap(); - - // Add a new block - let block = common::test_utils::produce_dummy_block(1, None, vec![tx.clone()]); - node_store - .update(&block, &[], &V03State::new(), None) - .unwrap(); - } - - // Re-open the store and verify that the transaction is still retrievable (which means it - // was cached correctly) - let node_store = SequencerStore::open_db(path, signing_key).unwrap(); - let output = node_store.get_transaction_by_hash(tx.hash()); - assert_eq!(Some((tx, 1)), output); - } } diff --git a/lez/sequencer/core/src/cross_zone_watcher.rs b/lez/sequencer/core/src/cross_zone_watcher.rs index 07bbc1410..43437f848 100644 --- a/lez/sequencer/core/src/cross_zone_watcher.rs +++ b/lez/sequencer/core/src/cross_zone_watcher.rs @@ -1,27 +1,34 @@ -use std::{sync::Arc, time::Duration}; +use std::time::Duration; -use common::{HashType, block::Block, transaction::LeeTransaction}; +use anyhow::{Context as _, Result}; +use common::{ + HashType, + block::{Block, PeerChainTip}, + transaction::LeeTransaction, +}; use cross_zone::{ EmissionSource, Link, StallState, alerts_at, build_dispatch_from_emission, equivocation_report, extract_emission, is_sequencer_only_program, link_to_tip, screen_peer_block, }; use cross_zone_inbox_core::message_key; use futures::{Stream, StreamExt as _}; +use kameo::actor::ActorRef; use lee::PublicKey; use log::{debug, error, warn}; use logos_blockchain_core::mantle::ops::channel::ChannelId; use logos_blockchain_zone_sdk::{ CommonHttpClient, Slot, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer, }; -use storage::sequencer::{ - RocksDBIO, - sequencer_cells::{PeerChainTip, PendingCrossZoneDispatchRecord}, +use sequencer_storage_actor::{ + StorageActorTrait, + protocol::{ + AddPendingCrossZoneDispatches, DeleteCrossZonePeerFloor, GetCrossZonePeerFloorBytes, + GetCrossZonePeerTip, PeerZoneKey, PendingCrossZoneDispatchRecord, + SetCrossZonePeerFloorBytes, SetCrossZonePeerTip, + }, }; use crate::{ - block_store::{ - clear_cross_zone_peer_floor, get_cross_zone_peer_floor, set_cross_zone_peer_floor, - }, config::{BedrockConfig, CrossZoneConfig}, task_group::TaskGroup, }; @@ -119,6 +126,56 @@ const fn resume_from(tip: Option, floor: Option) -> Resume { } } +/// This watcher's delivery floor on `peer_zone`'s channel. +/// +/// The highest slot every message of which was delivered, or `None` before it +/// has delivered anything from that peer. Stored as a little-endian `u64`, which +/// is why the encoding lives here rather than in the storage actor. +async fn get_cross_zone_peer_floor( + storage_ref: &ActorRef, + peer_zone: PeerZoneKey, +) -> Result> { + let Some(bytes) = storage_ref + .ask(GetCrossZonePeerFloorBytes { peer_zone }) + .await? + else { + return Ok(None); + }; + let bytes: [u8; 8] = bytes.as_slice().try_into().with_context(|| { + format!( + "Stored cross-zone peer floor is {} bytes, expected 8", + bytes.len() + ) + })?; + Ok(Some(Slot::new(u64::from_le_bytes(bytes)))) +} + +async fn set_cross_zone_peer_floor( + storage_ref: &ActorRef, + peer_zone: PeerZoneKey, + floor: Slot, +) -> Result<()> { + storage_ref + .ask(SetCrossZonePeerFloorBytes { + peer_zone, + bytes: floor.to_le_bytes().to_vec(), + }) + .await?; + Ok(()) +} + +/// Drops the stored floor so the watcher reads `peer_zone`'s channel from the +/// peer's genesis again. +async fn clear_cross_zone_peer_floor( + storage_ref: &ActorRef, + peer_zone: PeerZoneKey, +) -> Result<()> { + storage_ref + .ask(DeleteCrossZonePeerFloor { peer_zone }) + .await?; + Ok(()) +} + /// Spawns one watcher task per configured peer. /// /// Each task reads the peer's finalized blocks from Bedrock, recognizes outbound @@ -129,15 +186,13 @@ const fn resume_from(tip: Option, floor: Option) -> Resume { /// The returned group must be kept alive for as long as the watchers should /// run; dropping it stops them, and awaiting /// [`TaskGroup::shutdown`](crate::task_group::TaskGroup::shutdown) is what -/// proves they have stopped. Each watcher holds an `Arc`, so a -/// watcher still running keeps the `RocksDB` lock held and a restarting -/// sequencer cannot reopen its home directory. +/// proves they have stopped. #[must_use] -pub fn spawn_watchers( +pub fn spawn_watchers( bedrock_config: &BedrockConfig, cross_zone: &CrossZoneConfig, poll_interval: Duration, - dbio: &Arc, + storage_ref: &ActorRef, ) -> TaskGroup { let self_zone: [u8; 32] = *bedrock_config.channel_id.as_ref(); let mut tasks = Vec::new(); @@ -158,7 +213,7 @@ pub fn spawn_watchers( expected_pubkey, }, poll_interval, - Arc::clone(dbio), + storage_ref.clone(), ))); } @@ -169,11 +224,11 @@ pub fn spawn_watchers( clippy::infinite_loop, reason = "the peer watcher runs for the lifetime of the sequencer process" )] -async fn watch_peer( +async fn watch_peer( zone_indexer: ZoneIndexer, peer: PeerContext, poll_interval: Duration, - dbio: Arc, + storage_ref: ActorRef, ) { let peer_zone = peer.peer_zone; log::info!( @@ -186,7 +241,7 @@ async fn watch_peer( // key is content-addressed and the inbox no-ops a replay) but re-records // every already-delivered message, so without this a restart replayed the // peer's whole history into the store. - let floor = match get_cross_zone_peer_floor(&dbio, peer_zone) { + let floor = match get_cross_zone_peer_floor(&storage_ref, peer_zone).await { Ok(floor) => floor, Err(err) => { // Falling back to `None` would re-read the peer's whole history and @@ -202,7 +257,7 @@ async fn watch_peer( // The chain this watcher has already delivered from. Without it no block can // be told apart from one claiming an id it never reached, so a watcher that // cannot read it delivers nothing rather than guessing. - let mut tip = match dbio.get_cross_zone_peer_tip(peer_zone) { + let mut tip = match storage_ref.ask(GetCrossZonePeerTip { peer_zone }).await { Ok(tip) => tip, Err(err) => { error!( @@ -220,7 +275,7 @@ async fn watch_peer( ); // Durably, before reading anything, or a crash partway through the // rebuild resumes from the stale floor with nothing able to link. - if let Err(err) = clear_cross_zone_peer_floor(&dbio, peer_zone) { + if let Err(err) = clear_cross_zone_peer_floor(&storage_ref, peer_zone).await { error!( "Watcher could not clear the stale delivery floor for peer {}: {err:#}. Stopping this watcher rather than rebuilding its chain against a floor a restart would resume from.", hex::encode(peer_zone) @@ -251,7 +306,7 @@ async fn watch_peer( continue; } }; - let outcome = consume_peer_stream(stream, &peer, &dbio, &mut cursor, &mut tip).await; + let outcome = consume_peer_stream(stream, &peer, &storage_ref, &mut cursor, &mut tip).await; if let Some((slot, attempts)) = state.after_pass(outcome, cursor) && alerts_at(attempts) @@ -259,7 +314,10 @@ async fn watch_peer( error!( "Watcher for peer {} has been stuck at slot {slot:?} for {attempts} passes. Nothing from that peer is being delivered until it clears, and the delivery floor stays at {:?} so the slot keeps coming back.", hex::encode(peer_zone), - get_cross_zone_peer_floor(&dbio, peer_zone).ok().flatten() + get_cross_zone_peer_floor(&storage_ref, peer_zone) + .await + .ok() + .flatten() ); } // Reads on rather than stopping, since one such pass is ordinary, but a @@ -286,15 +344,15 @@ async fn watch_peer( /// /// Ending early holds the floor at the last slot consumed whole, so the next /// poll re-reads from there and a transient failure heals. -async fn consume_peer_stream( - stream: S, +async fn consume_peer_stream( + stream: Str, peer: &PeerContext, - dbio: &RocksDBIO, + storage_ref: &ActorRef, cursor: &mut Option, tip: &mut Option, ) -> PassOutcome where - S: Stream, + Str: Stream, { let mut stream = std::pin::pin!(stream); // The slot being consumed: every message of it seen so far is handled, but @@ -310,7 +368,7 @@ where if in_progress != Some(slot) { // A message from a later slot means the previous one completed. if let Some(done) = in_progress { - advance_cursor(dbio, peer.peer_zone, cursor, done); + advance_cursor(storage_ref, peer.peer_zone, cursor, done).await; } in_progress = Some(slot); } @@ -371,7 +429,7 @@ where ); } Link::Next(block_hash) => { - if !record_block_deliveries(&block, block_hash, peer, dbio) { + if !record_block_deliveries(&block, block_hash, peer, storage_ref).await { // Recording a delivery is what makes it survive the // mempool. Letting the pass finish here would move // the floor past this slot on a store that just @@ -392,7 +450,13 @@ where block_id: block.header.block_id, block_hash, }; - if let Err(err) = dbio.put_cross_zone_peer_tip(peer.peer_zone, next) { + if let Err(err) = storage_ref + .ask(SetCrossZonePeerTip { + peer_zone: peer.peer_zone, + tip: next, + }) + .await + { // Advancing only in memory would leave a restart // resuming from a floor above a tip, and every block // after it unlinkable. @@ -420,7 +484,7 @@ where // The stream drained cleanly, so the slot in progress completed too. if let Some(done) = in_progress { - advance_cursor(dbio, peer.peer_zone, cursor, done); + advance_cursor(storage_ref, peer.peer_zone, cursor, done).await; } if placed == 0 && skipped > 0 { return PassOutcome::Stranded; @@ -433,9 +497,14 @@ where /// /// A persist failure is only logged: the worst case is re-reading from the last /// stored slot after a restart, which delivery handles idempotently. -fn advance_cursor(dbio: &RocksDBIO, peer_zone: [u8; 32], cursor: &mut Option, slot: Slot) { +async fn advance_cursor( + storage_ref: &ActorRef, + peer_zone: [u8; 32], + cursor: &mut Option, + slot: Slot, +) { *cursor = Some(slot); - if let Err(err) = set_cross_zone_peer_floor(dbio, peer_zone, slot) { + if let Err(err) = set_cross_zone_peer_floor(storage_ref, peer_zone, slot).await { warn!( "Failed to persist watcher delivery floor for peer {}: {err:#}", hex::encode(peer_zone) @@ -451,11 +520,11 @@ fn advance_cursor(dbio: &RocksDBIO, peer_zone: [u8; 32], cursor: &mut Option( block: &Block, block_hash: HashType, peer: &PeerContext, - dbio: &RocksDBIO, + storage_ref: &ActorRef, ) -> bool { let peer_zone = peer.peer_zone; let self_zone = peer.self_zone; @@ -522,7 +591,12 @@ fn record_block_deliveries( } let offered = deliveries.len(); - match dbio.add_pending_cross_zone_dispatches(deliveries) { + match storage_ref + .ask(AddPendingCrossZoneDispatches { + dispatches: deliveries, + }) + .await + { // Fewer accepted than offered means the rest were recorded by an earlier // pass over the same slot, which the retry loop repeats for as long as // the slot stays stuck. @@ -558,12 +632,19 @@ fn record_block_deliveries( #[cfg(test)] mod tests { + use std::sync::{Arc, Mutex}; + use common::test_utils::produce_dummy_block; use cross_zone::test_utils::{linked_chain_to, ping_emission}; use futures::stream; + use kameo::actor::Spawn as _; use logos_blockchain_core::mantle::ops::channel::{MsgId, inscribe::Inscription}; use logos_blockchain_zone_sdk::ZoneBlock; - use storage::sequencer::{DB_META_PENDING_CROSS_ZONE_DISPATCH_COUNT_KEY, RocksDBIO}; + use sequencer_storage_actor::{ + StorageActor, + mock::MockStorageActor, + protocol::{GetPendingCrossZoneDispatches, RecordNewBlock}, + }; use tempfile::TempDir; use super::*; @@ -580,11 +661,25 @@ mod tests { } /// A store backed by a temp dir. The dir is returned so it outlives the db. - fn store() -> (TempDir, RocksDBIO) { + async fn store() -> (TempDir, ActorRef) { let dir = tempfile::tempdir().expect("temp dir"); - let genesis = produce_dummy_block(0, None, vec![]); - let dbio = RocksDBIO::create(dir.path(), &genesis, &lee::V03State::new()).expect("db"); - (dir, dbio) + let storage_ref = StorageActor::spawn(StorageActor::new(dir.path()).expect("open storage")); + seed_genesis(&storage_ref).await; + (dir, storage_ref) + } + + /// The watcher's messages need a database, not a chain, but the peer-tip + /// and dispatch cells share a store with one โ€” so seed it like a real node. + async fn seed_genesis(storage_ref: &ActorRef) { + storage_ref + .ask(RecordNewBlock { + block: produce_dummy_block(0, None, vec![]), + withdrawals: vec![], + state: Arc::new(lee::V03State::new()), + checkpoint_bytes: None, + }) + .await + .expect("seed genesis"); } /// A `ping_sender` emission addressed to `SELF_ZONE`. @@ -668,30 +763,58 @@ mod tests { /// The message keys recorded so far, sorted: the store keys each record by /// its message key, so no insertion order survives. - fn recorded_keys(dbio: &RocksDBIO) -> Vec<[u8; 32]> { - let mut keys: Vec<[u8; 32]> = dbio - .get_pending_cross_zone_dispatches() + async fn recorded_keys(storage_ref: &ActorRef) -> Vec<[u8; 32]> { + let mut keys = storage_ref + .ask(GetPendingCrossZoneDispatches) + .await .expect("pending dispatches readable") .into_iter() .map(|record| record.message_key) - .collect(); + .collect::>(); + keys.sort_unstable(); keys } - /// Makes every later pending-dispatch write fail, standing in for any - /// store failure before a delivery is durable: recording reads the count - /// first, so a count that will not decode is enough. - fn break_the_dispatch_store(dbio: &RocksDBIO) { - let cf = dbio - .db - .cf_handle(storage::CF_META_NAME) - .expect("meta column family"); - let key = - borsh::to_vec(&DB_META_PENDING_CROSS_ZONE_DISPATCH_COUNT_KEY).expect("key encodes"); - dbio.db - .put_cf(&cf, key, b"not a pending dispatch count") - .expect("write"); + /// A store that refuses every delivery write, standing in for any store + /// failure between reading a peer block and the delivery being durable. + fn store_refusing_deliveries() -> ActorRef { + let floor: Arc>>> = Arc::default(); + let tip: Arc>> = Arc::default(); + let mut storage = MockStorageActor::new(); + + storage + .expect_handle_add_pending_cross_zone_dispatches() + .returning(|_, _| { + Err(storage::error::DbError::db_interaction_error( + "the store refused the write".to_owned(), + ) + .into()) + }); + + let written_floor = Arc::clone(&floor); + storage + .expect_handle_set_cross_zone_peer_floor_bytes() + .returning(move |msg, _| { + *written_floor.lock().expect("floor cell") = Some(msg.bytes); + Ok(()) + }); + storage + .expect_handle_get_cross_zone_peer_floor_bytes() + .returning(move |_, _| Ok(floor.lock().expect("floor cell").clone())); + + let written_tip = Arc::clone(&tip); + storage + .expect_handle_set_cross_zone_peer_tip() + .returning(move |msg, _| { + *written_tip.lock().expect("tip cell") = Some(msg.tip); + Ok(()) + }); + storage + .expect_handle_get_cross_zone_peer_tip() + .returning(move |_, _| Ok(*tip.lock().expect("tip cell"))); + + MockStorageActor::spawn(storage) } /// Drives the state machine over a sequence of pass outcomes, with the read @@ -775,14 +898,14 @@ mod tests { #[tokio::test] async fn watcher_persists_its_cursor_as_it_consumes() { - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; let outcome = consume_peer_stream( stream::iter(vec![peer_block_msg(1, 0), peer_block_msg(2, 1)]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -791,13 +914,15 @@ mod tests { assert_eq!(outcome, PassOutcome::Drained); assert_eq!(cursor, Some(Slot::from(1))); assert_eq!( - get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(), + get_cross_zone_peer_floor(&storage_ref, PEER_ZONE) + .await + .unwrap(), Some(Slot::from(1)), "the cursor must be durable, not just in memory" ); let mut expected = vec![message_key(&PEER_ZONE, 1, 0), message_key(&PEER_ZONE, 2, 0)]; expected.sort_unstable(); - assert_eq!(recorded_keys(&dbio), expected); + assert_eq!(recorded_keys(&storage_ref).await, expected); } #[tokio::test] @@ -807,7 +932,7 @@ mod tests { // certainly refuse: the inbox is injected by this node alone, so a peer // naming it as a target is junk that would cost a pending slot and three // execution attempts. - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; @@ -818,7 +943,7 @@ mod tests { programs::cross_zone_inbox().id(), )]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -830,11 +955,13 @@ mod tests { "a message the watcher drops is not a failure" ); assert!( - recorded_keys(&dbio).is_empty(), + recorded_keys(&storage_ref).await.is_empty(), "a message aimed at a sequencer-only program must not be recorded" ); assert_eq!( - get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(), + get_cross_zone_peer_floor(&storage_ref, PEER_ZONE) + .await + .unwrap(), Some(Slot::from(0)), "the slot was fully read, so the floor still advances" ); @@ -844,7 +971,7 @@ mod tests { async fn a_delivery_to_an_unrelated_target_is_still_recorded() { // The watcher is not the authorization point any more. A target it knows // nothing about is recorded and delivered, and that target decides. - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; @@ -855,7 +982,7 @@ mod tests { programs::wrapped_token().id(), )]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -863,7 +990,7 @@ mod tests { assert_eq!(outcome, PassOutcome::Drained); assert_eq!( - recorded_keys(&dbio).len(), + recorded_keys(&storage_ref).await.len(), 1, "the watcher records it and lets the target refuse it" ); @@ -871,14 +998,14 @@ mod tests { #[tokio::test] async fn watcher_records_every_delivery_it_reads() { - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; consume_peer_stream( stream::iter(vec![peer_block_msg(1, 0)]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -888,7 +1015,10 @@ mod tests { // never re-read. The record is the whole of what survives that: block // production drains it, and it outlives a restart. It is dropped when // the delivery itself becomes irreversible, not when it is included. - let records = dbio.get_pending_cross_zone_dispatches().unwrap(); + let records = storage_ref + .ask(GetPendingCrossZoneDispatches) + .await + .unwrap(); assert_eq!(records.len(), 1, "the delivery must be recorded"); assert_eq!( records[0].message_key, @@ -907,20 +1037,23 @@ mod tests { #[tokio::test] async fn a_recorded_delivery_names_the_hash_the_watcher_validated() { - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; consume_peer_stream( stream::iter(vec![peer_block_msg(1, 0)]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) .await; - let records = dbio.get_pending_cross_zone_dispatches().unwrap(); + let records = storage_ref + .ask(GetPendingCrossZoneDispatches) + .await + .unwrap(); assert_eq!(records.len(), 1, "the delivery must be recorded"); let tx = borsh::from_slice::(&records[0].transaction).unwrap(); let LeeTransaction::Public(public_tx) = tx else { @@ -943,15 +1076,14 @@ mod tests { #[tokio::test] async fn a_delivery_that_cannot_be_recorded_holds_the_floor() { - let (_dir, dbio) = store(); - break_the_dispatch_store(&dbio); + let storage_ref = store_refusing_deliveries(); let mut cursor = None; let mut tip = None; let outcome = consume_peer_stream( stream::iter(vec![peer_block_msg(1, 0)]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -962,7 +1094,9 @@ mod tests { // rather than retried. assert_eq!(outcome, PassOutcome::Undelivered(Slot::from(0))); assert_eq!( - get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(), + get_cross_zone_peer_floor(&storage_ref, PEER_ZONE) + .await + .unwrap(), None, "the slot must stay re-readable" ); @@ -970,31 +1104,46 @@ mod tests { // crash in between makes the re-read see the block as already delivered // from, and its messages are never looked at again. assert_eq!(tip, None); - assert_eq!(dbio.get_cross_zone_peer_tip(PEER_ZONE).unwrap(), None); + assert_eq!( + storage_ref + .ask(GetCrossZonePeerTip { + peer_zone: PEER_ZONE + }) + .await + .unwrap(), + None + ); } #[tokio::test] async fn watcher_resumes_from_the_persisted_cursor_without_rereading() { - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; consume_peer_stream( stream::iter(vec![peer_block_msg(1, 0), peer_block_msg(2, 1)]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) .await; - assert_eq!(recorded_keys(&dbio).len(), 2); + assert_eq!(recorded_keys(&storage_ref).await.len(), 2); // Restart: a fresh watcher seeds both its cursor and its chain tip from // the store. One that had to rebuild the tip in memory would accept // whatever block arrived first. - let resumed = get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(); + let resumed = get_cross_zone_peer_floor(&storage_ref, PEER_ZONE) + .await + .unwrap(); assert_eq!(resumed, Some(Slot::from(1))); - let mut resumed_tip = dbio.get_cross_zone_peer_tip(PEER_ZONE).unwrap(); + let mut resumed_tip = storage_ref + .ask(GetCrossZonePeerTip { + peer_zone: PEER_ZONE, + }) + .await + .unwrap(); assert_eq!( resumed_tip, Some(tip_at(2)), @@ -1006,7 +1155,7 @@ mod tests { consume_peer_stream( stream::iter(vec![peer_block_msg(3, 2)]), &peer_context(), - &dbio, + &storage_ref, &mut resumed_cursor, &mut resumed_tip, ) @@ -1019,19 +1168,21 @@ mod tests { ]; expected.sort_unstable(); assert_eq!( - recorded_keys(&dbio), + recorded_keys(&storage_ref).await, expected, "only the unread block is recorded on the second pass" ); assert_eq!( - get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(), + get_cross_zone_peer_floor(&storage_ref, PEER_ZONE) + .await + .unwrap(), Some(Slot::from(2)) ); } #[tokio::test] async fn watcher_does_not_persist_past_an_undecodable_block() { - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; @@ -1042,7 +1193,7 @@ mod tests { peer_block_msg(3, 2), ]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -1052,11 +1203,13 @@ mod tests { // would drop its messages permanently rather than until the next restart. assert_eq!(outcome, PassOutcome::Undecodable(Slot::from(1))); assert_eq!( - get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(), + get_cross_zone_peer_floor(&storage_ref, PEER_ZONE) + .await + .unwrap(), Some(Slot::from(0)) ); assert_eq!( - recorded_keys(&dbio), + recorded_keys(&storage_ref).await, vec![message_key(&PEER_ZONE, 1, 0)], "the block after the failure is unread" ); @@ -1067,14 +1220,14 @@ mod tests { // One slot can carry several messages. Persisting after each message // would store a cursor the retry resumes past, so the message that // failed is never re-read and its delivery is lost for good. - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; let outcome = consume_peer_stream( stream::iter(vec![peer_block_msg(1, 4), undecodable_msg(4)]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -1082,8 +1235,16 @@ mod tests { assert_eq!(outcome, PassOutcome::Undecodable(Slot::from(4))); assert_eq!(cursor, None, "slot 4 is re-read whole on the next pass"); - assert_eq!(get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(), None); - assert_eq!(recorded_keys(&dbio), vec![message_key(&PEER_ZONE, 1, 0)]); + assert_eq!( + get_cross_zone_peer_floor(&storage_ref, PEER_ZONE) + .await + .unwrap(), + None + ); + assert_eq!( + recorded_keys(&storage_ref).await, + vec![message_key(&PEER_ZONE, 1, 0)] + ); } #[tokio::test] @@ -1091,7 +1252,7 @@ mod tests { // This used to be read past after twenty attempts, which advanced the // floor over the hole and lost those messages rather than delaying // them: nothing after a hole can link. Stopping keeps the slot readable. - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; @@ -1103,7 +1264,7 @@ mod tests { peer_block_msg(3, 2), ]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -1112,12 +1273,14 @@ mod tests { } assert_eq!( - recorded_keys(&dbio), + recorded_keys(&storage_ref).await, vec![message_key(&PEER_ZONE, 1, 0)], "no pass reads past the slot it cannot decode" ); assert_eq!( - get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(), + get_cross_zone_peer_floor(&storage_ref, PEER_ZONE) + .await + .unwrap(), Some(Slot::from(0)), "the floor stays below it, so a fixed decoder recovers the messages" ); @@ -1134,7 +1297,7 @@ mod tests { // // The honest blocks behind it still deliver: stopping here would cost // the peer one inscription to end its own deliveries for good. - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; let pre_burn = produce_dummy_block(5, None, vec![emission()]); @@ -1147,7 +1310,7 @@ mod tests { peer_block_msg(3, 3), ]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -1161,7 +1324,7 @@ mod tests { ]; expected.sort_unstable(); assert_eq!( - recorded_keys(&dbio), + recorded_keys(&storage_ref).await, expected, "the key the peer aimed to burn is never recorded, and nothing else is held up" ); @@ -1172,7 +1335,7 @@ mod tests { async fn a_second_block_at_a_delivered_id_is_not_delivered_from() { // Both claim id 2, so on chain both deliveries key on (PEER_ZONE, 2, 0) // and the second is a replay the inbox no-ops. - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; let equivocation = produce_dummy_block(2, Some(HashType([9; 32])), vec![emission()]); @@ -1184,7 +1347,7 @@ mod tests { block_msg(&equivocation, 2), ]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -1198,7 +1361,7 @@ mod tests { let mut expected = vec![message_key(&PEER_ZONE, 1, 0), message_key(&PEER_ZONE, 2, 0)]; expected.sort_unstable(); assert_eq!( - recorded_keys(&dbio), + recorded_keys(&storage_ref).await, expected, "one delivery per id, whatever the peer publishes under it" ); @@ -1209,7 +1372,7 @@ mod tests { async fn a_block_that_does_not_link_to_the_tip_is_not_delivered_from() { // Not on the chain we verified, so nothing is delivered from it, and // the honest block at that id still is when it lands. - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; let forked = produce_dummy_block(2, Some(HashType([9; 32])), vec![emission()]); @@ -1221,7 +1384,7 @@ mod tests { peer_block_msg(2, 2), ]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -1231,7 +1394,7 @@ mod tests { let mut expected = vec![message_key(&PEER_ZONE, 1, 0), message_key(&PEER_ZONE, 2, 0)]; expected.sort_unstable(); assert_eq!( - recorded_keys(&dbio), + recorded_keys(&storage_ref).await, expected, "the fork is passed over and the peer's own chain continues" ); @@ -1243,14 +1406,14 @@ mod tests { // A fresh watcher handed a mid-chain block has nothing to link it // against. Adopting it would let the peer choose where the chain starts // and burn every key below it with one block. - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; let outcome = consume_peer_stream( stream::iter(vec![peer_block_msg(2, 0)]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -1261,7 +1424,7 @@ mod tests { PassOutcome::Stranded, "a pass that placed nothing while passing blocks over is how a peer goes quiet" ); - assert!(recorded_keys(&dbio).is_empty()); + assert!(recorded_keys(&storage_ref).await.is_empty()); assert_eq!(tip, None); } @@ -1269,7 +1432,7 @@ mod tests { async fn a_tampered_header_hash_is_not_delivered_from() { // As correctly signed as any other block, since the signature does not // cover `header.hash`. Block 2 arriving behind it still delivers. - let (_dir, dbio) = store(); + let (_dir, storage_ref) = store().await; let mut cursor = None; let mut tip = None; let mut tampered = chain_block(2); @@ -1282,7 +1445,7 @@ mod tests { peer_block_msg(2, 2), ]), &peer_context(), - &dbio, + &storage_ref, &mut cursor, &mut tip, ) @@ -1291,7 +1454,7 @@ mod tests { assert_eq!(outcome, PassOutcome::Drained); let mut expected = vec![message_key(&PEER_ZONE, 1, 0), message_key(&PEER_ZONE, 2, 0)]; expected.sort_unstable(); - assert_eq!(recorded_keys(&dbio), expected); + assert_eq!(recorded_keys(&storage_ref).await, expected); assert_eq!(tip, Some(tip_at(2))); } @@ -1300,18 +1463,31 @@ mod tests { // The tip is written per block and the floor per slot, so a crash // partway through the rebuild would otherwise leave a floor far above a // tip of 1, and nothing read after that restart could link. - let (_dir, dbio) = store(); - set_cross_zone_peer_floor(&dbio, PEER_ZONE, Slot::from(5000)).unwrap(); + let (_dir, storage_ref) = store().await; + set_cross_zone_peer_floor(&storage_ref, PEER_ZONE, Slot::from(5000)) + .await + .unwrap(); - let floor = get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(); - let tip = dbio.get_cross_zone_peer_tip(PEER_ZONE).unwrap(); + let floor = get_cross_zone_peer_floor(&storage_ref, PEER_ZONE) + .await + .unwrap(); + let tip = storage_ref + .ask(GetCrossZonePeerTip { + peer_zone: PEER_ZONE, + }) + .await + .unwrap(); let resume = resume_from(tip, floor); assert_eq!(resume.cursor, None, "the rebuild reads from genesis"); assert!(resume.clear_floor); - clear_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(); + clear_cross_zone_peer_floor(&storage_ref, PEER_ZONE) + .await + .unwrap(); assert_eq!( - get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(), + get_cross_zone_peer_floor(&storage_ref, PEER_ZONE) + .await + .unwrap(), None, "and a crash mid-rebuild resumes from genesis too, not from slot 5000" ); diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 21a5a4ad0..49e619973 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -1,7 +1,7 @@ use std::{ collections::VecDeque, path::Path, - sync::{Arc, Mutex}, + sync::Arc, time::{Duration, Instant}, }; @@ -18,7 +18,7 @@ use common::{ use config::{GenesisAction, SequencerConfig}; use cross_zone_inbox_core::CrossZoneMessage; use futures::StreamExt as _; -use itertools::Itertools as _; +use kameo::actor::ActorRef; use lee::{AccountId, PublicTransaction, public_transaction::Message}; use lee_core::GENESIS_BLOCK_ID; use log::{debug, error, info, warn}; @@ -32,17 +32,17 @@ use mempool::{MemPool, MemPoolHandle}; #[cfg(feature = "mock")] pub use mock::SequencerCoreWithMockClients; use num_bigint::BigUint; -pub use storage::error::DbError; -// Re-exported because `cross_zone_dead_letters` returns it and the service -// crate does not depend on `storage`, so it could not otherwise name the type. -pub use storage::sequencer::sequencer_cells::DeadLetterDispatchRecord; -use storage::sequencer::{ - DispatchFailure, RocksDBIO, StoreUpdate, - sequencer_cells::{ - DispatchOrigin, PendingCrossZoneDispatchRecord, PendingDepositEventRecord, - WithdrawalReconciliationKey, ZoneAnchorRecord, +use sequencer_storage_actor::{ + StorageActor, StorageActorTrait, + protocol::{ + ApplyStoreUpdate, DeadLetterDispatchRecord, DispatchFailure, DispatchOrigin, + DropSettledCrossZoneDispatches, GetBlock, GetDeadLetterDispatches, GetFirstBlockId, + GetLatestBlockMeta, GetPendingCrossZoneDispatches, PendingCrossZoneDispatchRecord, + PendingDepositEventRecord, RecordNewBlock, SetZoneAnchor, WithdrawalReconciliationKey, + ZoneAnchorRecord, }, }; +use tokio::sync::Mutex; use tokio_retry::{Retry, strategy::FixedInterval}; use crate::{ @@ -123,11 +123,14 @@ struct DepositMetadata { recipient_id: lee::AccountId, } -pub struct SequencerCore { +pub struct SequencerCore< + BP: BlockPublisherTrait = ZoneSdkPublisher, + S: StorageActorTrait = StorageActor, +> { /// Two-tier chain state: production builds on its head; the publisher's /// `on_follow` sink feeds adopted/orphaned/finalized peer blocks into it. chain: Arc>, - store: SequencerStore, + store: SequencerStore, mempool: MemPool<(TransactionOrigin, LeeTransaction)>, sequencer_config: SequencerConfig, block_publisher: BP, @@ -139,86 +142,24 @@ pub struct SequencerCore { last_committee_submission_slot: Option, } -impl SequencerCore { +impl SequencerCore { const CHANNEL_PROBE_RETRIES: usize = 29; const CHANNEL_PROBE_RETRY_DELAY: Duration = Duration::from_secs(2); /// Channel slots between committee-config submissions; a margin over /// observed Bedrock confirmation lag. const COMMITTEE_SUBMISSION_COOLDOWN: SlotCount = 10; - /// Starts the sequencer using the provided configuration. - /// If an existing database is found, the sequencer state is loaded from it and - /// assumed to represent the correct latest state consistent with Bedrock-finalized data. - /// If no database is found, the sequencer performs a fresh start from genesis, - /// initializing its state with the accounts defined in the configuration file. - fn open_or_create_store( - config: &SequencerConfig, - bootstrap_sequencer_key: Option, - ) -> (SequencerStore, lee::V03State) { - let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); - let db_path = config.db_path(); - - if db_path.exists() { - let store = SequencerStore::open_db(&db_path, signing_key).unwrap_or_else(|err| { - panic!( - "Failed to open database at {} with error: {err}", - db_path.display() - ) - }); - let state = store - .get_lee_state() - .expect("Failed to read state from store"); - (store, state) - } else { - let legacy = config.home.join("rocksdb"); - if legacy.exists() { - warn!( - "Ignoring pre-channel-suffix database at {}; rename it to {} to resume it", - legacy.display(), - db_path.display() - ); - } - warn!( - "Database not found at {}, starting from genesis", - db_path.display() - ); - - let (genesis_state, genesis_txs) = build_genesis_state(config, bootstrap_sequencer_key); - - let hashable_data = HashableBlockData { - block_id: GENESIS_BLOCK_ID, - transactions: genesis_txs, - prev_block_hash: HashType([0; 32]), - timestamp: 0, - }; - let genesis_block = hashable_data.into_pending_block(&signing_key); - - let store = SequencerStore::create_db_with_genesis( - &db_path, - &genesis_block, - &genesis_state, - signing_key, - ) - .expect("Failed to create database with genesis block"); - - // Incrementing count for genesis. - sequencer_core_metrics::increment_blocks_produced_total(); - - (store, genesis_state) - } - } - /// Rebuilds the two-tier [`ChainState`]: the final tier from the persisted /// final snapshot (pre-genesis state when absent), the head tier by replaying /// every stored block above it, so a post-restart orphan can still revert. - fn restore_chain_state( + async fn restore_chain_state( config: &SequencerConfig, - store: &SequencerStore, + store: &SequencerStore, stored_head_state: &lee::V03State, ) -> ChainState { let final_snapshot = store - .dbio() .get_final_snapshot() + .await .expect("Failed to read final snapshot from store"); let (final_state, final_tip) = match final_snapshot { Some((state, meta)) => (state, Some(Tip::from(meta))), @@ -229,9 +170,11 @@ impl SequencerCore { let mut head_blocks = store .get_all_blocks() - .filter_ok(|block| block.header.block_id > boundary) - .collect::, _>>() - .expect("Failed to read blocks from store while restoring chain state"); + .await + .expect("Failed to read blocks from store while restoring chain state") + .into_iter() + .filter(|block| block.header.block_id > boundary) + .collect::>(); head_blocks.sort_unstable_by_key(|block| block.header.block_id); let mut chain = ChainState::from_final(final_state, final_tip); @@ -252,8 +195,43 @@ impl SequencerCore { chain } + /// Seeds the storage actor's database with this zone's genesis when it + /// holds no chain yet. + async fn seed_genesis_if_absent( + storage_ref: &ActorRef, + signing_key: &lee::PrivateKey, + bootstrap_sequencer_key: Option, + config: &SequencerConfig, + ) { + let first_block_id = storage_ref + .ask(GetFirstBlockId) + .await + .expect("Failed to read the first block id"); + if first_block_id.is_some() { + return; + } + + let (block, state) = genesis_block_and_state(signing_key, bootstrap_sequencer_key, config); + storage_ref + .ask(RecordNewBlock { + block, + withdrawals: Vec::new(), + state: Arc::new(state), + checkpoint_bytes: None, + }) + .await + .expect("Failed to seed the database with the genesis block"); + + sequencer_core_metrics::increment_blocks_produced_total(); + } + + #[expect( + clippy::cognitive_complexity, + reason = "Slop has won the battle, but our war is not over" + )] pub async fn start_from_config( config: SequencerConfig, + storage_ref: ActorRef, ) -> (Self, MemPoolHandle<(TransactionOrigin, LeeTransaction)>) { sequencer_core_metrics::init(); @@ -287,8 +265,18 @@ impl SequencerCore { info!("Channel does not exist yet; starting it as channel creator"); } let bootstrap_sequencer_key = (!channel_already_exists).then_some(own_sequencer_key); + let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); + Self::seed_genesis_if_absent(&storage_ref, &signing_key, bootstrap_sequencer_key, &config) + .await; - let (store, state) = Self::open_or_create_store(&config, bootstrap_sequencer_key); + let store = SequencerStore::new(storage_ref, signing_key) + .await + .expect("Failed to open sequencer store"); + let state = store + .get_lee_state() + .await + .expect("Failed to read state from store") + .expect("Store holds a chain but no state"); assert!( committee_discovery::config_is_readable(&state), @@ -296,12 +284,13 @@ impl SequencerCore { one this sequencer can operate on" ); - let chain = Arc::new(Mutex::new(Self::restore_chain_state( - &config, &store, &state, - ))); + let chain = Arc::new(Mutex::new( + Self::restore_chain_state(&config, &store, &state).await, + )); let initial_checkpoint = store .get_zone_checkpoint() + .await .expect("Failed to load zone-sdk checkpoint"); let is_fresh_start = initial_checkpoint.is_none(); @@ -313,7 +302,11 @@ impl SequencerCore { bedrock_signing_key, config.retry_pending_blocks_timeout, initial_checkpoint, - Self::on_follow(store.dbio(), Arc::clone(&chain), mempool_handle.clone()), + Self::on_follow( + store.storage_ref().clone(), + Arc::clone(&chain), + mempool_handle.clone(), + ), ) .await .expect("Failed to initialize Block Publisher"); @@ -328,7 +321,7 @@ impl SequencerCore { &config.bedrock_config, cross_zone, config.block_create_timeout, - &store.dbio(), + store.storage_ref(), ) }); // Before producing, verify our local state still belongs to the chain @@ -347,10 +340,12 @@ impl SequencerCore { // store written before the mark existed starts in. if let Some(tip) = store .latest_block_meta() + .await .expect("Failed to read latest block meta") { store .raise_published_high_water(tip.id) + .await .expect("Failed to seed published high water mark"); } @@ -361,9 +356,11 @@ impl SequencerCore { if is_fresh_start && channel_absent { let mut pending_blocks = store .get_all_blocks() - .filter_ok(|block| matches!(block.bedrock_status, BedrockStatus::Pending)) - .collect::, _>>() - .expect("Failed to read blocks from store while republishing on fresh start"); + .await + .expect("Failed to read blocks from store while republishing on fresh start") + .into_iter() + .filter(|block| matches!(block.bedrock_status, BedrockStatus::Pending)) + .collect::>(); pending_blocks.sort_unstable_by_key(|block| block.header.block_id); assert!( @@ -397,6 +394,7 @@ impl SequencerCore { last_checkpoint = Some(outcome.checkpoint); store .raise_published_high_water(block.header.block_id) + .await .expect("Failed to persist published high water mark"); } @@ -406,6 +404,7 @@ impl SequencerCore { if let Some(checkpoint) = last_checkpoint { store .set_zone_checkpoint(&checkpoint) + .await .expect("Failed to persist checkpoint after republishing on fresh start"); } } @@ -420,8 +419,8 @@ impl SequencerCore { last_committee_submission_slot: None, }; - sequencer_core_metrics::record_chain_height(sequencer_core.chain_height()); - record_dead_letter_gauge(&sequencer_core.store.dbio()); + sequencer_core_metrics::record_chain_height(sequencer_core.chain_height().await); + record_dead_letter_gauge(sequencer_core.store.storage_ref()).await; (sequencer_core, mempool_handle) } @@ -436,12 +435,13 @@ impl SequencerCore { /// this sequencer is the one that must bootstrap-publish its own blocks. async fn verify_and_reconstruct( publisher: &BP, - store: &SequencerStore, + store: &SequencerStore, chain: &Mutex, is_fresh_start: bool, ) -> Result { let anchor_record = store .get_zone_anchor() + .await .context("Failed to read zone anchor")?; let after_slot = anchor_record @@ -468,6 +468,7 @@ impl SequencerCore { // channel activity in a prior run. let local_tip = store .latest_block_meta() + .await .context("Failed to read latest block meta")? .map(|meta| meta.id); let had_checkpoint_before_start = !is_fresh_start; @@ -535,8 +536,8 @@ impl SequencerCore { // Locked per message (not across the stream `await`): concurrent // follow events interleave safely โ€” both paths apply idempotently // and persist under this same lock. - let mut chain = chain.lock().expect("chain state mutex poisoned"); - Self::apply_reconstructed_block(store, &mut chain, &block, slot)?; + let mut chain = chain.lock().await; + Self::apply_reconstructed_block(store.storage_ref(), &mut chain, &block, slot).await?; } // The channel exists once it has a tip; only when it has none is this @@ -551,14 +552,15 @@ impl SequencerCore { /// blocks we already hold, ignored when it conflicts at a height the final /// tier already settled, a validated continuation otherwise. Advances the /// persisted anchor to the block's slot. - fn apply_reconstructed_block( - store: &SequencerStore, + async fn apply_reconstructed_block( + storage_ref: &ActorRef, chain: &mut ChainState, block: &Block, slot: Slot, ) -> Result<()> { - let tip = store - .latest_block_meta() + let tip = storage_ref + .ask(GetLatestBlockMeta) + .await .context("Failed to read latest block meta")?; let block_id = block.header.block_id; let block_hash = block.header.hash; @@ -576,14 +578,16 @@ impl SequencerCore { // downstream would ever remove them. if let Some(tip) = &tip && block_id <= tip.id - && let Some(stored) = store - .get_block_at_id(block_id) + && let Some(stored) = storage_ref + .ask(GetBlock { block_id }) + .await .context("Failed to read stored block")? && stored.header.hash == block_hash { - settle_reconstructed_deliveries(store, &stored); - store - .set_zone_anchor(&record) + settle_reconstructed_deliveries(storage_ref, &stored).await; + storage_ref + .ask(SetZoneAnchor { anchor: record }) + .await .context("Failed to persist zone anchor")?; return Ok(()); } @@ -628,24 +632,29 @@ impl SequencerCore { .collect(); // The same for the deliveries it carries: the inbox has seen them, so // the drain would skip them anyway, and the records are owed nothing. - let finalized_dispatch_keys = settled_dispatch_keys(&store.dbio(), block); + let finalized_dispatch_keys = settled_dispatch_keys(storage_ref, block).await; // The tip meta stays pinned to the head tip even when the reconstructed // block lands below it, and the anchor only advances if the block // itself landed. let head_tip = chain.head_tip().map(|head| BlockMeta::from(&head)); let final_meta = chain.final_tip().map(|meta| BlockMeta::from(&meta)); - store - .dbio() - .store_update(&StoreUpdate { - blocks: &[(block, true)], - head_tip: head_tip.as_ref(), - final_snapshot: final_meta.as_ref().map(|meta| (chain.final_state(), meta)), - remove_deposit_records: &finalized_deposit_ids, - remove_dispatch_records: &finalized_dispatch_keys, - zone_anchor: Some(&record), - ..StoreUpdate::new(chain.head_state()) + storage_ref + .ask(ApplyStoreUpdate { + blocks: vec![(block.clone(), true)], + head_tip, + head_state: chain.share_head_state(), + final_snapshot: final_meta.map(|meta| (chain.share_final_state(), meta)), + remove_deposit_records: finalized_deposit_ids, + remove_dispatch_records: finalized_dispatch_keys, + zone_anchor: Some(record), + checkpoint: None, + finalized_up_to: None, + new_deposit_events: Vec::new(), + consumed_withdrawals: Vec::new(), + new_withdraw_intents: Vec::new(), }) + .await .context("Failed to persist reconstructed block")?; Ok(()) @@ -653,12 +662,17 @@ impl SequencerCore { /// Publisher sink adapter over [`apply_follow_update`]. fn on_follow( - dbio: Arc, + storage_ref: ActorRef, chain: Arc>, mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>, ) -> block_publisher::OnFollowSink { Box::new(move |update: block_publisher::FollowUpdate| { - apply_follow_update(&dbio, &chain, &mempool_handle, update); + let storage_ref = storage_ref.clone(); + let chain = Arc::clone(&chain); + let mempool_handle = mempool_handle.clone(); + Box::pin(async move { + apply_follow_update(&storage_ref, &chain, &mempool_handle, update).await; + }) }) } @@ -674,6 +688,7 @@ impl SequencerCore { committee_update, } = self .build_block_from_mempool(live_accredited_keys.as_deref()) + .await .context("Failed to build block from mempool transactions")?; let block_publisher::PublishOutcome { @@ -690,6 +705,7 @@ impl SequencerCore { // block below, so this height must never be published again. self.store .raise_published_high_water(block.header.block_id) + .await .context("Failed to persist published high water mark")?; // Independent Mantle tx, not bundled with the block above โ€” join/exit @@ -701,14 +717,11 @@ impl SequencerCore { .map(withdrawal_reconciliation_key) .collect(); - self.record_produced_block( - this_msg, - &block, - &withdrawal_reconciliation_keys, - &checkpoint, - )?; + let block_id = block.header.block_id; + self.record_produced_block(this_msg, block, withdrawal_reconciliation_keys, &checkpoint) + .await?; - Ok(block.header.block_id) + Ok(block_id) } /// Live committee snapshot for gating `FinalizeUnstake` inclusion and @@ -792,29 +805,30 @@ impl SequencerCore { /// `Parked` when the head reorged to a different parent), the canonical /// block is persisted by the follow path instead, and our invalidated /// inscription comes back via `orphaned`. - fn record_produced_block( + async fn record_produced_block( &mut self, this_msg: MsgId, - block: &Block, - withdrawal_reconciliation_keys: &[WithdrawalReconciliationKey], + block: Block, + withdrawal_reconciliation_keys: Vec, checkpoint: &block_publisher::SequencerCheckpoint, ) -> Result<()> { let checkpoint_bytes = block_store::checkpoint_bytes(checkpoint)?; - let mut chain = self.chain.lock().expect("chain state mutex poisoned"); - match chain.apply_produced(this_msg, block) { + let mut chain = self.chain.lock().await; + match chain.apply_produced(this_msg, &block) { AcceptOutcome::Applied => { - // Persisted under the lock so disk writes land in apply order - // with the follow path. - self.store.update( - block, - withdrawal_reconciliation_keys, - chain.head_state(), - Some(&checkpoint_bytes), - )?; + let block_id = block.header.block_id; + self.store + .record_new_block( + block, + withdrawal_reconciliation_keys, + chain.share_head_state(), + Some(checkpoint_bytes), + ) + .await?; sequencer_core_metrics::increment_blocks_produced_total(); - sequencer_core_metrics::record_chain_height(block.header.block_id); + sequencer_core_metrics::record_chain_height(block_id); } // Neither branch persists anything, checkpoint included: the // inscription it holds as pending belongs to a block that is not @@ -910,7 +924,11 @@ impl SequencerCore { true } - fn build_block_from_mempool( + #[expect( + clippy::cognitive_complexity, + reason = "Slop has won the battle, but our war is not over" + )] + async fn build_block_from_mempool( &mut self, live_accredited_keys: Option<&[sequencer_stake_core::SequencerKey]>, ) -> Result { @@ -923,8 +941,8 @@ impl SequencerCore { let mut settled = Vec::new(); let recorded_dispatches: Vec<_> = self .store - .dbio() - .get_pending_cross_zone_dispatches() + .pending_cross_zone_dispatches() + .await .context("Failed to load pending cross-zone dispatches")? .into_iter() .filter_map( @@ -958,7 +976,7 @@ impl SequencerCore { pending_dispatches, finalize_unstake_txs, ) = { - let chain = self.chain.lock().expect("chain state mutex poisoned"); + let chain = self.chain.lock().await; let tip = chain.head_tip(); let height = tip.as_ref().map_or(GENESIS_BLOCK_ID, |head| { head.block_id @@ -998,8 +1016,8 @@ impl SequencerCore { if !settled.is_empty() { if let Err(err) = self .store - .dbio() - .drop_settled_cross_zone_dispatches(&settled) + .drop_settled_cross_zone_dispatches(settled.clone()) + .await { // Only bookkeeping: the deliveries themselves are irreversible, // and the next turn tries again. @@ -1010,7 +1028,7 @@ impl SequencerCore { } // A settled delivery may be one this node had given up on, which // takes its dead letter with it. - record_dead_letter_gauge(&self.store.dbio()); + record_dead_letter_gauge(self.store.storage_ref()).await; } let mut valid_transactions = Vec::new(); @@ -1029,6 +1047,7 @@ impl SequencerCore { let pending_deposits: VecDeque = self .store .get_pending_deposit_events() + .await .context("Failed to load pending deposit events")? .into_iter() .filter(|record| !deposit_already_minted(&working_state, record.deposit_op_id)) @@ -1109,7 +1128,7 @@ impl SequencerCore { "Sequencer-drained transaction {tx_hash} cannot fit in any block under the \ {max_block_size} byte limit; giving up on it rather than stalling production", ); - self.count_dispatch_failure(&tx); + self.count_dispatch_failure(&tx).await; continue; } @@ -1164,7 +1183,7 @@ impl SequencerCore { // A failed transaction is simply left out of the block, except a // dispatch: that one is re-fed from the store every turn, so one // that can never execute would fail on every block for ever. - self.count_dispatch_failure(&tx); + self.count_dispatch_failure(&tx).await; } if valid_transactions.len() >= self.sequencer_config.max_num_tx_in_block { @@ -1209,23 +1228,18 @@ impl SequencerCore { /// Reads the current head state under the lock without cloning it, so callers /// reuse `V03State`'s own API (accounts, nonces, proofs) with no whole-state copy. - pub fn with_state(&self, f: impl FnOnce(&lee::V03State) -> R) -> R { - f(self - .chain - .lock() - .expect("chain state mutex poisoned") - .head_state()) + pub async fn with_state(&self, f: impl FnOnce(&lee::V03State) -> R) -> R { + f(self.chain.lock().await.head_state()) } - pub const fn block_store(&self) -> &SequencerStore { + pub const fn block_store(&self) -> &SequencerStore { &self.store } - #[must_use] - pub fn chain_height(&self) -> u64 { + pub async fn chain_height(&self) -> u64 { self.chain .lock() - .expect("chain state mutex poisoned") + .await .head_tip() .map_or(0, |tip| tip.block_id) } @@ -1240,20 +1254,20 @@ impl SequencerCore { /// [`StoreUpdate::finalized_up_to`]. Kept on the type for tests. // TODO: Delete blocks instead of marking them as finalized. Current // approach is used because we still have `GetBlockDataRequest`. - pub fn clean_finalized_blocks_from_db(&self, last_finalized_block_id: u64) -> Result<()> { + pub async fn clean_finalized_blocks_from_db(&self, last_finalized_block_id: u64) -> Result<()> { log::info!("Clearing pending blocks up to id: {last_finalized_block_id}"); self.store - .dbio() - .clean_pending_blocks_up_to(last_finalized_block_id)?; + .clean_pending_blocks_up_to(last_finalized_block_id) + .await?; Ok(()) } /// Returns the list of stored pending blocks. - pub fn get_pending_blocks(&self) -> Result> { + pub async fn get_pending_blocks(&self) -> Result> { Ok(self .store .get_all_blocks() - .collect::>>()? + .await? .into_iter() .filter(|block| matches!(block.bedrock_status, BedrockStatus::Pending)) .collect()) @@ -1300,7 +1314,7 @@ impl SequencerCore { /// can equally be a property of the moment, so give up only after several. /// Giving up moves the record to the dead letter: a peer cannot grow the /// pending list with deliveries that never execute, and it stays findable. - fn count_dispatch_failure(&self, tx: &LeeTransaction) { + async fn count_dispatch_failure(&self, tx: &LeeTransaction) { let Some(message) = extract_cross_zone_dispatch(tx) else { return; }; @@ -1316,12 +1330,12 @@ impl SequencerCore { }; match self .store - .dbio() .record_dispatch_failure(key, RETIRE_DISPATCH_AFTER_FAILURES, origin) + .await { Ok(DispatchFailure::Retired(record)) => { sequencer_core_metrics::increment_cross_zone_dispatches_retired_total(); - record_dead_letter_gauge(&self.store.dbio()); + record_dead_letter_gauge(self.store.storage_ref()).await; error!( "Giving up on cross-zone delivery {} from peer zone {} block {} transaction {} ({} bytes) after {} failed attempts. This node will not retry it; unless another sequencer carries it, the message is not delivered. Kept in the dead letter.", hex::encode(key), @@ -1354,10 +1368,9 @@ impl SequencerCore { /// Retained is read first so the pair can only skew towards a total that /// leads its list, an ordinary evicted or settled state. The other order /// would report entries against a total of zero. - pub fn cross_zone_dead_letters(&self) -> Result<(u64, Vec), DbError> { - let dbio = self.store.dbio(); - let retained = dbio.get_dead_letter_cross_zone_dispatches()?; - let total = dbio.get_dead_letter_cross_zone_dispatch_count()?; + pub async fn cross_zone_dead_letters(&self) -> Result<(u64, Vec)> { + let retained = self.store.dead_letter_dispatches().await?; + let total = self.store.dead_letter_dispatch_count().await?; Ok((total, retained)) } @@ -1382,11 +1395,10 @@ impl SequencerCore { } /// The height the next produced block would claim. - #[must_use] - pub fn next_block_height(&self) -> u64 { + pub async fn next_block_height(&self) -> u64 { self.chain .lock() - .expect("chain state mutex poisoned") + .await .head_tip() .map_or(GENESIS_BLOCK_ID, |tip| { tip.block_id @@ -1406,10 +1418,9 @@ impl SequencerCore { /// /// This is not a stall โ€” the head recovers by itself once the inscriptions /// we are protecting finalize and the final tier rebases onto them. - #[must_use] - pub fn rewound_below_published(&self) -> Option { - let high_water = self.store.published_high_water().ok().flatten()?; - (self.next_block_height() <= high_water).then_some(high_water) + pub async fn rewound_below_published(&self) -> Option { + let high_water = self.store.published_high_water().await.ok().flatten()?; + (self.next_block_height().await <= high_water).then_some(high_water) } /// Shared handle to the two-tier follow state, for tests to drive the @@ -1464,8 +1475,8 @@ fn dispatch_already_delivered(state: &lee::V03State, message: &CrossZoneMessage) /// Read from the store because the list falls as well as rises (eviction, and /// reconciliation when a delivery settles elsewhere). Costs a read and a decode, /// so call it only where one of those can have happened. -fn record_dead_letter_gauge(dbio: &RocksDBIO) { - match dbio.get_dead_letter_cross_zone_dispatches() { +async fn record_dead_letter_gauge(storage_ref: &ActorRef) { + match storage_ref.ask(GetDeadLetterDispatches).await { Ok(records) => { sequencer_core_metrics::record_cross_zone_dead_letter_dispatches(records.len()); } @@ -1487,8 +1498,8 @@ fn record_dead_letter_gauge(dbio: &RocksDBIO) { /// relies on a valid successor or a restart. `ChainState` never emits /// `AcceptOutcome::RetryableFailure` yet; adding retry parity here is a /// follow-up. -fn apply_follow_update( - dbio: &RocksDBIO, +async fn apply_follow_update( + storage_ref: &ActorRef, chain: &Mutex, mempool_handle: &MemPoolHandle<(TransactionOrigin, LeeTransaction)>, update: block_publisher::FollowUpdate, @@ -1522,7 +1533,7 @@ fn apply_follow_update( // The lock is held across the persist below so disk writes land in apply // order โ€” the produce path persists under this same lock. let (resubmit_txs, outcome, head_height) = { - let mut chain = chain.lock().expect("chain state mutex poisoned"); + let mut chain = chain.lock().await; // An orphan report rewinds the head to the earliest orphaned block and // prunes the store above it, which is how a run of our own inscriptions @@ -1619,6 +1630,7 @@ fn apply_follow_update( BlockMeta::from(&tip) }); let head_tip = chain.head_tip().map(|tip| BlockMeta::from(&tip)); + let head_tip_id = head_tip.as_ref().map_or(0, |tip| tip.id); // Every block at or below the highest finalized one is irreversible, so // stored blocks there can be marked finalized. @@ -1638,36 +1650,42 @@ fn apply_follow_update( // The same for cross-zone deliveries, keyed by message key: a record // goes once its own delivery is irreversible, never because another // block finalized at its height. - let finalized_dispatch_keys: Vec<[u8; 32]> = irreversible - .iter() - .flat_map(|block| settled_dispatch_keys(dbio, block)) - .collect(); + let mut finalized_dispatch_keys: Vec<[u8; 32]> = Vec::new(); + for block in &irreversible { + finalized_dispatch_keys.extend(settled_dispatch_keys(storage_ref, block).await); + } // A persist failure is fatal: the in-memory chain has already advanced, // and continuing would leave a permanent gap in the store. The `panic!` // ends the drive task, whose cancellation halts the node. - let outcome = dbio - .store_update(&StoreUpdate { - checkpoint: Some(&checkpoint_bytes), - blocks: &to_persist, - head_tip: head_tip.as_ref(), - final_snapshot: final_meta.as_ref().map(|meta| (chain.final_state(), meta)), + let outcome = storage_ref + .ask(ApplyStoreUpdate { + checkpoint: Some(checkpoint_bytes), + blocks: to_persist + .into_iter() + .map(|(block, fin)| (block.clone(), fin)) + .collect(), + head_tip, + head_state: chain.share_head_state(), + final_snapshot: final_meta.map(|meta| (chain.share_final_state(), meta)), finalized_up_to: last_finalized, - new_deposit_events: &deposit_records, - remove_deposit_records: &finalized_deposit_ids, - remove_dispatch_records: &finalized_dispatch_keys, - consumed_withdrawals: &consumed_withdrawals, - ..StoreUpdate::new(chain.head_state()) + new_deposit_events: deposit_records, + remove_deposit_records: finalized_deposit_ids, + remove_dispatch_records: finalized_dispatch_keys, + consumed_withdrawals, + new_withdraw_intents: Vec::new(), + zone_anchor: None, }) + .await .unwrap_or_else(|err| panic!("Failed to persist follow update: {err:#}")); - (resubmit_txs, outcome, head_tip.map_or(0, |tip| tip.id)) + (resubmit_txs, outcome, head_tip_id) }; sequencer_core_metrics::record_chain_height(head_height); // The runtime reconcile path: finalizing another sequencer's block drops the // dead letter of a delivery this node gave up on. - record_dead_letter_gauge(dbio); + record_dead_letter_gauge(storage_ref).await; if outcome.accepted_deposits > 0 { log::info!( @@ -1700,6 +1718,24 @@ fn apply_follow_update( } } +/// The genesis block and state `config` describes. +fn genesis_block_and_state( + signing_key: &lee::PrivateKey, + bootstrap_sequencer_key: Option, + config: &SequencerConfig, +) -> (Block, lee::V03State) { + let (genesis_state, genesis_txs) = build_genesis_state(config, bootstrap_sequencer_key); + let genesis_block = HashableBlockData { + block_id: GENESIS_BLOCK_ID, + transactions: genesis_txs, + prev_block_hash: HashType([0; 32]), + timestamp: 0, + } + .into_pending_block(signing_key); + + (genesis_block, genesis_state) +} + /// The pre-genesis state: `testnet_initial_state` plus the bridge-lock holdings, /// the only accounts seeded outside any transaction. Everything else, including /// the bootstrap sequencer's own stake, is applied as a genesis transaction in @@ -2002,7 +2038,7 @@ fn build_supply_account_genesis_transaction( let message = Message::try_new( faucet_program_id, vec![system_accounts::faucet_account_id(), recipient_vault_id], - vec![], + Vec::new(), faucet_core::Instruction::GenesisTransferVault { vault_program_id, recipient_id: *account_id, @@ -2010,7 +2046,7 @@ fn build_supply_account_genesis_transaction( }, ) .expect("Failed to serialize genesis transfer instruction"); - let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(vec![]); + let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(Vec::new()); PublicTransaction::new(message, witness_set) } @@ -2022,11 +2058,11 @@ fn build_supply_bridge_account_genesis_transaction(balance: u128) -> PublicTrans let message = Message::try_new( faucet_program_id, vec![system_accounts::faucet_account_id(), bridge_account_id], - vec![], + Vec::new(), faucet_core::Instruction::GenesisTransferDirect { amount: balance }, ) .expect("Failed to serialize bridge genesis transfer instruction"); - let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(vec![]); + let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(Vec::new()); PublicTransaction::new(message, witness_set) } @@ -2060,7 +2096,7 @@ fn build_bridge_deposit_tx_from_event(event: &PendingDepositEventRecord) -> Resu recipient_vault_id, receipt_id, ], - vec![], + Vec::new(), bridge_core::Instruction::Deposit { l1_deposit_op_id: event.deposit_op_id.0, vault_program_id, @@ -2070,7 +2106,7 @@ fn build_bridge_deposit_tx_from_event(event: &PendingDepositEventRecord) -> Resu ) .context("Failed to build bridge deposit message")?; - let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(vec![]); + let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(Vec::new()); Ok(LeeTransaction::Public(PublicTransaction::new( message, witness_set, @@ -2217,8 +2253,14 @@ fn extract_cross_zone_dispatch_key(tx: &LeeTransaction) -> Option<[u8; 32]> { /// indexer, which re-derives every delivery and halts, but the local record is /// the last copy of what we believed and it is about to be dropped either way. /// Saying so in the log is what makes the halt diagnosable. -fn settled_dispatch_keys(dbio: &RocksDBIO, block: &Block) -> Vec<[u8; 32]> { - let recorded = dbio.get_pending_cross_zone_dispatches().unwrap_or_default(); +async fn settled_dispatch_keys( + storage_ref: &ActorRef, + block: &Block, +) -> Vec<[u8; 32]> { + let recorded = storage_ref + .ask(GetPendingCrossZoneDispatches) + .await + .unwrap_or_default(); let (keys, forged) = classify_settled_deliveries(&recorded, block); for key in forged { error!( @@ -2263,12 +2305,18 @@ fn classify_settled_deliveries( /// /// A persist failure is only logged: the deliveries are already irreversible, so /// the worst case is a record the next drain drops instead. -fn settle_reconstructed_deliveries(store: &SequencerStore, block: &Block) { - let keys = settled_dispatch_keys(&store.dbio(), block); +async fn settle_reconstructed_deliveries( + storage_ref: &ActorRef, + block: &Block, +) { + let keys = settled_dispatch_keys(storage_ref, block).await; if keys.is_empty() { return; } - if let Err(err) = store.dbio().drop_settled_cross_zone_dispatches(&keys) { + if let Err(err) = storage_ref + .ask(DropSettledCrossZoneDispatches { message_keys: keys }) + .await + { warn!("Failed to settle reconstructed delivery records: {err:#}"); } } diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 567aea519..5bd2135f5 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -1,6 +1,6 @@ #![expect(clippy::shadow_unrelated, reason = "We don't care about it in tests")] -use std::{pin::pin, time::Duration}; +use std::{pin::pin, sync::Arc, time::Duration}; use common::{ HashType, @@ -8,6 +8,7 @@ use common::{ test_utils::sequencer_sign_key_for_testing, transaction::{LeeTransaction, clock_invocation}, }; +use kameo::actor::Spawn as _; use lee::{ Account, AccountId, Data, PrivateKey, PublicKey, PublicTransaction, V03State, program::Program, }; @@ -24,8 +25,12 @@ use logos_blockchain_key_management_system_service::keys::{Ed25519Key, ZkPublicK use logos_blockchain_zone_sdk::{Slot, sequencer::DepositInfo}; use mempool::MemPoolHandle; use ping_core::{ReceiverInstruction, ping_record_pda, receiver_config_account_id}; -use storage::sequencer::sequencer_cells::{ - DispatchOrigin, PendingCrossZoneDispatchRecord, PendingDepositEventRecord, +use sequencer_storage_actor::{ + StorageActor, + protocol::{ + AddPendingCrossZoneDispatches, AddPendingDepositEvent, DispatchOrigin, + PendingCrossZoneDispatchRecord, PendingDepositEventRecord, RecordNewBlock, + }, }; use tempfile::tempdir; use testnet_initial_state::{initial_pub_accounts_private_keys, initial_public_user_accounts}; @@ -34,7 +39,6 @@ use crate::{ MAX_DISPATCHES_PER_BLOCK, RETIRE_DISPATCH_AFTER_FAILURES, TransactionOrigin, apply_follow_update, block_publisher::FollowUpdate, - block_store::SequencerStore, build_bridge_deposit_tx_from_event, build_finalize_unstake_tx, build_genesis_state, classify_settled_deliveries, config::{ @@ -79,6 +83,17 @@ fn test_sequencer_key(seed: u8) -> sequencer_stake_core::SequencerKey { .expect("a Bedrock public key is a valid Ed25519 public key") } +async fn start_sequencer( + config: SequencerConfig, +) -> ( + SequencerCoreWithMockClients, + MemPoolHandle<(TransactionOrigin, LeeTransaction)>, +) { + let storage = StorageActor::new(&config.db_path()).expect("Failed to open database"); + let storage_ref = StorageActor::spawn(storage); + SequencerCoreWithMockClients::start_from_config(config, storage_ref).await +} + /// A follow update carrying nothing, to fill in the fields a test does not /// exercise via `..empty_follow_update()`. fn empty_follow_update() -> FollowUpdate { @@ -182,8 +197,7 @@ async fn common_setup_with_config( SequencerCoreWithMockClients, MemPoolHandle<(TransactionOrigin, LeeTransaction)>, ) { - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, mempool_handle) = start_sequencer(config).await; let tx = common::test_utils::produce_dummy_empty_transaction(); mempool_handle @@ -308,30 +322,33 @@ fn dispatches_in(block: &Block) -> Vec<[u8; 32]> { } /// The pending dispatch records a sequencer still holds. -fn pending_dispatches( +async fn pending_dispatches( sequencer: &SequencerCoreWithMockClients, ) -> Vec { sequencer - .store - .dbio() - .get_pending_cross_zone_dispatches() + .block_store() + .pending_cross_zone_dispatches() + .await .expect("pending dispatches readable") } #[tokio::test] async fn start_from_config() { let config = setup_sequencer_config(); - let (sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let (sequencer, _mempool_handle) = start_sequencer(config.clone()).await; - assert_eq!(sequencer.chain_height(), 1); + assert_eq!(sequencer.chain_height().await, 1); assert_eq!(sequencer.sequencer_config.max_num_tx_in_block, 10); let acc1_account_id = initial_public_user_accounts()[0].account_id; let acc2_account_id = initial_public_user_accounts()[1].account_id; - let balance_acc_1 = sequencer.with_state(|s| s.get_account_by_id(acc1_account_id).balance); - let balance_acc_2 = sequencer.with_state(|s| s.get_account_by_id(acc2_account_id).balance); + let balance_acc_1 = sequencer + .with_state(|s| s.get_account_by_id(acc1_account_id).balance) + .await; + let balance_acc_2 = sequencer + .with_state(|s| s.get_account_by_id(acc2_account_id).balance) + .await; assert_eq!(10000, balance_acc_1); assert_eq!(20000, balance_acc_2); @@ -355,18 +372,23 @@ async fn start_from_config_opens_existing_db_if_it_exists() { }; let genesis_block = genesis_hashable_data.into_pending_block(&signing_key); - SequencerStore::create_db_with_genesis( - &config.db_path(), - &genesis_block, - &genesis_state, - signing_key, - ) - .unwrap(); + let storage = StorageActor::new(&config.db_path()).unwrap(); + let storage_ref = StorageActor::spawn(storage); + storage_ref + .ask(RecordNewBlock { + block: genesis_block, + withdrawals: vec![], + state: Arc::new(genesis_state), + checkpoint_bytes: None, + }) + .await + .unwrap(); + storage_ref.stop_gracefully().await.unwrap(); + storage_ref.wait_for_shutdown().await; - let (sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; - assert_eq!(sequencer.chain_height(), 1); - assert!(sequencer.store.latest_block_meta().is_ok()); + let (sequencer, _mempool_handle) = start_sequencer(config).await; + assert_eq!(sequencer.chain_height().await, 1); + assert!(sequencer.store.last_block_id().await.is_ok()); } #[should_panic(expected = "Failed to open database")] @@ -382,7 +404,7 @@ async fn start_from_config_panics_when_db_open_returns_non_not_found_error() { // Force RocksDB open to fail with an IO error by placing a file at DB path. std::fs::write(&db_path, b"not-a-directory").unwrap(); - let _ = SequencerCoreWithMockClients::start_from_config(config).await; + let _ = start_sequencer(config).await; } #[tokio::test] @@ -394,10 +416,11 @@ async fn unfulfilled_deposit_events_are_drained_from_the_store_on_production() { let expected_amount = 1_u64; let recipient_id = initial_public_user_accounts()[0].account_id; - { - let (_sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config.clone()).await; - } + let storage_weak = { + let (sequencer, _mempool_handle) = start_sequencer(config.clone()).await; + sequencer.block_store().storage_ref().downgrade() + }; + storage_weak.wait_for_shutdown_with_result(|_| ()).await; let pending_event = PendingDepositEventRecord { deposit_op_id: HashType(deposit_op_id), @@ -407,21 +430,22 @@ async fn unfulfilled_deposit_events_are_drained_from_the_store_on_production() { }; { - let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); - let store = SequencerStore::open_db(&config.db_path(), signing_key).unwrap(); - - let inserted = store - .dbio() - .add_pending_deposit_event(pending_event) + let storage_ref = StorageActor::spawn(StorageActor::new(&config.db_path()).unwrap()); + let inserted = storage_ref + .ask(AddPendingDepositEvent { + event: pending_event, + }) + .await .unwrap(); assert!(inserted); + storage_ref.stop_gracefully().await.unwrap(); + storage_ref.wait_for_shutdown().await; } // The mint never goes through the mempool: the record is the queue, and // production drains it. That is what makes a restart โ€” or a follow event // arriving while a full mempool would have dropped the push โ€” lossless. - let (mut sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, _mempool_handle) = start_sequencer(config).await; assert!( sequencer.mempool.pop().is_none(), "deposit mints are drained from the store, never queued in the mempool" @@ -430,7 +454,8 @@ async fn unfulfilled_deposit_events_are_drained_from_the_store_on_production() { let block_id = sequencer.run_production_turn().await.unwrap(); let block = sequencer .store - .get_block_at_id(block_id) + .block_at_id(block_id) + .await .unwrap() .expect("produced block is stored"); assert!( @@ -448,13 +473,16 @@ async fn unfulfilled_deposit_events_are_drained_from_the_store_on_production() { sequencer .store .get_pending_deposit_events() + .await .unwrap() .iter() .any(|event| event.deposit_op_id == HashType(deposit_op_id)), "the record remains until the deposit finalizes" ); assert!( - sequencer.with_state(|state| deposit_already_minted(state, HashType(deposit_op_id))), + sequencer + .with_state(|state| deposit_already_minted(state, HashType(deposit_op_id))) + .await, "the deposit's receipt PDA marks it minted in head state" ); } @@ -466,26 +494,29 @@ async fn a_drained_deposit_is_not_minted_twice_across_turns() { let deposit_op_id = [17_u8; 32]; let recipient_id = initial_public_user_accounts()[0].account_id; - let (mut sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, _mempool_handle) = start_sequencer(config).await; sequencer - .store - .dbio() - .add_pending_deposit_event(PendingDepositEventRecord { - deposit_op_id: HashType(deposit_op_id), - source_tx_hash: HashType([7_u8; 32]), - amount: 1, - metadata: borsh::to_vec(&DepositMetadataForEncoding { recipient_id }).unwrap(), + .block_store() + .storage_ref() + .ask(AddPendingDepositEvent { + event: PendingDepositEventRecord { + deposit_op_id: HashType(deposit_op_id), + source_tx_hash: HashType([7_u8; 32]), + amount: 1, + metadata: borsh::to_vec(&DepositMetadataForEncoding { recipient_id }).unwrap(), + }, }) + .await .unwrap(); let first = sequencer.run_production_turn().await.unwrap(); let second = sequencer.run_production_turn().await.unwrap(); - let minted_in = |block_id: u64| { + let minted_in = async |block_id: u64| { sequencer .store - .get_block_at_id(block_id) + .block_at_id(block_id) + .await .unwrap() .expect("produced block is stored") .body @@ -495,9 +526,9 @@ async fn a_drained_deposit_is_not_minted_twice_across_turns() { .count() }; - assert_eq!(minted_in(first), 1); + assert_eq!(minted_in(first).await, 1); assert_eq!( - minted_in(second), + minted_in(second).await, 0, "the receipt PDA from the first mint must keep the drain from re-minting" ); @@ -515,31 +546,35 @@ async fn an_orphaned_deposit_is_reminted_exactly_once_in_the_replacement() { let deposit_op_id = [0x2c_u8; 32]; let amount = 500_u64; - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, mempool_handle) = start_sequencer(config).await; sequencer - .store - .dbio() - .add_pending_deposit_event(PendingDepositEventRecord { - deposit_op_id: HashType(deposit_op_id), - source_tx_hash: HashType([7_u8; 32]), - amount, - metadata: borsh::to_vec(&DepositMetadataForEncoding { recipient_id }).unwrap(), + .block_store() + .storage_ref() + .ask(AddPendingDepositEvent { + event: PendingDepositEventRecord { + deposit_op_id: HashType(deposit_op_id), + source_tx_hash: HashType([7_u8; 32]), + amount, + metadata: borsh::to_vec(&DepositMetadataForEncoding { recipient_id }).unwrap(), + }, }) + .await .unwrap(); // Produce the block that mints the deposit; its receipt marks it minted. sequencer.run_production_turn().await.unwrap(); - let minted_block = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + let minted_block = sequencer.store.block_at_id(2).await.unwrap().unwrap(); assert!( - sequencer.with_state(|s| deposit_already_minted(s, HashType(deposit_op_id))), + sequencer + .with_state(|s| deposit_already_minted(s, HashType(deposit_op_id))) + .await, "the first mint claims the receipt in head state" ); // Orphan that block. The receipt reverts with it โ€” nothing else tracks the // mint โ€” so the deposit reads as unminted again. apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { @@ -547,10 +582,17 @@ async fn an_orphaned_deposit_is_reminted_exactly_once_in_the_replacement() { orphaned: vec![(MsgId::from(minted_block.header.hash.0), minted_block)], ..empty_follow_update() }, + ) + .await; + assert_eq!( + sequencer.chain_height().await, + 1, + "the minting block is orphaned" ); - assert_eq!(sequencer.chain_height(), 1, "the minting block is orphaned"); assert!( - !sequencer.with_state(|s| deposit_already_minted(s, HashType(deposit_op_id))), + !sequencer + .with_state(|s| deposit_already_minted(s, HashType(deposit_op_id))) + .await, "the receipt reverts with the orphaned block" ); @@ -559,7 +601,8 @@ async fn an_orphaned_deposit_is_reminted_exactly_once_in_the_replacement() { let replacement = sequencer.run_production_turn().await.unwrap(); let mints = sequencer .store - .get_block_at_id(replacement) + .block_at_id(replacement) + .await .unwrap() .expect("replacement block is stored") .body @@ -573,7 +616,9 @@ async fn an_orphaned_deposit_is_reminted_exactly_once_in_the_replacement() { ); let vault_id = vault_core::compute_vault_account_id(programs::vault().id(), recipient_id); assert_eq!( - sequencer.with_state(|s| s.get_account_by_id(vault_id).balance), + sequencer + .with_state(|s| s.get_account_by_id(vault_id).balance) + .await, u128::from(amount), "the vault is credited exactly once across the reorg" ); @@ -592,8 +637,7 @@ async fn a_replayed_deposit_mint_no_ops_in_the_guest() { let deposit_op_id = [0x5a_u8; 32]; let amount = 500_u64; - let (sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (sequencer, _mempool_handle) = start_sequencer(config).await; let deposit_tx = build_bridge_deposit_tx_from_event(&PendingDepositEventRecord { deposit_op_id: HashType(deposit_op_id), @@ -607,7 +651,7 @@ async fn a_replayed_deposit_mint_no_ops_in_the_guest() { }; let vault_id = vault_core::compute_vault_account_id(programs::vault().id(), recipient_id); - let mut state = sequencer.chain().lock().unwrap().head_state().clone(); + let mut state = sequencer.chain().lock().await.head_state().clone(); // First mint: claims the receipt and credits the recipient vault. state @@ -640,13 +684,15 @@ async fn recorded_dispatches_are_drained_from_the_store_on_production() { let record = dispatch_record(7, ping_payload(&payload)); let key = record.message_key; - let (mut sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await; + let (mut sequencer, _mempool_handle) = start_sequencer(cross_zone_test_config()).await; assert_eq!( sequencer - .store - .dbio() - .add_pending_cross_zone_dispatches(vec![record]) + .block_store() + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: vec![record], + }) + .await .unwrap(), 1 ); @@ -662,7 +708,8 @@ async fn recorded_dispatches_are_drained_from_the_store_on_production() { let block_id = sequencer.run_production_turn().await.unwrap(); let block = sequencer .store - .get_block_at_id(block_id) + .block_at_id(block_id) + .await .unwrap() .expect("produced block is stored"); assert_eq!( @@ -673,7 +720,9 @@ async fn recorded_dispatches_are_drained_from_the_store_on_production() { let record_id = ping_record_pda(programs::ping_receiver().id()); assert_eq!( - sequencer.with_state(|state| state.get_account_by_id(record_id).data.into_inner()), + sequencer + .with_state(|state| state.get_account_by_id(record_id).data.into_inner()) + .await, payload, "the dispatch must reach its target program, not just sit in the block" ); @@ -682,6 +731,7 @@ async fn recorded_dispatches_are_drained_from_the_store_on_production() { // the inbox seen-set now in head state, not by any marker on the record. assert_eq!( pending_dispatches(&sequencer) + .await .iter() .map(|record| record.message_key) .collect::>(), @@ -698,36 +748,41 @@ async fn a_delivered_dispatch_is_skipped_on_the_next_turn() { let record = dispatch_record(11, ping_payload(b"once")); let key = record.message_key; - let (mut sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await; + let (mut sequencer, _mempool_handle) = start_sequencer(cross_zone_test_config()).await; sequencer - .store - .dbio() - .add_pending_cross_zone_dispatches(vec![record]) + .block_store() + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: vec![record], + }) + .await .unwrap(); let first = sequencer.run_production_turn().await.unwrap(); let second = sequencer.run_production_turn().await.unwrap(); - let delivered_in = |block_id: u64| { + let delivered_in = async |block_id: u64| { dispatches_in( &sequencer .store - .get_block_at_id(block_id) + .block_at_id(block_id) + .await .unwrap() .expect("produced block is stored"), ) }; - assert_eq!(delivered_in(first), vec![key]); + assert_eq!(delivered_in(first).await, vec![key]); assert!( - delivered_in(second).is_empty(), + delivered_in(second).await.is_empty(), "the inbox seen-set must keep the drain from re-delivering" ); let message = extract_cross_zone_dispatch(&dispatch_tx(11, ping_payload(b"once"))) .expect("the dispatch carries a cross-zone message"); assert!( - sequencer.with_state(|state| dispatch_already_delivered(state, &message)), + sequencer + .with_state(|state| dispatch_already_delivered(state, &message)) + .await, "the seen shard in head state is what the skip reads" ); } @@ -740,19 +795,22 @@ async fn a_dispatch_that_never_executes_is_given_up_on_after_repeated_failures() // so without a give-up policy it would fail on every block for ever. let record = dispatch_record(13, b"odd".to_vec()); - let (mut sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await; + let (mut sequencer, _mempool_handle) = start_sequencer(cross_zone_test_config()).await; sequencer - .store - .dbio() - .add_pending_cross_zone_dispatches(vec![record]) + .block_store() + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: vec![record], + }) + .await .unwrap(); for attempt in 1..RETIRE_DISPATCH_AFTER_FAILURES { let block_id = sequencer.run_production_turn().await.unwrap(); let block = sequencer .store - .get_block_at_id(block_id) + .block_at_id(block_id) + .await .unwrap() .expect("produced block is stored"); assert!( @@ -760,7 +818,7 @@ async fn a_dispatch_that_never_executes_is_given_up_on_after_repeated_failures() "a dispatch that fails to execute must not reach the block" ); - let records = pending_dispatches(&sequencer); + let records = pending_dispatches(&sequencer).await; assert_eq!(records.len(), 1); assert_eq!( records[0].failed_attempts, attempt, @@ -774,14 +832,13 @@ async fn a_dispatch_that_never_executes_is_given_up_on_after_repeated_failures() // list without bound. sequencer.run_production_turn().await.unwrap(); assert!( - pending_dispatches(&sequencer).is_empty(), + pending_dispatches(&sequencer).await.is_empty(), "giving up on a delivery must take its record out of the pending list" ); // The dead letter is the only record that this happened, and the origin is // what identifies which message stopped being attempted. - let dbio = sequencer.store.dbio(); - let dead_letters = dbio.get_dead_letter_cross_zone_dispatches().unwrap(); + let (dead_letter_count, dead_letters) = sequencer.cross_zone_dead_letters().await.unwrap(); assert_eq!(dead_letters.len(), 1); assert_eq!( dead_letters[0].origin, @@ -800,18 +857,23 @@ async fn a_dispatch_that_never_executes_is_given_up_on_after_repeated_failures() dead_letters[0].failed_attempts, RETIRE_DISPATCH_AFTER_FAILURES ); - assert_eq!(dbio.get_dead_letter_cross_zone_dispatch_count().unwrap(), 1); + assert_eq!(dead_letter_count, 1); // The same view the RPC serves, so an operator sees what the store holds. - let (total_retired, retained) = sequencer.cross_zone_dead_letters().unwrap(); + let (total_retired, retained) = sequencer.cross_zone_dead_letters().await.unwrap(); assert_eq!(total_retired, 1); assert_eq!(retained, dead_letters); // And nothing re-feeds it, so it stops costing a guest execution per block. let block_id = sequencer.run_production_turn().await.unwrap(); - let block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap(); + let block = sequencer + .store + .block_at_id(block_id) + .await + .unwrap() + .unwrap(); assert!(dispatches_in(&block).is_empty()); - assert!(pending_dispatches(&sequencer).is_empty()); + assert!(pending_dispatches(&sequencer).await.is_empty()); } #[tokio::test] @@ -824,45 +886,61 @@ async fn a_redelivered_record_is_dropped_once_its_delivery_is_irreversible() { let record = dispatch_record(29, ping_payload(b"again")); let key = record.message_key; - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await; + let (mut sequencer, mempool_handle) = start_sequencer(cross_zone_test_config()).await; sequencer - .store - .dbio() - .add_pending_cross_zone_dispatches(vec![record.clone()]) + .block_store() + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: vec![record.clone()], + }) + .await .unwrap(); let block_id = sequencer.run_production_turn().await.unwrap(); - let delivery_block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap(); + let delivery_block = sequencer + .store + .block_at_id(block_id) + .await + .unwrap() + .unwrap(); assert_eq!(dispatches_in(&delivery_block), vec![key]); apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { finalized: vec![(MsgId::from(delivery_block.header.hash.0), delivery_block)], ..empty_follow_update() }, - ); - assert!(pending_dispatches(&sequencer).is_empty()); + ) + .await; + assert!(pending_dispatches(&sequencer).await.is_empty()); // The watcher re-reads the slot and records it again. sequencer - .store - .dbio() - .add_pending_cross_zone_dispatches(vec![record]) + .block_store() + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: vec![record], + }) + .await .unwrap(); - assert_eq!(pending_dispatches(&sequencer).len(), 1); + assert_eq!(pending_dispatches(&sequencer).await.len(), 1); let block_id = sequencer.run_production_turn().await.unwrap(); - let block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap(); + let block = sequencer + .store + .block_at_id(block_id) + .await + .unwrap() + .unwrap(); assert!( dispatches_in(&block).is_empty(), "the delivery is already on the chain, so it must not be delivered again" ); assert!( - pending_dispatches(&sequencer).is_empty(), + pending_dispatches(&sequencer).await.is_empty(), "a record whose delivery is already irreversible must be dropped, not kept for ever" ); } @@ -876,12 +954,14 @@ async fn a_delivery_still_reversible_keeps_its_record() { let record = dispatch_record(31, ping_payload(b"pending")); let key = record.message_key; - let (mut sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await; + let (mut sequencer, _mempool_handle) = start_sequencer(cross_zone_test_config()).await; sequencer - .store - .dbio() - .add_pending_cross_zone_dispatches(vec![record]) + .block_store() + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: vec![record], + }) + .await .unwrap(); sequencer.run_production_turn().await.unwrap(); @@ -889,6 +969,7 @@ async fn a_delivery_still_reversible_keeps_its_record() { assert_eq!( pending_dispatches(&sequencer) + .await .iter() .map(|record| record.message_key) .collect::>(), @@ -944,12 +1025,14 @@ async fn a_delivery_too_large_for_any_block_does_not_stall_production() { let mut config = cross_zone_test_config(); config.max_block_size = bytesize::ByteSize::kib(4); - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, mempool_handle) = start_sequencer(config).await; sequencer - .store - .dbio() - .add_pending_cross_zone_dispatches(vec![record]) + .block_store() + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: vec![record], + }) + .await .unwrap(); let user_tx = common::test_utils::create_transaction_native_token_transfer( @@ -966,7 +1049,12 @@ async fn a_delivery_too_large_for_any_block_does_not_stall_production() { // Production must get past it to the mempool in the very first block. let block_id = sequencer.run_production_turn().await.unwrap(); - let block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap(); + let block = sequencer + .store + .block_at_id(block_id) + .await + .unwrap() + .unwrap(); assert!( block.body.transactions.contains(&user_tx), "an oversized drained delivery must not stop production reaching the mempool" @@ -978,7 +1066,7 @@ async fn a_delivery_too_large_for_any_block_does_not_stall_production() { sequencer.run_production_turn().await.unwrap(); } assert!( - pending_dispatches(&sequencer).is_empty(), + pending_dispatches(&sequencer).await.is_empty(), "a delivery that fits in no block must be given up on" ); } @@ -999,16 +1087,23 @@ async fn a_delivery_backlog_is_spread_across_blocks() { let mut config = cross_zone_test_config(); config.max_num_tx_in_block = backlog + 10; - let (mut sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, _mempool_handle) = start_sequencer(config).await; sequencer - .store - .dbio() - .add_pending_cross_zone_dispatches(records) + .block_store() + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: records, + }) + .await .unwrap(); let block_id = sequencer.run_production_turn().await.unwrap(); - let block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap(); + let block = sequencer + .store + .block_at_id(block_id) + .await + .unwrap() + .unwrap(); assert_eq!( dispatches_in(&block).len(), MAX_DISPATCHES_PER_BLOCK, @@ -1017,7 +1112,12 @@ async fn a_delivery_backlog_is_spread_across_blocks() { // Deferred, not dropped: the rest go in the next block. let block_id = sequencer.run_production_turn().await.unwrap(); - let block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap(); + let block = sequencer + .store + .block_at_id(block_id) + .await + .unwrap() + .unwrap(); assert_eq!(dispatches_in(&block).len(), 3); } @@ -1061,15 +1161,7 @@ async fn transaction_pre_check_native_transfer_other_signature() { let tx = tx.transaction_stateless_check().unwrap(); // Signature is not from sender. Execution fails - let result = tx.execute_check_on_state( - sequencer - .chain() - .lock() - .expect("chain mutex poisoned") - .head_state_mut(), - 0, - 0, - ); + let result = tx.execute_check_on_state(sequencer.chain().lock().await.head_state_mut(), 0, 0); assert!(matches!( result, @@ -1096,11 +1188,7 @@ async fn transaction_pre_check_native_transfer_sent_too_much() { assert!(result.is_ok()); let result = result.unwrap().execute_check_on_state( - sequencer - .chain() - .lock() - .expect("chain mutex poisoned") - .head_state_mut(), + sequencer.chain().lock().await.head_state_mut(), 0, 0, ); @@ -1125,19 +1213,15 @@ async fn transaction_execute_native_transfer() { acc1, 0, acc2, 100, &sign_key1, ); - tx.execute_check_on_state( - sequencer - .chain() - .lock() - .expect("chain mutex poisoned") - .head_state_mut(), - 0, - 0, - ) - .unwrap(); + tx.execute_check_on_state(sequencer.chain().lock().await.head_state_mut(), 0, 0) + .unwrap(); - let bal_from = sequencer.with_state(|s| s.get_account_by_id(acc1).balance); - let bal_to = sequencer.with_state(|s| s.get_account_by_id(acc2).balance); + let bal_from = sequencer + .with_state(|s| s.get_account_by_id(acc1).balance) + .await; + let bal_to = sequencer + .with_state(|s| s.get_account_by_id(acc2).balance) + .await; assert_eq!(bal_from, 9900); assert_eq!(bal_to, 20100); @@ -1174,7 +1258,7 @@ async fn push_tx_into_mempool_blocks_until_mempool_is_full() { #[tokio::test] async fn build_block_from_mempool() { let (mut sequencer, mempool_handle) = common_setup().await; - let genesis_height = sequencer.chain_height(); + let genesis_height = sequencer.chain_height().await; let tx = common::test_utils::produce_dummy_empty_transaction(); mempool_handle @@ -1182,10 +1266,10 @@ async fn build_block_from_mempool() { .await .unwrap(); - let result = sequencer.build_block_from_mempool(Some(&[])); + let result = sequencer.build_block_from_mempool(Some(&[])).await; assert!(result.is_ok()); // Building itself does not advance the head; only apply-after-publish does. - assert_eq!(sequencer.chain_height(), genesis_height); + assert_eq!(sequencer.chain_height().await, genesis_height); } #[test] @@ -1251,7 +1335,8 @@ async fn replay_transactions_are_rejected_in_the_same_block() { sequencer.run_production_turn().await.unwrap(); let block = sequencer .store - .get_block_at_id(sequencer.chain_height()) + .block_at_id(sequencer.chain_height().await) + .await .unwrap() .unwrap(); @@ -1286,7 +1371,8 @@ async fn replay_transactions_are_rejected_in_different_blocks() { sequencer.run_production_turn().await.unwrap(); let block = sequencer .store - .get_block_at_id(sequencer.chain_height()) + .block_at_id(sequencer.chain_height().await) + .await .unwrap() .unwrap(); assert_eq!( @@ -1305,7 +1391,8 @@ async fn replay_transactions_are_rejected_in_different_blocks() { sequencer.run_production_turn().await.unwrap(); let block = sequencer .store - .get_block_at_id(sequencer.chain_height()) + .block_at_id(sequencer.chain_height().await) + .await .unwrap() .unwrap(); // The replay is rejected, so only the clock tx is in the block. @@ -1327,9 +1414,8 @@ async fn restart_from_storage() { // In the following code block a transaction will be processed that moves `balance_to_move` // from `acc_1` to `acc_2`. The block created with that transaction will be kept stored in // the temporary directory for the block storage of this test. - { - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let storage_weak = { + let (mut sequencer, mempool_handle) = start_sequencer(config.clone()).await; let signing_key = create_signing_key_for_account1(); let tx = common::test_utils::create_transaction_native_token_transfer( @@ -1347,7 +1433,8 @@ async fn restart_from_storage() { sequencer.run_production_turn().await.unwrap(); let block = sequencer .store - .get_block_at_id(sequencer.chain_height()) + .block_at_id(sequencer.chain_height().await) + .await .unwrap() .unwrap(); assert_eq!( @@ -1357,14 +1444,19 @@ async fn restart_from_storage() { LeeTransaction::Public(clock_invocation(block.header.timestamp)) ] ); - } + sequencer.block_store().storage_ref().downgrade() + }; + storage_weak.wait_for_shutdown_with_result(|_| ()).await; // Instantiating a new sequencer from the same config. This should load the existing block // with the above transaction and update the state to reflect that. - let (sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config.clone()).await; - let balance_acc_1 = sequencer.with_state(|s| s.get_account_by_id(acc1_account_id).balance); - let balance_acc_2 = sequencer.with_state(|s| s.get_account_by_id(acc2_account_id).balance); + let (sequencer, _mempool_handle) = start_sequencer(config.clone()).await; + let balance_acc_1 = sequencer + .with_state(|s| s.get_account_by_id(acc1_account_id).balance) + .await; + let balance_acc_2 = sequencer + .with_state(|s| s.get_account_by_id(acc2_account_id).balance) + .await; // Balances should be consistent with the stored block assert_eq!( @@ -1380,19 +1472,17 @@ async fn restart_from_storage() { #[tokio::test] async fn get_pending_blocks() { let config = setup_sequencer_config(); - let (mut sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, _mempool_handle) = start_sequencer(config).await; sequencer.run_production_turn().await.unwrap(); sequencer.run_production_turn().await.unwrap(); sequencer.run_production_turn().await.unwrap(); - assert_eq!(sequencer.get_pending_blocks().unwrap().len(), 4); + assert_eq!(sequencer.get_pending_blocks().await.unwrap().len(), 4); } #[tokio::test] async fn delete_blocks() { let config = setup_sequencer_config(); - let (mut sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, _mempool_handle) = start_sequencer(config).await; sequencer.run_production_turn().await.unwrap(); sequencer.run_production_turn().await.unwrap(); sequencer.run_production_turn().await.unwrap(); @@ -1400,9 +1490,10 @@ async fn delete_blocks() { let last_finalized_block = 3; sequencer .clean_finalized_blocks_from_db(last_finalized_block) + .await .unwrap(); - assert_eq!(sequencer.get_pending_blocks().unwrap().len(), 1); + assert_eq!(sequencer.get_pending_blocks().await.unwrap().len(), 1); } #[tokio::test] @@ -1412,9 +1503,8 @@ async fn produce_block_with_correct_prev_meta_after_restart() { let acc2_account_id = initial_public_user_accounts()[1].account_id; // Step 1: Create initial database with some block metadata - let expected_prev_meta = { - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let (storage_weak, expected_prev_meta) = { + let (mut sequencer, mempool_handle) = start_sequencer(config.clone()).await; let signing_key = create_signing_key_for_account1(); @@ -1434,12 +1524,13 @@ async fn produce_block_with_correct_prev_meta_after_restart() { sequencer.run_production_turn().await.unwrap(); // Get the metadata of the last block produced - sequencer.store.latest_block_meta().unwrap().unwrap() + let meta = sequencer.store.latest_block_meta().await.unwrap().unwrap(); + (sequencer.block_store().storage_ref().downgrade(), meta) }; + storage_weak.wait_for_shutdown_with_result(|_| ()).await; // Step 2: Restart sequencer from the same storage - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let (mut sequencer, mempool_handle) = start_sequencer(config.clone()).await; // Step 3: Submit a new transaction let signing_key = create_signing_key_for_account1(); @@ -1462,7 +1553,8 @@ async fn produce_block_with_correct_prev_meta_after_restart() { // Step 5: Verify the new block has correct previous block metadata let new_block = sequencer .store - .get_block_at_id(sequencer.chain_height()) + .block_at_id(sequencer.chain_height().await) + .await .unwrap() .unwrap(); @@ -1514,7 +1606,8 @@ async fn transactions_touching_clock_account_are_dropped_from_block() { let block = sequencer .store - .get_block_at_id(sequencer.chain_height()) + .block_at_id(sequencer.chain_height().await) + .await .unwrap() .unwrap(); @@ -1569,7 +1662,8 @@ async fn user_tx_that_chain_calls_clock_is_dropped() { let block = sequencer .store - .get_block_at_id(sequencer.chain_height()) + .block_at_id(sequencer.chain_height().await) + .await .unwrap() .unwrap(); @@ -1588,12 +1682,14 @@ async fn block_production_aborts_when_clock_account_data_is_corrupted() { // Corrupt the clock 01 account data so the clock program panics on deserialization. let clock_account_id = system_accounts::clock_account_ids()[0]; - let mut corrupted = sequencer.with_state(|s| s.get_account_by_id(clock_account_id)); + let mut corrupted = sequencer + .with_state(|s| s.get_account_by_id(clock_account_id)) + .await; corrupted.data = vec![0xff; 3].try_into().unwrap(); sequencer .chain() .lock() - .expect("chain mutex poisoned") + .await .head_state_mut() .force_insert_account(clock_account_id, corrupted); @@ -2119,31 +2215,38 @@ fn resubmittable_txs_of_blocks_without_user_txs_is_empty() { #[tokio::test] async fn follow_update_persists_the_checkpoint_with_its_effects() { let config = setup_sequencer_config(); - let (sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; + let (sequencer, mempool_handle) = start_sequencer(config).await; let genesis_meta = sequencer .store .latest_block_meta() + .await .unwrap() .expect("genesis meta is set"); let peer_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![]); apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { adopted: vec![(MsgId::from([1; 32]), peer_block)], ..empty_follow_update() }, - ); + ) + .await; // The checkpoint is the sdk resume cursor; landing it without the block // would let a restart stream past a block the store never got. assert!( - sequencer.store.get_zone_checkpoint().unwrap().is_some(), + sequencer + .store + .get_zone_checkpoint() + .await + .unwrap() + .is_some(), "the event's checkpoint must be persisted alongside the block it covers" ); - assert!(sequencer.store.get_block_at_id(2).unwrap().is_some()); + assert!(sequencer.store.block_at_id(2).await.unwrap().is_some()); } /// The channel orphaning our own still-unfinalized blocks rewinds the head and @@ -2153,29 +2256,28 @@ async fn follow_update_persists_the_checkpoint_with_its_effects() { #[tokio::test] async fn head_rewound_below_published_height_blocks_production() { let config = setup_sequencer_config(); - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, mempool_handle) = start_sequencer(config).await; let first = sequencer.run_production_turn().await.unwrap(); let published_tip = sequencer.run_production_turn().await.unwrap(); assert_eq!( - sequencer.store.published_high_water().unwrap(), + sequencer.store.published_high_water().await.unwrap(), Some(published_tip), "publishing records the high water mark" ); assert!( - sequencer.rewound_below_published().is_none(), + sequencer.rewound_below_published().await.is_none(), "an intact head is free to produce" ); - let produced: Vec = [first, published_tip] - .into_iter() - .map(|id| sequencer.store.get_block_at_id(id).unwrap().unwrap()) - .collect(); + let mut produced: Vec = Vec::new(); + for id in [first, published_tip] { + produced.push(sequencer.store.block_at_id(id).await.unwrap().unwrap()); + } // The sdk reports both of them as orphaned. apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { @@ -2185,14 +2287,22 @@ async fn head_rewound_below_published_height_blocks_production() { .collect(), ..empty_follow_update() }, - ); + ) + .await; assert!( - sequencer.store.latest_block_meta().unwrap().unwrap().id < published_tip, + sequencer + .store + .latest_block_meta() + .await + .unwrap() + .unwrap() + .id + < published_tip, "the orphan report rewound the stored tip" ); assert_eq!( - sequencer.rewound_below_published(), + sequencer.rewound_below_published().await, Some(published_tip), "the mark outlives the pruning and blocks the turn" ); @@ -2200,7 +2310,7 @@ async fn head_rewound_below_published_height_blocks_production() { // Those inscriptions were on the channel all along: finalizing them rebases // the head onto them, and production is free again. The guard is a wait. apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { @@ -2210,11 +2320,12 @@ async fn head_rewound_below_published_height_blocks_production() { .collect(), ..empty_follow_update() }, - ); + ) + .await; - assert_eq!(sequencer.next_block_height(), published_tip + 1); + assert_eq!(sequencer.next_block_height().await, published_tip + 1); assert!( - sequencer.rewound_below_published().is_none(), + sequencer.rewound_below_published().await.is_none(), "a recovered head resumes producing" ); } @@ -2222,7 +2333,7 @@ async fn head_rewound_below_published_height_blocks_production() { #[tokio::test] async fn follow_update_records_deposits_for_the_production_drain() { let config = setup_sequencer_config(); - let (sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; + let (sequencer, mempool_handle) = start_sequencer(config).await; let recipient_id = initial_public_user_accounts()[0].account_id; let metadata = borsh::to_vec(&DepositMetadataForEncoding { recipient_id }).unwrap(); @@ -2237,16 +2348,17 @@ async fn follow_update_records_deposits_for_the_production_drain() { }; apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { deposits: vec![deposit], ..empty_follow_update() }, - ); + ) + .await; - let pending = sequencer.store.get_pending_deposit_events().unwrap(); + let pending = sequencer.store.get_pending_deposit_events().await.unwrap(); assert_eq!(pending.len(), 1); assert_eq!(pending[0].deposit_op_id, HashType([21; 32])); } @@ -2254,10 +2366,11 @@ async fn follow_update_records_deposits_for_the_production_drain() { #[tokio::test] async fn follow_adopted_peer_block_applies_and_persists() { let config = setup_sequencer_config(); - let (sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; + let (sequencer, mempool_handle) = start_sequencer(config).await; let genesis_meta = sequencer .store .latest_block_meta() + .await .unwrap() .expect("genesis meta is set"); @@ -2273,24 +2386,28 @@ async fn follow_adopted_peer_block_applies_and_persists() { let peer_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![tx]); apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { adopted: vec![(MsgId::from([1; 32]), peer_block.clone())], ..empty_follow_update() }, - ); + ) + .await; - assert_eq!(sequencer.chain_height(), 2); + assert_eq!(sequencer.chain_height().await, 2); let stored = sequencer .store - .get_block_at_id(2) + .block_at_id(2) + .await .unwrap() .expect("adopted peer block should be persisted"); assert_eq!(stored.header.hash, peer_block.header.hash); assert_eq!( - sequencer.with_state(|s| s.get_account_by_id(acc2).balance), + sequencer + .with_state(|s| s.get_account_by_id(acc2).balance) + .await, 20010 ); } @@ -2298,8 +2415,7 @@ async fn follow_adopted_peer_block_applies_and_persists() { #[tokio::test] async fn follow_redelivery_of_own_block_is_deduped() { let config = setup_sequencer_config(); - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, mempool_handle) = start_sequencer(config).await; let acc1 = initial_public_user_accounts()[0].account_id; let acc2 = initial_public_user_accounts()[1].account_id; @@ -2315,23 +2431,26 @@ async fn follow_redelivery_of_own_block_is_deduped() { .await .unwrap(); sequencer.run_production_turn().await.unwrap(); - let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + let block2 = sequencer.store.block_at_id(2).await.unwrap().unwrap(); // The channel redelivers our own block under the MsgId the mock publisher // assigned at publish time. apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { adopted: vec![(MsgId::from(block2.header.hash.0), block2)], ..empty_follow_update() }, - ); + ) + .await; - assert_eq!(sequencer.chain_height(), 2); + assert_eq!(sequencer.chain_height().await, 2); assert_eq!( - sequencer.with_state(|s| s.get_account_by_id(acc2).balance), + sequencer + .with_state(|s| s.get_account_by_id(acc2).balance) + .await, 20010, "the transfer must not be double-applied" ); @@ -2340,8 +2459,7 @@ async fn follow_redelivery_of_own_block_is_deduped() { #[tokio::test] async fn follow_orphan_reverts_head_and_requeues_user_txs() { let config = setup_sequencer_config(); - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, mempool_handle) = start_sequencer(config).await; let acc1 = initial_public_user_accounts()[0].account_id; let acc2 = initial_public_user_accounts()[1].account_id; @@ -2357,10 +2475,10 @@ async fn follow_orphan_reverts_head_and_requeues_user_txs() { .await .unwrap(); sequencer.run_production_turn().await.unwrap(); - let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + let block2 = sequencer.store.block_at_id(2).await.unwrap().unwrap(); apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { @@ -2368,11 +2486,14 @@ async fn follow_orphan_reverts_head_and_requeues_user_txs() { orphaned: vec![(MsgId::from(block2.header.hash.0), block2)], ..empty_follow_update() }, - ); + ) + .await; - assert_eq!(sequencer.chain_height(), 1); + assert_eq!(sequencer.chain_height().await, 1); assert_eq!( - sequencer.with_state(|s| s.get_account_by_id(acc1).balance), + sequencer + .with_state(|s| s.get_account_by_id(acc1).balance) + .await, 10000, "the orphaned transfer must be reverted from the head" ); @@ -2396,8 +2517,7 @@ async fn follow_orphan_of_a_finalized_block_requeues_nothing() { // included, so requeueing them would put them back in every block we // produce from then on. let config = setup_sequencer_config(); - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, mempool_handle) = start_sequencer(config).await; let acc1 = initial_public_user_accounts()[0].account_id; let acc2 = initial_public_user_accounts()[1].account_id; @@ -2413,34 +2533,38 @@ async fn follow_orphan_of_a_finalized_block_requeues_nothing() { .await .unwrap(); sequencer.run_production_turn().await.unwrap(); - let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + let block2 = sequencer.store.block_at_id(2).await.unwrap().unwrap(); apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { finalized: vec![(MsgId::from(block2.header.hash.0), block2.clone())], ..empty_follow_update() }, - ); + ) + .await; apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { orphaned: vec![(MsgId::from(block2.header.hash.0), block2)], ..empty_follow_update() }, - ); + ) + .await; assert_eq!( - sequencer.chain_height(), + sequencer.chain_height().await, 2, "an irreversible block cannot be reverted" ); assert_eq!( - sequencer.with_state(|s| s.get_account_by_id(acc2).balance), + sequencer + .with_state(|s| s.get_account_by_id(acc2).balance) + .await, 20010, "the finalized transfer stands" ); @@ -2453,8 +2577,7 @@ async fn follow_orphan_of_a_finalized_block_requeues_nothing() { #[tokio::test] async fn follow_finalized_own_block_moves_final_tier_and_marks_store() { let config = setup_sequencer_config(); - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, mempool_handle) = start_sequencer(config).await; let tx = common::test_utils::produce_dummy_empty_transaction(); mempool_handle @@ -2462,10 +2585,10 @@ async fn follow_finalized_own_block_moves_final_tier_and_marks_store() { .await .unwrap(); sequencer.run_production_turn().await.unwrap(); - let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + let block2 = sequencer.store.block_at_id(2).await.unwrap().unwrap(); apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { @@ -2474,17 +2597,18 @@ async fn follow_finalized_own_block_moves_final_tier_and_marks_store() { finalized: vec![(MsgId::from(block2.header.hash.0), block2)], ..empty_follow_update() }, - ); + ) + .await; let final_tip = sequencer .chain() .lock() - .expect("chain mutex poisoned") + .await .final_tip() .expect("final tip set"); assert_eq!(final_tip.block_id, 2); - assert_eq!(sequencer.chain_height(), 2, "head is unchanged"); - let stored = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + assert_eq!(sequencer.chain_height().await, 2, "head is unchanged"); + let stored = sequencer.store.block_at_id(2).await.unwrap().unwrap(); assert!(matches!(stored.bedrock_status, BedrockStatus::Finalized)); } @@ -2497,35 +2621,43 @@ async fn follow_finalized_delivery_drops_its_pending_record() { let record = dispatch_record(17, ping_payload(b"settled")); let key = record.message_key; - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await; + let (mut sequencer, mempool_handle) = start_sequencer(cross_zone_test_config()).await; sequencer - .store - .dbio() - .add_pending_cross_zone_dispatches(vec![record]) + .block_store() + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: vec![record], + }) + .await .unwrap(); let block_id = sequencer.run_production_turn().await.unwrap(); - let delivery_block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap(); + let delivery_block = sequencer + .store + .block_at_id(block_id) + .await + .unwrap() + .unwrap(); assert_eq!(dispatches_in(&delivery_block), vec![key]); assert_eq!( - pending_dispatches(&sequencer).len(), + pending_dispatches(&sequencer).await.len(), 1, "including the delivery is not enough to settle its record" ); apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { finalized: vec![(MsgId::from(delivery_block.header.hash.0), delivery_block)], ..empty_follow_update() }, - ); + ) + .await; assert!( - pending_dispatches(&sequencer).is_empty(), + pending_dispatches(&sequencer).await.is_empty(), "a delivery in an irreversible block settles its record" ); } @@ -2540,12 +2672,14 @@ async fn a_parked_finalized_block_does_not_drop_a_dispatch_record() { let key = record.message_key; let delivery = dispatch_tx(19, ping_payload(b"parked")); - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await; + let (mut sequencer, mempool_handle) = start_sequencer(cross_zone_test_config()).await; sequencer - .store - .dbio() - .add_pending_cross_zone_dispatches(vec![record]) + .block_store() + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: vec![record], + }) + .await .unwrap(); let tx = common::test_utils::produce_dummy_empty_transaction(); @@ -2561,17 +2695,19 @@ async fn a_parked_finalized_block_does_not_drop_a_dispatch_record() { common::test_utils::produce_dummy_block(9, Some(HashType([44; 32])), vec![delivery]); apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { finalized: vec![(MsgId::from([9; 32]), parked)], ..empty_follow_update() }, - ); + ) + .await; assert_eq!( pending_dispatches(&sequencer) + .await .iter() .map(|record| record.message_key) .collect::>(), @@ -2583,10 +2719,11 @@ async fn a_parked_finalized_block_does_not_drop_a_dispatch_record() { #[tokio::test] async fn follow_finalized_backfill_block_is_applied_and_marked_finalized() { let config = setup_sequencer_config(); - let (sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; + let (sequencer, mempool_handle) = start_sequencer(config).await; let genesis_meta = sequencer .store .latest_block_meta() + .await .unwrap() .expect("genesis meta is set"); @@ -2595,7 +2732,7 @@ async fn follow_finalized_backfill_block_is_applied_and_marked_finalized() { let peer_block = common::test_utils::produce_dummy_block(2, Some(genesis_meta.hash), vec![]); apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { @@ -2604,16 +2741,18 @@ async fn follow_finalized_backfill_block_is_applied_and_marked_finalized() { finalized: vec![(MsgId::from([2; 32]), peer_block.clone())], ..empty_follow_update() }, - ); + ) + .await; assert_eq!( - sequencer.chain_height(), + sequencer.chain_height().await, 2, "head mirrors final on backfill" ); let stored = sequencer .store - .get_block_at_id(2) + .block_at_id(2) + .await .unwrap() .expect("backfilled block should be persisted"); assert_eq!(stored.header.hash, peer_block.header.hash); @@ -2623,8 +2762,7 @@ async fn follow_finalized_backfill_block_is_applied_and_marked_finalized() { #[tokio::test] async fn parked_finalized_block_neither_sweeps_the_store_nor_drops_its_deposit_record() { let config = setup_sequencer_config(); - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, mempool_handle) = start_sequencer(config).await; // A produced block at head, still pending on the channel. let tx = common::test_utils::produce_dummy_empty_transaction(); @@ -2647,9 +2785,10 @@ async fn parked_finalized_block_neither_sweeps_the_store_nor_drops_its_deposit_r let deposit_tx = build_bridge_deposit_tx_from_event(&record).unwrap(); assert!( sequencer - .store - .dbio() - .add_pending_deposit_event(record) + .block_store() + .storage_ref() + .ask(AddPendingDepositEvent { event: record }) + .await .unwrap() ); @@ -2659,7 +2798,7 @@ async fn parked_finalized_block_neither_sweeps_the_store_nor_drops_its_deposit_r common::test_utils::produce_dummy_block(9, Some(HashType([44; 32])), vec![deposit_tx]); apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { @@ -2668,11 +2807,12 @@ async fn parked_finalized_block_neither_sweeps_the_store_nor_drops_its_deposit_r finalized: vec![(MsgId::from([9; 32]), parked)], ..empty_follow_update() }, - ); + ) + .await; // Nothing became irreversible, so the store must not be swept through the // parked block's height. - let stored = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + let stored = sequencer.store.block_at_id(2).await.unwrap().unwrap(); assert!( matches!(stored.bedrock_status, BedrockStatus::Pending), "a parked finalized block must not mark earlier blocks finalized" @@ -2683,6 +2823,7 @@ async fn parked_finalized_block_neither_sweeps_the_store_nor_drops_its_deposit_r sequencer .store .get_pending_deposit_events() + .await .unwrap() .iter() .any(|event| event.deposit_op_id == deposit_op_id), @@ -2697,9 +2838,8 @@ async fn restart_restores_head_tier_and_recovers_from_orphan() { let acc2 = initial_public_user_accounts()[1].account_id; // Produce block 2 (a user transfer), then "crash" before it finalizes. - let (tx, block2) = { - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let (storage_weak, tx, block2) = { + let (mut sequencer, mempool_handle) = start_sequencer(config.clone()).await; let tx = common::test_utils::create_transaction_native_token_transfer( acc1, 0, @@ -2712,22 +2852,27 @@ async fn restart_restores_head_tier_and_recovers_from_orphan() { .await .unwrap(); sequencer.run_production_turn().await.unwrap(); - (tx, sequencer.store.get_block_at_id(2).unwrap().unwrap()) + let block2 = sequencer.store.block_at_id(2).await.unwrap().unwrap(); + ( + sequencer.block_store().storage_ref().downgrade(), + tx, + block2, + ) }; + storage_weak.wait_for_shutdown_with_result(|_| ()).await; // Restart: nothing is finalized, so block 2 must come back as *head*, not // final โ€” the L1 can still orphan it. - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config.clone()).await; - assert_eq!(sequencer.chain_height(), 2); + let (mut sequencer, mempool_handle) = start_sequencer(config.clone()).await; + assert_eq!(sequencer.chain_height().await, 2); // The L1 orphans block 2 under its real MsgId (which we never persisted) // and adopts a competing empty block 2'. - let genesis = sequencer.store.get_block_at_id(1).unwrap().unwrap(); + let genesis = sequencer.store.block_at_id(1).await.unwrap().unwrap(); let block2_prime = common::test_utils::produce_dummy_block(2, Some(genesis.header.hash), vec![]); apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { @@ -2735,24 +2880,27 @@ async fn restart_restores_head_tier_and_recovers_from_orphan() { orphaned: vec![(MsgId::from([20; 32]), block2)], ..empty_follow_update() }, - ); + ) + .await; // The head reorged onto 2': transfer reverted, store overwritten, and the // orphaned user tx returned to the mempool. - assert_eq!(sequencer.chain_height(), 2); + assert_eq!(sequencer.chain_height().await, 2); let head_tip = sequencer .chain() .lock() - .expect("chain mutex poisoned") + .await .head_tip() .expect("head tip set"); assert_eq!(head_tip.hash, block2_prime.header.hash); assert_eq!( - sequencer.with_state(|s| s.get_account_by_id(acc1).balance), + sequencer + .with_state(|s| s.get_account_by_id(acc1).balance) + .await, 10000, "the orphaned transfer must be reverted" ); - let stored = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + let stored = sequencer.store.block_at_id(2).await.unwrap().unwrap(); assert_eq!(stored.header.hash, block2_prime.header.hash); let (origin, requeued) = sequencer .mempool @@ -2768,18 +2916,17 @@ async fn restart_reanchors_on_the_persisted_final_snapshot() { // Produce block 2 and follow its finalization, which persists the final // snapshot; then "crash". - { - let (mut sequencer, mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let storage_weak = { + let (mut sequencer, mempool_handle) = start_sequencer(config.clone()).await; let tx = common::test_utils::produce_dummy_empty_transaction(); mempool_handle .push((TransactionOrigin::User, tx)) .await .unwrap(); sequencer.run_production_turn().await.unwrap(); - let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + let block2 = sequencer.store.block_at_id(2).await.unwrap().unwrap(); apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { @@ -2788,15 +2935,17 @@ async fn restart_reanchors_on_the_persisted_final_snapshot() { finalized: vec![(MsgId::from(block2.header.hash.0), block2)], ..empty_follow_update() }, - ); - } + ) + .await; + sequencer.block_store().storage_ref().downgrade() + }; + storage_weak.wait_for_shutdown_with_result(|_| ()).await; // Restart: the final tier re-anchors on the snapshot instead of treating // the whole stored chain as final. - let (sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let (sequencer, _mempool_handle) = start_sequencer(config.clone()).await; let chain = sequencer.chain(); - let chain = chain.lock().expect("chain mutex poisoned"); + let chain = chain.lock().await; assert_eq!(chain.final_tip().expect("final tip set").block_id, 2); assert_eq!(chain.head_tip().expect("head tip set").block_id, 2); } @@ -2804,11 +2953,11 @@ async fn restart_reanchors_on_the_persisted_final_snapshot() { #[tokio::test] async fn record_produced_block_skips_persistence_on_lost_race() { let config = setup_sequencer_config(); - let (mut sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, _mempool_handle) = start_sequencer(config).await; let genesis_meta = sequencer .store .latest_block_meta() + .await .unwrap() .expect("genesis meta is set"); @@ -2817,7 +2966,7 @@ async fn record_produced_block_skips_persistence_on_lost_race() { sequencer .chain() .lock() - .expect("chain mutex poisoned") + .await .apply_adopted(MsgId::from([9; 32]), &peer_block); // Our competing block at the same height: same parent, different content. @@ -2834,51 +2983,48 @@ async fn record_produced_block_skips_persistence_on_lost_race() { sequencer .record_produced_block( MsgId::from(our_block.header.hash.0), - &our_block, - &[], + our_block.clone(), + vec![], &mock_checkpoint(), ) + .await .unwrap(); // The lost-race block must not reach the store; the head keeps the peer block. - assert!(sequencer.store.get_block_at_id(2).unwrap().is_none()); - let head_tip = sequencer - .chain() - .lock() - .expect("chain mutex poisoned") - .head_tip() - .expect("head tip"); + assert!(sequencer.store.block_at_id(2).await.unwrap().is_none()); + let head_tip = sequencer.chain().lock().await.head_tip().expect("head tip"); assert_eq!(head_tip.hash, peer_block.header.hash); } #[tokio::test] async fn record_produced_block_skips_persistence_when_block_no_longer_chains() { let config = setup_sequencer_config(); - let (mut sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config).await; + let (mut sequencer, _mempool_handle) = start_sequencer(config).await; // The head reorged under us: our block's parent is no longer the tip. let stale = common::test_utils::produce_dummy_block(2, Some(HashType([9; 32])), vec![]); sequencer .record_produced_block( MsgId::from(stale.header.hash.0), - &stale, - &[], + stale.clone(), + vec![], &mock_checkpoint(), ) + .await .unwrap(); - assert!(sequencer.store.get_block_at_id(2).unwrap().is_none()); - assert_eq!(sequencer.chain_height(), 1, "head is unchanged"); + assert!(sequencer.store.block_at_id(2).await.unwrap().is_none()); + assert_eq!(sequencer.chain_height().await, 1, "head is unchanged"); } #[tokio::test] async fn follow_update_persists_blocks_meta_and_state_atomically() { let config = setup_sequencer_config(); - let (sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; + let (sequencer, mempool_handle) = start_sequencer(config).await; let genesis_meta = sequencer .store .latest_block_meta() + .await .unwrap() .expect("genesis meta is set"); @@ -2896,7 +3042,7 @@ async fn follow_update_persists_blocks_meta_and_state_atomically() { // One update carrying several blocks: both adopted, block 2 also finalized. apply_follow_update( - &sequencer.store.dbio(), + sequencer.block_store().storage_ref(), &sequencer.chain(), &mempool_handle, FollowUpdate { @@ -2908,23 +3054,27 @@ async fn follow_update_persists_blocks_meta_and_state_atomically() { finalized: vec![(MsgId::from([2; 32]), block2)], ..empty_follow_update() }, - ); + ) + .await; // Blocks, tip meta and state all reflect the end of the batch: a late // finalized entry for an earlier block must not drag the tip meta back. let meta = sequencer .store .latest_block_meta() + .await .unwrap() .expect("meta is set"); assert_eq!(meta.id, 3); assert_eq!(meta.hash, block3.header.hash); - let stored2 = sequencer.store.get_block_at_id(2).unwrap().unwrap(); + let stored2 = sequencer.store.block_at_id(2).await.unwrap().unwrap(); assert!(matches!(stored2.bedrock_status, BedrockStatus::Finalized)); let stored_balance = sequencer .store .get_lee_state() + .await .unwrap() + .expect("the store holds a chain") .get_account_by_id(acc2) .balance; assert_eq!(stored_balance, 20010); diff --git a/lez/sequencer/core/src/tests/reconstruction.rs b/lez/sequencer/core/src/tests/reconstruction.rs index f2802ec0b..d61d12480 100644 --- a/lez/sequencer/core/src/tests/reconstruction.rs +++ b/lez/sequencer/core/src/tests/reconstruction.rs @@ -4,13 +4,15 @@ reason = "We don't care about it in tests" )] -use std::sync::Mutex; - use chain_state::ChainState; use common::block::Block; use logos_blockchain_core::mantle::ops::channel::{MsgId, inscribe::Inscription}; use logos_blockchain_zone_sdk::{Slot, ZoneBlock, ZoneMessage}; -use storage::sequencer::sequencer_cells::ZoneAnchorRecord; +use sequencer_storage_actor::{ + StorageActorTrait, + protocol::{AddPendingCrossZoneDispatches, AddPendingDepositEvent, ZoneAnchorRecord}, +}; +use tokio::sync::Mutex; use super::*; use crate::{ @@ -19,13 +21,35 @@ use crate::{ /// Fresh `(store, chain)` pair for a reconstruction target, as /// `start_from_config` would build them before the publisher starts. -fn fresh_store_and_chain(config: &SequencerConfig) -> (SequencerStore, Mutex) { +// TODO: Revisit this function, it relies to heavily on internal implementation details. +// Will be possible to address this once block publisher is moved to another actor(-s). +async fn fresh_store_and_chain( + config: &SequencerConfig, +) -> (SequencerStore, Mutex) { + let storage = StorageActor::new(&config.db_path()).expect("Failed to initialize storage actor"); + let storage_ref = StorageActor::spawn(storage); + // What `start_from_config` does before it opens a store, mirrored here + // because these cases drive `verify_and_reconstruct` directly. + let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); let bootstrap_sequencer_key = Some(test_bootstrap_sequencer_key(config)); - let (store, state) = - SequencerCore::::open_or_create_store(config, bootstrap_sequencer_key); - let chain = Mutex::new(SequencerCore::::restore_chain_state( - config, &store, &state, - )); + SequencerCore::::seed_genesis_if_absent( + &storage_ref, + &signing_key, + bootstrap_sequencer_key, + config, + ) + .await; + let store = SequencerStore::new(storage_ref, signing_key) + .await + .expect("open store"); + let state = store + .get_lee_state() + .await + .expect("read state") + .expect("seeded store holds a state"); + let chain = Mutex::new( + SequencerCore::::restore_chain_state(config, &store, &state).await, + ); (store, chain) } @@ -40,35 +64,49 @@ fn block_to_channel_message(block: &Block, slot: u64) -> (ZoneMessage, Slot) { /// Collects a sequencer's whole chain (genesis..=tip) into a canned channel, /// one block per slot at `slot_step` spacing. -fn channel_from_store(store: &SequencerStore, slot_step: u64) -> Vec<(ZoneMessage, Slot)> { +async fn channel_from_store( + store: &SequencerStore, + slot_step: u64, +) -> Vec<(ZoneMessage, Slot)> { let genesis_id = store.genesis_id(); - let tip_id = store.latest_block_meta().expect("tip").expect("present").id; - (genesis_id..=tip_id) - .enumerate() - .map(|(index, id)| { - let block = store.get_block_at_id(id).expect("read").expect("present"); - block_to_channel_message(&block, (index as u64 + 1) * slot_step) - }) - .collect() + let tip_id = store + .latest_block_meta() + .await + .expect("tip") + .expect("present") + .id; + let mut messages = Vec::new(); + for (index, id) in (genesis_id..=tip_id).enumerate() { + let block = store.block_at_id(id).await.expect("read").expect("present"); + messages.push(block_to_channel_message( + &block, + (index as u64 + 1) * slot_step, + )); + } + messages } #[tokio::test] async fn reconstructs_missing_channel_blocks_into_fresh_store() { // Sequencer A produces a few blocks; treat its chain as the channel. let config_a = setup_sequencer_config(); - let (mut seq_a, _handle_a) = - SequencerCoreWithMockClients::start_from_config(config_a.clone()).await; + let (mut seq_a, _handle_a) = start_sequencer(config_a.clone()).await; seq_a.run_production_turn().await.unwrap(); seq_a.run_production_turn().await.unwrap(); - let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap(); + let tip_a = seq_a + .block_store() + .latest_block_meta() + .await + .unwrap() + .unwrap(); - let messages = channel_from_store(seq_a.block_store(), 10); + let messages = channel_from_store(seq_a.block_store(), 10).await; let tip_slot = messages.last().unwrap().1; let channel_id = config_a.bedrock_config.channel_id; // Sequencer B starts from a fresh store and reconstructs A's chain. let config_b = setup_sequencer_config(); - let (store_b, chain_b) = fresh_store_and_chain(&config_b); + let (store_b, chain_b) = fresh_store_and_chain(&config_b).await; let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages); let channel_was_empty = SequencerCore::::verify_and_reconstruct( @@ -78,13 +116,13 @@ async fn reconstructs_missing_channel_blocks_into_fresh_store() { .expect("reconstruct"); assert!(!channel_was_empty); - let tip_b = store_b.latest_block_meta().unwrap().unwrap(); + let tip_b = store_b.latest_block_meta().await.unwrap().unwrap(); assert_eq!(tip_b.id, tip_a.id); assert_eq!(tip_b.hash, tip_a.hash); // State matches: initial account balances agree with sequencer A. - let state_b = chain_b.lock().unwrap().head_state().clone(); - let state_a = seq_a.chain().lock().unwrap().head_state().clone(); + let state_b = chain_b.lock().await.head_state().clone(); + let state_a = seq_a.chain().lock().await.head_state().clone(); for account in initial_public_user_accounts() { assert_eq!( state_b.get_account_by_id(account.account_id).balance, @@ -92,7 +130,11 @@ async fn reconstructs_missing_channel_blocks_into_fresh_store() { ); } - let anchor = store_b.get_zone_anchor().unwrap().expect("anchor recorded"); + let anchor = store_b + .get_zone_anchor() + .await + .unwrap() + .expect("anchor recorded"); assert_eq!(anchor.block_id, tip_a.id); assert_eq!(anchor.slot, tip_slot.into_inner()); @@ -103,24 +145,28 @@ async fn reconstructs_missing_channel_blocks_into_fresh_store() { .await .expect("reconstruct idempotent"); assert!(!channel_was_empty); - assert_eq!(store_b.latest_block_meta().unwrap().unwrap().id, tip_a.id); + assert_eq!( + store_b.latest_block_meta().await.unwrap().unwrap().id, + tip_a.id + ); } #[tokio::test] async fn fails_when_channel_serves_a_divergent_block() { let config = setup_sequencer_config(); - let (store, chain) = fresh_store_and_chain(&config); + let (store, chain) = fresh_store_and_chain(&config).await; // Anchor on the local genesis at some slot. let genesis_id = store.genesis_id(); - let genesis = store.get_block_at_id(genesis_id).unwrap().unwrap(); + let genesis = store.block_at_id(genesis_id).await.unwrap().unwrap(); let anchor_slot = 100_u64; store - .set_zone_anchor(&ZoneAnchorRecord { + .set_zone_anchor(ZoneAnchorRecord { slot: anchor_slot, block_id: genesis_id, hash: genesis.header.hash, }) + .await .unwrap(); // The channel serves a different block at the anchor id/slot. @@ -142,15 +188,16 @@ async fn fails_when_channel_serves_a_divergent_block() { #[tokio::test] async fn fails_when_channel_is_missing() { let config = setup_sequencer_config(); - let (store, chain) = fresh_store_and_chain(&config); + let (store, chain) = fresh_store_and_chain(&config).await; let genesis_id = store.genesis_id(); - let genesis = store.get_block_at_id(genesis_id).unwrap().unwrap(); + let genesis = store.block_at_id(genesis_id).await.unwrap().unwrap(); store - .set_zone_anchor(&ZoneAnchorRecord { + .set_zone_anchor(ZoneAnchorRecord { slot: 100, block_id: genesis_id, hash: genesis.header.hash, }) + .await .unwrap(); // Anchor present, but the channel does not exist on the connected chain. @@ -169,11 +216,15 @@ async fn fails_when_channel_is_missing() { #[tokio::test] async fn fails_when_channel_reinscribes_genesis_with_a_different_hash() { let config = setup_sequencer_config(); - let (store, chain) = fresh_store_and_chain(&config); + let (store, chain) = fresh_store_and_chain(&config).await; // Fresh store, no anchor. The channel serves a genesis at the same id but a // different hash โ€” a foreign chain reinscribing genesis. - let mut reinscribed = store.get_block_at_id(store.genesis_id()).unwrap().unwrap(); + let mut reinscribed = store + .block_at_id(store.genesis_id()) + .await + .unwrap() + .unwrap(); reinscribed.header.hash = HashType([0xAB_u8; 32]); let messages = vec![block_to_channel_message(&reinscribed, 10)]; @@ -195,7 +246,7 @@ async fn fails_when_channel_reinscribes_genesis_with_a_different_hash() { async fn fails_when_a_below_tip_channel_block_does_not_validate() { // A sequencer that committed blocks past genesis but never recorded an anchor. let config = setup_sequencer_config(); - let (mut seq, _handle) = SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let (mut seq, _handle) = start_sequencer(config.clone()).await; seq.run_production_turn().await.unwrap(); seq.run_production_turn().await.unwrap(); @@ -206,7 +257,8 @@ async fn fails_when_a_below_tip_channel_block_does_not_validate() { let below_tip_id = seq.block_store().genesis_id() + 1; let mut block = seq .block_store() - .get_block_at_id(below_tip_id) + .block_at_id(below_tip_id) + .await .unwrap() .unwrap(); block.header.hash = HashType([0xCD_u8; 32]); @@ -230,12 +282,16 @@ async fn fails_when_a_below_tip_channel_block_does_not_validate() { #[tokio::test] async fn fails_when_a_channel_block_is_numbered_below_genesis() { let config = setup_sequencer_config(); - let (store, chain) = fresh_store_and_chain(&config); + let (store, chain) = fresh_store_and_chain(&config).await; // A block numbered below our genesis โ€” a foreign chain with a lower // numbering. Nothing local sits at that id, so it goes straight to // validation and parks there. - let mut foreign = store.get_block_at_id(store.genesis_id()).unwrap().unwrap(); + let mut foreign = store + .block_at_id(store.genesis_id()) + .await + .unwrap() + .unwrap(); foreign.header.block_id = store.genesis_id() - 1; let messages = vec![block_to_channel_message(&foreign, 10)]; @@ -256,10 +312,14 @@ async fn fails_when_a_channel_block_is_numbered_below_genesis() { #[tokio::test] async fn fails_when_a_channel_block_does_not_extend_the_tip() { let config = setup_sequencer_config(); - let (store, chain) = fresh_store_and_chain(&config); + let (store, chain) = fresh_store_and_chain(&config).await; // A block claiming an id far past genesis does not chain onto the local tip. - let mut orphan = store.get_block_at_id(store.genesis_id()).unwrap().unwrap(); + let mut orphan = store + .block_at_id(store.genesis_id()) + .await + .unwrap() + .unwrap(); orphan.header.block_id = store.genesis_id() + 5; let messages = vec![block_to_channel_message(&orphan, 10)]; @@ -289,39 +349,42 @@ async fn fails_when_a_channel_block_does_not_extend_the_tip() { async fn reconstruction_ignores_a_duplicate_height_the_final_tier_settled() { // Sequencer A's chain is what the channel finalized. let config_a = setup_sequencer_config(); - let (mut seq_a, _mempool_a) = - SequencerCoreWithMockClients::start_from_config(config_a.clone()).await; + let (mut seq_a, _mempool_a) = start_sequencer(config_a.clone()).await; seq_a.run_production_turn().await.unwrap(); - let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap(); - let mut messages = channel_from_store(seq_a.block_store(), 10); + let tip_a = seq_a + .block_store() + .latest_block_meta() + .await + .unwrap() + .unwrap(); + let mut messages = channel_from_store(seq_a.block_store(), 10).await; let settled_slot = messages.last().unwrap().1; // Sequencer B: the cold-start backfill finalizes A's chain into its store. - let (seq_b, mempool_b) = - SequencerCoreWithMockClients::start_from_config(setup_sequencer_config()).await; - let finalized: Vec<(MsgId, Block)> = (seq_b.block_store().genesis_id()..=tip_a.id) - .map(|id| { - let block = seq_a.block_store().get_block_at_id(id).unwrap().unwrap(); - ( - MsgId::from([u8::try_from(id).expect("should be u8"); 32]), - block, - ) - }) - .collect(); + let (seq_b, mempool_b) = start_sequencer(setup_sequencer_config()).await; + let mut finalized: Vec<(MsgId, Block)> = Vec::new(); + for id in seq_b.block_store().genesis_id()..=tip_a.id { + let block = seq_a.block_store().block_at_id(id).await.unwrap().unwrap(); + finalized.push(( + MsgId::from([u8::try_from(id).expect("should be u8"); 32]), + block, + )); + } apply_follow_update( - &seq_b.store.dbio(), + seq_b.block_store().storage_ref(), &seq_b.chain(), &mempool_b, FollowUpdate { finalized, ..empty_follow_update() }, - ); + ) + .await; assert_eq!( seq_b .chain() .lock() - .unwrap() + .await .final_tip() .expect("backfill finalized A's chain") .block_id, @@ -331,7 +394,8 @@ async fn reconstruction_ignores_a_duplicate_height_the_final_tier_settled() { // A competitor published its own block at that same height. let parent = seq_a .block_store() - .get_block_at_id(tip_a.id - 1) + .block_at_id(tip_a.id - 1) + .await .unwrap() .unwrap(); let competitor = @@ -353,13 +417,19 @@ async fn reconstruction_ignores_a_duplicate_height_the_final_tier_settled() { .await .expect("a duplicate height the final tier settled must not abort startup"); - let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap(); + let tip_b = seq_b + .block_store() + .latest_block_meta() + .await + .unwrap() + .unwrap(); assert_eq!(tip_b.hash, tip_a.hash, "the finalized block stands"); // The anchor tracks the block we hold, never the one we dropped. let anchor = seq_b .block_store() .get_zone_anchor() + .await .unwrap() .expect("anchor"); assert_eq!(anchor.slot, settled_slot.into_inner()); @@ -372,31 +442,46 @@ async fn reconstruction_ignores_a_duplicate_height_the_final_tier_settled() { async fn reconstruction_replaces_a_conflicting_head_block_with_finalized_history() { // Sequencer A's chain is what the channel finalized. let config_a = setup_sequencer_config(); - let (mut seq_a, _mempool_a) = - SequencerCoreWithMockClients::start_from_config(config_a.clone()).await; + let (mut seq_a, _mempool_a) = start_sequencer(config_a.clone()).await; seq_a.run_production_turn().await.unwrap(); - let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap(); - let messages = channel_from_store(seq_a.block_store(), 10); + let tip_a = seq_a + .block_store() + .latest_block_meta() + .await + .unwrap() + .unwrap(); + let messages = channel_from_store(seq_a.block_store(), 10).await; let tip_slot = messages.last().unwrap().1; // Sequencer B adopted a competitor at that height and never saw it finalize. - let (seq_b, mempool_b) = - SequencerCoreWithMockClients::start_from_config(setup_sequencer_config()).await; - let genesis_b = seq_b.block_store().latest_block_meta().unwrap().unwrap(); + let (seq_b, mempool_b) = start_sequencer(setup_sequencer_config()).await; + let genesis_b = seq_b + .block_store() + .latest_block_meta() + .await + .unwrap() + .unwrap(); let competitor = common::test_utils::produce_dummy_block(tip_a.id, Some(genesis_b.hash), vec![]); assert_ne!(competitor.header.hash, tip_a.hash); apply_follow_update( - &seq_b.store.dbio(), + seq_b.block_store().storage_ref(), &seq_b.chain(), &mempool_b, FollowUpdate { adopted: vec![(MsgId::from([7_u8; 32]), competitor)], ..empty_follow_update() }, - ); + ) + .await; assert_eq!( - seq_b.block_store().latest_block_meta().unwrap().unwrap().id, + seq_b + .block_store() + .latest_block_meta() + .await + .unwrap() + .unwrap() + .id, tip_a.id, "the competitor is the head tip going in" ); @@ -415,7 +500,12 @@ async fn reconstruction_replaces_a_conflicting_head_block_with_finalized_history .await .expect("finalized history must replace a conflicting head block"); - let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap(); + let tip_b = seq_b + .block_store() + .latest_block_meta() + .await + .unwrap() + .unwrap(); assert_eq!(tip_b.id, tip_a.id); assert_eq!( tip_b.hash, tip_a.hash, @@ -499,7 +589,7 @@ fn deposit_event_record( // // Sequencer A produces a deposit block then a withdraw block. // let config_a = bridge_funded_config(); // let (mut seq_a, mempool_a) = -// SequencerCoreWithMockClients::start_from_config(config_a.clone()).await; +// start_sequencer(config_a.clone()).await; // let deposit_record = deposit_event_record(deposit_op_id, deposit_amount, recipient); // let deposit_tx = @@ -523,14 +613,14 @@ fn deposit_event_record( // .unwrap(); // seq_a.run_production_turn().await.unwrap(); -// let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap(); -// let messages = channel_from_store(seq_a.block_store(), 10); +// let tip_a = seq_a.block_store().latest_block_meta().await.unwrap().unwrap(); +// let messages = channel_from_store(seq_a.block_store(), 10).await; // let tip_slot = messages.last().unwrap().1; // let channel_id = config_a.bedrock_config.channel_id; // let config_b = bridge_funded_config(); // let (mut seq_b, _mempool_b) = -// SequencerCoreWithMockClients::start_from_config(config_b).await; +// start_sequencer(config_b).await; // // Backfill re-delivery: the deposit event is re-recorded as a pending record // // before reconstruction runs. The mint no longer flows through the mempool @@ -553,7 +643,7 @@ fn deposit_event_record( // .await // .expect("reconstruct"); -// let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap(); +// let tip_b = seq_b.block_store().latest_block_meta().await.unwrap().unwrap(); // assert_eq!(tip_b.id, tip_a.id); // assert_eq!(tip_b.hash, tip_a.hash); @@ -575,8 +665,8 @@ fn deposit_event_record( // let vault_id = vault_core::compute_vault_account_id(programs::vault().id(), recipient); // let bridge_id = system_accounts::bridge_account_id(); -// let state_b = seq_b.chain().lock().unwrap().head_state().clone(); -// let state_a = seq_a.chain().lock().unwrap().head_state().clone(); +// let state_b = seq_b.chain().lock().await.head_state().clone(); +// let state_a = seq_a.chain().lock().await.head_state().clone(); // for account in [vault_id, bridge_id, recipient] { // assert_eq!( // state_b.get_account_by_id(account).balance, @@ -592,7 +682,7 @@ fn deposit_event_record( // let produced = seq_b // .block_store() -// .get_block_at_id(tip_b.id + 1) +// .block_at_id(tip_b.id + 1) // .unwrap() // .expect("produced block present"); // assert!( @@ -631,7 +721,7 @@ fn deposit_event_record( // // Sequencer A produces a single withdraw block; treat its chain as the channel. // let config_a = bridge_funded_config(); // let (mut seq_a, mempool_a) = -// SequencerCoreWithMockClients::start_from_config(config_a.clone()).await; +// start_sequencer(config_a.clone()).await; // let withdraw_tx = build_public_withdraw_tx( // recipient, // 0, @@ -656,20 +746,20 @@ fn deposit_event_record( // "producing a withdraw must count it as unseen" // ); -// let messages = channel_from_store(seq_a.block_store(), 10); +// let messages = channel_from_store(seq_a.block_store(), 10).await; // let tip_slot = messages.last().unwrap().1; // let channel_id = config_a.bedrock_config.channel_id; // // Sequencer B reconstructs A's chain from a fresh store. // let config_b = bridge_funded_config(); -// let (store_b, chain_b) = fresh_store_and_chain(&config_b); +// let (store_b, chain_b) = fresh_store_and_chain(&config_b).await; // let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages); // SequencerCore::::verify_and_reconstruct(&mock_b, &store_b, &chain_b, // true) .await // .expect("reconstruct"); // assert!( -// !store_b.dbio().consume_unseen_withdraw_count(key).unwrap(), +// !dbio_b.consume_unseen_withdraw_count(key).unwrap(), // "reconstruction must not leave a phantom unseen-withdraw count" // ); // } @@ -687,8 +777,7 @@ async fn reconstruction_reconciles_already_finished_deposit() { // Sequencer A: a single block that fully processes the bridge deposit. let config_a = bridge_funded_config(); - let (mut seq_a, mempool_a) = - SequencerCoreWithMockClients::start_from_config(config_a.clone()).await; + let (mut seq_a, mempool_a) = start_sequencer(config_a.clone()).await; let deposit_record = deposit_event_record(deposit_op_id, deposit_amount, recipient); let deposit_tx = crate::build_bridge_deposit_tx_from_event(&deposit_record).expect("build deposit tx"); @@ -698,7 +787,7 @@ async fn reconstruction_reconciles_already_finished_deposit() { .unwrap(); seq_a.run_production_turn().await.unwrap(); - let messages = channel_from_store(seq_a.block_store(), 10); + let messages = channel_from_store(seq_a.block_store(), 10).await; let tip_slot = messages.last().unwrap().1; let channel_id = config_a.bedrock_config.channel_id; @@ -706,11 +795,14 @@ async fn reconstruction_reconciles_already_finished_deposit() { // pre-seeded, as the cold-start backfill would when it re-observes this // already-finalized deposit. let config_b = bridge_funded_config(); - let (store_b, chain_b) = fresh_store_and_chain(&config_b); + let (store_b, chain_b) = fresh_store_and_chain(&config_b).await; assert!( store_b - .dbio() - .add_pending_deposit_event(deposit_record.clone()) + .storage_ref() + .ask(AddPendingDepositEvent { + event: deposit_record.clone() + }) + .await .unwrap() ); @@ -724,7 +816,7 @@ async fn reconstruction_reconciles_already_finished_deposit() { assert_eq!( chain_b .lock() - .unwrap() + .await .head_state() .get_account_by_id(vault_id) .balance, @@ -736,14 +828,15 @@ async fn reconstruction_reconciles_already_finished_deposit() { // dropped the pending record backfill had re-delivered โ€” so the production // drain sees the deposit as minted and never re-emits it. assert!( - crate::deposit_already_minted( - chain_b.lock().unwrap().head_state(), - HashType(deposit_op_id) - ), + crate::deposit_already_minted(chain_b.lock().await.head_state(), HashType(deposit_op_id)), "the reconstructed deposit's receipt marks it minted" ); assert!( - store_b.get_pending_deposit_events().unwrap().is_empty(), + store_b + .get_pending_deposit_events() + .await + .unwrap() + .is_empty(), "reconstruction drops the finalized deposit's pending record" ); } @@ -761,29 +854,38 @@ async fn reconstructed_delivery_settles_its_pending_record() { // Sequencer A produces the block that carries the delivery. let config_a = cross_zone_test_config(); - let (mut seq_a, _mempool_a) = - SequencerCoreWithMockClients::start_from_config(config_a.clone()).await; + let (mut seq_a, _mempool_a) = start_sequencer(config_a.clone()).await; seq_a .block_store() - .dbio() - .add_pending_cross_zone_dispatches(vec![record.clone()]) + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: vec![record.clone()], + }) + .await .unwrap(); seq_a.run_production_turn().await.unwrap(); - let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap(); - let messages = channel_from_store(seq_a.block_store(), 10); + let tip_a = seq_a + .block_store() + .latest_block_meta() + .await + .unwrap() + .unwrap(); + let messages = channel_from_store(seq_a.block_store(), 10).await; let tip_slot = messages.last().unwrap().1; let channel_id = config_a.bedrock_config.channel_id; // Sequencer B holds the same record, as its own watcher would after reading // the peer block, and reconstructs A's chain from a fresh store. - let (mut seq_b, _mempool_b) = - SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await; + let (mut seq_b, _mempool_b) = start_sequencer(cross_zone_test_config()).await; assert_eq!( seq_b .block_store() - .dbio() - .add_pending_cross_zone_dispatches(vec![record]) + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: vec![record], + }) + .await .unwrap(), 1 ); @@ -798,14 +900,19 @@ async fn reconstructed_delivery_settles_its_pending_record() { .await .expect("reconstruct"); - let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap(); + let tip_b = seq_b + .block_store() + .latest_block_meta() + .await + .unwrap() + .unwrap(); assert_eq!(tip_b.id, tip_a.id); assert_eq!(tip_b.hash, tip_a.hash); assert!( seq_b - .block_store() - .dbio() - .get_pending_cross_zone_dispatches() + .store + .pending_cross_zone_dispatches() + .await .unwrap() .is_empty(), "reconstruction must settle the record of a delivery it replayed" @@ -814,14 +921,17 @@ async fn reconstructed_delivery_settles_its_pending_record() { // The delivery landed exactly once, and the next turn does not re-emit it. let record_id = ping_record_pda(programs::ping_receiver().id()); assert_eq!( - seq_b.with_state(|state| state.get_account_by_id(record_id).data.into_inner()), + seq_b + .with_state(|state| state.get_account_by_id(record_id).data.into_inner()) + .await, payload, "the reconstructed delivery must reach its target program" ); seq_b.run_production_turn().await.unwrap(); let produced = seq_b .block_store() - .get_block_at_id(tip_b.id + 1) + .block_at_id(tip_b.id + 1) + .await .unwrap() .expect("produced block present"); assert!( @@ -840,24 +950,27 @@ async fn a_verified_own_block_settles_its_delivery_records() { let record = dispatch_record(37, ping_payload(b"verified")); let key = record.message_key; - let (mut seq, _mempool) = - SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await; + let (mut seq, _mempool) = start_sequencer(cross_zone_test_config()).await; seq.block_store() - .dbio() - .add_pending_cross_zone_dispatches(vec![record]) + .storage_ref() + .ask(AddPendingCrossZoneDispatches { + dispatches: vec![record], + }) + .await .unwrap(); let block_id = seq.run_production_turn().await.unwrap(); let block = seq .block_store() - .get_block_at_id(block_id) + .block_at_id(block_id) + .await .unwrap() .unwrap(); assert_eq!(dispatches_in(&block), vec![key]); assert_eq!( - seq.block_store() - .dbio() - .get_pending_cross_zone_dispatches() + seq.store + .pending_cross_zone_dispatches() + .await .unwrap() .len(), 1, @@ -865,7 +978,7 @@ async fn a_verified_own_block_settles_its_delivery_records() { ); // The channel serves our own chain back, tip included. - let messages = channel_from_store(seq.block_store(), 10); + let messages = channel_from_store(seq.block_store(), 10).await; let tip_slot = messages.last().unwrap().1; let mock = MockBlockPublisher::with_canned_channel( seq.sequencer_config.bedrock_config.channel_id, @@ -879,9 +992,9 @@ async fn a_verified_own_block_settles_its_delivery_records() { .expect("reconstruct"); assert!( - seq.block_store() - .dbio() - .get_pending_cross_zone_dispatches() + seq.store + .pending_cross_zone_dispatches() + .await .unwrap() .is_empty(), "a delivery the channel confirms must not leave a record nothing can remove" @@ -895,19 +1008,28 @@ async fn committed_local_against_missing_channel_fails_without_anchor() { // anchor). Restarting it against a wiped/missing channel must still fail, // driven by the committed-blocks invariant rather than an anchor probe. let config = setup_sequencer_config(); - { - let (mut seq, _handle) = - SequencerCoreWithMockClients::start_from_config(config.clone()).await; + let storage_weak = { + let (mut seq, _handle) = start_sequencer(config.clone()).await; seq.run_production_turn().await.unwrap(); seq.run_production_turn().await.unwrap(); - assert!(seq.block_store().latest_block_meta().unwrap().unwrap().id > 1); - } // drop releases the store so we can reopen it + assert!( + seq.block_store() + .latest_block_meta() + .await + .unwrap() + .unwrap() + .id + > 1 + ); + seq.block_store().storage_ref().downgrade() + }; + storage_weak.wait_for_shutdown_with_result(|_| ()).await; // Reopen: blocks beyond genesis, no anchor. `is_fresh_start = false` stands in // for a checkpoint persisted by a prior sync (the mock never emits one). - let (store, chain) = fresh_store_and_chain(&config); - assert!(store.get_zone_anchor().unwrap().is_none()); - assert!(store.latest_block_meta().unwrap().unwrap().id > 1); + let (store, chain) = fresh_store_and_chain(&config).await; + assert!(store.get_zone_anchor().await.unwrap().is_none()); + assert!(store.latest_block_meta().await.unwrap().unwrap().id > 1); // The channel is gone: no tip, no messages. let mock = diff --git a/lez/sequencer/service/Cargo.toml b/lez/sequencer/service/Cargo.toml index c8edaec26..7fe6f4379 100644 --- a/lez/sequencer/service/Cargo.toml +++ b/lez/sequencer/service/Cargo.toml @@ -12,6 +12,7 @@ workspace = true sequencer_core = { workspace = true, features = ["testnet"] } sequencer_executor_actor.workspace = true sequencer_rpc_server_actor.workspace = true +sequencer_storage_actor.workspace = true lee = { workspace = true, optional = true } programs = { workspace = true, optional = true } system_accounts = { workspace = true, optional = true } diff --git a/lez/sequencer/service/src/lib.rs b/lez/sequencer/service/src/lib.rs index e6b819c09..537e2ff2d 100644 --- a/lez/sequencer/service/src/lib.rs +++ b/lez/sequencer/service/src/lib.rs @@ -9,6 +9,7 @@ pub use sequencer_core::config::*; use sequencer_core::load_or_create_signing_key; use sequencer_executor_actor::ExecutorActor; use sequencer_rpc_server_actor::RpcServerActor; +use sequencer_storage_actor::StorageActor; use tokio::select; use crate::actor_handle::ActorHandle; @@ -29,6 +30,7 @@ pub struct SequencerHandle { scheduler: ActorHandle, rpc_server: ActorHandle, executor: ActorHandle>, + storage: ActorHandle, addr: SocketAddr, /// Held for its lifetime: dropping it stops the gossip drive task. /// `None` when gossip is unconfigured. @@ -40,6 +42,7 @@ impl SequencerHandle { scheduler: ActorHandle, rpc_server: ActorHandle, executor: ActorHandle>, + storage: ActorHandle, addr: SocketAddr, gossip: Option, ) -> Self { @@ -47,6 +50,7 @@ impl SequencerHandle { scheduler, rpc_server, executor, + storage, addr, gossip, } @@ -59,6 +63,7 @@ impl SequencerHandle { scheduler, rpc_server, executor, + storage, addr: _, gossip: _, } = self; @@ -67,6 +72,7 @@ impl SequencerHandle { scheduler.shutdown().await; rpc_server.shutdown().await; executor.shutdown().await; + storage.shutdown().await; } /// Wait for any of the sequencer tasks to fail and return the error. @@ -79,6 +85,7 @@ impl SequencerHandle { executor, rpc_server, scheduler, + storage, addr: _, gossip: _, } = self; @@ -93,6 +100,9 @@ impl SequencerHandle { Err(err) = scheduler.failed() => { Err(err) } + Err(err) = storage.failed() => { + Err(err) + } } } @@ -106,11 +116,15 @@ impl SequencerHandle { executor, rpc_server, scheduler, + storage, addr: _, gossip: _, } = self; - executor.is_healthy() && rpc_server.is_healthy() && scheduler.is_healthy() + executor.is_healthy() + && rpc_server.is_healthy() + && scheduler.is_healthy() + && storage.is_healthy() } #[must_use] @@ -132,19 +146,21 @@ pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result None, Some(gossip_config) => { @@ -171,12 +187,13 @@ pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result Result DbResult { - let db_opts = Options::default(); - Self::open_inner(path, &db_opts) - } - - pub fn create(path: &Path, genesis_block: &Block, genesis_state: &V03State) -> DbResult { + /// Opens the database at `path`, creating an empty one when there is none. + /// + /// An empty one holds no chain: every read of the first or last block fails + /// until [`Self::write_genesis`] seeds it. + pub fn open_or_create(path: &Path) -> DbResult { let mut db_opts = Options::default(); db_opts.create_missing_column_families(true); db_opts.create_if_missing(true); - let dbio = Self::open_inner(path, &db_opts)?; - - let is_start_set = dbio.get_meta_is_first_block_set()?; - if !is_start_set { - let block_id = genesis_block.header.block_id; - // TODO: Shouldn't this be atomic (batched)? - dbio.put_meta_first_block_in_db(genesis_block)?; - dbio.put_meta_is_first_block_set()?; - dbio.put_meta_last_block_in_db(block_id)?; - dbio.put_meta_last_finalized_block_id(None)?; - dbio.put_meta_latest_block_meta(&BlockMeta { - id: genesis_block.header.block_id, - hash: genesis_block.header.hash, - })?; - dbio.put_lee_state_in_db(genesis_state)?; - } - - Ok(dbio) + Self::open_inner(path, &db_opts) } /// Dump every key/value pair across all column families into a [`DbDump`]. Column families are @@ -497,22 +478,22 @@ impl RocksDBIO { // Meta - pub fn get_meta_first_block_in_db(&self) -> DbResult { - self.get::(()).map(|cell| cell.0) + /// The id of the chain's first block, or `None` when this database holds no + /// chain yet โ€” nothing has written a genesis into it. + pub fn get_meta_first_block_in_db(&self) -> DbResult> { + Ok(self.get_opt::(())?.map(|cell| cell.0)) } - pub fn get_meta_last_block_in_db(&self) -> DbResult { - self.get::(()).map(|cell| cell.0) + /// The id of the chain's last block, or `None` when this database holds no + /// chain yet. + pub fn get_meta_last_block_in_db(&self) -> DbResult> { + Ok(self.get_opt::(())?.map(|cell| cell.0)) } pub fn get_meta_is_first_block_set(&self) -> DbResult { Ok(self.get_opt::(())?.is_some()) } - pub fn put_lee_state_in_db(&self, state: &V03State) -> DbResult<()> { - self.put(&LEEStateCellRef(state), ()) - } - pub fn put_lee_state_in_db_batch( &self, state: &V03State, @@ -521,42 +502,6 @@ impl RocksDBIO { self.put_batch(&LEEStateCellRef(state), (), batch) } - pub fn put_meta_first_block_in_db(&self, block: &Block) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_owned()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize first block id".to_owned()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let mut batch = WriteBatch::default(); - self.put_block(block, true, &mut batch)?; - self.db.write(batch).map_err(|rerr| { - DbError::rocksdb_cast_message( - rerr, - Some("Failed to write first block in db".to_owned()), - ) - })?; - - Ok(()) - } - - pub fn put_meta_last_block_in_db(&self, block_id: u64) -> DbResult<()> { - self.put(&LastBlockCell(block_id), ()) - } - fn put_meta_last_block_in_db_batch( &self, block_id: u64, @@ -565,16 +510,13 @@ impl RocksDBIO { self.put_batch(&LastBlockCell(block_id), (), batch) } - pub fn put_meta_last_finalized_block_id(&self, block_id: Option) -> DbResult<()> { - self.put(&LastFinalizedBlockIdCell(block_id), ()) - } - - pub fn put_meta_is_first_block_set(&self) -> DbResult<()> { - self.put(&FirstBlockSetCell(true), ()) - } - - fn put_meta_latest_block_meta(&self, block_meta: &BlockMeta) -> DbResult<()> { - self.put(&LatestBlockMetaCellRef(block_meta), ()) + fn put_meta_first_block_in_db_batch( + &self, + block_id: u64, + batch: &mut WriteBatch, + ) -> DbResult<()> { + self.put_batch(&FirstBlockCell(block_id), (), batch)?; + self.put_batch(&FirstBlockSetCell(true), (), batch) } fn put_meta_latest_block_meta_batch( @@ -1243,22 +1185,6 @@ impl RocksDBIO { Ok(unmatched.is_empty()) } - pub fn put_block(&self, block: &Block, first: bool, batch: &mut WriteBatch) -> DbResult<()> { - if !first { - // A produced block is the new head tip by construction: pin the - // tip meta and drop any stale higher blocks a preceding reorg left - // behind (mirrors `store_followed_blocks`). - let last_curr_block = self.get_meta_last_block_in_db()?; - for stale_id in block.header.block_id.saturating_add(1)..=last_curr_block { - self.delete_block_payload(stale_id, batch)?; - } - self.put_meta_last_block_in_db_batch(block.header.block_id, batch)?; - self.put_meta_latest_block_meta_batch(&BlockMeta::from(block), batch)?; - } - - self.put_block_payload(block, batch) - } - /// Stages deletion of a block payload into `batch`. fn delete_block_payload(&self, block_id: u64, batch: &mut WriteBatch) -> DbResult<()> { let cf_block = self.block_column(); @@ -1311,8 +1237,10 @@ impl RocksDBIO { self.put_batch(&FinalBlockMetaCellRef(meta), (), batch) } - pub fn get_lee_state(&self) -> DbResult { - self.get::(()).map(|val| val.0) + /// The state after the last applied block, or `None` when this database + /// holds no chain yet. + pub fn get_lee_state(&self) -> DbResult> { + Ok(self.get_opt::(())?.map(|val| val.0)) } pub fn delete_block(&self, block_id: u64) -> DbResult<()> { @@ -1350,6 +1278,15 @@ impl RocksDBIO { for block in to_write.values() { self.put_block_payload(block, &mut batch)?; } + + // The lowest block written to a store that holds no chain starts one. + // Nothing else records where a chain begins, and without it every read + // that walks from the first block has no lower bound. + if let Some((first_id, _)) = to_write.first_key_value() + && self.get_meta_first_block_in_db()?.is_none() + { + self.put_meta_first_block_in_db_batch(*first_id, &mut batch)?; + } self.db.write(batch).map_err(|rerr| { DbError::rocksdb_cast_message( rerr, @@ -1460,7 +1397,9 @@ impl RocksDBIO { zone_anchor, } = *update; - let last_block_in_db = self.get_meta_last_block_in_db()?; + // 0 stands in for "no chain yet": nothing to sweep above the new tip, + // and any tip this update pins is a change. + let last_block_in_db = self.get_meta_last_block_in_db()?.unwrap_or(0); let mut batch = WriteBatch::default(); if let Some(bytes) = checkpoint { @@ -1506,6 +1445,15 @@ impl RocksDBIO { self.put_block_payload(block, &mut batch)?; } + // The lowest block written to a store that holds no chain starts one. + // Nothing else records where a chain begins, and without it every read + // that walks from the first block has no lower bound. + if let Some((first_id, _)) = to_write.first_key_value() + && self.get_meta_first_block_in_db()?.is_none() + { + self.put_meta_first_block_in_db_batch(*first_id, &mut batch)?; + } + let accepted_deposits = self.stage_pending_deposit_events( new_deposit_events, remove_deposit_records, @@ -1551,7 +1499,7 @@ impl RocksDBIO { Ok(outcome) } - pub fn get_all_blocks(&self) -> impl Iterator> { + pub fn get_all_blocks(&self) -> impl Iterator> + '_ { let cf_block = self.block_column(); self.db .iterator_cf(&cf_block, rocksdb::IteratorMode::Start) diff --git a/lez/storage/src/sequencer/tests.rs b/lez/storage/src/sequencer/tests.rs index 392ffff93..5a53dd24f 100644 --- a/lez/storage/src/sequencer/tests.rs +++ b/lez/storage/src/sequencer/tests.rs @@ -24,7 +24,11 @@ fn state_with_balance(balance: u128) -> V03State { fn dbio_with_genesis(path: &Path) -> (RocksDBIO, Block) { let genesis = produce_dummy_block(1, None, vec![]); - let dbio = RocksDBIO::create(path, &genesis, &state_with_balance(100)).unwrap(); + let dbio = RocksDBIO::open_or_create(path).unwrap(); + // The same write any block takes: the first one into an empty store starts + // its chain. + dbio.atomic_update(&genesis, &[], &state_with_balance(100), None) + .unwrap(); (dbio, genesis) } @@ -68,6 +72,7 @@ fn sorted_dispatches( fn stored_balance(dbio: &RocksDBIO) -> u128 { dbio.get_lee_state() .unwrap() + .expect("the store holds a chain") .get_account_by_id(marker_id()) .balance } @@ -487,7 +492,7 @@ fn peer_chain_tips_round_trip_and_are_kept_per_peer() { // On disk, not in memory: a watcher that re-anchored on restart would take // whatever block reached it first, which is the id the attack picks. drop(dbio); - let reopened = RocksDBIO::open(temp_dir.path()).unwrap(); + let reopened = RocksDBIO::open_or_create(temp_dir.path()).unwrap(); assert_eq!( reopened.get_cross_zone_peer_tip(peer_a).unwrap(), Some(advanced) @@ -586,7 +591,7 @@ fn dispatch_records_survive_a_reopen() { // On disk, not in memory: the records are what stand between the watcher's // durable read floor and a lost delivery across a restart. drop(dbio); - let reopened = RocksDBIO::open(temp_dir.path()).unwrap(); + let reopened = RocksDBIO::open_or_create(temp_dir.path()).unwrap(); assert_eq!( sorted_dispatches(reopened.get_pending_cross_zone_dispatches().unwrap()), sorted_dispatches(vec![first, second]) @@ -612,7 +617,7 @@ fn a_legacy_dispatch_blob_is_migrated_into_per_message_entries_on_open() { .unwrap(); drop(dbio); - let migrated = RocksDBIO::open(temp_dir.path()).unwrap(); + let migrated = RocksDBIO::open_or_create(temp_dir.path()).unwrap(); assert_eq!( sorted_dispatches(migrated.get_pending_cross_zone_dispatches().unwrap()), sorted_dispatches(records.clone()), @@ -650,7 +655,7 @@ fn a_legacy_dispatch_blob_is_migrated_into_per_message_entries_on_open() { .unwrap(); drop(migrated); - let cleaned = RocksDBIO::open(temp_dir.path()).unwrap(); + let cleaned = RocksDBIO::open_or_create(temp_dir.path()).unwrap(); assert!( cleaned .db @@ -687,7 +692,7 @@ fn a_legacy_blob_over_live_entries_folds_additively() { .unwrap(); drop(dbio); - let merged = RocksDBIO::open(temp_dir.path()).unwrap(); + let merged = RocksDBIO::open_or_create(temp_dir.path()).unwrap(); assert_eq!( sorted_dispatches(merged.get_pending_cross_zone_dispatches().unwrap()), sorted_dispatches(vec![ @@ -1158,3 +1163,47 @@ fn produced_block_below_disk_head_pins_meta_and_prunes() { assert_eq!(meta.hash, block2b.header.hash); assert_eq!(stored_balance(&dbio), 400); } + +/// A database nothing has written a genesis into answers "no chain yet" rather +/// than failing: every one of these was a hard `get` that errored on absence. +#[test] +fn an_unseeded_store_reports_no_chain() { + let dir = tempfile::tempdir().expect("temp dir"); + let dbio = RocksDBIO::open_or_create(dir.path()).expect("open"); + + assert_eq!(dbio.get_meta_first_block_in_db().unwrap(), None); + assert_eq!(dbio.get_meta_last_block_in_db().unwrap(), None); + assert!(dbio.get_lee_state().unwrap().is_none()); + assert!(dbio.latest_block_meta().unwrap().is_none()); + assert!(dbio.get_final_snapshot().unwrap().is_none()); + assert!(!dbio.get_meta_is_first_block_set().unwrap()); + assert!(dbio.get_block(1).unwrap().is_none()); +} + +/// The first block written to an empty store starts its chain โ€” the property +/// that lets a genesis go in as an ordinary block write. +#[test] +fn the_first_block_written_starts_the_chain() { + let dir = tempfile::tempdir().expect("temp dir"); + let dbio = RocksDBIO::open_or_create(dir.path()).expect("open"); + assert_eq!(dbio.get_meta_first_block_in_db().unwrap(), None); + + let genesis = produce_dummy_block(1, None, vec![]); + dbio.atomic_update(&genesis, &[], &state_with_balance(100), None) + .expect("seed"); + + assert_eq!(dbio.get_meta_first_block_in_db().unwrap(), Some(1)); + assert_eq!(dbio.get_meta_last_block_in_db().unwrap(), Some(1)); + assert!(dbio.get_meta_is_first_block_set().unwrap()); + assert_eq!( + dbio.get_block(1).unwrap().unwrap().header.hash, + genesis.header.hash + ); + + // A later block extends the chain rather than restarting it. + let second = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + dbio.atomic_update(&second, &[], &state_with_balance(100), None) + .expect("extend"); + assert_eq!(dbio.get_meta_first_block_in_db().unwrap(), Some(1)); + assert_eq!(dbio.get_meta_last_block_in_db().unwrap(), Some(2)); +} diff --git a/test_fixtures/Cargo.toml b/test_fixtures/Cargo.toml index 4d9982123..d90681683 100644 --- a/test_fixtures/Cargo.toml +++ b/test_fixtures/Cargo.toml @@ -18,6 +18,7 @@ sequencer_core = { workspace = true, features = ["default", "testnet"] } sequencer_service.workspace = true sequencer_stake_core.workspace = true sequencer_service_rpc = { workspace = true, features = ["client"] } +sequencer_storage_actor.workspace = true wallet.workspace = true programs.workspace = true @@ -28,6 +29,7 @@ env_logger.workspace = true futures.workspace = true hex.workspace = true jsonrpsee = { workspace = true, features = ["ws-client"] } +kameo.workspace = true log.workspace = true num-bigint.workspace = true serde.workspace = true diff --git a/test_fixtures/fixtures/prebuilt_sequencer_db.dump b/test_fixtures/fixtures/prebuilt_sequencer_db.dump index a29c488d4..8a5277963 100644 Binary files a/test_fixtures/fixtures/prebuilt_sequencer_db.dump and b/test_fixtures/fixtures/prebuilt_sequencer_db.dump differ diff --git a/test_fixtures/src/bin/regenerate_test_fixture.rs b/test_fixtures/src/bin/regenerate_test_fixture.rs index 52e91bca6..99ba06643 100644 --- a/test_fixtures/src/bin/regenerate_test_fixture.rs +++ b/test_fixtures/src/bin/regenerate_test_fixture.rs @@ -3,11 +3,14 @@ #![expect(clippy::print_stdout, reason = "It's normal in this small cli")] -use std::{path::Path, time::Duration}; +use std::path::Path; use anyhow::{Context as _, Result}; -use lee::PrivateKey; -use sequencer_core::block_store::SequencerStore; +use kameo::actor::Spawn as _; +use sequencer_storage_actor::{ + StorageActor, + protocol::{DeleteZoneCheckpoint, DumpDb, ResetAllBlocksToPending}, +}; use test_fixtures::{ config, setup::{ @@ -75,22 +78,28 @@ async fn generate_prebuilt_fixture(dest: &Path) -> Result<()> { // Shut down gracefully to release the rocksdb lock before reopening the store. drop(wallet); - drop(sequencer_handle); + sequencer_handle.shutdown().await; let db_path = temp_sequencer_dir .path() .join(format!("rocksdb-{}", config::bedrock_channel_id())); - let store = open_store_with_retry(&db_path) + let storage = + StorageActor::new(&db_path).context("Failed to reopen sequencer storage after shutdown")?; + let storage_ref = StorageActor::spawn(storage); + storage_ref + .ask(DeleteZoneCheckpoint) .await - .context("Failed to reopen sequencer store after shutdown")?; - store - .delete_zone_checkpoint() .context("Failed to strip zone-sdk checkpoint from fixture database")?; - store - .reset_all_blocks_to_pending() + storage_ref + .ask(ResetAllBlocksToPending) + .await .context("Failed to reset fixture blocks to pending")?; - let dump = store.dump().context("Failed to dump fixture database")?; - drop(store); + let dump = storage_ref + .ask(DumpDb) + .await + .context("Failed to dump fixture database")?; + storage_ref.stop_gracefully().await?; + storage_ref.wait_for_shutdown_with_result(|_| ()).await; if let Some(parent) = dest.parent() { std::fs::create_dir_all(parent) @@ -104,24 +113,3 @@ async fn generate_prebuilt_fixture(dest: &Path) -> Result<()> { Ok(()) } - -/// Reopen the store, retrying while the shut-down sequencer's aborted tasks are still releasing -/// their db handles (the process holds the file lock until then). Must `await` between attempts. -async fn open_store_with_retry(db_path: &Path) -> Result { - let signing_key = PrivateKey::try_new(config::SEQUENCER_SIGNING_KEY) - .expect("Fixed sequencer signing key must be valid"); - - let mut last_err = None; - for _ in 0..100 { - // Let the runtime drop the aborted sequencer tasks before each attempt. - tokio::time::sleep(Duration::from_millis(100)).await; - match SequencerStore::open_db(db_path, signing_key.clone()) { - Ok(store) => return Ok(store), - Err(err) => last_err = Some(err), - } - } - Err(anyhow::anyhow!( - "Failed to open sequencer store at {} after retries: {last_err:?}", - db_path.display() - )) -} diff --git a/test_fixtures/src/setup.rs b/test_fixtures/src/setup.rs index 6844d4b6a..b08d34001 100644 --- a/test_fixtures/src/setup.rs +++ b/test_fixtures/src/setup.rs @@ -7,12 +7,10 @@ use anyhow::{Context as _, Result, bail}; use indexer_service::{ChannelId, IndexerHandle}; use lee::{AccountId, PrivateKey, PublicKey}; use log::{debug, warn}; -use sequencer_core::{ - block_publisher::ED25519_SECRET_KEY_SIZE, - block_store::{DbDump, SequencerStore}, -}; +use sequencer_core::block_publisher::ED25519_SECRET_KEY_SIZE; use sequencer_service::{GenesisAction, SequencerHandle}; use sequencer_service_rpc::{SequencerClient, SequencerClientBuilder}; +use sequencer_storage_actor::{StorageActor, protocol::DbDump}; use tempfile::TempDir; use testcontainers::compose::DockerCompose; use wallet::{ @@ -149,15 +147,13 @@ impl SequencerSetup { genesis } else { let dump = load_prebuilt_dump()?; - // `SequencerCore::open_or_create_store` looks for the channel-suffixed - // db under its home, so the restore has to land on the same name. + // The sequencer looks for the channel-suffixed db under its home, + // so the restore has to land on the same name. let dst = home.join(format!("rocksdb-{channel_id}")); - let _store = SequencerStore::restore_db_from_dump( - &dst, - &dump, - lee::PrivateKey::try_new(config::SEQUENCER_SIGNING_KEY)?, - ) - .context("Failed to restore prebuilt sequencer database from dump")?; + // Dropped right away: this only writes the database, which the + // sequencer opens for itself below. + let _storage = StorageActor::restore_from_dump(&dst, &dump) + .context("Failed to restore prebuilt sequencer database from dump")?; // TODO: Technically not correct, we should reconstruct the genesis transactions // from the dump, but this crutch doesn't affect anything for now Vec::new()