logos-blockchain/logos-execution-zone
1
0
Fork 0
mirror of https://github.com/logos-blockchain/logos-execution-zone.git synced 2026-08-25 03:11:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(sequencer): actor architecture, phase 2: Storage Actor

Browse Source
This commit is contained in:
Daniil Polyakov
2026-08-20 00:37:23 +03:00
parent 9a7a71abf2
commit 7249f2f881
33 changed files with 4149 additions and 1349 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+100 -4
View File
@@ -1278,9 +1278,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
version = "1.11.1"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
dependencies = [
"serde",
]
@@ -2548,6 +2548,12 @@ dependencies = [
"serde_json",
]
[[package]]
name = "downcast"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"
[[package]]
name = "downcast-rs"
version = "1.2.1"
@@ -3139,6 +3145,15 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fragile"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8878864ba14bb86e818a412bfd6f18f9eabd4ec0f008a28e8f7eb61db532fcf9"
dependencies = [
"futures-core",
]
[[package]]
name = "fs_extra"
version = "1.3.0"
@@ -6875,6 +6890,32 @@ dependencies = [
"sha3 0.11.0",
]
[[package]]
name = "mockall"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a6ceddfe3ce334925e96bf420fdb2dcee5bed6c632a168ece622676dadeaf8a"
dependencies = [
"cfg-if",
"downcast",
"fragile",
"mockall_derive",
"predicates",
"predicates-tree",
]
[[package]]
name = "mockall_derive"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cfe16fbe8a314aeec0b861ac24e60b1e123e97634bab045475b9d6a18416fd8"
dependencies = [
"cfg-if",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "module-lattice"
version = "0.2.3"
@@ -7861,6 +7902,32 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "predicates"
version = "3.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe"
dependencies = [
"anstyle",
"predicates-core",
]
[[package]]
name = "predicates-core"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144"
[[package]]
name = "predicates-tree"
version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2"
dependencies = [
"predicates-core",
"termtree",
]
[[package]]
name = "prettyplease"
version = "0.2.37"
@@ -9491,7 +9558,7 @@ dependencies = [
"futures",
"hex",
"humantime-serde",
"itertools 0.14.0",
"kameo",
"key_protocol",
"lee",
"lee_core",
@@ -9509,6 +9576,7 @@ dependencies = [
"risc0-zkvm",
"sequencer_core_metrics",
"sequencer_stake_core",
"sequencer_storage_actor",
"serde",
"serde_json",
"storage",
@@ -9541,15 +9609,19 @@ dependencies = [
"bytesize",
"common",
"env_logger",
"futures",
"hex",
"kameo",
"lee",
"lee_core",
"log",
"mempool",
"mockall",
"num-bigint 0.4.6",
"sequencer_core",
"storage",
"sequencer_stake_core",
"sequencer_storage_actor",
"system_accounts",
"tempfile",
"test_programs",
"thiserror 2.0.18",
@@ -9605,6 +9677,7 @@ dependencies = [
"sequencer_executor_actor",
"sequencer_rpc_server_actor",
"sequencer_stake_core",
"sequencer_storage_actor",
"system_accounts",
"tokio",
"tokio-util",
@@ -9649,6 +9722,21 @@ dependencies = [
"sequencer_stake_core",
]
[[package]]
name = "sequencer_storage_actor"
version = "0.1.0"
dependencies = [
"common",
"kameo",
"lee",
"lee_core",
"mockall",
"storage",
"tempfile",
"thiserror 2.0.18",
"tokio",
]
[[package]]
name = "serde"
version = "1.0.228"
@@ -10470,6 +10558,12 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "termtree"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
[[package]]
name = "test-case"
version = "3.3.1"
@@ -10516,6 +10610,7 @@ dependencies = [
"hex",
"indexer_service",
"jsonrpsee",
"kameo",
"key_protocol",
"lee",
"lee_core",
@@ -10527,6 +10622,7 @@ dependencies = [
"sequencer_service",
"sequencer_service_rpc",
"sequencer_stake_core",
"sequencer_storage_actor",
"serde",
"serde_json",
"tempfile",
Cargo.toml
+3
View File
@@ -25,6 +25,7 @@ members = [
"lez/sequencer/actors/executor",
"lez/sequencer/actors/rpc_server",
"lez/sequencer/actors/rpc_server/metrics",
"lez/sequencer/actors/storage",
"lez/indexer/core",
"lez/indexer/service",
"lez/indexer/service/protocol",
@@ -90,6 +91,7 @@ sequencer_service_rpc = { path = "lez/sequencer/service/rpc" }
sequencer_executor_actor = { path = "lez/sequencer/actors/executor" }
sequencer_rpc_server_actor = { path = "lez/sequencer/actors/rpc_server" }
sequencer_rpc_server_actor_metrics = { path = "lez/sequencer/actors/rpc_server/metrics" }
sequencer_storage_actor = { path = "lez/sequencer/actors/storage" }
sequencer_service = { path = "lez/sequencer/service" }
indexer_core = { path = "lez/indexer/core" }
indexer_service = { path = "lez/indexer/service" }
@@ -200,6 +202,7 @@ tokio-retry = "0.3.0"
schemars = "1.2"
async-stream = "0.3.6"
strum = { version = "0.28.0", features = ["derive"] }
mockall = "0.15.0"
logos-blockchain-common-http-client = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "97d5e104fc20ce6e2fb404f8a9364afcb181a3e5" }
logos-blockchain-key-management-system-service = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "97d5e104fc20ce6e2fb404f8a9364afcb181a3e5" }
lez/chain_state/src/chain.rs
+47 -21
View File
@@ -1,6 +1,8 @@
//! Two-tier chain state: a reorg-able `head` the sequencer builds on, plus an
//! irreversible `final` tier.
use std::sync::Arc;
use common::block::Block;
use lee::V03State;
use log::warn;
@@ -26,11 +28,11 @@ pub struct HeadEntry {
/// Only the final tier stalls: an invalid `adopted` block just freezes the
/// head tip and self-heals via reorg or finalization.
pub struct ChainState {
final_state: V03State,
final_state: Arc<V03State>,
final_tip: Option<Tip>,
final_stall: Option<StallReason>,
head_state: V03State,
head_state: Arc<V03State>,
head_blocks: Vec<HeadEntry>,
}
@@ -44,8 +46,9 @@ impl ChainState {
/// State restored from a persisted final tier; head mirrors final.
#[must_use]
pub fn from_final(final_state: V03State, final_tip: Option<Tip>) -> Self {
let final_state = Arc::new(final_state);
Self {
head_state: final_state.clone(),
head_state: Arc::clone(&final_state),
final_state,
final_tip,
head_blocks: Vec::new(),
@@ -55,22 +58,36 @@ impl ChainState {
/// State the sequencer builds its next block on.
#[must_use]
pub const fn head_state(&self) -> &V03State {
pub fn head_state(&self) -> &V03State {
&self.head_state
}
/// A shared handle on the head state, for callers that need to own it.
#[must_use]
pub fn share_head_state(&self) -> Arc<V03State> {
Arc::clone(&self.head_state)
}
/// Mutable access to the head state. Bypasses the `head_blocks` invariant, so
/// it is meant for tests and low-level callers.
///
/// Copies the state while a handle from [`Self::share_head_state`] is alive.
#[must_use]
pub const fn head_state_mut(&mut self) -> &mut V03State {
&mut self.head_state
pub fn head_state_mut(&mut self) -> &mut V03State {
Arc::make_mut(&mut self.head_state)
}
#[must_use]
pub const fn final_state(&self) -> &V03State {
pub fn final_state(&self) -> &V03State {
&self.final_state
}
/// A shared handle on the final state, for callers that need to own it.
#[must_use]
pub fn share_final_state(&self) -> Arc<V03State> {
Arc::clone(&self.final_state)
}
/// Parent the next produced block must chain on.
#[must_use]
pub fn head_tip(&self) -> Option<Tip> {
@@ -139,12 +156,12 @@ impl ChainState {
.position(|entry| entry.block.header.block_id >= block.header.block_id);
let (mut scratch, tip) = match reorg_at {
// continue from the tip
None => (self.head_state.clone(), self.head_tip()),
None => (Arc::clone(&self.head_state), self.head_tip()),
// reorg upto `idx`
Some(idx) => self.replay_head_prefix(idx),
};
match apply_block(tip.as_ref(), block, &mut scratch) {
match apply_block(tip.as_ref(), block, Arc::make_mut(&mut scratch)) {
Ok(()) => {
// now that `apply_block` succeeded, actually reorg the head
if let Some(idx) = reorg_at {
@@ -213,7 +230,11 @@ impl ChainState {
/// persisted — that would need a sidecar `block_id -> MsgId` cell); later
/// orphan/finalize events correlate by block hash.
pub fn restore_head_block(&mut self, block: Block) -> Result<(), BlockIngestError> {
apply_block(self.head_tip().as_ref(), &block, &mut self.head_state)?;
apply_block(
self.head_tip().as_ref(),
&block,
Arc::make_mut(&mut self.head_state),
)?;
let this_msg = MsgId::from(block.header.hash.0);
self.head_blocks.push(HeadEntry { this_msg, block });
Ok(())
@@ -250,8 +271,12 @@ impl ChainState {
fn finalize_through(&mut self, idx: usize) {
let finalized: Vec<HeadEntry> = self.head_blocks.drain(0..=idx).collect();
for entry in finalized {
apply_block(self.final_tip.as_ref(), &entry.block, &mut self.final_state)
.expect("validated head block must apply to the final tier");
apply_block(
self.final_tip.as_ref(),
&entry.block,
Arc::make_mut(&mut self.final_state),
)
.expect("validated head block must apply to the final tier");
self.final_tip = Some(Tip::from(&entry.block));
}
self.final_stall = None;
@@ -270,8 +295,8 @@ impl ChainState {
return AcceptOutcome::AlreadyApplied;
}
let mut scratch = self.final_state.clone();
match apply_block(self.final_tip.as_ref(), block, &mut scratch) {
let mut scratch = Arc::clone(&self.final_state);
match apply_block(self.final_tip.as_ref(), block, Arc::make_mut(&mut scratch)) {
Ok(()) => {
self.final_state = scratch;
self.final_tip = Some(Tip::from(block));
@@ -281,7 +306,7 @@ impl ChainState {
// orders orphans before their finalized replacement), so its
// txs are back in the caller's mempool.
self.head_blocks.clear();
self.head_state = self.final_state.clone();
self.head_state = Arc::clone(&self.final_state);
AcceptOutcome::Applied
}
Err(err) => {
@@ -297,11 +322,11 @@ impl ChainState {
}
/// State and tip after replaying `head_blocks[..count]` on the final tier.
fn replay_head_prefix(&self, count: usize) -> (V03State, Option<Tip>) {
let mut state = self.final_state.clone();
fn replay_head_prefix(&self, count: usize) -> (Arc<V03State>, Option<Tip>) {
let mut state = Arc::clone(&self.final_state);
let mut tip = self.final_tip.clone();
for entry in &self.head_blocks[..count] {
apply_block(tip.as_ref(), &entry.block, &mut state)
apply_block(tip.as_ref(), &entry.block, Arc::make_mut(&mut state))
.expect("validated head blocks must replay");
tip = Some(Tip::from(&entry.block));
}
@@ -337,14 +362,15 @@ mod tests {
/// `head_state` equals `final_state` replayed through `head_blocks`.
fn assert_head_matches_replay(chain: &ChainState) {
let mut state = chain.final_state.clone();
let mut state = Arc::clone(&chain.final_state);
let mut tip = chain.final_tip.clone();
for entry in &chain.head_blocks {
apply_block(tip.as_ref(), &entry.block, &mut state).expect("head blocks must replay");
apply_block(tip.as_ref(), &entry.block, Arc::make_mut(&mut state))
.expect("head blocks must replay");
tip = Some(Tip::from(&entry.block));
}
assert_eq!(
borsh::to_vec(&state).expect("state serializes"),
borsh::to_vec(state.as_ref()).expect("state serializes"),
borsh::to_vec(chain.head_state()).expect("state serializes"),
"head_state must equal final_state replayed through head_blocks"
);
lez/sequencer/actors/executor/Cargo.toml
+7 -2
View File
@@ -2,7 +2,7 @@
name = "sequencer_executor_actor"
version = "0.1.0"
edition = "2024"
license = { workspace = true }
license.workspace = true
[lints]
workspace = true
@@ -12,11 +12,12 @@ sequencer_core.workspace = true
common.workspace = true
lee_core.workspace = true
mempool.workspace = true
storage.workspace = true
sequencer_storage_actor.workspace = true
kameo.workspace = true
tokio.workspace = true
tokio-util.workspace = true
futures.workspace = true
log.workspace = true
anyhow.workspace = true
thiserror.workspace = true
@@ -25,9 +26,13 @@ hex.workspace = true
[dev-dependencies]
lee.workspace = true
sequencer_core = { workspace = true, features = ["mock"] }
sequencer_storage_actor = { workspace = true, features = ["mock"] }
system_accounts.workspace = true
sequencer_stake_core.workspace = true
test_programs.workspace = true
env_logger.workspace = true
mockall.workspace = true
tempfile.workspace = true
bytesize.workspace = true
num-bigint.workspace = true
lez/sequencer/actors/executor/src/actor.rs
+93 -48
View File
@@ -1,4 +1,5 @@
use common::{block::Block, transaction::LeeTransaction};
use futures::{TryFutureExt as _, TryStreamExt as _, future::ready, stream::FuturesOrdered};
use kameo::{
Actor,
actor::{ActorRef, WeakActorRef},
@@ -18,6 +19,7 @@ use sequencer_core::{
config::SequencerConfig,
task_group::TaskGroup,
};
use sequencer_storage_actor::{StorageActor, StorageActorTrait};
use tokio::select;
use tokio_util::sync::CancellationToken;
@@ -32,9 +34,10 @@ use crate::{
};
// TODO: Remove `BP` once this part is moved to a separate actor
pub struct ExecutorActor<BP: BlockPublisherTrait> {
sequencer: SequencerCore<BP>,
pub struct ExecutorActor<BP: BlockPublisherTrait, S: StorageActorTrait = StorageActor> {
mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>,
sequencer: SequencerCore<BP, S>,
storage_ref: ActorRef<S>,
// --- TODO: Remove these fields below ---
/// Cancelled when the publisher's drive task terminates (e.g. a panicked
@@ -46,16 +49,20 @@ pub struct ExecutorActor<BP: BlockPublisherTrait> {
background_tasks: Vec<TaskGroup>,
}
impl<BP: BlockPublisherTrait> ExecutorActor<BP> {
pub async fn new(config: SequencerConfig) -> Self {
let (sequencer, mempool_handle) = SequencerCore::<BP>::start_from_config(config).await;
impl<BP: BlockPublisherTrait, S: StorageActorTrait> ExecutorActor<BP, S> {
pub async fn new(config: SequencerConfig, storage_ref: ActorRef<S>) -> Self {
// TODO: Leave storage_ref as a top-level field only in `ExecutorActor`,
// while moving `SequencerCore` code into this actor.
let (sequencer, mempool_handle) =
SequencerCore::<BP, S>::start_from_config(config, storage_ref.clone()).await;
let driver_cancellation = sequencer.block_publisher().driver_cancellation();
let background_tasks = sequencer.background_tasks();
Self {
sequencer,
mempool_handle,
sequencer,
storage_ref,
driver_cancellation,
background_tasks,
}
@@ -69,7 +76,9 @@ impl<BP: BlockPublisherTrait> ExecutorActor<BP> {
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Actor for ExecutorActor<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait> Actor
for ExecutorActor<BP, S>
{
type Args = Self;
type Error = Error;
@@ -116,7 +125,9 @@ impl<BP: BlockPublisherTrait + Send + 'static> Actor for ExecutorActor<BP> {
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Message<ProduceBlock> for ExecutorActor<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait> Message<ProduceBlock>
for ExecutorActor<BP, S>
{
type Reply = Result<()>;
async fn handle(
@@ -134,11 +145,11 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<ProduceBlock> for Executo
// channel would carry two chains from there and nothing resolves that.
// The head rewinds under us when the sdk orphans our own unfinalized
// blocks, and recovers once they finalize, so this is a wait.
if let Some(high_water) = self.sequencer.rewound_below_published() {
if let Some(high_water) = self.sequencer.rewound_below_published().await {
warn!(
"Skipping turn: head rewound to {} but block {high_water} is already inscribed; \
waiting for the channel to restore it",
self.sequencer.next_block_height().saturating_sub(1),
self.sequencer.next_block_height().await.saturating_sub(1),
);
return Ok(());
}
@@ -161,7 +172,9 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<ProduceBlock> for Executo
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Message<Transaction> for ExecutorActor<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait> Message<Transaction>
for ExecutorActor<BP, S>
{
type Reply = Result<()>;
async fn handle(
@@ -175,7 +188,9 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<Transaction> for Executor
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Message<GetBlock> for ExecutorActor<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait> Message<GetBlock>
for ExecutorActor<BP, S>
{
type Reply = Result<Option<Block>>;
async fn handle(
@@ -183,14 +198,16 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<GetBlock> for ExecutorAct
GetBlock { block_id }: GetBlock,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.sequencer
.block_store()
.get_block_at_id(block_id)
self.storage_ref
.ask(sequencer_storage_actor::protocol::GetBlock { block_id })
.await
.map_err(Into::into)
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Message<GetBlockRange> for ExecutorActor<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait> Message<GetBlockRange>
for ExecutorActor<BP, S>
{
type Reply = Result<Vec<Block>>;
async fn handle(
@@ -199,18 +216,23 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<GetBlockRange> for Execut
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
range
.map_while(|block_id| {
self.sequencer
.block_store()
.get_block_at_id(block_id)
.map(|block_id| {
self.storage_ref
.ask(sequencer_storage_actor::protocol::GetBlock { block_id })
.into_future()
.map_err(Into::into)
.transpose()
})
.collect::<Result<Vec<_>>>()
.collect::<FuturesOrdered<_>>()
.try_take_while(|block_opt| ready(Ok(block_opt.is_some())))
.try_filter_map(|block_opt| ready(Ok(block_opt)))
.try_collect()
.await
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Message<GetLastBlockId> for ExecutorActor<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait> Message<GetLastBlockId>
for ExecutorActor<BP, S>
{
type Reply = Result<BlockId>;
async fn handle(
@@ -218,11 +240,13 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<GetLastBlockId> for Execu
GetLastBlockId: GetLastBlockId,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
Ok(self.sequencer.chain_height())
Ok(self.sequencer.chain_height().await)
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Message<GetAccountBalance> for ExecutorActor<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait>
Message<GetAccountBalance> for ExecutorActor<BP, S>
{
type Reply = Balance;
async fn handle(
@@ -232,10 +256,13 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<GetAccountBalance> for Ex
) -> Self::Reply {
self.sequencer
.with_state(|state| state.get_account_by_id(account_id).balance)
.await
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Message<GetTransaction> for ExecutorActor<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait> Message<GetTransaction>
for ExecutorActor<BP, S>
{
type Reply = Option<(LeeTransaction, BlockId)>;
async fn handle(
@@ -246,10 +273,13 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<GetTransaction> for Execu
self.sequencer
.block_store()
.get_transaction_by_hash(tx_hash)
.await
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Message<GetAccountNonces> for ExecutorActor<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait>
Message<GetAccountNonces> for ExecutorActor<BP, S>
{
type Reply = Vec<Nonce>;
async fn handle(
@@ -257,16 +287,20 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<GetAccountNonces> for Exe
GetAccountNonces { account_ids }: GetAccountNonces,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.sequencer.with_state(|state| {
account_ids
.into_iter()
.map(|account_id| state.get_account_by_id(account_id).nonce)
.collect()
})
self.sequencer
.with_state(|state| {
account_ids
.into_iter()
.map(|account_id| state.get_account_by_id(account_id).nonce)
.collect()
})
.await
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Message<GetProofsAndRoot> for ExecutorActor<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait>
Message<GetProofsAndRoot> for ExecutorActor<BP, S>
{
type Reply = (
Vec<Option<lee_core::MembershipProof>>,
lee_core::CommitmentSetDigest,
@@ -277,17 +311,21 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<GetProofsAndRoot> for Exe
GetProofsAndRoot { commitments }: GetProofsAndRoot,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.sequencer.with_state(|state| {
let proofs = commitments
.iter()
.map(|commitment| state.get_proof_for_commitment(commitment))
.collect();
(proofs, state.commitment_root())
})
self.sequencer
.with_state(|state| {
let proofs = commitments
.iter()
.map(|commitment| state.get_proof_for_commitment(commitment))
.collect();
(proofs, state.commitment_root())
})
.await
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Message<GetAccount> for ExecutorActor<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait> Message<GetAccount>
for ExecutorActor<BP, S>
{
type Reply = GetAccountReply;
async fn handle(
@@ -298,12 +336,15 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<GetAccount> for ExecutorA
GetAccountReply {
account: self
.sequencer
.with_state(|state| state.get_account_by_id(account_id)),
.with_state(|state| state.get_account_by_id(account_id))
.await,
}
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Message<GetChannelId> for ExecutorActor<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait> Message<GetChannelId>
for ExecutorActor<BP, S>
{
type Reply = GetChannelIdReply;
async fn handle(
@@ -317,8 +358,8 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<GetChannelId> for Executo
}
}
impl<BP: BlockPublisherTrait + Send + 'static> Message<GetCrossZoneDeadLetters>
for ExecutorActor<BP>
impl<BP: BlockPublisherTrait + Send + Sync + 'static, S: StorageActorTrait>
Message<GetCrossZoneDeadLetters> for ExecutorActor<BP, S>
{
type Reply = Result<GetCrossZoneDeadLettersReply>;
@@ -327,10 +368,14 @@ impl<BP: BlockPublisherTrait + Send + 'static> Message<GetCrossZoneDeadLetters>
GetCrossZoneDeadLetters: GetCrossZoneDeadLetters,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
let (total_retired, retained) = self.sequencer.cross_zone_dead_letters()?;
let (total_retired, retained) = self
.sequencer
.cross_zone_dead_letters()
.await
.map_err(Error::CrossZoneDeadLettersUnavailable)?;
Ok(GetCrossZoneDeadLettersReply {
total_retired,
retained,
retained: retained.into_iter().collect(),
})
}
}
lez/sequencer/actors/executor/src/error.rs
+19 -4
View File
@@ -9,9 +9,24 @@ pub enum Error {
#[error("The mempool is full")]
MempoolIsFull,
#[error("Storage error")]
StorageError(#[from] storage::error::DbError),
#[error("Storage request failed")]
StorageRequestFailed(
#[source] kameo::error::SendError<NoMatter, sequencer_storage_actor::error::Error>,
),
#[error(transparent)]
BlockProductionFailed(anyhow::Error),
#[error("Block production failed")]
BlockProductionFailed(#[source] anyhow::Error),
#[error("Failed to read the cross-zone dead letter")]
CrossZoneDeadLettersUnavailable(#[source] anyhow::Error),
}
/// A dummy struct replacing message type in [`kameo::error::SendError`]
/// as we don't want to expose the message type in the public API.
pub struct NoMatter;
impl<M> From<kameo::error::SendError<M, sequencer_storage_actor::error::Error>> for Error {
fn from(err: kameo::error::SendError<M, sequencer_storage_actor::error::Error>) -> Self {
Self::StorageRequestFailed(err.map_msg(|_| NoMatter))
}
}
lez/sequencer/actors/executor/src/protocol.rs
+1 -2
View File
@@ -6,7 +6,6 @@ use lee_core::{
BlockId, Commitment,
account::{Account, AccountId},
};
use sequencer_core::DeadLetterDispatchRecord;
#[derive(Copy, Clone)]
pub struct ProduceBlock;
@@ -62,5 +61,5 @@ pub struct GetCrossZoneDeadLetters;
#[derive(Reply)]
pub struct GetCrossZoneDeadLettersReply {
pub total_retired: u64,
pub retained: Vec<DeadLetterDispatchRecord>,
pub retained: Vec<sequencer_storage_actor::protocol::DeadLetterDispatchRecord>,
}
lez/sequencer/actors/executor/src/tests.rs
+112 -6
View File
@@ -1,23 +1,31 @@
use std::collections::BTreeMap;
use anyhow::Result;
use bytesize::ByteSize;
use common::transaction::LeeTransaction;
use common::{
HashType,
block::{BedrockStatus, Block, BlockBody, BlockHeader, BlockMeta},
transaction::LeeTransaction,
};
use kameo::{actor::Spawn as _, error::SendError};
use lee::{
AccountId, PrivateKey, PublicKey, PublicTransaction,
Account, AccountId, PrivateKey, PublicKey, PublicTransaction, Signature, V03State,
public_transaction::{Message, WitnessSet},
};
use mockall::predicate::{always, eq};
use num_bigint::BigUint;
use sequencer_core::{
config::{BedrockConfig, SequencerConfig},
mock::MockBlockPublisher,
};
use sequencer_storage_actor::mock::MockStorageActor;
use tempfile::TempDir;
use tokio::test;
use crate::{ExecutorActor, protocol};
fn sequencer_config() -> (SequencerConfig, tempfile::TempDir) {
let home = tempfile::tempdir().expect("Failed to create tmp home dir");
fn sequencer_config() -> (SequencerConfig, TempDir) {
let home = TempDir::new().expect("Failed to create temporary home directory");
let config = SequencerConfig {
home: home.path().to_path_buf(),
max_num_tx_in_block: 10,
@@ -62,13 +70,111 @@ fn test_transaction() -> LeeTransaction {
PublicTransaction::new(message, witness_set).into()
}
fn prepare_mock_storage_with_empty_genesis() -> MockStorageActor {
let genesis_block_meta = BlockMeta {
id: 1,
hash: HashType::default(),
};
let genesis_block = Block {
header: BlockHeader {
block_id: genesis_block_meta.id,
prev_block_hash: HashType::default(),
hash: genesis_block_meta.hash,
timestamp: 0,
signature: Signature { value: [0; 64] },
},
body: BlockBody {
transactions: vec![],
},
bedrock_status: BedrockStatus::Pending,
};
let state = V03State::new().with_public_accounts([(
system_accounts::sequencer_stake_config_account_id(),
Account {
data: sequencer_stake_core::SequencerStakeConfig {
minimum_sequencer_stake: 0,
entries: BTreeMap::new(),
}
.to_bytes()
.try_into()
.expect("Sequencer stake config must fit into Data"),
..Account::default()
},
)]);
let mut mock_storage = MockStorageActor::new();
mock_storage
.expect_handle_get_first_block_id()
.returning(|_, _| Ok(Some(1)));
mock_storage
.expect_handle_get_last_block_id()
.returning(|_, _| Ok(Some(1)));
let genesis_block_clone = genesis_block.clone();
mock_storage
.expect_handle_get_block()
.with(
eq(sequencer_storage_actor::protocol::GetBlock { block_id: 1 }),
always(),
)
.returning(move |_, _| Ok(Some(genesis_block_clone.clone())));
let state_clone = state.clone();
mock_storage
.expect_handle_get_lee_state()
.returning(move |_, _| Ok(Some(state_clone.clone())));
let genesis_block_meta_clone = genesis_block_meta.clone();
mock_storage
.expect_handle_get_final_snapshot()
.returning(move |_, _| Ok(Some((state.clone(), genesis_block_meta_clone.clone()))));
mock_storage
.expect_handle_get_all_blocks()
.returning(move |_, _| Ok(vec![genesis_block.clone()]));
mock_storage
.expect_handle_get_zone_checkpoint_bytes()
.returning(|_, _| Ok(None));
mock_storage
.expect_handle_get_zone_anchor()
.returning(|_, _| Ok(None));
mock_storage
.expect_handle_get_latest_block_meta()
.returning(move |_, _| Ok(Some(genesis_block_meta.clone())));
mock_storage
.expect_handle_raise_published_high_water()
.returning(|_, _| Ok(()));
mock_storage
.expect_handle_get_dead_letter_dispatches()
.returning(|_, _| Ok(vec![]));
mock_storage
}
#[test]
async fn handle_transaction_fails_on_full_mempool() -> Result<()> {
let _res = env_logger::try_init();
let (config, _home) = sequencer_config();
let mempool_max_size = config.mempool_max_size;
let executor = ExecutorActor::spawn(ExecutorActor::<MockBlockPublisher>::new(config).await);
let mock_storage = prepare_mock_storage_with_empty_genesis();
let storage_ref = MockStorageActor::spawn(mock_storage);
let executor = ExecutorActor::spawn(
ExecutorActor::<MockBlockPublisher, _>::new(config, storage_ref.clone()).await,
);
storage_ref
.tell(sequencer_storage_actor::mock::Checkpoint)
.await?;
// Fill mempool
for _ in 0..mempool_max_size {
lez/sequencer/actors/rpc_server/Cargo.toml
+1 -1
View File
@@ -2,7 +2,7 @@
name = "sequencer_rpc_server_actor"
version = "0.1.0"
edition = "2024"
license = { workspace = true }
license.workspace = true
[lints]
workspace = true
lez/sequencer/actors/rpc_server/src/actor.rs
+2 -2
View File
@@ -20,10 +20,10 @@ pub struct RpcServerActor {
}
impl RpcServerActor {
pub async fn new<BP: BlockPublisherTrait + Send + 'static>(
executor_ref: ActorRef<sequencer_executor_actor::ExecutorActor<BP>>,
pub async fn new<BP: BlockPublisherTrait + Send + Sync + 'static>(
listen_addr: SocketAddr,
max_block_size: ByteSize,
executor_ref: ActorRef<sequencer_executor_actor::ExecutorActor<BP>>,
gossip_tx_publisher: Option<GossipTxPublisher>,
) -> Result<Self> {
let server = jsonrpsee::server::ServerBuilder::with_config(
lez/sequencer/actors/rpc_server/src/actor/service.rs
+5 -3
View File
@@ -14,13 +14,13 @@ use sequencer_service_protocol::{
CrossZoneDeadLetter, CrossZoneDeadLetterReport, HashType, MembershipProof, Nonce, ProgramId,
};
pub struct Service<BP: BlockPublisherTrait + Send + 'static> {
pub struct Service<BP: BlockPublisherTrait + Send + Sync + 'static> {
executor_ref: ActorRef<sequencer_executor_actor::ExecutorActor<BP>>,
max_block_size: ByteSize,
gossip_tx_publisher: Option<GossipTxPublisher>,
}
impl<BP: BlockPublisherTrait + Send + 'static> Service<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static> Service<BP> {
pub fn new(
executor_ref: ActorRef<sequencer_executor_actor::ExecutorActor<BP>>,
max_block_size: ByteSize,
@@ -37,7 +37,9 @@ impl<BP: BlockPublisherTrait + Send + 'static> Service<BP> {
}
#[async_trait]
impl<BP: BlockPublisherTrait + Send + 'static> sequencer_service_rpc::RpcServer for Service<BP> {
impl<BP: BlockPublisherTrait + Send + Sync + 'static> sequencer_service_rpc::RpcServer
for Service<BP>
{
async fn send_transaction(&self, tx: LeeTransaction) -> Result<HashType, ErrorObjectOwned> {
sequencer_rpc_server_actor_metrics::increment_submitted_transactions_total();
lez/sequencer/actors/storage/Cargo.toml
+25
View File
@@ -0,0 +1,25 @@
[package]
name = "sequencer_storage_actor"
version = "0.1.0"
edition = "2024"
license.workspace = true
[lints]
workspace = true
[features]
mock = ["dep:mockall"]
[dependencies]
lee_core.workspace = true
lee.workspace = true
common.workspace = true
storage.workspace = true
kameo.workspace = true
thiserror.workspace = true
mockall = { workspace = true, optional = true }
[dev-dependencies]
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
tempfile.workspace = true
lez/sequencer/actors/storage/src/actor.rs
+651
View File
@@ -0,0 +1,651 @@
use std::path::Path;
use common::block::{Block, BlockMeta, PeerChainTip};
use kameo::{
Actor,
actor::{ActorRef, WeakActorRef},
error::ActorStopReason,
message::{Context, Message},
};
use lee::V03State;
use lee_core::BlockId;
use storage::sequencer::RocksDBIO;
use crate::{
Result, StorageActorTrait,
error::Error,
protocol::{
AddPendingCrossZoneDispatches, AddPendingDepositEvent, ApplyStoreUpdate,
CleanPendingBlocksUpTo, ConsumeUnseenWithdrawCount, DbDump, DeadLetterDispatchRecord,
DeleteBlock, DeleteCrossZonePeerFloor, DeleteZoneCheckpoint, DispatchFailure,
DropSettledCrossZoneDispatches, DumpDb, GetAllBlocks, GetBlock, GetCrossZonePeerFloorBytes,
GetCrossZonePeerTip, GetDeadLetterDispatchCount, GetDeadLetterDispatches, GetFinalSnapshot,
GetFirstBlockId, GetLastBlockId, GetLatestBlockMeta, GetLeeState,
GetPendingCrossZoneDispatches, GetPendingDepositEvents, GetPublishedHighWater,
GetZoneAnchor, GetZoneCheckpointBytes, MarkBlockAsFinalized,
PendingCrossZoneDispatchRecord, PendingDepositEventRecord, RaisePublishedHighWater,
RecordDispatchFailure, RecordNewBlock, ResetAllBlocksToPending, SetCrossZonePeerFloorBytes,
SetCrossZonePeerTip, SetZoneAnchor, SetZoneCheckpointBytes, StoreUpdateOutcome,
ZoneAnchorRecord,
},
};
pub struct StorageActor {
/// `None` after [`Actor::on_stop`] closed the database.
dbio: Option<RocksDBIO>,
}
impl StorageActor {
/// Creates a new `StorageActor` with a database at `location`.
/// If the database does not exist, it will be created.
pub fn new(location: &Path) -> Result<Self> {
let dbio = RocksDBIO::open_or_create(location)?;
Ok(Self { dbio: Some(dbio) })
}
/// Creates a fresh database at `location` from `dump`.
pub fn restore_from_dump(location: &Path, dump: &DbDump) -> Result<Self> {
let dbio = RocksDBIO::restore_from_dump(location, dump.as_db_dump())?;
Ok(Self { dbio: Some(dbio) })
}
/// The open database.
///
/// # Panics
///
/// If called after the actor has stopped, which can't happen for message handlers as kameo
/// stops delivering messages before [`Actor::on_stop`].
const fn dbio(&self) -> &RocksDBIO {
self.dbio
.as_ref()
.expect("Database is closed, the actor has already stopped")
}
}
impl StorageActorTrait for StorageActor {}
impl Actor for StorageActor {
type Args = Self;
type Error = Error;
async fn on_start(args: Self::Args, _actor_ref: ActorRef<Self>) -> Result<Self> {
Ok(args)
}
/// Closes the database, releasing the rocksdb lock on its directory.
async fn on_stop(
&mut self,
_actor_ref: WeakActorRef<Self>,
_reason: ActorStopReason,
) -> Result<()> {
drop(self.dbio.take());
Ok(())
}
}
impl Message<RecordNewBlock> for StorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
RecordNewBlock {
block,
withdrawals,
state,
checkpoint_bytes,
}: RecordNewBlock,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
let withdrawals = withdrawals.into_iter().map(Into::into).collect::<Vec<_>>();
self.dbio()
.atomic_update(&block, &withdrawals, &state, checkpoint_bytes.as_deref())
.map_err(Into::into)
}
}
impl Message<GetBlock> for StorageActor {
type Reply = Result<Option<Block>>;
async fn handle(
&mut self,
GetBlock { block_id }: GetBlock,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio().get_block(block_id).map_err(Into::into)
}
}
impl Message<GetAllBlocks> for StorageActor {
type Reply = Result<Vec<Block>>;
async fn handle(
&mut self,
GetAllBlocks: GetAllBlocks,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.get_all_blocks()
.collect::<storage::DbResult<Vec<_>>>()
.map_err(Into::into)
}
}
impl Message<DeleteBlock> for StorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
DeleteBlock { block_id }: DeleteBlock,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio().delete_block(block_id).map_err(Into::into)
}
}
impl Message<MarkBlockAsFinalized> for StorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
MarkBlockAsFinalized { block_id }: MarkBlockAsFinalized,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.mark_block_as_finalized(block_id)
.map_err(Into::into)
}
}
impl Message<ResetAllBlocksToPending> for StorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
ResetAllBlocksToPending: ResetAllBlocksToPending,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.reset_all_blocks_to_pending()
.map_err(Into::into)
}
}
impl Message<GetFirstBlockId> for StorageActor {
type Reply = Result<Option<BlockId>>;
async fn handle(
&mut self,
GetFirstBlockId: GetFirstBlockId,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio().get_meta_first_block_in_db().map_err(Into::into)
}
}
impl Message<GetLastBlockId> for StorageActor {
type Reply = Result<Option<BlockId>>;
async fn handle(
&mut self,
GetLastBlockId: GetLastBlockId,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio().get_meta_last_block_in_db().map_err(Into::into)
}
}
impl Message<GetLatestBlockMeta> for StorageActor {
type Reply = Result<Option<BlockMeta>>;
async fn handle(
&mut self,
GetLatestBlockMeta: GetLatestBlockMeta,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio().latest_block_meta().map_err(Into::into)
}
}
impl Message<GetLeeState> for StorageActor {
type Reply = Result<Option<V03State>>;
async fn handle(
&mut self,
GetLeeState: GetLeeState,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio().get_lee_state().map_err(Into::into)
}
}
impl Message<GetZoneCheckpointBytes> for StorageActor {
type Reply = Result<Option<Vec<u8>>>;
async fn handle(
&mut self,
GetZoneCheckpointBytes: GetZoneCheckpointBytes,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.get_zone_sdk_checkpoint_bytes()
.map_err(Into::into)
}
}
impl Message<SetZoneCheckpointBytes> for StorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
SetZoneCheckpointBytes { bytes }: SetZoneCheckpointBytes,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.put_zone_sdk_checkpoint_bytes(&bytes)
.map_err(Into::into)
}
}
impl Message<DeleteZoneCheckpoint> for StorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
DeleteZoneCheckpoint: DeleteZoneCheckpoint,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.delete_zone_sdk_checkpoint_bytes()
.map_err(Into::into)
}
}
impl Message<GetZoneAnchor> for StorageActor {
type Reply = Result<Option<ZoneAnchorRecord>>;
async fn handle(
&mut self,
GetZoneAnchor: GetZoneAnchor,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.get_zone_anchor()
.map(|anchor| anchor.map(Into::into))
.map_err(Into::into)
}
}
impl Message<SetZoneAnchor> for StorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
SetZoneAnchor { anchor }: SetZoneAnchor,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.put_zone_anchor(&anchor.into())
.map_err(Into::into)
}
}
impl Message<GetPublishedHighWater> for StorageActor {
type Reply = Result<Option<BlockId>>;
async fn handle(
&mut self,
GetPublishedHighWater: GetPublishedHighWater,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio().published_high_water().map_err(Into::into)
}
}
impl Message<RaisePublishedHighWater> for StorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
RaisePublishedHighWater { block_id }: RaisePublishedHighWater,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.raise_published_high_water(block_id)
.map_err(Into::into)
}
}
impl Message<GetPendingDepositEvents> for StorageActor {
type Reply = Result<Vec<PendingDepositEventRecord>>;
async fn handle(
&mut self,
GetPendingDepositEvents: GetPendingDepositEvents,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.get_pending_deposit_events()
.map(|events| events.into_iter().map(Into::into).collect())
.map_err(Into::into)
}
}
impl Message<DumpDb> for StorageActor {
type Reply = Result<DbDump>;
async fn handle(
&mut self,
DumpDb: DumpDb,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio().dump_all().map(Into::into).map_err(Into::into)
}
}
impl Message<ApplyStoreUpdate> for StorageActor {
type Reply = Result<StoreUpdateOutcome>;
async fn handle(
&mut self,
msg: ApplyStoreUpdate,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
let ApplyStoreUpdate {
checkpoint,
blocks,
head_tip,
head_state,
final_snapshot,
finalized_up_to,
new_deposit_events,
remove_deposit_records,
remove_dispatch_records,
consumed_withdrawals,
new_withdraw_intents,
zone_anchor,
} = msg;
let blocks = blocks
.iter()
.map(|(block, finalized)| (block, *finalized))
.collect::<Vec<_>>();
let zone_anchor = zone_anchor.map(Into::into);
let new_deposit_events = new_deposit_events
.into_iter()
.map(Into::into)
.collect::<Vec<_>>();
let consumed_withdrawals = consumed_withdrawals
.into_iter()
.map(Into::into)
.collect::<Vec<_>>();
let new_withdraw_intents = new_withdraw_intents
.into_iter()
.map(Into::into)
.collect::<Vec<_>>();
let update = storage::sequencer::StoreUpdate {
checkpoint: checkpoint.as_deref(),
blocks: &blocks,
head_tip: head_tip.as_ref(),
head_state: &head_state,
final_snapshot: final_snapshot
.as_ref()
.map(|(state, meta)| (state.as_ref(), meta)),
finalized_up_to,
new_deposit_events: &new_deposit_events,
remove_deposit_records: &remove_deposit_records,
remove_dispatch_records: &remove_dispatch_records,
consumed_withdrawals: &consumed_withdrawals,
new_withdraw_intents: &new_withdraw_intents,
zone_anchor: zone_anchor.as_ref(),
};
self.dbio()
.store_update(&update)
.map(Into::into)
.map_err(Into::into)
}
}
impl Message<CleanPendingBlocksUpTo> for StorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
CleanPendingBlocksUpTo { last_finalized }: CleanPendingBlocksUpTo,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.clean_pending_blocks_up_to(last_finalized)
.map_err(Into::into)
}
}
impl Message<GetFinalSnapshot> for StorageActor {
type Reply = Result<Option<(V03State, BlockMeta)>>;
async fn handle(
&mut self,
GetFinalSnapshot: GetFinalSnapshot,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio().get_final_snapshot().map_err(Into::into)
}
}
impl Message<AddPendingDepositEvent> for StorageActor {
type Reply = Result<bool>;
async fn handle(
&mut self,
AddPendingDepositEvent { event }: AddPendingDepositEvent,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.add_pending_deposit_event(event.into())
.map_err(Into::into)
}
}
impl Message<ConsumeUnseenWithdrawCount> for StorageActor {
type Reply = Result<bool>;
async fn handle(
&mut self,
ConsumeUnseenWithdrawCount { withdrawal }: ConsumeUnseenWithdrawCount,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.consume_unseen_withdraw_count(withdrawal.into())
.map_err(Into::into)
}
}
impl Message<GetPendingCrossZoneDispatches> for StorageActor {
type Reply = Result<Vec<PendingCrossZoneDispatchRecord>>;
async fn handle(
&mut self,
GetPendingCrossZoneDispatches: GetPendingCrossZoneDispatches,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.get_pending_cross_zone_dispatches()
.map(|records| records.into_iter().map(Into::into).collect())
.map_err(Into::into)
}
}
impl Message<AddPendingCrossZoneDispatches> for StorageActor {
type Reply = Result<usize>;
async fn handle(
&mut self,
AddPendingCrossZoneDispatches { dispatches }: AddPendingCrossZoneDispatches,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
let dispatches = dispatches.into_iter().map(Into::into).collect();
self.dbio()
.add_pending_cross_zone_dispatches(dispatches)
.map_err(Into::into)
}
}
impl Message<DropSettledCrossZoneDispatches> for StorageActor {
type Reply = Result<usize>;
async fn handle(
&mut self,
DropSettledCrossZoneDispatches { message_keys }: DropSettledCrossZoneDispatches,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.drop_settled_cross_zone_dispatches(&message_keys)
.map_err(Into::into)
}
}
impl Message<RecordDispatchFailure> for StorageActor {
type Reply = Result<DispatchFailure>;
async fn handle(
&mut self,
RecordDispatchFailure {
message_key,
retire_at,
origin,
}: RecordDispatchFailure,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.record_dispatch_failure(message_key, retire_at, origin.into())
.map(Into::into)
.map_err(Into::into)
}
}
impl Message<GetDeadLetterDispatches> for StorageActor {
type Reply = Result<Vec<DeadLetterDispatchRecord>>;
async fn handle(
&mut self,
GetDeadLetterDispatches: GetDeadLetterDispatches,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.get_dead_letter_cross_zone_dispatches()
.map(|records| records.into_iter().map(Into::into).collect())
.map_err(Into::into)
}
}
impl Message<GetDeadLetterDispatchCount> for StorageActor {
type Reply = Result<u64>;
async fn handle(
&mut self,
GetDeadLetterDispatchCount: GetDeadLetterDispatchCount,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.get_dead_letter_cross_zone_dispatch_count()
.map_err(Into::into)
}
}
impl Message<GetCrossZonePeerFloorBytes> for StorageActor {
type Reply = Result<Option<Vec<u8>>>;
async fn handle(
&mut self,
GetCrossZonePeerFloorBytes { peer_zone }: GetCrossZonePeerFloorBytes,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.get_cross_zone_peer_floor_bytes(peer_zone)
.map_err(Into::into)
}
}
impl Message<SetCrossZonePeerFloorBytes> for StorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
SetCrossZonePeerFloorBytes { peer_zone, bytes }: SetCrossZonePeerFloorBytes,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.put_cross_zone_peer_floor_bytes(peer_zone, &bytes)
.map_err(Into::into)
}
}
impl Message<DeleteCrossZonePeerFloor> for StorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
DeleteCrossZonePeerFloor { peer_zone }: DeleteCrossZonePeerFloor,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.delete_cross_zone_peer_floor(peer_zone)
.map_err(Into::into)
}
}
impl Message<GetCrossZonePeerTip> for StorageActor {
type Reply = Result<Option<PeerChainTip>>;
async fn handle(
&mut self,
GetCrossZonePeerTip { peer_zone }: GetCrossZonePeerTip,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.get_cross_zone_peer_tip(peer_zone)
.map_err(Into::into)
}
}
impl Message<SetCrossZonePeerTip> for StorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
SetCrossZonePeerTip { peer_zone, tip }: SetCrossZonePeerTip,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.dbio()
.put_cross_zone_peer_tip(peer_zone, tip)
.map_err(Into::into)
}
}
#[cfg(test)]
mod tests {
use kameo::actor::Spawn as _;
use super::*;
/// Holding the task's output keeps the stopped actor's state alive, which is exactly the
/// situation `on_stop` exists for: the lock has to be gone by the time the shutdown result
/// resolves, not by the time the state happens to be dropped.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stopped_actor_releases_the_database_lock() {
let dir = tempfile::tempdir().expect("Failed to create temp dir");
let prepared = StorageActor::prepare();
let actor_ref = prepared.actor_ref().clone();
let join_handle = prepared.spawn(StorageActor::new(dir.path()).expect("Failed to open db"));
actor_ref
.stop_gracefully()
.await
.expect("Failed to stop the actor");
actor_ref.wait_for_shutdown_with_result(|_| ()).await;
StorageActor::new(dir.path()).expect("Database lock must be released once the actor stops");
drop(join_handle);
}
}
lez/sequencer/actors/storage/src/error.rs
+5
View File
@@ -0,0 +1,5 @@
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("Database error")]
DatabaseError(#[from] storage::error::DbError),
}
lez/sequencer/actors/storage/src/lib.rs
+13
View File
@@ -0,0 +1,13 @@
//! Storage Actor is responsible for persisting data in local database.
pub use actor::StorageActor;
pub use r#trait::StorageActorTrait;
pub mod actor;
pub mod error;
#[cfg(feature = "mock")]
pub mod mock;
pub mod protocol;
pub mod r#trait;
pub type Result<T> = std::result::Result<T, error::Error>;
lez/sequencer/actors/storage/src/mock.rs
+709
View File
@@ -0,0 +1,709 @@
#![expect(
clippy::struct_field_names,
reason = "`handle*` prefix is used for convenience with `Message` trait"
)]
use common::block::{Block, BlockMeta, PeerChainTip};
use kameo::{
Actor, Reply,
actor::ActorRef,
message::{Context, Message},
};
use lee::V03State;
use lee_core::BlockId;
use crate::{
Result, StorageActorTrait,
error::Error,
protocol::{
AddPendingCrossZoneDispatches, AddPendingDepositEvent, ApplyStoreUpdate,
CleanPendingBlocksUpTo, ConsumeUnseenWithdrawCount, DbDump, DeadLetterDispatchRecord,
DeleteBlock, DeleteCrossZonePeerFloor, DeleteZoneCheckpoint, DispatchFailure,
DropSettledCrossZoneDispatches, DumpDb, GetAllBlocks, GetBlock, GetCrossZonePeerFloorBytes,
GetCrossZonePeerTip, GetDeadLetterDispatchCount, GetDeadLetterDispatches, GetFinalSnapshot,
GetFirstBlockId, GetLastBlockId, GetLatestBlockMeta, GetLeeState,
GetPendingCrossZoneDispatches, GetPendingDepositEvents, GetPublishedHighWater,
GetZoneAnchor, GetZoneCheckpointBytes, MarkBlockAsFinalized,
PendingCrossZoneDispatchRecord, PendingDepositEventRecord, RaisePublishedHighWater,
RecordDispatchFailure, RecordNewBlock, ResetAllBlocksToPending, SetCrossZonePeerFloorBytes,
SetCrossZonePeerTip, SetZoneAnchor, SetZoneCheckpointBytes, StoreUpdateOutcome,
ZoneAnchorRecord,
},
};
mockall::mock! {
pub StorageActor {
pub fn handle_record_new_block(
&mut self,
msg: RecordNewBlock,
ctx: &mut Context<Self, Result<()>>
) -> Result<()>;
pub fn handle_get_block(
&mut self,
msg: GetBlock,
ctx: &mut Context<Self, Result<Option<Block>>>
) -> Result<Option<Block>>;
pub fn handle_get_all_blocks(
&mut self,
msg: GetAllBlocks,
ctx: &mut Context<Self, Result<Vec<Block>>>
) -> Result<Vec<Block>>;
pub fn handle_delete_block(
&mut self,
msg: DeleteBlock,
ctx: &mut Context<Self, Result<()>>
) -> Result<()>;
pub fn handle_mark_block_as_finalized(
&mut self,
msg: MarkBlockAsFinalized,
ctx: &mut Context<Self, Result<()>>
) -> Result<()>;
pub fn handle_reset_all_blocks_to_pending(
&mut self,
msg: ResetAllBlocksToPending,
ctx: &mut Context<Self, Result<()>>
) -> Result<()>;
pub fn handle_get_first_block_id(
&mut self,
msg: GetFirstBlockId,
ctx: &mut Context<Self, Result<Option<BlockId>>>
) -> Result<Option<BlockId>>;
pub fn handle_get_last_block_id(
&mut self,
msg: GetLastBlockId,
ctx: &mut Context<Self, Result<Option<BlockId>>>
) -> Result<Option<BlockId>>;
pub fn handle_get_latest_block_meta(
&mut self,
msg: GetLatestBlockMeta,
ctx: &mut Context<Self, Result<Option<BlockMeta>>>
) -> Result<Option<BlockMeta>>;
pub fn handle_get_lee_state(
&mut self,
msg: GetLeeState,
ctx: &mut Context<Self, Result<Option<V03State>>>
) -> Result<Option<V03State>>;
pub fn handle_get_zone_checkpoint_bytes(
&mut self,
msg: GetZoneCheckpointBytes,
ctx: &mut Context<Self, Result<Option<Vec<u8>>>>
) -> Result<Option<Vec<u8>>>;
pub fn handle_set_zone_checkpoint_bytes(
&mut self,
msg: SetZoneCheckpointBytes,
ctx: &mut Context<Self, Result<()>>
) -> Result<()>;
pub fn handle_delete_zone_checkpoint(
&mut self,
msg: DeleteZoneCheckpoint,
ctx: &mut Context<Self, Result<()>>
) -> Result<()>;
pub fn handle_get_zone_anchor(
&mut self,
msg: GetZoneAnchor,
ctx: &mut Context<Self, Result<Option<ZoneAnchorRecord>>>
) -> Result<Option<ZoneAnchorRecord>>;
pub fn handle_set_zone_anchor(
&mut self,
msg: SetZoneAnchor,
ctx: &mut Context<Self, Result<()>>
) -> Result<()>;
pub fn handle_get_published_high_water(
&mut self,
msg: GetPublishedHighWater,
ctx: &mut Context<Self, Result<Option<BlockId>>>
) -> Result<Option<BlockId>>;
pub fn handle_raise_published_high_water(
&mut self,
msg: RaisePublishedHighWater,
ctx: &mut Context<Self, Result<()>>
) -> Result<()>;
pub fn handle_get_pending_deposit_events(
&mut self,
msg: GetPendingDepositEvents,
ctx: &mut Context<Self, Result<Vec<PendingDepositEventRecord>>>
) -> Result<Vec<PendingDepositEventRecord>>;
pub fn handle_apply_store_update(
&mut self,
msg: ApplyStoreUpdate,
ctx: &mut Context<Self, Result<StoreUpdateOutcome>>
) -> Result<StoreUpdateOutcome>;
pub fn handle_clean_pending_blocks_up_to(
&mut self,
msg: CleanPendingBlocksUpTo,
ctx: &mut Context<Self, Result<()>>
) -> Result<()>;
pub fn handle_get_final_snapshot(
&mut self,
msg: GetFinalSnapshot,
ctx: &mut Context<Self, Result<Option<(V03State, BlockMeta)>>>
) -> Result<Option<(V03State, BlockMeta)>>;
pub fn handle_add_pending_deposit_event(
&mut self,
msg: AddPendingDepositEvent,
ctx: &mut Context<Self, Result<bool>>
) -> Result<bool>;
pub fn handle_consume_unseen_withdraw_count(
&mut self,
msg: ConsumeUnseenWithdrawCount,
ctx: &mut Context<Self, Result<bool>>
) -> Result<bool>;
pub fn handle_get_pending_cross_zone_dispatches(
&mut self,
msg: GetPendingCrossZoneDispatches,
ctx: &mut Context<Self, Result<Vec<PendingCrossZoneDispatchRecord>>>
) -> Result<Vec<PendingCrossZoneDispatchRecord>>;
pub fn handle_add_pending_cross_zone_dispatches(
&mut self,
msg: AddPendingCrossZoneDispatches,
ctx: &mut Context<Self, Result<usize>>
) -> Result<usize>;
pub fn handle_drop_settled_cross_zone_dispatches(
&mut self,
msg: DropSettledCrossZoneDispatches,
ctx: &mut Context<Self, Result<usize>>
) -> Result<usize>;
pub fn handle_record_dispatch_failure(
&mut self,
msg: RecordDispatchFailure,
ctx: &mut Context<Self, Result<DispatchFailure>>
) -> Result<DispatchFailure>;
pub fn handle_get_dead_letter_dispatches(
&mut self,
msg: GetDeadLetterDispatches,
ctx: &mut Context<Self, Result<Vec<DeadLetterDispatchRecord>>>
) -> Result<Vec<DeadLetterDispatchRecord>>;
pub fn handle_get_dead_letter_dispatch_count(
&mut self,
msg: GetDeadLetterDispatchCount,
ctx: &mut Context<Self, Result<u64>>
) -> Result<u64>;
pub fn handle_get_cross_zone_peer_floor_bytes(
&mut self,
msg: GetCrossZonePeerFloorBytes,
ctx: &mut Context<Self, Result<Option<Vec<u8>>>>
) -> Result<Option<Vec<u8>>>;
pub fn handle_set_cross_zone_peer_floor_bytes(
&mut self,
msg: SetCrossZonePeerFloorBytes,
ctx: &mut Context<Self, Result<()>>
) -> Result<()>;
pub fn handle_delete_cross_zone_peer_floor(
&mut self,
msg: DeleteCrossZonePeerFloor,
ctx: &mut Context<Self, Result<()>>
) -> Result<()>;
pub fn handle_get_cross_zone_peer_tip(
&mut self,
msg: GetCrossZonePeerTip,
ctx: &mut Context<Self, Result<Option<PeerChainTip>>>
) -> Result<Option<common::block::PeerChainTip>>;
pub fn handle_set_cross_zone_peer_tip(
&mut self,
msg: SetCrossZonePeerTip,
ctx: &mut Context<Self, Result<()>>
) -> Result<()>;
pub fn handle_dump_db(
&mut self,
msg: DumpDb,
ctx: &mut Context<Self, Result<DbDump>>
) -> Result<DbDump>;
}
}
impl StorageActorTrait for MockStorageActor {}
impl Actor for MockStorageActor {
type Args = Self;
type Error = Error;
async fn on_start(args: Self::Args, _actor_ref: ActorRef<Self>) -> Result<Self> {
Ok(args)
}
}
/// Special message to trigger [`MockStorageActor::checkpoint()`].
pub struct Checkpoint;
impl Message<Checkpoint> for MockStorageActor {
type Reply = ();
async fn handle(
&mut self,
Checkpoint: Checkpoint,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.checkpoint();
}
}
/// Special message to [`std::mem::replace()`] the inner state of [`MockStorageActor`] with a new
/// one, returning old state.
/// This is useful for testing, to swap in a new mock with different expectations.
pub struct Replace {
pub mock: MockStorageActor,
}
#[derive(Reply)]
pub struct ReplaceReply {
pub old_mock: MockStorageActor,
}
impl Message<Replace> for MockStorageActor {
type Reply = ReplaceReply;
async fn handle(
&mut self,
Replace { mock }: Replace,
_ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
let old_mock = std::mem::replace(self, mock);
ReplaceReply { old_mock }
}
}
impl Message<RecordNewBlock> for MockStorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
msg: RecordNewBlock,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_record_new_block(msg, ctx)
}
}
impl Message<GetBlock> for MockStorageActor {
type Reply = Result<Option<Block>>;
async fn handle(&mut self, msg: GetBlock, ctx: &mut Context<Self, Self::Reply>) -> Self::Reply {
self.handle_get_block(msg, ctx)
}
}
impl Message<GetAllBlocks> for MockStorageActor {
type Reply = Result<Vec<Block>>;
async fn handle(
&mut self,
msg: GetAllBlocks,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_all_blocks(msg, ctx)
}
}
impl Message<DeleteBlock> for MockStorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
msg: DeleteBlock,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_delete_block(msg, ctx)
}
}
impl Message<MarkBlockAsFinalized> for MockStorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
msg: MarkBlockAsFinalized,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_mark_block_as_finalized(msg, ctx)
}
}
impl Message<ResetAllBlocksToPending> for MockStorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
msg: ResetAllBlocksToPending,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_reset_all_blocks_to_pending(msg, ctx)
}
}
impl Message<GetFirstBlockId> for MockStorageActor {
type Reply = Result<Option<BlockId>>;
async fn handle(
&mut self,
msg: GetFirstBlockId,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_first_block_id(msg, ctx)
}
}
impl Message<GetLastBlockId> for MockStorageActor {
type Reply = Result<Option<BlockId>>;
async fn handle(
&mut self,
msg: GetLastBlockId,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_last_block_id(msg, ctx)
}
}
impl Message<GetLatestBlockMeta> for MockStorageActor {
type Reply = Result<Option<BlockMeta>>;
async fn handle(
&mut self,
msg: GetLatestBlockMeta,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_latest_block_meta(msg, ctx)
}
}
impl Message<GetLeeState> for MockStorageActor {
type Reply = Result<Option<V03State>>;
async fn handle(
&mut self,
msg: GetLeeState,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_lee_state(msg, ctx)
}
}
impl Message<GetZoneCheckpointBytes> for MockStorageActor {
type Reply = Result<Option<Vec<u8>>>;
async fn handle(
&mut self,
msg: GetZoneCheckpointBytes,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_zone_checkpoint_bytes(msg, ctx)
}
}
impl Message<SetZoneCheckpointBytes> for MockStorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
msg: SetZoneCheckpointBytes,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_set_zone_checkpoint_bytes(msg, ctx)
}
}
impl Message<DeleteZoneCheckpoint> for MockStorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
msg: DeleteZoneCheckpoint,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_delete_zone_checkpoint(msg, ctx)
}
}
impl Message<GetZoneAnchor> for MockStorageActor {
type Reply = Result<Option<ZoneAnchorRecord>>;
async fn handle(
&mut self,
msg: GetZoneAnchor,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_zone_anchor(msg, ctx)
}
}
impl Message<SetZoneAnchor> for MockStorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
msg: SetZoneAnchor,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_set_zone_anchor(msg, ctx)
}
}
impl Message<GetPublishedHighWater> for MockStorageActor {
type Reply = Result<Option<BlockId>>;
async fn handle(
&mut self,
msg: GetPublishedHighWater,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_published_high_water(msg, ctx)
}
}
impl Message<RaisePublishedHighWater> for MockStorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
msg: RaisePublishedHighWater,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_raise_published_high_water(msg, ctx)
}
}
impl Message<GetPendingDepositEvents> for MockStorageActor {
type Reply = Result<Vec<PendingDepositEventRecord>>;
async fn handle(
&mut self,
msg: GetPendingDepositEvents,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_pending_deposit_events(msg, ctx)
}
}
impl Message<DumpDb> for MockStorageActor {
type Reply = Result<DbDump>;
async fn handle(&mut self, msg: DumpDb, ctx: &mut Context<Self, Self::Reply>) -> Self::Reply {
self.handle_dump_db(msg, ctx)
}
}
impl Message<ApplyStoreUpdate> for MockStorageActor {
type Reply = Result<StoreUpdateOutcome>;
async fn handle(
&mut self,
msg: ApplyStoreUpdate,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_apply_store_update(msg, ctx)
}
}
impl Message<CleanPendingBlocksUpTo> for MockStorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
msg: CleanPendingBlocksUpTo,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_clean_pending_blocks_up_to(msg, ctx)
}
}
impl Message<GetFinalSnapshot> for MockStorageActor {
type Reply = Result<Option<(V03State, BlockMeta)>>;
async fn handle(
&mut self,
msg: GetFinalSnapshot,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_final_snapshot(msg, ctx)
}
}
impl Message<AddPendingDepositEvent> for MockStorageActor {
type Reply = Result<bool>;
async fn handle(
&mut self,
msg: AddPendingDepositEvent,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_add_pending_deposit_event(msg, ctx)
}
}
impl Message<ConsumeUnseenWithdrawCount> for MockStorageActor {
type Reply = Result<bool>;
async fn handle(
&mut self,
msg: ConsumeUnseenWithdrawCount,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_consume_unseen_withdraw_count(msg, ctx)
}
}
impl Message<GetPendingCrossZoneDispatches> for MockStorageActor {
type Reply = Result<Vec<PendingCrossZoneDispatchRecord>>;
async fn handle(
&mut self,
msg: GetPendingCrossZoneDispatches,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_pending_cross_zone_dispatches(msg, ctx)
}
}
impl Message<AddPendingCrossZoneDispatches> for MockStorageActor {
type Reply = Result<usize>;
async fn handle(
&mut self,
msg: AddPendingCrossZoneDispatches,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_add_pending_cross_zone_dispatches(msg, ctx)
}
}
impl Message<DropSettledCrossZoneDispatches> for MockStorageActor {
type Reply = Result<usize>;
async fn handle(
&mut self,
msg: DropSettledCrossZoneDispatches,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_drop_settled_cross_zone_dispatches(msg, ctx)
}
}
impl Message<RecordDispatchFailure> for MockStorageActor {
type Reply = Result<DispatchFailure>;
async fn handle(
&mut self,
msg: RecordDispatchFailure,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_record_dispatch_failure(msg, ctx)
}
}
impl Message<GetDeadLetterDispatches> for MockStorageActor {
type Reply = Result<Vec<DeadLetterDispatchRecord>>;
async fn handle(
&mut self,
msg: GetDeadLetterDispatches,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_dead_letter_dispatches(msg, ctx)
}
}
impl Message<GetDeadLetterDispatchCount> for MockStorageActor {
type Reply = Result<u64>;
async fn handle(
&mut self,
msg: GetDeadLetterDispatchCount,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_dead_letter_dispatch_count(msg, ctx)
}
}
impl Message<GetCrossZonePeerFloorBytes> for MockStorageActor {
type Reply = Result<Option<Vec<u8>>>;
async fn handle(
&mut self,
msg: GetCrossZonePeerFloorBytes,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_cross_zone_peer_floor_bytes(msg, ctx)
}
}
impl Message<SetCrossZonePeerFloorBytes> for MockStorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
msg: SetCrossZonePeerFloorBytes,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_set_cross_zone_peer_floor_bytes(msg, ctx)
}
}
impl Message<DeleteCrossZonePeerFloor> for MockStorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
msg: DeleteCrossZonePeerFloor,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_delete_cross_zone_peer_floor(msg, ctx)
}
}
impl Message<GetCrossZonePeerTip> for MockStorageActor {
type Reply = Result<Option<PeerChainTip>>;
async fn handle(
&mut self,
msg: GetCrossZonePeerTip,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_get_cross_zone_peer_tip(msg, ctx)
}
}
impl Message<SetCrossZonePeerTip> for MockStorageActor {
type Reply = Result<()>;
async fn handle(
&mut self,
msg: SetCrossZonePeerTip,
ctx: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.handle_set_cross_zone_peer_tip(msg, ctx)
}
}
lez/sequencer/actors/storage/src/protocol.rs
+506
View File
@@ -0,0 +1,506 @@
// TODO: Instead of `From` conversions from `protocol` types to `cell` types consider moving
// `storage` crate as sub-module in `actor` while removing `cell` types and implementing
// encoding/decoding for `protocol` structures.
use std::sync::Arc;
use common::{
HashType,
block::{Block, BlockMeta, PeerChainTip},
};
use lee::V03State;
use lee_core::BlockId;
use storage::sequencer::{self as db, sequencer_cells as db_cells};
use crate::Result;
/// Persists `block` with the effects it covers.
pub struct RecordNewBlock {
pub block: Block,
pub withdrawals: Vec<WithdrawalReconciliationKey>,
pub state: Arc<V03State>,
pub checkpoint_bytes: Option<Vec<u8>>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GetBlock {
pub block_id: BlockId,
}
pub struct GetAllBlocks;
pub struct DeleteBlock {
pub block_id: BlockId,
}
pub struct MarkBlockAsFinalized {
pub block_id: BlockId,
}
pub struct ResetAllBlocksToPending;
pub struct GetFirstBlockId;
pub struct GetLastBlockId;
pub struct GetLatestBlockMeta;
pub struct GetLeeState;
pub struct GetZoneCheckpointBytes;
pub struct SetZoneCheckpointBytes {
pub bytes: Vec<u8>,
}
pub struct DeleteZoneCheckpoint;
pub struct GetZoneAnchor;
pub struct SetZoneAnchor {
pub anchor: ZoneAnchorRecord,
}
pub struct GetPublishedHighWater;
/// Raises the published high water mark to `block_id`, never lowering it.
pub struct RaisePublishedHighWater {
pub block_id: BlockId,
}
pub struct GetPendingDepositEvents;
pub struct AddPendingDepositEvent {
pub event: PendingDepositEventRecord,
}
pub struct GetFinalSnapshot;
/// Marks every stored pending block at or below `last_finalized` as finalized.
pub struct CleanPendingBlocksUpTo {
pub last_finalized: BlockId,
}
pub struct ConsumeUnseenWithdrawCount {
pub withdrawal: WithdrawalReconciliationKey,
}
pub struct GetPendingCrossZoneDispatches;
pub struct AddPendingCrossZoneDispatches {
pub dispatches: Vec<PendingCrossZoneDispatchRecord>,
}
pub struct DropSettledCrossZoneDispatches {
pub message_keys: Vec<[u8; 32]>,
}
pub struct RecordDispatchFailure {
pub message_key: [u8; 32],
pub retire_at: u32,
pub origin: DispatchOrigin,
}
pub struct GetDeadLetterDispatches;
pub struct GetDeadLetterDispatchCount;
pub struct GetCrossZonePeerFloorBytes {
pub peer_zone: PeerZoneKey,
}
pub struct SetCrossZonePeerFloorBytes {
pub peer_zone: PeerZoneKey,
pub bytes: Vec<u8>,
}
pub struct DeleteCrossZonePeerFloor {
pub peer_zone: PeerZoneKey,
}
pub struct GetCrossZonePeerTip {
pub peer_zone: PeerZoneKey,
}
pub struct SetCrossZonePeerTip {
pub peer_zone: PeerZoneKey,
pub tip: PeerChainTip,
}
pub struct DumpDb;
/// Everything one sequencer event writes.
// TODO: Consider removing [`SetZoneCheckpointBytes`] in favor of this.
// TODO: How does this stack with [`RecordNewBlock`] ?
// TODO: Reconsider options here.
pub struct ApplyStoreUpdate {
/// Serialized zone-sdk checkpoint for this event.
pub checkpoint: Option<Vec<u8>>,
/// `(block, finalized)` payloads to write.
pub blocks: Vec<(Block, bool)>,
/// Head tip to pin the stored chain to; `None` only for an empty chain.
pub head_tip: Option<BlockMeta>,
/// State after the last applied block.
pub head_state: Arc<V03State>,
/// `(state, meta)` of the final tier, when it advanced.
pub final_snapshot: Option<(Arc<V03State>, BlockMeta)>,
/// Highest block id this event made irreversible: stored blocks at or below
/// it become finalized.
pub finalized_up_to: Option<BlockId>,
/// Deposit events observed on L1, recorded unless already pending.
pub new_deposit_events: Vec<PendingDepositEventRecord>,
/// Deposit op ids whose mint finalized: their pending records are dropped.
pub remove_deposit_records: Vec<HashType>,
/// Message keys whose delivery finalized: their pending records are dropped.
pub remove_dispatch_records: Vec<[u8; 32]>,
/// L1 withdraw events to reconcile against the local unseen counters.
pub consumed_withdrawals: Vec<WithdrawalReconciliationKey>,
/// L2 withdraw intents this update raises, awaiting their L1 event.
pub new_withdraw_intents: Vec<WithdrawalReconciliationKey>,
/// Advance the channel-read anchor.
pub zone_anchor: Option<ZoneAnchorRecord>,
}
/// Zone id of a cross-zone peer.
pub type PeerZoneKey = [u8; 32];
/// Identity of one withdrawal: the id of the channel note it releases.
///
/// Shared by the intent recorded when the sequencer publishes a withdrawal and
/// the Bedrock event that later reports it.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct WithdrawalReconciliationKey {
pub released_note_id: [u8; 32],
}
impl From<WithdrawalReconciliationKey> for db_cells::WithdrawalReconciliationKey {
fn from(WithdrawalReconciliationKey { released_note_id }: WithdrawalReconciliationKey) -> Self {
Self { released_note_id }
}
}
impl From<db_cells::WithdrawalReconciliationKey> for WithdrawalReconciliationKey {
fn from(
db_cells::WithdrawalReconciliationKey { released_note_id }: db_cells::WithdrawalReconciliationKey,
) -> Self {
Self { released_note_id }
}
}
/// The last channel block read back and verified from Bedrock: the anchor for
/// the startup consistency check and the resume point for reconstruction.
///
/// `slot` is the raw L1 inscription slot; the caller converts to and from the
/// zone-sdk `Slot`, which does not derive borsh and so cannot be stored.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ZoneAnchorRecord {
pub slot: u64,
pub block_id: u64,
pub hash: HashType,
}
impl From<ZoneAnchorRecord> for db_cells::ZoneAnchorRecord {
fn from(
ZoneAnchorRecord {
slot,
block_id,
hash,
}: ZoneAnchorRecord,
) -> Self {
Self {
slot,
block_id,
hash,
}
}
}
impl From<db_cells::ZoneAnchorRecord> for ZoneAnchorRecord {
fn from(
db_cells::ZoneAnchorRecord {
slot,
block_id,
hash,
}: db_cells::ZoneAnchorRecord,
) -> Self {
Self {
slot,
block_id,
hash,
}
}
}
/// An L1 deposit event observed but not yet seen finalized.
///
/// Purely a liveness queue: whether to emit a mint is decided against chain
/// state, and the record is dropped once its mint finalizes.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PendingDepositEventRecord {
pub deposit_op_id: HashType,
pub source_tx_hash: HashType,
pub amount: u64,
pub metadata: Vec<u8>,
}
impl From<PendingDepositEventRecord> for db_cells::PendingDepositEventRecord {
fn from(
PendingDepositEventRecord {
deposit_op_id,
source_tx_hash,
amount,
metadata,
}: PendingDepositEventRecord,
) -> Self {
Self {
deposit_op_id,
source_tx_hash,
amount,
metadata,
}
}
}
impl From<db_cells::PendingDepositEventRecord> for PendingDepositEventRecord {
fn from(
db_cells::PendingDepositEventRecord {
deposit_op_id,
source_tx_hash,
amount,
metadata,
}: db_cells::PendingDepositEventRecord,
) -> Self {
Self {
deposit_op_id,
source_tx_hash,
amount,
metadata,
}
}
}
/// A cross-zone delivery read off a peer block but not yet known to be
/// irreversibly delivered.
///
/// The watcher's delivery floor is durable, so once it advances past a peer
/// block that block is never re-read; this record stands in its place. It
/// carries no "submitted" mark: it is dropped when the delivery finalizes, and
/// re-including one meanwhile is harmless because the inbox no-ops a replay on
/// chain.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PendingCrossZoneDispatchRecord {
/// Content-addressed replay key of the delivered message, and this record's
/// identity.
pub message_key: [u8; 32],
/// The borsh-encoded dispatch transaction, so production can re-feed it
/// without re-reading the peer channel.
pub transaction: Vec<u8>,
/// Production attempts that ended in an execution failure. Past a threshold
/// the record leaves this list for a [`DeadLetterDispatchRecord`].
pub failed_attempts: u32,
}
impl PendingCrossZoneDispatchRecord {
/// A delivery the watcher has just read: never attempted.
#[must_use]
pub const fn recorded(message_key: [u8; 32], transaction: Vec<u8>) -> Self {
Self {
message_key,
transaction,
failed_attempts: 0,
}
}
}
impl From<PendingCrossZoneDispatchRecord> for db_cells::PendingCrossZoneDispatchRecord {
fn from(
PendingCrossZoneDispatchRecord {
message_key,
transaction,
failed_attempts,
}: PendingCrossZoneDispatchRecord,
) -> Self {
Self {
message_key,
transaction,
failed_attempts,
}
}
}
impl From<db_cells::PendingCrossZoneDispatchRecord> for PendingCrossZoneDispatchRecord {
fn from(
db_cells::PendingCrossZoneDispatchRecord {
message_key,
transaction,
failed_attempts,
}: db_cells::PendingCrossZoneDispatchRecord,
) -> Self {
Self {
message_key,
transaction,
failed_attempts,
}
}
}
/// Which peer message a delivery carried, kept so a lost one can be traced back
/// to the peer block it was in.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DispatchOrigin {
pub src_zone: PeerZoneKey,
pub src_block_id: u64,
pub src_tx_index: u32,
}
impl From<DispatchOrigin> for db_cells::DispatchOrigin {
fn from(
DispatchOrigin {
src_zone,
src_block_id,
src_tx_index,
}: DispatchOrigin,
) -> Self {
Self {
src_zone,
src_block_id,
src_tx_index,
}
}
}
impl From<db_cells::DispatchOrigin> for DispatchOrigin {
fn from(
db_cells::DispatchOrigin {
src_zone,
src_block_id,
src_tx_index,
}: db_cells::DispatchOrigin,
) -> Self {
Self {
src_zone,
src_block_id,
src_tx_index,
}
}
}
/// A cross-zone delivery this node has given up on.
///
/// A dispatch that fails execution is left out of the block, so nothing on
/// chain records that it was attempted; this is the only durable trace. It
/// identifies the message rather than carrying it: the peer block and index are
/// enough to read it back off the channel.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DeadLetterDispatchRecord {
pub message_key: [u8; 32],
pub origin: DispatchOrigin,
/// Attempts made before giving up, so the record carries the policy that was
/// in force at the time.
pub failed_attempts: u32,
/// Size of the delivery transaction that would not execute, the diagnostic
/// for size-related failures.
pub transaction_bytes: u32,
}
impl From<db_cells::DeadLetterDispatchRecord> for DeadLetterDispatchRecord {
fn from(
db_cells::DeadLetterDispatchRecord {
message_key,
origin,
failed_attempts,
transaction_bytes,
}: db_cells::DeadLetterDispatchRecord,
) -> Self {
Self {
message_key,
origin: origin.into(),
failed_attempts,
transaction_bytes,
}
}
}
/// What counting a failed production attempt did to a delivery's record, the
/// reply to [`RecordDispatchFailure`].
///
/// Three outcomes rather than a bool: only one means this node stopped trying,
/// and a settled delivery has no record, so it is [`Self::Absent`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DispatchFailure {
/// Counted; the delivery is still pending and will be attempted again.
Retried { failed_attempts: u32 },
/// Given up on: moved out of the pending list and into the dead letter.
Retired(Box<DeadLetterDispatchRecord>),
/// No pending record, so nothing was counted and nothing was given up on.
Absent,
}
impl From<db::DispatchFailure> for DispatchFailure {
fn from(value: db::DispatchFailure) -> Self {
match value {
db::DispatchFailure::Retried { failed_attempts } => Self::Retried { failed_attempts },
db::DispatchFailure::Retired(record) => Self::Retired(Box::new((*record).into())),
db::DispatchFailure::Absent => Self::Absent,
}
}
}
/// What [`ApplyStoreUpdate`] observed while staging, for the caller to act on
/// *after* the write committed.
#[derive(Debug, Default)]
pub struct StoreUpdateOutcome {
/// How many deposit events were newly recorded; the rest were already
/// pending, and so already owed.
pub accepted_deposits: usize,
/// Withdraw events with no matching local unseen counter, one entry per
/// unmatched occurrence.
pub unmatched_withdrawals: Vec<WithdrawalReconciliationKey>,
}
impl From<db::StoreUpdateOutcome> for StoreUpdateOutcome {
fn from(
db::StoreUpdateOutcome {
accepted_deposits,
unmatched_withdrawals,
}: db::StoreUpdateOutcome,
) -> Self {
Self {
accepted_deposits,
unmatched_withdrawals: unmatched_withdrawals.into_iter().map(Into::into).collect(),
}
}
}
/// Schema-agnostic snapshot of a whole store, opaque by design.
pub struct DbDump(db::DbDump);
impl DbDump {
/// Serializes the dump to a compressed blob.
pub fn to_bytes(&self) -> Result<Vec<u8>> {
self.0.to_bytes().map_err(Into::into)
}
/// Reads back a dump produced by [`Self::to_bytes`].
pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
db::DbDump::from_bytes(bytes).map(Self).map_err(Into::into)
}
/// The wrapped dump, for the restore path.
pub(crate) const fn as_db_dump(&self) -> &db::DbDump {
&self.0
}
}
impl From<db::DbDump> for DbDump {
fn from(value: db::DbDump) -> Self {
Self(value)
}
}
lez/sequencer/actors/storage/src/trait.rs
+63
View File
@@ -0,0 +1,63 @@
use common::block::{Block, BlockMeta, PeerChainTip};
use kameo::{Actor, message::Message};
use lee::V03State;
use lee_core::BlockId;
use crate::{
Result,
error::Error,
protocol::{
AddPendingCrossZoneDispatches, AddPendingDepositEvent, ApplyStoreUpdate,
CleanPendingBlocksUpTo, ConsumeUnseenWithdrawCount, DbDump, DeadLetterDispatchRecord,
DeleteBlock, DeleteCrossZonePeerFloor, DeleteZoneCheckpoint, DispatchFailure,
DropSettledCrossZoneDispatches, DumpDb, GetAllBlocks, GetBlock, GetCrossZonePeerFloorBytes,
GetCrossZonePeerTip, GetDeadLetterDispatchCount, GetDeadLetterDispatches, GetFinalSnapshot,
GetFirstBlockId, GetLastBlockId, GetLatestBlockMeta, GetLeeState,
GetPendingCrossZoneDispatches, GetPendingDepositEvents, GetPublishedHighWater,
GetZoneAnchor, GetZoneCheckpointBytes, MarkBlockAsFinalized,
PendingCrossZoneDispatchRecord, PendingDepositEventRecord, RaisePublishedHighWater,
RecordDispatchFailure, RecordNewBlock, ResetAllBlocksToPending, SetCrossZonePeerFloorBytes,
SetCrossZonePeerTip, SetZoneAnchor, SetZoneCheckpointBytes, StoreUpdateOutcome,
ZoneAnchorRecord,
},
};
pub trait StorageActorTrait:
Actor<Args = Self, Error = Error>
+ Message<RecordNewBlock, Reply = Result<()>>
+ Message<ApplyStoreUpdate, Reply = Result<StoreUpdateOutcome>>
+ Message<GetBlock, Reply = Result<Option<Block>>>
+ Message<GetAllBlocks, Reply = Result<Vec<Block>>>
+ Message<DeleteBlock, Reply = Result<()>>
+ Message<MarkBlockAsFinalized, Reply = Result<()>>
+ Message<CleanPendingBlocksUpTo, Reply = Result<()>>
+ Message<ResetAllBlocksToPending, Reply = Result<()>>
+ Message<GetFirstBlockId, Reply = Result<Option<BlockId>>>
+ Message<GetLastBlockId, Reply = Result<Option<BlockId>>>
+ Message<GetLatestBlockMeta, Reply = Result<Option<BlockMeta>>>
+ Message<GetLeeState, Reply = Result<Option<V03State>>>
+ Message<GetFinalSnapshot, Reply = Result<Option<(V03State, BlockMeta)>>>
+ Message<GetZoneCheckpointBytes, Reply = Result<Option<Vec<u8>>>>
+ Message<SetZoneCheckpointBytes, Reply = Result<()>>
+ Message<DeleteZoneCheckpoint, Reply = Result<()>>
+ Message<GetZoneAnchor, Reply = Result<Option<ZoneAnchorRecord>>>
+ Message<SetZoneAnchor, Reply = Result<()>>
+ Message<GetPublishedHighWater, Reply = Result<Option<BlockId>>>
+ Message<RaisePublishedHighWater, Reply = Result<()>>
+ Message<GetPendingDepositEvents, Reply = Result<Vec<PendingDepositEventRecord>>>
+ Message<AddPendingDepositEvent, Reply = Result<bool>>
+ Message<ConsumeUnseenWithdrawCount, Reply = Result<bool>>
+ Message<GetPendingCrossZoneDispatches, Reply = Result<Vec<PendingCrossZoneDispatchRecord>>>
+ Message<AddPendingCrossZoneDispatches, Reply = Result<usize>>
+ Message<DropSettledCrossZoneDispatches, Reply = Result<usize>>
+ Message<RecordDispatchFailure, Reply = Result<DispatchFailure>>
+ Message<GetDeadLetterDispatches, Reply = Result<Vec<DeadLetterDispatchRecord>>>
+ Message<GetDeadLetterDispatchCount, Reply = Result<u64>>
+ Message<GetCrossZonePeerFloorBytes, Reply = Result<Option<Vec<u8>>>>
+ Message<SetCrossZonePeerFloorBytes, Reply = Result<()>>
+ Message<DeleteCrossZonePeerFloor, Reply = Result<()>>
+ Message<GetCrossZonePeerTip, Reply = Result<Option<PeerChainTip>>>
+ Message<SetCrossZonePeerTip, Reply = Result<()>>
+ Message<DumpDb, Reply = Result<DbDump>>
{
}
lez/sequencer/core/Cargo.toml
+4 -2
View File
@@ -13,7 +13,7 @@ lee_core.workspace = true
chain_state.workspace = true
common.workspace = true
sequencer_core_metrics = { workspace = true, features = ["record"] }
storage.workspace = true
sequencer_storage_actor.workspace = true
mempool.workspace = true
logos-blockchain-zone-sdk.workspace = true
testnet_initial_state.workspace = true
@@ -30,6 +30,7 @@ sequencer_stake_core.workspace = true
logos-blockchain-key-management-system-service.workspace = true
logos-blockchain-core.workspace = true
logos-blockchain-http-api-common.workspace = true
kameo.workspace = true
anyhow.workspace = true
serde.workspace = true
serde_json.workspace = true
@@ -48,7 +49,6 @@ url.workspace = true
num-bigint.workspace = true
risc0-zkvm.workspace = true
futures.workspace = true
itertools.workspace = true
libp2p.workspace = true
[features]
@@ -61,6 +61,8 @@ mdns = []
[dev-dependencies]
cross_zone = { workspace = true, features = ["test-utils"] }
sequencer_storage_actor = { workspace = true, features = ["mock"] }
storage.workspace = true
futures.workspace = true
test_programs.workspace = true
lee = { workspace = true, features = ["test-utils"] }
lez/sequencer/core/src/block_publisher.rs
+3 -7
View File
@@ -2,7 +2,7 @@ use std::time::Duration;
use anyhow::{Context as _, Result, anyhow, ensure};
use common::block::Block;
use futures::Stream;
use futures::{Stream, future::BoxFuture};
use log::{info, warn};
pub use logos_blockchain_core::mantle::{
ledger::NoteId,
@@ -77,7 +77,7 @@ pub struct FollowUpdate {
/// Sink for the follow path: apply the channel delta to chain state and
/// persist the whole event in one write.
pub type OnFollowSink = Box<dyn Fn(FollowUpdate) + Send + 'static>;
pub type OnFollowSink = Box<dyn Fn(FollowUpdate) -> BoxFuture<'static, ()> + Send + Sync + 'static>;
/// What one publish produced.
pub struct PublishOutcome {
@@ -417,10 +417,6 @@ impl BlockPublisherTrait for ZoneSdkPublisher {
}
}
// Nothing is awaited here: an await in this
// arm blocks the same task `publish_block`
// needs, and a non-turn sequencer never
// drains what it would be waiting on.
on_follow(FollowUpdate {
checkpoint,
adopted,
@@ -428,7 +424,7 @@ impl BlockPublisherTrait for ZoneSdkPublisher {
finalized: finalized_blocks,
deposits,
withdrawals,
});
}).await;
}
Event::Ready => {}
Event::TurnNotification { notification } => {
lez/sequencer/core/src/block_store.rs
+320 -285
View File
@@ -1,4 +1,4 @@
use std::{collections::HashMap, path::Path, sync::Arc};
use std::{collections::HashMap, sync::Arc};
use anyhow::{Context as _, Result};
use common::{
@@ -6,122 +6,134 @@ use common::{
block::{Block, BlockMeta},
transaction::LeeTransaction,
};
use kameo::actor::ActorRef;
use lee::V03State;
use lee_core::BlockId;
use logos_blockchain_zone_sdk::{Slot, sequencer::SequencerCheckpoint};
use storage::sequencer::{
RocksDBIO,
sequencer_cells::{
PeerZoneKey, PendingDepositEventRecord, WithdrawalReconciliationKey, ZoneAnchorRecord,
use logos_blockchain_zone_sdk::sequencer::SequencerCheckpoint;
use sequencer_storage_actor::{
StorageActorTrait,
protocol::{
CleanPendingBlocksUpTo, DeadLetterDispatchRecord, DeleteBlock, DeleteZoneCheckpoint,
DispatchFailure, DispatchOrigin, DropSettledCrossZoneDispatches, GetAllBlocks, GetBlock,
GetDeadLetterDispatchCount, GetDeadLetterDispatches, GetFinalSnapshot, GetFirstBlockId,
GetLastBlockId, GetLatestBlockMeta, GetLeeState, GetPendingCrossZoneDispatches,
GetPendingDepositEvents, GetPublishedHighWater, GetZoneAnchor, GetZoneCheckpointBytes,
MarkBlockAsFinalized, PendingCrossZoneDispatchRecord, PendingDepositEventRecord,
RaisePublishedHighWater, RecordDispatchFailure, RecordNewBlock, ResetAllBlocksToPending,
SetZoneAnchor, SetZoneCheckpointBytes, WithdrawalReconciliationKey, ZoneAnchorRecord,
},
};
pub use storage::{DbResult, sequencer::DbDump};
pub struct SequencerStore {
dbio: Arc<RocksDBIO>,
// TODO: Refactor this to be `Cache` (or remove entirely) and move other fields to
// [`crate::SequencerCore`].
pub struct SequencerStore<S: StorageActorTrait> {
storage_ref: ActorRef<S>,
// TODO: Consider adding the hashmap to the database for faster recovery.
tx_hash_to_block_map: HashMap<HashType, BlockId>,
genesis_id: u64,
signing_key: lee::PrivateKey,
}
impl SequencerStore {
/// Open existing database at the given location. Fails if no database is found.
pub fn open_db(location: &Path, signing_key: lee::PrivateKey) -> DbResult<Self> {
let dbio = Arc::new(RocksDBIO::open(location)?);
Self::from_dbio_and_signing_key(dbio, signing_key)
}
impl<S: StorageActorTrait> SequencerStore<S> {
pub async fn new(storage_ref: ActorRef<S>, signing_key: lee::PrivateKey) -> Result<Self> {
let genesis_id = storage_ref
.ask(GetFirstBlockId)
.await?
.context("Store holds no chain; it must be seeded with a genesis block first")?;
/// Create a fresh rocksdb at `location` from `dump`.
pub fn restore_db_from_dump(
location: &Path,
dump: &DbDump,
signing_key: lee::PrivateKey,
) -> DbResult<Self> {
let dbio = Arc::new(RocksDBIO::restore_from_dump(location, dump)?);
Self::from_dbio_and_signing_key(dbio, signing_key)
}
/// Starting database at the start of new chain.
/// Creates files if necessary.
///
/// ATTENTION: Will overwrite genesis block.
pub fn create_db_with_genesis(
location: &Path,
genesis_block: &Block,
genesis_state: &V03State,
signing_key: lee::PrivateKey,
) -> DbResult<Self> {
let dbio = Arc::new(RocksDBIO::create(location, genesis_block, genesis_state)?);
let genesis_id = dbio.get_meta_first_block_in_db()?;
let tx_hash_to_block_map = block_to_transactions_map(genesis_block);
Ok(Self {
dbio,
tx_hash_to_block_map,
genesis_id,
signing_key,
})
}
fn from_dbio_and_signing_key(
dbio: Arc<RocksDBIO>,
signing_key: lee::PrivateKey,
) -> DbResult<Self> {
let genesis_id = dbio.get_meta_first_block_in_db()?;
let last_id = dbio.latest_block_meta()?.map(|meta| meta.id);
let last_id = storage_ref
.ask(GetLastBlockId)
.await?
.context("Store holds a first block but no last one")?;
let mut tx_hash_to_block_map = HashMap::new();
if let Some(last_id) = last_id {
log::info!("Preparing block cache");
for i in genesis_id..=last_id {
let block = dbio
.get_block(i)?
.expect("Block should be present in the database");
log::info!("Preparing block cache");
for i in genesis_id..=last_id {
let block = storage_ref
.ask(GetBlock { block_id: i })
.await?
.expect("Block should be present in the database");
tx_hash_to_block_map.extend(block_to_transactions_map(&block));
}
log::info!(
"Block cache prepared. Total blocks in cache: {}",
tx_hash_to_block_map.len()
);
tx_hash_to_block_map.extend(block_to_transactions_map(&block));
}
log::info!(
"Block cache prepared. Total blocks in cache: {}",
tx_hash_to_block_map.len()
);
Ok(Self {
dbio,
storage_ref,
tx_hash_to_block_map,
genesis_id,
signing_key,
})
}
/// Shared handle to the underlying rocksdb. Used to persist the zone-sdk
/// checkpoint from the sequencer's drive task without needing &mut to the
/// store.
#[must_use]
pub fn dbio(&self) -> Arc<RocksDBIO> {
Arc::clone(&self.dbio)
pub async fn block_at_id(&self, id: u64) -> Result<Option<Block>> {
self.storage_ref
.ask(GetBlock { block_id: id })
.await
.map_err(Into::into)
}
pub fn get_block_at_id(&self, id: u64) -> DbResult<Option<Block>> {
self.dbio.get_block(id)
pub async fn get_all_blocks(&self) -> Result<Vec<Block>> {
self.storage_ref.ask(GetAllBlocks).await.map_err(Into::into)
}
pub fn delete_block_at_id(&mut self, block_id: u64) -> DbResult<()> {
self.dbio.delete_block(block_id)
pub async fn delete_block_at_id(&mut self, block_id: u64) -> Result<()> {
self.storage_ref
.ask(DeleteBlock { block_id })
.await
.map_err(Into::into)
}
pub fn mark_block_as_finalized(&mut self, block_id: u64) -> DbResult<()> {
self.dbio.mark_block_as_finalized(block_id)
pub async fn mark_block_as_finalized(&mut self, block_id: u64) -> Result<()> {
self.storage_ref
.ask(MarkBlockAsFinalized { block_id })
.await
.map_err(Into::into)
}
/// Reset every stored block to `Pending` so the next fresh start republishes the whole chain.
pub async fn reset_all_blocks_to_pending(&self) -> Result<()> {
self.storage_ref
.ask(ResetAllBlocksToPending)
.await
.map_err(Into::into)
}
/// Persists `block` with the effects it covers and extends the transaction
/// cache with the block's transactions.
pub async fn record_new_block(
&mut self,
block: Block,
withdrawals: Vec<WithdrawalReconciliationKey>,
state: Arc<V03State>,
checkpoint_bytes: Option<Vec<u8>>,
) -> Result<()> {
let new_transactions_map = block_to_transactions_map(&block);
self.storage_ref
.ask(RecordNewBlock {
block,
withdrawals,
state,
checkpoint_bytes,
})
.await?;
self.tx_hash_to_block_map.extend(new_transactions_map);
Ok(())
}
/// Returns the transaction corresponding to the given hash, if it exists in the blockchain.
#[must_use]
pub fn get_transaction_by_hash(&self, hash: HashType) -> Option<(LeeTransaction, BlockId)> {
pub async fn get_transaction_by_hash(
&self,
hash: HashType,
) -> Option<(LeeTransaction, BlockId)> {
let block_id = *self.tx_hash_to_block_map.get(&hash)?;
let block = self
.get_block_at_id(block_id)
.block_at_id(block_id)
.await
.ok()
.flatten()
.expect("Block should be present since the hash is in the map");
@@ -135,8 +147,19 @@ impl SequencerStore {
);
}
pub fn latest_block_meta(&self) -> DbResult<Option<BlockMeta>> {
self.dbio.latest_block_meta()
/// The id of the chain's last block, or `None` on a store holding no chain.
pub async fn last_block_id(&self) -> Result<Option<BlockId>> {
self.storage_ref
.ask(GetLastBlockId)
.await
.map_err(Into::into)
}
pub async fn latest_block_meta(&self) -> Result<Option<BlockMeta>> {
self.storage_ref
.ask(GetLatestBlockMeta)
.await
.map_err(Into::into)
}
#[must_use]
@@ -149,45 +172,22 @@ impl SequencerStore {
&self.signing_key
}
pub fn get_all_blocks(&self) -> impl Iterator<Item = DbResult<Block>> {
self.dbio.get_all_blocks()
}
pub(crate) fn update(
&mut self,
block: &Block,
withdrawals: &[WithdrawalReconciliationKey],
state: &V03State,
checkpoint: Option<&[u8]>,
) -> DbResult<()> {
let new_transactions_map = block_to_transactions_map(block);
self.dbio
.atomic_update(block, withdrawals, state, checkpoint)?;
self.tx_hash_to_block_map.extend(new_transactions_map);
Ok(())
}
pub fn get_lee_state(&self) -> DbResult<V03State> {
self.dbio.get_lee_state()
/// The state after the last applied block, or `None` on a store holding no
/// chain.
pub async fn get_lee_state(&self) -> Result<Option<V03State>> {
self.storage_ref.ask(GetLeeState).await.map_err(Into::into)
}
/// Remove the persisted zone-sdk checkpoint so the next startup is treated as a fresh start.
pub fn delete_zone_checkpoint(&self) -> DbResult<()> {
self.dbio.delete_zone_sdk_checkpoint_bytes()
pub async fn delete_zone_checkpoint(&self) -> Result<()> {
self.storage_ref
.ask(DeleteZoneCheckpoint)
.await
.map_err(Into::into)
}
/// Reset every stored block to `Pending` so the next fresh start republishes the whole chain.
pub fn reset_all_blocks_to_pending(&self) -> DbResult<()> {
self.dbio.reset_all_blocks_to_pending()
}
/// Single-blob [`DbDump`] of the whole store; restore with [`Self::restore_db_from_dump`].
pub fn dump(&self) -> DbResult<DbDump> {
self.dbio.dump_all()
}
pub fn get_zone_checkpoint(&self) -> Result<Option<SequencerCheckpoint>> {
let Some(bytes) = self.dbio.get_zone_sdk_checkpoint_bytes()? else {
pub async fn get_zone_checkpoint(&self) -> Result<Option<SequencerCheckpoint>> {
let Some(bytes) = self.storage_ref.ask(GetZoneCheckpointBytes).await? else {
return Ok(None);
};
let checkpoint: SequencerCheckpoint = serde_json::from_slice(&bytes)
@@ -197,36 +197,131 @@ impl SequencerStore {
/// Persists `checkpoint` on its own. Only valid when the effects it covers
/// are already durable — otherwise it must ride in the same write as them,
/// via [`storage::sequencer::StoreUpdate`].
pub fn set_zone_checkpoint(&self, checkpoint: &SequencerCheckpoint) -> Result<()> {
self.dbio
.put_zone_sdk_checkpoint_bytes(&checkpoint_bytes(checkpoint)?)?;
/// via [`Self::record_new_block`].
pub async fn set_zone_checkpoint(&self, checkpoint: &SequencerCheckpoint) -> Result<()> {
self.storage_ref
.ask(SetZoneCheckpointBytes {
bytes: checkpoint_bytes(checkpoint)?,
})
.await?;
Ok(())
}
/// The last channel block read back and verified from Bedrock (L1 slot +
/// `id`/`hash`), or `None` before any block has been read from the channel.
pub fn get_zone_anchor(&self) -> DbResult<Option<ZoneAnchorRecord>> {
self.dbio.get_zone_anchor()
pub async fn get_zone_anchor(&self) -> Result<Option<ZoneAnchorRecord>> {
self.storage_ref
.ask(GetZoneAnchor)
.await
.map_err(Into::into)
}
pub fn set_zone_anchor(&self, anchor: &ZoneAnchorRecord) -> DbResult<()> {
self.dbio.put_zone_anchor(anchor)
pub async fn set_zone_anchor(&self, anchor: ZoneAnchorRecord) -> Result<()> {
self.storage_ref
.ask(SetZoneAnchor { anchor })
.await
.map_err(Into::into)
}
/// The highest block id ever inscribed on the channel by this sequencer,
/// or `None` before it has published anything.
pub fn published_high_water(&self) -> DbResult<Option<u64>> {
self.dbio.published_high_water()
pub async fn published_high_water(&self) -> Result<Option<u64>> {
self.storage_ref
.ask(GetPublishedHighWater)
.await
.map_err(Into::into)
}
/// Raises the published high water mark to `block_id`, never lowering it.
pub fn raise_published_high_water(&self, block_id: u64) -> DbResult<()> {
self.dbio.raise_published_high_water(block_id)
pub async fn raise_published_high_water(&self, block_id: u64) -> Result<()> {
self.storage_ref
.ask(RaisePublishedHighWater { block_id })
.await
.map_err(Into::into)
}
pub fn get_pending_deposit_events(&self) -> DbResult<Vec<PendingDepositEventRecord>> {
self.dbio.get_pending_deposit_events()
pub async fn get_pending_deposit_events(&self) -> Result<Vec<PendingDepositEventRecord>> {
self.storage_ref
.ask(GetPendingDepositEvents)
.await
.map_err(Into::into)
}
/// The persisted final-tier `(state, meta)`, or `None` before anything
/// finalized.
pub async fn get_final_snapshot(&self) -> Result<Option<(V03State, BlockMeta)>> {
self.storage_ref
.ask(GetFinalSnapshot)
.await
.map_err(Into::into)
}
/// Marks every stored pending block at or below `last_finalized` as
/// finalized.
pub async fn clean_pending_blocks_up_to(&self, last_finalized: BlockId) -> Result<()> {
self.storage_ref
.ask(CleanPendingBlocksUpTo { last_finalized })
.await
.map_err(Into::into)
}
pub async fn pending_cross_zone_dispatches(
&self,
) -> Result<Vec<PendingCrossZoneDispatchRecord>> {
self.storage_ref
.ask(GetPendingCrossZoneDispatches)
.await
.map_err(Into::into)
}
pub async fn drop_settled_cross_zone_dispatches(
&self,
message_keys: Vec<[u8; 32]>,
) -> Result<usize> {
self.storage_ref
.ask(DropSettledCrossZoneDispatches { message_keys })
.await
.map_err(Into::into)
}
/// Counts one failed production attempt against `message_key`, giving up on
/// it once `retire_at` accumulate.
pub async fn record_dispatch_failure(
&self,
message_key: [u8; 32],
retire_at: u32,
origin: DispatchOrigin,
) -> Result<DispatchFailure> {
self.storage_ref
.ask(RecordDispatchFailure {
message_key,
retire_at,
origin,
})
.await
.map_err(Into::into)
}
pub async fn dead_letter_dispatches(&self) -> Result<Vec<DeadLetterDispatchRecord>> {
self.storage_ref
.ask(GetDeadLetterDispatches)
.await
.map_err(Into::into)
}
pub async fn dead_letter_dispatch_count(&self) -> Result<u64> {
self.storage_ref
.ask(GetDeadLetterDispatchCount)
.await
.map_err(Into::into)
}
/// The handle to the actor behind this store, for the paths that hold no
/// store of their own: the publisher's follow sink and the cross-zone
/// watchers, each of which outlives any one caller.
#[must_use]
pub const fn storage_ref(&self) -> &ActorRef<S> {
&self.storage_ref
}
}
@@ -245,240 +340,180 @@ pub(crate) fn block_to_transactions_map(block: &Block) -> HashMap<HashType, u64>
.collect()
}
/// A cross-zone watcher's delivery floor on `peer_zone`'s channel.
///
/// The highest slot every message of which was delivered, or `None` before it
/// has delivered anything from that peer. Stored as a little-endian `u64`.
///
/// Free functions rather than only [`SequencerStore`] methods because each
/// watcher runs as its own spawned task and holds an `Arc<RocksDBIO>`;
/// `SequencerStore` is not `Clone`.
pub fn get_cross_zone_peer_floor(dbio: &RocksDBIO, peer_zone: PeerZoneKey) -> Result<Option<Slot>> {
let Some(bytes) = dbio.get_cross_zone_peer_floor_bytes(peer_zone)? else {
return Ok(None);
};
let bytes: [u8; 8] = bytes.as_slice().try_into().with_context(|| {
format!(
"Stored cross-zone peer floor is {} bytes, expected 8",
bytes.len()
)
})?;
Ok(Some(Slot::new(u64::from_le_bytes(bytes))))
}
pub fn set_cross_zone_peer_floor(
dbio: &RocksDBIO,
peer_zone: PeerZoneKey,
floor: Slot,
) -> Result<()> {
dbio.put_cross_zone_peer_floor_bytes(peer_zone, &floor.to_le_bytes())?;
Ok(())
}
/// Drops the stored floor so the watcher reads `peer_zone`'s channel from the
/// peer's genesis again.
pub fn clear_cross_zone_peer_floor(dbio: &RocksDBIO, peer_zone: PeerZoneKey) -> Result<()> {
dbio.delete_cross_zone_peer_floor(peer_zone)?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::path::Path;
use common::{block::HashableBlockData, test_utils::sequencer_sign_key_for_testing};
use kameo::actor::Spawn as _;
use sequencer_storage_actor::StorageActor;
use tempfile::tempdir;
use super::*;
#[test]
fn get_transaction_by_hash() {
let temp_dir = tempdir().unwrap();
let path = temp_dir.path();
let signing_key = sequencer_sign_key_for_testing();
let genesis_block_hashable_data = HashableBlockData {
fn genesis_block(signing_key: &lee::PrivateKey) -> Block {
HashableBlockData {
block_id: 0,
prev_block_hash: HashType([0; 32]),
timestamp: 0,
transactions: vec![],
};
}
.into_pending_block(signing_key)
}
let genesis_block = genesis_block_hashable_data.into_pending_block(&signing_key);
// Start an empty node store
let mut node_store = SequencerStore::create_db_with_genesis(
path,
&genesis_block,
&testnet_initial_state::initial_state(),
signing_key,
/// Creates a fresh database at `path` seeded with `genesis` and opens a
/// store on the actor serving it.
async fn create_store(
path: &Path,
genesis: &Block,
signing_key: lee::PrivateKey,
) -> SequencerStore<StorageActor> {
let storage_ref = StorageActor::spawn(StorageActor::new(path).unwrap());
storage_ref
.ask(RecordNewBlock {
block: genesis.clone(),
withdrawals: vec![],
state: Arc::new(testnet_initial_state::initial_state()),
checkpoint_bytes: None,
})
.await
.unwrap();
SequencerStore::new(storage_ref, signing_key).await.unwrap()
}
async fn open_store(path: &Path, signing_key: lee::PrivateKey) -> SequencerStore<StorageActor> {
let storage_ref = StorageActor::spawn(StorageActor::new(path).unwrap());
SequencerStore::new(storage_ref, signing_key).await.unwrap()
}
#[tokio::test]
async fn get_transaction_by_hash() {
let temp_dir = tempdir().unwrap();
let signing_key = sequencer_sign_key_for_testing();
let mut store = create_store(
temp_dir.path(),
&genesis_block(&signing_key),
signing_key.clone(),
)
.unwrap();
.await;
let tx = common::test_utils::produce_dummy_empty_transaction();
let block = common::test_utils::produce_dummy_block(1, None, vec![tx.clone()]);
// Try retrieve a tx that's not in the chain yet.
let retrieved_tx = node_store.get_transaction_by_hash(tx.hash());
assert_eq!(None, retrieved_tx);
assert_eq!(store.get_transaction_by_hash(tx.hash()).await, None);
// Add the block with the transaction
let dummy_state = V03State::new();
node_store.update(&block, &[], &dummy_state, None).unwrap();
store
.record_new_block(block.clone(), vec![], Arc::new(V03State::new()), None)
.await
.unwrap();
// Try again
let output = node_store.get_transaction_by_hash(tx.hash());
let output = store.get_transaction_by_hash(tx.hash()).await;
assert_eq!(Some((tx, 1)), output);
}
#[test]
fn latest_block_meta_returns_genesis_meta_initially() {
#[tokio::test]
async fn latest_block_meta_returns_genesis_meta_initially() {
let temp_dir = tempdir().unwrap();
let path = temp_dir.path();
let signing_key = sequencer_sign_key_for_testing();
let genesis = genesis_block(&signing_key);
let genesis_hash = genesis.header.hash;
let genesis_block_hashable_data = HashableBlockData {
block_id: 0,
prev_block_hash: HashType([0; 32]),
timestamp: 0,
transactions: vec![],
};
let genesis_block = genesis_block_hashable_data.into_pending_block(&signing_key);
let genesis_hash = genesis_block.header.hash;
let node_store = SequencerStore::create_db_with_genesis(
path,
&genesis_block,
&testnet_initial_state::initial_state(),
signing_key,
)
.unwrap();
let store = create_store(temp_dir.path(), &genesis, signing_key).await;
// Verify that initially the latest block hash equals genesis hash
let latest_meta = node_store.latest_block_meta().unwrap().unwrap();
let latest_meta = store.latest_block_meta().await.unwrap().unwrap();
assert_eq!(latest_meta.hash, genesis_hash);
}
#[test]
fn latest_block_meta_updates_after_new_block() {
#[tokio::test]
async fn latest_block_meta_updates_after_new_block() {
let temp_dir = tempdir().unwrap();
let path = temp_dir.path();
let signing_key = sequencer_sign_key_for_testing();
let genesis_block_hashable_data = HashableBlockData {
block_id: 0,
prev_block_hash: HashType([0; 32]),
timestamp: 0,
transactions: vec![],
};
let genesis_block = genesis_block_hashable_data.into_pending_block(&signing_key);
let mut node_store = SequencerStore::create_db_with_genesis(
path,
&genesis_block,
&testnet_initial_state::initial_state(),
signing_key,
let mut store = create_store(
temp_dir.path(),
&genesis_block(&signing_key),
signing_key.clone(),
)
.unwrap();
.await;
// Add a new block
let tx = common::test_utils::produce_dummy_empty_transaction();
let block = common::test_utils::produce_dummy_block(1, None, vec![tx]);
let block_hash = block.header.hash;
let dummy_state = V03State::new();
node_store.update(&block, &[], &dummy_state, None).unwrap();
store
.record_new_block(block.clone(), vec![], Arc::new(V03State::new()), None)
.await
.unwrap();
// Verify that the latest block meta now equals the new block's hash
let latest_meta = node_store.latest_block_meta().unwrap().unwrap();
let latest_meta = store.latest_block_meta().await.unwrap().unwrap();
assert_eq!(latest_meta.hash, block_hash);
}
#[test]
fn mark_block_finalized() {
#[tokio::test]
async fn mark_block_finalized() {
let temp_dir = tempdir().unwrap();
let path = temp_dir.path();
let signing_key = sequencer_sign_key_for_testing();
let genesis_block_hashable_data = HashableBlockData {
block_id: 0,
prev_block_hash: HashType([0; 32]),
timestamp: 0,
transactions: vec![],
};
let genesis_block = genesis_block_hashable_data.into_pending_block(&signing_key);
let mut node_store = SequencerStore::create_db_with_genesis(
path,
&genesis_block,
&testnet_initial_state::initial_state(),
signing_key,
let mut store = create_store(
temp_dir.path(),
&genesis_block(&signing_key),
signing_key.clone(),
)
.unwrap();
.await;
// Add a new block with Pending status
let tx = common::test_utils::produce_dummy_empty_transaction();
let block = common::test_utils::produce_dummy_block(1, None, vec![tx]);
let block_id = block.header.block_id;
let dummy_state = V03State::new();
node_store.update(&block, &[], &dummy_state, None).unwrap();
store
.record_new_block(block.clone(), vec![], Arc::new(V03State::new()), None)
.await
.unwrap();
// Verify initial status is Pending
let retrieved_block = node_store.get_block_at_id(block_id).unwrap().unwrap();
let retrieved_block = store.block_at_id(block_id).await.unwrap().unwrap();
assert!(matches!(
retrieved_block.bedrock_status,
common::block::BedrockStatus::Pending
));
// Mark block as finalized
node_store.mark_block_as_finalized(block_id).unwrap();
store.mark_block_as_finalized(block_id).await.unwrap();
// Verify status is now Finalized
let finalized_block = node_store.get_block_at_id(block_id).unwrap().unwrap();
let finalized_block = store.block_at_id(block_id).await.unwrap().unwrap();
assert!(matches!(
finalized_block.bedrock_status,
common::block::BedrockStatus::Finalized
));
}
#[test]
fn open_existing_db_caches_transactions() {
#[tokio::test]
async fn open_existing_db_caches_transactions() {
let temp_dir = tempdir().unwrap();
let path = temp_dir.path();
let signing_key = sequencer_sign_key_for_testing();
let genesis_block_hashable_data = HashableBlockData {
block_id: 0,
prev_block_hash: HashType([0; 32]),
timestamp: 0,
transactions: vec![],
};
let genesis_block = genesis_block_hashable_data.into_pending_block(&signing_key);
let tx = common::test_utils::produce_dummy_empty_transaction();
{
// Create a scope to drop the first store after creating the db
let mut node_store = SequencerStore::create_db_with_genesis(
path,
&genesis_block,
&testnet_initial_state::initial_state(),
signing_key.clone(),
)
.unwrap();
// Add a new block
let storage_weak = {
let mut store =
create_store(path, &genesis_block(&signing_key), signing_key.clone()).await;
let block = common::test_utils::produce_dummy_block(1, None, vec![tx.clone()]);
node_store
.update(&block, &[], &V03State::new(), None)
store
.record_new_block(block.clone(), vec![], Arc::new(V03State::new()), None)
.await
.unwrap();
}
store.storage_ref().downgrade()
};
storage_weak.wait_for_shutdown_with_result(|_| ()).await;
// Re-open the store and verify that the transaction is still retrievable (which means it
// was cached correctly)
let node_store = SequencerStore::open_db(path, signing_key).unwrap();
let output = node_store.get_transaction_by_hash(tx.hash());
let store = open_store(path, signing_key).await;
let output = store.get_transaction_by_hash(tx.hash()).await;
assert_eq!(Some((tx, 1)), output);
}
}
lez/sequencer/core/src/cross_zone_watcher.rs
+295 -119
View File
@@ -1,27 +1,34 @@
use std::{sync::Arc, time::Duration};
use std::time::Duration;
use common::{HashType, block::Block, transaction::LeeTransaction};
use anyhow::{Context as _, Result};
use common::{
HashType,
block::{Block, PeerChainTip},
transaction::LeeTransaction,
};
use cross_zone::{
EmissionSource, Link, StallState, alerts_at, build_dispatch_from_emission, equivocation_report,
extract_emission, is_sequencer_only_program, link_to_tip, screen_peer_block,
};
use cross_zone_inbox_core::message_key;
use futures::{Stream, StreamExt as _};
use kameo::actor::ActorRef;
use lee::PublicKey;
use log::{debug, error, warn};
use logos_blockchain_core::mantle::ops::channel::ChannelId;
use logos_blockchain_zone_sdk::{
CommonHttpClient, Slot, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer,
};
use storage::sequencer::{
RocksDBIO,
sequencer_cells::{PeerChainTip, PendingCrossZoneDispatchRecord},
use sequencer_storage_actor::{
StorageActorTrait,
protocol::{
AddPendingCrossZoneDispatches, DeleteCrossZonePeerFloor, GetCrossZonePeerFloorBytes,
GetCrossZonePeerTip, PeerZoneKey, PendingCrossZoneDispatchRecord,
SetCrossZonePeerFloorBytes, SetCrossZonePeerTip,
},
};
use crate::{
block_store::{
clear_cross_zone_peer_floor, get_cross_zone_peer_floor, set_cross_zone_peer_floor,
},
config::{BedrockConfig, CrossZoneConfig},
task_group::TaskGroup,
};
@@ -119,6 +126,56 @@ const fn resume_from(tip: Option<PeerChainTip>, floor: Option<Slot>) -> Resume {
}
}
/// This watcher's delivery floor on `peer_zone`'s channel.
///
/// The highest slot every message of which was delivered, or `None` before it
/// has delivered anything from that peer. Stored as a little-endian `u64`, which
/// is why the encoding lives here rather than in the storage actor.
async fn get_cross_zone_peer_floor<S: StorageActorTrait>(
storage_ref: &ActorRef<S>,
peer_zone: PeerZoneKey,
) -> Result<Option<Slot>> {
let Some(bytes) = storage_ref
.ask(GetCrossZonePeerFloorBytes { peer_zone })
.await?
else {
return Ok(None);
};
let bytes: [u8; 8] = bytes.as_slice().try_into().with_context(|| {
format!(
"Stored cross-zone peer floor is {} bytes, expected 8",
bytes.len()
)
})?;
Ok(Some(Slot::new(u64::from_le_bytes(bytes))))
}
async fn set_cross_zone_peer_floor<S: StorageActorTrait>(
storage_ref: &ActorRef<S>,
peer_zone: PeerZoneKey,
floor: Slot,
) -> Result<()> {
storage_ref
.ask(SetCrossZonePeerFloorBytes {
peer_zone,
bytes: floor.to_le_bytes().to_vec(),
})
.await?;
Ok(())
}
/// Drops the stored floor so the watcher reads `peer_zone`'s channel from the
/// peer's genesis again.
async fn clear_cross_zone_peer_floor<S: StorageActorTrait>(
storage_ref: &ActorRef<S>,
peer_zone: PeerZoneKey,
) -> Result<()> {
storage_ref
.ask(DeleteCrossZonePeerFloor { peer_zone })
.await?;
Ok(())
}
/// Spawns one watcher task per configured peer.
///
/// Each task reads the peer's finalized blocks from Bedrock, recognizes outbound
@@ -129,15 +186,13 @@ const fn resume_from(tip: Option<PeerChainTip>, floor: Option<Slot>) -> Resume {
/// The returned group must be kept alive for as long as the watchers should
/// run; dropping it stops them, and awaiting
/// [`TaskGroup::shutdown`](crate::task_group::TaskGroup::shutdown) is what
/// proves they have stopped. Each watcher holds an `Arc<RocksDBIO>`, so a
/// watcher still running keeps the `RocksDB` lock held and a restarting
/// sequencer cannot reopen its home directory.
/// proves they have stopped.
#[must_use]
pub fn spawn_watchers(
pub fn spawn_watchers<S: StorageActorTrait>(
bedrock_config: &BedrockConfig,
cross_zone: &CrossZoneConfig,
poll_interval: Duration,
dbio: &Arc<RocksDBIO>,
storage_ref: &ActorRef<S>,
) -> TaskGroup {
let self_zone: [u8; 32] = *bedrock_config.channel_id.as_ref();
let mut tasks = Vec::new();
@@ -158,7 +213,7 @@ pub fn spawn_watchers(
expected_pubkey,
},
poll_interval,
Arc::clone(dbio),
storage_ref.clone(),
)));
}
@@ -169,11 +224,11 @@ pub fn spawn_watchers(
clippy::infinite_loop,
reason = "the peer watcher runs for the lifetime of the sequencer process"
)]
async fn watch_peer(
async fn watch_peer<S: StorageActorTrait>(
zone_indexer: ZoneIndexer<NodeHttpClient>,
peer: PeerContext,
poll_interval: Duration,
dbio: Arc<RocksDBIO>,
storage_ref: ActorRef<S>,
) {
let peer_zone = peer.peer_zone;
log::info!(
@@ -186,7 +241,7 @@ async fn watch_peer(
// key is content-addressed and the inbox no-ops a replay) but re-records
// every already-delivered message, so without this a restart replayed the
// peer's whole history into the store.
let floor = match get_cross_zone_peer_floor(&dbio, peer_zone) {
let floor = match get_cross_zone_peer_floor(&storage_ref, peer_zone).await {
Ok(floor) => floor,
Err(err) => {
// Falling back to `None` would re-read the peer's whole history and
@@ -202,7 +257,7 @@ async fn watch_peer(
// The chain this watcher has already delivered from. Without it no block can
// be told apart from one claiming an id it never reached, so a watcher that
// cannot read it delivers nothing rather than guessing.
let mut tip = match dbio.get_cross_zone_peer_tip(peer_zone) {
let mut tip = match storage_ref.ask(GetCrossZonePeerTip { peer_zone }).await {
Ok(tip) => tip,
Err(err) => {
error!(
@@ -220,7 +275,7 @@ async fn watch_peer(
);
// Durably, before reading anything, or a crash partway through the
// rebuild resumes from the stale floor with nothing able to link.
if let Err(err) = clear_cross_zone_peer_floor(&dbio, peer_zone) {
if let Err(err) = clear_cross_zone_peer_floor(&storage_ref, peer_zone).await {
error!(
"Watcher could not clear the stale delivery floor for peer {}: {err:#}. Stopping this watcher rather than rebuilding its chain against a floor a restart would resume from.",
hex::encode(peer_zone)
@@ -251,7 +306,7 @@ async fn watch_peer(
continue;
}
};
let outcome = consume_peer_stream(stream, &peer, &dbio, &mut cursor, &mut tip).await;
let outcome = consume_peer_stream(stream, &peer, &storage_ref, &mut cursor, &mut tip).await;
if let Some((slot, attempts)) = state.after_pass(outcome, cursor)
&& alerts_at(attempts)
@@ -259,7 +314,10 @@ async fn watch_peer(
error!(
"Watcher for peer {} has been stuck at slot {slot:?} for {attempts} passes. Nothing from that peer is being delivered until it clears, and the delivery floor stays at {:?} so the slot keeps coming back.",
hex::encode(peer_zone),
get_cross_zone_peer_floor(&dbio, peer_zone).ok().flatten()
get_cross_zone_peer_floor(&storage_ref, peer_zone)
.await
.ok()
.flatten()
);
}
// Reads on rather than stopping, since one such pass is ordinary, but a
@@ -286,15 +344,15 @@ async fn watch_peer(
///
/// Ending early holds the floor at the last slot consumed whole, so the next
/// poll re-reads from there and a transient failure heals.
async fn consume_peer_stream<S>(
stream: S,
async fn consume_peer_stream<Str, S: StorageActorTrait>(
stream: Str,
peer: &PeerContext,
dbio: &RocksDBIO,
storage_ref: &ActorRef<S>,
cursor: &mut Option<Slot>,
tip: &mut Option<PeerChainTip>,
) -> PassOutcome
where
S: Stream<Item = (ZoneMessage, Slot)>,
Str: Stream<Item = (ZoneMessage, Slot)>,
{
let mut stream = std::pin::pin!(stream);
// The slot being consumed: every message of it seen so far is handled, but
@@ -310,7 +368,7 @@ where
if in_progress != Some(slot) {
// A message from a later slot means the previous one completed.
if let Some(done) = in_progress {
advance_cursor(dbio, peer.peer_zone, cursor, done);
advance_cursor(storage_ref, peer.peer_zone, cursor, done).await;
}
in_progress = Some(slot);
}
@@ -371,7 +429,7 @@ where
);
}
Link::Next(block_hash) => {
if !record_block_deliveries(&block, block_hash, peer, dbio) {
if !record_block_deliveries(&block, block_hash, peer, storage_ref).await {
// Recording a delivery is what makes it survive the
// mempool. Letting the pass finish here would move
// the floor past this slot on a store that just
@@ -392,7 +450,13 @@ where
block_id: block.header.block_id,
block_hash,
};
if let Err(err) = dbio.put_cross_zone_peer_tip(peer.peer_zone, next) {
if let Err(err) = storage_ref
.ask(SetCrossZonePeerTip {
peer_zone: peer.peer_zone,
tip: next,
})
.await
{
// Advancing only in memory would leave a restart
// resuming from a floor above a tip, and every block
// after it unlinkable.
@@ -420,7 +484,7 @@ where
// The stream drained cleanly, so the slot in progress completed too.
if let Some(done) = in_progress {
advance_cursor(dbio, peer.peer_zone, cursor, done);
advance_cursor(storage_ref, peer.peer_zone, cursor, done).await;
}
if placed == 0 && skipped > 0 {
return PassOutcome::Stranded;
@@ -433,9 +497,14 @@ where
///
/// A persist failure is only logged: the worst case is re-reading from the last
/// stored slot after a restart, which delivery handles idempotently.
fn advance_cursor(dbio: &RocksDBIO, peer_zone: [u8; 32], cursor: &mut Option<Slot>, slot: Slot) {
async fn advance_cursor<S: StorageActorTrait>(
storage_ref: &ActorRef<S>,
peer_zone: [u8; 32],
cursor: &mut Option<Slot>,
slot: Slot,
) {
*cursor = Some(slot);
if let Err(err) = set_cross_zone_peer_floor(dbio, peer_zone, slot) {
if let Err(err) = set_cross_zone_peer_floor(storage_ref, peer_zone, slot).await {
warn!(
"Failed to persist watcher delivery floor for peer {}: {err:#}",
hex::encode(peer_zone)
@@ -451,11 +520,11 @@ fn advance_cursor(dbio: &RocksDBIO, peer_zone: [u8; 32], cursor: &mut Option<Slo
///
/// `block_hash` is the value [`screen_peer_block`] recomputed from the block's
/// own contents, not `block.header.hash`, which the signature does not cover.
fn record_block_deliveries(
async fn record_block_deliveries<S: StorageActorTrait>(
block: &Block,
block_hash: HashType,
peer: &PeerContext,
dbio: &RocksDBIO,
storage_ref: &ActorRef<S>,
) -> bool {
let peer_zone = peer.peer_zone;
let self_zone = peer.self_zone;
@@ -522,7 +591,12 @@ fn record_block_deliveries(
}
let offered = deliveries.len();
match dbio.add_pending_cross_zone_dispatches(deliveries) {
match storage_ref
.ask(AddPendingCrossZoneDispatches {
dispatches: deliveries,
})
.await
{
// Fewer accepted than offered means the rest were recorded by an earlier
// pass over the same slot, which the retry loop repeats for as long as
// the slot stays stuck.
@@ -558,12 +632,19 @@ fn record_block_deliveries(
#[cfg(test)]
mod tests {
use std::sync::{Arc, Mutex};
use common::test_utils::produce_dummy_block;
use cross_zone::test_utils::{linked_chain_to, ping_emission};
use futures::stream;
use kameo::actor::Spawn as _;
use logos_blockchain_core::mantle::ops::channel::{MsgId, inscribe::Inscription};
use logos_blockchain_zone_sdk::ZoneBlock;
use storage::sequencer::{DB_META_PENDING_CROSS_ZONE_DISPATCH_COUNT_KEY, RocksDBIO};
use sequencer_storage_actor::{
StorageActor,
mock::MockStorageActor,
protocol::{GetPendingCrossZoneDispatches, RecordNewBlock},
};
use tempfile::TempDir;
use super::*;
@@ -580,11 +661,25 @@ mod tests {
}
/// A store backed by a temp dir. The dir is returned so it outlives the db.
fn store() -> (TempDir, RocksDBIO) {
async fn store() -> (TempDir, ActorRef<StorageActor>) {
let dir = tempfile::tempdir().expect("temp dir");
let genesis = produce_dummy_block(0, None, vec![]);
let dbio = RocksDBIO::create(dir.path(), &genesis, &lee::V03State::new()).expect("db");
(dir, dbio)
let storage_ref = StorageActor::spawn(StorageActor::new(dir.path()).expect("open storage"));
seed_genesis(&storage_ref).await;
(dir, storage_ref)
}
/// The watcher's messages need a database, not a chain, but the peer-tip
/// and dispatch cells share a store with one — so seed it like a real node.
async fn seed_genesis(storage_ref: &ActorRef<StorageActor>) {
storage_ref
.ask(RecordNewBlock {
block: produce_dummy_block(0, None, vec![]),
withdrawals: vec![],
state: Arc::new(lee::V03State::new()),
checkpoint_bytes: None,
})
.await
.expect("seed genesis");
}
/// A `ping_sender` emission addressed to `SELF_ZONE`.
@@ -668,30 +763,58 @@ mod tests {
/// The message keys recorded so far, sorted: the store keys each record by
/// its message key, so no insertion order survives.
fn recorded_keys(dbio: &RocksDBIO) -> Vec<[u8; 32]> {
let mut keys: Vec<[u8; 32]> = dbio
.get_pending_cross_zone_dispatches()
async fn recorded_keys(storage_ref: &ActorRef<StorageActor>) -> Vec<[u8; 32]> {
let mut keys = storage_ref
.ask(GetPendingCrossZoneDispatches)
.await
.expect("pending dispatches readable")
.into_iter()
.map(|record| record.message_key)
.collect();
.collect::<Vec<_>>();
keys.sort_unstable();
keys
}
/// Makes every later pending-dispatch write fail, standing in for any
/// store failure before a delivery is durable: recording reads the count
/// first, so a count that will not decode is enough.
fn break_the_dispatch_store(dbio: &RocksDBIO) {
let cf = dbio
.db
.cf_handle(storage::CF_META_NAME)
.expect("meta column family");
let key =
borsh::to_vec(&DB_META_PENDING_CROSS_ZONE_DISPATCH_COUNT_KEY).expect("key encodes");
dbio.db
.put_cf(&cf, key, b"not a pending dispatch count")
.expect("write");
/// A store that refuses every delivery write, standing in for any store
/// failure between reading a peer block and the delivery being durable.
fn store_refusing_deliveries() -> ActorRef<MockStorageActor> {
let floor: Arc<Mutex<Option<Vec<u8>>>> = Arc::default();
let tip: Arc<Mutex<Option<PeerChainTip>>> = Arc::default();
let mut storage = MockStorageActor::new();
storage
.expect_handle_add_pending_cross_zone_dispatches()
.returning(|_, _| {
Err(storage::error::DbError::db_interaction_error(
"the store refused the write".to_owned(),
)
.into())
});
let written_floor = Arc::clone(&floor);
storage
.expect_handle_set_cross_zone_peer_floor_bytes()
.returning(move |msg, _| {
*written_floor.lock().expect("floor cell") = Some(msg.bytes);
Ok(())
});
storage
.expect_handle_get_cross_zone_peer_floor_bytes()
.returning(move |_, _| Ok(floor.lock().expect("floor cell").clone()));
let written_tip = Arc::clone(&tip);
storage
.expect_handle_set_cross_zone_peer_tip()
.returning(move |msg, _| {
*written_tip.lock().expect("tip cell") = Some(msg.tip);
Ok(())
});
storage
.expect_handle_get_cross_zone_peer_tip()
.returning(move |_, _| Ok(*tip.lock().expect("tip cell")));
MockStorageActor::spawn(storage)
}
/// Drives the state machine over a sequence of pass outcomes, with the read
@@ -775,14 +898,14 @@ mod tests {
#[tokio::test]
async fn watcher_persists_its_cursor_as_it_consumes() {
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
let outcome = consume_peer_stream(
stream::iter(vec![peer_block_msg(1, 0), peer_block_msg(2, 1)]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -791,13 +914,15 @@ mod tests {
assert_eq!(outcome, PassOutcome::Drained);
assert_eq!(cursor, Some(Slot::from(1)));
assert_eq!(
get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(),
get_cross_zone_peer_floor(&storage_ref, PEER_ZONE)
.await
.unwrap(),
Some(Slot::from(1)),
"the cursor must be durable, not just in memory"
);
let mut expected = vec![message_key(&PEER_ZONE, 1, 0), message_key(&PEER_ZONE, 2, 0)];
expected.sort_unstable();
assert_eq!(recorded_keys(&dbio), expected);
assert_eq!(recorded_keys(&storage_ref).await, expected);
}
#[tokio::test]
@@ -807,7 +932,7 @@ mod tests {
// certainly refuse: the inbox is injected by this node alone, so a peer
// naming it as a target is junk that would cost a pending slot and three
// execution attempts.
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
@@ -818,7 +943,7 @@ mod tests {
programs::cross_zone_inbox().id(),
)]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -830,11 +955,13 @@ mod tests {
"a message the watcher drops is not a failure"
);
assert!(
recorded_keys(&dbio).is_empty(),
recorded_keys(&storage_ref).await.is_empty(),
"a message aimed at a sequencer-only program must not be recorded"
);
assert_eq!(
get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(),
get_cross_zone_peer_floor(&storage_ref, PEER_ZONE)
.await
.unwrap(),
Some(Slot::from(0)),
"the slot was fully read, so the floor still advances"
);
@@ -844,7 +971,7 @@ mod tests {
async fn a_delivery_to_an_unrelated_target_is_still_recorded() {
// The watcher is not the authorization point any more. A target it knows
// nothing about is recorded and delivered, and that target decides.
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
@@ -855,7 +982,7 @@ mod tests {
programs::wrapped_token().id(),
)]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -863,7 +990,7 @@ mod tests {
assert_eq!(outcome, PassOutcome::Drained);
assert_eq!(
recorded_keys(&dbio).len(),
recorded_keys(&storage_ref).await.len(),
1,
"the watcher records it and lets the target refuse it"
);
@@ -871,14 +998,14 @@ mod tests {
#[tokio::test]
async fn watcher_records_every_delivery_it_reads() {
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
consume_peer_stream(
stream::iter(vec![peer_block_msg(1, 0)]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -888,7 +1015,10 @@ mod tests {
// never re-read. The record is the whole of what survives that: block
// production drains it, and it outlives a restart. It is dropped when
// the delivery itself becomes irreversible, not when it is included.
let records = dbio.get_pending_cross_zone_dispatches().unwrap();
let records = storage_ref
.ask(GetPendingCrossZoneDispatches)
.await
.unwrap();
assert_eq!(records.len(), 1, "the delivery must be recorded");
assert_eq!(
records[0].message_key,
@@ -907,20 +1037,23 @@ mod tests {
#[tokio::test]
async fn a_recorded_delivery_names_the_hash_the_watcher_validated() {
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
consume_peer_stream(
stream::iter(vec![peer_block_msg(1, 0)]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
.await;
let records = dbio.get_pending_cross_zone_dispatches().unwrap();
let records = storage_ref
.ask(GetPendingCrossZoneDispatches)
.await
.unwrap();
assert_eq!(records.len(), 1, "the delivery must be recorded");
let tx = borsh::from_slice::<LeeTransaction>(&records[0].transaction).unwrap();
let LeeTransaction::Public(public_tx) = tx else {
@@ -943,15 +1076,14 @@ mod tests {
#[tokio::test]
async fn a_delivery_that_cannot_be_recorded_holds_the_floor() {
let (_dir, dbio) = store();
break_the_dispatch_store(&dbio);
let storage_ref = store_refusing_deliveries();
let mut cursor = None;
let mut tip = None;
let outcome = consume_peer_stream(
stream::iter(vec![peer_block_msg(1, 0)]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -962,7 +1094,9 @@ mod tests {
// rather than retried.
assert_eq!(outcome, PassOutcome::Undelivered(Slot::from(0)));
assert_eq!(
get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(),
get_cross_zone_peer_floor(&storage_ref, PEER_ZONE)
.await
.unwrap(),
None,
"the slot must stay re-readable"
);
@@ -970,31 +1104,46 @@ mod tests {
// crash in between makes the re-read see the block as already delivered
// from, and its messages are never looked at again.
assert_eq!(tip, None);
assert_eq!(dbio.get_cross_zone_peer_tip(PEER_ZONE).unwrap(), None);
assert_eq!(
storage_ref
.ask(GetCrossZonePeerTip {
peer_zone: PEER_ZONE
})
.await
.unwrap(),
None
);
}
#[tokio::test]
async fn watcher_resumes_from_the_persisted_cursor_without_rereading() {
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
consume_peer_stream(
stream::iter(vec![peer_block_msg(1, 0), peer_block_msg(2, 1)]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
.await;
assert_eq!(recorded_keys(&dbio).len(), 2);
assert_eq!(recorded_keys(&storage_ref).await.len(), 2);
// Restart: a fresh watcher seeds both its cursor and its chain tip from
// the store. One that had to rebuild the tip in memory would accept
// whatever block arrived first.
let resumed = get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap();
let resumed = get_cross_zone_peer_floor(&storage_ref, PEER_ZONE)
.await
.unwrap();
assert_eq!(resumed, Some(Slot::from(1)));
let mut resumed_tip = dbio.get_cross_zone_peer_tip(PEER_ZONE).unwrap();
let mut resumed_tip = storage_ref
.ask(GetCrossZonePeerTip {
peer_zone: PEER_ZONE,
})
.await
.unwrap();
assert_eq!(
resumed_tip,
Some(tip_at(2)),
@@ -1006,7 +1155,7 @@ mod tests {
consume_peer_stream(
stream::iter(vec![peer_block_msg(3, 2)]),
&peer_context(),
&dbio,
&storage_ref,
&mut resumed_cursor,
&mut resumed_tip,
)
@@ -1019,19 +1168,21 @@ mod tests {
];
expected.sort_unstable();
assert_eq!(
recorded_keys(&dbio),
recorded_keys(&storage_ref).await,
expected,
"only the unread block is recorded on the second pass"
);
assert_eq!(
get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(),
get_cross_zone_peer_floor(&storage_ref, PEER_ZONE)
.await
.unwrap(),
Some(Slot::from(2))
);
}
#[tokio::test]
async fn watcher_does_not_persist_past_an_undecodable_block() {
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
@@ -1042,7 +1193,7 @@ mod tests {
peer_block_msg(3, 2),
]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -1052,11 +1203,13 @@ mod tests {
// would drop its messages permanently rather than until the next restart.
assert_eq!(outcome, PassOutcome::Undecodable(Slot::from(1)));
assert_eq!(
get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(),
get_cross_zone_peer_floor(&storage_ref, PEER_ZONE)
.await
.unwrap(),
Some(Slot::from(0))
);
assert_eq!(
recorded_keys(&dbio),
recorded_keys(&storage_ref).await,
vec![message_key(&PEER_ZONE, 1, 0)],
"the block after the failure is unread"
);
@@ -1067,14 +1220,14 @@ mod tests {
// One slot can carry several messages. Persisting after each message
// would store a cursor the retry resumes past, so the message that
// failed is never re-read and its delivery is lost for good.
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
let outcome = consume_peer_stream(
stream::iter(vec![peer_block_msg(1, 4), undecodable_msg(4)]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -1082,8 +1235,16 @@ mod tests {
assert_eq!(outcome, PassOutcome::Undecodable(Slot::from(4)));
assert_eq!(cursor, None, "slot 4 is re-read whole on the next pass");
assert_eq!(get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(), None);
assert_eq!(recorded_keys(&dbio), vec![message_key(&PEER_ZONE, 1, 0)]);
assert_eq!(
get_cross_zone_peer_floor(&storage_ref, PEER_ZONE)
.await
.unwrap(),
None
);
assert_eq!(
recorded_keys(&storage_ref).await,
vec![message_key(&PEER_ZONE, 1, 0)]
);
}
#[tokio::test]
@@ -1091,7 +1252,7 @@ mod tests {
// This used to be read past after twenty attempts, which advanced the
// floor over the hole and lost those messages rather than delaying
// them: nothing after a hole can link. Stopping keeps the slot readable.
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
@@ -1103,7 +1264,7 @@ mod tests {
peer_block_msg(3, 2),
]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -1112,12 +1273,14 @@ mod tests {
}
assert_eq!(
recorded_keys(&dbio),
recorded_keys(&storage_ref).await,
vec![message_key(&PEER_ZONE, 1, 0)],
"no pass reads past the slot it cannot decode"
);
assert_eq!(
get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(),
get_cross_zone_peer_floor(&storage_ref, PEER_ZONE)
.await
.unwrap(),
Some(Slot::from(0)),
"the floor stays below it, so a fixed decoder recovers the messages"
);
@@ -1134,7 +1297,7 @@ mod tests {
//
// The honest blocks behind it still deliver: stopping here would cost
// the peer one inscription to end its own deliveries for good.
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
let pre_burn = produce_dummy_block(5, None, vec![emission()]);
@@ -1147,7 +1310,7 @@ mod tests {
peer_block_msg(3, 3),
]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -1161,7 +1324,7 @@ mod tests {
];
expected.sort_unstable();
assert_eq!(
recorded_keys(&dbio),
recorded_keys(&storage_ref).await,
expected,
"the key the peer aimed to burn is never recorded, and nothing else is held up"
);
@@ -1172,7 +1335,7 @@ mod tests {
async fn a_second_block_at_a_delivered_id_is_not_delivered_from() {
// Both claim id 2, so on chain both deliveries key on (PEER_ZONE, 2, 0)
// and the second is a replay the inbox no-ops.
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
let equivocation = produce_dummy_block(2, Some(HashType([9; 32])), vec![emission()]);
@@ -1184,7 +1347,7 @@ mod tests {
block_msg(&equivocation, 2),
]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -1198,7 +1361,7 @@ mod tests {
let mut expected = vec![message_key(&PEER_ZONE, 1, 0), message_key(&PEER_ZONE, 2, 0)];
expected.sort_unstable();
assert_eq!(
recorded_keys(&dbio),
recorded_keys(&storage_ref).await,
expected,
"one delivery per id, whatever the peer publishes under it"
);
@@ -1209,7 +1372,7 @@ mod tests {
async fn a_block_that_does_not_link_to_the_tip_is_not_delivered_from() {
// Not on the chain we verified, so nothing is delivered from it, and
// the honest block at that id still is when it lands.
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
let forked = produce_dummy_block(2, Some(HashType([9; 32])), vec![emission()]);
@@ -1221,7 +1384,7 @@ mod tests {
peer_block_msg(2, 2),
]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -1231,7 +1394,7 @@ mod tests {
let mut expected = vec![message_key(&PEER_ZONE, 1, 0), message_key(&PEER_ZONE, 2, 0)];
expected.sort_unstable();
assert_eq!(
recorded_keys(&dbio),
recorded_keys(&storage_ref).await,
expected,
"the fork is passed over and the peer's own chain continues"
);
@@ -1243,14 +1406,14 @@ mod tests {
// A fresh watcher handed a mid-chain block has nothing to link it
// against. Adopting it would let the peer choose where the chain starts
// and burn every key below it with one block.
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
let outcome = consume_peer_stream(
stream::iter(vec![peer_block_msg(2, 0)]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -1261,7 +1424,7 @@ mod tests {
PassOutcome::Stranded,
"a pass that placed nothing while passing blocks over is how a peer goes quiet"
);
assert!(recorded_keys(&dbio).is_empty());
assert!(recorded_keys(&storage_ref).await.is_empty());
assert_eq!(tip, None);
}
@@ -1269,7 +1432,7 @@ mod tests {
async fn a_tampered_header_hash_is_not_delivered_from() {
// As correctly signed as any other block, since the signature does not
// cover `header.hash`. Block 2 arriving behind it still delivers.
let (_dir, dbio) = store();
let (_dir, storage_ref) = store().await;
let mut cursor = None;
let mut tip = None;
let mut tampered = chain_block(2);
@@ -1282,7 +1445,7 @@ mod tests {
peer_block_msg(2, 2),
]),
&peer_context(),
&dbio,
&storage_ref,
&mut cursor,
&mut tip,
)
@@ -1291,7 +1454,7 @@ mod tests {
assert_eq!(outcome, PassOutcome::Drained);
let mut expected = vec![message_key(&PEER_ZONE, 1, 0), message_key(&PEER_ZONE, 2, 0)];
expected.sort_unstable();
assert_eq!(recorded_keys(&dbio), expected);
assert_eq!(recorded_keys(&storage_ref).await, expected);
assert_eq!(tip, Some(tip_at(2)));
}
@@ -1300,18 +1463,31 @@ mod tests {
// The tip is written per block and the floor per slot, so a crash
// partway through the rebuild would otherwise leave a floor far above a
// tip of 1, and nothing read after that restart could link.
let (_dir, dbio) = store();
set_cross_zone_peer_floor(&dbio, PEER_ZONE, Slot::from(5000)).unwrap();
let (_dir, storage_ref) = store().await;
set_cross_zone_peer_floor(&storage_ref, PEER_ZONE, Slot::from(5000))
.await
.unwrap();
let floor = get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap();
let tip = dbio.get_cross_zone_peer_tip(PEER_ZONE).unwrap();
let floor = get_cross_zone_peer_floor(&storage_ref, PEER_ZONE)
.await
.unwrap();
let tip = storage_ref
.ask(GetCrossZonePeerTip {
peer_zone: PEER_ZONE,
})
.await
.unwrap();
let resume = resume_from(tip, floor);
assert_eq!(resume.cursor, None, "the rebuild reads from genesis");
assert!(resume.clear_floor);
clear_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap();
clear_cross_zone_peer_floor(&storage_ref, PEER_ZONE)
.await
.unwrap();
assert_eq!(
get_cross_zone_peer_floor(&dbio, PEER_ZONE).unwrap(),
get_cross_zone_peer_floor(&storage_ref, PEER_ZONE)
.await
.unwrap(),
None,
"and a crash mid-rebuild resumes from genesis too, not from slot 5000"
);
lez/sequencer/core/src/lib.rs
+258 -210
View File
@@ -1,7 +1,7 @@
use std::{
collections::VecDeque,
path::Path,
sync::{Arc, Mutex},
sync::Arc,
time::{Duration, Instant},
};
@@ -18,7 +18,7 @@ use common::{
use config::{GenesisAction, SequencerConfig};
use cross_zone_inbox_core::CrossZoneMessage;
use futures::StreamExt as _;
use itertools::Itertools as _;
use kameo::actor::ActorRef;
use lee::{AccountId, PublicTransaction, public_transaction::Message};
use lee_core::GENESIS_BLOCK_ID;
use log::{debug, error, info, warn};
@@ -32,17 +32,17 @@ use mempool::{MemPool, MemPoolHandle};
#[cfg(feature = "mock")]
pub use mock::SequencerCoreWithMockClients;
use num_bigint::BigUint;
pub use storage::error::DbError;
// Re-exported because `cross_zone_dead_letters` returns it and the service
// crate does not depend on `storage`, so it could not otherwise name the type.
pub use storage::sequencer::sequencer_cells::DeadLetterDispatchRecord;
use storage::sequencer::{
DispatchFailure, RocksDBIO, StoreUpdate,
sequencer_cells::{
DispatchOrigin, PendingCrossZoneDispatchRecord, PendingDepositEventRecord,
WithdrawalReconciliationKey, ZoneAnchorRecord,
use sequencer_storage_actor::{
StorageActor, StorageActorTrait,
protocol::{
ApplyStoreUpdate, DeadLetterDispatchRecord, DispatchFailure, DispatchOrigin,
DropSettledCrossZoneDispatches, GetBlock, GetDeadLetterDispatches, GetFirstBlockId,
GetLatestBlockMeta, GetPendingCrossZoneDispatches, PendingCrossZoneDispatchRecord,
PendingDepositEventRecord, RecordNewBlock, SetZoneAnchor, WithdrawalReconciliationKey,
ZoneAnchorRecord,
},
};
use tokio::sync::Mutex;
use tokio_retry::{Retry, strategy::FixedInterval};
use crate::{
@@ -123,11 +123,14 @@ struct DepositMetadata {
recipient_id: lee::AccountId,
}
pub struct SequencerCore<BP: BlockPublisherTrait = ZoneSdkPublisher> {
pub struct SequencerCore<
BP: BlockPublisherTrait = ZoneSdkPublisher,
S: StorageActorTrait = StorageActor,
> {
/// Two-tier chain state: production builds on its head; the publisher's
/// `on_follow` sink feeds adopted/orphaned/finalized peer blocks into it.
chain: Arc<Mutex<ChainState>>,
store: SequencerStore,
store: SequencerStore<S>,
mempool: MemPool<(TransactionOrigin, LeeTransaction)>,
sequencer_config: SequencerConfig,
block_publisher: BP,
@@ -139,86 +142,24 @@ pub struct SequencerCore<BP: BlockPublisherTrait = ZoneSdkPublisher> {
last_committee_submission_slot: Option<Slot>,
}
impl<BP: BlockPublisherTrait> SequencerCore<BP> {
impl<BP: BlockPublisherTrait, S: StorageActorTrait> SequencerCore<BP, S> {
const CHANNEL_PROBE_RETRIES: usize = 29;
const CHANNEL_PROBE_RETRY_DELAY: Duration = Duration::from_secs(2);
/// Channel slots between committee-config submissions; a margin over
/// observed Bedrock confirmation lag.
const COMMITTEE_SUBMISSION_COOLDOWN: SlotCount = 10;
/// Starts the sequencer using the provided configuration.
/// If an existing database is found, the sequencer state is loaded from it and
/// assumed to represent the correct latest state consistent with Bedrock-finalized data.
/// If no database is found, the sequencer performs a fresh start from genesis,
/// initializing its state with the accounts defined in the configuration file.
fn open_or_create_store(
config: &SequencerConfig,
bootstrap_sequencer_key: Option<sequencer_stake_core::SequencerKey>,
) -> (SequencerStore, lee::V03State) {
let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap();
let db_path = config.db_path();
if db_path.exists() {
let store = SequencerStore::open_db(&db_path, signing_key).unwrap_or_else(|err| {
panic!(
"Failed to open database at {} with error: {err}",
db_path.display()
)
});
let state = store
.get_lee_state()
.expect("Failed to read state from store");
(store, state)
} else {
let legacy = config.home.join("rocksdb");
if legacy.exists() {
warn!(
"Ignoring pre-channel-suffix database at {}; rename it to {} to resume it",
legacy.display(),
db_path.display()
);
}
warn!(
"Database not found at {}, starting from genesis",
db_path.display()
);
let (genesis_state, genesis_txs) = build_genesis_state(config, bootstrap_sequencer_key);
let hashable_data = HashableBlockData {
block_id: GENESIS_BLOCK_ID,
transactions: genesis_txs,
prev_block_hash: HashType([0; 32]),
timestamp: 0,
};
let genesis_block = hashable_data.into_pending_block(&signing_key);
let store = SequencerStore::create_db_with_genesis(
&db_path,
&genesis_block,
&genesis_state,
signing_key,
)
.expect("Failed to create database with genesis block");
// Incrementing count for genesis.
sequencer_core_metrics::increment_blocks_produced_total();
(store, genesis_state)
}
}
/// Rebuilds the two-tier [`ChainState`]: the final tier from the persisted
/// final snapshot (pre-genesis state when absent), the head tier by replaying
/// every stored block above it, so a post-restart orphan can still revert.
fn restore_chain_state(
async fn restore_chain_state(
config: &SequencerConfig,
store: &SequencerStore,
store: &SequencerStore<S>,
stored_head_state: &lee::V03State,
) -> ChainState {
let final_snapshot = store
.dbio()
.get_final_snapshot()
.await
.expect("Failed to read final snapshot from store");
let (final_state, final_tip) = match final_snapshot {
Some((state, meta)) => (state, Some(Tip::from(meta))),
@@ -229,9 +170,11 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
let mut head_blocks = store
.get_all_blocks()
.filter_ok(|block| block.header.block_id > boundary)
.collect::<Result<Vec<_>, _>>()
.expect("Failed to read blocks from store while restoring chain state");
.await
.expect("Failed to read blocks from store while restoring chain state")
.into_iter()
.filter(|block| block.header.block_id > boundary)
.collect::<Vec<_>>();
head_blocks.sort_unstable_by_key(|block| block.header.block_id);
let mut chain = ChainState::from_final(final_state, final_tip);
@@ -252,8 +195,43 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
chain
}
/// Seeds the storage actor's database with this zone's genesis when it
/// holds no chain yet.
async fn seed_genesis_if_absent(
storage_ref: &ActorRef<S>,
signing_key: &lee::PrivateKey,
bootstrap_sequencer_key: Option<sequencer_stake_core::SequencerKey>,
config: &SequencerConfig,
) {
let first_block_id = storage_ref
.ask(GetFirstBlockId)
.await
.expect("Failed to read the first block id");
if first_block_id.is_some() {
return;
}
let (block, state) = genesis_block_and_state(signing_key, bootstrap_sequencer_key, config);
storage_ref
.ask(RecordNewBlock {
block,
withdrawals: Vec::new(),
state: Arc::new(state),
checkpoint_bytes: None,
})
.await
.expect("Failed to seed the database with the genesis block");
sequencer_core_metrics::increment_blocks_produced_total();
}
#[expect(
clippy::cognitive_complexity,
reason = "Slop has won the battle, but our war is not over"
)]
pub async fn start_from_config(
config: SequencerConfig,
storage_ref: ActorRef<S>,
) -> (Self, MemPoolHandle<(TransactionOrigin, LeeTransaction)>) {
sequencer_core_metrics::init();
@@ -287,8 +265,18 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
info!("Channel does not exist yet; starting it as channel creator");
}
let bootstrap_sequencer_key = (!channel_already_exists).then_some(own_sequencer_key);
let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap();
Self::seed_genesis_if_absent(&storage_ref, &signing_key, bootstrap_sequencer_key, &config)
.await;
let (store, state) = Self::open_or_create_store(&config, bootstrap_sequencer_key);
let store = SequencerStore::new(storage_ref, signing_key)
.await
.expect("Failed to open sequencer store");
let state = store
.get_lee_state()
.await
.expect("Failed to read state from store")
.expect("Store holds a chain but no state");
assert!(
committee_discovery::config_is_readable(&state),
@@ -296,12 +284,13 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
one this sequencer can operate on"
);
let chain = Arc::new(Mutex::new(Self::restore_chain_state(
&config, &store, &state,
)));
let chain = Arc::new(Mutex::new(
Self::restore_chain_state(&config, &store, &state).await,
));
let initial_checkpoint = store
.get_zone_checkpoint()
.await
.expect("Failed to load zone-sdk checkpoint");
let is_fresh_start = initial_checkpoint.is_none();
@@ -313,7 +302,11 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
bedrock_signing_key,
config.retry_pending_blocks_timeout,
initial_checkpoint,
Self::on_follow(store.dbio(), Arc::clone(&chain), mempool_handle.clone()),
Self::on_follow(
store.storage_ref().clone(),
Arc::clone(&chain),
mempool_handle.clone(),
),
)
.await
.expect("Failed to initialize Block Publisher");
@@ -328,7 +321,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
&config.bedrock_config,
cross_zone,
config.block_create_timeout,
&store.dbio(),
store.storage_ref(),
)
});
// Before producing, verify our local state still belongs to the chain
@@ -347,10 +340,12 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
// store written before the mark existed starts in.
if let Some(tip) = store
.latest_block_meta()
.await
.expect("Failed to read latest block meta")
{
store
.raise_published_high_water(tip.id)
.await
.expect("Failed to seed published high water mark");
}
@@ -361,9 +356,11 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
if is_fresh_start && channel_absent {
let mut pending_blocks = store
.get_all_blocks()
.filter_ok(|block| matches!(block.bedrock_status, BedrockStatus::Pending))
.collect::<Result<Vec<_>, _>>()
.expect("Failed to read blocks from store while republishing on fresh start");
.await
.expect("Failed to read blocks from store while republishing on fresh start")
.into_iter()
.filter(|block| matches!(block.bedrock_status, BedrockStatus::Pending))
.collect::<Vec<_>>();
pending_blocks.sort_unstable_by_key(|block| block.header.block_id);
assert!(
@@ -397,6 +394,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
last_checkpoint = Some(outcome.checkpoint);
store
.raise_published_high_water(block.header.block_id)
.await
.expect("Failed to persist published high water mark");
}
@@ -406,6 +404,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
if let Some(checkpoint) = last_checkpoint {
store
.set_zone_checkpoint(&checkpoint)
.await
.expect("Failed to persist checkpoint after republishing on fresh start");
}
}
@@ -420,8 +419,8 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
last_committee_submission_slot: None,
};
sequencer_core_metrics::record_chain_height(sequencer_core.chain_height());
record_dead_letter_gauge(&sequencer_core.store.dbio());
sequencer_core_metrics::record_chain_height(sequencer_core.chain_height().await);
record_dead_letter_gauge(sequencer_core.store.storage_ref()).await;
(sequencer_core, mempool_handle)
}
@@ -436,12 +435,13 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
/// this sequencer is the one that must bootstrap-publish its own blocks.
async fn verify_and_reconstruct(
publisher: &BP,
store: &SequencerStore,
store: &SequencerStore<S>,
chain: &Mutex<ChainState>,
is_fresh_start: bool,
) -> Result<bool> {
let anchor_record = store
.get_zone_anchor()
.await
.context("Failed to read zone anchor")?;
let after_slot = anchor_record
@@ -468,6 +468,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
// channel activity in a prior run.
let local_tip = store
.latest_block_meta()
.await
.context("Failed to read latest block meta")?
.map(|meta| meta.id);
let had_checkpoint_before_start = !is_fresh_start;
@@ -535,8 +536,8 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
// Locked per message (not across the stream `await`): concurrent
// follow events interleave safely — both paths apply idempotently
// and persist under this same lock.
let mut chain = chain.lock().expect("chain state mutex poisoned");
Self::apply_reconstructed_block(store, &mut chain, &block, slot)?;
let mut chain = chain.lock().await;
Self::apply_reconstructed_block(store.storage_ref(), &mut chain, &block, slot).await?;
}
// The channel exists once it has a tip; only when it has none is this
@@ -551,14 +552,15 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
/// blocks we already hold, ignored when it conflicts at a height the final
/// tier already settled, a validated continuation otherwise. Advances the
/// persisted anchor to the block's slot.
fn apply_reconstructed_block(
store: &SequencerStore,
async fn apply_reconstructed_block(
storage_ref: &ActorRef<S>,
chain: &mut ChainState,
block: &Block,
slot: Slot,
) -> Result<()> {
let tip = store
.latest_block_meta()
let tip = storage_ref
.ask(GetLatestBlockMeta)
.await
.context("Failed to read latest block meta")?;
let block_id = block.header.block_id;
let block_hash = block.header.hash;
@@ -576,14 +578,16 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
// downstream would ever remove them.
if let Some(tip) = &tip
&& block_id <= tip.id
&& let Some(stored) = store
.get_block_at_id(block_id)
&& let Some(stored) = storage_ref
.ask(GetBlock { block_id })
.await
.context("Failed to read stored block")?
&& stored.header.hash == block_hash
{
settle_reconstructed_deliveries(store, &stored);
store
.set_zone_anchor(&record)
settle_reconstructed_deliveries(storage_ref, &stored).await;
storage_ref
.ask(SetZoneAnchor { anchor: record })
.await
.context("Failed to persist zone anchor")?;
return Ok(());
}
@@ -628,24 +632,29 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
.collect();
// The same for the deliveries it carries: the inbox has seen them, so
// the drain would skip them anyway, and the records are owed nothing.
let finalized_dispatch_keys = settled_dispatch_keys(&store.dbio(), block);
let finalized_dispatch_keys = settled_dispatch_keys(storage_ref, block).await;
// The tip meta stays pinned to the head tip even when the reconstructed
// block lands below it, and the anchor only advances if the block
// itself landed.
let head_tip = chain.head_tip().map(|head| BlockMeta::from(&head));
let final_meta = chain.final_tip().map(|meta| BlockMeta::from(&meta));
store
.dbio()
.store_update(&StoreUpdate {
blocks: &[(block, true)],
head_tip: head_tip.as_ref(),
final_snapshot: final_meta.as_ref().map(|meta| (chain.final_state(), meta)),
remove_deposit_records: &finalized_deposit_ids,
remove_dispatch_records: &finalized_dispatch_keys,
zone_anchor: Some(&record),
..StoreUpdate::new(chain.head_state())
storage_ref
.ask(ApplyStoreUpdate {
blocks: vec![(block.clone(), true)],
head_tip,
head_state: chain.share_head_state(),
final_snapshot: final_meta.map(|meta| (chain.share_final_state(), meta)),
remove_deposit_records: finalized_deposit_ids,
remove_dispatch_records: finalized_dispatch_keys,
zone_anchor: Some(record),
checkpoint: None,
finalized_up_to: None,
new_deposit_events: Vec::new(),
consumed_withdrawals: Vec::new(),
new_withdraw_intents: Vec::new(),
})
.await
.context("Failed to persist reconstructed block")?;
Ok(())
@@ -653,12 +662,17 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
/// Publisher sink adapter over [`apply_follow_update`].
fn on_follow(
dbio: Arc<RocksDBIO>,
storage_ref: ActorRef<S>,
chain: Arc<Mutex<ChainState>>,
mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>,
) -> block_publisher::OnFollowSink {
Box::new(move |update: block_publisher::FollowUpdate| {
apply_follow_update(&dbio, &chain, &mempool_handle, update);
let storage_ref = storage_ref.clone();
let chain = Arc::clone(&chain);
let mempool_handle = mempool_handle.clone();
Box::pin(async move {
apply_follow_update(&storage_ref, &chain, &mempool_handle, update).await;
})
})
}
@@ -674,6 +688,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
committee_update,
} = self
.build_block_from_mempool(live_accredited_keys.as_deref())
.await
.context("Failed to build block from mempool transactions")?;
let block_publisher::PublishOutcome {
@@ -690,6 +705,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
// block below, so this height must never be published again.
self.store
.raise_published_high_water(block.header.block_id)
.await
.context("Failed to persist published high water mark")?;
// Independent Mantle tx, not bundled with the block above — join/exit
@@ -701,14 +717,11 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
.map(withdrawal_reconciliation_key)
.collect();
self.record_produced_block(
this_msg,
&block,
&withdrawal_reconciliation_keys,
&checkpoint,
)?;
let block_id = block.header.block_id;
self.record_produced_block(this_msg, block, withdrawal_reconciliation_keys, &checkpoint)
.await?;
Ok(block.header.block_id)
Ok(block_id)
}
/// Live committee snapshot for gating `FinalizeUnstake` inclusion and
@@ -792,29 +805,30 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
/// `Parked` when the head reorged to a different parent), the canonical
/// block is persisted by the follow path instead, and our invalidated
/// inscription comes back via `orphaned`.
fn record_produced_block(
async fn record_produced_block(
&mut self,
this_msg: MsgId,
block: &Block,
withdrawal_reconciliation_keys: &[WithdrawalReconciliationKey],
block: Block,
withdrawal_reconciliation_keys: Vec<WithdrawalReconciliationKey>,
checkpoint: &block_publisher::SequencerCheckpoint,
) -> Result<()> {
let checkpoint_bytes = block_store::checkpoint_bytes(checkpoint)?;
let mut chain = self.chain.lock().expect("chain state mutex poisoned");
match chain.apply_produced(this_msg, block) {
let mut chain = self.chain.lock().await;
match chain.apply_produced(this_msg, &block) {
AcceptOutcome::Applied => {
// Persisted under the lock so disk writes land in apply order
// with the follow path.
self.store.update(
block,
withdrawal_reconciliation_keys,
chain.head_state(),
Some(&checkpoint_bytes),
)?;
let block_id = block.header.block_id;
self.store
.record_new_block(
block,
withdrawal_reconciliation_keys,
chain.share_head_state(),
Some(checkpoint_bytes),
)
.await?;
sequencer_core_metrics::increment_blocks_produced_total();
sequencer_core_metrics::record_chain_height(block.header.block_id);
sequencer_core_metrics::record_chain_height(block_id);
}
// Neither branch persists anything, checkpoint included: the
// inscription it holds as pending belongs to a block that is not
@@ -910,7 +924,11 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
true
}
fn build_block_from_mempool(
#[expect(
clippy::cognitive_complexity,
reason = "Slop has won the battle, but our war is not over"
)]
async fn build_block_from_mempool(
&mut self,
live_accredited_keys: Option<&[sequencer_stake_core::SequencerKey]>,
) -> Result<BlockWithMeta> {
@@ -923,8 +941,8 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
let mut settled = Vec::new();
let recorded_dispatches: Vec<_> = self
.store
.dbio()
.get_pending_cross_zone_dispatches()
.pending_cross_zone_dispatches()
.await
.context("Failed to load pending cross-zone dispatches")?
.into_iter()
.filter_map(
@@ -958,7 +976,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
pending_dispatches,
finalize_unstake_txs,
) = {
let chain = self.chain.lock().expect("chain state mutex poisoned");
let chain = self.chain.lock().await;
let tip = chain.head_tip();
let height = tip.as_ref().map_or(GENESIS_BLOCK_ID, |head| {
head.block_id
@@ -998,8 +1016,8 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
if !settled.is_empty() {
if let Err(err) = self
.store
.dbio()
.drop_settled_cross_zone_dispatches(&settled)
.drop_settled_cross_zone_dispatches(settled.clone())
.await
{
// Only bookkeeping: the deliveries themselves are irreversible,
// and the next turn tries again.
@@ -1010,7 +1028,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
}
// A settled delivery may be one this node had given up on, which
// takes its dead letter with it.
record_dead_letter_gauge(&self.store.dbio());
record_dead_letter_gauge(self.store.storage_ref()).await;
}
let mut valid_transactions = Vec::new();
@@ -1029,6 +1047,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
let pending_deposits: VecDeque<LeeTransaction> = self
.store
.get_pending_deposit_events()
.await
.context("Failed to load pending deposit events")?
.into_iter()
.filter(|record| !deposit_already_minted(&working_state, record.deposit_op_id))
@@ -1109,7 +1128,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
"Sequencer-drained transaction {tx_hash} cannot fit in any block under the \
{max_block_size} byte limit; giving up on it rather than stalling production",
);
self.count_dispatch_failure(&tx);
self.count_dispatch_failure(&tx).await;
continue;
}
@@ -1164,7 +1183,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
// A failed transaction is simply left out of the block, except a
// dispatch: that one is re-fed from the store every turn, so one
// that can never execute would fail on every block for ever.
self.count_dispatch_failure(&tx);
self.count_dispatch_failure(&tx).await;
}
if valid_transactions.len() >= self.sequencer_config.max_num_tx_in_block {
@@ -1209,23 +1228,18 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
/// Reads the current head state under the lock without cloning it, so callers
/// reuse `V03State`'s own API (accounts, nonces, proofs) with no whole-state copy.
pub fn with_state<R>(&self, f: impl FnOnce(&lee::V03State) -> R) -> R {
f(self
.chain
.lock()
.expect("chain state mutex poisoned")
.head_state())
pub async fn with_state<R>(&self, f: impl FnOnce(&lee::V03State) -> R) -> R {
f(self.chain.lock().await.head_state())
}
pub const fn block_store(&self) -> &SequencerStore {
pub const fn block_store(&self) -> &SequencerStore<S> {
&self.store
}
#[must_use]
pub fn chain_height(&self) -> u64 {
pub async fn chain_height(&self) -> u64 {
self.chain
.lock()
.expect("chain state mutex poisoned")
.await
.head_tip()
.map_or(0, |tip| tip.block_id)
}
@@ -1240,20 +1254,20 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
/// [`StoreUpdate::finalized_up_to`]. Kept on the type for tests.
// TODO: Delete blocks instead of marking them as finalized. Current
// approach is used because we still have `GetBlockDataRequest`.
pub fn clean_finalized_blocks_from_db(&self, last_finalized_block_id: u64) -> Result<()> {
pub async fn clean_finalized_blocks_from_db(&self, last_finalized_block_id: u64) -> Result<()> {
log::info!("Clearing pending blocks up to id: {last_finalized_block_id}");
self.store
.dbio()
.clean_pending_blocks_up_to(last_finalized_block_id)?;
.clean_pending_blocks_up_to(last_finalized_block_id)
.await?;
Ok(())
}
/// Returns the list of stored pending blocks.
pub fn get_pending_blocks(&self) -> Result<Vec<Block>> {
pub async fn get_pending_blocks(&self) -> Result<Vec<Block>> {
Ok(self
.store
.get_all_blocks()
.collect::<block_store::DbResult<Vec<Block>>>()?
.await?
.into_iter()
.filter(|block| matches!(block.bedrock_status, BedrockStatus::Pending))
.collect())
@@ -1300,7 +1314,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
/// can equally be a property of the moment, so give up only after several.
/// Giving up moves the record to the dead letter: a peer cannot grow the
/// pending list with deliveries that never execute, and it stays findable.
fn count_dispatch_failure(&self, tx: &LeeTransaction) {
async fn count_dispatch_failure(&self, tx: &LeeTransaction) {
let Some(message) = extract_cross_zone_dispatch(tx) else {
return;
};
@@ -1316,12 +1330,12 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
};
match self
.store
.dbio()
.record_dispatch_failure(key, RETIRE_DISPATCH_AFTER_FAILURES, origin)
.await
{
Ok(DispatchFailure::Retired(record)) => {
sequencer_core_metrics::increment_cross_zone_dispatches_retired_total();
record_dead_letter_gauge(&self.store.dbio());
record_dead_letter_gauge(self.store.storage_ref()).await;
error!(
"Giving up on cross-zone delivery {} from peer zone {} block {} transaction {} ({} bytes) after {} failed attempts. This node will not retry it; unless another sequencer carries it, the message is not delivered. Kept in the dead letter.",
hex::encode(key),
@@ -1354,10 +1368,9 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
/// Retained is read first so the pair can only skew towards a total that
/// leads its list, an ordinary evicted or settled state. The other order
/// would report entries against a total of zero.
pub fn cross_zone_dead_letters(&self) -> Result<(u64, Vec<DeadLetterDispatchRecord>), DbError> {
let dbio = self.store.dbio();
let retained = dbio.get_dead_letter_cross_zone_dispatches()?;
let total = dbio.get_dead_letter_cross_zone_dispatch_count()?;
pub async fn cross_zone_dead_letters(&self) -> Result<(u64, Vec<DeadLetterDispatchRecord>)> {
let retained = self.store.dead_letter_dispatches().await?;
let total = self.store.dead_letter_dispatch_count().await?;
Ok((total, retained))
}
@@ -1382,11 +1395,10 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
}
/// The height the next produced block would claim.
#[must_use]
pub fn next_block_height(&self) -> u64 {
pub async fn next_block_height(&self) -> u64 {
self.chain
.lock()
.expect("chain state mutex poisoned")
.await
.head_tip()
.map_or(GENESIS_BLOCK_ID, |tip| {
tip.block_id
@@ -1406,10 +1418,9 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
///
/// This is not a stall — the head recovers by itself once the inscriptions
/// we are protecting finalize and the final tier rebases onto them.
#[must_use]
pub fn rewound_below_published(&self) -> Option<u64> {
let high_water = self.store.published_high_water().ok().flatten()?;
(self.next_block_height() <= high_water).then_some(high_water)
pub async fn rewound_below_published(&self) -> Option<u64> {
let high_water = self.store.published_high_water().await.ok().flatten()?;
(self.next_block_height().await <= high_water).then_some(high_water)
}
/// Shared handle to the two-tier follow state, for tests to drive the
@@ -1464,8 +1475,8 @@ fn dispatch_already_delivered(state: &lee::V03State, message: &CrossZoneMessage)
/// Read from the store because the list falls as well as rises (eviction, and
/// reconciliation when a delivery settles elsewhere). Costs a read and a decode,
/// so call it only where one of those can have happened.
fn record_dead_letter_gauge(dbio: &RocksDBIO) {
match dbio.get_dead_letter_cross_zone_dispatches() {
async fn record_dead_letter_gauge<S: StorageActorTrait>(storage_ref: &ActorRef<S>) {
match storage_ref.ask(GetDeadLetterDispatches).await {
Ok(records) => {
sequencer_core_metrics::record_cross_zone_dead_letter_dispatches(records.len());
}
@@ -1487,8 +1498,8 @@ fn record_dead_letter_gauge(dbio: &RocksDBIO) {
/// relies on a valid successor or a restart. `ChainState` never emits
/// `AcceptOutcome::RetryableFailure` yet; adding retry parity here is a
/// follow-up.
fn apply_follow_update(
dbio: &RocksDBIO,
async fn apply_follow_update<S: StorageActorTrait>(
storage_ref: &ActorRef<S>,
chain: &Mutex<ChainState>,
mempool_handle: &MemPoolHandle<(TransactionOrigin, LeeTransaction)>,
update: block_publisher::FollowUpdate,
@@ -1522,7 +1533,7 @@ fn apply_follow_update(
// The lock is held across the persist below so disk writes land in apply
// order — the produce path persists under this same lock.
let (resubmit_txs, outcome, head_height) = {
let mut chain = chain.lock().expect("chain state mutex poisoned");
let mut chain = chain.lock().await;
// An orphan report rewinds the head to the earliest orphaned block and
// prunes the store above it, which is how a run of our own inscriptions
@@ -1619,6 +1630,7 @@ fn apply_follow_update(
BlockMeta::from(&tip)
});
let head_tip = chain.head_tip().map(|tip| BlockMeta::from(&tip));
let head_tip_id = head_tip.as_ref().map_or(0, |tip| tip.id);
// Every block at or below the highest finalized one is irreversible, so
// stored blocks there can be marked finalized.
@@ -1638,36 +1650,42 @@ fn apply_follow_update(
// The same for cross-zone deliveries, keyed by message key: a record
// goes once its own delivery is irreversible, never because another
// block finalized at its height.
let finalized_dispatch_keys: Vec<[u8; 32]> = irreversible
.iter()
.flat_map(|block| settled_dispatch_keys(dbio, block))
.collect();
let mut finalized_dispatch_keys: Vec<[u8; 32]> = Vec::new();
for block in &irreversible {
finalized_dispatch_keys.extend(settled_dispatch_keys(storage_ref, block).await);
}
// A persist failure is fatal: the in-memory chain has already advanced,
// and continuing would leave a permanent gap in the store. The `panic!`
// ends the drive task, whose cancellation halts the node.
let outcome = dbio
.store_update(&StoreUpdate {
checkpoint: Some(&checkpoint_bytes),
blocks: &to_persist,
head_tip: head_tip.as_ref(),
final_snapshot: final_meta.as_ref().map(|meta| (chain.final_state(), meta)),
let outcome = storage_ref
.ask(ApplyStoreUpdate {
checkpoint: Some(checkpoint_bytes),
blocks: to_persist
.into_iter()
.map(|(block, fin)| (block.clone(), fin))
.collect(),
head_tip,
head_state: chain.share_head_state(),
final_snapshot: final_meta.map(|meta| (chain.share_final_state(), meta)),
finalized_up_to: last_finalized,
new_deposit_events: &deposit_records,
remove_deposit_records: &finalized_deposit_ids,
remove_dispatch_records: &finalized_dispatch_keys,
consumed_withdrawals: &consumed_withdrawals,
..StoreUpdate::new(chain.head_state())
new_deposit_events: deposit_records,
remove_deposit_records: finalized_deposit_ids,
remove_dispatch_records: finalized_dispatch_keys,
consumed_withdrawals,
new_withdraw_intents: Vec::new(),
zone_anchor: None,
})
.await
.unwrap_or_else(|err| panic!("Failed to persist follow update: {err:#}"));
(resubmit_txs, outcome, head_tip.map_or(0, |tip| tip.id))
(resubmit_txs, outcome, head_tip_id)
};
sequencer_core_metrics::record_chain_height(head_height);
// The runtime reconcile path: finalizing another sequencer's block drops the
// dead letter of a delivery this node gave up on.
record_dead_letter_gauge(dbio);
record_dead_letter_gauge(storage_ref).await;
if outcome.accepted_deposits > 0 {
log::info!(
@@ -1700,6 +1718,24 @@ fn apply_follow_update(
}
}
/// The genesis block and state `config` describes.
fn genesis_block_and_state(
signing_key: &lee::PrivateKey,
bootstrap_sequencer_key: Option<sequencer_stake_core::SequencerKey>,
config: &SequencerConfig,
) -> (Block, lee::V03State) {
let (genesis_state, genesis_txs) = build_genesis_state(config, bootstrap_sequencer_key);
let genesis_block = HashableBlockData {
block_id: GENESIS_BLOCK_ID,
transactions: genesis_txs,
prev_block_hash: HashType([0; 32]),
timestamp: 0,
}
.into_pending_block(signing_key);
(genesis_block, genesis_state)
}
/// The pre-genesis state: `testnet_initial_state` plus the bridge-lock holdings,
/// the only accounts seeded outside any transaction. Everything else, including
/// the bootstrap sequencer's own stake, is applied as a genesis transaction in
@@ -2002,7 +2038,7 @@ fn build_supply_account_genesis_transaction(
let message = Message::try_new(
faucet_program_id,
vec![system_accounts::faucet_account_id(), recipient_vault_id],
vec![],
Vec::new(),
faucet_core::Instruction::GenesisTransferVault {
vault_program_id,
recipient_id: *account_id,
@@ -2010,7 +2046,7 @@ fn build_supply_account_genesis_transaction(
},
)
.expect("Failed to serialize genesis transfer instruction");
let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(vec![]);
let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(Vec::new());
PublicTransaction::new(message, witness_set)
}
@@ -2022,11 +2058,11 @@ fn build_supply_bridge_account_genesis_transaction(balance: u128) -> PublicTrans
let message = Message::try_new(
faucet_program_id,
vec![system_accounts::faucet_account_id(), bridge_account_id],
vec![],
Vec::new(),
faucet_core::Instruction::GenesisTransferDirect { amount: balance },
)
.expect("Failed to serialize bridge genesis transfer instruction");
let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(vec![]);
let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(Vec::new());
PublicTransaction::new(message, witness_set)
}
@@ -2060,7 +2096,7 @@ fn build_bridge_deposit_tx_from_event(event: &PendingDepositEventRecord) -> Resu
recipient_vault_id,
receipt_id,
],
vec![],
Vec::new(),
bridge_core::Instruction::Deposit {
l1_deposit_op_id: event.deposit_op_id.0,
vault_program_id,
@@ -2070,7 +2106,7 @@ fn build_bridge_deposit_tx_from_event(event: &PendingDepositEventRecord) -> Resu
)
.context("Failed to build bridge deposit message")?;
let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(vec![]);
let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(Vec::new());
Ok(LeeTransaction::Public(PublicTransaction::new(
message,
witness_set,
@@ -2217,8 +2253,14 @@ fn extract_cross_zone_dispatch_key(tx: &LeeTransaction) -> Option<[u8; 32]> {
/// indexer, which re-derives every delivery and halts, but the local record is
/// the last copy of what we believed and it is about to be dropped either way.
/// Saying so in the log is what makes the halt diagnosable.
fn settled_dispatch_keys(dbio: &RocksDBIO, block: &Block) -> Vec<[u8; 32]> {
let recorded = dbio.get_pending_cross_zone_dispatches().unwrap_or_default();
async fn settled_dispatch_keys<S: StorageActorTrait>(
storage_ref: &ActorRef<S>,
block: &Block,
) -> Vec<[u8; 32]> {
let recorded = storage_ref
.ask(GetPendingCrossZoneDispatches)
.await
.unwrap_or_default();
let (keys, forged) = classify_settled_deliveries(&recorded, block);
for key in forged {
error!(
@@ -2263,12 +2305,18 @@ fn classify_settled_deliveries(
///
/// A persist failure is only logged: the deliveries are already irreversible, so
/// the worst case is a record the next drain drops instead.
fn settle_reconstructed_deliveries(store: &SequencerStore, block: &Block) {
let keys = settled_dispatch_keys(&store.dbio(), block);
async fn settle_reconstructed_deliveries<S: StorageActorTrait>(
storage_ref: &ActorRef<S>,
block: &Block,
) {
let keys = settled_dispatch_keys(storage_ref, block).await;
if keys.is_empty() {
return;
}
if let Err(err) = store.dbio().drop_settled_cross_zone_dispatches(&keys) {
if let Err(err) = storage_ref
.ask(DropSettledCrossZoneDispatches { message_keys: keys })
.await
{
warn!("Failed to settle reconstructed delivery records: {err:#}");
}
}
lez/sequencer/core/src/tests.rs
+484 -334
View File
File diff suppressed because it is too large Load Diff
lez/sequencer/core/src/tests/reconstruction.rs
+255 -133
View File
@@ -4,13 +4,15 @@
reason = "We don't care about it in tests"
)]
use std::sync::Mutex;
use chain_state::ChainState;
use common::block::Block;
use logos_blockchain_core::mantle::ops::channel::{MsgId, inscribe::Inscription};
use logos_blockchain_zone_sdk::{Slot, ZoneBlock, ZoneMessage};
use storage::sequencer::sequencer_cells::ZoneAnchorRecord;
use sequencer_storage_actor::{
StorageActorTrait,
protocol::{AddPendingCrossZoneDispatches, AddPendingDepositEvent, ZoneAnchorRecord},
};
use tokio::sync::Mutex;
use super::*;
use crate::{
@@ -19,13 +21,35 @@ use crate::{
/// Fresh `(store, chain)` pair for a reconstruction target, as
/// `start_from_config` would build them before the publisher starts.
fn fresh_store_and_chain(config: &SequencerConfig) -> (SequencerStore, Mutex<ChainState>) {
// TODO: Revisit this function, it relies to heavily on internal implementation details.
// Will be possible to address this once block publisher is moved to another actor(-s).
async fn fresh_store_and_chain(
config: &SequencerConfig,
) -> (SequencerStore<StorageActor>, Mutex<ChainState>) {
let storage = StorageActor::new(&config.db_path()).expect("Failed to initialize storage actor");
let storage_ref = StorageActor::spawn(storage);
// What `start_from_config` does before it opens a store, mirrored here
// because these cases drive `verify_and_reconstruct` directly.
let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap();
let bootstrap_sequencer_key = Some(test_bootstrap_sequencer_key(config));
let (store, state) =
SequencerCore::<MockBlockPublisher>::open_or_create_store(config, bootstrap_sequencer_key);
let chain = Mutex::new(SequencerCore::<MockBlockPublisher>::restore_chain_state(
config, &store, &state,
));
SequencerCore::<MockBlockPublisher>::seed_genesis_if_absent(
&storage_ref,
&signing_key,
bootstrap_sequencer_key,
config,
)
.await;
let store = SequencerStore::new(storage_ref, signing_key)
.await
.expect("open store");
let state = store
.get_lee_state()
.await
.expect("read state")
.expect("seeded store holds a state");
let chain = Mutex::new(
SequencerCore::<MockBlockPublisher>::restore_chain_state(config, &store, &state).await,
);
(store, chain)
}
@@ -40,35 +64,49 @@ fn block_to_channel_message(block: &Block, slot: u64) -> (ZoneMessage, Slot) {
/// Collects a sequencer's whole chain (genesis..=tip) into a canned channel,
/// one block per slot at `slot_step` spacing.
fn channel_from_store(store: &SequencerStore, slot_step: u64) -> Vec<(ZoneMessage, Slot)> {
async fn channel_from_store<S: StorageActorTrait>(
store: &SequencerStore<S>,
slot_step: u64,
) -> Vec<(ZoneMessage, Slot)> {
let genesis_id = store.genesis_id();
let tip_id = store.latest_block_meta().expect("tip").expect("present").id;
(genesis_id..=tip_id)
.enumerate()
.map(|(index, id)| {
let block = store.get_block_at_id(id).expect("read").expect("present");
block_to_channel_message(&block, (index as u64 + 1) * slot_step)
})
.collect()
let tip_id = store
.latest_block_meta()
.await
.expect("tip")
.expect("present")
.id;
let mut messages = Vec::new();
for (index, id) in (genesis_id..=tip_id).enumerate() {
let block = store.block_at_id(id).await.expect("read").expect("present");
messages.push(block_to_channel_message(
&block,
(index as u64 + 1) * slot_step,
));
}
messages
}
#[tokio::test]
async fn reconstructs_missing_channel_blocks_into_fresh_store() {
// Sequencer A produces a few blocks; treat its chain as the channel.
let config_a = setup_sequencer_config();
let (mut seq_a, _handle_a) =
SequencerCoreWithMockClients::start_from_config(config_a.clone()).await;
let (mut seq_a, _handle_a) = start_sequencer(config_a.clone()).await;
seq_a.run_production_turn().await.unwrap();
seq_a.run_production_turn().await.unwrap();
let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap();
let tip_a = seq_a
.block_store()
.latest_block_meta()
.await
.unwrap()
.unwrap();
let messages = channel_from_store(seq_a.block_store(), 10);
let messages = channel_from_store(seq_a.block_store(), 10).await;
let tip_slot = messages.last().unwrap().1;
let channel_id = config_a.bedrock_config.channel_id;
// Sequencer B starts from a fresh store and reconstructs A's chain.
let config_b = setup_sequencer_config();
let (store_b, chain_b) = fresh_store_and_chain(&config_b);
let (store_b, chain_b) = fresh_store_and_chain(&config_b).await;
let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages);
let channel_was_empty = SequencerCore::<MockBlockPublisher>::verify_and_reconstruct(
@@ -78,13 +116,13 @@ async fn reconstructs_missing_channel_blocks_into_fresh_store() {
.expect("reconstruct");
assert!(!channel_was_empty);
let tip_b = store_b.latest_block_meta().unwrap().unwrap();
let tip_b = store_b.latest_block_meta().await.unwrap().unwrap();
assert_eq!(tip_b.id, tip_a.id);
assert_eq!(tip_b.hash, tip_a.hash);
// State matches: initial account balances agree with sequencer A.
let state_b = chain_b.lock().unwrap().head_state().clone();
let state_a = seq_a.chain().lock().unwrap().head_state().clone();
let state_b = chain_b.lock().await.head_state().clone();
let state_a = seq_a.chain().lock().await.head_state().clone();
for account in initial_public_user_accounts() {
assert_eq!(
state_b.get_account_by_id(account.account_id).balance,
@@ -92,7 +130,11 @@ async fn reconstructs_missing_channel_blocks_into_fresh_store() {
);
}
let anchor = store_b.get_zone_anchor().unwrap().expect("anchor recorded");
let anchor = store_b
.get_zone_anchor()
.await
.unwrap()
.expect("anchor recorded");
assert_eq!(anchor.block_id, tip_a.id);
assert_eq!(anchor.slot, tip_slot.into_inner());
@@ -103,24 +145,28 @@ async fn reconstructs_missing_channel_blocks_into_fresh_store() {
.await
.expect("reconstruct idempotent");
assert!(!channel_was_empty);
assert_eq!(store_b.latest_block_meta().unwrap().unwrap().id, tip_a.id);
assert_eq!(
store_b.latest_block_meta().await.unwrap().unwrap().id,
tip_a.id
);
}
#[tokio::test]
async fn fails_when_channel_serves_a_divergent_block() {
let config = setup_sequencer_config();
let (store, chain) = fresh_store_and_chain(&config);
let (store, chain) = fresh_store_and_chain(&config).await;
// Anchor on the local genesis at some slot.
let genesis_id = store.genesis_id();
let genesis = store.get_block_at_id(genesis_id).unwrap().unwrap();
let genesis = store.block_at_id(genesis_id).await.unwrap().unwrap();
let anchor_slot = 100_u64;
store
.set_zone_anchor(&ZoneAnchorRecord {
.set_zone_anchor(ZoneAnchorRecord {
slot: anchor_slot,
block_id: genesis_id,
hash: genesis.header.hash,
})
.await
.unwrap();
// The channel serves a different block at the anchor id/slot.
@@ -142,15 +188,16 @@ async fn fails_when_channel_serves_a_divergent_block() {
#[tokio::test]
async fn fails_when_channel_is_missing() {
let config = setup_sequencer_config();
let (store, chain) = fresh_store_and_chain(&config);
let (store, chain) = fresh_store_and_chain(&config).await;
let genesis_id = store.genesis_id();
let genesis = store.get_block_at_id(genesis_id).unwrap().unwrap();
let genesis = store.block_at_id(genesis_id).await.unwrap().unwrap();
store
.set_zone_anchor(&ZoneAnchorRecord {
.set_zone_anchor(ZoneAnchorRecord {
slot: 100,
block_id: genesis_id,
hash: genesis.header.hash,
})
.await
.unwrap();
// Anchor present, but the channel does not exist on the connected chain.
@@ -169,11 +216,15 @@ async fn fails_when_channel_is_missing() {
#[tokio::test]
async fn fails_when_channel_reinscribes_genesis_with_a_different_hash() {
let config = setup_sequencer_config();
let (store, chain) = fresh_store_and_chain(&config);
let (store, chain) = fresh_store_and_chain(&config).await;
// Fresh store, no anchor. The channel serves a genesis at the same id but a
// different hash — a foreign chain reinscribing genesis.
let mut reinscribed = store.get_block_at_id(store.genesis_id()).unwrap().unwrap();
let mut reinscribed = store
.block_at_id(store.genesis_id())
.await
.unwrap()
.unwrap();
reinscribed.header.hash = HashType([0xAB_u8; 32]);
let messages = vec![block_to_channel_message(&reinscribed, 10)];
@@ -195,7 +246,7 @@ async fn fails_when_channel_reinscribes_genesis_with_a_different_hash() {
async fn fails_when_a_below_tip_channel_block_does_not_validate() {
// A sequencer that committed blocks past genesis but never recorded an anchor.
let config = setup_sequencer_config();
let (mut seq, _handle) = SequencerCoreWithMockClients::start_from_config(config.clone()).await;
let (mut seq, _handle) = start_sequencer(config.clone()).await;
seq.run_production_turn().await.unwrap();
seq.run_production_turn().await.unwrap();
@@ -206,7 +257,8 @@ async fn fails_when_a_below_tip_channel_block_does_not_validate() {
let below_tip_id = seq.block_store().genesis_id() + 1;
let mut block = seq
.block_store()
.get_block_at_id(below_tip_id)
.block_at_id(below_tip_id)
.await
.unwrap()
.unwrap();
block.header.hash = HashType([0xCD_u8; 32]);
@@ -230,12 +282,16 @@ async fn fails_when_a_below_tip_channel_block_does_not_validate() {
#[tokio::test]
async fn fails_when_a_channel_block_is_numbered_below_genesis() {
let config = setup_sequencer_config();
let (store, chain) = fresh_store_and_chain(&config);
let (store, chain) = fresh_store_and_chain(&config).await;
// A block numbered below our genesis — a foreign chain with a lower
// numbering. Nothing local sits at that id, so it goes straight to
// validation and parks there.
let mut foreign = store.get_block_at_id(store.genesis_id()).unwrap().unwrap();
let mut foreign = store
.block_at_id(store.genesis_id())
.await
.unwrap()
.unwrap();
foreign.header.block_id = store.genesis_id() - 1;
let messages = vec![block_to_channel_message(&foreign, 10)];
@@ -256,10 +312,14 @@ async fn fails_when_a_channel_block_is_numbered_below_genesis() {
#[tokio::test]
async fn fails_when_a_channel_block_does_not_extend_the_tip() {
let config = setup_sequencer_config();
let (store, chain) = fresh_store_and_chain(&config);
let (store, chain) = fresh_store_and_chain(&config).await;
// A block claiming an id far past genesis does not chain onto the local tip.
let mut orphan = store.get_block_at_id(store.genesis_id()).unwrap().unwrap();
let mut orphan = store
.block_at_id(store.genesis_id())
.await
.unwrap()
.unwrap();
orphan.header.block_id = store.genesis_id() + 5;
let messages = vec![block_to_channel_message(&orphan, 10)];
@@ -289,39 +349,42 @@ async fn fails_when_a_channel_block_does_not_extend_the_tip() {
async fn reconstruction_ignores_a_duplicate_height_the_final_tier_settled() {
// Sequencer A's chain is what the channel finalized.
let config_a = setup_sequencer_config();
let (mut seq_a, _mempool_a) =
SequencerCoreWithMockClients::start_from_config(config_a.clone()).await;
let (mut seq_a, _mempool_a) = start_sequencer(config_a.clone()).await;
seq_a.run_production_turn().await.unwrap();
let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap();
let mut messages = channel_from_store(seq_a.block_store(), 10);
let tip_a = seq_a
.block_store()
.latest_block_meta()
.await
.unwrap()
.unwrap();
let mut messages = channel_from_store(seq_a.block_store(), 10).await;
let settled_slot = messages.last().unwrap().1;
// Sequencer B: the cold-start backfill finalizes A's chain into its store.
let (seq_b, mempool_b) =
SequencerCoreWithMockClients::start_from_config(setup_sequencer_config()).await;
let finalized: Vec<(MsgId, Block)> = (seq_b.block_store().genesis_id()..=tip_a.id)
.map(|id| {
let block = seq_a.block_store().get_block_at_id(id).unwrap().unwrap();
(
MsgId::from([u8::try_from(id).expect("should be u8"); 32]),
block,
)
})
.collect();
let (seq_b, mempool_b) = start_sequencer(setup_sequencer_config()).await;
let mut finalized: Vec<(MsgId, Block)> = Vec::new();
for id in seq_b.block_store().genesis_id()..=tip_a.id {
let block = seq_a.block_store().block_at_id(id).await.unwrap().unwrap();
finalized.push((
MsgId::from([u8::try_from(id).expect("should be u8"); 32]),
block,
));
}
apply_follow_update(
&seq_b.store.dbio(),
seq_b.block_store().storage_ref(),
&seq_b.chain(),
&mempool_b,
FollowUpdate {
finalized,
..empty_follow_update()
},
);
)
.await;
assert_eq!(
seq_b
.chain()
.lock()
.unwrap()
.await
.final_tip()
.expect("backfill finalized A's chain")
.block_id,
@@ -331,7 +394,8 @@ async fn reconstruction_ignores_a_duplicate_height_the_final_tier_settled() {
// A competitor published its own block at that same height.
let parent = seq_a
.block_store()
.get_block_at_id(tip_a.id - 1)
.block_at_id(tip_a.id - 1)
.await
.unwrap()
.unwrap();
let competitor =
@@ -353,13 +417,19 @@ async fn reconstruction_ignores_a_duplicate_height_the_final_tier_settled() {
.await
.expect("a duplicate height the final tier settled must not abort startup");
let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap();
let tip_b = seq_b
.block_store()
.latest_block_meta()
.await
.unwrap()
.unwrap();
assert_eq!(tip_b.hash, tip_a.hash, "the finalized block stands");
// The anchor tracks the block we hold, never the one we dropped.
let anchor = seq_b
.block_store()
.get_zone_anchor()
.await
.unwrap()
.expect("anchor");
assert_eq!(anchor.slot, settled_slot.into_inner());
@@ -372,31 +442,46 @@ async fn reconstruction_ignores_a_duplicate_height_the_final_tier_settled() {
async fn reconstruction_replaces_a_conflicting_head_block_with_finalized_history() {
// Sequencer A's chain is what the channel finalized.
let config_a = setup_sequencer_config();
let (mut seq_a, _mempool_a) =
SequencerCoreWithMockClients::start_from_config(config_a.clone()).await;
let (mut seq_a, _mempool_a) = start_sequencer(config_a.clone()).await;
seq_a.run_production_turn().await.unwrap();
let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap();
let messages = channel_from_store(seq_a.block_store(), 10);
let tip_a = seq_a
.block_store()
.latest_block_meta()
.await
.unwrap()
.unwrap();
let messages = channel_from_store(seq_a.block_store(), 10).await;
let tip_slot = messages.last().unwrap().1;
// Sequencer B adopted a competitor at that height and never saw it finalize.
let (seq_b, mempool_b) =
SequencerCoreWithMockClients::start_from_config(setup_sequencer_config()).await;
let genesis_b = seq_b.block_store().latest_block_meta().unwrap().unwrap();
let (seq_b, mempool_b) = start_sequencer(setup_sequencer_config()).await;
let genesis_b = seq_b
.block_store()
.latest_block_meta()
.await
.unwrap()
.unwrap();
let competitor =
common::test_utils::produce_dummy_block(tip_a.id, Some(genesis_b.hash), vec![]);
assert_ne!(competitor.header.hash, tip_a.hash);
apply_follow_update(
&seq_b.store.dbio(),
seq_b.block_store().storage_ref(),
&seq_b.chain(),
&mempool_b,
FollowUpdate {
adopted: vec![(MsgId::from([7_u8; 32]), competitor)],
..empty_follow_update()
},
);
)
.await;
assert_eq!(
seq_b.block_store().latest_block_meta().unwrap().unwrap().id,
seq_b
.block_store()
.latest_block_meta()
.await
.unwrap()
.unwrap()
.id,
tip_a.id,
"the competitor is the head tip going in"
);
@@ -415,7 +500,12 @@ async fn reconstruction_replaces_a_conflicting_head_block_with_finalized_history
.await
.expect("finalized history must replace a conflicting head block");
let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap();
let tip_b = seq_b
.block_store()
.latest_block_meta()
.await
.unwrap()
.unwrap();
assert_eq!(tip_b.id, tip_a.id);
assert_eq!(
tip_b.hash, tip_a.hash,
@@ -499,7 +589,7 @@ fn deposit_event_record(
// // Sequencer A produces a deposit block then a withdraw block.
// let config_a = bridge_funded_config();
// let (mut seq_a, mempool_a) =
// SequencerCoreWithMockClients::start_from_config(config_a.clone()).await;
// start_sequencer(config_a.clone()).await;
// let deposit_record = deposit_event_record(deposit_op_id, deposit_amount, recipient);
// let deposit_tx =
@@ -523,14 +613,14 @@ fn deposit_event_record(
// .unwrap();
// seq_a.run_production_turn().await.unwrap();
// let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap();
// let messages = channel_from_store(seq_a.block_store(), 10);
// let tip_a = seq_a.block_store().latest_block_meta().await.unwrap().unwrap();
// let messages = channel_from_store(seq_a.block_store(), 10).await;
// let tip_slot = messages.last().unwrap().1;
// let channel_id = config_a.bedrock_config.channel_id;
// let config_b = bridge_funded_config();
// let (mut seq_b, _mempool_b) =
// SequencerCoreWithMockClients::start_from_config(config_b).await;
// start_sequencer(config_b).await;
// // Backfill re-delivery: the deposit event is re-recorded as a pending record
// // before reconstruction runs. The mint no longer flows through the mempool
@@ -553,7 +643,7 @@ fn deposit_event_record(
// .await
// .expect("reconstruct");
// let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap();
// let tip_b = seq_b.block_store().latest_block_meta().await.unwrap().unwrap();
// assert_eq!(tip_b.id, tip_a.id);
// assert_eq!(tip_b.hash, tip_a.hash);
@@ -575,8 +665,8 @@ fn deposit_event_record(
// let vault_id = vault_core::compute_vault_account_id(programs::vault().id(), recipient);
// let bridge_id = system_accounts::bridge_account_id();
// let state_b = seq_b.chain().lock().unwrap().head_state().clone();
// let state_a = seq_a.chain().lock().unwrap().head_state().clone();
// let state_b = seq_b.chain().lock().await.head_state().clone();
// let state_a = seq_a.chain().lock().await.head_state().clone();
// for account in [vault_id, bridge_id, recipient] {
// assert_eq!(
// state_b.get_account_by_id(account).balance,
@@ -592,7 +682,7 @@ fn deposit_event_record(
// let produced = seq_b
// .block_store()
// .get_block_at_id(tip_b.id + 1)
// .block_at_id(tip_b.id + 1)
// .unwrap()
// .expect("produced block present");
// assert!(
@@ -631,7 +721,7 @@ fn deposit_event_record(
// // Sequencer A produces a single withdraw block; treat its chain as the channel.
// let config_a = bridge_funded_config();
// let (mut seq_a, mempool_a) =
// SequencerCoreWithMockClients::start_from_config(config_a.clone()).await;
// start_sequencer(config_a.clone()).await;
// let withdraw_tx = build_public_withdraw_tx(
// recipient,
// 0,
@@ -656,20 +746,20 @@ fn deposit_event_record(
// "producing a withdraw must count it as unseen"
// );
// let messages = channel_from_store(seq_a.block_store(), 10);
// let messages = channel_from_store(seq_a.block_store(), 10).await;
// let tip_slot = messages.last().unwrap().1;
// let channel_id = config_a.bedrock_config.channel_id;
// // Sequencer B reconstructs A's chain from a fresh store.
// let config_b = bridge_funded_config();
// let (store_b, chain_b) = fresh_store_and_chain(&config_b);
// let (store_b, chain_b) = fresh_store_and_chain(&config_b).await;
// let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages);
// SequencerCore::<MockBlockPublisher>::verify_and_reconstruct(&mock_b, &store_b, &chain_b,
// true) .await
// .expect("reconstruct");
// assert!(
// !store_b.dbio().consume_unseen_withdraw_count(key).unwrap(),
// !dbio_b.consume_unseen_withdraw_count(key).unwrap(),
// "reconstruction must not leave a phantom unseen-withdraw count"
// );
// }
@@ -687,8 +777,7 @@ async fn reconstruction_reconciles_already_finished_deposit() {
// Sequencer A: a single block that fully processes the bridge deposit.
let config_a = bridge_funded_config();
let (mut seq_a, mempool_a) =
SequencerCoreWithMockClients::start_from_config(config_a.clone()).await;
let (mut seq_a, mempool_a) = start_sequencer(config_a.clone()).await;
let deposit_record = deposit_event_record(deposit_op_id, deposit_amount, recipient);
let deposit_tx =
crate::build_bridge_deposit_tx_from_event(&deposit_record).expect("build deposit tx");
@@ -698,7 +787,7 @@ async fn reconstruction_reconciles_already_finished_deposit() {
.unwrap();
seq_a.run_production_turn().await.unwrap();
let messages = channel_from_store(seq_a.block_store(), 10);
let messages = channel_from_store(seq_a.block_store(), 10).await;
let tip_slot = messages.last().unwrap().1;
let channel_id = config_a.bedrock_config.channel_id;
@@ -706,11 +795,14 @@ async fn reconstruction_reconciles_already_finished_deposit() {
// pre-seeded, as the cold-start backfill would when it re-observes this
// already-finalized deposit.
let config_b = bridge_funded_config();
let (store_b, chain_b) = fresh_store_and_chain(&config_b);
let (store_b, chain_b) = fresh_store_and_chain(&config_b).await;
assert!(
store_b
.dbio()
.add_pending_deposit_event(deposit_record.clone())
.storage_ref()
.ask(AddPendingDepositEvent {
event: deposit_record.clone()
})
.await
.unwrap()
);
@@ -724,7 +816,7 @@ async fn reconstruction_reconciles_already_finished_deposit() {
assert_eq!(
chain_b
.lock()
.unwrap()
.await
.head_state()
.get_account_by_id(vault_id)
.balance,
@@ -736,14 +828,15 @@ async fn reconstruction_reconciles_already_finished_deposit() {
// dropped the pending record backfill had re-delivered — so the production
// drain sees the deposit as minted and never re-emits it.
assert!(
crate::deposit_already_minted(
chain_b.lock().unwrap().head_state(),
HashType(deposit_op_id)
),
crate::deposit_already_minted(chain_b.lock().await.head_state(), HashType(deposit_op_id)),
"the reconstructed deposit's receipt marks it minted"
);
assert!(
store_b.get_pending_deposit_events().unwrap().is_empty(),
store_b
.get_pending_deposit_events()
.await
.unwrap()
.is_empty(),
"reconstruction drops the finalized deposit's pending record"
);
}
@@ -761,29 +854,38 @@ async fn reconstructed_delivery_settles_its_pending_record() {
// Sequencer A produces the block that carries the delivery.
let config_a = cross_zone_test_config();
let (mut seq_a, _mempool_a) =
SequencerCoreWithMockClients::start_from_config(config_a.clone()).await;
let (mut seq_a, _mempool_a) = start_sequencer(config_a.clone()).await;
seq_a
.block_store()
.dbio()
.add_pending_cross_zone_dispatches(vec![record.clone()])
.storage_ref()
.ask(AddPendingCrossZoneDispatches {
dispatches: vec![record.clone()],
})
.await
.unwrap();
seq_a.run_production_turn().await.unwrap();
let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap();
let messages = channel_from_store(seq_a.block_store(), 10);
let tip_a = seq_a
.block_store()
.latest_block_meta()
.await
.unwrap()
.unwrap();
let messages = channel_from_store(seq_a.block_store(), 10).await;
let tip_slot = messages.last().unwrap().1;
let channel_id = config_a.bedrock_config.channel_id;
// Sequencer B holds the same record, as its own watcher would after reading
// the peer block, and reconstructs A's chain from a fresh store.
let (mut seq_b, _mempool_b) =
SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await;
let (mut seq_b, _mempool_b) = start_sequencer(cross_zone_test_config()).await;
assert_eq!(
seq_b
.block_store()
.dbio()
.add_pending_cross_zone_dispatches(vec![record])
.storage_ref()
.ask(AddPendingCrossZoneDispatches {
dispatches: vec![record],
})
.await
.unwrap(),
1
);
@@ -798,14 +900,19 @@ async fn reconstructed_delivery_settles_its_pending_record() {
.await
.expect("reconstruct");
let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap();
let tip_b = seq_b
.block_store()
.latest_block_meta()
.await
.unwrap()
.unwrap();
assert_eq!(tip_b.id, tip_a.id);
assert_eq!(tip_b.hash, tip_a.hash);
assert!(
seq_b
.block_store()
.dbio()
.get_pending_cross_zone_dispatches()
.store
.pending_cross_zone_dispatches()
.await
.unwrap()
.is_empty(),
"reconstruction must settle the record of a delivery it replayed"
@@ -814,14 +921,17 @@ async fn reconstructed_delivery_settles_its_pending_record() {
// The delivery landed exactly once, and the next turn does not re-emit it.
let record_id = ping_record_pda(programs::ping_receiver().id());
assert_eq!(
seq_b.with_state(|state| state.get_account_by_id(record_id).data.into_inner()),
seq_b
.with_state(|state| state.get_account_by_id(record_id).data.into_inner())
.await,
payload,
"the reconstructed delivery must reach its target program"
);
seq_b.run_production_turn().await.unwrap();
let produced = seq_b
.block_store()
.get_block_at_id(tip_b.id + 1)
.block_at_id(tip_b.id + 1)
.await
.unwrap()
.expect("produced block present");
assert!(
@@ -840,24 +950,27 @@ async fn a_verified_own_block_settles_its_delivery_records() {
let record = dispatch_record(37, ping_payload(b"verified"));
let key = record.message_key;
let (mut seq, _mempool) =
SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await;
let (mut seq, _mempool) = start_sequencer(cross_zone_test_config()).await;
seq.block_store()
.dbio()
.add_pending_cross_zone_dispatches(vec![record])
.storage_ref()
.ask(AddPendingCrossZoneDispatches {
dispatches: vec![record],
})
.await
.unwrap();
let block_id = seq.run_production_turn().await.unwrap();
let block = seq
.block_store()
.get_block_at_id(block_id)
.block_at_id(block_id)
.await
.unwrap()
.unwrap();
assert_eq!(dispatches_in(&block), vec![key]);
assert_eq!(
seq.block_store()
.dbio()
.get_pending_cross_zone_dispatches()
seq.store
.pending_cross_zone_dispatches()
.await
.unwrap()
.len(),
1,
@@ -865,7 +978,7 @@ async fn a_verified_own_block_settles_its_delivery_records() {
);
// The channel serves our own chain back, tip included.
let messages = channel_from_store(seq.block_store(), 10);
let messages = channel_from_store(seq.block_store(), 10).await;
let tip_slot = messages.last().unwrap().1;
let mock = MockBlockPublisher::with_canned_channel(
seq.sequencer_config.bedrock_config.channel_id,
@@ -879,9 +992,9 @@ async fn a_verified_own_block_settles_its_delivery_records() {
.expect("reconstruct");
assert!(
seq.block_store()
.dbio()
.get_pending_cross_zone_dispatches()
seq.store
.pending_cross_zone_dispatches()
.await
.unwrap()
.is_empty(),
"a delivery the channel confirms must not leave a record nothing can remove"
@@ -895,19 +1008,28 @@ async fn committed_local_against_missing_channel_fails_without_anchor() {
// anchor). Restarting it against a wiped/missing channel must still fail,
// driven by the committed-blocks invariant rather than an anchor probe.
let config = setup_sequencer_config();
{
let (mut seq, _handle) =
SequencerCoreWithMockClients::start_from_config(config.clone()).await;
let storage_weak = {
let (mut seq, _handle) = start_sequencer(config.clone()).await;
seq.run_production_turn().await.unwrap();
seq.run_production_turn().await.unwrap();
assert!(seq.block_store().latest_block_meta().unwrap().unwrap().id > 1);
} // drop releases the store so we can reopen it
assert!(
seq.block_store()
.latest_block_meta()
.await
.unwrap()
.unwrap()
.id
> 1
);
seq.block_store().storage_ref().downgrade()
};
storage_weak.wait_for_shutdown_with_result(|_| ()).await;
// Reopen: blocks beyond genesis, no anchor. `is_fresh_start = false` stands in
// for a checkpoint persisted by a prior sync (the mock never emits one).
let (store, chain) = fresh_store_and_chain(&config);
assert!(store.get_zone_anchor().unwrap().is_none());
assert!(store.latest_block_meta().unwrap().unwrap().id > 1);
let (store, chain) = fresh_store_and_chain(&config).await;
assert!(store.get_zone_anchor().await.unwrap().is_none());
assert!(store.latest_block_meta().await.unwrap().unwrap().id > 1);
// The channel is gone: no tip, no messages.
let mock =
lez/sequencer/service/Cargo.toml
+1
View File
@@ -12,6 +12,7 @@ workspace = true
sequencer_core = { workspace = true, features = ["testnet"] }
sequencer_executor_actor.workspace = true
sequencer_rpc_server_actor.workspace = true
sequencer_storage_actor.workspace = true
lee = { workspace = true, optional = true }
programs = { workspace = true, optional = true }
system_accounts = { workspace = true, optional = true }
lez/sequencer/service/src/lib.rs
+26 -8
View File
@@ -9,6 +9,7 @@ pub use sequencer_core::config::*;
use sequencer_core::load_or_create_signing_key;
use sequencer_executor_actor::ExecutorActor;
use sequencer_rpc_server_actor::RpcServerActor;
use sequencer_storage_actor::StorageActor;
use tokio::select;
use crate::actor_handle::ActorHandle;
@@ -29,6 +30,7 @@ pub struct SequencerHandle {
scheduler: ActorHandle<Scheduler>,
rpc_server: ActorHandle<RpcServerActor>,
executor: ActorHandle<ExecutorActor<BlockPublisher>>,
storage: ActorHandle<StorageActor>,
addr: SocketAddr,
/// Held for its lifetime: dropping it stops the gossip drive task.
/// `None` when gossip is unconfigured.
@@ -40,6 +42,7 @@ impl SequencerHandle {
scheduler: ActorHandle<Scheduler>,
rpc_server: ActorHandle<RpcServerActor>,
executor: ActorHandle<ExecutorActor<BlockPublisher>>,
storage: ActorHandle<StorageActor>,
addr: SocketAddr,
gossip: Option<sequencer_core::gossip::GossipNetwork>,
) -> Self {
@@ -47,6 +50,7 @@ impl SequencerHandle {
scheduler,
rpc_server,
executor,
storage,
addr,
gossip,
}
@@ -59,6 +63,7 @@ impl SequencerHandle {
scheduler,
rpc_server,
executor,
storage,
addr: _,
gossip: _,
} = self;
@@ -67,6 +72,7 @@ impl SequencerHandle {
scheduler.shutdown().await;
rpc_server.shutdown().await;
executor.shutdown().await;
storage.shutdown().await;
}
/// Wait for any of the sequencer tasks to fail and return the error.
@@ -79,6 +85,7 @@ impl SequencerHandle {
executor,
rpc_server,
scheduler,
storage,
addr: _,
gossip: _,
} = self;
@@ -93,6 +100,9 @@ impl SequencerHandle {
Err(err) = scheduler.failed() => {
Err(err)
}
Err(err) = storage.failed() => {
Err(err)
}
}
}
@@ -106,11 +116,15 @@ impl SequencerHandle {
executor,
rpc_server,
scheduler,
storage,
addr: _,
gossip: _,
} = self;
executor.is_healthy() && rpc_server.is_healthy() && scheduler.is_healthy()
executor.is_healthy()
&& rpc_server.is_healthy()
&& scheduler.is_healthy()
&& storage.is_healthy()
}
#[must_use]
@@ -132,19 +146,21 @@ pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result<Seq
let block_timeout = config.block_create_timeout;
let max_block_size = config.max_block_size;
// Captured before `config` moves into the executor; gossip needs them after.
let gossip_config = config.gossip.clone();
let bedrock_config = config.bedrock_config.clone();
let sequencer_home = config.home.clone();
let executor = ExecutorActor::new(config).await;
let storage =
StorageActor::new(&config.db_path()).context("Failed to initialize Storage Actor")?;
let storage_ref = StorageActor::spawn(storage);
info!("Storage Actor spawned");
let executor = ExecutorActor::new(config, storage_ref.clone()).await;
let mempool_handle = executor.mempool_handle();
let executor_ref = ExecutorActor::spawn(executor);
info!("Executor Actor spawned");
// Gossip is constructed only when configured; a `None` config means no
// sockets and no tasks. Startup failure here is a hard error
// (misconfiguration); after startup, gossip never halts the node.
// TODO: Should be a separate actor
let gossip_network = match gossip_config {
None => None,
Some(gossip_config) => {
@@ -171,12 +187,13 @@ pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result<Seq
.map(sequencer_core::gossip::GossipNetwork::tx_publisher);
let rpc_server = RpcServerActor::new(
executor_ref.clone(),
listen_addr,
max_block_size,
executor_ref.clone(),
tx_publisher,
)
.await?;
.await
.context("Failed to initialize RPC Server Actor")?;
let addr = rpc_server.addr();
let rpc_server_ref = RpcServerActor::spawn(rpc_server);
info!("RPC Server Actor spawned");
@@ -199,6 +216,7 @@ pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result<Seq
ActorHandle::new(scheduler_ref),
ActorHandle::new(rpc_server_ref),
ActorHandle::new(executor_ref),
ActorHandle::new(storage_ref),
addr,
gossip_network,
))
lez/storage/src/sequencer/mod.rs
+55 -107
View File
@@ -17,22 +17,21 @@ use rocksdb::{
};
use crate::{
CF_BLOCK_NAME, CF_META_NAME, DB_META_FIRST_BLOCK_IN_DB_KEY, DBIO, DbResult,
CF_BLOCK_NAME, CF_META_NAME, DBIO, DbResult,
cells::shared_cells::{BlockCell, FirstBlockCell, FirstBlockSetCell, LastBlockCell},
error::DbError,
sequencer::sequencer_cells::{
DeadLetterCrossZoneDispatchCountCell, DeadLetterCrossZoneDispatchesCellOwned,
DeadLetterCrossZoneDispatchesCellRef, DeadLetterDispatchRecord, DispatchOrigin,
FinalBlockMetaCellOwned, FinalBlockMetaCellRef, FinalLeeStateCellOwned,
FinalLeeStateCellRef, LEEStateCellOwned, LEEStateCellRef, LastFinalizedBlockIdCell,
LatestBlockMetaCellOwned, LatestBlockMetaCellRef,
LegacyPendingCrossZoneDispatchesCellOwned, PeerChainTip, PeerFloorCellOwned,
PeerFloorCellRef, PeerTipCell, PeerZoneKey, PendingCrossZoneDispatchCellOwned,
PendingCrossZoneDispatchCellRef, PendingCrossZoneDispatchCountCell,
PendingCrossZoneDispatchRecord, PendingDepositEventRecord, PendingDepositEventsCellOwned,
PendingDepositEventsCellRef, PublishedHighWaterCell, UnseenWithdrawCountCell,
WithdrawalReconciliationKey, ZoneAnchorCell, ZoneAnchorRecord, ZoneSdkCheckpointCellOwned,
ZoneSdkCheckpointCellRef,
FinalLeeStateCellRef, LEEStateCellOwned, LEEStateCellRef, LatestBlockMetaCellOwned,
LatestBlockMetaCellRef, LegacyPendingCrossZoneDispatchesCellOwned, PeerChainTip,
PeerFloorCellOwned, PeerFloorCellRef, PeerTipCell, PeerZoneKey,
PendingCrossZoneDispatchCellOwned, PendingCrossZoneDispatchCellRef,
PendingCrossZoneDispatchCountCell, PendingCrossZoneDispatchRecord,
PendingDepositEventRecord, PendingDepositEventsCellOwned, PendingDepositEventsCellRef,
PublishedHighWaterCell, UnseenWithdrawCountCell, WithdrawalReconciliationKey,
ZoneAnchorCell, ZoneAnchorRecord, ZoneSdkCheckpointCellOwned, ZoneSdkCheckpointCellRef,
},
};
@@ -271,33 +270,15 @@ impl RocksDBIO {
.unwrap_or_else(PoisonError::into_inner)
}
pub fn open(path: &Path) -> DbResult<Self> {
let db_opts = Options::default();
Self::open_inner(path, &db_opts)
}
pub fn create(path: &Path, genesis_block: &Block, genesis_state: &V03State) -> DbResult<Self> {
/// Opens the database at `path`, creating an empty one when there is none.
///
/// An empty one holds no chain: every read of the first or last block fails
/// until [`Self::write_genesis`] seeds it.
pub fn open_or_create(path: &Path) -> DbResult<Self> {
let mut db_opts = Options::default();
db_opts.create_missing_column_families(true);
db_opts.create_if_missing(true);
let dbio = Self::open_inner(path, &db_opts)?;
let is_start_set = dbio.get_meta_is_first_block_set()?;
if !is_start_set {
let block_id = genesis_block.header.block_id;
// TODO: Shouldn't this be atomic (batched)?
dbio.put_meta_first_block_in_db(genesis_block)?;
dbio.put_meta_is_first_block_set()?;
dbio.put_meta_last_block_in_db(block_id)?;
dbio.put_meta_last_finalized_block_id(None)?;
dbio.put_meta_latest_block_meta(&BlockMeta {
id: genesis_block.header.block_id,
hash: genesis_block.header.hash,
})?;
dbio.put_lee_state_in_db(genesis_state)?;
}
Ok(dbio)
Self::open_inner(path, &db_opts)
}
/// Dump every key/value pair across all column families into a [`DbDump`]. Column families are
@@ -497,22 +478,22 @@ impl RocksDBIO {
// Meta
pub fn get_meta_first_block_in_db(&self) -> DbResult<u64> {
self.get::<FirstBlockCell>(()).map(|cell| cell.0)
/// The id of the chain's first block, or `None` when this database holds no
/// chain yet — nothing has written a genesis into it.
pub fn get_meta_first_block_in_db(&self) -> DbResult<Option<u64>> {
Ok(self.get_opt::<FirstBlockCell>(())?.map(|cell| cell.0))
}
pub fn get_meta_last_block_in_db(&self) -> DbResult<u64> {
self.get::<LastBlockCell>(()).map(|cell| cell.0)
/// The id of the chain's last block, or `None` when this database holds no
/// chain yet.
pub fn get_meta_last_block_in_db(&self) -> DbResult<Option<u64>> {
Ok(self.get_opt::<LastBlockCell>(())?.map(|cell| cell.0))
}
pub fn get_meta_is_first_block_set(&self) -> DbResult<bool> {
Ok(self.get_opt::<FirstBlockSetCell>(())?.is_some())
}
pub fn put_lee_state_in_db(&self, state: &V03State) -> DbResult<()> {
self.put(&LEEStateCellRef(state), ())
}
pub fn put_lee_state_in_db_batch(
&self,
state: &V03State,
@@ -521,42 +502,6 @@ impl RocksDBIO {
self.put_batch(&LEEStateCellRef(state), (), batch)
}
pub fn put_meta_first_block_in_db(&self, block: &Block) -> DbResult<()> {
let cf_meta = self.meta_column();
self.db
.put_cf(
&cf_meta,
borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| {
DbError::borsh_cast_message(
err,
Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_owned()),
)
})?,
borsh::to_vec(&block.header.block_id).map_err(|err| {
DbError::borsh_cast_message(
err,
Some("Failed to serialize first block id".to_owned()),
)
})?,
)
.map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?;
let mut batch = WriteBatch::default();
self.put_block(block, true, &mut batch)?;
self.db.write(batch).map_err(|rerr| {
DbError::rocksdb_cast_message(
rerr,
Some("Failed to write first block in db".to_owned()),
)
})?;
Ok(())
}
pub fn put_meta_last_block_in_db(&self, block_id: u64) -> DbResult<()> {
self.put(&LastBlockCell(block_id), ())
}
fn put_meta_last_block_in_db_batch(
&self,
block_id: u64,
@@ -565,16 +510,13 @@ impl RocksDBIO {
self.put_batch(&LastBlockCell(block_id), (), batch)
}
pub fn put_meta_last_finalized_block_id(&self, block_id: Option<u64>) -> DbResult<()> {
self.put(&LastFinalizedBlockIdCell(block_id), ())
}
pub fn put_meta_is_first_block_set(&self) -> DbResult<()> {
self.put(&FirstBlockSetCell(true), ())
}
fn put_meta_latest_block_meta(&self, block_meta: &BlockMeta) -> DbResult<()> {
self.put(&LatestBlockMetaCellRef(block_meta), ())
fn put_meta_first_block_in_db_batch(
&self,
block_id: u64,
batch: &mut WriteBatch,
) -> DbResult<()> {
self.put_batch(&FirstBlockCell(block_id), (), batch)?;
self.put_batch(&FirstBlockSetCell(true), (), batch)
}
fn put_meta_latest_block_meta_batch(
@@ -1243,22 +1185,6 @@ impl RocksDBIO {
Ok(unmatched.is_empty())
}
pub fn put_block(&self, block: &Block, first: bool, batch: &mut WriteBatch) -> DbResult<()> {
if !first {
// A produced block is the new head tip by construction: pin the
// tip meta and drop any stale higher blocks a preceding reorg left
// behind (mirrors `store_followed_blocks`).
let last_curr_block = self.get_meta_last_block_in_db()?;
for stale_id in block.header.block_id.saturating_add(1)..=last_curr_block {
self.delete_block_payload(stale_id, batch)?;
}
self.put_meta_last_block_in_db_batch(block.header.block_id, batch)?;
self.put_meta_latest_block_meta_batch(&BlockMeta::from(block), batch)?;
}
self.put_block_payload(block, batch)
}
/// Stages deletion of a block payload into `batch`.
fn delete_block_payload(&self, block_id: u64, batch: &mut WriteBatch) -> DbResult<()> {
let cf_block = self.block_column();
@@ -1311,8 +1237,10 @@ impl RocksDBIO {
self.put_batch(&FinalBlockMetaCellRef(meta), (), batch)
}
pub fn get_lee_state(&self) -> DbResult<V03State> {
self.get::<LEEStateCellOwned>(()).map(|val| val.0)
/// The state after the last applied block, or `None` when this database
/// holds no chain yet.
pub fn get_lee_state(&self) -> DbResult<Option<V03State>> {
Ok(self.get_opt::<LEEStateCellOwned>(())?.map(|val| val.0))
}
pub fn delete_block(&self, block_id: u64) -> DbResult<()> {
@@ -1350,6 +1278,15 @@ impl RocksDBIO {
for block in to_write.values() {
self.put_block_payload(block, &mut batch)?;
}
// The lowest block written to a store that holds no chain starts one.
// Nothing else records where a chain begins, and without it every read
// that walks from the first block has no lower bound.
if let Some((first_id, _)) = to_write.first_key_value()
&& self.get_meta_first_block_in_db()?.is_none()
{
self.put_meta_first_block_in_db_batch(*first_id, &mut batch)?;
}
self.db.write(batch).map_err(|rerr| {
DbError::rocksdb_cast_message(
rerr,
@@ -1460,7 +1397,9 @@ impl RocksDBIO {
zone_anchor,
} = *update;
let last_block_in_db = self.get_meta_last_block_in_db()?;
// 0 stands in for "no chain yet": nothing to sweep above the new tip,
// and any tip this update pins is a change.
let last_block_in_db = self.get_meta_last_block_in_db()?.unwrap_or(0);
let mut batch = WriteBatch::default();
if let Some(bytes) = checkpoint {
@@ -1506,6 +1445,15 @@ impl RocksDBIO {
self.put_block_payload(block, &mut batch)?;
}
// The lowest block written to a store that holds no chain starts one.
// Nothing else records where a chain begins, and without it every read
// that walks from the first block has no lower bound.
if let Some((first_id, _)) = to_write.first_key_value()
&& self.get_meta_first_block_in_db()?.is_none()
{
self.put_meta_first_block_in_db_batch(*first_id, &mut batch)?;
}
let accepted_deposits = self.stage_pending_deposit_events(
new_deposit_events,
remove_deposit_records,
lez/storage/src/sequencer/tests.rs
+55 -6
View File
@@ -24,7 +24,11 @@ fn state_with_balance(balance: u128) -> V03State {
fn dbio_with_genesis(path: &Path) -> (RocksDBIO, Block) {
let genesis = produce_dummy_block(1, None, vec![]);
let dbio = RocksDBIO::create(path, &genesis, &state_with_balance(100)).unwrap();
let dbio = RocksDBIO::open_or_create(path).unwrap();
// The same write any block takes: the first one into an empty store starts
// its chain.
dbio.atomic_update(&genesis, &[], &state_with_balance(100), None)
.unwrap();
(dbio, genesis)
}
@@ -68,6 +72,7 @@ fn sorted_dispatches(
fn stored_balance(dbio: &RocksDBIO) -> u128 {
dbio.get_lee_state()
.unwrap()
.expect("the store holds a chain")
.get_account_by_id(marker_id())
.balance
}
@@ -487,7 +492,7 @@ fn peer_chain_tips_round_trip_and_are_kept_per_peer() {
// On disk, not in memory: a watcher that re-anchored on restart would take
// whatever block reached it first, which is the id the attack picks.
drop(dbio);
let reopened = RocksDBIO::open(temp_dir.path()).unwrap();
let reopened = RocksDBIO::open_or_create(temp_dir.path()).unwrap();
assert_eq!(
reopened.get_cross_zone_peer_tip(peer_a).unwrap(),
Some(advanced)
@@ -586,7 +591,7 @@ fn dispatch_records_survive_a_reopen() {
// On disk, not in memory: the records are what stand between the watcher's
// durable read floor and a lost delivery across a restart.
drop(dbio);
let reopened = RocksDBIO::open(temp_dir.path()).unwrap();
let reopened = RocksDBIO::open_or_create(temp_dir.path()).unwrap();
assert_eq!(
sorted_dispatches(reopened.get_pending_cross_zone_dispatches().unwrap()),
sorted_dispatches(vec![first, second])
@@ -612,7 +617,7 @@ fn a_legacy_dispatch_blob_is_migrated_into_per_message_entries_on_open() {
.unwrap();
drop(dbio);
let migrated = RocksDBIO::open(temp_dir.path()).unwrap();
let migrated = RocksDBIO::open_or_create(temp_dir.path()).unwrap();
assert_eq!(
sorted_dispatches(migrated.get_pending_cross_zone_dispatches().unwrap()),
sorted_dispatches(records.clone()),
@@ -650,7 +655,7 @@ fn a_legacy_dispatch_blob_is_migrated_into_per_message_entries_on_open() {
.unwrap();
drop(migrated);
let cleaned = RocksDBIO::open(temp_dir.path()).unwrap();
let cleaned = RocksDBIO::open_or_create(temp_dir.path()).unwrap();
assert!(
cleaned
.db
@@ -687,7 +692,7 @@ fn a_legacy_blob_over_live_entries_folds_additively() {
.unwrap();
drop(dbio);
let merged = RocksDBIO::open(temp_dir.path()).unwrap();
let merged = RocksDBIO::open_or_create(temp_dir.path()).unwrap();
assert_eq!(
sorted_dispatches(merged.get_pending_cross_zone_dispatches().unwrap()),
sorted_dispatches(vec![
@@ -1158,3 +1163,47 @@ fn produced_block_below_disk_head_pins_meta_and_prunes() {
assert_eq!(meta.hash, block2b.header.hash);
assert_eq!(stored_balance(&dbio), 400);
}
/// A database nothing has written a genesis into answers "no chain yet" rather
/// than failing: every one of these was a hard `get` that errored on absence.
#[test]
fn an_unseeded_store_reports_no_chain() {
let dir = tempfile::tempdir().expect("temp dir");
let dbio = RocksDBIO::open_or_create(dir.path()).expect("open");
assert_eq!(dbio.get_meta_first_block_in_db().unwrap(), None);
assert_eq!(dbio.get_meta_last_block_in_db().unwrap(), None);
assert!(dbio.get_lee_state().unwrap().is_none());
assert!(dbio.latest_block_meta().unwrap().is_none());
assert!(dbio.get_final_snapshot().unwrap().is_none());
assert!(!dbio.get_meta_is_first_block_set().unwrap());
assert!(dbio.get_block(1).unwrap().is_none());
}
/// The first block written to an empty store starts its chain — the property
/// that lets a genesis go in as an ordinary block write.
#[test]
fn the_first_block_written_starts_the_chain() {
let dir = tempfile::tempdir().expect("temp dir");
let dbio = RocksDBIO::open_or_create(dir.path()).expect("open");
assert_eq!(dbio.get_meta_first_block_in_db().unwrap(), None);
let genesis = produce_dummy_block(1, None, vec![]);
dbio.atomic_update(&genesis, &[], &state_with_balance(100), None)
.expect("seed");
assert_eq!(dbio.get_meta_first_block_in_db().unwrap(), Some(1));
assert_eq!(dbio.get_meta_last_block_in_db().unwrap(), Some(1));
assert!(dbio.get_meta_is_first_block_set().unwrap());
assert_eq!(
dbio.get_block(1).unwrap().unwrap().header.hash,
genesis.header.hash
);
// A later block extends the chain rather than restarting it.
let second = produce_dummy_block(2, Some(genesis.header.hash), vec![]);
dbio.atomic_update(&second, &[], &state_with_balance(100), None)
.expect("extend");
assert_eq!(dbio.get_meta_first_block_in_db().unwrap(), Some(1));
assert_eq!(dbio.get_meta_last_block_in_db().unwrap(), Some(2));
}
test_fixtures/Cargo.toml
+2
View File
@@ -18,6 +18,7 @@ sequencer_core = { workspace = true, features = ["default", "testnet"] }
sequencer_service.workspace = true
sequencer_stake_core.workspace = true
sequencer_service_rpc = { workspace = true, features = ["client"] }
sequencer_storage_actor.workspace = true
wallet.workspace = true
programs.workspace = true
@@ -28,6 +29,7 @@ env_logger.workspace = true
futures.workspace = true
hex.workspace = true
jsonrpsee = { workspace = true, features = ["ws-client"] }
kameo.workspace = true
log.workspace = true
num-bigint.workspace = true
serde.workspace = true
test_fixtures/fixtures/prebuilt_sequencer_db.dump
BIN
View File
Binary file not shown.
test_fixtures/src/bin/regenerate_test_fixture.rs
+21 -33
View File
@@ -3,11 +3,14 @@
#![expect(clippy::print_stdout, reason = "It's normal in this small cli")]
use std::{path::Path, time::Duration};
use std::path::Path;
use anyhow::{Context as _, Result};
use lee::PrivateKey;
use sequencer_core::block_store::SequencerStore;
use kameo::actor::Spawn as _;
use sequencer_storage_actor::{
StorageActor,
protocol::{DeleteZoneCheckpoint, DumpDb, ResetAllBlocksToPending},
};
use test_fixtures::{
config,
setup::{
@@ -75,22 +78,28 @@ async fn generate_prebuilt_fixture(dest: &Path) -> Result<()> {
// Shut down gracefully to release the rocksdb lock before reopening the store.
drop(wallet);
drop(sequencer_handle);
sequencer_handle.shutdown().await;
let db_path = temp_sequencer_dir
.path()
.join(format!("rocksdb-{}", config::bedrock_channel_id()));
let store = open_store_with_retry(&db_path)
let storage =
StorageActor::new(&db_path).context("Failed to reopen sequencer storage after shutdown")?;
let storage_ref = StorageActor::spawn(storage);
storage_ref
.ask(DeleteZoneCheckpoint)
.await
.context("Failed to reopen sequencer store after shutdown")?;
store
.delete_zone_checkpoint()
.context("Failed to strip zone-sdk checkpoint from fixture database")?;
store
.reset_all_blocks_to_pending()
storage_ref
.ask(ResetAllBlocksToPending)
.await
.context("Failed to reset fixture blocks to pending")?;
let dump = store.dump().context("Failed to dump fixture database")?;
drop(store);
let dump = storage_ref
.ask(DumpDb)
.await
.context("Failed to dump fixture database")?;
storage_ref.stop_gracefully().await?;
storage_ref.wait_for_shutdown_with_result(|_| ()).await;
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent)
@@ -104,24 +113,3 @@ async fn generate_prebuilt_fixture(dest: &Path) -> Result<()> {
Ok(())
}
/// Reopen the store, retrying while the shut-down sequencer's aborted tasks are still releasing
/// their db handles (the process holds the file lock until then). Must `await` between attempts.
async fn open_store_with_retry(db_path: &Path) -> Result<SequencerStore> {
let signing_key = PrivateKey::try_new(config::SEQUENCER_SIGNING_KEY)
.expect("Fixed sequencer signing key must be valid");
let mut last_err = None;
for _ in 0..100 {
// Let the runtime drop the aborted sequencer tasks before each attempt.
tokio::time::sleep(Duration::from_millis(100)).await;
match SequencerStore::open_db(db_path, signing_key.clone()) {
Ok(store) => return Ok(store),
Err(err) => last_err = Some(err),
}
}
Err(anyhow::anyhow!(
"Failed to open sequencer store at {} after retries: {last_err:?}",
db_path.display()
))
}
test_fixtures/src/setup.rs
+8 -12
View File
@@ -7,12 +7,10 @@ use anyhow::{Context as _, Result, bail};
use indexer_service::{ChannelId, IndexerHandle};
use lee::{AccountId, PrivateKey, PublicKey};
use log::{debug, warn};
use sequencer_core::{
block_publisher::ED25519_SECRET_KEY_SIZE,
block_store::{DbDump, SequencerStore},
};
use sequencer_core::block_publisher::ED25519_SECRET_KEY_SIZE;
use sequencer_service::{GenesisAction, SequencerHandle};
use sequencer_service_rpc::{SequencerClient, SequencerClientBuilder};
use sequencer_storage_actor::{StorageActor, protocol::DbDump};
use tempfile::TempDir;
use testcontainers::compose::DockerCompose;
use wallet::{
@@ -149,15 +147,13 @@ impl SequencerSetup {
genesis
} else {
let dump = load_prebuilt_dump()?;
// `SequencerCore::open_or_create_store` looks for the channel-suffixed
// db under its home, so the restore has to land on the same name.
// The sequencer looks for the channel-suffixed db under its home,
// so the restore has to land on the same name.
let dst = home.join(format!("rocksdb-{channel_id}"));
let _store = SequencerStore::restore_db_from_dump(
&dst,
&dump,
lee::PrivateKey::try_new(config::SEQUENCER_SIGNING_KEY)?,
)
.context("Failed to restore prebuilt sequencer database from dump")?;
// Dropped right away: this only writes the database, which the
// sequencer opens for itself below.
let _storage = StorageActor::restore_from_dump(&dst, &dump)
.context("Failed to restore prebuilt sequencer database from dump")?;
// TODO: Technically not correct, we should reconstruct the genesis transactions
// from the dump, but this crutch doesn't affect anything for now
Vec::new()