use std::{path::Path, sync::Arc, time::Instant}; use anyhow::{Context as _, Result, anyhow}; use borsh::BorshDeserialize; use common::{ HashType, block::{BedrockStatus, Block, HashableBlockData}, transaction::{LeeTransaction, clock_invocation}, }; use config::{GenesisAction, SequencerConfig}; use itertools::Itertools as _; use lee::{AccountId, PublicTransaction, public_transaction::Message}; use lee_core::GENESIS_BLOCK_ID; use log::{error, info, warn}; use logos_blockchain_key_management_system_service::keys::{ED25519_SECRET_KEY_SIZE, Ed25519Key}; use logos_blockchain_zone_sdk::sequencer::{DepositInfo, WithdrawArg}; use mempool::{MemPool, MemPoolHandle}; #[cfg(feature = "mock")] pub use mock::SequencerCoreWithMockClients; use num_bigint::BigUint; pub use storage::error::DbError; use storage::sequencer::{ RocksDBIO, sequencer_cells::{PendingDepositEventRecord, WithdrawalReconciliationKey}, }; use crate::{ block_publisher::{BlockPublisherTrait, ZoneSdkPublisher}, block_store::SequencerStore, }; pub mod block_publisher; pub mod block_store; pub mod config; #[cfg(feature = "mock")] pub mod mock; /// The origin of a transaction. #[derive(Clone, Copy)] pub enum TransactionOrigin { /// Basic transactions submitted by users via RPC. User, /// Transactions generated by the sequencer itself. Sequencer, } #[derive(Clone, Debug, BorshDeserialize)] struct DepositMetadata { recipient_id: lee::AccountId, } impl DepositMetadata { fn decode(bytes: &[u8]) -> Result { Self::try_from_slice(bytes) } } pub struct SequencerCore { state: lee::V03State, store: SequencerStore, mempool: MemPool<(TransactionOrigin, LeeTransaction)>, sequencer_config: SequencerConfig, chain_height: u64, block_publisher: BP, } impl SequencerCore { /// Starts the sequencer using the provided configuration. /// If an existing database is found, the sequencer state is loaded from it and /// assumed to represent the correct latest state consistent with Bedrock-finalized data. /// If no database is found, the sequencer performs a fresh start from genesis, /// initializing its state with the accounts defined in the configuration file. fn open_or_create_store(config: &SequencerConfig) -> (SequencerStore, lee::V03State, Block) { let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); let db_path = config.home.join("rocksdb"); if db_path.exists() { let store = SequencerStore::open_db(&db_path, signing_key).unwrap_or_else(|err| { panic!( "Failed to open database at {} with error: {err}", db_path.display() ) }); let state = store .get_lee_state() .expect("Failed to read state from store"); let genesis_block = store .get_block_at_id(store.genesis_id()) .expect("Failed to read genesis block from store") .expect("Genesis block not found in store"); (store, state, genesis_block) } else { warn!( "Database not found at {}, starting from genesis", db_path.display() ); let (genesis_state, genesis_txs) = build_genesis_state(config); let hashable_data = HashableBlockData { block_id: GENESIS_BLOCK_ID, transactions: genesis_txs, prev_block_hash: HashType([0; 32]), timestamp: 0, }; let genesis_block = hashable_data.into_pending_block(&signing_key); let store = SequencerStore::create_db_with_genesis( &db_path, &genesis_block, &genesis_state, signing_key, ) .expect("Failed to create database with genesis block"); (store, genesis_state, genesis_block) } } pub async fn start_from_config( config: SequencerConfig, ) -> (Self, MemPoolHandle<(TransactionOrigin, LeeTransaction)>) { let bedrock_signing_key = load_or_create_signing_key(&config.home.join("bedrock_signing_key")) .expect("Failed to load or create bedrock signing key"); let (store, state, _genesis_block) = Self::open_or_create_store(&config); let latest_block_meta = store .latest_block_meta() .expect("Failed to read latest block meta from store"); let initial_checkpoint = store .get_zone_checkpoint() .expect("Failed to load zone-sdk checkpoint"); let is_fresh_start = initial_checkpoint.is_none(); let (mempool, mempool_handle) = MemPool::new(config.mempool_max_size); replay_unfulfilled_deposit_events(&store, mempool_handle.clone()); let block_publisher = BP::new( &config.bedrock_config, bedrock_signing_key, config.retry_pending_blocks_timeout, initial_checkpoint, Self::on_checkpoint(store.dbio()), Self::on_finalized_block(store.dbio()), Self::on_deposit_event(store.dbio(), mempool_handle.clone()), Self::on_withdraw_event(store.dbio()), ) .await .expect("Failed to initialize Block Publisher"); // Fresh start (no checkpoint): republish all pending blocks if is_fresh_start { let mut pending_blocks = store .get_all_blocks() .filter_ok(|block| matches!(block.bedrock_status, BedrockStatus::Pending)) .collect::, _>>() .expect("Failed to read blocks from store while republishing on fresh start"); pending_blocks.sort_unstable_by_key(|block| block.header.block_id); assert!( pending_blocks .first() .is_none_or(|block| block.header.block_id == GENESIS_BLOCK_ID), "First pending block on fresh start should be the genesis block" ); for block in &pending_blocks { block_publisher .publish_block(block, vec![]) .await .unwrap_or_else(|err| { panic!( "Failed to publish block {} on fresh start: {err:#}", block.header.block_id ) }); } } let sequencer_core = Self { state, store, mempool, chain_height: latest_block_meta.id, sequencer_config: config, block_publisher, }; (sequencer_core, mempool_handle) } fn on_checkpoint(dbio: Arc) -> block_publisher::CheckpointSink { Box::new(move |cp| { let bytes = match serde_json::to_vec(&cp) { Ok(b) => b, Err(err) => { error!("Failed to serialize zone-sdk checkpoint: {err:#}"); return; } }; if let Err(err) = dbio.put_zone_sdk_checkpoint_bytes(&bytes) { error!("Failed to persist zone-sdk checkpoint: {err:#}"); } }) } fn on_finalized_block(dbio: Arc) -> block_publisher::FinalizedBlockSink { Box::new(move |block_id| { // NOTE: Theoretically Zone SDK may report finalization happening multiple times for the // same block. In practice this is very unlikely to happen. For that to // happen Sequencer should crash between receiving Finalized and Checkpoint events while // these events happen very fast (because Checkpoints are generated by Zone SDK // locally). if let Err(err) = dbio.clean_pending_blocks_up_to(block_id) { error!("Failed to mark pending blocks finalized up to {block_id}: {err:#}"); } match dbio.remove_fulfilled_pending_deposit_events_up_to_block(block_id) { Ok(0) => {} Ok(removed) => { info!( "Removed {removed} fulfilled pending deposit events up to finalized block {block_id}" ); } Err(err) => { error!( "Failed to remove fulfilled pending deposit events up to block {block_id}: {err:#}" ); } } }) } fn on_deposit_event( dbio: Arc, mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>, ) -> block_publisher::OnDepositEventSink { Box::new(move |deposit| { // NOTE: Theoretically Zone SDK may report multiple identical deposits. In practice this // is very unlikely to happen. For that to happen Sequencer should crash // between receiving Deposit and Checkpoint events while these events happen // very fast (because Checkpoints are generated by Zone SDK locally). let dbio = Arc::clone(&dbio); let mempool_handle = mempool_handle.clone(); Box::pin(async move { let id_hex = hex::encode(deposit.op_id); info!("Observed Bedrock Deposit event with id: {id_hex}"); let event_record = pending_deposit_event_record(&deposit); match dbio.add_pending_deposit_event(event_record.clone()) { Ok(true) => {} Ok(false) => { info!( "Deposit event {id_hex} already persisted as unfulfilled, skipping duplicate enqueue", ); return; } Err(err) => { error!( "Failed to persist unfulfilled deposit event {id_hex} before enqueue: {err:#}. Deposit will be lost.", ); return; } } let tx = match build_bridge_deposit_tx_from_event(&event_record) { Ok(tx) => tx, Err(err) => { error!( "Failed to build transaction from Bedrock deposit event {id_hex}: {err:#}. Deposit will be lost.", ); return; } }; if let Err(err) = mempool_handle .push((TransactionOrigin::Sequencer, tx)) .await { error!( "Failed to queue sequencer transaction built from finalized Bedrock event: {err:#}. Deposit will be lost." ); } }) }) } fn on_withdraw_event(dbio: Arc) -> block_publisher::OnWithdrawEventSink { Box::new(move |withdraw| { let dbio = Arc::clone(&dbio); Box::pin(async move { let hash_encoded = hex::encode(withdraw.tx_hash.as_ref()); let withdraw_key = match withdraw_event_reconciliation_key(&withdraw.op.outputs) { Ok(key) => key, Err(err) => { error!( "Failed to build reconciliation key for Bedrock Withdraw event with tx_hash {hash_encoded}: {err:#}" ); return; } }; match dbio.consume_unseen_withdraw_count(withdraw_key) { Ok(true) => { info!("Validated Bedrock Withdraw event with tx_hash: {hash_encoded}"); } Ok(false) => warn!( "Unexpected Bedrock Withdraw event with tx_hash {hash_encoded}: no matching unseen withdraw found" ), Err(err) => error!( "Failed to reconcile Bedrock Withdraw event with tx_hash {hash_encoded}: {err:#}" ), } }) }) } /// Produces a new block from mempool transactions and publishes it via zone-sdk. pub async fn produce_new_block(&mut self) -> Result { let BlockWithMeta { block, deposit_event_ids, withdrawals, } = self .build_block_from_mempool() .context("Failed to build block from mempool transactions")?; let withdrawal_reconciliation_keys = withdrawals .iter() .map(|withdraw| withdraw_event_reconciliation_key(&withdraw.outputs)) .collect::>() .context("Failed to build reconciliation keys for block withdrawals")?; self.block_publisher .publish_block(&block, withdrawals) .await .context("Failed to publish block to Bedrock")?; self.store.update( &block, &deposit_event_ids, withdrawal_reconciliation_keys, &self.state, )?; Ok(self.chain_height) } /// Validates and applies a single mempool transaction to the current state. /// Returns `Ok(true)` if the transaction was valid and applied, `Ok(false)` if /// it was skipped due to validation failure. fn apply_mempool_transaction( &mut self, origin: TransactionOrigin, tx: &LeeTransaction, block_height: u64, timestamp: u64, deposit_event_ids: &mut Vec, withdrawals: &mut Vec, ) -> Result { let tx_hash = tx.hash(); match origin { TransactionOrigin::User => { let validated_diff = match tx.validate_on_state( &self.state, block_height, timestamp, ) { Ok(diff) => diff, Err(err) => { error!( "Transaction with hash {tx_hash} failed execution check with error: {err:#?}, skipping it", ); return Ok(false); } }; if let Some(withdraw_data) = extract_bridge_withdraw_data(tx) { withdrawals.push(withdraw_data); } self.state.apply_state_diff(validated_diff); } TransactionOrigin::Sequencer => { let LeeTransaction::Public(public_tx) = tx else { panic!("Sequencer may only generate Public transactions, found {tx:#?}"); }; if let Some(deposit_op_id) = extract_bridge_deposit_id(tx) { deposit_event_ids.push(deposit_op_id); } self.state .transition_from_public_transaction(public_tx, block_height, timestamp) .context("Failed to execute sequencer-generated transaction")?; } } info!("Validated transaction with hash {tx_hash}, including it in block"); Ok(true) } fn build_block_from_mempool(&mut self) -> Result { let now = Instant::now(); let new_block_height = self.next_block_id(); let mut valid_transactions = Vec::new(); let mut deposit_event_ids = Vec::new(); let mut withdrawals = Vec::new(); let max_block_size = usize::try_from(self.sequencer_config.max_block_size.as_u64()) .expect("`max_block_size` should fit into usize"); let latest_block_meta = self .store .latest_block_meta() .context("Failed to get latest block meta from store")?; let new_block_timestamp = u64::try_from(chrono::Utc::now().timestamp_millis()) .expect("Timestamp must be positive"); let clock_tx = clock_invocation(new_block_timestamp); let clock_lee_tx = LeeTransaction::Public(clock_tx.clone()); while let Some((origin, tx)) = self.mempool.pop() { let tx_hash = tx.hash(); let temp_valid_transactions = [ valid_transactions.as_slice(), std::slice::from_ref(&tx), std::slice::from_ref(&clock_lee_tx), ] .concat(); let temp_hashable_data = HashableBlockData { block_id: new_block_height, transactions: temp_valid_transactions, prev_block_hash: latest_block_meta.hash, timestamp: new_block_timestamp, }; let block_size = borsh::to_vec(&temp_hashable_data) .context("Failed to serialize block for size check")? .len(); if block_size > max_block_size { warn!( "Transaction with hash {tx_hash} deferred to next block: \ block size {block_size} bytes would exceed limit of {max_block_size} bytes", ); self.mempool.push_front((origin, tx)); break; } if self.apply_mempool_transaction( origin, &tx, new_block_height, new_block_timestamp, &mut deposit_event_ids, &mut withdrawals, )? { valid_transactions.push(tx); } if valid_transactions.len() >= self.sequencer_config.max_num_tx_in_block { break; } } self.state .transition_from_public_transaction(&clock_tx, new_block_height, new_block_timestamp) .context("Clock transaction failed. Aborting block production.")?; valid_transactions.push(clock_lee_tx); let hashable_data = HashableBlockData { block_id: new_block_height, transactions: valid_transactions, prev_block_hash: latest_block_meta.hash, timestamp: new_block_timestamp, }; let block = hashable_data .clone() .into_pending_block(self.store.signing_key()); self.chain_height = new_block_height; log::info!( "Created block with {} transactions in {} seconds", hashable_data.transactions.len(), now.elapsed().as_secs() ); Ok(BlockWithMeta { block, deposit_event_ids, withdrawals, }) } pub const fn state(&self) -> &lee::V03State { &self.state } pub const fn block_store(&self) -> &SequencerStore { &self.store } pub const fn chain_height(&self) -> u64 { self.chain_height } pub const fn sequencer_config(&self) -> &SequencerConfig { &self.sequencer_config } /// Marks all pending blocks with `block_id <= last_finalized_block_id` as /// finalized. Idempotent. Production callers don't invoke this directly — /// it's wired up in `start_from_config` to the publisher's /// `on_finalized_block` sink, which fires on `Event::TxsFinalized` / /// `Event::FinalizedInscriptions`. Kept on the type for tests. // TODO: Delete blocks instead of marking them as finalized. Current // approach is used because we still have `GetBlockDataRequest`. pub fn clean_finalized_blocks_from_db(&self, last_finalized_block_id: u64) -> Result<()> { info!("Clearing pending blocks up to id: {last_finalized_block_id}"); self.store .dbio() .clean_pending_blocks_up_to(last_finalized_block_id)?; Ok(()) } /// Returns the list of stored pending blocks. pub fn get_pending_blocks(&self) -> Result> { Ok(self .store .get_all_blocks() .collect::>>()? .into_iter() .filter(|block| matches!(block.bedrock_status, BedrockStatus::Pending)) .collect()) } pub fn block_publisher(&self) -> BP { self.block_publisher.clone() } fn next_block_id(&self) -> u64 { self.chain_height .checked_add(1) .unwrap_or_else(|| panic!("Max block height reached: {}", self.chain_height)) } } struct BlockWithMeta { block: Block, deposit_event_ids: Vec, withdrawals: Vec, } /// Checks the database for any pending deposit events that have not yet been marked as submitted in /// a block, and re-queues them in the mempool in a separate async task for inclusion in the next /// block. fn replay_unfulfilled_deposit_events( store: &SequencerStore, mempool_handle: MemPoolHandle<(TransactionOrigin, LeeTransaction)>, ) { let replay_records: Vec = store .get_unfulfilled_deposit_events() .expect("Failed to load unfulfilled deposit events") .into_iter() .filter(|record| record.submitted_in_block_id.is_none()) .collect(); if replay_records.is_empty() { return; } info!( "Found {} unfulfilled deposit events in DB, re-queueing", replay_records.len() ); tokio::spawn(async move { for record in replay_records { let tx = match build_bridge_deposit_tx_from_event(&record) { Ok(tx) => tx, Err(err) => { warn!( "Skipping replay of pending deposit event {} due to tx build failure: {err:#}", hex::encode(record.deposit_op_id) ); continue; } }; if let Err(err) = mempool_handle .push((TransactionOrigin::Sequencer, tx)) .await { error!( "Failed to re-queue unfulfilled deposit event {} from DB: {err:#}", hex::encode(record.deposit_op_id) ); break; } } }); } /// Builds the initial genesis state from `testnet_initial_state` plus configured genesis /// transactions. Returns the final state and the list of [`LeeTransaction`]s that should be /// committed to the genesis block so external observers can replay them. fn build_genesis_state(config: &SequencerConfig) -> (lee::V03State, Vec) { #[cfg(not(feature = "testnet"))] let mut state = testnet_initial_state::initial_state(); #[cfg(feature = "testnet")] let mut state = testnet_initial_state::initial_state_testnet(); let genesis_txs = config .genesis .iter() .map(|genesis_tx| match genesis_tx { GenesisAction::SupplyAccount { account_id, balance, } => build_supply_account_genesis_transaction(account_id, *balance), GenesisAction::SupplyBridgeAccount { balance } => { build_supply_bridge_account_genesis_transaction(*balance) } }) .chain(std::iter::once(clock_invocation(0))) .inspect(|tx| { state .transition_from_public_transaction(tx, GENESIS_BLOCK_ID, 0) .expect("Failed to execute genesis transaction"); }) .map(LeeTransaction::Public) .collect(); (state, genesis_txs) } fn build_supply_account_genesis_transaction( account_id: &AccountId, balance: u128, ) -> PublicTransaction { let faucet_program_id = programs::faucet().id(); let vault_program_id = programs::vault().id(); let recipient_vault_id = vault_core::compute_vault_account_id(vault_program_id, *account_id); let message = Message::try_new( faucet_program_id, vec![system_accounts::faucet_account_id(), recipient_vault_id], vec![], faucet_core::Instruction::GenesisTransferVault { vault_program_id, recipient_id: *account_id, amount: balance, }, ) .expect("Failed to serialize genesis transfer instruction"); let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(vec![]); PublicTransaction::new(message, witness_set) } fn build_supply_bridge_account_genesis_transaction(balance: u128) -> PublicTransaction { let faucet_program_id = programs::faucet().id(); let bridge_account_id = system_accounts::bridge_account_id(); let message = Message::try_new( faucet_program_id, vec![system_accounts::faucet_account_id(), bridge_account_id], vec![], faucet_core::Instruction::GenesisTransferDirect { amount: balance }, ) .expect("Failed to serialize bridge genesis transfer instruction"); let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(vec![]); PublicTransaction::new(message, witness_set) } fn pending_deposit_event_record(deposit: &DepositInfo) -> PendingDepositEventRecord { PendingDepositEventRecord { deposit_op_id: HashType(deposit.op_id), source_tx_hash: HashType(deposit.tx_hash.0), amount: deposit.amount, metadata: deposit.metadata.clone().into(), submitted_in_block_id: None, } } fn build_bridge_deposit_tx_from_event(event: &PendingDepositEventRecord) -> Result { let metadata = DepositMetadata::decode(&event.metadata) .context("Failed to decode finalized Bedrock deposit metadata")?; let bridge_program_id = programs::bridge().id(); let vault_program_id = programs::vault().id(); let recipient_vault_id = vault_core::compute_vault_account_id(vault_program_id, metadata.recipient_id); let message = Message::try_new( bridge_program_id, vec![system_accounts::bridge_account_id(), recipient_vault_id], vec![], bridge_core::Instruction::Deposit { l1_deposit_op_id: event.deposit_op_id.0, vault_program_id, recipient_id: metadata.recipient_id, amount: event.amount, }, ) .context("Failed to build bridge deposit message")?; let witness_set = lee::public_transaction::WitnessSet::from_raw_parts(vec![]); Ok(LeeTransaction::Public(PublicTransaction::new( message, witness_set, ))) } #[must_use] fn extract_bridge_deposit_id(tx: &LeeTransaction) -> Option { let LeeTransaction::Public(tx) = tx else { return None; }; let message = tx.message(); if message.program_id != programs::bridge().id() { return None; } let instruction = risc0_zkvm::serde::from_slice::(&message.instruction_data) .ok()?; match instruction { bridge_core::Instruction::Deposit { l1_deposit_op_id, .. } => Some(HashType(l1_deposit_op_id)), bridge_core::Instruction::Withdraw { .. } => None, } } #[must_use] fn extract_bridge_withdraw_data(tx: &LeeTransaction) -> Option { let LeeTransaction::Public(tx) = tx else { return None; }; let message = tx.message(); if message.program_id != programs::bridge().id() { return None; } let instruction = risc0_zkvm::serde::from_slice::(&message.instruction_data) .ok()?; let bridge_core::Instruction::Withdraw { amount, bedrock_account_pk, } = instruction else { return None; }; let recipient_pk = logos_blockchain_key_management_system_service::keys::ZkPublicKey::from( BigUint::from_bytes_le(&bedrock_account_pk), ); Some(WithdrawArg { outputs: logos_blockchain_core::mantle::ledger::Outputs::new( logos_blockchain_core::mantle::Note::new(amount, recipient_pk), ), }) } fn withdraw_event_reconciliation_key( outputs: &logos_blockchain_core::mantle::ledger::Outputs, ) -> Result { let [note] = outputs.as_ref().as_slice() else { return Err(anyhow!( "Unsupported withdraw output count for reconciliation: {}", outputs.len() )); }; // `extract_bridge_withdraw_data` maps [u8;32] LE -> BigUint -> ZkPublicKey. // Reconcile by reversing that direction here. let mut bedrock_account_pk = BigUint::from(note.pk.into_inner()).to_bytes_le(); if bedrock_account_pk.len() > 32 { return Err(anyhow!( "Withdraw recipient public key is too large: {} bytes", bedrock_account_pk.len() )); } bedrock_account_pk.resize(32, 0); let bedrock_account_pk: [u8; 32] = bedrock_account_pk .try_into() .expect("Public key bytes were padded/truncated to 32 bytes"); Ok(WithdrawalReconciliationKey { amount: note.value, bedrock_account_pk, }) } /// Load signing key from file or generate a new one if it doesn't exist. fn load_or_create_signing_key(path: &Path) -> Result { if path.exists() { let key_bytes = std::fs::read(path)?; let key_array: [u8; ED25519_SECRET_KEY_SIZE] = key_bytes .try_into() .map_err(|_bytes| anyhow!("Found key with incorrect length"))?; Ok(Ed25519Key::from_bytes(&key_array)) } else { let mut key_bytes = [0_u8; ED25519_SECRET_KEY_SIZE]; rand::RngCore::fill_bytes(&mut rand::thread_rng(), &mut key_bytes); // Create parent directory if it doesn't exist if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } std::fs::write(path, key_bytes)?; Ok(Ed25519Key::from_bytes(&key_bytes)) } } #[cfg(test)] #[cfg(feature = "mock")] mod tests;