diff --git a/lez/sequencer/core/src/block_store.rs b/lez/sequencer/core/src/block_store.rs index df61b38b..d9699f2f 100644 --- a/lez/sequencer/core/src/block_store.rs +++ b/lez/sequencer/core/src/block_store.rs @@ -265,6 +265,13 @@ pub fn set_cross_zone_peer_floor( Ok(()) } +/// Drops the stored floor so the watcher reads `peer_zone`'s channel from the +/// peer's genesis again. +pub fn clear_cross_zone_peer_floor(dbio: &RocksDBIO, peer_zone: PeerZoneKey) -> Result<()> { + dbio.delete_cross_zone_peer_floor(peer_zone)?; + Ok(()) +} + #[cfg(test)] mod tests { use common::{block::HashableBlockData, test_utils::sequencer_sign_key_for_testing}; diff --git a/lez/storage/src/sequencer/mod.rs b/lez/storage/src/sequencer/mod.rs index ba6deaf9..7c8fcfad 100644 --- a/lez/storage/src/sequencer/mod.rs +++ b/lez/storage/src/sequencer/mod.rs @@ -22,12 +22,12 @@ use crate::{ sequencer::sequencer_cells::{ FinalBlockMetaCellOwned, FinalBlockMetaCellRef, FinalLeeStateCellOwned, FinalLeeStateCellRef, LEEStateCellOwned, LEEStateCellRef, LastFinalizedBlockIdCell, - LatestBlockMetaCellOwned, LatestBlockMetaCellRef, PeerFloorCellOwned, PeerFloorCellRef, - PeerZoneKey, PendingCrossZoneDispatchRecord, PendingCrossZoneDispatchesCellOwned, - PendingCrossZoneDispatchesCellRef, PendingDepositEventRecord, - PendingDepositEventsCellOwned, PendingDepositEventsCellRef, UnseenWithdrawCountCell, - WithdrawalReconciliationKey, ZoneAnchorCell, ZoneAnchorRecord, ZoneSdkCheckpointCellOwned, - ZoneSdkCheckpointCellRef, + LatestBlockMetaCellOwned, LatestBlockMetaCellRef, PeerChainTip, PeerFloorCellOwned, + PeerFloorCellRef, PeerTipCell, PeerZoneKey, PendingCrossZoneDispatchRecord, + PendingCrossZoneDispatchesCellOwned, PendingCrossZoneDispatchesCellRef, + PendingDepositEventRecord, PendingDepositEventsCellOwned, PendingDepositEventsCellRef, + UnseenWithdrawCountCell, WithdrawalReconciliationKey, ZoneAnchorCell, ZoneAnchorRecord, + ZoneSdkCheckpointCellOwned, ZoneSdkCheckpointCellRef, }, }; @@ -49,6 +49,9 @@ pub const DB_META_PENDING_DEPOSIT_EVENTS_KEY: &str = "pending_deposit_events"; /// Key base for storing a cross-zone watcher's delivery floor on one peer /// channel (opaque bytes). Keyed per peer zone. pub const DB_META_CROSS_ZONE_PEER_FLOOR_KEY: &str = "cross_zone_peer_floor"; +/// Key base for storing the last peer block a cross-zone watcher delivered +/// from, as an id and hash pair. Keyed per peer zone. +pub const DB_META_CROSS_ZONE_PEER_TIP_KEY: &str = "cross_zone_peer_tip"; /// Key base for storing cross-zone deliveries the watcher has recorded but /// which are not yet known to be irreversibly delivered. pub const DB_META_PENDING_CROSS_ZONE_DISPATCHES_KEY: &str = "pending_cross_zone_dispatches"; @@ -611,6 +614,38 @@ impl RocksDBIO { self.put(&PeerFloorCellRef(bytes), peer_zone) } + /// The last peer block one cross-zone watcher delivered from, or `None` + /// before it has delivered anything from that peer. + /// + /// Write it only after that block's deliveries are recorded: a crash in + /// between leaves a tip past deliveries that were never made, and nothing + /// re-reads them. + pub fn get_cross_zone_peer_tip( + &self, + peer_zone: PeerZoneKey, + ) -> DbResult> { + Ok(self.get_opt::(peer_zone)?.map(|cell| cell.0)) + } + + pub fn put_cross_zone_peer_tip( + &self, + peer_zone: PeerZoneKey, + tip: PeerChainTip, + ) -> DbResult<()> { + self.put(&PeerTipCell(tip), peer_zone) + } + + /// Forgets one peer's delivery floor, so its watcher reads that channel from + /// the peer's genesis again. Only sound while that peer has no stored tip: + /// with one, the re-read starts below a tip nothing it reads can link to. + /// + /// A floor above a tip is unusable either way, since the first block read is + /// too far ahead to link, so clearing the floor is what makes rebuilding a + /// tip survive a crash halfway through. + pub fn delete_cross_zone_peer_floor(&self, peer_zone: PeerZoneKey) -> DbResult<()> { + self.del::(peer_zone) + } + pub fn get_pending_cross_zone_dispatches( &self, ) -> DbResult> { diff --git a/lez/storage/src/sequencer/sequencer_cells.rs b/lez/storage/src/sequencer/sequencer_cells.rs index 521fff5e..d3e1e55e 100644 --- a/lez/storage/src/sequencer/sequencer_cells.rs +++ b/lez/storage/src/sequencer/sequencer_cells.rs @@ -8,10 +8,11 @@ use crate::{ error::DbError, sequencer::{ CF_LEE_STATE_NAME, DB_FINAL_BLOCK_META_KEY, DB_FINAL_LEE_STATE_KEY, DB_LEE_STATE_KEY, - DB_META_CROSS_ZONE_PEER_FLOOR_KEY, DB_META_LAST_FINALIZED_BLOCK_ID, - DB_META_LATEST_BLOCK_META_KEY, DB_META_PENDING_CROSS_ZONE_DISPATCHES_KEY, - DB_META_PENDING_DEPOSIT_EVENTS_KEY, DB_META_UNSEEN_WITHDRAW_COUNT_KEY, - DB_META_ZONE_CURSOR_KEY, DB_META_ZONE_SDK_CHECKPOINT_KEY, + DB_META_CROSS_ZONE_PEER_FLOOR_KEY, DB_META_CROSS_ZONE_PEER_TIP_KEY, + DB_META_LAST_FINALIZED_BLOCK_ID, DB_META_LATEST_BLOCK_META_KEY, + DB_META_PENDING_CROSS_ZONE_DISPATCHES_KEY, DB_META_PENDING_DEPOSIT_EVENTS_KEY, + DB_META_UNSEEN_WITHDRAW_COUNT_KEY, DB_META_ZONE_CURSOR_KEY, + DB_META_ZONE_SDK_CHECKPOINT_KEY, }, }; @@ -421,6 +422,57 @@ impl SimpleWritableCell for PeerFloorCellRef<'_> { } } +/// The last peer block a cross-zone watcher delivered from, and the link the +/// next one has to carry. +/// +/// `block_hash` is the recomputed hash, not `header.hash` as read: the +/// signature does not cover that field, so a signed block may carry a bogus one +/// and break the link against the peer's next honest block. +/// +/// Durable, not in-memory: a watcher that re-anchored on restart would accept a +/// block claiming any id. +#[derive(Debug, Clone, Copy, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +pub struct PeerChainTip { + pub block_id: u64, + pub block_hash: HashType, +} + +#[derive(Debug, BorshSerialize, BorshDeserialize)] +pub struct PeerTipCell(pub PeerChainTip); + +impl SimpleStorableCell for PeerTipCell { + type KeyParams = PeerZoneKey; + + const CELL_NAME: &'static str = DB_META_CROSS_ZONE_PEER_TIP_KEY; + const CF_NAME: &'static str = CF_META_NAME; + + /// Folds the peer zone into the key so each peer keeps its own tip. + fn key_constructor(peer_zone: Self::KeyParams) -> DbResult> { + borsh::to_vec(&(Self::CELL_NAME, peer_zone)).map_err(|err| { + DbError::borsh_cast_message( + err, + Some(format!( + "Failed to serialize {:?} key params", + Self::CELL_NAME + )), + ) + }) + } +} + +impl SimpleReadableCell for PeerTipCell {} + +impl SimpleWritableCell for PeerTipCell { + fn value_constructor(&self) -> DbResult> { + borsh::to_vec(&self).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize cross-zone peer tip cell".to_owned()), + ) + }) + } +} + /// Identity of one withdrawal, shared by the intent recorded when the /// sequencer publishes it and the Bedrock Withdraw event that later reports /// it: the id of the channel note the withdrawal releases. diff --git a/lez/storage/src/sequencer/tests.rs b/lez/storage/src/sequencer/tests.rs index 4f71ab77..385f7ffe 100644 --- a/lez/storage/src/sequencer/tests.rs +++ b/lez/storage/src/sequencer/tests.rs @@ -410,6 +410,77 @@ fn finalized_deposit_records_are_removed_by_op_id() { assert_eq!(stored, vec![second]); } +#[test] +fn peer_chain_tips_round_trip_and_are_kept_per_peer() { + let temp_dir = tempdir().unwrap(); + let (dbio, _genesis) = dbio_with_genesis(temp_dir.path()); + + let peer_a = [1_u8; 32]; + let peer_b = [2_u8; 32]; + assert_eq!(dbio.get_cross_zone_peer_tip(peer_a).unwrap(), None); + + let tip_a = PeerChainTip { + block_id: 7, + block_hash: HashType([9; 32]), + }; + let tip_b = PeerChainTip { + block_id: 3, + block_hash: HashType([4; 32]), + }; + // The floor shares this peer key with the tip. Asserting the tip alone + // passes even when the two cells occupy one key space. + dbio.put_cross_zone_peer_floor_bytes(peer_a, &11_u64.to_le_bytes()) + .unwrap(); + dbio.put_cross_zone_peer_tip(peer_a, tip_a).unwrap(); + dbio.put_cross_zone_peer_tip(peer_b, tip_b).unwrap(); + + // One tip per peer: a shared key would let one peer's chain decide which + // blocks another peer's watcher accepts. + assert_eq!(dbio.get_cross_zone_peer_tip(peer_a).unwrap(), Some(tip_a)); + assert_eq!(dbio.get_cross_zone_peer_tip(peer_b).unwrap(), Some(tip_b)); + assert_eq!( + dbio.get_cross_zone_peer_floor_bytes(peer_a).unwrap(), + Some(11_u64.to_le_bytes().to_vec()), + "the tip must not land in the floor's key space" + ); + + let advanced = PeerChainTip { + block_id: 8, + block_hash: HashType([10; 32]), + }; + dbio.put_cross_zone_peer_tip(peer_a, advanced).unwrap(); + assert_eq!( + dbio.get_cross_zone_peer_tip(peer_a).unwrap(), + Some(advanced) + ); + assert_eq!(dbio.get_cross_zone_peer_tip(peer_b).unwrap(), Some(tip_b)); + + // Clearing the floor is how a watcher with no tip rebuilds one, so it has + // to leave the tip alone: the two share the peer key and differ only in + // their key base. + dbio.delete_cross_zone_peer_floor(peer_a).unwrap(); + assert_eq!(dbio.get_cross_zone_peer_floor_bytes(peer_a).unwrap(), None); + assert_eq!( + dbio.get_cross_zone_peer_tip(peer_a).unwrap(), + Some(advanced) + ); + dbio.delete_cross_zone_peer_floor(peer_a) + .expect("clearing a floor that is already gone is not an error"); + + // On disk, not in memory: a watcher that re-anchored on restart would take + // whatever block reached it first, which is the id the attack picks. + drop(dbio); + let reopened = RocksDBIO::open(temp_dir.path()).unwrap(); + assert_eq!( + reopened.get_cross_zone_peer_tip(peer_a).unwrap(), + Some(advanced) + ); + assert_eq!( + reopened.get_cross_zone_peer_tip(peer_b).unwrap(), + Some(tip_b) + ); +} + #[test] fn dispatch_records_round_trip_and_dedupe_by_message_key() { let temp_dir = tempdir().unwrap();