feat(storage): add a durable per-peer cross-zone chain tip

lez/sequencer/core/src/block_store.rs

@@ -265,6 +265,13 @@ pub fn set_cross_zone_peer_floor(
     Ok(())
 }
 
+/// Drops the stored floor so the watcher reads `peer_zone`'s channel from the
+/// peer's genesis again.
+pub fn clear_cross_zone_peer_floor(dbio: &RocksDBIO, peer_zone: PeerZoneKey) -> Result<()> {
+    dbio.delete_cross_zone_peer_floor(peer_zone)?;
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use common::{block::HashableBlockData, test_utils::sequencer_sign_key_for_testing};

lez/storage/src/sequencer/mod.rs

@@ -22,12 +22,12 @@ use crate::{
     sequencer::sequencer_cells::{
         FinalBlockMetaCellOwned, FinalBlockMetaCellRef, FinalLeeStateCellOwned,
         FinalLeeStateCellRef, LEEStateCellOwned, LEEStateCellRef, LastFinalizedBlockIdCell,
-        LatestBlockMetaCellOwned, LatestBlockMetaCellRef, PeerFloorCellOwned, PeerFloorCellRef,
-        PeerZoneKey, PendingCrossZoneDispatchRecord, PendingCrossZoneDispatchesCellOwned,
-        PendingCrossZoneDispatchesCellRef, PendingDepositEventRecord,
-        PendingDepositEventsCellOwned, PendingDepositEventsCellRef, UnseenWithdrawCountCell,
-        WithdrawalReconciliationKey, ZoneAnchorCell, ZoneAnchorRecord, ZoneSdkCheckpointCellOwned,
-        ZoneSdkCheckpointCellRef,
+        LatestBlockMetaCellOwned, LatestBlockMetaCellRef, PeerChainTip, PeerFloorCellOwned,
+        PeerFloorCellRef, PeerTipCell, PeerZoneKey, PendingCrossZoneDispatchRecord,
+        PendingCrossZoneDispatchesCellOwned, PendingCrossZoneDispatchesCellRef,
+        PendingDepositEventRecord, PendingDepositEventsCellOwned, PendingDepositEventsCellRef,
+        UnseenWithdrawCountCell, WithdrawalReconciliationKey, ZoneAnchorCell, ZoneAnchorRecord,
+        ZoneSdkCheckpointCellOwned, ZoneSdkCheckpointCellRef,
     },
 };
 
@@ -49,6 +49,9 @@ pub const DB_META_PENDING_DEPOSIT_EVENTS_KEY: &str = "pending_deposit_events";
 /// Key base for storing a cross-zone watcher's delivery floor on one peer
 /// channel (opaque bytes). Keyed per peer zone.
 pub const DB_META_CROSS_ZONE_PEER_FLOOR_KEY: &str = "cross_zone_peer_floor";
+/// Key base for storing the last peer block a cross-zone watcher delivered
+/// from, as an id and hash pair. Keyed per peer zone.
+pub const DB_META_CROSS_ZONE_PEER_TIP_KEY: &str = "cross_zone_peer_tip";
 /// Key base for storing cross-zone deliveries the watcher has recorded but
 /// which are not yet known to be irreversibly delivered.
 pub const DB_META_PENDING_CROSS_ZONE_DISPATCHES_KEY: &str = "pending_cross_zone_dispatches";
@@ -611,6 +614,38 @@ impl RocksDBIO {
         self.put(&PeerFloorCellRef(bytes), peer_zone)
     }
 
+    /// The last peer block one cross-zone watcher delivered from, or `None`
+    /// before it has delivered anything from that peer.
+    ///
+    /// Write it only after that block's deliveries are recorded: a crash in
+    /// between leaves a tip past deliveries that were never made, and nothing
+    /// re-reads them.
+    pub fn get_cross_zone_peer_tip(
+        &self,
+        peer_zone: PeerZoneKey,
+    ) -> DbResult<Option<PeerChainTip>> {
+        Ok(self.get_opt::<PeerTipCell>(peer_zone)?.map(|cell| cell.0))
+    }
+
+    pub fn put_cross_zone_peer_tip(
+        &self,
+        peer_zone: PeerZoneKey,
+        tip: PeerChainTip,
+    ) -> DbResult<()> {
+        self.put(&PeerTipCell(tip), peer_zone)
+    }
+
+    /// Forgets one peer's delivery floor, so its watcher reads that channel from
+    /// the peer's genesis again. Only sound while that peer has no stored tip:
+    /// with one, the re-read starts below a tip nothing it reads can link to.
+    ///
+    /// A floor above a tip is unusable either way, since the first block read is
+    /// too far ahead to link, so clearing the floor is what makes rebuilding a
+    /// tip survive a crash halfway through.
+    pub fn delete_cross_zone_peer_floor(&self, peer_zone: PeerZoneKey) -> DbResult<()> {
+        self.del::<PeerFloorCellOwned>(peer_zone)
+    }
+
     pub fn get_pending_cross_zone_dispatches(
         &self,
     ) -> DbResult<Vec<PendingCrossZoneDispatchRecord>> {

lez/storage/src/sequencer/sequencer_cells.rs

@@ -8,10 +8,11 @@ use crate::{
     error::DbError,
     sequencer::{
         CF_LEE_STATE_NAME, DB_FINAL_BLOCK_META_KEY, DB_FINAL_LEE_STATE_KEY, DB_LEE_STATE_KEY,
-        DB_META_CROSS_ZONE_PEER_FLOOR_KEY, DB_META_LAST_FINALIZED_BLOCK_ID,
-        DB_META_LATEST_BLOCK_META_KEY, DB_META_PENDING_CROSS_ZONE_DISPATCHES_KEY,
-        DB_META_PENDING_DEPOSIT_EVENTS_KEY, DB_META_UNSEEN_WITHDRAW_COUNT_KEY,
-        DB_META_ZONE_CURSOR_KEY, DB_META_ZONE_SDK_CHECKPOINT_KEY,
+        DB_META_CROSS_ZONE_PEER_FLOOR_KEY, DB_META_CROSS_ZONE_PEER_TIP_KEY,
+        DB_META_LAST_FINALIZED_BLOCK_ID, DB_META_LATEST_BLOCK_META_KEY,
+        DB_META_PENDING_CROSS_ZONE_DISPATCHES_KEY, DB_META_PENDING_DEPOSIT_EVENTS_KEY,
+        DB_META_UNSEEN_WITHDRAW_COUNT_KEY, DB_META_ZONE_CURSOR_KEY,
+        DB_META_ZONE_SDK_CHECKPOINT_KEY,
     },
 };
 
@@ -421,6 +422,57 @@ impl SimpleWritableCell for PeerFloorCellRef<'_> {
     }
 }
 
+/// The last peer block a cross-zone watcher delivered from, and the link the
+/// next one has to carry.
+///
+/// `block_hash` is the recomputed hash, not `header.hash` as read: the
+/// signature does not cover that field, so a signed block may carry a bogus one
+/// and break the link against the peer's next honest block.
+///
+/// Durable, not in-memory: a watcher that re-anchored on restart would accept a
+/// block claiming any id.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, BorshSerialize, BorshDeserialize)]
+pub struct PeerChainTip {
+    pub block_id: u64,
+    pub block_hash: HashType,
+}
+
+#[derive(Debug, BorshSerialize, BorshDeserialize)]
+pub struct PeerTipCell(pub PeerChainTip);
+
+impl SimpleStorableCell for PeerTipCell {
+    type KeyParams = PeerZoneKey;
+
+    const CELL_NAME: &'static str = DB_META_CROSS_ZONE_PEER_TIP_KEY;
+    const CF_NAME: &'static str = CF_META_NAME;
+
+    /// Folds the peer zone into the key so each peer keeps its own tip.
+    fn key_constructor(peer_zone: Self::KeyParams) -> DbResult<Vec<u8>> {
+        borsh::to_vec(&(Self::CELL_NAME, peer_zone)).map_err(|err| {
+            DbError::borsh_cast_message(
+                err,
+                Some(format!(
+                    "Failed to serialize {:?} key params",
+                    Self::CELL_NAME
+                )),
+            )
+        })
+    }
+}
+
+impl SimpleReadableCell for PeerTipCell {}
+
+impl SimpleWritableCell for PeerTipCell {
+    fn value_constructor(&self) -> DbResult<Vec<u8>> {
+        borsh::to_vec(&self).map_err(|err| {
+            DbError::borsh_cast_message(
+                err,
+                Some("Failed to serialize cross-zone peer tip cell".to_owned()),
+            )
+        })
+    }
+}
+
 /// Identity of one withdrawal, shared by the intent recorded when the
 /// sequencer publishes it and the Bedrock Withdraw event that later reports
 /// it: the id of the channel note the withdrawal releases.

lez/storage/src/sequencer/tests.rs

@@ -410,6 +410,77 @@ fn finalized_deposit_records_are_removed_by_op_id() {
     assert_eq!(stored, vec![second]);
 }
 
+#[test]
+fn peer_chain_tips_round_trip_and_are_kept_per_peer() {
+    let temp_dir = tempdir().unwrap();
+    let (dbio, _genesis) = dbio_with_genesis(temp_dir.path());
+
+    let peer_a = [1_u8; 32];
+    let peer_b = [2_u8; 32];
+    assert_eq!(dbio.get_cross_zone_peer_tip(peer_a).unwrap(), None);
+
+    let tip_a = PeerChainTip {
+        block_id: 7,
+        block_hash: HashType([9; 32]),
+    };
+    let tip_b = PeerChainTip {
+        block_id: 3,
+        block_hash: HashType([4; 32]),
+    };
+    // The floor shares this peer key with the tip. Asserting the tip alone
+    // passes even when the two cells occupy one key space.
+    dbio.put_cross_zone_peer_floor_bytes(peer_a, &11_u64.to_le_bytes())
+        .unwrap();
+    dbio.put_cross_zone_peer_tip(peer_a, tip_a).unwrap();
+    dbio.put_cross_zone_peer_tip(peer_b, tip_b).unwrap();
+
+    // One tip per peer: a shared key would let one peer's chain decide which
+    // blocks another peer's watcher accepts.
+    assert_eq!(dbio.get_cross_zone_peer_tip(peer_a).unwrap(), Some(tip_a));
+    assert_eq!(dbio.get_cross_zone_peer_tip(peer_b).unwrap(), Some(tip_b));
+    assert_eq!(
+        dbio.get_cross_zone_peer_floor_bytes(peer_a).unwrap(),
+        Some(11_u64.to_le_bytes().to_vec()),
+        "the tip must not land in the floor's key space"
+    );
+
+    let advanced = PeerChainTip {
+        block_id: 8,
+        block_hash: HashType([10; 32]),
+    };
+    dbio.put_cross_zone_peer_tip(peer_a, advanced).unwrap();
+    assert_eq!(
+        dbio.get_cross_zone_peer_tip(peer_a).unwrap(),
+        Some(advanced)
+    );
+    assert_eq!(dbio.get_cross_zone_peer_tip(peer_b).unwrap(), Some(tip_b));
+
+    // Clearing the floor is how a watcher with no tip rebuilds one, so it has
+    // to leave the tip alone: the two share the peer key and differ only in
+    // their key base.
+    dbio.delete_cross_zone_peer_floor(peer_a).unwrap();
+    assert_eq!(dbio.get_cross_zone_peer_floor_bytes(peer_a).unwrap(), None);
+    assert_eq!(
+        dbio.get_cross_zone_peer_tip(peer_a).unwrap(),
+        Some(advanced)
+    );
+    dbio.delete_cross_zone_peer_floor(peer_a)
+        .expect("clearing a floor that is already gone is not an error");
+
+    // On disk, not in memory: a watcher that re-anchored on restart would take
+    // whatever block reached it first, which is the id the attack picks.
+    drop(dbio);
+    let reopened = RocksDBIO::open(temp_dir.path()).unwrap();
+    assert_eq!(
+        reopened.get_cross_zone_peer_tip(peer_a).unwrap(),
+        Some(advanced)
+    );
+    assert_eq!(
+        reopened.get_cross_zone_peer_tip(peer_b).unwrap(),
+        Some(tip_b)
+    );
+}
+
 #[test]
 fn dispatch_records_round_trip_and_dedupe_by_message_key() {
     let temp_dir = tempdir().unwrap();