From 3426f1f8a98f3829b1a81d42cd371e521b9c2efd Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Mon, 9 Mar 2026 11:15:57 +0200 Subject: [PATCH 01/13] fix: minor restructurization --- storage/src/indexer.rs | 1325 ------------------------- storage/src/indexer/mod.rs | 605 +++++++++++ storage/src/indexer/read_multi_get.rs | 5 + storage/src/indexer/read_once.rs | 424 ++++++++ storage/src/indexer/write_batch.rs | 78 ++ storage/src/indexer/write_once.rs | 244 +++++ 6 files changed, 1356 insertions(+), 1325 deletions(-) delete mode 100644 storage/src/indexer.rs create mode 100644 storage/src/indexer/mod.rs create mode 100644 storage/src/indexer/read_multi_get.rs create mode 100644 storage/src/indexer/read_once.rs create mode 100644 storage/src/indexer/write_batch.rs create mode 100644 storage/src/indexer/write_once.rs diff --git a/storage/src/indexer.rs b/storage/src/indexer.rs deleted file mode 100644 index 2c37ab0f..00000000 --- a/storage/src/indexer.rs +++ /dev/null @@ -1,1325 +0,0 @@ -use std::{collections::HashMap, ops::Div, path::Path, sync::Arc}; - -use common::{block::Block, transaction::NSSATransaction}; -use nssa::V02State; -use rocksdb::{ - BoundColumnFamily, ColumnFamilyDescriptor, DBWithThreadMode, MultiThreaded, Options, WriteBatch, -}; - -use crate::error::DbError; - -/// Maximal size of stored blocks in base -/// -/// Used to control db size -/// -/// Currently effectively unbounded. -pub const BUFF_SIZE_ROCKSDB: usize = usize::MAX; - -/// Size of stored blocks cache in memory -/// -/// Keeping small to not run out of memory -pub const CACHE_SIZE: usize = 1000; - -/// Key base for storing metainformation about id of first block in db -pub const DB_META_FIRST_BLOCK_IN_DB_KEY: &str = "first_block_in_db"; -/// Key base for storing metainformation about id of last current block in db -pub const DB_META_LAST_BLOCK_IN_DB_KEY: &str = "last_block_in_db"; -/// Key base for storing metainformation about id of last observed L1 lib header in db -pub const DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY: &str = - "last_observed_l1_lib_header_in_db"; -/// Key base for storing metainformation which describe if first block has been set -pub const DB_META_FIRST_BLOCK_SET_KEY: &str = "first_block_set"; -/// Key base for storing metainformation about the last breakpoint -pub const DB_META_LAST_BREAKPOINT_ID: &str = "last_breakpoint_id"; - -/// Interval between state breakpoints -pub const BREAKPOINT_INTERVAL: u64 = 100; - -/// Name of block column family -pub const CF_BLOCK_NAME: &str = "cf_block"; -/// Name of meta column family -pub const CF_META_NAME: &str = "cf_meta"; -/// Name of breakpoint column family -pub const CF_BREAKPOINT_NAME: &str = "cf_breakpoint"; -/// Name of hash to id map column family -pub const CF_HASH_TO_ID: &str = "cf_hash_to_id"; -/// Name of tx hash to id map column family -pub const CF_TX_TO_ID: &str = "cf_tx_to_id"; -/// Name of account meta column family -pub const CF_ACC_META: &str = "cf_acc_meta"; -/// Name of account id to tx hash map column family -pub const CF_ACC_TO_TX: &str = "cf_acc_to_tx"; - -pub type DbResult = Result; - -fn closest_breakpoint_id(block_id: u64) -> u64 { - block_id.saturating_sub(1).div(BREAKPOINT_INTERVAL) -} - -pub struct RocksDBIO { - pub db: DBWithThreadMode, -} - -impl RocksDBIO { - pub fn open_or_create(path: &Path, start_data: Option<(Block, V02State)>) -> DbResult { - let mut cf_opts = Options::default(); - cf_opts.set_max_write_buffer_number(16); - // ToDo: Add more column families for different data - let cfb = ColumnFamilyDescriptor::new(CF_BLOCK_NAME, cf_opts.clone()); - let cfmeta = ColumnFamilyDescriptor::new(CF_META_NAME, cf_opts.clone()); - let cfbreakpoint = ColumnFamilyDescriptor::new(CF_BREAKPOINT_NAME, cf_opts.clone()); - let cfhti = ColumnFamilyDescriptor::new(CF_HASH_TO_ID, cf_opts.clone()); - let cftti = ColumnFamilyDescriptor::new(CF_TX_TO_ID, cf_opts.clone()); - let cfameta = ColumnFamilyDescriptor::new(CF_ACC_META, cf_opts.clone()); - let cfatt = ColumnFamilyDescriptor::new(CF_ACC_TO_TX, cf_opts.clone()); - - let mut db_opts = Options::default(); - db_opts.create_missing_column_families(true); - db_opts.create_if_missing(true); - let db = DBWithThreadMode::::open_cf_descriptors( - &db_opts, - path, - vec![cfb, cfmeta, cfbreakpoint, cfhti, cftti, cfameta, cfatt], - ); - - let dbio = Self { - // There is no point in handling this from runner code - db: db.unwrap(), - }; - - let is_start_set = dbio.get_meta_is_first_block_set()?; - - if is_start_set { - Ok(dbio) - } else if let Some((block, initial_state)) = start_data { - let block_id = block.header.block_id; - dbio.put_meta_last_block_in_db(block_id)?; - dbio.put_meta_first_block_in_db(block)?; - dbio.put_meta_is_first_block_set()?; - - // First breakpoint setup - dbio.put_breakpoint(0, initial_state)?; - dbio.put_meta_last_breakpoint_id(0)?; - - Ok(dbio) - } else { - // Here we are trying to start a DB without a block, one should not do it. - unreachable!() - } - } - - pub fn destroy(path: &Path) -> DbResult<()> { - let mut cf_opts = Options::default(); - cf_opts.set_max_write_buffer_number(16); - // ToDo: Add more column families for different data - let _cfb = ColumnFamilyDescriptor::new(CF_BLOCK_NAME, cf_opts.clone()); - let _cfmeta = ColumnFamilyDescriptor::new(CF_META_NAME, cf_opts.clone()); - let _cfsnapshot = ColumnFamilyDescriptor::new(CF_BREAKPOINT_NAME, cf_opts.clone()); - let _cfhti = ColumnFamilyDescriptor::new(CF_HASH_TO_ID, cf_opts.clone()); - let _cftti = ColumnFamilyDescriptor::new(CF_TX_TO_ID, cf_opts.clone()); - let _cfameta = ColumnFamilyDescriptor::new(CF_ACC_META, cf_opts.clone()); - let _cfatt = ColumnFamilyDescriptor::new(CF_ACC_TO_TX, cf_opts.clone()); - - let mut db_opts = Options::default(); - db_opts.create_missing_column_families(true); - db_opts.create_if_missing(true); - DBWithThreadMode::::destroy(&db_opts, path) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) - } - - // Columns - - pub fn meta_column(&self) -> Arc> { - self.db.cf_handle(CF_META_NAME).unwrap() - } - - pub fn block_column(&self) -> Arc> { - self.db.cf_handle(CF_BLOCK_NAME).unwrap() - } - - pub fn breakpoint_column(&self) -> Arc> { - self.db.cf_handle(CF_BREAKPOINT_NAME).unwrap() - } - - pub fn hash_to_id_column(&self) -> Arc> { - self.db.cf_handle(CF_HASH_TO_ID).unwrap() - } - - pub fn tx_hash_to_id_column(&self) -> Arc> { - self.db.cf_handle(CF_TX_TO_ID).unwrap() - } - - pub fn account_id_to_tx_hash_column(&self) -> Arc> { - self.db.cf_handle(CF_ACC_TO_TX).unwrap() - } - - pub fn account_meta_column(&self) -> Arc> { - self.db.cf_handle(CF_ACC_META).unwrap() - } - - // Meta - - pub fn get_meta_first_block_in_db(&self) -> DbResult { - let cf_meta = self.meta_column(); - let res = self - .db - .get_cf( - &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to deserialize first block".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "First block not found".to_string(), - )) - } - } - - pub fn get_meta_last_block_in_db(&self) -> DbResult { - let cf_meta = self.meta_column(); - let res = self - .db - .get_cf( - &cf_meta, - borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to deserialize last block".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "Last block not found".to_string(), - )) - } - } - - pub fn get_meta_last_observed_l1_lib_header_in_db(&self) -> DbResult> { - let cf_meta = self.meta_column(); - let res = self - .db - .get_cf( - &cf_meta, - borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err( - |err| { - DbError::borsh_cast_message( - err, - Some( - "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" - .to_string(), - ), - ) - }, - )?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - res.map(|data| { - borsh::from_slice::<[u8; 32]>(&data).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to deserialize last l1 lib header".to_string()), - ) - }) - }) - .transpose() - } - - pub fn get_meta_is_first_block_set(&self) -> DbResult { - let cf_meta = self.meta_column(); - let res = self - .db - .get_cf( - &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - Ok(res.is_some()) - } - - pub fn get_meta_last_breakpoint_id(&self) -> DbResult { - let cf_meta = self.meta_column(); - let res = self - .db - .get_cf( - &cf_meta, - borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to deserialize last breakpoint id".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "Last breakpoint id not found".to_string(), - )) - } - } - - pub fn put_meta_first_block_in_db(&self, block: Block) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize first block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - self.put_block(block, [0; 32])?; - Ok(()) - } - - pub fn put_meta_last_block_in_db(&self, block_id: u64) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), - ) - })?, - borsh::to_vec(&block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize last block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - Ok(()) - } - - pub fn put_meta_last_observed_l1_lib_header_in_db( - &self, - l1_lib_header: [u8; 32], - ) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err( - |err| { - DbError::borsh_cast_message( - err, - Some( - "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" - .to_string(), - ), - ) - }, - )?, - borsh::to_vec(&l1_lib_header).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize last l1 block header".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - Ok(()) - } - - pub fn put_meta_last_breakpoint_id(&self, br_id: u64) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), - ) - })?, - borsh::to_vec(&br_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize last block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - Ok(()) - } - - pub fn put_meta_is_first_block_set(&self) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), - ) - })?, - [1u8; 1], - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - Ok(()) - } - - // Block - - pub fn put_block(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { - let cf_block = self.block_column(); - let cf_hti = self.hash_to_id_column(); - let cf_tti: Arc> = self.tx_hash_to_id_column(); - - // ToDo: rewrite this with write batching - - self.db - .put_cf( - &cf_block, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - borsh::to_vec(&block).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block data".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let last_curr_block = self.get_meta_last_block_in_db()?; - - if block.header.block_id > last_curr_block { - self.put_meta_last_block_in_db(block.header.block_id)?; - self.put_meta_last_observed_l1_lib_header_in_db(l1_lib_header)?; - } - - self.db - .put_cf( - &cf_hti, - borsh::to_vec(&block.header.hash).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block hash".to_string()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let mut acc_to_tx_map: HashMap<[u8; 32], Vec<[u8; 32]>> = HashMap::new(); - - for tx in block.body.transactions { - let tx_hash = tx.hash(); - - self.db - .put_cf( - &cf_tti, - borsh::to_vec(&tx_hash).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize tx hash".to_string()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let acc_ids = tx - .affected_public_account_ids() - .into_iter() - .map(|account_id| account_id.into_value()) - .collect::>(); - - for acc_id in acc_ids { - acc_to_tx_map - .entry(acc_id) - .and_modify(|tx_hashes| tx_hashes.push(tx_hash.into())) - .or_insert(vec![tx_hash.into()]); - } - } - - for (acc_id, tx_hashes) in acc_to_tx_map { - self.put_account_transactions(acc_id, tx_hashes)?; - } - - if block.header.block_id.is_multiple_of(BREAKPOINT_INTERVAL) { - self.put_next_breakpoint()?; - } - - Ok(()) - } - - pub fn get_block(&self, block_id: u64) -> DbResult { - let cf_block = self.block_column(); - let res = self - .db - .get_cf( - &cf_block, - borsh::to_vec(&block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize block data".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "Block on this id not found".to_string(), - )) - } - } - - pub fn get_block_batch(&self, before: Option, limit: u64) -> DbResult> { - let cf_block = self.block_column(); - let mut block_batch = vec![]; - - // Determine the starting block ID - let start_block_id = if let Some(before_id) = before { - before_id.saturating_sub(1) - } else { - // Get the latest block ID - self.get_meta_last_block_in_db()? - }; - - // ToDo: Multi get this - - for i in 0..limit { - let block_id = start_block_id.saturating_sub(i); - if block_id == 0 { - break; - } - - let res = self - .db - .get_cf( - &cf_block, - borsh::to_vec(&block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let block = if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize block data".to_string()), - ) - })?) - } else { - // Block not found, assuming that previous one was the last - break; - }?; - - block_batch.push(block); - } - - Ok(block_batch) - } - - // State - - pub fn put_breakpoint(&self, br_id: u64, breakpoint: V02State) -> DbResult<()> { - let cf_br = self.breakpoint_column(); - - self.db - .put_cf( - &cf_br, - borsh::to_vec(&br_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize breakpoint id".to_string()), - ) - })?, - borsh::to_vec(&breakpoint).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize breakpoint data".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) - } - - pub fn get_breakpoint(&self, br_id: u64) -> DbResult { - let cf_br = self.breakpoint_column(); - let res = self - .db - .get_cf( - &cf_br, - borsh::to_vec(&br_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize breakpoint id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize breakpoint data".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "Breakpoint on this id not found".to_string(), - )) - } - } - - pub fn calculate_state_for_id(&self, block_id: u64) -> DbResult { - let last_block = self.get_meta_last_block_in_db()?; - - if block_id <= last_block { - let br_id = closest_breakpoint_id(block_id); - let mut breakpoint = self.get_breakpoint(br_id)?; - - // ToDo: update it to handle any genesis id - // right now works correctly only if genesis_id < BREAKPOINT_INTERVAL - let start = if br_id != 0 { - BREAKPOINT_INTERVAL * br_id - } else { - self.get_meta_first_block_in_db()? - }; - - for id in start..=block_id { - let block = self.get_block(id)?; - - for transaction in block.body.transactions { - transaction - .transaction_stateless_check() - .map_err(|err| { - DbError::db_interaction_error(format!( - "transaction pre check failed with err {err:?}" - )) - })? - .execute_check_on_state(&mut breakpoint) - .map_err(|err| { - DbError::db_interaction_error(format!( - "transaction execution failed with err {err:?}" - )) - })?; - } - } - - Ok(breakpoint) - } else { - Err(DbError::db_interaction_error( - "Block on this id not found".to_string(), - )) - } - } - - pub fn final_state(&self) -> DbResult { - self.calculate_state_for_id(self.get_meta_last_block_in_db()?) - } - - pub fn put_next_breakpoint(&self) -> DbResult<()> { - let last_block = self.get_meta_last_block_in_db()?; - let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; - let block_to_break_id = next_breakpoint_id * BREAKPOINT_INTERVAL; - - if block_to_break_id <= last_block { - let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; - - self.put_breakpoint(next_breakpoint_id, next_breakpoint)?; - self.put_meta_last_breakpoint_id(next_breakpoint_id) - } else { - Err(DbError::db_interaction_error( - "Breakpoint not yet achieved".to_string(), - )) - } - } - - // Mappings - - pub fn get_block_id_by_hash(&self, hash: [u8; 32]) -> DbResult { - let cf_hti = self.hash_to_id_column(); - let res = self - .db - .get_cf( - &cf_hti, - borsh::to_vec(&hash).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block hash".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize block id".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "Block on this hash not found".to_string(), - )) - } - } - - pub fn get_block_id_by_tx_hash(&self, tx_hash: [u8; 32]) -> DbResult { - let cf_tti = self.tx_hash_to_id_column(); - let res = self - .db - .get_cf( - &cf_tti, - borsh::to_vec(&tx_hash).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize transaction hash".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize block id".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "Block for this tx hash not found".to_string(), - )) - } - } - - // Accounts meta - - fn update_acc_meta_batch( - &self, - acc_id: [u8; 32], - num_tx: u64, - write_batch: &mut WriteBatch, - ) -> DbResult<()> { - let cf_ameta = self.account_meta_column(); - - write_batch.put_cf( - &cf_ameta, - borsh::to_vec(&acc_id).map_err(|err| { - DbError::borsh_cast_message(err, Some("Failed to serialize account id".to_string())) - })?, - borsh::to_vec(&num_tx).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize acc metadata".to_string()), - ) - })?, - ); - - Ok(()) - } - - fn get_acc_meta_num_tx(&self, acc_id: [u8; 32]) -> DbResult> { - let cf_ameta = self.account_meta_column(); - let res = self.db.get_cf(&cf_ameta, acc_id).map_err(|rerr| { - DbError::rocksdb_cast_message(rerr, Some("Failed to read from acc meta cf".to_string())) - })?; - - res.map(|data| { - borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message(serr, Some("Failed to deserialize num tx".to_string())) - }) - }) - .transpose() - } - - // Account - - pub fn put_account_transactions( - &self, - acc_id: [u8; 32], - tx_hashes: Vec<[u8; 32]>, - ) -> DbResult<()> { - let acc_num_tx = self.get_acc_meta_num_tx(acc_id)?.unwrap_or(0); - let cf_att = self.account_id_to_tx_hash_column(); - let mut write_batch = WriteBatch::new(); - - for (tx_id, tx_hash) in tx_hashes.iter().enumerate() { - let put_id = acc_num_tx + tx_id as u64; - - let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { - DbError::borsh_cast_message( - berr, - Some("Failed to serialize account id".to_string()), - ) - })?; - let suffix = borsh::to_vec(&put_id).map_err(|berr| { - DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) - })?; - - prefix.extend_from_slice(&suffix); - - write_batch.put_cf( - &cf_att, - prefix, - borsh::to_vec(tx_hash).map_err(|berr| { - DbError::borsh_cast_message( - berr, - Some("Failed to serialize tx hash".to_string()), - ) - })?, - ); - } - - self.update_acc_meta_batch( - acc_id, - acc_num_tx + (tx_hashes.len() as u64), - &mut write_batch, - )?; - - self.db.write(write_batch).map_err(|rerr| { - DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_string())) - }) - } - - fn get_acc_transaction_hashes( - &self, - acc_id: [u8; 32], - offset: u64, - limit: u64, - ) -> DbResult> { - let cf_att = self.account_id_to_tx_hash_column(); - let mut tx_batch = vec![]; - - // ToDo: Multi get this - - for tx_id in offset..(offset + limit) { - let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { - DbError::borsh_cast_message( - berr, - Some("Failed to serialize account id".to_string()), - ) - })?; - let suffix = borsh::to_vec(&tx_id).map_err(|berr| { - DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) - })?; - - prefix.extend_from_slice(&suffix); - - let res = self - .db - .get_cf(&cf_att, prefix) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let tx_hash = if let Some(data) = res { - Ok(borsh::from_slice::<[u8; 32]>(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize tx_hash".to_string()), - ) - })?) - } else { - // Tx hash not found, assuming that previous one was the last - break; - }?; - - tx_batch.push(tx_hash); - } - - Ok(tx_batch) - } - - pub fn get_acc_transactions( - &self, - acc_id: [u8; 32], - offset: u64, - limit: u64, - ) -> DbResult> { - let mut tx_batch = vec![]; - - for tx_hash in self.get_acc_transaction_hashes(acc_id, offset, limit)? { - let block_id = self.get_block_id_by_tx_hash(tx_hash)?; - let block = self.get_block(block_id)?; - - let transaction = block - .body - .transactions - .iter() - .find(|tx| tx.hash().0 == tx_hash) - .ok_or(DbError::db_interaction_error(format!( - "Missing transaction in block {} with hash {:#?}", - block.header.block_id, tx_hash - )))?; - - tx_batch.push(transaction.clone()); - } - - Ok(tx_batch) - } -} - -#[cfg(test)] -mod tests { - use nssa::AccountId; - use tempfile::tempdir; - - use super::*; - - fn genesis_block() -> Block { - common::test_utils::produce_dummy_block(1, None, vec![]) - } - - fn acc1() -> AccountId { - AccountId::new([ - 148, 179, 206, 253, 199, 51, 82, 86, 232, 2, 152, 122, 80, 243, 54, 207, 237, 112, 83, - 153, 44, 59, 204, 49, 128, 84, 160, 227, 216, 149, 97, 102, - ]) - } - - fn acc2() -> AccountId { - AccountId::new([ - 30, 145, 107, 3, 207, 73, 192, 230, 160, 63, 238, 207, 18, 69, 54, 216, 103, 244, 92, - 94, 124, 248, 42, 16, 141, 19, 119, 18, 14, 226, 140, 204, - ]) - } - - fn acc1_sign_key() -> nssa::PrivateKey { - nssa::PrivateKey::try_new([1; 32]).unwrap() - } - - fn acc2_sign_key() -> nssa::PrivateKey { - nssa::PrivateKey::try_new([2; 32]).unwrap() - } - - fn initial_state() -> V02State { - nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]) - } - - fn transfer(amount: u128, nonce: u128, direction: bool) -> NSSATransaction { - let from; - let to; - let sign_key; - - if direction { - from = acc1(); - to = acc2(); - sign_key = acc1_sign_key(); - } else { - from = acc2(); - to = acc1(); - sign_key = acc2_sign_key(); - } - - common::test_utils::create_transaction_native_token_transfer( - from, nonce, to, amount, sign_key, - ) - } - - #[test] - fn test_start_db() { - let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); - - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) - .unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let first_id = dbio.get_meta_first_block_in_db().unwrap(); - let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); - let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); - let last_block = dbio.get_block(1).unwrap(); - let breakpoint = dbio.get_breakpoint(0).unwrap(); - let final_state = dbio.final_state().unwrap(); - - assert_eq!(last_id, 1); - assert_eq!(first_id, 1); - assert!(is_first_set); - assert_eq!(last_br_id, 0); - assert_eq!(last_block.header.hash, genesis_block().header.hash); - assert_eq!( - breakpoint.get_account_by_id(acc1()), - final_state.get_account_by_id(acc1()) - ); - assert_eq!( - breakpoint.get_account_by_id(acc2()), - final_state.get_account_by_id(acc2()) - ); - } - - #[test] - fn test_one_block_insertion() { - let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); - - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) - .unwrap(); - - let prev_hash = genesis_block().header.hash; - let transfer_tx = transfer(1, 0, true); - let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - - dbio.put_block(block, [1; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let first_id = dbio.get_meta_first_block_in_db().unwrap(); - let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); - let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - let breakpoint = dbio.get_breakpoint(0).unwrap(); - let final_state = dbio.final_state().unwrap(); - - assert_eq!(last_id, 2); - assert_eq!(first_id, 1); - assert!(is_first_set); - assert_eq!(last_br_id, 0); - assert_ne!(last_block.header.hash, genesis_block().header.hash); - assert_eq!( - breakpoint.get_account_by_id(acc1()).balance - - final_state.get_account_by_id(acc1()).balance, - 1 - ); - assert_eq!( - final_state.get_account_by_id(acc2()).balance - - breakpoint.get_account_by_id(acc2()).balance, - 1 - ); - } - - #[test] - fn test_new_breakpoint() { - let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); - - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) - .unwrap(); - - for i in 1..BREAKPOINT_INTERVAL { - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, (i - 1) as u128, true); - let block = - common::test_utils::produce_dummy_block(i + 1, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [i as u8; 32]).unwrap(); - } - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let first_id = dbio.get_meta_first_block_in_db().unwrap(); - let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); - let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - let prev_breakpoint = dbio.get_breakpoint(0).unwrap(); - let breakpoint = dbio.get_breakpoint(1).unwrap(); - let final_state = dbio.final_state().unwrap(); - - assert_eq!(last_id, 100); - assert_eq!(first_id, 1); - assert!(is_first_set); - assert_eq!(last_br_id, 1); - assert_ne!(last_block.header.hash, genesis_block().header.hash); - assert_eq!( - prev_breakpoint.get_account_by_id(acc1()).balance - - final_state.get_account_by_id(acc1()).balance, - 99 - ); - assert_eq!( - final_state.get_account_by_id(acc2()).balance - - prev_breakpoint.get_account_by_id(acc2()).balance, - 99 - ); - assert_eq!( - breakpoint.get_account_by_id(acc1()), - final_state.get_account_by_id(acc1()) - ); - assert_eq!( - breakpoint.get_account_by_id(acc2()), - final_state.get_account_by_id(acc2()) - ); - } - - #[test] - fn test_simple_maps() { - let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); - - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) - .unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 0, true); - let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - - let control_hash1 = block.header.hash; - - dbio.put_block(block, [1; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 1, true); - let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); - - let control_hash2 = block.header.hash; - - dbio.put_block(block, [2; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 2, true); - - let control_tx_hash1 = transfer_tx.hash(); - - let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [3; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 3, true); - - let control_tx_hash2 = transfer_tx.hash(); - - let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [4; 32]).unwrap(); - - let control_block_id1 = dbio.get_block_id_by_hash(control_hash1.0).unwrap(); - let control_block_id2 = dbio.get_block_id_by_hash(control_hash2.0).unwrap(); - let control_block_id3 = dbio.get_block_id_by_tx_hash(control_tx_hash1.0).unwrap(); - let control_block_id4 = dbio.get_block_id_by_tx_hash(control_tx_hash2.0).unwrap(); - - assert_eq!(control_block_id1, 2); - assert_eq!(control_block_id2, 3); - assert_eq!(control_block_id3, 4); - assert_eq!(control_block_id4, 5); - } - - #[test] - fn test_block_batch() { - let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); - - let mut block_res = vec![]; - - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) - .unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 0, true); - let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - - block_res.push(block.clone()); - dbio.put_block(block, [1; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 1, true); - let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); - - block_res.push(block.clone()); - dbio.put_block(block, [2; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 2, true); - - let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - block_res.push(block.clone()); - dbio.put_block(block, [3; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 3, true); - - let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - block_res.push(block.clone()); - dbio.put_block(block, [4; 32]).unwrap(); - - let block_hashes_mem: Vec<[u8; 32]> = - block_res.into_iter().map(|bl| bl.header.hash.0).collect(); - - // Get blocks before ID 6 (i.e., starting from 5 going backwards), limit 4 - // This should return blocks 5, 4, 3, 2 in descending order - let mut batch_res = dbio.get_block_batch(Some(6), 4).unwrap(); - batch_res.reverse(); // Reverse to match ascending order for comparison - - let block_hashes_db: Vec<[u8; 32]> = - batch_res.into_iter().map(|bl| bl.header.hash.0).collect(); - - assert_eq!(block_hashes_mem, block_hashes_db); - - let block_hashes_mem_limited = &block_hashes_mem[1..]; - - // Get blocks before ID 6, limit 3 - // This should return blocks 5, 4, 3 in descending order - let mut batch_res_limited = dbio.get_block_batch(Some(6), 3).unwrap(); - batch_res_limited.reverse(); // Reverse to match ascending order for comparison - - let block_hashes_db_limited: Vec<[u8; 32]> = batch_res_limited - .into_iter() - .map(|bl| bl.header.hash.0) - .collect(); - - assert_eq!(block_hashes_mem_limited, block_hashes_db_limited.as_slice()); - } - - #[test] - fn test_account_map() { - let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); - - let mut tx_hash_res = vec![]; - - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) - .unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 0, true); - - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - - dbio.put_block(block, [1; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 1, true); - - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); - - dbio.put_block(block, [2; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 2, true); - - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - - dbio.put_block(block, [3; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 3, true); - - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - - dbio.put_block(block, [4; 32]).unwrap(); - - let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 4).unwrap(); - let acc1_tx_hashes: Vec<[u8; 32]> = acc1_tx.into_iter().map(|tx| tx.hash().0).collect(); - - assert_eq!(acc1_tx_hashes, tx_hash_res); - - let acc1_tx_limited = dbio.get_acc_transactions(*acc1().value(), 1, 4).unwrap(); - let acc1_tx_limited_hashes: Vec<[u8; 32]> = - acc1_tx_limited.into_iter().map(|tx| tx.hash().0).collect(); - - assert_eq!(acc1_tx_limited_hashes.as_slice(), &tx_hash_res[1..]) - } -} diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs new file mode 100644 index 00000000..311746e6 --- /dev/null +++ b/storage/src/indexer/mod.rs @@ -0,0 +1,605 @@ +use std::{collections::HashMap, ops::Div, path::Path, sync::Arc}; + +use common::{block::Block, transaction::NSSATransaction}; +use nssa::V02State; +use rocksdb::{ + BoundColumnFamily, ColumnFamilyDescriptor, DBWithThreadMode, MultiThreaded, Options, WriteBatch, +}; + +use crate::error::DbError; + +pub mod read_once; +pub mod write_once; +pub mod read_multi_get; +pub mod write_batch; + +/// Maximal size of stored blocks in base +/// +/// Used to control db size +/// +/// Currently effectively unbounded. +pub const BUFF_SIZE_ROCKSDB: usize = usize::MAX; + +/// Size of stored blocks cache in memory +/// +/// Keeping small to not run out of memory +pub const CACHE_SIZE: usize = 1000; + +/// Key base for storing metainformation about id of first block in db +pub const DB_META_FIRST_BLOCK_IN_DB_KEY: &str = "first_block_in_db"; +/// Key base for storing metainformation about id of last current block in db +pub const DB_META_LAST_BLOCK_IN_DB_KEY: &str = "last_block_in_db"; +/// Key base for storing metainformation about id of last observed L1 lib header in db +pub const DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY: &str = + "last_observed_l1_lib_header_in_db"; +/// Key base for storing metainformation which describe if first block has been set +pub const DB_META_FIRST_BLOCK_SET_KEY: &str = "first_block_set"; +/// Key base for storing metainformation about the last breakpoint +pub const DB_META_LAST_BREAKPOINT_ID: &str = "last_breakpoint_id"; + +/// Interval between state breakpoints +pub const BREAKPOINT_INTERVAL: u64 = 100; + +/// Name of block column family +pub const CF_BLOCK_NAME: &str = "cf_block"; +/// Name of meta column family +pub const CF_META_NAME: &str = "cf_meta"; +/// Name of breakpoint column family +pub const CF_BREAKPOINT_NAME: &str = "cf_breakpoint"; +/// Name of hash to id map column family +pub const CF_HASH_TO_ID: &str = "cf_hash_to_id"; +/// Name of tx hash to id map column family +pub const CF_TX_TO_ID: &str = "cf_tx_to_id"; +/// Name of account meta column family +pub const CF_ACC_META: &str = "cf_acc_meta"; +/// Name of account id to tx hash map column family +pub const CF_ACC_TO_TX: &str = "cf_acc_to_tx"; + +pub type DbResult = Result; + +fn closest_breakpoint_id(block_id: u64) -> u64 { + block_id.saturating_sub(1).div(BREAKPOINT_INTERVAL) +} + +pub struct RocksDBIO { + pub db: DBWithThreadMode, +} + +impl RocksDBIO { + pub fn open_or_create(path: &Path, start_data: Option<(Block, V02State)>) -> DbResult { + let mut cf_opts = Options::default(); + cf_opts.set_max_write_buffer_number(16); + // ToDo: Add more column families for different data + let cfb = ColumnFamilyDescriptor::new(CF_BLOCK_NAME, cf_opts.clone()); + let cfmeta = ColumnFamilyDescriptor::new(CF_META_NAME, cf_opts.clone()); + let cfbreakpoint = ColumnFamilyDescriptor::new(CF_BREAKPOINT_NAME, cf_opts.clone()); + let cfhti = ColumnFamilyDescriptor::new(CF_HASH_TO_ID, cf_opts.clone()); + let cftti = ColumnFamilyDescriptor::new(CF_TX_TO_ID, cf_opts.clone()); + let cfameta = ColumnFamilyDescriptor::new(CF_ACC_META, cf_opts.clone()); + let cfatt = ColumnFamilyDescriptor::new(CF_ACC_TO_TX, cf_opts.clone()); + + let mut db_opts = Options::default(); + db_opts.create_missing_column_families(true); + db_opts.create_if_missing(true); + let db = DBWithThreadMode::::open_cf_descriptors( + &db_opts, + path, + vec![cfb, cfmeta, cfbreakpoint, cfhti, cftti, cfameta, cfatt], + ); + + let dbio = Self { + // There is no point in handling this from runner code + db: db.unwrap(), + }; + + let is_start_set = dbio.get_meta_is_first_block_set()?; + + if is_start_set { + Ok(dbio) + } else if let Some((block, initial_state)) = start_data { + let block_id = block.header.block_id; + dbio.put_meta_last_block_in_db(block_id)?; + dbio.put_meta_first_block_in_db(block)?; + dbio.put_meta_is_first_block_set()?; + + // First breakpoint setup + dbio.put_breakpoint(0, initial_state)?; + dbio.put_meta_last_breakpoint_id(0)?; + + Ok(dbio) + } else { + // Here we are trying to start a DB without a block, one should not do it. + unreachable!() + } + } + + pub fn destroy(path: &Path) -> DbResult<()> { + let mut cf_opts = Options::default(); + cf_opts.set_max_write_buffer_number(16); + // ToDo: Add more column families for different data + let _cfb = ColumnFamilyDescriptor::new(CF_BLOCK_NAME, cf_opts.clone()); + let _cfmeta = ColumnFamilyDescriptor::new(CF_META_NAME, cf_opts.clone()); + let _cfsnapshot = ColumnFamilyDescriptor::new(CF_BREAKPOINT_NAME, cf_opts.clone()); + let _cfhti = ColumnFamilyDescriptor::new(CF_HASH_TO_ID, cf_opts.clone()); + let _cftti = ColumnFamilyDescriptor::new(CF_TX_TO_ID, cf_opts.clone()); + let _cfameta = ColumnFamilyDescriptor::new(CF_ACC_META, cf_opts.clone()); + let _cfatt = ColumnFamilyDescriptor::new(CF_ACC_TO_TX, cf_opts.clone()); + + let mut db_opts = Options::default(); + db_opts.create_missing_column_families(true); + db_opts.create_if_missing(true); + DBWithThreadMode::::destroy(&db_opts, path) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) + } + + // Columns + + pub fn meta_column(&self) -> Arc> { + self.db.cf_handle(CF_META_NAME).unwrap() + } + + pub fn block_column(&self) -> Arc> { + self.db.cf_handle(CF_BLOCK_NAME).unwrap() + } + + pub fn breakpoint_column(&self) -> Arc> { + self.db.cf_handle(CF_BREAKPOINT_NAME).unwrap() + } + + pub fn hash_to_id_column(&self) -> Arc> { + self.db.cf_handle(CF_HASH_TO_ID).unwrap() + } + + pub fn tx_hash_to_id_column(&self) -> Arc> { + self.db.cf_handle(CF_TX_TO_ID).unwrap() + } + + pub fn account_id_to_tx_hash_column(&self) -> Arc> { + self.db.cf_handle(CF_ACC_TO_TX).unwrap() + } + + pub fn account_meta_column(&self) -> Arc> { + self.db.cf_handle(CF_ACC_META).unwrap() + } + + //State + + pub fn calculate_state_for_id(&self, block_id: u64) -> DbResult { + let last_block = self.get_meta_last_block_in_db()?; + + if block_id <= last_block { + let br_id = closest_breakpoint_id(block_id); + let mut breakpoint = self.get_breakpoint(br_id)?; + + // ToDo: update it to handle any genesis id + // right now works correctly only if genesis_id < BREAKPOINT_INTERVAL + let start = if br_id != 0 { + BREAKPOINT_INTERVAL * br_id + } else { + self.get_meta_first_block_in_db()? + }; + + for id in start..=block_id { + let block = self.get_block(id)?; + + for transaction in block.body.transactions { + transaction + .transaction_stateless_check() + .map_err(|err| { + DbError::db_interaction_error(format!( + "transaction pre check failed with err {err:?}" + )) + })? + .execute_check_on_state(&mut breakpoint) + .map_err(|err| { + DbError::db_interaction_error(format!( + "transaction execution failed with err {err:?}" + )) + })?; + } + } + + Ok(breakpoint) + } else { + Err(DbError::db_interaction_error( + "Block on this id not found".to_string(), + )) + } + } + + pub fn final_state(&self) -> DbResult { + self.calculate_state_for_id(self.get_meta_last_block_in_db()?) + } +} + +#[cfg(test)] +mod tests { + use nssa::AccountId; + use tempfile::tempdir; + + use super::*; + + fn genesis_block() -> Block { + common::test_utils::produce_dummy_block(1, None, vec![]) + } + + fn acc1() -> AccountId { + AccountId::new([ + 148, 179, 206, 253, 199, 51, 82, 86, 232, 2, 152, 122, 80, 243, 54, 207, 237, 112, 83, + 153, 44, 59, 204, 49, 128, 84, 160, 227, 216, 149, 97, 102, + ]) + } + + fn acc2() -> AccountId { + AccountId::new([ + 30, 145, 107, 3, 207, 73, 192, 230, 160, 63, 238, 207, 18, 69, 54, 216, 103, 244, 92, + 94, 124, 248, 42, 16, 141, 19, 119, 18, 14, 226, 140, 204, + ]) + } + + fn acc1_sign_key() -> nssa::PrivateKey { + nssa::PrivateKey::try_new([1; 32]).unwrap() + } + + fn acc2_sign_key() -> nssa::PrivateKey { + nssa::PrivateKey::try_new([2; 32]).unwrap() + } + + fn initial_state() -> V02State { + nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]) + } + + fn transfer(amount: u128, nonce: u128, direction: bool) -> NSSATransaction { + let from; + let to; + let sign_key; + + if direction { + from = acc1(); + to = acc2(); + sign_key = acc1_sign_key(); + } else { + from = acc2(); + to = acc1(); + sign_key = acc2_sign_key(); + } + + common::test_utils::create_transaction_native_token_transfer( + from, nonce, to, amount, sign_key, + ) + } + + #[test] + fn test_start_db() { + let temp_dir = tempdir().unwrap(); + let temdir_path = temp_dir.path(); + + let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) + .unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let first_id = dbio.get_meta_first_block_in_db().unwrap(); + let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); + let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); + let last_block = dbio.get_block(1).unwrap(); + let breakpoint = dbio.get_breakpoint(0).unwrap(); + let final_state = dbio.final_state().unwrap(); + + assert_eq!(last_id, 1); + assert_eq!(first_id, 1); + assert!(is_first_set); + assert_eq!(last_br_id, 0); + assert_eq!(last_block.header.hash, genesis_block().header.hash); + assert_eq!( + breakpoint.get_account_by_id(acc1()), + final_state.get_account_by_id(acc1()) + ); + assert_eq!( + breakpoint.get_account_by_id(acc2()), + final_state.get_account_by_id(acc2()) + ); + } + + #[test] + fn test_one_block_insertion() { + let temp_dir = tempdir().unwrap(); + let temdir_path = temp_dir.path(); + + let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) + .unwrap(); + + let prev_hash = genesis_block().header.hash; + let transfer_tx = transfer(1, 0, true); + let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); + + dbio.put_block(block, [1; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let first_id = dbio.get_meta_first_block_in_db().unwrap(); + let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); + let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + let breakpoint = dbio.get_breakpoint(0).unwrap(); + let final_state = dbio.final_state().unwrap(); + + assert_eq!(last_id, 2); + assert_eq!(first_id, 1); + assert!(is_first_set); + assert_eq!(last_br_id, 0); + assert_ne!(last_block.header.hash, genesis_block().header.hash); + assert_eq!( + breakpoint.get_account_by_id(acc1()).balance + - final_state.get_account_by_id(acc1()).balance, + 1 + ); + assert_eq!( + final_state.get_account_by_id(acc2()).balance + - breakpoint.get_account_by_id(acc2()).balance, + 1 + ); + } + + #[test] + fn test_new_breakpoint() { + let temp_dir = tempdir().unwrap(); + let temdir_path = temp_dir.path(); + + let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) + .unwrap(); + + for i in 1..BREAKPOINT_INTERVAL { + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, (i - 1) as u128, true); + let block = + common::test_utils::produce_dummy_block(i + 1, Some(prev_hash), vec![transfer_tx]); + dbio.put_block(block, [i as u8; 32]).unwrap(); + } + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let first_id = dbio.get_meta_first_block_in_db().unwrap(); + let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); + let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + let prev_breakpoint = dbio.get_breakpoint(0).unwrap(); + let breakpoint = dbio.get_breakpoint(1).unwrap(); + let final_state = dbio.final_state().unwrap(); + + assert_eq!(last_id, 100); + assert_eq!(first_id, 1); + assert!(is_first_set); + assert_eq!(last_br_id, 1); + assert_ne!(last_block.header.hash, genesis_block().header.hash); + assert_eq!( + prev_breakpoint.get_account_by_id(acc1()).balance + - final_state.get_account_by_id(acc1()).balance, + 99 + ); + assert_eq!( + final_state.get_account_by_id(acc2()).balance + - prev_breakpoint.get_account_by_id(acc2()).balance, + 99 + ); + assert_eq!( + breakpoint.get_account_by_id(acc1()), + final_state.get_account_by_id(acc1()) + ); + assert_eq!( + breakpoint.get_account_by_id(acc2()), + final_state.get_account_by_id(acc2()) + ); + } + + #[test] + fn test_simple_maps() { + let temp_dir = tempdir().unwrap(); + let temdir_path = temp_dir.path(); + + let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) + .unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 0, true); + let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); + + let control_hash1 = block.header.hash; + + dbio.put_block(block, [1; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 1, true); + let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); + + let control_hash2 = block.header.hash; + + dbio.put_block(block, [2; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 2, true); + + let control_tx_hash1 = transfer_tx.hash(); + + let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); + dbio.put_block(block, [3; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 3, true); + + let control_tx_hash2 = transfer_tx.hash(); + + let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); + dbio.put_block(block, [4; 32]).unwrap(); + + let control_block_id1 = dbio.get_block_id_by_hash(control_hash1.0).unwrap(); + let control_block_id2 = dbio.get_block_id_by_hash(control_hash2.0).unwrap(); + let control_block_id3 = dbio.get_block_id_by_tx_hash(control_tx_hash1.0).unwrap(); + let control_block_id4 = dbio.get_block_id_by_tx_hash(control_tx_hash2.0).unwrap(); + + assert_eq!(control_block_id1, 2); + assert_eq!(control_block_id2, 3); + assert_eq!(control_block_id3, 4); + assert_eq!(control_block_id4, 5); + } + + #[test] + fn test_block_batch() { + let temp_dir = tempdir().unwrap(); + let temdir_path = temp_dir.path(); + + let mut block_res = vec![]; + + let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) + .unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 0, true); + let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); + + block_res.push(block.clone()); + dbio.put_block(block, [1; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 1, true); + let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); + + block_res.push(block.clone()); + dbio.put_block(block, [2; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 2, true); + + let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); + block_res.push(block.clone()); + dbio.put_block(block, [3; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 3, true); + + let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); + block_res.push(block.clone()); + dbio.put_block(block, [4; 32]).unwrap(); + + let block_hashes_mem: Vec<[u8; 32]> = + block_res.into_iter().map(|bl| bl.header.hash.0).collect(); + + // Get blocks before ID 6 (i.e., starting from 5 going backwards), limit 4 + // This should return blocks 5, 4, 3, 2 in descending order + let mut batch_res = dbio.get_block_batch(Some(6), 4).unwrap(); + batch_res.reverse(); // Reverse to match ascending order for comparison + + let block_hashes_db: Vec<[u8; 32]> = + batch_res.into_iter().map(|bl| bl.header.hash.0).collect(); + + assert_eq!(block_hashes_mem, block_hashes_db); + + let block_hashes_mem_limited = &block_hashes_mem[1..]; + + // Get blocks before ID 6, limit 3 + // This should return blocks 5, 4, 3 in descending order + let mut batch_res_limited = dbio.get_block_batch(Some(6), 3).unwrap(); + batch_res_limited.reverse(); // Reverse to match ascending order for comparison + + let block_hashes_db_limited: Vec<[u8; 32]> = batch_res_limited + .into_iter() + .map(|bl| bl.header.hash.0) + .collect(); + + assert_eq!(block_hashes_mem_limited, block_hashes_db_limited.as_slice()); + } + + #[test] + fn test_account_map() { + let temp_dir = tempdir().unwrap(); + let temdir_path = temp_dir.path(); + + let mut tx_hash_res = vec![]; + + let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) + .unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 0, true); + + tx_hash_res.push(transfer_tx.hash().0); + + let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); + + dbio.put_block(block, [1; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 1, true); + + tx_hash_res.push(transfer_tx.hash().0); + + let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); + + dbio.put_block(block, [2; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 2, true); + + tx_hash_res.push(transfer_tx.hash().0); + + let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); + + dbio.put_block(block, [3; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 3, true); + + tx_hash_res.push(transfer_tx.hash().0); + + let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); + + dbio.put_block(block, [4; 32]).unwrap(); + + let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 4).unwrap(); + let acc1_tx_hashes: Vec<[u8; 32]> = acc1_tx.into_iter().map(|tx| tx.hash().0).collect(); + + assert_eq!(acc1_tx_hashes, tx_hash_res); + + let acc1_tx_limited = dbio.get_acc_transactions(*acc1().value(), 1, 4).unwrap(); + let acc1_tx_limited_hashes: Vec<[u8; 32]> = + acc1_tx_limited.into_iter().map(|tx| tx.hash().0).collect(); + + assert_eq!(acc1_tx_limited_hashes.as_slice(), &tx_hash_res[1..]) + } +} diff --git a/storage/src/indexer/read_multi_get.rs b/storage/src/indexer/read_multi_get.rs new file mode 100644 index 00000000..15124682 --- /dev/null +++ b/storage/src/indexer/read_multi_get.rs @@ -0,0 +1,5 @@ +use super::*; + +impl RocksDBIO { + +} \ No newline at end of file diff --git a/storage/src/indexer/read_once.rs b/storage/src/indexer/read_once.rs new file mode 100644 index 00000000..1091dbc1 --- /dev/null +++ b/storage/src/indexer/read_once.rs @@ -0,0 +1,424 @@ +use super::*; + +impl RocksDBIO { + //Meta + + pub fn get_meta_first_block_in_db(&self) -> DbResult { + let cf_meta = self.meta_column(); + let res = self + .db + .get_cf( + &cf_meta, + borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to deserialize first block".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "First block not found".to_string(), + )) + } + } + + pub fn get_meta_last_block_in_db(&self) -> DbResult { + let cf_meta = self.meta_column(); + let res = self + .db + .get_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to deserialize last block".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "Last block not found".to_string(), + )) + } + } + + pub fn get_meta_last_observed_l1_lib_header_in_db(&self) -> DbResult> { + let cf_meta = self.meta_column(); + let res = self + .db + .get_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err( + |err| { + DbError::borsh_cast_message( + err, + Some( + "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" + .to_string(), + ), + ) + }, + )?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + res.map(|data| { + borsh::from_slice::<[u8; 32]>(&data).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to deserialize last l1 lib header".to_string()), + ) + }) + }) + .transpose() + } + + pub fn get_meta_is_first_block_set(&self) -> DbResult { + let cf_meta = self.meta_column(); + let res = self + .db + .get_cf( + &cf_meta, + borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + Ok(res.is_some()) + } + + pub fn get_meta_last_breakpoint_id(&self) -> DbResult { + let cf_meta = self.meta_column(); + let res = self + .db + .get_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to deserialize last breakpoint id".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "Last breakpoint id not found".to_string(), + )) + } + } + + //Block + + pub fn get_block(&self, block_id: u64) -> DbResult { + let cf_block = self.block_column(); + let res = self + .db + .get_cf( + &cf_block, + borsh::to_vec(&block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize block data".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "Block on this id not found".to_string(), + )) + } + } + + pub fn get_block_batch(&self, before: Option, limit: u64) -> DbResult> { + let cf_block = self.block_column(); + let mut block_batch = vec![]; + + // Determine the starting block ID + let start_block_id = if let Some(before_id) = before { + before_id.saturating_sub(1) + } else { + // Get the latest block ID + self.get_meta_last_block_in_db()? + }; + + // ToDo: Multi get this + + for i in 0..limit { + let block_id = start_block_id.saturating_sub(i); + if block_id == 0 { + break; + } + + let res = self + .db + .get_cf( + &cf_block, + borsh::to_vec(&block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let block = if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize block data".to_string()), + ) + })?) + } else { + // Block not found, assuming that previous one was the last + break; + }?; + + block_batch.push(block); + } + + Ok(block_batch) + } + + //State + + pub fn get_breakpoint(&self, br_id: u64) -> DbResult { + let cf_br = self.breakpoint_column(); + let res = self + .db + .get_cf( + &cf_br, + borsh::to_vec(&br_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize breakpoint id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize breakpoint data".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "Breakpoint on this id not found".to_string(), + )) + } + } + + pub fn put_next_breakpoint(&self) -> DbResult<()> { + let last_block = self.get_meta_last_block_in_db()?; + let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; + let block_to_break_id = next_breakpoint_id * BREAKPOINT_INTERVAL; + + if block_to_break_id <= last_block { + let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; + + self.put_breakpoint(next_breakpoint_id, next_breakpoint)?; + self.put_meta_last_breakpoint_id(next_breakpoint_id) + } else { + Err(DbError::db_interaction_error( + "Breakpoint not yet achieved".to_string(), + )) + } + } + + // Mappings + + pub fn get_block_id_by_hash(&self, hash: [u8; 32]) -> DbResult { + let cf_hti = self.hash_to_id_column(); + let res = self + .db + .get_cf( + &cf_hti, + borsh::to_vec(&hash).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block hash".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize block id".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "Block on this hash not found".to_string(), + )) + } + } + + pub fn get_block_id_by_tx_hash(&self, tx_hash: [u8; 32]) -> DbResult { + let cf_tti = self.tx_hash_to_id_column(); + let res = self + .db + .get_cf( + &cf_tti, + borsh::to_vec(&tx_hash).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize transaction hash".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize block id".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "Block for this tx hash not found".to_string(), + )) + } + } + + // Accounts meta + + pub(crate) fn get_acc_meta_num_tx(&self, acc_id: [u8; 32]) -> DbResult> { + let cf_ameta = self.account_meta_column(); + let res = self.db.get_cf(&cf_ameta, acc_id).map_err(|rerr| { + DbError::rocksdb_cast_message(rerr, Some("Failed to read from acc meta cf".to_string())) + })?; + + res.map(|data| { + borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message(serr, Some("Failed to deserialize num tx".to_string())) + }) + }) + .transpose() + } + + // Account + + fn get_acc_transaction_hashes( + &self, + acc_id: [u8; 32], + offset: u64, + limit: u64, + ) -> DbResult> { + let cf_att = self.account_id_to_tx_hash_column(); + let mut tx_batch = vec![]; + + // ToDo: Multi get this + + for tx_id in offset..(offset + limit) { + let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { + DbError::borsh_cast_message( + berr, + Some("Failed to serialize account id".to_string()), + ) + })?; + let suffix = borsh::to_vec(&tx_id).map_err(|berr| { + DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) + })?; + + prefix.extend_from_slice(&suffix); + + let res = self + .db + .get_cf(&cf_att, prefix) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let tx_hash = if let Some(data) = res { + Ok(borsh::from_slice::<[u8; 32]>(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize tx_hash".to_string()), + ) + })?) + } else { + // Tx hash not found, assuming that previous one was the last + break; + }?; + + tx_batch.push(tx_hash); + } + + Ok(tx_batch) + } + + pub fn get_acc_transactions( + &self, + acc_id: [u8; 32], + offset: u64, + limit: u64, + ) -> DbResult> { + let mut tx_batch = vec![]; + + for tx_hash in self.get_acc_transaction_hashes(acc_id, offset, limit)? { + let block_id = self.get_block_id_by_tx_hash(tx_hash)?; + let block = self.get_block(block_id)?; + + let transaction = block + .body + .transactions + .iter() + .find(|tx| tx.hash().0 == tx_hash) + .ok_or(DbError::db_interaction_error(format!( + "Missing transaction in block {} with hash {:#?}", + block.header.block_id, tx_hash + )))?; + + tx_batch.push(transaction.clone()); + } + + Ok(tx_batch) + } +} \ No newline at end of file diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs new file mode 100644 index 00000000..2bb11a85 --- /dev/null +++ b/storage/src/indexer/write_batch.rs @@ -0,0 +1,78 @@ +use super::*; + +impl RocksDBIO { + // Accounts meta + + pub(crate) fn update_acc_meta_batch( + &self, + acc_id: [u8; 32], + num_tx: u64, + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + let cf_ameta = self.account_meta_column(); + + write_batch.put_cf( + &cf_ameta, + borsh::to_vec(&acc_id).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize account id".to_string())) + })?, + borsh::to_vec(&num_tx).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize acc metadata".to_string()), + ) + })?, + ); + + Ok(()) + } + + // Account + + pub fn put_account_transactions( + &self, + acc_id: [u8; 32], + tx_hashes: Vec<[u8; 32]>, + ) -> DbResult<()> { + let acc_num_tx = self.get_acc_meta_num_tx(acc_id)?.unwrap_or(0); + let cf_att = self.account_id_to_tx_hash_column(); + let mut write_batch = WriteBatch::new(); + + for (tx_id, tx_hash) in tx_hashes.iter().enumerate() { + let put_id = acc_num_tx + tx_id as u64; + + let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { + DbError::borsh_cast_message( + berr, + Some("Failed to serialize account id".to_string()), + ) + })?; + let suffix = borsh::to_vec(&put_id).map_err(|berr| { + DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) + })?; + + prefix.extend_from_slice(&suffix); + + write_batch.put_cf( + &cf_att, + prefix, + borsh::to_vec(tx_hash).map_err(|berr| { + DbError::borsh_cast_message( + berr, + Some("Failed to serialize tx hash".to_string()), + ) + })?, + ); + } + + self.update_acc_meta_batch( + acc_id, + acc_num_tx + (tx_hashes.len() as u64), + &mut write_batch, + )?; + + self.db.write(write_batch).map_err(|rerr| { + DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_string())) + }) + } +} \ No newline at end of file diff --git a/storage/src/indexer/write_once.rs b/storage/src/indexer/write_once.rs new file mode 100644 index 00000000..71946f63 --- /dev/null +++ b/storage/src/indexer/write_once.rs @@ -0,0 +1,244 @@ +use super::*; + +impl RocksDBIO { + //Meta + + pub fn put_meta_first_block_in_db(&self, block: Block) -> DbResult<()> { + let cf_meta = self.meta_column(); + self.db + .put_cf( + &cf_meta, + borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize first block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + self.put_block(block, [0; 32])?; + Ok(()) + } + + pub fn put_meta_last_block_in_db(&self, block_id: u64) -> DbResult<()> { + let cf_meta = self.meta_column(); + self.db + .put_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + borsh::to_vec(&block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize last block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + Ok(()) + } + + pub fn put_meta_last_observed_l1_lib_header_in_db( + &self, + l1_lib_header: [u8; 32], + ) -> DbResult<()> { + let cf_meta = self.meta_column(); + self.db + .put_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err( + |err| { + DbError::borsh_cast_message( + err, + Some( + "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" + .to_string(), + ), + ) + }, + )?, + borsh::to_vec(&l1_lib_header).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize last l1 block header".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + Ok(()) + } + + pub fn put_meta_last_breakpoint_id(&self, br_id: u64) -> DbResult<()> { + let cf_meta = self.meta_column(); + self.db + .put_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), + ) + })?, + borsh::to_vec(&br_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize last block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + Ok(()) + } + + pub fn put_meta_is_first_block_set(&self) -> DbResult<()> { + let cf_meta = self.meta_column(); + self.db + .put_cf( + &cf_meta, + borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), + ) + })?, + [1u8; 1], + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + Ok(()) + } + + //Block + + pub fn put_block(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { + let cf_block = self.block_column(); + let cf_hti = self.hash_to_id_column(); + let cf_tti: Arc> = self.tx_hash_to_id_column(); + + // ToDo: rewrite this with write batching + + self.db + .put_cf( + &cf_block, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + borsh::to_vec(&block).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block data".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let last_curr_block = self.get_meta_last_block_in_db()?; + + if block.header.block_id > last_curr_block { + self.put_meta_last_block_in_db(block.header.block_id)?; + self.put_meta_last_observed_l1_lib_header_in_db(l1_lib_header)?; + } + + self.db + .put_cf( + &cf_hti, + borsh::to_vec(&block.header.hash).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block hash".to_string()), + ) + })?, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let mut acc_to_tx_map: HashMap<[u8; 32], Vec<[u8; 32]>> = HashMap::new(); + + for tx in block.body.transactions { + let tx_hash = tx.hash(); + + self.db + .put_cf( + &cf_tti, + borsh::to_vec(&tx_hash).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize tx hash".to_string()), + ) + })?, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let acc_ids = tx + .affected_public_account_ids() + .into_iter() + .map(|account_id| account_id.into_value()) + .collect::>(); + + for acc_id in acc_ids { + acc_to_tx_map + .entry(acc_id) + .and_modify(|tx_hashes| tx_hashes.push(tx_hash.into())) + .or_insert(vec![tx_hash.into()]); + } + } + + for (acc_id, tx_hashes) in acc_to_tx_map { + self.put_account_transactions(acc_id, tx_hashes)?; + } + + if block.header.block_id.is_multiple_of(BREAKPOINT_INTERVAL) { + self.put_next_breakpoint()?; + } + + Ok(()) + } + + //State + + pub fn put_breakpoint(&self, br_id: u64, breakpoint: V02State) -> DbResult<()> { + let cf_br = self.breakpoint_column(); + + self.db + .put_cf( + &cf_br, + borsh::to_vec(&br_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize breakpoint id".to_string()), + ) + })?, + borsh::to_vec(&breakpoint).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize breakpoint data".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) + } +} \ No newline at end of file From 53b26064ebdb2e3ace9410290cf8e7f869d8adb6 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Mon, 9 Mar 2026 16:35:03 +0200 Subject: [PATCH 02/13] feat: multi gets --- storage/src/indexer/mod.rs | 6 +- storage/src/indexer/read_multi_get.rs | 349 +++++++++++++++++++++++++- storage/src/indexer/read_once.rs | 136 +--------- storage/src/indexer/write_batch.rs | 2 +- storage/src/indexer/write_once.rs | 8 +- 5 files changed, 359 insertions(+), 142 deletions(-) diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index 311746e6..c445e6bb 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -8,10 +8,10 @@ use rocksdb::{ use crate::error::DbError; -pub mod read_once; -pub mod write_once; pub mod read_multi_get; +pub mod read_once; pub mod write_batch; +pub mod write_once; /// Maximal size of stored blocks in base /// @@ -162,7 +162,7 @@ impl RocksDBIO { self.db.cf_handle(CF_ACC_META).unwrap() } - //State + // State pub fn calculate_state_for_id(&self, block_id: u64) -> DbResult { let last_block = self.get_meta_last_block_in_db()?; diff --git a/storage/src/indexer/read_multi_get.rs b/storage/src/indexer/read_multi_get.rs index 15124682..9fd68382 100644 --- a/storage/src/indexer/read_multi_get.rs +++ b/storage/src/indexer/read_multi_get.rs @@ -1,5 +1,350 @@ use super::*; +#[derive(Debug, Clone)] +pub struct DBMetadata { + pub first_block_in_db: u64, + pub last_block_in_db: u64, + pub last_observed_l1_lib_header_in_db: [u8; 32], + pub is_first_block_set: bool, + pub last_breakpoint_id: u64, +} + impl RocksDBIO { - -} \ No newline at end of file + fn meta_keys_list() -> DbResult>> { + let mut keys = vec![]; + + keys.push( + borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + ); + keys.push(borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), + ) + })?); + keys.push( + borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some( + "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" + .to_string(), + ), + ) + })?, + ); + keys.push(borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), + ) + })?); + keys.push(borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), + ) + })?); + + Ok(keys) + } + + fn read_meta_all(&self) -> DbResult> { + let cf_meta = self.meta_column(); + + let multi_get_res = self.db.multi_get_cf( + RocksDBIO::meta_keys_list()? + .into_iter() + .map(|key| (&cf_meta, key)), + ); + + let Some(first_block_in_db_raw) = + multi_get_res[0] + .as_ref() + .map_err(|err| DbError::RocksDbError { + error: err.clone(), + additional_info: Some("Failed to read first_block_in_db".to_string()), + })? + else { + return Ok(None); + }; + let Some(last_block_in_db_raw) = + multi_get_res[1] + .as_ref() + .map_err(|err| DbError::RocksDbError { + error: err.clone(), + additional_info: Some("Failed to read last_block_in_db".to_string()), + })? + else { + return Ok(None); + }; + let Some(last_observed_l1_lib_header_in_db_raw) = + multi_get_res[2] + .as_ref() + .map_err(|err| DbError::RocksDbError { + error: err.clone(), + additional_info: Some( + "Failed to read last_observed_l1_lib_header_in_db".to_string(), + ), + })? + else { + return Ok(None); + }; + let is_first_block_set = multi_get_res[3] + .as_ref() + .map_err(|err| DbError::RocksDbError { + error: err.clone(), + additional_info: Some("Failed to read is_first_block_set".to_string()), + })? + .is_some(); + let Some(last_breakpoint_id_raw) = + multi_get_res[4] + .as_ref() + .clone() + .map_err(|err| DbError::RocksDbError { + error: err.clone(), + additional_info: Some("Failed to read last_breakpoint_id".to_string()), + })? + else { + return Ok(None); + }; + + let first_block_in_db = borsh::from_slice::(first_block_in_db_raw).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to deserialize first block".to_string())) + })?; + let last_block_in_db = borsh::from_slice::(last_block_in_db_raw).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to deserialize last block".to_string())) + })?; + let last_observed_l1_lib_header_in_db = borsh::from_slice::<[u8; 32]>( + last_observed_l1_lib_header_in_db_raw, + ) + .map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to deserialize last l1 lib header".to_string()), + ) + })?; + let last_breakpoint_id = + borsh::from_slice::(last_breakpoint_id_raw).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to deserialize last breakpoint id".to_string()), + ) + })?; + + Ok(Some(DBMetadata { + first_block_in_db, + last_block_in_db, + last_observed_l1_lib_header_in_db, + is_first_block_set, + last_breakpoint_id, + })) + } + + pub fn get_block_batch(&self, before: Option, limit: u64) -> DbResult> { + let mut seq = vec![]; + + // Determine the starting block ID + let start_block_id = if let Some(before_id) = before { + before_id.saturating_sub(1) + } else { + // Get the latest block ID + self.get_meta_last_block_in_db()? + }; + + for i in 0..limit { + let block_id = start_block_id.saturating_sub(i); + if block_id == 0 { + break; + } + seq.push(block_id); + } + + self.get_block_batch_seq(seq.into_iter()) + } + + /// Get block batch from a sequence + /// + /// Currently assumes non-decreasing sequence + /// + /// ToDo: Add suport of arbitrary sequences + fn get_block_batch_seq(&self, seq: impl Iterator) -> DbResult> { + let cf_block = self.block_column(); + + // Keys setup + let mut keys = vec![]; + for block_id in seq { + keys.push(( + &cf_block, + borsh::to_vec(&block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + )); + } + + let multi_get_res = self.db.multi_get_cf(keys); + + // Keys parsing + let mut block_batch = vec![]; + for res in multi_get_res { + let res = res.map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let block = if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize block data".to_string()), + ) + })?) + } else { + // Block not found, assuming that previous one was the last + break; + }?; + + block_batch.push(block); + } + + Ok(block_batch) + } + + /// Get block ids by txs + /// + /// Transactions must be sorted by time of arrival + /// + /// ToDo: There may be multiple transactions in one block + /// so this method can take redundant reads. + /// Need to update signature and implementation. + fn get_block_ids_by_tx_vec(&self, tx_vec: &[[u8; 32]]) -> DbResult> { + let cf_tti = self.tx_hash_to_id_column(); + + // Keys setup + let mut keys = vec![]; + for tx_hash in tx_vec { + keys.push(( + &cf_tti, + borsh::to_vec(tx_hash).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize tx_hash".to_string()), + ) + })?, + )); + } + + let multi_get_res = self.db.multi_get_cf(keys); + + // Keys parsing + let mut block_id_batch = vec![]; + for res in multi_get_res { + let res = res.map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let block_id = if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize block id".to_string()), + ) + })?) + } else { + // Block not found, assuming that previous one was the last + break; + }?; + + block_id_batch.push(block_id); + } + + Ok(block_id_batch) + } + + // Account + + pub(crate) fn get_acc_transaction_hashes( + &self, + acc_id: [u8; 32], + offset: u64, + limit: u64, + ) -> DbResult> { + let cf_att = self.account_id_to_tx_hash_column(); + let mut tx_batch = vec![]; + + // Keys preparation + let mut keys = vec![]; + for tx_id in offset..(offset + limit) { + let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { + DbError::borsh_cast_message( + berr, + Some("Failed to serialize account id".to_string()), + ) + })?; + let suffix = borsh::to_vec(&tx_id).map_err(|berr| { + DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) + })?; + + prefix.extend_from_slice(&suffix); + + keys.push((&cf_att, prefix)); + } + + let multi_get_res = self.db.multi_get_cf(keys); + + for res in multi_get_res { + let res = res.map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let tx_hash = if let Some(data) = res { + Ok(borsh::from_slice::<[u8; 32]>(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize tx_hash".to_string()), + ) + })?) + } else { + // Tx hash not found, assuming that previous one was the last + break; + }?; + + tx_batch.push(tx_hash); + } + + Ok(tx_batch) + } + + pub fn get_acc_transactions( + &self, + acc_id: [u8; 32], + offset: u64, + limit: u64, + ) -> DbResult> { + let mut tx_batch = vec![]; + + let tx_hashes = self.get_acc_transaction_hashes(acc_id, offset, limit)?; + + let associated_blocks_multi_get = self + .get_block_batch_seq(self.get_block_ids_by_tx_vec(&tx_hashes)?.into_iter())? + .into_iter() + .zip(tx_hashes); + + for (block, tx_hash) in associated_blocks_multi_get { + let transaction = block + .body + .transactions + .iter() + .find(|tx| tx.hash().0 == tx_hash) + .ok_or(DbError::db_interaction_error(format!( + "Missing transaction in block {} with hash {:#?}", + block.header.block_id, tx_hash + )))?; + + tx_batch.push(transaction.clone()); + } + + Ok(tx_batch) + } +} diff --git a/storage/src/indexer/read_once.rs b/storage/src/indexer/read_once.rs index 1091dbc1..e544d277 100644 --- a/storage/src/indexer/read_once.rs +++ b/storage/src/indexer/read_once.rs @@ -1,7 +1,7 @@ use super::*; impl RocksDBIO { - //Meta + // Meta pub fn get_meta_first_block_in_db(&self) -> DbResult { let cf_meta = self.meta_column(); @@ -139,7 +139,7 @@ impl RocksDBIO { } } - //Block + // Block pub fn get_block(&self, block_id: u64) -> DbResult { let cf_block = self.block_column(); @@ -170,58 +170,7 @@ impl RocksDBIO { } } - pub fn get_block_batch(&self, before: Option, limit: u64) -> DbResult> { - let cf_block = self.block_column(); - let mut block_batch = vec![]; - - // Determine the starting block ID - let start_block_id = if let Some(before_id) = before { - before_id.saturating_sub(1) - } else { - // Get the latest block ID - self.get_meta_last_block_in_db()? - }; - - // ToDo: Multi get this - - for i in 0..limit { - let block_id = start_block_id.saturating_sub(i); - if block_id == 0 { - break; - } - - let res = self - .db - .get_cf( - &cf_block, - borsh::to_vec(&block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let block = if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize block data".to_string()), - ) - })?) - } else { - // Block not found, assuming that previous one was the last - break; - }?; - - block_batch.push(block); - } - - Ok(block_batch) - } - - //State + // State pub fn get_breakpoint(&self, br_id: u64) -> DbResult { let cf_br = self.breakpoint_column(); @@ -344,81 +293,4 @@ impl RocksDBIO { }) .transpose() } - - // Account - - fn get_acc_transaction_hashes( - &self, - acc_id: [u8; 32], - offset: u64, - limit: u64, - ) -> DbResult> { - let cf_att = self.account_id_to_tx_hash_column(); - let mut tx_batch = vec![]; - - // ToDo: Multi get this - - for tx_id in offset..(offset + limit) { - let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { - DbError::borsh_cast_message( - berr, - Some("Failed to serialize account id".to_string()), - ) - })?; - let suffix = borsh::to_vec(&tx_id).map_err(|berr| { - DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) - })?; - - prefix.extend_from_slice(&suffix); - - let res = self - .db - .get_cf(&cf_att, prefix) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let tx_hash = if let Some(data) = res { - Ok(borsh::from_slice::<[u8; 32]>(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize tx_hash".to_string()), - ) - })?) - } else { - // Tx hash not found, assuming that previous one was the last - break; - }?; - - tx_batch.push(tx_hash); - } - - Ok(tx_batch) - } - - pub fn get_acc_transactions( - &self, - acc_id: [u8; 32], - offset: u64, - limit: u64, - ) -> DbResult> { - let mut tx_batch = vec![]; - - for tx_hash in self.get_acc_transaction_hashes(acc_id, offset, limit)? { - let block_id = self.get_block_id_by_tx_hash(tx_hash)?; - let block = self.get_block(block_id)?; - - let transaction = block - .body - .transactions - .iter() - .find(|tx| tx.hash().0 == tx_hash) - .ok_or(DbError::db_interaction_error(format!( - "Missing transaction in block {} with hash {:#?}", - block.header.block_id, tx_hash - )))?; - - tx_batch.push(transaction.clone()); - } - - Ok(tx_batch) - } -} \ No newline at end of file +} diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs index 2bb11a85..254e7dc9 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_batch.rs @@ -75,4 +75,4 @@ impl RocksDBIO { DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_string())) }) } -} \ No newline at end of file +} diff --git a/storage/src/indexer/write_once.rs b/storage/src/indexer/write_once.rs index 71946f63..b4309c78 100644 --- a/storage/src/indexer/write_once.rs +++ b/storage/src/indexer/write_once.rs @@ -1,7 +1,7 @@ use super::*; impl RocksDBIO { - //Meta + // Meta pub fn put_meta_first_block_in_db(&self, block: Block) -> DbResult<()> { let cf_meta = self.meta_column(); @@ -118,7 +118,7 @@ impl RocksDBIO { Ok(()) } - //Block + // Block pub fn put_block(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { let cf_block = self.block_column(); @@ -218,7 +218,7 @@ impl RocksDBIO { Ok(()) } - //State + // State pub fn put_breakpoint(&self, br_id: u64, breakpoint: V02State) -> DbResult<()> { let cf_br = self.breakpoint_column(); @@ -241,4 +241,4 @@ impl RocksDBIO { ) .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) } -} \ No newline at end of file +} From b57c2ac33d8c105fe7bf345b5d6f585dcf7c98a4 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Wed, 11 Mar 2026 18:37:54 +0200 Subject: [PATCH 03/13] feat: batch writes --- storage/src/indexer/read_multi_get.rs | 4 +- storage/src/indexer/read_once.rs | 17 -- storage/src/indexer/write_batch.rs | 286 ++++++++++++++++++++++++++ storage/src/indexer/write_once.rs | 17 ++ 4 files changed, 305 insertions(+), 19 deletions(-) diff --git a/storage/src/indexer/read_multi_get.rs b/storage/src/indexer/read_multi_get.rs index 9fd68382..6bfb28cc 100644 --- a/storage/src/indexer/read_multi_get.rs +++ b/storage/src/indexer/read_multi_get.rs @@ -10,7 +10,7 @@ pub struct DBMetadata { } impl RocksDBIO { - fn meta_keys_list() -> DbResult>> { + pub fn meta_keys_list() -> DbResult>> { let mut keys = vec![]; keys.push( @@ -54,7 +54,7 @@ impl RocksDBIO { Ok(keys) } - fn read_meta_all(&self) -> DbResult> { + pub fn read_meta_all(&self) -> DbResult> { let cf_meta = self.meta_column(); let multi_get_res = self.db.multi_get_cf( diff --git a/storage/src/indexer/read_once.rs b/storage/src/indexer/read_once.rs index e544d277..8fe035fd 100644 --- a/storage/src/indexer/read_once.rs +++ b/storage/src/indexer/read_once.rs @@ -201,23 +201,6 @@ impl RocksDBIO { } } - pub fn put_next_breakpoint(&self) -> DbResult<()> { - let last_block = self.get_meta_last_block_in_db()?; - let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; - let block_to_break_id = next_breakpoint_id * BREAKPOINT_INTERVAL; - - if block_to_break_id <= last_block { - let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; - - self.put_breakpoint(next_breakpoint_id, next_breakpoint)?; - self.put_meta_last_breakpoint_id(next_breakpoint_id) - } else { - Err(DbError::db_interaction_error( - "Breakpoint not yet achieved".to_string(), - )) - } - } - // Mappings pub fn get_block_id_by_hash(&self, hash: [u8; 32]) -> DbResult { diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs index 254e7dc9..9febd605 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_batch.rs @@ -75,4 +75,290 @@ impl RocksDBIO { DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_string())) }) } + + pub fn put_account_transactions_dependant( + &self, + acc_id: [u8; 32], + tx_hashes: Vec<[u8; 32]>, + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + let acc_num_tx = self.get_acc_meta_num_tx(acc_id)?.unwrap_or(0); + let cf_att = self.account_id_to_tx_hash_column(); + + for (tx_id, tx_hash) in tx_hashes.iter().enumerate() { + let put_id = acc_num_tx + tx_id as u64; + + let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { + DbError::borsh_cast_message( + berr, + Some("Failed to serialize account id".to_string()), + ) + })?; + let suffix = borsh::to_vec(&put_id).map_err(|berr| { + DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) + })?; + + prefix.extend_from_slice(&suffix); + + write_batch.put_cf( + &cf_att, + prefix, + borsh::to_vec(tx_hash).map_err(|berr| { + DbError::borsh_cast_message( + berr, + Some("Failed to serialize tx hash".to_string()), + ) + })?, + ); + } + + self.update_acc_meta_batch(acc_id, acc_num_tx + (tx_hashes.len() as u64), write_batch)?; + + Ok(()) + } + + // Meta + + pub fn put_meta_first_block_in_db_batch(&self, block: Block) -> DbResult<()> { + let cf_meta = self.meta_column(); + self.db + .put_cf( + &cf_meta, + borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize first block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + self.put_block_batch(block, [0; 32])?; + Ok(()) + } + + pub fn put_meta_last_block_in_db_batch( + &self, + block_id: u64, + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + let cf_meta = self.meta_column(); + write_batch.put_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + borsh::to_vec(&block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize last block id".to_string()), + ) + })?, + ); + Ok(()) + } + + pub fn put_meta_last_observed_l1_lib_header_in_db_batch( + &self, + l1_lib_header: [u8; 32], + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + let cf_meta = self.meta_column(); + write_batch.put_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some( + "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" + .to_string(), + ), + ) + })?, + borsh::to_vec(&l1_lib_header).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize last l1 block header".to_string()), + ) + })?, + ); + Ok(()) + } + + pub fn put_meta_last_breakpoint_id_batch( + &self, + br_id: u64, + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + let cf_meta = self.meta_column(); + write_batch.put_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), + ) + })?, + borsh::to_vec(&br_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize last block id".to_string()), + ) + })?, + ); + Ok(()) + } + + pub fn put_meta_is_first_block_set_batch(&self, write_batch: &mut WriteBatch) -> DbResult<()> { + let cf_meta = self.meta_column(); + write_batch.put_cf( + &cf_meta, + borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), + ) + })?, + [1u8; 1], + ); + Ok(()) + } + + // Block + + pub fn put_block_batch(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { + let cf_block = self.block_column(); + let cf_hti = self.hash_to_id_column(); + let cf_tti: Arc> = self.tx_hash_to_id_column(); + let last_curr_block = self.get_meta_last_block_in_db()?; + let mut write_batch = WriteBatch::default(); + + // ToDo: rewrite this with write batching + + write_batch.put_cf( + &cf_block, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize block id".to_string())) + })?, + borsh::to_vec(&block).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize block data".to_string())) + })?, + ); + + if block.header.block_id > last_curr_block { + self.put_meta_last_block_in_db_batch(block.header.block_id, &mut write_batch)?; + self.put_meta_last_observed_l1_lib_header_in_db_batch(l1_lib_header, &mut write_batch)?; + } + + write_batch.put_cf( + &cf_hti, + borsh::to_vec(&block.header.hash).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize block hash".to_string())) + })?, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize block id".to_string())) + })?, + ); + + let mut acc_to_tx_map: HashMap<[u8; 32], Vec<[u8; 32]>> = HashMap::new(); + + for tx in block.body.transactions { + let tx_hash = tx.hash(); + + write_batch.put_cf( + &cf_tti, + borsh::to_vec(&tx_hash).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize tx hash".to_string()), + ) + })?, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + ); + + let acc_ids = tx + .affected_public_account_ids() + .into_iter() + .map(|account_id| account_id.into_value()) + .collect::>(); + + for acc_id in acc_ids { + acc_to_tx_map + .entry(acc_id) + .and_modify(|tx_hashes| tx_hashes.push(tx_hash.into())) + .or_insert(vec![tx_hash.into()]); + } + } + + for (acc_id, tx_hashes) in acc_to_tx_map { + self.put_account_transactions_dependant(acc_id, tx_hashes, &mut write_batch)?; + } + + if block.header.block_id.is_multiple_of(BREAKPOINT_INTERVAL) { + self.put_next_breakpoint_batch(&mut write_batch)?; + } + + self.db.write(write_batch).map_err(|rerr| { + DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_string())) + }) + } + + // State + + pub fn put_breakpoint_batch( + &self, + br_id: u64, + breakpoint: V02State, + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + let cf_br = self.breakpoint_column(); + + write_batch.put_cf( + &cf_br, + borsh::to_vec(&br_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize breakpoint id".to_string()), + ) + })?, + borsh::to_vec(&breakpoint).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize breakpoint data".to_string()), + ) + })?, + ); + Ok(()) + } + + pub fn put_next_breakpoint_batch(&self, write_batch: &mut WriteBatch) -> DbResult<()> { + let last_block = self.get_meta_last_block_in_db()?; + let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; + let block_to_break_id = next_breakpoint_id * BREAKPOINT_INTERVAL; + + if block_to_break_id <= last_block { + let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; + + self.put_breakpoint_batch(next_breakpoint_id, next_breakpoint, write_batch)?; + self.put_meta_last_breakpoint_id_batch(next_breakpoint_id, write_batch) + } else { + Err(DbError::db_interaction_error( + "Breakpoint not yet achieved".to_string(), + )) + } + } } diff --git a/storage/src/indexer/write_once.rs b/storage/src/indexer/write_once.rs index b4309c78..bf00fce5 100644 --- a/storage/src/indexer/write_once.rs +++ b/storage/src/indexer/write_once.rs @@ -241,4 +241,21 @@ impl RocksDBIO { ) .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) } + + pub fn put_next_breakpoint(&self) -> DbResult<()> { + let last_block = self.get_meta_last_block_in_db()?; + let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; + let block_to_break_id = next_breakpoint_id * BREAKPOINT_INTERVAL; + + if block_to_break_id <= last_block { + let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; + + self.put_breakpoint(next_breakpoint_id, next_breakpoint)?; + self.put_meta_last_breakpoint_id(next_breakpoint_id) + } else { + Err(DbError::db_interaction_error( + "Breakpoint not yet achieved".to_string(), + )) + } + } } From d8542e089fab3b2e3aef93b8a8023593d69b236f Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Thu, 12 Mar 2026 15:53:38 +0200 Subject: [PATCH 04/13] fix: various fixes and cleanup --- storage/src/indexer/mod.rs | 83 +++++++++------ storage/src/indexer/read_multi_get.rs | 147 +------------------------- storage/src/indexer/write_batch.rs | 51 +-------- 3 files changed, 58 insertions(+), 223 deletions(-) diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index c445e6bb..f07980a9 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -99,6 +99,7 @@ impl RocksDBIO { } else if let Some((block, initial_state)) = start_data { let block_id = block.header.block_id; dbio.put_meta_last_block_in_db(block_id)?; + dbio.put_meta_last_observed_l1_lib_header_in_db(block.bedrock_parent_id)?; dbio.put_meta_first_block_in_db(block)?; dbio.put_meta_is_first_block_set()?; @@ -179,9 +180,7 @@ impl RocksDBIO { self.get_meta_first_block_in_db()? }; - for id in start..=block_id { - let block = self.get_block(id)?; - + for block in self.get_block_batch_seq((start + 1)..=block_id)? { for transaction in block.body.transactions { transaction .transaction_stateless_check() @@ -347,7 +346,7 @@ mod tests { let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) .unwrap(); - for i in 1..BREAKPOINT_INTERVAL { + for i in 1..(BREAKPOINT_INTERVAL + 1) { let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -355,7 +354,7 @@ mod tests { let transfer_tx = transfer(1, (i - 1) as u128, true); let block = common::test_utils::produce_dummy_block(i + 1, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [i as u8; 32]).unwrap(); + dbio.put_block_batch(block, [i as u8; 32]).unwrap(); } let last_id = dbio.get_meta_last_block_in_db().unwrap(); @@ -367,7 +366,7 @@ mod tests { let breakpoint = dbio.get_breakpoint(1).unwrap(); let final_state = dbio.final_state().unwrap(); - assert_eq!(last_id, 100); + assert_eq!(last_id, 101); assert_eq!(first_id, 1); assert!(is_first_set); assert_eq!(last_br_id, 1); @@ -375,20 +374,22 @@ mod tests { assert_eq!( prev_breakpoint.get_account_by_id(acc1()).balance - final_state.get_account_by_id(acc1()).balance, - 99 + 100 ); assert_eq!( final_state.get_account_by_id(acc2()).balance - prev_breakpoint.get_account_by_id(acc2()).balance, - 99 + 100 ); assert_eq!( - breakpoint.get_account_by_id(acc1()), - final_state.get_account_by_id(acc1()) + breakpoint.get_account_by_id(acc1()).balance + - final_state.get_account_by_id(acc1()).balance, + 1 ); assert_eq!( - breakpoint.get_account_by_id(acc2()), - final_state.get_account_by_id(acc2()) + final_state.get_account_by_id(acc2()).balance + - breakpoint.get_account_by_id(acc2()).balance, + 1 ); } @@ -531,6 +532,14 @@ mod tests { .collect(); assert_eq!(block_hashes_mem_limited, block_hashes_db_limited.as_slice()); + + let block_batch_seq = dbio.get_block_batch_seq(1..=5).unwrap(); + let block_batch_ids = block_batch_seq + .into_iter() + .map(|block| block.header.block_id) + .collect::>(); + + assert_eq!(block_batch_ids, vec![1, 2, 3, 4, 5]); } #[test] @@ -538,20 +547,25 @@ mod tests { let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let mut tx_hash_res = vec![]; - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) .unwrap(); + let mut tx_hash_res = vec![]; + let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 0, true); + let transfer_tx1 = transfer(1, 0, true); + let transfer_tx2 = transfer(1, 1, true); + tx_hash_res.push(transfer_tx1.hash().0); + tx_hash_res.push(transfer_tx2.hash().0); - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); + let block = common::test_utils::produce_dummy_block( + 2, + Some(prev_hash), + vec![transfer_tx1, transfer_tx2], + ); dbio.put_block(block, [1; 32]).unwrap(); @@ -559,11 +573,16 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 1, true); + let transfer_tx1 = transfer(1, 2, true); + let transfer_tx2 = transfer(1, 3, true); + tx_hash_res.push(transfer_tx1.hash().0); + tx_hash_res.push(transfer_tx2.hash().0); - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); + let block = common::test_utils::produce_dummy_block( + 3, + Some(prev_hash), + vec![transfer_tx1, transfer_tx2], + ); dbio.put_block(block, [2; 32]).unwrap(); @@ -571,11 +590,16 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 2, true); + let transfer_tx1 = transfer(1, 4, true); + let transfer_tx2 = transfer(1, 5, true); + tx_hash_res.push(transfer_tx1.hash().0); + tx_hash_res.push(transfer_tx2.hash().0); - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); + let block = common::test_utils::produce_dummy_block( + 4, + Some(prev_hash), + vec![transfer_tx1, transfer_tx2], + ); dbio.put_block(block, [3; 32]).unwrap(); @@ -583,15 +607,14 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 3, true); - + let transfer_tx = transfer(1, 6, true); tx_hash_res.push(transfer_tx.hash().0); let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); dbio.put_block(block, [4; 32]).unwrap(); - let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 4).unwrap(); + let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 7).unwrap(); let acc1_tx_hashes: Vec<[u8; 32]> = acc1_tx.into_iter().map(|tx| tx.hash().0).collect(); assert_eq!(acc1_tx_hashes, tx_hash_res); @@ -600,6 +623,6 @@ mod tests { let acc1_tx_limited_hashes: Vec<[u8; 32]> = acc1_tx_limited.into_iter().map(|tx| tx.hash().0).collect(); - assert_eq!(acc1_tx_limited_hashes.as_slice(), &tx_hash_res[1..]) + assert_eq!(acc1_tx_limited_hashes.as_slice(), &tx_hash_res[1..5]) } } diff --git a/storage/src/indexer/read_multi_get.rs b/storage/src/indexer/read_multi_get.rs index 6bfb28cc..981bc18e 100644 --- a/storage/src/indexer/read_multi_get.rs +++ b/storage/src/indexer/read_multi_get.rs @@ -1,151 +1,6 @@ use super::*; -#[derive(Debug, Clone)] -pub struct DBMetadata { - pub first_block_in_db: u64, - pub last_block_in_db: u64, - pub last_observed_l1_lib_header_in_db: [u8; 32], - pub is_first_block_set: bool, - pub last_breakpoint_id: u64, -} - impl RocksDBIO { - pub fn meta_keys_list() -> DbResult>> { - let mut keys = vec![]; - - keys.push( - borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), - ) - })?, - ); - keys.push(borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), - ) - })?); - keys.push( - borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some( - "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" - .to_string(), - ), - ) - })?, - ); - keys.push(borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), - ) - })?); - keys.push(borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), - ) - })?); - - Ok(keys) - } - - pub fn read_meta_all(&self) -> DbResult> { - let cf_meta = self.meta_column(); - - let multi_get_res = self.db.multi_get_cf( - RocksDBIO::meta_keys_list()? - .into_iter() - .map(|key| (&cf_meta, key)), - ); - - let Some(first_block_in_db_raw) = - multi_get_res[0] - .as_ref() - .map_err(|err| DbError::RocksDbError { - error: err.clone(), - additional_info: Some("Failed to read first_block_in_db".to_string()), - })? - else { - return Ok(None); - }; - let Some(last_block_in_db_raw) = - multi_get_res[1] - .as_ref() - .map_err(|err| DbError::RocksDbError { - error: err.clone(), - additional_info: Some("Failed to read last_block_in_db".to_string()), - })? - else { - return Ok(None); - }; - let Some(last_observed_l1_lib_header_in_db_raw) = - multi_get_res[2] - .as_ref() - .map_err(|err| DbError::RocksDbError { - error: err.clone(), - additional_info: Some( - "Failed to read last_observed_l1_lib_header_in_db".to_string(), - ), - })? - else { - return Ok(None); - }; - let is_first_block_set = multi_get_res[3] - .as_ref() - .map_err(|err| DbError::RocksDbError { - error: err.clone(), - additional_info: Some("Failed to read is_first_block_set".to_string()), - })? - .is_some(); - let Some(last_breakpoint_id_raw) = - multi_get_res[4] - .as_ref() - .clone() - .map_err(|err| DbError::RocksDbError { - error: err.clone(), - additional_info: Some("Failed to read last_breakpoint_id".to_string()), - })? - else { - return Ok(None); - }; - - let first_block_in_db = borsh::from_slice::(first_block_in_db_raw).map_err(|err| { - DbError::borsh_cast_message(err, Some("Failed to deserialize first block".to_string())) - })?; - let last_block_in_db = borsh::from_slice::(last_block_in_db_raw).map_err(|err| { - DbError::borsh_cast_message(err, Some("Failed to deserialize last block".to_string())) - })?; - let last_observed_l1_lib_header_in_db = borsh::from_slice::<[u8; 32]>( - last_observed_l1_lib_header_in_db_raw, - ) - .map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to deserialize last l1 lib header".to_string()), - ) - })?; - let last_breakpoint_id = - borsh::from_slice::(last_breakpoint_id_raw).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to deserialize last breakpoint id".to_string()), - ) - })?; - - Ok(Some(DBMetadata { - first_block_in_db, - last_block_in_db, - last_observed_l1_lib_header_in_db, - is_first_block_set, - last_breakpoint_id, - })) - } - pub fn get_block_batch(&self, before: Option, limit: u64) -> DbResult> { let mut seq = vec![]; @@ -173,7 +28,7 @@ impl RocksDBIO { /// Currently assumes non-decreasing sequence /// /// ToDo: Add suport of arbitrary sequences - fn get_block_batch_seq(&self, seq: impl Iterator) -> DbResult> { + pub fn get_block_batch_seq(&self, seq: impl Iterator) -> DbResult> { let cf_block = self.block_column(); // Keys setup diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs index 9febd605..b98e539a 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_batch.rs @@ -308,57 +308,14 @@ impl RocksDBIO { self.put_account_transactions_dependant(acc_id, tx_hashes, &mut write_batch)?; } - if block.header.block_id.is_multiple_of(BREAKPOINT_INTERVAL) { - self.put_next_breakpoint_batch(&mut write_batch)?; - } - self.db.write(write_batch).map_err(|rerr| { DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_string())) - }) - } + })?; - // State + if block.header.block_id.is_multiple_of(BREAKPOINT_INTERVAL) { + self.put_next_breakpoint()?; + } - pub fn put_breakpoint_batch( - &self, - br_id: u64, - breakpoint: V02State, - write_batch: &mut WriteBatch, - ) -> DbResult<()> { - let cf_br = self.breakpoint_column(); - - write_batch.put_cf( - &cf_br, - borsh::to_vec(&br_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize breakpoint id".to_string()), - ) - })?, - borsh::to_vec(&breakpoint).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize breakpoint data".to_string()), - ) - })?, - ); Ok(()) } - - pub fn put_next_breakpoint_batch(&self, write_batch: &mut WriteBatch) -> DbResult<()> { - let last_block = self.get_meta_last_block_in_db()?; - let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; - let block_to_break_id = next_breakpoint_id * BREAKPOINT_INTERVAL; - - if block_to_break_id <= last_block { - let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; - - self.put_breakpoint_batch(next_breakpoint_id, next_breakpoint, write_batch)?; - self.put_meta_last_breakpoint_id_batch(next_breakpoint_id, write_batch) - } else { - Err(DbError::db_interaction_error( - "Breakpoint not yet achieved".to_string(), - )) - } - } } From 247094ca614ebd7d404ea1bb94fdb305699188b4 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Thu, 12 Mar 2026 16:21:04 +0200 Subject: [PATCH 05/13] fix: batching usage in indexer service --- indexer/core/src/block_store.rs | 2 +- storage/src/indexer/mod.rs | 28 +++---- storage/src/indexer/write_once.rs | 124 ------------------------------ 3 files changed, 15 insertions(+), 139 deletions(-) diff --git a/indexer/core/src/block_store.rs b/indexer/core/src/block_store.rs index 681b63c8..7844b947 100644 --- a/indexer/core/src/block_store.rs +++ b/indexer/core/src/block_store.rs @@ -118,6 +118,6 @@ impl IndexerStore { // to represent correct block finality block.bedrock_status = BedrockStatus::Finalized; - Ok(self.dbio.put_block(block, l1_header.into())?) + Ok(self.dbio.put_block_batch(block, l1_header.into())?) } } diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index f07980a9..db47c0a6 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -100,7 +100,7 @@ impl RocksDBIO { let block_id = block.header.block_id; dbio.put_meta_last_block_in_db(block_id)?; dbio.put_meta_last_observed_l1_lib_header_in_db(block.bedrock_parent_id)?; - dbio.put_meta_first_block_in_db(block)?; + dbio.put_meta_first_block_in_db_batch(block)?; dbio.put_meta_is_first_block_set()?; // First breakpoint setup @@ -311,7 +311,7 @@ mod tests { let transfer_tx = transfer(1, 0, true); let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [1; 32]).unwrap(); + dbio.put_block_batch(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let first_id = dbio.get_meta_first_block_in_db().unwrap(); @@ -410,7 +410,7 @@ mod tests { let control_hash1 = block.header.hash; - dbio.put_block(block, [1; 32]).unwrap(); + dbio.put_block_batch(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -421,7 +421,7 @@ mod tests { let control_hash2 = block.header.hash; - dbio.put_block(block, [2; 32]).unwrap(); + dbio.put_block_batch(block, [2; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -432,7 +432,7 @@ mod tests { let control_tx_hash1 = transfer_tx.hash(); let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [3; 32]).unwrap(); + dbio.put_block_batch(block, [3; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -443,7 +443,7 @@ mod tests { let control_tx_hash2 = transfer_tx.hash(); let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [4; 32]).unwrap(); + dbio.put_block_batch(block, [4; 32]).unwrap(); let control_block_id1 = dbio.get_block_id_by_hash(control_hash1.0).unwrap(); let control_block_id2 = dbio.get_block_id_by_hash(control_hash2.0).unwrap(); @@ -474,7 +474,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(block, [1; 32]).unwrap(); + dbio.put_block_batch(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -484,7 +484,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(block, [2; 32]).unwrap(); + dbio.put_block_batch(block, [2; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -494,7 +494,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(block, [3; 32]).unwrap(); + dbio.put_block_batch(block, [3; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -504,7 +504,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(block, [4; 32]).unwrap(); + dbio.put_block_batch(block, [4; 32]).unwrap(); let block_hashes_mem: Vec<[u8; 32]> = block_res.into_iter().map(|bl| bl.header.hash.0).collect(); @@ -567,7 +567,7 @@ mod tests { vec![transfer_tx1, transfer_tx2], ); - dbio.put_block(block, [1; 32]).unwrap(); + dbio.put_block_batch(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -584,7 +584,7 @@ mod tests { vec![transfer_tx1, transfer_tx2], ); - dbio.put_block(block, [2; 32]).unwrap(); + dbio.put_block_batch(block, [2; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -601,7 +601,7 @@ mod tests { vec![transfer_tx1, transfer_tx2], ); - dbio.put_block(block, [3; 32]).unwrap(); + dbio.put_block_batch(block, [3; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -612,7 +612,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [4; 32]).unwrap(); + dbio.put_block_batch(block, [4; 32]).unwrap(); let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 7).unwrap(); let acc1_tx_hashes: Vec<[u8; 32]> = acc1_tx.into_iter().map(|tx| tx.hash().0).collect(); diff --git a/storage/src/indexer/write_once.rs b/storage/src/indexer/write_once.rs index bf00fce5..ab600f84 100644 --- a/storage/src/indexer/write_once.rs +++ b/storage/src/indexer/write_once.rs @@ -3,30 +3,6 @@ use super::*; impl RocksDBIO { // Meta - pub fn put_meta_first_block_in_db(&self, block: Block) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize first block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - self.put_block(block, [0; 32])?; - Ok(()) - } - pub fn put_meta_last_block_in_db(&self, block_id: u64) -> DbResult<()> { let cf_meta = self.meta_column(); self.db @@ -118,106 +94,6 @@ impl RocksDBIO { Ok(()) } - // Block - - pub fn put_block(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { - let cf_block = self.block_column(); - let cf_hti = self.hash_to_id_column(); - let cf_tti: Arc> = self.tx_hash_to_id_column(); - - // ToDo: rewrite this with write batching - - self.db - .put_cf( - &cf_block, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - borsh::to_vec(&block).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block data".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let last_curr_block = self.get_meta_last_block_in_db()?; - - if block.header.block_id > last_curr_block { - self.put_meta_last_block_in_db(block.header.block_id)?; - self.put_meta_last_observed_l1_lib_header_in_db(l1_lib_header)?; - } - - self.db - .put_cf( - &cf_hti, - borsh::to_vec(&block.header.hash).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block hash".to_string()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let mut acc_to_tx_map: HashMap<[u8; 32], Vec<[u8; 32]>> = HashMap::new(); - - for tx in block.body.transactions { - let tx_hash = tx.hash(); - - self.db - .put_cf( - &cf_tti, - borsh::to_vec(&tx_hash).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize tx hash".to_string()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let acc_ids = tx - .affected_public_account_ids() - .into_iter() - .map(|account_id| account_id.into_value()) - .collect::>(); - - for acc_id in acc_ids { - acc_to_tx_map - .entry(acc_id) - .and_modify(|tx_hashes| tx_hashes.push(tx_hash.into())) - .or_insert(vec![tx_hash.into()]); - } - } - - for (acc_id, tx_hashes) in acc_to_tx_map { - self.put_account_transactions(acc_id, tx_hashes)?; - } - - if block.header.block_id.is_multiple_of(BREAKPOINT_INTERVAL) { - self.put_next_breakpoint()?; - } - - Ok(()) - } - // State pub fn put_breakpoint(&self, br_id: u64, breakpoint: V02State) -> DbResult<()> { From 7b20d70169ab6a8743be8ecb34fc4d5b85c2642c Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Fri, 13 Mar 2026 14:27:54 +0200 Subject: [PATCH 06/13] fix: header storing fix --- integration_tests/tests/indexer.rs | 2 +- storage/src/indexer/mod.rs | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/integration_tests/tests/indexer.rs b/integration_tests/tests/indexer.rs index ad169790..5422c292 100644 --- a/integration_tests/tests/indexer.rs +++ b/integration_tests/tests/indexer.rs @@ -12,7 +12,7 @@ use tokio::test; use wallet::cli::{Command, programs::native_token_transfer::AuthTransferSubcommand}; /// Timeout in milliseconds to reliably await for block finalization -const L2_TO_L1_TIMEOUT_MILLIS: u64 = 600000; +const L2_TO_L1_TIMEOUT_MILLIS: u64 = 300000; #[test] async fn indexer_test_run() -> Result<()> { diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index db47c0a6..b3e2f5b7 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -99,7 +99,6 @@ impl RocksDBIO { } else if let Some((block, initial_state)) = start_data { let block_id = block.header.block_id; dbio.put_meta_last_block_in_db(block_id)?; - dbio.put_meta_last_observed_l1_lib_header_in_db(block.bedrock_parent_id)?; dbio.put_meta_first_block_in_db_batch(block)?; dbio.put_meta_is_first_block_set()?; @@ -279,6 +278,7 @@ mod tests { let last_id = dbio.get_meta_last_block_in_db().unwrap(); let first_id = dbio.get_meta_first_block_in_db().unwrap(); let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); + let last_observed_l1_header = dbio.get_meta_last_observed_l1_lib_header_in_db().unwrap(); let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); let last_block = dbio.get_block(1).unwrap(); let breakpoint = dbio.get_breakpoint(0).unwrap(); @@ -286,6 +286,7 @@ mod tests { assert_eq!(last_id, 1); assert_eq!(first_id, 1); + assert_eq!(last_observed_l1_header, None); assert!(is_first_set); assert_eq!(last_br_id, 0); assert_eq!(last_block.header.hash, genesis_block().header.hash); @@ -315,6 +316,10 @@ mod tests { let last_id = dbio.get_meta_last_block_in_db().unwrap(); let first_id = dbio.get_meta_first_block_in_db().unwrap(); + let last_observed_l1_header = dbio + .get_meta_last_observed_l1_lib_header_in_db() + .unwrap() + .unwrap(); let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -323,6 +328,7 @@ mod tests { assert_eq!(last_id, 2); assert_eq!(first_id, 1); + assert_eq!(last_observed_l1_header, [1; 32]); assert!(is_first_set); assert_eq!(last_br_id, 0); assert_ne!(last_block.header.hash, genesis_block().header.hash); From adfb1156b1af13ab533807a168a8e763f7cd2daa Mon Sep 17 00:00:00 2001 From: Pravdyvy <46261001+Pravdyvy@users.noreply.github.com> Date: Tue, 17 Mar 2026 15:20:59 +0200 Subject: [PATCH 07/13] Update storage/src/indexer/write_batch.rs Co-authored-by: Sergio Chouhy <41742639+schouhy@users.noreply.github.com> --- storage/src/indexer/write_batch.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs index b98e539a..2387a424 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_batch.rs @@ -242,7 +242,6 @@ impl RocksDBIO { let last_curr_block = self.get_meta_last_block_in_db()?; let mut write_batch = WriteBatch::default(); - // ToDo: rewrite this with write batching write_batch.put_cf( &cf_block, From 6e7a6f8703821c29069b0462aa9cb9f0098491e4 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Tue, 17 Mar 2026 15:23:41 +0200 Subject: [PATCH 08/13] fix: fmt --- storage/src/indexer/write_batch.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs index 2387a424..9777addb 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_batch.rs @@ -242,7 +242,6 @@ impl RocksDBIO { let last_curr_block = self.get_meta_last_block_in_db()?; let mut write_batch = WriteBatch::default(); - write_batch.put_cf( &cf_block, borsh::to_vec(&block.header.block_id).map_err(|err| { From a00018e64eb79c9f36a4d32b4bd8f2574b168b75 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Tue, 17 Mar 2026 15:40:55 +0200 Subject: [PATCH 09/13] fix: suggestions 1 --- indexer/core/src/block_store.rs | 2 +- storage/src/indexer/mod.rs | 44 +++++++++++++++--------------- storage/src/indexer/write_batch.rs | 4 +-- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/indexer/core/src/block_store.rs b/indexer/core/src/block_store.rs index 7844b947..681b63c8 100644 --- a/indexer/core/src/block_store.rs +++ b/indexer/core/src/block_store.rs @@ -118,6 +118,6 @@ impl IndexerStore { // to represent correct block finality block.bedrock_status = BedrockStatus::Finalized; - Ok(self.dbio.put_block_batch(block, l1_header.into())?) + Ok(self.dbio.put_block(block, l1_header.into())?) } } diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index b3e2f5b7..7ac682c5 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -89,7 +89,7 @@ impl RocksDBIO { let dbio = Self { // There is no point in handling this from runner code - db: db.unwrap(), + db: db.expect("We should have permissions to open DB"), }; let is_start_set = dbio.get_meta_is_first_block_set()?; @@ -135,31 +135,31 @@ impl RocksDBIO { // Columns pub fn meta_column(&self) -> Arc> { - self.db.cf_handle(CF_META_NAME).unwrap() + self.db.cf_handle(CF_META_NAME).expect("Meta column should exist") } pub fn block_column(&self) -> Arc> { - self.db.cf_handle(CF_BLOCK_NAME).unwrap() + self.db.cf_handle(CF_BLOCK_NAME).expect("Block column should exist") } pub fn breakpoint_column(&self) -> Arc> { - self.db.cf_handle(CF_BREAKPOINT_NAME).unwrap() + self.db.cf_handle(CF_BREAKPOINT_NAME).expect("Breakpoint column should exist") } pub fn hash_to_id_column(&self) -> Arc> { - self.db.cf_handle(CF_HASH_TO_ID).unwrap() + self.db.cf_handle(CF_HASH_TO_ID).expect("Hash to id map column should exist") } pub fn tx_hash_to_id_column(&self) -> Arc> { - self.db.cf_handle(CF_TX_TO_ID).unwrap() + self.db.cf_handle(CF_TX_TO_ID).expect("Tx hash to id map column should exist") } pub fn account_id_to_tx_hash_column(&self) -> Arc> { - self.db.cf_handle(CF_ACC_TO_TX).unwrap() + self.db.cf_handle(CF_ACC_TO_TX).expect("Account id to tx map column should exist") } pub fn account_meta_column(&self) -> Arc> { - self.db.cf_handle(CF_ACC_META).unwrap() + self.db.cf_handle(CF_ACC_META).expect("Account meta column should exist") } // State @@ -312,7 +312,7 @@ mod tests { let transfer_tx = transfer(1, 0, true); let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - dbio.put_block_batch(block, [1; 32]).unwrap(); + dbio.put_block(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let first_id = dbio.get_meta_first_block_in_db().unwrap(); @@ -360,7 +360,7 @@ mod tests { let transfer_tx = transfer(1, (i - 1) as u128, true); let block = common::test_utils::produce_dummy_block(i + 1, Some(prev_hash), vec![transfer_tx]); - dbio.put_block_batch(block, [i as u8; 32]).unwrap(); + dbio.put_block(block, [i as u8; 32]).unwrap(); } let last_id = dbio.get_meta_last_block_in_db().unwrap(); @@ -416,7 +416,7 @@ mod tests { let control_hash1 = block.header.hash; - dbio.put_block_batch(block, [1; 32]).unwrap(); + dbio.put_block(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -427,7 +427,7 @@ mod tests { let control_hash2 = block.header.hash; - dbio.put_block_batch(block, [2; 32]).unwrap(); + dbio.put_block(block, [2; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -438,7 +438,7 @@ mod tests { let control_tx_hash1 = transfer_tx.hash(); let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - dbio.put_block_batch(block, [3; 32]).unwrap(); + dbio.put_block(block, [3; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -449,7 +449,7 @@ mod tests { let control_tx_hash2 = transfer_tx.hash(); let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - dbio.put_block_batch(block, [4; 32]).unwrap(); + dbio.put_block(block, [4; 32]).unwrap(); let control_block_id1 = dbio.get_block_id_by_hash(control_hash1.0).unwrap(); let control_block_id2 = dbio.get_block_id_by_hash(control_hash2.0).unwrap(); @@ -480,7 +480,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block_batch(block, [1; 32]).unwrap(); + dbio.put_block(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -490,7 +490,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block_batch(block, [2; 32]).unwrap(); + dbio.put_block(block, [2; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -500,7 +500,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block_batch(block, [3; 32]).unwrap(); + dbio.put_block(block, [3; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -510,7 +510,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block_batch(block, [4; 32]).unwrap(); + dbio.put_block(block, [4; 32]).unwrap(); let block_hashes_mem: Vec<[u8; 32]> = block_res.into_iter().map(|bl| bl.header.hash.0).collect(); @@ -573,7 +573,7 @@ mod tests { vec![transfer_tx1, transfer_tx2], ); - dbio.put_block_batch(block, [1; 32]).unwrap(); + dbio.put_block(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -590,7 +590,7 @@ mod tests { vec![transfer_tx1, transfer_tx2], ); - dbio.put_block_batch(block, [2; 32]).unwrap(); + dbio.put_block(block, [2; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -607,7 +607,7 @@ mod tests { vec![transfer_tx1, transfer_tx2], ); - dbio.put_block_batch(block, [3; 32]).unwrap(); + dbio.put_block(block, [3; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -618,7 +618,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - dbio.put_block_batch(block, [4; 32]).unwrap(); + dbio.put_block(block, [4; 32]).unwrap(); let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 7).unwrap(); let acc1_tx_hashes: Vec<[u8; 32]> = acc1_tx.into_iter().map(|tx| tx.hash().0).collect(); diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs index 9777addb..73d2c01d 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_batch.rs @@ -139,7 +139,7 @@ impl RocksDBIO { ) .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - self.put_block_batch(block, [0; 32])?; + self.put_block(block, [0; 32])?; Ok(()) } @@ -235,7 +235,7 @@ impl RocksDBIO { // Block - pub fn put_block_batch(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { + pub fn put_block(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { let cf_block = self.block_column(); let cf_hti = self.hash_to_id_column(); let cf_tti: Arc> = self.tx_hash_to_id_column(); From dfb9c709417dd7b878f3f0c821ddcd48b6770207 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Tue, 17 Mar 2026 15:44:36 +0200 Subject: [PATCH 10/13] fix: fmt --- storage/src/indexer/mod.rs | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index 7ac682c5..232a217c 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -135,31 +135,45 @@ impl RocksDBIO { // Columns pub fn meta_column(&self) -> Arc> { - self.db.cf_handle(CF_META_NAME).expect("Meta column should exist") + self.db + .cf_handle(CF_META_NAME) + .expect("Meta column should exist") } pub fn block_column(&self) -> Arc> { - self.db.cf_handle(CF_BLOCK_NAME).expect("Block column should exist") + self.db + .cf_handle(CF_BLOCK_NAME) + .expect("Block column should exist") } pub fn breakpoint_column(&self) -> Arc> { - self.db.cf_handle(CF_BREAKPOINT_NAME).expect("Breakpoint column should exist") + self.db + .cf_handle(CF_BREAKPOINT_NAME) + .expect("Breakpoint column should exist") } pub fn hash_to_id_column(&self) -> Arc> { - self.db.cf_handle(CF_HASH_TO_ID).expect("Hash to id map column should exist") + self.db + .cf_handle(CF_HASH_TO_ID) + .expect("Hash to id map column should exist") } pub fn tx_hash_to_id_column(&self) -> Arc> { - self.db.cf_handle(CF_TX_TO_ID).expect("Tx hash to id map column should exist") + self.db + .cf_handle(CF_TX_TO_ID) + .expect("Tx hash to id map column should exist") } pub fn account_id_to_tx_hash_column(&self) -> Arc> { - self.db.cf_handle(CF_ACC_TO_TX).expect("Account id to tx map column should exist") + self.db + .cf_handle(CF_ACC_TO_TX) + .expect("Account id to tx map column should exist") } pub fn account_meta_column(&self) -> Arc> { - self.db.cf_handle(CF_ACC_META).expect("Account meta column should exist") + self.db + .cf_handle(CF_ACC_META) + .expect("Account meta column should exist") } // State From 5b99f0dabd832d4032d7dcbadd69bc6d24fd11b8 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Wed, 18 Mar 2026 17:52:28 +0200 Subject: [PATCH 11/13] fix: lint fix --- storage/src/indexer/mod.rs | 192 +++++++++++------- .../{read_multi_get.rs => read_multiple.rs} | 46 +++-- storage/src/indexer/read_once.rs | 7 +- .../{write_batch.rs => write_atomic.rs} | 39 +++- .../{write_once.rs => write_non_atomic.rs} | 14 +- 5 files changed, 187 insertions(+), 111 deletions(-) rename storage/src/indexer/{read_multi_get.rs => read_multiple.rs} (81%) rename storage/src/indexer/{write_batch.rs => write_atomic.rs} (87%) rename storage/src/indexer/{write_once.rs => write_non_atomic.rs} (90%) diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index 12faf96d..c5d47c1f 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -8,10 +8,10 @@ use rocksdb::{ use crate::error::DbError; -pub mod read_multi_get; +pub mod read_multiple; pub mod read_once; -pub mod write_batch; -pub mod write_once; +pub mod write_atomic; +pub mod write_non_atomic; /// Maximal size of stored blocks in base. /// @@ -57,13 +57,6 @@ pub const CF_ACC_TO_TX: &str = "cf_acc_to_tx"; pub type DbResult = Result; -fn closest_breakpoint_id(block_id: u64) -> u64 { - block_id - .saturating_sub(1) - .checked_div(u64::from(BREAKPOINT_INTERVAL)) - .expect("Breakpoint interval is not zero") -} - pub struct RocksDBIO { pub db: DBWithThreadMode, } @@ -184,7 +177,9 @@ impl RocksDBIO { self.get_meta_first_block_in_db()? }; - for block in self.get_block_batch_seq((start + 1)..=block_id)? { + for block in self.get_block_batch_seq( + start.checked_add(1).expect("Will be lesser that u64::MAX")..=block_id, + )? { for transaction in block.body.transactions { transaction .transaction_stateless_check() @@ -215,11 +210,17 @@ impl RocksDBIO { } } -#[allow(clippy::shadow_unrelated)] +fn closest_breakpoint_id(block_id: u64) -> u64 { + block_id + .saturating_sub(1) + .checked_div(u64::from(BREAKPOINT_INTERVAL)) + .expect("Breakpoint interval is not zero") +} + +#[expect(clippy::shadow_unrelated, reason = "Fine for tests")] #[cfg(test)] mod tests { - use common::transaction::NSSATransaction; - use nssa::AccountId; + use nssa::{AccountId, PublicKey}; use tempfile::tempdir; use super::*; @@ -228,20 +229,6 @@ mod tests { common::test_utils::produce_dummy_block(1, None, vec![]) } - fn acc1() -> AccountId { - AccountId::new([ - 148, 179, 206, 253, 199, 51, 82, 86, 232, 2, 152, 122, 80, 243, 54, 207, 237, 112, 83, - 153, 44, 59, 204, 49, 128, 84, 160, 227, 216, 149, 97, 102, - ]) - } - - fn acc2() -> AccountId { - AccountId::new([ - 30, 145, 107, 3, 207, 73, 192, 230, 160, 63, 238, 207, 18, 69, 54, 216, 103, 244, 92, - 94, 124, 248, 42, 16, 141, 19, 119, 18, 14, 226, 140, 204, - ]) - } - fn acc1_sign_key() -> nssa::PrivateKey { nssa::PrivateKey::try_new([1; 32]).unwrap() } @@ -250,28 +237,12 @@ mod tests { nssa::PrivateKey::try_new([2; 32]).unwrap() } - fn initial_state() -> V02State { - nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]) + fn acc1() -> AccountId { + AccountId::from(&PublicKey::new_from_private_key(&acc1_sign_key())) } - fn transfer(amount: u128, nonce: u128, direction: bool) -> NSSATransaction { - let from; - let to; - let sign_key; - - if direction { - from = acc1(); - to = acc2(); - sign_key = acc1_sign_key(); - } else { - from = acc2(); - to = acc1(); - sign_key = acc2_sign_key(); - } - - common::test_utils::create_transaction_native_token_transfer( - from, nonce, to, amount, &sign_key, - ) + fn acc2() -> AccountId { + AccountId::from(&PublicKey::new_from_private_key(&acc2_sign_key())) } #[test] @@ -279,8 +250,12 @@ mod tests { let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = - RocksDBIO::open_or_create(temdir_path, &genesis_block(), &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create( + temdir_path, + &genesis_block(), + &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + ) + .unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let first_id = dbio.get_meta_first_block_in_db().unwrap(); @@ -312,11 +287,20 @@ mod tests { let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = - RocksDBIO::open_or_create(temdir_path, &genesis_block(), &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create( + temdir_path, + &genesis_block(), + &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + ) + .unwrap(); let prev_hash = genesis_block().header.hash; - let transfer_tx = transfer(1, 0, true); + let from = acc1(); + let to = acc2(); + let sign_key = acc1_sign_key(); + + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 0, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); dbio.put_block(&block, [1; 32]).unwrap(); @@ -356,15 +340,30 @@ mod tests { let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = - RocksDBIO::open_or_create(temdir_path, &genesis_block(), &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create( + temdir_path, + &genesis_block(), + &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + ) + .unwrap(); + + let from = acc1(); + let to = acc2(); + let sign_key = acc1_sign_key(); for i in 1..=BREAKPOINT_INTERVAL { let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, (i - 1).into(), true); + + let transfer_tx = common::test_utils::create_transaction_native_token_transfer( + from, + (i - 1).into(), + to, + 1, + &sign_key, + ); let block = common::test_utils::produce_dummy_block( (i + 1).into(), Some(prev_hash), @@ -414,14 +413,23 @@ mod tests { let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = - RocksDBIO::open_or_create(temdir_path, &genesis_block(), &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create( + temdir_path, + &genesis_block(), + &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + ) + .unwrap(); + + let from = acc1(); + let to = acc2(); + let sign_key = acc1_sign_key(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 0, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 0, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); let control_hash1 = block.header.hash; @@ -432,7 +440,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 1, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 1, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); let control_hash2 = block.header.hash; @@ -443,7 +452,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 2, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 2, to, 1, &sign_key); let control_tx_hash1 = transfer_tx.hash(); @@ -454,7 +464,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 3, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 3, to, 1, &sign_key); let control_tx_hash2 = transfer_tx.hash(); @@ -479,14 +490,23 @@ mod tests { let mut block_res = vec![]; - let dbio = - RocksDBIO::open_or_create(temdir_path, &genesis_block(), &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create( + temdir_path, + &genesis_block(), + &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + ) + .unwrap(); + + let from = acc1(); + let to = acc2(); + let sign_key = acc1_sign_key(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 0, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 0, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); @@ -496,7 +516,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 1, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 1, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); @@ -506,7 +527,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 2, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 2, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); @@ -516,7 +538,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 3, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 3, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); @@ -563,8 +586,16 @@ mod tests { let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = - RocksDBIO::open_or_create(temdir_path, &genesis_block(), &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create( + temdir_path, + &genesis_block(), + &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + ) + .unwrap(); + + let from = acc1(); + let to = acc2(); + let sign_key = acc1_sign_key(); let mut tx_hash_res = vec![]; @@ -572,8 +603,10 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx1 = transfer(1, 0, true); - let transfer_tx2 = transfer(1, 1, true); + let transfer_tx1 = + common::test_utils::create_transaction_native_token_transfer(from, 0, to, 1, &sign_key); + let transfer_tx2 = + common::test_utils::create_transaction_native_token_transfer(from, 1, to, 1, &sign_key); tx_hash_res.push(transfer_tx1.hash().0); tx_hash_res.push(transfer_tx2.hash().0); @@ -589,8 +622,10 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx1 = transfer(1, 2, true); - let transfer_tx2 = transfer(1, 3, true); + let transfer_tx1 = + common::test_utils::create_transaction_native_token_transfer(from, 2, to, 1, &sign_key); + let transfer_tx2 = + common::test_utils::create_transaction_native_token_transfer(from, 3, to, 1, &sign_key); tx_hash_res.push(transfer_tx1.hash().0); tx_hash_res.push(transfer_tx2.hash().0); @@ -606,8 +641,10 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx1 = transfer(1, 4, true); - let transfer_tx2 = transfer(1, 5, true); + let transfer_tx1 = + common::test_utils::create_transaction_native_token_transfer(from, 4, to, 1, &sign_key); + let transfer_tx2 = + common::test_utils::create_transaction_native_token_transfer(from, 5, to, 1, &sign_key); tx_hash_res.push(transfer_tx1.hash().0); tx_hash_res.push(transfer_tx2.hash().0); @@ -623,7 +660,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 6, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 6, to, 1, &sign_key); tx_hash_res.push(transfer_tx.hash().0); let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); diff --git a/storage/src/indexer/read_multi_get.rs b/storage/src/indexer/read_multiple.rs similarity index 81% rename from storage/src/indexer/read_multi_get.rs rename to storage/src/indexer/read_multiple.rs index 66fe25d7..866fc7b0 100644 --- a/storage/src/indexer/read_multi_get.rs +++ b/storage/src/indexer/read_multiple.rs @@ -1,7 +1,8 @@ use common::transaction::NSSATransaction; -use super::*; +use super::{Block, DbError, DbResult, RocksDBIO}; +#[expect(clippy::multiple_inherent_impl, reason = "Readability")] impl RocksDBIO { pub fn get_block_batch(&self, before: Option, limit: u64) -> DbResult> { let mut seq = vec![]; @@ -25,11 +26,11 @@ impl RocksDBIO { self.get_block_batch_seq(seq.into_iter()) } - /// Get block batch from a sequence + /// Get block batch from a sequence. /// - /// Currently assumes non-decreasing sequence + /// Currently assumes non-decreasing sequence. /// - /// ToDo: Add suport of arbitrary sequences + /// `ToDo`: Add suport of arbitrary sequences. pub fn get_block_batch_seq(&self, seq: impl Iterator) -> DbResult> { let cf_block = self.block_column(); @@ -72,11 +73,9 @@ impl RocksDBIO { Ok(block_batch) } - /// Get block ids by txs + /// Get block ids by txs. /// - /// Transactions must be sorted by time of arrival - /// - /// ToDo: There may be multiple transactions in one block + /// `ToDo`: There may be multiple transactions in one block /// so this method can take redundant reads. /// Need to update signature and implementation. fn get_block_ids_by_tx_vec(&self, tx_vec: &[[u8; 32]]) -> DbResult> { @@ -98,18 +97,21 @@ impl RocksDBIO { // Keys parsing let mut block_id_batch = vec![]; for res in multi_get_res { - let res = res.map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + let res = res + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))? + .ok_or_else(|| { + DbError::db_interaction_error( + "Tx to block id mapping do not contain transaction from vec".to_owned(), + ) + })?; - let block_id = if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { + let block_id = { + Ok(borsh::from_slice::(&res).map_err(|serr| { DbError::borsh_cast_message( serr, Some("Failed to deserialize block id".to_owned()), ) })?) - } else { - // Block not found, assuming that previous one was the last - break; }?; block_id_batch.push(block_id); @@ -131,7 +133,11 @@ impl RocksDBIO { // Keys preparation let mut keys = vec![]; - for tx_id in offset..(offset + limit) { + for tx_id in offset + ..offset + .checked_add(limit) + .expect("Transaction limit should be lesser than u64::MAX") + { let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { DbError::borsh_cast_message(berr, Some("Failed to serialize account id".to_owned())) })?; @@ -188,10 +194,12 @@ impl RocksDBIO { .transactions .iter() .find(|tx| tx.hash().0 == tx_hash) - .ok_or(DbError::db_interaction_error(format!( - "Missing transaction in block {} with hash {:#?}", - block.header.block_id, tx_hash - )))?; + .ok_or_else(|| { + DbError::db_interaction_error(format!( + "Missing transaction in block {} with hash {:#?}", + block.header.block_id, tx_hash + )) + })?; tx_batch.push(transaction.clone()); } diff --git a/storage/src/indexer/read_once.rs b/storage/src/indexer/read_once.rs index 40929fd6..f966f349 100644 --- a/storage/src/indexer/read_once.rs +++ b/storage/src/indexer/read_once.rs @@ -1,5 +1,10 @@ -use super::*; +use super::{ + Block, DB_META_FIRST_BLOCK_IN_DB_KEY, DB_META_FIRST_BLOCK_SET_KEY, + DB_META_LAST_BLOCK_IN_DB_KEY, DB_META_LAST_BREAKPOINT_ID, + DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DbError, DbResult, RocksDBIO, V02State, +}; +#[expect(clippy::multiple_inherent_impl, reason = "Readability")] impl RocksDBIO { // Meta diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_atomic.rs similarity index 87% rename from storage/src/indexer/write_batch.rs rename to storage/src/indexer/write_atomic.rs index a2d5b46b..161d763a 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_atomic.rs @@ -2,8 +2,13 @@ use std::collections::HashMap; use rocksdb::WriteBatch; -use super::*; +use super::{ + Arc, BREAKPOINT_INTERVAL, Block, BoundColumnFamily, DB_META_FIRST_BLOCK_IN_DB_KEY, + DB_META_FIRST_BLOCK_SET_KEY, DB_META_LAST_BLOCK_IN_DB_KEY, DB_META_LAST_BREAKPOINT_ID, + DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DbError, DbResult, RocksDBIO, +}; +#[expect(clippy::multiple_inherent_impl, reason = "Readability")] impl RocksDBIO { // Accounts meta @@ -36,14 +41,16 @@ impl RocksDBIO { pub fn put_account_transactions( &self, acc_id: [u8; 32], - tx_hashes: Vec<[u8; 32]>, + tx_hashes: &[[u8; 32]], ) -> DbResult<()> { let acc_num_tx = self.get_acc_meta_num_tx(acc_id)?.unwrap_or(0); let cf_att = self.account_id_to_tx_hash_column(); let mut write_batch = WriteBatch::new(); for (tx_id, tx_hash) in tx_hashes.iter().enumerate() { - let put_id = acc_num_tx + tx_id as u64; + let put_id = acc_num_tx + .checked_add(tx_id.try_into().expect("Must fit into u64")) + .expect("Tx count should be lesser that u64::MAX"); let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { DbError::borsh_cast_message(berr, Some("Failed to serialize account id".to_owned())) @@ -68,7 +75,9 @@ impl RocksDBIO { self.update_acc_meta_batch( acc_id, - acc_num_tx + (tx_hashes.len() as u64), + acc_num_tx + .checked_add(tx_hashes.len().try_into().expect("Must fit into u64")) + .expect("Tx count should be lesser that u64::MAX"), &mut write_batch, )?; @@ -80,14 +89,16 @@ impl RocksDBIO { pub fn put_account_transactions_dependant( &self, acc_id: [u8; 32], - tx_hashes: Vec<[u8; 32]>, + tx_hashes: &[[u8; 32]], write_batch: &mut WriteBatch, ) -> DbResult<()> { let acc_num_tx = self.get_acc_meta_num_tx(acc_id)?.unwrap_or(0); let cf_att = self.account_id_to_tx_hash_column(); for (tx_id, tx_hash) in tx_hashes.iter().enumerate() { - let put_id = acc_num_tx + tx_id as u64; + let put_id = acc_num_tx + .checked_add(tx_id.try_into().expect("Must fit into u64")) + .expect("Tx count should be lesser that u64::MAX"); let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { DbError::borsh_cast_message(berr, Some("Failed to serialize account id".to_owned())) @@ -110,7 +121,13 @@ impl RocksDBIO { ); } - self.update_acc_meta_batch(acc_id, acc_num_tx + (tx_hashes.len() as u64), write_batch)?; + self.update_acc_meta_batch( + acc_id, + acc_num_tx + .checked_add(tx_hashes.len().try_into().expect("Must fit into u64")) + .expect("Tx count should be lesser that u64::MAX"), + write_batch, + )?; Ok(()) } @@ -226,7 +243,7 @@ impl RocksDBIO { Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_owned()), ) })?, - [1u8; 1], + [1_u8; 1], ); Ok(()) } @@ -286,14 +303,14 @@ impl RocksDBIO { let acc_ids = tx .affected_public_account_ids() .into_iter() - .map(|account_id| account_id.into_value()) + .map(nssa::AccountId::into_value) .collect::>(); for acc_id in acc_ids { acc_to_tx_map .entry(acc_id) .and_modify(|tx_hashes| tx_hashes.push(tx_hash.into())) - .or_insert(vec![tx_hash.into()]); + .or_insert_with(|| vec![tx_hash.into()]); } } @@ -302,7 +319,7 @@ impl RocksDBIO { reason = "RocksDB will keep ordering persistent" )] for (acc_id, tx_hashes) in acc_to_tx_map { - self.put_account_transactions_dependant(acc_id, tx_hashes, &mut write_batch)?; + self.put_account_transactions_dependant(acc_id, &tx_hashes, &mut write_batch)?; } self.db.write(write_batch).map_err(|rerr| { diff --git a/storage/src/indexer/write_once.rs b/storage/src/indexer/write_non_atomic.rs similarity index 90% rename from storage/src/indexer/write_once.rs rename to storage/src/indexer/write_non_atomic.rs index 0edd7dfa..84fc7de5 100644 --- a/storage/src/indexer/write_once.rs +++ b/storage/src/indexer/write_non_atomic.rs @@ -1,5 +1,10 @@ -use super::*; +use super::{ + BREAKPOINT_INTERVAL, DB_META_FIRST_BLOCK_SET_KEY, DB_META_LAST_BLOCK_IN_DB_KEY, + DB_META_LAST_BREAKPOINT_ID, DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DbError, + DbResult, RocksDBIO, V02State, +}; +#[expect(clippy::multiple_inherent_impl, reason = "Readability")] impl RocksDBIO { // Meta @@ -88,7 +93,7 @@ impl RocksDBIO { Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_owned()), ) })?, - [1u8; 1], + [1_u8; 1], ) .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; Ok(()) @@ -120,7 +125,10 @@ impl RocksDBIO { pub fn put_next_breakpoint(&self) -> DbResult<()> { let last_block = self.get_meta_last_block_in_db()?; - let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; + let next_breakpoint_id = self + .get_meta_last_breakpoint_id()? + .checked_add(1) + .expect("Breakpoint Id will be lesser than u64::MAX"); let block_to_break_id = next_breakpoint_id .checked_mul(u64::from(BREAKPOINT_INTERVAL)) .expect("Reached maximum breakpoint id"); From 1eae20ba3f55ddde292a4b4de96706ba80661ec4 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Mon, 23 Mar 2026 18:16:53 +0200 Subject: [PATCH 12/13] fix: merge fixes --- storage/src/indexer/mod.rs | 20 ++++++++++---------- storage/src/indexer/read_once.rs | 6 +++--- storage/src/indexer/write_non_atomic.rs | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index bb84069a..299d3484 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -1,7 +1,7 @@ use std::{path::Path, sync::Arc}; use common::block::Block; -use nssa::V02State; +use nssa::V03State; use rocksdb::{ BoundColumnFamily, ColumnFamilyDescriptor, DBWithThreadMode, MultiThreaded, Options, }; @@ -65,7 +65,7 @@ impl RocksDBIO { pub fn open_or_create( path: &Path, genesis_block: &Block, - initial_state: &V02State, + initial_state: &V03State, ) -> DbResult { let mut cf_opts = Options::default(); cf_opts.set_max_write_buffer_number(16); @@ -160,7 +160,7 @@ impl RocksDBIO { // State - pub fn calculate_state_for_id(&self, block_id: u64) -> DbResult { + pub fn calculate_state_for_id(&self, block_id: u64) -> DbResult { let last_block = self.get_meta_last_block_in_db()?; if block_id <= last_block { @@ -205,7 +205,7 @@ impl RocksDBIO { } } - pub fn final_state(&self) -> DbResult { + pub fn final_state(&self) -> DbResult { self.calculate_state_for_id(self.get_meta_last_block_in_db()?) } } @@ -253,7 +253,7 @@ mod tests { let dbio = RocksDBIO::open_or_create( temdir_path, &genesis_block(), - &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + &nssa::V03State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), ) .unwrap(); @@ -290,7 +290,7 @@ mod tests { let dbio = RocksDBIO::open_or_create( temdir_path, &genesis_block(), - &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + &nssa::V03State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), ) .unwrap(); @@ -343,7 +343,7 @@ mod tests { let dbio = RocksDBIO::open_or_create( temdir_path, &genesis_block(), - &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + &nssa::V03State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), ) .unwrap(); @@ -416,7 +416,7 @@ mod tests { let dbio = RocksDBIO::open_or_create( temdir_path, &genesis_block(), - &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + &nssa::V03State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), ) .unwrap(); @@ -499,7 +499,7 @@ mod tests { let dbio = RocksDBIO::open_or_create( temdir_path, &genesis_block(), - &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + &nssa::V03State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), ) .unwrap(); @@ -595,7 +595,7 @@ mod tests { let dbio = RocksDBIO::open_or_create( temdir_path, &genesis_block(), - &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + &nssa::V03State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), ) .unwrap(); diff --git a/storage/src/indexer/read_once.rs b/storage/src/indexer/read_once.rs index 19aee681..74d1afe9 100644 --- a/storage/src/indexer/read_once.rs +++ b/storage/src/indexer/read_once.rs @@ -1,7 +1,7 @@ use super::{ Block, DB_META_FIRST_BLOCK_IN_DB_KEY, DB_META_FIRST_BLOCK_SET_KEY, DB_META_LAST_BLOCK_IN_DB_KEY, DB_META_LAST_BREAKPOINT_ID, - DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DbError, DbResult, RocksDBIO, V02State, + DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DbError, DbResult, RocksDBIO, V03State, }; #[expect(clippy::multiple_inherent_impl, reason = "Readability")] @@ -175,7 +175,7 @@ impl RocksDBIO { // State - pub fn get_breakpoint(&self, br_id: u64) -> DbResult { + pub fn get_breakpoint(&self, br_id: u64) -> DbResult { let cf_br = self.breakpoint_column(); let res = self .db @@ -191,7 +191,7 @@ impl RocksDBIO { .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { + Ok(borsh::from_slice::(&data).map_err(|serr| { DbError::borsh_cast_message( serr, Some("Failed to deserialize breakpoint data".to_owned()), diff --git a/storage/src/indexer/write_non_atomic.rs b/storage/src/indexer/write_non_atomic.rs index 84fc7de5..17c1be18 100644 --- a/storage/src/indexer/write_non_atomic.rs +++ b/storage/src/indexer/write_non_atomic.rs @@ -1,7 +1,7 @@ use super::{ BREAKPOINT_INTERVAL, DB_META_FIRST_BLOCK_SET_KEY, DB_META_LAST_BLOCK_IN_DB_KEY, DB_META_LAST_BREAKPOINT_ID, DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DbError, - DbResult, RocksDBIO, V02State, + DbResult, RocksDBIO, V03State, }; #[expect(clippy::multiple_inherent_impl, reason = "Readability")] @@ -101,7 +101,7 @@ impl RocksDBIO { // State - pub fn put_breakpoint(&self, br_id: u64, breakpoint: &V02State) -> DbResult<()> { + pub fn put_breakpoint(&self, br_id: u64, breakpoint: &V03State) -> DbResult<()> { let cf_br = self.breakpoint_column(); self.db From c3723776c6e7f74f05d0993ee8eae3eba52a01cf Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Mon, 23 Mar 2026 19:21:42 +0200 Subject: [PATCH 13/13] fix: deny fix --- Cargo.lock | 107 ++++++----------------------------------------------- 1 file changed, 12 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e0461fd..228ab10c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -629,9 +629,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "astral-tokio-tar" -version = "0.5.6" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec179a06c1769b1e42e1e2cbe74c7dcdb3d6383c838454d063eaac5bbb7ebbe5" +checksum = "3c23f3af104b40a3430ccb90ed5f7bd877a8dc5c26fc92fde51a22b40890dcf9" dependencies = [ "filetime", "futures-core", @@ -1939,7 +1939,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 2.0.117", + "syn 1.0.109", ] [[package]] @@ -5930,7 +5930,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -5943,7 +5943,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -6024,7 +6024,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -6919,7 +6919,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 0.26.11", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6930,9 +6930,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "ring", "rustls-pki-types", @@ -7842,9 +7842,9 @@ dependencies = [ [[package]] name = "testcontainers" -version = "0.27.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c0624faaa317c56d6d19136580be889677259caf5c897941c6f446b4655068" +checksum = "0bd36b06a2a6c0c3c81a83be1ab05fe86460d054d4d51bf513bc56b3e15bdc22" dependencies = [ "astral-tokio-tar", "async-trait", @@ -9038,24 +9038,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -9089,30 +9071,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -9125,12 +9090,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -9143,12 +9102,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -9161,24 +9114,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -9191,12 +9132,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -9209,12 +9144,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -9227,12 +9156,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -9245,12 +9168,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.7.15"