From 3426f1f8a98f3829b1a81d42cd371e521b9c2efd Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Mon, 9 Mar 2026 11:15:57 +0200 Subject: [PATCH 01/22] fix: minor restructurization --- storage/src/indexer.rs | 1325 ------------------------- storage/src/indexer/mod.rs | 605 +++++++++++ storage/src/indexer/read_multi_get.rs | 5 + storage/src/indexer/read_once.rs | 424 ++++++++ storage/src/indexer/write_batch.rs | 78 ++ storage/src/indexer/write_once.rs | 244 +++++ 6 files changed, 1356 insertions(+), 1325 deletions(-) delete mode 100644 storage/src/indexer.rs create mode 100644 storage/src/indexer/mod.rs create mode 100644 storage/src/indexer/read_multi_get.rs create mode 100644 storage/src/indexer/read_once.rs create mode 100644 storage/src/indexer/write_batch.rs create mode 100644 storage/src/indexer/write_once.rs diff --git a/storage/src/indexer.rs b/storage/src/indexer.rs deleted file mode 100644 index 2c37ab0f..00000000 --- a/storage/src/indexer.rs +++ /dev/null @@ -1,1325 +0,0 @@ -use std::{collections::HashMap, ops::Div, path::Path, sync::Arc}; - -use common::{block::Block, transaction::NSSATransaction}; -use nssa::V02State; -use rocksdb::{ - BoundColumnFamily, ColumnFamilyDescriptor, DBWithThreadMode, MultiThreaded, Options, WriteBatch, -}; - -use crate::error::DbError; - -/// Maximal size of stored blocks in base -/// -/// Used to control db size -/// -/// Currently effectively unbounded. -pub const BUFF_SIZE_ROCKSDB: usize = usize::MAX; - -/// Size of stored blocks cache in memory -/// -/// Keeping small to not run out of memory -pub const CACHE_SIZE: usize = 1000; - -/// Key base for storing metainformation about id of first block in db -pub const DB_META_FIRST_BLOCK_IN_DB_KEY: &str = "first_block_in_db"; -/// Key base for storing metainformation about id of last current block in db -pub const DB_META_LAST_BLOCK_IN_DB_KEY: &str = "last_block_in_db"; -/// Key base for storing metainformation about id of last observed L1 lib header in db -pub const DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY: &str = - "last_observed_l1_lib_header_in_db"; -/// Key base for storing metainformation which describe if first block has been set -pub const DB_META_FIRST_BLOCK_SET_KEY: &str = "first_block_set"; -/// Key base for storing metainformation about the last breakpoint -pub const DB_META_LAST_BREAKPOINT_ID: &str = "last_breakpoint_id"; - -/// Interval between state breakpoints -pub const BREAKPOINT_INTERVAL: u64 = 100; - -/// Name of block column family -pub const CF_BLOCK_NAME: &str = "cf_block"; -/// Name of meta column family -pub const CF_META_NAME: &str = "cf_meta"; -/// Name of breakpoint column family -pub const CF_BREAKPOINT_NAME: &str = "cf_breakpoint"; -/// Name of hash to id map column family -pub const CF_HASH_TO_ID: &str = "cf_hash_to_id"; -/// Name of tx hash to id map column family -pub const CF_TX_TO_ID: &str = "cf_tx_to_id"; -/// Name of account meta column family -pub const CF_ACC_META: &str = "cf_acc_meta"; -/// Name of account id to tx hash map column family -pub const CF_ACC_TO_TX: &str = "cf_acc_to_tx"; - -pub type DbResult = Result; - -fn closest_breakpoint_id(block_id: u64) -> u64 { - block_id.saturating_sub(1).div(BREAKPOINT_INTERVAL) -} - -pub struct RocksDBIO { - pub db: DBWithThreadMode, -} - -impl RocksDBIO { - pub fn open_or_create(path: &Path, start_data: Option<(Block, V02State)>) -> DbResult { - let mut cf_opts = Options::default(); - cf_opts.set_max_write_buffer_number(16); - // ToDo: Add more column families for different data - let cfb = ColumnFamilyDescriptor::new(CF_BLOCK_NAME, cf_opts.clone()); - let cfmeta = ColumnFamilyDescriptor::new(CF_META_NAME, cf_opts.clone()); - let cfbreakpoint = ColumnFamilyDescriptor::new(CF_BREAKPOINT_NAME, cf_opts.clone()); - let cfhti = ColumnFamilyDescriptor::new(CF_HASH_TO_ID, cf_opts.clone()); - let cftti = ColumnFamilyDescriptor::new(CF_TX_TO_ID, cf_opts.clone()); - let cfameta = ColumnFamilyDescriptor::new(CF_ACC_META, cf_opts.clone()); - let cfatt = ColumnFamilyDescriptor::new(CF_ACC_TO_TX, cf_opts.clone()); - - let mut db_opts = Options::default(); - db_opts.create_missing_column_families(true); - db_opts.create_if_missing(true); - let db = DBWithThreadMode::::open_cf_descriptors( - &db_opts, - path, - vec![cfb, cfmeta, cfbreakpoint, cfhti, cftti, cfameta, cfatt], - ); - - let dbio = Self { - // There is no point in handling this from runner code - db: db.unwrap(), - }; - - let is_start_set = dbio.get_meta_is_first_block_set()?; - - if is_start_set { - Ok(dbio) - } else if let Some((block, initial_state)) = start_data { - let block_id = block.header.block_id; - dbio.put_meta_last_block_in_db(block_id)?; - dbio.put_meta_first_block_in_db(block)?; - dbio.put_meta_is_first_block_set()?; - - // First breakpoint setup - dbio.put_breakpoint(0, initial_state)?; - dbio.put_meta_last_breakpoint_id(0)?; - - Ok(dbio) - } else { - // Here we are trying to start a DB without a block, one should not do it. - unreachable!() - } - } - - pub fn destroy(path: &Path) -> DbResult<()> { - let mut cf_opts = Options::default(); - cf_opts.set_max_write_buffer_number(16); - // ToDo: Add more column families for different data - let _cfb = ColumnFamilyDescriptor::new(CF_BLOCK_NAME, cf_opts.clone()); - let _cfmeta = ColumnFamilyDescriptor::new(CF_META_NAME, cf_opts.clone()); - let _cfsnapshot = ColumnFamilyDescriptor::new(CF_BREAKPOINT_NAME, cf_opts.clone()); - let _cfhti = ColumnFamilyDescriptor::new(CF_HASH_TO_ID, cf_opts.clone()); - let _cftti = ColumnFamilyDescriptor::new(CF_TX_TO_ID, cf_opts.clone()); - let _cfameta = ColumnFamilyDescriptor::new(CF_ACC_META, cf_opts.clone()); - let _cfatt = ColumnFamilyDescriptor::new(CF_ACC_TO_TX, cf_opts.clone()); - - let mut db_opts = Options::default(); - db_opts.create_missing_column_families(true); - db_opts.create_if_missing(true); - DBWithThreadMode::::destroy(&db_opts, path) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) - } - - // Columns - - pub fn meta_column(&self) -> Arc> { - self.db.cf_handle(CF_META_NAME).unwrap() - } - - pub fn block_column(&self) -> Arc> { - self.db.cf_handle(CF_BLOCK_NAME).unwrap() - } - - pub fn breakpoint_column(&self) -> Arc> { - self.db.cf_handle(CF_BREAKPOINT_NAME).unwrap() - } - - pub fn hash_to_id_column(&self) -> Arc> { - self.db.cf_handle(CF_HASH_TO_ID).unwrap() - } - - pub fn tx_hash_to_id_column(&self) -> Arc> { - self.db.cf_handle(CF_TX_TO_ID).unwrap() - } - - pub fn account_id_to_tx_hash_column(&self) -> Arc> { - self.db.cf_handle(CF_ACC_TO_TX).unwrap() - } - - pub fn account_meta_column(&self) -> Arc> { - self.db.cf_handle(CF_ACC_META).unwrap() - } - - // Meta - - pub fn get_meta_first_block_in_db(&self) -> DbResult { - let cf_meta = self.meta_column(); - let res = self - .db - .get_cf( - &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to deserialize first block".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "First block not found".to_string(), - )) - } - } - - pub fn get_meta_last_block_in_db(&self) -> DbResult { - let cf_meta = self.meta_column(); - let res = self - .db - .get_cf( - &cf_meta, - borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to deserialize last block".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "Last block not found".to_string(), - )) - } - } - - pub fn get_meta_last_observed_l1_lib_header_in_db(&self) -> DbResult> { - let cf_meta = self.meta_column(); - let res = self - .db - .get_cf( - &cf_meta, - borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err( - |err| { - DbError::borsh_cast_message( - err, - Some( - "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" - .to_string(), - ), - ) - }, - )?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - res.map(|data| { - borsh::from_slice::<[u8; 32]>(&data).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to deserialize last l1 lib header".to_string()), - ) - }) - }) - .transpose() - } - - pub fn get_meta_is_first_block_set(&self) -> DbResult { - let cf_meta = self.meta_column(); - let res = self - .db - .get_cf( - &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - Ok(res.is_some()) - } - - pub fn get_meta_last_breakpoint_id(&self) -> DbResult { - let cf_meta = self.meta_column(); - let res = self - .db - .get_cf( - &cf_meta, - borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to deserialize last breakpoint id".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "Last breakpoint id not found".to_string(), - )) - } - } - - pub fn put_meta_first_block_in_db(&self, block: Block) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize first block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - self.put_block(block, [0; 32])?; - Ok(()) - } - - pub fn put_meta_last_block_in_db(&self, block_id: u64) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), - ) - })?, - borsh::to_vec(&block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize last block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - Ok(()) - } - - pub fn put_meta_last_observed_l1_lib_header_in_db( - &self, - l1_lib_header: [u8; 32], - ) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err( - |err| { - DbError::borsh_cast_message( - err, - Some( - "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" - .to_string(), - ), - ) - }, - )?, - borsh::to_vec(&l1_lib_header).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize last l1 block header".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - Ok(()) - } - - pub fn put_meta_last_breakpoint_id(&self, br_id: u64) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), - ) - })?, - borsh::to_vec(&br_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize last block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - Ok(()) - } - - pub fn put_meta_is_first_block_set(&self) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), - ) - })?, - [1u8; 1], - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - Ok(()) - } - - // Block - - pub fn put_block(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { - let cf_block = self.block_column(); - let cf_hti = self.hash_to_id_column(); - let cf_tti: Arc> = self.tx_hash_to_id_column(); - - // ToDo: rewrite this with write batching - - self.db - .put_cf( - &cf_block, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - borsh::to_vec(&block).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block data".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let last_curr_block = self.get_meta_last_block_in_db()?; - - if block.header.block_id > last_curr_block { - self.put_meta_last_block_in_db(block.header.block_id)?; - self.put_meta_last_observed_l1_lib_header_in_db(l1_lib_header)?; - } - - self.db - .put_cf( - &cf_hti, - borsh::to_vec(&block.header.hash).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block hash".to_string()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let mut acc_to_tx_map: HashMap<[u8; 32], Vec<[u8; 32]>> = HashMap::new(); - - for tx in block.body.transactions { - let tx_hash = tx.hash(); - - self.db - .put_cf( - &cf_tti, - borsh::to_vec(&tx_hash).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize tx hash".to_string()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let acc_ids = tx - .affected_public_account_ids() - .into_iter() - .map(|account_id| account_id.into_value()) - .collect::>(); - - for acc_id in acc_ids { - acc_to_tx_map - .entry(acc_id) - .and_modify(|tx_hashes| tx_hashes.push(tx_hash.into())) - .or_insert(vec![tx_hash.into()]); - } - } - - for (acc_id, tx_hashes) in acc_to_tx_map { - self.put_account_transactions(acc_id, tx_hashes)?; - } - - if block.header.block_id.is_multiple_of(BREAKPOINT_INTERVAL) { - self.put_next_breakpoint()?; - } - - Ok(()) - } - - pub fn get_block(&self, block_id: u64) -> DbResult { - let cf_block = self.block_column(); - let res = self - .db - .get_cf( - &cf_block, - borsh::to_vec(&block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize block data".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "Block on this id not found".to_string(), - )) - } - } - - pub fn get_block_batch(&self, before: Option, limit: u64) -> DbResult> { - let cf_block = self.block_column(); - let mut block_batch = vec![]; - - // Determine the starting block ID - let start_block_id = if let Some(before_id) = before { - before_id.saturating_sub(1) - } else { - // Get the latest block ID - self.get_meta_last_block_in_db()? - }; - - // ToDo: Multi get this - - for i in 0..limit { - let block_id = start_block_id.saturating_sub(i); - if block_id == 0 { - break; - } - - let res = self - .db - .get_cf( - &cf_block, - borsh::to_vec(&block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let block = if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize block data".to_string()), - ) - })?) - } else { - // Block not found, assuming that previous one was the last - break; - }?; - - block_batch.push(block); - } - - Ok(block_batch) - } - - // State - - pub fn put_breakpoint(&self, br_id: u64, breakpoint: V02State) -> DbResult<()> { - let cf_br = self.breakpoint_column(); - - self.db - .put_cf( - &cf_br, - borsh::to_vec(&br_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize breakpoint id".to_string()), - ) - })?, - borsh::to_vec(&breakpoint).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize breakpoint data".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) - } - - pub fn get_breakpoint(&self, br_id: u64) -> DbResult { - let cf_br = self.breakpoint_column(); - let res = self - .db - .get_cf( - &cf_br, - borsh::to_vec(&br_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize breakpoint id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize breakpoint data".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "Breakpoint on this id not found".to_string(), - )) - } - } - - pub fn calculate_state_for_id(&self, block_id: u64) -> DbResult { - let last_block = self.get_meta_last_block_in_db()?; - - if block_id <= last_block { - let br_id = closest_breakpoint_id(block_id); - let mut breakpoint = self.get_breakpoint(br_id)?; - - // ToDo: update it to handle any genesis id - // right now works correctly only if genesis_id < BREAKPOINT_INTERVAL - let start = if br_id != 0 { - BREAKPOINT_INTERVAL * br_id - } else { - self.get_meta_first_block_in_db()? - }; - - for id in start..=block_id { - let block = self.get_block(id)?; - - for transaction in block.body.transactions { - transaction - .transaction_stateless_check() - .map_err(|err| { - DbError::db_interaction_error(format!( - "transaction pre check failed with err {err:?}" - )) - })? - .execute_check_on_state(&mut breakpoint) - .map_err(|err| { - DbError::db_interaction_error(format!( - "transaction execution failed with err {err:?}" - )) - })?; - } - } - - Ok(breakpoint) - } else { - Err(DbError::db_interaction_error( - "Block on this id not found".to_string(), - )) - } - } - - pub fn final_state(&self) -> DbResult { - self.calculate_state_for_id(self.get_meta_last_block_in_db()?) - } - - pub fn put_next_breakpoint(&self) -> DbResult<()> { - let last_block = self.get_meta_last_block_in_db()?; - let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; - let block_to_break_id = next_breakpoint_id * BREAKPOINT_INTERVAL; - - if block_to_break_id <= last_block { - let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; - - self.put_breakpoint(next_breakpoint_id, next_breakpoint)?; - self.put_meta_last_breakpoint_id(next_breakpoint_id) - } else { - Err(DbError::db_interaction_error( - "Breakpoint not yet achieved".to_string(), - )) - } - } - - // Mappings - - pub fn get_block_id_by_hash(&self, hash: [u8; 32]) -> DbResult { - let cf_hti = self.hash_to_id_column(); - let res = self - .db - .get_cf( - &cf_hti, - borsh::to_vec(&hash).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block hash".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize block id".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "Block on this hash not found".to_string(), - )) - } - } - - pub fn get_block_id_by_tx_hash(&self, tx_hash: [u8; 32]) -> DbResult { - let cf_tti = self.tx_hash_to_id_column(); - let res = self - .db - .get_cf( - &cf_tti, - borsh::to_vec(&tx_hash).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize transaction hash".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize block id".to_string()), - ) - })?) - } else { - Err(DbError::db_interaction_error( - "Block for this tx hash not found".to_string(), - )) - } - } - - // Accounts meta - - fn update_acc_meta_batch( - &self, - acc_id: [u8; 32], - num_tx: u64, - write_batch: &mut WriteBatch, - ) -> DbResult<()> { - let cf_ameta = self.account_meta_column(); - - write_batch.put_cf( - &cf_ameta, - borsh::to_vec(&acc_id).map_err(|err| { - DbError::borsh_cast_message(err, Some("Failed to serialize account id".to_string())) - })?, - borsh::to_vec(&num_tx).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize acc metadata".to_string()), - ) - })?, - ); - - Ok(()) - } - - fn get_acc_meta_num_tx(&self, acc_id: [u8; 32]) -> DbResult> { - let cf_ameta = self.account_meta_column(); - let res = self.db.get_cf(&cf_ameta, acc_id).map_err(|rerr| { - DbError::rocksdb_cast_message(rerr, Some("Failed to read from acc meta cf".to_string())) - })?; - - res.map(|data| { - borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message(serr, Some("Failed to deserialize num tx".to_string())) - }) - }) - .transpose() - } - - // Account - - pub fn put_account_transactions( - &self, - acc_id: [u8; 32], - tx_hashes: Vec<[u8; 32]>, - ) -> DbResult<()> { - let acc_num_tx = self.get_acc_meta_num_tx(acc_id)?.unwrap_or(0); - let cf_att = self.account_id_to_tx_hash_column(); - let mut write_batch = WriteBatch::new(); - - for (tx_id, tx_hash) in tx_hashes.iter().enumerate() { - let put_id = acc_num_tx + tx_id as u64; - - let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { - DbError::borsh_cast_message( - berr, - Some("Failed to serialize account id".to_string()), - ) - })?; - let suffix = borsh::to_vec(&put_id).map_err(|berr| { - DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) - })?; - - prefix.extend_from_slice(&suffix); - - write_batch.put_cf( - &cf_att, - prefix, - borsh::to_vec(tx_hash).map_err(|berr| { - DbError::borsh_cast_message( - berr, - Some("Failed to serialize tx hash".to_string()), - ) - })?, - ); - } - - self.update_acc_meta_batch( - acc_id, - acc_num_tx + (tx_hashes.len() as u64), - &mut write_batch, - )?; - - self.db.write(write_batch).map_err(|rerr| { - DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_string())) - }) - } - - fn get_acc_transaction_hashes( - &self, - acc_id: [u8; 32], - offset: u64, - limit: u64, - ) -> DbResult> { - let cf_att = self.account_id_to_tx_hash_column(); - let mut tx_batch = vec![]; - - // ToDo: Multi get this - - for tx_id in offset..(offset + limit) { - let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { - DbError::borsh_cast_message( - berr, - Some("Failed to serialize account id".to_string()), - ) - })?; - let suffix = borsh::to_vec(&tx_id).map_err(|berr| { - DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) - })?; - - prefix.extend_from_slice(&suffix); - - let res = self - .db - .get_cf(&cf_att, prefix) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let tx_hash = if let Some(data) = res { - Ok(borsh::from_slice::<[u8; 32]>(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize tx_hash".to_string()), - ) - })?) - } else { - // Tx hash not found, assuming that previous one was the last - break; - }?; - - tx_batch.push(tx_hash); - } - - Ok(tx_batch) - } - - pub fn get_acc_transactions( - &self, - acc_id: [u8; 32], - offset: u64, - limit: u64, - ) -> DbResult> { - let mut tx_batch = vec![]; - - for tx_hash in self.get_acc_transaction_hashes(acc_id, offset, limit)? { - let block_id = self.get_block_id_by_tx_hash(tx_hash)?; - let block = self.get_block(block_id)?; - - let transaction = block - .body - .transactions - .iter() - .find(|tx| tx.hash().0 == tx_hash) - .ok_or(DbError::db_interaction_error(format!( - "Missing transaction in block {} with hash {:#?}", - block.header.block_id, tx_hash - )))?; - - tx_batch.push(transaction.clone()); - } - - Ok(tx_batch) - } -} - -#[cfg(test)] -mod tests { - use nssa::AccountId; - use tempfile::tempdir; - - use super::*; - - fn genesis_block() -> Block { - common::test_utils::produce_dummy_block(1, None, vec![]) - } - - fn acc1() -> AccountId { - AccountId::new([ - 148, 179, 206, 253, 199, 51, 82, 86, 232, 2, 152, 122, 80, 243, 54, 207, 237, 112, 83, - 153, 44, 59, 204, 49, 128, 84, 160, 227, 216, 149, 97, 102, - ]) - } - - fn acc2() -> AccountId { - AccountId::new([ - 30, 145, 107, 3, 207, 73, 192, 230, 160, 63, 238, 207, 18, 69, 54, 216, 103, 244, 92, - 94, 124, 248, 42, 16, 141, 19, 119, 18, 14, 226, 140, 204, - ]) - } - - fn acc1_sign_key() -> nssa::PrivateKey { - nssa::PrivateKey::try_new([1; 32]).unwrap() - } - - fn acc2_sign_key() -> nssa::PrivateKey { - nssa::PrivateKey::try_new([2; 32]).unwrap() - } - - fn initial_state() -> V02State { - nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]) - } - - fn transfer(amount: u128, nonce: u128, direction: bool) -> NSSATransaction { - let from; - let to; - let sign_key; - - if direction { - from = acc1(); - to = acc2(); - sign_key = acc1_sign_key(); - } else { - from = acc2(); - to = acc1(); - sign_key = acc2_sign_key(); - } - - common::test_utils::create_transaction_native_token_transfer( - from, nonce, to, amount, sign_key, - ) - } - - #[test] - fn test_start_db() { - let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); - - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) - .unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let first_id = dbio.get_meta_first_block_in_db().unwrap(); - let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); - let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); - let last_block = dbio.get_block(1).unwrap(); - let breakpoint = dbio.get_breakpoint(0).unwrap(); - let final_state = dbio.final_state().unwrap(); - - assert_eq!(last_id, 1); - assert_eq!(first_id, 1); - assert!(is_first_set); - assert_eq!(last_br_id, 0); - assert_eq!(last_block.header.hash, genesis_block().header.hash); - assert_eq!( - breakpoint.get_account_by_id(acc1()), - final_state.get_account_by_id(acc1()) - ); - assert_eq!( - breakpoint.get_account_by_id(acc2()), - final_state.get_account_by_id(acc2()) - ); - } - - #[test] - fn test_one_block_insertion() { - let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); - - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) - .unwrap(); - - let prev_hash = genesis_block().header.hash; - let transfer_tx = transfer(1, 0, true); - let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - - dbio.put_block(block, [1; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let first_id = dbio.get_meta_first_block_in_db().unwrap(); - let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); - let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - let breakpoint = dbio.get_breakpoint(0).unwrap(); - let final_state = dbio.final_state().unwrap(); - - assert_eq!(last_id, 2); - assert_eq!(first_id, 1); - assert!(is_first_set); - assert_eq!(last_br_id, 0); - assert_ne!(last_block.header.hash, genesis_block().header.hash); - assert_eq!( - breakpoint.get_account_by_id(acc1()).balance - - final_state.get_account_by_id(acc1()).balance, - 1 - ); - assert_eq!( - final_state.get_account_by_id(acc2()).balance - - breakpoint.get_account_by_id(acc2()).balance, - 1 - ); - } - - #[test] - fn test_new_breakpoint() { - let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); - - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) - .unwrap(); - - for i in 1..BREAKPOINT_INTERVAL { - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, (i - 1) as u128, true); - let block = - common::test_utils::produce_dummy_block(i + 1, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [i as u8; 32]).unwrap(); - } - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let first_id = dbio.get_meta_first_block_in_db().unwrap(); - let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); - let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - let prev_breakpoint = dbio.get_breakpoint(0).unwrap(); - let breakpoint = dbio.get_breakpoint(1).unwrap(); - let final_state = dbio.final_state().unwrap(); - - assert_eq!(last_id, 100); - assert_eq!(first_id, 1); - assert!(is_first_set); - assert_eq!(last_br_id, 1); - assert_ne!(last_block.header.hash, genesis_block().header.hash); - assert_eq!( - prev_breakpoint.get_account_by_id(acc1()).balance - - final_state.get_account_by_id(acc1()).balance, - 99 - ); - assert_eq!( - final_state.get_account_by_id(acc2()).balance - - prev_breakpoint.get_account_by_id(acc2()).balance, - 99 - ); - assert_eq!( - breakpoint.get_account_by_id(acc1()), - final_state.get_account_by_id(acc1()) - ); - assert_eq!( - breakpoint.get_account_by_id(acc2()), - final_state.get_account_by_id(acc2()) - ); - } - - #[test] - fn test_simple_maps() { - let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); - - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) - .unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 0, true); - let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - - let control_hash1 = block.header.hash; - - dbio.put_block(block, [1; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 1, true); - let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); - - let control_hash2 = block.header.hash; - - dbio.put_block(block, [2; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 2, true); - - let control_tx_hash1 = transfer_tx.hash(); - - let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [3; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 3, true); - - let control_tx_hash2 = transfer_tx.hash(); - - let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [4; 32]).unwrap(); - - let control_block_id1 = dbio.get_block_id_by_hash(control_hash1.0).unwrap(); - let control_block_id2 = dbio.get_block_id_by_hash(control_hash2.0).unwrap(); - let control_block_id3 = dbio.get_block_id_by_tx_hash(control_tx_hash1.0).unwrap(); - let control_block_id4 = dbio.get_block_id_by_tx_hash(control_tx_hash2.0).unwrap(); - - assert_eq!(control_block_id1, 2); - assert_eq!(control_block_id2, 3); - assert_eq!(control_block_id3, 4); - assert_eq!(control_block_id4, 5); - } - - #[test] - fn test_block_batch() { - let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); - - let mut block_res = vec![]; - - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) - .unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 0, true); - let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - - block_res.push(block.clone()); - dbio.put_block(block, [1; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 1, true); - let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); - - block_res.push(block.clone()); - dbio.put_block(block, [2; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 2, true); - - let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - block_res.push(block.clone()); - dbio.put_block(block, [3; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 3, true); - - let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - block_res.push(block.clone()); - dbio.put_block(block, [4; 32]).unwrap(); - - let block_hashes_mem: Vec<[u8; 32]> = - block_res.into_iter().map(|bl| bl.header.hash.0).collect(); - - // Get blocks before ID 6 (i.e., starting from 5 going backwards), limit 4 - // This should return blocks 5, 4, 3, 2 in descending order - let mut batch_res = dbio.get_block_batch(Some(6), 4).unwrap(); - batch_res.reverse(); // Reverse to match ascending order for comparison - - let block_hashes_db: Vec<[u8; 32]> = - batch_res.into_iter().map(|bl| bl.header.hash.0).collect(); - - assert_eq!(block_hashes_mem, block_hashes_db); - - let block_hashes_mem_limited = &block_hashes_mem[1..]; - - // Get blocks before ID 6, limit 3 - // This should return blocks 5, 4, 3 in descending order - let mut batch_res_limited = dbio.get_block_batch(Some(6), 3).unwrap(); - batch_res_limited.reverse(); // Reverse to match ascending order for comparison - - let block_hashes_db_limited: Vec<[u8; 32]> = batch_res_limited - .into_iter() - .map(|bl| bl.header.hash.0) - .collect(); - - assert_eq!(block_hashes_mem_limited, block_hashes_db_limited.as_slice()); - } - - #[test] - fn test_account_map() { - let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); - - let mut tx_hash_res = vec![]; - - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) - .unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 0, true); - - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - - dbio.put_block(block, [1; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 1, true); - - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); - - dbio.put_block(block, [2; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 2, true); - - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - - dbio.put_block(block, [3; 32]).unwrap(); - - let last_id = dbio.get_meta_last_block_in_db().unwrap(); - let last_block = dbio.get_block(last_id).unwrap(); - - let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 3, true); - - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - - dbio.put_block(block, [4; 32]).unwrap(); - - let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 4).unwrap(); - let acc1_tx_hashes: Vec<[u8; 32]> = acc1_tx.into_iter().map(|tx| tx.hash().0).collect(); - - assert_eq!(acc1_tx_hashes, tx_hash_res); - - let acc1_tx_limited = dbio.get_acc_transactions(*acc1().value(), 1, 4).unwrap(); - let acc1_tx_limited_hashes: Vec<[u8; 32]> = - acc1_tx_limited.into_iter().map(|tx| tx.hash().0).collect(); - - assert_eq!(acc1_tx_limited_hashes.as_slice(), &tx_hash_res[1..]) - } -} diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs new file mode 100644 index 00000000..311746e6 --- /dev/null +++ b/storage/src/indexer/mod.rs @@ -0,0 +1,605 @@ +use std::{collections::HashMap, ops::Div, path::Path, sync::Arc}; + +use common::{block::Block, transaction::NSSATransaction}; +use nssa::V02State; +use rocksdb::{ + BoundColumnFamily, ColumnFamilyDescriptor, DBWithThreadMode, MultiThreaded, Options, WriteBatch, +}; + +use crate::error::DbError; + +pub mod read_once; +pub mod write_once; +pub mod read_multi_get; +pub mod write_batch; + +/// Maximal size of stored blocks in base +/// +/// Used to control db size +/// +/// Currently effectively unbounded. +pub const BUFF_SIZE_ROCKSDB: usize = usize::MAX; + +/// Size of stored blocks cache in memory +/// +/// Keeping small to not run out of memory +pub const CACHE_SIZE: usize = 1000; + +/// Key base for storing metainformation about id of first block in db +pub const DB_META_FIRST_BLOCK_IN_DB_KEY: &str = "first_block_in_db"; +/// Key base for storing metainformation about id of last current block in db +pub const DB_META_LAST_BLOCK_IN_DB_KEY: &str = "last_block_in_db"; +/// Key base for storing metainformation about id of last observed L1 lib header in db +pub const DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY: &str = + "last_observed_l1_lib_header_in_db"; +/// Key base for storing metainformation which describe if first block has been set +pub const DB_META_FIRST_BLOCK_SET_KEY: &str = "first_block_set"; +/// Key base for storing metainformation about the last breakpoint +pub const DB_META_LAST_BREAKPOINT_ID: &str = "last_breakpoint_id"; + +/// Interval between state breakpoints +pub const BREAKPOINT_INTERVAL: u64 = 100; + +/// Name of block column family +pub const CF_BLOCK_NAME: &str = "cf_block"; +/// Name of meta column family +pub const CF_META_NAME: &str = "cf_meta"; +/// Name of breakpoint column family +pub const CF_BREAKPOINT_NAME: &str = "cf_breakpoint"; +/// Name of hash to id map column family +pub const CF_HASH_TO_ID: &str = "cf_hash_to_id"; +/// Name of tx hash to id map column family +pub const CF_TX_TO_ID: &str = "cf_tx_to_id"; +/// Name of account meta column family +pub const CF_ACC_META: &str = "cf_acc_meta"; +/// Name of account id to tx hash map column family +pub const CF_ACC_TO_TX: &str = "cf_acc_to_tx"; + +pub type DbResult = Result; + +fn closest_breakpoint_id(block_id: u64) -> u64 { + block_id.saturating_sub(1).div(BREAKPOINT_INTERVAL) +} + +pub struct RocksDBIO { + pub db: DBWithThreadMode, +} + +impl RocksDBIO { + pub fn open_or_create(path: &Path, start_data: Option<(Block, V02State)>) -> DbResult { + let mut cf_opts = Options::default(); + cf_opts.set_max_write_buffer_number(16); + // ToDo: Add more column families for different data + let cfb = ColumnFamilyDescriptor::new(CF_BLOCK_NAME, cf_opts.clone()); + let cfmeta = ColumnFamilyDescriptor::new(CF_META_NAME, cf_opts.clone()); + let cfbreakpoint = ColumnFamilyDescriptor::new(CF_BREAKPOINT_NAME, cf_opts.clone()); + let cfhti = ColumnFamilyDescriptor::new(CF_HASH_TO_ID, cf_opts.clone()); + let cftti = ColumnFamilyDescriptor::new(CF_TX_TO_ID, cf_opts.clone()); + let cfameta = ColumnFamilyDescriptor::new(CF_ACC_META, cf_opts.clone()); + let cfatt = ColumnFamilyDescriptor::new(CF_ACC_TO_TX, cf_opts.clone()); + + let mut db_opts = Options::default(); + db_opts.create_missing_column_families(true); + db_opts.create_if_missing(true); + let db = DBWithThreadMode::::open_cf_descriptors( + &db_opts, + path, + vec![cfb, cfmeta, cfbreakpoint, cfhti, cftti, cfameta, cfatt], + ); + + let dbio = Self { + // There is no point in handling this from runner code + db: db.unwrap(), + }; + + let is_start_set = dbio.get_meta_is_first_block_set()?; + + if is_start_set { + Ok(dbio) + } else if let Some((block, initial_state)) = start_data { + let block_id = block.header.block_id; + dbio.put_meta_last_block_in_db(block_id)?; + dbio.put_meta_first_block_in_db(block)?; + dbio.put_meta_is_first_block_set()?; + + // First breakpoint setup + dbio.put_breakpoint(0, initial_state)?; + dbio.put_meta_last_breakpoint_id(0)?; + + Ok(dbio) + } else { + // Here we are trying to start a DB without a block, one should not do it. + unreachable!() + } + } + + pub fn destroy(path: &Path) -> DbResult<()> { + let mut cf_opts = Options::default(); + cf_opts.set_max_write_buffer_number(16); + // ToDo: Add more column families for different data + let _cfb = ColumnFamilyDescriptor::new(CF_BLOCK_NAME, cf_opts.clone()); + let _cfmeta = ColumnFamilyDescriptor::new(CF_META_NAME, cf_opts.clone()); + let _cfsnapshot = ColumnFamilyDescriptor::new(CF_BREAKPOINT_NAME, cf_opts.clone()); + let _cfhti = ColumnFamilyDescriptor::new(CF_HASH_TO_ID, cf_opts.clone()); + let _cftti = ColumnFamilyDescriptor::new(CF_TX_TO_ID, cf_opts.clone()); + let _cfameta = ColumnFamilyDescriptor::new(CF_ACC_META, cf_opts.clone()); + let _cfatt = ColumnFamilyDescriptor::new(CF_ACC_TO_TX, cf_opts.clone()); + + let mut db_opts = Options::default(); + db_opts.create_missing_column_families(true); + db_opts.create_if_missing(true); + DBWithThreadMode::::destroy(&db_opts, path) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) + } + + // Columns + + pub fn meta_column(&self) -> Arc> { + self.db.cf_handle(CF_META_NAME).unwrap() + } + + pub fn block_column(&self) -> Arc> { + self.db.cf_handle(CF_BLOCK_NAME).unwrap() + } + + pub fn breakpoint_column(&self) -> Arc> { + self.db.cf_handle(CF_BREAKPOINT_NAME).unwrap() + } + + pub fn hash_to_id_column(&self) -> Arc> { + self.db.cf_handle(CF_HASH_TO_ID).unwrap() + } + + pub fn tx_hash_to_id_column(&self) -> Arc> { + self.db.cf_handle(CF_TX_TO_ID).unwrap() + } + + pub fn account_id_to_tx_hash_column(&self) -> Arc> { + self.db.cf_handle(CF_ACC_TO_TX).unwrap() + } + + pub fn account_meta_column(&self) -> Arc> { + self.db.cf_handle(CF_ACC_META).unwrap() + } + + //State + + pub fn calculate_state_for_id(&self, block_id: u64) -> DbResult { + let last_block = self.get_meta_last_block_in_db()?; + + if block_id <= last_block { + let br_id = closest_breakpoint_id(block_id); + let mut breakpoint = self.get_breakpoint(br_id)?; + + // ToDo: update it to handle any genesis id + // right now works correctly only if genesis_id < BREAKPOINT_INTERVAL + let start = if br_id != 0 { + BREAKPOINT_INTERVAL * br_id + } else { + self.get_meta_first_block_in_db()? + }; + + for id in start..=block_id { + let block = self.get_block(id)?; + + for transaction in block.body.transactions { + transaction + .transaction_stateless_check() + .map_err(|err| { + DbError::db_interaction_error(format!( + "transaction pre check failed with err {err:?}" + )) + })? + .execute_check_on_state(&mut breakpoint) + .map_err(|err| { + DbError::db_interaction_error(format!( + "transaction execution failed with err {err:?}" + )) + })?; + } + } + + Ok(breakpoint) + } else { + Err(DbError::db_interaction_error( + "Block on this id not found".to_string(), + )) + } + } + + pub fn final_state(&self) -> DbResult { + self.calculate_state_for_id(self.get_meta_last_block_in_db()?) + } +} + +#[cfg(test)] +mod tests { + use nssa::AccountId; + use tempfile::tempdir; + + use super::*; + + fn genesis_block() -> Block { + common::test_utils::produce_dummy_block(1, None, vec![]) + } + + fn acc1() -> AccountId { + AccountId::new([ + 148, 179, 206, 253, 199, 51, 82, 86, 232, 2, 152, 122, 80, 243, 54, 207, 237, 112, 83, + 153, 44, 59, 204, 49, 128, 84, 160, 227, 216, 149, 97, 102, + ]) + } + + fn acc2() -> AccountId { + AccountId::new([ + 30, 145, 107, 3, 207, 73, 192, 230, 160, 63, 238, 207, 18, 69, 54, 216, 103, 244, 92, + 94, 124, 248, 42, 16, 141, 19, 119, 18, 14, 226, 140, 204, + ]) + } + + fn acc1_sign_key() -> nssa::PrivateKey { + nssa::PrivateKey::try_new([1; 32]).unwrap() + } + + fn acc2_sign_key() -> nssa::PrivateKey { + nssa::PrivateKey::try_new([2; 32]).unwrap() + } + + fn initial_state() -> V02State { + nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]) + } + + fn transfer(amount: u128, nonce: u128, direction: bool) -> NSSATransaction { + let from; + let to; + let sign_key; + + if direction { + from = acc1(); + to = acc2(); + sign_key = acc1_sign_key(); + } else { + from = acc2(); + to = acc1(); + sign_key = acc2_sign_key(); + } + + common::test_utils::create_transaction_native_token_transfer( + from, nonce, to, amount, sign_key, + ) + } + + #[test] + fn test_start_db() { + let temp_dir = tempdir().unwrap(); + let temdir_path = temp_dir.path(); + + let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) + .unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let first_id = dbio.get_meta_first_block_in_db().unwrap(); + let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); + let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); + let last_block = dbio.get_block(1).unwrap(); + let breakpoint = dbio.get_breakpoint(0).unwrap(); + let final_state = dbio.final_state().unwrap(); + + assert_eq!(last_id, 1); + assert_eq!(first_id, 1); + assert!(is_first_set); + assert_eq!(last_br_id, 0); + assert_eq!(last_block.header.hash, genesis_block().header.hash); + assert_eq!( + breakpoint.get_account_by_id(acc1()), + final_state.get_account_by_id(acc1()) + ); + assert_eq!( + breakpoint.get_account_by_id(acc2()), + final_state.get_account_by_id(acc2()) + ); + } + + #[test] + fn test_one_block_insertion() { + let temp_dir = tempdir().unwrap(); + let temdir_path = temp_dir.path(); + + let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) + .unwrap(); + + let prev_hash = genesis_block().header.hash; + let transfer_tx = transfer(1, 0, true); + let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); + + dbio.put_block(block, [1; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let first_id = dbio.get_meta_first_block_in_db().unwrap(); + let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); + let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + let breakpoint = dbio.get_breakpoint(0).unwrap(); + let final_state = dbio.final_state().unwrap(); + + assert_eq!(last_id, 2); + assert_eq!(first_id, 1); + assert!(is_first_set); + assert_eq!(last_br_id, 0); + assert_ne!(last_block.header.hash, genesis_block().header.hash); + assert_eq!( + breakpoint.get_account_by_id(acc1()).balance + - final_state.get_account_by_id(acc1()).balance, + 1 + ); + assert_eq!( + final_state.get_account_by_id(acc2()).balance + - breakpoint.get_account_by_id(acc2()).balance, + 1 + ); + } + + #[test] + fn test_new_breakpoint() { + let temp_dir = tempdir().unwrap(); + let temdir_path = temp_dir.path(); + + let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) + .unwrap(); + + for i in 1..BREAKPOINT_INTERVAL { + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, (i - 1) as u128, true); + let block = + common::test_utils::produce_dummy_block(i + 1, Some(prev_hash), vec![transfer_tx]); + dbio.put_block(block, [i as u8; 32]).unwrap(); + } + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let first_id = dbio.get_meta_first_block_in_db().unwrap(); + let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); + let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + let prev_breakpoint = dbio.get_breakpoint(0).unwrap(); + let breakpoint = dbio.get_breakpoint(1).unwrap(); + let final_state = dbio.final_state().unwrap(); + + assert_eq!(last_id, 100); + assert_eq!(first_id, 1); + assert!(is_first_set); + assert_eq!(last_br_id, 1); + assert_ne!(last_block.header.hash, genesis_block().header.hash); + assert_eq!( + prev_breakpoint.get_account_by_id(acc1()).balance + - final_state.get_account_by_id(acc1()).balance, + 99 + ); + assert_eq!( + final_state.get_account_by_id(acc2()).balance + - prev_breakpoint.get_account_by_id(acc2()).balance, + 99 + ); + assert_eq!( + breakpoint.get_account_by_id(acc1()), + final_state.get_account_by_id(acc1()) + ); + assert_eq!( + breakpoint.get_account_by_id(acc2()), + final_state.get_account_by_id(acc2()) + ); + } + + #[test] + fn test_simple_maps() { + let temp_dir = tempdir().unwrap(); + let temdir_path = temp_dir.path(); + + let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) + .unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 0, true); + let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); + + let control_hash1 = block.header.hash; + + dbio.put_block(block, [1; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 1, true); + let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); + + let control_hash2 = block.header.hash; + + dbio.put_block(block, [2; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 2, true); + + let control_tx_hash1 = transfer_tx.hash(); + + let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); + dbio.put_block(block, [3; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 3, true); + + let control_tx_hash2 = transfer_tx.hash(); + + let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); + dbio.put_block(block, [4; 32]).unwrap(); + + let control_block_id1 = dbio.get_block_id_by_hash(control_hash1.0).unwrap(); + let control_block_id2 = dbio.get_block_id_by_hash(control_hash2.0).unwrap(); + let control_block_id3 = dbio.get_block_id_by_tx_hash(control_tx_hash1.0).unwrap(); + let control_block_id4 = dbio.get_block_id_by_tx_hash(control_tx_hash2.0).unwrap(); + + assert_eq!(control_block_id1, 2); + assert_eq!(control_block_id2, 3); + assert_eq!(control_block_id3, 4); + assert_eq!(control_block_id4, 5); + } + + #[test] + fn test_block_batch() { + let temp_dir = tempdir().unwrap(); + let temdir_path = temp_dir.path(); + + let mut block_res = vec![]; + + let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) + .unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 0, true); + let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); + + block_res.push(block.clone()); + dbio.put_block(block, [1; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 1, true); + let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); + + block_res.push(block.clone()); + dbio.put_block(block, [2; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 2, true); + + let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); + block_res.push(block.clone()); + dbio.put_block(block, [3; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 3, true); + + let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); + block_res.push(block.clone()); + dbio.put_block(block, [4; 32]).unwrap(); + + let block_hashes_mem: Vec<[u8; 32]> = + block_res.into_iter().map(|bl| bl.header.hash.0).collect(); + + // Get blocks before ID 6 (i.e., starting from 5 going backwards), limit 4 + // This should return blocks 5, 4, 3, 2 in descending order + let mut batch_res = dbio.get_block_batch(Some(6), 4).unwrap(); + batch_res.reverse(); // Reverse to match ascending order for comparison + + let block_hashes_db: Vec<[u8; 32]> = + batch_res.into_iter().map(|bl| bl.header.hash.0).collect(); + + assert_eq!(block_hashes_mem, block_hashes_db); + + let block_hashes_mem_limited = &block_hashes_mem[1..]; + + // Get blocks before ID 6, limit 3 + // This should return blocks 5, 4, 3 in descending order + let mut batch_res_limited = dbio.get_block_batch(Some(6), 3).unwrap(); + batch_res_limited.reverse(); // Reverse to match ascending order for comparison + + let block_hashes_db_limited: Vec<[u8; 32]> = batch_res_limited + .into_iter() + .map(|bl| bl.header.hash.0) + .collect(); + + assert_eq!(block_hashes_mem_limited, block_hashes_db_limited.as_slice()); + } + + #[test] + fn test_account_map() { + let temp_dir = tempdir().unwrap(); + let temdir_path = temp_dir.path(); + + let mut tx_hash_res = vec![]; + + let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) + .unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 0, true); + + tx_hash_res.push(transfer_tx.hash().0); + + let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); + + dbio.put_block(block, [1; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 1, true); + + tx_hash_res.push(transfer_tx.hash().0); + + let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); + + dbio.put_block(block, [2; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 2, true); + + tx_hash_res.push(transfer_tx.hash().0); + + let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); + + dbio.put_block(block, [3; 32]).unwrap(); + + let last_id = dbio.get_meta_last_block_in_db().unwrap(); + let last_block = dbio.get_block(last_id).unwrap(); + + let prev_hash = last_block.header.hash; + let transfer_tx = transfer(1, 3, true); + + tx_hash_res.push(transfer_tx.hash().0); + + let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); + + dbio.put_block(block, [4; 32]).unwrap(); + + let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 4).unwrap(); + let acc1_tx_hashes: Vec<[u8; 32]> = acc1_tx.into_iter().map(|tx| tx.hash().0).collect(); + + assert_eq!(acc1_tx_hashes, tx_hash_res); + + let acc1_tx_limited = dbio.get_acc_transactions(*acc1().value(), 1, 4).unwrap(); + let acc1_tx_limited_hashes: Vec<[u8; 32]> = + acc1_tx_limited.into_iter().map(|tx| tx.hash().0).collect(); + + assert_eq!(acc1_tx_limited_hashes.as_slice(), &tx_hash_res[1..]) + } +} diff --git a/storage/src/indexer/read_multi_get.rs b/storage/src/indexer/read_multi_get.rs new file mode 100644 index 00000000..15124682 --- /dev/null +++ b/storage/src/indexer/read_multi_get.rs @@ -0,0 +1,5 @@ +use super::*; + +impl RocksDBIO { + +} \ No newline at end of file diff --git a/storage/src/indexer/read_once.rs b/storage/src/indexer/read_once.rs new file mode 100644 index 00000000..1091dbc1 --- /dev/null +++ b/storage/src/indexer/read_once.rs @@ -0,0 +1,424 @@ +use super::*; + +impl RocksDBIO { + //Meta + + pub fn get_meta_first_block_in_db(&self) -> DbResult { + let cf_meta = self.meta_column(); + let res = self + .db + .get_cf( + &cf_meta, + borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to deserialize first block".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "First block not found".to_string(), + )) + } + } + + pub fn get_meta_last_block_in_db(&self) -> DbResult { + let cf_meta = self.meta_column(); + let res = self + .db + .get_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to deserialize last block".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "Last block not found".to_string(), + )) + } + } + + pub fn get_meta_last_observed_l1_lib_header_in_db(&self) -> DbResult> { + let cf_meta = self.meta_column(); + let res = self + .db + .get_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err( + |err| { + DbError::borsh_cast_message( + err, + Some( + "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" + .to_string(), + ), + ) + }, + )?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + res.map(|data| { + borsh::from_slice::<[u8; 32]>(&data).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to deserialize last l1 lib header".to_string()), + ) + }) + }) + .transpose() + } + + pub fn get_meta_is_first_block_set(&self) -> DbResult { + let cf_meta = self.meta_column(); + let res = self + .db + .get_cf( + &cf_meta, + borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + Ok(res.is_some()) + } + + pub fn get_meta_last_breakpoint_id(&self) -> DbResult { + let cf_meta = self.meta_column(); + let res = self + .db + .get_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to deserialize last breakpoint id".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "Last breakpoint id not found".to_string(), + )) + } + } + + //Block + + pub fn get_block(&self, block_id: u64) -> DbResult { + let cf_block = self.block_column(); + let res = self + .db + .get_cf( + &cf_block, + borsh::to_vec(&block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize block data".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "Block on this id not found".to_string(), + )) + } + } + + pub fn get_block_batch(&self, before: Option, limit: u64) -> DbResult> { + let cf_block = self.block_column(); + let mut block_batch = vec![]; + + // Determine the starting block ID + let start_block_id = if let Some(before_id) = before { + before_id.saturating_sub(1) + } else { + // Get the latest block ID + self.get_meta_last_block_in_db()? + }; + + // ToDo: Multi get this + + for i in 0..limit { + let block_id = start_block_id.saturating_sub(i); + if block_id == 0 { + break; + } + + let res = self + .db + .get_cf( + &cf_block, + borsh::to_vec(&block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let block = if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize block data".to_string()), + ) + })?) + } else { + // Block not found, assuming that previous one was the last + break; + }?; + + block_batch.push(block); + } + + Ok(block_batch) + } + + //State + + pub fn get_breakpoint(&self, br_id: u64) -> DbResult { + let cf_br = self.breakpoint_column(); + let res = self + .db + .get_cf( + &cf_br, + borsh::to_vec(&br_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize breakpoint id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize breakpoint data".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "Breakpoint on this id not found".to_string(), + )) + } + } + + pub fn put_next_breakpoint(&self) -> DbResult<()> { + let last_block = self.get_meta_last_block_in_db()?; + let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; + let block_to_break_id = next_breakpoint_id * BREAKPOINT_INTERVAL; + + if block_to_break_id <= last_block { + let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; + + self.put_breakpoint(next_breakpoint_id, next_breakpoint)?; + self.put_meta_last_breakpoint_id(next_breakpoint_id) + } else { + Err(DbError::db_interaction_error( + "Breakpoint not yet achieved".to_string(), + )) + } + } + + // Mappings + + pub fn get_block_id_by_hash(&self, hash: [u8; 32]) -> DbResult { + let cf_hti = self.hash_to_id_column(); + let res = self + .db + .get_cf( + &cf_hti, + borsh::to_vec(&hash).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block hash".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize block id".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "Block on this hash not found".to_string(), + )) + } + } + + pub fn get_block_id_by_tx_hash(&self, tx_hash: [u8; 32]) -> DbResult { + let cf_tti = self.tx_hash_to_id_column(); + let res = self + .db + .get_cf( + &cf_tti, + borsh::to_vec(&tx_hash).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize transaction hash".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize block id".to_string()), + ) + })?) + } else { + Err(DbError::db_interaction_error( + "Block for this tx hash not found".to_string(), + )) + } + } + + // Accounts meta + + pub(crate) fn get_acc_meta_num_tx(&self, acc_id: [u8; 32]) -> DbResult> { + let cf_ameta = self.account_meta_column(); + let res = self.db.get_cf(&cf_ameta, acc_id).map_err(|rerr| { + DbError::rocksdb_cast_message(rerr, Some("Failed to read from acc meta cf".to_string())) + })?; + + res.map(|data| { + borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message(serr, Some("Failed to deserialize num tx".to_string())) + }) + }) + .transpose() + } + + // Account + + fn get_acc_transaction_hashes( + &self, + acc_id: [u8; 32], + offset: u64, + limit: u64, + ) -> DbResult> { + let cf_att = self.account_id_to_tx_hash_column(); + let mut tx_batch = vec![]; + + // ToDo: Multi get this + + for tx_id in offset..(offset + limit) { + let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { + DbError::borsh_cast_message( + berr, + Some("Failed to serialize account id".to_string()), + ) + })?; + let suffix = borsh::to_vec(&tx_id).map_err(|berr| { + DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) + })?; + + prefix.extend_from_slice(&suffix); + + let res = self + .db + .get_cf(&cf_att, prefix) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let tx_hash = if let Some(data) = res { + Ok(borsh::from_slice::<[u8; 32]>(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize tx_hash".to_string()), + ) + })?) + } else { + // Tx hash not found, assuming that previous one was the last + break; + }?; + + tx_batch.push(tx_hash); + } + + Ok(tx_batch) + } + + pub fn get_acc_transactions( + &self, + acc_id: [u8; 32], + offset: u64, + limit: u64, + ) -> DbResult> { + let mut tx_batch = vec![]; + + for tx_hash in self.get_acc_transaction_hashes(acc_id, offset, limit)? { + let block_id = self.get_block_id_by_tx_hash(tx_hash)?; + let block = self.get_block(block_id)?; + + let transaction = block + .body + .transactions + .iter() + .find(|tx| tx.hash().0 == tx_hash) + .ok_or(DbError::db_interaction_error(format!( + "Missing transaction in block {} with hash {:#?}", + block.header.block_id, tx_hash + )))?; + + tx_batch.push(transaction.clone()); + } + + Ok(tx_batch) + } +} \ No newline at end of file diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs new file mode 100644 index 00000000..2bb11a85 --- /dev/null +++ b/storage/src/indexer/write_batch.rs @@ -0,0 +1,78 @@ +use super::*; + +impl RocksDBIO { + // Accounts meta + + pub(crate) fn update_acc_meta_batch( + &self, + acc_id: [u8; 32], + num_tx: u64, + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + let cf_ameta = self.account_meta_column(); + + write_batch.put_cf( + &cf_ameta, + borsh::to_vec(&acc_id).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize account id".to_string())) + })?, + borsh::to_vec(&num_tx).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize acc metadata".to_string()), + ) + })?, + ); + + Ok(()) + } + + // Account + + pub fn put_account_transactions( + &self, + acc_id: [u8; 32], + tx_hashes: Vec<[u8; 32]>, + ) -> DbResult<()> { + let acc_num_tx = self.get_acc_meta_num_tx(acc_id)?.unwrap_or(0); + let cf_att = self.account_id_to_tx_hash_column(); + let mut write_batch = WriteBatch::new(); + + for (tx_id, tx_hash) in tx_hashes.iter().enumerate() { + let put_id = acc_num_tx + tx_id as u64; + + let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { + DbError::borsh_cast_message( + berr, + Some("Failed to serialize account id".to_string()), + ) + })?; + let suffix = borsh::to_vec(&put_id).map_err(|berr| { + DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) + })?; + + prefix.extend_from_slice(&suffix); + + write_batch.put_cf( + &cf_att, + prefix, + borsh::to_vec(tx_hash).map_err(|berr| { + DbError::borsh_cast_message( + berr, + Some("Failed to serialize tx hash".to_string()), + ) + })?, + ); + } + + self.update_acc_meta_batch( + acc_id, + acc_num_tx + (tx_hashes.len() as u64), + &mut write_batch, + )?; + + self.db.write(write_batch).map_err(|rerr| { + DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_string())) + }) + } +} \ No newline at end of file diff --git a/storage/src/indexer/write_once.rs b/storage/src/indexer/write_once.rs new file mode 100644 index 00000000..71946f63 --- /dev/null +++ b/storage/src/indexer/write_once.rs @@ -0,0 +1,244 @@ +use super::*; + +impl RocksDBIO { + //Meta + + pub fn put_meta_first_block_in_db(&self, block: Block) -> DbResult<()> { + let cf_meta = self.meta_column(); + self.db + .put_cf( + &cf_meta, + borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize first block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + self.put_block(block, [0; 32])?; + Ok(()) + } + + pub fn put_meta_last_block_in_db(&self, block_id: u64) -> DbResult<()> { + let cf_meta = self.meta_column(); + self.db + .put_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + borsh::to_vec(&block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize last block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + Ok(()) + } + + pub fn put_meta_last_observed_l1_lib_header_in_db( + &self, + l1_lib_header: [u8; 32], + ) -> DbResult<()> { + let cf_meta = self.meta_column(); + self.db + .put_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err( + |err| { + DbError::borsh_cast_message( + err, + Some( + "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" + .to_string(), + ), + ) + }, + )?, + borsh::to_vec(&l1_lib_header).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize last l1 block header".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + Ok(()) + } + + pub fn put_meta_last_breakpoint_id(&self, br_id: u64) -> DbResult<()> { + let cf_meta = self.meta_column(); + self.db + .put_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), + ) + })?, + borsh::to_vec(&br_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize last block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + Ok(()) + } + + pub fn put_meta_is_first_block_set(&self) -> DbResult<()> { + let cf_meta = self.meta_column(); + self.db + .put_cf( + &cf_meta, + borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), + ) + })?, + [1u8; 1], + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + Ok(()) + } + + //Block + + pub fn put_block(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { + let cf_block = self.block_column(); + let cf_hti = self.hash_to_id_column(); + let cf_tti: Arc> = self.tx_hash_to_id_column(); + + // ToDo: rewrite this with write batching + + self.db + .put_cf( + &cf_block, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + borsh::to_vec(&block).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block data".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let last_curr_block = self.get_meta_last_block_in_db()?; + + if block.header.block_id > last_curr_block { + self.put_meta_last_block_in_db(block.header.block_id)?; + self.put_meta_last_observed_l1_lib_header_in_db(l1_lib_header)?; + } + + self.db + .put_cf( + &cf_hti, + borsh::to_vec(&block.header.hash).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block hash".to_string()), + ) + })?, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let mut acc_to_tx_map: HashMap<[u8; 32], Vec<[u8; 32]>> = HashMap::new(); + + for tx in block.body.transactions { + let tx_hash = tx.hash(); + + self.db + .put_cf( + &cf_tti, + borsh::to_vec(&tx_hash).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize tx hash".to_string()), + ) + })?, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let acc_ids = tx + .affected_public_account_ids() + .into_iter() + .map(|account_id| account_id.into_value()) + .collect::>(); + + for acc_id in acc_ids { + acc_to_tx_map + .entry(acc_id) + .and_modify(|tx_hashes| tx_hashes.push(tx_hash.into())) + .or_insert(vec![tx_hash.into()]); + } + } + + for (acc_id, tx_hashes) in acc_to_tx_map { + self.put_account_transactions(acc_id, tx_hashes)?; + } + + if block.header.block_id.is_multiple_of(BREAKPOINT_INTERVAL) { + self.put_next_breakpoint()?; + } + + Ok(()) + } + + //State + + pub fn put_breakpoint(&self, br_id: u64, breakpoint: V02State) -> DbResult<()> { + let cf_br = self.breakpoint_column(); + + self.db + .put_cf( + &cf_br, + borsh::to_vec(&br_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize breakpoint id".to_string()), + ) + })?, + borsh::to_vec(&breakpoint).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize breakpoint data".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) + } +} \ No newline at end of file From 53b26064ebdb2e3ace9410290cf8e7f869d8adb6 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Mon, 9 Mar 2026 16:35:03 +0200 Subject: [PATCH 02/22] feat: multi gets --- storage/src/indexer/mod.rs | 6 +- storage/src/indexer/read_multi_get.rs | 349 +++++++++++++++++++++++++- storage/src/indexer/read_once.rs | 136 +--------- storage/src/indexer/write_batch.rs | 2 +- storage/src/indexer/write_once.rs | 8 +- 5 files changed, 359 insertions(+), 142 deletions(-) diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index 311746e6..c445e6bb 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -8,10 +8,10 @@ use rocksdb::{ use crate::error::DbError; -pub mod read_once; -pub mod write_once; pub mod read_multi_get; +pub mod read_once; pub mod write_batch; +pub mod write_once; /// Maximal size of stored blocks in base /// @@ -162,7 +162,7 @@ impl RocksDBIO { self.db.cf_handle(CF_ACC_META).unwrap() } - //State + // State pub fn calculate_state_for_id(&self, block_id: u64) -> DbResult { let last_block = self.get_meta_last_block_in_db()?; diff --git a/storage/src/indexer/read_multi_get.rs b/storage/src/indexer/read_multi_get.rs index 15124682..9fd68382 100644 --- a/storage/src/indexer/read_multi_get.rs +++ b/storage/src/indexer/read_multi_get.rs @@ -1,5 +1,350 @@ use super::*; +#[derive(Debug, Clone)] +pub struct DBMetadata { + pub first_block_in_db: u64, + pub last_block_in_db: u64, + pub last_observed_l1_lib_header_in_db: [u8; 32], + pub is_first_block_set: bool, + pub last_breakpoint_id: u64, +} + impl RocksDBIO { - -} \ No newline at end of file + fn meta_keys_list() -> DbResult>> { + let mut keys = vec![]; + + keys.push( + borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + ); + keys.push(borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), + ) + })?); + keys.push( + borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some( + "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" + .to_string(), + ), + ) + })?, + ); + keys.push(borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), + ) + })?); + keys.push(borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), + ) + })?); + + Ok(keys) + } + + fn read_meta_all(&self) -> DbResult> { + let cf_meta = self.meta_column(); + + let multi_get_res = self.db.multi_get_cf( + RocksDBIO::meta_keys_list()? + .into_iter() + .map(|key| (&cf_meta, key)), + ); + + let Some(first_block_in_db_raw) = + multi_get_res[0] + .as_ref() + .map_err(|err| DbError::RocksDbError { + error: err.clone(), + additional_info: Some("Failed to read first_block_in_db".to_string()), + })? + else { + return Ok(None); + }; + let Some(last_block_in_db_raw) = + multi_get_res[1] + .as_ref() + .map_err(|err| DbError::RocksDbError { + error: err.clone(), + additional_info: Some("Failed to read last_block_in_db".to_string()), + })? + else { + return Ok(None); + }; + let Some(last_observed_l1_lib_header_in_db_raw) = + multi_get_res[2] + .as_ref() + .map_err(|err| DbError::RocksDbError { + error: err.clone(), + additional_info: Some( + "Failed to read last_observed_l1_lib_header_in_db".to_string(), + ), + })? + else { + return Ok(None); + }; + let is_first_block_set = multi_get_res[3] + .as_ref() + .map_err(|err| DbError::RocksDbError { + error: err.clone(), + additional_info: Some("Failed to read is_first_block_set".to_string()), + })? + .is_some(); + let Some(last_breakpoint_id_raw) = + multi_get_res[4] + .as_ref() + .clone() + .map_err(|err| DbError::RocksDbError { + error: err.clone(), + additional_info: Some("Failed to read last_breakpoint_id".to_string()), + })? + else { + return Ok(None); + }; + + let first_block_in_db = borsh::from_slice::(first_block_in_db_raw).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to deserialize first block".to_string())) + })?; + let last_block_in_db = borsh::from_slice::(last_block_in_db_raw).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to deserialize last block".to_string())) + })?; + let last_observed_l1_lib_header_in_db = borsh::from_slice::<[u8; 32]>( + last_observed_l1_lib_header_in_db_raw, + ) + .map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to deserialize last l1 lib header".to_string()), + ) + })?; + let last_breakpoint_id = + borsh::from_slice::(last_breakpoint_id_raw).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to deserialize last breakpoint id".to_string()), + ) + })?; + + Ok(Some(DBMetadata { + first_block_in_db, + last_block_in_db, + last_observed_l1_lib_header_in_db, + is_first_block_set, + last_breakpoint_id, + })) + } + + pub fn get_block_batch(&self, before: Option, limit: u64) -> DbResult> { + let mut seq = vec![]; + + // Determine the starting block ID + let start_block_id = if let Some(before_id) = before { + before_id.saturating_sub(1) + } else { + // Get the latest block ID + self.get_meta_last_block_in_db()? + }; + + for i in 0..limit { + let block_id = start_block_id.saturating_sub(i); + if block_id == 0 { + break; + } + seq.push(block_id); + } + + self.get_block_batch_seq(seq.into_iter()) + } + + /// Get block batch from a sequence + /// + /// Currently assumes non-decreasing sequence + /// + /// ToDo: Add suport of arbitrary sequences + fn get_block_batch_seq(&self, seq: impl Iterator) -> DbResult> { + let cf_block = self.block_column(); + + // Keys setup + let mut keys = vec![]; + for block_id in seq { + keys.push(( + &cf_block, + borsh::to_vec(&block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + )); + } + + let multi_get_res = self.db.multi_get_cf(keys); + + // Keys parsing + let mut block_batch = vec![]; + for res in multi_get_res { + let res = res.map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let block = if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize block data".to_string()), + ) + })?) + } else { + // Block not found, assuming that previous one was the last + break; + }?; + + block_batch.push(block); + } + + Ok(block_batch) + } + + /// Get block ids by txs + /// + /// Transactions must be sorted by time of arrival + /// + /// ToDo: There may be multiple transactions in one block + /// so this method can take redundant reads. + /// Need to update signature and implementation. + fn get_block_ids_by_tx_vec(&self, tx_vec: &[[u8; 32]]) -> DbResult> { + let cf_tti = self.tx_hash_to_id_column(); + + // Keys setup + let mut keys = vec![]; + for tx_hash in tx_vec { + keys.push(( + &cf_tti, + borsh::to_vec(tx_hash).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize tx_hash".to_string()), + ) + })?, + )); + } + + let multi_get_res = self.db.multi_get_cf(keys); + + // Keys parsing + let mut block_id_batch = vec![]; + for res in multi_get_res { + let res = res.map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let block_id = if let Some(data) = res { + Ok(borsh::from_slice::(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize block id".to_string()), + ) + })?) + } else { + // Block not found, assuming that previous one was the last + break; + }?; + + block_id_batch.push(block_id); + } + + Ok(block_id_batch) + } + + // Account + + pub(crate) fn get_acc_transaction_hashes( + &self, + acc_id: [u8; 32], + offset: u64, + limit: u64, + ) -> DbResult> { + let cf_att = self.account_id_to_tx_hash_column(); + let mut tx_batch = vec![]; + + // Keys preparation + let mut keys = vec![]; + for tx_id in offset..(offset + limit) { + let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { + DbError::borsh_cast_message( + berr, + Some("Failed to serialize account id".to_string()), + ) + })?; + let suffix = borsh::to_vec(&tx_id).map_err(|berr| { + DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) + })?; + + prefix.extend_from_slice(&suffix); + + keys.push((&cf_att, prefix)); + } + + let multi_get_res = self.db.multi_get_cf(keys); + + for res in multi_get_res { + let res = res.map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + let tx_hash = if let Some(data) = res { + Ok(borsh::from_slice::<[u8; 32]>(&data).map_err(|serr| { + DbError::borsh_cast_message( + serr, + Some("Failed to deserialize tx_hash".to_string()), + ) + })?) + } else { + // Tx hash not found, assuming that previous one was the last + break; + }?; + + tx_batch.push(tx_hash); + } + + Ok(tx_batch) + } + + pub fn get_acc_transactions( + &self, + acc_id: [u8; 32], + offset: u64, + limit: u64, + ) -> DbResult> { + let mut tx_batch = vec![]; + + let tx_hashes = self.get_acc_transaction_hashes(acc_id, offset, limit)?; + + let associated_blocks_multi_get = self + .get_block_batch_seq(self.get_block_ids_by_tx_vec(&tx_hashes)?.into_iter())? + .into_iter() + .zip(tx_hashes); + + for (block, tx_hash) in associated_blocks_multi_get { + let transaction = block + .body + .transactions + .iter() + .find(|tx| tx.hash().0 == tx_hash) + .ok_or(DbError::db_interaction_error(format!( + "Missing transaction in block {} with hash {:#?}", + block.header.block_id, tx_hash + )))?; + + tx_batch.push(transaction.clone()); + } + + Ok(tx_batch) + } +} diff --git a/storage/src/indexer/read_once.rs b/storage/src/indexer/read_once.rs index 1091dbc1..e544d277 100644 --- a/storage/src/indexer/read_once.rs +++ b/storage/src/indexer/read_once.rs @@ -1,7 +1,7 @@ use super::*; impl RocksDBIO { - //Meta + // Meta pub fn get_meta_first_block_in_db(&self) -> DbResult { let cf_meta = self.meta_column(); @@ -139,7 +139,7 @@ impl RocksDBIO { } } - //Block + // Block pub fn get_block(&self, block_id: u64) -> DbResult { let cf_block = self.block_column(); @@ -170,58 +170,7 @@ impl RocksDBIO { } } - pub fn get_block_batch(&self, before: Option, limit: u64) -> DbResult> { - let cf_block = self.block_column(); - let mut block_batch = vec![]; - - // Determine the starting block ID - let start_block_id = if let Some(before_id) = before { - before_id.saturating_sub(1) - } else { - // Get the latest block ID - self.get_meta_last_block_in_db()? - }; - - // ToDo: Multi get this - - for i in 0..limit { - let block_id = start_block_id.saturating_sub(i); - if block_id == 0 { - break; - } - - let res = self - .db - .get_cf( - &cf_block, - borsh::to_vec(&block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let block = if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize block data".to_string()), - ) - })?) - } else { - // Block not found, assuming that previous one was the last - break; - }?; - - block_batch.push(block); - } - - Ok(block_batch) - } - - //State + // State pub fn get_breakpoint(&self, br_id: u64) -> DbResult { let cf_br = self.breakpoint_column(); @@ -344,81 +293,4 @@ impl RocksDBIO { }) .transpose() } - - // Account - - fn get_acc_transaction_hashes( - &self, - acc_id: [u8; 32], - offset: u64, - limit: u64, - ) -> DbResult> { - let cf_att = self.account_id_to_tx_hash_column(); - let mut tx_batch = vec![]; - - // ToDo: Multi get this - - for tx_id in offset..(offset + limit) { - let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { - DbError::borsh_cast_message( - berr, - Some("Failed to serialize account id".to_string()), - ) - })?; - let suffix = borsh::to_vec(&tx_id).map_err(|berr| { - DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) - })?; - - prefix.extend_from_slice(&suffix); - - let res = self - .db - .get_cf(&cf_att, prefix) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let tx_hash = if let Some(data) = res { - Ok(borsh::from_slice::<[u8; 32]>(&data).map_err(|serr| { - DbError::borsh_cast_message( - serr, - Some("Failed to deserialize tx_hash".to_string()), - ) - })?) - } else { - // Tx hash not found, assuming that previous one was the last - break; - }?; - - tx_batch.push(tx_hash); - } - - Ok(tx_batch) - } - - pub fn get_acc_transactions( - &self, - acc_id: [u8; 32], - offset: u64, - limit: u64, - ) -> DbResult> { - let mut tx_batch = vec![]; - - for tx_hash in self.get_acc_transaction_hashes(acc_id, offset, limit)? { - let block_id = self.get_block_id_by_tx_hash(tx_hash)?; - let block = self.get_block(block_id)?; - - let transaction = block - .body - .transactions - .iter() - .find(|tx| tx.hash().0 == tx_hash) - .ok_or(DbError::db_interaction_error(format!( - "Missing transaction in block {} with hash {:#?}", - block.header.block_id, tx_hash - )))?; - - tx_batch.push(transaction.clone()); - } - - Ok(tx_batch) - } -} \ No newline at end of file +} diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs index 2bb11a85..254e7dc9 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_batch.rs @@ -75,4 +75,4 @@ impl RocksDBIO { DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_string())) }) } -} \ No newline at end of file +} diff --git a/storage/src/indexer/write_once.rs b/storage/src/indexer/write_once.rs index 71946f63..b4309c78 100644 --- a/storage/src/indexer/write_once.rs +++ b/storage/src/indexer/write_once.rs @@ -1,7 +1,7 @@ use super::*; impl RocksDBIO { - //Meta + // Meta pub fn put_meta_first_block_in_db(&self, block: Block) -> DbResult<()> { let cf_meta = self.meta_column(); @@ -118,7 +118,7 @@ impl RocksDBIO { Ok(()) } - //Block + // Block pub fn put_block(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { let cf_block = self.block_column(); @@ -218,7 +218,7 @@ impl RocksDBIO { Ok(()) } - //State + // State pub fn put_breakpoint(&self, br_id: u64, breakpoint: V02State) -> DbResult<()> { let cf_br = self.breakpoint_column(); @@ -241,4 +241,4 @@ impl RocksDBIO { ) .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) } -} \ No newline at end of file +} From b57c2ac33d8c105fe7bf345b5d6f585dcf7c98a4 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Wed, 11 Mar 2026 18:37:54 +0200 Subject: [PATCH 03/22] feat: batch writes --- storage/src/indexer/read_multi_get.rs | 4 +- storage/src/indexer/read_once.rs | 17 -- storage/src/indexer/write_batch.rs | 286 ++++++++++++++++++++++++++ storage/src/indexer/write_once.rs | 17 ++ 4 files changed, 305 insertions(+), 19 deletions(-) diff --git a/storage/src/indexer/read_multi_get.rs b/storage/src/indexer/read_multi_get.rs index 9fd68382..6bfb28cc 100644 --- a/storage/src/indexer/read_multi_get.rs +++ b/storage/src/indexer/read_multi_get.rs @@ -10,7 +10,7 @@ pub struct DBMetadata { } impl RocksDBIO { - fn meta_keys_list() -> DbResult>> { + pub fn meta_keys_list() -> DbResult>> { let mut keys = vec![]; keys.push( @@ -54,7 +54,7 @@ impl RocksDBIO { Ok(keys) } - fn read_meta_all(&self) -> DbResult> { + pub fn read_meta_all(&self) -> DbResult> { let cf_meta = self.meta_column(); let multi_get_res = self.db.multi_get_cf( diff --git a/storage/src/indexer/read_once.rs b/storage/src/indexer/read_once.rs index e544d277..8fe035fd 100644 --- a/storage/src/indexer/read_once.rs +++ b/storage/src/indexer/read_once.rs @@ -201,23 +201,6 @@ impl RocksDBIO { } } - pub fn put_next_breakpoint(&self) -> DbResult<()> { - let last_block = self.get_meta_last_block_in_db()?; - let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; - let block_to_break_id = next_breakpoint_id * BREAKPOINT_INTERVAL; - - if block_to_break_id <= last_block { - let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; - - self.put_breakpoint(next_breakpoint_id, next_breakpoint)?; - self.put_meta_last_breakpoint_id(next_breakpoint_id) - } else { - Err(DbError::db_interaction_error( - "Breakpoint not yet achieved".to_string(), - )) - } - } - // Mappings pub fn get_block_id_by_hash(&self, hash: [u8; 32]) -> DbResult { diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs index 254e7dc9..9febd605 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_batch.rs @@ -75,4 +75,290 @@ impl RocksDBIO { DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_string())) }) } + + pub fn put_account_transactions_dependant( + &self, + acc_id: [u8; 32], + tx_hashes: Vec<[u8; 32]>, + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + let acc_num_tx = self.get_acc_meta_num_tx(acc_id)?.unwrap_or(0); + let cf_att = self.account_id_to_tx_hash_column(); + + for (tx_id, tx_hash) in tx_hashes.iter().enumerate() { + let put_id = acc_num_tx + tx_id as u64; + + let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { + DbError::borsh_cast_message( + berr, + Some("Failed to serialize account id".to_string()), + ) + })?; + let suffix = borsh::to_vec(&put_id).map_err(|berr| { + DbError::borsh_cast_message(berr, Some("Failed to serialize tx id".to_string())) + })?; + + prefix.extend_from_slice(&suffix); + + write_batch.put_cf( + &cf_att, + prefix, + borsh::to_vec(tx_hash).map_err(|berr| { + DbError::borsh_cast_message( + berr, + Some("Failed to serialize tx hash".to_string()), + ) + })?, + ); + } + + self.update_acc_meta_batch(acc_id, acc_num_tx + (tx_hashes.len() as u64), write_batch)?; + + Ok(()) + } + + // Meta + + pub fn put_meta_first_block_in_db_batch(&self, block: Block) -> DbResult<()> { + let cf_meta = self.meta_column(); + self.db + .put_cf( + &cf_meta, + borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize first block id".to_string()), + ) + })?, + ) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + + self.put_block_batch(block, [0; 32])?; + Ok(()) + } + + pub fn put_meta_last_block_in_db_batch( + &self, + block_id: u64, + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + let cf_meta = self.meta_column(); + write_batch.put_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), + ) + })?, + borsh::to_vec(&block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize last block id".to_string()), + ) + })?, + ); + Ok(()) + } + + pub fn put_meta_last_observed_l1_lib_header_in_db_batch( + &self, + l1_lib_header: [u8; 32], + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + let cf_meta = self.meta_column(); + write_batch.put_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some( + "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" + .to_string(), + ), + ) + })?, + borsh::to_vec(&l1_lib_header).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize last l1 block header".to_string()), + ) + })?, + ); + Ok(()) + } + + pub fn put_meta_last_breakpoint_id_batch( + &self, + br_id: u64, + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + let cf_meta = self.meta_column(); + write_batch.put_cf( + &cf_meta, + borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), + ) + })?, + borsh::to_vec(&br_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize last block id".to_string()), + ) + })?, + ); + Ok(()) + } + + pub fn put_meta_is_first_block_set_batch(&self, write_batch: &mut WriteBatch) -> DbResult<()> { + let cf_meta = self.meta_column(); + write_batch.put_cf( + &cf_meta, + borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), + ) + })?, + [1u8; 1], + ); + Ok(()) + } + + // Block + + pub fn put_block_batch(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { + let cf_block = self.block_column(); + let cf_hti = self.hash_to_id_column(); + let cf_tti: Arc> = self.tx_hash_to_id_column(); + let last_curr_block = self.get_meta_last_block_in_db()?; + let mut write_batch = WriteBatch::default(); + + // ToDo: rewrite this with write batching + + write_batch.put_cf( + &cf_block, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize block id".to_string())) + })?, + borsh::to_vec(&block).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize block data".to_string())) + })?, + ); + + if block.header.block_id > last_curr_block { + self.put_meta_last_block_in_db_batch(block.header.block_id, &mut write_batch)?; + self.put_meta_last_observed_l1_lib_header_in_db_batch(l1_lib_header, &mut write_batch)?; + } + + write_batch.put_cf( + &cf_hti, + borsh::to_vec(&block.header.hash).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize block hash".to_string())) + })?, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize block id".to_string())) + })?, + ); + + let mut acc_to_tx_map: HashMap<[u8; 32], Vec<[u8; 32]>> = HashMap::new(); + + for tx in block.body.transactions { + let tx_hash = tx.hash(); + + write_batch.put_cf( + &cf_tti, + borsh::to_vec(&tx_hash).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize tx hash".to_string()), + ) + })?, + borsh::to_vec(&block.header.block_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize block id".to_string()), + ) + })?, + ); + + let acc_ids = tx + .affected_public_account_ids() + .into_iter() + .map(|account_id| account_id.into_value()) + .collect::>(); + + for acc_id in acc_ids { + acc_to_tx_map + .entry(acc_id) + .and_modify(|tx_hashes| tx_hashes.push(tx_hash.into())) + .or_insert(vec![tx_hash.into()]); + } + } + + for (acc_id, tx_hashes) in acc_to_tx_map { + self.put_account_transactions_dependant(acc_id, tx_hashes, &mut write_batch)?; + } + + if block.header.block_id.is_multiple_of(BREAKPOINT_INTERVAL) { + self.put_next_breakpoint_batch(&mut write_batch)?; + } + + self.db.write(write_batch).map_err(|rerr| { + DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_string())) + }) + } + + // State + + pub fn put_breakpoint_batch( + &self, + br_id: u64, + breakpoint: V02State, + write_batch: &mut WriteBatch, + ) -> DbResult<()> { + let cf_br = self.breakpoint_column(); + + write_batch.put_cf( + &cf_br, + borsh::to_vec(&br_id).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize breakpoint id".to_string()), + ) + })?, + borsh::to_vec(&breakpoint).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize breakpoint data".to_string()), + ) + })?, + ); + Ok(()) + } + + pub fn put_next_breakpoint_batch(&self, write_batch: &mut WriteBatch) -> DbResult<()> { + let last_block = self.get_meta_last_block_in_db()?; + let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; + let block_to_break_id = next_breakpoint_id * BREAKPOINT_INTERVAL; + + if block_to_break_id <= last_block { + let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; + + self.put_breakpoint_batch(next_breakpoint_id, next_breakpoint, write_batch)?; + self.put_meta_last_breakpoint_id_batch(next_breakpoint_id, write_batch) + } else { + Err(DbError::db_interaction_error( + "Breakpoint not yet achieved".to_string(), + )) + } + } } diff --git a/storage/src/indexer/write_once.rs b/storage/src/indexer/write_once.rs index b4309c78..bf00fce5 100644 --- a/storage/src/indexer/write_once.rs +++ b/storage/src/indexer/write_once.rs @@ -241,4 +241,21 @@ impl RocksDBIO { ) .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) } + + pub fn put_next_breakpoint(&self) -> DbResult<()> { + let last_block = self.get_meta_last_block_in_db()?; + let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; + let block_to_break_id = next_breakpoint_id * BREAKPOINT_INTERVAL; + + if block_to_break_id <= last_block { + let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; + + self.put_breakpoint(next_breakpoint_id, next_breakpoint)?; + self.put_meta_last_breakpoint_id(next_breakpoint_id) + } else { + Err(DbError::db_interaction_error( + "Breakpoint not yet achieved".to_string(), + )) + } + } } From d8542e089fab3b2e3aef93b8a8023593d69b236f Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Thu, 12 Mar 2026 15:53:38 +0200 Subject: [PATCH 04/22] fix: various fixes and cleanup --- storage/src/indexer/mod.rs | 83 +++++++++------ storage/src/indexer/read_multi_get.rs | 147 +------------------------- storage/src/indexer/write_batch.rs | 51 +-------- 3 files changed, 58 insertions(+), 223 deletions(-) diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index c445e6bb..f07980a9 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -99,6 +99,7 @@ impl RocksDBIO { } else if let Some((block, initial_state)) = start_data { let block_id = block.header.block_id; dbio.put_meta_last_block_in_db(block_id)?; + dbio.put_meta_last_observed_l1_lib_header_in_db(block.bedrock_parent_id)?; dbio.put_meta_first_block_in_db(block)?; dbio.put_meta_is_first_block_set()?; @@ -179,9 +180,7 @@ impl RocksDBIO { self.get_meta_first_block_in_db()? }; - for id in start..=block_id { - let block = self.get_block(id)?; - + for block in self.get_block_batch_seq((start + 1)..=block_id)? { for transaction in block.body.transactions { transaction .transaction_stateless_check() @@ -347,7 +346,7 @@ mod tests { let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) .unwrap(); - for i in 1..BREAKPOINT_INTERVAL { + for i in 1..(BREAKPOINT_INTERVAL + 1) { let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -355,7 +354,7 @@ mod tests { let transfer_tx = transfer(1, (i - 1) as u128, true); let block = common::test_utils::produce_dummy_block(i + 1, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [i as u8; 32]).unwrap(); + dbio.put_block_batch(block, [i as u8; 32]).unwrap(); } let last_id = dbio.get_meta_last_block_in_db().unwrap(); @@ -367,7 +366,7 @@ mod tests { let breakpoint = dbio.get_breakpoint(1).unwrap(); let final_state = dbio.final_state().unwrap(); - assert_eq!(last_id, 100); + assert_eq!(last_id, 101); assert_eq!(first_id, 1); assert!(is_first_set); assert_eq!(last_br_id, 1); @@ -375,20 +374,22 @@ mod tests { assert_eq!( prev_breakpoint.get_account_by_id(acc1()).balance - final_state.get_account_by_id(acc1()).balance, - 99 + 100 ); assert_eq!( final_state.get_account_by_id(acc2()).balance - prev_breakpoint.get_account_by_id(acc2()).balance, - 99 + 100 ); assert_eq!( - breakpoint.get_account_by_id(acc1()), - final_state.get_account_by_id(acc1()) + breakpoint.get_account_by_id(acc1()).balance + - final_state.get_account_by_id(acc1()).balance, + 1 ); assert_eq!( - breakpoint.get_account_by_id(acc2()), - final_state.get_account_by_id(acc2()) + final_state.get_account_by_id(acc2()).balance + - breakpoint.get_account_by_id(acc2()).balance, + 1 ); } @@ -531,6 +532,14 @@ mod tests { .collect(); assert_eq!(block_hashes_mem_limited, block_hashes_db_limited.as_slice()); + + let block_batch_seq = dbio.get_block_batch_seq(1..=5).unwrap(); + let block_batch_ids = block_batch_seq + .into_iter() + .map(|block| block.header.block_id) + .collect::>(); + + assert_eq!(block_batch_ids, vec![1, 2, 3, 4, 5]); } #[test] @@ -538,20 +547,25 @@ mod tests { let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let mut tx_hash_res = vec![]; - let dbio = RocksDBIO::open_or_create(temdir_path, Some((genesis_block(), initial_state()))) .unwrap(); + let mut tx_hash_res = vec![]; + let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 0, true); + let transfer_tx1 = transfer(1, 0, true); + let transfer_tx2 = transfer(1, 1, true); + tx_hash_res.push(transfer_tx1.hash().0); + tx_hash_res.push(transfer_tx2.hash().0); - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); + let block = common::test_utils::produce_dummy_block( + 2, + Some(prev_hash), + vec![transfer_tx1, transfer_tx2], + ); dbio.put_block(block, [1; 32]).unwrap(); @@ -559,11 +573,16 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 1, true); + let transfer_tx1 = transfer(1, 2, true); + let transfer_tx2 = transfer(1, 3, true); + tx_hash_res.push(transfer_tx1.hash().0); + tx_hash_res.push(transfer_tx2.hash().0); - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); + let block = common::test_utils::produce_dummy_block( + 3, + Some(prev_hash), + vec![transfer_tx1, transfer_tx2], + ); dbio.put_block(block, [2; 32]).unwrap(); @@ -571,11 +590,16 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 2, true); + let transfer_tx1 = transfer(1, 4, true); + let transfer_tx2 = transfer(1, 5, true); + tx_hash_res.push(transfer_tx1.hash().0); + tx_hash_res.push(transfer_tx2.hash().0); - tx_hash_res.push(transfer_tx.hash().0); - - let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); + let block = common::test_utils::produce_dummy_block( + 4, + Some(prev_hash), + vec![transfer_tx1, transfer_tx2], + ); dbio.put_block(block, [3; 32]).unwrap(); @@ -583,15 +607,14 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 3, true); - + let transfer_tx = transfer(1, 6, true); tx_hash_res.push(transfer_tx.hash().0); let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); dbio.put_block(block, [4; 32]).unwrap(); - let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 4).unwrap(); + let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 7).unwrap(); let acc1_tx_hashes: Vec<[u8; 32]> = acc1_tx.into_iter().map(|tx| tx.hash().0).collect(); assert_eq!(acc1_tx_hashes, tx_hash_res); @@ -600,6 +623,6 @@ mod tests { let acc1_tx_limited_hashes: Vec<[u8; 32]> = acc1_tx_limited.into_iter().map(|tx| tx.hash().0).collect(); - assert_eq!(acc1_tx_limited_hashes.as_slice(), &tx_hash_res[1..]) + assert_eq!(acc1_tx_limited_hashes.as_slice(), &tx_hash_res[1..5]) } } diff --git a/storage/src/indexer/read_multi_get.rs b/storage/src/indexer/read_multi_get.rs index 6bfb28cc..981bc18e 100644 --- a/storage/src/indexer/read_multi_get.rs +++ b/storage/src/indexer/read_multi_get.rs @@ -1,151 +1,6 @@ use super::*; -#[derive(Debug, Clone)] -pub struct DBMetadata { - pub first_block_in_db: u64, - pub last_block_in_db: u64, - pub last_observed_l1_lib_header_in_db: [u8; 32], - pub is_first_block_set: bool, - pub last_breakpoint_id: u64, -} - impl RocksDBIO { - pub fn meta_keys_list() -> DbResult>> { - let mut keys = vec![]; - - keys.push( - borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), - ) - })?, - ); - keys.push(borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), - ) - })?); - keys.push( - borsh::to_vec(&DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some( - "Failed to serialize DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY" - .to_string(), - ), - ) - })?, - ); - keys.push(borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), - ) - })?); - keys.push(borsh::to_vec(&DB_META_LAST_BREAKPOINT_ID).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_LAST_BREAKPOINT_ID".to_string()), - ) - })?); - - Ok(keys) - } - - pub fn read_meta_all(&self) -> DbResult> { - let cf_meta = self.meta_column(); - - let multi_get_res = self.db.multi_get_cf( - RocksDBIO::meta_keys_list()? - .into_iter() - .map(|key| (&cf_meta, key)), - ); - - let Some(first_block_in_db_raw) = - multi_get_res[0] - .as_ref() - .map_err(|err| DbError::RocksDbError { - error: err.clone(), - additional_info: Some("Failed to read first_block_in_db".to_string()), - })? - else { - return Ok(None); - }; - let Some(last_block_in_db_raw) = - multi_get_res[1] - .as_ref() - .map_err(|err| DbError::RocksDbError { - error: err.clone(), - additional_info: Some("Failed to read last_block_in_db".to_string()), - })? - else { - return Ok(None); - }; - let Some(last_observed_l1_lib_header_in_db_raw) = - multi_get_res[2] - .as_ref() - .map_err(|err| DbError::RocksDbError { - error: err.clone(), - additional_info: Some( - "Failed to read last_observed_l1_lib_header_in_db".to_string(), - ), - })? - else { - return Ok(None); - }; - let is_first_block_set = multi_get_res[3] - .as_ref() - .map_err(|err| DbError::RocksDbError { - error: err.clone(), - additional_info: Some("Failed to read is_first_block_set".to_string()), - })? - .is_some(); - let Some(last_breakpoint_id_raw) = - multi_get_res[4] - .as_ref() - .clone() - .map_err(|err| DbError::RocksDbError { - error: err.clone(), - additional_info: Some("Failed to read last_breakpoint_id".to_string()), - })? - else { - return Ok(None); - }; - - let first_block_in_db = borsh::from_slice::(first_block_in_db_raw).map_err(|err| { - DbError::borsh_cast_message(err, Some("Failed to deserialize first block".to_string())) - })?; - let last_block_in_db = borsh::from_slice::(last_block_in_db_raw).map_err(|err| { - DbError::borsh_cast_message(err, Some("Failed to deserialize last block".to_string())) - })?; - let last_observed_l1_lib_header_in_db = borsh::from_slice::<[u8; 32]>( - last_observed_l1_lib_header_in_db_raw, - ) - .map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to deserialize last l1 lib header".to_string()), - ) - })?; - let last_breakpoint_id = - borsh::from_slice::(last_breakpoint_id_raw).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to deserialize last breakpoint id".to_string()), - ) - })?; - - Ok(Some(DBMetadata { - first_block_in_db, - last_block_in_db, - last_observed_l1_lib_header_in_db, - is_first_block_set, - last_breakpoint_id, - })) - } - pub fn get_block_batch(&self, before: Option, limit: u64) -> DbResult> { let mut seq = vec![]; @@ -173,7 +28,7 @@ impl RocksDBIO { /// Currently assumes non-decreasing sequence /// /// ToDo: Add suport of arbitrary sequences - fn get_block_batch_seq(&self, seq: impl Iterator) -> DbResult> { + pub fn get_block_batch_seq(&self, seq: impl Iterator) -> DbResult> { let cf_block = self.block_column(); // Keys setup diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs index 9febd605..b98e539a 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_batch.rs @@ -308,57 +308,14 @@ impl RocksDBIO { self.put_account_transactions_dependant(acc_id, tx_hashes, &mut write_batch)?; } - if block.header.block_id.is_multiple_of(BREAKPOINT_INTERVAL) { - self.put_next_breakpoint_batch(&mut write_batch)?; - } - self.db.write(write_batch).map_err(|rerr| { DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_string())) - }) - } + })?; - // State + if block.header.block_id.is_multiple_of(BREAKPOINT_INTERVAL) { + self.put_next_breakpoint()?; + } - pub fn put_breakpoint_batch( - &self, - br_id: u64, - breakpoint: V02State, - write_batch: &mut WriteBatch, - ) -> DbResult<()> { - let cf_br = self.breakpoint_column(); - - write_batch.put_cf( - &cf_br, - borsh::to_vec(&br_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize breakpoint id".to_string()), - ) - })?, - borsh::to_vec(&breakpoint).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize breakpoint data".to_string()), - ) - })?, - ); Ok(()) } - - pub fn put_next_breakpoint_batch(&self, write_batch: &mut WriteBatch) -> DbResult<()> { - let last_block = self.get_meta_last_block_in_db()?; - let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; - let block_to_break_id = next_breakpoint_id * BREAKPOINT_INTERVAL; - - if block_to_break_id <= last_block { - let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; - - self.put_breakpoint_batch(next_breakpoint_id, next_breakpoint, write_batch)?; - self.put_meta_last_breakpoint_id_batch(next_breakpoint_id, write_batch) - } else { - Err(DbError::db_interaction_error( - "Breakpoint not yet achieved".to_string(), - )) - } - } } From 247094ca614ebd7d404ea1bb94fdb305699188b4 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Thu, 12 Mar 2026 16:21:04 +0200 Subject: [PATCH 05/22] fix: batching usage in indexer service --- indexer/core/src/block_store.rs | 2 +- storage/src/indexer/mod.rs | 28 +++---- storage/src/indexer/write_once.rs | 124 ------------------------------ 3 files changed, 15 insertions(+), 139 deletions(-) diff --git a/indexer/core/src/block_store.rs b/indexer/core/src/block_store.rs index 681b63c8..7844b947 100644 --- a/indexer/core/src/block_store.rs +++ b/indexer/core/src/block_store.rs @@ -118,6 +118,6 @@ impl IndexerStore { // to represent correct block finality block.bedrock_status = BedrockStatus::Finalized; - Ok(self.dbio.put_block(block, l1_header.into())?) + Ok(self.dbio.put_block_batch(block, l1_header.into())?) } } diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index f07980a9..db47c0a6 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -100,7 +100,7 @@ impl RocksDBIO { let block_id = block.header.block_id; dbio.put_meta_last_block_in_db(block_id)?; dbio.put_meta_last_observed_l1_lib_header_in_db(block.bedrock_parent_id)?; - dbio.put_meta_first_block_in_db(block)?; + dbio.put_meta_first_block_in_db_batch(block)?; dbio.put_meta_is_first_block_set()?; // First breakpoint setup @@ -311,7 +311,7 @@ mod tests { let transfer_tx = transfer(1, 0, true); let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [1; 32]).unwrap(); + dbio.put_block_batch(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let first_id = dbio.get_meta_first_block_in_db().unwrap(); @@ -410,7 +410,7 @@ mod tests { let control_hash1 = block.header.hash; - dbio.put_block(block, [1; 32]).unwrap(); + dbio.put_block_batch(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -421,7 +421,7 @@ mod tests { let control_hash2 = block.header.hash; - dbio.put_block(block, [2; 32]).unwrap(); + dbio.put_block_batch(block, [2; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -432,7 +432,7 @@ mod tests { let control_tx_hash1 = transfer_tx.hash(); let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [3; 32]).unwrap(); + dbio.put_block_batch(block, [3; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -443,7 +443,7 @@ mod tests { let control_tx_hash2 = transfer_tx.hash(); let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [4; 32]).unwrap(); + dbio.put_block_batch(block, [4; 32]).unwrap(); let control_block_id1 = dbio.get_block_id_by_hash(control_hash1.0).unwrap(); let control_block_id2 = dbio.get_block_id_by_hash(control_hash2.0).unwrap(); @@ -474,7 +474,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(block, [1; 32]).unwrap(); + dbio.put_block_batch(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -484,7 +484,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(block, [2; 32]).unwrap(); + dbio.put_block_batch(block, [2; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -494,7 +494,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(block, [3; 32]).unwrap(); + dbio.put_block_batch(block, [3; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -504,7 +504,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(block, [4; 32]).unwrap(); + dbio.put_block_batch(block, [4; 32]).unwrap(); let block_hashes_mem: Vec<[u8; 32]> = block_res.into_iter().map(|bl| bl.header.hash.0).collect(); @@ -567,7 +567,7 @@ mod tests { vec![transfer_tx1, transfer_tx2], ); - dbio.put_block(block, [1; 32]).unwrap(); + dbio.put_block_batch(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -584,7 +584,7 @@ mod tests { vec![transfer_tx1, transfer_tx2], ); - dbio.put_block(block, [2; 32]).unwrap(); + dbio.put_block_batch(block, [2; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -601,7 +601,7 @@ mod tests { vec![transfer_tx1, transfer_tx2], ); - dbio.put_block(block, [3; 32]).unwrap(); + dbio.put_block_batch(block, [3; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -612,7 +612,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(block, [4; 32]).unwrap(); + dbio.put_block_batch(block, [4; 32]).unwrap(); let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 7).unwrap(); let acc1_tx_hashes: Vec<[u8; 32]> = acc1_tx.into_iter().map(|tx| tx.hash().0).collect(); diff --git a/storage/src/indexer/write_once.rs b/storage/src/indexer/write_once.rs index bf00fce5..ab600f84 100644 --- a/storage/src/indexer/write_once.rs +++ b/storage/src/indexer/write_once.rs @@ -3,30 +3,6 @@ use super::*; impl RocksDBIO { // Meta - pub fn put_meta_first_block_in_db(&self, block: Block) -> DbResult<()> { - let cf_meta = self.meta_column(); - self.db - .put_cf( - &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize first block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - self.put_block(block, [0; 32])?; - Ok(()) - } - pub fn put_meta_last_block_in_db(&self, block_id: u64) -> DbResult<()> { let cf_meta = self.meta_column(); self.db @@ -118,106 +94,6 @@ impl RocksDBIO { Ok(()) } - // Block - - pub fn put_block(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { - let cf_block = self.block_column(); - let cf_hti = self.hash_to_id_column(); - let cf_tti: Arc> = self.tx_hash_to_id_column(); - - // ToDo: rewrite this with write batching - - self.db - .put_cf( - &cf_block, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - borsh::to_vec(&block).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block data".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let last_curr_block = self.get_meta_last_block_in_db()?; - - if block.header.block_id > last_curr_block { - self.put_meta_last_block_in_db(block.header.block_id)?; - self.put_meta_last_observed_l1_lib_header_in_db(l1_lib_header)?; - } - - self.db - .put_cf( - &cf_hti, - borsh::to_vec(&block.header.hash).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block hash".to_string()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let mut acc_to_tx_map: HashMap<[u8; 32], Vec<[u8; 32]>> = HashMap::new(); - - for tx in block.body.transactions { - let tx_hash = tx.hash(); - - self.db - .put_cf( - &cf_tti, - borsh::to_vec(&tx_hash).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize tx hash".to_string()), - ) - })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize block id".to_string()), - ) - })?, - ) - .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - - let acc_ids = tx - .affected_public_account_ids() - .into_iter() - .map(|account_id| account_id.into_value()) - .collect::>(); - - for acc_id in acc_ids { - acc_to_tx_map - .entry(acc_id) - .and_modify(|tx_hashes| tx_hashes.push(tx_hash.into())) - .or_insert(vec![tx_hash.into()]); - } - } - - for (acc_id, tx_hashes) in acc_to_tx_map { - self.put_account_transactions(acc_id, tx_hashes)?; - } - - if block.header.block_id.is_multiple_of(BREAKPOINT_INTERVAL) { - self.put_next_breakpoint()?; - } - - Ok(()) - } - // State pub fn put_breakpoint(&self, br_id: u64, breakpoint: V02State) -> DbResult<()> { From 7b20d70169ab6a8743be8ecb34fc4d5b85c2642c Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Fri, 13 Mar 2026 14:27:54 +0200 Subject: [PATCH 06/22] fix: header storing fix --- integration_tests/tests/indexer.rs | 2 +- storage/src/indexer/mod.rs | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/integration_tests/tests/indexer.rs b/integration_tests/tests/indexer.rs index ad169790..5422c292 100644 --- a/integration_tests/tests/indexer.rs +++ b/integration_tests/tests/indexer.rs @@ -12,7 +12,7 @@ use tokio::test; use wallet::cli::{Command, programs::native_token_transfer::AuthTransferSubcommand}; /// Timeout in milliseconds to reliably await for block finalization -const L2_TO_L1_TIMEOUT_MILLIS: u64 = 600000; +const L2_TO_L1_TIMEOUT_MILLIS: u64 = 300000; #[test] async fn indexer_test_run() -> Result<()> { diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index db47c0a6..b3e2f5b7 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -99,7 +99,6 @@ impl RocksDBIO { } else if let Some((block, initial_state)) = start_data { let block_id = block.header.block_id; dbio.put_meta_last_block_in_db(block_id)?; - dbio.put_meta_last_observed_l1_lib_header_in_db(block.bedrock_parent_id)?; dbio.put_meta_first_block_in_db_batch(block)?; dbio.put_meta_is_first_block_set()?; @@ -279,6 +278,7 @@ mod tests { let last_id = dbio.get_meta_last_block_in_db().unwrap(); let first_id = dbio.get_meta_first_block_in_db().unwrap(); let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); + let last_observed_l1_header = dbio.get_meta_last_observed_l1_lib_header_in_db().unwrap(); let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); let last_block = dbio.get_block(1).unwrap(); let breakpoint = dbio.get_breakpoint(0).unwrap(); @@ -286,6 +286,7 @@ mod tests { assert_eq!(last_id, 1); assert_eq!(first_id, 1); + assert_eq!(last_observed_l1_header, None); assert!(is_first_set); assert_eq!(last_br_id, 0); assert_eq!(last_block.header.hash, genesis_block().header.hash); @@ -315,6 +316,10 @@ mod tests { let last_id = dbio.get_meta_last_block_in_db().unwrap(); let first_id = dbio.get_meta_first_block_in_db().unwrap(); + let last_observed_l1_header = dbio + .get_meta_last_observed_l1_lib_header_in_db() + .unwrap() + .unwrap(); let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -323,6 +328,7 @@ mod tests { assert_eq!(last_id, 2); assert_eq!(first_id, 1); + assert_eq!(last_observed_l1_header, [1; 32]); assert!(is_first_set); assert_eq!(last_br_id, 0); assert_ne!(last_block.header.hash, genesis_block().header.hash); From 02953999aacc4cc479fb7a29d2b74d9ee6c117b9 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Fri, 13 Mar 2026 18:23:39 +0200 Subject: [PATCH 07/22] feat: general initial state --- Cargo.lock | 3 + common/src/block.rs | 16 -- indexer/core/Cargo.toml | 1 + indexer/core/src/config.rs | 9 +- indexer/core/src/lib.rs | 40 +--- key_protocol/src/initial_state.rs | 363 ++++++++++++++++++++++++++++++ key_protocol/src/lib.rs | 1 + sequencer_core/Cargo.toml | 1 + sequencer_core/src/config.rs | 9 +- sequencer_core/src/lib.rs | 161 +++---------- sequencer_rpc/Cargo.toml | 1 + sequencer_rpc/src/process.rs | 91 ++------ wallet/src/chain_storage.rs | 99 +------- wallet/src/cli/config.rs | 3 +- wallet/src/config.rs | 340 ++-------------------------- wallet/src/helperfunctions.rs | 13 +- 16 files changed, 473 insertions(+), 678 deletions(-) create mode 100644 key_protocol/src/initial_state.rs diff --git a/Cargo.lock b/Cargo.lock index 33f810e0..da5b8c35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3749,6 +3749,7 @@ dependencies = [ "common", "futures", "humantime-serde", + "key_protocol", "log", "logos-blockchain-core", "nssa", @@ -7462,6 +7463,7 @@ dependencies = [ "futures", "humantime-serde", "jsonrpsee", + "key_protocol", "log", "logos-blockchain-core", "logos-blockchain-key-management-system-service", @@ -7493,6 +7495,7 @@ dependencies = [ "futures", "hex", "itertools 0.14.0", + "key_protocol", "log", "mempool", "nssa", diff --git a/common/src/block.rs b/common/src/block.rs index 0343435b..8c59b0ea 100644 --- a/common/src/block.rs +++ b/common/src/block.rs @@ -1,6 +1,4 @@ use borsh::{BorshDeserialize, BorshSerialize}; -use nssa::AccountId; -use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256, digest::FixedOutput}; use crate::{HashType, transaction::NSSATransaction}; @@ -109,20 +107,6 @@ impl From for HashableBlockData { } } -/// Helper struct for account (de-)serialization -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AccountInitialData { - pub account_id: AccountId, - pub balance: u128, -} - -/// Helper struct to (de-)serialize initial commitments -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CommitmentsInitialData { - pub npk: nssa_core::NullifierPublicKey, - pub account: nssa_core::account::Account, -} - #[cfg(test)] mod tests { use crate::{HashType, block::HashableBlockData, test_utils}; diff --git a/indexer/core/Cargo.toml b/indexer/core/Cargo.toml index 792fb4b7..1f6c41a2 100644 --- a/indexer/core/Cargo.toml +++ b/indexer/core/Cargo.toml @@ -10,6 +10,7 @@ bedrock_client.workspace = true nssa.workspace = true nssa_core.workspace = true storage.workspace = true +key_protocol.workspace = true anyhow.workspace = true log.workspace = true diff --git a/indexer/core/src/config.rs b/indexer/core/src/config.rs index 95e6147c..fad10ec5 100644 --- a/indexer/core/src/config.rs +++ b/indexer/core/src/config.rs @@ -7,10 +7,7 @@ use std::{ use anyhow::{Context as _, Result}; pub use bedrock_client::BackoffConfig; -use common::{ - block::{AccountInitialData, CommitmentsInitialData}, - config::BasicAuth, -}; +use common::config::BasicAuth; use humantime_serde; pub use logos_blockchain_core::mantle::ops::channel::ChannelId; use serde::{Deserialize, Serialize}; @@ -29,10 +26,6 @@ pub struct ClientConfig { pub struct IndexerConfig { /// Home dir of sequencer storage pub home: PathBuf, - /// List of initial accounts data - pub initial_accounts: Vec, - /// List of initial commitments - pub initial_commitments: Vec, /// Sequencers signing key pub signing_key: [u8; 32], #[serde(with = "humantime_serde")] diff --git a/indexer/core/src/lib.rs b/indexer/core/src/lib.rs index 6d56eb18..002a6c91 100644 --- a/indexer/core/src/lib.rs +++ b/indexer/core/src/lib.rs @@ -2,9 +2,11 @@ use std::collections::VecDeque; use anyhow::Result; use bedrock_client::{BedrockClient, HeaderId}; -use common::block::{Block, HashableBlockData}; -// ToDo: Remove after testnet -use common::{HashType, PINATA_BASE58}; +use common::{ + HashType, + block::{Block, HashableBlockData}, +}; +use key_protocol::initial_state::initial_state_testnet; use log::{debug, error, info}; use logos_blockchain_core::mantle::{ Op, SignedMantleTx, @@ -54,36 +56,8 @@ impl IndexerCore { let channel_genesis_msg_id = [0; 32]; let start_block = hashable_data.into_pending_block(&signing_key, channel_genesis_msg_id); - // This is a troubling moment, because changes in key protocol can - // affect this. And indexer can not reliably ask this data from sequencer - // because indexer must be independent from it. - // ToDo: move initial state generation into common and use the same method - // for indexer and sequencer. This way both services buit at same version - // could be in sync. - let initial_commitments: Vec = config - .initial_commitments - .iter() - .map(|init_comm_data| { - let npk = &init_comm_data.npk; - - let mut acc = init_comm_data.account.clone(); - - acc.program_owner = nssa::program::Program::authenticated_transfer_program().id(); - - nssa_core::Commitment::new(npk, &acc) - }) - .collect(); - - let init_accs: Vec<(nssa::AccountId, u128)> = config - .initial_accounts - .iter() - .map(|acc_data| (acc_data.account_id, acc_data.balance)) - .collect(); - - let mut state = nssa::V02State::new_with_genesis_accounts(&init_accs, &initial_commitments); - - // ToDo: Remove after testnet - state.add_pinata_program(PINATA_BASE58.parse().unwrap()); + // ToDo: replace with `initial_state()` after testnet + let state = initial_state_testnet(); let home = config.home.join("rocksdb"); diff --git a/key_protocol/src/initial_state.rs b/key_protocol/src/initial_state.rs new file mode 100644 index 00000000..8011d471 --- /dev/null +++ b/key_protocol/src/initial_state.rs @@ -0,0 +1,363 @@ +use common::PINATA_BASE58; +use nssa::{Account, AccountId, Data, PrivateKey, PublicKey, V02State}; +use nssa_core::{NullifierPublicKey, encryption::shared_key_derivation::Secp256k1Point}; +use serde::{Deserialize, Serialize}; + +use crate::key_management::{ + KeyChain, + secret_holders::{PrivateKeyHolder, SecretSpendingKey}, +}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PublicAccountPublicInitialData { + pub account_id: AccountId, + pub balance: u128, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PrivateAccountPublicInitialData { + pub npk: nssa_core::NullifierPublicKey, + pub account: nssa_core::account::Account, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PublicAccountPrivateInitialData { + pub account_id: nssa::AccountId, + pub pub_sign_key: nssa::PrivateKey, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PrivateAccountPrivateInitialData { + pub account_id: nssa::AccountId, + pub account: nssa_core::account::Account, + pub key_chain: KeyChain, +} + +pub fn initial_pub_accounts_private_keys() -> Vec { + let acc1_pub_sign_key = PrivateKey::try_new([ + 16, 162, 106, 154, 236, 125, 52, 184, 35, 100, 238, 174, 69, 197, 41, 77, 187, 10, 118, 75, + 0, 11, 148, 238, 185, 181, 133, 17, 220, 72, 124, 77, + ]) + .unwrap(); + + let acc2_pub_sign_key = PrivateKey::try_new([ + 113, 121, 64, 177, 204, 85, 229, 214, 178, 6, 109, 191, 29, 154, 63, 38, 242, 18, 244, 219, + 8, 208, 35, 136, 23, 127, 207, 237, 216, 169, 190, 27, + ]) + .unwrap(); + + vec![ + PublicAccountPrivateInitialData { + account_id: AccountId::from(&PublicKey::new_from_private_key(&acc1_pub_sign_key)), + pub_sign_key: acc1_pub_sign_key, + }, + PublicAccountPrivateInitialData { + account_id: AccountId::from(&PublicKey::new_from_private_key(&acc2_pub_sign_key)), + pub_sign_key: acc2_pub_sign_key, + }, + ] +} + +pub fn initial_priv_accounts_private_keys() -> Vec { + let key_chain_1 = KeyChain { + secret_spending_key: SecretSpendingKey([ + 93, 13, 190, 240, 250, 33, 108, 195, 176, 40, 144, 61, 4, 28, 58, 112, 53, 161, 42, + 238, 155, 27, 23, 176, 208, 121, 15, 229, 165, 180, 99, 143, + ]), + private_key_holder: PrivateKeyHolder { + nullifier_secret_key: [ + 25, 21, 186, 59, 180, 224, 101, 64, 163, 208, 228, 43, 13, 185, 100, 123, 156, 47, + 80, 179, 72, 51, 115, 11, 180, 99, 21, 201, 48, 194, 118, 144, + ], + viewing_secret_key: [ + 5, 85, 114, 119, 141, 187, 202, 170, 122, 253, 198, 81, 150, 8, 155, 21, 192, 65, + 24, 124, 116, 98, 110, 106, 137, 90, 165, 239, 80, 13, 222, 30, + ], + }, + nullifer_public_key: NullifierPublicKey([ + 167, 108, 50, 153, 74, 47, 151, 188, 140, 79, 195, 31, 181, 9, 40, 167, 201, 32, 175, + 129, 45, 245, 223, 193, 210, 170, 247, 128, 167, 140, 155, 129, + ]), + viewing_public_key: Secp256k1Point(vec![ + 2, 210, 206, 38, 213, 4, 182, 198, 220, 47, 93, 148, 61, 84, 148, 250, 158, 45, 8, 81, + 48, 80, 46, 230, 87, 210, 47, 204, 76, 58, 214, 167, 81, + ]), + }; + + let key_chain_2 = KeyChain { + secret_spending_key: SecretSpendingKey([ + 48, 175, 124, 10, 230, 240, 166, 14, 249, 254, 157, 226, 208, 124, 122, 177, 203, 139, + 192, 180, 43, 120, 55, 151, 50, 21, 113, 22, 254, 83, 148, 56, + ]), + private_key_holder: PrivateKeyHolder { + nullifier_secret_key: [ + 99, 82, 190, 140, 234, 10, 61, 163, 15, 211, 179, 54, 70, 166, 87, 5, 182, 68, 117, + 244, 217, 23, 99, 9, 4, 177, 230, 125, 109, 91, 160, 30, + ], + viewing_secret_key: [ + 205, 32, 76, 251, 255, 236, 96, 119, 61, 111, 65, 100, 75, 218, 12, 22, 17, 170, + 55, 226, 21, 154, 161, 34, 208, 74, 27, 1, 119, 13, 88, 128, + ], + }, + nullifer_public_key: NullifierPublicKey([ + 32, 67, 72, 164, 106, 53, 66, 239, 141, 15, 52, 230, 136, 177, 2, 236, 207, 243, 134, + 135, 210, 143, 87, 232, 215, 128, 194, 120, 113, 224, 4, 165, + ]), + viewing_public_key: Secp256k1Point(vec![ + 2, 79, 110, 46, 203, 29, 206, 205, 18, 86, 27, 189, 104, 103, 113, 181, 110, 53, 78, + 172, 11, 171, 190, 18, 126, 214, 81, 77, 192, 154, 58, 195, 238, + ]), + }; + + vec![ + PrivateAccountPrivateInitialData { + account_id: AccountId::from(&key_chain_1.nullifer_public_key), + account: Account { + program_owner: [0, 0, 0, 0, 0, 0, 0, 0], + balance: 10000, + data: Data::default(), + nonce: 0, + }, + key_chain: key_chain_1, + }, + PrivateAccountPrivateInitialData { + account_id: AccountId::from(&key_chain_2.nullifer_public_key), + account: Account { + program_owner: [0, 0, 0, 0, 0, 0, 0, 0], + balance: 20000, + data: Data::default(), + nonce: 0, + }, + key_chain: key_chain_2, + }, + ] +} + +pub fn initial_commitments() -> Vec { + initial_priv_accounts_private_keys() + .into_iter() + .map(|data| PrivateAccountPublicInitialData { + npk: data.key_chain.nullifer_public_key.clone(), + account: data.account.clone(), + }) + .collect() +} + +pub fn initial_accounts() -> Vec { + let initial_account_ids = initial_pub_accounts_private_keys() + .into_iter() + .map(|data| data.account_id) + .collect::>(); + + vec![ + PublicAccountPublicInitialData { + account_id: initial_account_ids[0], + balance: 10000, + }, + PublicAccountPublicInitialData { + account_id: initial_account_ids[1], + balance: 20000, + }, + ] +} + +pub fn initial_state() -> V02State { + let initial_commitments: Vec = initial_commitments() + .iter() + .map(|init_comm_data| { + let npk = &init_comm_data.npk; + + let mut acc = init_comm_data.account.clone(); + + acc.program_owner = nssa::program::Program::authenticated_transfer_program().id(); + + nssa_core::Commitment::new(npk, &acc) + }) + .collect(); + + let init_accs: Vec<(nssa::AccountId, u128)> = initial_accounts() + .iter() + .map(|acc_data| (acc_data.account_id, acc_data.balance)) + .collect(); + + nssa::V02State::new_with_genesis_accounts(&init_accs, &initial_commitments) +} + +pub fn initial_state_testnet() -> V02State { + let mut state = initial_state(); + + state.add_pinata_program(PINATA_BASE58.parse().unwrap()); + + state +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use super::*; + + #[test] + fn test_pub_state_consistency() { + let init_accs_private_data = initial_pub_accounts_private_keys(); + let init_accs_pub_data = initial_accounts(); + + assert_eq!( + init_accs_private_data[0].account_id, + init_accs_pub_data[0].account_id + ); + + assert_eq!( + init_accs_private_data[1].account_id, + init_accs_pub_data[1].account_id + ); + + assert_eq!( + init_accs_pub_data[0], + PublicAccountPublicInitialData { + account_id: AccountId::from_str("6iArKUXxhUJqS7kCaPNhwMWt3ro71PDyBj7jwAyE2VQV") + .unwrap(), + balance: 10000, + } + ); + + assert_eq!( + init_accs_pub_data[1], + PublicAccountPublicInitialData { + account_id: AccountId::from_str("7wHg9sbJwc6h3NP1S9bekfAzB8CHifEcxKswCKUt3YQo") + .unwrap(), + balance: 20000, + } + ) + } + + #[test] + fn test_private_state_consistency() { + let init_private_accs_keys = initial_priv_accounts_private_keys(); + let init_comms = initial_commitments(); + + assert_eq!( + init_private_accs_keys[0] + .key_chain + .secret_spending_key + .produce_private_key_holder(None) + .nullifier_secret_key, + init_private_accs_keys[0] + .key_chain + .private_key_holder + .nullifier_secret_key + ); + assert_eq!( + init_private_accs_keys[0] + .key_chain + .secret_spending_key + .produce_private_key_holder(None) + .viewing_secret_key, + init_private_accs_keys[0] + .key_chain + .private_key_holder + .viewing_secret_key + ); + assert_eq!( + init_private_accs_keys[0] + .key_chain + .private_key_holder + .generate_nullifier_public_key(), + init_private_accs_keys[0].key_chain.nullifer_public_key + ); + assert_eq!( + init_private_accs_keys[0] + .key_chain + .private_key_holder + .generate_viewing_public_key(), + init_private_accs_keys[0].key_chain.viewing_public_key + ); + + assert_eq!( + init_private_accs_keys[1] + .key_chain + .secret_spending_key + .produce_private_key_holder(None) + .nullifier_secret_key, + init_private_accs_keys[1] + .key_chain + .private_key_holder + .nullifier_secret_key + ); + assert_eq!( + init_private_accs_keys[1] + .key_chain + .secret_spending_key + .produce_private_key_holder(None) + .viewing_secret_key, + init_private_accs_keys[1] + .key_chain + .private_key_holder + .viewing_secret_key + ); + assert_eq!( + init_private_accs_keys[1] + .key_chain + .private_key_holder + .generate_nullifier_public_key(), + init_private_accs_keys[1].key_chain.nullifer_public_key + ); + assert_eq!( + init_private_accs_keys[1] + .key_chain + .private_key_holder + .generate_viewing_public_key(), + init_private_accs_keys[1].key_chain.viewing_public_key + ); + + assert_eq!( + init_private_accs_keys[0].account_id.to_string(), + "5ya25h4Xc9GAmrGB2WrTEnEWtQKJwRwQx3Xfo2tucNcE" + ); + assert_eq!( + init_private_accs_keys[1].account_id.to_string(), + "E8HwiTyQe4H9HK7icTvn95HQMnzx49mP9A2ddtMLpNaN" + ); + + assert_eq!( + init_private_accs_keys[0].key_chain.nullifer_public_key, + init_comms[0].npk + ); + assert_eq!( + init_private_accs_keys[1].key_chain.nullifer_public_key, + init_comms[1].npk + ); + + assert_eq!( + init_comms[0], + PrivateAccountPublicInitialData { + npk: NullifierPublicKey([ + 167, 108, 50, 153, 74, 47, 151, 188, 140, 79, 195, 31, 181, 9, 40, 167, 201, + 32, 175, 129, 45, 245, 223, 193, 210, 170, 247, 128, 167, 140, 155, 129, + ]), + account: Account { + program_owner: [0, 0, 0, 0, 0, 0, 0, 0], + balance: 10000, + data: Data::default(), + nonce: 0, + }, + } + ); + + assert_eq!( + init_comms[1], + PrivateAccountPublicInitialData { + npk: NullifierPublicKey([ + 32, 67, 72, 164, 106, 53, 66, 239, 141, 15, 52, 230, 136, 177, 2, 236, 207, + 243, 134, 135, 210, 143, 87, 232, 215, 128, 194, 120, 113, 224, 4, 165, + ]), + account: Account { + program_owner: [0, 0, 0, 0, 0, 0, 0, 0], + balance: 20000, + data: Data::default(), + nonce: 0, + }, + } + ) + } +} diff --git a/key_protocol/src/lib.rs b/key_protocol/src/lib.rs index 1a52c202..7c056ec8 100644 --- a/key_protocol/src/lib.rs +++ b/key_protocol/src/lib.rs @@ -1,2 +1,3 @@ +pub mod initial_state; pub mod key_management; pub mod key_protocol_core; diff --git a/sequencer_core/Cargo.toml b/sequencer_core/Cargo.toml index e939c7ae..1d4e604e 100644 --- a/sequencer_core/Cargo.toml +++ b/sequencer_core/Cargo.toml @@ -11,6 +11,7 @@ common.workspace = true storage.workspace = true mempool.workspace = true bedrock_client.workspace = true +key_protocol.workspace = true base58.workspace = true anyhow.workspace = true diff --git a/sequencer_core/src/config.rs b/sequencer_core/src/config.rs index 003b82e8..05a0dc72 100644 --- a/sequencer_core/src/config.rs +++ b/sequencer_core/src/config.rs @@ -8,10 +8,7 @@ use std::{ use anyhow::Result; use bedrock_client::BackoffConfig; use bytesize::ByteSize; -use common::{ - block::{AccountInitialData, CommitmentsInitialData}, - config::BasicAuth, -}; +use common::config::BasicAuth; use humantime_serde; use logos_blockchain_core::mantle::ops::channel::ChannelId; use serde::{Deserialize, Serialize}; @@ -43,10 +40,6 @@ pub struct SequencerConfig { pub retry_pending_blocks_timeout: Duration, /// Port to listen pub port: u16, - /// List of initial accounts data - pub initial_accounts: Vec, - /// List of initial commitments - pub initial_commitments: Vec, /// Sequencer own signing key pub signing_key: [u8; 32], /// Bedrock configuration options diff --git a/sequencer_core/src/lib.rs b/sequencer_core/src/lib.rs index 083728bf..fdb65d2a 100644 --- a/sequencer_core/src/lib.rs +++ b/sequencer_core/src/lib.rs @@ -10,6 +10,7 @@ use common::{ transaction::NSSATransaction, }; use config::SequencerConfig; +use key_protocol::initial_state::initial_state; use log::{error, info, warn}; use logos_blockchain_key_management_system_service::keys::{ED25519_SECRET_KEY_SIZE, Ed25519Key}; use mempool::{MemPool, MemPoolHandle}; @@ -98,30 +99,9 @@ impl SequencerCore { info!( - "No database found when starting the sequencer. Creating a fresh new with the initial data in config" + "No database found when starting the sequencer. Creating a fresh new with the initial data" ); - let initial_commitments: Vec = config - .initial_commitments - .iter() - .map(|init_comm_data| { - let npk = &init_comm_data.npk; - - let mut acc = init_comm_data.account.clone(); - - acc.program_owner = - nssa::program::Program::authenticated_transfer_program().id(); - - nssa_core::Commitment::new(npk, &acc) - }) - .collect(); - - let init_accs: Vec<(nssa::AccountId, u128)> = config - .initial_accounts - .iter() - .map(|acc_data| (acc_data.account_id, acc_data.balance)) - .collect(); - - nssa::V02State::new_with_genesis_accounts(&init_accs, &initial_commitments) + initial_state() } }; @@ -360,26 +340,20 @@ fn load_or_create_signing_key(path: &Path) -> Result { #[cfg(all(test, feature = "mock"))] mod tests { - use std::{pin::pin, str::FromStr as _, time::Duration}; + use std::{pin::pin, time::Duration}; - use base58::ToBase58; use bedrock_client::BackoffConfig; - use common::{ - block::AccountInitialData, test_utils::sequencer_sign_key_for_testing, - transaction::NSSATransaction, - }; + use common::{test_utils::sequencer_sign_key_for_testing, transaction::NSSATransaction}; + use key_protocol::initial_state::{initial_accounts, initial_pub_accounts_private_keys}; use logos_blockchain_core::mantle::ops::channel::ChannelId; use mempool::MemPoolHandle; - use nssa::{AccountId, PrivateKey}; use crate::{ config::{BedrockConfig, SequencerConfig}, mock::SequencerCoreWithMockClients, }; - fn setup_sequencer_config_variable_initial_accounts( - initial_accounts: Vec, - ) -> SequencerConfig { + fn setup_sequencer_config() -> SequencerConfig { let tempdir = tempfile::tempdir().unwrap(); let home = tempdir.path().to_path_buf(); @@ -393,8 +367,6 @@ mod tests { mempool_max_size: 10000, block_create_timeout: Duration::from_secs(1), port: 8080, - initial_accounts, - initial_commitments: vec![], signing_key: *sequencer_sign_key_for_testing().value(), bedrock_config: BedrockConfig { backoff: BackoffConfig { @@ -410,38 +382,12 @@ mod tests { } } - fn setup_sequencer_config() -> SequencerConfig { - let acc1_account_id: Vec = vec![ - 148, 179, 206, 253, 199, 51, 82, 86, 232, 2, 152, 122, 80, 243, 54, 207, 237, 112, 83, - 153, 44, 59, 204, 49, 128, 84, 160, 227, 216, 149, 97, 102, - ]; - - let acc2_account_id: Vec = vec![ - 30, 145, 107, 3, 207, 73, 192, 230, 160, 63, 238, 207, 18, 69, 54, 216, 103, 244, 92, - 94, 124, 248, 42, 16, 141, 19, 119, 18, 14, 226, 140, 204, - ]; - - let initial_acc1 = AccountInitialData { - account_id: AccountId::from_str(&acc1_account_id.to_base58()).unwrap(), - balance: 10000, - }; - - let initial_acc2 = AccountInitialData { - account_id: AccountId::from_str(&acc2_account_id.to_base58()).unwrap(), - balance: 20000, - }; - - let initial_accounts = vec![initial_acc1, initial_acc2]; - - setup_sequencer_config_variable_initial_accounts(initial_accounts) - } - fn create_signing_key_for_account1() -> nssa::PrivateKey { - nssa::PrivateKey::try_new([1; 32]).unwrap() + initial_pub_accounts_private_keys()[0].pub_sign_key.clone() } fn create_signing_key_for_account2() -> nssa::PrivateKey { - nssa::PrivateKey::try_new([2; 32]).unwrap() + initial_pub_accounts_private_keys()[1].pub_sign_key.clone() } async fn common_setup() -> (SequencerCoreWithMockClients, MemPoolHandle) { @@ -475,8 +421,8 @@ mod tests { assert_eq!(sequencer.sequencer_config.max_num_tx_in_block, 10); assert_eq!(sequencer.sequencer_config.port, 8080); - let acc1_account_id = config.initial_accounts[0].account_id; - let acc2_account_id = config.initial_accounts[1].account_id; + let acc1_account_id = initial_accounts()[0].account_id; + let acc2_account_id = initial_accounts()[1].account_id; let balance_acc_1 = sequencer.state.get_account_by_id(acc1_account_id).balance; let balance_acc_2 = sequencer.state.get_account_by_id(acc2_account_id).balance; @@ -485,47 +431,6 @@ mod tests { assert_eq!(20000, balance_acc_2); } - #[tokio::test] - async fn test_start_different_intial_accounts_balances() { - let acc1_account_id: Vec = vec![ - 27, 132, 197, 86, 123, 18, 100, 64, 153, 93, 62, 213, 170, 186, 5, 101, 215, 30, 24, - 52, 96, 72, 25, 255, 156, 23, 245, 233, 213, 221, 7, 143, - ]; - - let acc2_account_id: Vec = vec![ - 77, 75, 108, 209, 54, 16, 50, 202, 155, 210, 174, 185, 217, 0, 170, 77, 69, 217, 234, - 216, 10, 201, 66, 51, 116, 196, 81, 167, 37, 77, 7, 102, - ]; - - let initial_acc1 = AccountInitialData { - account_id: AccountId::from_str(&acc1_account_id.to_base58()).unwrap(), - balance: 10000, - }; - - let initial_acc2 = AccountInitialData { - account_id: AccountId::from_str(&acc2_account_id.to_base58()).unwrap(), - balance: 20000, - }; - - let initial_accounts = vec![initial_acc1, initial_acc2]; - - let config = setup_sequencer_config_variable_initial_accounts(initial_accounts); - let (sequencer, _mempool_handle) = - SequencerCoreWithMockClients::start_from_config(config.clone()).await; - - let acc1_account_id = config.initial_accounts[0].account_id; - let acc2_account_id = config.initial_accounts[1].account_id; - - assert_eq!( - 10000, - sequencer.state.get_account_by_id(acc1_account_id).balance - ); - assert_eq!( - 20000, - sequencer.state.get_account_by_id(acc2_account_id).balance - ); - } - #[test] fn test_transaction_pre_check_pass() { let tx = common::test_utils::produce_dummy_empty_transaction(); @@ -536,10 +441,10 @@ mod tests { #[tokio::test] async fn test_transaction_pre_check_native_transfer_valid() { - let (sequencer, _mempool_handle) = common_setup().await; + let (_sequencer, _mempool_handle) = common_setup().await; - let acc1 = sequencer.sequencer_config.initial_accounts[0].account_id; - let acc2 = sequencer.sequencer_config.initial_accounts[1].account_id; + let acc1 = initial_accounts()[0].account_id; + let acc2 = initial_accounts()[1].account_id; let sign_key1 = create_signing_key_for_account1(); @@ -555,8 +460,8 @@ mod tests { async fn test_transaction_pre_check_native_transfer_other_signature() { let (mut sequencer, _mempool_handle) = common_setup().await; - let acc1 = sequencer.sequencer_config.initial_accounts[0].account_id; - let acc2 = sequencer.sequencer_config.initial_accounts[1].account_id; + let acc1 = initial_accounts()[0].account_id; + let acc2 = initial_accounts()[1].account_id; let sign_key2 = create_signing_key_for_account2(); @@ -580,8 +485,8 @@ mod tests { async fn test_transaction_pre_check_native_transfer_sent_too_much() { let (mut sequencer, _mempool_handle) = common_setup().await; - let acc1 = sequencer.sequencer_config.initial_accounts[0].account_id; - let acc2 = sequencer.sequencer_config.initial_accounts[1].account_id; + let acc1 = initial_accounts()[0].account_id; + let acc2 = initial_accounts()[1].account_id; let sign_key1 = create_signing_key_for_account1(); @@ -607,8 +512,8 @@ mod tests { async fn test_transaction_execute_native_transfer() { let (mut sequencer, _mempool_handle) = common_setup().await; - let acc1 = sequencer.sequencer_config.initial_accounts[0].account_id; - let acc2 = sequencer.sequencer_config.initial_accounts[1].account_id; + let acc1 = initial_accounts()[0].account_id; + let acc2 = initial_accounts()[1].account_id; let sign_key1 = create_signing_key_for_account1(); @@ -669,8 +574,8 @@ mod tests { async fn test_replay_transactions_are_rejected_in_the_same_block() { let (mut sequencer, mempool_handle) = common_setup().await; - let acc1 = sequencer.sequencer_config.initial_accounts[0].account_id; - let acc2 = sequencer.sequencer_config.initial_accounts[1].account_id; + let acc1 = initial_accounts()[0].account_id; + let acc2 = initial_accounts()[1].account_id; let sign_key1 = create_signing_key_for_account1(); @@ -701,8 +606,8 @@ mod tests { async fn test_replay_transactions_are_rejected_in_different_blocks() { let (mut sequencer, mempool_handle) = common_setup().await; - let acc1 = sequencer.sequencer_config.initial_accounts[0].account_id; - let acc2 = sequencer.sequencer_config.initial_accounts[1].account_id; + let acc1 = initial_accounts()[0].account_id; + let acc2 = initial_accounts()[1].account_id; let sign_key1 = create_signing_key_for_account1(); @@ -736,8 +641,8 @@ mod tests { #[tokio::test] async fn test_restart_from_storage() { let config = setup_sequencer_config(); - let acc1_account_id = config.initial_accounts[0].account_id; - let acc2_account_id = config.initial_accounts[1].account_id; + let acc1_account_id = initial_accounts()[0].account_id; + let acc2_account_id = initial_accounts()[1].account_id; let balance_to_move = 13; // In the following code block a transaction will be processed that moves `balance_to_move` @@ -746,7 +651,7 @@ mod tests { { let (mut sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config.clone()).await; - let signing_key = PrivateKey::try_new([1; 32]).unwrap(); + let signing_key = create_signing_key_for_account1(); let tx = common::test_utils::create_transaction_native_token_transfer( acc1_account_id, @@ -777,11 +682,11 @@ mod tests { // Balances should be consistent with the stored block assert_eq!( balance_acc_1, - config.initial_accounts[0].balance - balance_to_move + initial_accounts()[0].balance - balance_to_move ); assert_eq!( balance_acc_2, - config.initial_accounts[1].balance + balance_to_move + initial_accounts()[1].balance + balance_to_move ); } @@ -828,15 +733,15 @@ mod tests { #[tokio::test] async fn test_produce_block_with_correct_prev_meta_after_restart() { let config = setup_sequencer_config(); - let acc1_account_id = config.initial_accounts[0].account_id; - let acc2_account_id = config.initial_accounts[1].account_id; + let acc1_account_id = initial_accounts()[0].account_id; + let acc2_account_id = initial_accounts()[1].account_id; // Step 1: Create initial database with some block metadata let expected_prev_meta = { let (mut sequencer, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config.clone()).await; - let signing_key = PrivateKey::try_new([1; 32]).unwrap(); + let signing_key = create_signing_key_for_account1(); // Add a transaction and produce a block to set up block metadata let tx = common::test_utils::create_transaction_native_token_transfer( @@ -861,7 +766,7 @@ mod tests { SequencerCoreWithMockClients::start_from_config(config.clone()).await; // Step 3: Submit a new transaction - let signing_key = PrivateKey::try_new([1; 32]).unwrap(); + let signing_key = create_signing_key_for_account1(); let tx = common::test_utils::create_transaction_native_token_transfer( acc1_account_id, 1, // Next nonce diff --git a/sequencer_rpc/Cargo.toml b/sequencer_rpc/Cargo.toml index 42aa978f..09641a00 100644 --- a/sequencer_rpc/Cargo.toml +++ b/sequencer_rpc/Cargo.toml @@ -10,6 +10,7 @@ common.workspace = true mempool.workspace = true sequencer_core = { workspace = true } bedrock_client.workspace = true +key_protocol.workspace = true anyhow.workspace = true serde_json.workspace = true diff --git a/sequencer_rpc/src/process.rs b/sequencer_rpc/src/process.rs index b3dca691..3fb7a6d9 100644 --- a/sequencer_rpc/src/process.rs +++ b/sequencer_rpc/src/process.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use actix_web::Error as HttpError; use base64::{Engine, engine::general_purpose}; use common::{ - block::{AccountInitialData, HashableBlockData}, + block::HashableBlockData, rpc_primitives::{ errors::RpcError, message::{Message, Request}, @@ -23,6 +23,7 @@ use common::{ transaction::{NSSATransaction, TransactionMalformationError}, }; use itertools::Itertools as _; +use key_protocol::initial_state::initial_accounts; use log::warn; use nssa::{self, program::Program}; use sequencer_core::{ @@ -201,13 +202,7 @@ impl JsonHandler let _get_initial_testnet_accounts_request = GetInitialTestnetAccountsRequest::parse(Some(request.params))?; - let initial_accounts: Vec = { - let state = self.sequencer_state.lock().await; - - state.sequencer_config().initial_accounts.clone() - }; - - respond(initial_accounts) + respond(initial_accounts()) } /// Returns the balance of the account at the given account_id. @@ -340,16 +335,14 @@ impl JsonHandler #[cfg(test)] mod tests { - use std::{str::FromStr as _, sync::Arc, time::Duration}; + use std::{sync::Arc, time::Duration}; - use base58::ToBase58; use base64::{Engine, engine::general_purpose}; use bedrock_client::BackoffConfig; use common::{ - block::AccountInitialData, config::BasicAuth, test_utils::sequencer_sign_key_for_testing, - transaction::NSSATransaction, + config::BasicAuth, test_utils::sequencer_sign_key_for_testing, transaction::NSSATransaction, }; - use nssa::AccountId; + use key_protocol::initial_state::{initial_accounts, initial_pub_accounts_private_keys}; use sequencer_core::{ config::{BedrockConfig, SequencerConfig}, mock::{MockBlockSettlementClient, MockIndexerClient, SequencerCoreWithMockClients}, @@ -366,27 +359,6 @@ mod tests { fn sequencer_config_for_tests() -> SequencerConfig { let tempdir = tempdir().unwrap(); let home = tempdir.path().to_path_buf(); - let acc1_id: Vec = vec![ - 148, 179, 206, 253, 199, 51, 82, 86, 232, 2, 152, 122, 80, 243, 54, 207, 237, 112, 83, - 153, 44, 59, 204, 49, 128, 84, 160, 227, 216, 149, 97, 102, - ]; - - let acc2_id: Vec = vec![ - 30, 145, 107, 3, 207, 73, 192, 230, 160, 63, 238, 207, 18, 69, 54, 216, 103, 244, 92, - 94, 124, 248, 42, 16, 141, 19, 119, 18, 14, 226, 140, 204, - ]; - - let initial_acc1 = AccountInitialData { - account_id: AccountId::from_str(&acc1_id.to_base58()).unwrap(), - balance: 10000, - }; - - let initial_acc2 = AccountInitialData { - account_id: AccountId::from_str(&acc2_id.to_base58()).unwrap(), - balance: 20000, - }; - - let initial_accounts = vec![initial_acc1, initial_acc2]; SequencerConfig { home, @@ -398,8 +370,6 @@ mod tests { mempool_max_size: 1000, block_create_timeout: Duration::from_secs(1), port: 8080, - initial_accounts, - initial_commitments: vec![], signing_key: *sequencer_sign_key_for_testing().value(), retry_pending_blocks_timeout: Duration::from_secs(60 * 4), bedrock_config: BedrockConfig { @@ -418,30 +388,18 @@ mod tests { } } - async fn components_for_tests() -> ( - JsonHandlerWithMockClients, - Vec, - NSSATransaction, - ) { + async fn components_for_tests() -> (JsonHandlerWithMockClients, NSSATransaction) { let config = sequencer_config_for_tests(); let (mut sequencer_core, mempool_handle) = SequencerCoreWithMockClients::start_from_config(config).await; - let initial_accounts = sequencer_core.sequencer_config().initial_accounts.clone(); - let signing_key = nssa::PrivateKey::try_new([1; 32]).unwrap(); + let signing_key = initial_pub_accounts_private_keys()[0].pub_sign_key.clone(); let balance_to_move = 10; let tx = common::test_utils::create_transaction_native_token_transfer( - AccountId::from_str( - &[ - 148, 179, 206, 253, 199, 51, 82, 86, 232, 2, 152, 122, 80, 243, 54, 207, 237, - 112, 83, 153, 44, 59, 204, 49, 128, 84, 160, 227, 216, 149, 97, 102, - ] - .to_base58(), - ) - .unwrap(), + initial_accounts()[0].account_id, 0, - AccountId::from_str(&[2; 32].to_base58()).unwrap(), + initial_accounts()[1].account_id, balance_to_move, signing_key, ); @@ -464,7 +422,6 @@ mod tests { mempool_handle, max_block_size, }, - initial_accounts, tx, ) } @@ -494,7 +451,7 @@ mod tests { #[actix_web::test] async fn test_get_account_balance_for_non_existent_account() { - let (json_handler, _, _) = components_for_tests().await; + let (json_handler, _) = components_for_tests().await; let request = serde_json::json!({ "jsonrpc": "2.0", "method": "get_account_balance", @@ -516,7 +473,7 @@ mod tests { #[actix_web::test] async fn test_get_account_balance_for_invalid_base58() { - let (json_handler, _, _) = components_for_tests().await; + let (json_handler, _) = components_for_tests().await; let request = serde_json::json!({ "jsonrpc": "2.0", "method": "get_account_balance", @@ -546,7 +503,7 @@ mod tests { #[actix_web::test] async fn test_get_account_balance_for_invalid_length() { - let (json_handler, _, _) = components_for_tests().await; + let (json_handler, _) = components_for_tests().await; let request = serde_json::json!({ "jsonrpc": "2.0", "method": "get_account_balance", @@ -576,9 +533,9 @@ mod tests { #[actix_web::test] async fn test_get_account_balance_for_existing_account() { - let (json_handler, initial_accounts, _) = components_for_tests().await; + let (json_handler, _) = components_for_tests().await; - let acc1_id = initial_accounts[0].account_id; + let acc1_id = initial_accounts()[0].account_id; let request = serde_json::json!({ "jsonrpc": "2.0", @@ -601,7 +558,7 @@ mod tests { #[actix_web::test] async fn test_get_accounts_nonces_for_non_existent_account() { - let (json_handler, _, _) = components_for_tests().await; + let (json_handler, _) = components_for_tests().await; let request = serde_json::json!({ "jsonrpc": "2.0", "method": "get_accounts_nonces", @@ -623,10 +580,10 @@ mod tests { #[actix_web::test] async fn test_get_accounts_nonces_for_existent_account() { - let (json_handler, initial_accounts, _) = components_for_tests().await; + let (json_handler, _) = components_for_tests().await; - let acc1_id = initial_accounts[0].account_id; - let acc2_id = initial_accounts[1].account_id; + let acc1_id = initial_accounts()[0].account_id; + let acc2_id = initial_accounts()[1].account_id; let request = serde_json::json!({ "jsonrpc": "2.0", @@ -649,7 +606,7 @@ mod tests { #[actix_web::test] async fn test_get_account_data_for_non_existent_account() { - let (json_handler, _, _) = components_for_tests().await; + let (json_handler, _) = components_for_tests().await; let request = serde_json::json!({ "jsonrpc": "2.0", "method": "get_account", @@ -676,7 +633,7 @@ mod tests { #[actix_web::test] async fn test_get_transaction_by_hash_for_non_existent_hash() { - let (json_handler, _, _) = components_for_tests().await; + let (json_handler, _) = components_for_tests().await; let request = serde_json::json!({ "jsonrpc": "2.0", "method": "get_transaction_by_hash", @@ -698,7 +655,7 @@ mod tests { #[actix_web::test] async fn test_get_transaction_by_hash_for_invalid_hex() { - let (json_handler, _, _) = components_for_tests().await; + let (json_handler, _) = components_for_tests().await; let request = serde_json::json!({ "jsonrpc": "2.0", "method": "get_transaction_by_hash", @@ -729,7 +686,7 @@ mod tests { #[actix_web::test] async fn test_get_transaction_by_hash_for_invalid_length() { - let (json_handler, _, _) = components_for_tests().await; + let (json_handler, _) = components_for_tests().await; let request = serde_json::json!({ "jsonrpc": "2.0", "method": "get_transaction_by_hash", @@ -760,7 +717,7 @@ mod tests { #[actix_web::test] async fn test_get_transaction_by_hash_for_existing_transaction() { - let (json_handler, _, tx) = components_for_tests().await; + let (json_handler, tx) = components_for_tests().await; let tx_hash_hex = hex::encode(tx.hash()); let expected_base64_encoded = general_purpose::STANDARD.encode(borsh::to_vec(&tx).unwrap()); diff --git a/wallet/src/chain_storage.rs b/wallet/src/chain_storage.rs index a4b22a31..b0794fc3 100644 --- a/wallet/src/chain_storage.rs +++ b/wallet/src/chain_storage.rs @@ -95,7 +95,7 @@ impl WalletChainStore { let mut public_init_acc_map = BTreeMap::new(); let mut private_init_acc_map = BTreeMap::new(); - for init_acc_data in config.initial_accounts.clone() { + for init_acc_data in InitialAccountData::create_initial_accounts_data() { match init_acc_data { InitialAccountData::Public(data) => { public_init_acc_map.insert(data.account_id, data.pub_sign_key); @@ -162,101 +162,7 @@ mod tests { }; use super::*; - use crate::config::{ - InitialAccountData, PersistentAccountDataPrivate, PersistentAccountDataPublic, - }; - - fn create_initial_accounts() -> Vec { - let initial_acc1 = serde_json::from_str( - r#"{ - "Public": { - "account_id": "6iArKUXxhUJqS7kCaPNhwMWt3ro71PDyBj7jwAyE2VQV", - "pub_sign_key": [ - 16, - 162, - 106, - 154, - 236, - 125, - 52, - 184, - 35, - 100, - 238, - 174, - 69, - 197, - 41, - 77, - 187, - 10, - 118, - 75, - 0, - 11, - 148, - 238, - 185, - 181, - 133, - 17, - 220, - 72, - 124, - 77 - ] - } - }"#, - ) - .unwrap(); - - let initial_acc2 = serde_json::from_str( - r#"{ - "Public": { - "account_id": "7wHg9sbJwc6h3NP1S9bekfAzB8CHifEcxKswCKUt3YQo", - "pub_sign_key": [ - 113, - 121, - 64, - 177, - 204, - 85, - 229, - 214, - 178, - 6, - 109, - 191, - 29, - 154, - 63, - 38, - 242, - 18, - 244, - 219, - 8, - 208, - 35, - 136, - 23, - 127, - 207, - 237, - 216, - 169, - 190, - 27 - ] - } - }"#, - ) - .unwrap(); - - let initial_accounts = vec![initial_acc1, initial_acc2]; - - initial_accounts - } + use crate::config::{PersistentAccountDataPrivate, PersistentAccountDataPublic}; fn create_sample_wallet_config() -> WalletConfig { WalletConfig { @@ -266,7 +172,6 @@ mod tests { seq_tx_poll_max_blocks: 5, seq_poll_max_retries: 10, seq_block_poll_max_amount: 100, - initial_accounts: create_initial_accounts(), basic_auth: None, } } diff --git a/wallet/src/cli/config.rs b/wallet/src/cli/config.rs index bc0e3662..6366a143 100644 --- a/wallet/src/cli/config.rs +++ b/wallet/src/cli/config.rs @@ -4,6 +4,7 @@ use clap::Subcommand; use crate::{ WalletCore, cli::{SubcommandReturnValue, WalletSubcommand}, + config::InitialAccountData, }; /// Represents generic config CLI subcommand @@ -68,7 +69,7 @@ impl WalletSubcommand for ConfigSubcommand { ); } "initial_accounts" => { - println!("{:#?}", wallet_core.storage.wallet_config.initial_accounts); + println!("{:#?}", InitialAccountData::create_initial_accounts_data()); } "basic_auth" => { if let Some(basic_auth) = &wallet_core.storage.wallet_config.basic_auth diff --git a/wallet/src/config.rs b/wallet/src/config.rs index 3780a065..09d0e3a7 100644 --- a/wallet/src/config.rs +++ b/wallet/src/config.rs @@ -8,9 +8,12 @@ use std::{ use anyhow::{Context as _, Result}; use common::config::BasicAuth; use humantime_serde; -use key_protocol::key_management::{ - KeyChain, - key_tree::{ +use key_protocol::{ + initial_state::{ + PrivateAccountPrivateInitialData, PublicAccountPrivateInitialData, + initial_priv_accounts_private_keys, initial_pub_accounts_private_keys, + }, + key_management::key_tree::{ chain_index::ChainIndex, keys_private::ChildKeysPrivate, keys_public::ChildKeysPublic, }, }; @@ -18,12 +21,6 @@ use log::warn; use serde::{Deserialize, Serialize}; use url::Url; -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InitialAccountDataPublic { - pub account_id: nssa::AccountId, - pub pub_sign_key: nssa::PrivateKey, -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PersistentAccountDataPublic { pub account_id: nssa::AccountId, @@ -31,13 +28,6 @@ pub struct PersistentAccountDataPublic { pub data: ChildKeysPublic, } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InitialAccountDataPrivate { - pub account_id: nssa::AccountId, - pub account: nssa_core::account::Account, - pub key_chain: KeyChain, -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PersistentAccountDataPrivate { pub account_id: nssa::AccountId, @@ -51,8 +41,21 @@ pub struct PersistentAccountDataPrivate { #[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, Serialize, Deserialize)] pub enum InitialAccountData { - Public(InitialAccountDataPublic), - Private(InitialAccountDataPrivate), + Public(PublicAccountPrivateInitialData), + Private(PrivateAccountPrivateInitialData), +} + +impl InitialAccountData { + pub(crate) fn create_initial_accounts_data() -> Vec { + let pub_data = initial_pub_accounts_private_keys(); + let priv_data = initial_priv_accounts_private_keys(); + + pub_data + .into_iter() + .map(Into::into) + .chain(priv_data.into_iter().map(Into::into)) + .collect() + } } // Big difference in enum variants sizes @@ -130,14 +133,14 @@ impl PersistentAccountData { } } -impl From for InitialAccountData { - fn from(value: InitialAccountDataPublic) -> Self { +impl From for InitialAccountData { + fn from(value: PublicAccountPrivateInitialData) -> Self { Self::Public(value) } } -impl From for InitialAccountData { - fn from(value: InitialAccountDataPrivate) -> Self { +impl From for InitialAccountData { + fn from(value: PrivateAccountPrivateInitialData) -> Self { Self::Private(value) } } @@ -195,8 +198,6 @@ pub struct WalletConfig { pub seq_poll_max_retries: u64, /// Max amount of blocks to poll in one request pub seq_block_poll_max_amount: u64, - /// Initial accounts for wallet - pub initial_accounts: Vec, /// Basic authentication credentials #[serde(skip_serializing_if = "Option::is_none")] pub basic_auth: Option, @@ -212,291 +213,6 @@ impl Default for WalletConfig { seq_poll_max_retries: 5, seq_block_poll_max_amount: 100, basic_auth: None, - initial_accounts: { - let init_acc_json = r#" - [ - { - "Public": { - "account_id": "6iArKUXxhUJqS7kCaPNhwMWt3ro71PDyBj7jwAyE2VQV", - "pub_sign_key": [ - 16, - 162, - 106, - 154, - 236, - 125, - 52, - 184, - 35, - 100, - 238, - 174, - 69, - 197, - 41, - 77, - 187, - 10, - 118, - 75, - 0, - 11, - 148, - 238, - 185, - 181, - 133, - 17, - 220, - 72, - 124, - 77 - ] - } - }, - { - "Public": { - "account_id": "7wHg9sbJwc6h3NP1S9bekfAzB8CHifEcxKswCKUt3YQo", - "pub_sign_key": [ - 113, - 121, - 64, - 177, - 204, - 85, - 229, - 214, - 178, - 6, - 109, - 191, - 29, - 154, - 63, - 38, - 242, - 18, - 244, - 219, - 8, - 208, - 35, - 136, - 23, - 127, - 207, - 237, - 216, - 169, - 190, - 27 - ] - } - }, - { - "Private": { - "account_id": "FpdcxBrMkHWqXCBQ6FG98eYfWGY6jWZRsKNSi1FwDMxy", - "account": { - "program_owner": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0 - ], - "balance": 10000, - "data": [], - "nonce": 0 - }, - "key_chain": { - "secret_spending_key": [ - 239, - 27, - 159, - 83, - 199, - 194, - 132, - 33, - 20, - 28, - 217, - 103, - 101, - 57, - 27, - 125, - 84, - 57, - 19, - 86, - 98, - 135, - 161, - 221, - 108, - 125, - 152, - 174, - 161, - 64, - 16, - 200 - ], - "private_key_holder": { - "nullifier_secret_key": [ - 71, - 195, - 16, - 119, - 0, - 98, - 35, - 106, - 139, - 82, - 145, - 50, - 27, - 140, - 206, - 19, - 53, - 122, - 166, - 76, - 195, - 0, - 16, - 19, - 21, - 143, - 155, - 119, - 9, - 200, - 81, - 105 - ], - "viewing_secret_key": [ - 5, - 117, - 221, - 27, - 236, - 199, - 53, - 22, - 249, - 231, - 98, - 147, - 213, - 116, - 191, - 82, - 188, - 148, - 175, - 98, - 139, - 52, - 232, - 249, - 220, - 217, - 83, - 58, - 112, - 155, - 197, - 196 - ] - }, - "nullifer_public_key": [ - 177, - 64, - 1, - 11, - 87, - 38, - 254, - 159, - 231, - 165, - 1, - 94, - 64, - 137, - 243, - 76, - 249, - 101, - 251, - 129, - 33, - 101, - 189, - 30, - 42, - 11, - 191, - 34, - 103, - 186, - 227, - 230 - ], - "viewing_public_key": [ - 2, 69, 126, 43, 158, 209, 172, 144, 23, 185, 208, 25, 163, 166, 176, 200, 225, 251, 106, 211, 4, 199, 112, 243, 207, 144, 135, 56, 157, 167, 32, 219, 38] - } - } - }, - { - "Private": { - "account_id": "E8HwiTyQe4H9HK7icTvn95HQMnzx49mP9A2ddtMLpNaN", - "account": { - "program_owner": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0 - ], - "balance": 20000, - "data": [], - "nonce": 0 - }, - "key_chain": { - "secret_spending_key": [ - 48, 175, 124, 10, 230, 240, 166, 14, 249, 254, 157, 226, 208, 124, 122, 177, 203, 139, 192, 180, 43, 120, 55, 151, 50, 21, 113, 22, 254, 83, 148, 56], - "private_key_holder": { - "nullifier_secret_key": [ - 99, 82, 190, 140, 234, 10, 61, 163, 15, 211, 179, 54, 70, 166, 87, 5, 182, 68, 117, 244, 217, 23, 99, 9, 4, 177, 230, 125, 109, 91, 160, 30 - ], - "viewing_secret_key": [ - 205, 32, 76, 251, 255, 236, 96, 119, 61, 111, 65, 100, 75, 218, 12, 22, 17, 170, 55, 226, 21, 154, 161, 34, 208, 74, 27, 1, 119, 13, 88, 128 - ] - }, - "nullifer_public_key": [ - 32, 67, 72, 164, 106, 53, 66, 239, 141, 15, 52, 230, 136, 177, 2, 236, 207, 243, 134, 135, 210, 143, 87, 232, 215, 128, 194, 120, 113, 224, 4, 165 - ], - "viewing_public_key": [ - 2, 79, 110, 46, 203, 29, 206, 205, 18, 86, 27, 189, 104, 103, 113, 181, 110, 53, 78, 172, 11, 171, 190, 18, 126, 214, 81, 77, 192, 154, 58, 195, 238 - ] - } - } - } - ] - "#; - serde_json::from_str(init_acc_json).unwrap() - }, } } } @@ -546,7 +262,6 @@ impl WalletConfig { seq_tx_poll_max_blocks, seq_poll_max_retries, seq_block_poll_max_amount, - initial_accounts, basic_auth, } = self; @@ -557,7 +272,6 @@ impl WalletConfig { seq_tx_poll_max_blocks: o_seq_tx_poll_max_blocks, seq_poll_max_retries: o_seq_poll_max_retries, seq_block_poll_max_amount: o_seq_block_poll_max_amount, - initial_accounts: o_initial_accounts, basic_auth: o_basic_auth, } = overrides; @@ -585,10 +299,6 @@ impl WalletConfig { warn!("Overriding wallet config 'seq_block_poll_max_amount' to {v}"); *seq_block_poll_max_amount = v; } - if let Some(v) = o_initial_accounts { - warn!("Overriding wallet config 'initial_accounts' to {v:#?}"); - *initial_accounts = v; - } if let Some(v) = o_basic_auth { warn!("Overriding wallet config 'basic_auth' to {v:#?}"); *basic_auth = v; diff --git a/wallet/src/helperfunctions.rs b/wallet/src/helperfunctions.rs index 20c04968..f7978f87 100644 --- a/wallet/src/helperfunctions.rs +++ b/wallet/src/helperfunctions.rs @@ -2,7 +2,10 @@ use std::{collections::HashMap, path::PathBuf, str::FromStr}; use anyhow::Result; use base64::{Engine, engine::general_purpose::STANDARD as BASE64}; -use key_protocol::key_protocol_core::NSSAUserData; +use key_protocol::{ + initial_state::{PrivateAccountPrivateInitialData, PublicAccountPrivateInitialData}, + key_protocol_core::NSSAUserData, +}; use nssa::Account; use nssa_core::account::Nonce; use rand::{RngCore, rngs::OsRng}; @@ -11,8 +14,8 @@ use serde::Serialize; use crate::{ HOME_DIR_ENV_VAR, config::{ - InitialAccountData, InitialAccountDataPrivate, InitialAccountDataPublic, Label, - PersistentAccountDataPrivate, PersistentAccountDataPublic, PersistentStorage, + InitialAccountData, Label, PersistentAccountDataPrivate, PersistentAccountDataPublic, + PersistentStorage, }, }; @@ -89,7 +92,7 @@ pub fn produce_data_for_storage( for (account_id, key) in &user_data.default_pub_account_signing_keys { vec_for_storage.push( - InitialAccountData::Public(InitialAccountDataPublic { + InitialAccountData::Public(PublicAccountPrivateInitialData { account_id: *account_id, pub_sign_key: key.clone(), }) @@ -99,7 +102,7 @@ pub fn produce_data_for_storage( for (account_id, (key_chain, account)) in &user_data.default_user_private_accounts { vec_for_storage.push( - InitialAccountData::Private(InitialAccountDataPrivate { + InitialAccountData::Private(PrivateAccountPrivateInitialData { account_id: *account_id, account: account.clone(), key_chain: key_chain.clone(), From a9a0c386ad68dd63065962c3366d502095acaa55 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Mon, 16 Mar 2026 15:15:35 +0200 Subject: [PATCH 08/22] fix: integartion tests fix --- indexer/core/src/config.rs | 7 +++++ indexer/core/src/lib.rs | 47 ++++++++++++++++++++++++++++++--- integration_tests/src/config.rs | 35 +++++++++++++----------- sequencer_core/src/config.rs | 7 +++++ sequencer_core/src/lib.rs | 42 ++++++++++++++++++++++++++++- sequencer_rpc/src/process.rs | 2 ++ wallet/src/chain_storage.rs | 44 ++++++++++++++++++++++-------- wallet/src/config.rs | 9 +++++++ 8 files changed, 162 insertions(+), 31 deletions(-) diff --git a/indexer/core/src/config.rs b/indexer/core/src/config.rs index fad10ec5..2b4bfa47 100644 --- a/indexer/core/src/config.rs +++ b/indexer/core/src/config.rs @@ -9,6 +9,9 @@ use anyhow::{Context as _, Result}; pub use bedrock_client::BackoffConfig; use common::config::BasicAuth; use humantime_serde; +use key_protocol::initial_state::{ + PrivateAccountPublicInitialData, PublicAccountPublicInitialData, +}; pub use logos_blockchain_core::mantle::ops::channel::ChannelId; use serde::{Deserialize, Serialize}; use url::Url; @@ -32,6 +35,10 @@ pub struct IndexerConfig { pub consensus_info_polling_interval: Duration, pub bedrock_client_config: ClientConfig, pub channel_id: ChannelId, + #[serde(skip_serializing_if = "Option::is_none")] + pub initial_accounts: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub initial_commitments: Option>, } impl IndexerConfig { diff --git a/indexer/core/src/lib.rs b/indexer/core/src/lib.rs index 002a6c91..0ab190d5 100644 --- a/indexer/core/src/lib.rs +++ b/indexer/core/src/lib.rs @@ -3,7 +3,7 @@ use std::collections::VecDeque; use anyhow::Result; use bedrock_client::{BedrockClient, HeaderId}; use common::{ - HashType, + HashType, PINATA_BASE58, block::{Block, HashableBlockData}, }; use key_protocol::initial_state::initial_state_testnet; @@ -12,6 +12,7 @@ use logos_blockchain_core::mantle::{ Op, SignedMantleTx, ops::channel::{ChannelId, inscribe::InscriptionOp}, }; +use nssa::V02State; use crate::{block_store::IndexerStore, config::IndexerConfig}; @@ -56,8 +57,48 @@ impl IndexerCore { let channel_genesis_msg_id = [0; 32]; let start_block = hashable_data.into_pending_block(&signing_key, channel_genesis_msg_id); - // ToDo: replace with `initial_state()` after testnet - let state = initial_state_testnet(); + let initial_commitments: Option> = config + .initial_commitments + .clone() + .map(|initial_commitments| { + initial_commitments + .iter() + .map(|init_comm_data| { + let npk = &init_comm_data.npk; + + let mut acc = init_comm_data.account.clone(); + + acc.program_owner = + nssa::program::Program::authenticated_transfer_program().id(); + + nssa_core::Commitment::new(npk, &acc) + }) + .collect() + }); + + let init_accs: Option> = + config.initial_accounts.clone().map(|initial_accounts| { + initial_accounts + .iter() + .map(|acc_data| (acc_data.account_id, acc_data.balance)) + .collect() + }); + + // If initial commitments or accounts are present in config, need to construct state from + // them + let state = if initial_commitments.is_some() || init_accs.is_some() { + let mut state = V02State::new_with_genesis_accounts( + &init_accs.unwrap_or_default(), + &initial_commitments.unwrap_or_default(), + ); + + // ToDo: Remove after testnet + state.add_pinata_program(PINATA_BASE58.parse().unwrap()); + + state + } else { + initial_state_testnet() + }; let home = config.home.join("rocksdb"); diff --git a/integration_tests/src/config.rs b/integration_tests/src/config.rs index 8dd18a25..e1922cc4 100644 --- a/integration_tests/src/config.rs +++ b/integration_tests/src/config.rs @@ -2,16 +2,19 @@ use std::{net::SocketAddr, path::PathBuf, time::Duration}; use anyhow::{Context, Result}; use bytesize::ByteSize; -use common::block::{AccountInitialData, CommitmentsInitialData}; use indexer_service::{BackoffConfig, ChannelId, ClientConfig, IndexerConfig}; -use key_protocol::key_management::KeyChain; +use key_protocol::{ + initial_state::{ + PrivateAccountPrivateInitialData, PrivateAccountPublicInitialData, + PublicAccountPrivateInitialData, PublicAccountPublicInitialData, + }, + key_management::KeyChain, +}; use nssa::{Account, AccountId, PrivateKey, PublicKey}; use nssa_core::{account::Data, program::DEFAULT_PROGRAM_ID}; use sequencer_core::config::{BedrockConfig, SequencerConfig}; use url::Url; -use wallet::config::{ - InitialAccountData, InitialAccountDataPrivate, InitialAccountDataPublic, WalletConfig, -}; +use wallet::config::{InitialAccountData, WalletConfig}; pub fn indexer_config( bedrock_addr: SocketAddr, @@ -30,8 +33,8 @@ pub fn indexer_config( max_retries: 10, }, }, - initial_accounts: initial_data.sequencer_initial_accounts(), - initial_commitments: initial_data.sequencer_initial_commitments(), + initial_accounts: Some(initial_data.sequencer_initial_accounts()), + initial_commitments: Some(initial_data.sequencer_initial_commitments()), signing_key: [37; 32], channel_id: bedrock_channel_id(), }) @@ -81,8 +84,8 @@ pub fn sequencer_config( block_create_timeout, retry_pending_blocks_timeout: Duration::from_secs(120), port: 0, - initial_accounts: initial_data.sequencer_initial_accounts(), - initial_commitments: initial_data.sequencer_initial_commitments(), + initial_accounts: Some(initial_data.sequencer_initial_accounts()), + initial_commitments: Some(initial_data.sequencer_initial_commitments()), signing_key: [37; 32], bedrock_config: BedrockConfig { backoff: BackoffConfig { @@ -111,7 +114,7 @@ pub fn wallet_config( seq_tx_poll_max_blocks: 15, seq_poll_max_retries: 10, seq_block_poll_max_amount: 100, - initial_accounts: initial_data.wallet_initial_accounts(), + initial_accounts: Some(initial_data.wallet_initial_accounts()), basic_auth: None, }) } @@ -184,13 +187,13 @@ impl InitialData { } } - fn sequencer_initial_accounts(&self) -> Vec { + fn sequencer_initial_accounts(&self) -> Vec { self.public_accounts .iter() .map(|(priv_key, balance)| { let pub_key = PublicKey::new_from_private_key(priv_key); let account_id = AccountId::from(&pub_key); - AccountInitialData { + PublicAccountPublicInitialData { account_id, balance: *balance, } @@ -198,10 +201,10 @@ impl InitialData { .collect() } - fn sequencer_initial_commitments(&self) -> Vec { + fn sequencer_initial_commitments(&self) -> Vec { self.private_accounts .iter() - .map(|(key_chain, account)| CommitmentsInitialData { + .map(|(key_chain, account)| PrivateAccountPublicInitialData { npk: key_chain.nullifer_public_key.clone(), account: account.clone(), }) @@ -214,14 +217,14 @@ impl InitialData { .map(|(priv_key, _)| { let pub_key = PublicKey::new_from_private_key(priv_key); let account_id = AccountId::from(&pub_key); - InitialAccountData::Public(InitialAccountDataPublic { + InitialAccountData::Public(PublicAccountPrivateInitialData { account_id, pub_sign_key: priv_key.clone(), }) }) .chain(self.private_accounts.iter().map(|(key_chain, account)| { let account_id = AccountId::from(&key_chain.nullifer_public_key); - InitialAccountData::Private(InitialAccountDataPrivate { + InitialAccountData::Private(PrivateAccountPrivateInitialData { account_id, account: account.clone(), key_chain: key_chain.clone(), diff --git a/sequencer_core/src/config.rs b/sequencer_core/src/config.rs index 05a0dc72..e92c3794 100644 --- a/sequencer_core/src/config.rs +++ b/sequencer_core/src/config.rs @@ -10,6 +10,9 @@ use bedrock_client::BackoffConfig; use bytesize::ByteSize; use common::config::BasicAuth; use humantime_serde; +use key_protocol::initial_state::{ + PrivateAccountPublicInitialData, PublicAccountPublicInitialData, +}; use logos_blockchain_core::mantle::ops::channel::ChannelId; use serde::{Deserialize, Serialize}; use url::Url; @@ -46,6 +49,10 @@ pub struct SequencerConfig { pub bedrock_config: BedrockConfig, /// Indexer RPC URL pub indexer_rpc_url: Url, + #[serde(skip_serializing_if = "Option::is_none")] + pub initial_accounts: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub initial_commitments: Option>, } #[derive(Clone, Serialize, Deserialize)] diff --git a/sequencer_core/src/lib.rs b/sequencer_core/src/lib.rs index fdb65d2a..9a103a5f 100644 --- a/sequencer_core/src/lib.rs +++ b/sequencer_core/src/lib.rs @@ -14,6 +14,7 @@ use key_protocol::initial_state::initial_state; use log::{error, info, warn}; use logos_blockchain_key_management_system_service::keys::{ED25519_SECRET_KEY_SIZE, Ed25519Key}; use mempool::{MemPool, MemPoolHandle}; +use nssa::V02State; use crate::{ block_settlement_client::{BlockSettlementClient, BlockSettlementClientTrait, MsgId}, @@ -101,7 +102,44 @@ impl SequencerCore> = config + .initial_commitments + .clone() + .map(|initial_commitments| { + initial_commitments + .iter() + .map(|init_comm_data| { + let npk = &init_comm_data.npk; + + let mut acc = init_comm_data.account.clone(); + + acc.program_owner = + nssa::program::Program::authenticated_transfer_program().id(); + + nssa_core::Commitment::new(npk, &acc) + }) + .collect() + }); + + let init_accs: Option> = + config.initial_accounts.clone().map(|initial_accounts| { + initial_accounts + .iter() + .map(|acc_data| (acc_data.account_id, acc_data.balance)) + .collect() + }); + + // If initial commitments or accounts are present in config, need to construct state + // from them + if initial_commitments.is_some() || init_accs.is_some() { + V02State::new_with_genesis_accounts( + &init_accs.unwrap_or_default(), + &initial_commitments.unwrap_or_default(), + ) + } else { + initial_state() + } } }; @@ -379,6 +417,8 @@ mod tests { }, retry_pending_blocks_timeout: Duration::from_secs(60 * 4), indexer_rpc_url: "ws://localhost:8779".parse().unwrap(), + initial_accounts: None, + initial_commitments: None, } } diff --git a/sequencer_rpc/src/process.rs b/sequencer_rpc/src/process.rs index 3fb7a6d9..75ace675 100644 --- a/sequencer_rpc/src/process.rs +++ b/sequencer_rpc/src/process.rs @@ -385,6 +385,8 @@ mod tests { }), }, indexer_rpc_url: "ws://localhost:8779".parse().unwrap(), + initial_accounts: None, + initial_commitments: None, } } diff --git a/wallet/src/chain_storage.rs b/wallet/src/chain_storage.rs index b0794fc3..2c998c8a 100644 --- a/wallet/src/chain_storage.rs +++ b/wallet/src/chain_storage.rs @@ -95,18 +95,39 @@ impl WalletChainStore { let mut public_init_acc_map = BTreeMap::new(); let mut private_init_acc_map = BTreeMap::new(); - for init_acc_data in InitialAccountData::create_initial_accounts_data() { - match init_acc_data { - InitialAccountData::Public(data) => { - public_init_acc_map.insert(data.account_id, data.pub_sign_key); + // If initial accounts are present in config, need to construct state from them + if let Some(initial_accounts) = config.initial_accounts.clone() { + for init_acc_data in initial_accounts { + match init_acc_data { + InitialAccountData::Public(data) => { + public_init_acc_map.insert(data.account_id, data.pub_sign_key); + } + InitialAccountData::Private(data) => { + let mut account = data.account; + // TODO: Program owner is only known after code is compiled and can't be set + // in the config. Therefore we overwrite it here on + // startup. Fix this when program id can be fetched + // from the node and queried from the wallet. + account.program_owner = Program::authenticated_transfer_program().id(); + private_init_acc_map.insert(data.account_id, (data.key_chain, account)); + } } - InitialAccountData::Private(data) => { - let mut account = data.account; - // TODO: Program owner is only known after code is compiled and can't be set in - // the config. Therefore we overwrite it here on startup. Fix this when program - // id can be fetched from the node and queried from the wallet. - account.program_owner = Program::authenticated_transfer_program().id(); - private_init_acc_map.insert(data.account_id, (data.key_chain, account)); + } + } else { + for init_acc_data in InitialAccountData::create_initial_accounts_data() { + match init_acc_data { + InitialAccountData::Public(data) => { + public_init_acc_map.insert(data.account_id, data.pub_sign_key); + } + InitialAccountData::Private(data) => { + let mut account = data.account; + // TODO: Program owner is only known after code is compiled and can't be set + // in the config. Therefore we overwrite it here on + // startup. Fix this when program id can be fetched + // from the node and queried from the wallet. + account.program_owner = Program::authenticated_transfer_program().id(); + private_init_acc_map.insert(data.account_id, (data.key_chain, account)); + } } } } @@ -173,6 +194,7 @@ mod tests { seq_poll_max_retries: 10, seq_block_poll_max_amount: 100, basic_auth: None, + initial_accounts: None, } } diff --git a/wallet/src/config.rs b/wallet/src/config.rs index 09d0e3a7..a5422d17 100644 --- a/wallet/src/config.rs +++ b/wallet/src/config.rs @@ -201,6 +201,8 @@ pub struct WalletConfig { /// Basic authentication credentials #[serde(skip_serializing_if = "Option::is_none")] pub basic_auth: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub initial_accounts: Option>, } impl Default for WalletConfig { @@ -213,6 +215,7 @@ impl Default for WalletConfig { seq_poll_max_retries: 5, seq_block_poll_max_amount: 100, basic_auth: None, + initial_accounts: None, } } } @@ -263,6 +266,7 @@ impl WalletConfig { seq_poll_max_retries, seq_block_poll_max_amount, basic_auth, + initial_accounts, } = self; let WalletConfigOverrides { @@ -273,6 +277,7 @@ impl WalletConfig { seq_poll_max_retries: o_seq_poll_max_retries, seq_block_poll_max_amount: o_seq_block_poll_max_amount, basic_auth: o_basic_auth, + initial_accounts: o_initial_accounts, } = overrides; if let Some(v) = o_override_rust_log { @@ -303,5 +308,9 @@ impl WalletConfig { warn!("Overriding wallet config 'basic_auth' to {v:#?}"); *basic_auth = v; } + if let Some(v) = o_initial_accounts { + warn!("Overriding wallet config 'initial_accounts' to {v:#?}"); + *initial_accounts = v; + } } } From 6b6b3a0fe8a3690d281155887016fc1d85347295 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Mon, 16 Mar 2026 15:56:26 +0200 Subject: [PATCH 09/22] fix: dep fix --- Cargo.lock | 2 -- sequencer_core/Cargo.toml | 1 - sequencer_rpc/Cargo.toml | 1 - 3 files changed, 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2e2a65c..f326b1a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7454,7 +7454,6 @@ name = "sequencer_core" version = "0.1.0" dependencies = [ "anyhow", - "base58", "bedrock_client", "borsh", "bytesize", @@ -7486,7 +7485,6 @@ dependencies = [ "actix-cors", "actix-web", "anyhow", - "base58", "base64 0.22.1", "bedrock_client", "borsh", diff --git a/sequencer_core/Cargo.toml b/sequencer_core/Cargo.toml index 1d4e604e..fa740c42 100644 --- a/sequencer_core/Cargo.toml +++ b/sequencer_core/Cargo.toml @@ -13,7 +13,6 @@ mempool.workspace = true bedrock_client.workspace = true key_protocol.workspace = true -base58.workspace = true anyhow.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/sequencer_rpc/Cargo.toml b/sequencer_rpc/Cargo.toml index 09641a00..c4907e49 100644 --- a/sequencer_rpc/Cargo.toml +++ b/sequencer_rpc/Cargo.toml @@ -18,7 +18,6 @@ log.workspace = true serde.workspace = true actix-cors.workspace = true futures.workspace = true -base58.workspace = true hex.workspace = true tempfile.workspace = true base64.workspace = true From b5afcba4c4527d1583c74ced14d36d22b45b7906 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Mon, 16 Mar 2026 17:02:22 +0200 Subject: [PATCH 10/22] fix: moved constants into consts --- key_protocol/src/initial_state.rs | 180 +++++++++++++++++------------- 1 file changed, 104 insertions(+), 76 deletions(-) diff --git a/key_protocol/src/initial_state.rs b/key_protocol/src/initial_state.rs index 8011d471..3af9182f 100644 --- a/key_protocol/src/initial_state.rs +++ b/key_protocol/src/initial_state.rs @@ -8,6 +8,74 @@ use crate::key_management::{ secret_holders::{PrivateKeyHolder, SecretSpendingKey}, }; +const PRIVATE_KEY_PUB_ACC_A: [u8; 32] = [ + 16, 162, 106, 154, 236, 125, 52, 184, 35, 100, 238, 174, 69, 197, 41, 77, 187, 10, 118, 75, 0, + 11, 148, 238, 185, 181, 133, 17, 220, 72, 124, 77, +]; + +const PRIVATE_KEY_PUB_ACC_B: [u8; 32] = [ + 113, 121, 64, 177, 204, 85, 229, 214, 178, 6, 109, 191, 29, 154, 63, 38, 242, 18, 244, 219, 8, + 208, 35, 136, 23, 127, 207, 237, 216, 169, 190, 27, +]; + +const SSK_PRIV_ACC_A: [u8; 32] = [ + 93, 13, 190, 240, 250, 33, 108, 195, 176, 40, 144, 61, 4, 28, 58, 112, 53, 161, 42, 238, 155, + 27, 23, 176, 208, 121, 15, 229, 165, 180, 99, 143, +]; + +const SSK_PRIV_ACC_B: [u8; 32] = [ + 48, 175, 124, 10, 230, 240, 166, 14, 249, 254, 157, 226, 208, 124, 122, 177, 203, 139, 192, + 180, 43, 120, 55, 151, 50, 21, 113, 22, 254, 83, 148, 56, +]; + +const NSK_PRIV_ACC_A: [u8; 32] = [ + 25, 21, 186, 59, 180, 224, 101, 64, 163, 208, 228, 43, 13, 185, 100, 123, 156, 47, 80, 179, 72, + 51, 115, 11, 180, 99, 21, 201, 48, 194, 118, 144, +]; + +const NSK_PRIV_ACC_B: [u8; 32] = [ + 99, 82, 190, 140, 234, 10, 61, 163, 15, 211, 179, 54, 70, 166, 87, 5, 182, 68, 117, 244, 217, + 23, 99, 9, 4, 177, 230, 125, 109, 91, 160, 30, +]; + +const VSK_PRIV_ACC_A: [u8; 32] = [ + 5, 85, 114, 119, 141, 187, 202, 170, 122, 253, 198, 81, 150, 8, 155, 21, 192, 65, 24, 124, 116, + 98, 110, 106, 137, 90, 165, 239, 80, 13, 222, 30, +]; + +const VSK_PRIV_ACC_B: [u8; 32] = [ + 205, 32, 76, 251, 255, 236, 96, 119, 61, 111, 65, 100, 75, 218, 12, 22, 17, 170, 55, 226, 21, + 154, 161, 34, 208, 74, 27, 1, 119, 13, 88, 128, +]; + +const VPK_PRIV_ACC_A: [u8; 33] = [ + 2, 210, 206, 38, 213, 4, 182, 198, 220, 47, 93, 148, 61, 84, 148, 250, 158, 45, 8, 81, 48, 80, + 46, 230, 87, 210, 47, 204, 76, 58, 214, 167, 81, +]; + +const VPK_PRIV_ACC_B: [u8; 33] = [ + 2, 79, 110, 46, 203, 29, 206, 205, 18, 86, 27, 189, 104, 103, 113, 181, 110, 53, 78, 172, 11, + 171, 190, 18, 126, 214, 81, 77, 192, 154, 58, 195, 238, +]; + +const NPK_PRIV_ACC_A: [u8; 32] = [ + 167, 108, 50, 153, 74, 47, 151, 188, 140, 79, 195, 31, 181, 9, 40, 167, 201, 32, 175, 129, 45, + 245, 223, 193, 210, 170, 247, 128, 167, 140, 155, 129, +]; + +const NPK_PRIV_ACC_B: [u8; 32] = [ + 32, 67, 72, 164, 106, 53, 66, 239, 141, 15, 52, 230, 136, 177, 2, 236, 207, 243, 134, 135, 210, + 143, 87, 232, 215, 128, 194, 120, 113, 224, 4, 165, +]; + +const DEFAULT_PROGRAM_OWNER: [u32; 8] = [0, 0, 0, 0, 0, 0, 0, 0]; + +const PUB_ACC_A_INITIAL_BALANCE: u128 = 10000; +const PUB_ACC_B_INITIAL_BALANCE: u128 = 20000; + +const PRIV_ACC_A_INITIAL_BALANCE: u128 = 10000; +const PRIV_ACC_B_INITIAL_BALANCE: u128 = 20000; + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct PublicAccountPublicInitialData { pub account_id: AccountId, @@ -34,17 +102,9 @@ pub struct PrivateAccountPrivateInitialData { } pub fn initial_pub_accounts_private_keys() -> Vec { - let acc1_pub_sign_key = PrivateKey::try_new([ - 16, 162, 106, 154, 236, 125, 52, 184, 35, 100, 238, 174, 69, 197, 41, 77, 187, 10, 118, 75, - 0, 11, 148, 238, 185, 181, 133, 17, 220, 72, 124, 77, - ]) - .unwrap(); + let acc1_pub_sign_key = PrivateKey::try_new(PRIVATE_KEY_PUB_ACC_A).unwrap(); - let acc2_pub_sign_key = PrivateKey::try_new([ - 113, 121, 64, 177, 204, 85, 229, 214, 178, 6, 109, 191, 29, 154, 63, 38, 242, 18, 244, 219, - 8, 208, 35, 136, 23, 127, 207, 237, 216, 169, 190, 27, - ]) - .unwrap(); + let acc2_pub_sign_key = PrivateKey::try_new(PRIVATE_KEY_PUB_ACC_B).unwrap(); vec![ PublicAccountPrivateInitialData { @@ -60,61 +120,31 @@ pub fn initial_pub_accounts_private_keys() -> Vec Vec { let key_chain_1 = KeyChain { - secret_spending_key: SecretSpendingKey([ - 93, 13, 190, 240, 250, 33, 108, 195, 176, 40, 144, 61, 4, 28, 58, 112, 53, 161, 42, - 238, 155, 27, 23, 176, 208, 121, 15, 229, 165, 180, 99, 143, - ]), + secret_spending_key: SecretSpendingKey(SSK_PRIV_ACC_A), private_key_holder: PrivateKeyHolder { - nullifier_secret_key: [ - 25, 21, 186, 59, 180, 224, 101, 64, 163, 208, 228, 43, 13, 185, 100, 123, 156, 47, - 80, 179, 72, 51, 115, 11, 180, 99, 21, 201, 48, 194, 118, 144, - ], - viewing_secret_key: [ - 5, 85, 114, 119, 141, 187, 202, 170, 122, 253, 198, 81, 150, 8, 155, 21, 192, 65, - 24, 124, 116, 98, 110, 106, 137, 90, 165, 239, 80, 13, 222, 30, - ], + nullifier_secret_key: NSK_PRIV_ACC_A, + viewing_secret_key: VSK_PRIV_ACC_A, }, - nullifer_public_key: NullifierPublicKey([ - 167, 108, 50, 153, 74, 47, 151, 188, 140, 79, 195, 31, 181, 9, 40, 167, 201, 32, 175, - 129, 45, 245, 223, 193, 210, 170, 247, 128, 167, 140, 155, 129, - ]), - viewing_public_key: Secp256k1Point(vec![ - 2, 210, 206, 38, 213, 4, 182, 198, 220, 47, 93, 148, 61, 84, 148, 250, 158, 45, 8, 81, - 48, 80, 46, 230, 87, 210, 47, 204, 76, 58, 214, 167, 81, - ]), + nullifer_public_key: NullifierPublicKey(NPK_PRIV_ACC_A), + viewing_public_key: Secp256k1Point(VPK_PRIV_ACC_A.to_vec()), }; let key_chain_2 = KeyChain { - secret_spending_key: SecretSpendingKey([ - 48, 175, 124, 10, 230, 240, 166, 14, 249, 254, 157, 226, 208, 124, 122, 177, 203, 139, - 192, 180, 43, 120, 55, 151, 50, 21, 113, 22, 254, 83, 148, 56, - ]), + secret_spending_key: SecretSpendingKey(SSK_PRIV_ACC_B), private_key_holder: PrivateKeyHolder { - nullifier_secret_key: [ - 99, 82, 190, 140, 234, 10, 61, 163, 15, 211, 179, 54, 70, 166, 87, 5, 182, 68, 117, - 244, 217, 23, 99, 9, 4, 177, 230, 125, 109, 91, 160, 30, - ], - viewing_secret_key: [ - 205, 32, 76, 251, 255, 236, 96, 119, 61, 111, 65, 100, 75, 218, 12, 22, 17, 170, - 55, 226, 21, 154, 161, 34, 208, 74, 27, 1, 119, 13, 88, 128, - ], + nullifier_secret_key: NSK_PRIV_ACC_B, + viewing_secret_key: VSK_PRIV_ACC_B, }, - nullifer_public_key: NullifierPublicKey([ - 32, 67, 72, 164, 106, 53, 66, 239, 141, 15, 52, 230, 136, 177, 2, 236, 207, 243, 134, - 135, 210, 143, 87, 232, 215, 128, 194, 120, 113, 224, 4, 165, - ]), - viewing_public_key: Secp256k1Point(vec![ - 2, 79, 110, 46, 203, 29, 206, 205, 18, 86, 27, 189, 104, 103, 113, 181, 110, 53, 78, - 172, 11, 171, 190, 18, 126, 214, 81, 77, 192, 154, 58, 195, 238, - ]), + nullifer_public_key: NullifierPublicKey(NPK_PRIV_ACC_B), + viewing_public_key: Secp256k1Point(VPK_PRIV_ACC_B.to_vec()), }; vec![ PrivateAccountPrivateInitialData { account_id: AccountId::from(&key_chain_1.nullifer_public_key), account: Account { - program_owner: [0, 0, 0, 0, 0, 0, 0, 0], - balance: 10000, + program_owner: DEFAULT_PROGRAM_OWNER, + balance: PRIV_ACC_A_INITIAL_BALANCE, data: Data::default(), nonce: 0, }, @@ -123,8 +153,8 @@ pub fn initial_priv_accounts_private_keys() -> Vec Vec { vec![ PublicAccountPublicInitialData { account_id: initial_account_ids[0], - balance: 10000, + balance: PUB_ACC_A_INITIAL_BALANCE, }, PublicAccountPublicInitialData { account_id: initial_account_ids[1], - balance: 20000, + balance: PUB_ACC_B_INITIAL_BALANCE, }, ] } @@ -197,6 +227,12 @@ mod tests { use super::*; + const PUB_ACC_A_TEXT_ADDR: &str = "6iArKUXxhUJqS7kCaPNhwMWt3ro71PDyBj7jwAyE2VQV"; + const PUB_ACC_B_TEXT_ADDR: &str = "7wHg9sbJwc6h3NP1S9bekfAzB8CHifEcxKswCKUt3YQo"; + + const PRIV_ACC_A_TEXT_ADDR: &str = "5ya25h4Xc9GAmrGB2WrTEnEWtQKJwRwQx3Xfo2tucNcE"; + const PRIV_ACC_B_TEXT_ADDR: &str = "E8HwiTyQe4H9HK7icTvn95HQMnzx49mP9A2ddtMLpNaN"; + #[test] fn test_pub_state_consistency() { let init_accs_private_data = initial_pub_accounts_private_keys(); @@ -215,18 +251,16 @@ mod tests { assert_eq!( init_accs_pub_data[0], PublicAccountPublicInitialData { - account_id: AccountId::from_str("6iArKUXxhUJqS7kCaPNhwMWt3ro71PDyBj7jwAyE2VQV") - .unwrap(), - balance: 10000, + account_id: AccountId::from_str(PUB_ACC_A_TEXT_ADDR).unwrap(), + balance: PUB_ACC_A_INITIAL_BALANCE, } ); assert_eq!( init_accs_pub_data[1], PublicAccountPublicInitialData { - account_id: AccountId::from_str("7wHg9sbJwc6h3NP1S9bekfAzB8CHifEcxKswCKUt3YQo") - .unwrap(), - balance: 20000, + account_id: AccountId::from_str(PUB_ACC_B_TEXT_ADDR).unwrap(), + balance: PUB_ACC_B_INITIAL_BALANCE, } ) } @@ -312,11 +346,11 @@ mod tests { assert_eq!( init_private_accs_keys[0].account_id.to_string(), - "5ya25h4Xc9GAmrGB2WrTEnEWtQKJwRwQx3Xfo2tucNcE" + PRIV_ACC_A_TEXT_ADDR ); assert_eq!( init_private_accs_keys[1].account_id.to_string(), - "E8HwiTyQe4H9HK7icTvn95HQMnzx49mP9A2ddtMLpNaN" + PRIV_ACC_B_TEXT_ADDR ); assert_eq!( @@ -331,13 +365,10 @@ mod tests { assert_eq!( init_comms[0], PrivateAccountPublicInitialData { - npk: NullifierPublicKey([ - 167, 108, 50, 153, 74, 47, 151, 188, 140, 79, 195, 31, 181, 9, 40, 167, 201, - 32, 175, 129, 45, 245, 223, 193, 210, 170, 247, 128, 167, 140, 155, 129, - ]), + npk: NullifierPublicKey(NPK_PRIV_ACC_A), account: Account { - program_owner: [0, 0, 0, 0, 0, 0, 0, 0], - balance: 10000, + program_owner: DEFAULT_PROGRAM_OWNER, + balance: PRIV_ACC_A_INITIAL_BALANCE, data: Data::default(), nonce: 0, }, @@ -347,13 +378,10 @@ mod tests { assert_eq!( init_comms[1], PrivateAccountPublicInitialData { - npk: NullifierPublicKey([ - 32, 67, 72, 164, 106, 53, 66, 239, 141, 15, 52, 230, 136, 177, 2, 236, 207, - 243, 134, 135, 210, 143, 87, 232, 215, 128, 194, 120, 113, 224, 4, 165, - ]), + npk: NullifierPublicKey(NPK_PRIV_ACC_B), account: Account { - program_owner: [0, 0, 0, 0, 0, 0, 0, 0], - balance: 20000, + program_owner: DEFAULT_PROGRAM_OWNER, + balance: PRIV_ACC_B_INITIAL_BALANCE, data: Data::default(), nonce: 0, }, From adfb1156b1af13ab533807a168a8e763f7cd2daa Mon Sep 17 00:00:00 2001 From: Pravdyvy <46261001+Pravdyvy@users.noreply.github.com> Date: Tue, 17 Mar 2026 15:20:59 +0200 Subject: [PATCH 11/22] Update storage/src/indexer/write_batch.rs Co-authored-by: Sergio Chouhy <41742639+schouhy@users.noreply.github.com> --- storage/src/indexer/write_batch.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs index b98e539a..2387a424 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_batch.rs @@ -242,7 +242,6 @@ impl RocksDBIO { let last_curr_block = self.get_meta_last_block_in_db()?; let mut write_batch = WriteBatch::default(); - // ToDo: rewrite this with write batching write_batch.put_cf( &cf_block, From 6e7a6f8703821c29069b0462aa9cb9f0098491e4 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Tue, 17 Mar 2026 15:23:41 +0200 Subject: [PATCH 12/22] fix: fmt --- storage/src/indexer/write_batch.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs index 2387a424..9777addb 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_batch.rs @@ -242,7 +242,6 @@ impl RocksDBIO { let last_curr_block = self.get_meta_last_block_in_db()?; let mut write_batch = WriteBatch::default(); - write_batch.put_cf( &cf_block, borsh::to_vec(&block.header.block_id).map_err(|err| { From a00018e64eb79c9f36a4d32b4bd8f2574b168b75 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Tue, 17 Mar 2026 15:40:55 +0200 Subject: [PATCH 13/22] fix: suggestions 1 --- indexer/core/src/block_store.rs | 2 +- storage/src/indexer/mod.rs | 44 +++++++++++++++--------------- storage/src/indexer/write_batch.rs | 4 +-- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/indexer/core/src/block_store.rs b/indexer/core/src/block_store.rs index 7844b947..681b63c8 100644 --- a/indexer/core/src/block_store.rs +++ b/indexer/core/src/block_store.rs @@ -118,6 +118,6 @@ impl IndexerStore { // to represent correct block finality block.bedrock_status = BedrockStatus::Finalized; - Ok(self.dbio.put_block_batch(block, l1_header.into())?) + Ok(self.dbio.put_block(block, l1_header.into())?) } } diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index b3e2f5b7..7ac682c5 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -89,7 +89,7 @@ impl RocksDBIO { let dbio = Self { // There is no point in handling this from runner code - db: db.unwrap(), + db: db.expect("We should have permissions to open DB"), }; let is_start_set = dbio.get_meta_is_first_block_set()?; @@ -135,31 +135,31 @@ impl RocksDBIO { // Columns pub fn meta_column(&self) -> Arc> { - self.db.cf_handle(CF_META_NAME).unwrap() + self.db.cf_handle(CF_META_NAME).expect("Meta column should exist") } pub fn block_column(&self) -> Arc> { - self.db.cf_handle(CF_BLOCK_NAME).unwrap() + self.db.cf_handle(CF_BLOCK_NAME).expect("Block column should exist") } pub fn breakpoint_column(&self) -> Arc> { - self.db.cf_handle(CF_BREAKPOINT_NAME).unwrap() + self.db.cf_handle(CF_BREAKPOINT_NAME).expect("Breakpoint column should exist") } pub fn hash_to_id_column(&self) -> Arc> { - self.db.cf_handle(CF_HASH_TO_ID).unwrap() + self.db.cf_handle(CF_HASH_TO_ID).expect("Hash to id map column should exist") } pub fn tx_hash_to_id_column(&self) -> Arc> { - self.db.cf_handle(CF_TX_TO_ID).unwrap() + self.db.cf_handle(CF_TX_TO_ID).expect("Tx hash to id map column should exist") } pub fn account_id_to_tx_hash_column(&self) -> Arc> { - self.db.cf_handle(CF_ACC_TO_TX).unwrap() + self.db.cf_handle(CF_ACC_TO_TX).expect("Account id to tx map column should exist") } pub fn account_meta_column(&self) -> Arc> { - self.db.cf_handle(CF_ACC_META).unwrap() + self.db.cf_handle(CF_ACC_META).expect("Account meta column should exist") } // State @@ -312,7 +312,7 @@ mod tests { let transfer_tx = transfer(1, 0, true); let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - dbio.put_block_batch(block, [1; 32]).unwrap(); + dbio.put_block(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let first_id = dbio.get_meta_first_block_in_db().unwrap(); @@ -360,7 +360,7 @@ mod tests { let transfer_tx = transfer(1, (i - 1) as u128, true); let block = common::test_utils::produce_dummy_block(i + 1, Some(prev_hash), vec![transfer_tx]); - dbio.put_block_batch(block, [i as u8; 32]).unwrap(); + dbio.put_block(block, [i as u8; 32]).unwrap(); } let last_id = dbio.get_meta_last_block_in_db().unwrap(); @@ -416,7 +416,7 @@ mod tests { let control_hash1 = block.header.hash; - dbio.put_block_batch(block, [1; 32]).unwrap(); + dbio.put_block(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -427,7 +427,7 @@ mod tests { let control_hash2 = block.header.hash; - dbio.put_block_batch(block, [2; 32]).unwrap(); + dbio.put_block(block, [2; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -438,7 +438,7 @@ mod tests { let control_tx_hash1 = transfer_tx.hash(); let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - dbio.put_block_batch(block, [3; 32]).unwrap(); + dbio.put_block(block, [3; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -449,7 +449,7 @@ mod tests { let control_tx_hash2 = transfer_tx.hash(); let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - dbio.put_block_batch(block, [4; 32]).unwrap(); + dbio.put_block(block, [4; 32]).unwrap(); let control_block_id1 = dbio.get_block_id_by_hash(control_hash1.0).unwrap(); let control_block_id2 = dbio.get_block_id_by_hash(control_hash2.0).unwrap(); @@ -480,7 +480,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block_batch(block, [1; 32]).unwrap(); + dbio.put_block(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -490,7 +490,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block_batch(block, [2; 32]).unwrap(); + dbio.put_block(block, [2; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -500,7 +500,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block_batch(block, [3; 32]).unwrap(); + dbio.put_block(block, [3; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -510,7 +510,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block_batch(block, [4; 32]).unwrap(); + dbio.put_block(block, [4; 32]).unwrap(); let block_hashes_mem: Vec<[u8; 32]> = block_res.into_iter().map(|bl| bl.header.hash.0).collect(); @@ -573,7 +573,7 @@ mod tests { vec![transfer_tx1, transfer_tx2], ); - dbio.put_block_batch(block, [1; 32]).unwrap(); + dbio.put_block(block, [1; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -590,7 +590,7 @@ mod tests { vec![transfer_tx1, transfer_tx2], ); - dbio.put_block_batch(block, [2; 32]).unwrap(); + dbio.put_block(block, [2; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -607,7 +607,7 @@ mod tests { vec![transfer_tx1, transfer_tx2], ); - dbio.put_block_batch(block, [3; 32]).unwrap(); + dbio.put_block(block, [3; 32]).unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); @@ -618,7 +618,7 @@ mod tests { let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); - dbio.put_block_batch(block, [4; 32]).unwrap(); + dbio.put_block(block, [4; 32]).unwrap(); let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 7).unwrap(); let acc1_tx_hashes: Vec<[u8; 32]> = acc1_tx.into_iter().map(|tx| tx.hash().0).collect(); diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_batch.rs index 9777addb..73d2c01d 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_batch.rs @@ -139,7 +139,7 @@ impl RocksDBIO { ) .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - self.put_block_batch(block, [0; 32])?; + self.put_block(block, [0; 32])?; Ok(()) } @@ -235,7 +235,7 @@ impl RocksDBIO { // Block - pub fn put_block_batch(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { + pub fn put_block(&self, block: Block, l1_lib_header: [u8; 32]) -> DbResult<()> { let cf_block = self.block_column(); let cf_hti = self.hash_to_id_column(); let cf_tti: Arc> = self.tx_hash_to_id_column(); From dfb9c709417dd7b878f3f0c821ddcd48b6770207 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Tue, 17 Mar 2026 15:44:36 +0200 Subject: [PATCH 14/22] fix: fmt --- storage/src/indexer/mod.rs | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index 7ac682c5..232a217c 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -135,31 +135,45 @@ impl RocksDBIO { // Columns pub fn meta_column(&self) -> Arc> { - self.db.cf_handle(CF_META_NAME).expect("Meta column should exist") + self.db + .cf_handle(CF_META_NAME) + .expect("Meta column should exist") } pub fn block_column(&self) -> Arc> { - self.db.cf_handle(CF_BLOCK_NAME).expect("Block column should exist") + self.db + .cf_handle(CF_BLOCK_NAME) + .expect("Block column should exist") } pub fn breakpoint_column(&self) -> Arc> { - self.db.cf_handle(CF_BREAKPOINT_NAME).expect("Breakpoint column should exist") + self.db + .cf_handle(CF_BREAKPOINT_NAME) + .expect("Breakpoint column should exist") } pub fn hash_to_id_column(&self) -> Arc> { - self.db.cf_handle(CF_HASH_TO_ID).expect("Hash to id map column should exist") + self.db + .cf_handle(CF_HASH_TO_ID) + .expect("Hash to id map column should exist") } pub fn tx_hash_to_id_column(&self) -> Arc> { - self.db.cf_handle(CF_TX_TO_ID).expect("Tx hash to id map column should exist") + self.db + .cf_handle(CF_TX_TO_ID) + .expect("Tx hash to id map column should exist") } pub fn account_id_to_tx_hash_column(&self) -> Arc> { - self.db.cf_handle(CF_ACC_TO_TX).expect("Account id to tx map column should exist") + self.db + .cf_handle(CF_ACC_TO_TX) + .expect("Account id to tx map column should exist") } pub fn account_meta_column(&self) -> Arc> { - self.db.cf_handle(CF_ACC_META).expect("Account meta column should exist") + self.db + .cf_handle(CF_ACC_META) + .expect("Account meta column should exist") } // State From 5b99f0dabd832d4032d7dcbadd69bc6d24fd11b8 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Wed, 18 Mar 2026 17:52:28 +0200 Subject: [PATCH 15/22] fix: lint fix --- storage/src/indexer/mod.rs | 192 +++++++++++------- .../{read_multi_get.rs => read_multiple.rs} | 46 +++-- storage/src/indexer/read_once.rs | 7 +- .../{write_batch.rs => write_atomic.rs} | 39 +++- .../{write_once.rs => write_non_atomic.rs} | 14 +- 5 files changed, 187 insertions(+), 111 deletions(-) rename storage/src/indexer/{read_multi_get.rs => read_multiple.rs} (81%) rename storage/src/indexer/{write_batch.rs => write_atomic.rs} (87%) rename storage/src/indexer/{write_once.rs => write_non_atomic.rs} (90%) diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index 12faf96d..c5d47c1f 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -8,10 +8,10 @@ use rocksdb::{ use crate::error::DbError; -pub mod read_multi_get; +pub mod read_multiple; pub mod read_once; -pub mod write_batch; -pub mod write_once; +pub mod write_atomic; +pub mod write_non_atomic; /// Maximal size of stored blocks in base. /// @@ -57,13 +57,6 @@ pub const CF_ACC_TO_TX: &str = "cf_acc_to_tx"; pub type DbResult = Result; -fn closest_breakpoint_id(block_id: u64) -> u64 { - block_id - .saturating_sub(1) - .checked_div(u64::from(BREAKPOINT_INTERVAL)) - .expect("Breakpoint interval is not zero") -} - pub struct RocksDBIO { pub db: DBWithThreadMode, } @@ -184,7 +177,9 @@ impl RocksDBIO { self.get_meta_first_block_in_db()? }; - for block in self.get_block_batch_seq((start + 1)..=block_id)? { + for block in self.get_block_batch_seq( + start.checked_add(1).expect("Will be lesser that u64::MAX")..=block_id, + )? { for transaction in block.body.transactions { transaction .transaction_stateless_check() @@ -215,11 +210,17 @@ impl RocksDBIO { } } -#[allow(clippy::shadow_unrelated)] +fn closest_breakpoint_id(block_id: u64) -> u64 { + block_id + .saturating_sub(1) + .checked_div(u64::from(BREAKPOINT_INTERVAL)) + .expect("Breakpoint interval is not zero") +} + +#[expect(clippy::shadow_unrelated, reason = "Fine for tests")] #[cfg(test)] mod tests { - use common::transaction::NSSATransaction; - use nssa::AccountId; + use nssa::{AccountId, PublicKey}; use tempfile::tempdir; use super::*; @@ -228,20 +229,6 @@ mod tests { common::test_utils::produce_dummy_block(1, None, vec![]) } - fn acc1() -> AccountId { - AccountId::new([ - 148, 179, 206, 253, 199, 51, 82, 86, 232, 2, 152, 122, 80, 243, 54, 207, 237, 112, 83, - 153, 44, 59, 204, 49, 128, 84, 160, 227, 216, 149, 97, 102, - ]) - } - - fn acc2() -> AccountId { - AccountId::new([ - 30, 145, 107, 3, 207, 73, 192, 230, 160, 63, 238, 207, 18, 69, 54, 216, 103, 244, 92, - 94, 124, 248, 42, 16, 141, 19, 119, 18, 14, 226, 140, 204, - ]) - } - fn acc1_sign_key() -> nssa::PrivateKey { nssa::PrivateKey::try_new([1; 32]).unwrap() } @@ -250,28 +237,12 @@ mod tests { nssa::PrivateKey::try_new([2; 32]).unwrap() } - fn initial_state() -> V02State { - nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]) + fn acc1() -> AccountId { + AccountId::from(&PublicKey::new_from_private_key(&acc1_sign_key())) } - fn transfer(amount: u128, nonce: u128, direction: bool) -> NSSATransaction { - let from; - let to; - let sign_key; - - if direction { - from = acc1(); - to = acc2(); - sign_key = acc1_sign_key(); - } else { - from = acc2(); - to = acc1(); - sign_key = acc2_sign_key(); - } - - common::test_utils::create_transaction_native_token_transfer( - from, nonce, to, amount, &sign_key, - ) + fn acc2() -> AccountId { + AccountId::from(&PublicKey::new_from_private_key(&acc2_sign_key())) } #[test] @@ -279,8 +250,12 @@ mod tests { let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = - RocksDBIO::open_or_create(temdir_path, &genesis_block(), &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create( + temdir_path, + &genesis_block(), + &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + ) + .unwrap(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let first_id = dbio.get_meta_first_block_in_db().unwrap(); @@ -312,11 +287,20 @@ mod tests { let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = - RocksDBIO::open_or_create(temdir_path, &genesis_block(), &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create( + temdir_path, + &genesis_block(), + &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + ) + .unwrap(); let prev_hash = genesis_block().header.hash; - let transfer_tx = transfer(1, 0, true); + let from = acc1(); + let to = acc2(); + let sign_key = acc1_sign_key(); + + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 0, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); dbio.put_block(&block, [1; 32]).unwrap(); @@ -356,15 +340,30 @@ mod tests { let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = - RocksDBIO::open_or_create(temdir_path, &genesis_block(), &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create( + temdir_path, + &genesis_block(), + &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + ) + .unwrap(); + + let from = acc1(); + let to = acc2(); + let sign_key = acc1_sign_key(); for i in 1..=BREAKPOINT_INTERVAL { let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, (i - 1).into(), true); + + let transfer_tx = common::test_utils::create_transaction_native_token_transfer( + from, + (i - 1).into(), + to, + 1, + &sign_key, + ); let block = common::test_utils::produce_dummy_block( (i + 1).into(), Some(prev_hash), @@ -414,14 +413,23 @@ mod tests { let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = - RocksDBIO::open_or_create(temdir_path, &genesis_block(), &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create( + temdir_path, + &genesis_block(), + &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + ) + .unwrap(); + + let from = acc1(); + let to = acc2(); + let sign_key = acc1_sign_key(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 0, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 0, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); let control_hash1 = block.header.hash; @@ -432,7 +440,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 1, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 1, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); let control_hash2 = block.header.hash; @@ -443,7 +452,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 2, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 2, to, 1, &sign_key); let control_tx_hash1 = transfer_tx.hash(); @@ -454,7 +464,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 3, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 3, to, 1, &sign_key); let control_tx_hash2 = transfer_tx.hash(); @@ -479,14 +490,23 @@ mod tests { let mut block_res = vec![]; - let dbio = - RocksDBIO::open_or_create(temdir_path, &genesis_block(), &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create( + temdir_path, + &genesis_block(), + &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + ) + .unwrap(); + + let from = acc1(); + let to = acc2(); + let sign_key = acc1_sign_key(); let last_id = dbio.get_meta_last_block_in_db().unwrap(); let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 0, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 0, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); @@ -496,7 +516,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 1, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 1, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); @@ -506,7 +527,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 2, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 2, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); @@ -516,7 +538,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 3, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 3, to, 1, &sign_key); let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); @@ -563,8 +586,16 @@ mod tests { let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = - RocksDBIO::open_or_create(temdir_path, &genesis_block(), &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create( + temdir_path, + &genesis_block(), + &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + ) + .unwrap(); + + let from = acc1(); + let to = acc2(); + let sign_key = acc1_sign_key(); let mut tx_hash_res = vec![]; @@ -572,8 +603,10 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx1 = transfer(1, 0, true); - let transfer_tx2 = transfer(1, 1, true); + let transfer_tx1 = + common::test_utils::create_transaction_native_token_transfer(from, 0, to, 1, &sign_key); + let transfer_tx2 = + common::test_utils::create_transaction_native_token_transfer(from, 1, to, 1, &sign_key); tx_hash_res.push(transfer_tx1.hash().0); tx_hash_res.push(transfer_tx2.hash().0); @@ -589,8 +622,10 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx1 = transfer(1, 2, true); - let transfer_tx2 = transfer(1, 3, true); + let transfer_tx1 = + common::test_utils::create_transaction_native_token_transfer(from, 2, to, 1, &sign_key); + let transfer_tx2 = + common::test_utils::create_transaction_native_token_transfer(from, 3, to, 1, &sign_key); tx_hash_res.push(transfer_tx1.hash().0); tx_hash_res.push(transfer_tx2.hash().0); @@ -606,8 +641,10 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx1 = transfer(1, 4, true); - let transfer_tx2 = transfer(1, 5, true); + let transfer_tx1 = + common::test_utils::create_transaction_native_token_transfer(from, 4, to, 1, &sign_key); + let transfer_tx2 = + common::test_utils::create_transaction_native_token_transfer(from, 5, to, 1, &sign_key); tx_hash_res.push(transfer_tx1.hash().0); tx_hash_res.push(transfer_tx2.hash().0); @@ -623,7 +660,8 @@ mod tests { let last_block = dbio.get_block(last_id).unwrap(); let prev_hash = last_block.header.hash; - let transfer_tx = transfer(1, 6, true); + let transfer_tx = + common::test_utils::create_transaction_native_token_transfer(from, 6, to, 1, &sign_key); tx_hash_res.push(transfer_tx.hash().0); let block = common::test_utils::produce_dummy_block(5, Some(prev_hash), vec![transfer_tx]); diff --git a/storage/src/indexer/read_multi_get.rs b/storage/src/indexer/read_multiple.rs similarity index 81% rename from storage/src/indexer/read_multi_get.rs rename to storage/src/indexer/read_multiple.rs index 66fe25d7..866fc7b0 100644 --- a/storage/src/indexer/read_multi_get.rs +++ b/storage/src/indexer/read_multiple.rs @@ -1,7 +1,8 @@ use common::transaction::NSSATransaction; -use super::*; +use super::{Block, DbError, DbResult, RocksDBIO}; +#[expect(clippy::multiple_inherent_impl, reason = "Readability")] impl RocksDBIO { pub fn get_block_batch(&self, before: Option, limit: u64) -> DbResult> { let mut seq = vec![]; @@ -25,11 +26,11 @@ impl RocksDBIO { self.get_block_batch_seq(seq.into_iter()) } - /// Get block batch from a sequence + /// Get block batch from a sequence. /// - /// Currently assumes non-decreasing sequence + /// Currently assumes non-decreasing sequence. /// - /// ToDo: Add suport of arbitrary sequences + /// `ToDo`: Add suport of arbitrary sequences. pub fn get_block_batch_seq(&self, seq: impl Iterator) -> DbResult> { let cf_block = self.block_column(); @@ -72,11 +73,9 @@ impl RocksDBIO { Ok(block_batch) } - /// Get block ids by txs + /// Get block ids by txs. /// - /// Transactions must be sorted by time of arrival - /// - /// ToDo: There may be multiple transactions in one block + /// `ToDo`: There may be multiple transactions in one block /// so this method can take redundant reads. /// Need to update signature and implementation. fn get_block_ids_by_tx_vec(&self, tx_vec: &[[u8; 32]]) -> DbResult> { @@ -98,18 +97,21 @@ impl RocksDBIO { // Keys parsing let mut block_id_batch = vec![]; for res in multi_get_res { - let res = res.map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; + let res = res + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))? + .ok_or_else(|| { + DbError::db_interaction_error( + "Tx to block id mapping do not contain transaction from vec".to_owned(), + ) + })?; - let block_id = if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { + let block_id = { + Ok(borsh::from_slice::(&res).map_err(|serr| { DbError::borsh_cast_message( serr, Some("Failed to deserialize block id".to_owned()), ) })?) - } else { - // Block not found, assuming that previous one was the last - break; }?; block_id_batch.push(block_id); @@ -131,7 +133,11 @@ impl RocksDBIO { // Keys preparation let mut keys = vec![]; - for tx_id in offset..(offset + limit) { + for tx_id in offset + ..offset + .checked_add(limit) + .expect("Transaction limit should be lesser than u64::MAX") + { let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { DbError::borsh_cast_message(berr, Some("Failed to serialize account id".to_owned())) })?; @@ -188,10 +194,12 @@ impl RocksDBIO { .transactions .iter() .find(|tx| tx.hash().0 == tx_hash) - .ok_or(DbError::db_interaction_error(format!( - "Missing transaction in block {} with hash {:#?}", - block.header.block_id, tx_hash - )))?; + .ok_or_else(|| { + DbError::db_interaction_error(format!( + "Missing transaction in block {} with hash {:#?}", + block.header.block_id, tx_hash + )) + })?; tx_batch.push(transaction.clone()); } diff --git a/storage/src/indexer/read_once.rs b/storage/src/indexer/read_once.rs index 40929fd6..f966f349 100644 --- a/storage/src/indexer/read_once.rs +++ b/storage/src/indexer/read_once.rs @@ -1,5 +1,10 @@ -use super::*; +use super::{ + Block, DB_META_FIRST_BLOCK_IN_DB_KEY, DB_META_FIRST_BLOCK_SET_KEY, + DB_META_LAST_BLOCK_IN_DB_KEY, DB_META_LAST_BREAKPOINT_ID, + DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DbError, DbResult, RocksDBIO, V02State, +}; +#[expect(clippy::multiple_inherent_impl, reason = "Readability")] impl RocksDBIO { // Meta diff --git a/storage/src/indexer/write_batch.rs b/storage/src/indexer/write_atomic.rs similarity index 87% rename from storage/src/indexer/write_batch.rs rename to storage/src/indexer/write_atomic.rs index a2d5b46b..161d763a 100644 --- a/storage/src/indexer/write_batch.rs +++ b/storage/src/indexer/write_atomic.rs @@ -2,8 +2,13 @@ use std::collections::HashMap; use rocksdb::WriteBatch; -use super::*; +use super::{ + Arc, BREAKPOINT_INTERVAL, Block, BoundColumnFamily, DB_META_FIRST_BLOCK_IN_DB_KEY, + DB_META_FIRST_BLOCK_SET_KEY, DB_META_LAST_BLOCK_IN_DB_KEY, DB_META_LAST_BREAKPOINT_ID, + DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DbError, DbResult, RocksDBIO, +}; +#[expect(clippy::multiple_inherent_impl, reason = "Readability")] impl RocksDBIO { // Accounts meta @@ -36,14 +41,16 @@ impl RocksDBIO { pub fn put_account_transactions( &self, acc_id: [u8; 32], - tx_hashes: Vec<[u8; 32]>, + tx_hashes: &[[u8; 32]], ) -> DbResult<()> { let acc_num_tx = self.get_acc_meta_num_tx(acc_id)?.unwrap_or(0); let cf_att = self.account_id_to_tx_hash_column(); let mut write_batch = WriteBatch::new(); for (tx_id, tx_hash) in tx_hashes.iter().enumerate() { - let put_id = acc_num_tx + tx_id as u64; + let put_id = acc_num_tx + .checked_add(tx_id.try_into().expect("Must fit into u64")) + .expect("Tx count should be lesser that u64::MAX"); let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { DbError::borsh_cast_message(berr, Some("Failed to serialize account id".to_owned())) @@ -68,7 +75,9 @@ impl RocksDBIO { self.update_acc_meta_batch( acc_id, - acc_num_tx + (tx_hashes.len() as u64), + acc_num_tx + .checked_add(tx_hashes.len().try_into().expect("Must fit into u64")) + .expect("Tx count should be lesser that u64::MAX"), &mut write_batch, )?; @@ -80,14 +89,16 @@ impl RocksDBIO { pub fn put_account_transactions_dependant( &self, acc_id: [u8; 32], - tx_hashes: Vec<[u8; 32]>, + tx_hashes: &[[u8; 32]], write_batch: &mut WriteBatch, ) -> DbResult<()> { let acc_num_tx = self.get_acc_meta_num_tx(acc_id)?.unwrap_or(0); let cf_att = self.account_id_to_tx_hash_column(); for (tx_id, tx_hash) in tx_hashes.iter().enumerate() { - let put_id = acc_num_tx + tx_id as u64; + let put_id = acc_num_tx + .checked_add(tx_id.try_into().expect("Must fit into u64")) + .expect("Tx count should be lesser that u64::MAX"); let mut prefix = borsh::to_vec(&acc_id).map_err(|berr| { DbError::borsh_cast_message(berr, Some("Failed to serialize account id".to_owned())) @@ -110,7 +121,13 @@ impl RocksDBIO { ); } - self.update_acc_meta_batch(acc_id, acc_num_tx + (tx_hashes.len() as u64), write_batch)?; + self.update_acc_meta_batch( + acc_id, + acc_num_tx + .checked_add(tx_hashes.len().try_into().expect("Must fit into u64")) + .expect("Tx count should be lesser that u64::MAX"), + write_batch, + )?; Ok(()) } @@ -226,7 +243,7 @@ impl RocksDBIO { Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_owned()), ) })?, - [1u8; 1], + [1_u8; 1], ); Ok(()) } @@ -286,14 +303,14 @@ impl RocksDBIO { let acc_ids = tx .affected_public_account_ids() .into_iter() - .map(|account_id| account_id.into_value()) + .map(nssa::AccountId::into_value) .collect::>(); for acc_id in acc_ids { acc_to_tx_map .entry(acc_id) .and_modify(|tx_hashes| tx_hashes.push(tx_hash.into())) - .or_insert(vec![tx_hash.into()]); + .or_insert_with(|| vec![tx_hash.into()]); } } @@ -302,7 +319,7 @@ impl RocksDBIO { reason = "RocksDB will keep ordering persistent" )] for (acc_id, tx_hashes) in acc_to_tx_map { - self.put_account_transactions_dependant(acc_id, tx_hashes, &mut write_batch)?; + self.put_account_transactions_dependant(acc_id, &tx_hashes, &mut write_batch)?; } self.db.write(write_batch).map_err(|rerr| { diff --git a/storage/src/indexer/write_once.rs b/storage/src/indexer/write_non_atomic.rs similarity index 90% rename from storage/src/indexer/write_once.rs rename to storage/src/indexer/write_non_atomic.rs index 0edd7dfa..84fc7de5 100644 --- a/storage/src/indexer/write_once.rs +++ b/storage/src/indexer/write_non_atomic.rs @@ -1,5 +1,10 @@ -use super::*; +use super::{ + BREAKPOINT_INTERVAL, DB_META_FIRST_BLOCK_SET_KEY, DB_META_LAST_BLOCK_IN_DB_KEY, + DB_META_LAST_BREAKPOINT_ID, DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DbError, + DbResult, RocksDBIO, V02State, +}; +#[expect(clippy::multiple_inherent_impl, reason = "Readability")] impl RocksDBIO { // Meta @@ -88,7 +93,7 @@ impl RocksDBIO { Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_owned()), ) })?, - [1u8; 1], + [1_u8; 1], ) .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; Ok(()) @@ -120,7 +125,10 @@ impl RocksDBIO { pub fn put_next_breakpoint(&self) -> DbResult<()> { let last_block = self.get_meta_last_block_in_db()?; - let next_breakpoint_id = self.get_meta_last_breakpoint_id()? + 1; + let next_breakpoint_id = self + .get_meta_last_breakpoint_id()? + .checked_add(1) + .expect("Breakpoint Id will be lesser than u64::MAX"); let block_to_break_id = next_breakpoint_id .checked_mul(u64::from(BREAKPOINT_INTERVAL)) .expect("Reached maximum breakpoint id"); From a0ab6ad92dd7dfcc572a25fdb1152cc554fcdca5 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Wed, 18 Mar 2026 18:32:05 +0200 Subject: [PATCH 16/22] fix: lint fix --- indexer/core/src/config.rs | 2 +- integration_tests/src/config.rs | 4 +- key_protocol/src/initial_state.rs | 18 ++++--- sequencer_core/src/config.rs | 2 +- sequencer_core/src/lib.rs | 81 +++++++++++++++---------------- sequencer_rpc/src/process.rs | 5 +- wallet/src/config.rs | 22 ++++----- wallet/src/helperfunctions.rs | 8 +-- 8 files changed, 72 insertions(+), 70 deletions(-) diff --git a/indexer/core/src/config.rs b/indexer/core/src/config.rs index e45870df..5eb8dd92 100644 --- a/indexer/core/src/config.rs +++ b/indexer/core/src/config.rs @@ -29,7 +29,7 @@ pub struct ClientConfig { pub struct IndexerConfig { /// Home dir of sequencer storage. pub home: PathBuf, - /// Sequencers signing key + /// Sequencers signing key. pub signing_key: [u8; 32], #[serde(with = "humantime_serde")] pub consensus_info_polling_interval: Duration, diff --git a/integration_tests/src/config.rs b/integration_tests/src/config.rs index cce50db1..875e1652 100644 --- a/integration_tests/src/config.rs +++ b/integration_tests/src/config.rs @@ -142,11 +142,11 @@ impl InitialData { }) .chain(self.private_accounts.iter().map(|(key_chain, account)| { let account_id = AccountId::from(&key_chain.nullifer_public_key); - InitialAccountData::Private(PrivateAccountPrivateInitialData { + InitialAccountData::Private(Box::new(PrivateAccountPrivateInitialData { account_id, account: account.clone(), key_chain: key_chain.clone(), - }) + })) })) .collect() } diff --git a/key_protocol/src/initial_state.rs b/key_protocol/src/initial_state.rs index 3af9182f..2150bd4c 100644 --- a/key_protocol/src/initial_state.rs +++ b/key_protocol/src/initial_state.rs @@ -101,6 +101,7 @@ pub struct PrivateAccountPrivateInitialData { pub key_chain: KeyChain, } +#[must_use] pub fn initial_pub_accounts_private_keys() -> Vec { let acc1_pub_sign_key = PrivateKey::try_new(PRIVATE_KEY_PUB_ACC_A).unwrap(); @@ -118,6 +119,7 @@ pub fn initial_pub_accounts_private_keys() -> Vec Vec { let key_chain_1 = KeyChain { secret_spending_key: SecretSpendingKey(SSK_PRIV_ACC_A), @@ -163,16 +165,18 @@ pub fn initial_priv_accounts_private_keys() -> Vec Vec { initial_priv_accounts_private_keys() .into_iter() .map(|data| PrivateAccountPublicInitialData { npk: data.key_chain.nullifer_public_key.clone(), - account: data.account.clone(), + account: data.account, }) .collect() } +#[must_use] pub fn initial_accounts() -> Vec { let initial_account_ids = initial_pub_accounts_private_keys() .into_iter() @@ -191,6 +195,7 @@ pub fn initial_accounts() -> Vec { ] } +#[must_use] pub fn initial_state() -> V02State { let initial_commitments: Vec = initial_commitments() .iter() @@ -213,6 +218,7 @@ pub fn initial_state() -> V02State { nssa::V02State::new_with_genesis_accounts(&init_accs, &initial_commitments) } +#[must_use] pub fn initial_state_testnet() -> V02State { let mut state = initial_state(); @@ -223,7 +229,7 @@ pub fn initial_state_testnet() -> V02State { #[cfg(test)] mod tests { - use std::str::FromStr; + use std::str::FromStr as _; use super::*; @@ -234,7 +240,7 @@ mod tests { const PRIV_ACC_B_TEXT_ADDR: &str = "E8HwiTyQe4H9HK7icTvn95HQMnzx49mP9A2ddtMLpNaN"; #[test] - fn test_pub_state_consistency() { + fn pub_state_consistency() { let init_accs_private_data = initial_pub_accounts_private_keys(); let init_accs_pub_data = initial_accounts(); @@ -262,11 +268,11 @@ mod tests { account_id: AccountId::from_str(PUB_ACC_B_TEXT_ADDR).unwrap(), balance: PUB_ACC_B_INITIAL_BALANCE, } - ) + ); } #[test] - fn test_private_state_consistency() { + fn private_state_consistency() { let init_private_accs_keys = initial_priv_accounts_private_keys(); let init_comms = initial_commitments(); @@ -386,6 +392,6 @@ mod tests { nonce: 0, }, } - ) + ); } } diff --git a/sequencer_core/src/config.rs b/sequencer_core/src/config.rs index 7e8343fa..46ee6945 100644 --- a/sequencer_core/src/config.rs +++ b/sequencer_core/src/config.rs @@ -43,7 +43,7 @@ pub struct SequencerConfig { pub retry_pending_blocks_timeout: Duration, /// Port to listen. pub port: u16, - /// Sequencer own signing key + /// Sequencer own signing key. pub signing_key: [u8; 32], /// Bedrock configuration options. pub bedrock_config: BedrockConfig, diff --git a/sequencer_core/src/lib.rs b/sequencer_core/src/lib.rs index 30a63108..5f5653da 100644 --- a/sequencer_core/src/lib.rs +++ b/sequencer_core/src/lib.rs @@ -94,53 +94,50 @@ impl SequencerCore { - info!("Found local database. Loading state and pending blocks from it."); - state - } - None => { - info!( - "No database found when starting the sequencer. Creating a fresh new with the initial data" - ); + let mut state = if let Some(state) = store.get_nssa_state() { + info!("Found local database. Loading state and pending blocks from it."); + state + } else { + info!( + "No database found when starting the sequencer. Creating a fresh new with the initial data" + ); - let initial_commitments: Option> = config - .initial_commitments - .clone() - .map(|initial_commitments| { - initial_commitments - .iter() - .map(|init_comm_data| { - let npk = &init_comm_data.npk; + let initial_commitments: Option> = config + .initial_commitments + .clone() + .map(|initial_commitments| { + initial_commitments + .iter() + .map(|init_comm_data| { + let npk = &init_comm_data.npk; - let mut acc = init_comm_data.account.clone(); + let mut acc = init_comm_data.account.clone(); - acc.program_owner = - nssa::program::Program::authenticated_transfer_program().id(); + acc.program_owner = + nssa::program::Program::authenticated_transfer_program().id(); - nssa_core::Commitment::new(npk, &acc) - }) - .collect() - }); + nssa_core::Commitment::new(npk, &acc) + }) + .collect() + }); - let init_accs: Option> = - config.initial_accounts.clone().map(|initial_accounts| { - initial_accounts - .iter() - .map(|acc_data| (acc_data.account_id, acc_data.balance)) - .collect() - }); + let init_accs: Option> = + config.initial_accounts.clone().map(|initial_accounts| { + initial_accounts + .iter() + .map(|acc_data| (acc_data.account_id, acc_data.balance)) + .collect() + }); - // If initial commitments or accounts are present in config, need to construct state - // from them - if initial_commitments.is_some() || init_accs.is_some() { - V02State::new_with_genesis_accounts( - &init_accs.unwrap_or_default(), - &initial_commitments.unwrap_or_default(), - ) - } else { - initial_state() - } + // If initial commitments or accounts are present in config, need to construct state + // from them + if initial_commitments.is_some() || init_accs.is_some() { + V02State::new_with_genesis_accounts( + &init_accs.unwrap_or_default(), + &initial_commitments.unwrap_or_default(), + ) + } else { + initial_state() } }; @@ -485,7 +482,7 @@ mod tests { } #[tokio::test] - async fn test_transaction_pre_check_native_transfer_valid() { + async fn transaction_pre_check_native_transfer_valid() { let (_sequencer, _mempool_handle) = common_setup().await; let acc1 = initial_accounts()[0].account_id; diff --git a/sequencer_rpc/src/process.rs b/sequencer_rpc/src/process.rs index 443eb6f8..ce27e3f7 100644 --- a/sequencer_rpc/src/process.rs +++ b/sequencer_rpc/src/process.rs @@ -197,7 +197,8 @@ impl JsonHandler /// Returns the initial accounts for testnet. /// `ToDo`: Useful only for testnet and needs to be removed later. - async fn get_initial_testnet_accounts(&self, request: Request) -> Result { + #[expect(clippy::unused_self, reason = "Let all methods be uniform")] + fn get_initial_testnet_accounts(&self, request: Request) -> Result { let _get_initial_testnet_accounts_request = GetInitialTestnetAccountsRequest::parse(Some(request.params))?; @@ -320,7 +321,7 @@ impl JsonHandler GET_BLOCK_RANGE => self.process_get_block_range_data(request).await, GET_GENESIS => self.process_get_genesis(request).await, GET_LAST_BLOCK => self.process_get_last_block(request).await, - GET_INITIAL_TESTNET_ACCOUNTS => self.get_initial_testnet_accounts(request).await, + GET_INITIAL_TESTNET_ACCOUNTS => self.get_initial_testnet_accounts(request), GET_ACCOUNT_BALANCE => self.process_get_account_balance(request).await, GET_ACCOUNTS_NONCES => self.process_get_accounts_nonces(request).await, GET_ACCOUNT => self.process_get_account(request).await, diff --git a/wallet/src/config.rs b/wallet/src/config.rs index 6774044b..146b1a21 100644 --- a/wallet/src/config.rs +++ b/wallet/src/config.rs @@ -41,10 +41,18 @@ pub struct PersistentAccountDataPrivate { #[derive(Debug, Clone, Serialize, Deserialize)] pub enum InitialAccountData { Public(PublicAccountPrivateInitialData), - Private(PrivateAccountPrivateInitialData), + Private(Box), } impl InitialAccountData { + #[must_use] + pub const fn account_id(&self) -> nssa::AccountId { + match &self { + Self::Public(acc) => acc.account_id, + Self::Private(acc) => acc.account_id, + } + } + pub(crate) fn create_initial_accounts_data() -> Vec { let pub_data = initial_pub_accounts_private_keys(); let priv_data = initial_priv_accounts_private_keys(); @@ -117,16 +125,6 @@ impl PersistentStorage { } } -impl InitialAccountData { - #[must_use] - pub fn account_id(&self) -> nssa::AccountId { - match &self { - Self::Public(acc) => acc.account_id, - Self::Private(acc) => acc.account_id, - } - } -} - impl PersistentAccountData { #[must_use] pub fn account_id(&self) -> nssa::AccountId { @@ -146,7 +144,7 @@ impl From for InitialAccountData { impl From for InitialAccountData { fn from(value: PrivateAccountPrivateInitialData) -> Self { - Self::Private(value) + Self::Private(Box::new(value)) } } diff --git a/wallet/src/helperfunctions.rs b/wallet/src/helperfunctions.rs index 02e85fd7..ceeb467f 100644 --- a/wallet/src/helperfunctions.rs +++ b/wallet/src/helperfunctions.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, path::PathBuf, str::FromStr as _}; -use anyhow::{Context, Result}; -use base58::ToBase58; +use anyhow::{Context as _, Result}; +use base58::ToBase58 as _; use key_protocol::{ initial_state::{PrivateAccountPrivateInitialData, PublicAccountPrivateInitialData}, key_protocol_core::NSSAUserData, @@ -132,11 +132,11 @@ pub fn produce_data_for_storage( for (account_id, (key_chain, account)) in &user_data.default_user_private_accounts { vec_for_storage.push( - InitialAccountData::Private(PrivateAccountPrivateInitialData { + InitialAccountData::Private(Box::new(PrivateAccountPrivateInitialData { account_id: *account_id, account: account.clone(), key_chain: key_chain.clone(), - }) + })) .into(), ); } From 2c9361c6b51f61d9c07f2e7b87a3c1530d3a3b33 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Thu, 19 Mar 2026 13:01:43 +0200 Subject: [PATCH 17/22] fix: suggestions 1 --- indexer/core/src/config.rs | 4 ++-- indexer/core/src/lib.rs | 8 +++++--- integration_tests/src/config.rs | 12 ++++++------ sequencer_core/src/config.rs | 4 ++-- sequencer_core/src/lib.rs | 12 +++++++----- sequencer_rpc/src/process.rs | 4 ++-- 6 files changed, 24 insertions(+), 20 deletions(-) diff --git a/indexer/core/src/config.rs b/indexer/core/src/config.rs index 5eb8dd92..fced57d9 100644 --- a/indexer/core/src/config.rs +++ b/indexer/core/src/config.rs @@ -36,9 +36,9 @@ pub struct IndexerConfig { pub bedrock_client_config: ClientConfig, pub channel_id: ChannelId, #[serde(skip_serializing_if = "Option::is_none")] - pub initial_accounts: Option>, + pub initial_public_accounts: Option>, #[serde(skip_serializing_if = "Option::is_none")] - pub initial_commitments: Option>, + pub initial_private_accounts: Option>, } impl IndexerConfig { diff --git a/indexer/core/src/lib.rs b/indexer/core/src/lib.rs index 1f20e822..ca17bd64 100644 --- a/indexer/core/src/lib.rs +++ b/indexer/core/src/lib.rs @@ -58,7 +58,7 @@ impl IndexerCore { let genesis_block = hashable_data.into_pending_block(&signing_key, channel_genesis_msg_id); let initial_commitments: Option> = config - .initial_commitments + .initial_private_accounts .clone() .map(|initial_commitments| { initial_commitments @@ -76,8 +76,10 @@ impl IndexerCore { .collect() }); - let init_accs: Option> = - config.initial_accounts.clone().map(|initial_accounts| { + let init_accs: Option> = config + .initial_public_accounts + .clone() + .map(|initial_accounts| { initial_accounts .iter() .map(|acc_data| (acc_data.account_id, acc_data.balance)) diff --git a/integration_tests/src/config.rs b/integration_tests/src/config.rs index 875e1652..98181d26 100644 --- a/integration_tests/src/config.rs +++ b/integration_tests/src/config.rs @@ -105,7 +105,7 @@ impl InitialData { } } - fn sequencer_initial_accounts(&self) -> Vec { + fn sequencer_initial_public_accounts(&self) -> Vec { self.public_accounts .iter() .map(|(priv_key, balance)| { @@ -119,7 +119,7 @@ impl InitialData { .collect() } - fn sequencer_initial_commitments(&self) -> Vec { + fn sequencer_initial_private_accounts(&self) -> Vec { self.private_accounts .iter() .map(|(key_chain, account)| PrivateAccountPublicInitialData { @@ -184,8 +184,8 @@ pub fn indexer_config( max_retries: 10, }, }, - initial_accounts: Some(initial_data.sequencer_initial_accounts()), - initial_commitments: Some(initial_data.sequencer_initial_commitments()), + initial_public_accounts: Some(initial_data.sequencer_initial_public_accounts()), + initial_private_accounts: Some(initial_data.sequencer_initial_private_accounts()), signing_key: [37; 32], channel_id: bedrock_channel_id(), }) @@ -216,8 +216,8 @@ pub fn sequencer_config( block_create_timeout, retry_pending_blocks_timeout: Duration::from_secs(120), port: 0, - initial_accounts: Some(initial_data.sequencer_initial_accounts()), - initial_commitments: Some(initial_data.sequencer_initial_commitments()), + initial_public_accounts: Some(initial_data.sequencer_initial_public_accounts()), + initial_private_accounts: Some(initial_data.sequencer_initial_private_accounts()), signing_key: [37; 32], bedrock_config: BedrockConfig { backoff: BackoffConfig { diff --git a/sequencer_core/src/config.rs b/sequencer_core/src/config.rs index 46ee6945..571ead1a 100644 --- a/sequencer_core/src/config.rs +++ b/sequencer_core/src/config.rs @@ -50,9 +50,9 @@ pub struct SequencerConfig { /// Indexer RPC URL. pub indexer_rpc_url: Url, #[serde(skip_serializing_if = "Option::is_none")] - pub initial_accounts: Option>, + pub initial_public_accounts: Option>, #[serde(skip_serializing_if = "Option::is_none")] - pub initial_commitments: Option>, + pub initial_private_accounts: Option>, } #[derive(Clone, Serialize, Deserialize)] diff --git a/sequencer_core/src/lib.rs b/sequencer_core/src/lib.rs index 5f5653da..22a388dd 100644 --- a/sequencer_core/src/lib.rs +++ b/sequencer_core/src/lib.rs @@ -103,7 +103,7 @@ impl SequencerCore> = config - .initial_commitments + .initial_private_accounts .clone() .map(|initial_commitments| { initial_commitments @@ -121,8 +121,10 @@ impl SequencerCore> = - config.initial_accounts.clone().map(|initial_accounts| { + let init_accs: Option> = config + .initial_public_accounts + .clone() + .map(|initial_accounts| { initial_accounts .iter() .map(|acc_data| (acc_data.account_id, acc_data.balance)) @@ -419,8 +421,8 @@ mod tests { }, retry_pending_blocks_timeout: Duration::from_secs(60 * 4), indexer_rpc_url: "ws://localhost:8779".parse().unwrap(), - initial_accounts: None, - initial_commitments: None, + initial_public_accounts: None, + initial_private_accounts: None, } } diff --git a/sequencer_rpc/src/process.rs b/sequencer_rpc/src/process.rs index ce27e3f7..d662dacd 100644 --- a/sequencer_rpc/src/process.rs +++ b/sequencer_rpc/src/process.rs @@ -385,8 +385,8 @@ mod tests { }), }, indexer_rpc_url: "ws://localhost:8779".parse().unwrap(), - initial_accounts: None, - initial_commitments: None, + initial_public_accounts: None, + initial_private_accounts: None, } } From afa0a63c955fbde4603921e823bcc48d8499ea7e Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Thu, 19 Mar 2026 18:01:15 +0200 Subject: [PATCH 18/22] fix: suggestions fix 2 --- Cargo.lock | 16 ++++++ Cargo.toml | 2 + indexer/core/Cargo.toml | 2 +- indexer/core/src/config.rs | 4 +- indexer/core/src/lib.rs | 6 +-- integration_tests/Cargo.toml | 1 + integration_tests/src/config.rs | 12 ++--- .../src/key_management/secret_holders.rs | 5 +- key_protocol/src/lib.rs | 1 - nssa/Cargo.toml | 2 +- programs/amm/Cargo.toml | 6 ++- sequencer_core/Cargo.toml | 1 + sequencer_core/src/config.rs | 4 +- sequencer_core/src/lib.rs | 4 +- sequencer_rpc/Cargo.toml | 1 + sequencer_rpc/src/process.rs | 4 +- testnet_initial_state/Cargo.toml | 16 ++++++ .../src/lib.rs | 9 ++-- wallet/Cargo.toml | 1 + wallet/src/chain_storage.rs | 49 +++++++------------ wallet/src/cli/config.rs | 12 ++++- wallet/src/config.rs | 14 +++--- wallet/src/helperfunctions.rs | 6 +-- 23 files changed, 100 insertions(+), 78 deletions(-) create mode 100644 testnet_initial_state/Cargo.toml rename key_protocol/src/initial_state.rs => testnet_initial_state/src/lib.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index c3c354a9..43a1264f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3716,6 +3716,7 @@ dependencies = [ "serde_json", "storage", "tempfile", + "testnet_initial_state", "tokio", "url", ] @@ -3844,6 +3845,7 @@ dependencies = [ "serde_json", "tempfile", "testcontainers", + "testnet_initial_state", "token_core", "tokio", "url", @@ -7449,6 +7451,7 @@ dependencies = [ "serde_json", "storage", "tempfile", + "testnet_initial_state", "tokio", "url", ] @@ -7476,6 +7479,7 @@ dependencies = [ "serde", "serde_json", "tempfile", + "testnet_initial_state", "tokio", ] @@ -8179,6 +8183,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "testnet_initial_state" +version = "0.1.0" +dependencies = [ + "common", + "key_protocol", + "nssa", + "nssa_core", + "serde", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -8976,6 +8991,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "testnet_initial_state", "token_core", "tokio", "url", diff --git a/Cargo.toml b/Cargo.toml index bcd11651..cec35b00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ members = [ "examples/program_deployment/methods", "examples/program_deployment/methods/guest", "bedrock_client", + "testnet_initial_state", ] [workspace.dependencies] @@ -57,6 +58,7 @@ amm_core = { path = "programs/amm/core" } amm_program = { path = "programs/amm" } test_program_methods = { path = "test_program_methods" } bedrock_client = { path = "bedrock_client" } +testnet_initial_state = { path = "testnet_initial_state" } tokio = { version = "1.50", features = [ "net", diff --git a/indexer/core/Cargo.toml b/indexer/core/Cargo.toml index 56a7c813..2d7ec258 100644 --- a/indexer/core/Cargo.toml +++ b/indexer/core/Cargo.toml @@ -14,6 +14,7 @@ nssa.workspace = true nssa_core.workspace = true storage.workspace = true key_protocol.workspace = true +testnet_initial_state.workspace = true anyhow.workspace = true log.workspace = true @@ -29,4 +30,3 @@ async-stream.workspace = true [dev-dependencies] tempfile.workspace = true - diff --git a/indexer/core/src/config.rs b/indexer/core/src/config.rs index fced57d9..291e54f5 100644 --- a/indexer/core/src/config.rs +++ b/indexer/core/src/config.rs @@ -9,11 +9,9 @@ use anyhow::{Context as _, Result}; pub use bedrock_client::BackoffConfig; use common::config::BasicAuth; use humantime_serde; -use key_protocol::initial_state::{ - PrivateAccountPublicInitialData, PublicAccountPublicInitialData, -}; pub use logos_blockchain_core::mantle::ops::channel::ChannelId; use serde::{Deserialize, Serialize}; +use testnet_initial_state::{PrivateAccountPublicInitialData, PublicAccountPublicInitialData}; use url::Url; #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/indexer/core/src/lib.rs b/indexer/core/src/lib.rs index ca17bd64..5c453743 100644 --- a/indexer/core/src/lib.rs +++ b/indexer/core/src/lib.rs @@ -6,13 +6,13 @@ use common::{ HashType, PINATA_BASE58, block::{Block, HashableBlockData}, }; -use key_protocol::initial_state::initial_state_testnet; use log::{debug, error, info}; use logos_blockchain_core::mantle::{ Op, SignedMantleTx, ops::channel::{ChannelId, inscribe::InscriptionOp}, }; use nssa::V02State; +use testnet_initial_state::initial_state_testnet; use crate::{block_store::IndexerStore, config::IndexerConfig}; @@ -59,7 +59,7 @@ impl IndexerCore { let initial_commitments: Option> = config .initial_private_accounts - .clone() + .as_ref() .map(|initial_commitments| { initial_commitments .iter() @@ -78,7 +78,7 @@ impl IndexerCore { let init_accs: Option> = config .initial_public_accounts - .clone() + .as_ref() .map(|initial_accounts| { initial_accounts .iter() diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index b18b782f..7f9ccab3 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -20,6 +20,7 @@ serde_json.workspace = true token_core.workspace = true indexer_service_rpc.workspace = true wallet-ffi.workspace = true +testnet_initial_state.workspace = true url.workspace = true diff --git a/integration_tests/src/config.rs b/integration_tests/src/config.rs index 98181d26..277c5682 100644 --- a/integration_tests/src/config.rs +++ b/integration_tests/src/config.rs @@ -3,16 +3,14 @@ use std::{net::SocketAddr, path::PathBuf, time::Duration}; use anyhow::{Context as _, Result}; use bytesize::ByteSize; use indexer_service::{BackoffConfig, ChannelId, ClientConfig, IndexerConfig}; -use key_protocol::{ - initial_state::{ - PrivateAccountPrivateInitialData, PrivateAccountPublicInitialData, - PublicAccountPrivateInitialData, PublicAccountPublicInitialData, - }, - key_management::KeyChain, -}; +use key_protocol::key_management::KeyChain; use nssa::{Account, AccountId, PrivateKey, PublicKey}; use nssa_core::{account::Data, program::DEFAULT_PROGRAM_ID}; use sequencer_core::config::{BedrockConfig, SequencerConfig}; +use testnet_initial_state::{ + PrivateAccountPrivateInitialData, PrivateAccountPublicInitialData, + PublicAccountPrivateInitialData, PublicAccountPublicInitialData, +}; use url::Url; use wallet::config::{InitialAccountData, WalletConfig}; diff --git a/key_protocol/src/key_management/secret_holders.rs b/key_protocol/src/key_management/secret_holders.rs index db39757e..049c88a1 100644 --- a/key_protocol/src/key_management/secret_holders.rs +++ b/key_protocol/src/key_management/secret_holders.rs @@ -20,17 +20,16 @@ pub struct SeedHolder { #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] /// Secret spending key object. Can produce `PrivateKeyHolder` objects. -pub struct SecretSpendingKey(pub(crate) [u8; 32]); +pub struct SecretSpendingKey(pub [u8; 32]); pub type ViewingSecretKey = Scalar; #[derive(Serialize, Deserialize, Debug, Clone)] /// Private key holder. Produces public keys. Can produce `account_id`. Can produce shared secret /// for recepient. -#[expect(clippy::partial_pub_fields, reason = "TODO: fix later")] pub struct PrivateKeyHolder { pub nullifier_secret_key: NullifierSecretKey, - pub(crate) viewing_secret_key: ViewingSecretKey, + pub viewing_secret_key: ViewingSecretKey, } impl SeedHolder { diff --git a/key_protocol/src/lib.rs b/key_protocol/src/lib.rs index 87c7b8e4..e3fe31cf 100644 --- a/key_protocol/src/lib.rs +++ b/key_protocol/src/lib.rs @@ -1,5 +1,4 @@ #![expect(clippy::print_stdout, reason = "TODO: fix later")] -pub mod initial_state; pub mod key_management; pub mod key_protocol_core; diff --git a/nssa/Cargo.toml b/nssa/Cargo.toml index e1b6805f..b50f189b 100644 --- a/nssa/Cargo.toml +++ b/nssa/Cargo.toml @@ -37,4 +37,4 @@ test-case = "3.3.1" [features] default = [] prove = ["risc0-zkvm/prove"] -test-utils = [] \ No newline at end of file +test-utils = [] diff --git a/programs/amm/Cargo.toml b/programs/amm/Cargo.toml index 449d5dcc..30074ac8 100644 --- a/programs/amm/Cargo.toml +++ b/programs/amm/Cargo.toml @@ -8,10 +8,12 @@ license = { workspace = true } workspace = true [dependencies] -nssa = { workspace = true, optional = true, features = ["test-utils"], default-features = true } +nssa = { workspace = true, optional = true, features = [ + "test-utils", +], default-features = true } nssa_core.workspace = true token_core.workspace = true amm_core.workspace = true [features] -nssa = ["dep:nssa"] \ No newline at end of file +nssa = ["dep:nssa"] diff --git a/sequencer_core/Cargo.toml b/sequencer_core/Cargo.toml index 9f29a581..3801945f 100644 --- a/sequencer_core/Cargo.toml +++ b/sequencer_core/Cargo.toml @@ -15,6 +15,7 @@ storage.workspace = true mempool.workspace = true bedrock_client.workspace = true key_protocol.workspace = true +testnet_initial_state.workspace = true anyhow.workspace = true serde.workspace = true diff --git a/sequencer_core/src/config.rs b/sequencer_core/src/config.rs index 571ead1a..d6db21ff 100644 --- a/sequencer_core/src/config.rs +++ b/sequencer_core/src/config.rs @@ -10,11 +10,9 @@ use bedrock_client::BackoffConfig; use bytesize::ByteSize; use common::config::BasicAuth; use humantime_serde; -use key_protocol::initial_state::{ - PrivateAccountPublicInitialData, PublicAccountPublicInitialData, -}; use logos_blockchain_core::mantle::ops::channel::ChannelId; use serde::{Deserialize, Serialize}; +use testnet_initial_state::{PrivateAccountPublicInitialData, PublicAccountPublicInitialData}; use url::Url; // TODO: Provide default values diff --git a/sequencer_core/src/lib.rs b/sequencer_core/src/lib.rs index 22a388dd..2bac5b08 100644 --- a/sequencer_core/src/lib.rs +++ b/sequencer_core/src/lib.rs @@ -10,13 +10,13 @@ use common::{ transaction::NSSATransaction, }; use config::SequencerConfig; -use key_protocol::initial_state::initial_state; use log::{error, info, warn}; use logos_blockchain_key_management_system_service::keys::{ED25519_SECRET_KEY_SIZE, Ed25519Key}; use mempool::{MemPool, MemPoolHandle}; #[cfg(feature = "mock")] pub use mock::SequencerCoreWithMockClients; use nssa::V02State; +use testnet_initial_state::initial_state; use crate::{ block_settlement_client::{BlockSettlementClient, BlockSettlementClientTrait, MsgId}, @@ -386,9 +386,9 @@ mod tests { use bedrock_client::BackoffConfig; use common::{test_utils::sequencer_sign_key_for_testing, transaction::NSSATransaction}; - use key_protocol::initial_state::{initial_accounts, initial_pub_accounts_private_keys}; use logos_blockchain_core::mantle::ops::channel::ChannelId; use mempool::MemPoolHandle; + use testnet_initial_state::{initial_accounts, initial_pub_accounts_private_keys}; use crate::{ config::{BedrockConfig, SequencerConfig}, diff --git a/sequencer_rpc/Cargo.toml b/sequencer_rpc/Cargo.toml index 4d724fd8..f6da68e8 100644 --- a/sequencer_rpc/Cargo.toml +++ b/sequencer_rpc/Cargo.toml @@ -14,6 +14,7 @@ mempool.workspace = true sequencer_core = { workspace = true } bedrock_client.workspace = true key_protocol.workspace = true +testnet_initial_state.workspace = true anyhow.workspace = true serde_json.workspace = true diff --git a/sequencer_rpc/src/process.rs b/sequencer_rpc/src/process.rs index d662dacd..4376b28a 100644 --- a/sequencer_rpc/src/process.rs +++ b/sequencer_rpc/src/process.rs @@ -23,13 +23,13 @@ use common::{ transaction::{NSSATransaction, TransactionMalformationError}, }; use itertools::Itertools as _; -use key_protocol::initial_state::initial_accounts; use log::warn; use nssa::{self, program::Program}; use sequencer_core::{ block_settlement_client::BlockSettlementClientTrait, indexer_client::IndexerClientTrait, }; use serde_json::Value; +use testnet_initial_state::initial_accounts; use super::{JsonHandler, respond, types::err_rpc::RpcErr}; @@ -342,13 +342,13 @@ mod tests { use common::{ config::BasicAuth, test_utils::sequencer_sign_key_for_testing, transaction::NSSATransaction, }; - use key_protocol::initial_state::{initial_accounts, initial_pub_accounts_private_keys}; use sequencer_core::{ config::{BedrockConfig, SequencerConfig}, mock::{MockBlockSettlementClient, MockIndexerClient, SequencerCoreWithMockClients}, }; use serde_json::Value; use tempfile::tempdir; + use testnet_initial_state::{initial_accounts, initial_pub_accounts_private_keys}; use tokio::sync::Mutex; use crate::rpc_handler; diff --git a/testnet_initial_state/Cargo.toml b/testnet_initial_state/Cargo.toml new file mode 100644 index 00000000..2b73f479 --- /dev/null +++ b/testnet_initial_state/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "testnet_initial_state" +version = "0.1.0" +edition = "2024" +license.workspace = true + +[dependencies] +key_protocol.workspace = true +nssa.workspace = true +nssa_core.workspace = true +common.workspace = true + +serde.workspace = true + +[lints] +workspace = true diff --git a/key_protocol/src/initial_state.rs b/testnet_initial_state/src/lib.rs similarity index 99% rename from key_protocol/src/initial_state.rs rename to testnet_initial_state/src/lib.rs index 2150bd4c..0c49b2e8 100644 --- a/key_protocol/src/initial_state.rs +++ b/testnet_initial_state/src/lib.rs @@ -1,12 +1,11 @@ use common::PINATA_BASE58; -use nssa::{Account, AccountId, Data, PrivateKey, PublicKey, V02State}; -use nssa_core::{NullifierPublicKey, encryption::shared_key_derivation::Secp256k1Point}; -use serde::{Deserialize, Serialize}; - -use crate::key_management::{ +use key_protocol::key_management::{ KeyChain, secret_holders::{PrivateKeyHolder, SecretSpendingKey}, }; +use nssa::{Account, AccountId, Data, PrivateKey, PublicKey, V02State}; +use nssa_core::{NullifierPublicKey, encryption::shared_key_derivation::Secp256k1Point}; +use serde::{Deserialize, Serialize}; const PRIVATE_KEY_PUB_ACC_A: [u8; 32] = [ 16, 162, 106, 154, 236, 125, 52, 184, 35, 100, 238, 174, 69, 197, 41, 77, 187, 10, 118, 75, 0, diff --git a/wallet/Cargo.toml b/wallet/Cargo.toml index 63e14bb6..ca6b8f1b 100644 --- a/wallet/Cargo.toml +++ b/wallet/Cargo.toml @@ -14,6 +14,7 @@ common.workspace = true key_protocol.workspace = true token_core.workspace = true amm_core.workspace = true +testnet_initial_state.workspace = true anyhow.workspace = true serde_json.workspace = true diff --git a/wallet/src/chain_storage.rs b/wallet/src/chain_storage.rs index 2dd9d64f..3e1a8de7 100644 --- a/wallet/src/chain_storage.rs +++ b/wallet/src/chain_storage.rs @@ -99,39 +99,24 @@ impl WalletChainStore { let mut public_init_acc_map = BTreeMap::new(); let mut private_init_acc_map = BTreeMap::new(); - // If initial accounts are present in config, need to construct state from them - if let Some(initial_accounts) = config.initial_accounts.clone() { - for init_acc_data in initial_accounts { - match init_acc_data { - InitialAccountData::Public(data) => { - public_init_acc_map.insert(data.account_id, data.pub_sign_key); - } - InitialAccountData::Private(data) => { - let mut account = data.account; - // TODO: Program owner is only known after code is compiled and can't be set - // in the config. Therefore we overwrite it here on - // startup. Fix this when program id can be fetched - // from the node and queried from the wallet. - account.program_owner = Program::authenticated_transfer_program().id(); - private_init_acc_map.insert(data.account_id, (data.key_chain, account)); - } + let initial_accounts = config + .initial_accounts + .clone() + .unwrap_or_else(InitialAccountData::create_initial_accounts_data); + + for init_acc_data in initial_accounts { + match init_acc_data { + InitialAccountData::Public(data) => { + public_init_acc_map.insert(data.account_id, data.pub_sign_key); } - } - } else { - for init_acc_data in InitialAccountData::create_initial_accounts_data() { - match init_acc_data { - InitialAccountData::Public(data) => { - public_init_acc_map.insert(data.account_id, data.pub_sign_key); - } - InitialAccountData::Private(data) => { - let mut account = data.account; - // TODO: Program owner is only known after code is compiled and can't be set - // in the config. Therefore we overwrite it here on - // startup. Fix this when program id can be fetched - // from the node and queried from the wallet. - account.program_owner = Program::authenticated_transfer_program().id(); - private_init_acc_map.insert(data.account_id, (data.key_chain, account)); - } + InitialAccountData::Private(data) => { + let mut account = data.account; + // TODO: Program owner is only known after code is compiled and can't be set + // in the config. Therefore we overwrite it here on + // startup. Fix this when program id can be fetched + // from the node and queried from the wallet. + account.program_owner = Program::authenticated_transfer_program().id(); + private_init_acc_map.insert(data.account_id, (data.key_chain, account)); } } } diff --git a/wallet/src/cli/config.rs b/wallet/src/cli/config.rs index ae70a63b..79b7aef2 100644 --- a/wallet/src/cli/config.rs +++ b/wallet/src/cli/config.rs @@ -69,7 +69,17 @@ impl WalletSubcommand for ConfigSubcommand { ); } "initial_accounts" => { - println!("{:#?}", InitialAccountData::create_initial_accounts_data()); + println!( + "{:#?}", + wallet_core + .storage + .wallet_config + .initial_accounts + .clone() + .unwrap_or_else( + InitialAccountData::create_initial_accounts_data + ) + ); } "basic_auth" => { if let Some(basic_auth) = &wallet_core.storage.wallet_config.basic_auth diff --git a/wallet/src/config.rs b/wallet/src/config.rs index 146b1a21..10a70030 100644 --- a/wallet/src/config.rs +++ b/wallet/src/config.rs @@ -8,17 +8,15 @@ use std::{ use anyhow::{Context as _, Result}; use common::config::BasicAuth; use humantime_serde; -use key_protocol::{ - initial_state::{ - PrivateAccountPrivateInitialData, PublicAccountPrivateInitialData, - initial_priv_accounts_private_keys, initial_pub_accounts_private_keys, - }, - key_management::key_tree::{ - chain_index::ChainIndex, keys_private::ChildKeysPrivate, keys_public::ChildKeysPublic, - }, +use key_protocol::key_management::key_tree::{ + chain_index::ChainIndex, keys_private::ChildKeysPrivate, keys_public::ChildKeysPublic, }; use log::warn; use serde::{Deserialize, Serialize}; +use testnet_initial_state::{ + PrivateAccountPrivateInitialData, PublicAccountPrivateInitialData, + initial_priv_accounts_private_keys, initial_pub_accounts_private_keys, +}; use url::Url; #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/wallet/src/helperfunctions.rs b/wallet/src/helperfunctions.rs index ceeb467f..e12fc444 100644 --- a/wallet/src/helperfunctions.rs +++ b/wallet/src/helperfunctions.rs @@ -2,14 +2,12 @@ use std::{collections::HashMap, path::PathBuf, str::FromStr as _}; use anyhow::{Context as _, Result}; use base58::ToBase58 as _; -use key_protocol::{ - initial_state::{PrivateAccountPrivateInitialData, PublicAccountPrivateInitialData}, - key_protocol_core::NSSAUserData, -}; +use key_protocol::key_protocol_core::NSSAUserData; use nssa::Account; use nssa_core::account::Nonce; use rand::{RngCore as _, rngs::OsRng}; use serde::Serialize; +use testnet_initial_state::{PrivateAccountPrivateInitialData, PublicAccountPrivateInitialData}; use crate::{ HOME_DIR_ENV_VAR, From e6cb4ecf9444ed8fd2b90c039836a2c345d4e49d Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Thu, 19 Mar 2026 19:47:47 +0200 Subject: [PATCH 19/22] fix: machete fix --- Cargo.lock | 3 --- indexer/core/Cargo.toml | 1 - sequencer_core/Cargo.toml | 1 - sequencer_rpc/Cargo.toml | 1 - 4 files changed, 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 43a1264f..1dba023f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3707,7 +3707,6 @@ dependencies = [ "common", "futures", "humantime-serde", - "key_protocol", "log", "logos-blockchain-core", "nssa", @@ -7439,7 +7438,6 @@ dependencies = [ "futures", "humantime-serde", "jsonrpsee", - "key_protocol", "log", "logos-blockchain-core", "logos-blockchain-key-management-system-service", @@ -7471,7 +7469,6 @@ dependencies = [ "futures", "hex", "itertools 0.14.0", - "key_protocol", "log", "mempool", "nssa", diff --git a/indexer/core/Cargo.toml b/indexer/core/Cargo.toml index 2d7ec258..33fe2d9d 100644 --- a/indexer/core/Cargo.toml +++ b/indexer/core/Cargo.toml @@ -13,7 +13,6 @@ bedrock_client.workspace = true nssa.workspace = true nssa_core.workspace = true storage.workspace = true -key_protocol.workspace = true testnet_initial_state.workspace = true anyhow.workspace = true diff --git a/sequencer_core/Cargo.toml b/sequencer_core/Cargo.toml index 3801945f..e1ff0895 100644 --- a/sequencer_core/Cargo.toml +++ b/sequencer_core/Cargo.toml @@ -14,7 +14,6 @@ common.workspace = true storage.workspace = true mempool.workspace = true bedrock_client.workspace = true -key_protocol.workspace = true testnet_initial_state.workspace = true anyhow.workspace = true diff --git a/sequencer_rpc/Cargo.toml b/sequencer_rpc/Cargo.toml index f6da68e8..afab1a89 100644 --- a/sequencer_rpc/Cargo.toml +++ b/sequencer_rpc/Cargo.toml @@ -13,7 +13,6 @@ common.workspace = true mempool.workspace = true sequencer_core = { workspace = true } bedrock_client.workspace = true -key_protocol.workspace = true testnet_initial_state.workspace = true anyhow.workspace = true From 1eae20ba3f55ddde292a4b4de96706ba80661ec4 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Mon, 23 Mar 2026 18:16:53 +0200 Subject: [PATCH 20/22] fix: merge fixes --- storage/src/indexer/mod.rs | 20 ++++++++++---------- storage/src/indexer/read_once.rs | 6 +++--- storage/src/indexer/write_non_atomic.rs | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/storage/src/indexer/mod.rs b/storage/src/indexer/mod.rs index bb84069a..299d3484 100644 --- a/storage/src/indexer/mod.rs +++ b/storage/src/indexer/mod.rs @@ -1,7 +1,7 @@ use std::{path::Path, sync::Arc}; use common::block::Block; -use nssa::V02State; +use nssa::V03State; use rocksdb::{ BoundColumnFamily, ColumnFamilyDescriptor, DBWithThreadMode, MultiThreaded, Options, }; @@ -65,7 +65,7 @@ impl RocksDBIO { pub fn open_or_create( path: &Path, genesis_block: &Block, - initial_state: &V02State, + initial_state: &V03State, ) -> DbResult { let mut cf_opts = Options::default(); cf_opts.set_max_write_buffer_number(16); @@ -160,7 +160,7 @@ impl RocksDBIO { // State - pub fn calculate_state_for_id(&self, block_id: u64) -> DbResult { + pub fn calculate_state_for_id(&self, block_id: u64) -> DbResult { let last_block = self.get_meta_last_block_in_db()?; if block_id <= last_block { @@ -205,7 +205,7 @@ impl RocksDBIO { } } - pub fn final_state(&self) -> DbResult { + pub fn final_state(&self) -> DbResult { self.calculate_state_for_id(self.get_meta_last_block_in_db()?) } } @@ -253,7 +253,7 @@ mod tests { let dbio = RocksDBIO::open_or_create( temdir_path, &genesis_block(), - &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + &nssa::V03State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), ) .unwrap(); @@ -290,7 +290,7 @@ mod tests { let dbio = RocksDBIO::open_or_create( temdir_path, &genesis_block(), - &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + &nssa::V03State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), ) .unwrap(); @@ -343,7 +343,7 @@ mod tests { let dbio = RocksDBIO::open_or_create( temdir_path, &genesis_block(), - &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + &nssa::V03State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), ) .unwrap(); @@ -416,7 +416,7 @@ mod tests { let dbio = RocksDBIO::open_or_create( temdir_path, &genesis_block(), - &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + &nssa::V03State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), ) .unwrap(); @@ -499,7 +499,7 @@ mod tests { let dbio = RocksDBIO::open_or_create( temdir_path, &genesis_block(), - &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + &nssa::V03State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), ) .unwrap(); @@ -595,7 +595,7 @@ mod tests { let dbio = RocksDBIO::open_or_create( temdir_path, &genesis_block(), - &nssa::V02State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), + &nssa::V03State::new_with_genesis_accounts(&[(acc1(), 10000), (acc2(), 20000)], &[]), ) .unwrap(); diff --git a/storage/src/indexer/read_once.rs b/storage/src/indexer/read_once.rs index 19aee681..74d1afe9 100644 --- a/storage/src/indexer/read_once.rs +++ b/storage/src/indexer/read_once.rs @@ -1,7 +1,7 @@ use super::{ Block, DB_META_FIRST_BLOCK_IN_DB_KEY, DB_META_FIRST_BLOCK_SET_KEY, DB_META_LAST_BLOCK_IN_DB_KEY, DB_META_LAST_BREAKPOINT_ID, - DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DbError, DbResult, RocksDBIO, V02State, + DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DbError, DbResult, RocksDBIO, V03State, }; #[expect(clippy::multiple_inherent_impl, reason = "Readability")] @@ -175,7 +175,7 @@ impl RocksDBIO { // State - pub fn get_breakpoint(&self, br_id: u64) -> DbResult { + pub fn get_breakpoint(&self, br_id: u64) -> DbResult { let cf_br = self.breakpoint_column(); let res = self .db @@ -191,7 +191,7 @@ impl RocksDBIO { .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; if let Some(data) = res { - Ok(borsh::from_slice::(&data).map_err(|serr| { + Ok(borsh::from_slice::(&data).map_err(|serr| { DbError::borsh_cast_message( serr, Some("Failed to deserialize breakpoint data".to_owned()), diff --git a/storage/src/indexer/write_non_atomic.rs b/storage/src/indexer/write_non_atomic.rs index 84fc7de5..17c1be18 100644 --- a/storage/src/indexer/write_non_atomic.rs +++ b/storage/src/indexer/write_non_atomic.rs @@ -1,7 +1,7 @@ use super::{ BREAKPOINT_INTERVAL, DB_META_FIRST_BLOCK_SET_KEY, DB_META_LAST_BLOCK_IN_DB_KEY, DB_META_LAST_BREAKPOINT_ID, DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DbError, - DbResult, RocksDBIO, V02State, + DbResult, RocksDBIO, V03State, }; #[expect(clippy::multiple_inherent_impl, reason = "Readability")] @@ -101,7 +101,7 @@ impl RocksDBIO { // State - pub fn put_breakpoint(&self, br_id: u64, breakpoint: &V02State) -> DbResult<()> { + pub fn put_breakpoint(&self, br_id: u64, breakpoint: &V03State) -> DbResult<()> { let cf_br = self.breakpoint_column(); self.db From c3723776c6e7f74f05d0993ee8eae3eba52a01cf Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Mon, 23 Mar 2026 19:21:42 +0200 Subject: [PATCH 21/22] fix: deny fix --- Cargo.lock | 107 ++++++----------------------------------------------- 1 file changed, 12 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e0461fd..228ab10c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -629,9 +629,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "astral-tokio-tar" -version = "0.5.6" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec179a06c1769b1e42e1e2cbe74c7dcdb3d6383c838454d063eaac5bbb7ebbe5" +checksum = "3c23f3af104b40a3430ccb90ed5f7bd877a8dc5c26fc92fde51a22b40890dcf9" dependencies = [ "filetime", "futures-core", @@ -1939,7 +1939,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 2.0.117", + "syn 1.0.109", ] [[package]] @@ -5930,7 +5930,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -5943,7 +5943,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -6024,7 +6024,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -6919,7 +6919,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 0.26.11", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6930,9 +6930,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "ring", "rustls-pki-types", @@ -7842,9 +7842,9 @@ dependencies = [ [[package]] name = "testcontainers" -version = "0.27.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c0624faaa317c56d6d19136580be889677259caf5c897941c6f446b4655068" +checksum = "0bd36b06a2a6c0c3c81a83be1ab05fe86460d054d4d51bf513bc56b3e15bdc22" dependencies = [ "astral-tokio-tar", "async-trait", @@ -9038,24 +9038,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -9089,30 +9071,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -9125,12 +9090,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -9143,12 +9102,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -9161,24 +9114,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -9191,12 +9132,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -9209,12 +9144,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -9227,12 +9156,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -9245,12 +9168,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.7.15" From ce136348e7a746a28b1e9380848aef1c88ac1593 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Mon, 23 Mar 2026 19:22:56 +0200 Subject: [PATCH 22/22] fix: deny fix --- Cargo.lock | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9f6e2154..6102fb5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -162,7 +162,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -173,7 +173,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -629,9 +629,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "astral-tokio-tar" -version = "0.5.6" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec179a06c1769b1e42e1e2cbe74c7dcdb3d6383c838454d063eaac5bbb7ebbe5" +checksum = "3c23f3af104b40a3430ccb90ed5f7bd877a8dc5c26fc92fde51a22b40890dcf9" dependencies = [ "filetime", "futures-core", @@ -1938,7 +1938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 2.0.117", + "syn 1.0.109", ] [[package]] @@ -2087,7 +2087,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2388,7 +2388,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5931,7 +5931,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -5944,7 +5944,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -6862,7 +6862,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6931,9 +6931,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "ring", "rustls-pki-types", @@ -7539,7 +7539,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -7790,7 +7790,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7843,9 +7843,9 @@ dependencies = [ [[package]] name = "testcontainers" -version = "0.27.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c0624faaa317c56d6d19136580be889677259caf5c897941c6f446b4655068" +checksum = "0bd36b06a2a6c0c3c81a83be1ab05fe86460d054d4d51bf513bc56b3e15bdc22" dependencies = [ "astral-tokio-tar", "async-trait", @@ -8954,7 +8954,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]]