From 4667424215a268ef486f1d9ecbf2ae2cf7e9b915 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Tue, 27 Jan 2026 13:36:49 +0200 Subject: [PATCH] feat: first diff modification --- storage/src/indexer.rs | 142 ++++++++++++++++++++--------------------- 1 file changed, 70 insertions(+), 72 deletions(-) diff --git a/storage/src/indexer.rs b/storage/src/indexer.rs index 5a21ecb2..8c461eb1 100644 --- a/storage/src/indexer.rs +++ b/storage/src/indexer.rs @@ -5,30 +5,27 @@ use rocksdb::{BoundColumnFamily, ColumnFamilyDescriptor, DBWithThreadMode, Multi use crate::error::DbError; -/// Maximal size of stored blocks in base +/// Maximal size of stored diff in base /// /// Used to control db size /// /// Currently effectively unbounded. pub const BUFF_SIZE_ROCKSDB: usize = usize::MAX; -/// Size of stored blocks cache in memory -/// -/// Keeping small to not run out of memory -pub const CACHE_SIZE: usize = 1000; +/// Delay in diffs between breakpoints +pub const BREAKPOINT_DELAY: usize = 100; -/// Key base for storing metainformation about id of first block in db -pub const DB_META_FIRST_BLOCK_IN_DB_KEY: &str = "first_block_in_db"; -/// Key base for storing metainformation about id of last current block in db -pub const DB_META_LAST_BLOCK_IN_DB_KEY: &str = "last_block_in_db"; -/// Key base for storing metainformation which describe if first block has been set -pub const DB_META_FIRST_BLOCK_SET_KEY: &str = "first_block_set"; +/// Key base for storing metainformation about id of first diff in db +pub const DB_META_FIRST_DIFF_IN_DB_KEY: &str = "first_diff_in_db"; +/// Key base for storing metainformation about id of last current diff in db +pub const DB_META_LAST_DIFF_IN_DB_KEY: &str = "last_diff_in_db"; +/// Key base for storing metainformation which describe if first diff has been set +pub const DB_META_FIRST_DIFF_SET_KEY: &str = "first_diff_set"; -/// Key base for storing snapshot which describe block id -pub const DB_SNAPSHOT_BLOCK_ID_KEY: &str = "block_id"; - -/// Name of block column family -pub const CF_BLOCK_NAME: &str = "cf_block"; +/// Name of diff column family +pub const CF_DIFF_NAME: &str = "cf_diff"; +/// Name of breakpoint coumn family +pub const CF_BREAKPOINT_NAME: &str = "cf_breakpoint"; /// Name of meta column family pub const CF_META_NAME: &str = "cf_meta"; @@ -39,12 +36,13 @@ pub struct RocksDBIO { } impl RocksDBIO { - pub fn open_or_create(path: &Path, start_block: Option) -> DbResult { + pub fn open_or_create(path: &Path, start_diff: Option) -> DbResult { let mut cf_opts = Options::default(); cf_opts.set_max_write_buffer_number(16); // ToDo: Add more column families for different data - let cfb = ColumnFamilyDescriptor::new(CF_BLOCK_NAME, cf_opts.clone()); + let cfdiff = ColumnFamilyDescriptor::new(CF_DIFF_NAME, cf_opts.clone()); let cfmeta = ColumnFamilyDescriptor::new(CF_META_NAME, cf_opts.clone()); + let cfbr = ColumnFamilyDescriptor::new(CF_BREAKPOINT_NAME, cf_opts.clone()); let mut db_opts = Options::default(); db_opts.create_missing_column_families(true); @@ -52,7 +50,7 @@ impl RocksDBIO { let db = DBWithThreadMode::::open_cf_descriptors( &db_opts, path, - vec![cfb, cfmeta], + vec![cfdiff, cfmeta, cfbr], ); let dbio = Self { @@ -60,19 +58,19 @@ impl RocksDBIO { db: db.unwrap(), }; - let is_start_set = dbio.get_meta_is_first_block_set()?; + let is_start_set = dbio.get_meta_is_first_diff_set()?; if is_start_set { Ok(dbio) - } else if let Some(block) = start_block { - let block_id = block.header.block_id; - dbio.put_meta_first_block_in_db(block)?; - dbio.put_meta_is_first_block_set()?; - dbio.put_meta_last_block_in_db(block_id)?; + } else if let Some(diff) = start_diff { + let diff_id = diff.header.diff_id; + dbio.put_meta_first_diff_in_db(diff)?; + dbio.put_meta_is_first_diff_set()?; + dbio.put_meta_last_diff_in_db(diff_id)?; Ok(dbio) } else { - // Here we are trying to start a DB without a block, one should not do it. + // Here we are trying to start a DB without a diff, one should not do it. unreachable!() } } @@ -81,7 +79,7 @@ impl RocksDBIO { let mut cf_opts = Options::default(); cf_opts.set_max_write_buffer_number(16); // ToDo: Add more column families for different data - let _cfb = ColumnFamilyDescriptor::new(CF_BLOCK_NAME, cf_opts.clone()); + let _cfb = ColumnFamilyDescriptor::new(CF_DIFF_NAME, cf_opts.clone()); let _cfmeta = ColumnFamilyDescriptor::new(CF_META_NAME, cf_opts.clone()); let mut db_opts = Options::default(); @@ -95,20 +93,20 @@ impl RocksDBIO { self.db.cf_handle(CF_META_NAME).unwrap() } - pub fn block_column(&self) -> Arc> { - self.db.cf_handle(CF_BLOCK_NAME).unwrap() + pub fn diff_column(&self) -> Arc> { + self.db.cf_handle(CF_DIFF_NAME).unwrap() } - pub fn get_meta_first_block_in_db(&self) -> DbResult { + pub fn get_meta_first_diff_in_db(&self) -> DbResult { let cf_meta = self.meta_column(); let res = self .db .get_cf( &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { + borsh::to_vec(&DB_META_FIRST_DIFF_IN_DB_KEY).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), + Some("Failed to serialize DB_META_FIRST_DIFF_IN_DB_KEY".to_string()), ) })?, ) @@ -118,26 +116,26 @@ impl RocksDBIO { Ok(borsh::from_slice::(&data).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to deserialize first block".to_string()), + Some("Failed to deserialize first diff".to_string()), ) })?) } else { Err(DbError::db_interaction_error( - "First block not found".to_string(), + "First diff not found".to_string(), )) } } - pub fn get_meta_last_block_in_db(&self) -> DbResult { + pub fn get_meta_last_diff_in_db(&self) -> DbResult { let cf_meta = self.meta_column(); let res = self .db .get_cf( &cf_meta, - borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { + borsh::to_vec(&DB_META_LAST_DIFF_IN_DB_KEY).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), + Some("Failed to serialize DB_META_LAST_DIFF_IN_DB_KEY".to_string()), ) })?, ) @@ -147,26 +145,26 @@ impl RocksDBIO { Ok(borsh::from_slice::(&data).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to deserialize last block".to_string()), + Some("Failed to deserialize last diff".to_string()), ) })?) } else { Err(DbError::db_interaction_error( - "Last block not found".to_string(), + "Last diff not found".to_string(), )) } } - pub fn get_meta_is_first_block_set(&self) -> DbResult { + pub fn get_meta_is_first_diff_set(&self) -> DbResult { let cf_meta = self.meta_column(); let res = self .db .get_cf( &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { + borsh::to_vec(&DB_META_FIRST_DIFF_SET_KEY).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), + Some("Failed to serialize DB_META_FIRST_DIFF_SET_KEY".to_string()), ) })?, ) @@ -175,45 +173,45 @@ impl RocksDBIO { Ok(res.is_some()) } - pub fn put_meta_first_block_in_db(&self, block: Block) -> DbResult<()> { + pub fn put_meta_first_diff_in_db(&self, diff: Block) -> DbResult<()> { let cf_meta = self.meta_column(); self.db .put_cf( &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_IN_DB_KEY).map_err(|err| { + borsh::to_vec(&DB_META_FIRST_DIFF_IN_DB_KEY).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to serialize DB_META_FIRST_BLOCK_IN_DB_KEY".to_string()), + Some("Failed to serialize DB_META_FIRST_DIFF_IN_DB_KEY".to_string()), ) })?, - borsh::to_vec(&block.header.block_id).map_err(|err| { + borsh::to_vec(&diff.header.diff_id).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to serialize first block id".to_string()), + Some("Failed to serialize first diff id".to_string()), ) })?, ) .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None))?; - self.put_block(block, true)?; + self.put_diff(diff, true)?; Ok(()) } - pub fn put_meta_last_block_in_db(&self, block_id: u64) -> DbResult<()> { + pub fn put_meta_last_diff_in_db(&self, diff_id: u64) -> DbResult<()> { let cf_meta = self.meta_column(); self.db .put_cf( &cf_meta, - borsh::to_vec(&DB_META_LAST_BLOCK_IN_DB_KEY).map_err(|err| { + borsh::to_vec(&DB_META_LAST_DIFF_IN_DB_KEY).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to serialize DB_META_LAST_BLOCK_IN_DB_KEY".to_string()), + Some("Failed to serialize DB_META_LAST_DIFF_IN_DB_KEY".to_string()), ) })?, - borsh::to_vec(&block_id).map_err(|err| { + borsh::to_vec(&diff_id).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to serialize last block id".to_string()), + Some("Failed to serialize last diff id".to_string()), ) })?, ) @@ -221,15 +219,15 @@ impl RocksDBIO { Ok(()) } - pub fn put_meta_is_first_block_set(&self) -> DbResult<()> { + pub fn put_meta_is_first_diff_set(&self) -> DbResult<()> { let cf_meta = self.meta_column(); self.db .put_cf( &cf_meta, - borsh::to_vec(&DB_META_FIRST_BLOCK_SET_KEY).map_err(|err| { + borsh::to_vec(&DB_META_FIRST_DIFF_SET_KEY).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to serialize DB_META_FIRST_BLOCK_SET_KEY".to_string()), + Some("Failed to serialize DB_META_FIRST_DIFF_SET_KEY".to_string()), ) })?, [1u8; 1], @@ -238,30 +236,30 @@ impl RocksDBIO { Ok(()) } - pub fn put_block(&self, block: Block, first: bool) -> DbResult<()> { - let cf_block = self.block_column(); + pub fn put_diff(&self, diff: Block, first: bool) -> DbResult<()> { + let cf_diff = self.diff_column(); if !first { - let last_curr_block = self.get_meta_last_block_in_db()?; + let last_curr_diff = self.get_meta_last_diff_in_db()?; - if block.header.block_id > last_curr_block { - self.put_meta_last_block_in_db(block.header.block_id)?; + if diff.header.diff_id > last_curr_diff { + self.put_meta_last_diff_in_db(diff.header.diff_id)?; } } self.db .put_cf( - &cf_block, - borsh::to_vec(&block.header.block_id).map_err(|err| { + &cf_diff, + borsh::to_vec(&diff.header.diff_id).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to serialize block id".to_string()), + Some("Failed to serialize diff id".to_string()), ) })?, - borsh::to_vec(&HashableBlockData::from(block)).map_err(|err| { + borsh::to_vec(&HashableBlockData::from(diff)).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to serialize block data".to_string()), + Some("Failed to serialize diff data".to_string()), ) })?, ) @@ -269,16 +267,16 @@ impl RocksDBIO { Ok(()) } - pub fn get_block(&self, block_id: u64) -> DbResult { - let cf_block = self.block_column(); + pub fn get_diff(&self, diff_id: u64) -> DbResult { + let cf_diff = self.diff_column(); let res = self .db .get_cf( - &cf_block, - borsh::to_vec(&block_id).map_err(|err| { + &cf_diff, + borsh::to_vec(&diff_id).map_err(|err| { DbError::borsh_cast_message( err, - Some("Failed to serialize block id".to_string()), + Some("Failed to serialize diff id".to_string()), ) })?, ) @@ -289,7 +287,7 @@ impl RocksDBIO { borsh::from_slice::(&data).map_err(|serr| { DbError::borsh_cast_message( serr, - Some("Failed to deserialize block data".to_string()), + Some("Failed to deserialize diff data".to_string()), ) })?, )