diff --git a/fluffy/tools/portal_bridge/portal_bridge_state.nim b/fluffy/tools/portal_bridge/portal_bridge_state.nim index b5692c6ee..cb0314f06 100644 --- a/fluffy/tools/portal_bridge/portal_bridge_state.nim +++ b/fluffy/tools/portal_bridge/portal_bridge_state.nim @@ -20,7 +20,7 @@ import ../../common/common_utils, ../../rpc/rpc_calls/rpc_trace_calls, ../../network/state/state_content, - ./state_bridge/[database, state_diff, world_state_helper, offers_builder], + ./state_bridge/[database, state_diff, world_state_helper], ./[portal_bridge_conf, portal_bridge_common] type BlockDataRef = ref object @@ -48,7 +48,7 @@ proc runBackfillCollectBlockDataLoop( var currentBlockNumber = startBlockNumber while true: - if currentBlockNumber mod 10000 == 0: + if currentBlockNumber mod 100000 == 0: info "Collecting block data for block number: ", blockNumber = currentBlockNumber let @@ -100,7 +100,7 @@ proc runBackfillCollectBlockDataLoop( proc runBackfillBuildBlockOffersLoop( blockDataQueue: AsyncQueue[BlockDataRef], - blockOffersQueue: AsyncQueue[BlockOffersRef], + #blockOffersQueue: AsyncQueue[BlockOffersRef], stateDir: string, ) {.async: (raises: [CancelledError]).} = info "Starting state backfill build block offers loop" @@ -125,24 +125,24 @@ proc runBackfillBuildBlockOffersLoop( "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" ) - var builder = OffersBuilderRef.init(ws, genesisBlockHash) - builder.buildBlockOffers() + # var builder = OffersBuilderRef.init(ws, genesisBlockHash) + # builder.buildBlockOffers() - await blockOffersQueue.addLast( - BlockOffersRef( - blockNumber: 0.uint64, - accountTrieOffers: builder.getAccountTrieOffers(), - contractTrieOffers: builder.getContractTrieOffers(), - contractCodeOffers: builder.getContractCodeOffers(), - ) - ) + # await blockOffersQueue.addLast( + # BlockOffersRef( + # blockNumber: 0.uint64, + # accountTrieOffers: builder.getAccountTrieOffers(), + # contractTrieOffers: builder.getContractTrieOffers(), + # contractCodeOffers: builder.getContractCodeOffers(), + # ) + # ) ws while true: let blockData = await blockDataQueue.popFirst() - if blockData.blockNumber mod 10000 == 0: + if blockData.blockNumber mod 100000 == 0: info "Building state for block number: ", blockNumber = blockData.blockNumber # For now all WorldStateRef functions need to be inside a transaction @@ -165,30 +165,30 @@ proc runBackfillBuildBlockOffersLoop( trace "State diffs successfully applied to block number:", blockNumber = blockData.blockNumber - var builder = OffersBuilderRef.init( - worldState, KeccakHash.fromBytes(blockData.blockObject.hash.bytes()) - ) - builder.buildBlockOffers() + # var builder = OffersBuilderRef.init( + # worldState, KeccakHash.fromBytes(blockData.blockObject.hash.bytes()) + # ) + # builder.buildBlockOffers() - await blockOffersQueue.addLast( - BlockOffersRef( - blockNumber: blockData.blockNumber, - accountTrieOffers: builder.getAccountTrieOffers(), - contractTrieOffers: builder.getContractTrieOffers(), - contractCodeOffers: builder.getContractCodeOffers(), - ) - ) + # await blockOffersQueue.addLast( + # BlockOffersRef( + # blockNumber: blockData.blockNumber, + # accountTrieOffers: builder.getAccountTrieOffers(), + # contractTrieOffers: builder.getContractTrieOffers(), + # contractCodeOffers: builder.getContractCodeOffers(), + # ) + # ) proc runBackfillMetricsLoop( blockDataQueue: AsyncQueue[BlockDataRef], - blockOffersQueue: AsyncQueue[BlockOffersRef], + #blockOffersQueue: AsyncQueue[BlockOffersRef], ) {.async: (raises: [CancelledError]).} = debug "Starting state backfill metrics loop" while true: await sleepAsync(10.seconds) info "Block data queue length: ", blockDataQueueLen = blockDataQueue.len() - info "Block offers queue length: ", blockOffersQueueLen = blockOffersQueue.len() + # info "Block offers queue length: ", blockOffersQueueLen = blockOffersQueue.len() proc runState*(config: PortalBridgeConf) = let @@ -215,19 +215,16 @@ proc runState*(config: PortalBridgeConf) = info "Starting state backfill from block number: ", startBlockNumber - let - blockDataQueue = newAsyncQueue[BlockDataRef](bufferSize) - blockOffersQueue = newAsyncQueue[BlockOffersRef](bufferSize) + let blockDataQueue = newAsyncQueue[BlockDataRef](bufferSize) + # blockOffersQueue = newAsyncQueue[BlockOffersRef](bufferSize) asyncSpawn runBackfillCollectBlockDataLoop( blockDataQueue, web3Client, startBlockNumber ) - asyncSpawn runBackfillBuildBlockOffersLoop( - blockDataQueue, blockOffersQueue, config.stateDir.string - ) + asyncSpawn runBackfillBuildBlockOffersLoop(blockDataQueue, config.stateDir.string) - asyncSpawn runBackfillMetricsLoop(blockDataQueue, blockOffersQueue) + asyncSpawn runBackfillMetricsLoop(blockDataQueue) while true: poll() diff --git a/fluffy/tools/portal_bridge/state_bridge/database.nim b/fluffy/tools/portal_bridge/state_bridge/database.nim index 06e18b1ac..ad18b5e02 100644 --- a/fluffy/tools/portal_bridge/state_bridge/database.nim +++ b/fluffy/tools/portal_bridge/state_bridge/database.nim @@ -21,18 +21,15 @@ const COL_FAMILY_NAMES = type AccountsBackendRef = ref object of RootObj cfHandle: ColFamilyHandleRef - tx: TransactionRef - updatedCache: TrieDatabaseRef + tx: TransactionRef #updatedCache: TrieDatabaseRef StorageBackendRef = ref object of RootObj cfHandle: ColFamilyHandleRef - tx: TransactionRef - updatedCache: TrieDatabaseRef + tx: TransactionRef #updatedCache: TrieDatabaseRef BytecodeBackendRef = ref object of RootObj cfHandle: ColFamilyHandleRef - tx: TransactionRef - updatedCache: TrieDatabaseRef + tx: TransactionRef #updatedCache: TrieDatabaseRef DatabaseBackendRef = AccountsBackendRef | StorageBackendRef | BytecodeBackendRef @@ -92,7 +89,7 @@ proc put( dbBackend: DatabaseBackendRef, key, val: openArray[byte] ) {.gcsafe, raises: [].} = doAssert dbBackend.tx.put(key, val, dbBackend.cfHandle).isOk() - dbBackend.updatedCache.put(key, val) + # dbBackend.updatedCache.put(key, val) proc get( dbBackend: DatabaseBackendRef, key: openArray[byte] @@ -120,14 +117,14 @@ proc getStorageBackend*(db: DatabaseRef): TrieDatabaseRef {.inline.} = proc getBytecodeBackend*(db: DatabaseRef): TrieDatabaseRef {.inline.} = trieDB(db.bytecodeBackend) -proc getAccountsUpdatedCache*(db: DatabaseRef): TrieDatabaseRef {.inline.} = - db.accountsBackend.updatedCache +# proc getAccountsUpdatedCache*(db: DatabaseRef): TrieDatabaseRef {.inline.} = +# db.accountsBackend.updatedCache -proc getStorageUpdatedCache*(db: DatabaseRef): TrieDatabaseRef {.inline.} = - db.storageBackend.updatedCache +# proc getStorageUpdatedCache*(db: DatabaseRef): TrieDatabaseRef {.inline.} = +# db.storageBackend.updatedCache -proc getBytecodeUpdatedCache*(db: DatabaseRef): TrieDatabaseRef {.inline.} = - db.bytecodeBackend.updatedCache +# proc getBytecodeUpdatedCache*(db: DatabaseRef): TrieDatabaseRef {.inline.} = +# db.bytecodeBackend.updatedCache proc beginTransaction*(db: DatabaseRef): Result[void, string] = if not db.pendingTransaction.isNil(): @@ -139,9 +136,9 @@ proc beginTransaction*(db: DatabaseRef): Result[void, string] = db.storageBackend.tx = tx db.bytecodeBackend.tx = tx - db.accountsBackend.updatedCache = newMemoryDB() - db.storageBackend.updatedCache = newMemoryDB() - db.bytecodeBackend.updatedCache = newMemoryDB() + # db.accountsBackend.updatedCache = newMemoryDB() + # db.storageBackend.updatedCache = newMemoryDB() + # db.bytecodeBackend.updatedCache = newMemoryDB() ok() diff --git a/fluffy/tools/portal_bridge/state_bridge/world_state.nim b/fluffy/tools/portal_bridge/state_bridge/world_state.nim index 39ce85454..55c46577b 100644 --- a/fluffy/tools/portal_bridge/state_bridge/world_state.nim +++ b/fluffy/tools/portal_bridge/state_bridge/world_state.nim @@ -53,7 +53,7 @@ type WorldStateRef* = ref object db: DatabaseRef - accountPreimages: TableRef[AddressHash, EthAddress] + #accountPreimages: TableRef[AddressHash, EthAddress] accountsTrie: HexaryTrie storageTries: TableRef[AddressHash, HexaryTrie] storageDb: TrieDatabaseRef @@ -62,7 +62,7 @@ type proc init*(T: type WorldStateRef, db: DatabaseRef): T = WorldStateRef( db: db, - accountPreimages: newTable[AddressHash, EthAddress](), + #accountPreimages: newTable[AddressHash, EthAddress](), accountsTrie: initHexaryTrie(db.getAccountsBackend(), isPruning = false), storageTries: newTable[AddressHash, HexaryTrie](), storageDb: db.getStorageBackend(), @@ -78,13 +78,13 @@ template toAccountKey*(address: EthAddress): AddressHash = template toStorageKey*(slotKey: UInt256): SlotKeyHash = keccakHash(toBytesBE(slotKey)) -proc getAccountPreImage*(state: WorldStateRef, accountKey: AddressHash): EthAddress = - doAssert(state.accountPreimages.contains(accountKey)) - state.accountPreimages.getOrDefault(accountKey) +# proc getAccountPreImage*(state: WorldStateRef, accountKey: AddressHash): EthAddress = +# doAssert(state.accountPreimages.contains(accountKey)) +# state.accountPreimages.getOrDefault(accountKey) proc getAccount*(state: WorldStateRef, address: EthAddress): AccountState = let accountKey = toAccountKey(address) - state.accountPreimages[accountKey] = address + # state.accountPreimages[accountKey] = address try: if state.accountsTrie.contains(accountKey.data): @@ -97,7 +97,7 @@ proc getAccount*(state: WorldStateRef, address: EthAddress): AccountState = proc setAccount*(state: WorldStateRef, address: EthAddress, accState: AccountState) = let accountKey = toAccountKey(address) - state.accountPreimages[accountKey] = address + # state.accountPreimages[accountKey] = address try: if not state.storageTries.contains(accountKey): @@ -126,7 +126,7 @@ proc setAccount*(state: WorldStateRef, address: EthAddress, accState: AccountSta proc deleteAccount*(state: WorldStateRef, address: EthAddress) = let accountKey = toAccountKey(address) - state.accountPreimages[accountKey] = address + # state.accountPreimages[accountKey] = address try: state.accountsTrie.del(accountKey.data) @@ -140,40 +140,40 @@ proc clearPreimages*(state: WorldStateRef) = #state.accountPreimages.clear() # Returns the account proofs for all the updated accounts from the last transaction -iterator updatedAccountProofs*(state: WorldStateRef): (EthAddress, seq[seq[byte]]) = - let trie = initHexaryTrie( - state.db.getAccountsUpdatedCache(), state.stateRoot(), isPruning = false - ) +# iterator updatedAccountProofs*(state: WorldStateRef): (EthAddress, seq[seq[byte]]) = +# let trie = initHexaryTrie( +# state.db.getAccountsUpdatedCache(), state.stateRoot(), isPruning = false +# ) - try: - for key in trie.keys(): - if key.len() == 0: - continue # skip the empty node created on initialization - let address = state.getAccountPreImage(KeccakHash.fromBytes(key)) - yield (address, trie.getBranch(key)) - except RlpError as e: - raiseAssert(e.msg) # should never happen unless the database is corrupted +# try: +# for key in trie.keys(): +# if key.len() == 0: +# continue # skip the empty node created on initialization +# let address = state.getAccountPreImage(KeccakHash.fromBytes(key)) +# yield (address, trie.getBranch(key)) +# except RlpError as e: +# raiseAssert(e.msg) # should never happen unless the database is corrupted -# Returns the storage proofs for the updated slots for the given account from the last transaction -iterator updatedStorageProofs*( - state: WorldStateRef, address: EthAddress -): (SlotKeyHash, seq[seq[byte]]) = - let accState = state.getAccount(address) +# # Returns the storage proofs for the updated slots for the given account from the last transaction +# iterator updatedStorageProofs*( +# state: WorldStateRef, address: EthAddress +# ): (SlotKeyHash, seq[seq[byte]]) = +# let accState = state.getAccount(address) - let trie = initHexaryTrie( - state.db.getStorageUpdatedCache(), accState.account.storageRoot, isPruning = false - ) +# let trie = initHexaryTrie( +# state.db.getStorageUpdatedCache(), accState.account.storageRoot, isPruning = false +# ) - try: - for key in trie.keys(): - if key.len() == 0: - continue # skip the empty node created on initialization - yield (KeccakHash.fromBytes(key), trie.getBranch(key)) - except RlpError as e: - raiseAssert(e.msg) # should never happen unless the database is corrupted +# try: +# for key in trie.keys(): +# if key.len() == 0: +# continue # skip the empty node created on initialization +# yield (KeccakHash.fromBytes(key), trie.getBranch(key)) +# except RlpError as e: +# raiseAssert(e.msg) # should never happen unless the database is corrupted -proc getUpdatedBytecode*(state: WorldStateRef, address: EthAddress): seq[byte] = - state.db.getBytecodeUpdatedCache().get(toAccountKey(address).data) +# proc getUpdatedBytecode*(state: WorldStateRef, address: EthAddress): seq[byte] = +# state.db.getBytecodeUpdatedCache().get(toAccountKey(address).data) # Returns the updated bytecode for all accounts from the last transaction # iterator updatedBytecode*(state: WorldStateRef): (EthAddress, seq[byte]) =