From c921135e2e780e03cf77b4e06062b390bece9961 Mon Sep 17 00:00:00 2001 From: Adam Spitz Date: Fri, 10 Mar 2023 13:42:37 -0500 Subject: [PATCH] Refactored to introduce distinct types AccountsTrie and StorageTrie. (#1490) This is in preparation for working on stateless mode; it's useful to have a clearer and more type-safe interface for accessing accounts versus slots. --- nimbus/db/accounts_cache.nim | 29 ++++++++-------- nimbus/db/distinct_tries.nim | 59 ++++++++++++++++++++++++++++++++ nimbus/db/state_db.nim | 37 ++++++++++---------- stateless/json_witness_gen.nim | 11 +++--- stateless/test_block_witness.nim | 10 +++--- stateless/test_witness_keys.nim | 18 +++++----- tests/macro_assembler.nim | 6 ++-- tests/test_accounts_cache.nim | 2 +- 8 files changed, 117 insertions(+), 55 deletions(-) create mode 100644 nimbus/db/distinct_tries.nim diff --git a/nimbus/db/accounts_cache.nim b/nimbus/db/accounts_cache.nim index b145b0164..14d716d18 100644 --- a/nimbus/db/accounts_cache.nim +++ b/nimbus/db/accounts_cache.nim @@ -3,6 +3,7 @@ import eth/[common, rlp], eth/trie/[hexary, db, trie_defs], ../constants, ../utils/utils, storage_types, ../../stateless/multi_keys, + ./distinct_tries, ./access_list as ac_access_list type @@ -31,7 +32,7 @@ type AccountsCache* = ref object db: TrieDatabaseRef - trie: SecureHexaryTrie + trie: AccountsTrie savePoint: SavePoint witnessCache: Table[EthAddress, WitnessData] isDirty: bool @@ -68,7 +69,7 @@ proc init*(x: typedesc[AccountsCache], db: TrieDatabaseRef, root: KeccakHash, pruneTrie: bool = true): AccountsCache = new result result.db = db - result.trie = initSecureHexaryTrie(db, root, pruneTrie) + result.trie = initAccountsTrie(db, root, pruneTrie) result.witnessCache = initTable[EthAddress, WitnessData]() discard result.beginSavepoint @@ -137,7 +138,7 @@ proc getAccount(ac: AccountsCache, address: EthAddress, shouldCreate = true): Re # not found in cache, look into state trie let recordFound = try: - ac.trie.get(address) + ac.trie.getAccountBytes(address) except RlpError: raiseAssert("No RlpError should occur on trie access for an address") if recordFound.len > 0: @@ -189,13 +190,13 @@ template createTrieKeyFromSlot(slot: UInt256): auto = # pad32(int_to_big_endian(slot)) # morally equivalent to toByteRange_Unnecessary but with different types -template getAccountTrie(db: TrieDatabaseRef, acc: RefAccount): auto = +template getStorageTrie(db: TrieDatabaseRef, acc: RefAccount): auto = # TODO: implement `prefix-db` to solve issue #228 permanently. # the `prefix-db` will automatically insert account address to the # underlying-db key without disturb how the trie works. # it will create virtual container for each account. # see nim-eth#9 - initSecureHexaryTrie(db, acc.account.storageRoot, false) + initStorageTrie(db, acc.account.storageRoot, false) proc originalStorageValue(acc: RefAccount, slot: UInt256, db: TrieDatabaseRef): UInt256 = # share the same original storage between multiple @@ -209,8 +210,8 @@ proc originalStorageValue(acc: RefAccount, slot: UInt256, db: TrieDatabaseRef): # Not in the original values cache - go to the DB. let slotAsKey = createTrieKeyFromSlot slot - accountTrie = getAccountTrie(db, acc) - foundRecord = accountTrie.get(slotAsKey) + storageTrie = getStorageTrie(db, acc) + foundRecord = storageTrie.getSlotBytes(slotAsKey) result = if foundRecord.len > 0: rlp.decode(foundRecord, UInt256) @@ -263,22 +264,22 @@ proc persistStorage(acc: RefAccount, db: TrieDatabaseRef, clearCache: bool) = if not clearCache and acc.originalStorage.isNil: acc.originalStorage = newTable[UInt256, UInt256]() - var accountTrie = getAccountTrie(db, acc) + var storageTrie = getStorageTrie(db, acc) for slot, value in acc.overlayStorage: let slotAsKey = createTrieKeyFromSlot slot if value > 0: let encodedValue = rlp.encode(value) - accountTrie.put(slotAsKey, encodedValue) + storageTrie.putSlotBytes(slotAsKey, encodedValue) else: - accountTrie.del(slotAsKey) + storageTrie.delSlotBytes(slotAsKey) # TODO: this can be disabled if we do not perform # accounts tracing # map slothash back to slot value # see iterator storage below - # slotHash can be obtained from accountTrie.put? + # slotHash can be obtained from storageTrie.putSlotBytes? let slotHash = keccakHash(slotAsKey) db.put(slotHashToSlotKey(slotHash.data).toOpenArray, rlp.encode(slot)) @@ -292,7 +293,7 @@ proc persistStorage(acc: RefAccount, db: TrieDatabaseRef, clearCache: bool) = acc.originalStorage.del(slot) acc.overlayStorage.clear() - acc.account.storageRoot = accountTrie.rootHash + acc.account.storageRoot = storageTrie.rootHash proc makeDirty(ac: AccountsCache, address: EthAddress, cloneStorage = true): RefAccount = ac.isDirty = true @@ -449,9 +450,9 @@ proc persist*(ac: AccountsCache, clearCache: bool = true) = # storageRoot must be updated first # before persisting account into merkle trie acc.persistStorage(ac.db, clearCache) - ac.trie.put address, rlp.encode(acc.account) + ac.trie.putAccountBytes address, rlp.encode(acc.account) of Remove: - ac.trie.del address + ac.trie.delAccountBytes address if not clearCache: cleanAccounts.incl address of DoNothing: diff --git a/nimbus/db/distinct_tries.nim b/nimbus/db/distinct_tries.nim new file mode 100644 index 000000000..c5e216d7e --- /dev/null +++ b/nimbus/db/distinct_tries.nim @@ -0,0 +1,59 @@ +# The point of this file is just to give a little more type-safety +# and clarity to our use of SecureHexaryTrie, by having distinct +# types for the big trie containing all the accounts and the little +# tries containing the storage for an individual account. +# +# It's nice to have all the accesses go through "getAccountBytes" +# rather than just "get" (which is hard to search for). Plus we +# may want to put in assertions to make sure that the nodes for +# the account are all present (in stateless mode), etc. + +import + std/typetraits, + eth/common, + eth/trie/[db, hexary] + +type + DB = TrieDatabaseRef + AccountsTrie* = distinct SecureHexaryTrie + StorageTrie* = distinct SecureHexaryTrie + +# I don't understand why "borrow" doesn't work here. --Adam +proc rootHash* (trie: AccountsTrie | StorageTrie): KeccakHash = distinctBase(trie).rootHash +proc rootHashHex*(trie: AccountsTrie | StorageTrie): string = distinctBase(trie).rootHashHex +proc db* (trie: AccountsTrie | StorageTrie): TrieDatabaseRef = distinctBase(trie).db +proc isPruning* (trie: AccountsTrie | StorageTrie): bool = distinctBase(trie).isPruning + + + +template initAccountsTrie*(db: DB, rootHash: KeccakHash, isPruning = true): AccountsTrie = + AccountsTrie(initSecureHexaryTrie(db, rootHash, isPruning)) + +template initAccountsTrie*(db: DB, isPruning = true): AccountsTrie = + AccountsTrie(initSecureHexaryTrie(db, isPruning)) + +proc getAccountBytes*(trie: AccountsTrie, address: EthAddress): seq[byte] = + SecureHexaryTrie(trie).get(address) + +proc putAccountBytes*(trie: var AccountsTrie, address: EthAddress, value: openArray[byte]) = + SecureHexaryTrie(trie).put(address, value) + +proc delAccountBytes*(trie: var AccountsTrie, address: EthAddress) = + SecureHexaryTrie(trie).del(address) + + + +template initStorageTrie*(db: DB, rootHash: KeccakHash, isPruning = true): StorageTrie = + StorageTrie(initSecureHexaryTrie(db, rootHash, isPruning)) + +template initStorageTrie*(db: DB, isPruning = true): StorageTrie = + StorageTrie(initSecureHexaryTrie(db, isPruning)) + +proc getSlotBytes*(trie: StorageTrie, slotAsKey: openArray[byte]): seq[byte] = + SecureHexaryTrie(trie).get(slotAsKey) + +proc putSlotBytes*(trie: var StorageTrie, slotAsKey: openArray[byte], value: openArray[byte]) = + SecureHexaryTrie(trie).put(slotAsKey, value) + +proc delSlotBytes*(trie: var StorageTrie, slotAsKey: openArray[byte]) = + SecureHexaryTrie(trie).del(slotAsKey) diff --git a/nimbus/db/state_db.nim b/nimbus/db/state_db.nim index 935ec2b7c..6aa35b325 100644 --- a/nimbus/db/state_db.nim +++ b/nimbus/db/state_db.nim @@ -8,7 +8,8 @@ import strformat, chronicles, eth/[common, rlp], eth/trie/[hexary, db, trie_defs], - ../constants, ../utils/utils, storage_types, sets + ../constants, ../utils/utils, storage_types, sets, + ./distinct_tries logScope: topics = "state_db" @@ -46,7 +47,7 @@ const type AccountStateDB* = ref object - trie: SecureHexaryTrie + trie: AccountsTrie originalRoot: KeccakHash # will be updated for every transaction transactionID: TransactionID when aleth_compat: @@ -67,12 +68,12 @@ proc rootHash*(db: AccountStateDB): KeccakHash = db.trie.rootHash proc `rootHash=`*(db: AccountStateDB, root: KeccakHash) = - db.trie = initSecureHexaryTrie(trieDB(db), root, db.trie.isPruning) + db.trie = initAccountsTrie(trieDB(db), root, db.trie.isPruning) proc newAccountStateDB*(backingStore: TrieDatabaseRef, root: KeccakHash, pruneTrie: bool): AccountStateDB = result.new() - result.trie = initSecureHexaryTrie(backingStore, root, pruneTrie) + result.trie = initAccountsTrie(backingStore, root, pruneTrie) result.originalRoot = root result.transactionID = backingStore.getTransactionID() when aleth_compat: @@ -82,20 +83,20 @@ proc getTrie*(db: AccountStateDB): HexaryTrie = HexaryTrie db.trie proc getSecureTrie*(db: AccountStateDB): SecureHexaryTrie = - db.trie + SecureHexaryTrie db.trie proc getAccount*(db: AccountStateDB, address: EthAddress): Account = - let recordFound = db.trie.get(address) + let recordFound = db.trie.getAccountBytes(address) if recordFound.len > 0: result = rlp.decode(recordFound, Account) else: result = newAccount() proc setAccount*(db: AccountStateDB, address: EthAddress, account: Account) = - db.trie.put(address, rlp.encode(account)) + db.trie.putAccountBytes(address, rlp.encode(account)) proc deleteAccount*(db: AccountStateDB, address: EthAddress) = - db.trie.del(address) + db.trie.delAccountBytes(address) proc getCodeHash*(db: AccountStateDB, address: EthAddress): Hash256 = let account = db.getAccount(address) @@ -124,13 +125,13 @@ template createTrieKeyFromSlot(slot: UInt256): auto = # pad32(int_to_big_endian(slot)) # morally equivalent to toByteRange_Unnecessary but with different types -template getAccountTrie(db: AccountStateDB, account: Account): auto = +template getStorageTrie(db: AccountStateDB, account: Account): auto = # TODO: implement `prefix-db` to solve issue #228 permanently. # the `prefix-db` will automatically insert account address to the # underlying-db key without disturb how the trie works. # it will create virtual container for each account. # see nim-eth#9 - initSecureHexaryTrie(trieDB(db), account.storageRoot, false) + initStorageTrie(trieDB(db), account.storageRoot, false) proc clearStorage*(db: AccountStateDB, address: EthAddress) = var account = db.getAccount(address) @@ -147,14 +148,14 @@ proc setStorage*(db: AccountStateDB, address: EthAddress, slot: UInt256, value: UInt256) = var account = db.getAccount(address) - var accountTrie = getAccountTrie(db, account) + var accountTrie = getStorageTrie(db, account) let slotAsKey = createTrieKeyFromSlot slot if value > 0: let encodedValue = rlp.encode(value) - accountTrie.put(slotAsKey, encodedValue) + accountTrie.putSlotBytes(slotAsKey, encodedValue) else: - accountTrie.del(slotAsKey) + accountTrie.delSlotBytes(slotAsKey) # map slothash back to slot value # see iterator storage below @@ -182,10 +183,10 @@ proc getStorage*(db: AccountStateDB, address: EthAddress, slot: UInt256): (UInt2 let account = db.getAccount(address) slotAsKey = createTrieKeyFromSlot slot - accountTrie = getAccountTrie(db, account) + storageTrie = getStorageTrie(db, account) let - foundRecord = accountTrie.get(slotAsKey) + foundRecord = storageTrie.getSlotBytes(slotAsKey) if foundRecord.len > 0: result = (rlp.decode(foundRecord, UInt256), true) @@ -232,10 +233,10 @@ proc dumpAccount*(db: AccountStateDB, addressS: string): string = return fmt"{addressS}: Storage: {db.getStorage(address, 0.u256)}; getAccount: {db.getAccount address}" proc accountExists*(db: AccountStateDB, address: EthAddress): bool = - db.trie.get(address).len > 0 + db.trie.getAccountBytes(address).len > 0 proc isEmptyAccount*(db: AccountStateDB, address: EthAddress): bool = - let recordFound = db.trie.get(address) + let recordFound = db.trie.getAccountBytes(address) assert(recordFound.len > 0) let account = rlp.decode(recordFound, Account) @@ -244,7 +245,7 @@ proc isEmptyAccount*(db: AccountStateDB, address: EthAddress): bool = account.nonce == 0 proc isDeadAccount*(db: AccountStateDB, address: EthAddress): bool = - let recordFound = db.trie.get(address) + let recordFound = db.trie.getAccountBytes(address) if recordFound.len > 0: let account = rlp.decode(recordFound, Account) result = account.codeHash == EMPTY_SHA3 and diff --git a/stateless/json_witness_gen.nim b/stateless/json_witness_gen.nim index 8a7685783..3a97d0906 100644 --- a/stateless/json_witness_gen.nim +++ b/stateless/json_witness_gen.nim @@ -2,7 +2,8 @@ import randutils, random, parseopt, strutils, os, eth/[common, rlp], eth/trie/[hexary, db, trie_defs], nimcrypto/sysrand, ../stateless/[json_from_tree], - ../nimbus/db/storage_types, ./witness_types, ./multi_keys + ../nimbus/db/[storage_types, distinct_tries], + ./witness_types, ./multi_keys type DB = TrieDatabaseRef @@ -38,12 +39,12 @@ proc randStorage(db: DB, numSlots: int): StorageKeys = if rand(0..1) == 0 or numSlots == 0: result = (emptyRlpHash, MultikeysRef(nil)) else: - var trie = initSecureHexaryTrie(db) + var trie = initStorageTrie(db) var keys = newSeq[StorageSlot](numSlots) for i in 0.. 0: let acc = rlp.decode(recordFound, Account) doAssert acc == account diff --git a/stateless/test_witness_keys.nim b/stateless/test_witness_keys.nim index 5c1b18b9b..4b32c9f31 100644 --- a/stateless/test_witness_keys.nim +++ b/stateless/test_witness_keys.nim @@ -3,7 +3,7 @@ import eth/[common, rlp], eth/trie/[hexary, db, trie_defs, nibbles], faststreams/inputs, nimcrypto/sysrand, ../stateless/[witness_from_tree, tree_from_witness], - ../nimbus/db/storage_types, ./witness_types, ./multi_keys + ../nimbus/db/[distinct_tries, storage_types], ./witness_types, ./multi_keys type DB = TrieDatabaseRef @@ -39,13 +39,13 @@ proc randStorage(db: DB): StorageKeys = if rand(0..1) == 0: result = (emptyRlpHash, MultikeysRef(nil)) else: - var trie = initSecureHexaryTrie(db) + var trie = initStorageTrie(db) let numPairs = rand(1..10) var keys = newSeq[StorageSlot](numPairs) for i in 0.. 0: let acc = rlp.decode(recordFound, Account) check acc == accs[i] @@ -261,7 +261,7 @@ proc witnessKeysMain*() = ] let m = initMultiKeys(keys, true) var memDB = newMemoryDB() - var trie = initSecureHexaryTrie(memDB) + var trie = initAccountsTrie(memDB) var acc = randAccount(memDB) var tt = initHexaryTrie(memDB) @@ -271,7 +271,7 @@ proc witnessKeysMain*() = let addrs = @[AccountKey(address: randAddress(), codeTouched: acc.codeTouched, storageKeys: m)] - trie.put(addrs[0].address, rlp.encode(acc.account)) + trie.putAccountBytes(addrs[0].address, rlp.encode(acc.account)) var mkeys = newMultiKeys(addrs) let rootHash = trie.rootHash diff --git a/tests/macro_assembler.nim b/tests/macro_assembler.nim index 659c74571..d64af994e 100644 --- a/tests/macro_assembler.nim +++ b/tests/macro_assembler.nim @@ -5,7 +5,7 @@ import import eth/trie/hexary, - ../nimbus/db/accounts_cache, + ../nimbus/db/[accounts_cache, distinct_tries], ../nimbus/evm/[async_operations, types], ../nimbus/vm_internals, ../nimbus/transaction/[call_common, call_evm], @@ -385,13 +385,13 @@ proc verifyAsmResult(vmState: BaseVMState, com: CommonRef, boa: Assembler, asmRe var storageRoot = stateDB.getStorageRoot(codeAddress) - trie = initSecureHexaryTrie(com.db.db, storageRoot) + trie = initStorageTrie(com.db.db, storageRoot) for kv in boa.storage: let key = kv[0].toHex() let val = kv[1].toHex() let keyBytes = (@(kv[0])) - let actual = trie.get(keyBytes).toHex() + let actual = trie.getSlotBytes(keyBytes).toHex() let zerosLen = 64 - (actual.len) let value = repeat('0', zerosLen) & actual if val != value: diff --git a/tests/test_accounts_cache.nim b/tests/test_accounts_cache.nim index be8b90a78..61614ad75 100644 --- a/tests/test_accounts_cache.nim +++ b/tests/test_accounts_cache.nim @@ -188,7 +188,7 @@ proc runTrial3crash(vmState: BaseVMState; inx: int; noisy = false) = # following function could be added to db/accounts_cache.nim: # # proc clobberRootHash*(ac: AccountsCache; root: KeccakHash; prune = true) = - # ac.trie = initSecureHexaryTrie(ac.db, rootHash, prune) + # ac.trie = initAccountsTrie(ac.db, rootHash, prune) # # Then, beginning this very function `runTrial3crash()` with #