Refactored to introduce distinct types AccountsTrie and StorageTrie. (#1490)

This is in preparation for working on stateless mode; it's useful to
have a clearer and more type-safe interface for accessing accounts
versus slots.
This commit is contained in:
Adam Spitz 2023-03-10 13:42:37 -05:00 committed by GitHub
parent 2f7f2dba2d
commit c921135e2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 117 additions and 55 deletions

View File

@ -3,6 +3,7 @@ import
eth/[common, rlp], eth/trie/[hexary, db, trie_defs],
../constants, ../utils/utils, storage_types,
../../stateless/multi_keys,
./distinct_tries,
./access_list as ac_access_list
type
@ -31,7 +32,7 @@ type
AccountsCache* = ref object
db: TrieDatabaseRef
trie: SecureHexaryTrie
trie: AccountsTrie
savePoint: SavePoint
witnessCache: Table[EthAddress, WitnessData]
isDirty: bool
@ -68,7 +69,7 @@ proc init*(x: typedesc[AccountsCache], db: TrieDatabaseRef,
root: KeccakHash, pruneTrie: bool = true): AccountsCache =
new result
result.db = db
result.trie = initSecureHexaryTrie(db, root, pruneTrie)
result.trie = initAccountsTrie(db, root, pruneTrie)
result.witnessCache = initTable[EthAddress, WitnessData]()
discard result.beginSavepoint
@ -137,7 +138,7 @@ proc getAccount(ac: AccountsCache, address: EthAddress, shouldCreate = true): Re
# not found in cache, look into state trie
let recordFound =
try:
ac.trie.get(address)
ac.trie.getAccountBytes(address)
except RlpError:
raiseAssert("No RlpError should occur on trie access for an address")
if recordFound.len > 0:
@ -189,13 +190,13 @@ template createTrieKeyFromSlot(slot: UInt256): auto =
# pad32(int_to_big_endian(slot))
# morally equivalent to toByteRange_Unnecessary but with different types
template getAccountTrie(db: TrieDatabaseRef, acc: RefAccount): auto =
template getStorageTrie(db: TrieDatabaseRef, acc: RefAccount): auto =
# TODO: implement `prefix-db` to solve issue #228 permanently.
# the `prefix-db` will automatically insert account address to the
# underlying-db key without disturb how the trie works.
# it will create virtual container for each account.
# see nim-eth#9
initSecureHexaryTrie(db, acc.account.storageRoot, false)
initStorageTrie(db, acc.account.storageRoot, false)
proc originalStorageValue(acc: RefAccount, slot: UInt256, db: TrieDatabaseRef): UInt256 =
# share the same original storage between multiple
@ -209,8 +210,8 @@ proc originalStorageValue(acc: RefAccount, slot: UInt256, db: TrieDatabaseRef):
# Not in the original values cache - go to the DB.
let
slotAsKey = createTrieKeyFromSlot slot
accountTrie = getAccountTrie(db, acc)
foundRecord = accountTrie.get(slotAsKey)
storageTrie = getStorageTrie(db, acc)
foundRecord = storageTrie.getSlotBytes(slotAsKey)
result = if foundRecord.len > 0:
rlp.decode(foundRecord, UInt256)
@ -263,22 +264,22 @@ proc persistStorage(acc: RefAccount, db: TrieDatabaseRef, clearCache: bool) =
if not clearCache and acc.originalStorage.isNil:
acc.originalStorage = newTable[UInt256, UInt256]()
var accountTrie = getAccountTrie(db, acc)
var storageTrie = getStorageTrie(db, acc)
for slot, value in acc.overlayStorage:
let slotAsKey = createTrieKeyFromSlot slot
if value > 0:
let encodedValue = rlp.encode(value)
accountTrie.put(slotAsKey, encodedValue)
storageTrie.putSlotBytes(slotAsKey, encodedValue)
else:
accountTrie.del(slotAsKey)
storageTrie.delSlotBytes(slotAsKey)
# TODO: this can be disabled if we do not perform
# accounts tracing
# map slothash back to slot value
# see iterator storage below
# slotHash can be obtained from accountTrie.put?
# slotHash can be obtained from storageTrie.putSlotBytes?
let slotHash = keccakHash(slotAsKey)
db.put(slotHashToSlotKey(slotHash.data).toOpenArray, rlp.encode(slot))
@ -292,7 +293,7 @@ proc persistStorage(acc: RefAccount, db: TrieDatabaseRef, clearCache: bool) =
acc.originalStorage.del(slot)
acc.overlayStorage.clear()
acc.account.storageRoot = accountTrie.rootHash
acc.account.storageRoot = storageTrie.rootHash
proc makeDirty(ac: AccountsCache, address: EthAddress, cloneStorage = true): RefAccount =
ac.isDirty = true
@ -449,9 +450,9 @@ proc persist*(ac: AccountsCache, clearCache: bool = true) =
# storageRoot must be updated first
# before persisting account into merkle trie
acc.persistStorage(ac.db, clearCache)
ac.trie.put address, rlp.encode(acc.account)
ac.trie.putAccountBytes address, rlp.encode(acc.account)
of Remove:
ac.trie.del address
ac.trie.delAccountBytes address
if not clearCache:
cleanAccounts.incl address
of DoNothing:

View File

@ -0,0 +1,59 @@
# The point of this file is just to give a little more type-safety
# and clarity to our use of SecureHexaryTrie, by having distinct
# types for the big trie containing all the accounts and the little
# tries containing the storage for an individual account.
#
# It's nice to have all the accesses go through "getAccountBytes"
# rather than just "get" (which is hard to search for). Plus we
# may want to put in assertions to make sure that the nodes for
# the account are all present (in stateless mode), etc.
import
std/typetraits,
eth/common,
eth/trie/[db, hexary]
type
DB = TrieDatabaseRef
AccountsTrie* = distinct SecureHexaryTrie
StorageTrie* = distinct SecureHexaryTrie
# I don't understand why "borrow" doesn't work here. --Adam
proc rootHash* (trie: AccountsTrie | StorageTrie): KeccakHash = distinctBase(trie).rootHash
proc rootHashHex*(trie: AccountsTrie | StorageTrie): string = distinctBase(trie).rootHashHex
proc db* (trie: AccountsTrie | StorageTrie): TrieDatabaseRef = distinctBase(trie).db
proc isPruning* (trie: AccountsTrie | StorageTrie): bool = distinctBase(trie).isPruning
template initAccountsTrie*(db: DB, rootHash: KeccakHash, isPruning = true): AccountsTrie =
AccountsTrie(initSecureHexaryTrie(db, rootHash, isPruning))
template initAccountsTrie*(db: DB, isPruning = true): AccountsTrie =
AccountsTrie(initSecureHexaryTrie(db, isPruning))
proc getAccountBytes*(trie: AccountsTrie, address: EthAddress): seq[byte] =
SecureHexaryTrie(trie).get(address)
proc putAccountBytes*(trie: var AccountsTrie, address: EthAddress, value: openArray[byte]) =
SecureHexaryTrie(trie).put(address, value)
proc delAccountBytes*(trie: var AccountsTrie, address: EthAddress) =
SecureHexaryTrie(trie).del(address)
template initStorageTrie*(db: DB, rootHash: KeccakHash, isPruning = true): StorageTrie =
StorageTrie(initSecureHexaryTrie(db, rootHash, isPruning))
template initStorageTrie*(db: DB, isPruning = true): StorageTrie =
StorageTrie(initSecureHexaryTrie(db, isPruning))
proc getSlotBytes*(trie: StorageTrie, slotAsKey: openArray[byte]): seq[byte] =
SecureHexaryTrie(trie).get(slotAsKey)
proc putSlotBytes*(trie: var StorageTrie, slotAsKey: openArray[byte], value: openArray[byte]) =
SecureHexaryTrie(trie).put(slotAsKey, value)
proc delSlotBytes*(trie: var StorageTrie, slotAsKey: openArray[byte]) =
SecureHexaryTrie(trie).del(slotAsKey)

View File

@ -8,7 +8,8 @@
import
strformat,
chronicles, eth/[common, rlp], eth/trie/[hexary, db, trie_defs],
../constants, ../utils/utils, storage_types, sets
../constants, ../utils/utils, storage_types, sets,
./distinct_tries
logScope:
topics = "state_db"
@ -46,7 +47,7 @@ const
type
AccountStateDB* = ref object
trie: SecureHexaryTrie
trie: AccountsTrie
originalRoot: KeccakHash # will be updated for every transaction
transactionID: TransactionID
when aleth_compat:
@ -67,12 +68,12 @@ proc rootHash*(db: AccountStateDB): KeccakHash =
db.trie.rootHash
proc `rootHash=`*(db: AccountStateDB, root: KeccakHash) =
db.trie = initSecureHexaryTrie(trieDB(db), root, db.trie.isPruning)
db.trie = initAccountsTrie(trieDB(db), root, db.trie.isPruning)
proc newAccountStateDB*(backingStore: TrieDatabaseRef,
root: KeccakHash, pruneTrie: bool): AccountStateDB =
result.new()
result.trie = initSecureHexaryTrie(backingStore, root, pruneTrie)
result.trie = initAccountsTrie(backingStore, root, pruneTrie)
result.originalRoot = root
result.transactionID = backingStore.getTransactionID()
when aleth_compat:
@ -82,20 +83,20 @@ proc getTrie*(db: AccountStateDB): HexaryTrie =
HexaryTrie db.trie
proc getSecureTrie*(db: AccountStateDB): SecureHexaryTrie =
db.trie
SecureHexaryTrie db.trie
proc getAccount*(db: AccountStateDB, address: EthAddress): Account =
let recordFound = db.trie.get(address)
let recordFound = db.trie.getAccountBytes(address)
if recordFound.len > 0:
result = rlp.decode(recordFound, Account)
else:
result = newAccount()
proc setAccount*(db: AccountStateDB, address: EthAddress, account: Account) =
db.trie.put(address, rlp.encode(account))
db.trie.putAccountBytes(address, rlp.encode(account))
proc deleteAccount*(db: AccountStateDB, address: EthAddress) =
db.trie.del(address)
db.trie.delAccountBytes(address)
proc getCodeHash*(db: AccountStateDB, address: EthAddress): Hash256 =
let account = db.getAccount(address)
@ -124,13 +125,13 @@ template createTrieKeyFromSlot(slot: UInt256): auto =
# pad32(int_to_big_endian(slot))
# morally equivalent to toByteRange_Unnecessary but with different types
template getAccountTrie(db: AccountStateDB, account: Account): auto =
template getStorageTrie(db: AccountStateDB, account: Account): auto =
# TODO: implement `prefix-db` to solve issue #228 permanently.
# the `prefix-db` will automatically insert account address to the
# underlying-db key without disturb how the trie works.
# it will create virtual container for each account.
# see nim-eth#9
initSecureHexaryTrie(trieDB(db), account.storageRoot, false)
initStorageTrie(trieDB(db), account.storageRoot, false)
proc clearStorage*(db: AccountStateDB, address: EthAddress) =
var account = db.getAccount(address)
@ -147,14 +148,14 @@ proc setStorage*(db: AccountStateDB,
address: EthAddress,
slot: UInt256, value: UInt256) =
var account = db.getAccount(address)
var accountTrie = getAccountTrie(db, account)
var accountTrie = getStorageTrie(db, account)
let slotAsKey = createTrieKeyFromSlot slot
if value > 0:
let encodedValue = rlp.encode(value)
accountTrie.put(slotAsKey, encodedValue)
accountTrie.putSlotBytes(slotAsKey, encodedValue)
else:
accountTrie.del(slotAsKey)
accountTrie.delSlotBytes(slotAsKey)
# map slothash back to slot value
# see iterator storage below
@ -182,10 +183,10 @@ proc getStorage*(db: AccountStateDB, address: EthAddress, slot: UInt256): (UInt2
let
account = db.getAccount(address)
slotAsKey = createTrieKeyFromSlot slot
accountTrie = getAccountTrie(db, account)
storageTrie = getStorageTrie(db, account)
let
foundRecord = accountTrie.get(slotAsKey)
foundRecord = storageTrie.getSlotBytes(slotAsKey)
if foundRecord.len > 0:
result = (rlp.decode(foundRecord, UInt256), true)
@ -232,10 +233,10 @@ proc dumpAccount*(db: AccountStateDB, addressS: string): string =
return fmt"{addressS}: Storage: {db.getStorage(address, 0.u256)}; getAccount: {db.getAccount address}"
proc accountExists*(db: AccountStateDB, address: EthAddress): bool =
db.trie.get(address).len > 0
db.trie.getAccountBytes(address).len > 0
proc isEmptyAccount*(db: AccountStateDB, address: EthAddress): bool =
let recordFound = db.trie.get(address)
let recordFound = db.trie.getAccountBytes(address)
assert(recordFound.len > 0)
let account = rlp.decode(recordFound, Account)
@ -244,7 +245,7 @@ proc isEmptyAccount*(db: AccountStateDB, address: EthAddress): bool =
account.nonce == 0
proc isDeadAccount*(db: AccountStateDB, address: EthAddress): bool =
let recordFound = db.trie.get(address)
let recordFound = db.trie.getAccountBytes(address)
if recordFound.len > 0:
let account = rlp.decode(recordFound, Account)
result = account.codeHash == EMPTY_SHA3 and

View File

@ -2,7 +2,8 @@ import
randutils, random, parseopt, strutils, os,
eth/[common, rlp], eth/trie/[hexary, db, trie_defs],
nimcrypto/sysrand, ../stateless/[json_from_tree],
../nimbus/db/storage_types, ./witness_types, ./multi_keys
../nimbus/db/[storage_types, distinct_tries],
./witness_types, ./multi_keys
type
DB = TrieDatabaseRef
@ -38,12 +39,12 @@ proc randStorage(db: DB, numSlots: int): StorageKeys =
if rand(0..1) == 0 or numSlots == 0:
result = (emptyRlpHash, MultikeysRef(nil))
else:
var trie = initSecureHexaryTrie(db)
var trie = initStorageTrie(db)
var keys = newSeq[StorageSlot](numSlots)
for i in 0..<numSlots:
keys[i] = randStorageSlot()
trie.put(keys[i], rlp.encode(randU256()))
trie.putSlotBytes(keys[i], rlp.encode(randU256()))
if rand(0..1) == 0:
result = (trie.rootHash, MultikeysRef(nil))
@ -65,7 +66,7 @@ proc randAddress(): EthAddress =
proc runGenerator(numPairs, numSlots: int): string =
var memDB = newMemoryDB()
var trie = initSecureHexaryTrie(memDB)
var trie = initAccountsTrie(memDB)
var addrs = newSeq[AccountKey](numPairs)
var accs = newSeq[Account](numPairs)
@ -73,7 +74,7 @@ proc runGenerator(numPairs, numSlots: int): string =
let acc = randAccount(memDB, numSlots)
addrs[i] = (randAddress(), acc.codeTouched, acc.storageKeys)
accs[i] = acc.account
trie.put(addrs[i].address, rlp.encode(accs[i]))
trie.putAccountBytes(addrs[i].address, rlp.encode(accs[i]))
var mkeys = newMultiKeys(addrs)
let rootHash = trie.rootHash

View File

@ -3,7 +3,7 @@ import
eth/[common, rlp], eth/trie/[hexary, db, trie_defs],
stew/byteutils,
../tests/[test_helpers, test_config],
../nimbus/db/accounts_cache, ./witness_types,
../nimbus/db/[accounts_cache, distinct_tries], ./witness_types,
../stateless/[witness_from_tree, tree_from_witness],
./multi_keys
@ -13,7 +13,7 @@ type
memDB: TrieDatabaseRef
proc testGetBranch(tester: Tester, rootHash: KeccakHash, testStatusIMPL: var TestStatus) =
var trie = initSecureHexaryTrie(tester.memdb, rootHash)
var trie = initAccountsTrie(tester.memdb, rootHash)
let flags = {wfEIP170}
try:
@ -30,10 +30,10 @@ proc testGetBranch(tester: Tester, rootHash: KeccakHash, testStatusIMPL: var Tes
var root = tb.buildTree()
check root.data == rootHash.data
let newTrie = initSecureHexaryTrie(tb.getDB(), root)
let newTrie = initAccountsTrie(tb.getDB(), root)
for kd in tester.keys.keys:
let account = rlp.decode(trie.get(kd.address), Account)
let recordFound = newTrie.get(kd.address)
let account = rlp.decode(trie.getAccountBytes(kd.address), Account)
let recordFound = newTrie.getAccountBytes(kd.address)
if recordFound.len > 0:
let acc = rlp.decode(recordFound, Account)
doAssert acc == account

View File

@ -3,7 +3,7 @@ import
eth/[common, rlp], eth/trie/[hexary, db, trie_defs, nibbles],
faststreams/inputs, nimcrypto/sysrand,
../stateless/[witness_from_tree, tree_from_witness],
../nimbus/db/storage_types, ./witness_types, ./multi_keys
../nimbus/db/[distinct_tries, storage_types], ./witness_types, ./multi_keys
type
DB = TrieDatabaseRef
@ -39,13 +39,13 @@ proc randStorage(db: DB): StorageKeys =
if rand(0..1) == 0:
result = (emptyRlpHash, MultikeysRef(nil))
else:
var trie = initSecureHexaryTrie(db)
var trie = initStorageTrie(db)
let numPairs = rand(1..10)
var keys = newSeq[StorageSlot](numPairs)
for i in 0..<numPairs:
keys[i] = randStorageSlot()
trie.put(keys[i], rlp.encode(randU256()))
trie.putSlotBytes(keys[i], rlp.encode(randU256()))
if rand(0..1) == 0:
result = (trie.rootHash, MultikeysRef(nil))
@ -69,7 +69,7 @@ proc runTest(numPairs: int, testStatusIMPL: var TestStatus,
addIdenticalKeys: bool = false, addInvalidKeys: static[bool] = false) =
var memDB = newMemoryDB()
var trie = initSecureHexaryTrie(memDB)
var trie = initAccountsTrie(memDB)
var addrs = newSeq[AccountKey](numPairs)
var accs = newSeq[Account](numPairs)
@ -77,7 +77,7 @@ proc runTest(numPairs: int, testStatusIMPL: var TestStatus,
let acc = randAccount(memDB)
addrs[i] = AccountKey(address: randAddress(), codeTouched: acc.codeTouched, storageKeys: acc.storageKeys)
accs[i] = acc.account
trie.put(addrs[i].address, rlp.encode(accs[i]))
trie.putAccountBytes(addrs[i].address, rlp.encode(accs[i]))
when addInvalidKeys:
# invalidAddress should not end up in block witness
@ -102,9 +102,9 @@ proc runTest(numPairs: int, testStatusIMPL: var TestStatus,
let root = tb.buildTree()
check root.data == rootHash.data
let newTrie = initSecureHexaryTrie(tb.getDB(), root)
let newTrie = initAccountsTrie(tb.getDB(), root)
for i in 0..<numPairs:
let recordFound = newTrie.get(addrs[i].address)
let recordFound = newTrie.getAccountBytes(addrs[i].address)
if recordFound.len > 0:
let acc = rlp.decode(recordFound, Account)
check acc == accs[i]
@ -261,7 +261,7 @@ proc witnessKeysMain*() =
]
let m = initMultiKeys(keys, true)
var memDB = newMemoryDB()
var trie = initSecureHexaryTrie(memDB)
var trie = initAccountsTrie(memDB)
var acc = randAccount(memDB)
var tt = initHexaryTrie(memDB)
@ -271,7 +271,7 @@ proc witnessKeysMain*() =
let addrs = @[AccountKey(address: randAddress(), codeTouched: acc.codeTouched, storageKeys: m)]
trie.put(addrs[0].address, rlp.encode(acc.account))
trie.putAccountBytes(addrs[0].address, rlp.encode(acc.account))
var mkeys = newMultiKeys(addrs)
let rootHash = trie.rootHash

View File

@ -5,7 +5,7 @@ import
import
eth/trie/hexary,
../nimbus/db/accounts_cache,
../nimbus/db/[accounts_cache, distinct_tries],
../nimbus/evm/[async_operations, types],
../nimbus/vm_internals,
../nimbus/transaction/[call_common, call_evm],
@ -385,13 +385,13 @@ proc verifyAsmResult(vmState: BaseVMState, com: CommonRef, boa: Assembler, asmRe
var
storageRoot = stateDB.getStorageRoot(codeAddress)
trie = initSecureHexaryTrie(com.db.db, storageRoot)
trie = initStorageTrie(com.db.db, storageRoot)
for kv in boa.storage:
let key = kv[0].toHex()
let val = kv[1].toHex()
let keyBytes = (@(kv[0]))
let actual = trie.get(keyBytes).toHex()
let actual = trie.getSlotBytes(keyBytes).toHex()
let zerosLen = 64 - (actual.len)
let value = repeat('0', zerosLen) & actual
if val != value:

View File

@ -188,7 +188,7 @@ proc runTrial3crash(vmState: BaseVMState; inx: int; noisy = false) =
# following function could be added to db/accounts_cache.nim:
#
# proc clobberRootHash*(ac: AccountsCache; root: KeccakHash; prune = true) =
# ac.trie = initSecureHexaryTrie(ac.db, rootHash, prune)
# ac.trie = initAccountsTrie(ac.db, rootHash, prune)
#
# Then, beginning this very function `runTrial3crash()` with
#