Merge pull request #511 from status-im/accounts_cache

stateless client experiment: switch `state_db` to `accounts_cache`
This commit is contained in:
andri lim 2020-06-02 09:31:07 +07:00 committed by GitHub
commit a5e9db6ee1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 116 additions and 71 deletions

View File

@ -23,12 +23,14 @@ type
originalStorage: TableRef[UInt256, UInt256]
overlayStorage: Table[UInt256, UInt256]
AccountsCache* = object
AccountsCache* = ref object
db: TrieDatabaseRef
trie: SecureHexaryTrie
savePoint: SavePoint
unrevertablyTouched: HashSet[EthAddress]
ReadOnlyStateDB* = distinct AccountsCache
TransactionState = enum
Pending
Committed
@ -46,6 +48,7 @@ proc beginSavepoint*(ac: var AccountsCache): SavePoint {.gcsafe.}
# The AccountsCache is modeled after TrieDatabase for it's transaction style
proc init*(x: typedesc[AccountsCache], db: TrieDatabaseRef,
root: KeccakHash, pruneTrie: bool = true): AccountsCache =
new result
result.db = db
result.trie = initSecureHexaryTrie(db, root, pruneTrie)
result.unrevertablyTouched = initHashSet[EthAddress]()
@ -142,13 +145,8 @@ proc isEmpty(acc: RefAccount): bool =
acc.account.balance.isZero and
acc.account.nonce == 0
proc exists(acc: RefAccount): bool =
if IsAlive notin acc.flags:
return false
if IsClone in acc.flags:
result = true
else:
result = IsNew notin acc.flags
template exists(acc: RefAccount): bool =
IsAlive in acc.flags
template createTrieKeyFromSlot(slot: UInt256): auto =
# XXX: This is too expensive. Similar to `createRangeFromAddress`
@ -390,6 +388,12 @@ proc removeEmptyAccounts*(ac: var AccountsCache) =
if acc.isEmpty:
acc.kill()
proc deleteAccount*(ac: var AccountsCache, address: EthAddress) =
# make sure all savepoints already committed
doAssert(ac.savePoint.parentSavePoint.isNil)
let acc = ac.getAccount(address)
acc.kill()
proc persist*(ac: var AccountsCache) =
# make sure all savepoint already committed
doAssert(ac.savePoint.parentSavePoint.isNil)
@ -412,10 +416,33 @@ proc persist*(ac: var AccountsCache) =
iterator storage*(ac: AccountsCache, address: EthAddress): (UInt256, UInt256) =
# beware that if the account not persisted,
# the storage root will not be updated
let storageRoot = ac.getAccount(address).account.storageRoot
var trie = initHexaryTrie(ac.db, storageRoot)
let acc = ac.getAccount(address, false)
if not acc.isNil:
let storageRoot = acc.account.storageRoot
var trie = initHexaryTrie(ac.db, storageRoot)
for slot, value in trie:
if slot.len != 0:
var keyData = ac.db.get(slotHashToSlotKey(slot).toOpenArray)
yield (rlp.decode(keyData, UInt256), rlp.decode(value, UInt256))
for slot, value in trie:
if slot.len != 0:
var keyData = ac.db.get(slotHashToSlotKey(slot).toOpenArray)
yield (rlp.decode(keyData, UInt256), rlp.decode(value, UInt256))
proc getStorageRoot*(ac: AccountsCache, address: EthAddress): Hash256 =
# beware that if the account not persisted,
# the storage root will not be updated
let acc = ac.getAccount(address, false)
if acc.isNil: emptyAcc.storageRoot
else: acc.account.storageRoot
proc rootHash*(db: ReadOnlyStateDB): KeccakHash {.borrow.}
proc getCodeHash*(db: ReadOnlyStateDB, address: EthAddress): Hash256 {.borrow.}
proc getStorageRoot*(db: ReadOnlyStateDB, address: EthAddress): Hash256 {.borrow.}
proc getBalance*(db: ReadOnlyStateDB, address: EthAddress): UInt256 {.borrow.}
proc getStorage*(db: ReadOnlyStateDB, address: EthAddress, slot: UInt256): UInt256 {.borrow.}
proc getNonce*(db: ReadOnlyStateDB, address: EthAddress): AccountNonce {.borrow.}
proc getCode*(db: ReadOnlyStateDB, address: EthAddress): seq[byte] {.borrow.}
proc getCodeSize*(db: ReadOnlyStateDB, address: EthAddress): int {.borrow.}
proc hasCodeOrNonce*(db: ReadOnlyStateDB, address: EthAddress): bool {.borrow.}
proc accountExists*(db: ReadOnlyStateDB, address: EthAddress): bool {.borrow.}
proc isDeadAccount*(db: ReadOnlyStateDB, address: EthAddress): bool {.borrow.}
proc isEmptyAccount*(db: ReadOnlyStateDB, address: EthAddress): bool {.borrow.}
proc getCommittedStorage*(db: ReadOnlyStateDB, address: EthAddress, slot: UInt256): UInt256 {.borrow.}

View File

@ -1,4 +1,4 @@
import eth/common, stew/byteutils, ../db/state_db
import eth/common, stew/byteutils, ../db/accounts_cache
const
# DAOForkBlockExtra is the block header extra-data field to set for the DAO fork
@ -135,7 +135,7 @@ const
# ApplyDAOHardFork modifies the state database according to the DAO hard-fork
# rules, transferring all balances of a set of DAO accounts to a single refund
# contract.
proc applyDAOHardFork*(statedb: var AccountStateDB) =
proc applyDAOHardFork*(statedb: var AccountsCache) =
const zero = 0.u256
# Move every DAO account and extra-balance account funds into the refund contract
for address in DAODrainList:

View File

@ -1,6 +1,6 @@
import options, sets,
eth/[common, bloom, trie/db], chronicles, nimcrypto,
../db/[db_chain, state_db],
../db/[db_chain, accounts_cache],
../utils, ../constants, ../transaction,
../vm_state, ../vm_types, ../vm_state_transactions,
../vm/[computation, message],
@ -42,7 +42,8 @@ proc processTransaction*(tx: Transaction, sender: EthAddress, vmState: BaseVMSta
debug "state clearing", account
db.deleteAccount(account)
vmState.accountDb.updateOriginalRoot()
#vmState.accountDb.updateOriginalRoot()
vmState.accountDb.persist()
type
# TODO: these types need to be removed
@ -142,6 +143,7 @@ proc processBlock*(chainDB: BaseChainDB, header: BlockHeader, body: BlockBody, v
# Reward beneficiary
vmState.mutateStateDB:
db.addBalance(header.coinbase, mainReward)
db.persist()
let stateDb = vmState.accountDb
if header.stateRoot != stateDb.rootHash:

View File

@ -13,7 +13,7 @@ import
eth/[common, keys, rlp, p2p], nimcrypto,
../transaction, ../config, ../vm_state, ../constants, ../vm_types,
../vm_state_transactions, ../utils,
../db/[db_chain, state_db],
../db/[db_chain, accounts_cache],
rpc_types, rpc_utils, ../vm/[message, computation],
../vm/interpreter/vm_forks
@ -166,9 +166,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
let
accountDb = accountDbFromTag(quantityTag)
addrBytes = data.toAddress
storage = accountDb.getStorage(addrBytes, quantity.u256)
if storage[1]:
result = storage[0]
result = accountDb.getStorage(addrBytes, quantity.u256)
rpcsrv.rpc("eth_getTransactionCount") do(data: EthAddressStr, quantityTag: string) -> AccountNonce:
## Returns the number of transactions sent from an address.

View File

@ -1,5 +1,5 @@
import
db/[db_chain, state_db, capturedb], eth/common, utils, json,
db/[db_chain, accounts_cache, capturedb], eth/common, utils, json,
constants, vm_state, vm_types, transaction, p2p/executor,
eth/trie/db, nimcrypto, strutils,
chronicles, rpc/hexstrings, launcher,
@ -33,18 +33,23 @@ proc toJson*(receipts: seq[Receipt]): JsonNode =
for receipt in receipts:
result.add receipt.toJson
proc captureAccount(n: JsonNode, db: AccountStateDB, address: EthAddress, name: string) =
proc captureAccount(n: JsonNode, db: AccountsCache, address: EthAddress, name: string) =
var jaccount = newJObject()
jaccount["name"] = %name
jaccount["address"] = %("0x" & $address)
let account = db.getAccount(address)
jaccount["nonce"] = %(encodeQuantity(account.nonce).toLowerAscii)
jaccount["balance"] = %("0x" & account.balance.toHex)
let nonce = db.getNonce(address)
let balance = db.getBalance(address)
let codeHash = db.getCodeHash(address)
let storageRoot = db.getStorageRoot(address)
jaccount["nonce"] = %(encodeQuantity(nonce).toLowerAscii)
jaccount["balance"] = %("0x" & balance.toHex)
let code = db.getCode(address)
jaccount["codeHash"] = %("0x" & ($account.codeHash).toLowerAscii)
jaccount["codeHash"] = %("0x" & ($codeHash).toLowerAscii)
jaccount["code"] = %("0x" & toHex(code, true))
jaccount["storageRoot"] = %("0x" & ($account.storageRoot).toLowerAscii)
jaccount["storageRoot"] = %("0x" & ($storageRoot).toLowerAscii)
var storage = newJObject()
for key, value in db.storage(address):
@ -102,6 +107,7 @@ proc traceTransaction*(chainDB: BaseChainDB, header: BlockHeader,
before.captureAccount(stateDb, sender, senderName)
before.captureAccount(stateDb, recipient, recipientName)
before.captureAccount(stateDb, header.coinbase, minerName)
stateDb.persist()
stateDiff["beforeRoot"] = %($stateDb.rootHash)
beforeRoot = stateDb.rootHash
@ -112,11 +118,12 @@ proc traceTransaction*(chainDB: BaseChainDB, header: BlockHeader,
after.captureAccount(stateDb, recipient, recipientName)
after.captureAccount(stateDb, header.coinbase, minerName)
vmState.removeTracedAccounts(sender, recipient, header.coinbase)
stateDb.persist()
stateDiff["afterRoot"] = %($stateDb.rootHash)
break
# internal transactions:
var stateBefore = newAccountStateDB(captureTrieDB, beforeRoot, chainDB.pruneTrie)
var stateBefore = AccountsCache.init(captureTrieDB, beforeRoot, chainDB.pruneTrie)
for idx, acc in tracedAccountsPairs(vmState):
before.captureAccount(stateBefore, acc, internalTxName & $idx)
@ -146,7 +153,7 @@ proc dumpBlockState*(db: BaseChainDB, header: BlockHeader, body: BlockBody, dump
var
before = newJArray()
after = newJArray()
stateBefore = newAccountStateDB(captureTrieDB, parent.stateRoot, db.pruneTrie)
stateBefore = AccountsCache.init(captureTrieDB, parent.stateRoot, db.pruneTrie)
for idx, tx in body.transactions:
let sender = tx.getSender

View File

@ -10,7 +10,7 @@ import
sets, eth/[common, keys], eth/trie/db as triedb,
../constants, ../errors, ../vm_state, ../vm_types,
./interpreter/[opcode_values, gas_meter, gas_costs, vm_forks],
./code_stream, ./memory, ./message, ./stack, ../db/[state_db, db_chain],
./code_stream, ./memory, ./message, ./stack, ../db/[accounts_cache, db_chain],
../utils/header, stew/[byteutils, ranges/ptr_arith], precompiles,
transaction_tracer, ../utils
@ -90,7 +90,7 @@ template getStorage*(c: Computation, slot: Uint256): Uint256 =
when evmc_enabled:
c.host.getStorage(c.msg.contractAddress, slot)
else:
c.vmState.readOnlyStateDB.getStorage(c.msg.contractAddress, slot)[0]
c.vmState.readOnlyStateDB.getStorage(c.msg.contractAddress, slot)
template getBalance*(c: Computation, address: EthAddress): Uint256 =
when evmc_enabled:
@ -102,7 +102,7 @@ template getCodeSize*(c: Computation, address: EthAddress): uint =
when evmc_enabled:
c.host.getCodeSize(address)
else:
uint(c.vmState.readOnlyStateDB.getCode(address).len)
uint(c.vmState.readOnlyStateDB.getCodeSize(address))
template getCodeHash*(c: Computation, address: EthAddress): Hash256 =
when evmc_enabled:
@ -179,18 +179,16 @@ proc isSuicided*(c: Computation, address: EthAddress): bool =
result = address in c.suicides
proc snapshot*(c: Computation) =
c.dbsnapshot.transaction = c.vmState.chaindb.db.beginTransaction()
c.dbsnapshot.intermediateRoot = c.vmState.accountDb.rootHash
c.savePoint = c.vmState.accountDb.beginSavePoint()
proc commit*(c: Computation) =
c.dbsnapshot.transaction.commit()
c.vmState.accountDb.commit(c.savePoint)
proc dispose*(c: Computation) {.inline.} =
c.dbsnapshot.transaction.dispose()
c.vmState.accountDb.dispose(c.savePoint)
proc rollback*(c: Computation) =
c.dbsnapshot.transaction.rollback()
c.vmState.accountDb.rootHash = c.dbsnapshot.intermediateRoot
c.vmState.accountDb.rollback(c.savePoint)
proc setError*(c: Computation, msg: string, burnsGas = false) {.inline.} =
c.error = Error(info: msg, burnsGas: burnsGas)

View File

@ -12,7 +12,7 @@ import
./gas_meter, ./gas_costs, ./opcode_values, ./vm_forks,
../memory, ../stack, ../code_stream, ../computation,
../../vm_state, ../../errors, ../../constants, ../../vm_types,
../../db/[db_chain, state_db]
../../db/[db_chain, accounts_cache]
when defined(evmc_enabled):
import ../evmc_api, ../evmc_helpers, evmc/evmc

View File

@ -1,7 +1,7 @@
import
json, strutils, sets, hashes,
chronicles, nimcrypto, eth/common, stint,
../vm_types, memory, stack, ../db/state_db,
../vm_types, memory, stack, ../db/accounts_cache,
eth/trie/hexary,
./interpreter/opcode_values
@ -100,7 +100,7 @@ proc traceOpCodeEnded*(tracer: var TransactionTracer, c: Computation, op: Op, la
if c.msg.depth < tracer.storageKeys.len:
var stateDB = c.vmState.accountDb
for key in tracer.storage(c.msg.depth):
let (value, _) = stateDB.getStorage(c.msg.contractAddress, key)
let value = stateDB.getStorage(c.msg.contractAddress, key)
storage[key.dumpHex] = %(value.dumpHex)
j["storage"] = storage

View File

@ -9,7 +9,7 @@ import
macros, strformat, tables, sets, options,
eth/common,
vm/interpreter/[vm_forks, gas_costs],
./constants, ./db/[db_chain, state_db],
./constants, ./db/[db_chain, accounts_cache],
./utils, json, vm_types, vm/transaction_tracer,
./config
@ -36,7 +36,7 @@ proc init*(self: BaseVMState, prevStateRoot: Hash256, header: BlockHeader,
self.tracer.initTracer(tracerFlags)
self.tracingEnabled = TracerFlags.EnableTracing in tracerFlags
self.logEntries = @[]
self.accountDb = newAccountStateDB(chainDB.db, prevStateRoot, chainDB.pruneTrie)
self.accountDb = AccountsCache.init(chainDB.db, prevStateRoot, chainDB.pruneTrie)
self.touchedAccounts = initHashSet[EthAddress]()
proc newBaseVMState*(prevStateRoot: Hash256, header: BlockHeader,

View File

@ -7,12 +7,13 @@
import
options, sets,
eth/common, chronicles, ./db/state_db,
eth/common, chronicles, ./db/accounts_cache,
transaction, vm_types, vm_state,
./vm/[computation, interpreter]
proc validateTransaction*(vmState: BaseVMState, tx: Transaction, sender: EthAddress, fork: Fork): bool =
let account = vmState.readOnlyStateDB.getAccount(sender)
let balance = vmState.readOnlyStateDB.getBalance(sender)
let nonce = vmState.readOnlyStateDB.getNonce(sender)
if vmState.cumulativeGasUsed + tx.gasLimit > vmState.blockHeader.gasLimit:
debug "invalid tx: block header gasLimit reached",
@ -22,9 +23,9 @@ proc validateTransaction*(vmState: BaseVMState, tx: Transaction, sender: EthAddr
return
let totalCost = tx.gasLimit.u256 * tx.gasPrice.u256 + tx.value
if totalCost > account.balance:
if totalCost > balance:
debug "invalid tx: not enough cash",
available=account.balance,
available=balance,
require=totalCost
return
@ -34,10 +35,10 @@ proc validateTransaction*(vmState: BaseVMState, tx: Transaction, sender: EthAddr
require=tx.intrinsicGas(fork)
return
if tx.accountNonce != account.nonce:
if tx.accountNonce != nonce:
debug "invalid tx: account nonce mismatch",
txNonce=tx.accountnonce,
accountNonce=account.nonce
accountNonce=nonce
return
result = true

View File

@ -10,7 +10,7 @@ import
options, json, sets,
./vm/[memory, stack, code_stream],
./vm/interpreter/[gas_costs, opcode_values, vm_forks], # TODO - will be hidden at a lower layer
./db/[db_chain, state_db]
./db/[db_chain, accounts_cache]
when defined(evmc_enabled):
import ./vm/evmc_api
@ -26,7 +26,7 @@ type
tracer* : TransactionTracer
logEntries* : seq[Log]
receipts* : seq[Receipt]
accountDb* : AccountStateDB
accountDb* : AccountsCache
cumulativeGasUsed*: GasInt
touchedAccounts*: HashSet[EthAddress]
suicides* : HashSet[EthAddress]
@ -55,10 +55,6 @@ type
accounts*: HashSet[EthAddress]
storageKeys*: seq[HashSet[Uint256]]
Snapshot* = object
transaction*: DbTransaction
intermediateRoot*: Hash256
Computation* = ref object
# The execution computation
vmState*: BaseVMState
@ -75,7 +71,7 @@ type
touchedAccounts*: HashSet[EthAddress]
suicides*: HashSet[EthAddress]
logEntries*: seq[Log]
dbsnapshot*: Snapshot
savePoint*: SavePoint
instr*: Op
opIndex*: int

View File

@ -7,7 +7,7 @@ import
import
options, json, os, eth/trie/[db, hexary],
../nimbus/[vm_state, vm_types, transaction, utils],
../nimbus/db/[db_chain, state_db],
../nimbus/db/[db_chain, accounts_cache],
../nimbus/vm_state_transactions,
../nimbus/vm/interpreter/vm_forks,
../nimbus/vm/[message, computation, memory]
@ -301,10 +301,12 @@ proc runVM*(blockNumber: Uint256, chainDB: BaseChainDB, boa: Assembler): bool =
error "different memory value", idx=i, expected=mem, actual=actual
return false
var stateDB = computation.vmState.accountDb
stateDB.persist()
var
stateDB = computation.vmState.accountDb
account = stateDB.getAccount(computation.msg.contractAddress)
trie = initSecureHexaryTrie(chainDB.db, account.storageRoot)
storageRoot = stateDB.getStorageRoot(computation.msg.contractAddress)
trie = initSecureHexaryTrie(chainDB.db, storageRoot)
for kv in boa.storage:
let key = kv[0].toHex()

View File

@ -14,7 +14,7 @@ import
../premix/parser, test_config,
../nimbus/vm/interpreter/vm_forks,
../nimbus/[vm_state, utils, vm_types, errors, transaction, constants],
../nimbus/db/[db_chain, state_db],
../nimbus/db/[db_chain, accounts_cache],
../nimbus/utils/header,
../nimbus/p2p/[executor, dao],
../nimbus/config
@ -299,6 +299,7 @@ proc assignBlockRewards(minedBlock: PlainBlock, vmState: BaseVMState, fork: Fork
# Reward beneficiary
vmState.mutateStateDB:
db.addBalance(minedBlock.header.coinbase, mainReward)
db.persist()
let stateDb = vmState.accountDb
if minedBlock.header.stateRoot != stateDb.rootHash:
@ -698,6 +699,7 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus, debugMode = fal
vmState.mutateStateDB:
setupStateDB(fixture["pre"], db)
db.persist()
let obtainedHash = $(vmState.readOnlyStateDB.rootHash)
check obtainedHash == $(tester.genesisBlockHeader.stateRoot)

View File

@ -14,7 +14,7 @@ import
../nimbus/transaction,
../nimbus/[vm_state, vm_types, utils],
../nimbus/vm/interpreter,
../nimbus/db/[db_chain, state_db]
../nimbus/db/[db_chain, accounts_cache]
type
Tester = object
@ -96,7 +96,10 @@ proc testFixtureIndexes(tester: Tester, testStatusIMPL: var TestStatus) =
vmState.mutateStateDB:
setupStateDB(tester.pre, db)
vmState.accountDB.updateOriginalRoot()
# this is an important step when using accounts_cache
# it will affect the account storage's location
# during the next call to `getComittedStorage`
db.persist()
defer:
let obtainedHash = "0x" & `$`(vmState.readOnlyStateDB.rootHash).toLowerAscii
@ -128,6 +131,12 @@ proc testFixtureIndexes(tester: Tester, testStatusIMPL: var TestStatus) =
if db.isEmptyAccount(miner):
db.deleteAccount(miner)
# this is an important step when using accounts_cache
# it will affect the account storage's location
# during the next call to `getComittedStorage`
# and the result of rootHash
db.persist()
proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus,
trace = false, debugMode = false, supportedForks: set[Fork] = supportedForks) =
var tester: Tester

View File

@ -11,7 +11,7 @@ import
testutils/markdown_reports,
../nimbus/[config, transaction, utils, errors],
../nimbus/vm/interpreter/vm_forks,
../nimbus/db/state_db
../nimbus/db/accounts_cache
func revmap(x: Table[Fork, string]): Table[string, Fork] =
result = initTable[string, Fork]()
@ -135,7 +135,7 @@ func getHexadecimalInt*(j: JsonNode): int64 =
data = fromHex(StUInt[64], j.getStr)
result = cast[int64](data)
proc setupStateDB*(wantedState: JsonNode, stateDB: var AccountStateDB) =
proc setupStateDB*(wantedState: JsonNode, stateDB: var AccountsCache) =
for ac, accountData in wantedState:
let account = ethAddressFromHex(ac)
for slot, value in accountData{"storage"}:
@ -157,9 +157,9 @@ proc verifyStateDB*(wantedState: JsonNode, stateDB: ReadOnlyStateDB) =
slotId = UInt256.fromHex slot
wantedValue = UInt256.fromHex value.getStr
let (actualValue, found) = stateDB.getStorage(account, slotId)
if not found:
raise newException(ValidationError, "account not found: " & ac)
let actualValue = stateDB.getStorage(account, slotId)
#if not found:
# raise newException(ValidationError, "account not found: " & ac)
if actualValue != wantedValue:
raise newException(ValidationError, &"{ac} storageDiff: [{slot}] {actualValue.toHex} != {wantedValue.toHex}")

View File

@ -840,3 +840,6 @@ proc opMemoryMain*() =
"0x00"
"0x00"
"0x0000000000000000000000000000002000000000000000000000000000000000"
when isMainModule:
opMemoryMain()

View File

@ -11,7 +11,7 @@ import
eth/[rlp, keys], eth/trie/db, eth/p2p/rlpx_protocols/eth_protocol,
../nimbus/rpc/[common, p2p, hexstrings, rpc_types],
../nimbus/[constants, vm_state, config, genesis],
../nimbus/db/[state_db, db_chain, storage_types],
../nimbus/db/[accounts_cache, db_chain, storage_types],
../nimbus/p2p/chain,
./rpcclient/test_hexstrings, ./test_helpers