nimbus-eth1/tests/test_accounts_cache.nim

376 lines
12 KiB
Nim
Raw Normal View History

# Nimbus
# Copyright (c) 2018-2019 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
import
std/[os, sequtils, strformat, strutils, tables],
../nimbus/[chain_config, config, constants, genesis],
../nimbus/db/[accounts_cache, db_chain],
../nimbus/p2p/chain,
../nimbus/transaction,
../nimbus/vm_state,
../nimbus/vm_types,
Redesign of BaseVMState descriptor (#923) * Redesign of BaseVMState descriptor why: BaseVMState provides an environment for executing transactions. The current descriptor also provides data that cannot generally be known within the execution environment, e.g. the total gasUsed which is available not before after all transactions have finished. Also, the BaseVMState constructor has been replaced by a constructor that does not need pre-initialised input of the account database. also: Previous constructor and some fields are provided with a deprecated annotation (producing a lot of noise.) * Replace legacy directives in production sources * Replace legacy directives in unit test sources * fix CI (missing premix update) * Remove legacy directives * chase CI problem * rebased * Re-introduce 'AccountsCache' constructor optimisation for 'BaseVmState' re-initialisation why: Constructing a new 'AccountsCache' descriptor can be avoided sometimes when the current state root is properly positioned already. Such a feature existed already as the update function 'initStateDB()' for the 'BaseChanDB' where the accounts cache was linked into this desctiptor. The function 'initStateDB()' was removed and re-implemented into the 'BaseVmState' constructor without optimisation. The old version was of restricted use as a wrong accounts cache state would unconditionally throw an exception rather than conceptually ask for a remedy. The optimised 'BaseVmState' re-initialisation has been implemented for the 'persistBlocks()' function. also: moved some test helpers to 'test/replay' folder * Remove unused & undocumented fields from Chain descriptor why: Reduces attack surface in general & improves reading the code.
2022-01-18 16:19:32 +00:00
./replay/undump,
eth/[common, p2p, trie/db],
unittest2
type
CaptureSpecs = tuple
network: NetworkID
file: string
numBlocks: int
numTxs: int
const
baseDir = [".", "tests", ".." / "tests", $DirSep] # path containg repo
repoDir = ["replay", "status", "test_clique"] # alternative repo paths
goerliCapture: CaptureSpecs = (
network: GoerliNet,
Redesign of BaseVMState descriptor (#923) * Redesign of BaseVMState descriptor why: BaseVMState provides an environment for executing transactions. The current descriptor also provides data that cannot generally be known within the execution environment, e.g. the total gasUsed which is available not before after all transactions have finished. Also, the BaseVMState constructor has been replaced by a constructor that does not need pre-initialised input of the account database. also: Previous constructor and some fields are provided with a deprecated annotation (producing a lot of noise.) * Replace legacy directives in production sources * Replace legacy directives in unit test sources * fix CI (missing premix update) * Remove legacy directives * chase CI problem * rebased * Re-introduce 'AccountsCache' constructor optimisation for 'BaseVmState' re-initialisation why: Constructing a new 'AccountsCache' descriptor can be avoided sometimes when the current state root is properly positioned already. Such a feature existed already as the update function 'initStateDB()' for the 'BaseChanDB' where the accounts cache was linked into this desctiptor. The function 'initStateDB()' was removed and re-implemented into the 'BaseVmState' constructor without optimisation. The old version was of restricted use as a wrong accounts cache state would unconditionally throw an exception rather than conceptually ask for a remedy. The optimised 'BaseVmState' re-initialisation has been implemented for the 'persistBlocks()' function. also: moved some test helpers to 'test/replay' folder * Remove unused & undocumented fields from Chain descriptor why: Reduces attack surface in general & improves reading the code.
2022-01-18 16:19:32 +00:00
file: "goerli68161.txt.gz",
numBlocks: 5500, # unconditionally load blocks
numTxs: 10) # txs following (not in block chain)
goerliCapture1: CaptureSpecs = (
GoerliNet, goerliCapture.file, 5500, 10000)
mainCapture: CaptureSpecs = (
MainNet, "mainnet843841.txt.gz", 50000, 3000)
var
xdb: BaseChainDB
txs: seq[Transaction]
txi: seq[int] # selected index into txs[] (crashable sender addresses)
# ------------------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------------------
proc findFilePath(file: string): string =
result = "?unknown?" / file
for dir in baseDir:
for repo in repoDir:
let path = dir / repo / file
if path.fileExists:
return path
proc pp*(a: EthAddress): string =
a.mapIt(it.toHex(2)).join[32 .. 39].toLowerAscii
proc pp*(tx: Transaction): string =
# "(" & tx.ecRecover.value.pp & "," & $tx.nonce & ")"
"(" & tx.getSender.pp & "," & $tx.nonce & ")"
proc pp*(h: KeccakHash): string =
h.data.mapIt(it.toHex(2)).join[52 .. 63].toLowerAscii
proc pp*(tx: Transaction; vmState: BaseVMState): string =
let address = tx.getSender
"(" & address.pp &
"," & $tx.nonce &
";" & $vmState.readOnlyStateDB.getNonce(address) &
"," & $vmState.readOnlyStateDB.getBalance(address) &
")"
# ------------------------------------------------------------------------------
# Private functions
# ------------------------------------------------------------------------------
proc blockChainForTesting*(network: NetworkID): BaseChainDB =
result = newBaseChainDB(
newMemoryDb(),
id = network,
params = network.networkParams)
result.populateProgress
initializeEmptyDB(result)
proc importBlocks(cdb: BaseChainDB; h: seq[BlockHeader]; b: seq[BlockBody]) =
if cdb.newChain.persistBlocks(h,b) != ValidationResult.OK:
raiseAssert "persistBlocks() failed at block #" & $h[0].blockNumber
proc getVmState(cdb: BaseChainDB; number: BlockNumber): BaseVMState =
Redesign of BaseVMState descriptor (#923) * Redesign of BaseVMState descriptor why: BaseVMState provides an environment for executing transactions. The current descriptor also provides data that cannot generally be known within the execution environment, e.g. the total gasUsed which is available not before after all transactions have finished. Also, the BaseVMState constructor has been replaced by a constructor that does not need pre-initialised input of the account database. also: Previous constructor and some fields are provided with a deprecated annotation (producing a lot of noise.) * Replace legacy directives in production sources * Replace legacy directives in unit test sources * fix CI (missing premix update) * Remove legacy directives * chase CI problem * rebased * Re-introduce 'AccountsCache' constructor optimisation for 'BaseVmState' re-initialisation why: Constructing a new 'AccountsCache' descriptor can be avoided sometimes when the current state root is properly positioned already. Such a feature existed already as the update function 'initStateDB()' for the 'BaseChanDB' where the accounts cache was linked into this desctiptor. The function 'initStateDB()' was removed and re-implemented into the 'BaseVmState' constructor without optimisation. The old version was of restricted use as a wrong accounts cache state would unconditionally throw an exception rather than conceptually ask for a remedy. The optimised 'BaseVmState' re-initialisation has been implemented for the 'persistBlocks()' function. also: moved some test helpers to 'test/replay' folder * Remove unused & undocumented fields from Chain descriptor why: Reduces attack surface in general & improves reading the code.
2022-01-18 16:19:32 +00:00
BaseVMState.new(cdb.getBlockHeader(number), cdb)
# ------------------------------------------------------------------------------
# Crash test function, finding out about how the transaction framework works ..
# ------------------------------------------------------------------------------
proc modBalance(ac: var AccountsCache, address: EthAddress) =
## This function is crucial for profucing the crash. If must
## modify the balance so that the database gets written.
# ac.blindBalanceSetter(address)
ac.addBalance(address, 1.u256)
proc runTrial2ok(vmState: BaseVMState; inx: int) =
## Run two blocks, the first one with *rollback*.
let eAddr = txs[inx].getSender
block:
let accTx = vmState.stateDB.beginSavepoint
vmState.stateDB.modBalance(eAddr)
vmState.stateDB.rollback(accTx)
block:
let accTx = vmState.stateDB.beginSavepoint
vmState.stateDB.modBalance(eAddr)
vmState.stateDB.commit(accTx)
vmState.stateDB.persist(clearCache = false)
proc runTrial3(vmState: BaseVMState; inx: int; rollback: bool) =
## Run three blocks, the second one optionally with *rollback*.
let eAddr = txs[inx].getSender
block:
let accTx = vmState.stateDB.beginSavepoint
vmState.stateDB.modBalance(eAddr)
vmState.stateDB.commit(accTx)
vmState.stateDB.persist(clearCache = false)
block:
let accTx = vmState.stateDB.beginSavepoint
vmState.stateDB.modBalance(eAddr)
if rollback:
vmState.stateDB.rollback(accTx)
break
vmState.stateDB.commit(accTx)
vmState.stateDB.persist(clearCache = false)
block:
let accTx = vmState.stateDB.beginSavepoint
vmState.stateDB.modBalance(eAddr)
vmState.stateDB.commit(accTx)
vmState.stateDB.persist(clearCache = false)
proc runTrial3crash(vmState: BaseVMState; inx: int; noisy = false) =
## Run three blocks with extra db frames and *rollback*.
let eAddr = txs[inx].getSender
block:
let dbTx = xdb.db.beginTransaction()
block:
let accTx = vmState.stateDB.beginSavepoint
vmState.stateDB.modBalance(eAddr)
vmState.stateDB.commit(accTx)
vmState.stateDB.persist(clearCache = false)
block:
let accTx = vmState.stateDB.beginSavepoint
vmState.stateDB.modBalance(eAddr)
vmState.stateDB.rollback(accTx)
# The following statement will cause a crash at the next `persist()` call.
dbTx.rollback()
# In order to survive without an exception in the next `persist()` call, the
# following function could be added to db/accounts_cache.nim:
#
# proc clobberRootHash*(ac: AccountsCache; root: KeccakHash; prune = true) =
# ac.trie = initSecureHexaryTrie(ac.db, rootHash, prune)
#
# Then, beginning this very function `runTrial3crash()` with
#
# let stateRoot = vmState.stateDB.rootHash
#
# the survival statement would be to re-assign the state-root via
#
# vmState.stateDB.clobberRootHash(stateRoot)
#
# Also mind this comment from Andri:
#
# [..] but as a reminder, only reinit the ac.trie is not enough, you
# should consider the accounts in the cache too. if there is any accounts
# in the cache they must in sync with the new rootHash.
#
block:
let dbTx = xdb.db.beginTransaction()
block:
let accTx = vmState.stateDB.beginSavepoint
vmState.stateDB.modBalance(eAddr)
vmState.stateDB.commit(accTx)
try:
vmState.stateDB.persist(clearCache = false)
except AssertionError as e:
if noisy:
let msg = e.msg.rsplit($DirSep,1)[^1]
echo &"*** runVmExec({eAddr.pp}): {e.name}: {msg}"
dbTx.dispose()
raise e
vmState.stateDB.persist(clearCache = false)
dbTx.commit()
proc runTrial4(vmState: BaseVMState; inx: int; rollback: bool) =
## Like `runTrial3()` but with four blocks and extra db transaction frames.
let eAddr = txs[inx].getSender
block:
let dbTx = xdb.db.beginTransaction()
block:
let accTx = vmState.stateDB.beginSavepoint
vmState.stateDB.modBalance(eAddr)
vmState.stateDB.commit(accTx)
vmState.stateDB.persist(clearCache = false)
block:
let accTx = vmState.stateDB.beginSavepoint
vmState.stateDB.modBalance(eAddr)
vmState.stateDB.commit(accTx)
vmState.stateDB.persist(clearCache = false)
block:
let accTx = vmState.stateDB.beginSavepoint
vmState.stateDB.modBalance(eAddr)
if rollback:
vmState.stateDB.rollback(accTx)
break
vmState.stateDB.commit(accTx)
vmState.stateDB.persist(clearCache = false)
# There must be no dbTx.rollback() here unless `vmState.stateDB` is
# discarded and/or re-initialised.
dbTx.commit()
block:
let dbTx = xdb.db.beginTransaction()
block:
let accTx = vmState.stateDB.beginSavepoint
vmState.stateDB.modBalance(eAddr)
vmState.stateDB.commit(accTx)
vmState.stateDB.persist(clearCache = false)
dbTx.commit()
# ------------------------------------------------------------------------------
# Test Runner
# ------------------------------------------------------------------------------
proc runner(noisy = true; capture = goerliCapture) =
let
loadBlocks = capture.numBlocks.u256
loadTxs = capture.numTxs
fileInfo = capture.file.splitFile.name.split(".")[0]
filePath = capture.file.findFilePath
txs.reset
xdb = capture.network.blockChainForTesting
suite &"StateDB nesting scenarios":
var topNumber: BlockNumber
test &"Import from {fileInfo}":
# Import minimum amount of blocks, then collect transactions
for chain in filePath.undumpNextGroup:
let leadBlkNum = chain[0][0].blockNumber
topNumber = chain[0][^1].blockNumber
if loadTxs <= txs.len:
break
# Verify Genesis
if leadBlkNum == 0.u256:
doAssert chain[0][0] == xdb.getBlockHeader(0.u256)
continue
# Import block chain blocks
if leadBlkNum < loadBlocks:
xdb.importBlocks(chain[0],chain[1])
continue
# Import transactions
for inx in 0 ..< chain[0].len:
let blkTxs = chain[1][inx].transactions
# Continue importing up until first non-trivial block
if txs.len == 0 and blkTxs.len == 0:
xdb.importBlocks(@[chain[0][inx]],@[chain[1][inx]])
continue
# Load transactions
txs.add blkTxs
test &"Collect unique sender addresses from {txs.len} txs," &
&" head=#{xdb.getCanonicalHead.blockNumber}, top=#{topNumber}":
var seen: Table[EthAddress,bool]
for n,tx in txs:
let a = tx.getSender
if not seen.hasKey(a):
seen[a] = true
txi.add n
test &"Run {txi.len} two-step trials with rollback":
let dbTx = xdb.db.beginTransaction()
defer: dbTx.dispose()
for n in txi:
let vmState = xdb.getVmState(xdb.getCanonicalHead.blockNumber)
vmState.runTrial2ok(n)
test &"Run {txi.len} three-step trials with rollback":
let dbTx = xdb.db.beginTransaction()
defer: dbTx.dispose()
for n in txi:
let vmState = xdb.getVmState(xdb.getCanonicalHead.blockNumber)
vmState.runTrial3(n, rollback = true)
test &"Run {txi.len} three-step trials with extra db frame rollback" &
" throwing Exceptions":
let dbTx = xdb.db.beginTransaction()
defer: dbTx.dispose()
for n in txi:
let vmState = xdb.getVmState(xdb.getCanonicalHead.blockNumber)
expect AssertionError:
vmState.runTrial3crash(n, noisy)
test &"Run {txi.len} tree-step trials without rollback":
let dbTx = xdb.db.beginTransaction()
defer: dbTx.dispose()
for n in txi:
let vmState = xdb.getVmState(xdb.getCanonicalHead.blockNumber)
vmState.runTrial3(n, rollback = false)
test &"Run {txi.len} four-step trials with rollback and db frames":
let dbTx = xdb.db.beginTransaction()
defer: dbTx.dispose()
for n in txi:
let vmState = xdb.getVmState(xdb.getCanonicalHead.blockNumber)
vmState.runTrial4(n, rollback = true)
# ------------------------------------------------------------------------------
# Main function(s)
# ------------------------------------------------------------------------------
proc accountsCacheMain*(noisy = defined(debug)) =
noisy.runner
when isMainModule:
var noisy = defined(debug)
#noisy = true
noisy.runner # mainCapture
# noisy.runner goerliCapture2
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------