mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-02-10 19:17:16 +00:00
0b32078c4b
This PR consolidates the split header-body sequences into a single EthBlock
sequence and cleans up the fallout from that which significantly reduces
block processing overhead during import thanks to less garbage collection
and fewer copies of things all around.
Notably, since the number of headers must always match the number of bodies,
we also get rid of a pointless degree of freedom that in the future could
introduce unnecessary bugs.
* only read header and body from era file
* avoid several unnecessary copies along the block processing way
* simplify signatures, cleaning up unused arguemnts and returns
* use `stew/assign2` in a few strategic places where the generated
nim assignent is slow and add a few `move` to work around poor
analysis in nim 1.6 (will need to be revisited for 2.0)
```
stats-20240607_2223-a814aa0b.csv vs stats-20240608_0714-21c1d0a9.csv
bps_x bps_y tps_x tps_y bpsd tpsd timed
block_number
(498305, 713245] 1,540.52 1,809.73 2,361.58 2775.340189 17.63% 17.63% -14.92%
(713245, 928185] 730.36 865.26 1,715.90 2028.973852 18.01% 18.01% -15.21%
(928185, 1143126] 663.03 789.10 2,529.26 3032.490771 19.79% 19.79% -16.28%
(1143126, 1358066] 393.46 508.05 2,152.50 2777.578119 29.13% 29.13% -22.50%
(1358066, 1573007] 370.88 440.72 2,351.31 2791.896052 18.81% 18.81% -15.80%
(1573007, 1787947] 283.65 335.11 2,068.93 2441.373402 17.60% 17.60% -14.91%
(1787947, 2002888] 287.29 342.11 2,078.39 2474.179448 18.99% 18.99% -15.91%
(2002888, 2217828] 293.38 343.16 2,208.83 2584.77457 17.16% 17.16% -14.61%
(2217828, 2432769] 140.09 167.86 1,081.87 1296.336926 18.82% 18.82% -15.80%
blocks: 1934464, baseline: 3h13m1s, contender: 2h43m47s
bpsd (mean): 19.55%
tpsd (mean): 19.55%
Time (total): -29m13s, -15.14%
```
391 lines
12 KiB
Nim
391 lines
12 KiB
Nim
# Nimbus
|
|
# Copyright (c) 2018-2024 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or distributed except
|
|
# according to those terms.
|
|
|
|
import
|
|
std/[os, strformat, strutils, tables],
|
|
chronicles,
|
|
stew/byteutils,
|
|
../nimbus/db/ledger,
|
|
../nimbus/common/common,
|
|
../nimbus/core/chain,
|
|
../nimbus/transaction,
|
|
../nimbus/constants,
|
|
../nimbus/vm_state,
|
|
../nimbus/vm_types,
|
|
./replay/undump_blocks,
|
|
unittest2
|
|
|
|
type
|
|
CaptureSpecs = tuple
|
|
network: NetworkId
|
|
file: string
|
|
numBlocks: int
|
|
numTxs: int
|
|
|
|
const
|
|
baseDir = [".", "tests", ".." / "tests", $DirSep] # path containg repo
|
|
repoDir = ["replay", "status", "test_clique"] # alternative repo paths
|
|
|
|
goerliCapture: CaptureSpecs = (
|
|
network: GoerliNet,
|
|
file: "goerli68161.txt.gz",
|
|
numBlocks: 5500, # unconditionally load blocks
|
|
numTxs: 10) # txs following (not in block chain)
|
|
|
|
when false:
|
|
goerliCapture1: CaptureSpecs = (
|
|
GoerliNet, goerliCapture.file, 5500, 10000)
|
|
|
|
mainCapture: CaptureSpecs = (
|
|
MainNet, "mainnet843841.txt.gz", 50000, 3000)
|
|
|
|
var
|
|
xdb: CoreDbRef
|
|
txs: seq[Transaction]
|
|
txi: seq[int] # selected index into txs[] (crashable sender addresses)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc findFilePath(file: string): string =
|
|
result = "?unknown?" / file
|
|
for dir in baseDir:
|
|
for repo in repoDir:
|
|
let path = dir / repo / file
|
|
if path.fileExists:
|
|
return path
|
|
|
|
proc pp*(a: EthAddress): string =
|
|
a.toHex[32 .. 39].toLowerAscii
|
|
|
|
proc pp*(tx: Transaction): string =
|
|
# "(" & tx.ecRecover.value.pp & "," & $tx.nonce & ")"
|
|
"(" & tx.getSender.pp & "," & $tx.nonce & ")"
|
|
|
|
proc pp*(h: KeccakHash): string =
|
|
h.data.toHex[52 .. 63].toLowerAscii
|
|
|
|
proc pp*(tx: Transaction; vmState: BaseVMState): string =
|
|
let address = tx.getSender
|
|
"(" & address.pp &
|
|
"," & $tx.nonce &
|
|
";" & $vmState.readOnlyStateDB.getNonce(address) &
|
|
"," & $vmState.readOnlyStateDB.getBalance(address) &
|
|
")"
|
|
|
|
when false:
|
|
proc setTraceLevel =
|
|
discard
|
|
when defined(chronicles_runtime_filtering) and loggingEnabled:
|
|
setLogLevel(LogLevel.TRACE)
|
|
|
|
proc setErrorLevel =
|
|
discard
|
|
when defined(chronicles_runtime_filtering) and loggingEnabled:
|
|
setLogLevel(LogLevel.ERROR)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc blockChainForTesting*(network: NetworkId): CommonRef =
|
|
result = CommonRef.new(
|
|
newCoreDbRef LegacyDbMemory,
|
|
networkId = network,
|
|
params = network.networkParams)
|
|
initializeEmptyDb(result)
|
|
|
|
proc importBlocks(com: CommonRef; h: seq[BlockHeader]; b: seq[BlockBody]) =
|
|
if com.newChain.persistBlocks(h,b).isErr:
|
|
raiseAssert "persistBlocks() failed at block #" & $h[0].blockNumber
|
|
|
|
proc getVmState(com: CommonRef; number: BlockNumber): BaseVMState =
|
|
BaseVMState.new(com.db.getBlockHeader(number), com)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Crash test function, finding out about how the transaction framework works ..
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc modBalance(ac: LedgerRef, address: EthAddress) =
|
|
## This function is crucial for profucing the crash. If must
|
|
## modify the balance so that the database gets written.
|
|
# ac.blindBalanceSetter(address)
|
|
ac.addBalance(address, 1.u256)
|
|
|
|
|
|
proc runTrial2ok(vmState: BaseVMState; inx: int) =
|
|
## Run two blocks, the first one with *rollback*.
|
|
let eAddr = txs[inx].getSender
|
|
|
|
block:
|
|
let accTx = vmState.stateDB.beginSavepoint
|
|
vmState.stateDB.modBalance(eAddr)
|
|
vmState.stateDB.rollback(accTx)
|
|
|
|
block:
|
|
let accTx = vmState.stateDB.beginSavepoint
|
|
vmState.stateDB.modBalance(eAddr)
|
|
vmState.stateDB.commit(accTx)
|
|
|
|
vmState.stateDB.persist()
|
|
|
|
|
|
proc runTrial3(vmState: BaseVMState; inx: int; rollback: bool) =
|
|
## Run three blocks, the second one optionally with *rollback*.
|
|
let eAddr = txs[inx].getSender
|
|
|
|
block:
|
|
let accTx = vmState.stateDB.beginSavepoint
|
|
vmState.stateDB.modBalance(eAddr)
|
|
vmState.stateDB.commit(accTx)
|
|
vmState.stateDB.persist()
|
|
|
|
block:
|
|
let accTx = vmState.stateDB.beginSavepoint
|
|
vmState.stateDB.modBalance(eAddr)
|
|
|
|
if rollback:
|
|
vmState.stateDB.rollback(accTx)
|
|
break
|
|
|
|
vmState.stateDB.commit(accTx)
|
|
vmState.stateDB.persist()
|
|
|
|
block:
|
|
let accTx = vmState.stateDB.beginSavepoint
|
|
vmState.stateDB.modBalance(eAddr)
|
|
vmState.stateDB.commit(accTx)
|
|
vmState.stateDB.persist()
|
|
|
|
|
|
proc runTrial3crash(vmState: BaseVMState; inx: int; noisy = false) =
|
|
## Run three blocks with extra db frames and *rollback*.
|
|
let eAddr = txs[inx].getSender
|
|
|
|
block:
|
|
let dbTx = xdb.beginTransaction()
|
|
|
|
block:
|
|
let accTx = vmState.stateDB.beginSavepoint
|
|
vmState.stateDB.modBalance(eAddr)
|
|
vmState.stateDB.commit(accTx)
|
|
vmState.stateDB.persist()
|
|
|
|
block:
|
|
let accTx = vmState.stateDB.beginSavepoint
|
|
vmState.stateDB.modBalance(eAddr)
|
|
vmState.stateDB.rollback(accTx)
|
|
|
|
# The following statement will cause a crash at the next `persist()` call.
|
|
dbTx.rollback()
|
|
|
|
# In order to survive without an exception in the next `persist()` call, the
|
|
# following function could be added to `db/ledger`:
|
|
#
|
|
# proc clobberRootHash*(ac: LedgerRef; root: KeccakHash; prune = true) =
|
|
# ac.trie = initAccountsTrie(ac.db, rootHash, prune)
|
|
#
|
|
# Then, beginning this very function `runTrial3crash()` with
|
|
#
|
|
# let stateRoot = vmState.stateDB.rootHash
|
|
#
|
|
# the survival statement would be to re-assign the state-root via
|
|
#
|
|
# vmState.stateDB.clobberRootHash(stateRoot)
|
|
#
|
|
# Also mind this comment from Andri:
|
|
#
|
|
# [..] but as a reminder, only reinit the ac.trie is not enough, you
|
|
# should consider the accounts in the cache too. if there is any accounts
|
|
# in the cache they must in sync with the new rootHash.
|
|
#
|
|
block:
|
|
let dbTx = xdb.beginTransaction()
|
|
|
|
block:
|
|
let accTx = vmState.stateDB.beginSavepoint
|
|
vmState.stateDB.modBalance(eAddr)
|
|
vmState.stateDB.commit(accTx)
|
|
|
|
try:
|
|
vmState.stateDB.persist()
|
|
except AssertionDefect as e:
|
|
if noisy:
|
|
let msg = e.msg.rsplit($DirSep,1)[^1]
|
|
echo &"*** runVmExec({eAddr.pp}): {e.name}: {msg}"
|
|
dbTx.dispose()
|
|
raise e
|
|
|
|
vmState.stateDB.persist()
|
|
|
|
dbTx.commit()
|
|
|
|
|
|
proc runTrial4(vmState: BaseVMState; inx: int; rollback: bool) =
|
|
## Like `runTrial3()` but with four blocks and extra db transaction frames.
|
|
let eAddr = txs[inx].getSender
|
|
|
|
block:
|
|
let dbTx = xdb.beginTransaction()
|
|
|
|
block:
|
|
let accTx = vmState.stateDB.beginSavepoint
|
|
vmState.stateDB.modBalance(eAddr)
|
|
vmState.stateDB.commit(accTx)
|
|
vmState.stateDB.persist()
|
|
|
|
block:
|
|
let accTx = vmState.stateDB.beginSavepoint
|
|
vmState.stateDB.modBalance(eAddr)
|
|
vmState.stateDB.commit(accTx)
|
|
vmState.stateDB.persist()
|
|
|
|
block:
|
|
let accTx = vmState.stateDB.beginSavepoint
|
|
vmState.stateDB.modBalance(eAddr)
|
|
|
|
if rollback:
|
|
vmState.stateDB.rollback(accTx)
|
|
break
|
|
|
|
vmState.stateDB.commit(accTx)
|
|
vmState.stateDB.persist()
|
|
|
|
# There must be no dbTx.rollback() here unless `vmState.stateDB` is
|
|
# discarded and/or re-initialised.
|
|
dbTx.commit()
|
|
|
|
block:
|
|
let dbTx = xdb.beginTransaction()
|
|
|
|
block:
|
|
let accTx = vmState.stateDB.beginSavepoint
|
|
vmState.stateDB.modBalance(eAddr)
|
|
vmState.stateDB.commit(accTx)
|
|
vmState.stateDB.persist()
|
|
|
|
dbTx.commit()
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Test Runner
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc runner(noisy = true; capture = goerliCapture) =
|
|
let
|
|
loadBlocks = capture.numBlocks.u256
|
|
loadTxs = capture.numTxs
|
|
fileInfo = capture.file.splitFile.name.split(".")[0]
|
|
filePath = capture.file.findFilePath
|
|
com = capture.network.blockChainForTesting
|
|
|
|
txs.reset
|
|
xdb = com.db
|
|
|
|
suite &"StateDB nesting scenarios":
|
|
var topNumber: BlockNumber
|
|
|
|
test &"Import from {fileInfo}":
|
|
# Import minimum amount of blocks, then collect transactions
|
|
for chain in filePath.undumpBlocks:
|
|
let leadBlkNum = chain[0].header.blockNumber
|
|
topNumber = chain[^1].header.blockNumber
|
|
|
|
if loadTxs <= txs.len:
|
|
break
|
|
|
|
# Verify Genesis
|
|
if leadBlkNum == 0.u256:
|
|
doAssert chain[0][0] == xdb.getBlockHeader(0.u256)
|
|
continue
|
|
|
|
# Import block chain blocks
|
|
if leadBlkNum < loadBlocks:
|
|
com.importBlocks(chain)
|
|
continue
|
|
|
|
# Import transactions
|
|
for inx in 0 ..< chain.len:
|
|
let blkTxs = chain[inx].transactions
|
|
|
|
# Continue importing up until first non-trivial block
|
|
if txs.len == 0 and blkTxs.len == 0:
|
|
com.importBlocks([chain[inx]])
|
|
continue
|
|
|
|
# Load transactions
|
|
txs.add blkTxs
|
|
|
|
|
|
test &"Collect unique sender addresses from {txs.len} txs," &
|
|
&" head=#{xdb.getCanonicalHead.blockNumber}, top=#{topNumber}":
|
|
var seen: Table[EthAddress,bool]
|
|
for n,tx in txs:
|
|
let a = tx.getSender
|
|
if not seen.hasKey(a):
|
|
seen[a] = true
|
|
txi.add n
|
|
|
|
test &"Run {txi.len} two-step trials with rollback":
|
|
let dbTx = xdb.beginTransaction()
|
|
defer: dbTx.dispose()
|
|
for n in txi:
|
|
let vmState = com.getVmState(xdb.getCanonicalHead.blockNumber)
|
|
vmState.runTrial2ok(n)
|
|
|
|
test &"Run {txi.len} three-step trials with rollback":
|
|
let dbTx = xdb.beginTransaction()
|
|
defer: dbTx.dispose()
|
|
for n in txi:
|
|
let vmState = com.getVmState(xdb.getCanonicalHead.blockNumber)
|
|
vmState.runTrial3(n, rollback = true)
|
|
|
|
test &"Run {txi.len} three-step trials with extra db frame rollback" &
|
|
" throwing Exceptions":
|
|
let dbTx = xdb.beginTransaction()
|
|
defer: dbTx.dispose()
|
|
for n in txi:
|
|
let vmState = com.getVmState(xdb.getCanonicalHead.blockNumber)
|
|
expect AssertionDefect:
|
|
vmState.runTrial3crash(n, noisy)
|
|
|
|
test &"Run {txi.len} tree-step trials without rollback":
|
|
let dbTx = xdb.beginTransaction()
|
|
defer: dbTx.dispose()
|
|
for n in txi:
|
|
let vmState = com.getVmState(xdb.getCanonicalHead.blockNumber)
|
|
vmState.runTrial3(n, rollback = false)
|
|
|
|
test &"Run {txi.len} four-step trials with rollback and db frames":
|
|
let dbTx = xdb.beginTransaction()
|
|
defer: dbTx.dispose()
|
|
for n in txi:
|
|
let vmState = com.getVmState(xdb.getCanonicalHead.blockNumber)
|
|
vmState.runTrial4(n, rollback = true)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Main function(s)
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc accountsCacheMain*(noisy = defined(debug)) =
|
|
noisy.runner
|
|
|
|
when isMainModule:
|
|
var noisy = defined(debug)
|
|
#noisy = true
|
|
|
|
setErrorLevel()
|
|
noisy.runner # mainCapture
|
|
# noisy.runner goerliCapture2
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|