mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-03-01 12:20:49 +00:00
6dc2773957
* Normalised storage tree addressing in function prototypes detail: Argument list is always `<db> <account-path> <slot-path> ..` with both path arguments as `openArray[]` * Remove cruft * CoreDb internally Use full account paths rather than addresses * Update API logging * Use hashed account address only in prototypes why: This avoids unnecessary repeated hashing of the same account address. The burden of doing that is upon the application. In the case here, the ledger caches all kinds of stuff anyway so it is common sense to exploit that for account address hashes. caveat: Using `openArray[byte]` argument types for hashed accounts is inherently fragile. In non-release mode, a length verification `doAssert` is enabled by default. * No accPath in data record (use `AristoAccount` as `CoreDbAccount`) * Remove now unused `eAddr` field from ledger `AccountRef` type why: Is duplicate of lookup key * Avoid merging the account record/statement in the ledger twice.
370 lines
11 KiB
Nim
370 lines
11 KiB
Nim
# Nimbus
|
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or
|
|
# distributed except according to those terms.
|
|
|
|
## Testing `CoreDB` wrapper implementation
|
|
|
|
import
|
|
std/[os, strformat, strutils],
|
|
chronicles,
|
|
eth/common,
|
|
results,
|
|
unittest2,
|
|
../nimbus/db/opts,
|
|
../nimbus/db/core_db/persistent,
|
|
../nimbus/core/chain,
|
|
./replay/pp,
|
|
./test_coredb/[
|
|
coredb_test_xx, test_chainsync, test_coredb_helpers, test_helpers]
|
|
|
|
const
|
|
# If `true`, this compile time option set up `unittest2` for manual parsing
|
|
unittest2DisableParamFiltering {.booldefine.} = false
|
|
|
|
baseDir = [".", "..", ".."/"..", $DirSep]
|
|
repoDir = [".", "tests", "nimbus-eth1-blobs"]
|
|
subDir = ["replay", "test_coredb", "custom-network", "main-era1"]
|
|
|
|
# Reference file for finding some database directory base
|
|
sampleDirRefFile = "coredb_test_xx.nim"
|
|
|
|
dbTypeDefault = AristoDbMemory
|
|
|
|
let
|
|
# Standard test sample
|
|
memorySampleDefault = mainTest0m
|
|
persistentSampleDefault = mainTest2r
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
when unittest2DisableParamFiltering:
|
|
import algorithm
|
|
|
|
# Filter out local options and pass on the rest to `unittest2`
|
|
proc cmdLineConfig(): tuple[samples: seq[CaptureSpecs]] {.used.} =
|
|
## This helper allows to pass additional command line options to the
|
|
## unit test.
|
|
##
|
|
## Example:
|
|
## ::
|
|
## nim c -r ...\
|
|
## -d:unittest2DisableParamFiltering \
|
|
## ./tests/test_coredb.nim \
|
|
## --output-level=VERBOSE \
|
|
## --sample=goerli-lp,goerli-ar
|
|
## or
|
|
## ::
|
|
## nim c ... -d:unittest2DisableParamFiltering ./tests/test_coredb.nim
|
|
## ./tests/test_coredb.out --output-level=VERBOSE --sample=goerli-ar
|
|
## ...
|
|
##
|
|
## At the moment, only the `--sample=` additional option is provided.
|
|
##
|
|
# Define sample list from the command line (if any)
|
|
const optPfx = "--sample=" # Custom option with sample list
|
|
|
|
proc parseError(s = "") =
|
|
let msg = if 0 < s.len: "Unsupported \"" & optPfx & "\" list item: " & s
|
|
else: "Empty \"" & optPfx & " list"
|
|
echo "*** ", getAppFilename().splitFile.name, ": ", msg
|
|
echo " Available: ", allSamples.mapIt(it.name).sorted.join(" ")
|
|
quit(99)
|
|
|
|
var other: seq[string] # Options for manual parsing by `unittest2`
|
|
|
|
for arg in commandLineParams():
|
|
if optPfx.len <= arg.len and arg[0 ..< optPfx.len] == optPfx:
|
|
for w in arg[optPfx.len ..< arg.len].split(",").mapIt(it.strip):
|
|
block findSample:
|
|
for sample in allSamples:
|
|
if w.cmpIgnoreCase(sample.name) == 0:
|
|
result.samples.add sample
|
|
break findSample
|
|
w.parseError()
|
|
if result.samples.len == 0:
|
|
parseError()
|
|
else:
|
|
other.add arg
|
|
|
|
# Setup `unittest2`
|
|
other.parseParameters
|
|
|
|
else:
|
|
# Kill the compilation process iff the directive `cmdLineConfig()` is used
|
|
template cmdLineConfig(): untyped {.used.} =
|
|
{.error: "cmdLineConfig() needs compiler option "&
|
|
" -d:unittest2DisableParamFiltering".}
|
|
|
|
|
|
proc findFilePath(
|
|
file: string;
|
|
baseDir: openArray[string] = baseDir;
|
|
repoDir: openArray[string] = repoDir;
|
|
subDir: openArray[string] = subDir;
|
|
): Result[string,void] =
|
|
file.findFilePathHelper(baseDir, repoDir, subDir)
|
|
|
|
|
|
proc getTmpDir(sampleDir = sampleDirRefFile): string =
|
|
sampleDir.findFilePath.value.splitFile.dir
|
|
|
|
|
|
proc flushDbDir(s: string) =
|
|
if s != "":
|
|
let dataDir = s / "nimbus"
|
|
if (dataDir / "data").dirExists:
|
|
# Typically under Windows: there might be stale file locks.
|
|
try: dataDir.removeDir except CatchableError: discard
|
|
block dontClearUnlessEmpty:
|
|
for w in s.walkDir:
|
|
break dontClearUnlessEmpty
|
|
try: s.removeDir except CatchableError: discard
|
|
|
|
# ----------------
|
|
|
|
proc setTraceLevel {.used.} =
|
|
discard
|
|
when defined(chronicles_runtime_filtering) and loggingEnabled:
|
|
setLogLevel(LogLevel.TRACE)
|
|
|
|
proc setDebugLevel {.used.} =
|
|
discard
|
|
when defined(chronicles_runtime_filtering) and loggingEnabled:
|
|
setLogLevel(LogLevel.DEBUG)
|
|
|
|
proc setErrorLevel {.used.} =
|
|
discard
|
|
when defined(chronicles_runtime_filtering) and loggingEnabled:
|
|
setLogLevel(LogLevel.ERROR)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc initRunnerDB(
|
|
path: string;
|
|
specs: CaptureSpecs;
|
|
dbType: CdbTypeEx;
|
|
pruneHistory: bool;
|
|
): CommonRef =
|
|
let coreDB =
|
|
# Resolve for static `dbType`
|
|
case dbType:
|
|
of CdbAristoMemory: AristoDbMemory.newCoreDbRef()
|
|
of CdbAristoRocks: AristoDbRocks.newCoreDbRef(path, DbOptions.init())
|
|
of CdbAristoDualRocks: newCdbAriAristoDualRocks(path, DbOptions.init())
|
|
of CdbAristoVoid: AristoDbVoid.newCoreDbRef()
|
|
of CdbOoops: raiseAssert "Ooops"
|
|
|
|
when false: # or true:
|
|
setDebugLevel()
|
|
coreDB.trackLegaApi = true
|
|
coreDB.trackNewApi = true
|
|
|
|
var
|
|
params: NetworkParams
|
|
networkId: NetworkId
|
|
if specs.builtIn:
|
|
networkId = specs.network
|
|
params = networkId.networkParams()
|
|
else:
|
|
doAssert specs.genesis.findFilePath.value.loadNetworkParams(params)
|
|
networkId = params.config.chainId.NetworkId
|
|
|
|
result = CommonRef.new(
|
|
db = coreDB,
|
|
networkId = networkId,
|
|
params = params,
|
|
pruneHistory = pruneHistory)
|
|
|
|
result.initializeEmptyDb
|
|
|
|
setErrorLevel()
|
|
coreDB.trackNewApi = false
|
|
coreDB.trackLedgerApi =false
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Test Runners: accounts and accounts storages
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc chainSyncRunner(
|
|
noisy = true;
|
|
capture = memorySampleDefault;
|
|
dbType = CdbTypeEx(0);
|
|
pruneHistory = false;
|
|
profilingOk = false;
|
|
finalDiskCleanUpOk = true;
|
|
enaLoggingOk = false;
|
|
lastOneExtraOk = true;
|
|
oldLogAlign = false;
|
|
) =
|
|
|
|
## Test backend database and ledger
|
|
let
|
|
fileInfo = capture.files[0]
|
|
.splitFile.name.split(".")[0]
|
|
.strip(leading=false, chars={'0'..'9'})
|
|
filePaths = capture.files.mapIt(it.findFilePath(baseDir,repoDir).value)
|
|
baseDir = getTmpDir() / capture.dbName & "-chain-sync"
|
|
dbDir = baseDir / "tmp"
|
|
numBlocks = capture.numBlocks
|
|
numBlocksInfo = if numBlocks == high(int): "all" else: $numBlocks
|
|
|
|
dbType = block:
|
|
# Decreasing priority: dbType, capture.dbType, dbTypeDefault
|
|
var effDbType = dbTypeDefault.to(CdbTypeEx)
|
|
if dbType != CdbTypeEx(0):
|
|
effDbType = dbType
|
|
elif capture.dbType != CoreDbType(0):
|
|
effDbType = capture.dbType.to(CdbTypeEx)
|
|
effDbType
|
|
|
|
persistent = dbType in CdbTypeExPersistent
|
|
|
|
defer:
|
|
if persistent: baseDir.flushDbDir
|
|
|
|
suite &"CoreDB and LedgerRef API on {fileInfo}, {dbType}":
|
|
|
|
test &"Ledger API {numBlocksInfo} blocks":
|
|
let
|
|
com = initRunnerDB(dbDir, capture, dbType, pruneHistory)
|
|
defer:
|
|
com.db.finish(eradicate = finalDiskCleanUpOk)
|
|
if profilingOk: noisy.test_chainSyncProfilingPrint numBlocks
|
|
if persistent and finalDiskCleanUpOk: dbDir.flushDbDir
|
|
|
|
if noisy:
|
|
com.db.trackNewApi = true
|
|
com.db.trackNewApi = true
|
|
com.db.trackLedgerApi = true
|
|
|
|
check noisy.test_chainSync(filePaths, com, numBlocks,
|
|
lastOneExtra=lastOneExtraOk, enaLogging=enaLoggingOk,
|
|
oldLogAlign=oldLogAlign)
|
|
|
|
|
|
proc persistentSyncPreLoadAndResumeRunner(
|
|
noisy = true;
|
|
capture = persistentSampleDefault;
|
|
dbType = CdbTypeEx(0);
|
|
profilingOk = false;
|
|
pruneHistory = false;
|
|
finalDiskCleanUpOk = true;
|
|
enaLoggingOk = false;
|
|
lastOneExtraOk = true;
|
|
oldLogAlign = false;
|
|
) =
|
|
## Test backend database and ledger
|
|
let
|
|
filePaths = capture.files.mapIt(it.findFilePath(baseDir,repoDir).value)
|
|
baseDir = getTmpDir() / capture.dbName & "-chain-sync"
|
|
dbDir = baseDir / "tmp"
|
|
|
|
dbType = block:
|
|
# Decreasing priority: dbType, capture.dbType, dbTypeDefault
|
|
var effDbType = dbTypeDefault.to(CdbTypeEx)
|
|
if dbType != CdbTypeEx(0):
|
|
effDbType = dbType
|
|
elif capture.dbType != CoreDbType(0):
|
|
effDbType = capture.dbType.to(CdbTypeEx)
|
|
effDbType
|
|
|
|
doAssert dbType in CdbTypeExPersistent
|
|
defer: baseDir.flushDbDir
|
|
|
|
let
|
|
firstPart = min(capture.numBlocks div 2, 200_000)
|
|
secndPart = capture.numBlocks
|
|
secndPartInfo = if secndPart == high(int): "all" else: $secndPart
|
|
|
|
suite &"CoreDB pre-load and resume test ..{firstPart}..{secndPartInfo}":
|
|
|
|
test "Populate db by initial sample parts":
|
|
let
|
|
com = initRunnerDB(dbDir, capture, dbType, pruneHistory)
|
|
defer:
|
|
com.db.finish(eradicate = finalDiskCleanUpOk)
|
|
if profilingOk: noisy.test_chainSyncProfilingPrint firstPart
|
|
|
|
if noisy:
|
|
com.db.trackNewApi = true
|
|
com.db.trackNewApi = true
|
|
com.db.trackLedgerApi = true
|
|
|
|
check noisy.test_chainSync(filePaths, com, firstPart,
|
|
lastOneExtra=lastOneExtraOk, enaLogging=enaLoggingOk,
|
|
oldLogAlign=oldLogAlign)
|
|
|
|
test &"Continue with rest of sample":
|
|
let
|
|
com = initRunnerDB(dbDir, capture, dbType, pruneHistory)
|
|
defer:
|
|
com.db.finish(eradicate = finalDiskCleanUpOk)
|
|
if profilingOk: noisy.test_chainSyncProfilingPrint secndPart
|
|
if finalDiskCleanUpOk: dbDir.flushDbDir
|
|
|
|
if noisy:
|
|
com.db.trackNewApi = true
|
|
com.db.trackNewApi = true
|
|
com.db.trackLedgerApi = true
|
|
|
|
check noisy.test_chainSync(filePaths, com, secndPart,
|
|
lastOneExtra=lastOneExtraOk, enaLogging=enaLoggingOk,
|
|
oldLogAlign=oldLogAlign)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Main function(s)
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc coreDbMain*(noisy = defined(debug)) =
|
|
noisy.chainSyncRunner()
|
|
noisy.persistentSyncPreLoadAndResumeRunner()
|
|
|
|
when isMainModule:
|
|
const
|
|
noisy {.used.} = defined(debug) or true
|
|
var
|
|
sampleList: seq[CaptureSpecs]
|
|
|
|
setErrorLevel()
|
|
|
|
when true and false:
|
|
false.coreDbMain()
|
|
|
|
# This one uses the readily available dump: `bulkTest0` and some huge replay
|
|
# dumps `bulkTest2`, `bulkTest3`, .. from the `nimbus-eth1-blobs` package.
|
|
# For specs see `tests/test_coredb/bulk_test_xx.nim`.
|
|
|
|
sampleList = cmdLineConfig().samples
|
|
if sampleList.len == 0:
|
|
sampleList = @[memorySampleDefault]
|
|
|
|
when true: # and false:
|
|
import std/times
|
|
var state: (Duration, int)
|
|
for n,capture in sampleList:
|
|
noisy.profileSection("@sample #" & $n, state):
|
|
noisy.chainSyncRunner(
|
|
#dbType = CdbAristoDualRocks,
|
|
capture = capture,
|
|
pruneHistory = true,
|
|
#profilingOk = true,
|
|
#finalDiskCleanUpOk = false,
|
|
oldLogAlign = true
|
|
)
|
|
|
|
noisy.say "***", "total: ", state[0].pp, " sections: ", state[1]
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|