mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-01-10 20:36:31 +00:00
1ffac5b2ea
details: Persistent pruning would not restore the `emptyRlp` value for the root node when the database becomes empty. This effects to an assertion exception next time the DB is accessed. As most unit tests run on the memory DB, this case slipped through unnoticed for a while (see also issue #9.)
301 lines
10 KiB
Nim
301 lines
10 KiB
Nim
# Nimbus - Types, data structures and shared utilities used in network sync
|
|
#
|
|
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or
|
|
# distributed except according to those terms.
|
|
|
|
## Snap sync components tester and TDD environment
|
|
|
|
import
|
|
std/[distros, os, strformat, strutils, tables],
|
|
chronicles,
|
|
eth/[common, p2p],
|
|
rocksdb,
|
|
unittest2,
|
|
../nimbus/db/core_db/persistent,
|
|
../nimbus/core/chain,
|
|
../nimbus/sync/snap/range_desc,
|
|
../nimbus/sync/snap/worker/db/hexary_desc,
|
|
./replay/pp,
|
|
./test_rocksdb_timing/[bulk_test_xx, test_db_timing]
|
|
|
|
const
|
|
baseDir = [".", "..", ".."/"..", $DirSep]
|
|
repoDir = [".", "tests"/"replay", "tests"/"test_sync_snap",
|
|
"nimbus-eth1-blobs"/"replay"]
|
|
|
|
# Reference file for finding the database directory
|
|
sampleDirRefFile = "sample0.txt.gz"
|
|
|
|
# Standard test samples
|
|
bChainCapture = bulkTest0
|
|
|
|
# Number of database slots (needed for timing tests)
|
|
nTestDbInstances = 9
|
|
|
|
type
|
|
TestDbs = object
|
|
## Provide enough spare empty databases
|
|
persistent: bool
|
|
dbDir: string
|
|
baseDir: string # for cleanup
|
|
subDir: string # for cleanup
|
|
cdb: array[nTestDbInstances,CoreDbRef]
|
|
|
|
when defined(linux):
|
|
# The `detectOs(Ubuntu)` directive is not Windows compatible, causes an
|
|
# error when running the system command `lsb_release -d` in the background.
|
|
let isUbuntu32bit = detectOs(Ubuntu) and int.sizeof == 4
|
|
else:
|
|
const isUbuntu32bit = false
|
|
|
|
let
|
|
# There was a problem with the Github/CI which results in spurious crashes
|
|
# when leaving the `runner()` if the persistent ChainDBRef initialisation
|
|
# was present, see `test_custom_network` for more details.
|
|
disablePersistentDB = isUbuntu32bit
|
|
|
|
var
|
|
xTmpDir: string
|
|
xDbs: TestDbs # for repeated storage/overwrite tests
|
|
xTab32: Table[ByteArray32,Blob] # extracted data
|
|
xTab33: Table[ByteArray33,Blob]
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc findFilePath(file: string;
|
|
baseDir, repoDir: openArray[string]): Result[string,void] =
|
|
for dir in baseDir:
|
|
for repo in repoDir:
|
|
let path = dir / repo / file
|
|
if path.fileExists:
|
|
return ok(path)
|
|
echo "*** File not found \"", file, "\"."
|
|
err()
|
|
|
|
proc getTmpDir(sampleDir = sampleDirRefFile): string =
|
|
sampleDir.findFilePath(baseDir,repoDir).value.splitFile.dir
|
|
|
|
proc setTraceLevel {.used.} =
|
|
discard
|
|
when defined(chronicles_runtime_filtering) and loggingEnabled:
|
|
setLogLevel(LogLevel.TRACE)
|
|
|
|
proc setErrorLevel {.used.} =
|
|
discard
|
|
when defined(chronicles_runtime_filtering) and loggingEnabled:
|
|
setLogLevel(LogLevel.ERROR)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc flushDbDir(s: string; subDir = "") =
|
|
if s != "":
|
|
let baseDir = s / "tmp"
|
|
for n in 0 ..< nTestDbInstances:
|
|
let instDir = if subDir == "": baseDir / $n else: baseDir / subDir / $n
|
|
if (instDir / "nimbus" / "data").dirExists:
|
|
# Typically under Windows: there might be stale file locks.
|
|
try: instDir.removeDir except CatchableError: discard
|
|
try: (baseDir / subDir).removeDir except CatchableError: discard
|
|
block dontClearUnlessEmpty:
|
|
for w in baseDir.walkDir:
|
|
break dontClearUnlessEmpty
|
|
try: baseDir.removeDir except CatchableError: discard
|
|
|
|
|
|
proc flushDbs(db: TestDbs) =
|
|
if db.persistent:
|
|
for n in 0 ..< nTestDbInstances:
|
|
if db.cdb[n].isNil or db.cdb[n].dbType != LegacyDbPersistent:
|
|
break
|
|
db.cdb[n].backend.toRocksStoreRef.store.db.rocksdb_close
|
|
db.baseDir.flushDbDir(db.subDir)
|
|
|
|
proc testDbs(
|
|
workDir: string;
|
|
subDir: string;
|
|
instances: int;
|
|
persistent: bool;
|
|
): TestDbs =
|
|
if disablePersistentDB or workDir == "" or not persistent:
|
|
result.persistent = false
|
|
result.dbDir = "*notused*"
|
|
else:
|
|
result.persistent = true
|
|
result.baseDir = workDir
|
|
result.subDir = subDir
|
|
if subDir != "":
|
|
result.dbDir = workDir / "tmp" / subDir
|
|
else:
|
|
result.dbDir = workDir / "tmp"
|
|
if result.persistent:
|
|
workDir.flushDbDir(subDir)
|
|
for n in 0 ..< min(result.cdb.len, instances):
|
|
result.cdb[n] = newCoreDbRef(LegacyDbPersistent, result.dbDir / $n)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Test Runners: database timing tests
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc importRunner(noisy = true; persistent = true; capture = bChainCapture) =
|
|
let
|
|
fileInfo = capture.file.splitFile.name.split(".")[0]
|
|
filePath = capture.file.findFilePath(baseDir,repoDir).value
|
|
tmpDir = getTmpDir()
|
|
db = tmpDir.testDbs(capture.name & "-import", instances=1, persistent)
|
|
numBlocks = capture.numBlocks
|
|
numBlocksInfo = if numBlocks == high(int): "" else: $numBlocks & " "
|
|
loadNoise = noisy
|
|
|
|
defer:
|
|
db.flushDbs
|
|
|
|
suite &"RocksDB: using {fileInfo} capture for testing db timings":
|
|
var ddb: CommonRef # perstent DB on disk
|
|
|
|
test &"Create persistent ChainDBRef on {tmpDir}":
|
|
ddb = CommonRef.new(
|
|
db = if db.persistent: db.cdb[0] else: newCoreDbRef(LegacyDbMemory),
|
|
networkId = capture.network,
|
|
pruneTrie = true,
|
|
params = capture.network.networkParams)
|
|
ddb.initializeEmptyDb
|
|
|
|
test &"Storing {numBlocksInfo}persistent blocks from dump":
|
|
noisy.test_dbTimingUndumpBlocks(filePath, ddb, numBlocks, loadNoise)
|
|
|
|
test "Extract key-value records into memory tables via rocksdb iterator":
|
|
if db.cdb[0].backend.toRocksStoreRef.isNil:
|
|
skip() # not persistent => db.cdb[0] is nil
|
|
else:
|
|
noisy.test_dbTimingRockySetup(xTab32, xTab33, db.cdb[0])
|
|
|
|
|
|
proc dbTimingRunner(noisy = true; persistent = true; cleanUp = true) =
|
|
let
|
|
fullNoise = false
|
|
var
|
|
emptyDb = "empty"
|
|
|
|
# Allows to repeat storing on existing data
|
|
if not xDbs.cdb[0].isNil:
|
|
emptyDb = "pre-loaded"
|
|
else:
|
|
xTmpDir = getTmpDir()
|
|
xDbs = xTmpDir.testDbs(
|
|
"timing-runner", instances=nTestDbInstances, persistent)
|
|
|
|
defer:
|
|
if cleanUp:
|
|
xDbs.flushDbs
|
|
xDbs.reset
|
|
|
|
suite &"RocksDB: storage tests on {emptyDb} databases":
|
|
#
|
|
# `xDbs` instance slots layout:
|
|
#
|
|
# * cdb[0] -- direct db, key length 32, no transaction
|
|
# * cdb[1] -- direct db, key length 32 as 33, no transaction
|
|
#
|
|
# * cdb[2] -- direct db, key length 32, transaction based
|
|
# * cdb[3] -- direct db, key length 32 as 33, transaction based
|
|
#
|
|
# * cdb[4] -- direct db, key length 33, no transaction
|
|
# * cdb[5] -- direct db, key length 33, transaction based
|
|
#
|
|
# * cdb[6] -- rocksdb, key length 32
|
|
# * cdb[7] -- rocksdb, key length 32 as 33
|
|
# * cdb[8] -- rocksdb, key length 33
|
|
#
|
|
doAssert 9 <= nTestDbInstances
|
|
doAssert not xDbs.cdb[8].isNil
|
|
|
|
let
|
|
storeDir32 = &"Directly store {xTab32.len} records"
|
|
storeDir33 = &"Directly store {xTab33.len} records"
|
|
storeTx32 = &"Transactionally store directly {xTab32.len} records"
|
|
storeTx33 = &"Transactionally store directly {xTab33.len} records"
|
|
intoTrieDb = &"into {emptyDb} trie db"
|
|
|
|
storeRks32 = &"Store {xTab32.len} records"
|
|
storeRks33 = &"Store {xTab33.len} records"
|
|
intoRksDb = &"into {emptyDb} rocksdb table"
|
|
|
|
if xTab32.len == 0 or xTab33.len == 0:
|
|
test &"Both tables with 32 byte keys(size={xTab32.len}), " &
|
|
&"33 byte keys(size={xTab32.len}) must be non-empty":
|
|
skip()
|
|
else:
|
|
test &"{storeDir32} (key length 32) {intoTrieDb}":
|
|
noisy.test_dbTimingStoreDirect32(xTab32, xDbs.cdb[0])
|
|
|
|
test &"{storeDir32} (key length 33) {intoTrieDb}":
|
|
noisy.test_dbTimingStoreDirectly32as33(xTab32, xDbs.cdb[1])
|
|
|
|
test &"{storeTx32} (key length 32) {intoTrieDb}":
|
|
noisy.test_dbTimingStoreTx32(xTab32, xDbs.cdb[2])
|
|
|
|
test &"{storeTx32} (key length 33) {intoTrieDb}":
|
|
noisy.test_dbTimingStoreTx32as33(xTab32, xDbs.cdb[3])
|
|
|
|
test &"{storeDir33} (key length 33) {intoTrieDb}":
|
|
noisy.test_dbTimingDirect33(xTab33, xDbs.cdb[4])
|
|
|
|
test &"{storeTx33} (key length 33) {intoTrieDb}":
|
|
noisy.test_dbTimingTx33(xTab33, xDbs.cdb[5])
|
|
|
|
if xDbs.cdb[0].backend.toRocksStoreRef.isNil:
|
|
test "The rocksdb interface must be available": skip()
|
|
else:
|
|
test &"{storeRks32} (key length 32) {intoRksDb}":
|
|
noisy.test_dbTimingRocky32(xTab32, xDbs.cdb[6], fullNoise)
|
|
|
|
test &"{storeRks32} (key length 33) {intoRksDb}":
|
|
noisy.test_dbTimingRocky32as33(xTab32, xDbs.cdb[7], fullNoise)
|
|
|
|
test &"{storeRks33} (key length 33) {intoRksDb}":
|
|
noisy.test_dbTimingRocky33(xTab33, xDbs.cdb[8], fullNoise)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Main function(s)
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc rocksDbTimingMain*(noisy = defined(debug)) =
|
|
noisy.importRunner() # small sample, just verify functionality
|
|
noisy.dbTimingRunner()
|
|
|
|
when isMainModule:
|
|
const
|
|
noisy = defined(debug) or true
|
|
|
|
#setTraceLevel()
|
|
setErrorLevel()
|
|
|
|
# This one uses the readily available dump: `bulkTest0` and some huge replay
|
|
# dumps `bulkTest2`, `bulkTest3`, .. from the `nimbus-eth1-blobs` package.
|
|
# For specs see `tests/test_rocksdb_timing/bulk_test_xx.nim`.
|
|
var testList = @[bulkTest0]
|
|
when true and false:
|
|
testList &= @[bulkTest1, bulkTest2, bulkTest3]
|
|
|
|
for test in testList:
|
|
noisy.showElapsed("importRunner()"):
|
|
noisy.importRunner(capture = test)
|
|
|
|
noisy.showElapsed("dbTimingRunner()"):
|
|
true.dbTimingRunner(cleanUp = false)
|
|
true.dbTimingRunner()
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|