314 lines
10 KiB
Nim
314 lines
10 KiB
Nim
# Nimbus - Types, data structures and shared utilities used in network sync
|
|
#
|
|
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or
|
|
# distributed except according to those terms.
|
|
|
|
## Re-invented implementation for Merkle Patricia Tree named as Aristo Trie
|
|
|
|
import
|
|
std/[os, strformat, strutils],
|
|
chronicles,
|
|
eth/[common, p2p],
|
|
rocksdb,
|
|
unittest2,
|
|
../nimbus/db/aristo/[aristo_desc, aristo_merge],
|
|
../nimbus/db/core_db,
|
|
../nimbus/db/core_db/[legacy_persistent, persistent],
|
|
../nimbus/core/chain,
|
|
../nimbus/sync/snap/worker/db/[rocky_bulk_load, snapdb_accounts, snapdb_desc],
|
|
./replay/[pp, undump_accounts, undump_storages],
|
|
./test_sync_snap/[snap_test_xx, test_accounts, test_types],
|
|
./test_aristo/[test_backend, test_helpers, test_transcode, test_tx]
|
|
|
|
const
|
|
baseDir = [".", "..", ".."/"..", $DirSep]
|
|
repoDir = [".", "tests", "nimbus-eth1-blobs"]
|
|
subDir = ["replay", "test_sync_snap", "replay"/"snap"]
|
|
|
|
# Reference file for finding the database directory
|
|
sampleDirRefFile = "sample0.txt.gz"
|
|
|
|
# Standard test samples
|
|
accSample = snapTest0
|
|
storSample = snapTest4
|
|
|
|
# Number of database slots available
|
|
nTestDbInstances = 9
|
|
|
|
type
|
|
TestDbs = object
|
|
## Provide enough spare empty databases
|
|
persistent: bool
|
|
dbDir: string
|
|
baseDir: string # for cleanup
|
|
subDir: string # for cleanup
|
|
cdb: array[nTestDbInstances,CoreDbRef]
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc findFilePath(
|
|
file: string;
|
|
baseDir: openArray[string] = baseDir;
|
|
repoDir: openArray[string] = repoDir;
|
|
subDir: openArray[string] = subDir;
|
|
): Result[string,void] =
|
|
for dir in baseDir:
|
|
if dir.dirExists:
|
|
for repo in repoDir:
|
|
if (dir / repo).dirExists:
|
|
for sub in subDir:
|
|
if (dir / repo / sub).dirExists:
|
|
let path = dir / repo / sub / file
|
|
if path.fileExists:
|
|
return ok(path)
|
|
echo "*** File not found \"", file, "\"."
|
|
err()
|
|
|
|
proc getTmpDir(sampleDir = sampleDirRefFile): string =
|
|
sampleDir.findFilePath.value.splitFile.dir
|
|
|
|
proc setTraceLevel {.used.} =
|
|
discard
|
|
when defined(chronicles_runtime_filtering) and loggingEnabled:
|
|
setLogLevel(LogLevel.TRACE)
|
|
|
|
proc setErrorLevel {.used.} =
|
|
discard
|
|
when defined(chronicles_runtime_filtering) and loggingEnabled:
|
|
setLogLevel(LogLevel.ERROR)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc flushDbDir(s: string; subDir = "") =
|
|
if s != "":
|
|
let baseDir = s / "tmp"
|
|
for n in 0 ..< nTestDbInstances:
|
|
let instDir = if subDir == "": baseDir / $n else: baseDir / subDir / $n
|
|
if (instDir / "nimbus" / "data").dirExists:
|
|
# Typically under Windows: there might be stale file locks.
|
|
try: instDir.removeDir except CatchableError: discard
|
|
try: (baseDir / subDir).removeDir except CatchableError: discard
|
|
block dontClearUnlessEmpty:
|
|
for w in baseDir.walkDir:
|
|
break dontClearUnlessEmpty
|
|
try: baseDir.removeDir except CatchableError: discard
|
|
|
|
|
|
proc flushDbs(db: TestDbs) =
|
|
if db.persistent:
|
|
for n in 0 ..< nTestDbInstances:
|
|
if db.cdb[n].isNil or db.cdb[n].dbType != LegacyDbPersistent:
|
|
break
|
|
db.cdb[n].toLegacyBackend.rocksStoreRef.store.db.rocksdb_close
|
|
db.baseDir.flushDbDir(db.subDir)
|
|
|
|
proc testDbs(
|
|
workDir: string;
|
|
subDir: string;
|
|
instances: int;
|
|
persistent: bool;
|
|
): TestDbs =
|
|
if workDir == "" or not persistent:
|
|
result.persistent = false
|
|
result.dbDir = "*notused*"
|
|
else:
|
|
result.persistent = true
|
|
result.baseDir = workDir
|
|
result.subDir = subDir
|
|
if subDir != "":
|
|
result.dbDir = workDir / "tmp" / subDir
|
|
else:
|
|
result.dbDir = workDir / "tmp"
|
|
if result.persistent:
|
|
workDir.flushDbDir(subDir)
|
|
for n in 0 ..< min(result.cdb.len, instances):
|
|
result.cdb[n] = newCoreDbRef(LegacyDbPersistent, result.dbDir / $n)
|
|
|
|
proc snapDbRef(cdb: CoreDbRef; pers: bool): SnapDbRef =
|
|
if pers: SnapDbRef.init(cdb)
|
|
else: SnapDbRef.init(newCoreDbRef LegacyDbMemory)
|
|
|
|
proc snapDbAccountsRef(
|
|
cdb: CoreDbRef;
|
|
root: Hash256;
|
|
pers: bool;
|
|
): SnapDbAccountsRef =
|
|
SnapDbAccountsRef.init(cdb.snapDbRef(pers), root, Peer())
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Test Runners: accounts and accounts storages
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc miscRunner(noisy =true) =
|
|
suite &"Aristo: Miscellaneous tests":
|
|
test &"VertexID recyling lists":
|
|
noisy.test_transcodeVidRecycleLists()
|
|
|
|
|
|
proc transcodeRunner(noisy =true; sample=accSample; stopAfter=high(int)) =
|
|
let
|
|
accLst = sample.to(seq[UndumpAccounts])
|
|
root = accLst[0].root
|
|
tmpDir = getTmpDir()
|
|
db = tmpDir.testDbs(sample.name&"-transcode", instances=2, persistent=true)
|
|
info = if db.persistent: &"persistent db on \"{db.baseDir}\""
|
|
else: "in-memory db"
|
|
fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")
|
|
|
|
defer:
|
|
db.flushDbs
|
|
|
|
suite &"Aristo: transcoding {fileInfo} accounts for {info}":
|
|
|
|
# New common descriptor for this sub-group of tests
|
|
let
|
|
desc = db.cdb[0].snapDbAccountsRef(root, db.persistent)
|
|
hexaDb = desc.hexaDb
|
|
getFn = desc.getAccountFn
|
|
dbg = if noisy: hexaDb else: nil
|
|
|
|
# Borrowed from `test_sync_snap/test_accounts.nim`
|
|
test &"Importing {accLst.len} list items to persistent database":
|
|
if db.persistent:
|
|
accLst.test_accountsImport(desc, true)
|
|
else:
|
|
skip()
|
|
|
|
test "Trancoding database records: RLP, NodeRef, Blob, VertexRef":
|
|
noisy.showElapsed("test_transcoder()"):
|
|
noisy.test_transcodeAccounts(
|
|
db.cdb[0].toLegacyBackend.rocksStoreRef, stopAfter)
|
|
|
|
|
|
proc accountsRunner(
|
|
noisy = true;
|
|
sample = accSample;
|
|
resetDb = false;
|
|
cmpBackends = true;
|
|
) =
|
|
let
|
|
accLst = sample.to(seq[UndumpAccounts]).to(seq[ProofTrieData])
|
|
fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")
|
|
listMode = if resetDb: "" else: ", merged data lists"
|
|
baseDir = getTmpDir() / sample.name & "-accounts"
|
|
dbDir = baseDir / "tmp"
|
|
|
|
defer:
|
|
try: baseDir.removeDir except CatchableError: discard
|
|
|
|
suite &"Aristo: accounts data dump from {fileInfo}{listMode}":
|
|
|
|
test &"Merge {accLst.len} proof & account lists to database":
|
|
check noisy.testTxMergeProofAndKvpList(accLst, dbDir, resetDb)
|
|
|
|
test &"Compare {accLst.len} account lists on database backends":
|
|
if cmpBackends:
|
|
check noisy.testBackendConsistency(accLst, dbDir, resetDb)
|
|
else:
|
|
skip()
|
|
|
|
test &"Delete accounts database, successively {accLst.len} entries":
|
|
check noisy.testTxMergeAndDelete(accLst, dbDir)
|
|
|
|
|
|
proc storagesRunner(
|
|
noisy = true;
|
|
sample = storSample;
|
|
resetDb = false;
|
|
oops: KnownHasherFailure = @[];
|
|
cmpBackends = true;
|
|
) =
|
|
let
|
|
stoLst = sample.to(seq[UndumpStorages]).to(seq[ProofTrieData])
|
|
fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")
|
|
listMode = if resetDb: "" else: ", merged data lists"
|
|
baseDir = getTmpDir() / sample.name & "-storage"
|
|
dbDir = baseDir / "tmp"
|
|
|
|
defer:
|
|
try: baseDir.removeDir except CatchableError: discard
|
|
|
|
suite &"Aristo: storages data dump from {fileInfo}{listMode}":
|
|
|
|
test &"Merge {stoLst.len} proof & slots lists to database":
|
|
check noisy.testTxMergeProofAndKvpList(
|
|
stoLst, dbDir, resetDb, fileInfo, oops)
|
|
|
|
test &"Compare {stoLst.len} slot lists on database backends":
|
|
if cmpBackends:
|
|
check noisy.testBackendConsistency(stoLst, dbDir, resetDb)
|
|
else:
|
|
skip()
|
|
|
|
test &"Delete storage database, successively {stoLst.len} entries":
|
|
check noisy.testTxMergeAndDelete(stoLst, dbDir)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Main function(s)
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc aristoMain*(noisy = defined(debug)) =
|
|
noisy.miscRunner()
|
|
noisy.transcodeRunner()
|
|
noisy.accountsRunner()
|
|
noisy.storagesRunner()
|
|
|
|
when isMainModule:
|
|
const
|
|
noisy = defined(debug) or true
|
|
|
|
setErrorLevel()
|
|
|
|
when true: # and false:
|
|
noisy.miscRunner()
|
|
|
|
# Borrowed from `test_sync_snap.nim`
|
|
when true: # and false:
|
|
for n,sam in snapTestList:
|
|
noisy.transcodeRunner(sam)
|
|
for n,sam in snapTestStorageList:
|
|
noisy.transcodeRunner(sam)
|
|
|
|
# This one uses dumps from the external `nimbus-eth1-blob` repo
|
|
when true and false:
|
|
import ./test_sync_snap/snap_other_xx
|
|
noisy.showElapsed("@snap_other_xx"):
|
|
for n,sam in snapOtherList:
|
|
noisy.accountsRunner(sam, resetDb=true)
|
|
|
|
# This one usues dumps from the external `nimbus-eth1-blob` repo
|
|
when true and false:
|
|
import ./test_sync_snap/snap_storage_xx
|
|
let knownFailures: KnownHasherFailure = @[
|
|
("storages3__18__25_dump#12.27367",(3,HashifyExistingHashMismatch)),
|
|
("storages4__26__33_dump#12.23924",(6,HashifyExistingHashMismatch)),
|
|
("storages5__34__41_dump#10.20512",(1,HashifyRootHashMismatch)),
|
|
("storagesB__84__92_dump#7.9709", (7,HashifyExistingHashMismatch)),
|
|
("storagesD_102_109_dump#18.28287",(9,HashifyExistingHashMismatch)),
|
|
]
|
|
noisy.showElapsed("@snap_storage_xx"):
|
|
for n,sam in snapStorageList:
|
|
noisy.accountsRunner(sam, resetDb=true)
|
|
noisy.storagesRunner(sam, resetDb=true, oops=knownFailures)
|
|
|
|
when true: # and false:
|
|
for n,sam in snapTestList:
|
|
noisy.accountsRunner(sam)
|
|
for n,sam in snapTestStorageList:
|
|
noisy.accountsRunner(sam)
|
|
noisy.storagesRunner(sam)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|