mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-01-25 11:39:09 +00:00
93a72025a1
why: For the main tree with root vertex ID 1, the leaf nodes hold the account data. These accounts may link to sub trees the storage root node ID of which must be registered here. There is no reverse key lookup on the backend. note: These definitions are experimental. Also, there are some tests missing for validating Payload data conversions.
310 lines
10 KiB
Nim
310 lines
10 KiB
Nim
# Nimbus - Types, data structures and shared utilities used in network sync
|
|
#
|
|
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or
|
|
# distributed except according to those terms.
|
|
|
|
## Re-invented implementation for Merkle Patricia Tree named as Aristo Trie
|
|
|
|
import
|
|
std/[os, strformat, strutils],
|
|
chronicles,
|
|
eth/[common, p2p],
|
|
rocksdb,
|
|
unittest2,
|
|
../nimbus/db/select_backend,
|
|
../nimbus/db/aristo/[aristo_desc, aristo_merge],
|
|
../nimbus/core/chain,
|
|
../nimbus/sync/snap/worker/db/[rocky_bulk_load, snapdb_accounts, snapdb_desc],
|
|
./replay/[pp, undump_accounts, undump_storages],
|
|
./test_sync_snap/[snap_test_xx, test_accounts, test_types],
|
|
./test_aristo/[test_backend, test_helpers, test_transcode, test_tx]
|
|
|
|
const
|
|
baseDir = [".", "..", ".."/"..", $DirSep]
|
|
repoDir = [".", "tests", "nimbus-eth1-blobs"]
|
|
subDir = ["replay", "test_sync_snap", "replay"/"snap"]
|
|
|
|
# Reference file for finding the database directory
|
|
sampleDirRefFile = "sample0.txt.gz"
|
|
|
|
# Standard test samples
|
|
accSample = snapTest0
|
|
storSample = snapTest4
|
|
|
|
# Number of database slots available
|
|
nTestDbInstances = 9
|
|
|
|
# Dormant (may be set if persistent database causes problems)
|
|
disablePersistentDB = false
|
|
|
|
type
|
|
TestDbs = object
|
|
## Provide enough spare empty databases
|
|
persistent: bool
|
|
dbDir: string
|
|
baseDir: string # for cleanup
|
|
subDir: string # for cleanup
|
|
cdb: array[nTestDbInstances,ChainDb]
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc findFilePath(
|
|
file: string;
|
|
baseDir: openArray[string] = baseDir;
|
|
repoDir: openArray[string] = repoDir;
|
|
subDir: openArray[string] = subDir;
|
|
): Result[string,void] =
|
|
for dir in baseDir:
|
|
if dir.dirExists:
|
|
for repo in repoDir:
|
|
if (dir / repo).dirExists:
|
|
for sub in subDir:
|
|
if (dir / repo / sub).dirExists:
|
|
let path = dir / repo / sub / file
|
|
if path.fileExists:
|
|
return ok(path)
|
|
echo "*** File not found \"", file, "\"."
|
|
err()
|
|
|
|
proc getTmpDir(sampleDir = sampleDirRefFile): string =
|
|
sampleDir.findFilePath.value.splitFile.dir
|
|
|
|
proc setTraceLevel {.used.} =
|
|
discard
|
|
when defined(chronicles_runtime_filtering) and loggingEnabled:
|
|
setLogLevel(LogLevel.TRACE)
|
|
|
|
proc setErrorLevel {.used.} =
|
|
discard
|
|
when defined(chronicles_runtime_filtering) and loggingEnabled:
|
|
setLogLevel(LogLevel.ERROR)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc flushDbDir(s: string; subDir = "") =
|
|
if s != "":
|
|
let baseDir = s / "tmp"
|
|
for n in 0 ..< nTestDbInstances:
|
|
let instDir = if subDir == "": baseDir / $n else: baseDir / subDir / $n
|
|
if (instDir / "nimbus" / "data").dirExists:
|
|
# Typically under Windows: there might be stale file locks.
|
|
try: instDir.removeDir except CatchableError: discard
|
|
try: (baseDir / subDir).removeDir except CatchableError: discard
|
|
block dontClearUnlessEmpty:
|
|
for w in baseDir.walkDir:
|
|
break dontClearUnlessEmpty
|
|
try: baseDir.removeDir except CatchableError: discard
|
|
|
|
|
|
proc flushDbs(db: TestDbs) =
|
|
if db.persistent:
|
|
for n in 0 ..< nTestDbInstances:
|
|
if db.cdb[n].rocksStoreRef.isNil:
|
|
break
|
|
db.cdb[n].rocksStoreRef.store.db.rocksdb_close
|
|
db.baseDir.flushDbDir(db.subDir)
|
|
|
|
proc testDbs(
|
|
workDir: string;
|
|
subDir: string;
|
|
instances: int;
|
|
persistent: bool;
|
|
): TestDbs =
|
|
if disablePersistentDB or workDir == "" or not persistent:
|
|
result.persistent = false
|
|
result.dbDir = "*notused*"
|
|
else:
|
|
result.persistent = true
|
|
result.baseDir = workDir
|
|
result.subDir = subDir
|
|
if subDir != "":
|
|
result.dbDir = workDir / "tmp" / subDir
|
|
else:
|
|
result.dbDir = workDir / "tmp"
|
|
if result.persistent:
|
|
workDir.flushDbDir(subDir)
|
|
for n in 0 ..< min(result.cdb.len, instances):
|
|
result.cdb[n] = (result.dbDir / $n).newChainDB
|
|
|
|
proc snapDbRef(cdb: ChainDb; pers: bool): SnapDbRef =
|
|
if pers: SnapDbRef.init(cdb) else: SnapDbRef.init(newMemoryDB())
|
|
|
|
proc snapDbAccountsRef(cdb:ChainDb; root:Hash256; pers:bool):SnapDbAccountsRef =
|
|
SnapDbAccountsRef.init(cdb.snapDbRef(pers), root, Peer())
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Test Runners: accounts and accounts storages
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc miscRunner(noisy =true) =
|
|
suite &"Aristo: Miscellaneous tests":
|
|
test &"VertexID recyling lists":
|
|
noisy.test_transcodeVidRecycleLists()
|
|
|
|
|
|
proc transcodeRunner(noisy =true; sample=accSample; stopAfter=high(int)) =
|
|
let
|
|
accLst = sample.to(seq[UndumpAccounts])
|
|
root = accLst[0].root
|
|
tmpDir = getTmpDir()
|
|
db = tmpDir.testDbs(sample.name&"-transcode", instances=2, persistent=true)
|
|
info = if db.persistent: &"persistent db on \"{db.baseDir}\""
|
|
else: "in-memory db"
|
|
fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")
|
|
|
|
defer:
|
|
db.flushDbs
|
|
|
|
suite &"Aristo: transcoding {fileInfo} accounts for {info}":
|
|
|
|
# New common descriptor for this sub-group of tests
|
|
let
|
|
desc = db.cdb[0].snapDbAccountsRef(root, db.persistent)
|
|
hexaDb = desc.hexaDb
|
|
getFn = desc.getAccountFn
|
|
dbg = if noisy: hexaDb else: nil
|
|
|
|
# Borrowed from `test_sync_snap/test_accounts.nim`
|
|
test &"Importing {accLst.len} list items to persistent database":
|
|
if db.persistent:
|
|
accLst.test_accountsImport(desc, true)
|
|
else:
|
|
skip()
|
|
|
|
test "Trancoding database records: RLP, NodeRef, Blob, VertexRef":
|
|
noisy.showElapsed("test_transcoder()"):
|
|
noisy.test_transcodeAccounts(db.cdb[0].rocksStoreRef, stopAfter)
|
|
|
|
|
|
proc accountsRunner(
|
|
noisy = true;
|
|
sample = accSample;
|
|
resetDb = false;
|
|
cmpBackends = true;
|
|
) =
|
|
let
|
|
accLst = sample.to(seq[UndumpAccounts]).to(seq[ProofTrieData])
|
|
fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")
|
|
listMode = if resetDb: "" else: ", merged data lists"
|
|
baseDir = getTmpDir() / sample.name & "-accounts"
|
|
dbDir = baseDir / "tmp"
|
|
|
|
defer:
|
|
try: baseDir.removeDir except CatchableError: discard
|
|
|
|
suite &"Aristo: accounts data dump from {fileInfo}{listMode}":
|
|
|
|
test &"Merge {accLst.len} proof & account lists to database":
|
|
check noisy.testTxMergeProofAndKvpList(accLst, dbDir, resetDb)
|
|
|
|
test &"Compare {accLst.len} account lists on database backends":
|
|
if cmpBackends:
|
|
check noisy.testBackendConsistency(accLst, dbDir, resetDb)
|
|
else:
|
|
skip()
|
|
|
|
test &"Delete accounts database, successively {accLst.len} entries":
|
|
check noisy.testTxMergeAndDelete(accLst, dbDir)
|
|
|
|
|
|
proc storagesRunner(
|
|
noisy = true;
|
|
sample = storSample;
|
|
resetDb = false;
|
|
oops: KnownHasherFailure = @[];
|
|
cmpBackends = true;
|
|
) =
|
|
let
|
|
stoLst = sample.to(seq[UndumpStorages]).to(seq[ProofTrieData])
|
|
fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")
|
|
listMode = if resetDb: "" else: ", merged data lists"
|
|
baseDir = getTmpDir() / sample.name & "-storage"
|
|
dbDir = baseDir / "tmp"
|
|
|
|
defer:
|
|
try: baseDir.removeDir except CatchableError: discard
|
|
|
|
suite &"Aristo: storages data dump from {fileInfo}{listMode}":
|
|
|
|
test &"Merge {stoLst.len} proof & slots lists to database":
|
|
check noisy.testTxMergeProofAndKvpList(
|
|
stoLst, dbDir, resetDb, fileInfo, oops)
|
|
|
|
test &"Compare {stoLst.len} slot lists on database backends":
|
|
if cmpBackends:
|
|
check noisy.testBackendConsistency(stoLst, dbDir, resetDb)
|
|
else:
|
|
skip()
|
|
|
|
test &"Delete storage database, successively {stoLst.len} entries":
|
|
check noisy.testTxMergeAndDelete(stoLst, dbDir)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Main function(s)
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc aristoMain*(noisy = defined(debug)) =
|
|
noisy.miscRunner()
|
|
noisy.transcodeRunner()
|
|
noisy.accountsRunner()
|
|
noisy.storagesRunner()
|
|
|
|
when isMainModule:
|
|
const
|
|
noisy = defined(debug) or true
|
|
|
|
setErrorLevel()
|
|
|
|
when true: # and false:
|
|
noisy.miscRunner()
|
|
|
|
# Borrowed from `test_sync_snap.nim`
|
|
when true: # and false:
|
|
for n,sam in snapTestList:
|
|
noisy.transcodeRunner(sam)
|
|
for n,sam in snapTestStorageList:
|
|
noisy.transcodeRunner(sam)
|
|
|
|
# This one uses dumps from the external `nimbus-eth1-blob` repo
|
|
when true and false:
|
|
import ./test_sync_snap/snap_other_xx
|
|
noisy.showElapsed("@snap_other_xx"):
|
|
for n,sam in snapOtherList:
|
|
noisy.accountsRunner(sam, resetDb=true)
|
|
|
|
# This one usues dumps from the external `nimbus-eth1-blob` repo
|
|
when true and false:
|
|
import ./test_sync_snap/snap_storage_xx
|
|
let knownFailures: KnownHasherFailure = @[
|
|
("storages3__18__25_dump#12.27367",(3,HashifyExistingHashMismatch)),
|
|
("storages4__26__33_dump#12.23924",(6,HashifyExistingHashMismatch)),
|
|
("storages5__34__41_dump#10.20512",(1,HashifyRootHashMismatch)),
|
|
("storagesB__84__92_dump#7.9709", (7,HashifyExistingHashMismatch)),
|
|
("storagesD_102_109_dump#18.28287",(9,HashifyExistingHashMismatch)),
|
|
]
|
|
noisy.showElapsed("@snap_storage_xx"):
|
|
for n,sam in snapStorageList:
|
|
noisy.accountsRunner(sam, resetDb=true)
|
|
noisy.storagesRunner(sam, resetDb=true, oops=knownFailures)
|
|
|
|
when true: # and false:
|
|
for n,sam in snapTestList:
|
|
noisy.accountsRunner(sam)
|
|
for n,sam in snapTestStorageList:
|
|
noisy.accountsRunner(sam)
|
|
noisy.storagesRunner(sam)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|