# Nimbus - Types, data structures and shared utilities used in network sync # # Copyright (c) 2018-2021 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) # * MIT license ([LICENSE-MIT](LICENSE-MIT) or # http://opensource.org/licenses/MIT) # at your option. This file may not be copied, modified, or # distributed except according to those terms. ## Re-invented implementation for Merkle Patricia Tree named as Aristo Trie import std/[os, strformat, strutils], chronicles, eth/[common, p2p], rocksdb, unittest2, ../nimbus/db/select_backend, ../nimbus/db/aristo/[aristo_desc, aristo_merge], ../nimbus/core/chain, ../nimbus/sync/snap/worker/db/[rocky_bulk_load, snapdb_accounts, snapdb_desc], ./replay/[pp, undump_accounts, undump_storages], ./test_sync_snap/[snap_test_xx, test_accounts, test_types], ./test_aristo/[test_backend, test_helpers, test_transcode, test_tx] const baseDir = [".", "..", ".."/"..", $DirSep] repoDir = [".", "tests", "nimbus-eth1-blobs"] subDir = ["replay", "test_sync_snap", "replay"/"snap"] # Reference file for finding the database directory sampleDirRefFile = "sample0.txt.gz" # Standard test samples accSample = snapTest0 storSample = snapTest4 # Number of database slots available nTestDbInstances = 9 # Dormant (may be set if persistent database causes problems) disablePersistentDB = false type TestDbs = object ## Provide enough spare empty databases persistent: bool dbDir: string baseDir: string # for cleanup subDir: string # for cleanup cdb: array[nTestDbInstances,ChainDb] # ------------------------------------------------------------------------------ # Helpers # ------------------------------------------------------------------------------ proc findFilePath( file: string; baseDir: openArray[string] = baseDir; repoDir: openArray[string] = repoDir; subDir: openArray[string] = subDir; ): Result[string,void] = for dir in baseDir: if dir.dirExists: for repo in repoDir: if (dir / repo).dirExists: for sub in subDir: if (dir / repo / sub).dirExists: let path = dir / repo / sub / file if path.fileExists: return ok(path) echo "*** File not found \"", file, "\"." err() proc getTmpDir(sampleDir = sampleDirRefFile): string = sampleDir.findFilePath.value.splitFile.dir proc setTraceLevel {.used.} = discard when defined(chronicles_runtime_filtering) and loggingEnabled: setLogLevel(LogLevel.TRACE) proc setErrorLevel {.used.} = discard when defined(chronicles_runtime_filtering) and loggingEnabled: setLogLevel(LogLevel.ERROR) # ------------------------------------------------------------------------------ # Private functions # ------------------------------------------------------------------------------ proc flushDbDir(s: string; subDir = "") = if s != "": let baseDir = s / "tmp" for n in 0 ..< nTestDbInstances: let instDir = if subDir == "": baseDir / $n else: baseDir / subDir / $n if (instDir / "nimbus" / "data").dirExists: # Typically under Windows: there might be stale file locks. try: instDir.removeDir except CatchableError: discard try: (baseDir / subDir).removeDir except CatchableError: discard block dontClearUnlessEmpty: for w in baseDir.walkDir: break dontClearUnlessEmpty try: baseDir.removeDir except CatchableError: discard proc flushDbs(db: TestDbs) = if db.persistent: for n in 0 ..< nTestDbInstances: if db.cdb[n].rocksStoreRef.isNil: break db.cdb[n].rocksStoreRef.store.db.rocksdb_close db.baseDir.flushDbDir(db.subDir) proc testDbs( workDir: string; subDir: string; instances: int; persistent: bool; ): TestDbs = if disablePersistentDB or workDir == "" or not persistent: result.persistent = false result.dbDir = "*notused*" else: result.persistent = true result.baseDir = workDir result.subDir = subDir if subDir != "": result.dbDir = workDir / "tmp" / subDir else: result.dbDir = workDir / "tmp" if result.persistent: workDir.flushDbDir(subDir) for n in 0 ..< min(result.cdb.len, instances): result.cdb[n] = (result.dbDir / $n).newChainDB proc snapDbRef(cdb: ChainDb; pers: bool): SnapDbRef = if pers: SnapDbRef.init(cdb) else: SnapDbRef.init(newMemoryDB()) proc snapDbAccountsRef(cdb:ChainDb; root:Hash256; pers:bool):SnapDbAccountsRef = SnapDbAccountsRef.init(cdb.snapDbRef(pers), root, Peer()) # ------------------------------------------------------------------------------ # Test Runners: accounts and accounts storages # ------------------------------------------------------------------------------ proc miscRunner(noisy =true) = suite &"Aristo: Miscellaneous tests": test &"VertexID recyling lists": noisy.test_transcodeVidRecycleLists() proc transcodeRunner(noisy =true; sample=accSample; stopAfter=high(int)) = let accLst = sample.to(seq[UndumpAccounts]) root = accLst[0].root tmpDir = getTmpDir() db = tmpDir.testDbs(sample.name&"-transcode", instances=2, persistent=true) info = if db.persistent: &"persistent db on \"{db.baseDir}\"" else: "in-memory db" fileInfo = sample.file.splitPath.tail.replace(".txt.gz","") defer: db.flushDbs suite &"Aristo: transcoding {fileInfo} accounts for {info}": # New common descriptor for this sub-group of tests let desc = db.cdb[0].snapDbAccountsRef(root, db.persistent) hexaDb = desc.hexaDb getFn = desc.getAccountFn dbg = if noisy: hexaDb else: nil # Borrowed from `test_sync_snap/test_accounts.nim` test &"Importing {accLst.len} list items to persistent database": if db.persistent: accLst.test_accountsImport(desc, true) else: skip() test "Trancoding database records: RLP, NodeRef, Blob, VertexRef": noisy.showElapsed("test_transcoder()"): noisy.test_transcodeAccounts(db.cdb[0].rocksStoreRef, stopAfter) proc accountsRunner( noisy = true; sample = accSample; resetDb = false; cmpBackends = true; ) = let accLst = sample.to(seq[UndumpAccounts]).to(seq[ProofTrieData]) fileInfo = sample.file.splitPath.tail.replace(".txt.gz","") listMode = if resetDb: "" else: ", merged data lists" baseDir = getTmpDir() / sample.name & "-accounts" dbDir = baseDir / "tmp" defer: try: baseDir.removeDir except CatchableError: discard suite &"Aristo: accounts data dump from {fileInfo}{listMode}": test &"Merge {accLst.len} proof & account lists to database": check noisy.testTxMergeProofAndKvpList(accLst, dbDir, resetDb) test &"Compare {accLst.len} account lists on database backends": if cmpBackends: check noisy.testBackendConsistency(accLst, dbDir, resetDb) else: skip() test &"Delete accounts database, successively {accLst.len} entries": check noisy.testTxMergeAndDelete(accLst, dbDir) proc storagesRunner( noisy = true; sample = storSample; resetDb = false; oops: KnownHasherFailure = @[]; cmpBackends = true; ) = let stoLst = sample.to(seq[UndumpStorages]).to(seq[ProofTrieData]) fileInfo = sample.file.splitPath.tail.replace(".txt.gz","") listMode = if resetDb: "" else: ", merged data lists" baseDir = getTmpDir() / sample.name & "-storage" dbDir = baseDir / "tmp" defer: try: baseDir.removeDir except CatchableError: discard suite &"Aristo: storages data dump from {fileInfo}{listMode}": test &"Merge {stoLst.len} proof & slots lists to database": check noisy.testTxMergeProofAndKvpList( stoLst, dbDir, resetDb, fileInfo, oops) test &"Compare {stoLst.len} slot lists on database backends": if cmpBackends: check noisy.testBackendConsistency(stoLst, dbDir, resetDb) else: skip() test &"Delete storage database, successively {stoLst.len} entries": check noisy.testTxMergeAndDelete(stoLst, dbDir) # ------------------------------------------------------------------------------ # Main function(s) # ------------------------------------------------------------------------------ proc aristoMain*(noisy = defined(debug)) = noisy.miscRunner() noisy.transcodeRunner() noisy.accountsRunner() noisy.storagesRunner() when isMainModule: const noisy = defined(debug) or true setErrorLevel() when true and false: noisy.miscRunner() # Borrowed from `test_sync_snap.nim` when true and false: for n,sam in snapTestList: noisy.transcodeRunner(sam) for n,sam in snapTestStorageList: noisy.transcodeRunner(sam) # This one uses dumps from the external `nimbus-eth1-blob` repo when true and false: import ./test_sync_snap/snap_other_xx noisy.showElapsed("@snap_other_xx"): for n,sam in snapOtherList: noisy.accountsRunner(sam, resetDb=true) # This one usues dumps from the external `nimbus-eth1-blob` repo when true and false: import ./test_sync_snap/snap_storage_xx let knownFailures: KnownHasherFailure = @[ ("storages3__18__25_dump#12.27367",(3,HashifyExistingHashMismatch)), ("storages4__26__33_dump#12.23924",(6,HashifyExistingHashMismatch)), ("storages5__34__41_dump#10.20512",(1,HashifyRootHashMismatch)), ("storagesB__84__92_dump#7.9709", (7,HashifyExistingHashMismatch)), ("storagesD_102_109_dump#18.28287",(9,HashifyExistingHashMismatch)), ] noisy.showElapsed("@snap_storage_xx"): for n,sam in snapStorageList: noisy.accountsRunner(sam, resetDb=true) noisy.storagesRunner(sam, resetDb=true, oops=knownFailures) when true: # and false: for n,sam in snapTestList: noisy.accountsRunner(sam) for n,sam in snapTestStorageList: noisy.accountsRunner(sam) noisy.storagesRunner(sam) # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------