nimbus-eth1/tests/test_aristo.nim

233 lines
8.0 KiB
Nim
Raw Normal View History

Core db and aristo updates for destructor and tx logic (#1894) * Disable `TransactionID` related functions from `state_db.nim` why: Functions `getCommittedStorage()` and `updateOriginalRoot()` from the `state_db` module are nowhere used. The emulation of a legacy `TransactionID` type functionality is administratively expensive to provide by `Aristo` (the legacy DB version is only partially implemented, anyway). As there is no other place where `TransactionID`s are used, they will not be provided by the `Aristo` variant of the `CoreDb`. For the legacy DB API, nothing will change. * Fix copyright headers in source code * Get rid of compiler warning * Update Aristo code, remove unused `merge()` variant, export `hashify()` why: Adapt to upcoming `CoreDb` wrapper * Remove synced tx feature from `Aristo` why: + This feature allowed to synchronise transaction methods like begin, commit, and rollback for a group of descriptors. + The feature is over engineered and not needed for `CoreDb`, neither is it complete (some convergence features missing.) * Add debugging helpers to `Kvt` also: Update database iterator, add count variable yield argument similar to `Aristo`. * Provide optional destructors for `CoreDb` API why; For the upcoming Aristo wrapper, this allows to control when certain smart destruction and update can take place. The auto destructor works fine in general when the storage/cache strategy is known and acceptable when creating descriptors. * Add update option for `CoreDb` API function `hash()` why; The hash function is typically used to get the state root of the MPT. Due to lazy hashing, this might be not available on the `Aristo` DB. So the `update` function asks for re-hashing the gurrent state changes if needed. * Update API tracking log mode: `info` => `debug * Use shared `Kvt` descriptor in new Ledger API why: No need to create a new descriptor all the time
2023-11-16 19:35:03 +00:00
# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or
# distributed except according to those terms.
## Re-invented implementation for Merkle Patricia Tree named as Aristo Trie
import
std/[os, strformat, strutils],
chronicles,
eth/common,
results,
unittest2,
../nimbus/db/aristo/[aristo_desc, aristo_merge],
./replay/[pp, undump_accounts, undump_storages],
./test_sync_snap/[snap_test_xx, test_types],
./test_aristo/[test_backend, test_filter, test_helpers, test_misc, test_tx]
const
baseDir = [".", "..", ".."/"..", $DirSep]
repoDir = [".", "tests", "nimbus-eth1-blobs"]
subDir = ["replay", "test_sync_snap", "replay"/"snap"]
# Reference file for finding the database directory
sampleDirRefFile = "sample0.txt.gz"
# Standard test samples
accSample = snapTest0
storSample = snapTest4
# ------------------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------------------
proc findFilePath(
file: string;
baseDir: openArray[string] = baseDir;
repoDir: openArray[string] = repoDir;
subDir: openArray[string] = subDir;
): Result[string,void] =
for dir in baseDir:
if dir.dirExists:
for repo in repoDir:
if (dir / repo).dirExists:
for sub in subDir:
if (dir / repo / sub).dirExists:
let path = dir / repo / sub / file
if path.fileExists:
return ok(path)
echo "*** File not found \"", file, "\"."
err()
proc getTmpDir(sampleDir = sampleDirRefFile): string =
sampleDir.findFilePath.value.splitFile.dir
proc setTraceLevel {.used.} =
discard
when defined(chronicles_runtime_filtering) and loggingEnabled:
setLogLevel(LogLevel.TRACE)
proc setErrorLevel {.used.} =
discard
when defined(chronicles_runtime_filtering) and loggingEnabled:
setLogLevel(LogLevel.ERROR)
# ------------------------------------------------------------------------------
# Test Runners: accounts and accounts storages
# ------------------------------------------------------------------------------
proc miscRunner(
noisy = true;
qidSampleSize = QidSample;
) =
suite "Aristo: Miscellaneous tests":
test "VertexID recyling lists":
check noisy.testVidRecycleLists()
test &"Low level cascaded fifos API (sample size: {qidSampleSize})":
check noisy.testQidScheduler(sampleSize = qidSampleSize)
test &"High level cascaded fifos API (sample size: {qidSampleSize})":
check noisy.testFilterFifo(sampleSize = qidSampleSize)
Aristo db update for short nodes key edge cases (#1887) * Aristo: Provide key-value list signature calculator detail: Simple wrappers around `Aristo` core functionality * Update new API for `CoreDb` details: + Renamed new API functions `contains()` => `hasKey()` or `hasPath()` which disables the `in` operator on non-boolean `contains()` functions + The functions `get()` and `fetch()` always return a not-found error if there is no item, available. The new functions `getOrEmpty()` and `mergeOrEmpty()` return an an empty `Blob` if there is no such key found. * Rewrite `core_apps.nim` using new API from `CoreDb` * Use `Aristo` functionality for calculating Merkle signatures details: For debugging, the `VerifyAristoForMerkleRootCalc` can be set so that `Aristo` results will be verified against the legacy versions. * Provide general interface for Merkle signing key-value tables details: Export `Aristo` wrappers * Activate `CoreDb` tests why: Now, API seems to be stable enough for general tests. * Update `toHex()` usage why: Byteutils' `toHex()` is superior to `toSeq.mapIt(it.toHex(2)).join` * Split `aristo_transcode` => `aristo_serialise` + `aristo_blobify` why: + Different modules for different purposes + `aristo_serialise`: RLP encoding/decoding + `aristo_blobify`: Aristo database encoding/decoding * Compacted representation of small nodes' links instead of Keccak hashes why: Ethereum MPTs use Keccak hashes as node links if the size of an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded node value is used as a pseudo node link (rather than a hash.) Such a node is nor stored on key-value database. Rather the RLP encoded node value is stored instead of a lode link in a parent node instead. Only for the root hash, the top level node is always referred to by the hash. This feature needed an abstraction of the `HashKey` object which is now either a hash or a blob of length at most 31 bytes. This leaves two ways of representing an empty/void `HashKey` type, either as an empty blob of zero length, or the hash of an empty blob. * Update `CoreDb` interface (mainly reducing logger noise) * Fix copyright years (to make `Lint` happy)
2023-11-08 12:18:32 +00:00
test "Short keys and other patholgical cases":
check noisy.testShortKeys()
proc accountsRunner(
noisy = true;
sample = accSample;
resetDb = false;
cmpBackends = true;
persistent = true;
) =
let
accLst = sample.to(seq[UndumpAccounts]).to(seq[ProofTrieData])
fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")
listMode = if resetDb: "" else: ", merged dumps"
baseDir = getTmpDir() / sample.name & "-accounts"
dbDir = if persistent: baseDir / "tmp" else: ""
isPersistent = if persistent: "persistent DB" else: "mem DB only"
doRdbOk = (cmpBackends and 0 < dbDir.len)
cmpBeInfo = if doRdbOk: "persistent" else: "memory"
defer:
try: baseDir.removeDir except CatchableError: discard
suite &"Aristo: accounts data dump from {fileInfo}{listMode}, {isPersistent}":
test &"Merge {accLst.len} proof & account lists to database":
check noisy.testTxMergeProofAndKvpList(accLst, dbDir, resetDb)
test &"Compare {accLst.len} account lists on {cmpBeInfo}" &
" db backend vs. cache":
check noisy.testBackendConsistency(accLst, dbDir, resetDb)
test &"Delete accounts database successively, {accLst.len} lists":
check noisy.testTxMergeAndDeleteOneByOne(accLst, dbDir)
test &"Delete accounts database sub-trees, {accLst.len} lists":
check noisy.testTxMergeAndDeleteSubTree(accLst, dbDir)
2023-06-02 19:21:46 +00:00
test &"Distributed backend access {accLst.len} entries":
check noisy.testDistributedAccess(accLst, dbDir)
test &"Filter backlog management {accLst.len} entries":
check noisy.testFilterBacklog(accLst, rdbPath=dbDir)
proc storagesRunner(
noisy = true;
sample = storSample;
resetDb = false;
oops: KnownHasherFailure = @[];
cmpBackends = true;
persistent = true;
) =
let
stoLst = sample.to(seq[UndumpStorages]).to(seq[ProofTrieData])
fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")
listMode = if resetDb: "" else: ", merged dumps"
baseDir = getTmpDir() / sample.name & "-storage"
dbDir = if persistent: baseDir / "tmp" else: ""
isPersistent = if persistent: "persistent DB" else: "mem DB only"
doRdbOk = (cmpBackends and 0 < dbDir.len)
cmpBeInfo = if doRdbOk: "persistent" else: "memory"
defer:
try: baseDir.removeDir except CatchableError: discard
suite &"Aristo: storages data dump from {fileInfo}{listMode}, {isPersistent}":
test &"Merge {stoLst.len} proof & slots lists to database":
check noisy.testTxMergeProofAndKvpList(
stoLst, dbDir, resetDb, fileInfo, oops)
test &"Compare {stoLst.len} slot lists on {cmpBeInfo}" &
" db backend vs. cache":
check noisy.testBackendConsistency(stoLst, dbDir, resetDb)
test &"Delete storage database successively, {stoLst.len} lists":
check noisy.testTxMergeAndDeleteOneByOne(stoLst, dbDir)
test &"Delete storage database sub-trees, {stoLst.len} lists":
check noisy.testTxMergeAndDeleteSubTree(stoLst, dbDir)
2023-06-02 19:21:46 +00:00
test &"Distributed backend access {stoLst.len} entries":
check noisy.testDistributedAccess(stoLst, dbDir)
test &"Filter backlog management {stoLst.len} entries":
check noisy.testFilterBacklog(stoLst, rdbPath=dbDir)
# ------------------------------------------------------------------------------
# Main function(s)
# ------------------------------------------------------------------------------
proc aristoMain*(noisy = defined(debug)) =
noisy.miscRunner()
noisy.accountsRunner()
noisy.storagesRunner()
when isMainModule:
const
noisy = defined(debug) or true
setErrorLevel()
when true: # and false:
noisy.miscRunner(qidSampleSize = 1_000)
# This one uses dumps from the external `nimbus-eth1-blob` repo
when true and false:
import ./test_sync_snap/snap_other_xx
noisy.showElapsed("@snap_other_xx"):
for n,sam in snapOtherList:
2023-06-02 19:21:46 +00:00
noisy.accountsRunner(sam, resetDb=true)
# This one usues dumps from the external `nimbus-eth1-blob` repo
2023-06-02 19:21:46 +00:00
when true and false:
import ./test_sync_snap/snap_storage_xx
let knownFailures: KnownHasherFailure = @[
("storages3__18__25_dump#12.27367",(3,HashifyExistingHashMismatch)),
("storages4__26__33_dump#12.23924",(6,HashifyExistingHashMismatch)),
("storages5__34__41_dump#10.20512",(1,HashifyRootHashMismatch)),
("storagesB__84__92_dump#7.9709", (7,HashifyExistingHashMismatch)),
("storagesD_102_109_dump#18.28287",(9,HashifyExistingHashMismatch)),
]
noisy.showElapsed("@snap_storage_xx"):
for n,sam in snapStorageList:
2023-06-02 19:21:46 +00:00
noisy.accountsRunner(sam, resetDb=true)
noisy.storagesRunner(sam, resetDb=true, oops=knownFailures)
when true: # and false:
let persistent = false # or true
noisy.showElapsed("@snap_test_list"):
for n,sam in snapTestList:
noisy.accountsRunner(sam, persistent=persistent)
noisy.showElapsed("@snap_test_storage_list"):
for n,sam in snapTestStorageList:
noisy.accountsRunner(sam, persistent=persistent)
noisy.storagesRunner(sam, persistent=persistent)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------