nimbus-eth1/tests/test_aristo/test_backend.nim

254 lines
7.9 KiB
Nim

# Nimbus
# Copyright (c) 2018-2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or
# distributed except according to those terms.
## Aristo (aka Patricia) DB records merge test
import
std/[algorithm, sequtils, sets, tables],
eth/common,
stew/results,
unittest2,
../../nimbus/sync/protocol,
../../nimbus/db/aristo/aristo_init/[
aristo_memory, aristo_rocksdb],
../../nimbus/db/aristo/[
aristo_desc, aristo_debug, aristo_hashify, aristo_init, aristo_layer,
aristo_merge],
./test_helpers
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
proc mergeData(
db: AristoDbRef;
rootKey: HashKey;
rootVid: VertexID;
proof: openArray[SnapProof];
leafs: openArray[LeafTiePayload];
noisy: bool;
): bool =
## Simplified loop body of `test_mergeProofAndKvpList()`
if 0 < proof.len:
let rc = db.merge(rootKey, rootVid)
if rc.isErr:
check rc.error == AristoError(0)
return
let proved = db.merge(proof, rc.value)
if proved.error notin {AristoError(0),MergeHashKeyCachedAlready}:
check proved.error in {AristoError(0),MergeHashKeyCachedAlready}
return
let merged = db.merge leafs
if merged.error notin {AristoError(0), MergeLeafPathCachedAlready}:
check merged.error in {AristoError(0), MergeLeafPathCachedAlready}
return
block:
let rc = db.hashify # (noisy, true)
if rc.isErr:
when true: # and false:
noisy.say "***", "dataMerge(9)",
" nLeafs=", leafs.len,
"\n cache dump\n ", db.pp,
"\n backend dump\n ", db.to(TypedBackendRef).pp(db)
check rc.error == (VertexID(0),AristoError(0))
return
true
proc verify(
ly: AristoLayerRef; # Database layer
be: MemBackendRef|RdbBackendRef; # Backend
noisy: bool;
): bool =
## ..
let
beSTab = be.walkVtx.toSeq.mapIt((it[1],it[2])).toTable
beKMap = be.walkKey.toSeq.mapIt((it[1],it[2])).toTable
beIdg = be.walkIdg.toSeq
for vid in beSTab.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID):
let
nVtx = ly.sTab.getOrVoid vid
mVtx = beSTab.getOrVoid vid
if not nVtx.isValid and not mVtx.isValid:
check nVtx != VertexRef(nil)
check mVtx != VertexRef(nil)
return
if nVtx != mVtx:
noisy.say "***", "verify",
" beType=", be.typeof,
" vid=", vid.pp,
" nVtx=", nVtx.pp,
" mVtx=", mVtx.pp
check nVtx == mVtx
return
if beSTab.len != ly.sTab.len or
beKMap.len != ly.kMap.len:
check beSTab.len == ly.sTab.len
check beKMap.len == ly.kMap.len
return
true
# ------------------------------------------------------------------------------
# Public test function
# ------------------------------------------------------------------------------
proc test_backendConsistency*(
noisy: bool;
list: openArray[ProofTrieData]; # Test data
rdbPath: string; # Rocks DB storage directory
resetDb = false;
doRdbOk = true;
): bool =
## Import accounts
var
ndb = AristoDbRef() # Reference cache
mdb = AristoDbRef() # Memory backend database
rdb = AristoDbRef() # Rocks DB backend database
rootKey = HashKey.default
count = 0
defer:
rdb.finish(flush=true)
for n,w in list:
if w.root != rootKey or resetDB:
rootKey = w.root
count = 0
ndb = AristoDbRef.init BackendNone
mdb = AristoDbRef.init BackendMemory
if doRdbOk:
rdb.finish(flush=true)
let rc = AristoDbRef.init(BackendRocksDB,rdbPath)
if rc.isErr:
check rc.error == 0
return
rdb = rc.value
count.inc
check ndb.backend.isNil
check not mdb.backend.isNil
check doRdbOk or not rdb.backend.isNil
when true and false:
noisy.say "***", "beCon(1) <", n, "/", list.len-1, ">", " groups=", count
block:
let
rootVid = VertexID(1)
leafs = w.kvpLst.mapRootVid VertexID(1) # for merging it into main trie
block:
let ndbOk = ndb.mergeData(
rootKey, rootVid, w.proof, leafs, noisy=false)
if not ndbOk:
check ndbOk
return
block:
let mdbOk = mdb.mergeData(
rootKey, rootVid, w.proof, leafs, noisy=false)
if not mdbOk:
check mdbOk
return
if doRdbOk: # optional
let rdbOk = rdb.mergeData(
rootKey, rootVid, w.proof, leafs, noisy=false)
if not rdbOk:
check rdbOk
return
when true and false:
noisy.say "***", "beCon(2) <", n, "/", list.len-1, ">",
" groups=", count,
"\n cache dump\n ", ndb.pp,
"\n backend dump\n ", ndb.to(TypedBackendRef).pp(ndb),
"\n -------------",
"\n mdb cache\n ", mdb.pp,
"\n mdb backend\n ", mdb.to(TypedBackendRef).pp(ndb),
"\n -------------",
"\n rdb cache\n ", rdb.pp,
"\n rdb backend\n ", rdb.to(TypedBackendRef).pp(ndb),
"\n -------------"
when true and false:
noisy.say "***", "beCon(4) <", n, "/", list.len-1, ">", " groups=", count
var
mdbPreSaveCache, mdbPreSaveBackend: string
rdbPreSaveCache, rdbPreSaveBackend: string
when true: # and false:
mdbPreSaveCache = mdb.pp
mdbPreSaveBackend = mdb.to(MemBackendRef).pp(ndb)
rdbPreSaveCache = rdb.pp
rdbPreSaveBackend = rdb.to(RdbBackendRef).pp(ndb)
# Store onto backend database
block:
#noisy.say "***", "db-dump\n ", mdb.pp
let rc = mdb.save
if rc.isErr:
check rc.error == (0,0)
return
if doRdbOk:
let rc = rdb.save
if rc.isErr:
check rc.error == (0,0)
return
if not ndb.top.verify(mdb.to(MemBackendRef), noisy):
when true and false:
noisy.say "***", "beCon(4) <", n, "/", list.len-1, ">",
" groups=", count,
"\n ndb cache\n ", ndb.pp,
"\n ndb backend=", ndb.backend.isNil.not,
#"\n -------------",
#"\n mdb pre-save cache\n ", mdbPreSaveCache,
#"\n mdb pre-save backend\n ", mdbPreSaveBackend,
"\n -------------",
"\n mdb cache\n ", mdb.pp,
"\n mdb backend\n ", mdb.to(TypedBackendRef).pp(ndb),
"\n -------------"
return
if doRdbOk:
if not ndb.top.verify(rdb.to(RdbBackendRef), noisy):
when true and false:
noisy.say "***", "beCon(4) <", n, "/", list.len-1, ">",
" groups=", count,
"\n ndb cache\n ", ndb.pp,
"\n ndb backend=", ndb.backend.isNil.not,
"\n -------------",
"\n rdb pre-save cache\n ", rdbPreSaveCache,
"\n rdb pre-save backend\n ", rdbPreSaveBackend,
"\n -------------",
"\n rdb cache\n ", rdb.pp,
"\n rdb backend\n ", rdb.to(TypedBackendRef).pp(ndb),
#"\n -------------",
#"\n mdb cache\n ", mdb.pp,
#"\n mdb backend\n ", mdb.to(TypedBackendRef).pp(ndb),
"\n -------------"
return
when true and false:
noisy.say "***", "beCon(9) <", n, "/", list.len-1, ">", " groups=", count
true
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------