nimbus-eth1/tests/test_aristo/test_balancer.nim

364 lines
10 KiB
Nim

# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or
# distributed except according to those terms.
## Aristo (aka Patricia) DB records distributed backend access test.
##
import
eth/common,
results,
unittest2,
../../nimbus/db/opts,
../../nimbus/db/core_db/backend/aristo_rocksdb,
../../nimbus/db/aristo/[
aristo_check,
aristo_debug,
aristo_desc,
aristo_get,
aristo_persistent,
aristo_tx],
../replay/xcheck,
./test_helpers
type
LeafQuartet =
array[0..3, seq[LeafTiePayload]]
DbTriplet =
array[0..2, AristoDbRef]
const
testRootVid = VertexID(2)
## Need to reconfigure for the test, root ID 1 cannot be deleted as a trie
# ------------------------------------------------------------------------------
# Private debugging helpers
# ------------------------------------------------------------------------------
proc dump(pfx: string; dx: varargs[AristoDbRef]): string =
if 0 < dx.len:
result = "\n "
var
pfx = pfx
qfx = ""
if pfx.len == 0:
(pfx,qfx) = ("[","]")
elif 1 < dx.len:
pfx = pfx & "#"
for n in 0 ..< dx.len:
let n1 = n + 1
result &= pfx
if 1 < dx.len:
result &= $n1
result &= qfx & "\n " & dx[n].pp(backendOk=true) & "\n"
if n1 < dx.len:
result &= " ==========\n "
proc dump(dx: varargs[AristoDbRef]): string {.used.} =
"".dump dx
proc dump(w: DbTriplet): string {.used.} =
"db".dump(w[0], w[1], w[2])
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
iterator quadripartite(td: openArray[ProofTrieData]): LeafQuartet =
## ...
var collect: seq[seq[LeafTiePayload]]
for w in td:
let lst = w.kvpLst.mapRootVid testRootVid
if lst.len < 8:
if 2 < collect.len:
yield [collect[0], collect[1], collect[2], lst]
collect.setLen(0)
else:
collect.add lst
else:
if collect.len == 0:
let a = lst.len div 4
yield [lst[0 ..< a], lst[a ..< 2*a], lst[2*a ..< 3*a], lst[3*a .. ^1]]
else:
if collect.len == 1:
let a = lst.len div 3
yield [collect[0], lst[0 ..< a], lst[a ..< 2*a], lst[a .. ^1]]
elif collect.len == 2:
let a = lst.len div 2
yield [collect[0], collect[1], lst[0 ..< a], lst[a .. ^1]]
else:
yield [collect[0], collect[1], collect[2], lst]
collect.setLen(0)
proc dbTriplet(w: LeafQuartet; rdbPath: string): Result[DbTriplet,AristoError] =
let db = block:
if 0 < rdbPath.len:
let (dbOpts, cfOpts) = DbOptions.init().toRocksDb()
let rc = AristoDbRef.init(RdbBackendRef, rdbPath, DbOptions.init(), dbOpts, cfOpts, [])
xCheckRc rc.error == 0:
result = err(rc.error)
rc.value()[0]
else:
AristoDbRef.init MemBackendRef
# Set failed `xCheck()` error result
result = err(AristoError 1)
# Fill backend
block:
let report = db.mergeList w[0]
if report.error != 0:
db.finish(eradicate=true)
xCheck report.error == 0
let rc = db.persist()
xCheckRc rc.error == 0:
result = err(rc.error)
let dx = [db, db.forkTx(0).value, db.forkTx(0).value]
xCheck dx[0].nForked == 2
# Reduce unwanted tx layers
for n in 1 ..< dx.len:
xCheck dx[n].level == 1
xCheck dx[n].txTop.value.commit.isOk
# Clause (9) from `aristo/README.md` example
for n in 0 ..< dx.len:
let report = dx[n].mergeList w[n+1]
if report.error != 0:
db.finish(eradicate=true)
xCheck (n, report.error) == (n,0)
return ok(dx)
# ----------------------
proc cleanUp(dx: var DbTriplet) =
if not dx[0].isNil:
dx[0].finish(eradicate=true)
dx.reset
# ----------------------
proc isDbEq(a, b: LayerRef; db: AristoDbRef; noisy = true): bool =
## Verify that argument filter `a` has the same effect on the
## physical/unfiltered backend of `db` as argument filter `b`.
if a.isNil:
return b.isNil
if b.isNil:
return false
if unsafeAddr(a[]) != unsafeAddr(b[]):
if a.kMap.getOrVoid((testRootVid, testRootVid)) !=
b.kMap.getOrVoid((testRootVid, testRootVid)) or
a.vTop != b.vTop:
return false
# Void entries may differ unless on physical backend
var (aTab, bTab) = (a.sTab, b.sTab)
if aTab.len < bTab.len:
aTab.swap bTab
for (vid,aVtx) in aTab.pairs:
let bVtx = bTab.getOrVoid vid
bTab.del vid
if aVtx != bVtx:
if aVtx.isValid and bVtx.isValid:
return false
# The valid one must match the backend data
let rc = db.getVtxUbe vid
if rc.isErr:
return false
let vtx = if aVtx.isValid: aVtx else: bVtx
if vtx != rc.value:
return false
elif not vid.isValid and not bTab.hasKey vid:
let rc = db.getVtxUbe vid
if rc.isOk:
return false # Exists on backend but missing on `bTab[]`
elif rc.error != GetKeyNotFound:
return false # general error
if 0 < bTab.len:
noisy.say "***", "not dbEq:", "bTabLen=", bTab.len
return false
# Similar for `kMap[]`
var (aMap, bMap) = (a.kMap, b.kMap)
if aMap.len < bMap.len:
aMap.swap bMap
for (vid,aKey) in aMap.pairs:
let bKey = bMap.getOrVoid vid
bMap.del vid
if aKey != bKey:
if aKey.isValid and bKey.isValid:
return false
# The valid one must match the backend data
let rc = db.getKeyUbe vid
if rc.isErr:
return false
let key = if aKey.isValid: aKey else: bKey
if key != rc.value:
return false
elif not vid.isValid and not bMap.hasKey vid:
let rc = db.getKeyUbe vid
if rc.isOk:
return false # Exists on backend but missing on `bMap[]`
elif rc.error != GetKeyNotFound:
return false # general error
if 0 < bMap.len:
noisy.say "***", "not dbEq:", " bMapLen=", bMap.len
return false
true
# ----------------------
proc checkBeOk(
dx: DbTriplet;
forceCache = false;
noisy = true;
): bool =
## ..
for n in 0 ..< dx.len:
let rc = dx[n].checkBE()
xCheckRc rc.error == (0,0):
noisy.say "***", "db checkBE failed",
" n=", n, "/", dx.len-1
true
# ------------------------------------------------------------------------------
# Public test function
# ------------------------------------------------------------------------------
proc testBalancer*(
noisy: bool;
list: openArray[ProofTrieData];
rdbPath: string; # Rocks DB storage directory
): bool =
var n = 0
for w in list.quadripartite:
n.inc
# Resulting clause (11) filters from `aristo/README.md` example
# which will be used in the second part of the tests
var
c11Filter1 = LayerRef(nil)
c11Filter3 = LayerRef(nil)
# Work through clauses (8)..(11) from `aristo/README.md` example
block:
# Clause (8) from `aristo/README.md` example
var
dx = block:
let rc = dbTriplet(w, rdbPath)
xCheckRc rc.error == 0
rc.value
(db1, db2, db3) = (dx[0], dx[1], dx[2])
defer:
dx.cleanUp()
when false: # or true:
noisy.say "*** testDistributedAccess (1)", "n=", n # , dx.dump
# Clause (9) from `aristo/README.md` example
block:
let rc = db1.persist()
xCheckRc rc.error == 0
xCheck db1.balancer == LayerRef(nil)
xCheck db2.balancer == db3.balancer
block:
let rc = db2.stow() # non-persistent
xCheckRc rc.error == 0:
noisy.say "*** testDistributedAccess (3)", "n=", n, "db2".dump db2
xCheck db1.balancer == LayerRef(nil)
xCheck db2.balancer != db3.balancer
# Clause (11) from `aristo/README.md` example
discard db2.reCentre()
block:
let rc = db2.persist()
xCheckRc rc.error == 0
xCheck db2.balancer == LayerRef(nil)
# Check/verify backends
block:
let ok = dx.checkBeOk(noisy=noisy)
xCheck ok:
noisy.say "*** testDistributedAccess (4)", "n=", n, "db3".dump db3
# Capture filters from clause (11)
c11Filter1 = db1.balancer
c11Filter3 = db3.balancer
# Clean up
dx.cleanUp()
# ----------
# Work through clauses (12)..(15) from `aristo/README.md` example
block:
var
dy = block:
let rc = dbTriplet(w, rdbPath)
xCheckRc rc.error == 0
rc.value
(db1, db2, db3) = (dy[0], dy[1], dy[2])
defer:
dy.cleanUp()
# Build clause (12) from `aristo/README.md` example
discard db2.reCentre()
block:
let rc = db2.persist()
xCheckRc rc.error == 0
xCheck db2.balancer == LayerRef(nil)
xCheck db1.balancer == db3.balancer
# Clause (13) from `aristo/README.md` example
xCheck not db1.isCentre()
block:
let rc = db1.stow() # non-persistent
xCheckRc rc.error == 0
# Clause (14) from `aristo/README.md` check
let c11Fil1_eq_db1RoFilter = c11Filter1.isDbEq(db1.balancer, db1, noisy)
xCheck c11Fil1_eq_db1RoFilter:
noisy.say "*** testDistributedAccess (7)", "n=", n,
"db1".dump(db1),
""
# Clause (15) from `aristo/README.md` check
let c11Fil3_eq_db3RoFilter = c11Filter3.isDbEq(db3.balancer, db3, noisy)
xCheck c11Fil3_eq_db3RoFilter:
noisy.say "*** testDistributedAccess (8)", "n=", n,
"db3".dump(db3),
""
# Check/verify backends
block:
let ok = dy.checkBeOk(noisy=noisy)
xCheck ok
when false: # or true:
noisy.say "*** testDistributedAccess (9)", "n=", n # , dy.dump
true
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------