mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-01-12 13:24:21 +00:00
5ac362fe6f
* Aristo: Merge `delta_siblings` module into `deltaPersistent()` * Aristo: Add `isEmpty()` for canonical checking whether a layer is empty * Aristo: Merge `LayerDeltaRef` into `LayerObj` why: No need to maintain nested object refs anymore. Previously the `LayerDeltaRef` object had a companion `LayerFinalRef` which held non-delta layer information. * Kvt: Merge `LayerDeltaRef` into `LayerRef` why: No need to maintain nested object refs (as with `Aristo`) * Kvt: Re-write balancer logic similar to `Aristo` why: Although `Kvt` was a cheap copy of `Aristo` it sort of got out of sync and the balancer code was wrong. * Update iterator over forked peers why: Yield additional field `isLast` indicating that the last iteration cycle was approached. * Optimise balancer calculation. why: One can often avoid providing a new object containing the merge of two layers for the balancer. This avoids copying tables. In some cases this is replaced by `hasKey()` look ups though. One uses one of the two to combine and merges the other into the first. Of course, this needs some checks for making sure that none of the components to merge is eventually shared with something else. * Fix copyright year
364 lines
10 KiB
Nim
364 lines
10 KiB
Nim
# Nimbus
|
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or
|
|
# distributed except according to those terms.
|
|
|
|
## Aristo (aka Patricia) DB records distributed backend access test.
|
|
##
|
|
|
|
import
|
|
eth/common,
|
|
results,
|
|
unittest2,
|
|
../../nimbus/db/opts,
|
|
../../nimbus/db/core_db/backend/aristo_rocksdb,
|
|
../../nimbus/db/aristo/[
|
|
aristo_check,
|
|
aristo_debug,
|
|
aristo_desc,
|
|
aristo_get,
|
|
aristo_persistent,
|
|
aristo_tx],
|
|
../replay/xcheck,
|
|
./test_helpers
|
|
|
|
type
|
|
LeafQuartet =
|
|
array[0..3, seq[LeafTiePayload]]
|
|
|
|
DbTriplet =
|
|
array[0..2, AristoDbRef]
|
|
|
|
const
|
|
testRootVid = VertexID(2)
|
|
## Need to reconfigure for the test, root ID 1 cannot be deleted as a trie
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private debugging helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc dump(pfx: string; dx: varargs[AristoDbRef]): string =
|
|
if 0 < dx.len:
|
|
result = "\n "
|
|
var
|
|
pfx = pfx
|
|
qfx = ""
|
|
if pfx.len == 0:
|
|
(pfx,qfx) = ("[","]")
|
|
elif 1 < dx.len:
|
|
pfx = pfx & "#"
|
|
for n in 0 ..< dx.len:
|
|
let n1 = n + 1
|
|
result &= pfx
|
|
if 1 < dx.len:
|
|
result &= $n1
|
|
result &= qfx & "\n " & dx[n].pp(backendOk=true) & "\n"
|
|
if n1 < dx.len:
|
|
result &= " ==========\n "
|
|
|
|
proc dump(dx: varargs[AristoDbRef]): string {.used.} =
|
|
"".dump dx
|
|
|
|
proc dump(w: DbTriplet): string {.used.} =
|
|
"db".dump(w[0], w[1], w[2])
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
iterator quadripartite(td: openArray[ProofTrieData]): LeafQuartet =
|
|
## ...
|
|
var collect: seq[seq[LeafTiePayload]]
|
|
|
|
for w in td:
|
|
let lst = w.kvpLst.mapRootVid testRootVid
|
|
|
|
if lst.len < 8:
|
|
if 2 < collect.len:
|
|
yield [collect[0], collect[1], collect[2], lst]
|
|
collect.setLen(0)
|
|
else:
|
|
collect.add lst
|
|
else:
|
|
if collect.len == 0:
|
|
let a = lst.len div 4
|
|
yield [lst[0 ..< a], lst[a ..< 2*a], lst[2*a ..< 3*a], lst[3*a .. ^1]]
|
|
else:
|
|
if collect.len == 1:
|
|
let a = lst.len div 3
|
|
yield [collect[0], lst[0 ..< a], lst[a ..< 2*a], lst[a .. ^1]]
|
|
elif collect.len == 2:
|
|
let a = lst.len div 2
|
|
yield [collect[0], collect[1], lst[0 ..< a], lst[a .. ^1]]
|
|
else:
|
|
yield [collect[0], collect[1], collect[2], lst]
|
|
collect.setLen(0)
|
|
|
|
proc dbTriplet(w: LeafQuartet; rdbPath: string): Result[DbTriplet,AristoError] =
|
|
let db = block:
|
|
if 0 < rdbPath.len:
|
|
let (dbOpts, cfOpts) = DbOptions.init().toRocksDb()
|
|
let rc = AristoDbRef.init(RdbBackendRef, rdbPath, dbOpts, cfOpts, [])
|
|
xCheckRc rc.error == 0:
|
|
result = err(rc.error)
|
|
rc.value()[0]
|
|
else:
|
|
AristoDbRef.init MemBackendRef
|
|
|
|
# Set failed `xCheck()` error result
|
|
result = err(AristoError 1)
|
|
|
|
# Fill backend
|
|
block:
|
|
let report = db.mergeList w[0]
|
|
if report.error != 0:
|
|
db.finish(eradicate=true)
|
|
xCheck report.error == 0
|
|
let rc = db.persist()
|
|
xCheckRc rc.error == 0:
|
|
result = err(rc.error)
|
|
|
|
let dx = [db, db.forkTx(0).value, db.forkTx(0).value]
|
|
xCheck dx[0].nForked == 2
|
|
|
|
# Reduce unwanted tx layers
|
|
for n in 1 ..< dx.len:
|
|
xCheck dx[n].level == 1
|
|
xCheck dx[n].txTop.value.commit.isOk
|
|
|
|
# Clause (9) from `aristo/README.md` example
|
|
for n in 0 ..< dx.len:
|
|
let report = dx[n].mergeList w[n+1]
|
|
if report.error != 0:
|
|
db.finish(eradicate=true)
|
|
xCheck (n, report.error) == (n,0)
|
|
|
|
return ok(dx)
|
|
|
|
# ----------------------
|
|
|
|
proc cleanUp(dx: var DbTriplet) =
|
|
if not dx[0].isNil:
|
|
dx[0].finish(eradicate=true)
|
|
dx.reset
|
|
|
|
# ----------------------
|
|
|
|
proc isDbEq(a, b: LayerRef; db: AristoDbRef; noisy = true): bool =
|
|
## Verify that argument filter `a` has the same effect on the
|
|
## physical/unfiltered backend of `db` as argument filter `b`.
|
|
if a.isNil:
|
|
return b.isNil
|
|
if b.isNil:
|
|
return false
|
|
if unsafeAddr(a[]) != unsafeAddr(b[]):
|
|
if a.kMap.getOrVoid((testRootVid, testRootVid)) !=
|
|
b.kMap.getOrVoid((testRootVid, testRootVid)) or
|
|
a.vTop != b.vTop:
|
|
return false
|
|
|
|
# Void entries may differ unless on physical backend
|
|
var (aTab, bTab) = (a.sTab, b.sTab)
|
|
if aTab.len < bTab.len:
|
|
aTab.swap bTab
|
|
for (vid,aVtx) in aTab.pairs:
|
|
let bVtx = bTab.getOrVoid vid
|
|
bTab.del vid
|
|
|
|
if aVtx != bVtx:
|
|
if aVtx.isValid and bVtx.isValid:
|
|
return false
|
|
# The valid one must match the backend data
|
|
let rc = db.getVtxUbe vid
|
|
if rc.isErr:
|
|
return false
|
|
let vtx = if aVtx.isValid: aVtx else: bVtx
|
|
if vtx != rc.value:
|
|
return false
|
|
|
|
elif not vid.isValid and not bTab.hasKey vid:
|
|
let rc = db.getVtxUbe vid
|
|
if rc.isOk:
|
|
return false # Exists on backend but missing on `bTab[]`
|
|
elif rc.error != GetKeyNotFound:
|
|
return false # general error
|
|
|
|
if 0 < bTab.len:
|
|
noisy.say "***", "not dbEq:", "bTabLen=", bTab.len
|
|
return false
|
|
|
|
# Similar for `kMap[]`
|
|
var (aMap, bMap) = (a.kMap, b.kMap)
|
|
if aMap.len < bMap.len:
|
|
aMap.swap bMap
|
|
for (vid,aKey) in aMap.pairs:
|
|
let bKey = bMap.getOrVoid vid
|
|
bMap.del vid
|
|
|
|
if aKey != bKey:
|
|
if aKey.isValid and bKey.isValid:
|
|
return false
|
|
# The valid one must match the backend data
|
|
let rc = db.getKeyUbe vid
|
|
if rc.isErr:
|
|
return false
|
|
let key = if aKey.isValid: aKey else: bKey
|
|
if key != rc.value:
|
|
return false
|
|
|
|
elif not vid.isValid and not bMap.hasKey vid:
|
|
let rc = db.getKeyUbe vid
|
|
if rc.isOk:
|
|
return false # Exists on backend but missing on `bMap[]`
|
|
elif rc.error != GetKeyNotFound:
|
|
return false # general error
|
|
|
|
if 0 < bMap.len:
|
|
noisy.say "***", "not dbEq:", " bMapLen=", bMap.len
|
|
return false
|
|
|
|
true
|
|
|
|
# ----------------------
|
|
|
|
proc checkBeOk(
|
|
dx: DbTriplet;
|
|
forceCache = false;
|
|
noisy = true;
|
|
): bool =
|
|
## ..
|
|
for n in 0 ..< dx.len:
|
|
let rc = dx[n].checkBE()
|
|
xCheckRc rc.error == (0,0):
|
|
noisy.say "***", "db checkBE failed",
|
|
" n=", n, "/", dx.len-1
|
|
true
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public test function
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc testBalancer*(
|
|
noisy: bool;
|
|
list: openArray[ProofTrieData];
|
|
rdbPath: string; # Rocks DB storage directory
|
|
): bool =
|
|
var n = 0
|
|
for w in list.quadripartite:
|
|
n.inc
|
|
|
|
# Resulting clause (11) filters from `aristo/README.md` example
|
|
# which will be used in the second part of the tests
|
|
var
|
|
c11Filter1 = LayerRef(nil)
|
|
c11Filter3 = LayerRef(nil)
|
|
|
|
# Work through clauses (8)..(11) from `aristo/README.md` example
|
|
block:
|
|
|
|
# Clause (8) from `aristo/README.md` example
|
|
var
|
|
dx = block:
|
|
let rc = dbTriplet(w, rdbPath)
|
|
xCheckRc rc.error == 0
|
|
rc.value
|
|
(db1, db2, db3) = (dx[0], dx[1], dx[2])
|
|
defer:
|
|
dx.cleanUp()
|
|
|
|
when false: # or true:
|
|
noisy.say "*** testDistributedAccess (1)", "n=", n # , dx.dump
|
|
|
|
# Clause (9) from `aristo/README.md` example
|
|
block:
|
|
let rc = db1.persist()
|
|
xCheckRc rc.error == 0
|
|
xCheck db1.balancer == LayerRef(nil)
|
|
xCheck db2.balancer == db3.balancer
|
|
|
|
block:
|
|
let rc = db2.stow() # non-persistent
|
|
xCheckRc rc.error == 0:
|
|
noisy.say "*** testDistributedAccess (3)", "n=", n, "db2".dump db2
|
|
xCheck db1.balancer == LayerRef(nil)
|
|
xCheck db2.balancer != db3.balancer
|
|
|
|
# Clause (11) from `aristo/README.md` example
|
|
discard db2.reCentre()
|
|
block:
|
|
let rc = db2.persist()
|
|
xCheckRc rc.error == 0
|
|
xCheck db2.balancer == LayerRef(nil)
|
|
|
|
# Check/verify backends
|
|
block:
|
|
let ok = dx.checkBeOk(noisy=noisy)
|
|
xCheck ok:
|
|
noisy.say "*** testDistributedAccess (4)", "n=", n, "db3".dump db3
|
|
|
|
# Capture filters from clause (11)
|
|
c11Filter1 = db1.balancer
|
|
c11Filter3 = db3.balancer
|
|
|
|
# Clean up
|
|
dx.cleanUp()
|
|
|
|
# ----------
|
|
|
|
# Work through clauses (12)..(15) from `aristo/README.md` example
|
|
block:
|
|
var
|
|
dy = block:
|
|
let rc = dbTriplet(w, rdbPath)
|
|
xCheckRc rc.error == 0
|
|
rc.value
|
|
(db1, db2, db3) = (dy[0], dy[1], dy[2])
|
|
defer:
|
|
dy.cleanUp()
|
|
|
|
# Build clause (12) from `aristo/README.md` example
|
|
discard db2.reCentre()
|
|
block:
|
|
let rc = db2.persist()
|
|
xCheckRc rc.error == 0
|
|
xCheck db2.balancer == LayerRef(nil)
|
|
xCheck db1.balancer == db3.balancer
|
|
|
|
# Clause (13) from `aristo/README.md` example
|
|
xCheck not db1.isCentre()
|
|
block:
|
|
let rc = db1.stow() # non-persistent
|
|
xCheckRc rc.error == 0
|
|
|
|
# Clause (14) from `aristo/README.md` check
|
|
let c11Fil1_eq_db1RoFilter = c11Filter1.isDbEq(db1.balancer, db1, noisy)
|
|
xCheck c11Fil1_eq_db1RoFilter:
|
|
noisy.say "*** testDistributedAccess (7)", "n=", n,
|
|
"db1".dump(db1),
|
|
""
|
|
|
|
# Clause (15) from `aristo/README.md` check
|
|
let c11Fil3_eq_db3RoFilter = c11Filter3.isDbEq(db3.balancer, db3, noisy)
|
|
xCheck c11Fil3_eq_db3RoFilter:
|
|
noisy.say "*** testDistributedAccess (8)", "n=", n,
|
|
"db3".dump(db3),
|
|
""
|
|
# Check/verify backends
|
|
block:
|
|
let ok = dy.checkBeOk(noisy=noisy)
|
|
xCheck ok
|
|
|
|
when false: # or true:
|
|
noisy.say "*** testDistributedAccess (9)", "n=", n # , dy.dump
|
|
|
|
true
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|