mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-01-16 15:25:24 +00:00
58cde36656
This kind of data is not used except in tests where it is used only to create databases that don't match actual usage of aristo. Removing simplifies future optimizations that can focus on processing specific leaf types more efficiently. A casualty of this removal is some test code as well as some proof generation code that is unused - on the surface, it looks like it should be possible to port both of these to the more specific data types - doing so would ensure that a database written by one part of the codebase can interact with the other - as it stands, there is confusion on this point since using the proof generation code will result in a database of a shape that is incompatible with the rest of eth1.
437 lines
12 KiB
Nim
437 lines
12 KiB
Nim
# Nimbus
|
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or
|
|
# distributed except according to those terms.
|
|
|
|
## Aristo (aka Patricia) DB records transaction based merge test
|
|
|
|
import
|
|
std/[algorithm, bitops, sequtils, sets, tables],
|
|
eth/common,
|
|
results,
|
|
unittest2,
|
|
stew/endians2,
|
|
../../nimbus/db/opts,
|
|
../../nimbus/db/core_db/backend/aristo_rocksdb,
|
|
../../nimbus/db/aristo/[
|
|
aristo_check,
|
|
aristo_debug,
|
|
aristo_delete,
|
|
aristo_desc,
|
|
aristo_get,
|
|
aristo_hike,
|
|
aristo_init/persistent,
|
|
aristo_nearby,
|
|
aristo_part,
|
|
aristo_part/part_debug,
|
|
aristo_tx],
|
|
../replay/xcheck,
|
|
./test_helpers
|
|
|
|
type
|
|
PrngDesc = object
|
|
prng: uint32 ## random state
|
|
|
|
KnownHasherFailure* = seq[(string,(int,AristoError))]
|
|
## (<sample-name> & "#" <instance>, (<vertex-id>,<error-symbol>))
|
|
|
|
const
|
|
testRootVid = VertexID(2)
|
|
## Need to reconfigure for the test, root ID 1 cannot be deleted as a trie
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc posixPrngRand(state: var uint32): byte =
|
|
## POSIX.1-2001 example of a rand() implementation, see manual page rand(3).
|
|
state = state * 1103515245 + 12345;
|
|
let val = (state shr 16) and 32767 # mod 2^31
|
|
(val shr 8).byte # Extract second byte
|
|
|
|
proc rand[W: SomeInteger|VertexID](ap: var PrngDesc; T: type W): T =
|
|
var a: array[sizeof T,byte]
|
|
for n in 0 ..< sizeof T:
|
|
a[n] = ap.prng.posixPrngRand().byte
|
|
when sizeof(T) == 1:
|
|
let w = uint8.fromBytesBE(a).T
|
|
when sizeof(T) == 2:
|
|
let w = uint16.fromBytesBE(a).T
|
|
when sizeof(T) == 4:
|
|
let w = uint32.fromBytesBE(a).T
|
|
else:
|
|
let w = uint64.fromBytesBE(a).T
|
|
when T is SomeUnsignedInt:
|
|
# That way, `fromBytesBE()` can be applied to `uint`
|
|
result = w
|
|
else:
|
|
# That way the result is independent of endianness
|
|
(addr result).copyMem(unsafeAddr w, sizeof w)
|
|
|
|
proc init(T: type PrngDesc; seed: int): PrngDesc =
|
|
result.prng = (seed and 0x7fffffff).uint32
|
|
|
|
proc rand(td: var PrngDesc; top: int): int =
|
|
if 0 < top:
|
|
let mask = (1 shl (8 * sizeof(int) - top.countLeadingZeroBits)) - 1
|
|
for _ in 0 ..< 100:
|
|
let w = mask and td.rand(typeof(result))
|
|
if w < top:
|
|
return w
|
|
raiseAssert "Not here (!)"
|
|
|
|
# -----------------------
|
|
|
|
proc randomisedLeafs(
|
|
db: AristoDbRef;
|
|
ltys: HashSet[LeafTie];
|
|
td: var PrngDesc;
|
|
): Result[seq[(LeafTie,RootedVertexID)],(VertexID,AristoError)] =
|
|
var lvp: seq[(LeafTie,RootedVertexID)]
|
|
for lty in ltys:
|
|
var hike: Hike
|
|
?lty.hikeUp(db, Opt.none(VertexRef), hike)
|
|
lvp.add (lty,(hike.root, hike.legs[^1].wp.vid))
|
|
|
|
var lvp2 = lvp.sorted(
|
|
cmp = proc(a,b: (LeafTie,RootedVertexID)): int = cmp(a[0],b[0]))
|
|
if 2 < lvp2.len:
|
|
for n in 0 ..< lvp2.len-1:
|
|
let r = n + td.rand(lvp2.len - n)
|
|
lvp2[n].swap lvp2[r]
|
|
ok lvp2
|
|
|
|
proc innerCleanUp(db: var AristoDbRef): bool {.discardable.} =
|
|
## Defer action
|
|
if not db.isNil:
|
|
let rx = db.txTop()
|
|
if rx.isOk:
|
|
let rc = rx.value.collapse(commit=false)
|
|
xCheckRc rc.error == 0
|
|
db.finish(eradicate=true)
|
|
db = AristoDbRef(nil)
|
|
true
|
|
|
|
# --------------------------------
|
|
|
|
proc saveToBackend(
|
|
tx: var AristoTxRef;
|
|
relax: bool;
|
|
noisy: bool;
|
|
debugID: int;
|
|
): bool =
|
|
var db = tx.to(AristoDbRef)
|
|
|
|
# Verify context: nesting level must be 2 (i.e. two transactions)
|
|
xCheck tx.level == 2
|
|
|
|
block:
|
|
let rc = db.checkTop()
|
|
xCheckRc rc.error == (0,0)
|
|
|
|
# Commit and hashify the current layer
|
|
block:
|
|
let rc = tx.commit()
|
|
xCheckRc rc.error == 0
|
|
|
|
block:
|
|
let rc = db.txTop()
|
|
xCheckRc rc.error == 0
|
|
tx = rc.value
|
|
|
|
# Verify context: nesting level must be 1 (i.e. one transaction)
|
|
xCheck tx.level == 1
|
|
|
|
block:
|
|
let rc = db.checkBE()
|
|
xCheckRc rc.error == (0,0)
|
|
|
|
# Commit and save to backend
|
|
block:
|
|
let rc = tx.commit()
|
|
xCheckRc rc.error == 0
|
|
|
|
block:
|
|
let rc = db.txTop()
|
|
xCheckErr rc.value.level < 0 # force error
|
|
|
|
block:
|
|
let rc = db.schedStow()
|
|
xCheckRc rc.error == 0
|
|
|
|
block:
|
|
let rc = db.checkBE()
|
|
xCheckRc rc.error == (0,0):
|
|
noisy.say "***", "saveToBackend (8)", " debugID=", debugID
|
|
|
|
# Update layers to original level
|
|
tx = db.txBegin().value.to(AristoDbRef).txBegin().value
|
|
|
|
true
|
|
|
|
|
|
proc fwdWalkVerify(
|
|
db: AristoDbRef;
|
|
root: VertexID;
|
|
leftOver: HashSet[LeafTie];
|
|
noisy: bool;
|
|
debugID: int;
|
|
): bool =
|
|
let
|
|
nLeafs = leftOver.len
|
|
var
|
|
leftOver = leftOver
|
|
last = LeafTie()
|
|
n = 0
|
|
for (key,_) in db.rightPairs low(LeafTie,root):
|
|
xCheck key in leftOver:
|
|
noisy.say "*** fwdWalkVerify", "id=", n + (nLeafs + 1) * debugID
|
|
leftOver.excl key
|
|
last = key
|
|
n.inc
|
|
|
|
# Verify stop condition
|
|
if last.root == VertexID(0):
|
|
last = low(LeafTie,root)
|
|
elif last != high(LeafTie,root):
|
|
last = last.next
|
|
let rc = last.right db
|
|
xCheck rc.isErr
|
|
xCheck rc.error[1] == NearbyBeyondRange
|
|
xCheck n == nLeafs
|
|
|
|
true
|
|
|
|
proc revWalkVerify(
|
|
db: AristoDbRef;
|
|
root: VertexID;
|
|
leftOver: HashSet[LeafTie];
|
|
noisy: bool;
|
|
debugID: int;
|
|
): bool =
|
|
let
|
|
nLeafs = leftOver.len
|
|
var
|
|
leftOver = leftOver
|
|
last = LeafTie()
|
|
n = 0
|
|
for (key,_) in db.leftPairs high(LeafTie,root):
|
|
xCheck key in leftOver:
|
|
noisy.say "*** revWalkVerify", " id=", n + (nLeafs + 1) * debugID
|
|
leftOver.excl key
|
|
last = key
|
|
n.inc
|
|
|
|
# Verify stop condition
|
|
if last.root == VertexID(0):
|
|
last = high(LeafTie,root)
|
|
elif last != low(LeafTie,root):
|
|
last = last.prev
|
|
let rc = last.left db
|
|
xCheck rc.isErr
|
|
xCheck rc.error[1] == NearbyBeyondRange
|
|
xCheck n == nLeafs
|
|
|
|
true
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public test function
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc testTxMergeAndDeleteOneByOne*(
|
|
noisy: bool;
|
|
list: openArray[ProofTrieData];
|
|
rdbPath: string; # Rocks DB storage directory
|
|
): bool {.deprecated: "rewrite to use non-generic data".} =
|
|
# var
|
|
# prng = PrngDesc.init 42
|
|
# db = AristoDbRef(nil)
|
|
# fwdRevVfyToggle = true
|
|
# defer:
|
|
# if not db.isNil:
|
|
# db.finish(eradicate=true)
|
|
|
|
# for n,w in list:
|
|
# # Start with brand new persistent database.
|
|
# db = block:
|
|
# if 0 < rdbPath.len:
|
|
# let (dbOpts, cfOpts) = DbOptions.init().toRocksDb()
|
|
# let rc = AristoDbRef.init(RdbBackendRef, rdbPath, DbOptions.init(), dbOpts, cfOpts, [])
|
|
# xCheckRc rc.error == 0
|
|
# rc.value()[0]
|
|
# else:
|
|
# AristoDbRef.init(MemBackendRef)
|
|
|
|
# # Start transaction (double frame for testing)
|
|
# xCheck db.txTop.isErr
|
|
# var tx = db.txBegin().value.to(AristoDbRef).txBegin().value
|
|
# xCheck tx.isTop()
|
|
# xCheck tx.level == 2
|
|
|
|
# # Reset database so that the next round has a clean setup
|
|
# defer: db.innerCleanUp
|
|
|
|
# # Merge leaf data into main trie
|
|
# let kvpLeafs = block:
|
|
# var lst = w.kvpLst.mapRootVid testRootVid
|
|
# # The list might be reduced for isolation of particular properties,
|
|
# # e.g. lst.setLen(min(5,lst.len))
|
|
# lst
|
|
# for i,leaf in kvpLeafs:
|
|
# let rc = db.mergeGenericData leaf
|
|
# xCheckRc rc.error == 0
|
|
|
|
# # List of all leaf entries that should be on the database
|
|
# var leafsLeft = kvpLeafs.mapIt(it.leafTie).toHashSet
|
|
|
|
# # Provide a (reproducible) peudo-random copy of the leafs list
|
|
# let leafVidPairs = block:
|
|
# let rc = db.randomisedLeafs(leafsLeft, prng)
|
|
# xCheckRc rc.error == (0,0)
|
|
# rc.value
|
|
|
|
# # Trigger subsequent saving tasks in loop below
|
|
# let (saveMod, saveRest, relax) = block:
|
|
# if leafVidPairs.len < 17: (7, 3, false)
|
|
# elif leafVidPairs.len < 31: (11, 7, false)
|
|
# else: (leafVidPairs.len div 5, 11, true)
|
|
|
|
# # === Loop over leafs ===
|
|
# for u,lvp in leafVidPairs:
|
|
# let
|
|
# runID = n + list.len * u
|
|
# tailWalkVerify = 7 # + 999
|
|
# doSaveBeOk = ((u mod saveMod) == saveRest)
|
|
# (leaf, lid) = lvp
|
|
|
|
# if doSaveBeOk:
|
|
# let saveBeOk = tx.saveToBackend(relax=relax, noisy=noisy, runID)
|
|
# xCheck saveBeOk:
|
|
# noisy.say "***", "del1by1(2)",
|
|
# " u=", u,
|
|
# " n=", n, "/", list.len,
|
|
# "\n db\n ", db.pp(backendOk=true),
|
|
# ""
|
|
|
|
# # Delete leaf
|
|
# block:
|
|
# let rc = db.deleteGenericData(leaf.root, @(leaf.path))
|
|
# xCheckRc rc.error == 0
|
|
|
|
# # Update list of remaininf leafs
|
|
# leafsLeft.excl leaf
|
|
|
|
# let deletedVtx = tx.db.getVtx lid
|
|
# xCheck deletedVtx.isValid == false:
|
|
# noisy.say "***", "del1by1(8)"
|
|
|
|
# # Walking the database is too slow for large tables. So the hope is that
|
|
# # potential errors will not go away and rather pop up later, as well.
|
|
# if leafsLeft.len <= tailWalkVerify:
|
|
# if u < leafVidPairs.len-1:
|
|
# if fwdRevVfyToggle:
|
|
# fwdRevVfyToggle = false
|
|
# if not db.fwdWalkVerify(leaf.root, leafsLeft, noisy, runID):
|
|
# return
|
|
# else:
|
|
# fwdRevVfyToggle = true
|
|
# if not db.revWalkVerify(leaf.root, leafsLeft, noisy, runID):
|
|
# return
|
|
|
|
# when true and false:
|
|
# noisy.say "***", "del1by1(9)",
|
|
# " n=", n, "/", list.len,
|
|
# " nLeafs=", kvpLeafs.len
|
|
|
|
true
|
|
|
|
|
|
proc testTxMergeAndDeleteSubTree*(
|
|
noisy: bool;
|
|
list: openArray[ProofTrieData];
|
|
rdbPath: string; # Rocks DB storage directory
|
|
): bool {.deprecated: "rewrite to use non-generic data".} =
|
|
# var
|
|
# prng = PrngDesc.init 42
|
|
# db = AristoDbRef(nil)
|
|
# defer:
|
|
# if not db.isNil:
|
|
# db.finish(eradicate=true)
|
|
|
|
# for n,w in list:
|
|
# # Start with brand new persistent database.
|
|
# db = block:
|
|
# if 0 < rdbPath.len:
|
|
# let (dbOpts, cfOpts) = DbOptions.init().toRocksDb()
|
|
# let rc = AristoDbRef.init(RdbBackendRef, rdbPath, DbOptions.init(), dbOpts, cfOpts, [])
|
|
# xCheckRc rc.error == 0
|
|
# rc.value()[0]
|
|
# else:
|
|
# AristoDbRef.init(MemBackendRef)
|
|
|
|
# # Start transaction (double frame for testing)
|
|
# xCheck db.txTop.isErr
|
|
# var tx = db.txBegin().value.to(AristoDbRef).txBegin().value
|
|
# xCheck tx.isTop()
|
|
# xCheck tx.level == 2
|
|
|
|
# # Reset database so that the next round has a clean setup
|
|
# defer: db.innerCleanUp
|
|
|
|
# # Merge leaf data into main trie (w/vertex ID 2)
|
|
# let kvpLeafs = block:
|
|
# var lst = w.kvpLst.mapRootVid testRootVid
|
|
# # The list might be reduced for isolation of particular properties,
|
|
# # e.g. lst.setLen(min(5,lst.len))
|
|
# lst
|
|
# for i,leaf in kvpLeafs:
|
|
# let rc = db.mergeGenericData leaf
|
|
# xCheckRc rc.error == 0
|
|
|
|
# # List of all leaf entries that should be on the database
|
|
# var leafsLeft = kvpLeafs.mapIt(it.leafTie).toHashSet
|
|
|
|
# # Provide a (reproducible) peudo-random copy of the leafs list
|
|
# let leafVidPairs = block:
|
|
# let rc = db.randomisedLeafs(leafsLeft, prng)
|
|
# xCheckRc rc.error == (0,0)
|
|
# rc.value
|
|
# discard leafVidPairs
|
|
|
|
# # === delete sub-tree ===
|
|
# block:
|
|
# let saveBeOk = tx.saveToBackend(relax=false, noisy=noisy, 1+list.len*n)
|
|
# xCheck saveBeOk:
|
|
# noisy.say "***", "del(1)",
|
|
# " n=", n, "/", list.len,
|
|
# "\n db\n ", db.pp(backendOk=true),
|
|
# ""
|
|
# # Delete sub-tree
|
|
# block:
|
|
# let rc = db.deleteGenericTree testRootVid
|
|
# xCheckRc rc.error == 0:
|
|
# noisy.say "***", "del(2)",
|
|
# " n=", n, "/", list.len,
|
|
# "\n db\n ", db.pp(backendOk=true),
|
|
# ""
|
|
# block:
|
|
# let saveBeOk = tx.saveToBackend(relax=false, noisy=noisy, 2+list.len*n)
|
|
# xCheck saveBeOk:
|
|
# noisy.say "***", "del(3)",
|
|
# " n=", n, "/", list.len,
|
|
# "\n db\n ", db.pp(backendOk=true),
|
|
# ""
|
|
# when true and false:
|
|
# noisy.say "***", "del(9) n=", n, "/", list.len, " nLeafs=", kvpLeafs.len
|
|
|
|
true
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|