mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-01-23 02:29:26 +00:00
Aristo db fixes after storage slots dump tests added (#1595)
* Fix missing Merkle key removal in `merge()` * Accept optional root hash argument in `hashify()` why: For importing a full database, there will be no proof data except the root key. So this can be used to check and set the root key in the database descriptor. also: Associate vertex ID to `hashify()` error return code * Added Aristo Trie traversal function why: * step along leaf vertices in sorted order * tree/trie consistency checks when debugging * Enabled storage slots test data for Aristo DB
This commit is contained in:
Jordan Hrycaj
GitHub
parent
cbd593f514
commit
099444ab3f
@ -90,6 +90,7 @@ type
|
||||
HashifyExistingHashMismatch
|
||||
HashifyLeafToRootAllFailed
|
||||
HashifyRootHashMismatch
|
||||
HashifyRootVidMismatch
|
||||
|
||||
HashifyCheckRevCountMismatch
|
||||
HashifyCheckRevHashMismatch
|
||||
@ -103,4 +104,18 @@ type
|
||||
HashifyCheckVtxIncomplete
|
||||
HashifyCheckVtxLockWithoutKey
|
||||
|
||||
# Neighbour vertex, tree traversal `nearbyRight()` and `nearbyLeft()`
|
||||
NearbyBeyondRange
|
||||
NearbyBranchError
|
||||
NearbyDanglingLink
|
||||
NearbyEmptyHike
|
||||
NearbyExtensionError
|
||||
NearbyFailed
|
||||
NearbyBranchExpected
|
||||
NearbyLeafExpected
|
||||
NearbyNestingTooDeep
|
||||
NearbyPathTailUnexpected
|
||||
NearbyPathTailInxOverflow
|
||||
NearbyUnexpectedVtx
|
||||
|
||||
# End
|
||||
|
||||
@ -99,7 +99,7 @@ proc toNode(vtx: VertexRef; db: AristoDbRef): Result[NodeRef,void] =
|
||||
proc leafToRootHasher(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
hike: Hike; # Hike for labelling leaf..root
|
||||
): Result[int,AristoError] =
|
||||
): Result[int,(VertexID,AristoError)] =
|
||||
## Returns the index of the first node that could not be hashed
|
||||
for n in (hike.legs.len-1).countDown(0):
|
||||
let
|
||||
@ -120,7 +120,7 @@ proc leafToRootHasher(
|
||||
elif key != vfyKey:
|
||||
let error = HashifyExistingHashMismatch
|
||||
debug "hashify failed", vid=wp.vid, key, expected=vfyKey, error
|
||||
return err(error)
|
||||
return err((wp.vid,error))
|
||||
|
||||
ok -1 # all could be hashed
|
||||
|
||||
@ -141,13 +141,13 @@ proc hashifyClear*(
|
||||
|
||||
proc hashify*(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
): Result[NodeKey,AristoError] =
|
||||
rootKey = EMPTY_ROOT_KEY; # Optional root key
|
||||
): Result[NodeKey,(VertexID,AristoError)] =
|
||||
## Add keys to the `Patricia Trie` so that it becomes a `Merkle Patricia
|
||||
## Tree`. If successful, the function returns the key (aka Merkle hash) of
|
||||
## the root vertex.
|
||||
var
|
||||
fullPath = false
|
||||
rootKey: NodeKey
|
||||
thisRootKey = EMPTY_ROOT_KEY
|
||||
|
||||
# Width-first leaf-to-root traversal structure
|
||||
backLink: Table[VertexID,VertexID]
|
||||
@ -156,7 +156,7 @@ proc hashify*(
|
||||
for (pathTag,vid) in db.lTab.pairs:
|
||||
let hike = pathTag.hikeUp(db.lRoot,db)
|
||||
if hike.error != AristoError(0):
|
||||
return err(hike.error)
|
||||
return err((VertexID(0),hike.error))
|
||||
|
||||
# Hash as much of the `hike` as possible
|
||||
let n = block:
|
||||
@ -178,13 +178,22 @@ proc hashify*(
|
||||
for u in (n-1).countDown(1):
|
||||
backLink[hike.legs[u].wp.vid] = hike.legs[u-1].wp.vid
|
||||
|
||||
elif not fullPath:
|
||||
rootKey = db.kMap.getOrDefault(hike.legs[0].wp.vid, EMPTY_ROOT_KEY)
|
||||
fullPath = (rootKey != EMPTY_ROOT_KEY)
|
||||
elif thisRootKey == EMPTY_ROOT_KEY:
|
||||
let rootVid = hike.legs[0].wp.vid
|
||||
thisRootKey = db.kMap.getOrDefault(rootVid, EMPTY_ROOT_KEY)
|
||||
|
||||
if thisRootKey != EMPTY_ROOT_KEY:
|
||||
if rootKey != EMPTY_ROOT_KEY and rootKey != thisRootKey:
|
||||
return err((rootVid, HashifyRootHashMismatch))
|
||||
|
||||
if db.lRoot == VertexID(0):
|
||||
db.lRoot = rootVid
|
||||
elif db.lRoot != rootVid:
|
||||
return err((rootVid,HashifyRootVidMismatch))
|
||||
|
||||
# At least one full path leaf..root should have succeeded with labelling
|
||||
if not fullPath:
|
||||
return err(HashifyLeafToRootAllFailed)
|
||||
if thisRootKey == EMPTY_ROOT_KEY:
|
||||
return err((VertexID(0),HashifyLeafToRootAllFailed))
|
||||
|
||||
# Update remaining hashes
|
||||
var n = 0 # for logging
|
||||
@ -216,7 +225,7 @@ proc hashify*(
|
||||
let error = HashifyExistingHashMismatch
|
||||
debug "hashify failed", vid=fromVid, key=nodeKey,
|
||||
expected=fromKey.pp, error
|
||||
return err(error)
|
||||
return err((fromVid,error))
|
||||
|
||||
done.incl fromVid
|
||||
|
||||
@ -228,14 +237,14 @@ proc hashify*(
|
||||
# Make sure that the algorithm proceeds
|
||||
if done.len == 0:
|
||||
let error = HashifyCannotComplete
|
||||
return err(error)
|
||||
return err((VertexID(0),error))
|
||||
|
||||
# Clean up dups from `backLink` and restart `downMost`
|
||||
for vid in done.items:
|
||||
backLink.del vid
|
||||
downMost = redo
|
||||
|
||||
ok rootKey
|
||||
ok thisRootKey
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public debugging functions
|
||||
|
||||
@ -375,6 +375,13 @@ proc topIsEmptyAddLeaf(
|
||||
let nibble = hike.tail[0].int8
|
||||
if not rootVtx.bVid[nibble].isZero:
|
||||
return Hike(error: MergeRootBranchLinkBusy)
|
||||
|
||||
# Clear Merkle hashes (aka node keys) unless proof mode
|
||||
if db.pPrf.len == 0:
|
||||
db.clearMerkleKeys(hike, hike.root)
|
||||
elif hike.root in db.pPrf:
|
||||
return Hike(error: MergeBranchProofModeLock)
|
||||
|
||||
let
|
||||
leafVid = db.vidFetch
|
||||
leafVtx = VertexRef(
|
||||
|
||||
484
nimbus/db/aristo/aristo_nearby.nim
Normal file
484
nimbus/db/aristo/aristo_nearby.nim
Normal file
@ -0,0 +1,484 @@
|
||||
# nimbus-eth1
|
||||
# Copyright (c) 2021 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
## Aristo DB -- Patricia Trie traversal
|
||||
## ====================================
|
||||
##
|
||||
## This module provides tools to visit leaf vertices in a monotone order,
|
||||
## increasing or decreasing. These tools are intended for
|
||||
## * boundary proof verification
|
||||
## * step along leaf vertices in sorted order
|
||||
## * tree/trie consistency checks when debugging
|
||||
##
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import
|
||||
std/tables,
|
||||
eth/[common, trie/nibbles],
|
||||
stew/results,
|
||||
"."/[aristo_desc, aristo_error, aristo_get, aristo_hike, aristo_path]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc `<=`(a, b: NibblesSeq): bool =
|
||||
## Compare nibbles, different lengths are padded to the right with zeros
|
||||
let abMin = min(a.len, b.len)
|
||||
for n in 0 ..< abMin:
|
||||
if a[n] < b[n]:
|
||||
return true
|
||||
if b[n] < a[n]:
|
||||
return false
|
||||
# otherwise a[n] == b[n]
|
||||
|
||||
# Assuming zero for missing entries
|
||||
if b.len < a.len:
|
||||
for n in abMin + 1 ..< a.len:
|
||||
if 0 < a[n]:
|
||||
return false
|
||||
true
|
||||
|
||||
proc `<`(a, b: NibblesSeq): bool =
|
||||
not (b <= a)
|
||||
|
||||
# ------------------
|
||||
|
||||
proc branchNibbleMin*(vtx: VertexRef; minInx: int8): int8 =
|
||||
## Find the least index for an argument branch `vtx` link with index
|
||||
## greater or equal the argument `nibble`.
|
||||
if vtx.vType == Branch:
|
||||
for n in minInx .. 15:
|
||||
if not vtx.bVid[n].isZero:
|
||||
return n
|
||||
-1
|
||||
|
||||
proc branchNibbleMax*(vtx: VertexRef; maxInx: int8): int8 =
|
||||
## Find the greatest index for an argument branch `vtx` link with index
|
||||
## less or equal the argument `nibble`.
|
||||
if vtx.vType == Branch:
|
||||
for n in maxInx.countDown 0:
|
||||
if not vtx.bVid[n].isZero:
|
||||
return n
|
||||
-1
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc complete(
|
||||
hike: Hike; # Partially expanded path
|
||||
vid: VertexID; # Start ID
|
||||
db: AristoDbRef; # Database layer
|
||||
hikeLenMax: static[int]; # Beware of loops (if any)
|
||||
doLeast: static[bool]; # Direction: *least* or *most*
|
||||
): Hike =
|
||||
## Extend `hike` using least or last vertex without recursion.
|
||||
var
|
||||
vid = vid
|
||||
vtx = db.getVtx vid
|
||||
uHike = Hike(root: hike.root, legs: hike.legs)
|
||||
if vtx.isNil:
|
||||
return Hike(error: GetVtxNotFound)
|
||||
|
||||
while uHike.legs.len < hikeLenMax:
|
||||
var leg = Leg(wp: VidVtxPair(vid: vid, vtx: vtx), nibble: -1)
|
||||
|
||||
case vtx.vType:
|
||||
of Leaf:
|
||||
uHike.legs.add leg
|
||||
return uHike # done
|
||||
|
||||
of Extension:
|
||||
vid = vtx.eVid
|
||||
if not vid.isZero:
|
||||
vtx = db.getVtx vid
|
||||
if not vtx.isNil:
|
||||
uHike.legs.add leg
|
||||
continue
|
||||
return Hike(error: NearbyExtensionError) # Oops, no way
|
||||
|
||||
of Branch:
|
||||
when doLeast:
|
||||
leg.nibble = vtx.branchNibbleMin 0
|
||||
else:
|
||||
leg.nibble = vtx.branchNibbleMax 15
|
||||
if 0 <= leg.nibble:
|
||||
vid = vtx.bVid[leg.nibble]
|
||||
vtx = db.getVtx vid
|
||||
if not vtx.isNil:
|
||||
uHike.legs.add leg
|
||||
continue
|
||||
return Hike(error: NearbyBranchError) # Oops, no way
|
||||
|
||||
Hike(error: NearbyNestingTooDeep)
|
||||
|
||||
|
||||
proc zeroAdjust(
|
||||
hike: Hike; # Partially expanded path
|
||||
db: AristoDbRef; # Database layer
|
||||
doLeast: static[bool]; # Direction: *least* or *most*
|
||||
): Hike =
|
||||
## Adjust empty argument path to the first node entry to the right. Ths
|
||||
## applies is the argument `hike` is before the first entry in the database.
|
||||
## The result is a hike which is aligned with the first entry.
|
||||
proc accept(p: Hike; pfx: NibblesSeq): bool =
|
||||
when doLeast:
|
||||
p.tail <= pfx
|
||||
else:
|
||||
pfx <= p.tail
|
||||
|
||||
proc branchBorderNibble(w: VertexRef; n: int8): int8 =
|
||||
when doLeast:
|
||||
w.branchNibbleMin n
|
||||
else:
|
||||
w.branchNibbleMax n
|
||||
|
||||
proc toHike(pfx: NibblesSeq, root: VertexID, db: AristoDbRef): Hike =
|
||||
when doLeast:
|
||||
pfx.pathPfxPad(0).hikeUp(root, db)
|
||||
else:
|
||||
pfx.pathPfxPad(255).hikeUp(root, db)
|
||||
|
||||
if 0 < hike.legs.len:
|
||||
result = hike
|
||||
result.error = AristoError(0)
|
||||
return
|
||||
|
||||
let root = db.getVtx hike.root
|
||||
if not root.isNil:
|
||||
block fail:
|
||||
var pfx: NibblesSeq
|
||||
case root.vType:
|
||||
of Branch:
|
||||
# Find first non-dangling link and assign it
|
||||
if hike.tail.len == 0:
|
||||
break fail
|
||||
|
||||
let n = root.branchBorderNibble hike.tail[0].int8
|
||||
if n < 0:
|
||||
# Before or after the database range
|
||||
return Hike(error: NearbyBeyondRange)
|
||||
pfx = @[n.byte].initNibbleRange.slice(1)
|
||||
|
||||
of Extension:
|
||||
let ePfx = root.ePfx
|
||||
# Must be followed by a branch node
|
||||
if hike.tail.len < 2 or not hike.accept(ePfx):
|
||||
break fail
|
||||
let vtx = db.getVtx root.eVid
|
||||
if vtx.isNil:
|
||||
break fail
|
||||
let ePfxLen = ePfx.len
|
||||
if hike.tail.len <= ePfxLen:
|
||||
return Hike(error: NearbyPathTailInxOverflow)
|
||||
let tailPfx = hike.tail.slice(0,ePfxLen)
|
||||
when doLeast:
|
||||
if ePfx < tailPfx:
|
||||
return Hike(error: NearbyBeyondRange)
|
||||
else:
|
||||
if tailPfx < ePfx:
|
||||
return Hike(error: NearbyBeyondRange)
|
||||
pfx = ePfx
|
||||
|
||||
of Leaf:
|
||||
pfx = root.lPfx
|
||||
if not hike.accept(pfx):
|
||||
# Before or after the database range
|
||||
return Hike(error: NearbyBeyondRange)
|
||||
|
||||
var newHike = pfx.toHike(hike.root, db)
|
||||
if 0 < newHike.legs.len:
|
||||
newHike.error = AristoError(0)
|
||||
return newHike
|
||||
|
||||
Hike(error: NearbyEmptyHike)
|
||||
|
||||
|
||||
proc finalise(
|
||||
hike: Hike; # Partially expanded path
|
||||
db: AristoDbRef; # Database layer
|
||||
moveRight: static[bool]; # Direction of next node
|
||||
): Hike =
|
||||
## Handle some pathological cases after main processing failed
|
||||
proc beyond(p: Hike; pfx: NibblesSeq): bool =
|
||||
when moveRight:
|
||||
pfx < p.tail
|
||||
else:
|
||||
p.tail < pfx
|
||||
|
||||
proc branchBorderNibble(w: VertexRef): int8 =
|
||||
when moveRight:
|
||||
w.branchNibbleMax 15
|
||||
else:
|
||||
w.branchNibbleMin 0
|
||||
|
||||
# Just for completeness (this case should have been handled, already)
|
||||
if hike.legs.len == 0:
|
||||
return Hike(error: NearbyEmptyHike)
|
||||
|
||||
# Check whether the path is beyond the database range
|
||||
if 0 < hike.tail.len: # nothing to compare against, otherwise
|
||||
let top = hike.legs[^1]
|
||||
|
||||
# Note that only a `Branch` nodes has a non-zero nibble
|
||||
if 0 <= top.nibble and top.nibble == top.wp.vtx.branchBorderNibble:
|
||||
# Check the following up node
|
||||
let vtx = db.getVtx top.wp.vtx.bVid[top.nibble]
|
||||
if vtx.isNil:
|
||||
return Hike(error: NearbyDanglingLink)
|
||||
|
||||
var pfx: NibblesSeq
|
||||
case vtx.vType:
|
||||
of Leaf:
|
||||
pfx = vtx.lPfx
|
||||
of Extension:
|
||||
pfx = vtx.ePfx
|
||||
of Branch:
|
||||
pfx = @[vtx.branchBorderNibble.byte].initNibbleRange.slice(1)
|
||||
if hike.beyond pfx:
|
||||
return Hike(error: NearbyBeyondRange)
|
||||
|
||||
# Pathological cases
|
||||
# * finalise right: nfffff.. for n < f or
|
||||
# * finalise left: n00000.. for 0 < n
|
||||
if hike.legs[0].wp.vtx.vType == Branch or
|
||||
(1 < hike.legs.len and hike.legs[1].wp.vtx.vType == Branch):
|
||||
return Hike(error: NearbyFailed) # no more nodes
|
||||
|
||||
Hike(error: NearbyUnexpectedVtx) # error
|
||||
|
||||
|
||||
proc nearbyNext(
|
||||
hike: Hike; # Partially expanded path
|
||||
db: AristoDbRef; # Database layer
|
||||
hikeLenMax: static[int]; # Beware of loops (if any)
|
||||
moveRight: static[bool]; # Direction of next node
|
||||
): Hike =
|
||||
## Unified implementation of `nearbyRight()` and `nearbyLeft()`.
|
||||
proc accept(nibble: int8): bool =
|
||||
## Accept `nibble` unless on boundaty dependent on `moveRight`
|
||||
when moveRight:
|
||||
nibble < 15
|
||||
else:
|
||||
0 < nibble
|
||||
|
||||
proc accept(p: Hike; pfx: NibblesSeq): bool =
|
||||
when moveRight:
|
||||
p.tail <= pfx
|
||||
else:
|
||||
pfx <= p.tail
|
||||
|
||||
proc branchNibbleNext(w: VertexRef; n: int8): int8 =
|
||||
when moveRight:
|
||||
w.branchNibbleMin(n + 1)
|
||||
else:
|
||||
w.branchNibbleMax(n - 1)
|
||||
|
||||
# Some easy cases
|
||||
var hike = hike.zeroAdjust(db, doLeast=moveRight)
|
||||
if hike.error != AristoError(0):
|
||||
return hike
|
||||
|
||||
if hike.legs[^1].wp.vtx.vType == Extension:
|
||||
let vid = hike.legs[^1].wp.vtx.eVid
|
||||
return hike.complete(vid, db, hikeLenMax, doLeast=moveRight)
|
||||
|
||||
var
|
||||
uHike = hike
|
||||
start = true
|
||||
while 0 < uHike.legs.len:
|
||||
let top = uHike.legs[^1]
|
||||
case top.wp.vtx.vType:
|
||||
of Leaf:
|
||||
return uHike
|
||||
of Branch:
|
||||
if top.nibble < 0 or uHike.tail.len == 0:
|
||||
return Hike(error: NearbyUnexpectedVtx)
|
||||
of Extension:
|
||||
uHike.tail = top.wp.vtx.ePfx & uHike.tail
|
||||
uHike.legs.setLen(uHike.legs.len - 1)
|
||||
continue
|
||||
|
||||
var
|
||||
step = top
|
||||
let
|
||||
uHikeLen = uHike.legs.len # in case of backtracking
|
||||
uHikeTail = uHike.tail # in case of backtracking
|
||||
|
||||
# Look ahead checking next node
|
||||
if start:
|
||||
let vid = top.wp.vtx.bVid[top.nibble]
|
||||
if vid.isZero:
|
||||
return Hike(error: NearbyDanglingLink) # error
|
||||
|
||||
let vtx = db.getVtx vid
|
||||
if vtx.isNil:
|
||||
return Hike(error: GetVtxNotFound) # error
|
||||
|
||||
case vtx.vType
|
||||
of Leaf:
|
||||
if uHike.accept vtx.lPfx:
|
||||
return uHike.complete(vid, db, hikeLenMax, doLeast=moveRight)
|
||||
of Extension:
|
||||
if uHike.accept vtx.ePfx:
|
||||
return uHike.complete(vid, db, hikeLenMax, doLeast=moveRight)
|
||||
of Branch:
|
||||
let nibble = uHike.tail[0].int8
|
||||
if start and accept nibble:
|
||||
# Step down and complete with a branch link on the child node
|
||||
step = Leg(wp: VidVtxPair(vid: vid, vtx: vtx), nibble: nibble)
|
||||
uHike.legs.add step
|
||||
|
||||
# Find the next item to the right/left of the current top entry
|
||||
let n = step.wp.vtx.branchNibbleNext step.nibble
|
||||
if 0 <= n:
|
||||
uHike.legs[^1].nibble = n
|
||||
return uHike.complete(
|
||||
step.wp.vtx.bVid[n], db, hikeLenMax, doLeast=moveRight)
|
||||
|
||||
if start:
|
||||
# Retry without look ahead
|
||||
start = false
|
||||
|
||||
# Restore `uPath` (pop temporary extra step)
|
||||
if uHikeLen < uHike.legs.len:
|
||||
uHike.legs.setLen(uHikeLen)
|
||||
uHike.tail = uHikeTail
|
||||
else:
|
||||
# Pop current `Branch` node on top and append nibble to `tail`
|
||||
uHike.tail = @[top.nibble.byte].initNibbleRange.slice(1) & uHike.tail
|
||||
uHike.legs.setLen(uHike.legs.len - 1)
|
||||
# End while
|
||||
|
||||
# Handle some pathological cases
|
||||
return hike.finalise(db, moveRight)
|
||||
|
||||
|
||||
proc nearbyNext(
|
||||
baseTag: NodeTag; # Some node
|
||||
root: VertexID; # State root
|
||||
db: AristoDbRef; # Database layer
|
||||
hikeLenMax: static[int]; # Beware of loops (if any)
|
||||
moveRight:static[ bool]; # Direction of next node
|
||||
): Result[NodeTag,AristoError] =
|
||||
## Variant of `nearbyNext()`, convenience wrapper
|
||||
let hike = baseTag.hikeUp(root,db).nearbyNext(db, hikeLenMax, moveRight)
|
||||
if hike.error != AristoError(0):
|
||||
return err(hike.error)
|
||||
|
||||
if 0 < hike.legs.len and hike.legs[^1].wp.vtx.vType == Leaf:
|
||||
let rc = hike.legsTo(NibblesSeq).pathToKey
|
||||
if rc.isOk:
|
||||
return ok rc.value.to(NodeTag)
|
||||
return err(rc.error)
|
||||
|
||||
err(NearbyLeafExpected)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public functions, moving and right boundary proof
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc nearbyRight*(
|
||||
hike: Hike; # Partially expanded path
|
||||
db: AristoDbRef; # Database layer
|
||||
): Hike =
|
||||
## Extends the maximally extended argument nodes `hike` to the right (i.e.
|
||||
## with non-decreasing path value). This function does not backtrack if
|
||||
## there are dangling links in between. It will return an error in that case.
|
||||
##
|
||||
## If there is no more leaf node to the right of the argument `hike`, the
|
||||
## particular error code `NearbyBeyondRange` is returned.
|
||||
##
|
||||
## This code is intended to be used for verifying a left-bound proof to
|
||||
## verify that there is no leaf node *right* of a boundary path value.
|
||||
hike.nearbyNext(db, 64, moveRight=true)
|
||||
|
||||
proc nearbyRight*(
|
||||
nodeTag: NodeTag; # Some node
|
||||
root: VertexID; # State root
|
||||
db: AristoDbRef; # Database layer
|
||||
): Result[NodeTag,AristoError] =
|
||||
## Variant of `nearbyRight()` working with a `NodeTag` argument instead
|
||||
## of a `Hike`.
|
||||
nodeTag.nearbyNext(root, db, 64, moveRight=true)
|
||||
|
||||
proc nearbyLeft*(
|
||||
hike: Hike; # Partially expanded path
|
||||
db: AristoDbRef; # Database layer
|
||||
): Hike =
|
||||
## Similar to `nearbyRight()`.
|
||||
##
|
||||
## This code is intended to be used for verifying a right-bound proof to
|
||||
## verify that there is no leaf node *left* to a boundary path value.
|
||||
hike.nearbyNext(db, 64, moveRight=false)
|
||||
|
||||
proc nearbyLeft*(
|
||||
nodeTag: NodeTag; # Some node
|
||||
root: VertexID; # State root
|
||||
db: AristoDbRef; # Database layer
|
||||
): Result[NodeTag,AristoError] =
|
||||
## Similar to `nearbyRight()` for `NodeTag` argument instead
|
||||
## of a `Hike`.
|
||||
nodeTag.nearbyNext(root, db, 64, moveRight=false)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public debugging helpers
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc nearbyRightMissing*(
|
||||
hike: Hike; # Partially expanded path
|
||||
db: AristoDbRef; # Database layer
|
||||
): Result[bool,AristoError] =
|
||||
## Returns `true` if the maximally extended argument nodes `hike` is the
|
||||
## rightmost on the hexary trie database. It verifies that there is no more
|
||||
## leaf entry to the right of the argument `hike`. This function is an
|
||||
## an alternative to
|
||||
## ::
|
||||
## let rc = path.nearbyRight(db)
|
||||
## if rc.isOk:
|
||||
## # not at the end => false
|
||||
## ...
|
||||
## elif rc.error != NearbyBeyondRange:
|
||||
## # problem with database => error
|
||||
## ...
|
||||
## else:
|
||||
## # no nore nodes => true
|
||||
## ...
|
||||
## and is intended mainly for debugging.
|
||||
if hike.legs.len == 0:
|
||||
return err(NearbyEmptyHike)
|
||||
if 0 < hike.tail.len:
|
||||
return err(NearbyPathTailUnexpected)
|
||||
|
||||
let top = hike.legs[^1]
|
||||
if top.wp.vtx.vType != Branch or top.nibble < 0:
|
||||
return err(NearbyBranchError)
|
||||
|
||||
let vid = top.wp.vtx.bVid[top.nibble]
|
||||
if vid.isZero:
|
||||
return err(NearbyDanglingLink) # error
|
||||
|
||||
let vtx = db.getVtx vid
|
||||
if vtx.isNil:
|
||||
return err(GetVtxNotFound) # error
|
||||
|
||||
case vtx.vType
|
||||
of Leaf:
|
||||
return ok(vtx.lPfx < hike.tail)
|
||||
of Extension:
|
||||
return ok(vtx.ePfx < hike.tail)
|
||||
of Branch:
|
||||
return ok(vtx.branchNibbleMin(hike.tail[0].int8) < 0)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
# ------------------------------------------------------------------------------
|
||||
@ -11,6 +11,7 @@
|
||||
{.push raises: [].}
|
||||
|
||||
import
|
||||
std/sequtils,
|
||||
eth/[common, trie/nibbles],
|
||||
stew/results,
|
||||
"."/[aristo_constants, aristo_desc, aristo_error]
|
||||
@ -39,7 +40,7 @@ proc pathAsNibbles*(key: NodeKey): NibblesSeq =
|
||||
proc pathAsNibbles*(tag: NodeTag): NibblesSeq =
|
||||
tag.to(NodeKey).pathAsNibbles()
|
||||
|
||||
proc pathAsBlob*(keyOrTag: NodeKey|Nodetag): Blob =
|
||||
proc pathAsBlob*(keyOrTag: NodeKey|NodeTag): Blob =
|
||||
keyOrTag.pathAsNibbles.hexPrefixEncode(isLeaf=true)
|
||||
|
||||
|
||||
@ -64,6 +65,33 @@ proc pathToTag*(partPath: NibblesSeq|Blob): Result[NodeTag,AristoError] =
|
||||
return ok(rc.value.to(NodeTag))
|
||||
err(rc.error)
|
||||
|
||||
# --------------------
|
||||
|
||||
proc pathPfxPad*(pfx: NibblesSeq; dblNibble: static[byte]): NodeKey =
|
||||
## Extend (or cut) the argument nibbles sequence `pfx` for generating a
|
||||
## `NodeKey`.
|
||||
##
|
||||
## This function must be handled with some care regarding a meaningful value
|
||||
## for the `dblNibble` argument. Currently, only static values `0` and `255`
|
||||
## are allowed for padding. This is checked at compile time.
|
||||
static:
|
||||
doAssert dblNibble == 0 or dblNibble == 255
|
||||
|
||||
# Pad with zeroes
|
||||
var padded: NibblesSeq
|
||||
|
||||
let padLen = 64 - pfx.len
|
||||
if 0 <= padLen:
|
||||
padded = pfx & dblNibble.repeat(padlen div 2).mapIt(it.byte).initNibbleRange
|
||||
if (padLen and 1) == 1:
|
||||
padded = padded & @[dblNibble.byte].initNibbleRange.slice(1)
|
||||
else:
|
||||
let nope = seq[byte].default.initNibbleRange
|
||||
padded = pfx.slice(0,64) & nope # nope forces re-alignment
|
||||
|
||||
let bytes = padded.getBytes
|
||||
(addr result.ByteArray32[0]).copyMem(unsafeAddr bytes[0], bytes.len)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
@ -18,13 +18,12 @@ import
|
||||
rocksdb,
|
||||
unittest2,
|
||||
../nimbus/db/select_backend,
|
||||
../nimbus/db/aristo/[aristo_desc],
|
||||
../nimbus/db/aristo/[aristo_desc, aristo_error, aristo_merge],
|
||||
../nimbus/core/chain,
|
||||
../nimbus/sync/snap/worker/db/[
|
||||
hexary_desc, rocky_bulk_load, snapdb_accounts, snapdb_desc],
|
||||
./replay/[pp, undump_accounts],
|
||||
../nimbus/sync/snap/worker/db/[rocky_bulk_load, snapdb_accounts, snapdb_desc],
|
||||
./replay/[pp, undump_accounts, undump_storages],
|
||||
./test_sync_snap/[snap_test_xx, test_accounts, test_types],
|
||||
./test_aristo/[test_merge, test_transcode]
|
||||
./test_aristo/[test_helpers, test_merge, test_nearby, test_transcode]
|
||||
|
||||
const
|
||||
baseDir = [".", "..", ".."/"..", $DirSep]
|
||||
@ -36,6 +35,7 @@ const
|
||||
|
||||
# Standard test samples
|
||||
accSample = snapTest0
|
||||
storSample = snapTest4
|
||||
|
||||
# Number of database slots available
|
||||
nTestDbInstances = 9
|
||||
@ -91,22 +91,6 @@ proc setErrorLevel {.used.} =
|
||||
# Private functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc to(sample: AccountsSample; T: type seq[UndumpAccounts]): T =
|
||||
## Convert test data into usable in-memory format
|
||||
let file = sample.file.findFilePath.value
|
||||
var root: Hash256
|
||||
for w in file.undumpNextAccount:
|
||||
let n = w.seenAccounts - 1
|
||||
if n < sample.firstItem:
|
||||
continue
|
||||
if sample.lastItem < n:
|
||||
break
|
||||
if sample.firstItem == n:
|
||||
root = w.root
|
||||
elif w.root != root:
|
||||
break
|
||||
result.add w
|
||||
|
||||
proc flushDbDir(s: string; subDir = "") =
|
||||
if s != "":
|
||||
let baseDir = s / "tmp"
|
||||
@ -162,7 +146,7 @@ proc snapDbAccountsRef(cdb:ChainDb; root:Hash256; pers:bool):SnapDbAccountsRef =
|
||||
# Test Runners: accounts and accounts storages
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc transcodeRunner(noisy = true; sample = accSample; stopAfter = high(int)) =
|
||||
proc transcodeRunner(noisy =true; sample=accSample; stopAfter=high(int)) =
|
||||
let
|
||||
accLst = sample.to(seq[UndumpAccounts])
|
||||
root = accLst[0].root
|
||||
@ -199,19 +183,45 @@ proc transcodeRunner(noisy = true; sample = accSample; stopAfter = high(int)) =
|
||||
noisy.test_transcodeAccounts(db.cdb[0].rocksStoreRef, stopAfter)
|
||||
|
||||
|
||||
proc dataRunner(noisy = true; sample = accSample) =
|
||||
proc accountsRunner(noisy=true; sample=accSample, resetDb=false) =
|
||||
let
|
||||
accLst = sample.to(seq[UndumpAccounts])
|
||||
accLst = sample.to(seq[UndumpAccounts]).to(seq[ProofTrieData])
|
||||
fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")
|
||||
listMode = if resetDb: "" else: ", merged data lists"
|
||||
|
||||
suite &"Aristo: accounts data import from {fileInfo}":
|
||||
suite &"Aristo: accounts data dump from {fileInfo}{listMode}":
|
||||
|
||||
test &"Merge {accLst.len} account lists to database":
|
||||
noisy.test_mergeAccounts accLst
|
||||
noisy.test_mergeKvpList(accLst, resetDb)
|
||||
|
||||
test &"Merge {accLst.len} proof & account lists to database":
|
||||
noisy.test_mergeProofsAndAccounts accLst
|
||||
noisy.test_mergeProofAndKvpList(accLst, resetDb)
|
||||
|
||||
test &"Traverse accounts database w/{accLst.len} account lists":
|
||||
noisy.test_nearbyKvpList(accLst, resetDb)
|
||||
|
||||
|
||||
proc storagesRunner(
|
||||
noisy = true;
|
||||
sample = storSample;
|
||||
resetDb = false;
|
||||
oops: KnownHasherFailure = @[];
|
||||
) =
|
||||
let
|
||||
stoLst = sample.to(seq[UndumpStorages]).to(seq[ProofTrieData])
|
||||
fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")
|
||||
listMode = if resetDb: "" else: ", merged data lists"
|
||||
|
||||
suite &"Aristo: storages data dump from {fileInfo}{listMode}":
|
||||
|
||||
test &"Merge {stoLst.len} storage slot lists to database":
|
||||
noisy.test_mergeKvpList(stoLst, resetDb)
|
||||
|
||||
test &"Merge {stoLst.len} proof & slots lists to database":
|
||||
noisy.test_mergeProofAndKvpList(stoLst, resetDb, fileInfo, oops)
|
||||
|
||||
test &"Traverse storage slots database w/{stoLst.len} account lists":
|
||||
noisy.test_nearbyKvpList(stoLst, resetDb)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Main function(s)
|
||||
@ -219,14 +229,15 @@ proc dataRunner(noisy = true; sample = accSample) =
|
||||
|
||||
proc aristoMain*(noisy = defined(debug)) =
|
||||
noisy.transcodeRunner()
|
||||
noisy.dataRunner()
|
||||
noisy.accountsRunner()
|
||||
noisy.storagesRunner()
|
||||
|
||||
when isMainModule:
|
||||
const
|
||||
noisy = defined(debug) or true
|
||||
|
||||
# Borrowed from `test_sync_snap.nim`
|
||||
when true: # and false:
|
||||
when true and false:
|
||||
for n,sam in snapTestList:
|
||||
noisy.transcodeRunner(sam)
|
||||
for n,sam in snapTestStorageList:
|
||||
@ -235,22 +246,27 @@ when isMainModule:
|
||||
# This one uses dumps from the external `nimbus-eth1-blob` repo
|
||||
when true and false:
|
||||
import ./test_sync_snap/snap_other_xx
|
||||
noisy.showElapsed("dataRunner() @snap_other_xx"):
|
||||
noisy.showElapsed("@snap_other_xx"):
|
||||
for n,sam in snapOtherList:
|
||||
noisy.dataRunner(sam)
|
||||
noisy.accountsRunner(sam)
|
||||
|
||||
# This one usues dumps from the external `nimbus-eth1-blob` repo
|
||||
when true and false:
|
||||
when true: # and false:
|
||||
import ./test_sync_snap/snap_storage_xx
|
||||
noisy.showElapsed("dataRunner() @snap_storage_xx"):
|
||||
let knownFailures: KnownHasherFailure = @[
|
||||
("storages5__34__41_dump#10.20512",(VertexID(1),HashifyRootHashMismatch)),
|
||||
]
|
||||
noisy.showElapsed("@snap_storage_xx"):
|
||||
for n,sam in snapStorageList:
|
||||
noisy.dataRunner(sam)
|
||||
noisy.accountsRunner(sam)
|
||||
noisy.storagesRunner(sam,oops=knownFailures)
|
||||
|
||||
when true: # and false:
|
||||
for n,sam in snapTestList:
|
||||
noisy.dataRunner(sam)
|
||||
noisy.accountsRunner(sam)
|
||||
for n,sam in snapTestStorageList:
|
||||
noisy.dataRunner(sam)
|
||||
noisy.accountsRunner(sam)
|
||||
noisy.storagesRunner(sam)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
|
||||
@ -13,9 +13,41 @@ import
|
||||
std/sequtils,
|
||||
eth/common,
|
||||
rocksdb,
|
||||
../../nimbus/db/aristo/[aristo_desc, aristo_merge],
|
||||
../../nimbus/db/kvstore_rocksdb,
|
||||
../../nimbus/sync/snap/constants,
|
||||
../replay/pp
|
||||
../../nimbus/sync/protocol/snap/snap_types,
|
||||
../../nimbus/sync/snap/[constants, range_desc],
|
||||
../test_sync_snap/test_types,
|
||||
../replay/[pp, undump_accounts, undump_storages]
|
||||
|
||||
type
|
||||
ProofTrieData* = object
|
||||
root*: NodeKey
|
||||
id*: int
|
||||
proof*: seq[SnapProof]
|
||||
kvpLst*: seq[LeafKVP]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc to(w: UndumpAccounts; T: type ProofTrieData): T =
|
||||
T(root: w.root.to(NodeKey),
|
||||
proof: w.data.proof,
|
||||
kvpLst: w.data.accounts.mapIt(LeafKVP(
|
||||
pathTag: it.accKey.to(NodeTag),
|
||||
payload: PayloadRef(pType: BlobData, blob: it.accBlob))))
|
||||
|
||||
proc to(s: UndumpStorages; id: int; T: type seq[ProofTrieData]): T =
|
||||
for w in s.data.storages:
|
||||
result.add ProofTrieData(
|
||||
root: w.account.storageRoot.to(NodeKey),
|
||||
id: id,
|
||||
kvpLst: w.data.mapIt(LeafKVP(
|
||||
pathTag: it.slotHash.to(NodeTag),
|
||||
payload: PayloadRef(pType: BlobData, blob: it.slotData))))
|
||||
if 0 < result.len:
|
||||
result[^1].proof = s.data.proof
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public helpers
|
||||
@ -30,6 +62,47 @@ proc say*(noisy = false; pfx = "***"; args: varargs[string, `$`]) =
|
||||
else:
|
||||
echo pfx, args.toSeq.join
|
||||
|
||||
# -----------------------
|
||||
|
||||
proc to*(sample: AccountsSample; T: type seq[UndumpAccounts]): T =
|
||||
## Convert test data into usable in-memory format
|
||||
let file = sample.file.findFilePath.value
|
||||
var root: Hash256
|
||||
for w in file.undumpNextAccount:
|
||||
let n = w.seenAccounts - 1
|
||||
if n < sample.firstItem:
|
||||
continue
|
||||
if sample.lastItem < n:
|
||||
break
|
||||
if sample.firstItem == n:
|
||||
root = w.root
|
||||
elif w.root != root:
|
||||
break
|
||||
result.add w
|
||||
|
||||
proc to*(sample: AccountsSample; T: type seq[UndumpStorages]): T =
|
||||
## Convert test data into usable in-memory format
|
||||
let file = sample.file.findFilePath.value
|
||||
var root: Hash256
|
||||
for w in file.undumpNextStorages:
|
||||
let n = w.seenAccounts - 1 # storages selector based on accounts
|
||||
if n < sample.firstItem:
|
||||
continue
|
||||
if sample.lastItem < n:
|
||||
break
|
||||
if sample.firstItem == n:
|
||||
root = w.root
|
||||
elif w.root != root:
|
||||
break
|
||||
result.add w
|
||||
|
||||
proc to*(w: seq[UndumpAccounts]; T: type seq[ProofTrieData]): T =
|
||||
w.mapIt(it.to(ProofTrieData))
|
||||
|
||||
proc to*(s: seq[UndumpStorages]; T: type seq[ProofTrieData]): T =
|
||||
for n,w in s:
|
||||
result &= w.to(n,seq[ProofTrieData])
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public iterators
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
@ -12,28 +12,28 @@
|
||||
## Aristo (aka Patricia) DB records merge test
|
||||
|
||||
import
|
||||
std/sequtils,
|
||||
eth/common,
|
||||
stew/results,
|
||||
unittest2,
|
||||
../../nimbus/db/aristo/[
|
||||
aristo_desc, aristo_debug, aristo_error, aristo_hashify,
|
||||
aristo_desc, aristo_debug, aristo_error, aristo_get, aristo_hashify,
|
||||
aristo_hike, aristo_merge],
|
||||
../../nimbus/sync/snap/range_desc,
|
||||
../replay/undump_accounts,
|
||||
./test_helpers
|
||||
|
||||
type
|
||||
KnownHasherFailure* = seq[(string,(VertexID,AristoError))]
|
||||
## (<sample-name> & "#" <instance>, @[(<slot-id>, <error-symbol>)), ..])
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc to(w: PackedAccount; T: type LeafKVP): T =
|
||||
T(pathTag: w.accKey.to(NodeTag),
|
||||
payload: PayloadRef(pType: BlobData, blob: w.accBlob))
|
||||
|
||||
proc to[T](w: openArray[PackedAccount]; W: type seq[T]): W =
|
||||
w.toSeq.mapIt(it.to(T))
|
||||
|
||||
proc pp(w: tuple[merged: int, dups: int, error: AristoError]): string =
|
||||
result = "(merged: " & $w.merged & ", dups: " & $w.dups
|
||||
if w.error != AristoError(0):
|
||||
result &= ", error: " & $w.error
|
||||
result &= ")"
|
||||
|
||||
proc mergeStepwise(
|
||||
db: AristoDbRef;
|
||||
@ -100,7 +100,7 @@ proc mergeStepwise(
|
||||
check 0 < ekih.legs.len
|
||||
elif ekih.legs[^1].wp.vtx.vType != Leaf:
|
||||
check ekih.legs[^1].wp.vtx.vType == Leaf
|
||||
else:
|
||||
elif hike.error != MergeLeafPathCachedAlready:
|
||||
check ekih.legs[^1].wp.vtx.lData.blob == leaf.payload.blob
|
||||
|
||||
if db.lTab.len != lTabLen + merged:
|
||||
@ -118,44 +118,51 @@ proc mergeStepwise(
|
||||
# Public test function
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc test_mergeAccounts*(
|
||||
proc test_mergeKvpList*(
|
||||
noisy: bool;
|
||||
lst: openArray[UndumpAccounts];
|
||||
list: openArray[ProofTrieData];
|
||||
resetDb = false;
|
||||
) =
|
||||
let
|
||||
db = AristoDbRef()
|
||||
|
||||
for n,par in lst:
|
||||
var db = AristoDbRef()
|
||||
for n,w in list:
|
||||
if resetDb:
|
||||
db = AristoDbRef()
|
||||
let
|
||||
lstLen = list.len
|
||||
lTabLen = db.lTab.len
|
||||
leafs = par.data.accounts.to(seq[LeafKVP])
|
||||
leafs = w.kvpLst
|
||||
#prePreDb = db.pp
|
||||
added = db.merge leafs
|
||||
#added = db.mergeStepwise(leafs, noisy=false)
|
||||
#added = db.mergeStepwise(leafs, noisy=(6 < n))
|
||||
|
||||
check added.error == AristoError(0)
|
||||
check db.lTab.len == lTabLen + added.merged
|
||||
check added.merged + added.dups == leafs.len
|
||||
|
||||
let
|
||||
#preDb = db.pp
|
||||
preKMap = (db.kMap.len, db.pp(sTabOk=false, lTabOk=false))
|
||||
prePAmk = (db.pAmk.len, db.pAmk.pp(db))
|
||||
|
||||
block:
|
||||
let rc = db.hashify # (noisy=true)
|
||||
if rc.isErr: # or true:
|
||||
noisy.say "***", "<", n, "> db dump",
|
||||
noisy.say "***", "<", n, ">",
|
||||
" added=", added,
|
||||
" db dump",
|
||||
"\n pre-kMap(", preKMap[0], ")\n ", preKMap[1],
|
||||
#"\n pre-pre-DB", prePreDb, "\n --------\n pre-DB", preDb,
|
||||
"\n --------",
|
||||
"\n post-state ", db.pp,
|
||||
"\n"
|
||||
if rc.isErr:
|
||||
check rc.error == AristoError(0) # force message
|
||||
check rc.error == (VertexID(0),AristoError(0)) # force message
|
||||
return
|
||||
|
||||
block:
|
||||
let rc = db.hashifyCheck()
|
||||
if rc.isErr:
|
||||
noisy.say "***", "<", n, "/", lst.len-1, "> db dump",
|
||||
noisy.say "***", "<", n, "/", lstLen-1, "> db dump",
|
||||
"\n pre-kMap(", preKMap[0], ")\n ", preKMap[1],
|
||||
"\n --------",
|
||||
"\n pre-pAmk(", prePAmk[0], ")\n ", prePAmk[1],
|
||||
@ -166,65 +173,116 @@ proc test_mergeAccounts*(
|
||||
check rc == Result[void,(VertexID,AristoError)].ok()
|
||||
return
|
||||
|
||||
#noisy.say "***", "sample ",n,"/",lst.len-1," leafs merged: ", added.merged
|
||||
when true and false:
|
||||
noisy.say "***", "sample ", n, "/", lstLen-1,
|
||||
" leafs merged=", added.merged,
|
||||
" dup=", added.dups
|
||||
|
||||
|
||||
proc test_mergeProofsAndAccounts*(
|
||||
proc test_mergeProofAndKvpList*(
|
||||
noisy: bool;
|
||||
lst: openArray[UndumpAccounts];
|
||||
list: openArray[ProofTrieData];
|
||||
resetDb = false;
|
||||
idPfx = "";
|
||||
oops: KnownHasherFailure = @[];
|
||||
) =
|
||||
let
|
||||
db = AristoDbRef()
|
||||
var
|
||||
db = AristoDbRef(nil)
|
||||
rootKey = NodeKey.default
|
||||
count = 0
|
||||
for n,w in list:
|
||||
if resetDb or w.root != rootKey or w.proof.len == 0:
|
||||
db = AristoDbRef()
|
||||
rootKey = w.root
|
||||
count = 0
|
||||
count.inc
|
||||
|
||||
for n,par in lst:
|
||||
let
|
||||
testId = idPfx & "#" & $w.id & "." & $n
|
||||
oopsTab = oops.toTable
|
||||
lstLen = list.len
|
||||
sTabLen = db.sTab.len
|
||||
lTabLen = db.lTab.len
|
||||
leafs = par.data.accounts.to(seq[LeafKVP])
|
||||
leafs = w.kvpLst
|
||||
|
||||
noisy.say "***", "sample ", n, "/", lst.len-1, " start, nLeafs=", leafs.len
|
||||
when true and false:
|
||||
noisy.say "***", "sample <", n, "/", lstLen-1, ">",
|
||||
" groups=", count, " nLeafs=", leafs.len
|
||||
|
||||
let
|
||||
rootKey = par.root.to(NodeKey)
|
||||
proved = db.merge par.data.proof
|
||||
var proved: tuple[merged: int, dups: int, error: AristoError]
|
||||
if 0 < w.proof.len:
|
||||
proved = db.merge w.proof
|
||||
check proved.error in {AristoError(0),MergeNodeKeyCachedAlready}
|
||||
check w.proof.len == proved.merged + proved.dups
|
||||
check db.lTab.len == lTabLen
|
||||
check db.sTab.len == proved.merged + sTabLen
|
||||
check proved.merged < db.pAmk.len
|
||||
check proved.merged < db.kMap.len
|
||||
|
||||
check proved.error in {AristoError(0),MergeNodeKeyCachedAlready}
|
||||
check par.data.proof.len == proved.merged + proved.dups
|
||||
check db.lTab.len == lTabLen
|
||||
check db.sTab.len == proved.merged + sTabLen
|
||||
check proved.merged < db.pAmk.len
|
||||
check proved.merged < db.kMap.len
|
||||
|
||||
# Set up root ID
|
||||
db.lRoot = db.pAmk.getOrDefault(rootKey, VertexID(0))
|
||||
check db.lRoot != VertexID(0)
|
||||
|
||||
noisy.say "***", "sample ", n, "/", lst.len-1, " proved=", proved
|
||||
#noisy.say "***", "<", n, "/", lst.len-1, ">\n ", db.pp
|
||||
|
||||
let
|
||||
added = db.merge leafs
|
||||
#added = db.mergeStepwise(leafs, noisy=false)
|
||||
|
||||
check db.lTab.len == lTabLen + added.merged
|
||||
check added.merged + added.dups == leafs.len
|
||||
|
||||
block:
|
||||
if added.error notin {AristoError(0), MergeLeafPathCachedAlready}:
|
||||
noisy.say "***", "<", n, "/", lst.len-1, ">\n ", db.pp
|
||||
check added.error in {AristoError(0), MergeLeafPathCachedAlready}
|
||||
# Set up root ID
|
||||
db.lRoot = db.pAmk.getOrDefault(rootKey, VertexID(0))
|
||||
if db.lRoot == VertexID(0):
|
||||
check db.lRoot != VertexID(0)
|
||||
return
|
||||
|
||||
noisy.say "***", "sample ", n, "/", lst.len-1, " added=", added
|
||||
when true and false:
|
||||
noisy.say "***", "sample <", n, "/", lstLen-1, ">",
|
||||
" groups=", count, " nLeafs=", leafs.len, " proved=", proved
|
||||
|
||||
let
|
||||
merged = db.merge leafs
|
||||
#merged = db.mergeStepwise(leafs, noisy=false)
|
||||
|
||||
check db.lTab.len == lTabLen + merged.merged
|
||||
check merged.merged + merged.dups == leafs.len
|
||||
|
||||
if w.proof.len == 0:
|
||||
let vtx = db.getVtx VertexID(1)
|
||||
#check db.pAmk.getOrDefault(rootKey, VertexID(0)) != VertexID(0)
|
||||
|
||||
block:
|
||||
let rc = db.hashify # (noisy=false or (7 <= n))
|
||||
if rc.isErr:
|
||||
noisy.say "***", "<", n, "/", lst.len-1, ">\n ", db.pp
|
||||
check rc.error == AristoError(0)
|
||||
if merged.error notin {AristoError(0), MergeLeafPathCachedAlready}:
|
||||
noisy.say "***", "<", n, "/", lstLen-1, ">\n ", db.pp
|
||||
check merged.error in {AristoError(0), MergeLeafPathCachedAlready}
|
||||
return
|
||||
|
||||
noisy.say "***", "sample ",n,"/",lst.len-1," leafs merged: ", added.merged
|
||||
#noisy.say "***", "sample ", n, "/", lstLen-1, " merged=", merged
|
||||
|
||||
block:
|
||||
let
|
||||
preRoot = db.lRoot
|
||||
preDb = db.pp(sTabOk=false, lTabOk=false)
|
||||
rc = db.hashify rootKey
|
||||
|
||||
# Handle known errors
|
||||
if oopsTab.hasKey(testId):
|
||||
if rc.isOK:
|
||||
check rc.isErr
|
||||
return
|
||||
if oopsTab[testId] != rc.error:
|
||||
check oopsTab[testId] == rc.error
|
||||
return
|
||||
|
||||
# Otherwise, check for correctness
|
||||
elif rc.isErr:
|
||||
noisy.say "***", "<", n, "/", lstLen-1, ">",
|
||||
" testId=", testId,
|
||||
" groups=", count,
|
||||
"\n pre-DB",
|
||||
" lRoot=", preRoot.pp,
|
||||
"\n ", preDb,
|
||||
"\n --------",
|
||||
"\n ", db.pp
|
||||
check rc.error == (VertexID(0),AristoError(0))
|
||||
return
|
||||
|
||||
if db.lRoot == VertexID(0):
|
||||
check db.lRoot != VertexID(0)
|
||||
return
|
||||
|
||||
when true and false:
|
||||
noisy.say "***", "sample <", n, "/", lstLen-1, ">",
|
||||
" groups=", count, " proved=", proved.pp, " merged=", merged.pp
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
|
||||
161
tests/test_aristo/test_nearby.nim
Normal file
161
tests/test_aristo/test_nearby.nim
Normal file
@ -0,0 +1,161 @@
|
||||
# Nimbus - Types, data structures and shared utilities used in network sync
|
||||
#
|
||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or
|
||||
# distributed except according to those terms.
|
||||
|
||||
## Aristo (aka Patricia) DB records merge test
|
||||
|
||||
import
|
||||
std/[algorithm, sequtils, sets],
|
||||
eth/common,
|
||||
stew/results,
|
||||
unittest2,
|
||||
../../nimbus/db/aristo/[
|
||||
aristo_desc, aristo_debug, aristo_error, aristo_merge, aristo_nearby],
|
||||
../../nimbus/sync/snap/range_desc,
|
||||
./test_helpers
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc fwdWalkLeafsCompleteDB(
|
||||
db: AristoDbRef;
|
||||
tags: openArray[NodeTag];
|
||||
noisy: bool;
|
||||
): tuple[visited: int, error: AristoError] =
|
||||
let
|
||||
tLen = tags.len
|
||||
var
|
||||
error = AristoError(0)
|
||||
tag = (tags[0].u256 div 2).NodeTag
|
||||
n = 0
|
||||
while true:
|
||||
let rc = tag.nearbyRight(db.lRoot, db) # , noisy)
|
||||
#noisy.say "=================== ", n
|
||||
if rc.isErr:
|
||||
if rc.error != NearbyBeyondRange:
|
||||
noisy.say "***", "[", n, "/", tLen-1, "] fwd-walk error=", rc.error
|
||||
error = rc.error
|
||||
check rc.error == AristoError(0)
|
||||
elif n != tLen:
|
||||
error = AristoError(1)
|
||||
check n == tLen
|
||||
break
|
||||
if tLen <= n:
|
||||
noisy.say "***", "[", n, "/", tLen-1, "] fwd-walk -- ",
|
||||
" oops, too many leafs (index overflow)"
|
||||
error = AristoError(1)
|
||||
check n < tlen
|
||||
break
|
||||
if rc.value != tags[n]:
|
||||
noisy.say "***", "[", n, "/", tLen-1, "] fwd-walk -- leafs differ,",
|
||||
" got=", rc.value.pp(db),
|
||||
" wanted=", tags[n].pp(db) #, " db-dump\n ", db.pp
|
||||
error = AristoError(1)
|
||||
check rc.value == tags[n]
|
||||
break
|
||||
if rc.value < high(NodeTag):
|
||||
tag = (rc.value.u256 + 1).NodeTag
|
||||
n.inc
|
||||
|
||||
(n,error)
|
||||
|
||||
|
||||
proc revWalkLeafsCompleteDB(
|
||||
db: AristoDbRef;
|
||||
tags: openArray[NodeTag];
|
||||
noisy: bool;
|
||||
): tuple[visited: int, error: AristoError] =
|
||||
let
|
||||
tLen = tags.len
|
||||
var
|
||||
error = AristoError(0)
|
||||
delta = ((high(UInt256) - tags[^1].u256) div 2)
|
||||
tag = (tags[^1].u256 + delta).NodeTag
|
||||
n = tLen-1
|
||||
while true: # and false:
|
||||
let rc = tag.nearbyLeft(db.lRoot, db) # , noisy)
|
||||
if rc.isErr:
|
||||
if rc.error != NearbyBeyondRange:
|
||||
noisy.say "***", "[", n, "/", tLen-1, "] rev-walk error=", rc.error
|
||||
error = rc.error
|
||||
check rc.error == AristoError(0)
|
||||
elif n != -1:
|
||||
error = AristoError(1)
|
||||
check n == -1
|
||||
break
|
||||
if n < 0:
|
||||
noisy.say "***", "[", n, "/", tLen-1, "] rev-walk -- ",
|
||||
" oops, too many leafs (index underflow)"
|
||||
error = AristoError(1)
|
||||
check 0 <= n
|
||||
break
|
||||
if rc.value != tags[n]:
|
||||
noisy.say "***", "[", n, "/", tLen-1, "] rev-walk -- leafs differ,",
|
||||
" got=", rc.value.pp(db),
|
||||
" wanted=", tags[n]..pp(db) #, " db-dump\n ", db.pp
|
||||
error = AristoError(1)
|
||||
check rc.value == tags[n]
|
||||
break
|
||||
if low(NodeTag) < rc.value:
|
||||
tag = (rc.value.u256 - 1).NodeTag
|
||||
n.dec
|
||||
|
||||
(tLen-1 - n, error)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public test function
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc test_nearbyKvpList*(
|
||||
noisy: bool;
|
||||
list: openArray[ProofTrieData];
|
||||
resetDb = false;
|
||||
) =
|
||||
var
|
||||
db = AristoDbRef()
|
||||
tagSet: HashSet[NodeTag]
|
||||
for n,w in list:
|
||||
if resetDb:
|
||||
db = AristoDbRef()
|
||||
tagSet.reset
|
||||
let
|
||||
lstLen = list.len
|
||||
lTabLen = db.lTab.len
|
||||
leafs = w.kvpLst
|
||||
added = db.merge leafs
|
||||
|
||||
check added.error == AristoError(0)
|
||||
check db.lTab.len == lTabLen + added.merged
|
||||
check added.merged + added.dups == leafs.len
|
||||
|
||||
for w in leafs:
|
||||
tagSet.incl w.pathTag
|
||||
|
||||
let
|
||||
tags = tagSet.toSeq.sorted
|
||||
fwdWalk = db.fwdWalkLeafsCompleteDB(tags, noisy=true)
|
||||
revWalk = db.revWalkLeafsCompleteDB(tags, noisy=true)
|
||||
|
||||
check fwdWalk.error == AristoError(0)
|
||||
check revWalk.error == AristoError(0)
|
||||
check fwdWalk == revWalk
|
||||
|
||||
if {fwdWalk.error, revWalk.error} != {AristoError(0)}:
|
||||
noisy.say "***", "<", n, "/", lstLen-1, "> db dump",
|
||||
"\n post-state ", db.pp,
|
||||
"\n"
|
||||
break
|
||||
|
||||
#noisy.say "***", "sample ",n,"/",lstLen-1, " visited=", fwdWalk.visited
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
# ------------------------------------------------------------------------------
|
||||
Loading…
x
Reference in New Issue
Block a user