diff --git a/nimbus/db/aristo/aristo_error.nim b/nimbus/db/aristo/aristo_error.nim index 879603d68..fa0e2f933 100644 --- a/nimbus/db/aristo/aristo_error.nim +++ b/nimbus/db/aristo/aristo_error.nim @@ -90,6 +90,7 @@ type HashifyExistingHashMismatch HashifyLeafToRootAllFailed HashifyRootHashMismatch + HashifyRootVidMismatch HashifyCheckRevCountMismatch HashifyCheckRevHashMismatch @@ -103,4 +104,18 @@ type HashifyCheckVtxIncomplete HashifyCheckVtxLockWithoutKey + # Neighbour vertex, tree traversal `nearbyRight()` and `nearbyLeft()` + NearbyBeyondRange + NearbyBranchError + NearbyDanglingLink + NearbyEmptyHike + NearbyExtensionError + NearbyFailed + NearbyBranchExpected + NearbyLeafExpected + NearbyNestingTooDeep + NearbyPathTailUnexpected + NearbyPathTailInxOverflow + NearbyUnexpectedVtx + # End diff --git a/nimbus/db/aristo/aristo_hashify.nim b/nimbus/db/aristo/aristo_hashify.nim index 445b952d1..48ec48e20 100644 --- a/nimbus/db/aristo/aristo_hashify.nim +++ b/nimbus/db/aristo/aristo_hashify.nim @@ -99,7 +99,7 @@ proc toNode(vtx: VertexRef; db: AristoDbRef): Result[NodeRef,void] = proc leafToRootHasher( db: AristoDbRef; # Database, top layer hike: Hike; # Hike for labelling leaf..root - ): Result[int,AristoError] = + ): Result[int,(VertexID,AristoError)] = ## Returns the index of the first node that could not be hashed for n in (hike.legs.len-1).countDown(0): let @@ -120,7 +120,7 @@ proc leafToRootHasher( elif key != vfyKey: let error = HashifyExistingHashMismatch debug "hashify failed", vid=wp.vid, key, expected=vfyKey, error - return err(error) + return err((wp.vid,error)) ok -1 # all could be hashed @@ -141,13 +141,13 @@ proc hashifyClear*( proc hashify*( db: AristoDbRef; # Database, top layer - ): Result[NodeKey,AristoError] = + rootKey = EMPTY_ROOT_KEY; # Optional root key + ): Result[NodeKey,(VertexID,AristoError)] = ## Add keys to the `Patricia Trie` so that it becomes a `Merkle Patricia ## Tree`. If successful, the function returns the key (aka Merkle hash) of ## the root vertex. var - fullPath = false - rootKey: NodeKey + thisRootKey = EMPTY_ROOT_KEY # Width-first leaf-to-root traversal structure backLink: Table[VertexID,VertexID] @@ -156,7 +156,7 @@ proc hashify*( for (pathTag,vid) in db.lTab.pairs: let hike = pathTag.hikeUp(db.lRoot,db) if hike.error != AristoError(0): - return err(hike.error) + return err((VertexID(0),hike.error)) # Hash as much of the `hike` as possible let n = block: @@ -178,13 +178,22 @@ proc hashify*( for u in (n-1).countDown(1): backLink[hike.legs[u].wp.vid] = hike.legs[u-1].wp.vid - elif not fullPath: - rootKey = db.kMap.getOrDefault(hike.legs[0].wp.vid, EMPTY_ROOT_KEY) - fullPath = (rootKey != EMPTY_ROOT_KEY) + elif thisRootKey == EMPTY_ROOT_KEY: + let rootVid = hike.legs[0].wp.vid + thisRootKey = db.kMap.getOrDefault(rootVid, EMPTY_ROOT_KEY) + + if thisRootKey != EMPTY_ROOT_KEY: + if rootKey != EMPTY_ROOT_KEY and rootKey != thisRootKey: + return err((rootVid, HashifyRootHashMismatch)) + + if db.lRoot == VertexID(0): + db.lRoot = rootVid + elif db.lRoot != rootVid: + return err((rootVid,HashifyRootVidMismatch)) # At least one full path leaf..root should have succeeded with labelling - if not fullPath: - return err(HashifyLeafToRootAllFailed) + if thisRootKey == EMPTY_ROOT_KEY: + return err((VertexID(0),HashifyLeafToRootAllFailed)) # Update remaining hashes var n = 0 # for logging @@ -216,7 +225,7 @@ proc hashify*( let error = HashifyExistingHashMismatch debug "hashify failed", vid=fromVid, key=nodeKey, expected=fromKey.pp, error - return err(error) + return err((fromVid,error)) done.incl fromVid @@ -228,14 +237,14 @@ proc hashify*( # Make sure that the algorithm proceeds if done.len == 0: let error = HashifyCannotComplete - return err(error) + return err((VertexID(0),error)) # Clean up dups from `backLink` and restart `downMost` for vid in done.items: backLink.del vid downMost = redo - ok rootKey + ok thisRootKey # ------------------------------------------------------------------------------ # Public debugging functions diff --git a/nimbus/db/aristo/aristo_merge.nim b/nimbus/db/aristo/aristo_merge.nim index d34ab93e9..91ffbd621 100644 --- a/nimbus/db/aristo/aristo_merge.nim +++ b/nimbus/db/aristo/aristo_merge.nim @@ -375,6 +375,13 @@ proc topIsEmptyAddLeaf( let nibble = hike.tail[0].int8 if not rootVtx.bVid[nibble].isZero: return Hike(error: MergeRootBranchLinkBusy) + + # Clear Merkle hashes (aka node keys) unless proof mode + if db.pPrf.len == 0: + db.clearMerkleKeys(hike, hike.root) + elif hike.root in db.pPrf: + return Hike(error: MergeBranchProofModeLock) + let leafVid = db.vidFetch leafVtx = VertexRef( diff --git a/nimbus/db/aristo/aristo_nearby.nim b/nimbus/db/aristo/aristo_nearby.nim new file mode 100644 index 000000000..591db89b4 --- /dev/null +++ b/nimbus/db/aristo/aristo_nearby.nim @@ -0,0 +1,484 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Aristo DB -- Patricia Trie traversal +## ==================================== +## +## This module provides tools to visit leaf vertices in a monotone order, +## increasing or decreasing. These tools are intended for +## * boundary proof verification +## * step along leaf vertices in sorted order +## * tree/trie consistency checks when debugging +## + +{.push raises: [].} + +import + std/tables, + eth/[common, trie/nibbles], + stew/results, + "."/[aristo_desc, aristo_error, aristo_get, aristo_hike, aristo_path] + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +proc `<=`(a, b: NibblesSeq): bool = + ## Compare nibbles, different lengths are padded to the right with zeros + let abMin = min(a.len, b.len) + for n in 0 ..< abMin: + if a[n] < b[n]: + return true + if b[n] < a[n]: + return false + # otherwise a[n] == b[n] + + # Assuming zero for missing entries + if b.len < a.len: + for n in abMin + 1 ..< a.len: + if 0 < a[n]: + return false + true + +proc `<`(a, b: NibblesSeq): bool = + not (b <= a) + +# ------------------ + +proc branchNibbleMin*(vtx: VertexRef; minInx: int8): int8 = + ## Find the least index for an argument branch `vtx` link with index + ## greater or equal the argument `nibble`. + if vtx.vType == Branch: + for n in minInx .. 15: + if not vtx.bVid[n].isZero: + return n + -1 + +proc branchNibbleMax*(vtx: VertexRef; maxInx: int8): int8 = + ## Find the greatest index for an argument branch `vtx` link with index + ## less or equal the argument `nibble`. + if vtx.vType == Branch: + for n in maxInx.countDown 0: + if not vtx.bVid[n].isZero: + return n + -1 + +# ------------------------------------------------------------------------------ +# Private functions +# ------------------------------------------------------------------------------ + +proc complete( + hike: Hike; # Partially expanded path + vid: VertexID; # Start ID + db: AristoDbRef; # Database layer + hikeLenMax: static[int]; # Beware of loops (if any) + doLeast: static[bool]; # Direction: *least* or *most* + ): Hike = + ## Extend `hike` using least or last vertex without recursion. + var + vid = vid + vtx = db.getVtx vid + uHike = Hike(root: hike.root, legs: hike.legs) + if vtx.isNil: + return Hike(error: GetVtxNotFound) + + while uHike.legs.len < hikeLenMax: + var leg = Leg(wp: VidVtxPair(vid: vid, vtx: vtx), nibble: -1) + + case vtx.vType: + of Leaf: + uHike.legs.add leg + return uHike # done + + of Extension: + vid = vtx.eVid + if not vid.isZero: + vtx = db.getVtx vid + if not vtx.isNil: + uHike.legs.add leg + continue + return Hike(error: NearbyExtensionError) # Oops, no way + + of Branch: + when doLeast: + leg.nibble = vtx.branchNibbleMin 0 + else: + leg.nibble = vtx.branchNibbleMax 15 + if 0 <= leg.nibble: + vid = vtx.bVid[leg.nibble] + vtx = db.getVtx vid + if not vtx.isNil: + uHike.legs.add leg + continue + return Hike(error: NearbyBranchError) # Oops, no way + + Hike(error: NearbyNestingTooDeep) + + +proc zeroAdjust( + hike: Hike; # Partially expanded path + db: AristoDbRef; # Database layer + doLeast: static[bool]; # Direction: *least* or *most* + ): Hike = + ## Adjust empty argument path to the first node entry to the right. Ths + ## applies is the argument `hike` is before the first entry in the database. + ## The result is a hike which is aligned with the first entry. + proc accept(p: Hike; pfx: NibblesSeq): bool = + when doLeast: + p.tail <= pfx + else: + pfx <= p.tail + + proc branchBorderNibble(w: VertexRef; n: int8): int8 = + when doLeast: + w.branchNibbleMin n + else: + w.branchNibbleMax n + + proc toHike(pfx: NibblesSeq, root: VertexID, db: AristoDbRef): Hike = + when doLeast: + pfx.pathPfxPad(0).hikeUp(root, db) + else: + pfx.pathPfxPad(255).hikeUp(root, db) + + if 0 < hike.legs.len: + result = hike + result.error = AristoError(0) + return + + let root = db.getVtx hike.root + if not root.isNil: + block fail: + var pfx: NibblesSeq + case root.vType: + of Branch: + # Find first non-dangling link and assign it + if hike.tail.len == 0: + break fail + + let n = root.branchBorderNibble hike.tail[0].int8 + if n < 0: + # Before or after the database range + return Hike(error: NearbyBeyondRange) + pfx = @[n.byte].initNibbleRange.slice(1) + + of Extension: + let ePfx = root.ePfx + # Must be followed by a branch node + if hike.tail.len < 2 or not hike.accept(ePfx): + break fail + let vtx = db.getVtx root.eVid + if vtx.isNil: + break fail + let ePfxLen = ePfx.len + if hike.tail.len <= ePfxLen: + return Hike(error: NearbyPathTailInxOverflow) + let tailPfx = hike.tail.slice(0,ePfxLen) + when doLeast: + if ePfx < tailPfx: + return Hike(error: NearbyBeyondRange) + else: + if tailPfx < ePfx: + return Hike(error: NearbyBeyondRange) + pfx = ePfx + + of Leaf: + pfx = root.lPfx + if not hike.accept(pfx): + # Before or after the database range + return Hike(error: NearbyBeyondRange) + + var newHike = pfx.toHike(hike.root, db) + if 0 < newHike.legs.len: + newHike.error = AristoError(0) + return newHike + + Hike(error: NearbyEmptyHike) + + +proc finalise( + hike: Hike; # Partially expanded path + db: AristoDbRef; # Database layer + moveRight: static[bool]; # Direction of next node + ): Hike = + ## Handle some pathological cases after main processing failed + proc beyond(p: Hike; pfx: NibblesSeq): bool = + when moveRight: + pfx < p.tail + else: + p.tail < pfx + + proc branchBorderNibble(w: VertexRef): int8 = + when moveRight: + w.branchNibbleMax 15 + else: + w.branchNibbleMin 0 + + # Just for completeness (this case should have been handled, already) + if hike.legs.len == 0: + return Hike(error: NearbyEmptyHike) + + # Check whether the path is beyond the database range + if 0 < hike.tail.len: # nothing to compare against, otherwise + let top = hike.legs[^1] + + # Note that only a `Branch` nodes has a non-zero nibble + if 0 <= top.nibble and top.nibble == top.wp.vtx.branchBorderNibble: + # Check the following up node + let vtx = db.getVtx top.wp.vtx.bVid[top.nibble] + if vtx.isNil: + return Hike(error: NearbyDanglingLink) + + var pfx: NibblesSeq + case vtx.vType: + of Leaf: + pfx = vtx.lPfx + of Extension: + pfx = vtx.ePfx + of Branch: + pfx = @[vtx.branchBorderNibble.byte].initNibbleRange.slice(1) + if hike.beyond pfx: + return Hike(error: NearbyBeyondRange) + + # Pathological cases + # * finalise right: nfffff.. for n < f or + # * finalise left: n00000.. for 0 < n + if hike.legs[0].wp.vtx.vType == Branch or + (1 < hike.legs.len and hike.legs[1].wp.vtx.vType == Branch): + return Hike(error: NearbyFailed) # no more nodes + + Hike(error: NearbyUnexpectedVtx) # error + + +proc nearbyNext( + hike: Hike; # Partially expanded path + db: AristoDbRef; # Database layer + hikeLenMax: static[int]; # Beware of loops (if any) + moveRight: static[bool]; # Direction of next node + ): Hike = + ## Unified implementation of `nearbyRight()` and `nearbyLeft()`. + proc accept(nibble: int8): bool = + ## Accept `nibble` unless on boundaty dependent on `moveRight` + when moveRight: + nibble < 15 + else: + 0 < nibble + + proc accept(p: Hike; pfx: NibblesSeq): bool = + when moveRight: + p.tail <= pfx + else: + pfx <= p.tail + + proc branchNibbleNext(w: VertexRef; n: int8): int8 = + when moveRight: + w.branchNibbleMin(n + 1) + else: + w.branchNibbleMax(n - 1) + + # Some easy cases + var hike = hike.zeroAdjust(db, doLeast=moveRight) + if hike.error != AristoError(0): + return hike + + if hike.legs[^1].wp.vtx.vType == Extension: + let vid = hike.legs[^1].wp.vtx.eVid + return hike.complete(vid, db, hikeLenMax, doLeast=moveRight) + + var + uHike = hike + start = true + while 0 < uHike.legs.len: + let top = uHike.legs[^1] + case top.wp.vtx.vType: + of Leaf: + return uHike + of Branch: + if top.nibble < 0 or uHike.tail.len == 0: + return Hike(error: NearbyUnexpectedVtx) + of Extension: + uHike.tail = top.wp.vtx.ePfx & uHike.tail + uHike.legs.setLen(uHike.legs.len - 1) + continue + + var + step = top + let + uHikeLen = uHike.legs.len # in case of backtracking + uHikeTail = uHike.tail # in case of backtracking + + # Look ahead checking next node + if start: + let vid = top.wp.vtx.bVid[top.nibble] + if vid.isZero: + return Hike(error: NearbyDanglingLink) # error + + let vtx = db.getVtx vid + if vtx.isNil: + return Hike(error: GetVtxNotFound) # error + + case vtx.vType + of Leaf: + if uHike.accept vtx.lPfx: + return uHike.complete(vid, db, hikeLenMax, doLeast=moveRight) + of Extension: + if uHike.accept vtx.ePfx: + return uHike.complete(vid, db, hikeLenMax, doLeast=moveRight) + of Branch: + let nibble = uHike.tail[0].int8 + if start and accept nibble: + # Step down and complete with a branch link on the child node + step = Leg(wp: VidVtxPair(vid: vid, vtx: vtx), nibble: nibble) + uHike.legs.add step + + # Find the next item to the right/left of the current top entry + let n = step.wp.vtx.branchNibbleNext step.nibble + if 0 <= n: + uHike.legs[^1].nibble = n + return uHike.complete( + step.wp.vtx.bVid[n], db, hikeLenMax, doLeast=moveRight) + + if start: + # Retry without look ahead + start = false + + # Restore `uPath` (pop temporary extra step) + if uHikeLen < uHike.legs.len: + uHike.legs.setLen(uHikeLen) + uHike.tail = uHikeTail + else: + # Pop current `Branch` node on top and append nibble to `tail` + uHike.tail = @[top.nibble.byte].initNibbleRange.slice(1) & uHike.tail + uHike.legs.setLen(uHike.legs.len - 1) + # End while + + # Handle some pathological cases + return hike.finalise(db, moveRight) + + +proc nearbyNext( + baseTag: NodeTag; # Some node + root: VertexID; # State root + db: AristoDbRef; # Database layer + hikeLenMax: static[int]; # Beware of loops (if any) + moveRight:static[ bool]; # Direction of next node + ): Result[NodeTag,AristoError] = + ## Variant of `nearbyNext()`, convenience wrapper + let hike = baseTag.hikeUp(root,db).nearbyNext(db, hikeLenMax, moveRight) + if hike.error != AristoError(0): + return err(hike.error) + + if 0 < hike.legs.len and hike.legs[^1].wp.vtx.vType == Leaf: + let rc = hike.legsTo(NibblesSeq).pathToKey + if rc.isOk: + return ok rc.value.to(NodeTag) + return err(rc.error) + + err(NearbyLeafExpected) + +# ------------------------------------------------------------------------------ +# Public functions, moving and right boundary proof +# ------------------------------------------------------------------------------ + +proc nearbyRight*( + hike: Hike; # Partially expanded path + db: AristoDbRef; # Database layer + ): Hike = + ## Extends the maximally extended argument nodes `hike` to the right (i.e. + ## with non-decreasing path value). This function does not backtrack if + ## there are dangling links in between. It will return an error in that case. + ## + ## If there is no more leaf node to the right of the argument `hike`, the + ## particular error code `NearbyBeyondRange` is returned. + ## + ## This code is intended to be used for verifying a left-bound proof to + ## verify that there is no leaf node *right* of a boundary path value. + hike.nearbyNext(db, 64, moveRight=true) + +proc nearbyRight*( + nodeTag: NodeTag; # Some node + root: VertexID; # State root + db: AristoDbRef; # Database layer + ): Result[NodeTag,AristoError] = + ## Variant of `nearbyRight()` working with a `NodeTag` argument instead + ## of a `Hike`. + nodeTag.nearbyNext(root, db, 64, moveRight=true) + +proc nearbyLeft*( + hike: Hike; # Partially expanded path + db: AristoDbRef; # Database layer + ): Hike = + ## Similar to `nearbyRight()`. + ## + ## This code is intended to be used for verifying a right-bound proof to + ## verify that there is no leaf node *left* to a boundary path value. + hike.nearbyNext(db, 64, moveRight=false) + +proc nearbyLeft*( + nodeTag: NodeTag; # Some node + root: VertexID; # State root + db: AristoDbRef; # Database layer + ): Result[NodeTag,AristoError] = + ## Similar to `nearbyRight()` for `NodeTag` argument instead + ## of a `Hike`. + nodeTag.nearbyNext(root, db, 64, moveRight=false) + +# ------------------------------------------------------------------------------ +# Public debugging helpers +# ------------------------------------------------------------------------------ + +proc nearbyRightMissing*( + hike: Hike; # Partially expanded path + db: AristoDbRef; # Database layer + ): Result[bool,AristoError] = + ## Returns `true` if the maximally extended argument nodes `hike` is the + ## rightmost on the hexary trie database. It verifies that there is no more + ## leaf entry to the right of the argument `hike`. This function is an + ## an alternative to + ## :: + ## let rc = path.nearbyRight(db) + ## if rc.isOk: + ## # not at the end => false + ## ... + ## elif rc.error != NearbyBeyondRange: + ## # problem with database => error + ## ... + ## else: + ## # no nore nodes => true + ## ... + ## and is intended mainly for debugging. + if hike.legs.len == 0: + return err(NearbyEmptyHike) + if 0 < hike.tail.len: + return err(NearbyPathTailUnexpected) + + let top = hike.legs[^1] + if top.wp.vtx.vType != Branch or top.nibble < 0: + return err(NearbyBranchError) + + let vid = top.wp.vtx.bVid[top.nibble] + if vid.isZero: + return err(NearbyDanglingLink) # error + + let vtx = db.getVtx vid + if vtx.isNil: + return err(GetVtxNotFound) # error + + case vtx.vType + of Leaf: + return ok(vtx.lPfx < hike.tail) + of Extension: + return ok(vtx.ePfx < hike.tail) + of Branch: + return ok(vtx.branchNibbleMin(hike.tail[0].int8) < 0) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_path.nim b/nimbus/db/aristo/aristo_path.nim index 1ca8defd2..5b0d12abb 100644 --- a/nimbus/db/aristo/aristo_path.nim +++ b/nimbus/db/aristo/aristo_path.nim @@ -11,6 +11,7 @@ {.push raises: [].} import + std/sequtils, eth/[common, trie/nibbles], stew/results, "."/[aristo_constants, aristo_desc, aristo_error] @@ -39,7 +40,7 @@ proc pathAsNibbles*(key: NodeKey): NibblesSeq = proc pathAsNibbles*(tag: NodeTag): NibblesSeq = tag.to(NodeKey).pathAsNibbles() -proc pathAsBlob*(keyOrTag: NodeKey|Nodetag): Blob = +proc pathAsBlob*(keyOrTag: NodeKey|NodeTag): Blob = keyOrTag.pathAsNibbles.hexPrefixEncode(isLeaf=true) @@ -64,6 +65,33 @@ proc pathToTag*(partPath: NibblesSeq|Blob): Result[NodeTag,AristoError] = return ok(rc.value.to(NodeTag)) err(rc.error) +# -------------------- + +proc pathPfxPad*(pfx: NibblesSeq; dblNibble: static[byte]): NodeKey = + ## Extend (or cut) the argument nibbles sequence `pfx` for generating a + ## `NodeKey`. + ## + ## This function must be handled with some care regarding a meaningful value + ## for the `dblNibble` argument. Currently, only static values `0` and `255` + ## are allowed for padding. This is checked at compile time. + static: + doAssert dblNibble == 0 or dblNibble == 255 + + # Pad with zeroes + var padded: NibblesSeq + + let padLen = 64 - pfx.len + if 0 <= padLen: + padded = pfx & dblNibble.repeat(padlen div 2).mapIt(it.byte).initNibbleRange + if (padLen and 1) == 1: + padded = padded & @[dblNibble.byte].initNibbleRange.slice(1) + else: + let nope = seq[byte].default.initNibbleRange + padded = pfx.slice(0,64) & nope # nope forces re-alignment + + let bytes = padded.getBytes + (addr result.ByteArray32[0]).copyMem(unsafeAddr bytes[0], bytes.len) + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/tests/test_aristo.nim b/tests/test_aristo.nim index a06c2e289..7348ee15f 100644 --- a/tests/test_aristo.nim +++ b/tests/test_aristo.nim @@ -18,13 +18,12 @@ import rocksdb, unittest2, ../nimbus/db/select_backend, - ../nimbus/db/aristo/[aristo_desc], + ../nimbus/db/aristo/[aristo_desc, aristo_error, aristo_merge], ../nimbus/core/chain, - ../nimbus/sync/snap/worker/db/[ - hexary_desc, rocky_bulk_load, snapdb_accounts, snapdb_desc], - ./replay/[pp, undump_accounts], + ../nimbus/sync/snap/worker/db/[rocky_bulk_load, snapdb_accounts, snapdb_desc], + ./replay/[pp, undump_accounts, undump_storages], ./test_sync_snap/[snap_test_xx, test_accounts, test_types], - ./test_aristo/[test_merge, test_transcode] + ./test_aristo/[test_helpers, test_merge, test_nearby, test_transcode] const baseDir = [".", "..", ".."/"..", $DirSep] @@ -36,6 +35,7 @@ const # Standard test samples accSample = snapTest0 + storSample = snapTest4 # Number of database slots available nTestDbInstances = 9 @@ -91,22 +91,6 @@ proc setErrorLevel {.used.} = # Private functions # ------------------------------------------------------------------------------ -proc to(sample: AccountsSample; T: type seq[UndumpAccounts]): T = - ## Convert test data into usable in-memory format - let file = sample.file.findFilePath.value - var root: Hash256 - for w in file.undumpNextAccount: - let n = w.seenAccounts - 1 - if n < sample.firstItem: - continue - if sample.lastItem < n: - break - if sample.firstItem == n: - root = w.root - elif w.root != root: - break - result.add w - proc flushDbDir(s: string; subDir = "") = if s != "": let baseDir = s / "tmp" @@ -162,7 +146,7 @@ proc snapDbAccountsRef(cdb:ChainDb; root:Hash256; pers:bool):SnapDbAccountsRef = # Test Runners: accounts and accounts storages # ------------------------------------------------------------------------------ -proc transcodeRunner(noisy = true; sample = accSample; stopAfter = high(int)) = +proc transcodeRunner(noisy =true; sample=accSample; stopAfter=high(int)) = let accLst = sample.to(seq[UndumpAccounts]) root = accLst[0].root @@ -199,19 +183,45 @@ proc transcodeRunner(noisy = true; sample = accSample; stopAfter = high(int)) = noisy.test_transcodeAccounts(db.cdb[0].rocksStoreRef, stopAfter) -proc dataRunner(noisy = true; sample = accSample) = +proc accountsRunner(noisy=true; sample=accSample, resetDb=false) = let - accLst = sample.to(seq[UndumpAccounts]) + accLst = sample.to(seq[UndumpAccounts]).to(seq[ProofTrieData]) fileInfo = sample.file.splitPath.tail.replace(".txt.gz","") + listMode = if resetDb: "" else: ", merged data lists" - suite &"Aristo: accounts data import from {fileInfo}": + suite &"Aristo: accounts data dump from {fileInfo}{listMode}": test &"Merge {accLst.len} account lists to database": - noisy.test_mergeAccounts accLst + noisy.test_mergeKvpList(accLst, resetDb) test &"Merge {accLst.len} proof & account lists to database": - noisy.test_mergeProofsAndAccounts accLst + noisy.test_mergeProofAndKvpList(accLst, resetDb) + test &"Traverse accounts database w/{accLst.len} account lists": + noisy.test_nearbyKvpList(accLst, resetDb) + + +proc storagesRunner( + noisy = true; + sample = storSample; + resetDb = false; + oops: KnownHasherFailure = @[]; + ) = + let + stoLst = sample.to(seq[UndumpStorages]).to(seq[ProofTrieData]) + fileInfo = sample.file.splitPath.tail.replace(".txt.gz","") + listMode = if resetDb: "" else: ", merged data lists" + + suite &"Aristo: storages data dump from {fileInfo}{listMode}": + + test &"Merge {stoLst.len} storage slot lists to database": + noisy.test_mergeKvpList(stoLst, resetDb) + + test &"Merge {stoLst.len} proof & slots lists to database": + noisy.test_mergeProofAndKvpList(stoLst, resetDb, fileInfo, oops) + + test &"Traverse storage slots database w/{stoLst.len} account lists": + noisy.test_nearbyKvpList(stoLst, resetDb) # ------------------------------------------------------------------------------ # Main function(s) @@ -219,14 +229,15 @@ proc dataRunner(noisy = true; sample = accSample) = proc aristoMain*(noisy = defined(debug)) = noisy.transcodeRunner() - noisy.dataRunner() + noisy.accountsRunner() + noisy.storagesRunner() when isMainModule: const noisy = defined(debug) or true # Borrowed from `test_sync_snap.nim` - when true: # and false: + when true and false: for n,sam in snapTestList: noisy.transcodeRunner(sam) for n,sam in snapTestStorageList: @@ -235,22 +246,27 @@ when isMainModule: # This one uses dumps from the external `nimbus-eth1-blob` repo when true and false: import ./test_sync_snap/snap_other_xx - noisy.showElapsed("dataRunner() @snap_other_xx"): + noisy.showElapsed("@snap_other_xx"): for n,sam in snapOtherList: - noisy.dataRunner(sam) + noisy.accountsRunner(sam) # This one usues dumps from the external `nimbus-eth1-blob` repo - when true and false: + when true: # and false: import ./test_sync_snap/snap_storage_xx - noisy.showElapsed("dataRunner() @snap_storage_xx"): + let knownFailures: KnownHasherFailure = @[ + ("storages5__34__41_dump#10.20512",(VertexID(1),HashifyRootHashMismatch)), + ] + noisy.showElapsed("@snap_storage_xx"): for n,sam in snapStorageList: - noisy.dataRunner(sam) + noisy.accountsRunner(sam) + noisy.storagesRunner(sam,oops=knownFailures) when true: # and false: for n,sam in snapTestList: - noisy.dataRunner(sam) + noisy.accountsRunner(sam) for n,sam in snapTestStorageList: - noisy.dataRunner(sam) + noisy.accountsRunner(sam) + noisy.storagesRunner(sam) # ------------------------------------------------------------------------------ # End diff --git a/tests/test_aristo/test_helpers.nim b/tests/test_aristo/test_helpers.nim index 1a300c9ef..87b28b603 100644 --- a/tests/test_aristo/test_helpers.nim +++ b/tests/test_aristo/test_helpers.nim @@ -13,9 +13,41 @@ import std/sequtils, eth/common, rocksdb, + ../../nimbus/db/aristo/[aristo_desc, aristo_merge], ../../nimbus/db/kvstore_rocksdb, - ../../nimbus/sync/snap/constants, - ../replay/pp + ../../nimbus/sync/protocol/snap/snap_types, + ../../nimbus/sync/snap/[constants, range_desc], + ../test_sync_snap/test_types, + ../replay/[pp, undump_accounts, undump_storages] + +type + ProofTrieData* = object + root*: NodeKey + id*: int + proof*: seq[SnapProof] + kvpLst*: seq[LeafKVP] + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +proc to(w: UndumpAccounts; T: type ProofTrieData): T = + T(root: w.root.to(NodeKey), + proof: w.data.proof, + kvpLst: w.data.accounts.mapIt(LeafKVP( + pathTag: it.accKey.to(NodeTag), + payload: PayloadRef(pType: BlobData, blob: it.accBlob)))) + +proc to(s: UndumpStorages; id: int; T: type seq[ProofTrieData]): T = + for w in s.data.storages: + result.add ProofTrieData( + root: w.account.storageRoot.to(NodeKey), + id: id, + kvpLst: w.data.mapIt(LeafKVP( + pathTag: it.slotHash.to(NodeTag), + payload: PayloadRef(pType: BlobData, blob: it.slotData)))) + if 0 < result.len: + result[^1].proof = s.data.proof # ------------------------------------------------------------------------------ # Public helpers @@ -30,6 +62,47 @@ proc say*(noisy = false; pfx = "***"; args: varargs[string, `$`]) = else: echo pfx, args.toSeq.join +# ----------------------- + +proc to*(sample: AccountsSample; T: type seq[UndumpAccounts]): T = + ## Convert test data into usable in-memory format + let file = sample.file.findFilePath.value + var root: Hash256 + for w in file.undumpNextAccount: + let n = w.seenAccounts - 1 + if n < sample.firstItem: + continue + if sample.lastItem < n: + break + if sample.firstItem == n: + root = w.root + elif w.root != root: + break + result.add w + +proc to*(sample: AccountsSample; T: type seq[UndumpStorages]): T = + ## Convert test data into usable in-memory format + let file = sample.file.findFilePath.value + var root: Hash256 + for w in file.undumpNextStorages: + let n = w.seenAccounts - 1 # storages selector based on accounts + if n < sample.firstItem: + continue + if sample.lastItem < n: + break + if sample.firstItem == n: + root = w.root + elif w.root != root: + break + result.add w + +proc to*(w: seq[UndumpAccounts]; T: type seq[ProofTrieData]): T = + w.mapIt(it.to(ProofTrieData)) + +proc to*(s: seq[UndumpStorages]; T: type seq[ProofTrieData]): T = + for n,w in s: + result &= w.to(n,seq[ProofTrieData]) + # ------------------------------------------------------------------------------ # Public iterators # ------------------------------------------------------------------------------ diff --git a/tests/test_aristo/test_merge.nim b/tests/test_aristo/test_merge.nim index 996555d41..8723a01f4 100644 --- a/tests/test_aristo/test_merge.nim +++ b/tests/test_aristo/test_merge.nim @@ -12,28 +12,28 @@ ## Aristo (aka Patricia) DB records merge test import - std/sequtils, eth/common, stew/results, unittest2, ../../nimbus/db/aristo/[ - aristo_desc, aristo_debug, aristo_error, aristo_hashify, + aristo_desc, aristo_debug, aristo_error, aristo_get, aristo_hashify, aristo_hike, aristo_merge], ../../nimbus/sync/snap/range_desc, - ../replay/undump_accounts, ./test_helpers +type + KnownHasherFailure* = seq[(string,(VertexID,AristoError))] + ## ( & "#" , @[(, )), ..]) + # ------------------------------------------------------------------------------ # Private helpers # ------------------------------------------------------------------------------ -proc to(w: PackedAccount; T: type LeafKVP): T = - T(pathTag: w.accKey.to(NodeTag), - payload: PayloadRef(pType: BlobData, blob: w.accBlob)) - -proc to[T](w: openArray[PackedAccount]; W: type seq[T]): W = - w.toSeq.mapIt(it.to(T)) - +proc pp(w: tuple[merged: int, dups: int, error: AristoError]): string = + result = "(merged: " & $w.merged & ", dups: " & $w.dups + if w.error != AristoError(0): + result &= ", error: " & $w.error + result &= ")" proc mergeStepwise( db: AristoDbRef; @@ -100,7 +100,7 @@ proc mergeStepwise( check 0 < ekih.legs.len elif ekih.legs[^1].wp.vtx.vType != Leaf: check ekih.legs[^1].wp.vtx.vType == Leaf - else: + elif hike.error != MergeLeafPathCachedAlready: check ekih.legs[^1].wp.vtx.lData.blob == leaf.payload.blob if db.lTab.len != lTabLen + merged: @@ -118,44 +118,51 @@ proc mergeStepwise( # Public test function # ------------------------------------------------------------------------------ -proc test_mergeAccounts*( +proc test_mergeKvpList*( noisy: bool; - lst: openArray[UndumpAccounts]; + list: openArray[ProofTrieData]; + resetDb = false; ) = - let - db = AristoDbRef() - - for n,par in lst: + var db = AristoDbRef() + for n,w in list: + if resetDb: + db = AristoDbRef() let + lstLen = list.len lTabLen = db.lTab.len - leafs = par.data.accounts.to(seq[LeafKVP]) + leafs = w.kvpLst + #prePreDb = db.pp added = db.merge leafs - #added = db.mergeStepwise(leafs, noisy=false) + #added = db.mergeStepwise(leafs, noisy=(6 < n)) check added.error == AristoError(0) check db.lTab.len == lTabLen + added.merged check added.merged + added.dups == leafs.len let + #preDb = db.pp preKMap = (db.kMap.len, db.pp(sTabOk=false, lTabOk=false)) prePAmk = (db.pAmk.len, db.pAmk.pp(db)) block: let rc = db.hashify # (noisy=true) if rc.isErr: # or true: - noisy.say "***", "<", n, "> db dump", + noisy.say "***", "<", n, ">", + " added=", added, + " db dump", "\n pre-kMap(", preKMap[0], ")\n ", preKMap[1], + #"\n pre-pre-DB", prePreDb, "\n --------\n pre-DB", preDb, "\n --------", "\n post-state ", db.pp, "\n" if rc.isErr: - check rc.error == AristoError(0) # force message + check rc.error == (VertexID(0),AristoError(0)) # force message return block: let rc = db.hashifyCheck() if rc.isErr: - noisy.say "***", "<", n, "/", lst.len-1, "> db dump", + noisy.say "***", "<", n, "/", lstLen-1, "> db dump", "\n pre-kMap(", preKMap[0], ")\n ", preKMap[1], "\n --------", "\n pre-pAmk(", prePAmk[0], ")\n ", prePAmk[1], @@ -166,65 +173,116 @@ proc test_mergeAccounts*( check rc == Result[void,(VertexID,AristoError)].ok() return - #noisy.say "***", "sample ",n,"/",lst.len-1," leafs merged: ", added.merged + when true and false: + noisy.say "***", "sample ", n, "/", lstLen-1, + " leafs merged=", added.merged, + " dup=", added.dups -proc test_mergeProofsAndAccounts*( +proc test_mergeProofAndKvpList*( noisy: bool; - lst: openArray[UndumpAccounts]; + list: openArray[ProofTrieData]; + resetDb = false; + idPfx = ""; + oops: KnownHasherFailure = @[]; ) = - let - db = AristoDbRef() + var + db = AristoDbRef(nil) + rootKey = NodeKey.default + count = 0 + for n,w in list: + if resetDb or w.root != rootKey or w.proof.len == 0: + db = AristoDbRef() + rootKey = w.root + count = 0 + count.inc - for n,par in lst: let + testId = idPfx & "#" & $w.id & "." & $n + oopsTab = oops.toTable + lstLen = list.len sTabLen = db.sTab.len lTabLen = db.lTab.len - leafs = par.data.accounts.to(seq[LeafKVP]) + leafs = w.kvpLst - noisy.say "***", "sample ", n, "/", lst.len-1, " start, nLeafs=", leafs.len + when true and false: + noisy.say "***", "sample <", n, "/", lstLen-1, ">", + " groups=", count, " nLeafs=", leafs.len - let - rootKey = par.root.to(NodeKey) - proved = db.merge par.data.proof + var proved: tuple[merged: int, dups: int, error: AristoError] + if 0 < w.proof.len: + proved = db.merge w.proof + check proved.error in {AristoError(0),MergeNodeKeyCachedAlready} + check w.proof.len == proved.merged + proved.dups + check db.lTab.len == lTabLen + check db.sTab.len == proved.merged + sTabLen + check proved.merged < db.pAmk.len + check proved.merged < db.kMap.len - check proved.error in {AristoError(0),MergeNodeKeyCachedAlready} - check par.data.proof.len == proved.merged + proved.dups - check db.lTab.len == lTabLen - check db.sTab.len == proved.merged + sTabLen - check proved.merged < db.pAmk.len - check proved.merged < db.kMap.len - - # Set up root ID - db.lRoot = db.pAmk.getOrDefault(rootKey, VertexID(0)) - check db.lRoot != VertexID(0) - - noisy.say "***", "sample ", n, "/", lst.len-1, " proved=", proved - #noisy.say "***", "<", n, "/", lst.len-1, ">\n ", db.pp - - let - added = db.merge leafs - #added = db.mergeStepwise(leafs, noisy=false) - - check db.lTab.len == lTabLen + added.merged - check added.merged + added.dups == leafs.len - - block: - if added.error notin {AristoError(0), MergeLeafPathCachedAlready}: - noisy.say "***", "<", n, "/", lst.len-1, ">\n ", db.pp - check added.error in {AristoError(0), MergeLeafPathCachedAlready} + # Set up root ID + db.lRoot = db.pAmk.getOrDefault(rootKey, VertexID(0)) + if db.lRoot == VertexID(0): + check db.lRoot != VertexID(0) return - noisy.say "***", "sample ", n, "/", lst.len-1, " added=", added + when true and false: + noisy.say "***", "sample <", n, "/", lstLen-1, ">", + " groups=", count, " nLeafs=", leafs.len, " proved=", proved + + let + merged = db.merge leafs + #merged = db.mergeStepwise(leafs, noisy=false) + + check db.lTab.len == lTabLen + merged.merged + check merged.merged + merged.dups == leafs.len + + if w.proof.len == 0: + let vtx = db.getVtx VertexID(1) + #check db.pAmk.getOrDefault(rootKey, VertexID(0)) != VertexID(0) block: - let rc = db.hashify # (noisy=false or (7 <= n)) - if rc.isErr: - noisy.say "***", "<", n, "/", lst.len-1, ">\n ", db.pp - check rc.error == AristoError(0) + if merged.error notin {AristoError(0), MergeLeafPathCachedAlready}: + noisy.say "***", "<", n, "/", lstLen-1, ">\n ", db.pp + check merged.error in {AristoError(0), MergeLeafPathCachedAlready} return - noisy.say "***", "sample ",n,"/",lst.len-1," leafs merged: ", added.merged + #noisy.say "***", "sample ", n, "/", lstLen-1, " merged=", merged + + block: + let + preRoot = db.lRoot + preDb = db.pp(sTabOk=false, lTabOk=false) + rc = db.hashify rootKey + + # Handle known errors + if oopsTab.hasKey(testId): + if rc.isOK: + check rc.isErr + return + if oopsTab[testId] != rc.error: + check oopsTab[testId] == rc.error + return + + # Otherwise, check for correctness + elif rc.isErr: + noisy.say "***", "<", n, "/", lstLen-1, ">", + " testId=", testId, + " groups=", count, + "\n pre-DB", + " lRoot=", preRoot.pp, + "\n ", preDb, + "\n --------", + "\n ", db.pp + check rc.error == (VertexID(0),AristoError(0)) + return + + if db.lRoot == VertexID(0): + check db.lRoot != VertexID(0) + return + + when true and false: + noisy.say "***", "sample <", n, "/", lstLen-1, ">", + " groups=", count, " proved=", proved.pp, " merged=", merged.pp # ------------------------------------------------------------------------------ # End diff --git a/tests/test_aristo/test_nearby.nim b/tests/test_aristo/test_nearby.nim new file mode 100644 index 000000000..3346949c2 --- /dev/null +++ b/tests/test_aristo/test_nearby.nim @@ -0,0 +1,161 @@ +# Nimbus - Types, data structures and shared utilities used in network sync +# +# Copyright (c) 2018-2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or +# distributed except according to those terms. + +## Aristo (aka Patricia) DB records merge test + +import + std/[algorithm, sequtils, sets], + eth/common, + stew/results, + unittest2, + ../../nimbus/db/aristo/[ + aristo_desc, aristo_debug, aristo_error, aristo_merge, aristo_nearby], + ../../nimbus/sync/snap/range_desc, + ./test_helpers + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +proc fwdWalkLeafsCompleteDB( + db: AristoDbRef; + tags: openArray[NodeTag]; + noisy: bool; + ): tuple[visited: int, error: AristoError] = + let + tLen = tags.len + var + error = AristoError(0) + tag = (tags[0].u256 div 2).NodeTag + n = 0 + while true: + let rc = tag.nearbyRight(db.lRoot, db) # , noisy) + #noisy.say "=================== ", n + if rc.isErr: + if rc.error != NearbyBeyondRange: + noisy.say "***", "[", n, "/", tLen-1, "] fwd-walk error=", rc.error + error = rc.error + check rc.error == AristoError(0) + elif n != tLen: + error = AristoError(1) + check n == tLen + break + if tLen <= n: + noisy.say "***", "[", n, "/", tLen-1, "] fwd-walk -- ", + " oops, too many leafs (index overflow)" + error = AristoError(1) + check n < tlen + break + if rc.value != tags[n]: + noisy.say "***", "[", n, "/", tLen-1, "] fwd-walk -- leafs differ,", + " got=", rc.value.pp(db), + " wanted=", tags[n].pp(db) #, " db-dump\n ", db.pp + error = AristoError(1) + check rc.value == tags[n] + break + if rc.value < high(NodeTag): + tag = (rc.value.u256 + 1).NodeTag + n.inc + + (n,error) + + +proc revWalkLeafsCompleteDB( + db: AristoDbRef; + tags: openArray[NodeTag]; + noisy: bool; + ): tuple[visited: int, error: AristoError] = + let + tLen = tags.len + var + error = AristoError(0) + delta = ((high(UInt256) - tags[^1].u256) div 2) + tag = (tags[^1].u256 + delta).NodeTag + n = tLen-1 + while true: # and false: + let rc = tag.nearbyLeft(db.lRoot, db) # , noisy) + if rc.isErr: + if rc.error != NearbyBeyondRange: + noisy.say "***", "[", n, "/", tLen-1, "] rev-walk error=", rc.error + error = rc.error + check rc.error == AristoError(0) + elif n != -1: + error = AristoError(1) + check n == -1 + break + if n < 0: + noisy.say "***", "[", n, "/", tLen-1, "] rev-walk -- ", + " oops, too many leafs (index underflow)" + error = AristoError(1) + check 0 <= n + break + if rc.value != tags[n]: + noisy.say "***", "[", n, "/", tLen-1, "] rev-walk -- leafs differ,", + " got=", rc.value.pp(db), + " wanted=", tags[n]..pp(db) #, " db-dump\n ", db.pp + error = AristoError(1) + check rc.value == tags[n] + break + if low(NodeTag) < rc.value: + tag = (rc.value.u256 - 1).NodeTag + n.dec + + (tLen-1 - n, error) + +# ------------------------------------------------------------------------------ +# Public test function +# ------------------------------------------------------------------------------ + +proc test_nearbyKvpList*( + noisy: bool; + list: openArray[ProofTrieData]; + resetDb = false; + ) = + var + db = AristoDbRef() + tagSet: HashSet[NodeTag] + for n,w in list: + if resetDb: + db = AristoDbRef() + tagSet.reset + let + lstLen = list.len + lTabLen = db.lTab.len + leafs = w.kvpLst + added = db.merge leafs + + check added.error == AristoError(0) + check db.lTab.len == lTabLen + added.merged + check added.merged + added.dups == leafs.len + + for w in leafs: + tagSet.incl w.pathTag + + let + tags = tagSet.toSeq.sorted + fwdWalk = db.fwdWalkLeafsCompleteDB(tags, noisy=true) + revWalk = db.revWalkLeafsCompleteDB(tags, noisy=true) + + check fwdWalk.error == AristoError(0) + check revWalk.error == AristoError(0) + check fwdWalk == revWalk + + if {fwdWalk.error, revWalk.error} != {AristoError(0)}: + noisy.say "***", "<", n, "/", lstLen-1, "> db dump", + "\n post-state ", db.pp, + "\n" + break + + #noisy.say "***", "sample ",n,"/",lstLen-1, " visited=", fwdWalk.visited + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------