From 11bb33d0bcc1fa20bfd854e3bbadb3c41bdcc50b Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Fri, 2 Jun 2023 20:21:46 +0100 Subject: [PATCH] Added delete fuctionality (#1596) --- nimbus/db/aristo/aristo_delete.nim | 135 ++++++++++++++++++++++ nimbus/db/aristo/aristo_error.nim | 8 ++ nimbus/db/aristo/aristo_merge.nim | 3 + nimbus/db/aristo/aristo_nearby.nim | 26 ++--- tests/test_aristo.nim | 19 +++- tests/test_aristo/test_delete.nim | 177 +++++++++++++++++++++++++++++ 6 files changed, 349 insertions(+), 19 deletions(-) create mode 100644 nimbus/db/aristo/aristo_delete.nim create mode 100644 tests/test_aristo/test_delete.nim diff --git a/nimbus/db/aristo/aristo_delete.nim b/nimbus/db/aristo/aristo_delete.nim new file mode 100644 index 000000000..1d19a7969 --- /dev/null +++ b/nimbus/db/aristo/aristo_delete.nim @@ -0,0 +1,135 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Aristo DB -- Patricia Trie delete funcionality +## ============================================== +## +## Deleate by `Hike` type chain of vertices. + +{.push raises: [].} + +import + std/[sets, tables], + chronicles, + eth/[common, trie/nibbles], + stew/results, + "."/[aristo_constants, aristo_desc, aristo_error, aristo_get, aristo_hike, + aristo_path, aristo_vid] + +logScope: + topics = "aristo-delete" + +# ------------------------------------------------------------------------------ +# Private functions +# ------------------------------------------------------------------------------ + +proc branchStillNeeded(vtx: VertexRef): bool = + for n in 0 .. 15: + if vtx.bVid[n] != VertexID(0): + return true + +proc clearKey(db: AristoDbRef; vid: VertexID) = + let key = db.kMap.getOrDefault(vid, EMPTY_ROOT_KEY) + if key != EMPTY_ROOT_KEY: + db.kMap.del vid + db.pAmk.del key + +proc doneWith(db: AristoDbRef; vid: VertexID) = + # Remove entry + db.vidDispose vid + db.sTab.del vid + db.clearKey vid # Update Merkle hash + + +proc deleteImpl( + hike: Hike; # Fully expanded path + pathTag: NodeTag; # `Patricia Trie` path root-to-leaf + db: AristoDbRef; # Database, top layer + ): Result[void,(VertexID,AristoError)] = + ## Implementation of *delete* functionality. + if hike.error != AristoError(0): + if 0 < hike.legs.len: + return err((hike.legs[^1].wp.vid,hike.error)) + return err((VertexID(0),hike.error)) + + # doAssert 0 < hike.legs.len and hike.tail.len == 0 # as assured by `hikeUp()` + var inx = hike.legs.len - 1 + + # Remove leaf entry on the top + let lf = hike.legs[inx].wp + if lf.vtx.vType != Leaf: + return err((lf.vid,DelLeafExpexted)) + if lf.vid in db.pPrf: + return err((lf.vid, DelLeafLocked)) + db.doneWith lf.vid + inx.dec + + while 0 <= inx: + # Unlink child node + let br = hike.legs[inx].wp + if br.vtx.vType != Branch: + return err((br.vid,DelBranchExpexted)) + if br.vid in db.pPrf: + return err((br.vid, DelBranchLocked)) + br.vtx.bVid[hike.legs[inx].nibble] = VertexID(0) + + if br.vtx.branchStillNeeded: + db.clearKey br.vid + break + + # Remove this `Branch` entry + db.doneWith br.vid + inx.dec + + if inx < 0: + break + + # There might be an optional `Extension` to remove + let ext = hike.legs[inx].wp + if ext.vtx.vType == Extension: + if br.vid in db.pPrf: + return err((ext.vid, DelExtLocked)) + db.doneWith ext.vid + inx.dec + + # Delete leaf entry + db.lTab.del pathTag + if db.lTab.len == 0: + db.lRoot = VertexID(0) + + ok() + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc delete*( + hike: Hike; # Fully expanded chain of vertices + db: AristoDbRef; # Database, top layer + ): Result[void,(VertexID,AristoError)] = + ## Delete argument `hike` chain of vertices from the database + # Need path in order to remove it from `lTab[]` + let pathTag = block: + let rc = hike.to(NibblesSeq).pathToTag() + if rc.isErr: + return err((VertexID(0),DelPathTagError)) + rc.value + hike.deleteImpl(pathTag, db) + +proc delete*( + pathTag: NodeTag; # `Patricia Trie` path root-to-leaf + db: AristoDbRef; # Database, top layer + ): Result[void,(VertexID,AristoError)] = + ## Variant of `delete()` + pathTag.hikeUp(db.lRoot, db).deleteImpl(pathTag, db) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_error.nim b/nimbus/db/aristo/aristo_error.nim index fa0e2f933..fa92da9a4 100644 --- a/nimbus/db/aristo/aristo_error.nim +++ b/nimbus/db/aristo/aristo_error.nim @@ -118,4 +118,12 @@ type NearbyPathTailInxOverflow NearbyUnexpectedVtx + # Deletion of vertices, `delete()` + DelPathTagError + DelLeafExpexted + DelLeafLocked + DelBranchExpexted + DelBranchLocked + DelExtLocked + # End diff --git a/nimbus/db/aristo/aristo_merge.nim b/nimbus/db/aristo/aristo_merge.nim index 91ffbd621..0a040fb31 100644 --- a/nimbus/db/aristo/aristo_merge.nim +++ b/nimbus/db/aristo/aristo_merge.nim @@ -64,6 +64,9 @@ proc `xPfx=`(vtx: VertexRef, val: NibblesSeq) = of Branch: doAssert vtx.vType != Branch # Ooops +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ proc clearMerkleKeys( db: AristoDbRef; # Database, top layer diff --git a/nimbus/db/aristo/aristo_nearby.nim b/nimbus/db/aristo/aristo_nearby.nim index 591db89b4..c414c7ae5 100644 --- a/nimbus/db/aristo/aristo_nearby.nim +++ b/nimbus/db/aristo/aristo_nearby.nim @@ -75,7 +75,7 @@ proc branchNibbleMax*(vtx: VertexRef; maxInx: int8): int8 = # ------------------------------------------------------------------------------ proc complete( - hike: Hike; # Partially expanded path + hike: Hike; # Partially expanded chain of vertices vid: VertexID; # Start ID db: AristoDbRef; # Database layer hikeLenMax: static[int]; # Beware of loops (if any) @@ -123,7 +123,7 @@ proc complete( proc zeroAdjust( - hike: Hike; # Partially expanded path + hike: Hike; # Partially expanded chain of vertices db: AristoDbRef; # Database layer doLeast: static[bool]; # Direction: *least* or *most* ): Hike = @@ -204,9 +204,9 @@ proc zeroAdjust( proc finalise( - hike: Hike; # Partially expanded path + hike: Hike; # Partially expanded chain of vertices db: AristoDbRef; # Database layer - moveRight: static[bool]; # Direction of next node + moveRight: static[bool]; # Direction of next vertex ): Hike = ## Handle some pathological cases after main processing failed proc beyond(p: Hike; pfx: NibblesSeq): bool = @@ -258,10 +258,10 @@ proc finalise( proc nearbyNext( - hike: Hike; # Partially expanded path + hike: Hike; # Partially expanded chain of vertices db: AristoDbRef; # Database layer hikeLenMax: static[int]; # Beware of loops (if any) - moveRight: static[bool]; # Direction of next node + moveRight: static[bool]; # Direction of next vertex ): Hike = ## Unified implementation of `nearbyRight()` and `nearbyLeft()`. proc accept(nibble: int8): bool = @@ -364,11 +364,11 @@ proc nearbyNext( proc nearbyNext( - baseTag: NodeTag; # Some node + baseTag: NodeTag; # Some `Patricia Trie` path root: VertexID; # State root db: AristoDbRef; # Database layer hikeLenMax: static[int]; # Beware of loops (if any) - moveRight:static[ bool]; # Direction of next node + moveRight:static[ bool]; # Direction of next vertex ): Result[NodeTag,AristoError] = ## Variant of `nearbyNext()`, convenience wrapper let hike = baseTag.hikeUp(root,db).nearbyNext(db, hikeLenMax, moveRight) @@ -388,7 +388,7 @@ proc nearbyNext( # ------------------------------------------------------------------------------ proc nearbyRight*( - hike: Hike; # Partially expanded path + hike: Hike; # Partially expanded chain of vertices db: AristoDbRef; # Database layer ): Hike = ## Extends the maximally extended argument nodes `hike` to the right (i.e. @@ -403,7 +403,7 @@ proc nearbyRight*( hike.nearbyNext(db, 64, moveRight=true) proc nearbyRight*( - nodeTag: NodeTag; # Some node + nodeTag: NodeTag; # Some `Patricia Trie` path root: VertexID; # State root db: AristoDbRef; # Database layer ): Result[NodeTag,AristoError] = @@ -412,7 +412,7 @@ proc nearbyRight*( nodeTag.nearbyNext(root, db, 64, moveRight=true) proc nearbyLeft*( - hike: Hike; # Partially expanded path + hike: Hike; # Partially expanded chain of vertices db: AristoDbRef; # Database layer ): Hike = ## Similar to `nearbyRight()`. @@ -422,7 +422,7 @@ proc nearbyLeft*( hike.nearbyNext(db, 64, moveRight=false) proc nearbyLeft*( - nodeTag: NodeTag; # Some node + nodeTag: NodeTag; # Some `Patricia Trie` path root: VertexID; # State root db: AristoDbRef; # Database layer ): Result[NodeTag,AristoError] = @@ -435,7 +435,7 @@ proc nearbyLeft*( # ------------------------------------------------------------------------------ proc nearbyRightMissing*( - hike: Hike; # Partially expanded path + hike: Hike; # Partially expanded chain of vertices db: AristoDbRef; # Database layer ): Result[bool,AristoError] = ## Returns `true` if the maximally extended argument nodes `hike` is the diff --git a/tests/test_aristo.nim b/tests/test_aristo.nim index 7348ee15f..613cf7303 100644 --- a/tests/test_aristo.nim +++ b/tests/test_aristo.nim @@ -23,7 +23,8 @@ import ../nimbus/sync/snap/worker/db/[rocky_bulk_load, snapdb_accounts, snapdb_desc], ./replay/[pp, undump_accounts, undump_storages], ./test_sync_snap/[snap_test_xx, test_accounts, test_types], - ./test_aristo/[test_helpers, test_merge, test_nearby, test_transcode] + ./test_aristo/[ + test_delete, test_helpers, test_merge, test_nearby, test_transcode] const baseDir = [".", "..", ".."/"..", $DirSep] @@ -200,6 +201,9 @@ proc accountsRunner(noisy=true; sample=accSample, resetDb=false) = test &"Traverse accounts database w/{accLst.len} account lists": noisy.test_nearbyKvpList(accLst, resetDb) + test &"Delete accounts database, successively {accLst.len} entries": + noisy.test_delete accLst + proc storagesRunner( noisy = true; @@ -223,6 +227,9 @@ proc storagesRunner( test &"Traverse storage slots database w/{stoLst.len} account lists": noisy.test_nearbyKvpList(stoLst, resetDb) + test &"Delete storage database, successively {stoLst.len} entries": + noisy.test_delete stoLst + # ------------------------------------------------------------------------------ # Main function(s) # ------------------------------------------------------------------------------ @@ -237,7 +244,7 @@ when isMainModule: noisy = defined(debug) or true # Borrowed from `test_sync_snap.nim` - when true and false: + when true: # and false: for n,sam in snapTestList: noisy.transcodeRunner(sam) for n,sam in snapTestStorageList: @@ -248,18 +255,18 @@ when isMainModule: import ./test_sync_snap/snap_other_xx noisy.showElapsed("@snap_other_xx"): for n,sam in snapOtherList: - noisy.accountsRunner(sam) + noisy.accountsRunner(sam, resetDb=true) # This one usues dumps from the external `nimbus-eth1-blob` repo - when true: # and false: + when true and false: import ./test_sync_snap/snap_storage_xx let knownFailures: KnownHasherFailure = @[ ("storages5__34__41_dump#10.20512",(VertexID(1),HashifyRootHashMismatch)), ] noisy.showElapsed("@snap_storage_xx"): for n,sam in snapStorageList: - noisy.accountsRunner(sam) - noisy.storagesRunner(sam,oops=knownFailures) + noisy.accountsRunner(sam, resetDb=true) + noisy.storagesRunner(sam, resetDb=true, oops=knownFailures) when true: # and false: for n,sam in snapTestList: diff --git a/tests/test_aristo/test_delete.nim b/tests/test_aristo/test_delete.nim new file mode 100644 index 000000000..0fe8e0abe --- /dev/null +++ b/tests/test_aristo/test_delete.nim @@ -0,0 +1,177 @@ +# Nimbus - Types, data structures and shared utilities used in network sync +# +# Copyright (c) 2018-2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or +# distributed except according to those terms. + +## Aristo (aka Patricia) DB records merge test + +import + std/[algorithm, bitops, sequtils], + eth/common, + stew/results, + unittest2, + ../../nimbus/db/aristo/[ + aristo_desc, aristo_delete, aristo_error, + aristo_hashify, aristo_nearby, aristo_merge], + ../../nimbus/sync/snap/range_desc, + ./test_helpers + +type + TesterDesc = object + prng: uint32 ## random state + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +proc posixPrngRand(state: var uint32): byte = + ## POSIX.1-2001 example of a rand() implementation, see manual page rand(3). + state = state * 1103515245 + 12345; + let val = (state shr 16) and 32767 # mod 2^31 + (val shr 8).byte # Extract second byte + +proc rand[W: SomeInteger|VertexID](ap: var TesterDesc; T: type W): T = + var a: array[sizeof T,byte] + for n in 0 ..< sizeof T: + a[n] = ap.prng.posixPrngRand().byte + when sizeof(T) == 1: + let w = uint8.fromBytesBE(a).T + when sizeof(T) == 2: + let w = uint16.fromBytesBE(a).T + when sizeof(T) == 4: + let w = uint32.fromBytesBE(a).T + else: + let w = uint64.fromBytesBE(a).T + when T is SomeUnsignedInt: + # That way, `fromBytesBE()` can be applied to `uint` + result = w + else: + # That way the result is independent of endianness + (addr result).copyMem(unsafeAddr w, sizeof w) + +proc init(T: type TesterDesc; seed: int): TesterDesc = + result.prng = (seed and 0x7fffffff).uint32 + +proc rand(td: var TesterDesc; top: int): int = + if 0 < top: + let mask = (1 shl (8 * sizeof(int) - top.countLeadingZeroBits)) - 1 + for _ in 0 ..< 100: + let w = mask and td.rand(typeof(result)) + if w < top: + return w + raiseAssert "Not here (!)" + +# ----------------------- + +proc fwdWalkVerify( + db: AristoDbRef; + noisy: bool; + ): tuple[visited: int, error: AristoError] = + let + lTabLen = db.lTab.len + var + error = AristoError(0) + tag: NodeTag + n = 0 + while n < lTabLen + 1: + let rc = tag.nearbyRight(db.lRoot, db) # , noisy) + #noisy.say "=================== ", n + if rc.isErr: + if rc.error != NearbyBeyondRange: + noisy.say "***", "<", n, "/", lTabLen-1, "> fwd-walk error=", rc.error + error = rc.error + check rc.error == AristoError(0) + break + if rc.value < high(NodeTag): + tag = (rc.value.u256 + 1).NodeTag + n.inc + + if error != AristoError(0): + return (n,error) + + if n != lTabLen: + check n == lTabLen + return (-1, AristoError(1)) + + (0, AristoError(0)) + +# ------------------------------------------------------------------------------ +# Public test function +# ------------------------------------------------------------------------------ + +proc test_delete*( + noisy: bool; + list: openArray[ProofTrieData]; + ) = + var td = TesterDesc.init 42 + for n,w in list: + let + db = AristoDbRef() + lstLen = list.len + added = db.merge w.kvpLst + + if added.error != AristoError(0): + check added.error == AristoError(0) + return + + let rc = db.hashify + if rc.isErr: + check rc.error == (VertexID(0),AristoError(0)) + return + # Now `db` represents a (fully labelled) `Merkle Patricia Tree` + + # Provide a (reproducible) peudo-random copy of the leafs list + var leafs = db.lTab.keys.toSeq.mapIt(it.Uint256).sorted.mapIt(it.NodeTag) + if 2 < leafs.len: + for n in 0 ..< leafs.len-1: + let r = n + td.rand(leafs.len - n) + leafs[n].swap leafs[r] + + let uMax = leafs.len - 1 + for u,pathTag in leafs: + let rc = pathTag.delete(db) # , noisy=(tags.len < 2)) + + if rc.isErr: + check rc.error == (VertexID(0),AristoError(0)) + return + if pathTag in db.lTab: + check pathTag notin db.lTab + return + if uMax != db.lTab.len + u: + check uMax == db.lTab.len + u + return + + # Walking the database is too slow for large tables. So the hope is that + # potential errors will not go away and rather pop up later, as well. + const tailCheck = 999 + if uMax < u + tailCheck: + if u < uMax: + let vfy = db.fwdWalkVerify(noisy) + if vfy.error != AristoError(0): + check vfy == (0, AristoError(0)) + return + elif 0 < db.sTab.len: + check db.sTab.len == 0 + return + let rc = db.hashifyCheck(relax=true) + if rc.isErr: + check rc.error == (VertexID(0),AristoError(0)) + return + + when true and false: + if uMax < u + tailCheck or (u mod 777) == 3: + noisy.say "***", "step lTab=", db.lTab.len + + when true and false: + noisy.say "***", "sample <", n, "/", list.len-1, ">", + " lstLen=", w.kvpLst.len + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------