diff --git a/nimbus/db/aristo/aristo_check.nim b/nimbus/db/aristo/aristo_check.nim new file mode 100644 index 000000000..4f6652aa4 --- /dev/null +++ b/nimbus/db/aristo/aristo_check.nim @@ -0,0 +1,93 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Aristo DB -- Consistency checks +## =============================== +## +{.push raises: [].} + +import + std/[algorithm, sequtils, sets, tables], + eth/common, + stew/[interval_set, results], + ./aristo_init/[aristo_memory, aristo_rocksdb], + "."/[aristo_desc, aristo_get, aristo_init, aristo_vid], + ./aristo_hashify/hashify_helper, + ./aristo_check/[check_be, check_cache] + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc checkCache*( + db: AristoDb; # Database, top layer + relax = false; # Check existing hashes only + ): Result[void,(VertexID,AristoError)] = + ## Verify that the cache structure is correct as it would be after `merge()` + ## and `hashify()` operations. Unless `relaxed` is set `true` it would not + ## fully check against the backend, which is typically not applicable after + ## `delete()` operations. + ## + ## The following is verified: + ## + ## * Each `sTab[]` entry has a valid vertex which can be compiled as a node. + ## If `relax` is set `false`, the Merkle hashes are recompiled and must + ## match. + ## + ## * The hash table `kMap[]` and its inverse lookup table `pAmk[]` must + ## correnspond. + ## + if relax: + let rc = db.checkCacheRelaxed() + if rc.isErr: + return rc + else: + let rc = db.checkCacheStrict() + if rc.isErr: + return rc + + db.checkCacheCommon() + + +proc checkBE*( + db: AristoDb; # Database, top layer + relax = true; # Not re-compiling hashes if `true` + cache = true; # Also verify cache + ): Result[void,(VertexID,AristoError)] = + ## Veryfy database backend structure. If the argument `relax` is set `false`, + ## all necessary Merkle hashes are compiled and verified. If the argument + ## `cache` is set `true`, the cache is also checked so that a `safe()` + ## operation will leave the backend consistent. + ## + ## The following is verified: + ## + ## * Each vertex ID on the structural table can be represented as a Merkle + ## patricia Tree node. If `relax` is set `false`, the Merkle hashes are + ## all recompiled and must match. + ## + ## * The set of free vertex IDa as potentally suppliedby the ID generator + ## state is disjunct to the set of already used vertex IDs on the database. + ## Moreover, the union of both sets is equivalent to the set of positive + ## `uint64` numbers. + ## + if not db.backend.isNil: + let be = db.to(TypedBackendRef) + case be.kind: + of BackendMemory: + return be.MemBackendRef.checkBE(db, cache=cache, relax=relax) + of BackendRocksDB: + return be.RdbBackendRef.checkBE(db, cache=cache, relax=relax) + of BackendNone: + discard + ok() + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_check/check_be.nim b/nimbus/db/aristo/aristo_check/check_be.nim new file mode 100644 index 000000000..8ff12df9d --- /dev/null +++ b/nimbus/db/aristo/aristo_check/check_be.nim @@ -0,0 +1,175 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +{.push raises: [].} + +import + std/[algorithm, sequtils, sets, tables], + eth/common, + stew/interval_set, + ../aristo_hashify/hashify_helper, + ../aristo_init/[aristo_memory, aristo_rocksdb], + ".."/[aristo_desc, aristo_get, aristo_vid] + +const + Vid2 = @[VertexID(2)].toHashSet + +# ------------------------------------------------------------------------------ +# Private helper +# ------------------------------------------------------------------------------ + +proc invTo(s: IntervalSetRef[VertexID,uint64]; T: type HashSet[VertexID]): T = + ## Convert the complement of the argument list `s` to a set of vertex IDs + ## as it would appear with a vertex generator state list. + if s.total < high(uint64): + for w in s.increasing: + if w.maxPt == high(VertexID): + result.incl w.minPt # last interval + else: + for pt in w.minPt .. w.maxPt: + result.incl pt + +proc toNodeBe( + vtx: VertexRef; # Vertex to convert + db: AristoDb; # Database, top layer + ): Result[NodeRef,VertexID] = + ## Similar to `toNode()` but fetching from the backend only + case vtx.vType: + of Leaf: + return ok NodeRef(vType: Leaf, lPfx: vtx.lPfx, lData: vtx.lData) + of Branch: + let node = NodeRef(vType: Branch, bVid: vtx.bVid) + var missing: seq[VertexID] + for n in 0 .. 15: + let vid = vtx.bVid[n] + if vid.isValid: + let rc = db.getKeyBackend vid + if rc.isOk and rc.value.isValid: + node.key[n] = rc.value + else: + return err(vid) + else: + node.key[n] = VOID_HASH_KEY + return ok node + of Extension: + let + vid = vtx.eVid + rc = db.getKeyBackend vid + if rc.isOk and rc.value.isValid: + let node = NodeRef(vType: Extension, ePfx: vtx.ePfx, eVid: vid) + node.key[0] = rc.value + return ok node + return err(vid) + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc checkBE*[T]( + be: T; # backend descriptor + db: AristoDb; # Database, top layer + relax: bool; # Not compiling hashes if `true` + cache: bool; # Also verify cache + ): Result[void,(VertexID,AristoError)] = + ## Make sure that each vertex has a Merkle hash and vice versa. Also check + ## the vertex ID generator state. + let vids = IntervalSetRef[VertexID,uint64].init() + discard vids.merge Interval[VertexID,uint64].new(VertexID(1),high(VertexID)) + + for (_,vid,vtx) in be.walkVtx: + if not vtx.isValid: + return err((vid,CheckBeVtxInvalid)) + let rc = db.getKeyBackend vid + if rc.isErr or not rc.value.isValid: + return err((vid,CheckBeKeyMissing)) + + for (_,vid,key) in be.walkKey: + if not key.isvalid: + return err((vid,CheckBeKeyInvalid)) + let rc = db.getVtxBackend vid + if rc.isErr or not rc.value.isValid: + return err((vid,CheckBeVtxMissing)) + let rx = rc.value.toNodeBe db # backend only + if rx.isErr: + return err((vid,CheckBeKeyCantCompile)) + if not relax: + let expected = rx.value.toHashKey + if expected != key: + return err((vid,CheckBeKeyMismatch)) + discard vids.reduce Interval[VertexID,uint64].new(vid,vid) + + # Compare calculated state against database state + block: + # Extract vertex ID generator state + var vGen: HashSet[VertexID] + for (_,_,w) in be.walkIdg: + vGen = vGen + w.toHashSet + let + vGenExpected = vids.invTo(HashSet[VertexID]) + delta = vGenExpected -+- vGen # symmetric difference + if 0 < delta.len: + # Exclude fringe case when there is a single root vertex only + if vGenExpected != Vid2 or 0 < vGen.len: + return err((delta.toSeq.sorted[^1],CheckBeGarbledVGen)) + + # Check cache against backend + if cache: + + # Check structural table + for (vid,vtx) in db.top.sTab.pairs: + # A `kMap[]` entry must exist. + if not db.top.kMap.hasKey vid: + return err((vid,CheckBeCacheKeyMissing)) + if vtx.isValid: + # Register existing vid against backend generator state + discard vids.reduce Interval[VertexID,uint64].new(vid,vid) + else: + # Some vertex is to be deleted, the key must be empty + let lbl = db.top.kMap.getOrVoid vid + if lbl.isValid: + return err((vid,CheckBeCacheKeyNonEmpty)) + # There must be a representation on the backend DB + if db.getVtxBackend(vid).isErr: + return err((vid,CheckBeCacheVidUnsynced)) + # Register deleted vid against backend generator state + discard vids.merge Interval[VertexID,uint64].new(vid,vid) + + # Check key table + for (vid,lbl) in db.top.kMap.pairs: + let vtx = db.getVtx vid + if not db.top.sTab.hasKey(vid) and not vtx.isValid: + return err((vid,CheckBeCacheKeyDangling)) + if lbl.isValid and not relax: + if not vtx.isValid: + return err((vid,CheckBeCacheVtxDangling)) + let rc = vtx.toNode db # compile cache first + if rc.isErr: + return err((vid,CheckBeCacheKeyCantCompile)) + let expected = rc.value.toHashKey + if expected != lbl.key: + return err((vid,CheckBeCacheKeyMismatch)) + + # Check vGen + var tmp = AristoDB(top: AristoLayerRef(vGen: db.top.vGen)) + tmp.vidReorg() + let + vGen = tmp.top.vGen.toHashSet + vGenExpected = vids.invTo(HashSet[VertexID]) + delta = vGenExpected -+- vGen # symmetric difference + if 0 < delta.len: + # Exclude fringe case when there is a single root vertex only + if vGenExpected != Vid2 or 0 < vGen.len: + return err((delta.toSeq.sorted[^1],CheckBeCacheGarbledVGen)) + + ok() + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_check/check_cache.nim b/nimbus/db/aristo/aristo_check/check_cache.nim new file mode 100644 index 000000000..9c825eac3 --- /dev/null +++ b/nimbus/db/aristo/aristo_check/check_cache.nim @@ -0,0 +1,126 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +{.push raises: [].} + +import + std/[sequtils, sets, tables], + eth/common, + stew/results, + ../aristo_hashify/hashify_helper, + ".."/[aristo_desc, aristo_get] + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc checkCacheStrict*( + db: AristoDb; # Database, top layer + ): Result[void,(VertexID,AristoError)] = + for (vid,vtx) in db.top.sTab.pairs: + let rc = vtx.toNode db + if rc.isErr: + return err((vid,CheckStkVtxIncomplete)) + + let lbl = db.top.kMap.getOrVoid vid + if not lbl.isValid: + return err((vid,CheckStkVtxKeyMissing)) + if lbl.key != rc.value.toHashKey: + return err((vid,CheckStkVtxKeyMismatch)) + + let revVid = db.top.pAmk.getOrVoid lbl + if not revVid.isValid: + return err((vid,CheckStkRevKeyMissing)) + if revVid != vid: + return err((vid,CheckStkRevKeyMismatch)) + + if 0 < db.top.pAmk.len and db.top.pAmk.len < db.top.sTab.len: + # Cannot have less changes than cached entries + return err((VertexID(0),CheckStkVtxCountMismatch)) + + ok() + + +proc checkCacheRelaxed*( + db: AristoDb; # Database, top layer + ): Result[void,(VertexID,AristoError)] = + if 0 < db.top.pPrf.len: + for vid in db.top.pPrf: + let vtx = db.top.sTab.getOrVoid vid + if vtx.isValid: + let rc = vtx.toNode db + if rc.isErr: + return err((vid,CheckRlxVtxIncomplete)) + + let lbl = db.top.kMap.getOrVoid vid + if not lbl.isValid: + return err((vid,CheckRlxVtxKeyMissing)) + if lbl.key != rc.value.toHashKey: + return err((vid,CheckRlxVtxKeyMismatch)) + + let revVid = db.top.pAmk.getOrVoid lbl + if not revVid.isValid: + return err((vid,CheckRlxRevKeyMissing)) + if revVid != vid: + return err((vid,CheckRlxRevKeyMismatch)) + else: + # Is be a deleted entry + let rc = db.getVtxBackend vid + if rc.isErr: + return err((vid,CheckRlxVidVtxBeMissing)) + if not db.top.kMap.hasKey vid: + return err((vid,CheckRlxVtxEmptyKeyMissing)) + if db.top.kMap.getOrVoid(vid).isValid: + return err((vid,CheckRlxVtxEmptyKeyExpected)) + else: + for (vid,lbl) in db.top.kMap.pairs: + if lbl.isValid: # Otherwise to be deleted + let vtx = db.getVtx vid + if vtx.isValid: + let rc = vtx.toNode db + if rc.isOk: + if lbl.key != rc.value.toHashKey: + return err((vid,CheckRlxVtxKeyMismatch)) + + let revVid = db.top.pAmk.getOrVoid lbl + if not revVid.isValid: + return err((vid,CheckRlxRevKeyMissing)) + if revVid != vid: + return err((vid,CheckRlxRevKeyMissing)) + if revVid != vid: + return err((vid,CheckRlxRevKeyMismatch)) + ok() + + +proc checkCacheCommon*( + db: AristoDb; # Database, top layer + ): Result[void,(VertexID,AristoError)] = + # Some `kMap[]` entries may ne void indicating backend deletion + let kMapCount = db.top.kMap.values.toSeq.filterIt(it.isValid).len + + if db.top.pAmk.len != kMapCount: + var knownKeys: HashSet[VertexID] + for (key,vid) in db.top.pAmk.pairs: + if not db.top.kMap.hasKey(vid): + return err((vid,CheckAnyRevVtxMissing)) + if vid in knownKeys: + return err((vid,CheckAnyRevVtxDup)) + knownKeys.incl vid + return err((VertexID(0),CheckAnyRevCountMismatch)) # should not apply(!) + + for vid in db.top.pPrf: + if not db.top.kMap.hasKey(vid): + return err((vid,CheckAnyVtxLockWithoutKey)) + ok() + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ + diff --git a/nimbus/db/aristo/aristo_debug.nim b/nimbus/db/aristo/aristo_debug.nim index 8a33b57c5..46380c707 100644 --- a/nimbus/db/aristo/aristo_debug.nim +++ b/nimbus/db/aristo/aristo_debug.nim @@ -78,6 +78,9 @@ proc ppVid(vid: VertexID; pfx = true): string = else: result &= "ø" +proc ppVidList(vGen: openArray[VertexID]): string = + "[" & vGen.mapIt(it.ppVid).join(",") & "]" + proc vidCode(lbl: HashLabel, db: AristoDb): uint64 = if lbl.isValid: if not db.top.isNil: @@ -165,7 +168,7 @@ proc ppPayload(p: PayloadRef, db: AristoDb): string = proc ppVtx(nd: VertexRef, db: AristoDb, vid: VertexID): string = if not nd.isValid: - result = "n/a" + result = "ø" else: if db.top.isNil or not vid.isValid or vid in db.top.pPrf: result = ["L(", "X(", "B("][nd.vType.ord] @@ -186,6 +189,29 @@ proc ppVtx(nd: VertexRef, db: AristoDb, vid: VertexID): string = result &= "," result &= ")" +proc ppSTab( + sTab: Table[VertexID,VertexRef]; + db = AristoDb(); + indent = 4; + ): string = + "{" & sTab.sortedKeys + .mapIt((it, sTab.getOrVoid it)) + .mapIt("(" & it[0].ppVid & "," & it[1].ppVtx(db,it[0]) & ")") + .join("," & indent.toPfx(1)) & "}" + +proc ppLTab( + lTab: Table[LeafTie,VertexID]; + indent = 4; + ): string = + var db = AristoDb() + "{" & lTab.sortedKeys + .mapIt((it, lTab.getOrVoid it)) + .mapIt("(" & it[0].ppLeafTie(db) & "," & it[1].ppVid & ")") + .join("," & indent.toPfx(1)) & "}" + +proc ppPPrf(pPrf: HashSet[VertexID]): string = + "{" & pPrf.sortedKeys.mapIt(it.ppVid).join(",") & "}" + proc ppXMap*( db: AristoDb; kMap: Table[VertexID,HashLabel]; @@ -215,7 +241,7 @@ proc ppXMap*( s &= "(" & s s &= ",*" & $count else: - s &= "£r(!)" + s &= "£ø" if s[0] == '(': s &= ")" s & "," @@ -233,7 +259,11 @@ proc ppXMap*( r.inc if r != vid: if i+1 != n: - result &= ".. " & revKeys[n-1].ppRevlabel + if i+1 == n-1: + result &= pfx + else: + result &= ".. " + result &= revKeys[n-1].ppRevlabel result &= pfx & vid.ppRevlabel (i, r) = (n, vid) if i < revKeys.len - 1: @@ -261,13 +291,18 @@ proc ppXMap*( result &= pfx result &= cache[i][0].ppNtry for n in 1 ..< cache.len: - let w = cache[n] - r[0].inc - r[1].inc + let + m = cache[n-1] + w = cache[n] + r = (r[0]+1, r[1]+1, r[2]) if r != w or w[2]: if i+1 != n: - result &= ".. " & cache[n-1][0].ppNtry - result &= pfx & cache[n][0].ppNtry + if i+1 == n-1: + result &= pfx + else: + result &= ".. " + result &= m[0].ppNtry + result &= pfx & w[0].ppNtry (i, r) = (n, w) if i < cache.len - 1: if i+1 != cache.len - 1: @@ -296,6 +331,63 @@ proc ppBe[T](be: T; db: AristoDb; indent: int): string = $(1+it[0]) & "(" & it[1].ppVid & "," & it[2].ppKey & ")" ).join(pfx2) & "}" + +proc ppCache( + db: AristoDb; + vGenOk: bool; + sTabOk: bool; + lTabOk: bool; + kMapOk: bool; + pPrfOk: bool; + indent = 4; + ): string = + let + pfx1 = indent.toPfx + pfx2 = indent.toPfx(1) + tagOk = 1 < sTabOk.ord + lTabOk.ord + kMapOk.ord + pPrfOk.ord + vGenOk.ord + var + pfy = "" + + proc doPrefix(s: string; dataOk: bool): string = + var rc: string + if tagOk: + rc = pfy & s & (if dataOk: pfx2 else: " ") + pfy = pfx1 + else: + rc = pfy + pfy = pfx2 + rc + + if not db.top.isNil: + if vGenOk: + let + tLen = db.top.vGen.len + info = "vGen(" & $tLen & ")" + result &= info.doPrefix(0 < tLen) & db.top.vGen.ppVidList + if sTabOk: + let + tLen = db.top.sTab.len + info = "sTab(" & $tLen & ")" + result &= info.doPrefix(0 < tLen) & db.top.sTab.ppSTab(db,indent+1) + if lTabOk: + let + tlen = db.top.lTab.len + info = "lTab(" & $tLen & ")" + result &= info.doPrefix(0 < tLen) & db.top.lTab.ppLTab(indent+1) + if kMapOk: + let + tLen = db.top.kMap.len + ulen = db.top.pAmk.len + lInf = if tLen == uLen: $tLen else: $tLen & "," & $ulen + info = "kMap(" & lInf & ")" + result &= info.doPrefix(0 < tLen + uLen) + result &= db.ppXMap(db.top.kMap,db.top.pAmk,indent+1) + if pPrfOk: + let + tLen = db.top.pPrf.len + info = "pPrf(" & $tLen & ")" + result &= info.doPrefix(0 < tLen) & db.top.pPrf.ppPPrf + # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ @@ -326,7 +418,7 @@ proc pp*(vid: VertexID): string = vid.ppVid proc pp*(vGen: openArray[VertexID]): string = - "[" & vGen.mapIt(it.ppVid).join(",") & "]" + vGen.ppVidList proc pp*(p: PayloadRef, db = AristoDb()): string = p.ppPayload(db) @@ -372,22 +464,13 @@ proc pp*(nd: NodeRef): string = nd.pp(db) proc pp*(sTab: Table[VertexID,VertexRef]; db = AristoDb(); indent = 4): string = - "{" & sTab.sortedKeys - .mapIt((it, sTab.getOrVoid it)) - .filterIt(it[1].isValid) - .mapIt("(" & it[0].ppVid & "," & it[1].ppVtx(db,it[0]) & ")") - .join("," & indent.toPfx(1)) & "}" + sTab.ppSTab proc pp*(lTab: Table[LeafTie,VertexID]; indent = 4): string = - var db = AristoDb() - "{" & lTab.sortedKeys - .mapIt((it, lTab.getOrVoid it)) - .filterIt(it[1].isValid) - .mapIt("(" & it[0].ppLeafTie(db) & "," & it[1].ppVid & ")") - .join("," & indent.toPfx(1)) & "}" + lTab.ppLTab proc pp*(pPrf: HashSet[VertexID]): string = - "{" & pPrf.sortedKeys.mapIt(it.ppVid).join(",") & "}" + pPrf.ppPPrf proc pp*(leg: Leg; db = AristoDb()): string = result = "(" & leg.wp.vid.ppVid & "," @@ -454,49 +537,31 @@ proc pp*( proc pp*( db: AristoDb; - vGenOk = true; - sTabOk = true; - lTabOk = true; - kMapOk = true; - pPrfOk = true; indent = 4; ): string = - let - pfx1 = indent.toPfx - pfx2 = indent.toPfx(1) - tagOk = 1 < sTabOk.ord + lTabOk.ord + kMapOk.ord + pPrfOk.ord + vGenOk.ord - var - pfy = "" - - proc doPrefix(s: string): string = - var rc: string - if tagOk: - rc = pfy & s & pfx2 - pfy = pfx1 - else: - rc = pfy - pfy = pfx2 - rc - - if not db.top.isNil: - if vGenOk: - let info = "vGen(" & $db.top.vGen.len & ")" - result &= info.doPrefix & db.top.vGen.pp - if sTabOk: - let info = "sTab(" & $db.top.sTab.len & ")" - result &= info.doPrefix & db.top.sTab.pp(db,indent+1) - if lTabOk: - let info = "lTab(" & $db.top.lTab.len & ")" - result &= info.doPrefix & db.top.lTab.pp(indent+1) - if kMapOk: - let info = "kMap(" & $db.top.kMap.len & "," & $db.top.pAmk.len & ")" - result &= info.doPrefix & db.ppXMap(db.top.kMap,db.top.pAmk,indent+1) - if pPrfOk: - let info = "pPrf(" & $db.top.pPrf.len & ")" - result &= info.doPrefix & db.top.pPrf.pp + db.ppCache( + vGenOk=true, sTabOk=true, lTabOk=true, kMapOk=true, pPrfOk=true) proc pp*( - be: AristoTypedBackendRef; + db: AristoDb; + xTabOk: bool; + indent = 4; + ): string = + db.ppCache( + vGenOk=true, sTabOk=xTabOk, lTabOk=xTabOk, kMapOk=true, pPrfOk=true) + +proc pp*( + db: AristoDb; + xTabOk: bool; + kMapOk: bool; + other = false; + indent = 4; + ): string = + db.ppCache( + vGenOk=other, sTabOk=xTabOk, lTabOk=xTabOk, kMapOk=kMapOk, pPrfOk=other) + +proc pp*( + be: TypedBackendRef; db: AristoDb; indent = 4; ): string = diff --git a/nimbus/db/aristo/aristo_delete.nim b/nimbus/db/aristo/aristo_delete.nim index f9054d58d..ca945afee 100644 --- a/nimbus/db/aristo/aristo_delete.nim +++ b/nimbus/db/aristo/aristo_delete.nim @@ -26,13 +26,21 @@ logScope: topics = "aristo-delete" # ------------------------------------------------------------------------------ -# Private functions +# Private heplers # ------------------------------------------------------------------------------ -proc branchStillNeeded(vtx: VertexRef): bool = +proc branchStillNeeded(vtx: VertexRef): Result[int,void] = + ## Returns the nibble if there is only one reference left. + var nibble = -1 for n in 0 .. 15: if vtx.bVid[n].isValid: - return true + if 0 <= nibble: + return ok(-1) + nibble = n + if 0 <= nibble: + return ok(nibble) + # Oops, degenerated branch node + err() proc clearKey( db: AristoDb; # Database, top layer @@ -52,15 +60,208 @@ proc doneWith( vid: VertexID; # Vertex IDs to clear ) = # Remove entry - db.vidDispose vid # Will be propagated to backend - db.top.sTab.del vid + if db.getVtxBackend(vid).isOk: + db.top.sTab[vid] = VertexRef(nil) # Will be propagated to backend + else: + db.top.sTab.del vid + db.vidDispose vid db.clearKey vid +# ------------------------------------------------------------------------------ +# Private functions +# ------------------------------------------------------------------------------ + +proc collapseBranch( + db: AristoDb; # Database, top layer + hike: Hike; # Fully expanded path + nibble: byte; # Applicable link for `Branch` vertex + ): Result[void,(VertexID,AristoError)] = + ## Convert/merge vertices: + ## :: + ## current | becomes | condition + ## | | + ## ^3 ^2 | ^3 ^2 | + ## -------------------+---------------------+------------------ + ## Branch
Branch | Branch Branch | 2 < legs.len (1) + ## Ext
Branch | Branch | 2 < legs.len (2) + ##
Branch | Branch | legs.len == 2 (3) + ## + ## Depending on whether the parent `par` is an extension, merge `br` into + ## `par`. Otherwise replace `br` by an extension. + ## + let br = hike.legs[^2].wp + + var xt = VidVtxPair( # Rewrite `br` + vid: br.vid, + vtx: VertexRef( + vType: Extension, + ePfx: @[nibble].initNibbleRange.slice(1), + eVid: br.vtx.bVid[nibble])) + + if 2 < hike.legs.len: # (1) or (2) + let par = hike.legs[^3].wp + case par.vtx.vType: + of Branch: # (1) + # Replace `br` (use `xt` as-is) + discard + + of Extension: # (2) + # Merge `br` into ^3 (update `xt`) + db.doneWith xt.vid + xt.vid = par.vid + xt.vtx.ePfx = par.vtx.ePfx & xt.vtx.ePfx + + of Leaf: + return err((par.vid,DelLeafUnexpected)) + + else: # (3) + # Replace `br` (use `xt` as-is) + discard + + db.top.sTab[xt.vid] = xt.vtx + ok() + + +proc collapseExt( + db: AristoDb; # Database, top layer + hike: Hike; # Fully expanded path + nibble: byte; # Link for `Branch` vertex `^2` + vtx: VertexRef; # Follow up extension vertex (nibble) + ): Result[void,(VertexID,AristoError)] = + ## Convert/merge vertices: + ## :: + ## ^3 ^2 `vtx` | ^3 ^2 | + ## --------------------+-----------------------+------------------ + ## Branch
Ext | Branch | 2 < legs.len (1) + ## Ext
Ext | | 2 < legs.len (2) + ##
Ext | | legs.len == 2 (3) + ## + ## Merge `vtx` into `br` and unlink `vtx`. + ## + let br = hike.legs[^2].wp + + var xt = VidVtxPair( # Merge `vtx` into `br` + vid: br.vid, + vtx: VertexRef( + vType: Extension, + ePfx: @[nibble].initNibbleRange.slice(1) & vtx.ePfx, + eVid: vtx.eVid)) + db.doneWith br.vtx.bVid[nibble] # `vtx` is obsolete now + + if 2 < hike.legs.len: # (1) or (2) + let par = hike.legs[^3].wp + case par.vtx.vType: + of Branch: # (1) + # Replace `br` by `^2 & vtx` (use `xt` as-is) + discard + + of Extension: # (2) + # Replace ^3 by `^3 & ^2 & vtx` (update `xt`) + db.doneWith xt.vid + xt.vid = par.vid + xt.vtx.ePfx = par.vtx.ePfx & xt.vtx.ePfx + + of Leaf: + return err((par.vid,DelLeafUnexpected)) + + else: # (3) + # Replace ^2 by `^2 & vtx` (use `xt` as-is) + discard + + db.top.sTab[xt.vid] = xt.vtx + ok() + + +proc collapseLeaf( + db: AristoDb; # Database, top layer + hike: Hike; # Fully expanded path + nibble: byte; # Link for `Branch` vertex `^2` + vtx: VertexRef; # Follow up leaf vertex (from nibble) + ): Result[void,(VertexID,AristoError)] = + ## Convert/merge vertices: + ## :: + ## current | becomes | condition + ## | | + ## ^4 ^3 ^2 `vtx` | ^4 ^3 ^2 | + ## -------------------------+----------------------------+------------------ + ## .. Branch
Leaf | .. Branch | 2 < legs.len (1) + ## Branch Ext
Leaf | Branch | 3 < legs.len (2) + ## Ext
Leaf | | legs.len == 3 (3) + ##
Leaf | | legs.len == 2 (4) + ## + ## Merge `
` and `Leaf` replacing one and removing the other. + ## + let br = hike.legs[^2].wp + + var lf = VidVtxPair( # Merge `br` into `vtx` + vid: br.vtx.bVid[nibble], + vtx: VertexRef( + vType: Leaf, + lPfx: @[nibble].initNibbleRange.slice(1) & vtx.lPfx, + lData: vtx.lData)) + db.doneWith br.vid # `br` is obsolete now + db.clearKey lf.vid # `vtx` was modified + + if 2 < hike.legs.len: # (1), (2), or (3) + # Merge `br` into the leaf `vtx` and unlink `br`. + let par = hike.legs[^3].wp + case par.vtx.vType: + of Branch: # (1) + # Replace `vtx` by `^2 & vtx` (use `lf` as-is) + par.vtx.bVid[hike.legs[^3].nibble] = lf.vid + db.top.sTab[par.vid] = par.vtx + db.top.sTab[lf.vid] = lf.vtx + return ok() + + of Extension: # (2) or (3) + # Merge `^3` into `lf` but keep the leaf vertex ID unchanged. This + # avoids some `lTab[]` registry update. + lf.vtx.lPfx = par.vtx.ePfx & lf.vtx.lPfx + + if 3 < hike.legs.len: # (2) + # Grandparent exists + let gpr = hike.legs[^4].wp + if gpr.vtx.vType != Branch: + return err((gpr.vid,DelBranchExpexted)) + db.doneWith par.vid # `par` is obsolete now + gpr.vtx.bVid[hike.legs[^4].nibble] = lf.vid + db.top.sTab[gpr.vid] = gpr.vtx + db.top.sTab[lf.vid] = lf.vtx + return ok() + + # No grandparent, so ^3 is root vertex # (3) + db.top.sTab[par.vid] = lf.vtx + # Continue below + + of Leaf: + return err((par.vid,DelLeafUnexpected)) + + else: # (4) + # Replace ^2 by `^2 & vtx` (use `lf` as-is) + db.top.sTab[br.vid] = lf.vtx + # Continue below + + # Common part for setting up `lf` as root vertex # Rest of (3) or (4) + let rc = lf.vtx.lPfx.pathToTag + if rc.isErr: + return err((br.vid,rc.error)) + # + # No need to update the cache unless `lf` is present there. The leaf path + # as well as the value associated with the leaf path has not been changed. + let lfTie = LeafTie(root: hike.root, path: rc.value) + if db.top.lTab.hasKey lfTie: + db.top.lTab[lfTie] = lf.vid + + # Clean up stale leaf vertex which has moved to root position + db.doneWith lf.vid + ok() + +# ------------------------- proc deleteImpl( + db: AristoDb; # Database, top layer hike: Hike; # Fully expanded path lty: LeafTie; # `Patricia Trie` path root-to-leaf - db: AristoDb; # Database, top layer ): Result[void,(VertexID,AristoError)] = ## Implementation of *delete* functionality. if hike.error != AristoError(0): @@ -68,53 +269,58 @@ proc deleteImpl( return err((hike.legs[^1].wp.vid,hike.error)) return err((VertexID(0),hike.error)) - # doAssert 0 < hike.legs.len and hike.tail.len == 0 # as assured by `hikeUp()` + # Remove leaf entry on the top + let lf = hike.legs[^1].wp + if lf.vtx.vType != Leaf: + return err((lf.vid,DelLeafExpexted)) + if lf.vid in db.top.pPrf: + return err((lf.vid, DelLeafLocked)) + db.doneWith lf.vid - var lf: VidVtxPair - block: - var inx = hike.legs.len - 1 + if 1 < hike.legs.len: - # Remove leaf entry on the top - lf = hike.legs[inx].wp - if lf.vtx.vType != Leaf: - return err((lf.vid,DelLeafExpexted)) - if lf.vid in db.top.pPrf: - return err((lf.vid, DelLeafLocked)) - db.doneWith(lf.vid) - inx.dec + # Get current `Branch` vertex `br` + let br = hike.legs[^2].wp + if br.vtx.vType != Branch: + return err((br.vid,DelBranchExpexted)) - while 0 <= inx: - # Unlink child vertex - let br = hike.legs[inx].wp - if br.vtx.vType != Branch: - return err((br.vid,DelBranchExpexted)) - if br.vid in db.top.pPrf: - return err((br.vid, DelBranchLocked)) - br.vtx.bVid[hike.legs[inx].nibble] = VertexID(0) - db.top.sTab[br.vid] = br.vtx + # Unlink child vertex from structural table + br.vtx.bVid[hike.legs[^2].nibble] = VertexID(0) + db.top.sTab[br.vid] = br.vtx - if br.vtx.branchStillNeeded: - # Clear all keys up to the toot key - db.clearKey(br.vid) - while 0 < inx: - inx.dec - db.clearKey(hike.legs[inx].wp.vid) - break + # Clear all keys up to the root key + for n in 0 .. hike.legs.len - 2: + let vid = hike.legs[n].wp.vid + if vid in db.top.pPrf: + return err((vid, DelBranchLocked)) + db.clearKey vid - # Remove this `Branch` entry - db.doneWith(br.vid) - inx.dec + let nibble = block: + let rc = br.vtx.branchStillNeeded() + if rc.isErr: + return err((br.vid,DelBranchWithoutRefs)) + rc.value - if inx < 0: - break + # Convert to `Extension` or `Leaf` vertex + if 0 <= nibble: + # Get child vertex (there must be one after a `Branch` node) + let nxt = block: + let vid = br.vtx.bVid[nibble] + VidVtxPair(vid: vid, vtx: db.getVtx vid) + if not nxt.vtx.isValid: + return err((nxt.vid, DelVidStaleVtx)) - # There might be an optional `Extension` to remove - let ext = hike.legs[inx].wp - if ext.vtx.vType == Extension: - if br.vid in db.top.pPrf: - return err((ext.vid, DelExtLocked)) - db.doneWith(ext.vid) - inx.dec + # Collapse `Branch` vertex `br` depending on `nxt` vertex type + let rc = block: + case nxt.vtx.vType: + of Branch: + db.collapseBranch(hike, nibble.byte) + of Extension: + db.collapseExt(hike, nibble.byte, nxt.vtx) + of Leaf: + db.collapseLeaf(hike, nibble.byte, nxt.vtx) + if rc.isErr: + return err(rc.error) # Delete leaf entry let rc = db.getVtxBackend lf.vid @@ -132,24 +338,24 @@ proc deleteImpl( # ------------------------------------------------------------------------------ proc delete*( - hike: Hike; # Fully expanded chain of vertices db: AristoDb; # Database, top layer + hike: Hike; # Fully expanded chain of vertices ): Result[void,(VertexID,AristoError)] = ## Delete argument `hike` chain of vertices from the database # Need path in order to remove it from `lTab[]` - let lky = block: + let lty = block: let rc = hike.to(NibblesSeq).pathToTag() if rc.isErr: return err((VertexID(0),DelPathTagError)) LeafTie(root: hike.root, path: rc.value) - hike.deleteImpl(lky, db) + db.deleteImpl(hike, lty) proc delete*( - lty: LeafTie; # `Patricia Trie` path root-to-leaf db: AristoDb; # Database, top layer + lty: LeafTie; # `Patricia Trie` path root-to-leaf ): Result[void,(VertexID,AristoError)] = ## Variant of `delete()` - lty.hikeUp(db).deleteImpl(lty, db) + db.deleteImpl(lty.hikeUp(db), lty) # ------------------------------------------------------------------------------ # End diff --git a/nimbus/db/aristo/aristo_desc/aristo_error.nim b/nimbus/db/aristo/aristo_desc/aristo_error.nim index cb6c4ad38..2443753dd 100644 --- a/nimbus/db/aristo/aristo_desc/aristo_error.nim +++ b/nimbus/db/aristo/aristo_desc/aristo_error.nim @@ -24,8 +24,10 @@ type RlpOtherException # Data record transcoders, `deblobify()` and `blobify()` - BlobifyVtxExPathOverflow - BlobifyVtxLeafPathOverflow + BlobifyBranchMissingRefs + BlobifyExtMissingRefs + BlobifyExtPathOverflow + BlobifyLeafPathOverflow DeblobNilArgument DeblobUnknown @@ -87,18 +89,49 @@ type HashifyLeafToRootAllFailed HashifyRootHashMismatch HashifyRootVidMismatch + HashifyVidCircularDependence + HashifyVtxMissing - HashifyCheckRevCountMismatch - HashifyCheckRevHashMismatch - HashifyCheckRevHashMissing - HashifyCheckRevVtxDup - HashifyCheckRevVtxMissing - HashifyCheckVidVtxMismatch - HashifyCheckVtxCountMismatch - HashifyCheckVtxHashMismatch - HashifyCheckVtxHashMissing - HashifyCheckVtxIncomplete - HashifyCheckVtxLockWithoutKey + # Cache checker `checkCache()` + CheckStkVtxIncomplete + CheckStkVtxKeyMissing + CheckStkVtxKeyMismatch + CheckStkRevKeyMissing + CheckStkRevKeyMismatch + CheckStkVtxCountMismatch + + CheckRlxVidVtxMismatch + CheckRlxVtxIncomplete + CheckRlxVtxKeyMissing + CheckRlxVtxKeyMismatch + CheckRlxRevKeyMissing + CheckRlxRevKeyMismatch + CheckRlxVidVtxBeMissing + CheckRlxVtxEmptyKeyMissing + CheckRlxVtxEmptyKeyExpected + + CheckAnyRevVtxMissing + CheckAnyRevVtxDup + CheckAnyRevCountMismatch + CheckAnyVtxLockWithoutKey + + # Backend structural check `checkBE()` + CheckBeVtxInvalid + CheckBeKeyInvalid + CheckBeVtxMissing + CheckBeKeyMissing + CheckBeKeyCantCompile + CheckBeKeyMismatch + CheckBeGarbledVGen + + CheckBeCacheKeyMissing + CheckBeCacheKeyNonEmpty + CheckBeCacheVidUnsynced + CheckBeCacheKeyDangling + CheckBeCacheVtxDangling + CheckBeCacheKeyCantCompile + CheckBeCacheKeyMismatch + CheckBeCacheGarbledVGen # Neighbour vertex, tree traversal `nearbyRight()` and `nearbyLeft()` NearbyBeyondRange @@ -113,17 +146,22 @@ type NearbyPathTailUnexpected NearbyPathTailInxOverflow NearbyUnexpectedVtx + NearbyVidInvalid # Deletion of vertices, `delete()` DelPathTagError DelLeafExpexted DelLeafLocked + DelLeafUnexpected DelBranchExpexted DelBranchLocked + DelBranchWithoutRefs DelExtLocked + DelVidStaleVtx # Save permanently, `save()` SaveBackendMissing + SaveLeafVidRepurposed # Get functions form `aristo_get.nim` GetLeafNotFound diff --git a/nimbus/db/aristo/aristo_desc/aristo_types_identifiers.nim b/nimbus/db/aristo/aristo_desc/aristo_types_identifiers.nim index 9c5f178af..750fbcaeb 100644 --- a/nimbus/db/aristo/aristo_desc/aristo_types_identifiers.nim +++ b/nimbus/db/aristo/aristo_desc/aristo_types_identifiers.nim @@ -75,6 +75,7 @@ static: # ------------------------------------------------------------------------------ func `<`*(a, b: VertexID): bool {.borrow.} +func `<=`*(a, b: VertexID): bool {.borrow.} func `==`*(a, b: VertexID): bool {.borrow.} func cmp*(a, b: VertexID): int {.borrow.} func `$`*(a: VertexID): string = $a.uint64 @@ -82,6 +83,11 @@ func `$`*(a: VertexID): string = $a.uint64 func `==`*(a: VertexID; b: static[uint]): bool = a == VertexID(b) +# Scalar model extension for `IntervalSetRef[VertexID,uint64]` +proc `+`*(a: VertexID; b: uint64): VertexID = (a.uint64+b).VertexID +proc `-`*(a: VertexID; b: uint64): VertexID = (a.uint64-b).VertexID +proc `-`*(a, b: VertexID): uint64 = (a.uint64 - b.uint64) + # ------------------------------------------------------------------------------ # Public helpers: `HashID` scalar data model # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_desc/aristo_types_structural.nim b/nimbus/db/aristo/aristo_desc/aristo_types_structural.nim index 57b2e8e27..bfbae49a4 100644 --- a/nimbus/db/aristo/aristo_desc/aristo_types_structural.nim +++ b/nimbus/db/aristo/aristo_desc/aristo_types_structural.nim @@ -143,21 +143,24 @@ proc dup*(pld: PayloadRef): PayloadRef = proc dup*(vtx: VertexRef): VertexRef = ## Duplicate vertex. # Not using `deepCopy()` here (some `gc` needs `--deepcopy:on`.) - case vtx.vType: - of Leaf: - VertexRef( - vType: Leaf, - lPfx: vtx.lPfx, - lData: vtx.ldata.dup) - of Extension: - VertexRef( - vType: Extension, - ePfx: vtx.ePfx, - eVid: vtx.eVid) - of Branch: - VertexRef( - vType: Branch, - bVid: vtx.bVid) + if vtx.isNil: + VertexRef(nil) + else: + case vtx.vType: + of Leaf: + VertexRef( + vType: Leaf, + lPfx: vtx.lPfx, + lData: vtx.ldata.dup) + of Extension: + VertexRef( + vType: Extension, + ePfx: vtx.ePfx, + eVid: vtx.eVid) + of Branch: + VertexRef( + vType: Branch, + bVid: vtx.bVid) proc to*(node: NodeRef; T: type VertexRef): T = ## Extract a copy of the `VertexRef` part from a `NodeRef`. diff --git a/nimbus/db/aristo/aristo_hashify.nim b/nimbus/db/aristo/aristo_hashify.nim index 915732424..9682ee069 100644 --- a/nimbus/db/aristo/aristo_hashify.nim +++ b/nimbus/db/aristo/aristo_hashify.nim @@ -42,12 +42,12 @@ {.push raises: [].} import - std/[sequtils, sets, strutils, tables], + std/[algorithm, sequtils, sets, strutils, tables], chronicles, eth/common, - stew/results, - "."/[aristo_constants, aristo_desc, aristo_get, aristo_hike, - aristo_transcode, aristo_vid] + stew/[interval_set, results], + "."/[aristo_desc, aristo_get, aristo_hike, aristo_vid], + ./aristo_hashify/hashify_helper type BackVidValRef = ref object @@ -78,31 +78,6 @@ func isValid(brv: BackVidValRef): bool = # Private functions # ------------------------------------------------------------------------------ -proc toNode(vtx: VertexRef; db: AristoDb): Result[NodeRef,void] = - case vtx.vType: - of Leaf: - return ok NodeRef(vType: Leaf, lPfx: vtx.lPfx, lData: vtx.lData) - of Branch: - let node = NodeRef(vType: Branch, bVid: vtx.bVid) - for n in 0 .. 15: - if vtx.bVid[n].isValid: - let key = db.getKey vtx.bVid[n] - if key.isValid: - node.key[n] = key - continue - return err() - else: - node.key[n] = VOID_HASH_KEY - return ok node - of Extension: - if vtx.eVid.isValid: - let key = db.getKey vtx.eVid - if key.isValid: - let node = NodeRef(vType: Extension, ePfx: vtx.ePfx, eVid: vtx.eVid) - node.key[0] = key - return ok node - - proc updateHashKey( db: AristoDb; # Database, top layer root: VertexID; # Root ID @@ -174,13 +149,66 @@ proc leafToRootHasher( # Check against existing key, or store new key let - key = rc.value.encode.digestTo(HashKey) + key = rc.value.toHashKey rx = db.updateHashKey(hike.root, wp.vid, key, bg) if rx.isErr: return err((wp.vid,rx.error)) ok -1 # all could be hashed +# ------------------ + +proc deletedLeafHasher( + db: AristoDb; # Database, top layer + hike: Hike; # Hike for labelling leaf..root + ): Result[void,(VertexID,AristoError)] = + var + todo = hike.legs.reversed.mapIt(it.wp) + solved: HashSet[VertexID] + # Edge case for empty `hike` + if todo.len == 0: + let vtx = db.getVtx hike.root + if not vtx.isValid: + return err((hike.root,HashifyVtxMissing)) + todo = @[VidVtxPair(vid: hike.root, vtx: vtx)] + while 0 < todo.len: + var + delayed: seq[VidVtxPair] + didHere: HashSet[VertexID] # avoid duplicates + for wp in todo: + let rc = wp.vtx.toNode(db, stopEarly=false) + if rc.isOk: + let + expected = rc.value.toHashKey + key = db.getKey wp.vid + if key.isValid: + if key != expected: + return err((wp.vid,HashifyExistingHashMismatch)) + else: + db.vidAttach(HashLabel(root: hike.root, key: expected), wp.vid) + solved.incl wp.vid + else: + # Resolve follow up vertices first + for vid in rc.error: + let vtx = db.getVtx vid + if not vtx.isValid: + return err((vid,HashifyVtxMissing)) + if vid in solved: + discard wp.vtx.toNode(db, stopEarly=false) + return err((vid,HashifyVidCircularDependence)) + if vid notin didHere: + didHere.incl vid + delayed.add VidVtxPair(vid: vid, vtx: vtx) + + # Followed by this vertex which relies on the ones registered above. + if wp.vid notin didHere: + didHere.incl wp.vid + delayed.add wp + + todo = delayed + + ok() + # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ @@ -212,40 +240,48 @@ proc hashify*( for (lky,vid) in db.top.lTab.pairs: let hike = lky.hikeUp(db) - if hike.error != AristoError(0): - return err((vid,hike.error)) - roots.incl hike.root - - # Hash as much of the `hike` as possible - let n = block: - let rc = db.leafToRootHasher hike + # There might be deleted entries on the leaf table. If this is tha case, + # the Merkle hashes for the vertices in the `hike` can all be compiled. + if not vid.isValid: + let rc = db.deletedLeafHasher hike if rc.isErr: return err(rc.error) - rc.value - if 0 < n: - # Backtrack and register remaining nodes. Note that in case *n == 0*, the - # root vertex has not been fully resolved yet. - # - # hike.legs: (leg[0], leg[1], .., leg[n-1], leg[n], ..) - # | | | | - # | <---- | <---- | <---- | - # | | | - # | backLink[] | downMost | - # - downMost[hike.legs[n].wp.vid] = BackVidValRef( - root: hike.root, - onBe: hike.legs[n].backend, - toVid: hike.legs[n-1].wp.vid) - for u in (n-1).countDown(1): - backLink[hike.legs[u].wp.vid] = BackVidValRef( + elif hike.error != AristoError(0): + return err((vid,hike.error)) + + else: + # Hash as much of the `hike` as possible + let n = block: + let rc = db.leafToRootHasher hike + if rc.isErr: + return err(rc.error) + rc.value + + roots.incl hike.root + + if 0 < n: + # Backtrack and register remaining nodes. Note that in case *n == 0*, + # the root vertex has not been fully resolved yet. + # + # hike.legs: (leg[0], leg[1], .., leg[n-1], leg[n], ..) + # | | | | + # | <---- | <---- | <---- | + # | | | + # | backLink[] | downMost | + # + downMost[hike.legs[n].wp.vid] = BackVidValRef( root: hike.root, - onBe: hike.legs[u].backend, - toVid: hike.legs[u-1].wp.vid) - - elif n < 0: - completed.incl hike.root + onBe: hike.legs[n].backend, + toVid: hike.legs[n-1].wp.vid) + for u in (n-1).countDown(1): + backLink[hike.legs[u].wp.vid] = BackVidValRef( + root: hike.root, + onBe: hike.legs[u].backend, + toVid: hike.legs[u-1].wp.vid) + elif n < 0: + completed.incl hike.root # At least one full path leaf..root should have succeeded with labelling # for each root. @@ -263,7 +299,7 @@ proc hashify*( # references have Merkle hashes. # # Also `db.getVtx(vid)` => not nil as it was fetched earlier, already - let rc = db.getVtx(vid).toNode(db) + let rc = db.getVtx(vid).toNode db if rc.isErr: # Cannot complete with this vertex, so do it later redo[vid] = val @@ -271,7 +307,7 @@ proc hashify*( else: # Update Merkle hash let - key = rc.value.encode.digestTo(HashKey) + key = rc.value.toHashKey rx = db.updateHashKey(val.root, vid, key, val.onBe) if rx.isErr: return err((vid,rx.error)) @@ -295,95 +331,6 @@ proc hashify*( ok completed -# ------------------------------------------------------------------------------ -# Public debugging functions -# ------------------------------------------------------------------------------ - -proc hashifyCheck*( - db: AristoDb; # Database, top layer - relax = false; # Check existing hashes only - ): Result[void,(VertexID,AristoError)] = - ## Verify that the Merkle hash keys are either completely missing or - ## match all known vertices on the argument database layer `db`. - if not relax: - for (vid,vtx) in db.top.sTab.pairs: - let rc = vtx.toNode(db) - if rc.isErr: - return err((vid,HashifyCheckVtxIncomplete)) - - let lbl = db.top.kMap.getOrVoid vid - if not lbl.isValid: - return err((vid,HashifyCheckVtxHashMissing)) - if lbl.key != rc.value.encode.digestTo(HashKey): - return err((vid,HashifyCheckVtxHashMismatch)) - - let revVid = db.top.pAmk.getOrVoid lbl - if not revVid.isValid: - return err((vid,HashifyCheckRevHashMissing)) - if revVid != vid: - return err((vid,HashifyCheckRevHashMismatch)) - - elif 0 < db.top.pPrf.len: - for vid in db.top.pPrf: - let vtx = db.top.sTab.getOrVoid vid - if not vtx.isValid: - return err((vid,HashifyCheckVidVtxMismatch)) - - let rc = vtx.toNode(db) - if rc.isErr: - return err((vid,HashifyCheckVtxIncomplete)) - - let lbl = db.top.kMap.getOrVoid vid - if not lbl.isValid: - return err((vid,HashifyCheckVtxHashMissing)) - if lbl.key != rc.value.encode.digestTo(HashKey): - return err((vid,HashifyCheckVtxHashMismatch)) - - let revVid = db.top.pAmk.getOrVoid lbl - if not revVid.isValid: - return err((vid,HashifyCheckRevHashMissing)) - if revVid != vid: - return err((vid,HashifyCheckRevHashMismatch)) - - else: - for (vid,lbl) in db.top.kMap.pairs: - if lbl.isValid: # Otherwise to be deleted - let vtx = db.getVtx vid - if vtx.isValid: - let rc = vtx.toNode(db) - if rc.isOk: - if lbl.key != rc.value.encode.digestTo(HashKey): - return err((vid,HashifyCheckVtxHashMismatch)) - - let revVid = db.top.pAmk.getOrVoid lbl - if not revVid.isValid: - return err((vid,HashifyCheckRevHashMissing)) - if revVid != vid: - return err((vid,HashifyCheckRevHashMismatch)) - - # Some `kMap[]` entries may ne void indicating backend deletion - let kMapCount = db.top.kMap.values.toSeq.filterIt(it.isValid).len - - if db.top.pAmk.len != kMapCount: - var knownKeys: HashSet[VertexID] - for (key,vid) in db.top.pAmk.pairs: - if not db.top.kMap.hasKey(vid): - return err((vid,HashifyCheckRevVtxMissing)) - if vid in knownKeys: - return err((vid,HashifyCheckRevVtxDup)) - knownKeys.incl vid - return err((VertexID(0),HashifyCheckRevCountMismatch)) # should not apply(!) - - if 0 < db.top.pAmk.len and not relax and db.top.pAmk.len < db.top.sTab.len: - # Cannot have less changes than cached entries - return err((VertexID(0),HashifyCheckVtxCountMismatch)) - - for vid in db.top.pPrf: - if not db.top.kMap.hasKey(vid): - return err((vid,HashifyCheckVtxLockWithoutKey)) - - ok() - # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_hashify/hashify_helper.nim b/nimbus/db/aristo/aristo_hashify/hashify_helper.nim new file mode 100644 index 000000000..55ba406e3 --- /dev/null +++ b/nimbus/db/aristo/aristo_hashify/hashify_helper.nim @@ -0,0 +1,69 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + + +{.push raises: [].} + +import + #std/[tables], + eth/common, + stew/results, + ".."/[aristo_constants, aristo_desc, aristo_get, aristo_transcode] + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc toNode*( + vtx: VertexRef; # Vertex to convert + db: AristoDb; # Database, top layer + stopEarly = true; # Full list of missing links if `false` + ): Result[NodeRef,seq[VertexID]] = + ## Convert argument vertex to node + case vtx.vType: + of Leaf: + return ok NodeRef(vType: Leaf, lPfx: vtx.lPfx, lData: vtx.lData) + of Branch: + let node = NodeRef(vType: Branch, bVid: vtx.bVid) + var missing: seq[VertexID] + for n in 0 .. 15: + let vid = vtx.bVid[n] + if vid.isValid: + let key = db.getKey vid + if key.isValid: + node.key[n] = key + else: + missing.add vid + if stopEarly: + break + else: + node.key[n] = VOID_HASH_KEY + if 0 < missing.len: + return err(missing) + return ok node + of Extension: + let + vid = vtx.eVid + key = db.getKey vid + if key.isValid: + let node = NodeRef(vType: Extension, ePfx: vtx.ePfx, eVid: vid) + node.key[0] = key + return ok node + return err(@[vid]) + +# This function cannot go into `aristo_desc` as it depends on `aristo_transcode` +# which depends on `aristo_desc`. +proc toHashKey*(node: NodeRef): HashKey = + ## Convert argument `node` to Merkle hash key + node.encode.digestTo(HashKey) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_init.nim b/nimbus/db/aristo/aristo_init.nim index 6fda2dc4e..da44d5e35 100644 --- a/nimbus/db/aristo/aristo_init.nim +++ b/nimbus/db/aristo/aristo_init.nim @@ -21,7 +21,7 @@ import ./aristo_desc/aristo_types_backend export - AristoBackendType, AristoStorageType, AristoTypedBackendRef + AristoBackendType, AristoStorageType, TypedBackendRef # ------------------------------------------------------------------------------ # Public database constuctors, destructor @@ -91,7 +91,10 @@ proc finish*(db: var AristoDb; flush = false) = # ----------------- -proc to*[W: MemBackendRef|RdbBackendRef](db: AristoDb; T: type W): T = +proc to*[W: TypedBackendRef|MemBackendRef|RdbBackendRef]( + db: AristoDb; + T: type W; + ): T = ## Handy helper for lew-level access to some backend functionality db.backend.T diff --git a/nimbus/db/aristo/aristo_init/aristo_init_common.nim b/nimbus/db/aristo/aristo_init/aristo_init_common.nim index 0fd220ed9..f192f0b89 100644 --- a/nimbus/db/aristo/aristo_init/aristo_init_common.nim +++ b/nimbus/db/aristo/aristo_init/aristo_init_common.nim @@ -10,6 +10,7 @@ {.push raises: [].} import + ../aristo_desc, ../aristo_desc/aristo_types_backend const @@ -22,13 +23,19 @@ type BackendMemory BackendRocksDB - AristoTypedBackendRef* = ref object of AristoBackendRef + TypedBackendRef* = ref object of AristoBackendRef kind*: AristoBackendType ## Backend type identifier when verifyIxId: txGen: uint ## Transaction ID generator (for debugging) txId: uint ## Active transaction ID (for debugging) + TypedPutHdlErrRef* = ref object of RootRef + pfx*: AristoStorageType ## Error sub-table + vid*: VertexID ## Vertex ID where the error occured + code*: AristoError ## Error code (if any) + TypedPutHdlRef* = ref object of PutHdlRef + error*: TypedPutHdlErrRef ## Track error while collecting transaction when verifyIxId: txId: uint ## Transaction ID (for debugging) @@ -42,7 +49,7 @@ type # Public helpers # ------------------------------------------------------------------------------ -proc beginSession*(hdl: TypedPutHdlRef; db: AristoTypedBackendRef) = +proc beginSession*(hdl: TypedPutHdlRef; db: TypedBackendRef) = when verifyIxId: doAssert db.txId == 0 if db.txGen == 0: @@ -51,11 +58,11 @@ proc beginSession*(hdl: TypedPutHdlRef; db: AristoTypedBackendRef) = hdl.txId = db.txGen db.txGen.inc -proc verifySession*(hdl: TypedPutHdlRef; db: AristoTypedBackendRef) = +proc verifySession*(hdl: TypedPutHdlRef; db: TypedBackendRef) = when verifyIxId: doAssert db.txId == hdl.txId -proc finishSession*(hdl: TypedPutHdlRef; db: AristoTypedBackendRef) = +proc finishSession*(hdl: TypedPutHdlRef; db: TypedBackendRef) = when verifyIxId: doAssert db.txId == hdl.txId db.txId = 0 diff --git a/nimbus/db/aristo/aristo_init/aristo_memory.nim b/nimbus/db/aristo/aristo_init/aristo_memory.nim index 297fb11da..2fbb00dc6 100644 --- a/nimbus/db/aristo/aristo_init/aristo_memory.nim +++ b/nimbus/db/aristo/aristo_init/aristo_memory.nim @@ -28,6 +28,7 @@ import std/[algorithm, sequtils, tables], + chronicles, eth/common, stew/results, ../aristo_constants, @@ -37,7 +38,7 @@ import ./aristo_init_common type - MemBackendRef* = ref object of AristoTypedBackendRef + MemBackendRef* = ref object of TypedBackendRef ## Inheriting table so access can be extended for debugging purposes sTab: Table[VertexID,VertexRef] ## Structural vertex table making up a trie kMap: Table[VertexID,HashKey] ## Merkle hash key mapping @@ -53,6 +54,10 @@ type # Private helpers # ------------------------------------------------------------------------------ +template logTxt(info: static[string]): static[string] = + "MemoryDB " & info + + proc newSession(db: MemBackendRef): MemPutHdlRef = new result result.TypedPutHdlRef.beginSession db @@ -102,28 +107,43 @@ proc putVtxFn(db: MemBackendRef): PutVtxFn = result = proc(hdl: PutHdlRef; vrps: openArray[(VertexID,VertexRef)]) = let hdl = hdl.getSession db - for (vid,vtx) in vrps: - hdl.sTab[vid] = vtx.dup + if hdl.error.isNil: + for (vid,vtx) in vrps: + if not vtx.isNil: + let rc = vtx.blobify # verify data record + if rc.isErr: + hdl.error = TypedPutHdlErrRef( + pfx: VtxPfx, + vid: vid, + code: rc.error) + return + hdl.sTab[vid] = vtx.dup proc putKeyFn(db: MemBackendRef): PutKeyFn = result = proc(hdl: PutHdlRef; vkps: openArray[(VertexID,HashKey)]) = let hdl = hdl.getSession db - for (vid,key) in vkps: - hdl.kMap[vid] = key + if hdl.error.isNil: + for (vid,key) in vkps: + hdl.kMap[vid] = key proc putIdgFn(db: MemBackendRef): PutIdgFn = result = proc(hdl: PutHdlRef; vs: openArray[VertexID]) = let hdl = hdl.getSession db - hdl.vGen = vs.toSeq - hdl.vGenOk = true + if hdl.error.isNil: + hdl.vGen = vs.toSeq + hdl.vGenOk = true proc putEndFn(db: MemBackendRef): PutEndFn = result = proc(hdl: PutHdlRef): AristoError = let hdl = hdl.endSession db + if not hdl.error.isNil: + debug logTxt "putEndFn: failed", + pfx=hdl.error.pfx, vid=hdl.error.vid, error=hdl.error.code + return hdl.error.code for (vid,vtx) in hdl.sTab.pairs: if vtx.isValid: diff --git a/nimbus/db/aristo/aristo_init/aristo_rocksdb.nim b/nimbus/db/aristo/aristo_init/aristo_rocksdb.nim index 8a12a1b7f..92250567a 100644 --- a/nimbus/db/aristo/aristo_init/aristo_rocksdb.nim +++ b/nimbus/db/aristo/aristo_init/aristo_rocksdb.nim @@ -42,17 +42,11 @@ logScope: topics = "aristo-backend" type - RdbBackendRef* = ref object of AristoTypedBackendRef + RdbBackendRef* = ref object of TypedBackendRef rdb: RdbInst ## Allows low level access to database - RdbPutHdlErr = tuple - pfx: AristoStorageType ## Error sub-table - vid: VertexID ## Vertex ID where the error occured - code: AristoError ## Error code (if any) - RdbPutHdlRef = ref object of TypedPutHdlRef cache: RdbTabs ## Tranaction cache - error: RdbPutHdlErr ## Track error while collecting transaction const extraTraceMessages = false or true @@ -151,12 +145,15 @@ proc putVtxFn(db: RdbBackendRef): PutVtxFn = result = proc(hdl: PutHdlRef; vrps: openArray[(VertexID,VertexRef)]) = let hdl = hdl.getSession db - if hdl.error.code == AristoError(0): + if hdl.error.isNil: for (vid,vtx) in vrps: if vtx.isValid: let rc = vtx.blobify if rc.isErr: - hdl.error = (VtxPfx, vid, rc.error) + hdl.error = TypedPutHdlErrRef( + pfx: VtxPfx, + vid: vid, + code: rc.error) return hdl.cache[VtxPfx][vid] = rc.value else: @@ -166,7 +163,7 @@ proc putKeyFn(db: RdbBackendRef): PutKeyFn = result = proc(hdl: PutHdlRef; vkps: openArray[(VertexID,HashKey)]) = let hdl = hdl.getSession db - if hdl.error.code == AristoError(0): + if hdl.error.isNil: for (vid,key) in vkps: if key.isValid: hdl.cache[KeyPfx][vid] = key.to(Blob) @@ -178,7 +175,7 @@ proc putIdgFn(db: RdbBackendRef): PutIdgFn = result = proc(hdl: PutHdlRef; vs: openArray[VertexID]) = let hdl = hdl.getSession db - if hdl.error.code == AristoError(0): + if hdl.error.isNil: if 0 < vs.len: hdl.cache[IdgPfx][VertexID(0)] = vs.blobify else: @@ -189,7 +186,7 @@ proc putEndFn(db: RdbBackendRef): PutEndFn = result = proc(hdl: PutHdlRef): AristoError = let hdl = hdl.endSession db - if hdl.error.code != AristoError(0): + if not hdl.error.isNil: debug logTxt "putEndFn: failed", pfx=hdl.error.pfx, vid=hdl.error.vid, error=hdl.error.code return hdl.error.code diff --git a/nimbus/db/aristo/aristo_layer.nim b/nimbus/db/aristo/aristo_layer.nim index 23d182ff3..0bd9e4f9f 100644 --- a/nimbus/db/aristo/aristo_layer.nim +++ b/nimbus/db/aristo/aristo_layer.nim @@ -15,7 +15,7 @@ import std/[sequtils, tables], stew/results, - "."/[aristo_desc, aristo_get] + "."/[aristo_desc, aristo_get, aristo_vid] type DeltaHistoryRef* = ref object @@ -75,7 +75,7 @@ proc pop*(db: var AristoDb; merge = true): AristoError = proc save*( db: var AristoDb; # Database to be updated clear = true; # Clear current top level cache - ): Result[DeltaHistoryRef,AristoError] = + ): Result[DeltaHistoryRef,(VertexID,AristoError)] = ## Save top layer into persistent database. There is no check whether the ## current layer is fully consistent as a Merkle Patricia Tree. It is ## advised to run `hashify()` on the top layer before calling `save()`. @@ -88,9 +88,9 @@ proc save*( ## let be = db.backend if be.isNil: - return err(SaveBackendMissing) + return err((VertexID(0),SaveBackendMissing)) - let hst = DeltaHistoryRef() # Change history + let hst = DeltaHistoryRef() # Change history # Record changed `Leaf` nodes into the history table for (lky,vid) in db.top.lTab.pairs: @@ -99,14 +99,17 @@ proc save*( let rc = db.getVtxBackend vid if rc.isErr: if rc.error != GetVtxNotFound: - return err(rc.error) # Stop - hst.leafs[lky] = PayloadRef(nil) # So this is a new leaf vertex + return err((vid,rc.error)) # Stop + hst.leafs[lky] = PayloadRef(nil) # So this is a new leaf vertex elif rc.value.vType == Leaf: - hst.leafs[lky] = rc.value.lData # Record previous payload + hst.leafs[lky] = rc.value.lData # Record previous payload else: - hst.leafs[lky] = PayloadRef(nil) # Was re-puropsed as leaf vertex + return err((vid,SaveLeafVidRepurposed)) # Was re-puropsed else: - hst.leafs[lky] = PayloadRef(nil) # New leaf vertex + hst.leafs[lky] = PayloadRef(nil) # New leaf vertex + + # Compact recycled nodes + db.vidReorg() # Save structural and other table entries let txFrame = be.putBegFn() @@ -115,7 +118,7 @@ proc save*( be.putIdgFn(txFrame, db.top.vGen) let w = be.putEndFn txFrame if w != AristoError(0): - return err(w) + return err((VertexID(0),w)) # Delete stack and clear top db.stack.setLen(0) diff --git a/nimbus/db/aristo/aristo_merge.nim b/nimbus/db/aristo/aristo_merge.nim index f82d55325..5b90bb99d 100644 --- a/nimbus/db/aristo/aristo_merge.nim +++ b/nimbus/db/aristo/aristo_merge.nim @@ -150,7 +150,7 @@ proc insertBranch( return Hike(error: MergeBrLinkLeafGarbled) let - local = db.vidFetch + local = db.vidFetch(pristine = true) lty = LeafTie(root: hike.root, path: rc.value) db.top.lTab[lty] = local # update leaf path lookup cache db.top.sTab[local] = linkVtx @@ -168,7 +168,7 @@ proc insertBranch( forkVtx.bVid[linkInx] = local block: - let local = db.vidFetch + let local = db.vidFetch(pristine = true) forkVtx.bVid[leafInx] = local leafLeg.wp.vid = local leafLeg.wp.vtx = VertexRef( @@ -242,7 +242,7 @@ proc concatBranchAndLeaf( # Append leaf vertex let - vid = db.vidFetch + vid = db.vidFetch(pristine = true) vtx = VertexRef( vType: Leaf, lPfx: hike.tail.slice(1), @@ -364,7 +364,7 @@ proc topIsExtAddLeaf( return Hike(error: MergeBranchProofModeLock) let - vid = db.vidFetch + vid = db.vidFetch(pristine = true) vtx = VertexRef( vType: Leaf, lPfx: hike.tail.slice(1), @@ -396,7 +396,7 @@ proc topIsEmptyAddLeaf( return Hike(error: MergeBranchProofModeLock) let - leafVid = db.vidFetch + leafVid = db.vidFetch(pristine = true) leafVtx = VertexRef( vType: Leaf, lPfx: hike.tail.slice(1), diff --git a/nimbus/db/aristo/aristo_nearby.nim b/nimbus/db/aristo/aristo_nearby.nim index f279576b1..15cf08442 100644 --- a/nimbus/db/aristo/aristo_nearby.nim +++ b/nimbus/db/aristo/aristo_nearby.nim @@ -80,22 +80,23 @@ proc complete( db: AristoDb; # Database layer hikeLenMax: static[int]; # Beware of loops (if any) doLeast: static[bool]; # Direction: *least* or *most* - ): Hike = + ): Result[Hike,(VertexID,AristoError)] = ## Extend `hike` using least or last vertex without recursion. + if not vid.isValid: + return err((VertexID(0),NearbyVidInvalid)) var vid = vid vtx = db.getVtx vid uHike = Hike(root: hike.root, legs: hike.legs) if not vtx.isValid: - return Hike(error: GetVtxNotFound) + return err((vid,GetVtxNotFound)) while uHike.legs.len < hikeLenMax: var leg = Leg(wp: VidVtxPair(vid: vid, vtx: vtx), nibble: -1) - case vtx.vType: of Leaf: uHike.legs.add leg - return uHike # done + return ok(uHike) # done of Extension: vid = vtx.eVid @@ -104,7 +105,7 @@ proc complete( if vtx.isValid: uHike.legs.add leg continue - return Hike(error: NearbyExtensionError) # Oops, no way + return err((vid,NearbyExtensionError)) # Oops, no way of Branch: when doLeast: @@ -117,16 +118,16 @@ proc complete( if vtx.isValid: uHike.legs.add leg continue - return Hike(error: NearbyBranchError) # Oops, no way + return err((leg.wp.vid,NearbyBranchError)) # Oops, no way - Hike(error: NearbyNestingTooDeep) + err((VertexID(0),NearbyNestingTooDeep)) proc zeroAdjust( hike: Hike; # Partially expanded chain of vertices db: AristoDb; # Database layer doLeast: static[bool]; # Direction: *least* or *most* - ): Hike = + ): Result[Hike,(VertexID,AristoError)] = ## Adjust empty argument path to the first vertex entry to the right. Ths ## applies is the argument `hike` is before the first entry in the database. ## The result is a hike which is aligned with the first entry. @@ -149,9 +150,7 @@ proc zeroAdjust( pfx.pathPfxPad(255).hikeUp(root, db) if 0 < hike.legs.len: - result = hike - result.error = AristoError(0) - return + return ok(hike) let root = db.getVtx hike.root if root.isValid: @@ -166,7 +165,7 @@ proc zeroAdjust( let n = root.branchBorderNibble hike.tail[0].int8 if n < 0: # Before or after the database range - return Hike(error: NearbyBeyondRange) + return err((hike.root,NearbyBeyondRange)) pfx = @[n.byte].initNibbleRange.slice(1) of Extension: @@ -179,35 +178,34 @@ proc zeroAdjust( break fail let ePfxLen = ePfx.len if hike.tail.len <= ePfxLen: - return Hike(error: NearbyPathTailInxOverflow) + return err((root.eVid,NearbyPathTailInxOverflow)) let tailPfx = hike.tail.slice(0,ePfxLen) when doLeast: if ePfx < tailPfx: - return Hike(error: NearbyBeyondRange) + return err((root.eVid,NearbyBeyondRange)) else: if tailPfx < ePfx: - return Hike(error: NearbyBeyondRange) + return err((root.eVid,NearbyBeyondRange)) pfx = ePfx of Leaf: pfx = root.lPfx if not hike.accept(pfx): # Before or after the database range - return Hike(error: NearbyBeyondRange) + return err((hike.root,NearbyBeyondRange)) var newHike = pfx.toHike(hike.root, db) if 0 < newHike.legs.len: - newHike.error = AristoError(0) - return newHike + return ok(newHike) - Hike(error: NearbyEmptyHike) + err((VertexID(0),NearbyEmptyHike)) proc finalise( hike: Hike; # Partially expanded chain of vertices db: AristoDb; # Database layer moveRight: static[bool]; # Direction of next vertex - ): Hike = + ): Result[Hike,(VertexID,AristoError)] = ## Handle some pathological cases after main processing failed proc beyond(p: Hike; pfx: NibblesSeq): bool = when moveRight: @@ -223,7 +221,7 @@ proc finalise( # Just for completeness (this case should have been handled, already) if hike.legs.len == 0: - return Hike(error: NearbyEmptyHike) + return err((VertexID(0),NearbyEmptyHike)) # Check whether the path is beyond the database range if 0 < hike.tail.len: # nothing to compare against, otherwise @@ -232,9 +230,11 @@ proc finalise( # Note that only a `Branch` vertices has a non-zero nibble if 0 <= top.nibble and top.nibble == top.wp.vtx.branchBorderNibble: # Check the following up vertex - let vtx = db.getVtx top.wp.vtx.bVid[top.nibble] + let + vid = top.wp.vtx.bVid[top.nibble] + vtx = db.getVtx vid if not vtx.isValid: - return Hike(error: NearbyDanglingLink) + return err((vid,NearbyDanglingLink)) var pfx: NibblesSeq case vtx.vType: @@ -245,16 +245,16 @@ proc finalise( of Branch: pfx = @[vtx.branchBorderNibble.byte].initNibbleRange.slice(1) if hike.beyond pfx: - return Hike(error: NearbyBeyondRange) + return err((vid,NearbyBeyondRange)) # Pathological cases # * finalise right: nfffff.. for n < f or # * finalise left: n00000.. for 0 < n if hike.legs[0].wp.vtx.vType == Branch or (1 < hike.legs.len and hike.legs[1].wp.vtx.vType == Branch): - return Hike(error: NearbyFailed) # no more vertices + return err((VertexID(0),NearbyFailed)) # no more vertices - Hike(error: NearbyUnexpectedVtx) # error + err((hike.legs[^1].wp.vid,NearbyUnexpectedVtx)) # error proc nearbyNext( @@ -262,7 +262,7 @@ proc nearbyNext( db: AristoDb; # Database layer hikeLenMax: static[int]; # Beware of loops (if any) moveRight: static[bool]; # Direction of next vertex - ): Hike = + ): Result[Hike,(VertexID,AristoError)] = ## Unified implementation of `nearbyRight()` and `nearbyLeft()`. proc accept(nibble: int8): bool = ## Accept `nibble` unless on boundaty dependent on `moveRight` @@ -284,9 +284,11 @@ proc nearbyNext( w.branchNibbleMax(n - 1) # Some easy cases - var hike = hike.zeroAdjust(db, doLeast=moveRight) - if hike.error != AristoError(0): - return hike + let hike = block: + var rc = hike.zeroAdjust(db, doLeast=moveRight) + if rc.isErr: + return err(rc.error) + rc.value if hike.legs[^1].wp.vtx.vType == Extension: let vid = hike.legs[^1].wp.vtx.eVid @@ -299,10 +301,10 @@ proc nearbyNext( let top = uHike.legs[^1] case top.wp.vtx.vType: of Leaf: - return uHike + return ok(uHike) of Branch: if top.nibble < 0 or uHike.tail.len == 0: - return Hike(error: NearbyUnexpectedVtx) + return err((top.wp.vid,NearbyUnexpectedVtx)) of Extension: uHike.tail = top.wp.vtx.ePfx & uHike.tail uHike.legs.setLen(uHike.legs.len - 1) @@ -318,11 +320,11 @@ proc nearbyNext( if start: let vid = top.wp.vtx.bVid[top.nibble] if not vid.isValid: - return Hike(error: NearbyDanglingLink) # error + return err((top.wp.vid,NearbyDanglingLink)) # error let vtx = db.getVtx vid if not vtx.isValid: - return Hike(error: GetVtxNotFound) # error + return err((vid,GetVtxNotFound)) # error case vtx.vType of Leaf: @@ -360,36 +362,40 @@ proc nearbyNext( # End while # Handle some pathological cases - return hike.finalise(db, moveRight) + hike.finalise(db, moveRight) -proc nearbyNext( +proc nearbyNextLeafTie( lty: LeafTie; # Some `Patricia Trie` path db: AristoDb; # Database layer hikeLenMax: static[int]; # Beware of loops (if any) moveRight:static[bool]; # Direction of next vertex - ): Result[HashID,AristoError] = - ## Variant of `nearbyNext()`, convenience wrapper - let hike = lty.hikeUp(db).nearbyNext(db, hikeLenMax, moveRight) - if hike.error != AristoError(0): - return err(hike.error) + ): Result[HashID,(VertexID,AristoError)] = + ## Variant of `nearbyNext()`, convenience wrapper + let hike = block: + let rc = lty.hikeUp(db).nearbyNext(db, hikeLenMax, moveRight) + if rc.isErr: + return err(rc.error) + rc.value - if 0 < hike.legs.len and hike.legs[^1].wp.vtx.vType == Leaf: + if 0 < hike.legs.len: + if hike.legs[^1].wp.vtx.vType != Leaf: + return err((hike.legs[^1].wp.vid,NearbyLeafExpected)) let rc = hike.legsTo(NibblesSeq).pathToKey if rc.isOk: return ok rc.value.to(HashID) - return err(rc.error) + return err((VertexID(0),rc.error)) - err(NearbyLeafExpected) + err((VertexID(0),NearbyLeafExpected)) # ------------------------------------------------------------------------------ # Public functions, moving and right boundary proof # ------------------------------------------------------------------------------ -proc nearbyRight*( +proc right*( hike: Hike; # Partially expanded chain of vertices db: AristoDb; # Database layer - ): Hike = + ): Result[Hike,(VertexID,AristoError)] = ## Extends the maximally extended argument vertices `hike` to the right (i.e. ## with non-decreasing path value). This function does not backtrack if ## there are dangling links in between. It will return an error in that case. @@ -401,33 +407,33 @@ proc nearbyRight*( ## verify that there is no leaf vertex *right* of a boundary path value. hike.nearbyNext(db, 64, moveRight=true) -proc nearbyRight*( +proc right*( lty: LeafTie; # Some `Patricia Trie` path db: AristoDb; # Database layer - ): Result[LeafTie,AristoError] = + ): Result[LeafTie,(VertexID,AristoError)] = ## Variant of `nearbyRight()` working with a `HashID` argument instead ## of a `Hike`. - let rc = lty.nearbyNext(db, 64, moveRight=true) + let rc = lty.nearbyNextLeafTie(db, 64, moveRight=true) if rc.isErr: return err(rc.error) ok LeafTie(root: lty.root, path: rc.value) -proc nearbyLeft*( +proc left*( hike: Hike; # Partially expanded chain of vertices db: AristoDb; # Database layer - ): Hike = + ): Result[Hike,(VertexID,AristoError)] = ## Similar to `nearbyRight()`. ## ## This code is intended to be used for verifying a right-bound proof to ## verify that there is no leaf vertex *left* to a boundary path value. hike.nearbyNext(db, 64, moveRight=false) -proc nearbyLeft*( +proc left*( lty: LeafTie; # Some `Patricia Trie` path db: AristoDb; # Database layer - ): Result[LeafTie,AristoError] = + ): Result[LeafTie,(VertexID,AristoError)] = ## Similar to `nearbyRight()` for `HashID` argument instead of a `Hike`. - let rc = lty.nearbyNext(db, 64, moveRight=false) + let rc = lty.nearbyNextLeafTie(db, 64, moveRight=false) if rc.isErr: return err(rc.error) ok LeafTie(root: lty.root, path: rc.value) @@ -436,7 +442,7 @@ proc nearbyLeft*( # Public debugging helpers # ------------------------------------------------------------------------------ -proc nearbyRightMissing*( +proc rightMissing*( hike: Hike; # Partially expanded chain of vertices db: AristoDb; # Database layer ): Result[bool,AristoError] = diff --git a/nimbus/db/aristo/aristo_transcode.nim b/nimbus/db/aristo/aristo_transcode.nim index 1b37eb464..10610b8e6 100644 --- a/nimbus/db/aristo/aristo_transcode.nim +++ b/nimbus/db/aristo/aristo_transcode.nim @@ -127,10 +127,10 @@ proc append*(writer: var RlpWriter; node: NodeRef) = # Public db record transcoders # ------------------------------------------------------------------------------ -proc blobify*(node: VertexRef; data: var Blob): AristoError = - ## This function serialises the node argument to a database record. Contrary - ## to RLP based serialisation, these records aim to align on fixed byte - ## boundaries. +proc blobify*(vtx: VertexRef; data: var Blob): AristoError = + ## This function serialises the vertex argument to a database record. + ## Contrary to RLP based serialisation, these records aim to align on + ## fixed byte boundaries. ## :: ## Branch: ## uint64, ... -- list of up to 16 child vertices lookup keys @@ -152,7 +152,7 @@ proc blobify*(node: VertexRef; data: var Blob): AristoError = ## :: ## 8 * n * ((access shr (n * 4)) and 15) ## - case node.vType: + case vtx.vType: of Branch: var top = 0u64 @@ -160,30 +160,34 @@ proc blobify*(node: VertexRef; data: var Blob): AristoError = refs: Blob keys: Blob for n in 0..15: - if node.bVid[n].isValid: + if vtx.bVid[n].isValid: access = access or (1u16 shl n) - refs &= node.bVid[n].uint64.toBytesBE.toSeq + refs &= vtx.bVid[n].uint64.toBytesBE.toSeq + if refs.len < 16: + return BlobifyBranchMissingRefs data = refs & access.toBytesBE.toSeq & @[0u8] of Extension: let - pSegm = node.ePfx.hexPrefixEncode(isleaf = false) + pSegm = vtx.ePfx.hexPrefixEncode(isleaf = false) psLen = pSegm.len.byte if psLen == 0 or 33 < pslen: - return BlobifyVtxExPathOverflow - data = node.eVid.uint64.toBytesBE.toSeq & pSegm & @[0x80u8 or psLen] + return BlobifyExtPathOverflow + if not vtx.eVid.isValid: + return BlobifyExtMissingRefs + data = vtx.eVid.uint64.toBytesBE.toSeq & pSegm & @[0x80u8 or psLen] of Leaf: let - pSegm = node.lPfx.hexPrefixEncode(isleaf = true) + pSegm = vtx.lPfx.hexPrefixEncode(isleaf = true) psLen = pSegm.len.byte if psLen == 0 or 33 < psLen: - return BlobifyVtxLeafPathOverflow - data = node.lData.convertTo(Blob) & pSegm & @[0xC0u8 or psLen] + return BlobifyLeafPathOverflow + data = vtx.lData.convertTo(Blob) & pSegm & @[0xC0u8 or psLen] -proc blobify*(node: VertexRef): Result[Blob, AristoError] = +proc blobify*(vtx: VertexRef): Result[Blob, AristoError] = ## Variant of `blobify()` var data: Blob - info = node.blobify data + info = vtx.blobify data if info != AristoError(0): return err(info) ok(data) diff --git a/nimbus/db/aristo/aristo_vid.nim b/nimbus/db/aristo/aristo_vid.nim index 0b57487e4..8fe3d8a60 100644 --- a/nimbus/db/aristo/aristo_vid.nim +++ b/nimbus/db/aristo/aristo_vid.nim @@ -21,19 +21,23 @@ import # Public functions # ------------------------------------------------------------------------------ -proc vidFetch*(db: AristoDb): VertexID = - ## Create a new `VertexID`. Reusable *ID*s are kept in a list where the top - ## entry *ID0* has the property that any other *ID* larger *ID0* is also not +proc vidFetch*(db: AristoDb; pristine = false): VertexID = + ## Create a new `VertexID`. Reusable vertex *ID*s are kept in a list where + ## the top entry *ID* has the property that any other *ID* larger is also not ## not used on the database. + ## + ## The function prefers to return recycled vertex *ID*s if there are any. + ## When the argument `pristine` is set `true`, the function guarantees to + ## return a non-recycled, brand new vertex *ID* which is the preferred mode + ## when creating leaf vertices. let top = db.top - case top.vGen.len: - of 0: + if top.vGen.len == 0: # Note that `VertexID(1)` is the root of the main trie top.vGen = @[VertexID(3)] result = VertexID(2) - of 1: + elif top.vGen.len == 1 or pristine: result = top.vGen[^1] - top.vGen = @[VertexID(result.uint64 + 1)] + top.vGen[^1] = result + 1 else: result = top.vGen[^2] top.vGen[^2] = top.vGen[^1] diff --git a/tests/test_aristo.nim b/tests/test_aristo.nim index 69b2a003e..0a05a740b 100644 --- a/tests/test_aristo.nim +++ b/tests/test_aristo.nim @@ -222,7 +222,7 @@ proc accountsRunner( check noisy.test_nearbyKvpList(accLst, resetDb) test &"Delete accounts database, successively {accLst.len} entries": - check noisy.test_delete accLst + check noisy.test_delete(accLst, dbDir) proc storagesRunner( @@ -261,7 +261,7 @@ proc storagesRunner( check noisy.test_nearbyKvpList(stoLst, resetDb) test &"Delete storage database, successively {stoLst.len} entries": - check noisy.test_delete stoLst + check noisy.test_delete(stoLst, dbDir) # ------------------------------------------------------------------------------ # Main function(s) @@ -279,11 +279,11 @@ when isMainModule: setErrorLevel() - when true: # and false: + when true and false: noisy.miscRunner() # Borrowed from `test_sync_snap.nim` - when true: # and false: + when true and false: for n,sam in snapTestList: noisy.transcodeRunner(sam) for n,sam in snapTestStorageList: diff --git a/tests/test_aristo/test_backend.nim b/tests/test_aristo/test_backend.nim index 1fe2778a5..e46b6ff39 100644 --- a/tests/test_aristo/test_backend.nim +++ b/tests/test_aristo/test_backend.nim @@ -59,7 +59,7 @@ proc mergeData( noisy.say "***", "dataMerge(9)", " nLeafs=", leafs.len, "\n cache dump\n ", db.pp, - "\n backend dump\n ", db.backend.AristoTypedBackendRef.pp(db) + "\n backend dump\n ", db.to(TypedBackendRef).pp(db) check rc.error == (VertexID(0),AristoError(0)) return @@ -174,13 +174,13 @@ proc test_backendConsistency*( noisy.say "***", "beCon(2) <", n, "/", list.len-1, ">", " groups=", count, "\n cache dump\n ", ndb.pp, - "\n backend dump\n ", ndb.backend.AristoTypedBackendRef.pp(ndb), + "\n backend dump\n ", ndb.to(TypedBackendRef).pp(ndb), "\n -------------", "\n mdb cache\n ", mdb.pp, - "\n mdb backend\n ", mdb.to(MemBackendRef).pp(ndb), + "\n mdb backend\n ", mdb.to(TypedBackendRef).pp(ndb), "\n -------------", "\n rdb cache\n ", rdb.pp, - "\n rdb backend\n ", rdb.to(RdbBackendRef).pp(ndb), + "\n rdb backend\n ", rdb.to(TypedBackendRef).pp(ndb), "\n -------------" when true and false: @@ -200,7 +200,7 @@ proc test_backendConsistency*( #noisy.say "***", "db-dump\n ", mdb.pp let rc = mdb.save if rc.isErr: - check rc.error == AristoError(0) + check rc.error == (0,0) return rc.value @@ -208,11 +208,11 @@ proc test_backendConsistency*( let rdbHist = block: let rc = rdb.save if rc.isErr: - check rc.error == AristoError(0) + check rc.error == (0,0) return rc.value - if not ndb.top.verify(mdb.backend.MemBackendRef, noisy): + if not ndb.top.verify(mdb.to(MemBackendRef), noisy): when true and false: noisy.say "***", "beCon(4) <", n, "/", list.len-1, ">", " groups=", count, @@ -223,7 +223,7 @@ proc test_backendConsistency*( #"\n mdb pre-save backend\n ", mdbPreSaveBackend, "\n -------------", "\n mdb cache\n ", mdb.pp, - "\n mdb backend\n ", mdb.to(MemBackendRef).pp(ndb), + "\n mdb backend\n ", mdb.to(TypedBackendRef).pp(ndb), "\n -------------" return @@ -239,10 +239,10 @@ proc test_backendConsistency*( "\n rdb pre-save backend\n ", rdbPreSaveBackend, "\n -------------", "\n rdb cache\n ", rdb.pp, - "\n rdb backend\n ", rdb.to(RdbBackendRef).pp(ndb), + "\n rdb backend\n ", rdb.to(TypedBackendRef).pp(ndb), #"\n -------------", #"\n mdb cache\n ", mdb.pp, - #"\n mdb backend\n ", mdb.to(MemBackendRef).pp(ndb), + #"\n mdb backend\n ", mdb.to(TypedBackendRef).pp(ndb), "\n -------------" return diff --git a/tests/test_aristo/test_delete.nim b/tests/test_aristo/test_delete.nim index 9410959a8..5757cf707 100644 --- a/tests/test_aristo/test_delete.nim +++ b/tests/test_aristo/test_delete.nim @@ -12,13 +12,14 @@ ## Aristo (aka Patricia) DB records merge test import - std/[algorithm, bitops, sequtils], + std/[algorithm, bitops, sequtils, strutils, sets], eth/common, stew/results, unittest2, ../../nimbus/db/aristo/[ - aristo_desc, aristo_debug, aristo_delete, aristo_hashify, aristo_init, - aristo_nearby, aristo_merge], + aristo_check, aristo_desc, aristo_debug, aristo_delete, aristo_get, + aristo_hashify, aristo_hike, aristo_init, aristo_layer, aristo_nearby, + aristo_merge], ./test_helpers type @@ -32,6 +33,9 @@ type proc sortedKeys(lTab: Table[LeafTie,VertexID]): seq[LeafTie] = lTab.keys.toSeq.sorted(cmp = proc(a,b: LeafTie): int = cmp(a,b)) +proc pp(q: HashSet[LeafTie]): string = + "{" & q.toSeq.mapIt(it.pp).join(",") & "}" + # -------------- proc posixPrngRand(state: var uint32): byte = @@ -73,38 +77,153 @@ proc rand(td: var TesterDesc; top: int): int = # ----------------------- +proc randomisedLeafs(db: AristoDb; td: var TesterDesc): seq[LeafTie] = + result = db.top.lTab.sortedKeys + if 2 < result.len: + for n in 0 ..< result.len-1: + let r = n + td.rand(result.len - n) + result[n].swap result[r] + + +proc saveToBackend( + db: var AristoDb; + relax: bool; + noisy: bool; + debugID: int; + ): bool = + let + trigger = false # or (debugID == 340) + prePreCache = db.pp + prePreBe = db.to(TypedBackendRef).pp(db) + if trigger: + noisy.say "***", "saveToBackend =========================== ", debugID + block: + let rc = db.checkCache(relax=true) + if rc.isErr: + noisy.say "***", "saveToBackend (1) hashifyCheck", + " debugID=", debugID, + " error=", rc.error, + "\n cache\n ", db.pp, + "\n backend\n ", db.to(TypedBackendRef).pp(db), + "\n --------" + check rc.error == (0,0) + return + block: + let rc = db.hashify # (noisy = trigger) + if rc.isErr: + noisy.say "***", "saveToBackend (2) hashify", + " debugID=", debugID, + " error=", rc.error, + "\n pre-cache\n ", prePreCache, + "\n pre-be\n ", prePreBe, + "\n -------- hasify() -----", + "\n cache\n ", db.pp, + "\n backend\n ", db.to(TypedBackendRef).pp(db), + "\n --------" + check rc.error == (0,0) + return + let + preCache = db.pp + preBe = db.to(TypedBackendRef).pp(db) + block: + let rc = db.checkBE(relax=true) + if rc.isErr: + let noisy = true + noisy.say "***", "saveToBackend (3) checkBE", + " debugID=", debugID, + " error=", rc.error, + "\n cache\n ", db.pp, + "\n backend\n ", db.to(TypedBackendRef).pp(db), + "\n --------" + check rc.error == (0,0) + return + block: + let rc = db.save() + if rc.isErr: + check rc.error == (0,0) + return + block: + let rc = db.checkBE(relax=relax) + if rc.isErr: + let noisy = true + noisy.say "***", "saveToBackend (4) checkBE", + " debugID=", debugID, + " error=", rc.error, + "\n prePre-cache\n ", prePreCache, + "\n prePre-be\n ", prePreBe, + "\n -------- hashify() -----", + "\n pre-cache\n ", preCache, + "\n pre-be\n ", preBe, + "\n -------- save() --------", + "\n cache\n ", db.pp, + "\n backend\n ", db.to(TypedBackendRef).pp(db), + "\n --------" + check rc.error == (0,0) + return + + when true and false: + if trigger: + noisy.say "***", "saveToBackend (9)", + " debugID=", debugID, + "\n prePre-cache\n ", prePreCache, + "\n prePre-be\n ", prePreBe, + "\n -------- hashify() -----", + "\n pre-cache\n ", preCache, + "\n pre-be\n ", preBe, + "\n -------- save() --------", + "\n cache\n ", db.pp, + "\n backend\n ", db.to(TypedBackendRef).pp(db), + "\n --------" + true + + proc fwdWalkVerify( db: AristoDb; root: VertexID; + left: HashSet[LeafTie]; noisy: bool; - ): tuple[visited: int, error: AristoError] = + debugID: int; + ): tuple[visited: int, error: AristoError] = let - lTabLen = db.top.lTab.len + nLeafs = left.len var - error = AristoError(0) + lfLeft = left lty = LeafTie(root: root) n = 0 - while n < lTabLen + 1: - let rc = lty.nearbyRight(db) - #noisy.say "=================== ", n + + while n < nLeafs + 1: + let id = n + (nLeafs + 1) * debugID + noisy.say "NearbyBeyondRange =================== ", id + + let rc = lty.right db if rc.isErr: - if rc.error != NearbyBeyondRange: - noisy.say "***", "<", n, "/", lTabLen-1, "> fwd-walk error=", rc.error - error = rc.error - check rc.error == AristoError(0) - break + if rc.error[1] != NearbyBeyondRange or 0 < lfLeft.len: + noisy.say "***", "fwdWalkVerify (1) nearbyRight", + " n=", n, "/", nLeafs, + " lty=", lty.pp(db), + " error=", rc.error + check rc.error == (0,0) + return (n,rc.error[1]) + return (0, AristoError(0)) + + if rc.value notin lfLeft: + noisy.say "***", "fwdWalkVerify (2) lfLeft", + " n=", n, "/", nLeafs, + " lty=", lty.pp(db) + check rc.error == (0,0) + return (n,rc.error[1]) + if rc.value.path < high(HashID): lty.path = HashID(rc.value.path.u256 + 1) + + lfLeft.excl rc.value n.inc - if error != AristoError(0): - return (n,error) - - if n != lTabLen: - check n == lTabLen - return (-1, AristoError(1)) - - (0, AristoError(0)) + noisy.say "***", "fwdWalkVerify (9) oops", + " n=", n, "/", nLeafs, + " lfLeft=", lfLeft.pp + check n <= nLeafs + (-1, AristoError(1)) # ------------------------------------------------------------------------------ # Public test function @@ -113,77 +232,149 @@ proc fwdWalkVerify( proc test_delete*( noisy: bool; list: openArray[ProofTrieData]; - ): bool = - var td = TesterDesc.init 42 + rdbPath: string; # Rocks DB storage directory + ): bool = + var + td = TesterDesc.init 42 + db: AristoDb + defer: + db.finish(flush=true) + for n,w in list: + # Start with new database + db.finish(flush=true) + db = block: + let rc = AristoDb.init(BackendRocksDB,rdbPath) + if rc.isErr: + check rc.error == 0 + return + rc.value + + # Merge leaf data into main trie (w/vertex ID 1) let - db = AristoDb.init BackendNone # (top: AristoLayerRef()) - lstLen = list.len - leafs = w.kvpLst.mapRootVid VertexID(1) # merge into main trie + leafs = w.kvpLst.mapRootVid VertexID(1) added = db.merge leafs - preState = db.pp - - if added.error != AristoError(0): - check added.error == AristoError(0) + if added.error != 0: + check added.error == 0 return - let rc = db.hashify - if rc.isErr: - check rc.error == (VertexID(0),AristoError(0)) - return - # Now `db` represents a (fully labelled) `Merkle Patricia Tree` - # Provide a (reproducible) peudo-random copy of the leafs list - var leafTies = db.top.lTab.sortedKeys - if 2 < leafTies.len: - for n in 0 ..< leafTies.len-1: - let r = n + td.rand(leafTies.len - n) - leafTies[n].swap leafTies[r] + let leafTies = db.randomisedLeafs td + var leafsLeft = leafs.mapIt(it.leafTie).toHashSet - let uMax = leafTies.len - 1 + # Complete as `Merkle Patricia Tree` and save to backend, clears cache + block: + let saveBeOk = db.saveToBackend(relax=true, noisy=false, 0) + if not saveBeOk: + check saveBeOk + return + + # Trigger subsequent saving tasks in loop below + let (saveMod, saveRest, relax) = block: + if leafTies.len < 17: (7, 3, false) + elif leafTies.len < 31: (11, 7, false) + else: (leafTies.len div 5, 11, true) + + # Loop over leaf ties for u,leafTie in leafTies: - let rc = leafTie.delete db # ,noisy) + + # Get leaf vertex ID so making sure that it is on the database + let + runID = n + list.len * u + doSaveBeOk = ((u mod saveMod) == saveRest) # or true + trigger = false # or runID in {60,80} + tailWalkVerify = 20 # + 999 + leafVid = block: + let hike = leafTie.hikeUp(db) + if hike.error != 0: # Ooops + check hike.error == 0 + return + hike.legs[^1].wp.vid + + if doSaveBeOk: + when true and false: + noisy.say "***", "del(1)", + " n=", n, "/", list.len, + " u=", u, "/", leafTies.len, + " runID=", runID, + " relax=", relax, + " leafVid=", leafVid.pp + let saveBeOk = db.saveToBackend(relax=relax, noisy=noisy, runID) + if not saveBeOk: + noisy.say "***", "del(2)", + " n=", n, "/", list.len, + " u=", u, "/", leafTies.len, + " leafVid=", leafVid.pp + check saveBeOk + return + + # Delete leaf + let + preCache = db.pp + rc = db.delete leafTie if rc.isErr: - check rc.error == (VertexID(0),AristoError(0)) + check rc.error == (0,0) return - if leafTie in db.top.lTab: - check leafTie notin db.top.lTab - return - if uMax != db.top.lTab.len + u: - check uMax == db.top.lTab.len + u + + # Update list of remaininf leafs + leafsLeft.excl leafTie + + let leafVtx = db.getVtx leafVid + if leafVtx.isValid: + noisy.say "***", "del(3)", + " n=", n, "/", list.len, + " u=", u, "/", leafTies.len, + " runID=", runID, + " root=", leafTie.root.pp, + " leafVid=", leafVid.pp, + "\n --------", + "\n pre-cache\n ", preCache, + "\n --------", + "\n cache\n ", db.pp, + "\n backend\n ", db.to(TypedBackendRef).pp(db), + "\n --------" + check leafVtx.isValid == false return # Walking the database is too slow for large tables. So the hope is that # potential errors will not go away and rather pop up later, as well. - const tailCheck = 999 - if uMax < u + tailCheck: - if u < uMax: - let vfy = db.fwdWalkVerify(leafTie.root, noisy) - if vfy.error != AristoError(0): - check vfy == (0, AristoError(0)) + if leafsLeft.len <= tailWalkVerify: + if u < leafTies.len-1: + let + noisy = false + vfy = db.fwdWalkVerify(leafTie.root, leafsLeft, noisy, runID) + if vfy.error != AristoError(0): # or 7 <= u: + noisy.say "***", "del(5)", + " n=", n, "/", list.len, + " u=", u, "/", leafTies.len, + " runID=", runID, + " root=", leafTie.root.pp, + " leafVid=", leafVid.pp, + "\n leafVtx=", leafVtx.pp(db), + "\n --------", + "\n pre-cache\n ", preCache, + "\n -------- delete() -------", + "\n cache\n ", db.pp, + "\n backend\n ", db.to(TypedBackendRef).pp(db), + "\n --------" + check vfy == (0,0) return - elif 0 < db.top.sTab.len: - check db.top.sTab.len == 0 - return - let rc = db.hashifyCheck(relax=true) # ,noisy=true) - if rc.isErr: - noisy.say "***", "<", n, "/", lstLen-1, ">", - " item=", u, "/", uMax, - "\n --------", - "\n pre-DB\n ", preState, - "\n --------", - "\n cache\n ", db.pp, - "\n --------" - check rc.error == (VertexID(0),AristoError(0)) - return when true and false: - if uMax < u + tailCheck or (u mod 777) == 3: - noisy.say "***", "step lTab=", db.top.lTab.len + if trigger: + noisy.say "***", "del(8)", + " n=", n, "/", list.len, + " u=", u, "/", leafTies.len, + " runID=", runID, + "\n pre-cache\n ", preCache, + "\n -------- delete() -------", + "\n cache\n ", db.pp, + "\n backend\n ", db.to(TypedBackendRef).pp(db), + "\n --------" + + when true: # and false: + noisy.say "***", "del(9) n=", n, "/", list.len, " nLeafs=", leafs.len - when true and false: - noisy.say "***", "sample <", n, "/", list.len-1, ">", - " lstLen=", leafs.len true # ------------------------------------------------------------------------------ diff --git a/tests/test_aristo/test_helpers.nim b/tests/test_aristo/test_helpers.nim index 8875bbd20..0513027ef 100644 --- a/tests/test_aristo/test_helpers.nim +++ b/tests/test_aristo/test_helpers.nim @@ -94,6 +94,12 @@ proc say*(noisy = false; pfx = "***"; args: varargs[string, `$`]) = # Public helpers # ------------------------------------------------------------------------------ +proc `==`*[T: AristoError|VertexID](a: T, b: int): bool = + a == T(b) + +proc `==`*[S,T](a: (S,T), b: (int,int)): bool = + a == (S(b[0]), T(b[1])) + proc to*(sample: AccountsSample; T: type seq[UndumpAccounts]): T = ## Convert test data into usable in-memory format let file = sample.file.findFilePath.value diff --git a/tests/test_aristo/test_merge.nim b/tests/test_aristo/test_merge.nim index e3dd74cf8..afd1e972b 100644 --- a/tests/test_aristo/test_merge.nim +++ b/tests/test_aristo/test_merge.nim @@ -18,8 +18,8 @@ import unittest2, ../../nimbus/db/aristo/aristo_init/aristo_rocksdb, ../../nimbus/db/aristo/[ - aristo_desc, aristo_debug, aristo_get, aristo_hashify, aristo_init, - aristo_hike, aristo_layer, aristo_merge], + aristo_check, aristo_desc, aristo_debug, aristo_get, aristo_hashify, + aristo_init, aristo_hike, aristo_layer, aristo_merge], ./test_helpers type @@ -76,7 +76,7 @@ proc mergeStepwise( stopOk = true let hashesOk = block: - let rc = db.hashifyCheck(relax=true) + let rc = db.checkCache(relax=true) if rc.isOk: (VertexID(0),AristoError(0)) else: @@ -196,7 +196,7 @@ proc test_mergeKvpList*( "\n --------" block: - let rc = db.hashifyCheck() + let rc = db.checkCache() if rc.isErr: noisy.say "*** kvp(4)", "<", n, "/", lstLen-1, "> db dump", "\n pre-DB\n ", preDb, @@ -211,7 +211,7 @@ proc test_mergeKvpList*( let rdbHist = block: let rc = db.save if rc.isErr: - check rc.error == AristoError(0) + check rc.error == (0,0) return rc.value @@ -324,7 +324,7 @@ proc test_mergeProofAndKvpList*( block: let - preDb = db.pp(sTabOk=false, lTabOk=false) + preDb = db.pp(xTabOk=false) rc = db.hashify() # noisy=true) # Handle known errors @@ -354,7 +354,7 @@ proc test_mergeProofAndKvpList*( let rdbHist = block: let rc = db.save if rc.isErr: - check rc.error == AristoError(0) + check rc.error == (0,0) return rc.value diff --git a/tests/test_aristo/test_nearby.nim b/tests/test_aristo/test_nearby.nim index 39026ff1b..a11ae8d49 100644 --- a/tests/test_aristo/test_nearby.nim +++ b/tests/test_aristo/test_nearby.nim @@ -33,40 +33,37 @@ proc fwdWalkLeafsCompleteDB( let tLen = tags.len var - error = AristoError(0) lty = LeafTie(root: root, path: HashID(tags[0].u256 div 2)) n = 0 while true: - let rc = lty.nearbyRight(db) + let rc = lty.right(db) #noisy.say "=================== ", n if rc.isErr: - if rc.error != NearbyBeyondRange: + if rc.error[1] != NearbyBeyondRange: noisy.say "***", "[", n, "/", tLen-1, "] fwd-walk error=", rc.error - error = rc.error - check rc.error == AristoError(0) - elif n != tLen: - error = AristoError(1) + check rc.error == (0,0) + return (n,rc.error[1]) + if n != tLen: check n == tLen + return (n,AristoError(1)) break if tLen <= n: noisy.say "***", "[", n, "/", tLen-1, "] fwd-walk -- ", " oops, too many leafs (index overflow)" - error = AristoError(1) check n < tlen - break + return (n,AristoError(1)) if rc.value.path != tags[n]: noisy.say "***", "[", n, "/", tLen-1, "] fwd-walk -- leafs differ,", " got=", rc.value.pp(db), " wanted=", LeafTie(root: root, path: tags[n]).pp(db) #, # " db-dump\n ", db.pp - error = AristoError(1) check rc.value.path == tags[n] - break + return (n,AristoError(1)) if rc.value.path < high(HashID): lty.path = HashID(rc.value.path.u256 + 1) n.inc - (n,error) + (n,AristoError(0)) proc revWalkLeafsCompleteDB( @@ -78,39 +75,36 @@ proc revWalkLeafsCompleteDB( let tLen = tags.len var - error = AristoError(0) delta = ((high(UInt256) - tags[^1].u256) div 2) lty = LeafTie(root: root, path: HashID(tags[^1].u256 + delta)) n = tLen-1 while true: # and false: - let rc = lty.nearbyLeft(db) + let rc = lty.left(db) if rc.isErr: - if rc.error != NearbyBeyondRange: + if rc.error[1] != NearbyBeyondRange: noisy.say "***", "[", n, "/", tLen-1, "] rev-walk error=", rc.error - error = rc.error - check rc.error == AristoError(0) - elif n != -1: - error = AristoError(1) + check rc.error == (0,0) + return (n,rc.error[1]) + if n != -1: check n == -1 + return (n,AristoError(1)) break if n < 0: noisy.say "***", "[", n, "/", tLen-1, "] rev-walk -- ", " oops, too many leafs (index underflow)" - error = AristoError(1) check 0 <= n - break + return (n,AristoError(1)) if rc.value.path != tags[n]: noisy.say "***", "[", n, "/", tLen-1, "] rev-walk -- leafs differ,", " got=", rc.value.pp(db), " wanted=", tags[n]..pp(db) #, " db-dump\n ", db.pp - error = AristoError(1) check rc.value.path == tags[n] - break + return (n,AristoError(1)) if low(HashID) < rc.value.path: lty.path = HashID(rc.value.path.u256 - 1) n.dec - (tLen-1 - n, error) + (tLen-1 - n, AristoError(0)) # ------------------------------------------------------------------------------ # Public test function