From 69a158864cc44a69ebfbdf2b2649b21909d4179f Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Tue, 4 Jun 2024 15:05:13 +0000 Subject: [PATCH] Remove vid recycling feature (#2294) --- nimbus/db/aristo/aristo_blobify.nim | 54 ++++--- nimbus/db/aristo/aristo_check/check_be.nim | 113 ++++++--------- nimbus/db/aristo/aristo_check/check_top.nim | 17 ++- nimbus/db/aristo/aristo_debug.nim | 48 +++---- nimbus/db/aristo/aristo_delete.nim | 14 -- nimbus/db/aristo/aristo_delta.nim | 2 +- nimbus/db/aristo/aristo_delta/delta_merge.nim | 2 +- .../db/aristo/aristo_delta/delta_reverse.nim | 6 +- nimbus/db/aristo/aristo_desc.nim | 8 +- nimbus/db/aristo/aristo_desc/desc_backend.nim | 24 ++-- nimbus/db/aristo/aristo_desc/desc_error.nim | 7 +- .../db/aristo/aristo_desc/desc_structural.nim | 2 +- nimbus/db/aristo/aristo_get.nim | 16 +-- nimbus/db/aristo/aristo_init/init_common.nim | 2 +- nimbus/db/aristo/aristo_init/memory_db.nim | 37 +++-- nimbus/db/aristo/aristo_init/persistent.nim | 6 +- nimbus/db/aristo/aristo_init/rocks_db.nim | 35 +++-- nimbus/db/aristo/aristo_layers.nim | 8 +- nimbus/db/aristo/aristo_tx/tx_fork.nim | 6 +- nimbus/db/aristo/aristo_tx/tx_frame.nim | 4 +- nimbus/db/aristo/aristo_tx/tx_stow.nim | 19 ++- nimbus/db/aristo/aristo_vid.nim | 95 ++----------- tests/test_aristo.nim | 3 - tests/test_aristo/test_filter.nim | 2 +- tests/test_aristo/test_misc.nim | 132 +----------------- 25 files changed, 208 insertions(+), 454 deletions(-) diff --git a/nimbus/db/aristo/aristo_blobify.nim b/nimbus/db/aristo/aristo_blobify.nim index ae9804011..fd7dd6f9a 100644 --- a/nimbus/db/aristo/aristo_blobify.nim +++ b/nimbus/db/aristo/aristo_blobify.nim @@ -135,26 +135,24 @@ proc blobifyTo*(vtx: VertexRef; data: var Blob): Result[void,AristoError] = data &= [0xC0u8 or psLen] ok() - proc blobify*(vtx: VertexRef): Result[Blob, AristoError] = ## Variant of `blobify()` var data: Blob ? vtx.blobifyTo data ok(move(data)) -proc blobifyTo*(vGen: openArray[VertexID]; data: var Blob) = - ## This function serialises a list of vertex IDs. - ## :: - ## uint64, ... -- list of IDs - ## 0x7c -- marker(8) - ## - for w in vGen: - data &= w.uint64.toBytesBE - data.add 0x7Cu8 -proc blobify*(vGen: openArray[VertexID]): Blob = - ## Variant of `blobify()` - vGen.blobifyTo result +proc blobifyTo*(tuv: VertexID; data: var Blob) = + ## This function serialises a top used vertex ID. + data.setLen(9) + let w = tuv.uint64.toBytesBE + (addr data[0]).copyMem(unsafeAddr w[0], 8) + data[8] = 0x7Cu8 + +proc blobify*(tuv: VertexID): Blob = + ## Variant of `blobifyTo()` + tuv.blobifyTo result + proc blobifyTo*(lSst: SavedState; data: var Blob) = ## Serialise a last saved state record @@ -315,30 +313,28 @@ proc deblobify*( proc deblobifyTo*( data: openArray[byte]; - vGen: var seq[VertexID]; + tuv: var VertexID; ): Result[void,AristoError] = - ## De-serialise the data record encoded with `blobify()` into the vertex ID - ## generator argument `vGen`. + ## De-serialise a top level vertex ID. if data.len == 0: - vGen = @[] + tuv = VertexID(0) + elif data.len != 9: + return err(DeblobSizeGarbled) + elif data[^1] != 0x7c: + return err(DeblobWrongType) else: - if (data.len mod 8) != 1: - return err(DeblobSizeGarbled) - if data[^1] != 0x7c: - return err(DeblobWrongType) - for n in 0 ..< (data.len div 8): - let w = n * 8 - vGen.add (uint64.fromBytesBE data.toOpenArray(w, w+7)).VertexID + tuv = (uint64.fromBytesBE data.toOpenArray(0, 7)).VertexID ok() proc deblobify*( data: openArray[byte]; - T: type seq[VertexID]; + T: type VertexID; ): Result[T,AristoError] = - ## Variant of `deblobify()` for deserialising the vertex ID generator state - var vGen: T - ? data.deblobifyTo vGen - ok move(vGen) + ## Variant of `deblobify()` for deserialising a top level vertex ID. + var vTop: T + ? data.deblobifyTo vTop + ok move(vTop) + proc deblobifyTo*( data: openArray[byte]; diff --git a/nimbus/db/aristo/aristo_check/check_be.nim b/nimbus/db/aristo/aristo_check/check_be.nim index 399327ea0..994e5bd07 100644 --- a/nimbus/db/aristo/aristo_check/check_be.nim +++ b/nimbus/db/aristo/aristo_check/check_be.nim @@ -11,32 +11,18 @@ {.push raises: [].} import - std/[algorithm, sequtils, sets, tables, typetraits], + std/[sets, tables], eth/[common, trie/nibbles], + results, stew/interval_set, ../../aristo, ../aristo_walk/persistent, ".."/[aristo_desc, aristo_get, aristo_layers, aristo_serialise] -const - Vid2 = @[VertexID(LEAST_FREE_VID)].toHashSet - # ------------------------------------------------------------------------------ # Private helper # ------------------------------------------------------------------------------ -proc to(s: IntervalSetRef[VertexID,uint64]; T: type HashSet[VertexID]): T = - ## Convert the argument list `s` to a set of vertex IDs as it would appear - ## with a vertex generator state list. - if s.total < high(uint64): - for w in s.increasing: - if w.maxPt == high(VertexID): - result.incl w.minPt # last interval - else: - for pt in w.minPt .. w.maxPt: - if LEAST_FREE_VID <= pt.distinctBase: - result.incl pt - proc toNodeBE( vtx: VertexRef; # Vertex to convert db: AristoDbRef; # Database, top layer @@ -76,17 +62,6 @@ proc toNodeBE( return ok node return err(vid) -proc vidReorgAlways(vGen: seq[VertexID]): seq[VertexID] = - ## See `vidReorg()`, this one always sorts and optimises - ## - if 1 < vGen.len: - let lst = vGen.mapIt(uint64(it)).sorted(Descending).mapIt(VertexID(it)) - for n in 0 .. lst.len-2: - if lst[n].uint64 != lst[n+1].uint64 + 1: - return lst[n+1 .. lst.len-1] & @[lst[n]] - return @[lst[^1]] - vGen - # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ @@ -100,11 +75,11 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef]( ): Result[void,(VertexID,AristoError)] = ## Make sure that each vertex has a Merkle hash and vice versa. Also check ## the vertex ID generator state. - let vids = IntervalSetRef[VertexID,uint64].init() - discard vids.merge Interval[VertexID,uint64].new( - VertexID(LEAST_FREE_VID),high(VertexID)) + var topVidBe = VertexID(0) for (vid,vtx) in T.walkVtxBe db: + if topVidBe < vid: + topVidBe = vid if not vtx.isValid: return err((vid,CheckBeVtxInvalid)) let rc = db.getKeyBE vid @@ -127,6 +102,8 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef]( return err((vid,CheckBeVtxExtPfxMissing)) for (vid,key) in T.walkKeyBe db: + if topVidBe < vid: + topVidBe = vid if not key.isValid: return err((vid,CheckBeKeyInvalid)) let vtx = db.getVtxBE(vid).valueOr: @@ -137,29 +114,32 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef]( let expected = node.digestTo(HashKey) if expected != key: return err((vid,CheckBeKeyMismatch)) - discard vids.reduce Interval[VertexID,uint64].new(vid,vid) - # Compare calculated state against database state - block: - # Extract vertex ID generator state - let vGen = block: - let rc = db.getIdgBE() + # Compare calculated `vTop` against database state + if topVidBe.isValid: + let vidTuvBe = block: + let rc = db.getTuvBE() if rc.isOk: - rc.value.vidReorgAlways.toHashSet - elif rc.error == GetIdgNotFound: - EmptyVidSeq.toHashSet + rc.value + elif rc.error == GetTuvNotFound: + VertexID(0) else: return err((VertexID(0),rc.error)) - let - vGenExpected = vids.to(HashSet[VertexID]) - delta = vGenExpected -+- vGen # symmetric difference - if 0 < delta.len: - # Exclude fringe case when there is a single root vertex only - if vGenExpected != Vid2 or 0 < vGen.len: - return err((delta.toSeq.sorted[^1],CheckBeGarbledVGen)) + if vidTuvBe != topVidBe: + # All vertices and keys between `topVidBe` and `vidTuvBe` must have + # been deleted. + for vid in max(topVidBe + 1, VertexID(LEAST_FREE_VID)) .. vidTuvBe: + if db.getVtxBE(vid).isOk or db.getKeyBE(vid).isOk: + echo ">>>", + " topVidBe=", topVidBe, + " vidTuvBe=", vidTuvBe, + " vid=", vid + return err((vid,CheckBeGarbledVTop)) - # Check top layer cache against backend + # Check layer cache against backend if cache: + var topVidCache = VertexID(0) + let checkKeysOk = block: if db.dirty.len == 0: true @@ -170,6 +150,8 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef]( # Check structural table for (vid,vtx) in db.layersWalkVtx: + if vtx.isValid and topVidCache < vid: + topVidCache = vid let key = block: let rc = db.layersGetKey(vid) if rc.isOk: @@ -179,22 +161,19 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef]( return err((vid,CheckBeCacheKeyMissing)) else: VOID_HASH_KEY - if vtx.isValid: - # Register existing vid against backend generator state - discard vids.reduce Interval[VertexID,uint64].new(vid,vid) - else: + if not vtx.isValid: # Some vertex is to be deleted, the key must be empty if checkKeysOk and key.isValid: return err((vid,CheckBeCacheKeyNonEmpty)) # There must be a representation on the backend DB unless in a TX if db.getVtxBE(vid).isErr and db.stack.len == 0: return err((vid,CheckBeCacheVidUnsynced)) - # Register deleted vid against backend generator state - discard vids.merge Interval[VertexID,uint64].new(vid,vid) # Check key table var list: seq[VertexID] for (vid,key) in db.layersWalkKey: + if key.isValid and topVidCache < vid: + topVidCache = vid list.add vid let vtx = db.getVtx vid if db.layersGetVtx(vid).isErr and not vtx.isValid: @@ -209,22 +188,18 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef]( if expected != key: return err((vid,CheckBeCacheKeyMismatch)) - # Check vGen - let - vGen = db.vGen.vidReorgAlways.toHashSet - vGenExpected = vids.to(HashSet[VertexID]) - delta = vGenExpected -+- vGen # symmetric difference - if 0 < delta.len: - if vGen == Vid2 and vGenExpected.len == 0: - # Fringe case when the database is empty - discard - elif vGen.len == 0 and vGenExpected == Vid2: - # Fringe case when there is a single root vertex only - discard - else: - let delta = delta.toSeq - if delta.len != 1 or delta[0] != VertexID(1) or VertexID(1) in vGen: - return err((delta.sorted[^1],CheckBeCacheGarbledVGen)) + # Check vTop + if topVidCache.isValid and topVidCache != db.vTop: + # All vertices and keys between `topVidCache` and `db.vTop` must have + # been deleted. + for vid in max(db.vTop + 1, VertexID(LEAST_FREE_VID)) .. topVidCache: + if db.layersGetVtxOrVoid(vid).isValid or + db.layersGetKeyOrVoid(vid).isValid: + echo ">>>", + " topVidCache=", topVidCache, + " vTop=", db.vTop, + " vid=", vid + return err((db.vTop,CheckBeCacheGarbledVTop)) ok() diff --git a/nimbus/db/aristo/aristo_check/check_top.nim b/nimbus/db/aristo/aristo_check/check_top.nim index ce41558b5..11cb6ac60 100644 --- a/nimbus/db/aristo/aristo_check/check_top.nim +++ b/nimbus/db/aristo/aristo_check/check_top.nim @@ -11,7 +11,7 @@ {.push raises: [].} import - std/[sequtils, sets], + std/[sequtils, sets, typetraits], eth/[common, trie/nibbles], results, ".."/[aristo_desc, aristo_get, aristo_layers, aristo_serialise, aristo_utils] @@ -92,15 +92,17 @@ proc checkTopCommon*( let kMapCount = db.layersWalkKey.toSeq.mapIt(it[1]).filterIt(it.isValid).len kMapNilCount = db.layersWalkKey.toSeq.len - kMapCount - vGen = db.vGen.toHashSet - vGenMax = if vGen.len == 0: VertexID(0) else: db.vGen[^1] + vTop = db.vTop var + topVid = VertexID(0) stoRoots: HashSet[VertexID] # Collect leafs and check deleted entries var nNilVtx = 0 for (vid,vtx) in db.layersWalkVtx: if vtx.isValid: + if topVid < vid: + topVid = vid case vtx.vType: of Leaf: if vtx.lData.pType == AccountData: @@ -108,7 +110,7 @@ proc checkTopCommon*( if stoVid.isValid: if stoVid in stoRoots: return err((stoVid,CheckAnyVidSharedStorageRoot)) - if vGenMax.isValid and (vGenMax < stoVid or stoVid in vGen): + if vTop < stoVid: return err((stoVid,CheckAnyVidDeadStorageRoot)) stoRoots.incl stoVid of Branch: @@ -131,6 +133,13 @@ proc checkTopCommon*( if rc.value.isValid: return err((vid,CheckAnyVtxEmptyKeyExpected)) + if vTop.distinctBase < LEAST_FREE_VID: + # Verify that all vids are below `LEAST_FREE_VID` + if topVid.distinctBase < LEAST_FREE_VID: + for (vid,key) in db.layersWalkKey: + if key.isValid and LEAST_FREE_VID <= vid.distinctBase: + return err((topVid,CheckAnyVTopUnset)) + # If present, there are at least as many deleted hashes as there are deleted # vertices. if kMapNilCount != 0 and kMapNilCount < nNilVtx: diff --git a/nimbus/db/aristo/aristo_debug.nim b/nimbus/db/aristo/aristo_debug.nim index d62c3245e..997589ac3 100644 --- a/nimbus/db/aristo/aristo_debug.nim +++ b/nimbus/db/aristo/aristo_debug.nim @@ -17,7 +17,7 @@ import stew/[byteutils, interval_set], ./aristo_desc/desc_backend, ./aristo_init/[memory_db, memory_only, rocks_db], - "."/[aristo_constants, aristo_desc, aristo_hike, aristo_layers] + "."/[aristo_desc, aristo_hike, aristo_layers] # ------------------------------------------------------------------------------ # Private functions @@ -150,14 +150,14 @@ func ppCodeHash(h: Hash256): string = else: result &= h.data.toHex.squeeze(hex=true,ignLen=true) -proc ppVidList(vGen: openArray[VertexID]): string = +proc ppVidList(vLst: openArray[VertexID]): string = result = "[" - if vGen.len <= 250: - result &= vGen.mapIt(it.ppVid).join(",") + if vLst.len <= 250: + result &= vLst.mapIt(it.ppVid).join(",") else: - result &= vGen[0 .. 99].mapIt(it.ppVid).join(",") + result &= vLst[0 .. 99].mapIt(it.ppVid).join(",") result &= ",.." - result &= vGen[^100 .. ^1].mapIt(it.ppVid).join(",") + result &= vLst[^100 .. ^1].mapIt(it.ppVid).join(",") result &= "]" proc ppKey(key: HashKey; db: AristoDbRef; pfx = true): string = @@ -404,8 +404,7 @@ proc ppFilter( result &= " n/a" return result &= pfx & "src=" & fl.src.ppKey(db) - result &= pfx & "vGen" & pfx1 & "[" & - fl.vGen.mapIt(it.ppVid).join(",") & "]" + result &= pfx & "vTop=" & fl.vTop.ppVid result &= pfx & "sTab" & pfx1 & "{" for n,vid in fl.sTab.sortedKeys: let vtx = fl.sTab.getOrVoid vid @@ -426,13 +425,11 @@ proc ppBe[T](be: T; db: AristoDbRef; limit: int; indent: int): string = pfx2 = indent.toPfx(2) result = "<" & $be.kind & ">" var (dump,dataOk) = ("",false) - dump &= pfx & "vGen" block: - let q = be.getIdgFn().get(otherwise = EmptyVidSeq) - dump &= "(" & $q.len & ")" - if 0 < q.len: + let rc = be.getTuvFn() + if rc.isOk: + dump &= pfx & "vTop=" & rc.value.ppVid dataOk = true - dump &= pfx1 & q.ppVidList() block: dump &= pfx & "sTab" var (n, data) = (0, "") @@ -471,7 +468,7 @@ proc ppBe[T](be: T; db: AristoDbRef; limit: int; indent: int): string = proc ppLayer( layer: LayerRef; db: AristoDbRef; - vGenOk: bool; + vTopOk: bool; sTabOk: bool; kMapOk: bool; pPrfOk: bool; @@ -481,7 +478,7 @@ proc ppLayer( let pfx1 = indent.toPfx(1) pfx2 = indent.toPfx(2) - nOKs = vGenOk.ord + sTabOk.ord + kMapOk.ord + pPrfOk.ord + fRppOk.ord + nOKs = vTopOk.ord + sTabOk.ord + kMapOk.ord + pPrfOk.ord + fRppOk.ord tagOk = 1 < nOKs var pfy = "" @@ -489,7 +486,9 @@ proc ppLayer( proc doPrefix(s: string; dataOk: bool): string = var rc: string if tagOk: - rc = pfy & s & (if dataOk: pfx2 else: "") + rc = pfy + if 0 < s.len: + rc &= s & (if dataOk: pfx2 else: "") pfy = pfx1 else: rc = pfy @@ -499,11 +498,8 @@ proc ppLayer( if not layer.isNil: if 2 < nOKs: result &= "".doPrefix(false) - if vGenOk: - let - tLen = layer.delta.vGen.len - info = "vGen(" & $tLen & ")" - result &= info.doPrefix(0 < tLen) & layer.delta.vGen.ppVidList + if vTopOk: + result &= "".doPrefix(true) & "vTop=" & layer.delta.vTop.ppVid if sTabOk: let tLen = layer.delta.sTab.len @@ -562,8 +558,8 @@ proc pp*(lty: LeafTie, db = AristoDbRef(nil)): string = proc pp*(vid: VertexID): string = vid.ppVid -proc pp*(vGen: openArray[VertexID]): string = - vGen.ppVidList +proc pp*(vLst: openArray[VertexID]): string = + vLst.ppVidList proc pp*(p: PayloadRef, db = AristoDbRef(nil)): string = p.ppPayload(db.orDefault) @@ -681,7 +677,7 @@ proc pp*( indent = 4; ): string = layer.ppLayer( - db, vGenOk=true, sTabOk=true, kMapOk=true, pPrfOk=true, fRppOk=true) + db, vTopOk=true, sTabOk=true, kMapOk=true, pPrfOk=true, fRppOk=true) proc pp*( layer: LayerRef; @@ -690,7 +686,7 @@ proc pp*( indent = 4; ): string = layer.ppLayer( - db, vGenOk=true, sTabOk=xTabOk, kMapOk=true, pPrfOk=true, fRppOk=true) + db, vTopOk=true, sTabOk=xTabOk, kMapOk=true, pPrfOk=true, fRppOk=true) proc pp*( layer: LayerRef; @@ -701,7 +697,7 @@ proc pp*( indent = 4; ): string = layer.ppLayer( - db, vGenOk=other, sTabOk=xTabOk, kMapOk=kMapOk, pPrfOk=other, fRppOk=other) + db, vTopOk=other, sTabOk=xTabOk, kMapOk=kMapOk, pPrfOk=other, fRppOk=other) proc pp*( diff --git a/nimbus/db/aristo/aristo_delete.nim b/nimbus/db/aristo/aristo_delete.nim index 6a59c6ba9..abb62fa33 100644 --- a/nimbus/db/aristo/aristo_delete.nim +++ b/nimbus/db/aristo/aristo_delete.nim @@ -300,13 +300,6 @@ proc delSubTreeImpl( db.layersPutVtx(VertexID(1), wp.vid, leaf) db.layersResKey(VertexID(1), wp.vid) - # Squeeze list of recycled vertex IDs - # TODO this causes a reallocation of vGen which slows down subsequent - # additions to the list because the sequence must grow which entails a - # full copy in addition to this reorg itself - around block 2.5M this - # causes significant slowdown as the vid list is >1M entries long - # See also EIP-161 which is why there are so many deletions - # db.top.final.vGen = db.vGen.vidReorg() ok() @@ -398,13 +391,6 @@ proc deleteImpl( db.layersPutVtx(VertexID(1), wp.vid, leaf) db.layersResKey(VertexID(1), wp.vid) - # Squeeze list of recycled vertex IDs - # TODO this causes a reallocation of vGen which slows down subsequent - # additions to the list because the sequence must grow which entails a - # full copy in addition to this reorg itself - around block 2.5M this - # causes significant slowdown as the vid list is >1M entries long - # See also EIP-161 which is why there are so many deletions``` - # db.top.final.vGen = db.vGen.vidReorg() ok(emptySubTreeOk) # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_delta.nim b/nimbus/db/aristo/aristo_delta.nim index c2c4a4501..ed5e3a24f 100644 --- a/nimbus/db/aristo/aristo_delta.nim +++ b/nimbus/db/aristo/aristo_delta.nim @@ -77,7 +77,7 @@ proc deltaPersistent*( let writeBatch = be.putBegFn() be.putVtxFn(writeBatch, db.balancer.sTab.pairs.toSeq) be.putKeyFn(writeBatch, db.balancer.kMap.pairs.toSeq) - be.putIdgFn(writeBatch, db.balancer.vGen) + be.putTuvFn(writeBatch, db.balancer.vTop) be.putLstFn(writeBatch, lSst) ? be.putEndFn writeBatch # Finalise write batch diff --git a/nimbus/db/aristo/aristo_delta/delta_merge.nim b/nimbus/db/aristo/aristo_delta/delta_merge.nim index 78c9504c2..042f9946d 100644 --- a/nimbus/db/aristo/aristo_delta/delta_merge.nim +++ b/nimbus/db/aristo/aristo_delta/delta_merge.nim @@ -69,7 +69,7 @@ proc deltaMerge*( src: lower.src, sTab: lower.sTab, kMap: lower.kMap, - vGen: upper.vGen) + vTop: upper.vTop) for (vid,vtx) in upper.sTab.pairs: if vtx.isValid or not newFilter.sTab.hasKey vid: diff --git a/nimbus/db/aristo/aristo_delta/delta_reverse.nim b/nimbus/db/aristo/aristo_delta/delta_reverse.nim index 7066ac3de..0946f42fd 100644 --- a/nimbus/db/aristo/aristo_delta/delta_reverse.nim +++ b/nimbus/db/aristo/aristo_delta/delta_reverse.nim @@ -33,10 +33,10 @@ proc revFilter*( # Get vid generator state on backend block: - let rc = db.getIdgUbe() + let rc = db.getTuvUbe() if rc.isOk: - rev.vGen = rc.value - elif rc.error != GetIdgNotFound: + rev.vTop = rc.value + elif rc.error != GetTuvNotFound: return err((VertexID(0), rc.error)) # Calculate reverse changes for the `sTab[]` structural table diff --git a/nimbus/db/aristo/aristo_desc.nim b/nimbus/db/aristo/aristo_desc.nim index f31bd5f4c..1a46de3d1 100644 --- a/nimbus/db/aristo/aristo_desc.nim +++ b/nimbus/db/aristo/aristo_desc.nim @@ -203,12 +203,12 @@ proc fork*( if not noTopLayer: clone.top = LayerRef.init() if not db.balancer.isNil: - clone.top.delta.vGen = db.balancer.vGen + clone.top.delta.vTop = db.balancer.vTop else: - let rc = clone.backend.getIdgFn() + let rc = clone.backend.getTuvFn() if rc.isOk: - clone.top.delta.vGen = rc.value - elif rc.error != GetIdgNotFound: + clone.top.delta.vTop = rc.value + elif rc.error != GetTuvNotFound: return err(rc.error) # Add to peer list of clones diff --git a/nimbus/db/aristo/aristo_desc/desc_backend.nim b/nimbus/db/aristo/aristo_desc/desc_backend.nim index 7736719b8..1613f4de3 100644 --- a/nimbus/db/aristo/aristo_desc/desc_backend.nim +++ b/nimbus/db/aristo/aristo_desc/desc_backend.nim @@ -29,10 +29,10 @@ type ## Generic backend database retrieval function for a single ## `Aristo DB` hash lookup value. - GetIdgFn* = - proc(): Result[seq[VertexID],AristoError] {.gcsafe, raises: [].} - ## Generic backend database retrieval function for a the ID generator - ## `Aristo DB` state record. + GetTuvFn* = + proc(): Result[VertexID,AristoError] {.gcsafe, raises: [].} + ## Generic backend database retrieval function for the top used + ## vertex ID. GetLstFn* = proc(): Result[SavedState,AristoError] @@ -63,11 +63,11 @@ type ## Generic backend database bulk storage function, `VOID_HASH_KEY` ## values indicate that records should be deleted. - PutIdgFn* = - proc(hdl: PutHdlRef; vs: openArray[VertexID]) + PutTuvFn* = + proc(hdl: PutHdlRef; vs: VertexID) {.gcsafe, raises: [].} - ## Generic backend database ID generator state storage function. This - ## function replaces the current generator state. + ## Generic backend database ID generator storage function for the + ## top used vertex ID. PutLstFn* = proc(hdl: PutHdlRef; lst: SavedState) @@ -109,13 +109,13 @@ type ## Backend interface. getVtxFn*: GetVtxFn ## Read vertex record getKeyFn*: GetKeyFn ## Read Merkle hash/key - getIdgFn*: GetIdgFn ## Read vertex ID generator state + getTuvFn*: GetTuvFn ## Read top used vertex ID getLstFn*: GetLstFn ## Read saved state putBegFn*: PutBegFn ## Start bulk store session putVtxFn*: PutVtxFn ## Bulk store vertex records putKeyFn*: PutKeyFn ## Bulk store vertex hashes - putIdgFn*: PutIdgFn ## Store ID generator state + putTuvFn*: PutTuvFn ## Store top used vertex ID putLstFn*: PutLstFn ## Store saved state putEndFn*: PutEndFn ## Commit bulk store session @@ -125,13 +125,13 @@ type proc init*(trg: var BackendObj; src: BackendObj) = trg.getVtxFn = src.getVtxFn trg.getKeyFn = src.getKeyFn - trg.getIdgFn = src.getIdgFn + trg.getTuvFn = src.getTuvFn trg.getLstFn = src.getLstFn trg.putBegFn = src.putBegFn trg.putVtxFn = src.putVtxFn trg.putKeyFn = src.putKeyFn - trg.putIdgFn = src.putIdgFn + trg.putTuvFn = src.putTuvFn trg.putLstFn = src.putLstFn trg.putEndFn = src.putEndFn diff --git a/nimbus/db/aristo/aristo_desc/desc_error.nim b/nimbus/db/aristo/aristo_desc/desc_error.nim index 7a7c5a170..b69864832 100644 --- a/nimbus/db/aristo/aristo_desc/desc_error.nim +++ b/nimbus/db/aristo/aristo_desc/desc_error.nim @@ -151,6 +151,7 @@ type CheckAnyVtxBranchLinksMissing CheckAnyVtxExtPfxMissing CheckAnyVtxLockWithoutKey + CheckAnyVTopUnset # Backend structural check `checkBE()` CheckBeVtxInvalid @@ -161,7 +162,7 @@ type CheckBeKeyMissing CheckBeKeyCantCompile CheckBeKeyMismatch - CheckBeGarbledVGen + CheckBeGarbledVTop CheckBeCacheIsDirty CheckBeCacheKeyMissing @@ -171,7 +172,7 @@ type CheckBeCacheVtxDangling CheckBeCacheKeyCantCompile CheckBeCacheKeyMismatch - CheckBeCacheGarbledVGen + CheckBeCacheGarbledVTop CheckBeFifoSrcTrgMismatch CheckBeFifoTrgNotStateRoot @@ -229,7 +230,7 @@ type GetVtxNotFound GetKeyNotFound GetFilNotFound - GetIdgNotFound + GetTuvNotFound GetLstNotFound GetFqsNotFound diff --git a/nimbus/db/aristo/aristo_desc/desc_structural.nim b/nimbus/db/aristo/aristo_desc/desc_structural.nim index 028f6f6cb..c0474ef61 100644 --- a/nimbus/db/aristo/aristo_desc/desc_structural.nim +++ b/nimbus/db/aristo/aristo_desc/desc_structural.nim @@ -104,7 +104,7 @@ type src*: HashKey ## Only needed when used as a filter sTab*: Table[VertexID,VertexRef] ## Structural vertex table kMap*: Table[VertexID,HashKey] ## Merkle hash key mapping - vGen*: seq[VertexID] ## Recycling state for vertex IDs + vTop*: VertexID ## Last used vertex ID LayerFinalRef* = ref object ## Final tables fully supersede tables on lower layers when stacked as a diff --git a/nimbus/db/aristo/aristo_get.nim b/nimbus/db/aristo/aristo_get.nim index 545939b1e..a8b4e9bcd 100644 --- a/nimbus/db/aristo/aristo_get.nim +++ b/nimbus/db/aristo/aristo_get.nim @@ -22,14 +22,14 @@ import # Public functions # ------------------------------------------------------------------------------ -proc getIdgUbe*( +proc getTuvUbe*( db: AristoDbRef; - ): Result[seq[VertexID],AristoError] = + ): Result[VertexID,AristoError] = ## Get the ID generator state from the unfiltered backened if available. let be = db.backend if not be.isNil: - return be.getIdgFn() - err(GetIdgNotFound) + return be.getTuvFn() + err(GetTuvNotFound) proc getLstUbe*( db: AristoDbRef; @@ -62,13 +62,13 @@ proc getKeyUbe*( # ------------------ -proc getIdgBE*( +proc getTuvBE*( db: AristoDbRef; - ): Result[seq[VertexID],AristoError] = + ): Result[VertexID,AristoError] = ## Get the ID generator state the `backened` layer if available. if not db.balancer.isNil: - return ok(db.balancer.vGen) - db.getIdgUbe() + return ok(db.balancer.vTop) + db.getTuvUbe() proc getVtxBE*( db: AristoDbRef; diff --git a/nimbus/db/aristo/aristo_init/init_common.nim b/nimbus/db/aristo/aristo_init/init_common.nim index ab23a2ec6..d58df711c 100644 --- a/nimbus/db/aristo/aristo_init/init_common.nim +++ b/nimbus/db/aristo/aristo_init/init_common.nim @@ -62,7 +62,7 @@ type txId: uint ## Transaction ID (for debugging) const - AdmTabIdIdg* = AdminTabID(0) ## Access key for vertex ID generator state + AdmTabIdTuv* = AdminTabID(0) ## Access key for vertex ID generator state AdmTabIdLst* = AdminTabID(2) ## Access key for last state # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_init/memory_db.nim b/nimbus/db/aristo/aristo_init/memory_db.nim index ac484f5a7..d5dc60c8f 100644 --- a/nimbus/db/aristo/aristo_init/memory_db.nim +++ b/nimbus/db/aristo/aristo_init/memory_db.nim @@ -45,7 +45,7 @@ type ## Database sTab: Table[VertexID,Blob] ## Structural vertex table making up a trie kMap: Table[VertexID,HashKey] ## Merkle hash key mapping - vGen: Option[seq[VertexID]] ## ID generator state + tUvi: Option[VertexID] ## Top used vertex ID lSst: Option[SavedState] ## Last saved state MemBackendRef* = ref object of TypedBackendRef @@ -55,7 +55,7 @@ type MemPutHdlRef = ref object of TypedPutHdlRef sTab: Table[VertexID,Blob] kMap: Table[VertexID,HashKey] - vGen: Option[seq[VertexID]] + tUvi: Option[VertexID] lSst: Option[SavedState] when extraTraceMessages: @@ -108,12 +108,12 @@ proc getKeyFn(db: MemBackendRef): GetKeyFn = return ok key err(GetKeyNotFound) -proc getIdgFn(db: MemBackendRef): GetIdgFn = +proc getTuvFn(db: MemBackendRef): GetTuvFn = result = - proc(): Result[seq[VertexID],AristoError]= - if db.mdb.vGen.isSome: - return ok db.mdb.vGen.unsafeGet - err(GetIdgNotFound) + proc(): Result[VertexID,AristoError]= + if db.mdb.tUvi.isSome: + return ok db.mdb.tUvi.unsafeGet + err(GetTuvNotFound) proc getLstFn(db: MemBackendRef): GetLstFn = result = @@ -156,12 +156,12 @@ proc putKeyFn(db: MemBackendRef): PutKeyFn = for (vid,key) in vkps: hdl.kMap[vid] = key -proc putIdgFn(db: MemBackendRef): PutIdgFn = +proc putTuvFn(db: MemBackendRef): PutTuvFn = result = - proc(hdl: PutHdlRef; vs: openArray[VertexID]) = + proc(hdl: PutHdlRef; vs: VertexID) = let hdl = hdl.getSession db if hdl.error.isNil: - hdl.vGen = some(vs.toSeq) + hdl.tUvi = some(vs) proc putLstFn(db: MemBackendRef): PutLstFn = result = @@ -197,12 +197,9 @@ proc putEndFn(db: MemBackendRef): PutEndFn = else: db.mdb.kMap.del vid - if hdl.vGen.isSome: - let vGen = hdl.vGen.unsafeGet - if vGen.len == 0: - db.mdb.vGen = none(seq[VertexID]) - else: - db.mdb.vGen = some(vGen) + let tuv = hdl.tUvi.get(otherwise = VertexID(0)) + if tuv.isValid: + db.mdb.tUvi = some(tuv) if hdl.lSst.isSome: db.mdb.lSst = hdl.lSst @@ -232,13 +229,13 @@ proc memoryBackend*(): BackendRef = db.getVtxFn = getVtxFn db db.getKeyFn = getKeyFn db - db.getIdgFn = getIdgFn db + db.getTuvFn = getTuvFn db db.getLstFn = getLstFn db db.putBegFn = putBegFn db db.putVtxFn = putVtxFn db db.putKeyFn = putKeyFn db - db.putIdgFn = putIdgFn db + db.putTuvFn = putTuvFn db db.putLstFn = putLstFn db db.putEndFn = putEndFn db @@ -287,8 +284,8 @@ iterator walk*( ## ## Non-decodable entries are stepped over while the counter `n` of the ## yield record is still incremented. - if be.mdb.vGen.isSome: - yield(AdmPfx, AdmTabIdIdg.uint64, be.mdb.vGen.unsafeGet.blobify) + if be.mdb.tUvi.isSome: + yield(AdmPfx, AdmTabIdTuv.uint64, be.mdb.tUvi.unsafeGet.blobify) if be.mdb.lSst.isSome: yield(AdmPfx, AdmTabIdLst.uint64, be.mdb.lSst.unsafeGet.blobify) diff --git a/nimbus/db/aristo/aristo_init/persistent.nim b/nimbus/db/aristo/aristo_init/persistent.nim index 13b7ed40f..7f56bbc0a 100644 --- a/nimbus/db/aristo/aristo_init/persistent.nim +++ b/nimbus/db/aristo/aristo_init/persistent.nim @@ -38,15 +38,15 @@ proc newAristoRdbDbRef( ): Result[AristoDbRef, AristoError]= let be = ? rocksDbAristoBackend(basePath) - vGen = block: - let rc = be.getIdgFn() + vTop = block: + let rc = be.getTuvFn() if rc.isErr: be.closeFn(flush = false) return err(rc.error) rc.value ok AristoDbRef( top: LayerRef( - delta: LayerDeltaRef(vGen: vGen), + delta: LayerDeltaRef(vTop: vTop), final: LayerFinalRef()), backend: be) diff --git a/nimbus/db/aristo/aristo_init/rocks_db.nim b/nimbus/db/aristo/aristo_init/rocks_db.nim index 277fdec26..683570a25 100644 --- a/nimbus/db/aristo/aristo_init/rocks_db.nim +++ b/nimbus/db/aristo/aristo_init/rocks_db.nim @@ -110,23 +110,22 @@ proc getKeyFn(db: RdbBackendRef): GetKeyFn = err(GetKeyNotFound) -proc getIdgFn(db: RdbBackendRef): GetIdgFn = +proc getTuvFn(db: RdbBackendRef): GetTuvFn = result = - proc(): Result[seq[VertexID],AristoError]= + proc(): Result[VertexID,AristoError]= # Fetch serialised data record. - let data = db.rdb.getByPfx(AdmPfx, AdmTabIdIdg.uint64).valueOr: + let data = db.rdb.getByPfx(AdmPfx, AdmTabIdTuv.uint64).valueOr: when extraTraceMessages: - trace logTxt "getIdgFn: failed", error=error[0], info=error[1] + trace logTxt "getTuvFn: failed", error=error[0], info=error[1] return err(error[0]) # Decode data record if data.len == 0: - let w = EmptyVidSeq # Must be `let` - return ok w # Compiler error with `ok(EmptyVidSeq)` + return ok VertexID(0) # Decode data record - data.deblobify seq[VertexID] + data.deblobify VertexID proc getLstFn(db: RdbBackendRef): GetLstFn = result = @@ -200,18 +199,18 @@ proc putKeyFn(db: RdbBackendRef): PutKeyFn = code: error[1], info: error[2]) -proc putIdgFn(db: RdbBackendRef): PutIdgFn = +proc putTuvFn(db: RdbBackendRef): PutTuvFn = result = - proc(hdl: PutHdlRef; vs: openArray[VertexID]) = + proc(hdl: PutHdlRef; vs: VertexID) = let hdl = hdl.getSession db if hdl.error.isNil: - let data = if 0 < vs.len: vs.blobify else: EmptyBlob - db.rdb.putByPfx(AdmPfx, @[(AdmTabIdIdg.uint64, data)]).isOkOr: - hdl.error = TypedPutHdlErrRef( - pfx: AdmPfx, - aid: AdmTabIdIdg, - code: error[1], - info: error[2]) + if vs.isValid: + db.rdb.putByPfx(AdmPfx, @[(AdmTabIdTuv.uint64, vs.blobify)]).isOkOr: + hdl.error = TypedPutHdlErrRef( + pfx: AdmPfx, + aid: AdmTabIdTuv, + code: error[1], + info: error[2]) proc putLstFn(db: RdbBackendRef): PutLstFn = result = @@ -282,13 +281,13 @@ proc rocksDbAristoBackend*(path: string): Result[BackendRef,AristoError] = db.getVtxFn = getVtxFn db db.getKeyFn = getKeyFn db - db.getIdgFn = getIdgFn db + db.getTuvFn = getTuvFn db db.getLstFn = getLstFn db db.putBegFn = putBegFn db db.putVtxFn = putVtxFn db db.putKeyFn = putKeyFn db - db.putIdgFn = putIdgFn db + db.putTuvFn = putTuvFn db db.putLstFn = putLstFn db db.putEndFn = putEndFn db diff --git a/nimbus/db/aristo/aristo_layers.nim b/nimbus/db/aristo/aristo_layers.nim index 5191f522e..327e7f543 100644 --- a/nimbus/db/aristo/aristo_layers.nim +++ b/nimbus/db/aristo/aristo_layers.nim @@ -35,8 +35,8 @@ func dirty*(db: AristoDbRef): lent HashSet[VertexID] = func pPrf*(db: AristoDbRef): lent HashSet[VertexID] = db.top.final.pPrf -func vGen*(db: AristoDbRef): lent seq[VertexID] = - db.top.delta.vGen +func vTop*(db: AristoDbRef): VertexID = + db.top.delta.vTop # ------------------------------------------------------------------------------ # Public getters/helpers @@ -190,7 +190,7 @@ func layersMergeOnto*(src: LayerRef; trg: var LayerObj) = trg.delta.sTab[vid] = vtx for (vid,key) in src.delta.kMap.pairs: trg.delta.kMap[vid] = key - trg.delta.vGen = src.delta.vGen + trg.delta.vTop = src.delta.vTop func layersCc*(db: AristoDbRef; level = high(int)): LayerRef = @@ -207,7 +207,7 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef = delta: LayerDeltaRef( sTab: layers[0].delta.sTab.dup, # explicit dup for ref values kMap: layers[0].delta.kMap, - vGen: layers[^1].delta.vGen)) + vTop: layers[^1].delta.vTop)) # Consecutively merge other layers on top for n in 1 ..< layers.len: diff --git a/nimbus/db/aristo/aristo_tx/tx_fork.nim b/nimbus/db/aristo/aristo_tx/tx_fork.nim index 74993bbfb..e19005845 100644 --- a/nimbus/db/aristo/aristo_tx/tx_fork.nim +++ b/nimbus/db/aristo/aristo_tx/tx_fork.nim @@ -66,12 +66,12 @@ proc txFork*( # Provide new empty stack layer let stackLayer = block: - let rc = db.getIdgBE() + let rc = db.getTuvBE() if rc.isOk: LayerRef( - delta: LayerDeltaRef(vGen: rc.value), + delta: LayerDeltaRef(vTop: rc.value), final: LayerFinalRef()) - elif rc.error == GetIdgNotFound: + elif rc.error == GetTuvNotFound: LayerRef.init() else: return err(rc.error) diff --git a/nimbus/db/aristo/aristo_tx/tx_frame.nim b/nimbus/db/aristo/aristo_tx/tx_frame.nim index 9f066b41a..31bc28aeb 100644 --- a/nimbus/db/aristo/aristo_tx/tx_frame.nim +++ b/nimbus/db/aristo/aristo_tx/tx_frame.nim @@ -81,10 +81,10 @@ proc txFrameBegin*(db: AristoDbRef): Result[AristoTxRef,AristoError] = if db.txFrameLevel != db.stack.len: return err(TxStackGarbled) - let vGen = db.top.delta.vGen + let vTop = db.top.delta.vTop db.stack.add db.top db.top = LayerRef( - delta: LayerDeltaRef(vGen: vGen), + delta: LayerDeltaRef(vTop: vTop), final: db.top.final.dup, txUid: db.getTxUid) diff --git a/nimbus/db/aristo/aristo_tx/tx_stow.nim b/nimbus/db/aristo/aristo_tx/tx_stow.nim index 627e90f97..19d42b4ac 100644 --- a/nimbus/db/aristo/aristo_tx/tx_stow.nim +++ b/nimbus/db/aristo/aristo_tx/tx_stow.nim @@ -14,11 +14,10 @@ {.push raises: [].} import - std/[sets, tables], + std/tables, results, ../aristo_delta/delta_merge, - ".."/[aristo_desc, aristo_get, aristo_delta, aristo_layers, aristo_hashify, - aristo_vid] + ".."/[aristo_desc, aristo_get, aristo_delta, aristo_layers, aristo_hashify] # ------------------------------------------------------------------------------ # Private functions @@ -116,20 +115,20 @@ proc txStow*( db.topMerge(rc.value).isOkOr: return err(error) - # New empty top layer (probably with `snap` proofs and `vGen` carry over) + # New empty top layer (probably with `snap` proofs and `vTop` carry over) db.top = LayerRef( delta: LayerDeltaRef(), final: final) if db.balancer.isValid: - db.top.delta.vGen = db.balancer.vGen + db.top.delta.vTop = db.balancer.vTop else: - let rc = db.getIdgUbe() + let rc = db.getTuvUbe() if rc.isOk: - db.top.delta.vGen = rc.value + db.top.delta.vTop = rc.value else: - # It is OK if there was no `Idg`. Otherwise something serious happened + # It is OK if there was no `vTop`. Otherwise something serious happened # and there is no way to recover easily. - doAssert rc.error == GetIdgNotFound + doAssert rc.error == GetTuvNotFound elif db.top.delta.sTab.len != 0 and not db.top.delta.sTab.getOrVoid(VertexID(1)).isValid: @@ -142,7 +141,7 @@ proc txStow*( # New empty top layer (probably with `snap` proofs carry over) db.top = LayerRef( - delta: LayerDeltaRef(vGen: db.vGen), + delta: LayerDeltaRef(vTop: db.vTop), final: final, txUid: db.top.txUid) ok() diff --git a/nimbus/db/aristo/aristo_vid.nim b/nimbus/db/aristo/aristo_vid.nim index 440f0e481..2baa5dc24 100644 --- a/nimbus/db/aristo/aristo_vid.nim +++ b/nimbus/db/aristo/aristo_vid.nim @@ -10,102 +10,31 @@ ## Handle vertex IDs on the layered Aristo DB delta architecture ## ============================================================= - +## {.push raises: [].} import - std/[algorithm, sequtils, typetraits], - "."/[aristo_desc, aristo_layers] + std/typetraits, + ./aristo_desc # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ proc vidFetch*(db: AristoDbRef; pristine = false): VertexID = - ## Recycle or create a new `VertexID`. Reusable vertex *ID*s are kept in a - ## list where the top entry *ID* has the property that any other *ID* larger - ## is also not used on the database. + ## Fetch next vertex ID. ## - ## The function prefers to return recycled vertex *ID*s if there are any. - ## When the argument `pristine` is set `true`, the function guarantees to - ## return a non-recycled, brand new vertex *ID* which is the preferred mode - ## when creating leaf vertices. - ## - if db.vGen.len == 0: - # Note that `VertexID(1)` is the root of the main trie - db.top.delta.vGen = @[VertexID(LEAST_FREE_VID+1)] - result = VertexID(LEAST_FREE_VID) - elif db.vGen.len == 1 or pristine: - result = db.vGen[^1] - db.top.delta.vGen[^1] = result + 1 + if db.top.delta.vTop == 0: + db.top.delta.vTop = VertexID(LEAST_FREE_VID) else: - result = db.vGen[^2] - db.top.delta.vGen[^2] = db.top.delta.vGen[^1] - db.top.delta.vGen.setLen(db.vGen.len-1) - doAssert LEAST_FREE_VID <= result.distinctBase - - -proc vidPeek*(db: AristoDbRef): VertexID = - ## Like `new()` without consuming this *ID*. It will return the *ID* that - ## would be returned by the `new()` function. - ## - case db.vGen.len: - of 0: - VertexID(LEAST_FREE_VID) - of 1: - db.vGen[^1] - else: - db.vGen[^2] - + db.top.delta.vTop.inc + db.top.delta.vTop proc vidDispose*(db: AristoDbRef; vid: VertexID) = - ## Recycle the argument `vtxID` which is useful after deleting entries from - ## the vertex table to prevent the `VertexID` type key values small. - ## - if LEAST_FREE_VID <= vid.distinctBase: - if db.vGen.len == 0: - db.top.delta.vGen = @[vid] - else: - let topID = db.vGen[^1] - # Only store smaller numbers: all numberts larger than `topID` - # are free numbers - if vid < topID: - db.top.delta.vGen[^1] = vid - db.top.delta.vGen.add topID - - -proc vidReorg*(vGen: seq[VertexID]): seq[VertexID] = - ## Return a compacted version of the argument vertex ID generator state - ## `vGen`. The function removes redundant items from the recycle queue and - ## orders it in a way so that smaller `VertexID` numbers are re-used first. - ## - # Apply heuristic test to avoid unnecessary sorting - var reOrgOk = false - if 2 < vGen.len and vGen[0] < vGen[^2]: - if vGen.len < 10: - reOrgOk = true - elif vGen[0] < vGen[1] and vGen[^3] < vGen[^2]: - reOrgOk = true - - if reOrgOk: - let lst = vGen.mapIt(uint64(it)).sorted(Descending).mapIt(VertexID(it)) - for n in 0 .. lst.len-2: - if lst[n].uint64 != lst[n+1].uint64 + 1: - # All elements of the sequence `lst[0]`..`lst[n]` are in decreasing - # order with distance 1. Only the smallest item is needed and the - # rest can be removed (as long as distance is 1.) - # - # Example: - # 7, 6, 5, 3.. => 5, 3.. => @[3..] & @[5] - # ^ - # | - # n - # - return lst[n+1 .. lst.len-1] & @[lst[n]] - # Entries decrease continuously - return @[lst[^1]] - - vGen + ## Only top vertexIDs are disposed + if vid == db.top.delta.vTop and + LEAST_FREE_VID < db.top.delta.vTop.distinctBase: + db.top.delta.vTop.dec # ------------------------------------------------------------------------------ # End diff --git a/tests/test_aristo.nim b/tests/test_aristo.nim index 6cb327b4c..62feb059a 100644 --- a/tests/test_aristo.nim +++ b/tests/test_aristo.nim @@ -75,9 +75,6 @@ proc setErrorLevel {.used.} = proc miscRunner(noisy = true) = suite "Aristo: Miscellaneous tests": - test "VertexID recyling lists": - check noisy.testVidRecycleLists() - test "Short keys and other patholgical cases": check noisy.testShortKeys() diff --git a/tests/test_aristo/test_filter.nim b/tests/test_aristo/test_filter.nim index 256c5d43a..d14742088 100644 --- a/tests/test_aristo/test_filter.nim +++ b/tests/test_aristo/test_filter.nim @@ -153,7 +153,7 @@ proc isDbEq(a, b: LayerDeltaRef; db: AristoDbRef; noisy = true): bool = if unsafeAddr(a[]) != unsafeAddr(b[]): if a.src != b.src or a.kMap.getOrVoid(VertexID 1) != b.kMap.getOrVoid(VertexID 1) or - a.vGen != b.vGen: + a.vTop != b.vTop: return false # Void entries may differ unless on physical backend diff --git a/tests/test_aristo/test_misc.nim b/tests/test_aristo/test_misc.nim index b1992aaf5..f32873269 100644 --- a/tests/test_aristo/test_misc.nim +++ b/tests/test_aristo/test_misc.nim @@ -11,149 +11,19 @@ ## Aristo (aka Patricia) DB trancoder test import - std/[sequtils, sets], eth/common, results, stew/byteutils, - stew/endians2, unittest2, ../../nimbus/db/aristo, - ../../nimbus/db/aristo/[ - aristo_check, aristo_debug, aristo_desc, aristo_blobify, aristo_layers, - aristo_vid], + ../../nimbus/db/aristo/[aristo_check, aristo_debug, aristo_desc], ../replay/xcheck, ./test_helpers -type - TesterDesc = object - prng: uint32 ## random state - -# ------------------------------------------------------------------------------ -# Private helpers -# ------------------------------------------------------------------------------ - -proc posixPrngRand(state: var uint32): byte = - ## POSIX.1-2001 example of a rand() implementation, see manual page rand(3). - state = state * 1103515245 + 12345; - let val = (state shr 16) and 32767 # mod 2^31 - (val shr 8).byte # Extract second byte - -proc rand[W: SomeInteger|VertexID](ap: var TesterDesc; T: type W): T = - var a: array[sizeof T,byte] - for n in 0 ..< sizeof T: - a[n] = ap.prng.posixPrngRand().byte - when sizeof(T) == 1: - let w = uint8.fromBytesBE(a).T - when sizeof(T) == 2: - let w = uint16.fromBytesBE(a).T - when sizeof(T) == 4: - let w = uint32.fromBytesBE(a).T - else: - let w = uint64.fromBytesBE(a).T - when T is SomeUnsignedInt: - # That way, `fromBytesBE()` can be applied to `uint` - result = w - else: - # That way the result is independent of endianness - (addr result).copyMem(unsafeAddr w, sizeof w) - -proc vidRand(td: var TesterDesc; bits = 19): VertexID = - if bits < 64: - let - mask = (1u64 shl max(1,bits)) - 1 - rval = td.rand uint64 - (rval and mask).VertexID - else: - td.rand VertexID - -proc init(T: type TesterDesc; seed: int): TesterDesc = - result.prng = (seed and 0x7fffffff).uint32 - -proc `+`(a: VertexID, b: int): VertexID = - (a.uint64 + b.uint64).VertexID - # ------------------------------------------------------------------------------ # Public test function # ------------------------------------------------------------------------------ -proc testVidRecycleLists*(noisy = true; seed = 42): bool = - ## Transcode VID lists held in `AristoDb` descriptor - ## - var td = TesterDesc.init seed - let db = AristoDbRef.init() - - # Add some randum numbers - block: - let first = td.vidRand() - db.vidDispose first - - var - expectedVids = 1 - count = 1 - # Feed some numbers used and some discaded - while expectedVids < 5 or count < 5 + expectedVids: - count.inc - let vid = td.vidRand() - expectedVids += (vid < first).ord - db.vidDispose vid - - xCheck db.vGen.len == expectedVids: - noisy.say "***", "vids=", db.vGen.len, " discarded=", count-expectedVids - - # Serialise/deserialise - block: - let dbBlob = db.vGen.blobify - - # Deserialise - let - db1 = AristoDbRef.init() - rc = dbBlob.deblobify seq[VertexID] - xCheckRc rc.error == 0 - db1.top.delta.vGen = rc.value - - xCheck db.vGen == db1.vGen - - # Make sure that recycled numbers are fetched first - let topVid = db.vGen[^1] - while 1 < db.vGen.len: - let w = db.vidFetch() - xCheck w < topVid - xCheck db.vGen.len == 1 and db.vGen[0] == topVid - - # Get some consecutive vertex IDs - for n in 0 .. 5: - let w = db.vidFetch() - xCheck w == topVid + n - xCheck db.vGen.len == 1 - - # Repeat last test after clearing the cache - db.top.delta.vGen.setLen(0) - for n in 0 .. 5: - let w = db.vidFetch() - xCheck w == VertexID(LEAST_FREE_VID) + n # VertexID(1) is default root ID - xCheck db.vGen.len == 1 - - # Recycling and re-org tests - func toVQ(a: seq[int]): seq[VertexID] = a.mapIt(VertexID(LEAST_FREE_VID+it)) - - # Heuristic prevents from re-org - xCheck @[8, 7, 3, 4, 5, 9] .toVQ.vidReorg == @[8, 7, 3, 4, 5, 9] .toVQ - xCheck @[8, 7, 6, 3, 4, 5, 9] .toVQ.vidReorg == @[8, 7, 6, 3, 4, 5, 9].toVQ - xCheck @[5, 4, 3, 7] .toVQ.vidReorg == @[5, 4, 3, 7] .toVQ - xCheck @[5] .toVQ.vidReorg == @[5] .toVQ - xCheck @[3, 5] .toVQ.vidReorg == @[3, 5] .toVQ - xCheck @[4, 5] .toVQ.vidReorg == @[4, 5] .toVQ - - # performing re-org - xCheck @[5, 7, 3, 4, 8, 9] .toVQ.vidReorg == @[5, 4, 3, 7] .toVQ - xCheck @[5, 7, 6, 3, 4, 8, 9] .toVQ.vidReorg == @[3] .toVQ - xCheck @[3, 4, 5, 7] .toVQ.vidReorg == @[5, 4, 3, 7] .toVQ - - xCheck newSeq[VertexID](0).vidReorg().len == 0 - - true - - proc testShortKeys*( noisy = true; ): bool =