Remove vid recycling feature (#2294)
This commit is contained in:
parent
cc909c99f2
commit
69a158864c
|
@ -135,26 +135,24 @@ proc blobifyTo*(vtx: VertexRef; data: var Blob): Result[void,AristoError] =
|
||||||
data &= [0xC0u8 or psLen]
|
data &= [0xC0u8 or psLen]
|
||||||
ok()
|
ok()
|
||||||
|
|
||||||
|
|
||||||
proc blobify*(vtx: VertexRef): Result[Blob, AristoError] =
|
proc blobify*(vtx: VertexRef): Result[Blob, AristoError] =
|
||||||
## Variant of `blobify()`
|
## Variant of `blobify()`
|
||||||
var data: Blob
|
var data: Blob
|
||||||
? vtx.blobifyTo data
|
? vtx.blobifyTo data
|
||||||
ok(move(data))
|
ok(move(data))
|
||||||
|
|
||||||
proc blobifyTo*(vGen: openArray[VertexID]; data: var Blob) =
|
|
||||||
## This function serialises a list of vertex IDs.
|
|
||||||
## ::
|
|
||||||
## uint64, ... -- list of IDs
|
|
||||||
## 0x7c -- marker(8)
|
|
||||||
##
|
|
||||||
for w in vGen:
|
|
||||||
data &= w.uint64.toBytesBE
|
|
||||||
data.add 0x7Cu8
|
|
||||||
|
|
||||||
proc blobify*(vGen: openArray[VertexID]): Blob =
|
proc blobifyTo*(tuv: VertexID; data: var Blob) =
|
||||||
## Variant of `blobify()`
|
## This function serialises a top used vertex ID.
|
||||||
vGen.blobifyTo result
|
data.setLen(9)
|
||||||
|
let w = tuv.uint64.toBytesBE
|
||||||
|
(addr data[0]).copyMem(unsafeAddr w[0], 8)
|
||||||
|
data[8] = 0x7Cu8
|
||||||
|
|
||||||
|
proc blobify*(tuv: VertexID): Blob =
|
||||||
|
## Variant of `blobifyTo()`
|
||||||
|
tuv.blobifyTo result
|
||||||
|
|
||||||
|
|
||||||
proc blobifyTo*(lSst: SavedState; data: var Blob) =
|
proc blobifyTo*(lSst: SavedState; data: var Blob) =
|
||||||
## Serialise a last saved state record
|
## Serialise a last saved state record
|
||||||
|
@ -315,30 +313,28 @@ proc deblobify*(
|
||||||
|
|
||||||
proc deblobifyTo*(
|
proc deblobifyTo*(
|
||||||
data: openArray[byte];
|
data: openArray[byte];
|
||||||
vGen: var seq[VertexID];
|
tuv: var VertexID;
|
||||||
): Result[void,AristoError] =
|
): Result[void,AristoError] =
|
||||||
## De-serialise the data record encoded with `blobify()` into the vertex ID
|
## De-serialise a top level vertex ID.
|
||||||
## generator argument `vGen`.
|
|
||||||
if data.len == 0:
|
if data.len == 0:
|
||||||
vGen = @[]
|
tuv = VertexID(0)
|
||||||
|
elif data.len != 9:
|
||||||
|
return err(DeblobSizeGarbled)
|
||||||
|
elif data[^1] != 0x7c:
|
||||||
|
return err(DeblobWrongType)
|
||||||
else:
|
else:
|
||||||
if (data.len mod 8) != 1:
|
tuv = (uint64.fromBytesBE data.toOpenArray(0, 7)).VertexID
|
||||||
return err(DeblobSizeGarbled)
|
|
||||||
if data[^1] != 0x7c:
|
|
||||||
return err(DeblobWrongType)
|
|
||||||
for n in 0 ..< (data.len div 8):
|
|
||||||
let w = n * 8
|
|
||||||
vGen.add (uint64.fromBytesBE data.toOpenArray(w, w+7)).VertexID
|
|
||||||
ok()
|
ok()
|
||||||
|
|
||||||
proc deblobify*(
|
proc deblobify*(
|
||||||
data: openArray[byte];
|
data: openArray[byte];
|
||||||
T: type seq[VertexID];
|
T: type VertexID;
|
||||||
): Result[T,AristoError] =
|
): Result[T,AristoError] =
|
||||||
## Variant of `deblobify()` for deserialising the vertex ID generator state
|
## Variant of `deblobify()` for deserialising a top level vertex ID.
|
||||||
var vGen: T
|
var vTop: T
|
||||||
? data.deblobifyTo vGen
|
? data.deblobifyTo vTop
|
||||||
ok move(vGen)
|
ok move(vTop)
|
||||||
|
|
||||||
|
|
||||||
proc deblobifyTo*(
|
proc deblobifyTo*(
|
||||||
data: openArray[byte];
|
data: openArray[byte];
|
||||||
|
|
|
@ -11,32 +11,18 @@
|
||||||
{.push raises: [].}
|
{.push raises: [].}
|
||||||
|
|
||||||
import
|
import
|
||||||
std/[algorithm, sequtils, sets, tables, typetraits],
|
std/[sets, tables],
|
||||||
eth/[common, trie/nibbles],
|
eth/[common, trie/nibbles],
|
||||||
|
results,
|
||||||
stew/interval_set,
|
stew/interval_set,
|
||||||
../../aristo,
|
../../aristo,
|
||||||
../aristo_walk/persistent,
|
../aristo_walk/persistent,
|
||||||
".."/[aristo_desc, aristo_get, aristo_layers, aristo_serialise]
|
".."/[aristo_desc, aristo_get, aristo_layers, aristo_serialise]
|
||||||
|
|
||||||
const
|
|
||||||
Vid2 = @[VertexID(LEAST_FREE_VID)].toHashSet
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Private helper
|
# Private helper
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
proc to(s: IntervalSetRef[VertexID,uint64]; T: type HashSet[VertexID]): T =
|
|
||||||
## Convert the argument list `s` to a set of vertex IDs as it would appear
|
|
||||||
## with a vertex generator state list.
|
|
||||||
if s.total < high(uint64):
|
|
||||||
for w in s.increasing:
|
|
||||||
if w.maxPt == high(VertexID):
|
|
||||||
result.incl w.minPt # last interval
|
|
||||||
else:
|
|
||||||
for pt in w.minPt .. w.maxPt:
|
|
||||||
if LEAST_FREE_VID <= pt.distinctBase:
|
|
||||||
result.incl pt
|
|
||||||
|
|
||||||
proc toNodeBE(
|
proc toNodeBE(
|
||||||
vtx: VertexRef; # Vertex to convert
|
vtx: VertexRef; # Vertex to convert
|
||||||
db: AristoDbRef; # Database, top layer
|
db: AristoDbRef; # Database, top layer
|
||||||
|
@ -76,17 +62,6 @@ proc toNodeBE(
|
||||||
return ok node
|
return ok node
|
||||||
return err(vid)
|
return err(vid)
|
||||||
|
|
||||||
proc vidReorgAlways(vGen: seq[VertexID]): seq[VertexID] =
|
|
||||||
## See `vidReorg()`, this one always sorts and optimises
|
|
||||||
##
|
|
||||||
if 1 < vGen.len:
|
|
||||||
let lst = vGen.mapIt(uint64(it)).sorted(Descending).mapIt(VertexID(it))
|
|
||||||
for n in 0 .. lst.len-2:
|
|
||||||
if lst[n].uint64 != lst[n+1].uint64 + 1:
|
|
||||||
return lst[n+1 .. lst.len-1] & @[lst[n]]
|
|
||||||
return @[lst[^1]]
|
|
||||||
vGen
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Public functions
|
# Public functions
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
@ -100,11 +75,11 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
|
||||||
): Result[void,(VertexID,AristoError)] =
|
): Result[void,(VertexID,AristoError)] =
|
||||||
## Make sure that each vertex has a Merkle hash and vice versa. Also check
|
## Make sure that each vertex has a Merkle hash and vice versa. Also check
|
||||||
## the vertex ID generator state.
|
## the vertex ID generator state.
|
||||||
let vids = IntervalSetRef[VertexID,uint64].init()
|
var topVidBe = VertexID(0)
|
||||||
discard vids.merge Interval[VertexID,uint64].new(
|
|
||||||
VertexID(LEAST_FREE_VID),high(VertexID))
|
|
||||||
|
|
||||||
for (vid,vtx) in T.walkVtxBe db:
|
for (vid,vtx) in T.walkVtxBe db:
|
||||||
|
if topVidBe < vid:
|
||||||
|
topVidBe = vid
|
||||||
if not vtx.isValid:
|
if not vtx.isValid:
|
||||||
return err((vid,CheckBeVtxInvalid))
|
return err((vid,CheckBeVtxInvalid))
|
||||||
let rc = db.getKeyBE vid
|
let rc = db.getKeyBE vid
|
||||||
|
@ -127,6 +102,8 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
|
||||||
return err((vid,CheckBeVtxExtPfxMissing))
|
return err((vid,CheckBeVtxExtPfxMissing))
|
||||||
|
|
||||||
for (vid,key) in T.walkKeyBe db:
|
for (vid,key) in T.walkKeyBe db:
|
||||||
|
if topVidBe < vid:
|
||||||
|
topVidBe = vid
|
||||||
if not key.isValid:
|
if not key.isValid:
|
||||||
return err((vid,CheckBeKeyInvalid))
|
return err((vid,CheckBeKeyInvalid))
|
||||||
let vtx = db.getVtxBE(vid).valueOr:
|
let vtx = db.getVtxBE(vid).valueOr:
|
||||||
|
@ -137,29 +114,32 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
|
||||||
let expected = node.digestTo(HashKey)
|
let expected = node.digestTo(HashKey)
|
||||||
if expected != key:
|
if expected != key:
|
||||||
return err((vid,CheckBeKeyMismatch))
|
return err((vid,CheckBeKeyMismatch))
|
||||||
discard vids.reduce Interval[VertexID,uint64].new(vid,vid)
|
|
||||||
|
|
||||||
# Compare calculated state against database state
|
# Compare calculated `vTop` against database state
|
||||||
block:
|
if topVidBe.isValid:
|
||||||
# Extract vertex ID generator state
|
let vidTuvBe = block:
|
||||||
let vGen = block:
|
let rc = db.getTuvBE()
|
||||||
let rc = db.getIdgBE()
|
|
||||||
if rc.isOk:
|
if rc.isOk:
|
||||||
rc.value.vidReorgAlways.toHashSet
|
rc.value
|
||||||
elif rc.error == GetIdgNotFound:
|
elif rc.error == GetTuvNotFound:
|
||||||
EmptyVidSeq.toHashSet
|
VertexID(0)
|
||||||
else:
|
else:
|
||||||
return err((VertexID(0),rc.error))
|
return err((VertexID(0),rc.error))
|
||||||
let
|
if vidTuvBe != topVidBe:
|
||||||
vGenExpected = vids.to(HashSet[VertexID])
|
# All vertices and keys between `topVidBe` and `vidTuvBe` must have
|
||||||
delta = vGenExpected -+- vGen # symmetric difference
|
# been deleted.
|
||||||
if 0 < delta.len:
|
for vid in max(topVidBe + 1, VertexID(LEAST_FREE_VID)) .. vidTuvBe:
|
||||||
# Exclude fringe case when there is a single root vertex only
|
if db.getVtxBE(vid).isOk or db.getKeyBE(vid).isOk:
|
||||||
if vGenExpected != Vid2 or 0 < vGen.len:
|
echo ">>>",
|
||||||
return err((delta.toSeq.sorted[^1],CheckBeGarbledVGen))
|
" topVidBe=", topVidBe,
|
||||||
|
" vidTuvBe=", vidTuvBe,
|
||||||
|
" vid=", vid
|
||||||
|
return err((vid,CheckBeGarbledVTop))
|
||||||
|
|
||||||
# Check top layer cache against backend
|
# Check layer cache against backend
|
||||||
if cache:
|
if cache:
|
||||||
|
var topVidCache = VertexID(0)
|
||||||
|
|
||||||
let checkKeysOk = block:
|
let checkKeysOk = block:
|
||||||
if db.dirty.len == 0:
|
if db.dirty.len == 0:
|
||||||
true
|
true
|
||||||
|
@ -170,6 +150,8 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
|
||||||
|
|
||||||
# Check structural table
|
# Check structural table
|
||||||
for (vid,vtx) in db.layersWalkVtx:
|
for (vid,vtx) in db.layersWalkVtx:
|
||||||
|
if vtx.isValid and topVidCache < vid:
|
||||||
|
topVidCache = vid
|
||||||
let key = block:
|
let key = block:
|
||||||
let rc = db.layersGetKey(vid)
|
let rc = db.layersGetKey(vid)
|
||||||
if rc.isOk:
|
if rc.isOk:
|
||||||
|
@ -179,22 +161,19 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
|
||||||
return err((vid,CheckBeCacheKeyMissing))
|
return err((vid,CheckBeCacheKeyMissing))
|
||||||
else:
|
else:
|
||||||
VOID_HASH_KEY
|
VOID_HASH_KEY
|
||||||
if vtx.isValid:
|
if not vtx.isValid:
|
||||||
# Register existing vid against backend generator state
|
|
||||||
discard vids.reduce Interval[VertexID,uint64].new(vid,vid)
|
|
||||||
else:
|
|
||||||
# Some vertex is to be deleted, the key must be empty
|
# Some vertex is to be deleted, the key must be empty
|
||||||
if checkKeysOk and key.isValid:
|
if checkKeysOk and key.isValid:
|
||||||
return err((vid,CheckBeCacheKeyNonEmpty))
|
return err((vid,CheckBeCacheKeyNonEmpty))
|
||||||
# There must be a representation on the backend DB unless in a TX
|
# There must be a representation on the backend DB unless in a TX
|
||||||
if db.getVtxBE(vid).isErr and db.stack.len == 0:
|
if db.getVtxBE(vid).isErr and db.stack.len == 0:
|
||||||
return err((vid,CheckBeCacheVidUnsynced))
|
return err((vid,CheckBeCacheVidUnsynced))
|
||||||
# Register deleted vid against backend generator state
|
|
||||||
discard vids.merge Interval[VertexID,uint64].new(vid,vid)
|
|
||||||
|
|
||||||
# Check key table
|
# Check key table
|
||||||
var list: seq[VertexID]
|
var list: seq[VertexID]
|
||||||
for (vid,key) in db.layersWalkKey:
|
for (vid,key) in db.layersWalkKey:
|
||||||
|
if key.isValid and topVidCache < vid:
|
||||||
|
topVidCache = vid
|
||||||
list.add vid
|
list.add vid
|
||||||
let vtx = db.getVtx vid
|
let vtx = db.getVtx vid
|
||||||
if db.layersGetVtx(vid).isErr and not vtx.isValid:
|
if db.layersGetVtx(vid).isErr and not vtx.isValid:
|
||||||
|
@ -209,22 +188,18 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
|
||||||
if expected != key:
|
if expected != key:
|
||||||
return err((vid,CheckBeCacheKeyMismatch))
|
return err((vid,CheckBeCacheKeyMismatch))
|
||||||
|
|
||||||
# Check vGen
|
# Check vTop
|
||||||
let
|
if topVidCache.isValid and topVidCache != db.vTop:
|
||||||
vGen = db.vGen.vidReorgAlways.toHashSet
|
# All vertices and keys between `topVidCache` and `db.vTop` must have
|
||||||
vGenExpected = vids.to(HashSet[VertexID])
|
# been deleted.
|
||||||
delta = vGenExpected -+- vGen # symmetric difference
|
for vid in max(db.vTop + 1, VertexID(LEAST_FREE_VID)) .. topVidCache:
|
||||||
if 0 < delta.len:
|
if db.layersGetVtxOrVoid(vid).isValid or
|
||||||
if vGen == Vid2 and vGenExpected.len == 0:
|
db.layersGetKeyOrVoid(vid).isValid:
|
||||||
# Fringe case when the database is empty
|
echo ">>>",
|
||||||
discard
|
" topVidCache=", topVidCache,
|
||||||
elif vGen.len == 0 and vGenExpected == Vid2:
|
" vTop=", db.vTop,
|
||||||
# Fringe case when there is a single root vertex only
|
" vid=", vid
|
||||||
discard
|
return err((db.vTop,CheckBeCacheGarbledVTop))
|
||||||
else:
|
|
||||||
let delta = delta.toSeq
|
|
||||||
if delta.len != 1 or delta[0] != VertexID(1) or VertexID(1) in vGen:
|
|
||||||
return err((delta.sorted[^1],CheckBeCacheGarbledVGen))
|
|
||||||
|
|
||||||
ok()
|
ok()
|
||||||
|
|
||||||
|
|
|
@ -11,7 +11,7 @@
|
||||||
{.push raises: [].}
|
{.push raises: [].}
|
||||||
|
|
||||||
import
|
import
|
||||||
std/[sequtils, sets],
|
std/[sequtils, sets, typetraits],
|
||||||
eth/[common, trie/nibbles],
|
eth/[common, trie/nibbles],
|
||||||
results,
|
results,
|
||||||
".."/[aristo_desc, aristo_get, aristo_layers, aristo_serialise, aristo_utils]
|
".."/[aristo_desc, aristo_get, aristo_layers, aristo_serialise, aristo_utils]
|
||||||
|
@ -92,15 +92,17 @@ proc checkTopCommon*(
|
||||||
let
|
let
|
||||||
kMapCount = db.layersWalkKey.toSeq.mapIt(it[1]).filterIt(it.isValid).len
|
kMapCount = db.layersWalkKey.toSeq.mapIt(it[1]).filterIt(it.isValid).len
|
||||||
kMapNilCount = db.layersWalkKey.toSeq.len - kMapCount
|
kMapNilCount = db.layersWalkKey.toSeq.len - kMapCount
|
||||||
vGen = db.vGen.toHashSet
|
vTop = db.vTop
|
||||||
vGenMax = if vGen.len == 0: VertexID(0) else: db.vGen[^1]
|
|
||||||
var
|
var
|
||||||
|
topVid = VertexID(0)
|
||||||
stoRoots: HashSet[VertexID]
|
stoRoots: HashSet[VertexID]
|
||||||
|
|
||||||
# Collect leafs and check deleted entries
|
# Collect leafs and check deleted entries
|
||||||
var nNilVtx = 0
|
var nNilVtx = 0
|
||||||
for (vid,vtx) in db.layersWalkVtx:
|
for (vid,vtx) in db.layersWalkVtx:
|
||||||
if vtx.isValid:
|
if vtx.isValid:
|
||||||
|
if topVid < vid:
|
||||||
|
topVid = vid
|
||||||
case vtx.vType:
|
case vtx.vType:
|
||||||
of Leaf:
|
of Leaf:
|
||||||
if vtx.lData.pType == AccountData:
|
if vtx.lData.pType == AccountData:
|
||||||
|
@ -108,7 +110,7 @@ proc checkTopCommon*(
|
||||||
if stoVid.isValid:
|
if stoVid.isValid:
|
||||||
if stoVid in stoRoots:
|
if stoVid in stoRoots:
|
||||||
return err((stoVid,CheckAnyVidSharedStorageRoot))
|
return err((stoVid,CheckAnyVidSharedStorageRoot))
|
||||||
if vGenMax.isValid and (vGenMax < stoVid or stoVid in vGen):
|
if vTop < stoVid:
|
||||||
return err((stoVid,CheckAnyVidDeadStorageRoot))
|
return err((stoVid,CheckAnyVidDeadStorageRoot))
|
||||||
stoRoots.incl stoVid
|
stoRoots.incl stoVid
|
||||||
of Branch:
|
of Branch:
|
||||||
|
@ -131,6 +133,13 @@ proc checkTopCommon*(
|
||||||
if rc.value.isValid:
|
if rc.value.isValid:
|
||||||
return err((vid,CheckAnyVtxEmptyKeyExpected))
|
return err((vid,CheckAnyVtxEmptyKeyExpected))
|
||||||
|
|
||||||
|
if vTop.distinctBase < LEAST_FREE_VID:
|
||||||
|
# Verify that all vids are below `LEAST_FREE_VID`
|
||||||
|
if topVid.distinctBase < LEAST_FREE_VID:
|
||||||
|
for (vid,key) in db.layersWalkKey:
|
||||||
|
if key.isValid and LEAST_FREE_VID <= vid.distinctBase:
|
||||||
|
return err((topVid,CheckAnyVTopUnset))
|
||||||
|
|
||||||
# If present, there are at least as many deleted hashes as there are deleted
|
# If present, there are at least as many deleted hashes as there are deleted
|
||||||
# vertices.
|
# vertices.
|
||||||
if kMapNilCount != 0 and kMapNilCount < nNilVtx:
|
if kMapNilCount != 0 and kMapNilCount < nNilVtx:
|
||||||
|
|
|
@ -17,7 +17,7 @@ import
|
||||||
stew/[byteutils, interval_set],
|
stew/[byteutils, interval_set],
|
||||||
./aristo_desc/desc_backend,
|
./aristo_desc/desc_backend,
|
||||||
./aristo_init/[memory_db, memory_only, rocks_db],
|
./aristo_init/[memory_db, memory_only, rocks_db],
|
||||||
"."/[aristo_constants, aristo_desc, aristo_hike, aristo_layers]
|
"."/[aristo_desc, aristo_hike, aristo_layers]
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Private functions
|
# Private functions
|
||||||
|
@ -150,14 +150,14 @@ func ppCodeHash(h: Hash256): string =
|
||||||
else:
|
else:
|
||||||
result &= h.data.toHex.squeeze(hex=true,ignLen=true)
|
result &= h.data.toHex.squeeze(hex=true,ignLen=true)
|
||||||
|
|
||||||
proc ppVidList(vGen: openArray[VertexID]): string =
|
proc ppVidList(vLst: openArray[VertexID]): string =
|
||||||
result = "["
|
result = "["
|
||||||
if vGen.len <= 250:
|
if vLst.len <= 250:
|
||||||
result &= vGen.mapIt(it.ppVid).join(",")
|
result &= vLst.mapIt(it.ppVid).join(",")
|
||||||
else:
|
else:
|
||||||
result &= vGen[0 .. 99].mapIt(it.ppVid).join(",")
|
result &= vLst[0 .. 99].mapIt(it.ppVid).join(",")
|
||||||
result &= ",.."
|
result &= ",.."
|
||||||
result &= vGen[^100 .. ^1].mapIt(it.ppVid).join(",")
|
result &= vLst[^100 .. ^1].mapIt(it.ppVid).join(",")
|
||||||
result &= "]"
|
result &= "]"
|
||||||
|
|
||||||
proc ppKey(key: HashKey; db: AristoDbRef; pfx = true): string =
|
proc ppKey(key: HashKey; db: AristoDbRef; pfx = true): string =
|
||||||
|
@ -404,8 +404,7 @@ proc ppFilter(
|
||||||
result &= " n/a"
|
result &= " n/a"
|
||||||
return
|
return
|
||||||
result &= pfx & "src=" & fl.src.ppKey(db)
|
result &= pfx & "src=" & fl.src.ppKey(db)
|
||||||
result &= pfx & "vGen" & pfx1 & "[" &
|
result &= pfx & "vTop=" & fl.vTop.ppVid
|
||||||
fl.vGen.mapIt(it.ppVid).join(",") & "]"
|
|
||||||
result &= pfx & "sTab" & pfx1 & "{"
|
result &= pfx & "sTab" & pfx1 & "{"
|
||||||
for n,vid in fl.sTab.sortedKeys:
|
for n,vid in fl.sTab.sortedKeys:
|
||||||
let vtx = fl.sTab.getOrVoid vid
|
let vtx = fl.sTab.getOrVoid vid
|
||||||
|
@ -426,13 +425,11 @@ proc ppBe[T](be: T; db: AristoDbRef; limit: int; indent: int): string =
|
||||||
pfx2 = indent.toPfx(2)
|
pfx2 = indent.toPfx(2)
|
||||||
result = "<" & $be.kind & ">"
|
result = "<" & $be.kind & ">"
|
||||||
var (dump,dataOk) = ("",false)
|
var (dump,dataOk) = ("",false)
|
||||||
dump &= pfx & "vGen"
|
|
||||||
block:
|
block:
|
||||||
let q = be.getIdgFn().get(otherwise = EmptyVidSeq)
|
let rc = be.getTuvFn()
|
||||||
dump &= "(" & $q.len & ")"
|
if rc.isOk:
|
||||||
if 0 < q.len:
|
dump &= pfx & "vTop=" & rc.value.ppVid
|
||||||
dataOk = true
|
dataOk = true
|
||||||
dump &= pfx1 & q.ppVidList()
|
|
||||||
block:
|
block:
|
||||||
dump &= pfx & "sTab"
|
dump &= pfx & "sTab"
|
||||||
var (n, data) = (0, "")
|
var (n, data) = (0, "")
|
||||||
|
@ -471,7 +468,7 @@ proc ppBe[T](be: T; db: AristoDbRef; limit: int; indent: int): string =
|
||||||
proc ppLayer(
|
proc ppLayer(
|
||||||
layer: LayerRef;
|
layer: LayerRef;
|
||||||
db: AristoDbRef;
|
db: AristoDbRef;
|
||||||
vGenOk: bool;
|
vTopOk: bool;
|
||||||
sTabOk: bool;
|
sTabOk: bool;
|
||||||
kMapOk: bool;
|
kMapOk: bool;
|
||||||
pPrfOk: bool;
|
pPrfOk: bool;
|
||||||
|
@ -481,7 +478,7 @@ proc ppLayer(
|
||||||
let
|
let
|
||||||
pfx1 = indent.toPfx(1)
|
pfx1 = indent.toPfx(1)
|
||||||
pfx2 = indent.toPfx(2)
|
pfx2 = indent.toPfx(2)
|
||||||
nOKs = vGenOk.ord + sTabOk.ord + kMapOk.ord + pPrfOk.ord + fRppOk.ord
|
nOKs = vTopOk.ord + sTabOk.ord + kMapOk.ord + pPrfOk.ord + fRppOk.ord
|
||||||
tagOk = 1 < nOKs
|
tagOk = 1 < nOKs
|
||||||
var
|
var
|
||||||
pfy = ""
|
pfy = ""
|
||||||
|
@ -489,7 +486,9 @@ proc ppLayer(
|
||||||
proc doPrefix(s: string; dataOk: bool): string =
|
proc doPrefix(s: string; dataOk: bool): string =
|
||||||
var rc: string
|
var rc: string
|
||||||
if tagOk:
|
if tagOk:
|
||||||
rc = pfy & s & (if dataOk: pfx2 else: "")
|
rc = pfy
|
||||||
|
if 0 < s.len:
|
||||||
|
rc &= s & (if dataOk: pfx2 else: "")
|
||||||
pfy = pfx1
|
pfy = pfx1
|
||||||
else:
|
else:
|
||||||
rc = pfy
|
rc = pfy
|
||||||
|
@ -499,11 +498,8 @@ proc ppLayer(
|
||||||
if not layer.isNil:
|
if not layer.isNil:
|
||||||
if 2 < nOKs:
|
if 2 < nOKs:
|
||||||
result &= "<layer>".doPrefix(false)
|
result &= "<layer>".doPrefix(false)
|
||||||
if vGenOk:
|
if vTopOk:
|
||||||
let
|
result &= "".doPrefix(true) & "vTop=" & layer.delta.vTop.ppVid
|
||||||
tLen = layer.delta.vGen.len
|
|
||||||
info = "vGen(" & $tLen & ")"
|
|
||||||
result &= info.doPrefix(0 < tLen) & layer.delta.vGen.ppVidList
|
|
||||||
if sTabOk:
|
if sTabOk:
|
||||||
let
|
let
|
||||||
tLen = layer.delta.sTab.len
|
tLen = layer.delta.sTab.len
|
||||||
|
@ -562,8 +558,8 @@ proc pp*(lty: LeafTie, db = AristoDbRef(nil)): string =
|
||||||
proc pp*(vid: VertexID): string =
|
proc pp*(vid: VertexID): string =
|
||||||
vid.ppVid
|
vid.ppVid
|
||||||
|
|
||||||
proc pp*(vGen: openArray[VertexID]): string =
|
proc pp*(vLst: openArray[VertexID]): string =
|
||||||
vGen.ppVidList
|
vLst.ppVidList
|
||||||
|
|
||||||
proc pp*(p: PayloadRef, db = AristoDbRef(nil)): string =
|
proc pp*(p: PayloadRef, db = AristoDbRef(nil)): string =
|
||||||
p.ppPayload(db.orDefault)
|
p.ppPayload(db.orDefault)
|
||||||
|
@ -681,7 +677,7 @@ proc pp*(
|
||||||
indent = 4;
|
indent = 4;
|
||||||
): string =
|
): string =
|
||||||
layer.ppLayer(
|
layer.ppLayer(
|
||||||
db, vGenOk=true, sTabOk=true, kMapOk=true, pPrfOk=true, fRppOk=true)
|
db, vTopOk=true, sTabOk=true, kMapOk=true, pPrfOk=true, fRppOk=true)
|
||||||
|
|
||||||
proc pp*(
|
proc pp*(
|
||||||
layer: LayerRef;
|
layer: LayerRef;
|
||||||
|
@ -690,7 +686,7 @@ proc pp*(
|
||||||
indent = 4;
|
indent = 4;
|
||||||
): string =
|
): string =
|
||||||
layer.ppLayer(
|
layer.ppLayer(
|
||||||
db, vGenOk=true, sTabOk=xTabOk, kMapOk=true, pPrfOk=true, fRppOk=true)
|
db, vTopOk=true, sTabOk=xTabOk, kMapOk=true, pPrfOk=true, fRppOk=true)
|
||||||
|
|
||||||
proc pp*(
|
proc pp*(
|
||||||
layer: LayerRef;
|
layer: LayerRef;
|
||||||
|
@ -701,7 +697,7 @@ proc pp*(
|
||||||
indent = 4;
|
indent = 4;
|
||||||
): string =
|
): string =
|
||||||
layer.ppLayer(
|
layer.ppLayer(
|
||||||
db, vGenOk=other, sTabOk=xTabOk, kMapOk=kMapOk, pPrfOk=other, fRppOk=other)
|
db, vTopOk=other, sTabOk=xTabOk, kMapOk=kMapOk, pPrfOk=other, fRppOk=other)
|
||||||
|
|
||||||
|
|
||||||
proc pp*(
|
proc pp*(
|
||||||
|
|
|
@ -300,13 +300,6 @@ proc delSubTreeImpl(
|
||||||
db.layersPutVtx(VertexID(1), wp.vid, leaf)
|
db.layersPutVtx(VertexID(1), wp.vid, leaf)
|
||||||
db.layersResKey(VertexID(1), wp.vid)
|
db.layersResKey(VertexID(1), wp.vid)
|
||||||
|
|
||||||
# Squeeze list of recycled vertex IDs
|
|
||||||
# TODO this causes a reallocation of vGen which slows down subsequent
|
|
||||||
# additions to the list because the sequence must grow which entails a
|
|
||||||
# full copy in addition to this reorg itself - around block 2.5M this
|
|
||||||
# causes significant slowdown as the vid list is >1M entries long
|
|
||||||
# See also EIP-161 which is why there are so many deletions
|
|
||||||
# db.top.final.vGen = db.vGen.vidReorg()
|
|
||||||
ok()
|
ok()
|
||||||
|
|
||||||
|
|
||||||
|
@ -398,13 +391,6 @@ proc deleteImpl(
|
||||||
db.layersPutVtx(VertexID(1), wp.vid, leaf)
|
db.layersPutVtx(VertexID(1), wp.vid, leaf)
|
||||||
db.layersResKey(VertexID(1), wp.vid)
|
db.layersResKey(VertexID(1), wp.vid)
|
||||||
|
|
||||||
# Squeeze list of recycled vertex IDs
|
|
||||||
# TODO this causes a reallocation of vGen which slows down subsequent
|
|
||||||
# additions to the list because the sequence must grow which entails a
|
|
||||||
# full copy in addition to this reorg itself - around block 2.5M this
|
|
||||||
# causes significant slowdown as the vid list is >1M entries long
|
|
||||||
# See also EIP-161 which is why there are so many deletions```
|
|
||||||
# db.top.final.vGen = db.vGen.vidReorg()
|
|
||||||
ok(emptySubTreeOk)
|
ok(emptySubTreeOk)
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
|
|
@ -77,7 +77,7 @@ proc deltaPersistent*(
|
||||||
let writeBatch = be.putBegFn()
|
let writeBatch = be.putBegFn()
|
||||||
be.putVtxFn(writeBatch, db.balancer.sTab.pairs.toSeq)
|
be.putVtxFn(writeBatch, db.balancer.sTab.pairs.toSeq)
|
||||||
be.putKeyFn(writeBatch, db.balancer.kMap.pairs.toSeq)
|
be.putKeyFn(writeBatch, db.balancer.kMap.pairs.toSeq)
|
||||||
be.putIdgFn(writeBatch, db.balancer.vGen)
|
be.putTuvFn(writeBatch, db.balancer.vTop)
|
||||||
be.putLstFn(writeBatch, lSst)
|
be.putLstFn(writeBatch, lSst)
|
||||||
? be.putEndFn writeBatch # Finalise write batch
|
? be.putEndFn writeBatch # Finalise write batch
|
||||||
|
|
||||||
|
|
|
@ -69,7 +69,7 @@ proc deltaMerge*(
|
||||||
src: lower.src,
|
src: lower.src,
|
||||||
sTab: lower.sTab,
|
sTab: lower.sTab,
|
||||||
kMap: lower.kMap,
|
kMap: lower.kMap,
|
||||||
vGen: upper.vGen)
|
vTop: upper.vTop)
|
||||||
|
|
||||||
for (vid,vtx) in upper.sTab.pairs:
|
for (vid,vtx) in upper.sTab.pairs:
|
||||||
if vtx.isValid or not newFilter.sTab.hasKey vid:
|
if vtx.isValid or not newFilter.sTab.hasKey vid:
|
||||||
|
|
|
@ -33,10 +33,10 @@ proc revFilter*(
|
||||||
|
|
||||||
# Get vid generator state on backend
|
# Get vid generator state on backend
|
||||||
block:
|
block:
|
||||||
let rc = db.getIdgUbe()
|
let rc = db.getTuvUbe()
|
||||||
if rc.isOk:
|
if rc.isOk:
|
||||||
rev.vGen = rc.value
|
rev.vTop = rc.value
|
||||||
elif rc.error != GetIdgNotFound:
|
elif rc.error != GetTuvNotFound:
|
||||||
return err((VertexID(0), rc.error))
|
return err((VertexID(0), rc.error))
|
||||||
|
|
||||||
# Calculate reverse changes for the `sTab[]` structural table
|
# Calculate reverse changes for the `sTab[]` structural table
|
||||||
|
|
|
@ -203,12 +203,12 @@ proc fork*(
|
||||||
if not noTopLayer:
|
if not noTopLayer:
|
||||||
clone.top = LayerRef.init()
|
clone.top = LayerRef.init()
|
||||||
if not db.balancer.isNil:
|
if not db.balancer.isNil:
|
||||||
clone.top.delta.vGen = db.balancer.vGen
|
clone.top.delta.vTop = db.balancer.vTop
|
||||||
else:
|
else:
|
||||||
let rc = clone.backend.getIdgFn()
|
let rc = clone.backend.getTuvFn()
|
||||||
if rc.isOk:
|
if rc.isOk:
|
||||||
clone.top.delta.vGen = rc.value
|
clone.top.delta.vTop = rc.value
|
||||||
elif rc.error != GetIdgNotFound:
|
elif rc.error != GetTuvNotFound:
|
||||||
return err(rc.error)
|
return err(rc.error)
|
||||||
|
|
||||||
# Add to peer list of clones
|
# Add to peer list of clones
|
||||||
|
|
|
@ -29,10 +29,10 @@ type
|
||||||
## Generic backend database retrieval function for a single
|
## Generic backend database retrieval function for a single
|
||||||
## `Aristo DB` hash lookup value.
|
## `Aristo DB` hash lookup value.
|
||||||
|
|
||||||
GetIdgFn* =
|
GetTuvFn* =
|
||||||
proc(): Result[seq[VertexID],AristoError] {.gcsafe, raises: [].}
|
proc(): Result[VertexID,AristoError] {.gcsafe, raises: [].}
|
||||||
## Generic backend database retrieval function for a the ID generator
|
## Generic backend database retrieval function for the top used
|
||||||
## `Aristo DB` state record.
|
## vertex ID.
|
||||||
|
|
||||||
GetLstFn* =
|
GetLstFn* =
|
||||||
proc(): Result[SavedState,AristoError]
|
proc(): Result[SavedState,AristoError]
|
||||||
|
@ -63,11 +63,11 @@ type
|
||||||
## Generic backend database bulk storage function, `VOID_HASH_KEY`
|
## Generic backend database bulk storage function, `VOID_HASH_KEY`
|
||||||
## values indicate that records should be deleted.
|
## values indicate that records should be deleted.
|
||||||
|
|
||||||
PutIdgFn* =
|
PutTuvFn* =
|
||||||
proc(hdl: PutHdlRef; vs: openArray[VertexID])
|
proc(hdl: PutHdlRef; vs: VertexID)
|
||||||
{.gcsafe, raises: [].}
|
{.gcsafe, raises: [].}
|
||||||
## Generic backend database ID generator state storage function. This
|
## Generic backend database ID generator storage function for the
|
||||||
## function replaces the current generator state.
|
## top used vertex ID.
|
||||||
|
|
||||||
PutLstFn* =
|
PutLstFn* =
|
||||||
proc(hdl: PutHdlRef; lst: SavedState)
|
proc(hdl: PutHdlRef; lst: SavedState)
|
||||||
|
@ -109,13 +109,13 @@ type
|
||||||
## Backend interface.
|
## Backend interface.
|
||||||
getVtxFn*: GetVtxFn ## Read vertex record
|
getVtxFn*: GetVtxFn ## Read vertex record
|
||||||
getKeyFn*: GetKeyFn ## Read Merkle hash/key
|
getKeyFn*: GetKeyFn ## Read Merkle hash/key
|
||||||
getIdgFn*: GetIdgFn ## Read vertex ID generator state
|
getTuvFn*: GetTuvFn ## Read top used vertex ID
|
||||||
getLstFn*: GetLstFn ## Read saved state
|
getLstFn*: GetLstFn ## Read saved state
|
||||||
|
|
||||||
putBegFn*: PutBegFn ## Start bulk store session
|
putBegFn*: PutBegFn ## Start bulk store session
|
||||||
putVtxFn*: PutVtxFn ## Bulk store vertex records
|
putVtxFn*: PutVtxFn ## Bulk store vertex records
|
||||||
putKeyFn*: PutKeyFn ## Bulk store vertex hashes
|
putKeyFn*: PutKeyFn ## Bulk store vertex hashes
|
||||||
putIdgFn*: PutIdgFn ## Store ID generator state
|
putTuvFn*: PutTuvFn ## Store top used vertex ID
|
||||||
putLstFn*: PutLstFn ## Store saved state
|
putLstFn*: PutLstFn ## Store saved state
|
||||||
putEndFn*: PutEndFn ## Commit bulk store session
|
putEndFn*: PutEndFn ## Commit bulk store session
|
||||||
|
|
||||||
|
@ -125,13 +125,13 @@ type
|
||||||
proc init*(trg: var BackendObj; src: BackendObj) =
|
proc init*(trg: var BackendObj; src: BackendObj) =
|
||||||
trg.getVtxFn = src.getVtxFn
|
trg.getVtxFn = src.getVtxFn
|
||||||
trg.getKeyFn = src.getKeyFn
|
trg.getKeyFn = src.getKeyFn
|
||||||
trg.getIdgFn = src.getIdgFn
|
trg.getTuvFn = src.getTuvFn
|
||||||
trg.getLstFn = src.getLstFn
|
trg.getLstFn = src.getLstFn
|
||||||
|
|
||||||
trg.putBegFn = src.putBegFn
|
trg.putBegFn = src.putBegFn
|
||||||
trg.putVtxFn = src.putVtxFn
|
trg.putVtxFn = src.putVtxFn
|
||||||
trg.putKeyFn = src.putKeyFn
|
trg.putKeyFn = src.putKeyFn
|
||||||
trg.putIdgFn = src.putIdgFn
|
trg.putTuvFn = src.putTuvFn
|
||||||
trg.putLstFn = src.putLstFn
|
trg.putLstFn = src.putLstFn
|
||||||
trg.putEndFn = src.putEndFn
|
trg.putEndFn = src.putEndFn
|
||||||
|
|
||||||
|
|
|
@ -151,6 +151,7 @@ type
|
||||||
CheckAnyVtxBranchLinksMissing
|
CheckAnyVtxBranchLinksMissing
|
||||||
CheckAnyVtxExtPfxMissing
|
CheckAnyVtxExtPfxMissing
|
||||||
CheckAnyVtxLockWithoutKey
|
CheckAnyVtxLockWithoutKey
|
||||||
|
CheckAnyVTopUnset
|
||||||
|
|
||||||
# Backend structural check `checkBE()`
|
# Backend structural check `checkBE()`
|
||||||
CheckBeVtxInvalid
|
CheckBeVtxInvalid
|
||||||
|
@ -161,7 +162,7 @@ type
|
||||||
CheckBeKeyMissing
|
CheckBeKeyMissing
|
||||||
CheckBeKeyCantCompile
|
CheckBeKeyCantCompile
|
||||||
CheckBeKeyMismatch
|
CheckBeKeyMismatch
|
||||||
CheckBeGarbledVGen
|
CheckBeGarbledVTop
|
||||||
|
|
||||||
CheckBeCacheIsDirty
|
CheckBeCacheIsDirty
|
||||||
CheckBeCacheKeyMissing
|
CheckBeCacheKeyMissing
|
||||||
|
@ -171,7 +172,7 @@ type
|
||||||
CheckBeCacheVtxDangling
|
CheckBeCacheVtxDangling
|
||||||
CheckBeCacheKeyCantCompile
|
CheckBeCacheKeyCantCompile
|
||||||
CheckBeCacheKeyMismatch
|
CheckBeCacheKeyMismatch
|
||||||
CheckBeCacheGarbledVGen
|
CheckBeCacheGarbledVTop
|
||||||
|
|
||||||
CheckBeFifoSrcTrgMismatch
|
CheckBeFifoSrcTrgMismatch
|
||||||
CheckBeFifoTrgNotStateRoot
|
CheckBeFifoTrgNotStateRoot
|
||||||
|
@ -229,7 +230,7 @@ type
|
||||||
GetVtxNotFound
|
GetVtxNotFound
|
||||||
GetKeyNotFound
|
GetKeyNotFound
|
||||||
GetFilNotFound
|
GetFilNotFound
|
||||||
GetIdgNotFound
|
GetTuvNotFound
|
||||||
GetLstNotFound
|
GetLstNotFound
|
||||||
GetFqsNotFound
|
GetFqsNotFound
|
||||||
|
|
||||||
|
|
|
@ -104,7 +104,7 @@ type
|
||||||
src*: HashKey ## Only needed when used as a filter
|
src*: HashKey ## Only needed when used as a filter
|
||||||
sTab*: Table[VertexID,VertexRef] ## Structural vertex table
|
sTab*: Table[VertexID,VertexRef] ## Structural vertex table
|
||||||
kMap*: Table[VertexID,HashKey] ## Merkle hash key mapping
|
kMap*: Table[VertexID,HashKey] ## Merkle hash key mapping
|
||||||
vGen*: seq[VertexID] ## Recycling state for vertex IDs
|
vTop*: VertexID ## Last used vertex ID
|
||||||
|
|
||||||
LayerFinalRef* = ref object
|
LayerFinalRef* = ref object
|
||||||
## Final tables fully supersede tables on lower layers when stacked as a
|
## Final tables fully supersede tables on lower layers when stacked as a
|
||||||
|
|
|
@ -22,14 +22,14 @@ import
|
||||||
# Public functions
|
# Public functions
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
proc getIdgUbe*(
|
proc getTuvUbe*(
|
||||||
db: AristoDbRef;
|
db: AristoDbRef;
|
||||||
): Result[seq[VertexID],AristoError] =
|
): Result[VertexID,AristoError] =
|
||||||
## Get the ID generator state from the unfiltered backened if available.
|
## Get the ID generator state from the unfiltered backened if available.
|
||||||
let be = db.backend
|
let be = db.backend
|
||||||
if not be.isNil:
|
if not be.isNil:
|
||||||
return be.getIdgFn()
|
return be.getTuvFn()
|
||||||
err(GetIdgNotFound)
|
err(GetTuvNotFound)
|
||||||
|
|
||||||
proc getLstUbe*(
|
proc getLstUbe*(
|
||||||
db: AristoDbRef;
|
db: AristoDbRef;
|
||||||
|
@ -62,13 +62,13 @@ proc getKeyUbe*(
|
||||||
|
|
||||||
# ------------------
|
# ------------------
|
||||||
|
|
||||||
proc getIdgBE*(
|
proc getTuvBE*(
|
||||||
db: AristoDbRef;
|
db: AristoDbRef;
|
||||||
): Result[seq[VertexID],AristoError] =
|
): Result[VertexID,AristoError] =
|
||||||
## Get the ID generator state the `backened` layer if available.
|
## Get the ID generator state the `backened` layer if available.
|
||||||
if not db.balancer.isNil:
|
if not db.balancer.isNil:
|
||||||
return ok(db.balancer.vGen)
|
return ok(db.balancer.vTop)
|
||||||
db.getIdgUbe()
|
db.getTuvUbe()
|
||||||
|
|
||||||
proc getVtxBE*(
|
proc getVtxBE*(
|
||||||
db: AristoDbRef;
|
db: AristoDbRef;
|
||||||
|
|
|
@ -62,7 +62,7 @@ type
|
||||||
txId: uint ## Transaction ID (for debugging)
|
txId: uint ## Transaction ID (for debugging)
|
||||||
|
|
||||||
const
|
const
|
||||||
AdmTabIdIdg* = AdminTabID(0) ## Access key for vertex ID generator state
|
AdmTabIdTuv* = AdminTabID(0) ## Access key for vertex ID generator state
|
||||||
AdmTabIdLst* = AdminTabID(2) ## Access key for last state
|
AdmTabIdLst* = AdminTabID(2) ## Access key for last state
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
|
|
@ -45,7 +45,7 @@ type
|
||||||
## Database
|
## Database
|
||||||
sTab: Table[VertexID,Blob] ## Structural vertex table making up a trie
|
sTab: Table[VertexID,Blob] ## Structural vertex table making up a trie
|
||||||
kMap: Table[VertexID,HashKey] ## Merkle hash key mapping
|
kMap: Table[VertexID,HashKey] ## Merkle hash key mapping
|
||||||
vGen: Option[seq[VertexID]] ## ID generator state
|
tUvi: Option[VertexID] ## Top used vertex ID
|
||||||
lSst: Option[SavedState] ## Last saved state
|
lSst: Option[SavedState] ## Last saved state
|
||||||
|
|
||||||
MemBackendRef* = ref object of TypedBackendRef
|
MemBackendRef* = ref object of TypedBackendRef
|
||||||
|
@ -55,7 +55,7 @@ type
|
||||||
MemPutHdlRef = ref object of TypedPutHdlRef
|
MemPutHdlRef = ref object of TypedPutHdlRef
|
||||||
sTab: Table[VertexID,Blob]
|
sTab: Table[VertexID,Blob]
|
||||||
kMap: Table[VertexID,HashKey]
|
kMap: Table[VertexID,HashKey]
|
||||||
vGen: Option[seq[VertexID]]
|
tUvi: Option[VertexID]
|
||||||
lSst: Option[SavedState]
|
lSst: Option[SavedState]
|
||||||
|
|
||||||
when extraTraceMessages:
|
when extraTraceMessages:
|
||||||
|
@ -108,12 +108,12 @@ proc getKeyFn(db: MemBackendRef): GetKeyFn =
|
||||||
return ok key
|
return ok key
|
||||||
err(GetKeyNotFound)
|
err(GetKeyNotFound)
|
||||||
|
|
||||||
proc getIdgFn(db: MemBackendRef): GetIdgFn =
|
proc getTuvFn(db: MemBackendRef): GetTuvFn =
|
||||||
result =
|
result =
|
||||||
proc(): Result[seq[VertexID],AristoError]=
|
proc(): Result[VertexID,AristoError]=
|
||||||
if db.mdb.vGen.isSome:
|
if db.mdb.tUvi.isSome:
|
||||||
return ok db.mdb.vGen.unsafeGet
|
return ok db.mdb.tUvi.unsafeGet
|
||||||
err(GetIdgNotFound)
|
err(GetTuvNotFound)
|
||||||
|
|
||||||
proc getLstFn(db: MemBackendRef): GetLstFn =
|
proc getLstFn(db: MemBackendRef): GetLstFn =
|
||||||
result =
|
result =
|
||||||
|
@ -156,12 +156,12 @@ proc putKeyFn(db: MemBackendRef): PutKeyFn =
|
||||||
for (vid,key) in vkps:
|
for (vid,key) in vkps:
|
||||||
hdl.kMap[vid] = key
|
hdl.kMap[vid] = key
|
||||||
|
|
||||||
proc putIdgFn(db: MemBackendRef): PutIdgFn =
|
proc putTuvFn(db: MemBackendRef): PutTuvFn =
|
||||||
result =
|
result =
|
||||||
proc(hdl: PutHdlRef; vs: openArray[VertexID]) =
|
proc(hdl: PutHdlRef; vs: VertexID) =
|
||||||
let hdl = hdl.getSession db
|
let hdl = hdl.getSession db
|
||||||
if hdl.error.isNil:
|
if hdl.error.isNil:
|
||||||
hdl.vGen = some(vs.toSeq)
|
hdl.tUvi = some(vs)
|
||||||
|
|
||||||
proc putLstFn(db: MemBackendRef): PutLstFn =
|
proc putLstFn(db: MemBackendRef): PutLstFn =
|
||||||
result =
|
result =
|
||||||
|
@ -197,12 +197,9 @@ proc putEndFn(db: MemBackendRef): PutEndFn =
|
||||||
else:
|
else:
|
||||||
db.mdb.kMap.del vid
|
db.mdb.kMap.del vid
|
||||||
|
|
||||||
if hdl.vGen.isSome:
|
let tuv = hdl.tUvi.get(otherwise = VertexID(0))
|
||||||
let vGen = hdl.vGen.unsafeGet
|
if tuv.isValid:
|
||||||
if vGen.len == 0:
|
db.mdb.tUvi = some(tuv)
|
||||||
db.mdb.vGen = none(seq[VertexID])
|
|
||||||
else:
|
|
||||||
db.mdb.vGen = some(vGen)
|
|
||||||
|
|
||||||
if hdl.lSst.isSome:
|
if hdl.lSst.isSome:
|
||||||
db.mdb.lSst = hdl.lSst
|
db.mdb.lSst = hdl.lSst
|
||||||
|
@ -232,13 +229,13 @@ proc memoryBackend*(): BackendRef =
|
||||||
|
|
||||||
db.getVtxFn = getVtxFn db
|
db.getVtxFn = getVtxFn db
|
||||||
db.getKeyFn = getKeyFn db
|
db.getKeyFn = getKeyFn db
|
||||||
db.getIdgFn = getIdgFn db
|
db.getTuvFn = getTuvFn db
|
||||||
db.getLstFn = getLstFn db
|
db.getLstFn = getLstFn db
|
||||||
|
|
||||||
db.putBegFn = putBegFn db
|
db.putBegFn = putBegFn db
|
||||||
db.putVtxFn = putVtxFn db
|
db.putVtxFn = putVtxFn db
|
||||||
db.putKeyFn = putKeyFn db
|
db.putKeyFn = putKeyFn db
|
||||||
db.putIdgFn = putIdgFn db
|
db.putTuvFn = putTuvFn db
|
||||||
db.putLstFn = putLstFn db
|
db.putLstFn = putLstFn db
|
||||||
db.putEndFn = putEndFn db
|
db.putEndFn = putEndFn db
|
||||||
|
|
||||||
|
@ -287,8 +284,8 @@ iterator walk*(
|
||||||
##
|
##
|
||||||
## Non-decodable entries are stepped over while the counter `n` of the
|
## Non-decodable entries are stepped over while the counter `n` of the
|
||||||
## yield record is still incremented.
|
## yield record is still incremented.
|
||||||
if be.mdb.vGen.isSome:
|
if be.mdb.tUvi.isSome:
|
||||||
yield(AdmPfx, AdmTabIdIdg.uint64, be.mdb.vGen.unsafeGet.blobify)
|
yield(AdmPfx, AdmTabIdTuv.uint64, be.mdb.tUvi.unsafeGet.blobify)
|
||||||
if be.mdb.lSst.isSome:
|
if be.mdb.lSst.isSome:
|
||||||
yield(AdmPfx, AdmTabIdLst.uint64, be.mdb.lSst.unsafeGet.blobify)
|
yield(AdmPfx, AdmTabIdLst.uint64, be.mdb.lSst.unsafeGet.blobify)
|
||||||
|
|
||||||
|
|
|
@ -38,15 +38,15 @@ proc newAristoRdbDbRef(
|
||||||
): Result[AristoDbRef, AristoError]=
|
): Result[AristoDbRef, AristoError]=
|
||||||
let
|
let
|
||||||
be = ? rocksDbAristoBackend(basePath)
|
be = ? rocksDbAristoBackend(basePath)
|
||||||
vGen = block:
|
vTop = block:
|
||||||
let rc = be.getIdgFn()
|
let rc = be.getTuvFn()
|
||||||
if rc.isErr:
|
if rc.isErr:
|
||||||
be.closeFn(flush = false)
|
be.closeFn(flush = false)
|
||||||
return err(rc.error)
|
return err(rc.error)
|
||||||
rc.value
|
rc.value
|
||||||
ok AristoDbRef(
|
ok AristoDbRef(
|
||||||
top: LayerRef(
|
top: LayerRef(
|
||||||
delta: LayerDeltaRef(vGen: vGen),
|
delta: LayerDeltaRef(vTop: vTop),
|
||||||
final: LayerFinalRef()),
|
final: LayerFinalRef()),
|
||||||
backend: be)
|
backend: be)
|
||||||
|
|
||||||
|
|
|
@ -110,23 +110,22 @@ proc getKeyFn(db: RdbBackendRef): GetKeyFn =
|
||||||
|
|
||||||
err(GetKeyNotFound)
|
err(GetKeyNotFound)
|
||||||
|
|
||||||
proc getIdgFn(db: RdbBackendRef): GetIdgFn =
|
proc getTuvFn(db: RdbBackendRef): GetTuvFn =
|
||||||
result =
|
result =
|
||||||
proc(): Result[seq[VertexID],AristoError]=
|
proc(): Result[VertexID,AristoError]=
|
||||||
|
|
||||||
# Fetch serialised data record.
|
# Fetch serialised data record.
|
||||||
let data = db.rdb.getByPfx(AdmPfx, AdmTabIdIdg.uint64).valueOr:
|
let data = db.rdb.getByPfx(AdmPfx, AdmTabIdTuv.uint64).valueOr:
|
||||||
when extraTraceMessages:
|
when extraTraceMessages:
|
||||||
trace logTxt "getIdgFn: failed", error=error[0], info=error[1]
|
trace logTxt "getTuvFn: failed", error=error[0], info=error[1]
|
||||||
return err(error[0])
|
return err(error[0])
|
||||||
|
|
||||||
# Decode data record
|
# Decode data record
|
||||||
if data.len == 0:
|
if data.len == 0:
|
||||||
let w = EmptyVidSeq # Must be `let`
|
return ok VertexID(0)
|
||||||
return ok w # Compiler error with `ok(EmptyVidSeq)`
|
|
||||||
|
|
||||||
# Decode data record
|
# Decode data record
|
||||||
data.deblobify seq[VertexID]
|
data.deblobify VertexID
|
||||||
|
|
||||||
proc getLstFn(db: RdbBackendRef): GetLstFn =
|
proc getLstFn(db: RdbBackendRef): GetLstFn =
|
||||||
result =
|
result =
|
||||||
|
@ -200,18 +199,18 @@ proc putKeyFn(db: RdbBackendRef): PutKeyFn =
|
||||||
code: error[1],
|
code: error[1],
|
||||||
info: error[2])
|
info: error[2])
|
||||||
|
|
||||||
proc putIdgFn(db: RdbBackendRef): PutIdgFn =
|
proc putTuvFn(db: RdbBackendRef): PutTuvFn =
|
||||||
result =
|
result =
|
||||||
proc(hdl: PutHdlRef; vs: openArray[VertexID]) =
|
proc(hdl: PutHdlRef; vs: VertexID) =
|
||||||
let hdl = hdl.getSession db
|
let hdl = hdl.getSession db
|
||||||
if hdl.error.isNil:
|
if hdl.error.isNil:
|
||||||
let data = if 0 < vs.len: vs.blobify else: EmptyBlob
|
if vs.isValid:
|
||||||
db.rdb.putByPfx(AdmPfx, @[(AdmTabIdIdg.uint64, data)]).isOkOr:
|
db.rdb.putByPfx(AdmPfx, @[(AdmTabIdTuv.uint64, vs.blobify)]).isOkOr:
|
||||||
hdl.error = TypedPutHdlErrRef(
|
hdl.error = TypedPutHdlErrRef(
|
||||||
pfx: AdmPfx,
|
pfx: AdmPfx,
|
||||||
aid: AdmTabIdIdg,
|
aid: AdmTabIdTuv,
|
||||||
code: error[1],
|
code: error[1],
|
||||||
info: error[2])
|
info: error[2])
|
||||||
|
|
||||||
proc putLstFn(db: RdbBackendRef): PutLstFn =
|
proc putLstFn(db: RdbBackendRef): PutLstFn =
|
||||||
result =
|
result =
|
||||||
|
@ -282,13 +281,13 @@ proc rocksDbAristoBackend*(path: string): Result[BackendRef,AristoError] =
|
||||||
|
|
||||||
db.getVtxFn = getVtxFn db
|
db.getVtxFn = getVtxFn db
|
||||||
db.getKeyFn = getKeyFn db
|
db.getKeyFn = getKeyFn db
|
||||||
db.getIdgFn = getIdgFn db
|
db.getTuvFn = getTuvFn db
|
||||||
db.getLstFn = getLstFn db
|
db.getLstFn = getLstFn db
|
||||||
|
|
||||||
db.putBegFn = putBegFn db
|
db.putBegFn = putBegFn db
|
||||||
db.putVtxFn = putVtxFn db
|
db.putVtxFn = putVtxFn db
|
||||||
db.putKeyFn = putKeyFn db
|
db.putKeyFn = putKeyFn db
|
||||||
db.putIdgFn = putIdgFn db
|
db.putTuvFn = putTuvFn db
|
||||||
db.putLstFn = putLstFn db
|
db.putLstFn = putLstFn db
|
||||||
db.putEndFn = putEndFn db
|
db.putEndFn = putEndFn db
|
||||||
|
|
||||||
|
|
|
@ -35,8 +35,8 @@ func dirty*(db: AristoDbRef): lent HashSet[VertexID] =
|
||||||
func pPrf*(db: AristoDbRef): lent HashSet[VertexID] =
|
func pPrf*(db: AristoDbRef): lent HashSet[VertexID] =
|
||||||
db.top.final.pPrf
|
db.top.final.pPrf
|
||||||
|
|
||||||
func vGen*(db: AristoDbRef): lent seq[VertexID] =
|
func vTop*(db: AristoDbRef): VertexID =
|
||||||
db.top.delta.vGen
|
db.top.delta.vTop
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Public getters/helpers
|
# Public getters/helpers
|
||||||
|
@ -190,7 +190,7 @@ func layersMergeOnto*(src: LayerRef; trg: var LayerObj) =
|
||||||
trg.delta.sTab[vid] = vtx
|
trg.delta.sTab[vid] = vtx
|
||||||
for (vid,key) in src.delta.kMap.pairs:
|
for (vid,key) in src.delta.kMap.pairs:
|
||||||
trg.delta.kMap[vid] = key
|
trg.delta.kMap[vid] = key
|
||||||
trg.delta.vGen = src.delta.vGen
|
trg.delta.vTop = src.delta.vTop
|
||||||
|
|
||||||
|
|
||||||
func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
|
func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
|
||||||
|
@ -207,7 +207,7 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
|
||||||
delta: LayerDeltaRef(
|
delta: LayerDeltaRef(
|
||||||
sTab: layers[0].delta.sTab.dup, # explicit dup for ref values
|
sTab: layers[0].delta.sTab.dup, # explicit dup for ref values
|
||||||
kMap: layers[0].delta.kMap,
|
kMap: layers[0].delta.kMap,
|
||||||
vGen: layers[^1].delta.vGen))
|
vTop: layers[^1].delta.vTop))
|
||||||
|
|
||||||
# Consecutively merge other layers on top
|
# Consecutively merge other layers on top
|
||||||
for n in 1 ..< layers.len:
|
for n in 1 ..< layers.len:
|
||||||
|
|
|
@ -66,12 +66,12 @@ proc txFork*(
|
||||||
|
|
||||||
# Provide new empty stack layer
|
# Provide new empty stack layer
|
||||||
let stackLayer = block:
|
let stackLayer = block:
|
||||||
let rc = db.getIdgBE()
|
let rc = db.getTuvBE()
|
||||||
if rc.isOk:
|
if rc.isOk:
|
||||||
LayerRef(
|
LayerRef(
|
||||||
delta: LayerDeltaRef(vGen: rc.value),
|
delta: LayerDeltaRef(vTop: rc.value),
|
||||||
final: LayerFinalRef())
|
final: LayerFinalRef())
|
||||||
elif rc.error == GetIdgNotFound:
|
elif rc.error == GetTuvNotFound:
|
||||||
LayerRef.init()
|
LayerRef.init()
|
||||||
else:
|
else:
|
||||||
return err(rc.error)
|
return err(rc.error)
|
||||||
|
|
|
@ -81,10 +81,10 @@ proc txFrameBegin*(db: AristoDbRef): Result[AristoTxRef,AristoError] =
|
||||||
if db.txFrameLevel != db.stack.len:
|
if db.txFrameLevel != db.stack.len:
|
||||||
return err(TxStackGarbled)
|
return err(TxStackGarbled)
|
||||||
|
|
||||||
let vGen = db.top.delta.vGen
|
let vTop = db.top.delta.vTop
|
||||||
db.stack.add db.top
|
db.stack.add db.top
|
||||||
db.top = LayerRef(
|
db.top = LayerRef(
|
||||||
delta: LayerDeltaRef(vGen: vGen),
|
delta: LayerDeltaRef(vTop: vTop),
|
||||||
final: db.top.final.dup,
|
final: db.top.final.dup,
|
||||||
txUid: db.getTxUid)
|
txUid: db.getTxUid)
|
||||||
|
|
||||||
|
|
|
@ -14,11 +14,10 @@
|
||||||
{.push raises: [].}
|
{.push raises: [].}
|
||||||
|
|
||||||
import
|
import
|
||||||
std/[sets, tables],
|
std/tables,
|
||||||
results,
|
results,
|
||||||
../aristo_delta/delta_merge,
|
../aristo_delta/delta_merge,
|
||||||
".."/[aristo_desc, aristo_get, aristo_delta, aristo_layers, aristo_hashify,
|
".."/[aristo_desc, aristo_get, aristo_delta, aristo_layers, aristo_hashify]
|
||||||
aristo_vid]
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Private functions
|
# Private functions
|
||||||
|
@ -116,20 +115,20 @@ proc txStow*(
|
||||||
db.topMerge(rc.value).isOkOr:
|
db.topMerge(rc.value).isOkOr:
|
||||||
return err(error)
|
return err(error)
|
||||||
|
|
||||||
# New empty top layer (probably with `snap` proofs and `vGen` carry over)
|
# New empty top layer (probably with `snap` proofs and `vTop` carry over)
|
||||||
db.top = LayerRef(
|
db.top = LayerRef(
|
||||||
delta: LayerDeltaRef(),
|
delta: LayerDeltaRef(),
|
||||||
final: final)
|
final: final)
|
||||||
if db.balancer.isValid:
|
if db.balancer.isValid:
|
||||||
db.top.delta.vGen = db.balancer.vGen
|
db.top.delta.vTop = db.balancer.vTop
|
||||||
else:
|
else:
|
||||||
let rc = db.getIdgUbe()
|
let rc = db.getTuvUbe()
|
||||||
if rc.isOk:
|
if rc.isOk:
|
||||||
db.top.delta.vGen = rc.value
|
db.top.delta.vTop = rc.value
|
||||||
else:
|
else:
|
||||||
# It is OK if there was no `Idg`. Otherwise something serious happened
|
# It is OK if there was no `vTop`. Otherwise something serious happened
|
||||||
# and there is no way to recover easily.
|
# and there is no way to recover easily.
|
||||||
doAssert rc.error == GetIdgNotFound
|
doAssert rc.error == GetTuvNotFound
|
||||||
|
|
||||||
elif db.top.delta.sTab.len != 0 and
|
elif db.top.delta.sTab.len != 0 and
|
||||||
not db.top.delta.sTab.getOrVoid(VertexID(1)).isValid:
|
not db.top.delta.sTab.getOrVoid(VertexID(1)).isValid:
|
||||||
|
@ -142,7 +141,7 @@ proc txStow*(
|
||||||
|
|
||||||
# New empty top layer (probably with `snap` proofs carry over)
|
# New empty top layer (probably with `snap` proofs carry over)
|
||||||
db.top = LayerRef(
|
db.top = LayerRef(
|
||||||
delta: LayerDeltaRef(vGen: db.vGen),
|
delta: LayerDeltaRef(vTop: db.vTop),
|
||||||
final: final,
|
final: final,
|
||||||
txUid: db.top.txUid)
|
txUid: db.top.txUid)
|
||||||
ok()
|
ok()
|
||||||
|
|
|
@ -10,102 +10,31 @@
|
||||||
|
|
||||||
## Handle vertex IDs on the layered Aristo DB delta architecture
|
## Handle vertex IDs on the layered Aristo DB delta architecture
|
||||||
## =============================================================
|
## =============================================================
|
||||||
|
##
|
||||||
{.push raises: [].}
|
{.push raises: [].}
|
||||||
|
|
||||||
import
|
import
|
||||||
std/[algorithm, sequtils, typetraits],
|
std/typetraits,
|
||||||
"."/[aristo_desc, aristo_layers]
|
./aristo_desc
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Public functions
|
# Public functions
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
proc vidFetch*(db: AristoDbRef; pristine = false): VertexID =
|
proc vidFetch*(db: AristoDbRef; pristine = false): VertexID =
|
||||||
## Recycle or create a new `VertexID`. Reusable vertex *ID*s are kept in a
|
## Fetch next vertex ID.
|
||||||
## list where the top entry *ID* has the property that any other *ID* larger
|
|
||||||
## is also not used on the database.
|
|
||||||
##
|
##
|
||||||
## The function prefers to return recycled vertex *ID*s if there are any.
|
if db.top.delta.vTop == 0:
|
||||||
## When the argument `pristine` is set `true`, the function guarantees to
|
db.top.delta.vTop = VertexID(LEAST_FREE_VID)
|
||||||
## return a non-recycled, brand new vertex *ID* which is the preferred mode
|
|
||||||
## when creating leaf vertices.
|
|
||||||
##
|
|
||||||
if db.vGen.len == 0:
|
|
||||||
# Note that `VertexID(1)` is the root of the main trie
|
|
||||||
db.top.delta.vGen = @[VertexID(LEAST_FREE_VID+1)]
|
|
||||||
result = VertexID(LEAST_FREE_VID)
|
|
||||||
elif db.vGen.len == 1 or pristine:
|
|
||||||
result = db.vGen[^1]
|
|
||||||
db.top.delta.vGen[^1] = result + 1
|
|
||||||
else:
|
else:
|
||||||
result = db.vGen[^2]
|
db.top.delta.vTop.inc
|
||||||
db.top.delta.vGen[^2] = db.top.delta.vGen[^1]
|
db.top.delta.vTop
|
||||||
db.top.delta.vGen.setLen(db.vGen.len-1)
|
|
||||||
doAssert LEAST_FREE_VID <= result.distinctBase
|
|
||||||
|
|
||||||
|
|
||||||
proc vidPeek*(db: AristoDbRef): VertexID =
|
|
||||||
## Like `new()` without consuming this *ID*. It will return the *ID* that
|
|
||||||
## would be returned by the `new()` function.
|
|
||||||
##
|
|
||||||
case db.vGen.len:
|
|
||||||
of 0:
|
|
||||||
VertexID(LEAST_FREE_VID)
|
|
||||||
of 1:
|
|
||||||
db.vGen[^1]
|
|
||||||
else:
|
|
||||||
db.vGen[^2]
|
|
||||||
|
|
||||||
|
|
||||||
proc vidDispose*(db: AristoDbRef; vid: VertexID) =
|
proc vidDispose*(db: AristoDbRef; vid: VertexID) =
|
||||||
## Recycle the argument `vtxID` which is useful after deleting entries from
|
## Only top vertexIDs are disposed
|
||||||
## the vertex table to prevent the `VertexID` type key values small.
|
if vid == db.top.delta.vTop and
|
||||||
##
|
LEAST_FREE_VID < db.top.delta.vTop.distinctBase:
|
||||||
if LEAST_FREE_VID <= vid.distinctBase:
|
db.top.delta.vTop.dec
|
||||||
if db.vGen.len == 0:
|
|
||||||
db.top.delta.vGen = @[vid]
|
|
||||||
else:
|
|
||||||
let topID = db.vGen[^1]
|
|
||||||
# Only store smaller numbers: all numberts larger than `topID`
|
|
||||||
# are free numbers
|
|
||||||
if vid < topID:
|
|
||||||
db.top.delta.vGen[^1] = vid
|
|
||||||
db.top.delta.vGen.add topID
|
|
||||||
|
|
||||||
|
|
||||||
proc vidReorg*(vGen: seq[VertexID]): seq[VertexID] =
|
|
||||||
## Return a compacted version of the argument vertex ID generator state
|
|
||||||
## `vGen`. The function removes redundant items from the recycle queue and
|
|
||||||
## orders it in a way so that smaller `VertexID` numbers are re-used first.
|
|
||||||
##
|
|
||||||
# Apply heuristic test to avoid unnecessary sorting
|
|
||||||
var reOrgOk = false
|
|
||||||
if 2 < vGen.len and vGen[0] < vGen[^2]:
|
|
||||||
if vGen.len < 10:
|
|
||||||
reOrgOk = true
|
|
||||||
elif vGen[0] < vGen[1] and vGen[^3] < vGen[^2]:
|
|
||||||
reOrgOk = true
|
|
||||||
|
|
||||||
if reOrgOk:
|
|
||||||
let lst = vGen.mapIt(uint64(it)).sorted(Descending).mapIt(VertexID(it))
|
|
||||||
for n in 0 .. lst.len-2:
|
|
||||||
if lst[n].uint64 != lst[n+1].uint64 + 1:
|
|
||||||
# All elements of the sequence `lst[0]`..`lst[n]` are in decreasing
|
|
||||||
# order with distance 1. Only the smallest item is needed and the
|
|
||||||
# rest can be removed (as long as distance is 1.)
|
|
||||||
#
|
|
||||||
# Example:
|
|
||||||
# 7, 6, 5, 3.. => 5, 3.. => @[3..] & @[5]
|
|
||||||
# ^
|
|
||||||
# |
|
|
||||||
# n
|
|
||||||
#
|
|
||||||
return lst[n+1 .. lst.len-1] & @[lst[n]]
|
|
||||||
# Entries decrease continuously
|
|
||||||
return @[lst[^1]]
|
|
||||||
|
|
||||||
vGen
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# End
|
# End
|
||||||
|
|
|
@ -75,9 +75,6 @@ proc setErrorLevel {.used.} =
|
||||||
proc miscRunner(noisy = true) =
|
proc miscRunner(noisy = true) =
|
||||||
suite "Aristo: Miscellaneous tests":
|
suite "Aristo: Miscellaneous tests":
|
||||||
|
|
||||||
test "VertexID recyling lists":
|
|
||||||
check noisy.testVidRecycleLists()
|
|
||||||
|
|
||||||
test "Short keys and other patholgical cases":
|
test "Short keys and other patholgical cases":
|
||||||
check noisy.testShortKeys()
|
check noisy.testShortKeys()
|
||||||
|
|
||||||
|
|
|
@ -153,7 +153,7 @@ proc isDbEq(a, b: LayerDeltaRef; db: AristoDbRef; noisy = true): bool =
|
||||||
if unsafeAddr(a[]) != unsafeAddr(b[]):
|
if unsafeAddr(a[]) != unsafeAddr(b[]):
|
||||||
if a.src != b.src or
|
if a.src != b.src or
|
||||||
a.kMap.getOrVoid(VertexID 1) != b.kMap.getOrVoid(VertexID 1) or
|
a.kMap.getOrVoid(VertexID 1) != b.kMap.getOrVoid(VertexID 1) or
|
||||||
a.vGen != b.vGen:
|
a.vTop != b.vTop:
|
||||||
return false
|
return false
|
||||||
|
|
||||||
# Void entries may differ unless on physical backend
|
# Void entries may differ unless on physical backend
|
||||||
|
|
|
@ -11,149 +11,19 @@
|
||||||
## Aristo (aka Patricia) DB trancoder test
|
## Aristo (aka Patricia) DB trancoder test
|
||||||
|
|
||||||
import
|
import
|
||||||
std/[sequtils, sets],
|
|
||||||
eth/common,
|
eth/common,
|
||||||
results,
|
results,
|
||||||
stew/byteutils,
|
stew/byteutils,
|
||||||
stew/endians2,
|
|
||||||
unittest2,
|
unittest2,
|
||||||
../../nimbus/db/aristo,
|
../../nimbus/db/aristo,
|
||||||
../../nimbus/db/aristo/[
|
../../nimbus/db/aristo/[aristo_check, aristo_debug, aristo_desc],
|
||||||
aristo_check, aristo_debug, aristo_desc, aristo_blobify, aristo_layers,
|
|
||||||
aristo_vid],
|
|
||||||
../replay/xcheck,
|
../replay/xcheck,
|
||||||
./test_helpers
|
./test_helpers
|
||||||
|
|
||||||
type
|
|
||||||
TesterDesc = object
|
|
||||||
prng: uint32 ## random state
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
|
||||||
# Private helpers
|
|
||||||
# ------------------------------------------------------------------------------
|
|
||||||
|
|
||||||
proc posixPrngRand(state: var uint32): byte =
|
|
||||||
## POSIX.1-2001 example of a rand() implementation, see manual page rand(3).
|
|
||||||
state = state * 1103515245 + 12345;
|
|
||||||
let val = (state shr 16) and 32767 # mod 2^31
|
|
||||||
(val shr 8).byte # Extract second byte
|
|
||||||
|
|
||||||
proc rand[W: SomeInteger|VertexID](ap: var TesterDesc; T: type W): T =
|
|
||||||
var a: array[sizeof T,byte]
|
|
||||||
for n in 0 ..< sizeof T:
|
|
||||||
a[n] = ap.prng.posixPrngRand().byte
|
|
||||||
when sizeof(T) == 1:
|
|
||||||
let w = uint8.fromBytesBE(a).T
|
|
||||||
when sizeof(T) == 2:
|
|
||||||
let w = uint16.fromBytesBE(a).T
|
|
||||||
when sizeof(T) == 4:
|
|
||||||
let w = uint32.fromBytesBE(a).T
|
|
||||||
else:
|
|
||||||
let w = uint64.fromBytesBE(a).T
|
|
||||||
when T is SomeUnsignedInt:
|
|
||||||
# That way, `fromBytesBE()` can be applied to `uint`
|
|
||||||
result = w
|
|
||||||
else:
|
|
||||||
# That way the result is independent of endianness
|
|
||||||
(addr result).copyMem(unsafeAddr w, sizeof w)
|
|
||||||
|
|
||||||
proc vidRand(td: var TesterDesc; bits = 19): VertexID =
|
|
||||||
if bits < 64:
|
|
||||||
let
|
|
||||||
mask = (1u64 shl max(1,bits)) - 1
|
|
||||||
rval = td.rand uint64
|
|
||||||
(rval and mask).VertexID
|
|
||||||
else:
|
|
||||||
td.rand VertexID
|
|
||||||
|
|
||||||
proc init(T: type TesterDesc; seed: int): TesterDesc =
|
|
||||||
result.prng = (seed and 0x7fffffff).uint32
|
|
||||||
|
|
||||||
proc `+`(a: VertexID, b: int): VertexID =
|
|
||||||
(a.uint64 + b.uint64).VertexID
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Public test function
|
# Public test function
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
proc testVidRecycleLists*(noisy = true; seed = 42): bool =
|
|
||||||
## Transcode VID lists held in `AristoDb` descriptor
|
|
||||||
##
|
|
||||||
var td = TesterDesc.init seed
|
|
||||||
let db = AristoDbRef.init()
|
|
||||||
|
|
||||||
# Add some randum numbers
|
|
||||||
block:
|
|
||||||
let first = td.vidRand()
|
|
||||||
db.vidDispose first
|
|
||||||
|
|
||||||
var
|
|
||||||
expectedVids = 1
|
|
||||||
count = 1
|
|
||||||
# Feed some numbers used and some discaded
|
|
||||||
while expectedVids < 5 or count < 5 + expectedVids:
|
|
||||||
count.inc
|
|
||||||
let vid = td.vidRand()
|
|
||||||
expectedVids += (vid < first).ord
|
|
||||||
db.vidDispose vid
|
|
||||||
|
|
||||||
xCheck db.vGen.len == expectedVids:
|
|
||||||
noisy.say "***", "vids=", db.vGen.len, " discarded=", count-expectedVids
|
|
||||||
|
|
||||||
# Serialise/deserialise
|
|
||||||
block:
|
|
||||||
let dbBlob = db.vGen.blobify
|
|
||||||
|
|
||||||
# Deserialise
|
|
||||||
let
|
|
||||||
db1 = AristoDbRef.init()
|
|
||||||
rc = dbBlob.deblobify seq[VertexID]
|
|
||||||
xCheckRc rc.error == 0
|
|
||||||
db1.top.delta.vGen = rc.value
|
|
||||||
|
|
||||||
xCheck db.vGen == db1.vGen
|
|
||||||
|
|
||||||
# Make sure that recycled numbers are fetched first
|
|
||||||
let topVid = db.vGen[^1]
|
|
||||||
while 1 < db.vGen.len:
|
|
||||||
let w = db.vidFetch()
|
|
||||||
xCheck w < topVid
|
|
||||||
xCheck db.vGen.len == 1 and db.vGen[0] == topVid
|
|
||||||
|
|
||||||
# Get some consecutive vertex IDs
|
|
||||||
for n in 0 .. 5:
|
|
||||||
let w = db.vidFetch()
|
|
||||||
xCheck w == topVid + n
|
|
||||||
xCheck db.vGen.len == 1
|
|
||||||
|
|
||||||
# Repeat last test after clearing the cache
|
|
||||||
db.top.delta.vGen.setLen(0)
|
|
||||||
for n in 0 .. 5:
|
|
||||||
let w = db.vidFetch()
|
|
||||||
xCheck w == VertexID(LEAST_FREE_VID) + n # VertexID(1) is default root ID
|
|
||||||
xCheck db.vGen.len == 1
|
|
||||||
|
|
||||||
# Recycling and re-org tests
|
|
||||||
func toVQ(a: seq[int]): seq[VertexID] = a.mapIt(VertexID(LEAST_FREE_VID+it))
|
|
||||||
|
|
||||||
# Heuristic prevents from re-org
|
|
||||||
xCheck @[8, 7, 3, 4, 5, 9] .toVQ.vidReorg == @[8, 7, 3, 4, 5, 9] .toVQ
|
|
||||||
xCheck @[8, 7, 6, 3, 4, 5, 9] .toVQ.vidReorg == @[8, 7, 6, 3, 4, 5, 9].toVQ
|
|
||||||
xCheck @[5, 4, 3, 7] .toVQ.vidReorg == @[5, 4, 3, 7] .toVQ
|
|
||||||
xCheck @[5] .toVQ.vidReorg == @[5] .toVQ
|
|
||||||
xCheck @[3, 5] .toVQ.vidReorg == @[3, 5] .toVQ
|
|
||||||
xCheck @[4, 5] .toVQ.vidReorg == @[4, 5] .toVQ
|
|
||||||
|
|
||||||
# performing re-org
|
|
||||||
xCheck @[5, 7, 3, 4, 8, 9] .toVQ.vidReorg == @[5, 4, 3, 7] .toVQ
|
|
||||||
xCheck @[5, 7, 6, 3, 4, 8, 9] .toVQ.vidReorg == @[3] .toVQ
|
|
||||||
xCheck @[3, 4, 5, 7] .toVQ.vidReorg == @[5, 4, 3, 7] .toVQ
|
|
||||||
|
|
||||||
xCheck newSeq[VertexID](0).vidReorg().len == 0
|
|
||||||
|
|
||||||
true
|
|
||||||
|
|
||||||
|
|
||||||
proc testShortKeys*(
|
proc testShortKeys*(
|
||||||
noisy = true;
|
noisy = true;
|
||||||
): bool =
|
): bool =
|
||||||
|
|
Loading…
Reference in New Issue