status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Aristo db remove over engineered object type (#2027) 

* CoreDb: update test suite

* Aristo: Simplify reverse key map

why:
  The reverse key map `pAmk: (root,key) -> {vid,..}` as been simplified to
  `pAmk: key -> {vid,..}` as the state `root` domain argument is not used,
  anymore

* Aristo: Remove `HashLabel` object type and replace it by `HashKey`

why:
  The `HashLabel` object attaches a root hash to a hash key. This is
  nowhere used, anymore.

* Fix copyright
This commit is contained in:
Jordan Hrycaj 2024-02-14 19:11:59 +00:00 committed by GitHub
parent d5a54f66ee
commit 1b4a43c140
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 282 additions and 331 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
nimbus/db/aristo/aristo_check/check_be.nim
View File

@ -155,7 +155,7 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
# Check structural table # Check structural table
for (vid,vtx) in db.layersWalkVtx: for (vid,vtx) in db.layersWalkVtx:
let lbl = db.layersGetLabel(vid).valueOr: let key = db.layersGetKey(vid).valueOr:
# A `kMap[]` entry must exist. # A `kMap[]` entry must exist.
return err((vid,CheckBeCacheKeyMissing)) return err((vid,CheckBeCacheKeyMissing))
if vtx.isValid: if vtx.isValid:
@ -163,7 +163,7 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
discard vids.reduce Interval[VertexID,uint64].new(vid,vid) discard vids.reduce Interval[VertexID,uint64].new(vid,vid)
else: else:
# Some vertex is to be deleted, the key must be empty # Some vertex is to be deleted, the key must be empty
if lbl.isValid: if key.isValid:
return err((vid,CheckBeCacheKeyNonEmpty)) return err((vid,CheckBeCacheKeyNonEmpty))
# There must be a representation on the backend DB unless in a TX # There must be a representation on the backend DB unless in a TX
if db.getVtxBE(vid).isErr and db.stack.len == 0: if db.getVtxBE(vid).isErr and db.stack.len == 0:
@ -186,19 +186,19 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
# Check key table # Check key table
var list: seq[VertexID] var list: seq[VertexID]
for (vid,lbl) in db.layersWalkLabel: for (vid,key) in db.layersWalkKey:
list.add vid list.add vid
let vtx = db.getVtx vid let vtx = db.getVtx vid
if db.layersGetVtx(vid).isErr and not vtx.isValid: if db.layersGetVtx(vid).isErr and not vtx.isValid:
return err((vid,CheckBeCacheKeyDangling)) return err((vid,CheckBeCacheKeyDangling))
if not lbl.isValid or relax: if not key.isValid or relax:
continue continue
if not vtx.isValid: if not vtx.isValid:
return err((vid,CheckBeCacheVtxDangling)) return err((vid,CheckBeCacheVtxDangling))
let node = vtx.toNode(db).valueOr: # compile cache first let node = vtx.toNode(db).valueOr: # compile cache first
return err((vid,CheckBeCacheKeyCantCompile)) return err((vid,CheckBeCacheKeyCantCompile))
let expected = node.digestTo(HashKey) let expected = node.digestTo(HashKey)
if expected != lbl.key: if expected != key:
return err((vid,CheckBeCacheKeyMismatch)) return err((vid,CheckBeCacheKeyMismatch))
# Check vGen # Check vGen

54
nimbus/db/aristo/aristo_check/check_top.nim
View File

@ -1,5 +1,5 @@
# nimbus-eth1 # nimbus-eth1
# Copyright (c) 2023 Status Research & Development GmbH # Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of # Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0) # http://www.apache.org/licenses/LICENSE-2.0)
@ -27,42 +27,42 @@ proc checkTopStrict*(
# vertex ID (i.e. not deleted). # vertex ID (i.e. not deleted).
var zeroKeys: HashSet[VertexID] var zeroKeys: HashSet[VertexID]
for (vid,vtx) in db.layersWalkVtx: for (vid,vtx) in db.layersWalkVtx:
let lbl = db.layersGetLabelOrVoid vid let key = db.layersGetKeyOrVoid vid
if not vtx.isValid: if not vtx.isValid:
if lbl.isValid: if key.isValid:
return err((vid,CheckStkVtxKeyMismatch)) return err((vid,CheckStkVtxKeyMismatch))
else: # Empty key flags key is for update else: # Empty key flags key is for update
zeroKeys.incl vid zeroKeys.incl vid
elif lbl.isValid: elif key.isValid:
# So `vtx` and `lbl` exist # So `vtx` and `key` exist
let node = vtx.toNode(db).valueOr: let node = vtx.toNode(db).valueOr:
return err((vid,CheckStkVtxIncomplete)) return err((vid,CheckStkVtxIncomplete))
if lbl.key != node.digestTo(HashKey): if key != node.digestTo(HashKey):
return err((vid,CheckStkVtxKeyMismatch)) return err((vid,CheckStkVtxKeyMismatch))
let revVids = db.layersGetLebalOrVoid lbl let revVids = db.layersGetYekOrVoid key
if not revVids.isValid: if not revVids.isValid:
return err((vid,CheckStkRevKeyMissing)) return err((vid,CheckStkRevKeyMissing))
if vid notin revVids: if vid notin revVids:
return err((vid,CheckStkRevKeyMismatch)) return err((vid,CheckStkRevKeyMismatch))
elif not db.dirty or db.layersGetLabel(vid).isErr: elif not db.dirty or db.layersGetKey(vid).isErr:
# So `vtx` exists but not `lbl`, so cache is supposed dirty and the # So `vtx` exists but not `key`, so cache is supposed dirty and the
# vertex has a zero entry. # vertex has a zero entry.
return err((vid,CheckStkVtxKeyMissing)) return err((vid,CheckStkVtxKeyMissing))
else: # Empty key flags key is for update else: # Empty key flags key is for update
zeroKeys.incl vid zeroKeys.incl vid
for (vid,key) in db.layersWalkLabel: for (vid,key) in db.layersWalkKey:
if not key.isValid and vid notin zeroKeys: if not key.isValid and vid notin zeroKeys:
if not db.getVtx(vid).isValid: if not db.getVtx(vid).isValid:
return err((vid,CheckStkKeyStrayZeroEntry)) return err((vid,CheckStkKeyStrayZeroEntry))
let let
pAmkVtxCount = db.layersWalkLebal.toSeq.mapIt(it[1]).foldl(a + b.len, 0) pAmkVtxCount = db.layersWalkYek.toSeq.mapIt(it[1]).foldl(a + b.len, 0)
sTabVtxCount = db.layersWalkVtx.toSeq.mapIt(it[1]).filterIt(it.isValid).len sTabVtxCount = db.layersWalkVtx.toSeq.mapIt(it[1]).filterIt(it.isValid).len
# Non-zero values mist sum up the same # Non-zero values mist sum up the same
@ -73,7 +73,7 @@ proc checkTopStrict*(
proc checkTopProofMode*( proc checkTopProofMode*(
db: AristoDbRef; # Database, top layer db: AristoDbRef; # Database, top layer
): Result[void,(VertexID,AristoError)] = ): Result[void,(VertexID,AristoError)] =
if 0 < db.pPrf.len: if 0 < db.pPrf.len:
for vid in db.pPrf: for vid in db.pPrf:
@ -82,28 +82,28 @@ proc checkTopProofMode*(
let node = vtx.toNode(db).valueOr: let node = vtx.toNode(db).valueOr:
return err((vid,CheckRlxVtxIncomplete)) return err((vid,CheckRlxVtxIncomplete))
let lbl = db.layersGetlabelOrVoid vid let key = db.layersGetKeyOrVoid vid
if not lbl.isValid: if not key.isValid:
return err((vid,CheckRlxVtxKeyMissing)) return err((vid,CheckRlxVtxKeyMissing))
if lbl.key != node.digestTo(HashKey): if key != node.digestTo(HashKey):
return err((vid,CheckRlxVtxKeyMismatch)) return err((vid,CheckRlxVtxKeyMismatch))
let revVids = db.layersGetLebalOrVoid lbl let revVids = db.layersGetYekOrVoid key
if not revVids.isValid: if not revVids.isValid:
return err((vid,CheckRlxRevKeyMissing)) return err((vid,CheckRlxRevKeyMissing))
if vid notin revVids: if vid notin revVids:
return err((vid,CheckRlxRevKeyMismatch)) return err((vid,CheckRlxRevKeyMismatch))
else: else:
for (vid,lbl) in db.layersWalkLabel: for (vid,key) in db.layersWalkKey:
if lbl.isValid: # Otherwise to be deleted if key.isValid: # Otherwise to be deleted
let vtx = db.getVtx vid let vtx = db.getVtx vid
if vtx.isValid: if vtx.isValid:
let node = vtx.toNode(db).valueOr: let node = vtx.toNode(db).valueOr:
continue continue
if lbl.key != node.digestTo(HashKey): if key != node.digestTo(HashKey):
return err((vid,CheckRlxVtxKeyMismatch)) return err((vid,CheckRlxVtxKeyMismatch))
let revVids = db.layersGetLebalOrVoid lbl let revVids = db.layersGetYekOrVoid key
if not revVids.isValid: if not revVids.isValid:
return err((vid,CheckRlxRevKeyMissing)) return err((vid,CheckRlxRevKeyMissing))
if vid notin revVids: if vid notin revVids:
@ -115,8 +115,8 @@ proc checkTopCommon*(
): Result[void,(VertexID,AristoError)] = ): Result[void,(VertexID,AristoError)] =
# Some `kMap[]` entries may ne void indicating backend deletion # Some `kMap[]` entries may ne void indicating backend deletion
let let
kMapCount = db.layersWalkLabel.toSeq.mapIt(it[1]).filterIt(it.isValid).len kMapCount = db.layersWalkKey.toSeq.mapIt(it[1]).filterIt(it.isValid).len
kMapNilCount = db.layersWalkLabel.toSeq.len - kMapCount kMapNilCount = db.layersWalkKey.toSeq.len - kMapCount
# Collect leafs and check deleted entries # Collect leafs and check deleted entries
var nNilVtx = 0 var nNilVtx = 0
@ -139,7 +139,7 @@ proc checkTopCommon*(
return err((vid,CheckAnyVtxExtPfxMissing)) return err((vid,CheckAnyVtxExtPfxMissing))
else: else:
nNilVtx.inc nNilVtx.inc
let rc = db.layersGetLabel vid let rc = db.layersGetKey vid
if rc.isErr: if rc.isErr:
return err((vid,CheckAnyVtxEmptyKeyMissing)) return err((vid,CheckAnyVtxEmptyKeyMissing))
if rc.value.isValid: if rc.value.isValid:
@ -150,12 +150,12 @@ proc checkTopCommon*(
if kMapNilCount != 0 and kMapNilCount < nNilVtx: if kMapNilCount != 0 and kMapNilCount < nNilVtx:
return err((VertexID(0),CheckAnyVtxEmptyKeyMismatch)) return err((VertexID(0),CheckAnyVtxEmptyKeyMismatch))
let pAmkVtxCount = db.layersWalkLebal.toSeq.mapIt(it[1]).foldl(a + b.len, 0) let pAmkVtxCount = db.layersWalkYek.toSeq.mapIt(it[1]).foldl(a + b.len, 0)
if pAmkVtxCount != kMapCount: if pAmkVtxCount != kMapCount:
var knownKeys: HashSet[VertexID] var knownKeys: HashSet[VertexID]
for (key,vids) in db.layersWalkLebal: for (key,vids) in db.layersWalkYek:
for vid in vids: for vid in vids:
if db.layersGetLabel(vid).isErr: if db.layersGetKey(vid).isErr:
return err((vid,CheckAnyRevVtxMissing)) return err((vid,CheckAnyRevVtxMissing))
if vid in knownKeys: if vid in knownKeys:
return err((vid,CheckAnyRevVtxDup)) return err((vid,CheckAnyRevVtxDup))
@ -163,7 +163,7 @@ proc checkTopCommon*(
return err((VertexID(0),CheckAnyRevCountMismatch)) # should not apply(!) return err((VertexID(0),CheckAnyRevCountMismatch)) # should not apply(!)
for vid in db.pPrf: for vid in db.pPrf:
if db.layersGetLabel(vid).isErr: if db.layersGetKey(vid).isErr:
return err((vid,CheckAnyVtxLockWithoutKey)) return err((vid,CheckAnyVtxLockWithoutKey))
ok() ok()

3
nimbus/db/aristo/aristo_constants.nim
View File

@ -34,9 +34,6 @@ const
VOID_HASH_KEY* = HashKey() VOID_HASH_KEY* = HashKey()
## Void equivalent for Merkle hash value ## Void equivalent for Merkle hash value
VOID_HASH_LABEL* = HashLabel(key: VOID_HASH_KEY)
## Void equivalent for Merkle hash value
VOID_PATH_ID* = PathID() VOID_PATH_ID* = PathID()
## Void equivalent for Merkle hash value ## Void equivalent for Merkle hash value

190
nimbus/db/aristo/aristo_debug.nim
View File

@ -27,30 +27,24 @@ import
proc orDefault(db: AristoDbRef): AristoDbRef = proc orDefault(db: AristoDbRef): AristoDbRef =
if db.isNil: AristoDbRef(top: LayerRef.init()) else: db if db.isNil: AristoDbRef(top: LayerRef.init()) else: db
proc del(xMap: var VidsByLabelTab; lbl: HashLabel; vid: VertexID) = proc del(xMap: var VidsByKeyTab; key: HashKey; vid: VertexID) =
# Update `xMap` # Update `xMap`
var vidsLen = -1 var vidsLen = -1
xMap.withValue(lbl,value): xMap.withValue(key,value):
value[].excl vid value[].excl vid
vidsLen = value[].len vidsLen = value[].len
if vidsLen == 0: if vidsLen == 0:
xMap.del lbl xMap.del key
proc del(xMap: var VidsByLabelTab; lbl: HashLabel; vids: HashSet[VertexID]) = proc del(xMap: var VidsByKeyTab; key: HashKey; vids: HashSet[VertexID]) =
for vid in vids: for vid in vids:
xMap.del(lbl, vid) xMap.del(key, vid)
proc add(xMap: var VidsByLabelTab; lbl: HashLabel; vid: VertexID) = proc add(xMap: var VidsByKeyTab; key: Hashkey; vid: VertexID) =
xMap.withValue(lbl,value): xMap.withValue(key,value):
value[].incl vid value[].incl vid
do: # else if not found do: # else if not found
xMap[lbl] = @[vid].toHashSet xMap[key] = @[vid].toHashSet
func cmp(a, b: HashLabel): int =
if a.root != b.root:
a.root.cmp b.root
else:
a.key.cmp b.key
# -------------------------- # --------------------------
@ -63,22 +57,13 @@ proc toHexLsb(w: int8): string =
proc sortedKeys(lTab: Table[LeafTie,VertexID]): seq[LeafTie] = proc sortedKeys(lTab: Table[LeafTie,VertexID]): seq[LeafTie] =
lTab.keys.toSeq.sorted(cmp = proc(a,b: LeafTie): int = cmp(a,b)) lTab.keys.toSeq.sorted(cmp = proc(a,b: LeafTie): int = cmp(a,b))
proc sortedKeys(kMap: Table[VertexID,HashLabel]): seq[VertexID] = proc sortedKeys[T](tab: Table[VertexID,T]): seq[VertexID] =
kMap.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID) tab.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID)
proc sortedKeys(kMap: Table[VertexID,HashKey]): seq[VertexID] =
kMap.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID)
proc sortedKeys(sTab: Table[VertexID,VertexRef]): seq[VertexID] =
sTab.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID)
proc sortedKeys(pPrf: HashSet[VertexID]): seq[VertexID] = proc sortedKeys(pPrf: HashSet[VertexID]): seq[VertexID] =
pPrf.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID) pPrf.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID)
proc sortedKeys(pAmk: Table[HashLabel,VertexID]): seq[HashLabel] = proc sortedKeys[T](pAmk: Table[HashKey,T]): seq[HashKey] =
pAmk.keys.toSeq.sorted cmp
proc sortedKeys(pAmk: VidsByLabelTab): seq[HashLabel] =
pAmk.keys.toSeq.sorted cmp pAmk.keys.toSeq.sorted cmp
@ -118,42 +103,40 @@ proc stripZeros(a: string; toExp = false): string =
elif 2 < n: elif 2 < n:
result &= "↑" & $n result &= "↑" & $n
proc vidCode(lbl: HashLabel, db: AristoDbRef): uint64 = proc vidCode(key: HashKey, db: AristoDbRef): uint64 =
if lbl.isValid: if key.isValid:
block: block:
let vids = db.layersGetLebalOrVoid lbl let vids = db.layersGetYekOrVoid key
if vids.isValid: if vids.isValid:
db.xMap.del(lbl, vids) db.xMap.del(key, vids)
return vids.sortedKeys[0].uint64 return vids.sortedKeys[0].uint64
block: block:
let vids = db.xMap.getOrVoid lbl let vids = db.xMap.getOrVoid key
if vids.isValid: if vids.isValid:
return vids.sortedKeys[0].uint64 return vids.sortedKeys[0].uint64
# --------------------- # ---------------------
proc ppLabelOk( proc ppKeyOk(
db: AristoDbRef; db: AristoDbRef;
root: VertexID;
key: HashKey; key: HashKey;
vid: VertexID; vid: VertexID;
): string = ): string =
if key.isValid and vid.isValid: if key.isValid and vid.isValid:
let let
lbl = HashLabel(root: root, key: key) vids = db.layersGetYekOrVoid key
vids = db.layersGetLebalOrVoid lbl
if vids.isValid: if vids.isValid:
db.xMap.del(lbl, vids) db.xMap.del(key, vids)
if vid notin vids: if vid notin vids:
result = "(!)" result = "(!)"
return return
block: block:
let vids = db.xMap.getOrVoid lbl let vids = db.xMap.getOrVoid key
if vids.isValid: if vids.isValid:
if vid notin vids: if vid notin vids:
result = "(!)" result = "(!)"
return return
db.xMap.add(lbl,vid) db.xMap.add(key,vid)
proc ppVid(vid: VertexID; pfx = true): string = proc ppVid(vid: VertexID; pfx = true): string =
if pfx: if pfx:
@ -215,16 +198,15 @@ proc ppVidList(vGen: openArray[VertexID]): string =
#proc ppVidList(vGen: HashSet[VertexID]): string = #proc ppVidList(vGen: HashSet[VertexID]): string =
# "{" & vGen.sortedKeys.mapIt(it.ppVid).join(",") & "}" # "{" & vGen.sortedKeys.mapIt(it.ppVid).join(",") & "}"
proc ppKey(key: HashKey; db: AristoDbRef; root: VertexID; pfx = true): string = proc ppKey(key: HashKey; db: AristoDbRef; pfx = true): string =
proc getVids(): tuple[vids: HashSet[VertexID], xMapTag: string] = proc getVids(): tuple[vids: HashSet[VertexID], xMapTag: string] =
let lbl = HashLabel(root: root, key: key)
block: block:
let vids = db.layersGetLebalOrVoid lbl let vids = db.layersGetYekOrVoid key
if vids.isValid: if vids.isValid:
db.xMap.del(lbl, vids) db.xMap.del(key, vids)
return (vids, "") return (vids, "")
block: block:
let vids = db.xMap.getOrVoid lbl let vids = db.xMap.getOrVoid key
if vids.isValid: if vids.isValid:
return (vids, "+") return (vids, "+")
if pfx: if pfx:
@ -247,13 +229,6 @@ proc ppKey(key: HashKey; db: AristoDbRef; root: VertexID; pfx = true): string =
return return
result &= @key.toHex.squeeze(hex=true,ignLen=true) & tag result &= @key.toHex.squeeze(hex=true,ignLen=true) & tag
proc ppLabel(lbl: HashLabel; db: AristoDbRef): string =
if lbl.isValid:
"%" & ($lbl.root.toHex).stripZeros &
":" & lbl.key.ppKey(db, lbl.root, pfx=false)
else:
"%ø"
proc ppLeafTie(lty: LeafTie, db: AristoDbRef): string = proc ppLeafTie(lty: LeafTie, db: AristoDbRef): string =
let pfx = lty.path.to(NibblesSeq) let pfx = lty.path.to(NibblesSeq)
"@" & lty.root.ppVid(pfx=false) & ":" & "@" & lty.root.ppVid(pfx=false) & ":" &
@ -288,7 +263,7 @@ proc ppVtx(nd: VertexRef, db: AristoDbRef, vid: VertexID): string =
else: else:
if not vid.isValid or vid in db.pPrf: if not vid.isValid or vid in db.pPrf:
result = ["L(", "X(", "B("][nd.vType.ord] result = ["L(", "X(", "B("][nd.vType.ord]
elif db.layersGetLabel(vid).isOk: elif db.layersGetKey(vid).isOk:
result = ["l(", "x(", "b("][nd.vType.ord] result = ["l(", "x(", "b("][nd.vType.ord]
else: else:
result = ["ł(", "€(", "þ("][nd.vType.ord] result = ["ł(", "€(", "þ("][nd.vType.ord]
@ -330,8 +305,8 @@ proc ppPPrf(pPrf: HashSet[VertexID]): string =
proc ppXMap*( proc ppXMap*(
db: AristoDbRef; db: AristoDbRef;
kMap: Table[VertexID,HashLabel]; kMap: Table[VertexID,HashKey];
pAmk: VidsByLabelTab; pAmk: VidsByKeyTab;
indent: int; indent: int;
): string = ): string =
@ -351,24 +326,24 @@ proc ppXMap*(
if 1 < w.len: if 1 < w.len:
multi = multi + w multi = multi + w
# Vertex IDs without forward mapping `kMap: VertexID -> HashLabel` # Vertex IDs without forward mapping `kMap: VertexID -> HashKey`
var revOnly: Table[VertexID,HashLabel] var revOnly: Table[VertexID,HashKey]
for (lbl,vids) in pAmk.pairs: for (key,vids) in pAmk.pairs:
for vid in vids: for vid in vids:
if not kMap.hasKey vid: if not kMap.hasKey vid:
revOnly[vid] = lbl revOnly[vid] = key
let revKeys = let revKeys =
revOnly.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID) revOnly.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID)
proc ppNtry(n: uint64): string = proc ppNtry(n: uint64): string =
var s = VertexID(n).ppVid var s = VertexID(n).ppVid
let lbl = kMap.getOrVoid VertexID(n) let key = kMap.getOrVoid VertexID(n)
if lbl.isValid: if key.isValid:
let vids = pAmk.getOrVoid lbl let vids = pAmk.getOrVoid key
if VertexID(n) notin vids or 1 < vids.len: if VertexID(n) notin vids or 1 < vids.len:
s = "(" & s & "," & lbl.key.ppKey(db,lbl.root) s = "(" & s & "," & key.ppKey(db)
elif lbl.key.len < 32: elif key.len < 32:
s &= "[#" & $lbl.key.len & "]" s &= "[#" & $key.len & "]"
else: else:
s &= "£ø" s &= "£ø"
if s[0] == '(': if s[0] == '(':
@ -378,10 +353,10 @@ proc ppXMap*(
result = "{" result = "{"
# Extra reverse lookups # Extra reverse lookups
if 0 < revKeys.len: if 0 < revKeys.len:
proc ppRevlabel(vid: VertexID): string = proc ppRevKey(vid: VertexID): string =
"(ø," & revOnly.getOrVoid(vid).ppLabel(db) & ")" "(ø," & revOnly.getOrVoid(vid).ppkey(db) & ")"
var (i, r) = (0, revKeys[0]) var (i, r) = (0, revKeys[0])
result &= revKeys[0].ppRevlabel result &= revKeys[0].ppRevKey
for n in 1 ..< revKeys.len: for n in 1 ..< revKeys.len:
let vid = revKeys[n] let vid = revKeys[n]
r.inc r.inc
@ -391,24 +366,24 @@ proc ppXMap*(
result &= pfx result &= pfx
else: else:
result &= ".. " result &= ".. "
result &= revKeys[n-1].ppRevlabel result &= revKeys[n-1].ppRevKey
result &= pfx & vid.ppRevlabel result &= pfx & vid.ppRevKey
(i, r) = (n, vid) (i, r) = (n, vid)
if i < revKeys.len - 1: if i < revKeys.len - 1:
if i+1 != revKeys.len - 1: if i+1 != revKeys.len - 1:
result &= ".. " result &= ".. "
else: else:
result &= pfx result &= pfx
result &= revKeys[^1].ppRevlabel result &= revKeys[^1].ppRevKey
# Forward lookups # Forward lookups
var cache: seq[(uint64,uint64,bool)] var cache: seq[(uint64,uint64,bool)]
for vid in kMap.sortedKeys: for vid in kMap.sortedKeys:
let lbl = kMap.getOrVoid vid let key = kMap.getOrVoid vid
if lbl.isValid: if key.isValid:
cache.add (vid.uint64, lbl.vidCode(db), vid in multi) cache.add (vid.uint64, key.vidCode(db), vid in multi)
let vids = pAmk.getOrVoid lbl let vids = pAmk.getOrVoid key
if (0 < vids.len and vid notin vids) or lbl.key.len < 32: if (0 < vids.len and vid notin vids) or key.len < 32:
cache[^1][2] = true cache[^1][2] = true
else: else:
cache.add (vid.uint64, 0u64, true) cache.add (vid.uint64, 0u64, true)
@ -445,7 +420,6 @@ proc ppXMap*(
proc ppFilter( proc ppFilter(
fl: FilterRef; fl: FilterRef;
db: AristoDbRef; db: AristoDbRef;
root: VertexID;
indent: int; indent: int;
): string = ): string =
## Walk over filter tables ## Walk over filter tables
@ -458,8 +432,8 @@ proc ppFilter(
result &= " n/a" result &= " n/a"
return return
result &= pfx & "fid=" & fl.fid.ppFid result &= pfx & "fid=" & fl.fid.ppFid
result &= pfx & "src=" & fl.src.to(HashKey).ppKey(db,root) result &= pfx & "src=" & fl.src.to(HashKey).ppKey(db)
result &= pfx & "trg=" & fl.trg.to(HashKey).ppKey(db,root) result &= pfx & "trg=" & fl.trg.to(HashKey).ppKey(db)
result &= pfx & "vGen" & pfx1 & "[" & result &= pfx & "vGen" & pfx1 & "[" &
fl.vGen.mapIt(it.ppVid).join(",") & "]" fl.vGen.mapIt(it.ppVid).join(",") & "]"
result &= pfx & "sTab" & pfx1 & "{" result &= pfx & "sTab" & pfx1 & "{"
@ -471,10 +445,10 @@ proc ppFilter(
for n,vid in fl.kMap.sortedKeys: for n,vid in fl.kMap.sortedKeys:
let key = fl.kMap.getOrVoid vid let key = fl.kMap.getOrVoid vid
if 0 < n: result &= pfx2 if 0 < n: result &= pfx2
result &= $(1+n) & "(" & vid.ppVid & "," & key.ppKey(db,root) & ")" result &= $(1+n) & "(" & vid.ppVid & "," & key.ppKey(db) & ")"
result &= "}" result &= "}"
proc ppBe[T](be: T; db: AristoDbRef; root: VertexID; indent: int): string = proc ppBe[T](be: T; db: AristoDbRef; indent: int): string =
## Walk over backend tables ## Walk over backend tables
let let
pfx = indent.toPfx pfx = indent.toPfx
@ -497,7 +471,7 @@ proc ppBe[T](be: T; db: AristoDbRef; root: VertexID; indent: int): string =
for (vid,key) in be.walkKey: for (vid,key) in be.walkKey:
if 0 < n: result &= pfx2 if 0 < n: result &= pfx2
n.inc n.inc
result &= $n & "(" & vid.ppVid & "," & key.ppKey(db,root) & ")" result &= $n & "(" & vid.ppVid & "," & key.ppKey(db) & ")"
result &= "}" result &= "}"
proc ppLayer( proc ppLayer(
@ -577,13 +551,10 @@ proc pp*(w: Hash256): string =
w.data.toHex.squeeze(hex=true,ignLen=true) w.data.toHex.squeeze(hex=true,ignLen=true)
proc pp*(w: HashKey; sig: MerkleSignRef): string = proc pp*(w: HashKey; sig: MerkleSignRef): string =
w.ppKey(sig.db, sig.root) w.ppKey(sig.db)
proc pp*(w: HashKey; db = AristoDbRef(nil); root = VertexID(1)): string = proc pp*(w: HashKey; db = AristoDbRef(nil)): string =
w.ppKey(db.orDefault, root) w.ppKey(db.orDefault)
proc pp*(lbl: HashLabel, db = AristoDbRef(nil)): string =
lbl.ppLabel(db.orDefault)
proc pp*(lty: LeafTie, db = AristoDbRef(nil)): string = proc pp*(lty: LeafTie, db = AristoDbRef(nil)): string =
lty.ppLeafTie(db.orDefault) lty.ppLeafTie(db.orDefault)
@ -615,7 +586,7 @@ proc pp*(p: PayloadRef, db = AristoDbRef(nil)): string =
proc pp*(nd: VertexRef, db = AristoDbRef(nil)): string = proc pp*(nd: VertexRef, db = AristoDbRef(nil)): string =
nd.ppVtx(db.orDefault, VertexID(0)) nd.ppVtx(db.orDefault, VertexID(0))
proc pp*(nd: NodeRef; root: VertexID; db: AristoDbRef): string = proc pp*(nd: NodeRef; db: AristoDbRef): string =
if not nd.isValid: if not nd.isValid:
result = "n/a" result = "n/a"
elif nd.error != AristoError(0): elif nd.error != AristoError(0):
@ -628,21 +599,21 @@ proc pp*(nd: NodeRef; root: VertexID; db: AristoDbRef): string =
of Extension: of Extension:
result &= $nd.ePfx.ppPathPfx & "," & nd.eVid.ppVid & "," result &= $nd.ePfx.ppPathPfx & "," & nd.eVid.ppVid & ","
result &= nd.key[0].ppKey(db,root) result &= nd.key[0].ppKey(db)
result &= db.ppLabelOk(root, nd.key[0], nd.eVid) result &= db.ppKeyOk(nd.key[0], nd.eVid)
of Branch: of Branch:
result &= "[" result &= "["
for n in 0..15: for n in 0..15:
if nd.bVid[n].isValid or nd.key[n].isValid: if nd.bVid[n].isValid or nd.key[n].isValid:
result &= nd.bVid[n].ppVid result &= nd.bVid[n].ppVid
result &= db.ppLabelOk(root, nd.key[n], nd.bVid[n]) & "," result &= db.ppKeyOk(nd.key[n], nd.bVid[n]) & ","
result[^1] = ']' result[^1] = ']'
result &= ",[" result &= ",["
for n in 0..15: for n in 0..15:
if nd.bVid[n].isValid or nd.key[n].isValid: if nd.bVid[n].isValid or nd.key[n].isValid:
result &= nd.key[n].ppKey(db,root) result &= nd.key[n].ppKey(db)
result &= "," result &= ","
result[^1] = ']' result[^1] = ']'
result &= ")" result &= ")"
@ -680,11 +651,11 @@ proc pp*(leg: Leg; db = AristoDbRef(nil)): string =
let db = db.orDefault() let db = db.orDefault()
result = "(" & leg.wp.vid.ppVid & "," result = "(" & leg.wp.vid.ppVid & ","
block: block:
let lbl = db.layersGetLabelOrVoid leg.wp.vid let key = db.layersGetKeyOrVoid leg.wp.vid
if not lbl.isValid: if not key.isValid:
result &= "ø" result &= "ø"
elif leg.wp.vid notin db.layersGetLebalOrVoid lbl: elif leg.wp.vid notin db.layersGetYekOrVoid key:
result &= lbl.ppLabel(db) result &= key.ppKey(db)
result &= "," result &= ","
if 0 <= leg.nibble: if 0 <= leg.nibble:
result &= $leg.nibble.ppNibble result &= $leg.nibble.ppNibble
@ -704,32 +675,32 @@ proc pp*(hike: Hike; db = AristoDbRef(nil); indent = 4): string =
result &= pfx & "(" & hike.tail.ppPathPfx & ")" result &= pfx & "(" & hike.tail.ppPathPfx & ")"
result &= "]" result &= "]"
proc pp*(kMap: Table[VertexID,Hashlabel]; indent = 4): string = proc pp*(kMap: Table[VertexID,HashKey]; indent = 4): string =
let db = AristoDbRef(nil).orDefault let db = AristoDbRef(nil).orDefault
"{" & kMap.sortedKeys "{" & kMap.sortedKeys
.mapIt((it, kMap.getOrVoid it)) .mapIt((it, kMap.getOrVoid it))
.mapIt("(" & it[0].ppVid & "," & it[1].ppLabel(db) & ")") .mapIt("(" & it[0].ppVid & "," & it[1].ppKey(db) & ")")
.join("," & indent.toPfx(1)) & "}" .join("," & indent.toPfx(1)) & "}"
proc pp*(kMap: Table[VertexID,Hashlabel]; db: AristoDbRef; indent = 4): string = proc pp*(kMap: Table[VertexID,HashKey]; db: AristoDbRef; indent = 4): string =
db.ppXMap(kMap, db.layersCc.delta.pAmk, indent) db.ppXMap(kMap, db.layersCc.delta.pAmk, indent)
proc pp*( proc pp*(
pAmk: Table[HashLabel,VertexID]; pAmk: Table[HashKey,VertexID];
db = AristoDbRef(nil); db = AristoDbRef(nil);
indent = 4; indent = 4;
): string = ): string =
let db = db.orDefault let db = db.orDefault
"{" & pAmk.sortedkeys "{" & pAmk.sortedkeys
.mapIt((it, pAmk.getOrVoid it)) .mapIt((it, pAmk.getOrVoid it))
.mapIt("(" & it[0].ppLabel(db) & "," & it[1].ppVid & ")") .mapIt("(" & it[0].ppKey(db) & "," & it[1].ppVid & ")")
.join("," & indent.toPfx(1)) & "}" .join("," & indent.toPfx(1)) & "}"
proc pp*(pAmk: VidsByLabelTab; db = AristoDbRef(nil); indent = 4): string = proc pp*(pAmk: VidsByKeyTab; db = AristoDbRef(nil); indent = 4): string =
let db = db.orDefault let db = db.orDefault
"{" & pAmk.sortedkeys "{" & pAmk.sortedkeys
.mapIt((it, pAmk.getOrVoid it)) .mapIt((it, pAmk.getOrVoid it))
.mapIt("(" & it[0].ppLabel(db) & "," & it[1].ppVids & ")") .mapIt("(" & it[0].ppKey(db) & "," & it[1].ppVids & ")")
.join("," & indent.toPfx(1)) & "}" .join("," & indent.toPfx(1)) & "}"
# --------------------- # ---------------------
@ -792,29 +763,26 @@ proc pp*(
proc pp*( proc pp*(
filter: FilterRef; filter: FilterRef;
db = AristoDbRef(nil); db = AristoDbRef(nil);
root = VertexID(1);
indent = 4; indent = 4;
): string = ): string =
filter.ppFilter(db.orDefault(), root, indent) filter.ppFilter(db.orDefault(), indent)
proc pp*( proc pp*(
be: BackendRef; be: BackendRef;
db: AristoDbRef; db: AristoDbRef;
root = VertexID(1);
indent = 4; indent = 4;
): string = ): string =
result = db.roFilter.ppFilter(db, root, indent+1) & indent.toPfx result = db.roFilter.ppFilter(db, indent+1) & indent.toPfx
case be.kind: case be.kind:
of BackendMemory: of BackendMemory:
result &= be.MemBackendRef.ppBe(db, root, indent+1) result &= be.MemBackendRef.ppBe(db, indent+1)
of BackendRocksDB: of BackendRocksDB:
result &= be.RdbBackendRef.ppBe(db, root, indent+1) result &= be.RdbBackendRef.ppBe(db, indent+1)
of BackendVoid: of BackendVoid:
result &= "<NoBackend>" result &= "<NoBackend>"
proc pp*( proc pp*(
db: AristoDbRef; db: AristoDbRef;
root = VertexID(1);
indent = 4; indent = 4;
backendOk = false; backendOk = false;
filterOk = true; filterOk = true;
@ -842,13 +810,13 @@ proc pp*(
if backendOk: if backendOk:
result &= db.backend.pp(db) result &= db.backend.pp(db)
elif filterOk: elif filterOk:
result &= db.roFilter.ppFilter(db, root, indent+1) result &= db.roFilter.ppFilter(db, indent+1)
proc pp*(sdb: MerkleSignRef; indent = 4): string = proc pp*(sdb: MerkleSignRef; indent = 4): string =
"count=" & $sdb.count & "count=" & $sdb.count &
" root=" & sdb.root.pp & " root=" & sdb.root.pp &
" error=" & $sdb.error & " error=" & $sdb.error &
"\n db\n " & sdb.db.pp(root=sdb.root, indent=indent+1) "\n db\n " & sdb.db.pp(indent=indent+1)
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# End # End

4
nimbus/db/aristo/aristo_delete.nim
View File

@ -67,7 +67,7 @@ proc nullifyKey(
vid: VertexID; # Vertex IDs to clear vid: VertexID; # Vertex IDs to clear
) = ) =
# Register for void hash (to be recompiled) # Register for void hash (to be recompiled)
db.layersResLabel vid db.layersResKey vid
proc disposeOfVtx( proc disposeOfVtx(
db: AristoDbRef; # Database, top layer db: AristoDbRef; # Database, top layer
@ -75,7 +75,7 @@ proc disposeOfVtx(
) = ) =
# Remove entry # Remove entry
db.layersResVtx vid db.layersResVtx vid
db.layersResLabel vid db.layersResKey vid
db.vidDispose vid # Recycle ID db.vidDispose vid # Recycle ID
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------

8
nimbus/db/aristo/aristo_desc.nim
View File

@ -78,7 +78,7 @@ type
dudes: DudesRef ## Related DB descriptors dudes: DudesRef ## Related DB descriptors
# Debugging data below, might go away in future # Debugging data below, might go away in future
xMap*: VidsByLabelTab ## For pretty printing, extends `pAmk` xMap*: VidsByKeyTab ## For pretty printing, extends `pAmk`
AristoDbAction* = proc(db: AristoDbRef) {.gcsafe, raises: [].} AristoDbAction* = proc(db: AristoDbRef) {.gcsafe, raises: [].}
## Generic call back function/closure. ## Generic call back function/closure.
@ -90,9 +90,6 @@ type
func getOrVoid*[W](tab: Table[W,VertexRef]; w: W): VertexRef = func getOrVoid*[W](tab: Table[W,VertexRef]; w: W): VertexRef =
tab.getOrDefault(w, VertexRef(nil)) tab.getOrDefault(w, VertexRef(nil))
func getOrVoid*[W](tab: Table[W,HashLabel]; w: W): HashLabel =
tab.getOrDefault(w, VOID_HASH_LABEL)
func getOrVoid*[W](tab: Table[W,NodeRef]; w: W): NodeRef = func getOrVoid*[W](tab: Table[W,NodeRef]; w: W): NodeRef =
tab.getOrDefault(w, NodeRef(nil)) tab.getOrDefault(w, NodeRef(nil))
@ -134,9 +131,6 @@ func isValid*(key: HashKey): bool =
func isValid*(vid: VertexID): bool = func isValid*(vid: VertexID): bool =
vid != VertexID(0) vid != VertexID(0)
func isValid*(lbl: HashLabel): bool =
lbl.key.isValid
func isValid*(sqv: HashSet[VertexID]): bool = func isValid*(sqv: HashSet[VertexID]): bool =
sqv != EmptyVidSet sqv != EmptyVidSet

19
nimbus/db/aristo/aristo_desc/desc_identifiers.nim
View File

@ -86,18 +86,6 @@ type
root*: VertexID ## Root ID for the sub-trie root*: VertexID ## Root ID for the sub-trie
path*: PathID ## Path into the `Patricia Trie` path*: PathID ## Path into the `Patricia Trie`
HashLabel* = object
## Merkle hash key uniquely associated with a vertex ID. As hashes in a
## `Merkle Patricia Tree` are unique only on a particular sub-trie, the
## hash key is paired with the top vertex of the relevant sub-trie. This
## construction is similar to the one of a `LeafTie` object.
##
## Note that `HashLabel` objects have no representation in the
## `Aristo Trie`. They are used temporarily and in caches or backlog
## tables.
root*: VertexID ## Root ID for the sub-trie.
key*: HashKey ## Merkle hash or encoded small node data
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Chronicles formatters # Chronicles formatters
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -382,13 +370,6 @@ func hash*(a: HashKey): Hash =
h = h !& a.blob.hash h = h !& a.blob.hash
!$h !$h
func hash*(lbl: HashLabel): Hash =
## Table/KeyedQueue/HashSet mixin
var h: Hash = 0
h = h !& lbl.root.hash
h = h !& lbl.key.hash
!$h
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Miscellaneous helpers # Miscellaneous helpers
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------

8
nimbus/db/aristo/aristo_desc/desc_structural.nim
View File

@ -76,8 +76,8 @@ type
kMap*: Table[VertexID,HashKey] ## Filter Merkle hash key mapping kMap*: Table[VertexID,HashKey] ## Filter Merkle hash key mapping
vGen*: seq[VertexID] ## Filter unique vertex ID generator vGen*: seq[VertexID] ## Filter unique vertex ID generator
VidsByLabelTab* = Table[HashLabel,HashSet[VertexID]] VidsByKeyTab* = Table[HashKey,HashSet[VertexID]]
## Reverse lookup searching `VertexID` by the hash key/label. ## Reverse lookup searching `VertexID` by the hash key.
LayerDeltaRef* = ref object LayerDeltaRef* = ref object
## Delta layers are stacked implying a tables hierarchy. Table entries on ## Delta layers are stacked implying a tables hierarchy. Table entries on
@ -103,8 +103,8 @@ type
## inconsistent state that must be resolved. ## inconsistent state that must be resolved.
## ##
sTab*: Table[VertexID,VertexRef] ## Structural vertex table sTab*: Table[VertexID,VertexRef] ## Structural vertex table
kMap*: Table[VertexID,HashLabel] ## Merkle hash key mapping kMap*: Table[VertexID,HashKey] ## Merkle hash key mapping
pAmk*: VidsByLabelTab ## Reverse `kMap` entries, hash key lookup pAmk*: VidsByKeyTab ## Reverse `kMap` entries, hash key lookup
LayerFinalRef* = ref object LayerFinalRef* = ref object
## Final tables fully supersede tables on lower layers when stacked as a ## Final tables fully supersede tables on lower layers when stacked as a

2
nimbus/db/aristo/aristo_filter.nim
View File

@ -56,7 +56,7 @@ proc fwdFilter*(
ok FilterRef( ok FilterRef(
src: srcRoot, src: srcRoot,
sTab: layer.delta.sTab, sTab: layer.delta.sTab,
kMap: layer.delta.kMap.pairs.toSeq.mapIt((it[0],it[1].key)).toTable, kMap: layer.delta.kMap,
vGen: layer.final.vGen.vidReorg, # Compact recycled IDs vGen: layer.final.vGen.vidReorg, # Compact recycled IDs
trg: trgRoot) trg: trgRoot)

9
nimbus/db/aristo/aristo_filter/filter_helpers.nim
View File

@ -51,9 +51,9 @@ proc getLayerStateRoots*(
spr.be = sprBeKey.to(Hash256) spr.be = sprBeKey.to(Hash256)
spr.fg = block: spr.fg = block:
let lbl = delta.kMap.getOrVoid VertexID(1) let key = delta.kMap.getOrVoid VertexID(1)
if lbl.isValid: if key.isValid:
lbl.key.to(Hash256) key.to(Hash256)
else: else:
EMPTY_ROOT_HASH EMPTY_ROOT_HASH
if spr.fg.isValid: if spr.fg.isValid:
@ -68,8 +68,7 @@ proc getLayerStateRoots*(
return ok(spr) return ok(spr)
if chunkedMpt: if chunkedMpt:
let lbl = HashLabel(root: VertexID(1), key: sprBeKey) if VertexID(1) in delta.pAmk.getOrVoid sprBeKey:
if VertexID(1) in delta.pAmk.getOrVoid lbl:
spr.fg = spr.be spr.fg = spr.be
return ok(spr) return ok(spr)

6
nimbus/db/aristo/aristo_get.nim
View File

@ -166,12 +166,12 @@ proc getKeyRc*(db: AristoDbRef; vid: VertexID): Result[HashKey,AristoError] =
block body: block body:
let key = db.layersGetKey(vid).valueOr: let key = db.layersGetKey(vid).valueOr:
break body break body
# If there is a zero value label, the entry is either marked for being # If there is a zero key value, the entry is either marked for being
# updated or for deletion on the database. So check below. # updated or for deletion on the database. So check below.
if key.isValid: if key.isValid:
return ok key return ok key
# The zero value label does not refer to an update mark if there is no # The zero key value does not refer to an update mark if there is no
# valid vertex (either on the cache or the backend whatever comes first.) # valid vertex (either on the cache or the backend whatever comes first.)
let vtx = db.layersGetVtx(vid).valueOr: let vtx = db.layersGetVtx(vid).valueOr:
# There was no vertex on the cache. So there must be one the backend (the # There was no vertex on the cache. So there must be one the backend (the
@ -180,7 +180,7 @@ proc getKeyRc*(db: AristoDbRef; vid: VertexID): Result[HashKey,AristoError] =
if vtx.isValid: if vtx.isValid:
return err(GetKeyUpdateNeeded) return err(GetKeyUpdateNeeded)
else: else:
# The vertex is to be deleted. So is the value label. # The vertex is to be deleted. So is the value key.
return err(GetKeyNotFound) return err(GetKeyNotFound)
db.getKeyBE vid db.getKeyBE vid

11
nimbus/db/aristo/aristo_hashify.nim
View File

@ -25,7 +25,7 @@
## `lTab[]` entry with `(root-vertex,path,VertexID(0))`. ## `lTab[]` entry with `(root-vertex,path,VertexID(0))`.
## ##
## * All vertices where the key (aka Merkle hash) has changed must have a ## * All vertices where the key (aka Merkle hash) has changed must have a
## top layer cache `kMap[]` entry `(vertex-ID,VOID_HASH_LABEL)` indicating ## top layer cache `kMap[]` entry `(vertex-ID,VOID_HASH_KEY)` indicating
## that there is no key available for this vertex. This also applies for ## that there is no key available for this vertex. This also applies for
## backend verices where the key has changed while the structural logic ## backend verices where the key has changed while the structural logic
## did not change. ## did not change.
@ -333,8 +333,8 @@ proc hashify*(
# is the task to search for unresolved node keys and add glue paths to # is the task to search for unresolved node keys and add glue paths to
# the width-first schedule. # the width-first schedule.
var unresolved: HashSet[VertexID] var unresolved: HashSet[VertexID]
for (vid,lbl) in db.layersWalkLabel: for (vid,key) in db.layersWalkKey:
if not lbl.isValid and if not key.isValid and
vid notin wff: vid notin wff:
let rc = db.layersGetVtx vid let rc = db.layersGetVtx vid
if rc.isErr or rc.value.isValid: if rc.isErr or rc.value.isValid:
@ -390,8 +390,7 @@ proc hashify*(
# End `valueOr` terminates error clause # End `valueOr` terminates error clause
# Could resolve => update Merkle hash # Could resolve => update Merkle hash
let key = node.digestTo(HashKey) db.layersPutKey(vid, node.digestTo HashKey)
db.layersPutLabel(vid, HashLabel(root: val.root, key: key))
# Set follow up link for next round # Set follow up link for next round
wff.setNextLink(redo, val) wff.setNextLink(redo, val)
@ -410,7 +409,7 @@ proc hashify*(
# Convert root vertex to a node. # Convert root vertex to a node.
let node = db.getVtx(vid).toNode(db,stopEarly=false).valueOr: let node = db.getVtx(vid).toNode(db,stopEarly=false).valueOr:
return err((vid,HashifyRootNodeUnresolved)) return err((vid,HashifyRootNodeUnresolved))
db.layersPutLabel(vid, HashLabel(root: vid, key: node.digestTo(HashKey))) db.layersPutKey(vid, node.digestTo(HashKey))
wff.completed.incl vid wff.completed.incl vid
db.top.final.dirty = false # Mark top layer clean db.top.final.dirty = false # Mark top layer clean

173
nimbus/db/aristo/aristo_layers.nim
View File

@ -25,21 +25,21 @@ func dup(sTab: Table[VertexID,VertexRef]): Table[VertexID,VertexRef] =
for (k,v) in sTab.pairs: for (k,v) in sTab.pairs:
result[k] = v.dup result[k] = v.dup
func getLebalOrVoid(stack: seq[LayerRef]; lbl: HashLabel): HashSet[VertexID] = func getLebalOrVoid(stack: seq[LayerRef]; key: HashKey): HashSet[VertexID] =
# Helper: get next set of vertex IDs from stack. # Helper: get next set of vertex IDs from stack.
for w in stack.reversed: for w in stack.reversed:
w.delta.pAmk.withValue(lbl,value): w.delta.pAmk.withValue(key,value):
return value[] return value[]
proc recalcLebal(layer: var LayerObj) = proc recalcLebal(layer: var LayerObj) =
## Calculate reverse `kMap[]` for final (aka zero) layer ## Calculate reverse `kMap[]` for final (aka zero) layer
layer.delta.pAmk.clear layer.delta.pAmk.clear
for (vid,lbl) in layer.delta.kMap.pairs: for (vid,key) in layer.delta.kMap.pairs:
if lbl.isValid: if key.isValid:
layer.delta.pAmk.withValue(lbl, value): layer.delta.pAmk.withValue(key, value):
value[].incl vid value[].incl vid
do: do:
layer.delta.pAmk[lbl] = @[vid].toHashSet layer.delta.pAmk[key] = @[vid].toHashSet
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public getters: lazy value lookup for read only versions # Public getters: lazy value lookup for read only versions
@ -62,24 +62,25 @@ func dirty*(db: AristoDbRef): bool =
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
func nLayersVtx*(db: AristoDbRef): int = func nLayersVtx*(db: AristoDbRef): int =
## Number of vertex ID/vertex entries on the cache layers. This is an upper ## Number of vertex ID/vertex entries on the cache layers. This is an upper bound
## bound for the number of effective vertex ID mappings held on the cache ## for the number of effective vertex ID mappings held on the cache layers as
## layers as there might be duplicate entries for the same vertex ID on ## there might be duplicate entries for the same vertex ID on different layers.
## different layers. ##
db.stack.mapIt(it.delta.sTab.len).foldl(a + b, db.top.delta.sTab.len) db.stack.mapIt(it.delta.sTab.len).foldl(a + b, db.top.delta.sTab.len)
func nLayersLabel*(db: AristoDbRef): int = func nLayersKey*(db: AristoDbRef): int =
## Number of vertex ID/label entries on the cache layers. This is an upper ## Number of vertex ID/key entries on the cache layers. This is an upper bound
## bound for the number of effective vertex ID mappingss held on the cache ## for the number of effective vertex ID mappingss held on the cache layers as
## layers as there might be duplicate entries for the same vertex ID on ## there might be duplicate entries for the same vertex ID on different layers.
## different layers. ##
db.stack.mapIt(it.delta.kMap.len).foldl(a + b, db.top.delta.kMap.len) db.stack.mapIt(it.delta.kMap.len).foldl(a + b, db.top.delta.kMap.len)
func nLayersLebal*(db: AristoDbRef): int = func nLayersYek*(db: AristoDbRef): int =
## Number of label/vertex IDs reverse lookup entries on the cache layers. ## Number of key/vertex IDs reverse lookup entries on the cache layers. This
## This is an upper bound for the number of effective label mappingss held ## is an upper bound for the number of effective key mappingss held on the
## on the cache layers as there might be duplicate entries for the same label ## cache layers as there might be duplicate entries for the same key on
## on different layers. ## different layers.
##
db.stack.mapIt(it.delta.pAmk.len).foldl(a + b, db.top.delta.pAmk.len) db.stack.mapIt(it.delta.pAmk.len).foldl(a + b, db.top.delta.pAmk.len)
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -104,10 +105,9 @@ proc layersGetVtxOrVoid*(db: AristoDbRef; vid: VertexID): VertexRef =
db.layersGetVtx(vid).valueOr: VertexRef(nil) db.layersGetVtx(vid).valueOr: VertexRef(nil)
proc layersGetLabel*(db: AristoDbRef; vid: VertexID): Result[HashLabel,void] = proc layersGetKey*(db: AristoDbRef; vid: VertexID): Result[HashKey,void] =
## Find a hash label (containh the `HashKey`) on the cache layers. An ## Find a hash key on the cache layers. An `ok()` result might contain a void
## `ok()` result might contain a void hash label if it is stored on the ## hash key if it is stored on the cache that way.
## cache that way.
## ##
if db.top.delta.kMap.hasKey vid: if db.top.delta.kMap.hasKey vid:
# This is ok regardless of the `dirty` flag. If this vertex has become # This is ok regardless of the `dirty` flag. If this vertex has become
@ -121,43 +121,30 @@ proc layersGetLabel*(db: AristoDbRef; vid: VertexID): Result[HashLabel,void] =
err() err()
proc layersGetlabelOrVoid*(db: AristoDbRef; vid: VertexID): HashLabel =
## Simplified version of `layersGetLabel()`
db.layersGetLabel(vid).valueOr: VOID_HASH_LABEL
proc layersGetKey*(db: AristoDbRef; vid: VertexID): Result[HashKey,void] =
## Variant of `layersGetLabel()` for returning the `HashKey` part of the
## label only.
let lbl = db.layersGetLabel(vid).valueOr:
return err()
# Note that `lbl.isValid == lbl.key.isValid`
ok(lbl.key)
proc layersGetKeyOrVoid*(db: AristoDbRef; vid: VertexID): HashKey = proc layersGetKeyOrVoid*(db: AristoDbRef; vid: VertexID): HashKey =
## Simplified version of `layersGetKey()` ## Simplified version of `layersGetkey()`
db.layersGetKey(vid).valueOr: VOID_HASH_KEY db.layersGetKey(vid).valueOr: VOID_HASH_KEY
proc layersGetLebal*( proc layersGetYek*(
db: AristoDbRef; db: AristoDbRef;
lbl: HashLabel; key: HashKey;
): Result[HashSet[VertexID],void] = ): Result[HashSet[VertexID],void] =
## Inverse of `layersGetKey()`. For a given argumnt `lbl`, find all vertex ## Inverse of `layersGetKey()`. For a given argumnt `key`, finds all vertex IDs
## IDs that have `layersGetLbl()` return this very `lbl` value for the these ## that have `layersGetKey()` return this very `key` value for the argument
## vertex IDs. ## vertex IDs.
if db.top.delta.pAmk.hasKey lbl: if db.top.delta.pAmk.hasKey key:
return ok(db.top.delta.pAmk.getOrVoid lbl) return ok(db.top.delta.pAmk.getOrVoid key)
for w in db.stack.reversed: for w in db.stack.reversed:
if w.delta.pAmk.hasKey lbl: if w.delta.pAmk.hasKey key:
return ok(w.delta.pAmk.getOrVoid lbl) return ok(w.delta.pAmk.getOrVoid key)
err() err()
proc layersGetLebalOrVoid*(db: AristoDbRef; lbl: HashLabel): HashSet[VertexID] = proc layersGetYekOrVoid*(db: AristoDbRef; key: HashKey): HashSet[VertexID] =
## Simplified version of `layersGetVidsOrVoid()` ## Simplified version of `layersGetVidsOrVoid()`
db.layersGetLebal(lbl).valueOr: EmptyVidSet db.layersGetYek(key).valueOr: EmptyVidSet
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions: put variants # Public functions: put variants
@ -174,43 +161,43 @@ proc layersResVtx*(db: AristoDbRef; vid: VertexID) =
db.layersPutVtx(vid, VertexRef(nil)) db.layersPutVtx(vid, VertexRef(nil))
proc layersPutLabel*(db: AristoDbRef; vid: VertexID; lbl: HashLabel) = proc layersPutKey*(db: AristoDbRef; vid: VertexID; key: HashKey) =
## Store a (potentally void) hash label on the top layer ## Store a (potentally void) hash key on the top layer
# Get previous label # Get previous key
let blb = db.top.delta.kMap.getOrVoid vid let prvKey = db.top.delta.kMap.getOrVoid vid
# Update label on `label->vid` mapping table # Update key on `kMap:key->vid` mapping table
db.top.delta.kMap[vid] = lbl db.top.delta.kMap[vid] = key
db.top.final.dirty = true # Modified top cache layers db.top.final.dirty = true # Modified top cache layers
# Clear previous value on reverse table if it has changed # Clear previous value on reverse table if it has changed
if blb.isValid and blb != lbl: if prvKey.isValid and prvKey != key:
var vidsLen = -1 var vidsLen = -1
db.top.delta.pAmk.withValue(blb, value): db.top.delta.pAmk.withValue(prvKey, value):
value[].excl vid value[].excl vid
vidsLen = value[].len vidsLen = value[].len
do: # provide empty lookup do: # provide empty lookup
let vids = db.stack.getLebalOrVoid(blb) let vids = db.stack.getLebalOrVoid(prvKey)
if vids.isValid and vid in vids: if vids.isValid and vid in vids:
# This entry supersedes non-emtpty changed ones from lower levels # This entry supersedes non-emtpty changed ones from lower levels
db.top.delta.pAmk[blb] = vids - @[vid].toHashSet db.top.delta.pAmk[prvKey] = vids - @[vid].toHashSet
if vidsLen == 0 and not db.stack.getLebalOrVoid(blb).isValid: if vidsLen == 0 and not db.stack.getLebalOrVoid(prvKey).isValid:
# There is no non-emtpty entry on lower levels, so ledete this one # There is no non-emtpty entry on lower levels, so ledete this one
db.top.delta.pAmk.del blb db.top.delta.pAmk.del prvKey
# Add updated value on reverse table if non-zero # Add updated value on reverse table if non-zero
if lbl.isValid: if key.isValid:
db.top.delta.pAmk.withValue(lbl, value): db.top.delta.pAmk.withValue(key, value):
value[].incl vid value[].incl vid
do: # else if not found: need to merge with value set from lower layer do: # else if not found: need to merge with value set from lower layer
db.top.delta.pAmk[lbl] = db.stack.getLebalOrVoid(lbl) + @[vid].toHashSet db.top.delta.pAmk[key] = db.stack.getLebalOrVoid(key) + @[vid].toHashSet
proc layersResLabel*(db: AristoDbRef; vid: VertexID) = proc layersResKey*(db: AristoDbRef; vid: VertexID) =
## Shortcut for `db.layersPutLabel(vid, VOID_HASH_LABEL)`. It is sort of the ## Shortcut for `db.layersPutKey(vid, VOID_HASH_KEY)`. It is sort of the
## equivalent of a delete function. ## equivalent of a delete function.
db.layersPutLabel(vid, VOID_HASH_LABEL) db.layersPutKey(vid, VOID_HASH_KEY)
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions # Public functions
@ -225,18 +212,18 @@ proc layersMergeOnto*(src: LayerRef; trg: var LayerObj; stack: seq[LayerRef]) =
for (vid,vtx) in src.delta.sTab.pairs: for (vid,vtx) in src.delta.sTab.pairs:
trg.delta.sTab[vid] = vtx trg.delta.sTab[vid] = vtx
for (vid,lbl) in src.delta.kMap.pairs: for (vid,key) in src.delta.kMap.pairs:
trg.delta.kMap[vid] = lbl trg.delta.kMap[vid] = key
if stack.len == 0: if stack.len == 0:
# Re-calculate `pAmk[]` # Re-calculate `pAmk[]`
trg.recalcLebal() trg.recalcLebal()
else: else:
# Merge reverse `kMap[]` layers. Empty label image sets are ignored unless # Merge reverse `kMap[]` layers. Empty key set images are ignored unless
# they supersede non-empty values on the argument `stack[]`. # they supersede non-empty values on the argument `stack[]`.
for (lbl,vids) in src.delta.pAmk.pairs: for (key,vids) in src.delta.pAmk.pairs:
if 0 < vids.len or stack.getLebalOrVoid(lbl).isValid: if 0 < vids.len or stack.getLebalOrVoid(key).isValid:
trg.delta.pAmk[lbl] = vids trg.delta.pAmk[key] = vids
func layersCc*(db: AristoDbRef; level = high(int)): LayerRef = func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
@ -258,8 +245,8 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
for n in 1 ..< layers.len: for n in 1 ..< layers.len:
for (vid,vtx) in layers[n].delta.sTab.pairs: for (vid,vtx) in layers[n].delta.sTab.pairs:
result.delta.sTab[vid] = vtx result.delta.sTab[vid] = vtx
for (vid,lbl) in layers[n].delta.kMap.pairs: for (vid,key) in layers[n].delta.kMap.pairs:
result.delta.kMap[vid] = lbl result.delta.kMap[vid] = key
# Re-calculate `pAmk[]` # Re-calculate `pAmk[]`
result[].recalcLebal() result[].recalcLebal()
@ -298,37 +285,37 @@ iterator layersWalkVtx*(
yield (vid,vtx) yield (vid,vtx)
iterator layersWalkLabel*( iterator layersWalkKey*(
db: AristoDbRef; db: AristoDbRef;
): tuple[vid: VertexID, lbl: HashLabel] = ): tuple[vid: VertexID, key: HashKey] =
## Walk over all `(VertexID,HashLabel)` pairs on the cache layers. Note that ## Walk over all `(VertexID,HashKey)` pairs on the cache layers. Note that
## entries are unsorted. ## entries are unsorted.
var seen: HashSet[VertexID] var seen: HashSet[VertexID]
for (vid,lbl) in db.top.delta.kMap.pairs: for (vid,key) in db.top.delta.kMap.pairs:
yield (vid,lbl) yield (vid,key)
seen.incl vid seen.incl vid
for w in db.stack.reversed: for w in db.stack.reversed:
for (vid,lbl) in w.delta.kMap.pairs: for (vid,key) in w.delta.kMap.pairs:
if vid notin seen: if vid notin seen:
yield (vid,lbl) yield (vid,key)
seen.incl vid seen.incl vid
iterator layersWalkLebal*( iterator layersWalkYek*(
db: AristoDbRef; db: AristoDbRef;
): tuple[lbl: HashLabel, vids: HashSet[VertexID]] = ): tuple[key: HashKey, vids: HashSet[VertexID]] =
## Walk over `(HashLabel,HashSet[VertexID])` pairs. ## Walk over `(HashKey,HashSet[VertexID])` pairs.
var seen: HashSet[HashLabel] var seen: HashSet[HashKey]
for (lbl,vids) in db.top.delta.pAmk.pairs: for (key,vids) in db.top.delta.pAmk.pairs:
yield (lbl,vids) yield (key,vids)
seen.incl lbl seen.incl key
for w in db.stack.reversed: for w in db.stack.reversed:
for (lbl,vids) in w.delta.pAmk.pairs: for (key,vids) in w.delta.pAmk.pairs:
if lbl notin seen: if key notin seen:
yield (lbl,vids) yield (key,vids)
seen.incl lbl seen.incl key
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# End # End

43
nimbus/db/aristo/aristo_merge.nim
View File

@ -81,7 +81,7 @@ proc nullifyKey(
vid: VertexID; # Vertex IDs to clear vid: VertexID; # Vertex IDs to clear
) = ) =
# Register for void hash (to be recompiled) # Register for void hash (to be recompiled)
db.layersResLabel vid db.layersResKey vid
proc clearMerkleKeys( proc clearMerkleKeys(
db: AristoDbRef; # Database, top layer db: AristoDbRef; # Database, top layer
@ -97,7 +97,7 @@ proc setVtxAndKey(
vtx: VertexRef; # Vertex to add vtx: VertexRef; # Vertex to add
) = ) =
db.layersPutVtx(vid, vtx) db.layersPutVtx(vid, vtx)
db.layersResLabel vid db.layersResKey vid
# ----------- # -----------
@ -512,8 +512,7 @@ proc mergeNodeImpl(
# already. This is provided for if the `nodes` are processed in the right # already. This is provided for if the `nodes` are processed in the right
# order `root->.. ->leaf`. # order `root->.. ->leaf`.
let let
hashLbl = HashLabel(root: rootVid, key: hashKey) vids = db.layersGetYekOrVoid(hashKey).toSeq
vids = db.layersGetLebalOrVoid(hashLbl).toSeq
isRoot = rootVid in vids isRoot = rootVid in vids
if vids.len == 0: if vids.len == 0:
return err(MergeRevVidMustHaveBeenCached) return err(MergeRevVidMustHaveBeenCached)
@ -522,8 +521,8 @@ proc mergeNodeImpl(
return err(MergeHashKeyRevLookUpGarbled) return err(MergeHashKeyRevLookUpGarbled)
# Use the first vertex ID from the `vis` list as representant for all others # Use the first vertex ID from the `vis` list as representant for all others
let lbl = db.layersGetLabelOrVoid vids[0] let key = db.layersGetKeyOrVoid vids[0]
if lbl == hashLbl: if key == hashKey:
if db.layersGetVtx(vids[0]).isOk: if db.layersGetVtx(vids[0]).isOk:
for n in 1 ..< vids.len: for n in 1 ..< vids.len:
if db.layersGetVtx(vids[n]).isErr: if db.layersGetVtx(vids[n]).isErr:
@ -531,7 +530,7 @@ proc mergeNodeImpl(
# This is tyically considered OK # This is tyically considered OK
return err(MergeHashKeyCachedAlready) return err(MergeHashKeyCachedAlready)
# Otherwise proceed # Otherwise proceed
elif lbl.isValid: elif key.isValid:
# Different key assigned => error # Different key assigned => error
return err(MergeHashKeyDiffersFromCached) return err(MergeHashKeyDiffersFromCached)
@ -548,7 +547,7 @@ proc mergeNodeImpl(
# Verify that all `vids` entries are similar # Verify that all `vids` entries are similar
for n in 1 ..< vids.len: for n in 1 ..< vids.len:
let w = vids[n] let w = vids[n]
if lbl != db.layersGetLabelOrVoid(w) or db.layersGetVtx(w).isOk: if key != db.layersGetKeyOrVoid(w) or db.layersGetVtx(w).isOk:
return err(MergeHashKeyRevLookUpGarbled) return err(MergeHashKeyRevLookUpGarbled)
if not hasVtx: if not hasVtx:
# Prefer existing node which has all links available, already. # Prefer existing node which has all links available, already.
@ -556,31 +555,31 @@ proc mergeNodeImpl(
if u.isValid: if u.isValid:
(vtx, hasVtx) = (u, true) (vtx, hasVtx) = (u, true)
# The `vertexID <-> hashLabel` mappings need to be set up now (if any) # The `vertexID <-> hashKey` mappings need to be set up now (if any)
case node.vType: case node.vType:
of Leaf: of Leaf:
discard discard
of Extension: of Extension:
if node.key[0].isValid: if node.key[0].isValid:
let eLbl = HashLabel(root: rootVid, key: node.key[0]) let eKey = node.key[0]
if not hasVtx: if not hasVtx:
# Brand new reverse lookup link for this vertex # Brand new reverse lookup link for this vertex
vtx.eVid = db.vidFetch vtx.eVid = db.vidFetch
db.layersPutLabel(vtx.eVid, eLbl) db.layersPutKey(vtx.eVid, eKey)
elif not vtx.eVid.isValid: elif not vtx.eVid.isValid:
return err(MergeNodeVtxDiffersFromExisting) return err(MergeNodeVtxDiffersFromExisting)
db.layersPutLabel(vtx.eVid, eLbl) db.layersPutKey(vtx.eVid, eKey)
of Branch: of Branch:
for n in 0..15: for n in 0..15:
if node.key[n].isValid: if node.key[n].isValid:
let bLbl = HashLabel(root: rootVid, key: node.key[n]) let bKey = node.key[n]
if not hasVtx: if not hasVtx:
# Brand new reverse lookup link for this vertex # Brand new reverse lookup link for this vertex
vtx.bVid[n] = db.vidFetch vtx.bVid[n] = db.vidFetch
db.layersPutLabel(vtx.bVid[n], bLbl) db.layersPutKey(vtx.bVid[n], bKey)
elif not vtx.bVid[n].isValid: elif not vtx.bVid[n].isValid:
return err(MergeNodeVtxDiffersFromExisting) return err(MergeNodeVtxDiffersFromExisting)
db.layersPutLabel(vtx.bVid[n], bLbl) db.layersPutKey(vtx.bVid[n], bKey)
for w in vids: for w in vids:
db.top.final.pPrf.incl w db.top.final.pPrf.incl w
@ -824,13 +823,11 @@ proc merge*(
if 0 < chain.len and chain[^1] == rootKey: if 0 < chain.len and chain[^1] == rootKey:
chains.add chain chains.add chain
# Make sure that the reverse lookup for the root vertex label is available. # Make sure that the reverse lookup for the root vertex key is available.
block: block:
let let vids = db.layersGetYekOrVoid rootKey
lbl = HashLabel(root: rootVid, key: rootKey)
vids = db.layersGetLebalOrVoid lbl
if not vids.isValid: if not vids.isValid:
db.layersPutlabel(rootVid, lbl) db.layersPutKey(rootVid, rootKey)
# Process over chains in reverse mode starting with the root node. This # Process over chains in reverse mode starting with the root node. This
# allows the algorithm to find existing nodes on the backend. # allows the algorithm to find existing nodes on the backend.
@ -882,7 +879,7 @@ proc merge*(
return ok rootVid return ok rootVid
if not key.isValid: if not key.isValid:
db.layersPutLabel(rootVid, HashLabel(root: rootVid, key: rootLink)) db.layersPutKey(rootVid, rootLink)
return ok rootVid return ok rootVid
else: else:
let key = db.getKey VertexID(1) let key = db.getKey VertexID(1)
@ -891,13 +888,13 @@ proc merge*(
# Otherwise assign unless valid # Otherwise assign unless valid
if not key.isValid: if not key.isValid:
db.layersPutLabel(VertexID(1),HashLabel(root: VertexID(1), key: rootLink)) db.layersPutKey(VertexID(1), rootLink)
return ok VertexID(1) return ok VertexID(1)
# Create and assign a new root key # Create and assign a new root key
if not rootVid.isValid: if not rootVid.isValid:
let vid = db.vidFetch let vid = db.vidFetch
db.layersPutLabel(vid, HashLabel(root: vid, key: rootLink)) db.layersPutKey(vid, rootLink)
return ok vid return ok vid
err(MergeRootKeyDiffersForVid) err(MergeRootKeyDiffersForVid)

8
nimbus/db/aristo/aristo_utils.nim
View File

@ -108,10 +108,10 @@ proc toNode*(
## ##
proc getKey(db: AristoDbRef; vid: VertexID; beOk: bool): HashKey = proc getKey(db: AristoDbRef; vid: VertexID; beOk: bool): HashKey =
block body: block body:
let lbl = db.layersGetLabel(vid).valueOr: let key = db.layersGetKey(vid).valueOr:
break body break body
if lbl.isValid: if key.isValid:
return lbl.key return key
else: else:
return VOID_HASH_KEY return VOID_HASH_KEY
if beOk: if beOk:
@ -225,7 +225,7 @@ proc registerAccount*(
# Clear Merkle keys and store leaf record # Clear Merkle keys and store leaf record
for w in hike.legs.mapIt(it.wp.vid): for w in hike.legs.mapIt(it.wp.vid):
db.layersResLabel w db.layersResKey w
db.top.final.lTab[lty] = wp.vid db.top.final.lTab[lty] = wp.vid
ok() ok()

3
tests/test_aristo/test_filter.nim
View File

@ -76,8 +76,7 @@ func ppFil(w: FilterRef; db = AristoDbRef(nil)): string =
let n = key.to(UInt256) let n = key.to(UInt256)
if n == 0: "£ø" else: "£" & $n if n == 0: "£ø" else: "£" & $n
else: else:
let keyLink = HashKey.fromBytes(key.data).value HashKey.fromBytes(key.data).value.pp(db)
HashLabel(root: VertexID(1), key: keyLink).pp(db)
"(" & w.fid.pp & "," & w.src.qq(db) & "->" & w.trg.qq(db) & ")" "(" & w.fid.pp & "," & w.src.qq(db) & "->" & w.trg.qq(db) & ")"
func pp(qf: (QueueID,FilterRef); db = AristoDbRef(nil)): string = func pp(qf: (QueueID,FilterRef); db = AristoDbRef(nil)): string =

2
tests/test_aristo/test_helpers.nim
View File

@ -59,7 +59,7 @@ func pp*(
let let
pfx = indent.toPfx pfx = indent.toPfx
rootLink = w.root.to(HashKey) rootLink = w.root.to(HashKey)
result = "(" & HashLabel(root: rootID, key: rootLink).pp(db) result = "(" & rootLink.pp(db)
result &= "," & $w.id & ",[" & $w.proof.len & "]," result &= "," & $w.id & ",[" & $w.proof.len & "],"
result &= pfx & " [" result &= pfx & " ["
for n,kvp in w.kvpLst: for n,kvp in w.kvpLst:

4
tests/test_aristo/test_misc.nim
View File

@ -493,7 +493,7 @@ proc testShortKeys*(
"\n r=", r.pp(sig), "\n r=", r.pp(sig),
"\n ", sig.pp(), "\n ", sig.pp(),
"\n", "\n",
"\n pAmk=", sig.db.layersWalkLebal.toSeq.toTable.pp(sig.db), "\n pAmk=", sig.db.layersWalkYek.toSeq.toTable.pp(sig.db),
"\n" "\n"
let w = sig.merkleSignCommit().value let w = sig.merkleSignCommit().value
gossip.say "*** testShortkeys (2)", "n=", n, " inx=", inx, gossip.say "*** testShortkeys (2)", "n=", n, " inx=", inx,
@ -502,7 +502,7 @@ proc testShortKeys*(
"\n R=", w.pp(sig), "\n R=", w.pp(sig),
"\n ", sig.pp(), "\n ", sig.pp(),
"\n", "\n",
"\n pAmk=", sig.db.layersWalkLebal.toSeq.toTable.pp(sig.db), "\n pAmk=", sig.db.layersWalkYek.toSeq.toTable.pp(sig.db),
"\n", "\n",
"\n ----------------", "\n ----------------",
"\n" "\n"

4
tests/test_aristo/test_tx.nim
View File

@ -109,7 +109,7 @@ proc schedStow(
): Result[void,AristoError] = ): Result[void,AristoError] =
## Scheduled storage ## Scheduled storage
let let
layersMeter = db.nLayersVtx + db.nLayersLabel layersMeter = db.nLayersVtx() + db.nLayersKey()
filterMeter = if db.roFilter.isNil: 0 filterMeter = if db.roFilter.isNil: 0
else: db.roFilter.sTab.len + db.roFilter.kMap.len else: db.roFilter.sTab.len + db.roFilter.kMap.len
persistent = MaxFilterBulk < max(layersMeter, filterMeter) persistent = MaxFilterBulk < max(layersMeter, filterMeter)
@ -557,7 +557,7 @@ proc testTxMergeProofAndKvpList*(
xCheck w.proof.len == proved.merged + proved.dups xCheck w.proof.len == proved.merged + proved.dups
xCheck db.lTab.len == lTabLen xCheck db.lTab.len == lTabLen
xCheck db.nLayersVtx() <= proved.merged + sTabLen xCheck db.nLayersVtx() <= proved.merged + sTabLen
xCheck proved.merged < db.nLayersLebal() xCheck proved.merged < db.nLayersYek()
let let
merged = db.mergeList leafs merged = db.mergeList leafs

42
tests/test_coredb.nim
View File

@ -46,7 +46,24 @@ let
when unittest2DisableParamFiltering: when unittest2DisableParamFiltering:
# Filter out local options and pass on the rest to `unittest2` # Filter out local options and pass on the rest to `unittest2`
proc cmdLineConfig(): tuple[samples: seq[CaptureSpecs]] = proc cmdLineConfig(): tuple[samples: seq[CaptureSpecs]] =
## This helper allows to pass additional command line options to the
## unit test.
##
## Example:
## ::
## nim c -r ...\
## -d:unittest2DisableParamFiltering \
## ./tests/test_coredb.nim \
## --output-level=VERBOSE \
## --sample=goerli-lp,goerli-ar
## or
## ::
## nim c ... -d:unittest2DisableParamFiltering ./tests/test_coredb.nim
## ./tests/test_coredb.out --output-level=VERBOSE --sample=goerli-ar
## ...
##
## At the moment, only the `--sample=` additional option is provided.
##
# Define sample list from the command line (if any) # Define sample list from the command line (if any)
const optPfx = "--sample=" # Custom option with sample list const optPfx = "--sample=" # Custom option with sample list
@ -182,8 +199,10 @@ proc chainSyncRunner(
capture = bChainCapture; capture = bChainCapture;
dbType = CoreDbType(0); dbType = CoreDbType(0);
ldgType = ldgTypeDefault; ldgType = ldgTypeDefault;
enaLogging = false; profilingOk = false;
lastOneExtra = true; finalDiskCleanUpOk = true;
enaLoggingOk = false;
lastOneExtraOk = true;
) = ) =
## Test backend database and ledger ## Test backend database and ledger
@ -215,9 +234,9 @@ proc chainSyncRunner(
let let
com = initRunnerDB(dbDir, capture, dbType, ldgType) com = initRunnerDB(dbDir, capture, dbType, ldgType)
defer: defer:
com.db.finish(flush = true) com.db.finish(flush = finalDiskCleanUpOk)
#noisy.testChainSyncProfilingPrint numBlocks if profilingOk: noisy.testChainSyncProfilingPrint numBlocks
if persistent: dbDir.flushDbDir if persistent and finalDiskCleanUpOk: dbDir.flushDbDir
if noisy: if noisy:
com.db.trackNewApi = true com.db.trackNewApi = true
@ -226,7 +245,7 @@ proc chainSyncRunner(
com.db.localDbOnly = true com.db.localDbOnly = true
check noisy.testChainSync(filePaths, com, numBlocks, check noisy.testChainSync(filePaths, com, numBlocks,
lastOneExtra=lastOneExtra, enaLogging=enaLogging) lastOneExtra=lastOneExtraOk, enaLogging=enaLoggingOk)
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Main function(s) # Main function(s)
@ -258,10 +277,13 @@ when isMainModule:
for n,capture in sampleList: for n,capture in sampleList:
noisy.profileSection("@testList #" & $n, state): noisy.profileSection("@testList #" & $n, state):
noisy.chainSyncRunner( noisy.chainSyncRunner(
capture=capture, capture = capture,
#dbType = ..., #dbType = ..,
ldgType=LedgerCache, ldgType=LedgerCache,
#enaLogging = true #profilingOk = ..,
finalDiskCleanUpOk = false,
#enaLoggingOk = ..,
#lastOneExtraOk = ..,
) )
noisy.say "***", "total elapsed: ", state[0].pp, " sections: ", state[1] noisy.say "***", "total elapsed: ", state[0].pp, " sections: ", state[1]

10
tests/test_coredb/coredb_test_xx.nim
View File

@ -92,6 +92,14 @@ let
# To be compared against the proof-of-concept implementation as reference # To be compared against the proof-of-concept implementation as reference
legaTest0* = CaptureSpecs(
builtIn: true,
name: ariTest0.name.replace("-am", "-lm"),
network: ariTest0.network,
files: ariTest0.files,
numBlocks: ariTest0.numBlocks,
dbType: LegacyDbMemory)
legaTest1* = CaptureSpecs( legaTest1* = CaptureSpecs(
builtIn: true, builtIn: true,
name: ariTest1.name.replace("-ar", "-lp"), name: ariTest1.name.replace("-ar", "-lp"),
@ -112,6 +120,6 @@ let
allSamples* = [ allSamples* = [
bulkTest0, bulkTest1, bulkTest2, bulkTest3, bulkTest0, bulkTest1, bulkTest2, bulkTest3,
ariTest0, ariTest1, ariTest2, ariTest0, ariTest1, ariTest2,
legaTest1, legaTest2] legaTest0, legaTest1, legaTest2]
# End # End