status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

remove pPrf, fRpp (#2445) 

No longer used now that hashify is gone
This commit is contained in:
Jacek Sieka 2024-07-03 22:21:57 +02:00 committed by GitHub
parent 443c6d1f8e
commit b23795ab39
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 29 additions and 597 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
nimbus/db/aristo/aristo_check/check_top.nim
View File

@ -61,19 +61,6 @@ proc checkTopStrict*(
proc checkTopProofMode*( proc checkTopProofMode*(
db: AristoDbRef; # Database, top layer db: AristoDbRef; # Database, top layer
): Result[void,(VertexID,AristoError)] = ): Result[void,(VertexID,AristoError)] =
if 0 < db.pPrf.len:
for vid in db.pPrf:
let vtx = db.layersGetVtxOrVoid vid
if vtx.isValid:
let node = vtx.toNode(db).valueOr:
return err((vid,CheckRlxVtxIncomplete))
let key = db.layersGetKeyOrVoid vid
if not key.isValid:
return err((vid,CheckRlxVtxKeyMissing))
if key != node.digestTo(HashKey):
return err((vid,CheckRlxVtxKeyMismatch))
else:
for (vid,key) in db.layersWalkKey: for (vid,key) in db.layersWalkKey:
if key.isValid: # Otherwise to be deleted if key.isValid: # Otherwise to be deleted
let vtx = db.getVtx vid let vtx = db.getVtx vid
@ -145,9 +132,6 @@ proc checkTopCommon*(
if kMapNilCount != 0 and kMapNilCount < nNilVtx: if kMapNilCount != 0 and kMapNilCount < nNilVtx:
return err((VertexID(0),CheckAnyVtxEmptyKeyMismatch)) return err((VertexID(0),CheckAnyVtxEmptyKeyMismatch))
for vid in db.pPrf:
if db.layersGetKey(vid).isErr:
return err((vid,CheckAnyVtxLockWithoutKey))
ok() ok()
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------

77
nimbus/db/aristo/aristo_debug.nim
View File

@ -86,18 +86,6 @@ proc stripZeros(a: string; toExp = false): string =
elif 2 < n: elif 2 < n:
result &= "↑" & $n result &= "↑" & $n
proc vidCode(key: HashKey, db: AristoDbRef): uint64 =
if key.isValid:
block:
let vid = db.layerGetProofVidOrVoid key
if vid.isValid:
db.xMap.add(key, vid)
return vid.uint64
block:
let vids = db.xMap.getOrVoid key
if vids.isValid:
return vids.sortedKeys[0].uint64
# --------------------- # ---------------------
proc ppKeyOk( proc ppKeyOk(
@ -106,11 +94,6 @@ proc ppKeyOk(
vid: VertexID; vid: VertexID;
): string = ): string =
if key.isValid and vid.isValid: if key.isValid and vid.isValid:
block:
let vid = db.layerGetProofVidOrVoid key
if vid.isValid:
db.xMap.add(key, vid)
return
block: block:
let vids = db.xMap.getOrVoid key let vids = db.xMap.getOrVoid key
if vids.isValid: if vids.isValid:
@ -162,11 +145,6 @@ proc ppVidList(vLst: openArray[VertexID]): string =
proc ppKey(key: HashKey; db: AristoDbRef; pfx = true): string = proc ppKey(key: HashKey; db: AristoDbRef; pfx = true): string =
proc getVids(): tuple[vids: HashSet[VertexID], xMapTag: string] = proc getVids(): tuple[vids: HashSet[VertexID], xMapTag: string] =
block:
let vid = db.layerGetProofVidOrVoid key
if vid.isValid:
db.xMap.add(key, vid)
return (@[vid].toHashSet, "")
block: block:
let vids = db.xMap.getOrVoid key let vids = db.xMap.getOrVoid key
if vids.isValid: if vids.isValid:
@ -221,7 +199,7 @@ proc ppVtx(nd: VertexRef, db: AristoDbRef, vid: VertexID): string =
if not nd.isValid: if not nd.isValid:
result = "ø" result = "ø"
else: else:
if not vid.isValid or vid in db.pPrf: if not vid.isValid:
result = ["L(", "X(", "B("][nd.vType.ord] result = ["L(", "X(", "B("][nd.vType.ord]
elif db.layersGetKey(vid).isOk: elif db.layersGetKey(vid).isOk:
result = ["l(", "x(", "b("][nd.vType.ord] result = ["l(", "x(", "b("][nd.vType.ord]
@ -250,21 +228,6 @@ proc ppSTab(
.mapIt("(" & it[0].ppVid & "," & it[1].ppVtx(db,it[0]) & ")") .mapIt("(" & it[0].ppVid & "," & it[1].ppVtx(db,it[0]) & ")")
.join(indent.toPfx(1)) & "}" .join(indent.toPfx(1)) & "}"
proc ppPPrf(pPrf: HashSet[VertexID]): string =
result = "{"
if 0 < pPrf.len:
let isr = IntervalSetRef[VertexID,uint64].init()
for w in pPrf:
doAssert isr.merge(w,w) == 1
for iv in isr.increasing():
result &= iv.minPt.ppVid
if 1 < iv.len:
result &= ".. " & iv.maxPt.ppVid
result &= ", "
result.setlen(result.len - 2)
#result &= pPrf.sortedKeys.mapIt(it.ppVid).join(",")
result &= "}"
proc ppXMap*( proc ppXMap*(
db: AristoDbRef; db: AristoDbRef;
kMap: Table[VertexID,HashKey]; kMap: Table[VertexID,HashKey];
@ -341,10 +304,7 @@ proc ppXMap*(
for vid in kMap.sortedKeys: for vid in kMap.sortedKeys:
let key = kMap.getOrVoid vid let key = kMap.getOrVoid vid
if key.isValid: if key.isValid:
cache.add (vid.uint64, key.vidCode(db), vid in multi) discard # TODO obsolete - clean up?
let vids = db.xMap.getOrVoid key
if (0 < vids.len and vid notin vids) or key.len < 32:
cache[^1][2] = true
else: else:
cache.add (vid.uint64, 0u64, true) cache.add (vid.uint64, 0u64, true)
@ -377,16 +337,6 @@ proc ppXMap*(
else: else:
result &= "}" result &= "}"
proc ppFRpp(
fRpp: Table[HashKey,VertexID];
db: AristoDbRef;
indent = 4;
): string =
let
xMap = fRpp.pairs.toSeq.mapIt((it[1],it[0])).toTable
xStr = db.ppXMap(xMap, indent)
"<" & xStr[1..^2] & ">"
proc ppFilter( proc ppFilter(
fl: LayerDeltaRef; fl: LayerDeltaRef;
db: AristoDbRef; db: AristoDbRef;
@ -468,14 +418,12 @@ proc ppLayer(
vTopOk: bool; vTopOk: bool;
sTabOk: bool; sTabOk: bool;
kMapOk: bool; kMapOk: bool;
pPrfOk: bool;
fRppOk: bool;
indent = 4; indent = 4;
): string = ): string =
let let
pfx1 = indent.toPfx(1) pfx1 = indent.toPfx(1)
pfx2 = indent.toPfx(2) pfx2 = indent.toPfx(2)
nOKs = vTopOk.ord + sTabOk.ord + kMapOk.ord + pPrfOk.ord + fRppOk.ord nOKs = vTopOk.ord + sTabOk.ord + kMapOk.ord
tagOk = 1 < nOKs tagOk = 1 < nOKs
var var
pfy = "" pfy = ""
@ -510,16 +458,6 @@ proc ppLayer(
info = "kMap(" & lInf & ")" info = "kMap(" & lInf & ")"
result &= info.doPrefix(0 < tLen + uLen) result &= info.doPrefix(0 < tLen + uLen)
result &= db.ppXMap(layer.delta.kMap, indent+2) result &= db.ppXMap(layer.delta.kMap, indent+2)
if pPrfOk:
let
tLen = layer.final.pPrf.len
info = "pPrf(" & $tLen & ")"
result &= info.doPrefix(0 < tLen) & layer.final.pPrf.ppPPrf
if fRppOk:
let
tLen = layer.final.fRpp.len
info = "fRpp(" & $tLen & ")"
result &= info.doPrefix(0 < tLen) & layer.final.fRpp.ppFRpp(db,indent+2)
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions # Public functions
@ -610,9 +548,6 @@ proc pp*(
): string = ): string =
sTab.ppSTab(db.orDefault) sTab.ppSTab(db.orDefault)
proc pp*(pPrf: HashSet[VertexID]): string =
pPrf.ppPPrf
proc pp*(leg: Leg; db = AristoDbRef(nil)): string = proc pp*(leg: Leg; db = AristoDbRef(nil)): string =
let db = db.orDefault() let db = db.orDefault()
result = "(" & leg.wp.vid.ppVid & "," result = "(" & leg.wp.vid.ppVid & ","
@ -669,7 +604,7 @@ proc pp*(
indent = 4; indent = 4;
): string = ): string =
layer.ppLayer( layer.ppLayer(
db, vTopOk=true, sTabOk=true, kMapOk=true, pPrfOk=true, fRppOk=true) db, vTopOk=true, sTabOk=true, kMapOk=true)
proc pp*( proc pp*(
layer: LayerRef; layer: LayerRef;
@ -678,7 +613,7 @@ proc pp*(
indent = 4; indent = 4;
): string = ): string =
layer.ppLayer( layer.ppLayer(
db, vTopOk=true, sTabOk=xTabOk, kMapOk=true, pPrfOk=true, fRppOk=true) db, vTopOk=true, sTabOk=xTabOk, kMapOk=true)
proc pp*( proc pp*(
layer: LayerRef; layer: LayerRef;
@ -689,7 +624,7 @@ proc pp*(
indent = 4; indent = 4;
): string = ): string =
layer.ppLayer( layer.ppLayer(
db, vTopOk=other, sTabOk=xTabOk, kMapOk=kMapOk, pPrfOk=other, fRppOk=other) db, vTopOk=other, sTabOk=xTabOk, kMapOk=kMapOk)
proc pp*( proc pp*(

6
nimbus/db/aristo/aristo_delete.nim
View File

@ -16,7 +16,7 @@
{.push raises: [].} {.push raises: [].}
import import
std/[sets, typetraits], std/typetraits,
eth/common, eth/common,
results, results,
"."/[aristo_desc, aristo_fetch, aristo_get, aristo_hike, aristo_layers, "."/[aristo_desc, aristo_fetch, aristo_get, aristo_hike, aristo_layers,
@ -274,8 +274,6 @@ proc deleteImpl(
let lf = hike.legs[^1].wp let lf = hike.legs[^1].wp
if lf.vtx.vType != Leaf: if lf.vtx.vType != Leaf:
return err((lf.vid,DelLeafExpexted)) return err((lf.vid,DelLeafExpexted))
if lf.vid in db.pPrf:
return err((lf.vid, DelLeafLocked))
db.disposeOfVtx(hike.root, lf.vid) db.disposeOfVtx(hike.root, lf.vid)
@ -295,8 +293,6 @@ proc deleteImpl(
# Clear all Merkle hash keys up to the root key # Clear all Merkle hash keys up to the root key
for n in 0 .. hike.legs.len - 2: for n in 0 .. hike.legs.len - 2:
let vid = hike.legs[n].wp.vid let vid = hike.legs[n].wp.vid
if vid in db.top.final.pPrf:
return err((vid, DelBranchLocked))
db.layersResKey(hike.root, vid) db.layersResKey(hike.root, vid)
let nibble = block: let nibble = block:

23
nimbus/db/aristo/aristo_desc/desc_structural.nim
View File

@ -15,7 +15,7 @@
{.push raises: [].} {.push raises: [].}
import import
std/[hashes, sets, tables], std/[hashes, tables],
eth/common, eth/common,
"."/[desc_error, desc_identifiers] "."/[desc_error, desc_identifiers]
@ -114,23 +114,11 @@ type
accSids*: Table[Hash256, VertexID] ## Account path -> stoID accSids*: Table[Hash256, VertexID] ## Account path -> stoID
LayerFinalRef* = ref object
## Final tables fully supersede tables on lower layers when stacked as a
## whole. Missing entries on a higher layers are the final state (for the
## the top layer version of the table.)
##
## These structures are used for tables which are typically smaller then
## the ones on the `LayerDelta` object.
##
pPrf*: HashSet[VertexID] ## Locked vertices (proof nodes)
fRpp*: Table[HashKey,VertexID] ## Key lookup for `pPrf[]` (proof nodes)
LayerRef* = ref LayerObj LayerRef* = ref LayerObj
LayerObj* = object LayerObj* = object
## Hexary trie database layer structures. Any layer holds the full ## Hexary trie database layer structures. Any layer holds the full
## change relative to the backend. ## change relative to the backend.
delta*: LayerDeltaRef ## Most structural tables held as deltas delta*: LayerDeltaRef ## Most structural tables held as deltas
final*: LayerFinalRef ## Stored as latest version
txUid*: uint ## Transaction identifier if positive txUid*: uint ## Transaction identifier if positive
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -139,8 +127,7 @@ type
func init*(T: type LayerRef): T = func init*(T: type LayerRef): T =
## Constructor, returns empty layer ## Constructor, returns empty layer
T(delta: LayerDeltaRef(), T(delta: LayerDeltaRef())
final: LayerFinalRef())
func hash*(node: NodeRef): Hash = func hash*(node: NodeRef): Hash =
## Table/KeyedQueue/HashSet mixin ## Table/KeyedQueue/HashSet mixin
@ -271,12 +258,6 @@ func dup*(node: NodeRef): NodeRef =
bVid: node.bVid, bVid: node.bVid,
key: node.key) key: node.key)
func dup*(final: LayerFinalRef): LayerFinalRef =
## Duplicate final layer.
LayerFinalRef(
pPrf: final.pPrf,
fRpp: final.fRpp)
func dup*(wp: VidVtxPair): VidVtxPair = func dup*(wp: VidVtxPair): VidVtxPair =
## Safe copy of `wp` argument ## Safe copy of `wp` argument
VidVtxPair( VidVtxPair(

3
nimbus/db/aristo/aristo_init/persistent.nim
View File

@ -52,8 +52,7 @@ proc newAristoRdbDbRef(
rc.value rc.value
ok((AristoDbRef( ok((AristoDbRef(
top: LayerRef( top: LayerRef(
delta: LayerDeltaRef(vTop: vTop), delta: LayerDeltaRef(vTop: vTop)),
final: LayerFinalRef()),
backend: be), oCfs)) backend: be), oCfs))
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------

40
nimbus/db/aristo/aristo_layers.nim
View File

@ -29,9 +29,6 @@ func dup(sTab: Table[VertexID,VertexRef]): Table[VertexID,VertexRef] =
# Public getters: lazy value lookup for read only versions # Public getters: lazy value lookup for read only versions
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
func pPrf*(db: AristoDbRef): lent HashSet[VertexID] =
db.top.final.pPrf
func vTop*(db: AristoDbRef): VertexID = func vTop*(db: AristoDbRef): VertexID =
db.top.delta.vTop db.top.delta.vTop
@ -94,20 +91,6 @@ func layersGetKeyOrVoid*(db: AristoDbRef; vid: VertexID): HashKey =
## Simplified version of `layersGetKey()` ## Simplified version of `layersGetKey()`
db.layersGetKey(vid).valueOr: VOID_HASH_KEY db.layersGetKey(vid).valueOr: VOID_HASH_KEY
func layerGetProofKeyOrVoid*(db: AristoDbRef; vid: VertexID): HashKey =
## Get the hash key of a proof node if it was registered as such.
if vid in db.top.final.pPrf:
db.top.delta.kMap.getOrVoid vid
else:
VOID_HASH_KEY
func layerGetProofVidOrVoid*(db: AristoDbRef; key: HashKey): VertexID =
## Reverse look up for a registered proof node or a link key for such a
## node. The vertex for a returned vertex ID might not exist if the
## argument `key` refers to a link key of a registered proof node.
db.top.final.fRpp.getOrVoid key
func layersGetStoID*(db: AristoDbRef; accPath: Hash256): Opt[VertexID] = func layersGetStoID*(db: AristoDbRef; accPath: Hash256): Opt[VertexID] =
db.top.delta.accSids.withValue(accPath, item): db.top.delta.accSids.withValue(accPath, item):
return Opt.some(item[]) return Opt.some(item[])
@ -156,27 +139,6 @@ func layersResKey*(db: AristoDbRef; root: VertexID; vid: VertexID) =
## equivalent of a delete function. ## equivalent of a delete function.
db.layersPutKey(root, vid, VOID_HASH_KEY) db.layersPutKey(root, vid, VOID_HASH_KEY)
func layersPutProof*(db: AristoDbRef; vid: VertexID; key: HashKey) =
## Register a link key of a proof node.
let lKey = db.layersGetKeyOrVoid vid
if not lKey.isValid or lKey != key:
db.top.delta.kMap[vid] = key
db.top.final.fRpp[key] = vid
func layersPutProof*(
db: AristoDbRef;
vid: VertexID;
key: HashKey;
vtx: VertexRef;
) =
## Register a full proof node (not only a link key.)
let lVtx = db.layersGetVtxOrVoid vid
if not lVtx.isValid or lVtx != vtx:
db.top.delta.sTab[vid] = vtx
db.top.final.pPrf.incl vid
db.layersPutProof(vid, key)
func layersPutStoID*(db: AristoDbRef; accPath: Hash256; stoID: VertexID) = func layersPutStoID*(db: AristoDbRef; accPath: Hash256; stoID: VertexID) =
db.top.delta.accSids[accPath] = stoID db.top.delta.accSids[accPath] = stoID
@ -188,7 +150,6 @@ func layersMergeOnto*(src: LayerRef; trg: var LayerObj) =
## Merges the argument `src` into the argument `trg` and returns `trg`. For ## Merges the argument `src` into the argument `trg` and returns `trg`. For
## the result layer, the `txUid` value set to `0`. ## the result layer, the `txUid` value set to `0`.
## ##
trg.final = src.final
trg.txUid = 0 trg.txUid = 0
for (vid,vtx) in src.delta.sTab.pairs: for (vid,vtx) in src.delta.sTab.pairs:
@ -209,7 +170,6 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
# Set up initial layer (bottom layer) # Set up initial layer (bottom layer)
result = LayerRef( result = LayerRef(
final: layers[^1].final.dup, # Pre-merged/final values
delta: LayerDeltaRef( delta: LayerDeltaRef(
sTab: layers[0].delta.sTab.dup, # explicit dup for ref values sTab: layers[0].delta.sTab.dup, # explicit dup for ref values
kMap: layers[0].delta.kMap, kMap: layers[0].delta.kMap,

5
nimbus/db/aristo/aristo_merge.nim
View File

@ -29,10 +29,7 @@ import
eth/common, eth/common,
results, results,
"."/[aristo_desc, aristo_fetch, aristo_layers, aristo_utils, aristo_vid], "."/[aristo_desc, aristo_fetch, aristo_layers, aristo_utils, aristo_vid],
./aristo_merge/[merge_payload_helper, merge_proof] ./aristo_merge/merge_payload_helper
export
merge_proof
const const
MergeNoAction = {MergeLeafPathCachedAlready, MergeLeafPathOnBackendAlready} MergeNoAction = {MergeLeafPathCachedAlready, MergeLeafPathOnBackendAlready}

29
nimbus/db/aristo/aristo_merge/merge_payload_helper.nim
View File

@ -11,7 +11,7 @@
{.push raises: [].} {.push raises: [].}
import import
std/[sets, typetraits], std/typetraits,
eth/common, eth/common,
results, results,
".."/[aristo_desc, aristo_get, aristo_hike, aristo_layers, aristo_vid] ".."/[aristo_desc, aristo_get, aristo_hike, aristo_layers, aristo_vid]
@ -100,10 +100,7 @@ proc insertBranch(
# Install `forkVtx` # Install `forkVtx`
block: block:
# Clear Merkle hashes (aka hash keys) unless proof mode. # Clear Merkle hashes (aka hash keys) unless proof mode.
if db.pPrf.len == 0:
db.clearMerkleKeys(hike, linkID) db.clearMerkleKeys(hike, linkID)
elif linkID in db.pPrf:
return err(MergeNonBranchProofModeLock)
if linkVtx.vType == Leaf: if linkVtx.vType == Leaf:
# Double check path prefix # Double check path prefix
@ -196,10 +193,7 @@ proc concatBranchAndLeaf(
return err(MergeRootBranchLinkBusy) return err(MergeRootBranchLinkBusy)
# Clear Merkle hashes (aka hash keys) unless proof mode. # Clear Merkle hashes (aka hash keys) unless proof mode.
if db.pPrf.len == 0:
db.clearMerkleKeys(hike, brVid) db.clearMerkleKeys(hike, brVid)
elif brVid in db.pPrf:
return err(MergeBranchProofModeLock) # Ooops
# Append branch vertex # Append branch vertex
var okHike = Hike(root: hike.root, legs: hike.legs) var okHike = Hike(root: hike.root, legs: hike.legs)
@ -264,7 +258,6 @@ proc mergePayloadTopIsBranchAddLeaf(
# #
# <-------- immutable ------------> <---- mutable ----> .. # <-------- immutable ------------> <---- mutable ----> ..
# #
if db.pPrf.len == 0:
# Not much else that can be done here # Not much else that can be done here
raiseAssert "Dangling edge:" & raiseAssert "Dangling edge:" &
" pfx=" & $hike.legsTo(hike.legs.len-1,NibblesBuf) & " pfx=" & $hike.legsTo(hike.legs.len-1,NibblesBuf) &
@ -273,18 +266,6 @@ proc mergePayloadTopIsBranchAddLeaf(
" edge=" & $linkID & " edge=" & $linkID &
" tail=" & $hike.tail " tail=" & $hike.tail
# Reuse placeholder entry in table
let vtx = VertexRef(
vType: Leaf,
lPfx: hike.tail,
lData: payload)
db.setVtxAndKey(hike.root, linkID, vtx)
var okHike = Hike(root: hike.root, legs: hike.legs)
okHike.legs.add Leg(wp: VidVtxPair(vid: linkID, vtx: vtx), nibble: -1)
if parent notin db.pPrf:
db.layersResKey(hike.root, parent)
return ok(okHike)
if linkVtx.vType == Branch: if linkVtx.vType == Branch:
# Slot link to a branch vertex should be handled by `hikeUp()` # Slot link to a branch vertex should be handled by `hikeUp()`
# #
@ -347,10 +328,7 @@ proc mergePayloadTopIsExtAddLeaf(
return err(MergeRootBranchLinkBusy) return err(MergeRootBranchLinkBusy)
# Clear Merkle hashes (aka hash keys) unless proof mode # Clear Merkle hashes (aka hash keys) unless proof mode
if db.pPrf.len == 0:
db.clearMerkleKeys(hike, brVid) db.clearMerkleKeys(hike, brVid)
elif brVid in db.pPrf:
return err(MergeBranchProofModeLock)
let let
brDup = brVtx.dup brDup = brVtx.dup
@ -382,10 +360,7 @@ proc mergePayloadTopIsEmptyAddLeaf(
return err(MergeRootBranchLinkBusy) return err(MergeRootBranchLinkBusy)
# Clear Merkle hashes (aka hash keys) unless proof mode # Clear Merkle hashes (aka hash keys) unless proof mode
if db.pPrf.len == 0:
db.clearMerkleKeys(hike, hike.root) db.clearMerkleKeys(hike, hike.root)
elif hike.root in db.pPrf:
return err(MergeBranchProofModeLock)
let let
rootDup = rootVtx.dup rootDup = rootVtx.dup
@ -416,8 +391,6 @@ proc mergePayloadUpdate(
# Update payloads if they differ # Update payloads if they differ
if leafLeg.wp.vtx.lData != payload: if leafLeg.wp.vtx.lData != payload:
let vid = leafLeg.wp.vid let vid = leafLeg.wp.vid
if vid in db.pPrf:
return err(MergeLeafProofModeLock)
# Update accounts storage root which is handled implicitly # Update accounts storage root which is handled implicitly
if hike.root == VertexID(1): if hike.root == VertexID(1):

364
nimbus/db/aristo/aristo_merge/merge_proof.nim
View File

@ -1,364 +0,0 @@
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
{.push raises: [].}
import
std/[algorithm, sets, tables],
eth/common,
results,
stew/keyed_queue,
../../../sync/protocol/snap/snap_types,
".."/[aristo_desc, aristo_get, aristo_layers, aristo_serialise, aristo_vid]
# ------------------------------------------------------------------------------
# Private functions: add Merkle proof node
# ------------------------------------------------------------------------------
proc mergeNodeImpl(
db: AristoDbRef; # Database, top layer
hashKey: HashKey; # Merkel hash of node (or so)
node: NodeRef; # Node derived from RLP representation
rootVid: VertexID; # Current sub-trie
): Result[void,AristoError] =
## The function merges the argument hash key `lid` as expanded from the
## node RLP representation into the `Aristo Trie` database. The vertex is
## split off from the node and stored separately. So are the Merkle hashes.
## The vertex is labelled `locked`.
##
## The `node` argument is *not* checked, whether the vertex IDs have been
## allocated, already. If the node comes straight from the `decode()` RLP
## decoder as expected, these vertex IDs will be all zero.
##
## This function expects that the parent for the argument `node` has already
## been installed.
##
## Caveat:
## Proof of concept, not in production yet.
##
# Check for error after RLP decoding
doAssert node.error == AristoError(0)
# Verify arguments
if not rootVid.isValid:
return err(MergeRootKeyInvalid)
if not hashKey.isValid:
return err(MergeHashKeyInvalid)
# Make sure that the `vid<->key` reverse mapping is updated.
let vid = db.layerGetProofVidOrVoid hashKey
if not vid.isValid:
return err(MergeRevVidMustHaveBeenCached)
# Use the vertex ID `vid` to be populated by the argument root node
let key = db.layersGetKeyOrVoid vid
if key.isValid and key != hashKey:
return err(MergeHashKeyDiffersFromCached)
# Set up vertex.
let (vtx, newVtxFromNode) = block:
let vty = db.getVtx vid
if vty.isValid:
(vty, false)
else:
(node.to(VertexRef), true)
# The `vertexID <-> hashKey` mappings need to be set up now (if any)
case node.vType:
of Leaf:
# Check whether there is need to convert the payload to `Account` payload
if rootVid == VertexID(1) and newVtxFromNode:
try:
let
# `aristo_serialise.read()` always decodes raw data payloaf
acc = rlp.decode(node.lData.rawBlob, Account)
pyl = PayloadRef(
pType: AccountData,
account: AristoAccount(
nonce: acc.nonce,
balance: acc.balance,
codeHash: acc.codeHash))
if acc.storageRoot.isValid:
var sid = db.layerGetProofVidOrVoid acc.storageRoot.to(HashKey)
if not sid.isValid:
sid = db.vidFetch
db.layersPutProof(sid, acc.storageRoot.to(HashKey))
pyl.stoID = sid
vtx.lData = pyl
except RlpError:
return err(MergeNodeAccountPayloadError)
of Extension:
if node.key[0].isValid:
let eKey = node.key[0]
if newVtxFromNode:
vtx.eVid = db.layerGetProofVidOrVoid eKey
if not vtx.eVid.isValid:
# Brand new reverse lookup link for this vertex
vtx.eVid = db.vidFetch
elif not vtx.eVid.isValid:
return err(MergeNodeVidMissing)
else:
let yEke = db.getKey vtx.eVid
if yEke.isValid and eKey != yEke:
return err(MergeNodeVtxDiffersFromExisting)
db.layersPutProof(vtx.eVid, eKey)
of Branch:
for n in 0..15:
if node.key[n].isValid:
let bKey = node.key[n]
if newVtxFromNode:
vtx.bVid[n] = db.layerGetProofVidOrVoid bKey
if not vtx.bVid[n].isValid:
# Brand new reverse lookup link for this vertex
vtx.bVid[n] = db.vidFetch
elif not vtx.bVid[n].isValid:
return err(MergeNodeVidMissing)
else:
let yEkb = db.getKey vtx.bVid[n]
if yEkb.isValid and yEkb != bKey:
return err(MergeNodeVtxDiffersFromExisting)
db.layersPutProof(vtx.bVid[n], bKey)
# Store and lock vertex
db.layersPutProof(vid, key, vtx)
ok()
# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
proc mergeProof*(
db: AristoDbRef; # Database, top layer
proof: openArray[SnapProof]; # RLP encoded node records
rootVid = VertexID(0); # Current sub-trie
): Result[int, AristoError]
{.gcsafe, raises: [RlpError].} =
## The function merges the argument `proof` list of RLP encoded node records
## into the `Aristo Trie` database. This function is intended to be used with
## the proof nodes as returened by `snap/1` messages.
##
## If there is no root vertex ID passed, the function tries to find out what
## the root hashes are and allocates new vertices with static IDs `$2`, `$3`,
## etc.
##
## Caveat:
## Proof of concept, not in production yet.
##
proc update(
seen: var Table[HashKey,NodeRef];
todo: var KeyedQueueNV[NodeRef];
key: HashKey;
) {.gcsafe, raises: [RlpError].} =
## Check for embedded nodes, i.e. fully encoded node instead of a hash.
## They need to be treated as full nodes, here.
if key.isValid and key.len < 32:
let lid = key.data.digestTo(HashKey)
if not seen.hasKey lid:
let node = key.data.decode(NodeRef)
discard todo.append node
seen[lid] = node
let rootKey = block:
if rootVid.isValid:
let vidKey = db.getKey rootVid
if not vidKey.isValid:
return err(MergeRootKeyInvalid)
# Make sure that the reverse lookup for the root vertex key is available.
if not db.layerGetProofVidOrVoid(vidKey).isValid:
return err(MergeProofInitMissing)
vidKey
else:
VOID_HASH_KEY
# Expand and collect hash keys and nodes and parent indicator
var
nodeTab: Table[HashKey,NodeRef]
rootKeys: HashSet[HashKey] # Potential root node hashes
for w in proof:
let
key = w.Blob.digestTo(HashKey)
node = rlp.decode(w.Blob,NodeRef)
if node.error != AristoError(0):
return err(node.error)
nodeTab[key] = node
rootKeys.incl key
# Check for embedded nodes, i.e. fully encoded node instead of a hash.
# They will be added as full nodes to the `nodeTab[]`.
var embNodes: KeyedQueueNV[NodeRef]
discard embNodes.append node
while true:
let node = embNodes.shift.valueOr: break
case node.vType:
of Leaf:
discard
of Branch:
for n in 0 .. 15:
nodeTab.update(embNodes, node.key[n])
of Extension:
nodeTab.update(embNodes, node.key[0])
# Create a table with back links
var
backLink: Table[HashKey,HashKey]
blindNodes: HashSet[HashKey]
for (key,node) in nodeTab.pairs:
case node.vType:
of Leaf:
blindNodes.incl key
of Extension:
if nodeTab.hasKey node.key[0]:
backLink[node.key[0]] = key
rootKeys.excl node.key[0] # predecessor => not root
else:
blindNodes.incl key
of Branch:
var isBlind = true
for n in 0 .. 15:
if nodeTab.hasKey node.key[n]:
isBlind = false
backLink[node.key[n]] = key
rootKeys.excl node.key[n] # predecessor => not root
if isBlind:
blindNodes.incl key
# If it exists, the root key must be in the set `mayBeRoot` in order
# to work.
var roots: Table[HashKey,VertexID]
if rootVid.isValid:
if rootKey notin rootKeys:
return err(MergeRootKeyNotInProof)
roots[rootKey] = rootVid
elif rootKeys.len == 0:
return err(MergeRootKeysMissing)
else:
# Add static root keys different from VertexID(1)
var count = 2
for key in rootKeys.items:
while true:
# Check for already allocated nodes
let vid1 = db.layerGetProofVidOrVoid key
if vid1.isValid:
roots[key] = vid1
break
# Use the next free static free vertex ID
let vid2 = VertexID(count)
count.inc
if not db.getKey(vid2).isValid:
doAssert not db.layerGetProofVidOrVoid(key).isValid
db.layersPutProof(vid2, key)
roots[key] = vid2
break
if LEAST_FREE_VID <= count:
return err(MergeRootKeysOverflow)
# Run over blind nodes and build chains from a blind/bottom level node up
# to the root node. Select only chains that end up at the pre-defined root
# node.
var
accounts: seq[seq[HashKey]] # This one separated, to be processed last
chains: seq[seq[HashKey]]
for w in blindNodes:
# Build a chain of nodes up to the root node
var
chain: seq[HashKey]
nodeKey = w
while nodeKey.isValid and nodeTab.hasKey nodeKey:
chain.add nodeKey
nodeKey = backLink.getOrVoid nodeKey
if 0 < chain.len and chain[^1] in roots:
if roots.getOrVoid(chain[0]) == VertexID(1):
accounts.add chain
else:
chains.add chain
# Process over chains in reverse mode starting with the root node. This
# allows the algorithm to find existing nodes on the backend.
var
seen: HashSet[HashKey]
merged = 0
# Process the root ID which is common to all chains
for chain in chains & accounts:
let chainRootVid = roots.getOrVoid chain[^1]
for key in chain.reversed:
if key notin seen:
seen.incl key
let node = nodeTab.getOrVoid key
db.mergeNodeImpl(key, node, chainRootVid).isOkOr:
return err(error)
merged.inc
ok merged
proc mergeProof*(
db: AristoDbRef; # Database, top layer
rootHash: Hash256; # Merkle hash for root
rootVid = VertexID(0); # Optionally, force root vertex ID
): Result[VertexID,AristoError] =
## Set up a `rootKey` associated with a vertex ID for use with proof nodes.
##
## If argument `rootVid` is unset then a new dybamic root vertex (i.e.
## the ID will be at least `LEAST_FREE_VID`) will be installed.
##
## Otherwise, if the argument `rootVid` is set then a sub-trie with root
## `rootVid` is checked for. An error is returned if it is set up already
## with a different `rootHash`.
##
## Upon successful return, the vertex ID assigned to the root key is returned.
##
## Caveat:
## Proof of concept, not in production yet.
##
let rootKey = rootHash.to(HashKey)
if rootVid.isValid:
let key = db.getKey rootVid
if key.isValid:
if rootKey.isValid and key != rootKey:
# Cannot use installed root key differing from hash argument
return err(MergeRootKeyDiffersForVid)
# Confirm root ID and key for proof nodes processing
db.layersPutProof(rootVid, key) # note that `rootKey` might be void
return ok rootVid
if not rootHash.isValid:
return err(MergeRootArgsIncomplete)
if db.getVtx(rootVid).isValid:
# Cannot use verify root key for existing root vertex
return err(MergeRootKeyMissing)
# Confirm root ID and hash key for proof nodes processing
db.layersPutProof(rootVid, rootKey)
return ok rootVid
if not rootHash.isValid:
return err(MergeRootArgsIncomplete)
# Now there is no root vertex ID, only the hash argument.
# So Create and assign a new root key.
let vid = db.vidFetch
db.layersPutProof(vid, rootKey)
return ok vid
proc mergeProof*(
db: AristoDbRef; # Database, top layer
rootVid: VertexID; # Root ID
): Result[VertexID,AristoError] =
## Variant of `mergeProof()` for missing `rootHash`
db.mergeProof(EMPTY_ROOT_HASH, rootVid)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

3
nimbus/db/aristo/aristo_tx/tx_fork.nim
View File

@ -65,8 +65,7 @@ proc txFork*(
let rc = db.getTuvBE() let rc = db.getTuvBE()
if rc.isOk: if rc.isOk:
LayerRef( LayerRef(
delta: LayerDeltaRef(vTop: rc.value), delta: LayerDeltaRef(vTop: rc.value))
final: LayerFinalRef())
elif rc.error == GetTuvNotFound: elif rc.error == GetTuvNotFound:
LayerRef.init() LayerRef.init()
else: else:

1
nimbus/db/aristo/aristo_tx/tx_frame.nim
View File

@ -85,7 +85,6 @@ proc txFrameBegin*(db: AristoDbRef): Result[AristoTxRef,AristoError] =
db.stack.add db.top db.stack.add db.top
db.top = LayerRef( db.top = LayerRef(
delta: LayerDeltaRef(vTop: vTop), delta: LayerDeltaRef(vTop: vTop),
final: db.top.final.dup,
txUid: db.getTxUid) txUid: db.getTxUid)
db.txRef = AristoTxRef( db.txRef = AristoTxRef(

9
nimbus/db/aristo/aristo_tx/tx_stow.nim
View File

@ -100,11 +100,6 @@ proc txStow*(
if rc.isErr and rc.error != TxPrettyPointlessLayer: if rc.isErr and rc.error != TxPrettyPointlessLayer:
return err(rc.error) return err(rc.error)
# Special treatment for `snap` proofs (aka `chunkedMpt`)
let final =
if chunkedMpt: LayerFinalRef(fRpp: db.top.final.fRpp)
else: LayerFinalRef()
# Move/merge/install `top` layer onto `balancer` # Move/merge/install `top` layer onto `balancer`
if rc.isOk: if rc.isOk:
db.topMerge(rc.value).isOkOr: db.topMerge(rc.value).isOkOr:
@ -112,8 +107,7 @@ proc txStow*(
# New empty top layer (probably with `snap` proofs and `vTop` carry over) # New empty top layer (probably with `snap` proofs and `vTop` carry over)
db.top = LayerRef( db.top = LayerRef(
delta: LayerDeltaRef(), delta: LayerDeltaRef())
final: final)
if db.balancer.isValid: if db.balancer.isValid:
db.top.delta.vTop = db.balancer.vTop db.top.delta.vTop = db.balancer.vTop
else: else:
@ -137,7 +131,6 @@ proc txStow*(
# New empty top layer (probably with `snap` proofs carry over) # New empty top layer (probably with `snap` proofs carry over)
db.top = LayerRef( db.top = LayerRef(
delta: LayerDeltaRef(vTop: db.vTop), delta: LayerDeltaRef(vTop: db.vTop),
final: final,
txUid: db.top.txUid) txUid: db.top.txUid)
ok() ok()

1
nimbus/db/core_db/core_apps.nim
View File

@ -20,7 +20,6 @@ import
stew/byteutils, stew/byteutils,
"../.."/[errors, constants], "../.."/[errors, constants],
".."/[aristo, storage_types], ".."/[aristo, storage_types],
./backend/aristo_db,
"."/base "."/base
logScope: logScope:

14
tests/test_aristo/test_tx.nim
View File

@ -27,7 +27,6 @@ import
aristo_hike, aristo_hike,
aristo_init/persistent, aristo_init/persistent,
aristo_layers, aristo_layers,
aristo_merge,
aristo_nearby, aristo_nearby,
aristo_tx], aristo_tx],
../replay/xcheck, ../replay/xcheck,
@ -569,19 +568,6 @@ proc testTxMergeProofAndKvpList*(
# var lst = w.kvpLst.mapRootVid testRootVid # var lst = w.kvpLst.mapRootVid testRootVid
if 0 < w.proof.len:
let root = block:
let rc = db.mergeProof(rootKey, testRootVid)
xCheckRc rc.error == 0
rc.value
let nMerged = block:
let rc = db.mergeProof(w.proof, root)
xCheckRc rc.error == 0
rc.value
xCheck w.proof.len == nMerged
xCheck db.nLayersVtx() <= nMerged + sTabLen
let merged = db.mergeList leafs let merged = db.mergeList leafs
xCheck merged.error in {AristoError(0), MergeLeafPathCachedAlready} xCheck merged.error in {AristoError(0), MergeLeafPathCachedAlready}

5
tests/test_tracer_json.nim
View File

@ -70,11 +70,6 @@ proc preLoadAristoDb(cdb: CoreDbRef; jKvp: JsonNode; num: BlockNumber) =
# Set up production MPT # Set up production MPT
doAssert adb.mergeProof(proof).isOk doAssert adb.mergeProof(proof).isOk
# Remove locks so that hashify can re-assign changed nodes
adb.top.final.pPrf.clear
adb.top.final.fRpp.clear
# use tracerTestGen.nim to generate additional test data # use tracerTestGen.nim to generate additional test data
proc testFixtureImpl(node: JsonNode, testStatusIMPL: var TestStatus, memoryDB: CoreDbRef) = proc testFixtureImpl(node: JsonNode, testStatusIMPL: var TestStatus, memoryDB: CoreDbRef) =
setErrorLevel() setErrorLevel()