380 lines
13 KiB
Nim
380 lines
13 KiB
Nim
# nimbus-eth1
|
|
# Copyright (c) 2021 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or distributed
|
|
# except according to those terms.
|
|
|
|
import
|
|
std/[sequtils, strutils, tables],
|
|
chronicles,
|
|
eth/[common, trie/nibbles],
|
|
stew/results,
|
|
../../range_desc,
|
|
"."/[hexary_desc, hexary_paths]
|
|
|
|
{.push raises: [].}
|
|
|
|
logScope:
|
|
topics = "snap-db"
|
|
|
|
type
|
|
TrieNodeStatCtxRef* = ref object
|
|
## Context to resume searching for dangling links
|
|
case persistent*: bool
|
|
of true:
|
|
hddCtx*: seq[(NodeKey,NibblesSeq)]
|
|
else:
|
|
memCtx*: seq[(RepairKey,NibblesSeq)]
|
|
|
|
TrieNodeStat* = object
|
|
## Trie inspection report
|
|
dangling*: seq[NodeSpecs] ## Referes to nodes with incomplete refs
|
|
count*: uint64 ## Number of nodes visited
|
|
level*: uint8 ## Maximum nesting depth of dangling nodes
|
|
stopped*: bool ## Potential loop detected if `true`
|
|
resumeCtx*: TrieNodeStatCtxRef ## Context for resuming inspection
|
|
|
|
const
|
|
extraTraceMessages = false # or true
|
|
|
|
when extraTraceMessages:
|
|
import stew/byteutils
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private helpers, debugging
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc ppDangling(a: seq[NodeSpecs]; maxItems = 30): string =
|
|
proc ppBlob(w: Blob): string =
|
|
w.mapIt(it.toHex(2)).join.toLowerAscii
|
|
let
|
|
q = a.mapIt(it.partialPath.ppBlob)[0 ..< min(maxItems,a.len)]
|
|
andMore = if maxItems < a.len: ", ..[#" & $a.len & "].." else: ""
|
|
"{" & q.join(",") & andMore & "}"
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc convertTo(key: RepairKey; T: type NodeKey): T =
|
|
## Might be lossy, check before use
|
|
discard result.init(key.ByteArray33[1 .. 32])
|
|
|
|
proc convertTo(key: Blob; T: type NodeKey): T =
|
|
## Might be lossy, check before use
|
|
discard result.init(key)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc processLink(
|
|
db: HexaryTreeDbRef;
|
|
stats: var TrieNodeStat;
|
|
inspect: var seq[(RepairKey,NibblesSeq)];
|
|
trail: NibblesSeq;
|
|
child: RepairKey;
|
|
) =
|
|
## Helper for `hexaryInspect()`
|
|
if not child.isZero:
|
|
if not child.isNodeKey:
|
|
# Oops -- caught in the middle of a repair process? Just register
|
|
# this node
|
|
stats.dangling.add NodeSpecs(
|
|
partialPath: trail.hexPrefixEncode(isLeaf = false))
|
|
elif db.tab.hasKey(child):
|
|
inspect.add (child,trail)
|
|
else:
|
|
stats.dangling.add NodeSpecs(
|
|
partialPath: trail.hexPrefixEncode(isLeaf = false),
|
|
nodeKey: child.convertTo(NodeKey))
|
|
|
|
proc processLink(
|
|
getFn: HexaryGetFn;
|
|
stats: var TrieNodeStat;
|
|
inspect: var seq[(NodeKey,NibblesSeq)];
|
|
trail: NibblesSeq;
|
|
child: Rlp;
|
|
) {.gcsafe, raises: [RlpError]} =
|
|
## Ditto
|
|
if not child.isEmpty:
|
|
let childBlob = child.toBytes
|
|
if childBlob.len != 32:
|
|
# Oops -- that is wrong, although the only sensible action is to
|
|
# register the node and otherwise ignore it
|
|
stats.dangling.add NodeSpecs(
|
|
partialPath: trail.hexPrefixEncode(isLeaf = false))
|
|
else:
|
|
let childKey = childBlob.convertTo(NodeKey)
|
|
if 0 < child.toBytes.getFn().len:
|
|
inspect.add (childKey,trail)
|
|
else:
|
|
stats.dangling.add NodeSpecs(
|
|
partialPath: trail.hexPrefixEncode(isLeaf = false),
|
|
nodeKey: childKey)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc to*(resumeCtx: TrieNodeStatCtxRef; T: type seq[NodeSpecs]): T =
|
|
## Convert resumption context to nodes that can be used otherwise. This
|
|
## function might be useful for error recovery.
|
|
##
|
|
## Note: In a non-persistant case, temporary `RepairKey` type node specs
|
|
## that cannot be converted to `NodeKey` type nodes are silently dropped.
|
|
## This should be no problem as a hexary trie with `RepairKey` type node
|
|
## refs must be repaired or discarded anyway.
|
|
if resumeCtx.persistent:
|
|
for (key,trail) in resumeCtx.hddCtx:
|
|
result.add NodeSpecs(
|
|
partialPath: trail.hexPrefixEncode(isLeaf = false),
|
|
nodeKey: key)
|
|
else:
|
|
for (key,trail) in resumeCtx.memCtx:
|
|
if key.isNodeKey:
|
|
result.add NodeSpecs(
|
|
partialPath: trail.hexPrefixEncode(isLeaf = false),
|
|
nodeKey: key.convertTo(NodeKey))
|
|
|
|
|
|
proc hexaryInspectTrie*(
|
|
db: HexaryTreeDbRef; # Database
|
|
root: NodeKey; # State root
|
|
partialPaths: seq[Blob] = @[]; # Starting paths for search
|
|
resumeCtx: TrieNodeStatCtxRef = nil; # Context for resuming inspection
|
|
suspendAfter = high(uint64); # To be resumed
|
|
stopAtLevel = 64u8; # Width-first depth level
|
|
maxDangling = high(int); # Maximal number of dangling results
|
|
): TrieNodeStat
|
|
{.gcsafe, raises: [KeyError]} =
|
|
## Starting with the argument list `paths`, find all the non-leaf nodes in
|
|
## the hexary trie which have at least one node key reference missing in
|
|
## the trie database. The references for these nodes are collected and
|
|
## returned.
|
|
##
|
|
## * Argument `partialPaths` list entries that do not refer to an existing
|
|
## and allocated hexary trie node are silently ignored. So are enytries
|
|
## that not refer to either a valid extension or a branch type node.
|
|
##
|
|
## * This function traverses the hexary trie in *width-first* mode
|
|
## simultaneously for any entry of the argument `partialPaths` list. Abart
|
|
## from completing the search there are three conditions when the search
|
|
## pauses to return the current state (via `resumeCtx`, see next bullet
|
|
## point):
|
|
## + The depth level of the running algorithm exceeds `stopAtLevel`.
|
|
## + The number of visited nodes exceeds `suspendAfter`.
|
|
## + Te number of cunnently collected dangling nodes exceeds `maxDangling`.
|
|
## If the function pauses because the current depth exceeds `stopAtLevel`
|
|
## then the `stopped` flag of the result object will be set, as well.
|
|
##
|
|
## * When paused for some of the reasons listed above, the `resumeCtx` field
|
|
## of the result object contains the current state so that the function
|
|
## can resume searching from where is paused. An application using this
|
|
## feature could look like:
|
|
## ::
|
|
## var ctx = TrieNodeStatCtxRef()
|
|
## while not ctx.isNil:
|
|
## let state = hexaryInspectTrie(db, root, paths, resumeCtx=ctx, 1024)
|
|
## ...
|
|
## ctx = state.resumeCtx
|
|
##
|
|
let rootKey = root.to(RepairKey)
|
|
if not db.tab.hasKey(rootKey):
|
|
return TrieNodeStat()
|
|
|
|
var
|
|
reVisit: seq[(RepairKey,NibblesSeq)]
|
|
again: seq[(RepairKey,NibblesSeq)]
|
|
resumeOk = false
|
|
|
|
# Initialise lists from previous session
|
|
if not resumeCtx.isNil and
|
|
not resumeCtx.persistent and
|
|
0 < resumeCtx.memCtx.len:
|
|
resumeOk = true
|
|
reVisit = resumeCtx.memCtx
|
|
|
|
if partialPaths.len == 0 and not resumeOk:
|
|
reVisit.add (rootKey,EmptyNibbleRange)
|
|
else:
|
|
# Add argument paths
|
|
for w in partialPaths:
|
|
let (isLeaf,nibbles) = hexPrefixDecode w
|
|
if not isLeaf:
|
|
let rc = nibbles.hexaryPathNodeKey(rootKey, db, missingOk=false)
|
|
if rc.isOk:
|
|
reVisit.add (rc.value.to(RepairKey), nibbles)
|
|
|
|
# Stopping on `suspendAfter` has precedence over `stopAtLevel`
|
|
while 0 < reVisit.len and result.count <= suspendAfter:
|
|
if stopAtLevel < result.level:
|
|
result.stopped = true
|
|
break
|
|
|
|
for n in 0 ..< reVisit.len:
|
|
if suspendAfter < result.count or
|
|
maxDangling <= result.dangling.len:
|
|
# Swallow rest
|
|
again &= reVisit[n ..< reVisit.len]
|
|
break
|
|
|
|
let
|
|
(rKey, parentTrail) = reVisit[n]
|
|
node = db.tab[rKey]
|
|
# parent = rKey.convertTo(NodeKey) -- unused
|
|
|
|
case node.kind:
|
|
of Extension:
|
|
let
|
|
trail = parentTrail & node.ePfx
|
|
child = node.eLink
|
|
db.processLink(stats=result, inspect=again, trail, child)
|
|
of Branch:
|
|
for n in 0 ..< 16:
|
|
let
|
|
trail = parentTrail & @[n.byte].initNibbleRange.slice(1)
|
|
child = node.bLink[n]
|
|
db.processLink(stats=result, inspect=again, trail, child)
|
|
of Leaf:
|
|
# Ooops, forget node and key
|
|
discard
|
|
|
|
result.count.inc
|
|
# End `for`
|
|
|
|
result.level.inc
|
|
swap(reVisit, again)
|
|
again.setLen(0)
|
|
# End while
|
|
|
|
if 0 < reVisit.len:
|
|
result.resumeCtx = TrieNodeStatCtxRef(
|
|
persistent: false,
|
|
memCtx: reVisit)
|
|
|
|
|
|
proc hexaryInspectTrie*(
|
|
getFn: HexaryGetFn; # Database abstraction
|
|
rootKey: NodeKey; # State root
|
|
partialPaths: seq[Blob] = @[]; # Starting paths for search
|
|
resumeCtx: TrieNodeStatCtxRef = nil; # Context for resuming inspection
|
|
suspendAfter = high(uint64); # To be resumed
|
|
stopAtLevel = 64u8; # Width-first depth level
|
|
maxDangling = high(int); # Maximal number of dangling results
|
|
): TrieNodeStat
|
|
{.gcsafe, raises: [RlpError]} =
|
|
## Variant of `hexaryInspectTrie()` for persistent database.
|
|
when extraTraceMessages:
|
|
let nPaths = paths.len
|
|
|
|
let root = rootKey.to(Blob)
|
|
if root.getFn().len == 0:
|
|
when extraTraceMessages:
|
|
trace "Hexary inspect: missing root", nPaths, maxLeafPaths,
|
|
rootKey=root.toHex
|
|
return TrieNodeStat()
|
|
|
|
var
|
|
reVisit: seq[(NodeKey,NibblesSeq)]
|
|
again: seq[(NodeKey,NibblesSeq)]
|
|
resumeOk = false
|
|
|
|
# Initialise lists from previous session
|
|
if not resumeCtx.isNil and
|
|
resumeCtx.persistent and
|
|
0 < resumeCtx.hddCtx.len:
|
|
resumeOk = true
|
|
reVisit = resumeCtx.hddCtx
|
|
|
|
if partialPaths.len == 0 and not resumeOk:
|
|
reVisit.add (rootKey,EmptyNibbleRange)
|
|
else:
|
|
# Add argument paths
|
|
for w in partialPaths:
|
|
let (isLeaf,nibbles) = hexPrefixDecode w
|
|
if not isLeaf:
|
|
let rc = nibbles.hexaryPathNodeKey(rootKey, getFn, missingOk=false)
|
|
if rc.isOk:
|
|
reVisit.add (rc.value, nibbles)
|
|
|
|
# Stopping on `suspendAfter` has precedence over `stopAtLevel`
|
|
while 0 < reVisit.len and result.count <= suspendAfter:
|
|
when extraTraceMessages:
|
|
trace "Hexary inspect processing", nPaths, maxLeafPaths,
|
|
level=result.level, nReVisit=reVisit.len, nDangling=result.dangling.len
|
|
|
|
if stopAtLevel < result.level:
|
|
result.stopped = true
|
|
break
|
|
|
|
for n in 0 ..< reVisit.len:
|
|
if suspendAfter < result.count or
|
|
maxDangling <= result.dangling.len:
|
|
# Swallow rest
|
|
again = again & reVisit[n ..< reVisit.len]
|
|
break
|
|
|
|
let
|
|
(parent, parentTrail) = reVisit[n]
|
|
parentBlob = parent.to(Blob).getFn()
|
|
if parentBlob.len == 0:
|
|
# Ooops, forget node and key
|
|
continue
|
|
|
|
let nodeRlp = rlpFromBytes parentBlob
|
|
case nodeRlp.listLen:
|
|
of 2:
|
|
let (isLeaf,xPfx) = hexPrefixDecode nodeRlp.listElem(0).toBytes
|
|
if not isleaf:
|
|
let
|
|
trail = parentTrail & xPfx
|
|
child = nodeRlp.listElem(1)
|
|
getFn.processLink(stats=result, inspect=again, trail, child)
|
|
of 17:
|
|
for n in 0 ..< 16:
|
|
let
|
|
trail = parentTrail & @[n.byte].initNibbleRange.slice(1)
|
|
child = nodeRlp.listElem(n)
|
|
getFn.processLink(stats=result, inspect=again, trail, child)
|
|
else:
|
|
# Ooops, forget node and key
|
|
discard
|
|
|
|
result.count.inc
|
|
# End `for`
|
|
|
|
result.level.inc
|
|
swap(reVisit, again)
|
|
again.setLen(0)
|
|
# End while
|
|
|
|
if 0 < reVisit.len:
|
|
result.resumeCtx = TrieNodeStatCtxRef(
|
|
persistent: true,
|
|
hddCtx: reVisit)
|
|
|
|
when extraTraceMessages:
|
|
trace "Hexary inspect finished", nPaths, maxLeafPaths,
|
|
level=result.level, nResumeCtx=reVisit.len, nDangling=result.dangling.len,
|
|
maxLevel=stopAtLevel, stopped=result.stopped
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public functions, debugging
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc pp*(a: TrieNodeStat; db: HexaryTreeDbRef; maxItems = 30): string =
|
|
result = "(" & $a.level
|
|
if a.stopped:
|
|
result &= "stopped,"
|
|
result &= $a.dangling.len & "," &
|
|
a.dangling.ppDangling(maxItems) & ")"
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|