status-im/nimbus-eth1
2
0
Fork 0
mirror of https://github.com/status-im/nimbus-eth1.git synced 2025-01-11 21:04:11 +00:00
Code Issues Projects Releases Wiki Activity

Prepare snap server client test scenario cont4 (#1507)

Browse Source 
* Add state root to node steps path register `RPath` or `XPath`

why:
  Typically, the first node in the path register is the state root. There
  are occasions, when the path register is empty (i.e. there are no node
  references) which typically applies to a zero node key.

  In order to find the next node key greater than zero, the state root is
  is needed which is now part of the `RPath` or `XPath` data types.

* Extracted hexary tree debugging functions into separate files

* Update empty path fringe case for left/right node neighbour

why:
  When starting at zero, the node steps path register would be empty. So
  will any path that is before the fist non-zero link of a state root (if
  it is a `Branch` node.)

  The `hexaryNearbyRight()` or `hexaryNearbyLeft()` function required a
  non-zero node steps path register.  Now the first node is to be advanced
  starting at the first state root link if necessary.

* Simplify/reorg neighbour node finder

why:
  There was too mach code repetition for the cases
  * persistent or in-memory database
  * left or right move

details:
  Most algorithms apply for persistent and in-memory alike. Using
  templates/generic functions most of these algorithms can be stated
  in a unified way

* Update storage slots snap/1 handler

details:
  Minor changes to be more debugging friendly.

* Fix detection of full database for snap sync

* Docu: Snap sync test & debugging scenario
This commit is contained in:
Jordan Hrycaj 2023-03-17 14:46:50 +00:00 committed by GitHub
parent 11fc2de060
commit 15d0ccb39c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 1473 additions and 1377 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
nimbus/sync/handlers/snap.nim
View File

@ -11,7 +11,7 @@
{.push raises: [].}
import
std/sequtils,
std/[sequtils, strutils],
chronicles,
chronos,
eth/[p2p, trie/trie_defs],
@ -19,7 +19,7 @@ import
../../db/db_chain,
../../core/chain,
../snap/[constants, range_desc],
../snap/worker/db/[hexary_desc, hexary_paths, hexary_range],
../snap/worker/db/[hexary_desc, hexary_error, hexary_paths, hexary_range],
../protocol,
../protocol/snap/snap_types
@ -43,7 +43,7 @@ const
emptySnapStorageList = seq[SnapStorage].default
## Dummy list for empty slots
defaultElaFetchMax = 1500.milliseconds
defaultElaFetchMax = 990.milliseconds
## Fetching accounts or slots can be extensive, stop in the middle if
## it takes too long
@ -70,7 +70,7 @@ proc getAccountFn(
return proc(key: openArray[byte]): Blob =
db.get(key)
proc getStorageSlotsFn(
proc getStoSlotFn(
chain: ChainRef;
accKey: NodeKey;
): HexaryGetFn
@ -106,26 +106,37 @@ proc to(
proc mkNodeTagRange(
origin: openArray[byte];
limit: openArray[byte];
nAccounts = 1;
): Result[NodeTagRange,void] =
var (minPt, maxPt) = (low(NodeTag), high(NodeTag))
if 0 < origin.len or 0 < limit.len:
# Range applies only if there is exactly one account. A number of accounts
# different from 1 may be used by `getStorageRanges()`
if nAccounts == 0:
return err() # oops: no account
# Veriify range atguments
if not minPt.init(origin) or not maxPt.init(limit) or maxPt <= minPt:
when extraTraceMessages:
trace logTxt "mkNodeTagRange: malformed range", origin, limit
return err()
if 1 < nAccounts:
return ok(NodeTagRange.new(low(NodeTag), high(NodeTag)))
ok(NodeTagRange.new(minPt, maxPt))
proc fetchLeafRange(
ctx: SnapWireRef; # Handler descriptor
db: HexaryGetFn; # Database abstraction
getFn: HexaryGetFn; # Database abstraction
root: Hash256; # State root
iv: NodeTagRange; # Proofed range of leaf paths
replySizeMax: int; # Updated size counter for the raw list
stopAt: Moment; # Implies timeout
): Result[RangeProof,void]
): Result[RangeProof,HexaryError]
{.gcsafe, raises: [CatchableError].} =
# Assemble result Note that the size limit is the size of the leaf nodes
@ -136,44 +147,48 @@ proc fetchLeafRange(
sizeMax = replySizeMax - estimatedProofSize
now = Moment.now()
timeout = if now < stopAt: stopAt - now else: 1.milliseconds
rc = db.hexaryRangeLeafsProof(rootKey, iv, sizeMax, timeout)
rc = getFn.hexaryRangeLeafsProof(rootKey, iv, sizeMax, timeout)
if rc.isErr:
debug logTxt "fetchLeafRange: database problem",
error logTxt "fetchLeafRange: database problem",
iv, replySizeMax, error=rc.error
return err() # database error
let sizeOnWire = rc.value.leafsSize + rc.value.proofSize
return rc # database error
let sizeOnWire = rc.value.leafsSize + rc.value.proofSize
if sizeOnWire <= replySizeMax:
return ok(rc.value)
return rc
# Estimate the overhead size on wire needed for a single leaf tail item
const leafExtraSize = (sizeof RangeLeaf()) - (sizeof newSeq[Blob](0))
let nLeafs = rc.value.leafs.len
when extraTraceMessages:
trace logTxt "fetchLeafRange: reducing reply sample",
iv, sizeOnWire, replySizeMax, nLeafs
# Strip parts of leafs result and amend remainder by adding proof nodes
var
rpl = rc.value
leafsTop = rpl.leafs.len - 1
tailSize = 0
tailItems = 0
reduceBy = replySizeMax - sizeOnWire
while tailSize <= reduceBy and tailItems < leafsTop:
# Estimate the size on wire needed for the tail item
const extraSize = (sizeof RangeLeaf()) - (sizeof newSeq[Blob](0))
tailSize += rpl.leafs[leafsTop - tailItems].data.len + extraSize
var (tailSize, tailItems, reduceBy) = (0, 0, replySizeMax - sizeOnWire)
while tailSize <= reduceBy:
tailItems.inc
if leafsTop <= tailItems:
debug logTxt "fetchLeafRange: stripping leaf list failed",
iv, replySizeMax, leafsTop, tailItems
return err() # package size too small
if nLeafs <= tailItems:
when extraTraceMessages:
trace logTxt "fetchLeafRange: stripping leaf list failed",
iv, replySizeMax, nLeafs, tailItems
return err(DataSizeError) # empty tail (package size too small)
tailSize += rc.value.leafs[^tailItems].data.len + leafExtraSize
rpl.leafs.setLen(leafsTop - tailItems - 1) # chop off one more for slack
# Provide truncated leafs list
let
leafProof = db.hexaryRangeLeafsProof(rootKey, rpl)
leafProof = getFn.hexaryRangeLeafsProof(
rootKey, RangeProof(leafs: rc.value.leafs[0 ..< nLeafs - tailItems]))
strippedSizeOnWire = leafProof.leafsSize + leafProof.proofSize
if strippedSizeOnWire <= replySizeMax:
return ok(leafProof)
debug logTxt "fetchLeafRange: data size problem",
iv, replySizeMax, leafsTop, tailItems, strippedSizeOnWire
when extraTraceMessages:
trace logTxt "fetchLeafRange: data size problem",
iv, replySizeMax, nLeafs, tailItems, strippedSizeOnWire
err()
err(DataSizeError)
# ------------------------------------------------------------------------------
# Private functions: peer observer
@ -254,8 +269,8 @@ method getAccountRange*(
iv = block: # Calculate effective accounts range (if any)
let rc = origin.mkNodeTagRange limit
if rc.isErr:
return
rc.value # malformed interval
return # malformed interval
rc.value
db = ctx.chain.getAccountFn
stopAt = Moment.now() + ctx.elaFetchMax
@ -293,10 +308,10 @@ method getStorageRanges*(
let
iv = block: # Calculate effective slots range (if any)
let rc = origin.mkNodeTagRange limit
let rc = origin.mkNodeTagRange(limit, accounts.len)
if rc.isErr:
return
rc.value # malformed interval
return # malformed interval
rc.value
accGetFn = ctx.chain.getAccountFn
rootKey = root.to(NodeKey)
@ -331,19 +346,30 @@ method getStorageRanges*(
accDataLen=accData.len, stoRoot
continue
# Collect data slots for this account
# Stop unless there is enough space left
if sizeMax - dataAllocated <= estimatedProofSize:
break
# Prepare for data collection
let
db = ctx.chain.getStorageSlotsFn(accKey)
rc = ctx.fetchLeafRange(db, stoRoot, iv, sizeMax - dataAllocated, stopAt)
slotsGetFn = ctx.chain.getStoSlotFn(accKey)
sizeLeft = sizeMax - dataAllocated
# Collect data slots for this account
let rc = ctx.fetchLeafRange(slotsGetFn, stoRoot, iv, sizeLeft, stopAt)
if rc.isErr:
when extraTraceMessages:
trace logTxt "getStorageRanges: failed", iv, sizeMax, dataAllocated,
accDataLen=accData.len, stoRoot
trace logTxt "getStorageRanges: failed", iv, sizeMax, sizeLeft,
accDataLen=accData.len, stoRoot, error=rc.error
return # extraction failed
# Process data slots for this account
dataAllocated += rc.value.leafsSize
when extraTraceMessages:
if accounts.len == 1:
trace logTxt "getStorageRanges: single account", iv, accKey, stoRoot
#trace logTxt "getStorageRanges: data slots", iv, sizeMax, dataAllocated,
# accKey, stoRoot, nSlots=rc.value.leafs.len, nProof=rc.value.proof.len

62
nimbus/sync/snap/README.txt Normal file
View File

@ -0,0 +1,62 @@
Snap sync test & debugging scenario
===================================
Start snap/1 server
-------------------
# Enter nimbus directory for snap/1 protocol server.
cd server
# Tell nimbus to stop full sync after 2 mio blocks.
echo 2000000 > full-limit.txt
# Tell nimbus to use this predefined key ID
echo 123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0 > full-id.key
./build/nimbus \
--tcp-port:30319 --nat=None --sync-mode=full \
--protocols=snap --discovery=none \
--net-key=./full-id.key \
--sync-ctrl-file=./full-limit.txt \
--log-level:TRACE
# Wait for several hours until enough blocks have been downloaded so that
# snap sync data are available. The full 2 mio blocks are available if the
# log ticker shows something like
#
# INF 2023-03-17 [..] Sync statistics (suspended) topics="full-tick" [..] persistent=#2000080 [..]
#
# where the persistent=#2000080 field might vary
Start snap/1 client
-------------------
# Note: When the snap/1 server has enough blocks, the client can be started.
# Enter nimbus directory for snap/1 protocol server
cd client
# Tell nimbus to use this pivot block number. This number must be smaller
# than the 2000000 written into the file full-limit.txt above.
echo 600000 > snap/snap-update.txt.
# Tell nimbus to use this hard coded peer enode.
echo enode://192d7e7a302bd4ff27f48d7852621e0d3cb863a6dd67dd44e0314a25a3aa866837f0d2460b4444dc66e7b7a2cd56a2de1c31b2a2ba4e23549bf3ba3b0c4f2eb5@127.0.0.1:30319 > snap/full-servers.txt
./build/nimbus \
--tcp-port:30102 --nat=None --sync-mode=snap \
--protocols=none --discovery=none \
--static-peers-file=./full-servers.txt \
--sync-ctrl-file=./snap-update.txt \
--log-level:TRACE
Modifications while the programs are syncing
--------------------------------------------
# Increasing the number in the files full/full-limit.txt or
# snap/snap-update.txt will be recognised while running. Decreasing
# or removing will be ignored.

18
nimbus/sync/snap/worker.nim
View File

@ -225,14 +225,21 @@ proc runMulti*(buddy: SnapBuddyRef) {.async.} =
return # nothing to do
rc.value
pivot = "#" & $env.stateHeader.blockNumber # for logging
nStorQuAtStart = env.fetchStorageFull.len +
env.fetchStoragePart.len +
env.parkedStorage.len
buddy.only.pivotEnv = env
# Full sync processsing based on current snapshot
# -----------------------------------------------
if env.storageDone:
# Check whether this pivot is fully downloaded
if env.fetchAccounts.processed.isFull and nStorQuAtStart == 0:
trace "Snap full sync -- not implemented yet", peer, pivot
await sleepAsync(5.seconds)
# flip over to single mode for getting new instructins
buddy.ctrl.multiOk = false
return
# Snapshot sync processing
@ -248,9 +255,8 @@ proc runMulti*(buddy: SnapBuddyRef) {.async.} =
nAccounts {.used.} = env.nAccounts
nSlotLists {.used.} = env.nSlotLists
processed {.used.} = env.fetchAccounts.processed.fullFactor.toPC(2)
nStoQu {.used.} = env.fetchStorageFull.len + env.fetchStoragePart.len
trace "Multi sync runner", peer, pivot, nAccounts, nSlotLists, processed,
nStoQu
nStoQu=nStorQuAtStart
# This one is the syncing work horse which downloads the database
await env.execSnapSyncAction(buddy)
@ -260,7 +266,7 @@ proc runMulti*(buddy: SnapBuddyRef) {.async.} =
nAccounts = env.nAccounts
nSlotLists = env.nSlotLists
processed = env.fetchAccounts.processed.fullFactor.toPC(2)
nStoQu = env.fetchStorageFull.len + env.fetchStoragePart.len
nStoQuLater = env.fetchStorageFull.len + env.fetchStoragePart.len
if env.archived:
# Archive pivot if it became stale
@ -273,11 +279,11 @@ proc runMulti*(buddy: SnapBuddyRef) {.async.} =
let rc = env.saveCheckpoint(ctx)
if rc.isErr:
error "Failed to save recovery checkpoint", peer, pivot, nAccounts,
nSlotLists, processed, nStoQu, error=rc.error
nSlotLists, processed, nStoQu=nStoQuLater, error=rc.error
else:
when extraTraceMessages:
trace "Saved recovery checkpoint", peer, pivot, nAccounts, nSlotLists,
processed, nStoQu, blobSize=rc.value
processed, nStoQu=nStoQuLater, blobSize=rc.value
# ------------------------------------------------------------------------------
# End

666
nimbus/sync/snap/worker/db/hexary_debug.nim Normal file
View File

@ -0,0 +1,666 @@
# nimbus-eth1
# Copyright (c) 2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
## Find node paths in hexary tries.
{.push raises: [].}
import
std/[algorithm, sequtils, sets, strutils, tables, times],
chronos,
eth/[common, trie/nibbles],
stew/results,
../../range_desc,
"."/[hexary_desc, hexary_error]
proc next*(path: XPath; getFn: HexaryGetFn; minDepth = 64): XPath
{.gcsafe, raises: [CatchableError].}
proc prev*(path: XPath; getFn: HexaryGetFn; minDepth = 64): XPath
{.gcsafe, raises: [CatchableError].}
# ------------------------------------------------------------------------------
# Private pretty printing helpers
# ------------------------------------------------------------------------------
proc asDateTime(m: Moment): DateTime =
## Approximate UTC based `DateTime` for a `Moment`
let
utcNow = times.now().utc
momNow = Moment.now()
utcNow + initDuration(nanoseconds = (m - momNow).nanoseconds)
# --------------
proc toPfx(indent: int): string =
"\n" & " ".repeat(indent)
proc ppImpl(s: string; hex = false): string =
## For long strings print `begin..end` only
if hex:
let n = (s.len + 1) div 2
(if s.len < 20: s else: s[0 .. 5] & ".." & s[s.len-8 .. s.len-1]) &
"[" & (if 0 < n: "#" & $n else: "") & "]"
elif s.len <= 30:
s
else:
(if (s.len and 1) == 0: s[0 ..< 8] else: "0" & s[0 ..< 7]) &
"..(" & $s.len & ").." & s[s.len-16 ..< s.len]
proc ppImpl(key: RepairKey; db: HexaryTreeDbRef): string =
if key.isZero:
return "ø"
if not key.isNodekey:
var num: uint64
(addr num).copyMem(unsafeAddr key.ByteArray33[25], 8)
return "%" & $num
try:
if not disablePrettyKeys and not db.keyPp.isNil:
return db.keyPp(key)
except CatchableError:
discard
key.ByteArray33.toSeq.mapIt(it.toHex(2)).join.toLowerAscii
proc ppImpl(key: NodeKey; db: HexaryTreeDbRef): string =
key.to(RepairKey).ppImpl(db)
proc ppImpl(w: openArray[RepairKey]; db: HexaryTreeDbRef): string =
w.mapIt(it.ppImpl(db)).join(",")
proc ppImpl(w: openArray[Blob]; db: HexaryTreeDbRef): string =
var q: seq[RepairKey]
for a in w:
var key: RepairKey
discard key.init(a)
q.add key
q.ppImpl(db)
proc ppStr(blob: Blob): string =
if blob.len == 0: ""
else: blob.mapIt(it.toHex(2)).join.toLowerAscii.ppImpl(hex = true)
proc ppImpl(n: RNodeRef; db: HexaryTreeDbRef): string =
let so = n.state.ord
case n.kind:
of Leaf:
["l","ł","L","R"][so] & "(" & $n.lPfx & "," & n.lData.ppStr & ")"
of Extension:
["e","€","E","R"][so] & "(" & $n.ePfx & "," & n.eLink.ppImpl(db) & ")"
of Branch:
["b","þ","B","R"][so] & "(" & n.bLink.ppImpl(db) & "," & n.bData.ppStr & ")"
proc ppImpl(n: XNodeObj; db: HexaryTreeDbRef): string =
case n.kind:
of Leaf:
"l(" & $n.lPfx & "," & n.lData.ppStr & ")"
of Extension:
var key: RepairKey
discard key.init(n.eLink)
"e(" & $n.ePfx & "," & key.ppImpl(db) & ")"
of Branch:
"b(" & n.bLink[0..15].ppImpl(db) & "," & n.bLink[16].ppStr & ")"
proc ppImpl(w: RPathStep; db: HexaryTreeDbRef): string =
let
nibble = if 0 <= w.nibble: w.nibble.toHex(1).toLowerAscii else: "ø"
key = w.key.ppImpl(db)
"(" & key & "," & nibble & "," & w.node.ppImpl(db) & ")"
proc ppImpl(w: XPathStep; db: HexaryTreeDbRef): string =
let nibble = if 0 <= w.nibble: w.nibble.toHex(1).toLowerAscii else: "ø"
var key: RepairKey
discard key.init(w.key)
"(" & key.ppImpl(db) & "," & $nibble & "," & w.node.ppImpl(db) & ")"
proc ppImpl(db: HexaryTreeDbRef; root: NodeKey): seq[string] =
## Dump the entries from the a generic repair tree. This function assumes
## that mapped keys are printed `$###` if a node is locked or static, and
## some substitute for the first letter `$` otherwise (if they are mutable.)
proc toKey(s: string): uint64 =
try:
result = s[1 ..< s.len].parseUint
except ValueError as e:
raiseAssert "Ooops ppImpl(s=" & s & "): name=" & $e.name & " msg=" & e.msg
if s[0] != '$':
result = result or (1u64 shl 63)
proc cmpIt(x, y: (uint64,string)): int =
cmp(x[0],y[0])
var accu: seq[(uint64,string)]
if root.ByteArray32 != ByteArray32.default:
accu.add @[(0u64, "($0" & "," & root.ppImpl(db) & ")")]
for key,node in db.tab.pairs:
accu.add (
key.ppImpl(db).tokey,
"(" & key.ppImpl(db) & "," & node.ppImpl(db) & ")")
accu.sorted(cmpIt).mapIt(it[1])
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
proc getNibblesImpl(path: XPath; start = 0): NibblesSeq =
## Re-build the key path
for n in start ..< path.path.len:
let it = path.path[n]
case it.node.kind:
of Branch:
result = result & @[it.nibble.byte].initNibbleRange.slice(1)
of Extension:
result = result & it.node.ePfx
of Leaf:
result = result & it.node.lPfx
result = result & path.tail
proc getLeafData(path: XPath): Blob =
## Return the leaf data from a successful `XPath` computation (if any.)
## Note that this function also exists as `hexary_paths.leafData()` but
## the import of this file is avoided.
if path.tail.len == 0 and 0 < path.path.len:
let node = path.path[^1].node
case node.kind:
of Branch:
return node.bLink[16]
of Leaf:
return node.lData
of Extension:
discard
proc toBranchNode(
rlp: Rlp
): XNodeObj
{.gcsafe, raises: [RlpError].} =
var rlp = rlp
XNodeObj(kind: Branch, bLink: rlp.read(array[17,Blob]))
proc toLeafNode(
rlp: Rlp;
pSegm: NibblesSeq
): XNodeObj
{.gcsafe, raises: [RlpError].} =
XNodeObj(kind: Leaf, lPfx: pSegm, lData: rlp.listElem(1).toBytes)
proc toExtensionNode(
rlp: Rlp;
pSegm: NibblesSeq
): XNodeObj
{.gcsafe, raises: [RlpError].} =
XNodeObj(kind: Extension, ePfx: pSegm, eLink: rlp.listElem(1).toBytes)
proc to(node: XNodeObj; T: type RNodeRef): T =
case node.kind:
of Leaf:
result = T(
kind: Leaf,
lData: node.lData,
lPfx: node.lPfx)
of Extension:
result = T(
kind: Extension,
eLink: node.eLink.convertTo(RepairKey),
ePfx: node.ePfx)
of Branch:
result = T(
kind: Branch,
bData: node.bLink[16])
for n in 0 .. 15:
result.bLink[n] = node.bLink[n].convertTo(RepairKey)
# ------------------------------------------------------------------------------
# Private functions
# ------------------------------------------------------------------------------
proc pathLeast(
path: XPath;
key: Blob;
getFn: HexaryGetFn;
): XPath
{.gcsafe, raises: [CatchableError].} =
## For the partial path given, extend by branch nodes with least node
## indices.
result = path
result.tail = EmptyNibbleRange
result.depth = result.getNibblesImpl.len
var
key = key
value = key.getFn()
if value.len == 0:
return
while true:
block loopContinue:
let nodeRlp = rlpFromBytes value
case nodeRlp.listLen:
of 2:
let (isLeaf,pathSegment) = hexPrefixDecode nodeRlp.listElem(0).toBytes
# Leaf node
if isLeaf:
let node = nodeRlp.toLeafNode(pathSegment)
result.path.add XPathStep(key: key, node: node, nibble: -1)
result.depth += pathSegment.len
return # done ok
let node = nodeRlp.toExtensionNode(pathSegment)
if 0 < node.eLink.len:
value = node.eLink.getFn()
if 0 < value.len:
result.path.add XPathStep(key: key, node: node, nibble: -1)
result.depth += pathSegment.len
key = node.eLink
break loopContinue
of 17:
# Branch node
let node = nodeRlp.toBranchNode
if node.bLink[16].len != 0 and 64 <= result.depth:
result.path.add XPathStep(key: key, node: node, nibble: -1)
return # done ok
for inx in 0 .. 15:
let newKey = node.bLink[inx]
if 0 < newKey.len:
value = newKey.getFn()
if 0 < value.len:
result.path.add XPathStep(key: key, node: node, nibble: inx.int8)
result.depth.inc
key = newKey
break loopContinue
else:
discard
# Recurse (iteratively)
while true:
block loopRecurse:
# Modify last branch node and try again
if result.path[^1].node.kind == Branch:
for inx in result.path[^1].nibble+1 .. 15:
let newKey = result.path[^1].node.bLink[inx]
if 0 < newKey.len:
value = newKey.getFn()
if 0 < value.len:
result.path[^1].nibble = inx.int8
key = newKey
break loopContinue
# Failed, step back and try predecessor branch.
while path.path.len < result.path.len:
case result.path[^1].node.kind:
of Branch:
result.depth.dec
result.path.setLen(result.path.len - 1)
break loopRecurse
of Extension:
result.depth -= result.path[^1].node.ePfx.len
result.path.setLen(result.path.len - 1)
of Leaf:
return # Ooops
return # Failed
# Notreached
# End while
# Notreached
proc pathMost(
path: XPath;
key: Blob;
getFn: HexaryGetFn;
): XPath
{.gcsafe, raises: [CatchableError].} =
## For the partial path given, extend by branch nodes with greatest node
## indices.
result = path
result.tail = EmptyNibbleRange
result.depth = result.getNibblesImpl.len
var
key = key
value = key.getFn()
if value.len == 0:
return
while true:
block loopContinue:
let nodeRlp = rlpFromBytes value
case nodeRlp.listLen:
of 2:
let (isLeaf,pathSegment) = hexPrefixDecode nodeRlp.listElem(0).toBytes
# Leaf node
if isLeaf:
let node = nodeRlp.toLeafNode(pathSegment)
result.path.add XPathStep(key: key, node: node, nibble: -1)
result.depth += pathSegment.len
return # done ok
# Extension node
let node = nodeRlp.toExtensionNode(pathSegment)
if 0 < node.eLink.len:
value = node.eLink.getFn()
if 0 < value.len:
result.path.add XPathStep(key: key, node: node, nibble: -1)
result.depth += pathSegment.len
key = node.eLink
break loopContinue
of 17:
# Branch node
let node = nodeRlp.toBranchNode
if node.bLink[16].len != 0 and 64 <= result.depth:
result.path.add XPathStep(key: key, node: node, nibble: -1)
return # done ok
for inx in 15.countDown(0):
let newKey = node.bLink[inx]
if 0 < newKey.len:
value = newKey.getFn()
if 0 < value.len:
result.path.add XPathStep(key: key, node: node, nibble: inx.int8)
result.depth.inc
key = newKey
break loopContinue
else:
discard
# Recurse (iteratively)
while true:
block loopRecurse:
# Modify last branch node and try again
if result.path[^1].node.kind == Branch:
for inx in (result.path[^1].nibble-1).countDown(0):
let newKey = result.path[^1].node.bLink[inx]
if 0 < newKey.len:
value = newKey.getFn()
if 0 < value.len:
result.path[^1].nibble = inx.int8
key = newKey
break loopContinue
# Failed, step back and try predecessor branch.
while path.path.len < result.path.len:
case result.path[^1].node.kind:
of Branch:
result.depth.dec
result.path.setLen(result.path.len - 1)
break loopRecurse
of Extension:
result.depth -= result.path[^1].node.ePfx.len
result.path.setLen(result.path.len - 1)
of Leaf:
return # Ooops
return # Failed
# Notreached
# End while
# Notreached
# ---------------
proc fillFromLeft(
db: HexaryTreeDbRef; # Target in-memory database
rootKey: NodeKey; # State root for persistent source database
getFn: HexaryGetFn; # Source database abstraction
maxLeafs = 5000; # Error if more than this many leaf nodes
): Result[int,HexaryError]
{.gcsafe, raises: [CatchableError].} =
## Import persistent sub-tree into target database
# Find first least path
var
here = XPath(root: rootKey).pathLeast(rootkey.to(Blob), getFn)
countSteps = 0
if 0 < here.path.len:
while true:
countSteps.inc
# Import records
for step in here.path:
db.tab[step.key.convertTo(RepairKey)] = step.node.to(RNodeRef)
# Get next path
let topKey = here.path[^1].key
here = here.next(getFn)
# Check for end condition
if here.path.len == 0:
break
if topKey == here.path[^1].key:
return err(GarbledNextLeaf) # Ooops
if maxLeafs <= countSteps:
return err(LeafMaxExceeded)
ok(countSteps)
proc fillFromRight(
db: HexaryTreeDbRef; # Target in-memory database
rootKey: NodeKey; # State root for persistent source database
getFn: HexaryGetFn; # Source database abstraction
maxLeafs = 5000; # Error if more than this many leaf nodes
): Result[int,HexaryError]
{.gcsafe, raises: [CatchableError].} =
## Import persistent sub-tree into target database
# Find first least path
var
here = XPath(root: rootKey).pathMost(rootkey.to(Blob), getFn)
countSteps = 0
if 0 < here.path.len:
while true:
countSteps.inc
# Import records
for step in here.path:
db.tab[step.key.convertTo(RepairKey)] = step.node.to(RNodeRef)
# Get next path
let topKey = here.path[^1].key
here = here.prev(getFn)
# Check for end condition
if here.path.len == 0:
break
if topKey == here.path[^1].key:
return err(GarbledNextLeaf) # Ooops
if maxLeafs <= countSteps:
return err(LeafMaxExceeded)
ok(countSteps)
# ------------------------------------------------------------------------------
# Public functions, pretty printing
# ------------------------------------------------------------------------------
proc pp*(s: string; hex = false): string =
## For long strings print `begin..end` only
s.ppImpl(hex)
proc pp*(w: NibblesSeq): string =
$w
proc pp*(key: RepairKey): string =
## Raw key, for referenced key dump use `key.pp(db)` below
key.ByteArray33.toSeq.mapIt(it.toHex(2)).join.tolowerAscii
proc pp*(key: NodeKey): string =
## Raw key, for referenced key dump use `key.pp(db)` below
key.ByteArray32.toSeq.mapIt(it.toHex(2)).join.tolowerAscii
proc pp*(key: NodeKey|RepairKey; db: HexaryTreeDbRef): string =
key.ppImpl(db)
proc pp*(
w: RNodeRef|XNodeObj|RPathStep|XPathStep;
db: HexaryTreeDbRef;
): string =
w.ppImpl(db)
proc pp*(
w: openArray[RPathStep|XPathStep];
db:HexaryTreeDbRef;
delim: string;
): string =
w.toSeq.mapIt(it.ppImpl(db)).join(delim)
proc pp*(
w: openArray[RPathStep|XPathStep];
db: HexaryTreeDbRef;
indent = 4;
): string =
w.pp(db, indent.toPfx)
proc pp*(w: RPath|XPath; db: HexaryTreeDbRef; delim: string): string =
result = "<" & w.root.pp(db) & ">"
if 0 < w.path.len:
result &= delim & w.path.pp(db, delim)
result &= delim & "(" & $w.tail
when typeof(w) is XPath:
result &= "," & $w.depth
result &= ")"
proc pp*(w: RPath|XPath; db: HexaryTreeDbRef; indent=4): string =
w.pp(db, indent.toPfx)
proc pp*(db: HexaryTreeDbRef; root: NodeKey; delim: string): string =
## Dump the entries from the a generic accounts trie. These are
## key value pairs for
## ::
## Branch: ($1,b(<$2,$3,..,$17>,))
## Extension: ($18,e(832b5e..06e697,$19))
## Leaf: ($20,l(cc9b5d..1c3b4,f84401..f9e5129d[#70]))
##
## where keys are typically represented as `$<id>` or `¶<id>` or `ø`
## depending on whether a key is final (`$<id>`), temporary (`¶<id>`)
## or unset/missing (`ø`).
##
## The node types are indicated by a letter after the first key before
## the round brackets
## ::
## Branch: 'b', 'þ', or 'B'
## Extension: 'e', '€', or 'E'
## Leaf: 'l', 'ł', or 'L'
##
## Here a small letter indicates a `Static` node which was from the
## original `proofs` list, a capital letter indicates a `Mutable` node
## added on the fly which might need some change, and the decorated
## letters stand for `Locked` nodes which are like `Static` ones but
## added later (typically these nodes are update `Mutable` nodes.)
##
## Beware: dumping a large database is not recommended
db.ppImpl(root).join(delim)
proc pp*(db: HexaryTreeDbRef; root: NodeKey; indent=4): string =
## Dump the entries from the a generic repair tree.
db.pp(root, indent.toPfx)
proc pp*(m: Moment): string =
## Prints a moment in time similar to *chronicles* time format.
m.asDateTime.format "yyyy-MM-dd HH:mm:ss'.'fff'+00:00'"
# ------------------------------------------------------------------------------
# Public functions, traversal over partial tree in persistent database
# ------------------------------------------------------------------------------
proc next*(
path: XPath;
getFn: HexaryGetFn;
minDepth = 64;
): XPath
{.gcsafe, raises: [CatchableError].} =
## Advance the argument `path` to the next leaf node (if any.). The
## `minDepth` argument requires the result of `next()` to satisfy
## `minDepth <= next().getNibbles.len`.
var pLen = path.path.len
# Find the last branch in the path, increase link and step down
while 0 < pLen:
# Find branch none
pLen.dec
let it = path.path[pLen]
if it.node.kind == Branch and it.nibble < 15:
# Find the next item to the right in the branch list
for inx in (it.nibble + 1) .. 15:
let link = it.node.bLink[inx]
if link.len != 0:
let
branch = XPathStep(key: it.key, node: it.node, nibble: inx.int8)
walk = path.path[0 ..< pLen] & branch
newPath = XPath(root: path.root, path: walk).pathLeast(link, getFn)
if minDepth <= newPath.depth and 0 < newPath.getLeafData.len:
return newPath
proc prev*(
path: XPath;
getFn: HexaryGetFn;
minDepth = 64;
): XPath
{.gcsafe, raises: [CatchableError].} =
## Advance the argument `path` to the previous leaf node (if any.) The
## `minDepth` argument requires the result of `next()` to satisfy
## `minDepth <= next().getNibbles.len`.
var pLen = path.path.len
# Find the last branch in the path, decrease link and step down
while 0 < pLen:
# Find branch none
pLen.dec
let it = path.path[pLen]
if it.node.kind == Branch and 0 < it.nibble:
# Find the next item to the right in the branch list
for inx in (it.nibble - 1).countDown(0):
let link = it.node.bLink[inx]
if link.len != 0:
let
branch = XPathStep(key: it.key, node: it.node, nibble: inx.int8)
walk = path.path[0 ..< pLen] & branch
newPath = XPath(root: path.root, path: walk).pathMost(link,getFn)
if minDepth <= newPath.depth and 0 < newPath.getLeafData.len:
return newPath
proc fromPersistent*(
db: HexaryTreeDbRef; # Target in-memory database
rootKey: NodeKey; # State root for persistent source database
getFn: HexaryGetFn; # Source database abstraction
maxLeafs = 5000; # Error if more than this many leaf nodes
reverse = false; # Fill left to right by default
): Result[int,HexaryError]
{.gcsafe, raises: [CatchableError].} =
## Import persistent sub-tree into target database
if reverse:
db.fillFromLeft(rootKey, getFn, maxLeafs)
else:
db.fillFromRight(rootKey, getFn, maxLeafs)
proc fromPersistent*(
rootKey: NodeKey; # State root for persistent source database
getFn: HexaryGetFn; # Source database abstraction
maxLeafs = 5000; # Error if more than this many leaf nodes
reverse = false; # Fill left to right by default
): Result[HexaryTreeDbRef,HexaryError]
{.gcsafe, raises: [CatchableError].} =
## Variant of `fromPersistent()` for an ad-hoc table
let
db = HexaryTreeDbRef()
rc = db.fromPersistent(rootKey, getFn, maxLeafs, reverse)
if rc.isErr:
return err(rc.error)
ok(db)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

201
nimbus/sync/snap/worker/db/hexary_desc.nim
View File

@ -8,15 +8,15 @@
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
{.push raises: [].}
import
std/[algorithm, hashes, sequtils, sets, strutils, tables],
eth/[common, p2p, trie/nibbles],
std/[hashes, sequtils, sets, tables],
eth/[common, trie/nibbles],
stint,
../../range_desc,
./hexary_error
{.push raises: [].}
type
HexaryPpFn* =
proc(key: RepairKey): string {.gcsafe, raises: [CatchableError].}
@ -113,6 +113,7 @@ type
nibble*: int8 ## Branch node selector (if any)
RPath* = object
root*: RepairKey ## Root node needed when `path.len == 0`
path*: seq[RPathStep]
tail*: NibblesSeq ## Stands for non completed leaf path
@ -123,6 +124,7 @@ type
nibble*: int8 ## Branch node selector (if any)
XPath* = object
root*: NodeKey ## Root node needed when `path.len == 0`
path*: seq[XPathStep]
tail*: NibblesSeq ## Stands for non completed leaf path
depth*: int ## May indicate path length (typically 64)
@ -172,14 +174,6 @@ proc isZero*(a: RepairKey): bool {.gcsafe.}
# Private helpers
# ------------------------------------------------------------------------------
proc initImpl(key: var RepairKey; data: openArray[byte]): bool =
key.reset
if 0 < data.len and data.len <= 33:
let trg = addr key.ByteArray33[33 - data.len]
trg.copyMem(unsafeAddr data[0], data.len)
return true
proc append(writer: var RlpWriter, node: RNodeRef) =
## Mixin for RLP writer
proc appendOk(writer: var RlpWriter; key: RepairKey): bool =
@ -225,167 +219,16 @@ proc append(writer: var RlpWriter, node: XNodeObj) =
writer.append(node.lPfx.hexPrefixEncode(isleaf = true))
writer.append(node.lData)
# ------------------------------------------------------------------------------
# Private debugging helpers
# ------------------------------------------------------------------------------
proc to*(key: NodeKey; T: type RepairKey): T {.gcsafe.}
proc toPfx(indent: int): string =
"\n" & " ".repeat(indent)
proc ppImpl(s: string; hex = false): string =
## For long strings print `begin..end` only
if hex:
let n = (s.len + 1) div 2
(if s.len < 20: s else: s[0 .. 5] & ".." & s[s.len-8 .. s.len-1]) &
"[" & (if 0 < n: "#" & $n else: "") & "]"
elif s.len <= 30:
s
else:
(if (s.len and 1) == 0: s[0 ..< 8] else: "0" & s[0 ..< 7]) &
"..(" & $s.len & ").." & s[s.len-16 ..< s.len]
proc ppImpl(key: RepairKey; db: HexaryTreeDbRef): string =
if key.isZero:
return "ø"
if not key.isNodekey:
var num: uint64
(addr num).copyMem(unsafeAddr key.ByteArray33[25], 8)
return "%" & $num
try:
if not disablePrettyKeys and not db.keyPp.isNil:
return db.keyPp(key)
except CatchableError:
discard
key.ByteArray33.toSeq.mapIt(it.toHex(2)).join.toLowerAscii
proc ppImpl(key: NodeKey; db: HexaryTreeDbRef): string =
key.to(RepairKey).ppImpl(db)
proc ppImpl(w: openArray[RepairKey]; db: HexaryTreeDbRef): string =
w.mapIt(it.ppImpl(db)).join(",")
proc ppImpl(w: openArray[Blob]; db: HexaryTreeDbRef): string =
var q: seq[RepairKey]
for a in w:
var key: RepairKey
discard key.initImpl(a)
q.add key
q.ppImpl(db)
proc ppStr(blob: Blob): string =
if blob.len == 0: ""
else: blob.mapIt(it.toHex(2)).join.toLowerAscii.ppImpl(hex = true)
proc ppImpl(n: RNodeRef; db: HexaryTreeDbRef): string =
let so = n.state.ord
case n.kind:
of Leaf:
["l","ł","L","R"][so] & "(" & $n.lPfx & "," & n.lData.ppStr & ")"
of Extension:
["e","€","E","R"][so] & "(" & $n.ePfx & "," & n.eLink.ppImpl(db) & ")"
of Branch:
["b","þ","B","R"][so] & "(" & n.bLink.ppImpl(db) & "," & n.bData.ppStr & ")"
proc ppImpl(n: XNodeObj; db: HexaryTreeDbRef): string =
case n.kind:
of Leaf:
"l(" & $n.lPfx & "," & n.lData.ppStr & ")"
of Extension:
var key: RepairKey
discard key.initImpl(n.eLink)
"e(" & $n.ePfx & "," & key.ppImpl(db) & ")"
of Branch:
"b(" & n.bLink[0..15].ppImpl(db) & "," & n.bLink[16].ppStr & ")"
proc ppImpl(w: RPathStep; db: HexaryTreeDbRef): string =
let
nibble = if 0 <= w.nibble: w.nibble.toHex(1).toLowerAscii else: "ø"
key = w.key.ppImpl(db)
"(" & key & "," & nibble & "," & w.node.ppImpl(db) & ")"
proc ppImpl(w: XPathStep; db: HexaryTreeDbRef): string =
let nibble = if 0 <= w.nibble: w.nibble.toHex(1).toLowerAscii else: "ø"
var key: RepairKey
discard key.initImpl(w.key)
"(" & key.ppImpl(db) & "," & $nibble & "," & w.node.ppImpl(db) & ")"
proc ppImpl(db: HexaryTreeDbRef; root: NodeKey): seq[string] =
## Dump the entries from the a generic repair tree. This function assumes
## that mapped keys are printed `$###` if a node is locked or static, and
## some substitute for the first letter `$` otherwise (if they are mutable.)
proc toKey(s: string): uint64 =
try:
result = s[1 ..< s.len].parseUint
except ValueError as e:
raiseAssert "Ooops ppImpl(s=" & s & "): name=" & $e.name & " msg=" & e.msg
if s[0] != '$':
result = result or (1u64 shl 63)
proc cmpIt(x, y: (uint64,string)): int =
cmp(x[0],y[0])
var accu: seq[(uint64,string)]
if root.ByteArray32 != ByteArray32.default:
accu.add @[(0u64, "($0" & "," & root.ppImpl(db) & ")")]
for key,node in db.tab.pairs:
accu.add (
key.ppImpl(db).tokey,
"(" & key.ppImpl(db) & "," & node.ppImpl(db) & ")")
accu.sorted(cmpIt).mapIt(it[1])
# ------------------------------------------------------------------------------
# Public debugging helpers
# ------------------------------------------------------------------------------
proc pp*(s: string; hex = false): string =
## For long strings print `begin..end` only
s.ppImpl(hex)
proc pp*(w: NibblesSeq): string =
$w
proc pp*(key: RepairKey): string =
## Raw key, for referenced key dump use `key.pp(db)` below
key.ByteArray33.toSeq.mapIt(it.toHex(2)).join.tolowerAscii
proc pp*(key: NodeKey): string =
## Raw key, for referenced key dump use `key.pp(db)` below
key.ByteArray32.toSeq.mapIt(it.toHex(2)).join.tolowerAscii
proc pp*(key: NodeKey|RepairKey; db: HexaryTreeDbRef): string =
key.ppImpl(db)
proc pp*(
w: RNodeRef|XNodeObj|RPathStep|XPathStep;
db: HexaryTreeDbRef;
): string =
w.ppImpl(db)
proc pp*(w:openArray[RPathStep|XPathStep];db:HexaryTreeDbRef;indent=4): string =
w.toSeq.mapIt(it.ppImpl(db)).join(indent.toPfx)
proc pp*(w: RPath; db: HexaryTreeDbRef; indent=4): string =
w.path.pp(db,indent) & indent.toPfx & "(" & $w.tail & ")"
proc pp*(w: XPath; db: HexaryTreeDbRef; indent=4): string =
w.path.pp(db,indent) & indent.toPfx & "(" & $w.tail & "," & $w.depth & ")"
proc pp*(db: HexaryTreeDbRef; root: NodeKey; indent=4): string =
## Dump the entries from the a generic repair tree.
db.ppImpl(root).join(indent.toPfx)
proc pp*(db: HexaryTreeDbRef; indent=4): string =
## varinat of `pp()` above
db.ppImpl(NodeKey.default).join(indent.toPfx)
# ------------------------------------------------------------------------------
# Public constructor (or similar)
# ------------------------------------------------------------------------------
proc init*(key: var RepairKey; data: openArray[byte]): bool =
key.initImpl(data)
key.reset
if 0 < data.len and data.len <= 33:
let trg = addr key.ByteArray33[33 - data.len]
trg.copyMem(unsafeAddr data[0], data.len)
return true
proc newRepairKey*(db: HexaryTreeDbRef): RepairKey =
db.repairKeyGen.inc
@ -434,7 +277,7 @@ proc convertTo*(data: Blob; T: type NodeTag): T =
proc convertTo*(data: Blob; T: type RepairKey): T =
## Probably lossy conversion, use `init()` for safe conversion
discard result.initImpl(data)
discard result.init(data)
proc convertTo*(node: RNodeRef; T: type Blob): T =
## Write the node as an RLP-encoded blob
@ -455,6 +298,26 @@ proc convertTo*(nodeList: openArray[XNodeObj]; T: type Blob): T =
writer.append w
writer.finish
proc padPartialPath*(pfx: NibblesSeq; dblNibble: byte): NodeKey =
## Extend (or cut) `partialPath` nibbles sequence and generate `NodeKey`.
## This function must be handled with some care regarding a meaningful value
## for the `dblNibble` argument. Using values `0` or `255` is typically used
## to create the minimum or maximum envelope value from the `pfx` argument.
# Pad with zeroes
var padded: NibblesSeq
let padLen = 64 - pfx.len
if 0 <= padLen:
padded = pfx & dblNibble.repeat(padlen div 2).initNibbleRange
if (padLen and 1) == 1:
padded = padded & @[dblNibble].initNibbleRange.slice(1)
else:
let nope = seq[byte].default.initNibbleRange
padded = pfx.slice(0,64) & nope # nope forces re-alignment
let bytes = padded.getBytes
(addr result.ByteArray32[0]).copyMem(unsafeAddr bytes[0], bytes.len)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

23
nimbus/sync/snap/worker/db/hexary_envelope.nim
View File

@ -70,6 +70,9 @@
## * then there is a ``w = partialPath & w-ext`` in ``W`` with
## ``p-ext = w-ext & some-ext``.
##
{.push raises: [].}
import
std/[algorithm, sequtils, tables],
eth/[common, trie/nibbles],
@ -77,8 +80,6 @@ import
../../range_desc,
"."/[hexary_desc, hexary_error, hexary_nearby, hexary_paths]
{.push raises: [].}
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
@ -136,24 +137,6 @@ template noRlpErrorOops(info: static[string]; code: untyped) =
# Private functions
# ------------------------------------------------------------------------------
proc padPartialPath(pfx: NibblesSeq; dblNibble: byte): NodeKey =
## Extend (or cut) `partialPath` nibbles sequence and generate `NodeKey`
# Pad with zeroes
var padded: NibblesSeq
let padLen = 64 - pfx.len
if 0 <= padLen:
padded = pfx & dblNibble.repeat(padlen div 2).initNibbleRange
if (padLen and 1) == 1:
padded = padded & @[dblNibble].initNibbleRange.slice(1)
else:
let nope = seq[byte].default.initNibbleRange
padded = pfx.slice(0,64) & nope # nope forces re-alignment
let bytes = padded.getBytes
(addr result.ByteArray32[0]).copyMem(unsafeAddr bytes[0], bytes.len)
proc doDecomposeLeft(
envQ: RPath|XPath;
ivQ: RPath|XPath;

8
nimbus/sync/snap/worker/db/hexary_error.nim
View File

@ -28,6 +28,13 @@ type
TooManySlotAccounts
NoAccountsYet
# debug
LeafMaxExceeded
GarbledNextLeaf
# snap handler
DataSizeError
# range
LeafNodeExpected
FailedNextNode
@ -42,6 +49,7 @@ type
NearbyEmptyPath
NearbyLeafExpected
NearbyDanglingLink
NearbyPathTail
# envelope
DecomposeDegenerated

10
nimbus/sync/snap/worker/db/hexary_interpolate.nim
View File

@ -14,6 +14,8 @@
## purposes, it should be replaced by the new facility of the upcoming
## re-factored database layer.
{.push raises: [].}
import
std/[tables],
eth/[common, trie/nibbles],
@ -21,8 +23,6 @@ import
../../range_desc,
"."/[hexary_desc, hexary_error, hexary_paths]
{.push raises: [].}
type
RPathXStep = object
## Extended `RPathStep` needed for `NodeKey` assignmant
@ -114,6 +114,7 @@ proc rTreeExtendLeaf(
if not key.isNodeKey:
rPath.path[^1].node.bLink[nibble] = key
return RPath(
root: rPath.root,
path: rPath.path & RPathStep(key: key, node: leaf, nibble: -1),
tail: EmptyNibbleRange)
@ -129,7 +130,10 @@ proc rTreeExtendLeaf(
let
nibble = rPath.tail[0].int8
xStep = RPathStep(key: key, node: node, nibble: nibble)
xPath = RPath(path: rPath.path & xStep, tail: rPath.tail.slice(1))
xPath = RPath(
root: rPath.root,
path: rPath.path & xStep,
tail: rPath.tail.slice(1))
return db.rTreeExtendLeaf(xPath, db.newRepairKey())

1005
nimbus/sync/snap/worker/db/hexary_nearby.nim
View File

File diff suppressed because it is too large Load Diff

260
nimbus/sync/snap/worker/db/hexary_paths.nim
View File

@ -89,16 +89,15 @@ proc toExtensionNode(
# Private functions
# ------------------------------------------------------------------------------
proc pathExtend(
proc rootPathExtend(
path: RPath;
key: RepairKey;
db: HexaryTreeDbRef;
): RPath
{.gcsafe, raises: [KeyError].} =
## For the given path, extend to the longest possible repair tree `db`
## path following the argument `path.tail`.
result = path
var key = key
var key = path.root
while db.tab.hasKey(key):
let node = db.tab[key]
@ -127,15 +126,14 @@ proc pathExtend(
key = node.eLink
proc pathExtend(
proc rootPathExtend(
path: XPath;
key: Blob;
getFn: HexaryGetFn;
): XPath
{.gcsafe, raises: [CatchableError]} =
## Ditto for `XPath` rather than `RPath`
result = path
var key = key
var key = path.root.to(Blob)
while true:
let value = key.getFn()
if value.len == 0:
@ -187,187 +185,6 @@ proc pathExtend(
# end while
# notreached
proc pathLeast(
path: XPath;
key: Blob;
getFn: HexaryGetFn;
): XPath
{.gcsafe, raises: [CatchableError]} =
## For the partial path given, extend by branch nodes with least node
## indices.
result = path
result.tail = EmptyNibbleRange
result.depth = result.getNibblesImpl.len
var
key = key
value = key.getFn()
if value.len == 0:
return
while true:
block loopContinue:
let nodeRlp = rlpFromBytes value
case nodeRlp.listLen:
of 2:
let (isLeaf,pathSegment) = hexPrefixDecode nodeRlp.listElem(0).toBytes
# Leaf node
if isLeaf:
let node = nodeRlp.toLeafNode(pathSegment)
result.path.add XPathStep(key: key, node: node, nibble: -1)
result.depth += pathSegment.len
return # done ok
let node = nodeRlp.toExtensionNode(pathSegment)
if 0 < node.eLink.len:
value = node.eLink.getFn()
if 0 < value.len:
result.path.add XPathStep(key: key, node: node, nibble: -1)
result.depth += pathSegment.len
key = node.eLink
break loopContinue
of 17:
# Branch node
let node = nodeRlp.toBranchNode
if node.bLink[16].len != 0 and 64 <= result.depth:
result.path.add XPathStep(key: key, node: node, nibble: -1)
return # done ok
for inx in 0 .. 15:
let newKey = node.bLink[inx]
if 0 < newKey.len:
value = newKey.getFn()
if 0 < value.len:
result.path.add XPathStep(key: key, node: node, nibble: inx.int8)
result.depth.inc
key = newKey
break loopContinue
else:
discard
# Recurse (iteratively)
while true:
block loopRecurse:
# Modify last branch node and try again
if result.path[^1].node.kind == Branch:
for inx in result.path[^1].nibble+1 .. 15:
let newKey = result.path[^1].node.bLink[inx]
if 0 < newKey.len:
value = newKey.getFn()
if 0 < value.len:
result.path[^1].nibble = inx.int8
key = newKey
break loopContinue
# Failed, step back and try predecessor branch.
while path.path.len < result.path.len:
case result.path[^1].node.kind:
of Branch:
result.depth.dec
result.path.setLen(result.path.len - 1)
break loopRecurse
of Extension:
result.depth -= result.path[^1].node.ePfx.len
result.path.setLen(result.path.len - 1)
of Leaf:
return # Ooops
return # Failed
# Notreached
# End while
# Notreached
proc pathMost(
path: XPath;
key: Blob;
getFn: HexaryGetFn;
): XPath
{.gcsafe, raises: [CatchableError]} =
## For the partial path given, extend by branch nodes with greatest node
## indices.
result = path
result.tail = EmptyNibbleRange
result.depth = result.getNibblesImpl.len
var
key = key
value = key.getFn()
if value.len == 0:
return
while true:
block loopContinue:
let nodeRlp = rlpFromBytes value
case nodeRlp.listLen:
of 2:
let (isLeaf,pathSegment) = hexPrefixDecode nodeRlp.listElem(0).toBytes
# Leaf node
if isLeaf:
let node = nodeRlp.toLeafNode(pathSegment)
result.path.add XPathStep(key: key, node: node, nibble: -1)
result.depth += pathSegment.len
return # done ok
# Extension node
let node = nodeRlp.toExtensionNode(pathSegment)
if 0 < node.eLink.len:
value = node.eLink.getFn()
if 0 < value.len:
result.path.add XPathStep(key: key, node: node, nibble: -1)
result.depth += pathSegment.len
key = node.eLink
break loopContinue
of 17:
# Branch node
let node = nodeRlp.toBranchNode
if node.bLink[16].len != 0 and 64 <= result.depth:
result.path.add XPathStep(key: key, node: node, nibble: -1)
return # done ok
for inx in 15.countDown(0):
let newKey = node.bLink[inx]
if 0 < newKey.len:
value = newKey.getFn()
if 0 < value.len:
result.path.add XPathStep(key: key, node: node, nibble: inx.int8)
result.depth.inc
key = newKey
break loopContinue
else:
discard
# Recurse (iteratively)
while true:
block loopRecurse:
# Modify last branch node and try again
if result.path[^1].node.kind == Branch:
for inx in (result.path[^1].nibble-1).countDown(0):
let newKey = result.path[^1].node.bLink[inx]
if 0 < newKey.len:
value = newKey.getFn()
if 0 < value.len:
result.path[^1].nibble = inx.int8
key = newKey
break loopContinue
# Failed, step back and try predecessor branch.
while path.path.len < result.path.len:
case result.path[^1].node.kind:
of Branch:
result.depth.dec
result.path.setLen(result.path.len - 1)
break loopRecurse
of Extension:
result.depth -= result.path[^1].node.ePfx.len
result.path.setLen(result.path.len - 1)
of Leaf:
return # Ooops
return # Failed
# Notreached
# End while
# Notreached
# ------------------------------------------------------------------------------
# Public helpers
# ------------------------------------------------------------------------------
@ -432,7 +249,7 @@ proc hexaryPath*(
## Compute the longest possible repair tree `db` path matching the `nodeKey`
## nibbles. The `nodeNey` path argument comes before the `db` one for
## supporting a more functional notation.
RPath(tail: partialPath).pathExtend(rootKey.to(RepairKey), db)
RPath(root: rootKey.to(RepairKey), tail: partialPath).rootPathExtend(db)
proc hexaryPath*(
nodeKey: NodeKey;
@ -469,7 +286,7 @@ proc hexaryPath*(
): XPath
{.gcsafe, raises: [CatchableError]} =
## Compute the longest possible path on an arbitrary hexary trie.
XPath(tail: partialPath).pathExtend(rootKey.to(Blob), getFn)
XPath(root: rootKey, tail: partialPath).rootPathExtend(getFn)
proc hexaryPath*(
nodeKey: NodeKey;
@ -583,71 +400,6 @@ proc hexaryPathNodeKeys*(
.mapIt(it.value)
.toHashSet
# ------------------------------------------------------------------------------
# Public functions, traversal
# ------------------------------------------------------------------------------
proc next*(
path: XPath;
getFn: HexaryGetFn;
minDepth = 64;
): XPath
{.gcsafe, raises: [CatchableError]} =
## Advance the argument `path` to the next leaf node (if any.). The
## `minDepth` argument requires the result of `next()` to satisfy
## `minDepth <= next().getNibbles.len`.
var pLen = path.path.len
# Find the last branch in the path, increase link and step down
while 0 < pLen:
# Find branch none
pLen.dec
let it = path.path[pLen]
if it.node.kind == Branch and it.nibble < 15:
# Find the next item to the right in the branch list
for inx in (it.nibble + 1) .. 15:
let link = it.node.bLink[inx]
if link.len != 0:
let
branch = XPathStep(key: it.key, node: it.node, nibble: inx.int8)
walk = path.path[0 ..< pLen] & branch
newPath = XPath(path: walk).pathLeast(link, getFn)
if minDepth <= newPath.depth and 0 < newPath.leafData.len:
return newPath
proc prev*(
path: XPath;
getFn: HexaryGetFn;
minDepth = 64;
): XPath
{.gcsafe, raises: [CatchableError]} =
## Advance the argument `path` to the previous leaf node (if any.) The
## `minDepth` argument requires the result of `next()` to satisfy
## `minDepth <= next().getNibbles.len`.
var pLen = path.path.len
# Find the last branch in the path, decrease link and step down
while 0 < pLen:
# Find branch none
pLen.dec
let it = path.path[pLen]
if it.node.kind == Branch and 0 < it.nibble:
# Find the next item to the right in the branch list
for inx in (it.nibble - 1).countDown(0):
let link = it.node.bLink[inx]
if link.len != 0:
let
branch = XPathStep(key: it.key, node: it.node, nibble: inx.int8)
walk = path.path[0 ..< pLen] & branch
newPath = XPath(path: walk).pathMost(link, getFn)
if minDepth <= newPath.depth and 0 < newPath.leafData.len:
return newPath
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

14
nimbus/sync/snap/worker/db/hexary_range.nim
View File

@ -12,6 +12,7 @@
import
std/[sequtils, sets, tables],
chronicles,
chronos,
eth/[common, p2p, trie/nibbles],
stew/[byteutils, interval_set],
@ -109,8 +110,9 @@ template collectLeafs(
): auto =
## Collect trie database leafs prototype. This directive is provided as
## `template` for avoiding varying exceprion annotations.
var rc: Result[RangeProof,HexaryError]
var
rc: Result[RangeProof,HexaryError]
ttd = stopAt
block body:
let
nodeMax = maxPt(iv) # `inject` is for debugging (if any)
@ -121,9 +123,9 @@ template collectLeafs(
# Set up base node, the nearest node before `iv.minPt`
if 0.to(NodeTag) < nodeTag:
let rx = nodeTag.hexaryPath(rootKey,db).hexaryNearbyLeft(db)
let rx = nodeTag.hexaryNearbyLeft(rootKey, db)
if rx.isOk:
rls.base = getPartialPath(rx.value).convertTo(NodeKey).to(NodeTag)
rls.base = rx.value
elif rx.error notin {NearbyFailed,NearbyEmptyPath}:
rc = typeof(rc).err(rx.error)
break body
@ -149,7 +151,7 @@ template collectLeafs(
# Prevents from semi-endless looping
if rightTag <= prevTag and 0 < rls.leafs.len:
# Oops, should have been tackeled by `hexaryNearbyRight()`
# Oops, should have been tackled by `hexaryNearbyRight()`
rc = typeof(rc).err(FailedNextNode)
break body # stop here
@ -165,7 +167,7 @@ template collectLeafs(
key: rightKey,
data: xPath.leafData)
if timeIsOver(stopAt):
if timeIsOver(ttd):
break # timout
prevTag = nodeTag

76
nimbus/sync/snap/worker/db/snapdb_accounts.nim
View File

@ -11,7 +11,7 @@
{.push raises: [].}
import
std/[algorithm, sequtils, tables],
std/tables,
chronicles,
eth/[common, p2p, rlp, trie/nibbles],
stew/[byteutils, interval_set],
@ -43,15 +43,6 @@ proc getAccountFn*(ps: SnapDbAccountsRef): HexaryGetFn
proc to(h: Hash256; T: type NodeKey): T =
h.data.T
proc convertTo(data: openArray[byte]; T: type Hash256): T =
discard result.data.NodeKey.init(data) # size error => zero
template noKeyError(info: static[string]; code: untyped) =
try:
code
except KeyError as e:
raiseAssert "Not possible (" & info & "): " & e.msg
template noExceptionOops(info: static[string]; code: untyped) =
try:
code
@ -449,71 +440,6 @@ proc getAccountsData*(
SnapDbAccountsRef.init(
pv, root, Peer()).getAccountsData(path, persistent=true)
# ------------------------------------------------------------------------------
# Public functions: additional helpers
# ------------------------------------------------------------------------------
proc sortMerge*(base: openArray[NodeTag]): NodeTag =
## Helper for merging several `(NodeTag,seq[PackedAccount])` data sets
## so that there are no overlap which would be rejected by `merge()`.
##
## This function selects a `NodeTag` from a list.
result = high(NodeTag)
for w in base:
if w < result:
result = w
proc sortMerge*(acc: openArray[seq[PackedAccount]]): seq[PackedAccount] =
## Helper for merging several `(NodeTag,seq[PackedAccount])` data sets
## so that there are no overlap which would be rejected by `merge()`.
##
## This function flattens and sorts the argument account lists.
noKeyError("sortMergeAccounts"):
var accounts: Table[NodeTag,PackedAccount]
for accList in acc:
for item in accList:
accounts[item.accKey.to(NodeTag)] = item
result = toSeq(accounts.keys).sorted(cmp).mapIt(accounts[it])
proc getAccountsChainDb*(
ps: SnapDbAccountsRef;
accKey: NodeKey;
): Result[Account,HexaryError] =
## Fetch account via `ChainDBRef`
ps.getAccountsData(accKey, persistent = true)
proc nextAccountsChainDbKey*(
ps: SnapDbAccountsRef;
accKey: NodeKey;
): Result[NodeKey,HexaryError] =
## Fetch the account path on the `ChainDBRef`, the one next to the
## argument account key.
noExceptionOops("getChainDbAccount()"):
let path = accKey
.hexaryPath(ps.root, ps.getAccountFn)
.next(ps.getAccountFn)
.getNibbles
if 64 == path.len:
return ok(path.getBytes.convertTo(Hash256).to(NodeKey))
err(AccountNotFound)
proc prevAccountsChainDbKey*(
ps: SnapDbAccountsRef;
accKey: NodeKey;
): Result[NodeKey,HexaryError] =
## Fetch the account path on the `ChainDBRef`, the one before to the
## argument account.
noExceptionOops("getChainDbAccount()"):
let path = accKey
.hexaryPath(ps.root, ps.getAccountFn)
.prev(ps.getAccountFn)
.getNibbles
if 64 == path.len:
return ok(path.getBytes.convertTo(Hash256).to(NodeKey))
err(AccountNotFound)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

180
nimbus/sync/snap/worker/db/snapdb_debug.nim Normal file
View File

@ -0,0 +1,180 @@
# nimbus-eth1
# Copyright (c) 2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
{.push raises: [].}
import
std/[algorithm, sequtils, tables],
eth/[common, trie/nibbles],
stew/results,
../../range_desc,
"."/[hexary_debug, hexary_desc, hexary_error, hexary_paths, snapdb_desc]
# ------------------------------------------------------------------------------
# Private debugging helpers
# ------------------------------------------------------------------------------
template noPpError(info: static[string]; code: untyped) =
try:
code
except ValueError as e:
raiseAssert "Inconveivable (" & info & "): " & e.msg
except KeyError as e:
raiseAssert "Not possible (" & info & "): " & e.msg
except CatchableError as e:
raiseAssert "Ooops (" & info & ") " & $e.name & ": " & e.msg
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
proc convertTo(data: openArray[byte]; T: type Hash256): T =
discard result.data.NodeKey.init(data) # size error => zero
template noKeyError(info: static[string]; code: untyped) =
try:
code
except KeyError as e:
raiseAssert "Not possible (" & info & "): " & e.msg
template noExceptionOops(info: static[string]; code: untyped) =
try:
code
except KeyError as e:
raiseAssert "Not possible -- " & info & ": " & e.msg
except RlpError:
return err(RlpEncoding)
except CatchableError as e:
return err(AccountNotFound)
# ------------------------------------------------------------------------------
# Public functions, pretty printing
# ------------------------------------------------------------------------------
proc pp*(a: RepairKey; ps: SnapDbBaseRef): string =
if not ps.isNil:
let toKey = ps.hexaDb.keyPp
if not toKey.isNil:
try:
return a.toKey
except CatchableError:
discard
$a.ByteArray33
proc pp*(a: NodeKey; ps: SnapDbBaseRef): string =
if not ps.isNil:
let toKey = ps.hexaDb.keyPp
if not toKey.isNil:
try:
return a.to(RepairKey).toKey
except CatchableError:
discard
$a.ByteArray32
proc pp*(a: NodeTag; ps: SnapDbBaseRef): string =
a.to(NodeKey).pp(ps)
# ------------------------------------------------------------------------------
# Public constructor
# ------------------------------------------------------------------------------
proc init*(
T: type HexaryTreeDbRef;
): T =
## Constructor variant. It provides a `HexaryTreeDbRef()` with a key cache
## attached for pretty printing. So this one is mainly for debugging.
HexaryTreeDbRef.init(SnapDbRef())
# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
proc sortMerge*(base: openArray[NodeTag]): NodeTag =
## Helper for merging several `(NodeTag,seq[PackedAccount])` data sets
## so that there are no overlap which would be rejected by `merge()`.
##
## This function selects a `NodeTag` from a list.
result = high(NodeTag)
for w in base:
if w < result:
result = w
proc sortMerge*(acc: openArray[seq[PackedAccount]]): seq[PackedAccount] =
## Helper for merging several `(NodeTag,seq[PackedAccount])` data sets
## so that there are no overlap which would be rejected by `merge()`.
##
## This function flattens and sorts the argument account lists.
noKeyError("sortMergeAccounts"):
var accounts: Table[NodeTag,PackedAccount]
for accList in acc:
for item in accList:
accounts[item.accKey.to(NodeTag)] = item
result = toSeq(accounts.keys).sorted(cmp).mapIt(accounts[it])
proc nextAccountsChainDbKey*(
ps: SnapDbBaseRef;
accKey: NodeKey;
getFn: HexaryGetFn;
): Result[NodeKey,HexaryError] =
## Fetch the account path on the `ChainDBRef`, the one next to the
## argument account key.
noExceptionOops("getChainDbAccount()"):
let path = accKey
.hexaryPath(ps.root, getFn) # ps.getAccountFn)
.next(getFn) # ps.getAccountFn)
.getNibbles
if 64 == path.len:
return ok(path.getBytes.convertTo(Hash256).to(NodeKey))
err(AccountNotFound)
proc prevAccountsChainDbKey*(
ps: SnapDbBaseRef;
accKey: NodeKey;
getFn: HexaryGetFn;
): Result[NodeKey,HexaryError] =
## Fetch the account path on the `ChainDBRef`, the one before to the
## argument account.
noExceptionOops("getChainDbAccount()"):
let path = accKey
.hexaryPath(ps.root, getFn) # ps.getAccountFn)
.prev(getFn) # ps.getAccountFn)
.getNibbles
if 64 == path.len:
return ok(path.getBytes.convertTo(Hash256).to(NodeKey))
err(AccountNotFound)
# ------------------------------------------------------------------------------
# More debugging (and playing with the hexary database)
# ------------------------------------------------------------------------------
proc assignPrettyKeys*(xDb: HexaryTreeDbRef; root: NodeKey) =
## Prepare for pretty pringing/debugging. Run early enough this function
## sets the root key to `"$"`, for instance.
if not xDb.keyPp.isNil:
noPpError("validate(1)"):
# Make keys assigned in pretty order for printing
let rootKey = root.to(RepairKey)
discard xDb.keyPp rootKey
var keysList = toSeq(xDb.tab.keys)
if xDb.tab.hasKey(rootKey):
keysList = @[rootKey] & keysList
for key in keysList:
let node = xDb.tab[key]
discard xDb.keyPp key
case node.kind:
of Branch: (for w in node.bLink: discard xDb.keyPp w)
of Extension: discard xDb.keyPp node.eLink
of Leaf: discard
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

121
nimbus/sync/snap/worker/db/snapdb_desc.nim
View File

@ -11,13 +11,13 @@
{.push raises: [].}
import
std/[sequtils, tables],
std/tables,
chronicles,
eth/[common, p2p, trie/db, trie/nibbles],
../../../../db/[select_backend, storage_types],
../../../protocol,
../../range_desc,
"."/[hexary_desc, hexary_error, hexary_import, hexary_nearby,
"."/[hexary_debug, hexary_desc, hexary_error, hexary_import, hexary_nearby,
hexary_paths, rocky_bulk_load]
logScope:
@ -46,47 +46,20 @@ type
# Private debugging helpers
# ------------------------------------------------------------------------------
template noPpError(info: static[string]; code: untyped) =
template noKeyError(info: static[string]; code: untyped) =
try:
code
except ValueError as e:
raiseAssert "Inconveivable (" & info & "): " & e.msg
except KeyError as e:
raiseAssert "Not possible (" & info & "): " & e.msg
except CatchableError as e:
raiseAssert "Ooops (" & info & ") " & $e.name & ": " & e.msg
proc toKey(a: RepairKey; pv: SnapDbRef): uint =
if not a.isZero:
noPpError("pp(RepairKey)"):
if not pv.keyMap.hasKey(a):
pv.keyMap[a] = pv.keyMap.len.uint + 1
result = pv.keyMap[a]
proc toKey(a: RepairKey; ps: SnapDbBaseRef): uint =
a.toKey(ps.base)
proc toKey(a: NodeKey; ps: SnapDbBaseRef): uint =
a.to(RepairKey).toKey(ps)
#proc toKey(a: NodeTag; ps: SnapDbBaseRef): uint =
# a.to(NodeKey).toKey(ps)
proc ppImpl(a: RepairKey; pv: SnapDbRef): string =
"$" & $a.toKey(pv)
# ------------------------------------------------------------------------------
# Debugging, pretty printing
# ------------------------------------------------------------------------------
proc pp*(a: NodeKey; ps: SnapDbBaseRef): string =
if a.isZero: "ø" else:"$" & $a.toKey(ps)
proc pp*(a: RepairKey; ps: SnapDbBaseRef): string =
if a.isZero: "ø" elif a.isNodeKey: "$" & $a.toKey(ps) else: "@" & $a.toKey(ps)
proc pp*(a: NodeTag; ps: SnapDbBaseRef): string =
a.to(NodeKey).pp(ps)
proc keyPp(a: RepairKey; pv: SnapDbRef): string =
if a.isZero:
return "ø"
if not pv.keyMap.hasKey(a):
pv.keyMap[a] = pv.keyMap.len.uint + 1
result = if a.isNodeKey: "$" else: "@"
noKeyError("pp(RepairKey)"):
result &= $pv.keyMap[a]
# ------------------------------------------------------------------------------
# Private helper
@ -127,7 +100,7 @@ proc init*(
): T =
## Constructor for inner hexary trie database
let xDb = HexaryTreeDbRef()
xDb.keyPp = proc(key: RepairKey): string = key.ppImpl(pv) # will go away
xDb.keyPp = proc(key: RepairKey): string = key.keyPp(pv) # will go away
return xDb
proc init*(
@ -137,13 +110,6 @@ proc init*(
## Constructor variant
HexaryTreeDbRef.init(ps.base)
proc init*(
T: type HexaryTreeDbRef;
): T =
## Constructor variant. It provides a `HexaryTreeDbRef()` with a key key cache attached
## for pretty printing. So this one is mainly for debugging.
HexaryTreeDbRef.init(SnapDbRef())
# ---------------
proc init*(
@ -292,7 +258,10 @@ proc verifyNoMoreRight*(
let
root = root.to(RepairKey)
base = base.to(NodeKey)
if base.hexaryPath(root, xDb).hexaryNearbyRightMissing(xDb):
rc = base.hexaryPath(root, xDb).hexaryNearbyRightMissing(xDb)
if rc.isErr:
return err(rc.error)
if rc.value:
return ok()
let error = LowerBoundProofError
@ -300,64 +269,6 @@ proc verifyNoMoreRight*(
trace "verifyLeftmostBound()", peer, base=base.pp, error
err(error)
# ------------------------------------------------------------------------------
# Debugging (and playing with the hexary database)
# ------------------------------------------------------------------------------
proc assignPrettyKeys*(xDb: HexaryTreeDbRef; root: NodeKey) =
## Prepare for pretty pringing/debugging. Run early enough this function
## sets the root key to `"$"`, for instance.
if not xDb.keyPp.isNil:
noPpError("validate(1)"):
# Make keys assigned in pretty order for printing
let rootKey = root.to(RepairKey)
discard xDb.keyPp rootKey
var keysList = toSeq(xDb.tab.keys)
if xDb.tab.hasKey(rootKey):
keysList = @[rootKey] & keysList
for key in keysList:
let node = xDb.tab[key]
discard xDb.keyPp key
case node.kind:
of Branch: (for w in node.bLink: discard xDb.keyPp w)
of Extension: discard xDb.keyPp node.eLink
of Leaf: discard
proc dumpPath*(ps: SnapDbBaseRef; key: NodeTag): seq[string] =
## Pretty print helper compiling the path into the repair tree for the
## argument `key`.
noPpError("dumpPath"):
let rPath= key.hexaryPath(ps.root, ps.hexaDb)
result = rPath.path.mapIt(it.pp(ps.hexaDb)) & @["(" & rPath.tail.pp & ")"]
proc dumpHexaDB*(xDb: HexaryTreeDbRef; root: NodeKey; indent = 4): string =
## Dump the entries from the a generic accounts trie. These are
## key value pairs for
## ::
## Branch: ($1,b(<$2,$3,..,$17>,))
## Extension: ($18,e(832b5e..06e697,$19))
## Leaf: ($20,l(cc9b5d..1c3b4,f84401..f9e5129d[#70]))
##
## where keys are typically represented as `$<id>` or `¶<id>` or `ø`
## depending on whether a key is final (`$<id>`), temporary (`¶<id>`)
## or unset/missing (`ø`).
##
## The node types are indicated by a letter after the first key before
## the round brackets
## ::
## Branch: 'b', 'þ', or 'B'
## Extension: 'e', '€', or 'E'
## Leaf: 'l', 'ł', or 'L'
##
## Here a small letter indicates a `Static` node which was from the
## original `proofs` list, a capital letter indicates a `Mutable` node
## added on the fly which might need some change, and the decorated
## letters stand for `Locked` nodes which are like `Static` ones but
## added later (typically these nodes are update `Mutable` nodes.)
##
## Beware: dumping a large database is not recommended
xDb.pp(root, indent)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

1
nimbus/sync/snap/worker_desc.nim
View File

@ -65,7 +65,6 @@ type
fetchStorageFull*: SnapSlotsQueue ## Fetch storage trie for these accounts
fetchStoragePart*: SnapSlotsQueue ## Partial storage trie to com[plete
parkedStorage*: HashSet[NodeKey] ## Storage batch items in use
storageDone*: bool ## Done with storage, block sync next
# Info
nAccounts*: uint64 ## Imported # of accounts

2
tests/replay/undump_accounts.nim
View File

@ -99,8 +99,6 @@ proc dumpAccounts*(
iterator undumpNextAccount*(gzFile: string): UndumpAccounts =
var
line = ""
lno = 0
state = UndumpHeader
data: UndumpAccounts
nAccounts = 0u

5
tests/replay/undump_blocks.nim
View File

@ -24,9 +24,6 @@ template say(args: varargs[untyped]) =
# echo args
discard
proc toByteSeq(s: string): seq[byte] =
utils.fromHex(s)
# ------------------------------------------------------------------------------
# Public capture
# ------------------------------------------------------------------------------
@ -90,8 +87,6 @@ iterator undumpNextGroup*(gzFile: string): (seq[BlockHeader],seq[BlockBody]) =
var
headerQ: seq[BlockHeader]
bodyQ: seq[BlockBody]
line = ""
lno = 0
current = 0u
start = 0u
top = 0u

2
tests/replay/undump_storages.nim
View File

@ -104,8 +104,6 @@ proc dumpStorages*(
iterator undumpNextStorages*(gzFile: string): UndumpStorages =
var
line = ""
lno = 0
state = UndumpStoragesHeader
data: UndumpStorages
nAccounts = 0u

14
tests/test_sync_snap.nim
View File

@ -23,7 +23,7 @@ import
../nimbus/sync/snap/range_desc,
../nimbus/sync/snap/worker/db/[
hexary_desc, hexary_envelope, hexary_error, hexary_inspect, hexary_nearby,
hexary_paths, rocky_bulk_load, snapdb_accounts, snapdb_desc],
hexary_paths, rocky_bulk_load, snapdb_accounts, snapdb_debug, snapdb_desc],
./replay/[pp, undump_accounts, undump_storages],
./test_sync_snap/[
bulk_test_xx, snap_test_xx,
@ -91,12 +91,12 @@ proc findFilePath(file: string;
proc getTmpDir(sampleDir = sampleDirRefFile): string =
sampleDir.findFilePath(baseDir,repoDir).value.splitFile.dir
proc setTraceLevel =
proc setTraceLevel {.used.} =
discard
when defined(chronicles_runtime_filtering) and loggingEnabled:
setLogLevel(LogLevel.TRACE)
proc setErrorLevel =
proc setErrorLevel {.used.} =
discard
when defined(chronicles_runtime_filtering) and loggingEnabled:
setLogLevel(LogLevel.ERROR)
@ -144,12 +144,12 @@ proc flushDbDir(s: string; subDir = "") =
let instDir = if subDir == "": baseDir / $n else: baseDir / subDir / $n
if (instDir / "nimbus" / "data").dirExists:
# Typically under Windows: there might be stale file locks.
try: instDir.removeDir except: discard
try: (baseDir / subDir).removeDir except: discard
try: instDir.removeDir except CatchableError: discard
try: (baseDir / subDir).removeDir except CatchableError: discard
block dontClearUnlessEmpty:
for w in baseDir.walkDir:
break dontClearUnlessEmpty
try: baseDir.removeDir except: discard
try: baseDir.removeDir except CatchableError: discard
proc flushDbs(db: TestDbs) =
@ -233,7 +233,7 @@ proc accountsRunner(noisy = true; persistent = true; sample = accSample) =
hexaDb.assignPrettyKeys(root.to(NodeKey))
# Beware: dumping a large database is not recommended
# true.say "***", "database dump\n ", hexaDb.dumpHexaDB(root)
# true.say "***", "database dump\n ", hexaDb.pp(root.to(NodeKey))
test &"Retrieve accounts & proofs for previous account ranges":
if db.persistent:

48
tests/test_sync_snap/test_accounts.nim
View File

@ -64,7 +64,9 @@ import
../../nimbus/db/select_backend,
../../nimbus/sync/protocol,
../../nimbus/sync/snap/range_desc,
../../nimbus/sync/snap/worker/db/[snapdb_accounts, snapdb_desc],
../../nimbus/sync/snap/worker/db/[
hexary_debug, hexary_desc, hexary_error,
snapdb_accounts, snapdb_debug, snapdb_desc],
../replay/[pp, undump_accounts],
./test_helpers
@ -97,10 +99,13 @@ proc test_accountsMergeProofs*(
) =
## Merge account proofs
# Load/accumulate data from several samples (needs some particular sort)
let baseTag = inList.mapIt(it.base).sortMerge
let packed = PackedAccountRange(
accounts: inList.mapIt(it.data.accounts).sortMerge,
proof: inList.mapIt(it.data.proof).flatten)
let
getFn = desc.getAccountFn
baseTag = inList.mapIt(it.base).sortMerge
packed = PackedAccountRange(
accounts: inList.mapIt(it.data.accounts).sortMerge,
proof: inList.mapIt(it.data.proof).flatten)
nAccounts = packed.accounts.len
# Merging intervals will produce gaps, so the result is expected OK but
# different from `.isImportOk`
check desc.importAccounts(baseTag, packed, true).isOk
@ -114,21 +119,29 @@ proc test_accountsMergeProofs*(
# need to check for additional records only on either end of a range.
var keySet = packed.accounts.mapIt(it.accKey).toHashSet
for w in inList:
var key = desc.prevAccountsChainDbKey(w.data.accounts[0].accKey)
var key = desc.prevAccountsChainDbKey(w.data.accounts[0].accKey, getFn)
while key.isOk and key.value notin keySet:
keySet.incl key.value
let newKey = desc.prevAccountsChainDbKey(key.value)
let newKey = desc.prevAccountsChainDbKey(key.value, getFn)
check newKey != key
key = newKey
key = desc.nextAccountsChainDbKey(w.data.accounts[^1].accKey)
key = desc.nextAccountsChainDbKey(w.data.accounts[^1].accKey, getFn)
while key.isOk and key.value notin keySet:
keySet.incl key.value
let newKey = desc.nextAccountsChainDbKey(key.value)
let newKey = desc.nextAccountsChainDbKey(key.value, getFn)
check newKey != key
key = newKey
accKeys = toSeq(keySet).mapIt(it.to(NodeTag)).sorted(cmp)
.mapIt(it.to(NodeKey))
check packed.accounts.len <= accKeys.len
# Some database samples have a few more account keys which come in by the
# proof nodes.
check nAccounts <= accKeys.len
# Verify against table importer
let
xDb = HexaryTreeDbRef.init() # Can dump database with `.pp(xDb)`
rc = xDb.fromPersistent(desc.root, getFn, accKeys.len + 100)
check rc == Result[int,HexaryError].ok(accKeys.len)
proc test_accountsRevisitStoredItems*(
@ -137,6 +150,8 @@ proc test_accountsRevisitStoredItems*(
noisy = false;
) =
## Revisit stored items on ChainDBRef
let
getFn = desc.getAccountFn
var
nextAccount = accKeys[0]
prevAccount: NodeKey
@ -145,12 +160,13 @@ proc test_accountsRevisitStoredItems*(
count.inc
let
pfx = $count & "#"
byChainDB = desc.getAccountsChainDb(accKey)
byNextKey = desc.nextAccountsChainDbKey(accKey)
byPrevKey = desc.prevAccountsChainDbKey(accKey)
noisy.say "*** find",
"<", count, "> byChainDb=", byChainDB.pp
check byChainDB.isOk
byChainDB = desc.getAccountsData(accKey, persistent=true)
byNextKey = desc.nextAccountsChainDbKey(accKey, getFn)
byPrevKey = desc.prevAccountsChainDbKey(accKey, getFn)
if byChainDB.isErr:
noisy.say "*** find",
"<", count, "> byChainDb=", byChainDB.pp
check byChainDB.isOk
# Check `next` traversal funcionality. If `byNextKey.isOk` fails, the
# `nextAccount` value is still the old one and will be different from

28
tests/test_sync_snap/test_node_range.nim
View File

@ -19,8 +19,9 @@ import
../../nimbus/sync/[handlers, protocol, types],
../../nimbus/sync/snap/range_desc,
../../nimbus/sync/snap/worker/db/[
hexary_desc, hexary_envelope, hexary_error, hexary_interpolate,
hexary_nearby, hexary_paths, hexary_range, snapdb_accounts, snapdb_desc],
hexary_debug, hexary_desc, hexary_envelope, hexary_error,
hexary_interpolate, hexary_nearby, hexary_paths, hexary_range,
snapdb_accounts, snapdb_debug, snapdb_desc],
../replay/[pp, undump_accounts],
./test_helpers
@ -209,7 +210,6 @@ proc verifyRangeProof(
): Result[void,HexaryError] =
## Re-build temporary database and prove or disprove
let
dumpOk = dbg.isNil.not
noisy = dbg.isNil.not
xDb = HexaryTreeDbRef()
if not dbg.isNil:
@ -252,7 +252,7 @@ proc verifyRangeProof(
"\n\n last=", leafs[^1].key,
"\n ", leafs[^1].key.hexaryPath(rootKey,xDb).pp(dbg),
"\n\n database dump",
"\n ", xDb.dumpHexaDB(rootKey),
"\n ", xDb.pp(rootKey),
"\n"
# ------------------------------------------------------------------------------
@ -270,8 +270,6 @@ proc test_NodeRangeDecompose*(
# stray account nodes in the proof *before* the left boundary.
doAssert 2 < accKeys.len
const
isPersistent = db.type is HexaryTreeDbRef
let
rootKey = root.to(NodeKey)
baseTag = accKeys[0].to(NodeTag) + 1.u256
@ -392,12 +390,16 @@ proc test_NodeRangeProof*(
for n,w in inLst:
doAssert 1 < w.data.accounts.len
let
# Use the middle of the first two points as base
delta = (w.data.accounts[1].accKey.to(NodeTag) -
w.data.accounts[0].accKey.to(NodeTag)) div 2
base = w.data.accounts[0].accKey.to(NodeTag) + delta
first = w.data.accounts[0].accKey.to(NodeTag)
delta = (w.data.accounts[1].accKey.to(NodeTag) - first) div 2
# Use the middle of the first two points as base unless w.base is zero.
# This is needed as the range extractor needs the node before the `base`
# (if ateher is any) in order to assemble the proof. But this node might
# not be present in the partial database.
(base, start) = if w.base == 0.to(NodeTag): (w.base, 0)
else: (first + delta, 1)
# Assemble accounts list starting at the second item
accounts = w.data.accounts[1 ..< min(w.data.accounts.len,maxLen)]
accounts = w.data.accounts[start ..< min(w.data.accounts.len,maxLen)]
iv = NodeTagRange.new(base, accounts[^1].accKey.to(NodeTag))
rc = db.hexaryRangeLeafsProof(rootKey, iv)
check rc.isOk
@ -486,7 +488,7 @@ proc test_NodeRangeLeftBoundary*(
## Verify left side boundary checks
let
rootKey = inLst[0].root.to(NodeKey)
noisy = not dbg.isNil
noisy {.used.} = not dbg.isNil
# Assuming the `inLst` entries have been stored in the DB already
for n,w in inLst:
@ -505,7 +507,7 @@ proc test_NodeRangeLeftBoundary*(
check (n, j, leftKey) == (n, j, toLeftKey)
rootKey.printCompareLeftNearby(leftKey, rightKey, db, dbg)
return
noisy.say "***", "n=", n, " accounts=", accounts.len
# noisy.say "***", "n=", n, " accounts=", accounts.len
# ------------------------------------------------------------------------------
# End