Prepare snap server client test scenario cont4 (#1507)

* Add state root to node steps path register `RPath` or `XPath`

why:
  Typically, the first node in the path register is the state root. There
  are occasions, when the path register is empty (i.e. there are no node
  references) which typically applies to a zero node key.

  In order to find the next node key greater than zero, the state root is
  is needed which is now part of the `RPath` or `XPath` data types.

* Extracted hexary tree debugging functions into separate files

* Update empty path fringe case for left/right node neighbour

why:
  When starting at zero, the node steps path register would be empty. So
  will any path that is before the fist non-zero link of a state root (if
  it is a `Branch` node.)

  The `hexaryNearbyRight()` or `hexaryNearbyLeft()` function required a
  non-zero node steps path register.  Now the first node is to be advanced
  starting at the first state root link if necessary.

* Simplify/reorg neighbour node finder

why:
  There was too mach code repetition for the cases
  * persistent or in-memory database
  * left or right move

details:
  Most algorithms apply for persistent and in-memory alike. Using
  templates/generic functions most of these algorithms can be stated
  in a unified way

* Update storage slots snap/1 handler

details:
  Minor changes to be more debugging friendly.

* Fix detection of full database for snap sync

* Docu: Snap sync test & debugging scenario

nimbus/sync/handlers/snap.nim

@@ -11,7 +11,7 @@
 {.push raises: [].}
 
 import
-  std/sequtils,
+  std/[sequtils, strutils],
   chronicles,
   chronos,
   eth/[p2p, trie/trie_defs],
@@ -19,7 +19,7 @@ import
   ../../db/db_chain,
   ../../core/chain,
   ../snap/[constants, range_desc],
-  ../snap/worker/db/[hexary_desc, hexary_paths, hexary_range],
+  ../snap/worker/db/[hexary_desc, hexary_error, hexary_paths, hexary_range],
   ../protocol,
   ../protocol/snap/snap_types
 
@@ -43,7 +43,7 @@ const
   emptySnapStorageList = seq[SnapStorage].default
     ## Dummy list for empty slots
 
-  defaultElaFetchMax = 1500.milliseconds
+  defaultElaFetchMax = 990.milliseconds
     ## Fetching accounts or slots can be extensive, stop in the middle if
     ## it takes too long
 
@@ -70,7 +70,7 @@ proc getAccountFn(
   return proc(key: openArray[byte]): Blob =
     db.get(key)
 
-proc getStorageSlotsFn(
+proc getStoSlotFn(
     chain: ChainRef;
     accKey: NodeKey;
       ): HexaryGetFn
@@ -106,26 +106,37 @@ proc to(
 proc mkNodeTagRange(
     origin: openArray[byte];
     limit: openArray[byte];
+    nAccounts = 1;
       ): Result[NodeTagRange,void] =
   var (minPt, maxPt) = (low(NodeTag), high(NodeTag))
 
   if 0 < origin.len or 0 < limit.len:
+
+    # Range applies only if there is exactly one account. A number of accounts
+    # different from 1 may be used by `getStorageRanges()`
+    if nAccounts == 0:
+      return err() # oops: no account
+
+    # Veriify range atguments
     if not minPt.init(origin) or not maxPt.init(limit) or maxPt <= minPt:
       when extraTraceMessages:
         trace logTxt "mkNodeTagRange: malformed range", origin, limit
       return err()
 
+    if 1 < nAccounts:
+      return ok(NodeTagRange.new(low(NodeTag), high(NodeTag)))
+
   ok(NodeTagRange.new(minPt, maxPt))
 
 
 proc fetchLeafRange(
     ctx: SnapWireRef;                   # Handler descriptor
-    db: HexaryGetFn;                    # Database abstraction
+    getFn: HexaryGetFn;                 # Database abstraction
     root: Hash256;                      # State root
     iv: NodeTagRange;                   # Proofed range of leaf paths
     replySizeMax: int;                  # Updated size counter for the raw list
     stopAt: Moment;                     # Implies timeout
-      ): Result[RangeProof,void]
+      ): Result[RangeProof,HexaryError]
       {.gcsafe, raises: [CatchableError].} =
 
   # Assemble result Note that the size limit is the size of the leaf nodes
@@ -136,44 +147,48 @@ proc fetchLeafRange(
     sizeMax = replySizeMax - estimatedProofSize
     now = Moment.now()
     timeout = if now < stopAt: stopAt - now else: 1.milliseconds
-    rc = db.hexaryRangeLeafsProof(rootKey, iv, sizeMax, timeout)
+    rc = getFn.hexaryRangeLeafsProof(rootKey, iv, sizeMax, timeout)
   if rc.isErr:
-    debug logTxt "fetchLeafRange: database problem",
+    error logTxt "fetchLeafRange: database problem",
       iv, replySizeMax, error=rc.error
-    return err() # database error
+    return rc # database error
-  let sizeOnWire = rc.value.leafsSize + rc.value.proofSize
 
+  let sizeOnWire = rc.value.leafsSize + rc.value.proofSize
   if sizeOnWire <= replySizeMax:
-    return ok(rc.value)
+    return rc
+
+  # Estimate the overhead size on wire needed for a single leaf tail item
+  const leafExtraSize = (sizeof RangeLeaf()) - (sizeof newSeq[Blob](0))
+
+  let nLeafs = rc.value.leafs.len
+  when extraTraceMessages:
+    trace logTxt "fetchLeafRange: reducing reply sample",
+      iv, sizeOnWire, replySizeMax, nLeafs
 
   # Strip parts of leafs result and amend remainder by adding proof nodes
-  var
+  var (tailSize, tailItems, reduceBy) = (0, 0, replySizeMax - sizeOnWire)
-    rpl = rc.value
+  while tailSize <= reduceBy:
-    leafsTop = rpl.leafs.len - 1
-    tailSize = 0
-    tailItems = 0
-    reduceBy = replySizeMax - sizeOnWire
-  while tailSize <= reduceBy and tailItems < leafsTop:
-    # Estimate the size on wire needed for the tail item
-    const extraSize = (sizeof RangeLeaf()) - (sizeof newSeq[Blob](0))
-    tailSize += rpl.leafs[leafsTop - tailItems].data.len + extraSize
     tailItems.inc
-  if leafsTop <= tailItems:
+    if nLeafs <= tailItems:
-    debug logTxt "fetchLeafRange: stripping leaf list failed",
+      when extraTraceMessages:
-      iv, replySizeMax, leafsTop, tailItems
+        trace logTxt "fetchLeafRange: stripping leaf list failed",
-    return err() # package size too small
+          iv, replySizeMax, nLeafs, tailItems
+      return err(DataSizeError) # empty tail (package size too small)
+    tailSize += rc.value.leafs[^tailItems].data.len + leafExtraSize
 
-  rpl.leafs.setLen(leafsTop - tailItems - 1) # chop off one more for slack
+  # Provide truncated leafs list
   let
-    leafProof = db.hexaryRangeLeafsProof(rootKey, rpl)
+    leafProof = getFn.hexaryRangeLeafsProof(
+      rootKey, RangeProof(leafs: rc.value.leafs[0 ..< nLeafs - tailItems]))
     strippedSizeOnWire = leafProof.leafsSize + leafProof.proofSize
   if strippedSizeOnWire <= replySizeMax:
     return ok(leafProof)
 
-  debug logTxt "fetchLeafRange: data size problem",
+  when extraTraceMessages:
-    iv, replySizeMax, leafsTop, tailItems, strippedSizeOnWire
+    trace logTxt "fetchLeafRange: data size problem",
+      iv, replySizeMax, nLeafs, tailItems, strippedSizeOnWire
 
-  err()
+  err(DataSizeError)
 
 # ------------------------------------------------------------------------------
 # Private functions: peer observer
@@ -254,8 +269,8 @@ method getAccountRange*(
     iv = block: # Calculate effective accounts range (if any)
       let rc = origin.mkNodeTagRange limit
       if rc.isErr:
-        return
+        return # malformed interval
-      rc.value # malformed interval
+      rc.value
 
     db = ctx.chain.getAccountFn
     stopAt = Moment.now() + ctx.elaFetchMax
@@ -293,10 +308,10 @@ method getStorageRanges*(
 
   let
     iv = block: # Calculate effective slots range (if any)
-      let rc = origin.mkNodeTagRange limit
+      let rc = origin.mkNodeTagRange(limit, accounts.len)
       if rc.isErr:
-        return
+        return # malformed interval
-      rc.value # malformed interval
+      rc.value
 
     accGetFn = ctx.chain.getAccountFn
     rootKey = root.to(NodeKey)
@@ -331,19 +346,30 @@ method getStorageRanges*(
         accDataLen=accData.len, stoRoot
       continue
 
-    # Collect data slots for this account
+    # Stop unless there is enough space left
+    if sizeMax - dataAllocated <= estimatedProofSize:
+      break
+
+    # Prepare for data collection
     let
-      db = ctx.chain.getStorageSlotsFn(accKey)
+      slotsGetFn = ctx.chain.getStoSlotFn(accKey)
-      rc = ctx.fetchLeafRange(db, stoRoot, iv, sizeMax - dataAllocated, stopAt)
+      sizeLeft = sizeMax - dataAllocated
+
+    # Collect data slots for this account
+    let rc = ctx.fetchLeafRange(slotsGetFn, stoRoot, iv, sizeLeft, stopAt)
     if rc.isErr:
       when extraTraceMessages:
-        trace logTxt "getStorageRanges: failed", iv, sizeMax, dataAllocated,
+        trace logTxt "getStorageRanges: failed", iv, sizeMax, sizeLeft,
-          accDataLen=accData.len, stoRoot
+          accDataLen=accData.len, stoRoot, error=rc.error
       return # extraction failed
 
     # Process data slots for this account
     dataAllocated += rc.value.leafsSize
 
+    when extraTraceMessages:
+      if accounts.len == 1:
+        trace logTxt "getStorageRanges: single account", iv, accKey, stoRoot
+
     #trace logTxt "getStorageRanges: data slots", iv, sizeMax, dataAllocated,
     #  accKey, stoRoot, nSlots=rc.value.leafs.len, nProof=rc.value.proof.len
 

nimbus/sync/snap/README.txt

@@ -0,0 +1,62 @@
+Snap sync test & debugging scenario
+===================================
+
+
+Start snap/1 server
+-------------------
+
+  # Enter nimbus directory for snap/1 protocol server.
+  cd server
+
+  # Tell nimbus to stop full sync after 2 mio blocks.
+  echo 2000000 > full-limit.txt
+
+  # Tell nimbus to use this predefined key ID
+  echo 123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0 > full-id.key
+
+  ./build/nimbus \
+    --tcp-port:30319 --nat=None --sync-mode=full \
+	--protocols=snap --discovery=none \
+	--net-key=./full-id.key \
+	--sync-ctrl-file=./full-limit.txt \
+	--log-level:TRACE
+
+  # Wait for several hours until enough blocks have been downloaded so that
+  # snap sync data are available. The full 2 mio blocks are available if the
+  # log ticker shows something like
+  #
+  # INF 2023-03-17 [..] Sync statistics (suspended) topics="full-tick" [..] persistent=#2000080 [..]
+  #
+  # where the persistent=#2000080 field might vary
+
+
+Start snap/1 client
+-------------------
+
+  # Note: When the snap/1 server has enough blocks, the client can be started.
+
+  # Enter nimbus directory for snap/1 protocol server
+  cd client
+
+  # Tell nimbus to use this pivot block number. This number must be smaller
+  # than the 2000000 written into the file full-limit.txt above.
+  echo 600000 > snap/snap-update.txt.
+
+  # Tell nimbus to use this hard coded peer enode.
+  echo enode://192d7e7a302bd4ff27f48d7852621e0d3cb863a6dd67dd44e0314a25a3aa866837f0d2460b4444dc66e7b7a2cd56a2de1c31b2a2ba4e23549bf3ba3b0c4f2eb5@127.0.0.1:30319 > snap/full-servers.txt
+
+  ./build/nimbus \
+    --tcp-port:30102 --nat=None --sync-mode=snap \
+	--protocols=none --discovery=none \
+	--static-peers-file=./full-servers.txt \
+	--sync-ctrl-file=./snap-update.txt \
+	--log-level:TRACE
+
+
+Modifications while the programs are syncing
+--------------------------------------------
+
+  # Increasing the number in the files full/full-limit.txt or
+  # snap/snap-update.txt will be recognised while running. Decreasing
+  # or removing will be ignored.
+

nimbus/sync/snap/worker.nim

@@ -225,14 +225,21 @@ proc runMulti*(buddy: SnapBuddyRef) {.async.} =
         return # nothing to do
       rc.value
     pivot = "#" & $env.stateHeader.blockNumber # for logging
+    nStorQuAtStart = env.fetchStorageFull.len +
+                     env.fetchStoragePart.len +
+                     env.parkedStorage.len
 
   buddy.only.pivotEnv = env
 
   # Full sync processsing based on current snapshot
   # -----------------------------------------------
-  if env.storageDone:
+
+  # Check whether this pivot is fully downloaded
+  if env.fetchAccounts.processed.isFull and nStorQuAtStart == 0:
     trace "Snap full sync -- not implemented yet", peer, pivot
     await sleepAsync(5.seconds)
+    # flip over to single mode for getting new instructins
+    buddy.ctrl.multiOk = false
     return
 
   # Snapshot sync processing
@@ -248,9 +255,8 @@ proc runMulti*(buddy: SnapBuddyRef) {.async.} =
         nAccounts {.used.} = env.nAccounts
         nSlotLists {.used.} = env.nSlotLists
         processed {.used.} = env.fetchAccounts.processed.fullFactor.toPC(2)
-        nStoQu {.used.} = env.fetchStorageFull.len + env.fetchStoragePart.len
       trace "Multi sync runner", peer, pivot, nAccounts, nSlotLists, processed,
-        nStoQu
+        nStoQu=nStorQuAtStart
 
   # This one is the syncing work horse which downloads the database
   await env.execSnapSyncAction(buddy)
@@ -260,7 +266,7 @@ proc runMulti*(buddy: SnapBuddyRef) {.async.} =
     nAccounts = env.nAccounts
     nSlotLists = env.nSlotLists
     processed = env.fetchAccounts.processed.fullFactor.toPC(2)
-    nStoQu = env.fetchStorageFull.len + env.fetchStoragePart.len
+    nStoQuLater = env.fetchStorageFull.len + env.fetchStoragePart.len
 
   if env.archived:
     # Archive pivot if it became stale
@@ -273,11 +279,11 @@ proc runMulti*(buddy: SnapBuddyRef) {.async.} =
     let rc = env.saveCheckpoint(ctx)
     if rc.isErr:
       error "Failed to save recovery checkpoint", peer, pivot, nAccounts,
-       nSlotLists, processed, nStoQu, error=rc.error
+       nSlotLists, processed, nStoQu=nStoQuLater, error=rc.error
     else:
       when extraTraceMessages:
         trace "Saved recovery checkpoint", peer, pivot, nAccounts, nSlotLists,
-          processed, nStoQu, blobSize=rc.value
+          processed, nStoQu=nStoQuLater, blobSize=rc.value
 
 # ------------------------------------------------------------------------------
 # End

nimbus/sync/snap/worker/db/hexary_debug.nim

@@ -0,0 +1,666 @@
+# nimbus-eth1
+# Copyright (c) 2021 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+## Find node paths in hexary tries.
+
+{.push raises: [].}
+
+import
+  std/[algorithm, sequtils, sets, strutils, tables, times],
+  chronos,
+  eth/[common, trie/nibbles],
+  stew/results,
+  ../../range_desc,
+  "."/[hexary_desc, hexary_error]
+
+proc next*(path: XPath; getFn: HexaryGetFn; minDepth = 64): XPath
+    {.gcsafe, raises: [CatchableError].}
+
+proc prev*(path: XPath; getFn: HexaryGetFn; minDepth = 64): XPath
+    {.gcsafe, raises: [CatchableError].}
+
+# ------------------------------------------------------------------------------
+# Private pretty printing helpers
+# ------------------------------------------------------------------------------
+
+proc asDateTime(m: Moment): DateTime =
+  ## Approximate UTC based `DateTime` for a `Moment`
+  let
+    utcNow = times.now().utc
+    momNow = Moment.now()
+  utcNow + initDuration(nanoseconds = (m - momNow).nanoseconds)
+
+# --------------
+
+proc toPfx(indent: int): string =
+  "\n" & " ".repeat(indent)
+
+proc ppImpl(s: string; hex = false): string =
+  ## For long strings print `begin..end` only
+  if hex:
+    let n = (s.len + 1) div 2
+    (if s.len < 20: s else: s[0 .. 5] & ".." & s[s.len-8 .. s.len-1]) &
+      "[" & (if 0 < n: "#" & $n else: "") & "]"
+  elif s.len <= 30:
+    s
+  else:
+    (if (s.len and 1) == 0: s[0 ..< 8] else: "0" & s[0 ..< 7]) &
+      "..(" & $s.len & ").." & s[s.len-16 ..< s.len]
+
+proc ppImpl(key: RepairKey; db: HexaryTreeDbRef): string =
+  if key.isZero:
+    return "ø"
+  if not key.isNodekey:
+    var num: uint64
+    (addr num).copyMem(unsafeAddr key.ByteArray33[25], 8)
+    return "%" & $num
+  try:
+    if not disablePrettyKeys and not db.keyPp.isNil:
+      return db.keyPp(key)
+  except CatchableError:
+    discard
+  key.ByteArray33.toSeq.mapIt(it.toHex(2)).join.toLowerAscii
+
+proc ppImpl(key: NodeKey; db: HexaryTreeDbRef): string =
+  key.to(RepairKey).ppImpl(db)
+
+proc ppImpl(w: openArray[RepairKey]; db: HexaryTreeDbRef): string =
+  w.mapIt(it.ppImpl(db)).join(",")
+
+proc ppImpl(w: openArray[Blob]; db: HexaryTreeDbRef): string =
+  var q: seq[RepairKey]
+  for a in w:
+    var key: RepairKey
+    discard key.init(a)
+    q.add key
+  q.ppImpl(db)
+
+proc ppStr(blob: Blob): string =
+  if blob.len == 0: ""
+  else: blob.mapIt(it.toHex(2)).join.toLowerAscii.ppImpl(hex = true)
+
+proc ppImpl(n: RNodeRef; db: HexaryTreeDbRef): string =
+  let so = n.state.ord
+  case n.kind:
+  of Leaf:
+    ["l","ł","L","R"][so] & "(" & $n.lPfx & "," & n.lData.ppStr & ")"
+  of Extension:
+    ["e","€","E","R"][so] & "(" & $n.ePfx & "," & n.eLink.ppImpl(db) & ")"
+  of Branch:
+    ["b","þ","B","R"][so] & "(" & n.bLink.ppImpl(db) & "," & n.bData.ppStr & ")"
+
+proc ppImpl(n: XNodeObj; db: HexaryTreeDbRef): string =
+  case n.kind:
+  of Leaf:
+    "l(" & $n.lPfx & "," & n.lData.ppStr & ")"
+  of Extension:
+    var key: RepairKey
+    discard key.init(n.eLink)
+    "e(" & $n.ePfx & "," & key.ppImpl(db) & ")"
+  of Branch:
+    "b(" & n.bLink[0..15].ppImpl(db) & "," &  n.bLink[16].ppStr & ")"
+
+proc ppImpl(w: RPathStep; db: HexaryTreeDbRef): string =
+  let
+    nibble = if 0 <= w.nibble: w.nibble.toHex(1).toLowerAscii else: "ø"
+    key = w.key.ppImpl(db)
+  "(" & key & "," & nibble & "," & w.node.ppImpl(db) & ")"
+
+proc ppImpl(w: XPathStep; db: HexaryTreeDbRef): string =
+  let nibble = if 0 <= w.nibble: w.nibble.toHex(1).toLowerAscii else: "ø"
+  var key: RepairKey
+  discard key.init(w.key)
+  "(" & key.ppImpl(db) & "," & $nibble & "," & w.node.ppImpl(db) & ")"
+
+proc ppImpl(db: HexaryTreeDbRef; root: NodeKey): seq[string] =
+  ## Dump the entries from the a generic repair tree. This function assumes
+  ## that mapped keys are printed `$###` if a node is locked or static, and
+  ## some substitute for the first letter `$` otherwise (if they are mutable.)
+  proc toKey(s: string): uint64 =
+    try:
+      result = s[1 ..< s.len].parseUint
+    except ValueError as e:
+      raiseAssert "Ooops ppImpl(s=" & s & "): name=" & $e.name & " msg=" & e.msg
+    if s[0] != '$':
+      result = result or (1u64 shl 63)
+  proc cmpIt(x, y: (uint64,string)): int =
+    cmp(x[0],y[0])
+
+  var accu: seq[(uint64,string)]
+  if root.ByteArray32 != ByteArray32.default:
+    accu.add @[(0u64, "($0" & "," & root.ppImpl(db) & ")")]
+  for key,node in db.tab.pairs:
+    accu.add (
+      key.ppImpl(db).tokey,
+      "(" & key.ppImpl(db) & "," & node.ppImpl(db) & ")")
+
+  accu.sorted(cmpIt).mapIt(it[1])
+
+# ------------------------------------------------------------------------------
+# Private helpers
+# ------------------------------------------------------------------------------
+
+proc getNibblesImpl(path: XPath; start = 0): NibblesSeq =
+  ## Re-build the key path
+  for n in start ..< path.path.len:
+    let it = path.path[n]
+    case it.node.kind:
+    of Branch:
+      result = result & @[it.nibble.byte].initNibbleRange.slice(1)
+    of Extension:
+      result = result & it.node.ePfx
+    of Leaf:
+      result = result & it.node.lPfx
+  result = result & path.tail
+
+proc getLeafData(path: XPath): Blob =
+  ## Return the leaf data from a successful `XPath` computation (if any.)
+  ## Note that this function also exists as `hexary_paths.leafData()` but
+  ## the import of this file is avoided.
+  if path.tail.len == 0 and 0 < path.path.len:
+    let node = path.path[^1].node
+    case node.kind:
+    of Branch:
+      return node.bLink[16]
+    of Leaf:
+      return node.lData
+    of Extension:
+      discard
+
+proc toBranchNode(
+    rlp: Rlp
+      ): XNodeObj
+      {.gcsafe, raises: [RlpError].} =
+  var rlp = rlp
+  XNodeObj(kind: Branch, bLink: rlp.read(array[17,Blob]))
+
+proc toLeafNode(
+    rlp: Rlp;
+    pSegm: NibblesSeq
+      ): XNodeObj
+      {.gcsafe, raises: [RlpError].} =
+  XNodeObj(kind: Leaf, lPfx: pSegm, lData: rlp.listElem(1).toBytes)
+
+proc toExtensionNode(
+    rlp: Rlp;
+    pSegm: NibblesSeq
+      ): XNodeObj
+      {.gcsafe, raises: [RlpError].} =
+  XNodeObj(kind: Extension, ePfx: pSegm, eLink: rlp.listElem(1).toBytes)
+
+
+proc to(node: XNodeObj; T: type RNodeRef): T =
+  case node.kind:
+  of Leaf:
+    result = T(
+      kind: Leaf,
+      lData: node.lData,
+      lPfx: node.lPfx)
+  of Extension:
+    result = T(
+      kind: Extension,
+      eLink: node.eLink.convertTo(RepairKey),
+      ePfx: node.ePfx)
+  of Branch:
+    result = T(
+      kind: Branch,
+      bData: node.bLink[16])
+    for n in 0 .. 15:
+      result.bLink[n] = node.bLink[n].convertTo(RepairKey)
+
+# ------------------------------------------------------------------------------
+# Private functions
+# ------------------------------------------------------------------------------
+
+proc pathLeast(
+    path: XPath;
+    key: Blob;
+    getFn: HexaryGetFn;
+      ): XPath
+      {.gcsafe, raises: [CatchableError].} =
+  ## For the partial path given, extend by branch nodes with least node
+  ## indices.
+  result = path
+  result.tail = EmptyNibbleRange
+  result.depth = result.getNibblesImpl.len
+
+  var
+    key = key
+    value = key.getFn()
+  if value.len == 0:
+    return
+
+  while true:
+    block loopContinue:
+      let nodeRlp = rlpFromBytes value
+      case nodeRlp.listLen:
+      of 2:
+        let (isLeaf,pathSegment) = hexPrefixDecode nodeRlp.listElem(0).toBytes
+
+        # Leaf node
+        if isLeaf:
+          let node = nodeRlp.toLeafNode(pathSegment)
+          result.path.add XPathStep(key: key, node: node, nibble: -1)
+          result.depth += pathSegment.len
+          return # done ok
+
+        let node = nodeRlp.toExtensionNode(pathSegment)
+        if 0 < node.eLink.len:
+          value = node.eLink.getFn()
+          if 0 < value.len:
+            result.path.add XPathStep(key: key, node: node, nibble: -1)
+            result.depth += pathSegment.len
+            key = node.eLink
+            break loopContinue
+      of 17:
+        # Branch node
+        let node = nodeRlp.toBranchNode
+        if node.bLink[16].len != 0 and 64 <= result.depth:
+          result.path.add XPathStep(key: key, node: node, nibble: -1)
+          return # done ok
+
+        for inx in 0 .. 15:
+          let newKey = node.bLink[inx]
+          if 0 < newKey.len:
+            value = newKey.getFn()
+            if 0 < value.len:
+              result.path.add XPathStep(key: key, node: node, nibble: inx.int8)
+              result.depth.inc
+              key = newKey
+              break loopContinue
+      else:
+        discard
+
+      # Recurse (iteratively)
+      while true:
+        block loopRecurse:
+          # Modify last branch node and try again
+          if result.path[^1].node.kind == Branch:
+            for inx in result.path[^1].nibble+1 .. 15:
+              let newKey = result.path[^1].node.bLink[inx]
+              if 0 < newKey.len:
+                value = newKey.getFn()
+                if 0 < value.len:
+                  result.path[^1].nibble = inx.int8
+                  key = newKey
+                  break loopContinue
+          # Failed, step back and try predecessor branch.
+          while path.path.len < result.path.len:
+            case result.path[^1].node.kind:
+            of Branch:
+              result.depth.dec
+              result.path.setLen(result.path.len - 1)
+              break loopRecurse
+            of Extension:
+              result.depth -= result.path[^1].node.ePfx.len
+              result.path.setLen(result.path.len - 1)
+            of Leaf:
+              return # Ooops
+          return # Failed
+      # Notreached
+    # End while
+  # Notreached
+
+
+proc pathMost(
+    path: XPath;
+    key: Blob;
+    getFn: HexaryGetFn;
+      ): XPath
+      {.gcsafe, raises: [CatchableError].} =
+  ## For the partial path given, extend by branch nodes with greatest node
+  ## indices.
+  result = path
+  result.tail = EmptyNibbleRange
+  result.depth = result.getNibblesImpl.len
+
+  var
+    key = key
+    value = key.getFn()
+  if value.len == 0:
+    return
+
+  while true:
+    block loopContinue:
+      let nodeRlp = rlpFromBytes value
+      case nodeRlp.listLen:
+      of 2:
+        let (isLeaf,pathSegment) = hexPrefixDecode nodeRlp.listElem(0).toBytes
+
+        # Leaf node
+        if isLeaf:
+          let node = nodeRlp.toLeafNode(pathSegment)
+          result.path.add XPathStep(key: key, node: node, nibble: -1)
+          result.depth += pathSegment.len
+          return # done ok
+
+        # Extension node
+        let node = nodeRlp.toExtensionNode(pathSegment)
+        if 0 < node.eLink.len:
+          value = node.eLink.getFn()
+          if 0 < value.len:
+            result.path.add XPathStep(key: key, node: node, nibble: -1)
+            result.depth += pathSegment.len
+            key = node.eLink
+            break loopContinue
+      of 17:
+        # Branch node
+        let node = nodeRlp.toBranchNode
+        if node.bLink[16].len != 0 and 64 <= result.depth:
+          result.path.add XPathStep(key: key, node: node, nibble: -1)
+          return # done ok
+
+        for inx in 15.countDown(0):
+          let newKey = node.bLink[inx]
+          if 0 < newKey.len:
+            value = newKey.getFn()
+            if 0 < value.len:
+              result.path.add XPathStep(key: key, node: node, nibble: inx.int8)
+              result.depth.inc
+              key = newKey
+              break loopContinue
+      else:
+        discard
+
+      # Recurse (iteratively)
+      while true:
+        block loopRecurse:
+          # Modify last branch node and try again
+          if result.path[^1].node.kind == Branch:
+            for inx in (result.path[^1].nibble-1).countDown(0):
+              let newKey = result.path[^1].node.bLink[inx]
+              if 0 < newKey.len:
+                value = newKey.getFn()
+                if 0 < value.len:
+                  result.path[^1].nibble = inx.int8
+                  key = newKey
+                  break loopContinue
+          # Failed, step back and try predecessor branch.
+          while path.path.len < result.path.len:
+            case result.path[^1].node.kind:
+            of Branch:
+              result.depth.dec
+              result.path.setLen(result.path.len - 1)
+              break loopRecurse
+            of Extension:
+              result.depth -= result.path[^1].node.ePfx.len
+              result.path.setLen(result.path.len - 1)
+            of Leaf:
+              return # Ooops
+          return # Failed
+      # Notreached
+    # End while
+  # Notreached
+
+# ---------------
+
+proc fillFromLeft(
+    db: HexaryTreeDbRef;       # Target in-memory database
+    rootKey: NodeKey;          # State root for persistent source database
+    getFn: HexaryGetFn;        # Source database abstraction
+    maxLeafs = 5000;           # Error if more than this many leaf nodes
+      ): Result[int,HexaryError]
+      {.gcsafe, raises: [CatchableError].} =
+  ## Import persistent sub-tree into target database
+
+  # Find first least path
+  var
+    here = XPath(root: rootKey).pathLeast(rootkey.to(Blob), getFn)
+    countSteps = 0
+
+  if 0 < here.path.len:
+    while true:
+      countSteps.inc
+
+      # Import records
+      for step in here.path:
+        db.tab[step.key.convertTo(RepairKey)] = step.node.to(RNodeRef)
+
+      # Get next path
+      let topKey = here.path[^1].key
+      here = here.next(getFn)
+
+      # Check for end condition
+      if here.path.len == 0:
+        break
+      if topKey == here.path[^1].key:
+        return err(GarbledNextLeaf) # Ooops
+      if maxLeafs <= countSteps:
+        return err(LeafMaxExceeded)
+
+  ok(countSteps)
+
+proc fillFromRight(
+    db: HexaryTreeDbRef;       # Target in-memory database
+    rootKey: NodeKey;          # State root for persistent source database
+    getFn: HexaryGetFn;        # Source database abstraction
+    maxLeafs = 5000;           # Error if more than this many leaf nodes
+      ): Result[int,HexaryError]
+      {.gcsafe, raises: [CatchableError].} =
+  ## Import persistent sub-tree into target database
+
+  # Find first least path
+  var
+    here = XPath(root: rootKey).pathMost(rootkey.to(Blob), getFn)
+    countSteps = 0
+
+  if 0 < here.path.len:
+    while true:
+      countSteps.inc
+
+      # Import records
+      for step in here.path:
+        db.tab[step.key.convertTo(RepairKey)] = step.node.to(RNodeRef)
+
+      # Get next path
+      let topKey = here.path[^1].key
+      here = here.prev(getFn)
+
+      # Check for end condition
+      if here.path.len == 0:
+        break
+      if topKey == here.path[^1].key:
+        return err(GarbledNextLeaf) # Ooops
+      if maxLeafs <= countSteps:
+        return err(LeafMaxExceeded)
+
+  ok(countSteps)
+
+# ------------------------------------------------------------------------------
+# Public functions, pretty printing
+# ------------------------------------------------------------------------------
+
+proc pp*(s: string; hex = false): string =
+  ## For long strings print `begin..end` only
+  s.ppImpl(hex)
+
+proc pp*(w: NibblesSeq): string =
+  $w
+
+proc pp*(key: RepairKey): string =
+  ## Raw key, for referenced key dump use `key.pp(db)` below
+  key.ByteArray33.toSeq.mapIt(it.toHex(2)).join.tolowerAscii
+
+proc pp*(key: NodeKey): string =
+  ## Raw key, for referenced key dump use `key.pp(db)` below
+  key.ByteArray32.toSeq.mapIt(it.toHex(2)).join.tolowerAscii
+
+proc pp*(key: NodeKey|RepairKey; db: HexaryTreeDbRef): string =
+  key.ppImpl(db)
+
+proc pp*(
+    w: RNodeRef|XNodeObj|RPathStep|XPathStep;
+    db: HexaryTreeDbRef;
+      ): string =
+  w.ppImpl(db)
+
+proc pp*(
+    w: openArray[RPathStep|XPathStep];
+    db:HexaryTreeDbRef;
+    delim: string;
+      ): string =
+  w.toSeq.mapIt(it.ppImpl(db)).join(delim)
+
+proc pp*(
+    w: openArray[RPathStep|XPathStep];
+    db: HexaryTreeDbRef;
+    indent = 4;
+      ): string =
+  w.pp(db, indent.toPfx)
+
+proc pp*(w: RPath|XPath; db: HexaryTreeDbRef; delim: string): string =
+  result = "<" & w.root.pp(db) & ">"
+  if 0 < w.path.len:
+    result &= delim & w.path.pp(db, delim)
+  result &= delim & "(" & $w.tail
+  when typeof(w) is XPath:
+    result &= "," & $w.depth
+  result &= ")"
+
+proc pp*(w: RPath|XPath; db: HexaryTreeDbRef; indent=4): string =
+  w.pp(db, indent.toPfx)
+
+
+proc pp*(db: HexaryTreeDbRef; root: NodeKey; delim: string): string =
+  ## Dump the entries from the a generic accounts trie. These are
+  ## key value pairs for
+  ## ::
+  ##   Branch:    ($1,b(<$2,$3,..,$17>,))
+  ##   Extension: ($18,e(832b5e..06e697,$19))
+  ##   Leaf:      ($20,l(cc9b5d..1c3b4,f84401..f9e5129d[#70]))
+  ##
+  ## where keys are typically represented as `$<id>` or `¶<id>` or `ø`
+  ## depending on whether a key is final (`$<id>`), temporary (`¶<id>`)
+  ## or unset/missing (`ø`).
+  ##
+  ## The node types are indicated by a letter after the first key before
+  ## the round brackets
+  ## ::
+  ##   Branch:    'b', 'þ', or 'B'
+  ##   Extension: 'e', '€', or 'E'
+  ##   Leaf:      'l', 'ł', or 'L'
+  ##
+  ## Here a small letter indicates a `Static` node which was from the
+  ## original `proofs` list, a capital letter indicates a `Mutable` node
+  ## added on the fly which might need some change, and the decorated
+  ## letters stand for `Locked` nodes which are like `Static` ones but
+  ## added later (typically these nodes are update `Mutable` nodes.)
+  ##
+  ## Beware: dumping a large database is not recommended
+  db.ppImpl(root).join(delim)
+
+proc pp*(db: HexaryTreeDbRef; root: NodeKey; indent=4): string =
+  ## Dump the entries from the a generic repair tree.
+  db.pp(root, indent.toPfx)
+
+
+proc pp*(m: Moment): string =
+  ## Prints a moment in time similar to *chronicles* time format.
+  m.asDateTime.format "yyyy-MM-dd HH:mm:ss'.'fff'+00:00'"
+
+# ------------------------------------------------------------------------------
+# Public functions, traversal over partial tree in persistent database
+# ------------------------------------------------------------------------------
+
+proc next*(
+    path: XPath;
+    getFn: HexaryGetFn;
+    minDepth = 64;
+      ): XPath
+      {.gcsafe, raises: [CatchableError].} =
+  ## Advance the argument `path` to the next leaf node (if any.). The
+  ## `minDepth` argument requires the result of `next()` to satisfy
+  ## `minDepth <= next().getNibbles.len`.
+  var pLen = path.path.len
+
+  # Find the last branch in the path, increase link and step down
+  while 0 < pLen:
+
+    # Find branch none
+    pLen.dec
+
+    let it = path.path[pLen]
+    if it.node.kind == Branch and it.nibble < 15:
+
+      # Find the next item to the right in the branch list
+      for inx in (it.nibble + 1) .. 15:
+        let link = it.node.bLink[inx]
+        if link.len != 0:
+          let
+            branch = XPathStep(key: it.key, node: it.node, nibble: inx.int8)
+            walk = path.path[0 ..< pLen] & branch
+            newPath = XPath(root: path.root, path: walk).pathLeast(link, getFn)
+          if minDepth <= newPath.depth and 0 < newPath.getLeafData.len:
+            return newPath
+
+
+proc prev*(
+    path: XPath;
+    getFn: HexaryGetFn;
+    minDepth = 64;
+      ): XPath
+      {.gcsafe, raises: [CatchableError].} =
+  ## Advance the argument `path` to the previous leaf node (if any.) The
+  ## `minDepth` argument requires the result of `next()` to satisfy
+  ## `minDepth <= next().getNibbles.len`.
+  var pLen = path.path.len
+
+  # Find the last branch in the path, decrease link and step down
+  while 0 < pLen:
+
+    # Find branch none
+    pLen.dec
+    let it = path.path[pLen]
+    if it.node.kind == Branch and 0 < it.nibble:
+
+      # Find the next item to the right in the branch list
+      for inx in (it.nibble - 1).countDown(0):
+        let link = it.node.bLink[inx]
+        if link.len != 0:
+          let
+            branch = XPathStep(key: it.key, node: it.node, nibble: inx.int8)
+            walk = path.path[0 ..< pLen] & branch
+            newPath = XPath(root: path.root, path: walk).pathMost(link,getFn)
+          if minDepth <= newPath.depth and 0 < newPath.getLeafData.len:
+            return newPath
+
+
+proc fromPersistent*(
+    db: HexaryTreeDbRef;       # Target in-memory database
+    rootKey: NodeKey;          # State root for persistent source database
+    getFn: HexaryGetFn;        # Source database abstraction
+    maxLeafs = 5000;           # Error if more than this many leaf nodes
+    reverse = false;           # Fill left to right by default
+      ): Result[int,HexaryError]
+      {.gcsafe, raises: [CatchableError].} =
+  ## Import persistent sub-tree into target database
+  if reverse:
+    db.fillFromLeft(rootKey, getFn, maxLeafs)
+  else:
+    db.fillFromRight(rootKey, getFn, maxLeafs)
+
+proc fromPersistent*(
+    rootKey: NodeKey;          # State root for persistent source database
+    getFn: HexaryGetFn;        # Source database abstraction
+    maxLeafs = 5000;           # Error if more than this many leaf nodes
+    reverse = false;           # Fill left to right by default
+      ): Result[HexaryTreeDbRef,HexaryError]
+      {.gcsafe, raises: [CatchableError].} =
+  ## Variant of `fromPersistent()` for an ad-hoc table
+  let
+    db = HexaryTreeDbRef()
+    rc = db.fromPersistent(rootKey, getFn, maxLeafs, reverse)
+  if rc.isErr:
+    return err(rc.error)
+  ok(db)
+
+# ------------------------------------------------------------------------------
+# End
+# ------------------------------------------------------------------------------

nimbus/sync/snap/worker/db/hexary_desc.nim

@@ -8,15 +8,15 @@
 # at your option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
+{.push raises: [].}
+
 import
-  std/[algorithm, hashes, sequtils, sets, strutils, tables],
+  std/[hashes, sequtils, sets, tables],
-  eth/[common, p2p, trie/nibbles],
+  eth/[common, trie/nibbles],
   stint,
   ../../range_desc,
   ./hexary_error
 
-{.push raises: [].}
-
 type
   HexaryPpFn* =
     proc(key: RepairKey): string {.gcsafe, raises: [CatchableError].}
@@ -113,6 +113,7 @@ type
     nibble*: int8                   ## Branch node selector (if any)
 
   RPath* = object
+    root*: RepairKey                ## Root node needed when `path.len == 0`
     path*: seq[RPathStep]
     tail*: NibblesSeq               ## Stands for non completed leaf path
 
@@ -123,6 +124,7 @@ type
     nibble*: int8                   ## Branch node selector (if any)
 
   XPath* = object
+    root*: NodeKey                  ## Root node needed when `path.len == 0`
     path*: seq[XPathStep]
     tail*: NibblesSeq               ## Stands for non completed leaf path
     depth*: int                     ## May indicate path length (typically 64)
@@ -172,14 +174,6 @@ proc isZero*(a: RepairKey): bool {.gcsafe.}
 # Private helpers
 # ------------------------------------------------------------------------------
 
-proc initImpl(key: var RepairKey; data: openArray[byte]): bool =
-  key.reset
-  if 0 < data.len and data.len <= 33:
-    let trg = addr key.ByteArray33[33 - data.len]
-    trg.copyMem(unsafeAddr data[0], data.len)
-    return true
-
-
 proc append(writer: var RlpWriter, node: RNodeRef) =
   ## Mixin for RLP writer
   proc appendOk(writer: var RlpWriter; key: RepairKey): bool =
@@ -225,167 +219,16 @@ proc append(writer: var RlpWriter, node: XNodeObj) =
     writer.append(node.lPfx.hexPrefixEncode(isleaf = true))
     writer.append(node.lData)
 
-# ------------------------------------------------------------------------------
-# Private debugging helpers
-# ------------------------------------------------------------------------------
-
-proc to*(key: NodeKey; T: type RepairKey): T {.gcsafe.}
-
-proc toPfx(indent: int): string =
-  "\n" & " ".repeat(indent)
-
-proc ppImpl(s: string; hex = false): string =
-  ## For long strings print `begin..end` only
-  if hex:
-    let n = (s.len + 1) div 2
-    (if s.len < 20: s else: s[0 .. 5] & ".." & s[s.len-8 .. s.len-1]) &
-      "[" & (if 0 < n: "#" & $n else: "") & "]"
-  elif s.len <= 30:
-    s
-  else:
-    (if (s.len and 1) == 0: s[0 ..< 8] else: "0" & s[0 ..< 7]) &
-      "..(" & $s.len & ").." & s[s.len-16 ..< s.len]
-
-proc ppImpl(key: RepairKey; db: HexaryTreeDbRef): string =
-  if key.isZero:
-    return "ø"
-  if not key.isNodekey:
-    var num: uint64
-    (addr num).copyMem(unsafeAddr key.ByteArray33[25], 8)
-    return "%" & $num
-  try:
-    if not disablePrettyKeys and not db.keyPp.isNil:
-      return db.keyPp(key)
-  except CatchableError:
-    discard
-  key.ByteArray33.toSeq.mapIt(it.toHex(2)).join.toLowerAscii
-
-proc ppImpl(key: NodeKey; db: HexaryTreeDbRef): string =
-  key.to(RepairKey).ppImpl(db)
-
-proc ppImpl(w: openArray[RepairKey]; db: HexaryTreeDbRef): string =
-  w.mapIt(it.ppImpl(db)).join(",")
-
-proc ppImpl(w: openArray[Blob]; db: HexaryTreeDbRef): string =
-  var q: seq[RepairKey]
-  for a in w:
-    var key: RepairKey
-    discard key.initImpl(a)
-    q.add key
-  q.ppImpl(db)
-
-proc ppStr(blob: Blob): string =
-  if blob.len == 0: ""
-  else: blob.mapIt(it.toHex(2)).join.toLowerAscii.ppImpl(hex = true)
-
-proc ppImpl(n: RNodeRef; db: HexaryTreeDbRef): string =
-  let so = n.state.ord
-  case n.kind:
-  of Leaf:
-    ["l","ł","L","R"][so] & "(" & $n.lPfx & "," & n.lData.ppStr & ")"
-  of Extension:
-    ["e","€","E","R"][so] & "(" & $n.ePfx & "," & n.eLink.ppImpl(db) & ")"
-  of Branch:
-    ["b","þ","B","R"][so] & "(" & n.bLink.ppImpl(db) & "," & n.bData.ppStr & ")"
-
-proc ppImpl(n: XNodeObj; db: HexaryTreeDbRef): string =
-  case n.kind:
-  of Leaf:
-    "l(" & $n.lPfx & "," & n.lData.ppStr & ")"
-  of Extension:
-    var key: RepairKey
-    discard key.initImpl(n.eLink)
-    "e(" & $n.ePfx & "," & key.ppImpl(db) & ")"
-  of Branch:
-    "b(" & n.bLink[0..15].ppImpl(db) & "," &  n.bLink[16].ppStr & ")"
-
-proc ppImpl(w: RPathStep; db: HexaryTreeDbRef): string =
-  let
-    nibble = if 0 <= w.nibble: w.nibble.toHex(1).toLowerAscii else: "ø"
-    key = w.key.ppImpl(db)
-  "(" & key & "," & nibble & "," & w.node.ppImpl(db) & ")"
-
-proc ppImpl(w: XPathStep; db: HexaryTreeDbRef): string =
-  let nibble = if 0 <= w.nibble: w.nibble.toHex(1).toLowerAscii else: "ø"
-  var key: RepairKey
-  discard key.initImpl(w.key)
-  "(" & key.ppImpl(db) & "," & $nibble & "," & w.node.ppImpl(db) & ")"
-
-proc ppImpl(db: HexaryTreeDbRef; root: NodeKey): seq[string] =
-  ## Dump the entries from the a generic repair tree. This function assumes
-  ## that mapped keys are printed `$###` if a node is locked or static, and
-  ## some substitute for the first letter `$` otherwise (if they are mutable.)
-  proc toKey(s: string): uint64 =
-    try:
-      result = s[1 ..< s.len].parseUint
-    except ValueError as e:
-      raiseAssert "Ooops ppImpl(s=" & s & "): name=" & $e.name & " msg=" & e.msg
-    if s[0] != '$':
-      result = result or (1u64 shl 63)
-  proc cmpIt(x, y: (uint64,string)): int =
-    cmp(x[0],y[0])
-
-  var accu: seq[(uint64,string)]
-  if root.ByteArray32 != ByteArray32.default:
-    accu.add @[(0u64, "($0" & "," & root.ppImpl(db) & ")")]
-  for key,node in db.tab.pairs:
-    accu.add (
-      key.ppImpl(db).tokey,
-      "(" & key.ppImpl(db) & "," & node.ppImpl(db) & ")")
-
-  accu.sorted(cmpIt).mapIt(it[1])
-
-# ------------------------------------------------------------------------------
-# Public debugging helpers
-# ------------------------------------------------------------------------------
-
-proc pp*(s: string; hex = false): string =
-  ## For long strings print `begin..end` only
-  s.ppImpl(hex)
-
-proc pp*(w: NibblesSeq): string =
-  $w
-
-proc pp*(key: RepairKey): string =
-  ## Raw key, for referenced key dump use `key.pp(db)` below
-  key.ByteArray33.toSeq.mapIt(it.toHex(2)).join.tolowerAscii
-
-proc pp*(key: NodeKey): string =
-  ## Raw key, for referenced key dump use `key.pp(db)` below
-  key.ByteArray32.toSeq.mapIt(it.toHex(2)).join.tolowerAscii
-
-proc pp*(key: NodeKey|RepairKey; db: HexaryTreeDbRef): string =
-  key.ppImpl(db)
-
-proc pp*(
-    w: RNodeRef|XNodeObj|RPathStep|XPathStep;
-    db: HexaryTreeDbRef;
-      ): string =
-  w.ppImpl(db)
-
-proc pp*(w:openArray[RPathStep|XPathStep];db:HexaryTreeDbRef;indent=4): string =
-  w.toSeq.mapIt(it.ppImpl(db)).join(indent.toPfx)
-
-proc pp*(w: RPath; db: HexaryTreeDbRef; indent=4): string =
-  w.path.pp(db,indent) & indent.toPfx & "(" & $w.tail & ")"
-
-proc pp*(w: XPath; db: HexaryTreeDbRef; indent=4): string =
-  w.path.pp(db,indent) & indent.toPfx & "(" & $w.tail & "," & $w.depth & ")"
-
-proc pp*(db: HexaryTreeDbRef; root: NodeKey; indent=4): string =
-  ## Dump the entries from the a generic repair tree.
-  db.ppImpl(root).join(indent.toPfx)
-
-proc pp*(db: HexaryTreeDbRef; indent=4): string =
-  ## varinat of `pp()` above
-  db.ppImpl(NodeKey.default).join(indent.toPfx)
-
 # ------------------------------------------------------------------------------
 # Public constructor (or similar)
 # ------------------------------------------------------------------------------
 
 proc init*(key: var RepairKey; data: openArray[byte]): bool =
-  key.initImpl(data)
+  key.reset
+  if 0 < data.len and data.len <= 33:
+    let trg = addr key.ByteArray33[33 - data.len]
+    trg.copyMem(unsafeAddr data[0], data.len)
+    return true
 
 proc newRepairKey*(db: HexaryTreeDbRef): RepairKey =
   db.repairKeyGen.inc
@@ -434,7 +277,7 @@ proc convertTo*(data: Blob; T: type NodeTag): T =
 
 proc convertTo*(data: Blob; T: type RepairKey): T =
   ## Probably lossy conversion, use `init()` for safe conversion
-  discard result.initImpl(data)
+  discard result.init(data)
 
 proc convertTo*(node: RNodeRef; T: type Blob): T =
   ## Write the node as an RLP-encoded blob
@@ -455,6 +298,26 @@ proc convertTo*(nodeList: openArray[XNodeObj]; T: type Blob): T =
     writer.append w
   writer.finish
 
+proc padPartialPath*(pfx: NibblesSeq; dblNibble: byte): NodeKey =
+  ## Extend (or cut) `partialPath` nibbles sequence and generate `NodeKey`.
+  ## This function must be handled with some care regarding a meaningful value
+  ## for the `dblNibble` argument. Using values `0` or `255` is typically used
+  ## to create the minimum or maximum envelope value from the `pfx` argument.
+  # Pad with zeroes
+  var padded: NibblesSeq
+
+  let padLen = 64 - pfx.len
+  if 0 <= padLen:
+    padded = pfx & dblNibble.repeat(padlen div 2).initNibbleRange
+    if (padLen and 1) == 1:
+      padded = padded & @[dblNibble].initNibbleRange.slice(1)
+  else:
+    let nope = seq[byte].default.initNibbleRange
+    padded = pfx.slice(0,64) & nope # nope forces re-alignment
+
+  let bytes = padded.getBytes
+  (addr result.ByteArray32[0]).copyMem(unsafeAddr bytes[0], bytes.len)
+
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/sync/snap/worker/db/hexary_envelope.nim

@@ -70,6 +70,9 @@
 ## * then there is a ``w = partialPath & w-ext`` in ``W`` with
 ##   ``p-ext = w-ext & some-ext``.
 ##
+
+{.push raises: [].}
+
 import
   std/[algorithm, sequtils, tables],
   eth/[common, trie/nibbles],
@@ -77,8 +80,6 @@ import
   ../../range_desc,
   "."/[hexary_desc, hexary_error, hexary_nearby, hexary_paths]
 
-{.push raises: [].}
-
 # ------------------------------------------------------------------------------
 # Private helpers
 # ------------------------------------------------------------------------------
@@ -136,24 +137,6 @@ template noRlpErrorOops(info: static[string]; code: untyped) =
 # Private functions
 # ------------------------------------------------------------------------------
 
-proc padPartialPath(pfx: NibblesSeq; dblNibble: byte): NodeKey =
-  ## Extend (or cut) `partialPath` nibbles sequence and generate `NodeKey`
-  # Pad with zeroes
-  var padded: NibblesSeq
-
-  let padLen = 64 - pfx.len
-  if 0 <= padLen:
-    padded = pfx & dblNibble.repeat(padlen div 2).initNibbleRange
-    if (padLen and 1) == 1:
-      padded = padded & @[dblNibble].initNibbleRange.slice(1)
-  else:
-    let nope = seq[byte].default.initNibbleRange
-    padded = pfx.slice(0,64) & nope # nope forces re-alignment
-
-  let bytes = padded.getBytes
-  (addr result.ByteArray32[0]).copyMem(unsafeAddr bytes[0], bytes.len)
-
-
 proc doDecomposeLeft(
     envQ: RPath|XPath;
     ivQ: RPath|XPath;

nimbus/sync/snap/worker/db/hexary_error.nim

@@ -28,6 +28,13 @@ type
     TooManySlotAccounts
     NoAccountsYet
 
+    # debug
+    LeafMaxExceeded
+    GarbledNextLeaf
+
+    # snap handler
+    DataSizeError
+
     # range
     LeafNodeExpected
     FailedNextNode
@@ -42,6 +49,7 @@ type
     NearbyEmptyPath
     NearbyLeafExpected
     NearbyDanglingLink
+    NearbyPathTail
 
     # envelope
     DecomposeDegenerated

nimbus/sync/snap/worker/db/hexary_interpolate.nim

@@ -14,6 +14,8 @@
 ## purposes, it should be replaced by the new facility of the upcoming
 ## re-factored database layer.
 
+{.push raises: [].}
+
 import
   std/[tables],
   eth/[common, trie/nibbles],
@@ -21,8 +23,6 @@ import
   ../../range_desc,
   "."/[hexary_desc, hexary_error, hexary_paths]
 
-{.push raises: [].}
-
 type
   RPathXStep = object
     ## Extended `RPathStep` needed for `NodeKey` assignmant
@@ -114,6 +114,7 @@ proc rTreeExtendLeaf(
     if not key.isNodeKey:
       rPath.path[^1].node.bLink[nibble] = key
     return RPath(
+      root: rPath.root,
       path: rPath.path & RPathStep(key: key, node: leaf, nibble: -1),
       tail: EmptyNibbleRange)
 
@@ -129,7 +130,10 @@ proc rTreeExtendLeaf(
     let
       nibble = rPath.tail[0].int8
       xStep = RPathStep(key: key, node: node, nibble: nibble)
-      xPath = RPath(path: rPath.path & xStep, tail: rPath.tail.slice(1))
+      xPath = RPath(
+        root: rPath.root,
+        path: rPath.path & xStep,
+        tail: rPath.tail.slice(1))
     return db.rTreeExtendLeaf(xPath, db.newRepairKey())
 
 

nimbus/sync/snap/worker/db/hexary_paths.nim

@@ -89,16 +89,15 @@ proc toExtensionNode(
 # Private functions
 # ------------------------------------------------------------------------------
 
-proc pathExtend(
+proc rootPathExtend(
     path: RPath;
-    key: RepairKey;
     db: HexaryTreeDbRef;
       ): RPath
       {.gcsafe, raises: [KeyError].} =
   ## For the given path, extend to the longest possible repair tree `db`
   ## path following the argument `path.tail`.
   result = path
-  var key = key
+  var key = path.root
   while db.tab.hasKey(key):
     let node = db.tab[key]
 
@@ -127,15 +126,14 @@ proc pathExtend(
       key = node.eLink
 
 
-proc pathExtend(
+proc rootPathExtend(
     path: XPath;
-    key: Blob;
     getFn: HexaryGetFn;
       ): XPath
       {.gcsafe, raises: [CatchableError]} =
   ## Ditto for `XPath` rather than `RPath`
   result = path
-  var key = key
+  var key = path.root.to(Blob)
   while true:
     let value = key.getFn()
     if value.len == 0:
@@ -187,187 +185,6 @@ proc pathExtend(
     # end while
   # notreached
 
-
-proc pathLeast(
-    path: XPath;
-    key: Blob;
-    getFn: HexaryGetFn;
-      ): XPath
-      {.gcsafe, raises: [CatchableError]} =
-  ## For the partial path given, extend by branch nodes with least node
-  ## indices.
-  result = path
-  result.tail = EmptyNibbleRange
-  result.depth = result.getNibblesImpl.len
-
-  var
-    key = key
-    value = key.getFn()
-  if value.len == 0:
-    return
-
-  while true:
-    block loopContinue:
-      let nodeRlp = rlpFromBytes value
-      case nodeRlp.listLen:
-      of 2:
-        let (isLeaf,pathSegment) = hexPrefixDecode nodeRlp.listElem(0).toBytes
-
-        # Leaf node
-        if isLeaf:
-          let node = nodeRlp.toLeafNode(pathSegment)
-          result.path.add XPathStep(key: key, node: node, nibble: -1)
-          result.depth += pathSegment.len
-          return # done ok
-
-        let node = nodeRlp.toExtensionNode(pathSegment)
-        if 0 < node.eLink.len:
-          value = node.eLink.getFn()
-          if 0 < value.len:
-            result.path.add XPathStep(key: key, node: node, nibble: -1)
-            result.depth += pathSegment.len
-            key = node.eLink
-            break loopContinue
-      of 17:
-        # Branch node
-        let node = nodeRlp.toBranchNode
-        if node.bLink[16].len != 0 and 64 <= result.depth:
-          result.path.add XPathStep(key: key, node: node, nibble: -1)
-          return # done ok
-
-        for inx in 0 .. 15:
-          let newKey = node.bLink[inx]
-          if 0 < newKey.len:
-            value = newKey.getFn()
-            if 0 < value.len:
-              result.path.add XPathStep(key: key, node: node, nibble: inx.int8)
-              result.depth.inc
-              key = newKey
-              break loopContinue
-      else:
-        discard
-
-      # Recurse (iteratively)
-      while true:
-        block loopRecurse:
-          # Modify last branch node and try again
-          if result.path[^1].node.kind == Branch:
-            for inx in result.path[^1].nibble+1 .. 15:
-              let newKey = result.path[^1].node.bLink[inx]
-              if 0 < newKey.len:
-                value = newKey.getFn()
-                if 0 < value.len:
-                  result.path[^1].nibble = inx.int8
-                  key = newKey
-                  break loopContinue
-          # Failed, step back and try predecessor branch.
-          while path.path.len < result.path.len:
-            case result.path[^1].node.kind:
-            of Branch:
-              result.depth.dec
-              result.path.setLen(result.path.len - 1)
-              break loopRecurse
-            of Extension:
-              result.depth -= result.path[^1].node.ePfx.len
-              result.path.setLen(result.path.len - 1)
-            of Leaf:
-              return # Ooops
-          return # Failed
-      # Notreached
-    # End while
-  # Notreached
-
-
-proc pathMost(
-    path: XPath;
-    key: Blob;
-    getFn: HexaryGetFn;
-      ): XPath
-      {.gcsafe, raises: [CatchableError]} =
-  ## For the partial path given, extend by branch nodes with greatest node
-  ## indices.
-  result = path
-  result.tail = EmptyNibbleRange
-  result.depth = result.getNibblesImpl.len
-
-  var
-    key = key
-    value = key.getFn()
-  if value.len == 0:
-    return
-
-  while true:
-    block loopContinue:
-      let nodeRlp = rlpFromBytes value
-      case nodeRlp.listLen:
-      of 2:
-        let (isLeaf,pathSegment) = hexPrefixDecode nodeRlp.listElem(0).toBytes
-
-        # Leaf node
-        if isLeaf:
-          let node = nodeRlp.toLeafNode(pathSegment)
-          result.path.add XPathStep(key: key, node: node, nibble: -1)
-          result.depth += pathSegment.len
-          return # done ok
-
-        # Extension node
-        let node = nodeRlp.toExtensionNode(pathSegment)
-        if 0 < node.eLink.len:
-          value = node.eLink.getFn()
-          if 0 < value.len:
-            result.path.add XPathStep(key: key, node: node, nibble: -1)
-            result.depth += pathSegment.len
-            key = node.eLink
-            break loopContinue
-      of 17:
-        # Branch node
-        let node = nodeRlp.toBranchNode
-        if node.bLink[16].len != 0 and 64 <= result.depth:
-          result.path.add XPathStep(key: key, node: node, nibble: -1)
-          return # done ok
-
-        for inx in 15.countDown(0):
-          let newKey = node.bLink[inx]
-          if 0 < newKey.len:
-            value = newKey.getFn()
-            if 0 < value.len:
-              result.path.add XPathStep(key: key, node: node, nibble: inx.int8)
-              result.depth.inc
-              key = newKey
-              break loopContinue
-      else:
-        discard
-
-      # Recurse (iteratively)
-      while true:
-        block loopRecurse:
-          # Modify last branch node and try again
-          if result.path[^1].node.kind == Branch:
-            for inx in (result.path[^1].nibble-1).countDown(0):
-              let newKey = result.path[^1].node.bLink[inx]
-              if 0 < newKey.len:
-                value = newKey.getFn()
-                if 0 < value.len:
-                  result.path[^1].nibble = inx.int8
-                  key = newKey
-                  break loopContinue
-          # Failed, step back and try predecessor branch.
-          while path.path.len < result.path.len:
-            case result.path[^1].node.kind:
-            of Branch:
-              result.depth.dec
-              result.path.setLen(result.path.len - 1)
-              break loopRecurse
-            of Extension:
-              result.depth -= result.path[^1].node.ePfx.len
-              result.path.setLen(result.path.len - 1)
-            of Leaf:
-              return # Ooops
-          return # Failed
-      # Notreached
-    # End while
-  # Notreached
-
 # ------------------------------------------------------------------------------
 # Public helpers
 # ------------------------------------------------------------------------------
@@ -432,7 +249,7 @@ proc hexaryPath*(
   ## Compute the longest possible repair tree `db` path matching the `nodeKey`
   ## nibbles. The `nodeNey` path argument comes before the `db` one for
   ## supporting a more functional notation.
-  RPath(tail: partialPath).pathExtend(rootKey.to(RepairKey), db)
+  RPath(root: rootKey.to(RepairKey), tail: partialPath).rootPathExtend(db)
 
 proc hexaryPath*(
     nodeKey: NodeKey;
@@ -469,7 +286,7 @@ proc hexaryPath*(
       ): XPath
       {.gcsafe, raises: [CatchableError]} =
   ## Compute the longest possible path on an arbitrary hexary trie.
-  XPath(tail: partialPath).pathExtend(rootKey.to(Blob), getFn)
+  XPath(root: rootKey, tail: partialPath).rootPathExtend(getFn)
 
 proc hexaryPath*(
     nodeKey: NodeKey;
@@ -583,71 +400,6 @@ proc hexaryPathNodeKeys*(
     .mapIt(it.value)
     .toHashSet
 
-# ------------------------------------------------------------------------------
-# Public functions, traversal
-# ------------------------------------------------------------------------------
-
-proc next*(
-    path: XPath;
-    getFn: HexaryGetFn;
-    minDepth = 64;
-      ): XPath
-      {.gcsafe, raises: [CatchableError]} =
-  ## Advance the argument `path` to the next leaf node (if any.). The
-  ## `minDepth` argument requires the result of `next()` to satisfy
-  ## `minDepth <= next().getNibbles.len`.
-  var pLen = path.path.len
-
-  # Find the last branch in the path, increase link and step down
-  while 0 < pLen:
-
-    # Find branch none
-    pLen.dec
-
-    let it = path.path[pLen]
-    if it.node.kind == Branch and it.nibble < 15:
-
-      # Find the next item to the right in the branch list
-      for inx in (it.nibble + 1) .. 15:
-        let link = it.node.bLink[inx]
-        if link.len != 0:
-          let
-            branch = XPathStep(key: it.key, node: it.node, nibble: inx.int8)
-            walk = path.path[0 ..< pLen] & branch
-            newPath = XPath(path: walk).pathLeast(link, getFn)
-          if minDepth <= newPath.depth and 0 < newPath.leafData.len:
-            return newPath
-
-proc prev*(
-    path: XPath;
-    getFn: HexaryGetFn;
-    minDepth = 64;
-      ): XPath
-      {.gcsafe, raises: [CatchableError]} =
-  ## Advance the argument `path` to the previous leaf node (if any.) The
-  ## `minDepth` argument requires the result of `next()` to satisfy
-  ## `minDepth <= next().getNibbles.len`.
-  var pLen = path.path.len
-
-  # Find the last branch in the path, decrease link and step down
-  while 0 < pLen:
-
-    # Find branch none
-    pLen.dec
-    let it = path.path[pLen]
-    if it.node.kind == Branch and 0 < it.nibble:
-
-      # Find the next item to the right in the branch list
-      for inx in (it.nibble - 1).countDown(0):
-        let link = it.node.bLink[inx]
-        if link.len != 0:
-          let
-            branch = XPathStep(key: it.key, node: it.node, nibble: inx.int8)
-            walk = path.path[0 ..< pLen] & branch
-            newPath = XPath(path: walk).pathMost(link, getFn)
-          if minDepth <= newPath.depth and 0 < newPath.leafData.len:
-            return newPath
-
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/sync/snap/worker/db/hexary_range.nim

@@ -12,6 +12,7 @@
 
 import
   std/[sequtils, sets, tables],
+  chronicles,
   chronos,
   eth/[common, p2p, trie/nibbles],
   stew/[byteutils, interval_set],
@@ -109,8 +110,9 @@ template collectLeafs(
       ): auto =
   ## Collect trie database leafs prototype. This directive is provided as
   ## `template` for avoiding varying exceprion annotations.
-  var rc: Result[RangeProof,HexaryError]
+  var
-
+    rc: Result[RangeProof,HexaryError]
+    ttd = stopAt
   block body:
     let
       nodeMax = maxPt(iv) # `inject` is for debugging (if any)
@@ -121,9 +123,9 @@ template collectLeafs(
 
     # Set up base node, the nearest node before `iv.minPt`
     if 0.to(NodeTag) < nodeTag:
-      let rx = nodeTag.hexaryPath(rootKey,db).hexaryNearbyLeft(db)
+      let rx = nodeTag.hexaryNearbyLeft(rootKey, db)
       if rx.isOk:
-        rls.base = getPartialPath(rx.value).convertTo(NodeKey).to(NodeTag)
+        rls.base = rx.value
       elif rx.error notin {NearbyFailed,NearbyEmptyPath}:
         rc = typeof(rc).err(rx.error)
         break body
@@ -149,7 +151,7 @@ template collectLeafs(
 
       # Prevents from semi-endless looping
       if rightTag <= prevTag and 0 < rls.leafs.len:
-        # Oops, should have been tackeled by `hexaryNearbyRight()`
+        # Oops, should have been tackled by `hexaryNearbyRight()`
         rc = typeof(rc).err(FailedNextNode)
         break body # stop here
 
@@ -165,7 +167,7 @@ template collectLeafs(
         key:  rightKey,
         data: xPath.leafData)
 
-      if timeIsOver(stopAt):
+      if timeIsOver(ttd):
         break # timout
 
       prevTag = nodeTag

nimbus/sync/snap/worker/db/snapdb_accounts.nim

@@ -11,7 +11,7 @@
 {.push raises: [].}
 
 import
-  std/[algorithm, sequtils, tables],
+  std/tables,
   chronicles,
   eth/[common, p2p, rlp, trie/nibbles],
   stew/[byteutils, interval_set],
@@ -43,15 +43,6 @@ proc getAccountFn*(ps: SnapDbAccountsRef): HexaryGetFn
 proc to(h: Hash256; T: type NodeKey): T =
   h.data.T
 
-proc convertTo(data: openArray[byte]; T: type Hash256): T =
-  discard result.data.NodeKey.init(data) # size error => zero
-
-template noKeyError(info: static[string]; code: untyped) =
-  try:
-    code
-  except KeyError as e:
-    raiseAssert "Not possible (" & info & "): " & e.msg
-
 template noExceptionOops(info: static[string]; code: untyped) =
   try:
     code
@@ -449,71 +440,6 @@ proc getAccountsData*(
   SnapDbAccountsRef.init(
     pv, root, Peer()).getAccountsData(path, persistent=true)
 
-# ------------------------------------------------------------------------------
-# Public functions: additional helpers
-# ------------------------------------------------------------------------------
-
-proc sortMerge*(base: openArray[NodeTag]): NodeTag =
-  ## Helper for merging several `(NodeTag,seq[PackedAccount])` data sets
-  ## so that there are no overlap which would be rejected by `merge()`.
-  ##
-  ## This function selects a `NodeTag` from a list.
-  result = high(NodeTag)
-  for w in base:
-    if w < result:
-      result = w
-
-proc sortMerge*(acc: openArray[seq[PackedAccount]]): seq[PackedAccount] =
-  ## Helper for merging several `(NodeTag,seq[PackedAccount])` data sets
-  ## so that there are no overlap which would be rejected by `merge()`.
-  ##
-  ## This function flattens and sorts the argument account lists.
-  noKeyError("sortMergeAccounts"):
-    var accounts: Table[NodeTag,PackedAccount]
-    for accList in acc:
-      for item in accList:
-        accounts[item.accKey.to(NodeTag)] = item
-    result = toSeq(accounts.keys).sorted(cmp).mapIt(accounts[it])
-
-proc getAccountsChainDb*(
-    ps: SnapDbAccountsRef;
-    accKey: NodeKey;
-      ): Result[Account,HexaryError] =
-  ## Fetch account via `ChainDBRef`
-  ps.getAccountsData(accKey, persistent = true)
-
-proc nextAccountsChainDbKey*(
-    ps: SnapDbAccountsRef;
-    accKey: NodeKey;
-      ): Result[NodeKey,HexaryError] =
-  ## Fetch the account path on the `ChainDBRef`, the one next to the
-  ## argument account key.
-  noExceptionOops("getChainDbAccount()"):
-    let path = accKey
-               .hexaryPath(ps.root, ps.getAccountFn)
-               .next(ps.getAccountFn)
-               .getNibbles
-    if 64 == path.len:
-      return ok(path.getBytes.convertTo(Hash256).to(NodeKey))
-
-  err(AccountNotFound)
-
-proc prevAccountsChainDbKey*(
-    ps: SnapDbAccountsRef;
-    accKey: NodeKey;
-      ): Result[NodeKey,HexaryError] =
-  ## Fetch the account path on the `ChainDBRef`, the one before to the
-  ## argument account.
-  noExceptionOops("getChainDbAccount()"):
-    let path = accKey
-               .hexaryPath(ps.root, ps.getAccountFn)
-               .prev(ps.getAccountFn)
-               .getNibbles
-    if 64 == path.len:
-      return ok(path.getBytes.convertTo(Hash256).to(NodeKey))
-
-  err(AccountNotFound)
-
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/sync/snap/worker/db/snapdb_debug.nim

@@ -0,0 +1,180 @@
+# nimbus-eth1
+# Copyright (c) 2021 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+{.push raises: [].}
+
+import
+  std/[algorithm, sequtils, tables],
+  eth/[common, trie/nibbles],
+  stew/results,
+  ../../range_desc,
+  "."/[hexary_debug, hexary_desc, hexary_error, hexary_paths, snapdb_desc]
+
+# ------------------------------------------------------------------------------
+# Private debugging helpers
+# ------------------------------------------------------------------------------
+
+template noPpError(info: static[string]; code: untyped) =
+  try:
+    code
+  except ValueError as e:
+    raiseAssert "Inconveivable (" & info & "): " & e.msg
+  except KeyError as e:
+    raiseAssert "Not possible (" & info & "): " & e.msg
+  except CatchableError as e:
+    raiseAssert "Ooops (" & info & ") " & $e.name & ": " & e.msg
+
+# ------------------------------------------------------------------------------
+# Private helpers
+# ------------------------------------------------------------------------------
+
+proc convertTo(data: openArray[byte]; T: type Hash256): T =
+  discard result.data.NodeKey.init(data) # size error => zero
+
+template noKeyError(info: static[string]; code: untyped) =
+  try:
+    code
+  except KeyError as e:
+    raiseAssert "Not possible (" & info & "): " & e.msg
+
+template noExceptionOops(info: static[string]; code: untyped) =
+  try:
+    code
+  except KeyError as e:
+    raiseAssert "Not possible -- " & info & ": " & e.msg
+  except RlpError:
+    return err(RlpEncoding)
+  except CatchableError as e:
+    return err(AccountNotFound)
+
+# ------------------------------------------------------------------------------
+# Public functions, pretty printing
+# ------------------------------------------------------------------------------
+
+proc pp*(a: RepairKey; ps: SnapDbBaseRef): string =
+  if not ps.isNil:
+    let toKey = ps.hexaDb.keyPp
+    if not toKey.isNil:
+      try:
+        return a.toKey
+      except CatchableError:
+        discard
+  $a.ByteArray33
+
+proc pp*(a: NodeKey; ps: SnapDbBaseRef): string =
+  if not ps.isNil:
+    let toKey = ps.hexaDb.keyPp
+    if not toKey.isNil:
+      try:
+        return a.to(RepairKey).toKey
+      except CatchableError:
+        discard
+  $a.ByteArray32
+
+proc pp*(a: NodeTag; ps: SnapDbBaseRef): string =
+  a.to(NodeKey).pp(ps)
+
+# ------------------------------------------------------------------------------
+# Public constructor
+# ------------------------------------------------------------------------------
+
+proc init*(
+    T: type HexaryTreeDbRef;
+      ): T =
+  ## Constructor variant. It provides a `HexaryTreeDbRef()` with a key cache
+  ## attached for pretty printing. So this one is mainly for debugging.
+  HexaryTreeDbRef.init(SnapDbRef())
+
+# ------------------------------------------------------------------------------
+# Public functions
+# ------------------------------------------------------------------------------
+
+proc sortMerge*(base: openArray[NodeTag]): NodeTag =
+  ## Helper for merging several `(NodeTag,seq[PackedAccount])` data sets
+  ## so that there are no overlap which would be rejected by `merge()`.
+  ##
+  ## This function selects a `NodeTag` from a list.
+  result = high(NodeTag)
+  for w in base:
+    if w < result:
+      result = w
+
+proc sortMerge*(acc: openArray[seq[PackedAccount]]): seq[PackedAccount] =
+  ## Helper for merging several `(NodeTag,seq[PackedAccount])` data sets
+  ## so that there are no overlap which would be rejected by `merge()`.
+  ##
+  ## This function flattens and sorts the argument account lists.
+  noKeyError("sortMergeAccounts"):
+    var accounts: Table[NodeTag,PackedAccount]
+    for accList in acc:
+      for item in accList:
+        accounts[item.accKey.to(NodeTag)] = item
+    result = toSeq(accounts.keys).sorted(cmp).mapIt(accounts[it])
+
+proc nextAccountsChainDbKey*(
+    ps: SnapDbBaseRef;
+    accKey: NodeKey;
+    getFn: HexaryGetFn;
+      ): Result[NodeKey,HexaryError] =
+  ## Fetch the account path on the `ChainDBRef`, the one next to the
+  ## argument account key.
+  noExceptionOops("getChainDbAccount()"):
+    let path = accKey
+               .hexaryPath(ps.root, getFn) # ps.getAccountFn)
+               .next(getFn)                # ps.getAccountFn)
+               .getNibbles
+    if 64 == path.len:
+      return ok(path.getBytes.convertTo(Hash256).to(NodeKey))
+
+  err(AccountNotFound)
+
+proc prevAccountsChainDbKey*(
+    ps: SnapDbBaseRef;
+    accKey: NodeKey;
+    getFn: HexaryGetFn;
+      ): Result[NodeKey,HexaryError] =
+  ## Fetch the account path on the `ChainDBRef`, the one before to the
+  ## argument account.
+  noExceptionOops("getChainDbAccount()"):
+    let path = accKey
+               .hexaryPath(ps.root, getFn) # ps.getAccountFn)
+               .prev(getFn)                # ps.getAccountFn)
+               .getNibbles
+    if 64 == path.len:
+      return ok(path.getBytes.convertTo(Hash256).to(NodeKey))
+
+  err(AccountNotFound)
+
+# ------------------------------------------------------------------------------
+# More debugging (and playing with the hexary database)
+# ------------------------------------------------------------------------------
+
+proc assignPrettyKeys*(xDb: HexaryTreeDbRef; root: NodeKey) =
+  ## Prepare for pretty pringing/debugging. Run early enough this function
+  ## sets the root key to `"$"`, for instance.
+  if not xDb.keyPp.isNil:
+    noPpError("validate(1)"):
+      # Make keys assigned in pretty order for printing
+      let rootKey = root.to(RepairKey)
+      discard xDb.keyPp rootKey
+      var keysList = toSeq(xDb.tab.keys)
+      if xDb.tab.hasKey(rootKey):
+        keysList = @[rootKey] & keysList
+      for key in keysList:
+        let node = xDb.tab[key]
+        discard xDb.keyPp key
+        case node.kind:
+        of Branch: (for w in node.bLink: discard xDb.keyPp w)
+        of Extension: discard xDb.keyPp node.eLink
+        of Leaf: discard
+
+# ------------------------------------------------------------------------------
+# End
+# ------------------------------------------------------------------------------

nimbus/sync/snap/worker/db/snapdb_desc.nim

@@ -11,13 +11,13 @@
 {.push raises: [].}
 
 import
-  std/[sequtils, tables],
+  std/tables,
   chronicles,
   eth/[common, p2p, trie/db, trie/nibbles],
   ../../../../db/[select_backend, storage_types],
   ../../../protocol,
   ../../range_desc,
-  "."/[hexary_desc, hexary_error, hexary_import, hexary_nearby,
+  "."/[hexary_debug, hexary_desc, hexary_error, hexary_import, hexary_nearby,
        hexary_paths, rocky_bulk_load]
 
 logScope:
@@ -46,47 +46,20 @@ type
 # Private debugging helpers
 # ------------------------------------------------------------------------------
 
-template noPpError(info: static[string]; code: untyped) =
+template noKeyError(info: static[string]; code: untyped) =
   try:
     code
-  except ValueError as e:
-    raiseAssert "Inconveivable (" & info & "): " & e.msg
   except KeyError as e:
     raiseAssert "Not possible (" & info & "): " & e.msg
-  except CatchableError as e:
-    raiseAssert "Ooops (" & info & ") " & $e.name & ": " & e.msg
 
-proc toKey(a: RepairKey; pv: SnapDbRef): uint =
+proc keyPp(a: RepairKey; pv: SnapDbRef): string =
-  if not a.isZero:
+  if a.isZero:
-    noPpError("pp(RepairKey)"):
+    return "ø"
-      if not pv.keyMap.hasKey(a):
+  if not pv.keyMap.hasKey(a):
-        pv.keyMap[a] = pv.keyMap.len.uint + 1
+    pv.keyMap[a] = pv.keyMap.len.uint + 1
-      result = pv.keyMap[a]
+  result = if a.isNodeKey: "$" else: "@"
-
+  noKeyError("pp(RepairKey)"):
-proc toKey(a: RepairKey; ps: SnapDbBaseRef): uint =
+    result &= $pv.keyMap[a]
-  a.toKey(ps.base)
-
-proc toKey(a: NodeKey; ps: SnapDbBaseRef): uint =
-  a.to(RepairKey).toKey(ps)
-
-#proc toKey(a: NodeTag; ps: SnapDbBaseRef): uint =
-#  a.to(NodeKey).toKey(ps)
-
-proc ppImpl(a: RepairKey; pv: SnapDbRef): string =
-  "$" & $a.toKey(pv)
-
-# ------------------------------------------------------------------------------
-# Debugging, pretty printing
-# ------------------------------------------------------------------------------
-
-proc pp*(a: NodeKey; ps: SnapDbBaseRef): string =
-  if a.isZero: "ø" else:"$" & $a.toKey(ps)
-
-proc pp*(a: RepairKey; ps: SnapDbBaseRef): string =
-  if a.isZero: "ø" elif a.isNodeKey: "$" & $a.toKey(ps) else: "@" & $a.toKey(ps)
-
-proc pp*(a: NodeTag; ps: SnapDbBaseRef): string =
-  a.to(NodeKey).pp(ps)
 
 # ------------------------------------------------------------------------------
 # Private helper
@@ -127,7 +100,7 @@ proc init*(
       ): T =
   ## Constructor for inner hexary trie database
   let xDb = HexaryTreeDbRef()
-  xDb.keyPp = proc(key: RepairKey): string = key.ppImpl(pv) # will go away
+  xDb.keyPp = proc(key: RepairKey): string = key.keyPp(pv) # will go away
   return xDb
 
 proc init*(
@@ -137,13 +110,6 @@ proc init*(
   ## Constructor variant
   HexaryTreeDbRef.init(ps.base)
 
-proc init*(
-    T: type HexaryTreeDbRef;
-      ): T =
-  ## Constructor variant. It provides a `HexaryTreeDbRef()` with a key key cache attached
-  ## for pretty printing. So this one is mainly for debugging.
-  HexaryTreeDbRef.init(SnapDbRef())
-
 # ---------------
 
 proc init*(
@@ -292,7 +258,10 @@ proc verifyNoMoreRight*(
   let
     root = root.to(RepairKey)
     base = base.to(NodeKey)
-  if base.hexaryPath(root, xDb).hexaryNearbyRightMissing(xDb):
+    rc = base.hexaryPath(root, xDb).hexaryNearbyRightMissing(xDb)
+  if rc.isErr:
+    return err(rc.error)
+  if rc.value:
     return ok()
 
   let error = LowerBoundProofError
@@ -300,64 +269,6 @@ proc verifyNoMoreRight*(
     trace "verifyLeftmostBound()", peer, base=base.pp, error
   err(error)
 
-# ------------------------------------------------------------------------------
-# Debugging (and playing with the hexary database)
-# ------------------------------------------------------------------------------
-
-proc assignPrettyKeys*(xDb: HexaryTreeDbRef; root: NodeKey) =
-  ## Prepare for pretty pringing/debugging. Run early enough this function
-  ## sets the root key to `"$"`, for instance.
-  if not xDb.keyPp.isNil:
-    noPpError("validate(1)"):
-      # Make keys assigned in pretty order for printing
-      let rootKey = root.to(RepairKey)
-      discard xDb.keyPp rootKey
-      var keysList = toSeq(xDb.tab.keys)
-      if xDb.tab.hasKey(rootKey):
-        keysList = @[rootKey] & keysList
-      for key in keysList:
-        let node = xDb.tab[key]
-        discard xDb.keyPp key
-        case node.kind:
-        of Branch: (for w in node.bLink: discard xDb.keyPp w)
-        of Extension: discard xDb.keyPp node.eLink
-        of Leaf: discard
-
-proc dumpPath*(ps: SnapDbBaseRef; key: NodeTag): seq[string] =
-  ## Pretty print helper compiling the path into the repair tree for the
-  ## argument `key`.
-  noPpError("dumpPath"):
-    let rPath= key.hexaryPath(ps.root, ps.hexaDb)
-    result = rPath.path.mapIt(it.pp(ps.hexaDb)) & @["(" & rPath.tail.pp & ")"]
-
-proc dumpHexaDB*(xDb: HexaryTreeDbRef; root: NodeKey; indent = 4): string =
-  ## Dump the entries from the a generic accounts trie. These are
-  ## key value pairs for
-  ## ::
-  ##   Branch:    ($1,b(<$2,$3,..,$17>,))
-  ##   Extension: ($18,e(832b5e..06e697,$19))
-  ##   Leaf:      ($20,l(cc9b5d..1c3b4,f84401..f9e5129d[#70]))
-  ##
-  ## where keys are typically represented as `$<id>` or `¶<id>` or `ø`
-  ## depending on whether a key is final (`$<id>`), temporary (`¶<id>`)
-  ## or unset/missing (`ø`).
-  ##
-  ## The node types are indicated by a letter after the first key before
-  ## the round brackets
-  ## ::
-  ##   Branch:    'b', 'þ', or 'B'
-  ##   Extension: 'e', '€', or 'E'
-  ##   Leaf:      'l', 'ł', or 'L'
-  ##
-  ## Here a small letter indicates a `Static` node which was from the
-  ## original `proofs` list, a capital letter indicates a `Mutable` node
-  ## added on the fly which might need some change, and the decorated
-  ## letters stand for `Locked` nodes which are like `Static` ones but
-  ## added later (typically these nodes are update `Mutable` nodes.)
-  ##
-  ## Beware: dumping a large database is not recommended
-  xDb.pp(root, indent)
-
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/sync/snap/worker_desc.nim

@@ -65,7 +65,6 @@ type
     fetchStorageFull*: SnapSlotsQueue  ## Fetch storage trie for these accounts
     fetchStoragePart*: SnapSlotsQueue  ## Partial storage trie to com[plete
     parkedStorage*: HashSet[NodeKey]   ## Storage batch items in use
-    storageDone*: bool                 ## Done with storage, block sync next
 
     # Info
     nAccounts*: uint64                 ## Imported # of accounts

tests/replay/undump_accounts.nim

@@ -99,8 +99,6 @@ proc dumpAccounts*(
 
 iterator undumpNextAccount*(gzFile: string): UndumpAccounts =
   var
-    line = ""
-    lno = 0
     state = UndumpHeader
     data: UndumpAccounts
     nAccounts = 0u

tests/replay/undump_blocks.nim

@@ -24,9 +24,6 @@ template say(args: varargs[untyped]) =
   # echo args
   discard
 
-proc toByteSeq(s: string): seq[byte] =
-  utils.fromHex(s)
-
 # ------------------------------------------------------------------------------
 # Public capture
 # ------------------------------------------------------------------------------
@@ -90,8 +87,6 @@ iterator undumpNextGroup*(gzFile: string): (seq[BlockHeader],seq[BlockBody]) =
   var
     headerQ: seq[BlockHeader]
     bodyQ: seq[BlockBody]
-    line = ""
-    lno = 0
     current = 0u
     start = 0u
     top = 0u

tests/replay/undump_storages.nim

@@ -104,8 +104,6 @@ proc dumpStorages*(
 
 iterator undumpNextStorages*(gzFile: string): UndumpStorages =
   var
-    line = ""
-    lno = 0
     state = UndumpStoragesHeader
     data: UndumpStorages
     nAccounts = 0u

tests/test_sync_snap.nim

@@ -23,7 +23,7 @@ import
   ../nimbus/sync/snap/range_desc,
   ../nimbus/sync/snap/worker/db/[
     hexary_desc, hexary_envelope, hexary_error, hexary_inspect, hexary_nearby,
-    hexary_paths, rocky_bulk_load, snapdb_accounts, snapdb_desc],
+    hexary_paths, rocky_bulk_load, snapdb_accounts, snapdb_debug, snapdb_desc],
   ./replay/[pp, undump_accounts, undump_storages],
   ./test_sync_snap/[
     bulk_test_xx, snap_test_xx,
@@ -91,12 +91,12 @@ proc findFilePath(file: string;
 proc getTmpDir(sampleDir = sampleDirRefFile): string =
   sampleDir.findFilePath(baseDir,repoDir).value.splitFile.dir
 
-proc setTraceLevel =
+proc setTraceLevel {.used.} =
   discard
   when defined(chronicles_runtime_filtering) and loggingEnabled:
     setLogLevel(LogLevel.TRACE)
 
-proc setErrorLevel =
+proc setErrorLevel {.used.} =
   discard
   when defined(chronicles_runtime_filtering) and loggingEnabled:
     setLogLevel(LogLevel.ERROR)
@@ -144,12 +144,12 @@ proc flushDbDir(s: string; subDir = "") =
       let instDir = if subDir == "": baseDir / $n else: baseDir / subDir / $n
       if (instDir / "nimbus" / "data").dirExists:
         # Typically under Windows: there might be stale file locks.
-        try: instDir.removeDir except: discard
+        try: instDir.removeDir except CatchableError: discard
-    try: (baseDir / subDir).removeDir except: discard
+    try: (baseDir / subDir).removeDir except CatchableError: discard
     block dontClearUnlessEmpty:
       for w in baseDir.walkDir:
         break dontClearUnlessEmpty
-      try: baseDir.removeDir except: discard
+      try: baseDir.removeDir except CatchableError: discard
 
 
 proc flushDbs(db: TestDbs) =
@@ -233,7 +233,7 @@ proc accountsRunner(noisy = true;  persistent = true; sample = accSample) =
       hexaDb.assignPrettyKeys(root.to(NodeKey))
 
       # Beware: dumping a large database is not recommended
-      # true.say "***", "database dump\n    ", hexaDb.dumpHexaDB(root)
+      # true.say "***", "database dump\n    ", hexaDb.pp(root.to(NodeKey))
 
       test &"Retrieve accounts & proofs for previous account ranges":
         if db.persistent:

tests/test_sync_snap/test_accounts.nim

@@ -64,7 +64,9 @@ import
   ../../nimbus/db/select_backend,
   ../../nimbus/sync/protocol,
   ../../nimbus/sync/snap/range_desc,
-  ../../nimbus/sync/snap/worker/db/[snapdb_accounts, snapdb_desc],
+  ../../nimbus/sync/snap/worker/db/[
+    hexary_debug, hexary_desc, hexary_error,
+    snapdb_accounts, snapdb_debug, snapdb_desc],
   ../replay/[pp, undump_accounts],
   ./test_helpers
 
@@ -97,10 +99,13 @@ proc test_accountsMergeProofs*(
       ) =
   ## Merge account proofs
   # Load/accumulate data from several samples (needs some particular sort)
-  let baseTag = inList.mapIt(it.base).sortMerge
+  let
-  let packed = PackedAccountRange(
+    getFn = desc.getAccountFn
-    accounts: inList.mapIt(it.data.accounts).sortMerge,
+    baseTag = inList.mapIt(it.base).sortMerge
-    proof:    inList.mapIt(it.data.proof).flatten)
+    packed = PackedAccountRange(
+      accounts: inList.mapIt(it.data.accounts).sortMerge,
+      proof:    inList.mapIt(it.data.proof).flatten)
+    nAccounts = packed.accounts.len
   # Merging intervals will produce gaps, so the result is expected OK but
   # different from `.isImportOk`
   check desc.importAccounts(baseTag, packed, true).isOk
@@ -114,21 +119,29 @@ proc test_accountsMergeProofs*(
   # need to check for additional records only on either end of a range.
   var keySet = packed.accounts.mapIt(it.accKey).toHashSet
   for w in inList:
-    var key = desc.prevAccountsChainDbKey(w.data.accounts[0].accKey)
+    var key = desc.prevAccountsChainDbKey(w.data.accounts[0].accKey, getFn)
     while key.isOk and key.value notin keySet:
       keySet.incl key.value
-      let newKey = desc.prevAccountsChainDbKey(key.value)
+      let newKey = desc.prevAccountsChainDbKey(key.value, getFn)
       check newKey != key
       key = newKey
-    key = desc.nextAccountsChainDbKey(w.data.accounts[^1].accKey)
+    key = desc.nextAccountsChainDbKey(w.data.accounts[^1].accKey, getFn)
     while key.isOk and key.value notin keySet:
       keySet.incl key.value
-      let newKey = desc.nextAccountsChainDbKey(key.value)
+      let newKey = desc.nextAccountsChainDbKey(key.value, getFn)
       check newKey != key
       key = newKey
   accKeys = toSeq(keySet).mapIt(it.to(NodeTag)).sorted(cmp)
                          .mapIt(it.to(NodeKey))
-  check packed.accounts.len <= accKeys.len
+  # Some database samples have a few more account keys which come in by the
+  # proof nodes.
+  check nAccounts <= accKeys.len
+
+  # Verify against table importer
+  let
+    xDb = HexaryTreeDbRef.init() # Can dump database with `.pp(xDb)`
+    rc = xDb.fromPersistent(desc.root, getFn, accKeys.len + 100)
+  check rc == Result[int,HexaryError].ok(accKeys.len)
 
 
 proc test_accountsRevisitStoredItems*(
@@ -137,6 +150,8 @@ proc test_accountsRevisitStoredItems*(
     noisy = false;
       ) =
   ## Revisit stored items on ChainDBRef
+  let
+    getFn = desc.getAccountFn
   var
     nextAccount = accKeys[0]
     prevAccount: NodeKey
@@ -145,12 +160,13 @@ proc test_accountsRevisitStoredItems*(
     count.inc
     let
       pfx = $count & "#"
-      byChainDB = desc.getAccountsChainDb(accKey)
+      byChainDB = desc.getAccountsData(accKey, persistent=true)
-      byNextKey = desc.nextAccountsChainDbKey(accKey)
+      byNextKey = desc.nextAccountsChainDbKey(accKey, getFn)
-      byPrevKey = desc.prevAccountsChainDbKey(accKey)
+      byPrevKey = desc.prevAccountsChainDbKey(accKey, getFn)
-    noisy.say "*** find",
+    if byChainDB.isErr:
-      "<", count, "> byChainDb=", byChainDB.pp
+      noisy.say "*** find",
-    check byChainDB.isOk
+        "<", count, "> byChainDb=", byChainDB.pp
+      check byChainDB.isOk
 
     # Check `next` traversal funcionality. If `byNextKey.isOk` fails, the
     # `nextAccount` value is still the old one and will be different from

tests/test_sync_snap/test_node_range.nim

@@ -19,8 +19,9 @@ import
   ../../nimbus/sync/[handlers, protocol, types],
   ../../nimbus/sync/snap/range_desc,
   ../../nimbus/sync/snap/worker/db/[
-    hexary_desc, hexary_envelope,  hexary_error, hexary_interpolate,
+    hexary_debug, hexary_desc, hexary_envelope,  hexary_error,
-    hexary_nearby, hexary_paths, hexary_range, snapdb_accounts, snapdb_desc],
+    hexary_interpolate, hexary_nearby, hexary_paths, hexary_range,
+    snapdb_accounts, snapdb_debug, snapdb_desc],
   ../replay/[pp, undump_accounts],
   ./test_helpers
 
@@ -209,7 +210,6 @@ proc verifyRangeProof(
      ): Result[void,HexaryError] =
   ## Re-build temporary database and prove or disprove
   let
-    dumpOk = dbg.isNil.not
     noisy = dbg.isNil.not
     xDb = HexaryTreeDbRef()
   if not dbg.isNil:
@@ -252,7 +252,7 @@ proc verifyRangeProof(
       "\n\n    last=", leafs[^1].key,
       "\n    ", leafs[^1].key.hexaryPath(rootKey,xDb).pp(dbg),
       "\n\n    database dump",
-      "\n    ", xDb.dumpHexaDB(rootKey),
+      "\n    ", xDb.pp(rootKey),
       "\n"
 
 # ------------------------------------------------------------------------------
@@ -270,8 +270,6 @@ proc test_NodeRangeDecompose*(
   # stray account nodes in the proof *before* the left boundary.
   doAssert 2 < accKeys.len
 
-  const
-    isPersistent = db.type is HexaryTreeDbRef
   let
     rootKey = root.to(NodeKey)
     baseTag = accKeys[0].to(NodeTag) + 1.u256
@@ -392,12 +390,16 @@ proc test_NodeRangeProof*(
   for n,w in inLst:
     doAssert 1 < w.data.accounts.len
     let
-      # Use the middle of the first two points as base
+      first = w.data.accounts[0].accKey.to(NodeTag)
-      delta = (w.data.accounts[1].accKey.to(NodeTag) -
+      delta = (w.data.accounts[1].accKey.to(NodeTag) - first) div 2
-               w.data.accounts[0].accKey.to(NodeTag)) div 2
+      # Use the middle of the first two points as base unless w.base is zero.
-      base = w.data.accounts[0].accKey.to(NodeTag) + delta
+      # This is needed as the range extractor needs the node before the `base`
+      # (if ateher is any) in order to assemble the proof. But this node might
+      # not be present in the partial database.
+      (base, start) = if w.base == 0.to(NodeTag): (w.base, 0)
+                      else: (first + delta, 1)
       # Assemble accounts list starting at the second item
-      accounts = w.data.accounts[1 ..< min(w.data.accounts.len,maxLen)]
+      accounts = w.data.accounts[start ..< min(w.data.accounts.len,maxLen)]
       iv = NodeTagRange.new(base, accounts[^1].accKey.to(NodeTag))
       rc = db.hexaryRangeLeafsProof(rootKey, iv)
     check rc.isOk
@@ -486,7 +488,7 @@ proc test_NodeRangeLeftBoundary*(
   ## Verify left side boundary checks
   let
     rootKey = inLst[0].root.to(NodeKey)
-    noisy = not dbg.isNil
+    noisy {.used.} = not dbg.isNil
 
   # Assuming the `inLst` entries have been stored in the DB already
   for n,w in inLst:
@@ -505,7 +507,7 @@ proc test_NodeRangeLeftBoundary*(
         check (n, j, leftKey) == (n, j, toLeftKey)
         rootKey.printCompareLeftNearby(leftKey, rightKey, db, dbg)
         return
-    noisy.say "***", "n=", n, " accounts=", accounts.len
+    # noisy.say "***", "n=", n, " accounts=", accounts.len
 
 # ------------------------------------------------------------------------------
 # End