Snap sync fix trie interpolation fringe condition (#1457)

* Cosmetics

details:
+ Update doc generator
+ Fix key type representation in `hexary_desc` for debugging
+ Redefine `isImportOk()` as template for better `check()` line reporting

* Fix fringe condition when interpolating Merkle-Patricia tries

details:
  Small change with profound effect fixing some pathological condition
  that haunted the unit test set on large data sers. There is still one
  condition left which might well be due to an incomplete data set.

* Unit test proof nodes for node range extractor

* Unit tests to run on full extraction set

why:
  Left over from troubleshooting, range length was only 5

nimbus/makefile

@@ -30,15 +30,14 @@ MUFFLE     := 2>/dev/null
 endif
 
 # Support for external NIM compiler unless X=0
-ifneq ($(if $X,$X,1),0)
+ifneq ($(if $X,$X,0),0)
-# works with external nim 1.5.1 (not the local 1.2.10 one)
 PATH       := $(SAVED_PATH):$(NIM_PATH):$(NIMBLE_DIR)/bin
 else
 PATH       := $(NIM_PATH):$(NIMBLE_DIR)/bin:$(SAVED_PATH)
 endif
 
 # Compat version is used with external NIM compiler
-NIM_COMPAT := --useVersion:1.2
+# NIM_COMPAT := --useVersion:1.2
 
 # Name of NIMDOC compiler, test for newer version on host OS
 NIM_CMD    := nim
@@ -71,7 +70,7 @@ help::
 	echo "Usage: $(MAKE) <target> [<option> ..]"
 	echo
 	echo "<option>: V=1          -- verbose mode"
-	echo "          X=0          -- preferring local nim compiler (this repo)"
+	echo "          X=1          -- preferring local nim compiler (this repo)"
 	echo "          NIMFLAGS=..  -- additional flags for nim-docs generator"
 	echo
 	echo "<target>: docs         -- build NIM docs"
@@ -116,12 +115,9 @@ docs/ex/%.png : %.png
 	@mkdir -p $(dir $@)
 	@set -x; cp "$<" "$@"
 
-docs/dochack.js docs/nimdoc.out.css:
-	cp docs.static/$(notdir $@) $@
-
 .PHONY: docs-index-helper
 .SILENT: docs-index-helper
-docs-index-helper: docs/dochack.js docs/nimdoc.out.css
+docs-index-helper:
 	nim=$(NIM_EXE); \
 	$$nim --version | sed q ; set -x ;\
 	$$nim --skipProjCfg buildIndex -o:docs/theindex.html \
@@ -211,9 +207,6 @@ clean-test-exe:
 
 .SILENT: clean-docs
 clean-docs:
-	for f in docs/dochack.js docs/nimdoc.out.css; do \
-	  [ ! -f "$$f" ] || (set -x; rm -f "$$f"); \
-	done
 	for f in $(foreach f,$(EXE_FILES),docs/$f) \
 	         $(foreach f,$(PNG_FILES),docs/ex/$f) \
 	         $(foreach f,$(MD_FILES),docs/ex/$f) \
@@ -223,10 +216,6 @@ clean-docs:
 	  [ ! -f "$$f.idx"  ] || (set -x; rm -f "$$f.idx"); \
 	  [ ! -f "$$f.png"  ] || (set -x; rm -f "$$f.png"); \
 	done
-	find docs -name 'nimdoc?out.css' -print 2>/dev/null | \
-	  while read f_css ; do \
-	    [ ! -f "$$f_css"  ] || (set -x; rm -f "$$f_css"); \
-	  done
 	for d in $(shell find docs -depth -type d -print $(MUFFLE)); do \
 	  (set -x; rmdir "$$d" $(MUFFLE)) || true; \
 	done

nimbus/sync/snap/worker/db/hexary_desc.nim

@@ -163,6 +163,9 @@ static:
 var
   disablePrettyKeys* = false      ## Degugging, print raw keys if `true`
 
+proc isNodeKey*(a: RepairKey): bool {.gcsafe.}
+proc isZero*(a: RepairKey): bool {.gcsafe.}
+
 # ------------------------------------------------------------------------------
 # Private helpers
 # ------------------------------------------------------------------------------
@@ -196,6 +199,12 @@ proc ppImpl(s: string; hex = false): string =
       "..(" & $s.len & ").." & s[s.len-16 ..< s.len]
 
 proc ppImpl(key: RepairKey; db: HexaryTreeDbRef): string =
+  if key.isZero:
+    return "ø"
+  if not key.isNodekey:
+    var num: uint64
+    (addr num).copyMem(unsafeAddr key.ByteArray33[25], 8)
+    return "%" & $num
   try:
     if not disablePrettyKeys and not db.keyPp.isNil:
       return db.keyPp(key)
@@ -333,7 +342,11 @@ proc init*(key: var RepairKey; data: openArray[byte]): bool =
 
 proc newRepairKey*(db: HexaryTreeDbRef): RepairKey =
   db.repairKeyGen.inc
-  var src = db.repairKeyGen.toBytesBE
+  # Storing in proper endian handy for debugging (but not really important)
+  when cpuEndian == bigEndian:
+    var src = db.repairKeyGen.toBytesBE
+  else:
+    var src = db.repairKeyGen.toBytesLE
   (addr result.ByteArray33[25]).copyMem(addr src[0], 8)
   result.ByteArray33[0] = 1
 
@@ -355,8 +368,11 @@ proc to*(key: NodeKey; T: type NibblesSeq): T =
 proc to*(key: NodeKey; T: type RepairKey): T =
   (addr result.ByteArray33[1]).copyMem(unsafeAddr key.ByteArray32[0], 32)
 
-proc isZero*[T: NodeTag|NodeKey|RepairKey](a: T): bool =
+proc isZero*(a: RepairKey): bool =
-  a == T.default
+  a == typeof(a).default
+
+proc isZero*[T: NodeTag|NodeKey](a: T): bool =
+  a == typeof(a).default
 
 proc isNodeKey*(a: RepairKey): bool =
   a.ByteArray33[0] == 0

nimbus/sync/snap/worker/db/hexary_envelope.nim

@@ -144,7 +144,7 @@ proc padPartialPath(pfx: NibblesSeq; dblNibble: byte): NodeKey =
       padded = padded & @[dblNibble].initNibbleRange.slice(1)
   else:
     let nope = seq[byte].default.initNibbleRange
-    padded = pfx.slice(0,63) & nope # nope forces re-alignment
+    padded = pfx.slice(0,64) & nope # nope forces re-alignment
 
   let bytes = padded.getBytes
   (addr result.ByteArray32[0]).copyMem(unsafeAddr bytes[0], bytes.len)

nimbus/sync/snap/worker/db/hexary_interpolate.nim

@@ -34,13 +34,16 @@ type
 # Private debugging helpers
 # ------------------------------------------------------------------------------
 
-#proc pp(w: RPathXStep; db: HexaryTreeDbRef): string =
+when true:
-#  let y = if w.canLock: "lockOk" else: "noLock"
+  import std/[sequtils, strutils]
-#  "(" & $w.pos & "," & y & "," & w.step.pp(db) & ")"
 
-#proc pp(w: seq[RPathXStep]; db: HexaryTreeDbRef; indent = 4): string =
+  proc pp(w: RPathXStep; db: HexaryTreeDbRef): string =
-#  let pfx = "\n" & " ".repeat(indent)
+    let y = if w.canLock: "lockOk" else: "noLock"
-#  w.mapIt(it.pp(db)).join(pfx)
+    "(" & $w.pos & "," & y & "," & w.step.pp(db) & ")"
+
+  proc pp(w: seq[RPathXStep]; db: HexaryTreeDbRef; indent = 4): string =
+    let pfx = "\n" & " ".repeat(indent)
+    w.mapIt(it.pp(db)).join(pfx)
 
 # ------------------------------------------------------------------------------
 # Private helpers
@@ -285,7 +288,8 @@ proc rTreeInterpolate(
 proc rTreeUpdateKeys(
     rPath: RPath;
     db: HexaryTreeDbRef;
-      ): Result[void,bool] =
+      ): Result[void,bool]
+      {.gcsafe, raises: [KeyError].} =
   ## The argument `rPath` is assumed to organise database nodes as
   ##
   ##    root -> ... -> () -> () -> ... -> () -> () ...
@@ -343,7 +347,9 @@ proc rTreeUpdateKeys(
         of Leaf:
           discard
         if key != thisKey:
-          return err(false) # no changes were made
+          if not db.tab.hasKey(key) or
+             db.tab[key].state notin {Mutable,TmpRoot}:
+            return err(false) # no changes were made
 
         # Ok, replace database records by stack entries
         var lockOk = true

nimbus/sync/snap/worker/db/snapdb_accounts.nim

@@ -236,7 +236,7 @@ proc importAccounts*(
         return err(rc.error)
       accounts = rc.value
 
-    # Inspect trie for dangling nodes from prrof data (if any.)
+    # Inspect trie for dangling nodes from proof data (if any.)
     if 0 < data.proof.len:
       proofStats = ps.hexaDb.hexaryInspectTrie(ps.root)
 

nimbus/sync/snap/worker/db/snapdb_desc.nim

@@ -74,7 +74,7 @@ proc toKey(a: NodeKey; ps: SnapDbBaseRef): uint =
 #  a.to(NodeKey).toKey(ps)
 
 proc ppImpl(a: RepairKey; pv: SnapDbRef): string =
-  if a.isZero: "ø" else:"$" & $a.toKey(pv)
+  "$" & $a.toKey(pv)
 
 # ------------------------------------------------------------------------------
 # Debugging, pretty printing
@@ -297,23 +297,28 @@ proc verifyNoMoreRight*(
 # Debugging (and playing with the hexary database)
 # ------------------------------------------------------------------------------
 
-proc assignPrettyKeys*(ps: SnapDbBaseRef) =
+proc assignPrettyKeys*(xDb: HexaryTreeDbRef; root: NodeKey) =
   ## Prepare for pretty pringing/debugging. Run early enough this function
   ## sets the root key to `"$"`, for instance.
-  noPpError("validate(1)"):
+  if not xDb.keyPp.isNil:
-    # Make keys assigned in pretty order for printing
+    noPpError("validate(1)"):
-    var keysList = toSeq(ps.hexaDb.tab.keys)
+      # Make keys assigned in pretty order for printing
-    let rootKey = ps.root.to(RepairKey)
+      let rootKey = root.to(RepairKey)
-    discard rootKey.toKey(ps)
+      discard xDb.keyPp rootKey
-    if ps.hexaDb.tab.hasKey(rootKey):
+      var keysList = toSeq(xDb.tab.keys)
-      keysList = @[rootKey] & keysList
+      if xDb.tab.hasKey(rootKey):
-    for key in keysList:
+        keysList = @[rootKey] & keysList
-      let node = ps.hexaDb.tab[key]
+      for key in keysList:
-      discard key.toKey(ps)
+        let node = xDb.tab[key]
-      case node.kind:
+        discard xDb.keyPp key
-      of Branch: (for w in node.bLink: discard w.toKey(ps))
+        case node.kind:
-      of Extension: discard node.eLink.toKey(ps)
+        of Branch: (for w in node.bLink: discard xDb.keyPp w)
-      of Leaf: discard
+        of Extension: discard xDb.keyPp node.eLink
+        of Leaf: discard
+
+proc assignPrettyKeys*(ps: SnapDbBaseRef) =
+  ## Variant of `assignPrettyKeys()`
+  ps.hexaDb.assignPrettyKeys(ps.root)
 
 proc dumpPath*(ps: SnapDbBaseRef; key: NodeTag): seq[string] =
   ## Pretty print helper compiling the path into the repair tree for the

tests/test_sync_snap.nim

@@ -203,17 +203,20 @@ proc accountsRunner(noisy = true;  persistent = true; sample = accSample) =
         getFn = desc.getAccountFn
         dbg = if noisy: hexaDb else: nil
 
-      desc.assignPrettyKeys() # debugging, make sure that state root ~ "$0"
-
       test &"Proofing {accLst.len} list items for state root ..{root.pp}":
         accLst.test_accountsImport(desc, db.persistent)
 
+      # debugging, make sure that state root ~ "$0"
+      desc.assignPrettyKeys()
+
+      # Beware: dumping a large database is not recommended
+      # true.say "***", "database dump\n    ", desc.dumpHexaDB()
+
       test &"Retrieve accounts & proofs for previous account ranges":
-        let nPart = 3
         if db.persistent:
-          accLst.test_NodeRangeRightProofs(getFn, nPart, dbg)
+          accLst.test_NodeRangeProof(getFn, dbg)
         else:
-          accLst.test_NodeRangeRightProofs(hexaDB, nPart, dbg)
+          accLst.test_NodeRangeProof(hexaDB, dbg)
 
       test &"Verify left boundary checks":
         if db.persistent:
@@ -235,8 +238,6 @@ proc accountsRunner(noisy = true;  persistent = true; sample = accSample) =
 
       test &"Revisiting {accKeys.len} stored items on ChainDBRef":
         accKeys.test_accountsRevisitStoredItems(desc, noisy)
-        # Beware: dumping a large database is not recommended
-        # true.say "***", "database dump\n    ", desc.dumpHexaDB()
 
       test &"Decompose path prefix envelopes on {info}":
         let hexaDb = desc.hexaDb
@@ -286,6 +287,7 @@ proc storagesRunner(
     test &"Inspecting {stoLst.len} imported storages lists sub-tries":
       stoLst.test_storageSlotsTries(xdb, db.persistent, knownFailures,idPfx)
 
+
 proc inspectionRunner(
     noisy = true;
     persistent = true;
@@ -547,24 +549,20 @@ when isMainModule:
   #
 
   # This one uses dumps from the external `nimbus-eth1-blob` repo
-  when true: # and false:
+  when true and false:
     import ./test_sync_snap/snap_other_xx
     noisy.showElapsed("accountsRunner()"):
       for n,sam in snapOtherList:
         false.accountsRunner(persistent=true, sam)
-    #noisy.showElapsed("inspectRunner()"):
+    noisy.showElapsed("inspectRunner()"):
-    #  for n,sam in snapOtherHealingList:
+      for n,sam in snapOtherHealingList:
-    #    false.inspectionRunner(persistent=true, cascaded=false, sam)
+        false.inspectionRunner(persistent=true, cascaded=false, sam)
 
   # This one usues dumps from the external `nimbus-eth1-blob` repo
   when true and false:
     import ./test_sync_snap/snap_storage_xx
-    let knownFailures = @[
+    let knownFailures: KnownStorageFailure = @[
-      ("storages3__18__25_dump#11", @[( 233, RightBoundaryProofFailed)]),
-      ("storages4__26__33_dump#11", @[(1193, RightBoundaryProofFailed)]),
       ("storages5__34__41_dump#10", @[( 508, RootNodeMismatch)]),
-      ("storagesB__84__92_dump#6",  @[( 325, RightBoundaryProofFailed)]),
-      ("storagesD_102_109_dump#17", @[(1102, RightBoundaryProofFailed)]),
     ]
     noisy.showElapsed("storageRunner()"):
       for n,sam in snapStorageList:
@@ -572,14 +570,14 @@ when isMainModule:
 
   # This one uses readily available dumps
   when true: # and false:
-  #  false.inspectionRunner()
+    false.inspectionRunner()
     for n,sam in snapTestList:
       false.accountsRunner(persistent=false, sam)
       false.accountsRunner(persistent=true, sam)
     for n,sam in snapTestStorageList:
       false.accountsRunner(persistent=false, sam)
       false.accountsRunner(persistent=true, sam)
-  #    false.storagesRunner(persistent=true, sam)
+      false.storagesRunner(persistent=true, sam)
 
   # This one uses readily available dumps
   when true and false:

tests/test_sync_snap/snap_storage_xx.nim

@@ -9,19 +9,6 @@
 # at your option. This file may not be copied, modified, or
 # distributed except according to those terms.
 
-import
-  std/os,
-  ./test_types# Nimbus - Types, data structures and shared utilities used in network sync
-#
-# Copyright (c) 2018-2021 Status Research & Development GmbH
-# Licensed under either of
-#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
-#    http://www.apache.org/licenses/LICENSE-2.0)
-#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
-#    http://opensource.org/licenses/MIT)
-# at your option. This file may not be copied, modified, or
-# distributed except according to those terms.
-
 import
   std/os,
   ./test_types

tests/test_sync_snap/test_helpers.nim

@@ -19,17 +19,23 @@ import
   ../../nimbus/sync/snap/worker/db/[hexary_desc, snapdb_accounts],
   ../replay/pp
 
+type
+  KnownStorageFailure* = seq[(string,seq[(int,HexaryError)])]
+    ## (<sample-name> & "#" <instance>, @[(<slot-id>, <error-symbol>)), ..])
+
 # ------------------------------------------------------------------------------
 # Public helpers
 # ------------------------------------------------------------------------------
 
-proc isImportOk*(rc: Result[SnapAccountsGaps,HexaryError]): bool =
+template isImportOk*(rc: Result[SnapAccountsGaps,HexaryError]): bool =
   if rc.isErr:
     check rc.error == NothingSerious # prints an error if different
+    false
   elif 0 < rc.value.innerGaps.len:
     check rc.value.innerGaps == seq[NodeSpecs].default
+    false
   else:
-    return true
+    true
 
 proc lastTwo*(a: openArray[string]): seq[string] =
   if 1 < a.len: @[a[^2],a[^1]] else: a.toSeq

tests/test_sync_snap/test_node_range.nim

@@ -12,15 +12,16 @@
 ## Snap sync components tester and TDD environment
 
 import
-  std/[sequtils, strformat, strutils],
+  std/[sequtils, sets, strformat, strutils],
   eth/[common, p2p, rlp, trie/nibbles],
   stew/[byteutils, interval_set, results],
   unittest2,
   ../../nimbus/sync/types,
   ../../nimbus/sync/snap/range_desc,
   ../../nimbus/sync/snap/worker/db/[
-    hexary_desc, hexary_envelope,  hexary_error, hexary_nearby, hexary_paths,
+    hexary_desc, hexary_envelope,  hexary_error, hexary_interpolate,
-    hexary_range, snapdb_accounts, snapdb_desc],
+    hexary_import, hexary_nearby, hexary_paths, hexary_range,
+    snapdb_accounts, snapdb_desc],
   ../replay/[pp, undump_accounts],
   ./test_helpers
 
@@ -28,6 +29,18 @@ const
   cmaNlSp0 = ",\n" & repeat(" ",12)
   cmaNlSpc = ",\n" & repeat(" ",13)
 
+# ------------------------------------------------------------------------------
+# Private helpers
+# ------------------------------------------------------------------------------
+
+proc ppNodeKeys(a: openArray[Blob], dbg = HexaryTreeDbRef(nil)): string =
+  result = "["
+  if dbg.isNil:
+    result &= a.mapIt(it.digestTo(NodeKey).pp(collapse=true)).join(",")
+  else:
+    result &= a.mapIt(it.digestTo(NodeKey).pp(dbg)).join(",")
+  result &= "]"
+
 # ------------------------------------------------------------------------------
 # Private functions
 # ------------------------------------------------------------------------------
@@ -132,18 +145,14 @@ proc printCompareRightLeafs(
       else:
         check rightRc.error == HexaryError(0) # force error printing
 
-      noisy.say "\n***", "i=", i, "/", accounts.len,
+      if noisy: true.say "\n***", "i=", i, "/", accounts.len,
-        "\n",
+        "\n\n    prdKey=", prdKey,
-        "\n    prdKey=", prdKey,
         "\n    ", prdKey.hexaryPath(rootKey,db).pp(dbg),
-        "\n",
+        "\n\n    nxtKey=", nxtTag,
-        "\n    nxtKey=", nxtTag,
         "\n    ", nxtPath.pp(dbg),
-        "\n",
+        "\n\n    accKey=", accKey,
-        "\n    accKey=", accKey,
         "\n    ", accKey.hexaryPath(rootKey,db).pp(dbg),
-        "\n",
+        "\n\n    lfsKey=", lfsKey,
-        "\n    lfsKey=", lfsKey,
         "\n    ", lfsKey.hexaryPath(rootKey,db).pp(dbg),
         "\n"
       return
@@ -170,14 +179,12 @@ proc printCompareLeftNearby(
     if toLeftKey == leftKey:
       return
 
-  noisy.say "\n***",
+  if noisy: true.say "\n***",
     "    rightKey=", rightKey,
     "\n    ", rightKey.hexaryPath(rootKey,db).pp(dbg),
-    "\n",
+    "\n\n    leftKey=", leftKey,
-    "\n    leftKey=", leftKey,
     "\n    ", leftKey.hexaryPath(rootKey,db).pp(dbg),
-    "\n",
+    "\n\n    toLeftKey=", toLeftKey,
-    "\n    toLeftKey=", toLeftKey,
     "\n    ", toLeftKey.hexaryPath(rootKey,db).pp(dbg),
     "\n"
 
@@ -191,6 +198,52 @@ proc verifyAccountListSizes() =
       nonce:       high(uint64),
       balance:     high(UInt256)).repeat(n).encode.len
 
+
+proc verifyRangeProof(
+    rootKey: NodeKey;
+    leafs: seq[RangeLeaf];
+    proof: seq[Blob];
+    dbg = HexaryTreeDbRef(nil);
+     ): Result[void,HexaryError] =
+  ## Re-build temporary database and prove or disprove
+  let
+    dumpOk = dbg.isNil.not
+    noisy = dbg.isNil.not
+    xDb = HexaryTreeDbRef()
+  if not dbg.isNil:
+    xDb.keyPp = dbg.keyPp
+
+  # Import proof nodes
+  var unrefs, refs: HashSet[RepairKey] # values ignored
+  for rlpRec in proof:
+    let importError = xDb.hexaryImport(rlpRec, unrefs, refs).error
+    if importError != HexaryError(0):
+      check importError == HexaryError(0)
+      return err(importError)
+
+  # Build tree
+  var lItems = leafs.mapIt(RLeafSpecs(
+    pathTag: it.key.to(NodeTag),
+    payload: it.data))
+  let rc = xDb.hexaryInterpolate(rootKey, lItems)
+  if rc.isOk:
+    return ok()
+
+  if noisy:
+    true.say "\n***", "error=", rc.error,
+      #"\n",
+      #"\n    unrefs=[", unrefs.toSeq.mapIt(it.pp(dbg)).join(","), "]",
+      #"\n    refs=[", refs.toSeq.mapIt(it.pp(dbg)).join(","), "]",
+      "\n\n    proof=", proof.ppNodeKeys(dbg),
+      "\n\n    first=", leafs[0].key,
+      "\n    ", leafs[0].key.hexaryPath(rootKey,xDb).pp(dbg),
+      "\n\n    last=", leafs[^1].key,
+      "\n    ", leafs[^1].key.hexaryPath(rootKey,xDb).pp(dbg),
+      "\n\n    database dump",
+      "\n    ", xDb.dumpHexaDB(rootKey),
+      "\n"
+  rc
+
 # ------------------------------------------------------------------------------
 # Private functions, pretty printing
 # ------------------------------------------------------------------------------
@@ -322,10 +375,9 @@ proc test_NodeRangeDecompose*(
       if true: quit()
 
 
-proc test_NodeRangeRightProofs*(
+proc test_NodeRangeProof*(
     inLst: seq[UndumpAccounts];
     db: HexaryTreeDbRef|HexaryGetFn;         ## Database abstraction
-    nSplit = 0;                              ## Also split intervals (unused)
     dbg = HexaryTreeDbRef(nil);              ## Debugging env
       ) =
   ## Partition range and provide proofs suitable for `GetAccountRange` message
@@ -333,6 +385,7 @@ proc test_NodeRangeRightProofs*(
   let
     rootKey = inLst[0].root.to(NodeKey)
     noisy = not dbg.isNil
+    maxLen = high(int)
 
   # RLP does not allow static check
   verifyAccountListSizes()
@@ -340,25 +393,44 @@ proc test_NodeRangeRightProofs*(
   # Assuming the `inLst` entries have been stored in the DB already
   for n,w in inLst:
     let
-      iv = NodeTagRange.new(w.base, w.data.accounts[^1].accKey.to(NodeTag))
+      accounts = w.data.accounts[0 .. min(w.data.accounts.len,maxLen)-1]
-      rc = iv.hexaryRangeLeafsProof(rootKey, db, high(int))
+      iv = NodeTagRange.new(w.base, accounts[^1].accKey.to(NodeTag))
+      rc = iv.hexaryRangeLeafsProof(rootKey, db, accounts.len)
     check rc.isOk
     if rc.isErr:
       return
 
-    let
+    let leafs = rc.value.leafs
-      leafs = rc.value.leafs
-      accounts = w.data.accounts
     if leafs.len != accounts.len or accounts[^1].accKey != leafs[^1].key:
       noisy.say "***", "n=", n, " something went wrong .."
       check (n,leafs.len) == (n,accounts.len)
       rootKey.printCompareRightLeafs(w.base, accounts, leafs, db, dbg)
       return
 
-    # FIXME: verify that proof nodes are complete
+    # Import proof nodes and build trie
+    var rx = rootKey.verifyRangeProof(leafs, rc.value.proof)
+    if rx.isErr:
+      rx = rootKey.verifyRangeProof(leafs, rc.value.proof, dbg)
+      let
+        baseNbls =  iv.minPt.to(NodeKey).to(NibblesSeq)
+        lastNbls =  iv.maxPt.to(NodeKey).to(NibblesSeq)
+        nPfxNblsLen = baseNbls.sharedPrefixLen lastNbls
+        pfxNbls = baseNbls.slice(0, nPfxNblsLen)
+      noisy.say "***", "n=", n,
+        " leafs=", leafs.len,
+        " proof=", rc.value.proof.ppNodeKeys(dbg),
+        "\n\n   ",
+        " base=", iv.minPt,
+        "\n    ", iv.minPt.hexaryPath(rootKey,db).pp(dbg),
+        "\n\n   ",
+        " pfx=", pfxNbls,
+        " nPfx=", nPfxNblsLen,
+        "\n    ", pfxNbls.hexaryPath(rootKey,db).pp(dbg),
+        "\n"
+
+      check rx == typeof(rx).ok()
+      return
 
-    check rc.value.proof.len <= w.data.proof.len
-    check leafs[^1].key.to(NodeTag) <= iv.maxPt
     noisy.say "***", "n=", n,
       " leafs=", leafs.len,
       " proof=", rc.value.proof.len, "/", w.data.proof.len

tests/test_sync_snap/test_storage.nim

@@ -58,7 +58,7 @@ proc test_storageSlotsImport*(
     inList: seq[UndumpStorages];
     dbBase: SnapDbRef;
     persistent: bool;
-    ignore: seq[(string,seq[(int,HexaryError)])];
+    ignore: KnownStorageFailure;
     idPfx: string;
       ) =
   ## Import and merge storages lists
@@ -80,7 +80,7 @@ proc test_storageSlotsTries*(
     inList: seq[UndumpStorages];
     dbBase: SnapDbRef;
     persistent: bool;
-    ignore: seq[(string,seq[(int,HexaryError)])];
+    ignore: KnownStorageFailure;
     idPfx: string;
       ) =
   ## Inspecting imported storages lists sub-tries