Add portal proof functionality for non-existing keys/paths (#2610)

nimbus/db/aristo/aristo_api.nim

@@ -330,16 +330,20 @@ type
   AristoApiPartAccountTwig* =
     proc(db: AristoDbRef;
          accPath: Hash256;
-        ): Result[seq[Blob], AristoError]
+        ): Result[(seq[Blob],bool), AristoError]
         {.noRaise.}
       ## This function returns a chain of rlp-encoded nodes along the argument
-      ## path `(root,path)`.
+      ## path `(root,path)` followed by a `true` value if the `path` argument
+      ## exists in the database. If the argument `path` is not on the database,
+      ## a partial path will be returned follwed by a `false` value.
+      ##
+      ## Errors will only be returned for invalid paths.
 
   AristoApiPartGenericTwig* =
     proc(db: AristoDbRef;
          root: VertexID;
          path: openArray[byte];
-        ): Result[seq[Blob], AristoError]
+        ): Result[(seq[Blob],bool), AristoError]
         {.noRaise.}
       ## Variant of `partAccountTwig()`.
       ##
@@ -350,30 +354,33 @@ type
     proc(db: AristoDbRef;
          accPath: Hash256;
          stoPath: Hash256;
-        ): Result[seq[Blob], AristoError]
+        ): Result[(seq[Blob],bool), AristoError]
         {.noRaise.}
-      ## Variant of `partAccountTwig()`.
+      ## Variant of `partAccountTwig()`. Note that the function always returns
+      ## an error unless the `accPath` is valid.
 
   AristoApiPartUntwigGeneric* =
     proc(chain: openArray[Blob];
          root: Hash256;
          path: openArray[byte];
-        ): Result[Blob,AristoError]
+        ): Result[Opt[Blob],AristoError]
         {.noRaise.}
-      ## Follow and verify the argument `chain` up unlil the last entry
+      ## Follow and verify the argument `chain` up unlil the last entry which
-      ## which must be a leaf node. Extract the payload and pass it on
+      ## must be a leaf node. Extract the payload and pass it on as return
-      ## as return code.
+      ## code. If a `Opt.none()` result is returned then the `path` argument
+      ## does provably not exist relative to `chain`.
 
   AristoApiPartUntwigGenericOk* =
     proc(chain: openArray[Blob];
          root: Hash256;
          path: openArray[byte];
-         payload: openArray[byte];
+         payload: Opt[Blob];
         ): Result[void,AristoError]
         {.noRaise.}
-      ## Variant of `partUntwigGeneric()`. The function verifis the argument
+      ## Variant of `partUntwigGeneric()`. The function verifies the argument
       ## `chain` of rlp-encoded nodes against the `path` and `payload`
-      ## arguments.
+      ## arguments. If `payload` is passed `Opt.none()`, then the function is
+      ## subject to proving that the `path` does not exist relaive to `chain`.
       ##
       ## Note: This function provides a functionality comparable to the
       ## `isValidBranch()` function from `hexary.nim`.
@@ -382,7 +389,7 @@ type
     proc(chain: openArray[Blob];
          root: Hash256;
          path: Hash256;
-        ): Result[Blob,AristoError]
+        ): Result[Opt[Blob],AristoError]
         {.noRaise.}
       ## Variant of `partUntwigGeneric()`.
 
@@ -390,7 +397,7 @@ type
     proc(chain: openArray[Blob];
          root: Hash256;
          path: Hash256;
-         payload: openArray[byte];
+         payload: Opt[Blob];
         ): Result[void,AristoError]
         {.noRaise.}
       ## Variant of `partUntwigGenericOk()`.
@@ -974,7 +981,7 @@ func init*(
         result = api.partUntwigGeneric(a, b, c)
 
   profApi.partUntwigGenericOk =
-    proc(a: openArray[Blob]; b: Hash256; c, d: openArray[byte]): auto =
+    proc(a: openArray[Blob]; b:Hash256; c:openArray[byte]; d:Opt[Blob]): auto =
       AristoApiProfPartUntwigGenericOkFn.profileRunner:
         result = api.partUntwigGenericOk(a, b, c, d)
 
@@ -984,7 +991,7 @@ func init*(
         result = api.partUntwigPath(a, b, c)
 
   profApi.partUntwigPathOk =
-    proc(a: openArray[Blob]; b, c: Hash256; d: openArray[byte]): auto =
+    proc(a: openArray[Blob]; b, c: Hash256; d: Opt[Blob]): auto =
       AristoApiProfPartUntwigPathOkFn.profileRunner:
         result = api.partUntwigPathOk(a, b, c, d)
 

nimbus/db/aristo/aristo_check/check_twig.nim

@@ -27,7 +27,7 @@ proc checkTwig*(
   let
     proof = ? db.partGenericTwig(root, path)
     key = ? db.computeKey (root,root)
-    pyl = ? proof.partUntwigGeneric(key.to(Hash256), path)
+    pyl = ? proof[0].partUntwigGeneric(key.to(Hash256), path)
 
   ok()
 
@@ -40,7 +40,7 @@ proc checkTwig*(
     proof = ? db.partStorageTwig(accPath, stoPath)
     vid = ? db.fetchStorageID accPath
     key = ? db.computeKey (VertexID(1),vid)
-    pyl = ? proof.partUntwigPath(key.to(Hash256), stoPath)
+    pyl = ? proof[0].partUntwigPath(key.to(Hash256), stoPath)
 
   ok()
 

nimbus/db/aristo/aristo_desc/desc_error.nim

@@ -170,6 +170,7 @@ type
     PartChkVidTabCoreRootMissing
     PartChkVidTabVidMissing
     PartChnBranchPathExhausted
+    PartChnBranchVoidEdge
     PartChnExtPfxMismatch
     PartChnLeafPathMismatch
     PartChnNodeConvError

nimbus/db/aristo/aristo_part.nim

@@ -70,19 +70,28 @@ proc partGenericTwig*(
     db: AristoDbRef;
     root: VertexID;
     path: NibblesBuf;
-      ): Result[seq[Blob], AristoError] =
+      ): Result[(seq[Blob],bool), AristoError] =
   ## This function returns a chain of rlp-encoded nodes along the argument
-  ## path `(root,path)`.
+  ## path `(root,path)` followed by a `true` value if the `path` argument
+  ## exists in the database. If the argument `path` is not on the database,
+  ## a partial path will be returned follwed by a `false` value.
+  ##
+  ## Errors will only be returned for invalid paths.
   ##
   var chain: seq[Blob]
-  ? db.chainRlpNodes((root,root), path, chain)
+  let rc = db.chainRlpNodes((root,root), path, chain)
-  ok chain
+  if rc.isOk:
+    ok((chain, true))
+  elif rc.error in ChainRlpNodesNoEntry:
+    ok((chain, false))
+  else:
+    err(rc.error)
 
 proc partGenericTwig*(
     db: AristoDbRef;
     root: VertexID;
     path: openArray[byte];
-      ): Result[seq[Blob], AristoError] =
+      ): Result[(seq[Blob],bool), AristoError] =
   ## Variant of `partGenericTwig()`.
   ##
   ## Note: This function provides a functionality comparable to the
@@ -93,7 +102,7 @@ proc partGenericTwig*(
 proc partAccountTwig*(
     db: AristoDbRef;
     accPath: Hash256;
-      ): Result[seq[Blob], AristoError] =
+      ): Result[(seq[Blob],bool), AristoError] =
   ## Variant of `partGenericTwig()`.
   db.partGenericTwig(VertexID(1), NibblesBuf.fromBytes accPath.data)
 
@@ -101,8 +110,9 @@ proc partStorageTwig*(
     db: AristoDbRef;
     accPath: Hash256;
     stoPath: Hash256;
-      ): Result[seq[Blob], AristoError] =
+      ): Result[(seq[Blob],bool), AristoError] =
-  ## Variant of `partGenericTwig()`.
+  ## Variant of `partGenericTwig()`. Note that the function always returns an
+  ## error unless the `accPath` is valid.
   let vid = ? db.fetchStorageID accPath
   db.partGenericTwig(vid, NibblesBuf.fromBytes stoPath.data)
 
@@ -112,11 +122,19 @@ proc partUntwigGeneric*(
     chain: openArray[Blob];
     root: Hash256;
     path: openArray[byte];
-      ): Result[Blob,AristoError] =
+      ): Result[Opt[Blob],AristoError] =
-  ## Verify the chain of rlp-encoded nodes and return the payload.
+  ## Verify the chain of rlp-encoded nodes and return the payload. If a
+  ## `Opt.none()` result is returned then the `path` argument does provably
+  ## not exist relative to `chain`.
   try:
-    let nibbles = NibblesBuf.fromBytes path
+    let
-    return chain.trackRlpNodes(root.to(HashKey), nibbles, start=true)
+      nibbles = NibblesBuf.fromBytes path
+      rc = chain.trackRlpNodes(root.to(HashKey), nibbles, start=true)
+    if rc.isOk:
+      return ok(Opt.some rc.value)
+    if rc.error in TrackRlpNodesNoEntry:
+      return ok(Opt.none Blob)
+    return err(rc.error)
   except RlpError:
     return err(PartTrkRlpError)
 
@@ -124,7 +142,7 @@ proc partUntwigPath*(
     chain: openArray[Blob];
     root: Hash256;
     path: Hash256;
-      ): Result[Blob,AristoError] =
+      ): Result[Opt[Blob],AristoError] =
   ## Variant of `partUntwigGeneric()`.
   chain.partUntwigGeneric(root, path.data)
 
@@ -133,7 +151,7 @@ proc partUntwigGenericOk*(
     chain: openArray[Blob];
     root: Hash256;
     path: openArray[byte];
-    payload: openArray[byte];
+    payload: Opt[Blob];
       ): Result[void,AristoError] =
   ## Verify the argument `chain` of rlp-encoded nodes against the `path`
   ## and `payload` arguments.
@@ -150,7 +168,7 @@ proc partUntwigPathOk*(
     chain: openArray[Blob];
     root: Hash256;
     path: Hash256;
-    payload: openArray[byte];
+    payload: Opt[Blob];
       ): Result[void,AristoError] =
   ## Variant of `partUntwigGenericOk()`.
   chain.partUntwigGenericOk(root, path.data, payload)

nimbus/db/aristo/aristo_part/part_chain_rlp.nim

@@ -15,6 +15,16 @@ import
   results,
   ".."/[aristo_desc, aristo_get, aristo_utils, aristo_compute, aristo_serialise]
 
+const
+  ChainRlpNodesNoEntry* = {
+    PartChnLeafPathMismatch, PartChnExtPfxMismatch, PartChnBranchVoidEdge}
+    ## Partial path errors that can be used to proof that a path does
+    ## not exists.
+
+  TrackRlpNodesNoEntry* = {PartTrkLinkExpected, PartTrkLeafPfxMismatch}
+    ## This is the opposite of `ChainRlpNodesNoEntry` when verifying that a
+    ## node does not exist.
+
 # ------------------------------------------------------------------------------
 # Public functions
 # ------------------------------------------------------------------------------
@@ -53,6 +63,8 @@ proc chainRlpNodes*(
       let
         nibble = path[nChewOff]
         rest = path.slice(nChewOff+1)
+      if not vtx.bVid[nibble].isValid:
+        return err(PartChnBranchVoidEdge)
       # Recursion!
       db.chainRlpNodes((rvid.root,vtx.bVid[nibble]), rest, chain)
 

nimbus/db/core_db/base.nim

@@ -212,7 +212,7 @@ proc verify*(
     proof: openArray[Blob];
     root: Hash256;
     path: openArray[byte];
-      ): CoreDbRc[Blob] =
+      ): CoreDbRc[Opt[Blob]] =
   ## This function os the counterpart of any of the `proof()` functions. Given
   ## the argument chain of rlp-encoded nodes `proof`, this function verifies
   ## that the chain represents a partial MPT starting with a root node state
@@ -243,7 +243,7 @@ proc verifyOk*(
     proof: openArray[Blob];
     root: Hash256;
     path: openArray[byte];
-    payload: openArray[byte];
+    payload: Opt[Blob];
       ): CoreDbRc[void] =
   ## Variant of `verify()` which directly checks the argument `payload`
   ## against what would be the return code in `verify()`.
@@ -267,7 +267,7 @@ proc verify*(
     proof: openArray[Blob];
     root: Hash256;
     path: Hash256;
-      ): CoreDbRc[Blob] =
+      ): CoreDbRc[Opt[Blob]] =
   ## Variant of `verify()`.
   template mpt: untyped =
     when db is CoreDbRef:
@@ -288,7 +288,7 @@ proc verifyOk*(
     proof: openArray[Blob];
     root: Hash256;
     path: Hash256;
-    payload: openArray[byte];
+    payload: Opt[Blob];
       ): CoreDbRc[void] =
   ## Variant of `verifyOk()`.
   template mpt: untyped =
@@ -432,9 +432,12 @@ proc getGeneric*(
 proc proof*(
     mpt: CoreDbMptRef;
     key: openArray[byte];
-      ): CoreDbRc[seq[Blob]] =
+      ): CoreDbRc[(seq[Blob],bool)] =
   ## On the generic MPT, collect the nodes along the `key` interpreted as
-  ## path. Return these path nodes as a chain of rlp-encoded blobs.
+  ## path. Return these path nodes as a chain of rlp-encoded blobs followed
+  ## by a bool value which is `true` if the `key` path exists in the database,
+  ## and `false` otherwise. In the latter case, the chain of rlp-encoded blobs
+  ## are the nodes proving that the `key` path does not exist.
   ##
   mpt.setTrackNewApi MptProofFn
   result = block:
@@ -547,9 +550,12 @@ proc getAccounts*(ctx: CoreDbCtxRef): CoreDbAccRef =
 proc proof*(
     acc: CoreDbAccRef;
     accPath: Hash256;
-      ): CoreDbRc[seq[Blob]] =
+      ): CoreDbRc[(seq[Blob],bool)] =
   ## On the accounts MPT, collect the nodes along the `accPath` interpreted as
-  ## path. Return these path nodes as a chain of rlp-encoded blobs.
+  ## path. Return these path nodes as a chain of rlp-encoded blobs followed
+  ## by a bool value which is `true` if the `key` path exists in the database,
+  ## and `false` otherwise. In the latter case, the chain of rlp-encoded blobs
+  ## are the nodes proving that the `key` path does not exist.
   ##
   acc.setTrackNewApi AccProofFn
   result = block:
@@ -671,10 +677,16 @@ proc slotProof*(
     acc: CoreDbAccRef;
     accPath: Hash256;
     stoPath: Hash256;
-      ): CoreDbRc[seq[Blob]] =
+      ): CoreDbRc[(seq[Blob],bool)] =
   ## On the storage MPT related to the argument account `acPath`, collect the
   ## nodes along the `stoPath` interpreted as path. Return these path nodes as
-  ## a chain of rlp-encoded blobs.
+  ## a chain of rlp-encoded blobs followed by a bool value which is `true` if
+  ## the `key` path exists in the database, and `false` otherwise. In the
+  ## latter case, the chain of rlp-encoded blobs are the nodes proving that
+  ## the `key` path does not exist.
+  ##
+  ## Note that the function always returns an error unless the `accPath` is
+  ## valid.
   ##
   acc.setTrackNewApi AccSlotProofFn
   result = block:

nimbus/db/core_db/base/api_tracking.nim

@@ -132,6 +132,11 @@ func toStr(rc: CoreDbRc[seq[Blob]]): string =
   if rc.isOk: "ok([" & rc.value.mapIt("[#" & $it.len & "]").join(",") & "])"
   else: "err(" & rc.error.toStr & ")"
 
+func toStr(rc: CoreDbRc[(seq[Blob],bool)]): string =
+  if rc.isOk: "ok([" & rc.value[0].mapIt("[#" & $it.len & "]").join(",") &
+                             "]," & $rc.value[1] & ")"
+  else: "err(" & rc.error.toStr & ")"
+
 func toStr(rc: CoreDbRc[Hash256]): string =
   if rc.isOk: "ok(" & rc.value.toStr & ")" else: "err(" & rc.error.toStr & ")"
 

tests/test_aristo/test_portal_proof.nim

@@ -25,6 +25,7 @@ import
 type
   ProofData = ref object
     chain: seq[Blob]
+    missing: bool
     error: AristoError
     hike: Hike
 
@@ -122,6 +123,14 @@ func asExtension(b: Blob; path: Hash256): Blob =
   else:
     b
 
+when false:
+  # just keep for potential debugging
+  proc sq(s: string): string =
+    ## For long strings print `begin..end` only
+    let n = (s.len + 1) div 2
+    result = if s.len < 20: s else: s[0 .. 5] & ".." & s[s.len-8 .. ^1]
+    result &= "[" & (if 0 < n: "#" & $n else: "") & "]"
+
 # ------------------------------------------------------------------------------
 # Private test functions
 # ------------------------------------------------------------------------------
@@ -152,13 +161,33 @@ proc testCreatePortalProof(node: JsonNode, testStatusIMPL: var TestStatus) =
   # Create proof chains
   for (path,proof) in sample.pairs:
     let rc = ps.db.partAccountTwig path
-    check rc.isOk == (proof.error == AristoError 0)
+    if proof.error == AristoError(0):
-    if rc.isOk:
+      check rc.isOk and rc.value[1] == true
-      proof.chain = rc.value
+      proof.chain = rc.value[0]
+    elif proof.error != HikeBranchMissingEdge:
+      # Note that this is a partial data base and in this case the proof for a
+      # non-existing entry might not work properly when the vertex is missing.
+      check rc.isOk and rc.value[1] == false
+      proof.chain = rc.value[0]
+      proof.missing = true
 
   # Verify proof chains
   for (path,proof) in sample.pairs:
-    if proof.error == AristoError 0:
+    if proof.missing:
+      # Proof for missing entries
+      let
+        rVid = proof.hike.root
+        root = ps.db.getKey((rVid,rVid)).to(Hash256)
+        chain = proof.chain
+
+      block:
+        let rc = proof.chain.partUntwigPath(root, path)
+        check rc.isOk and rc.value.isNone
+
+      # Just for completeness (same a above combined into a single function)
+      check proof.chain.partUntwigPathOk(root, path, Opt.none Blob).isOk
+
+    elif proof.error == AristoError 0:
       let
         rVid = proof.hike.root
         pyl = proof.hike.legs[^1].wp.vtx.lData.payloadAsBlob(ps)
@@ -174,9 +203,8 @@ proc testCreatePortalProof(node: JsonNode, testStatusIMPL: var TestStatus) =
         # Create the same proof again which must result into the same as before
         block:
           let rc = pq.db.partAccountTwig path
-          check rc.isOk
+          if rc.isOk and rc.value[1] == true:
-          if rc.isOk:
+            check rc.value[0] == proof.chain
-            check rc.value == proof.chain
 
         # Verify proof
         let root = pq.db.getKey((rVid,rVid)).to(Hash256)
@@ -184,10 +212,10 @@ proc testCreatePortalProof(node: JsonNode, testStatusIMPL: var TestStatus) =
           let rc = proof.chain.partUntwigPath(root, path)
           check rc.isOk
           if rc.isOk:
-            check rc.value == pyl
+            check rc.value == Opt.some(pyl)
 
         # Just for completeness (same a above combined into a single function)
-        check proof.chain.partUntwigPathOk(root, path, pyl).isOk
+        check proof.chain.partUntwigPathOk(root, path, Opt.some pyl).isOk
 
       # Extension nodes are rare, so there is one created, inserted and the
       # previous test repeated.
@@ -204,18 +232,18 @@ proc testCreatePortalProof(node: JsonNode, testStatusIMPL: var TestStatus) =
         # Re-create proof again
         block:
           let rc = pq.db.partAccountTwig path
-          check rc.isOk
+          check rc.isOk and rc.value[1] == true
-          if rc.isOk:
+          if rc.isOk and rc.value[1] == true:
-            check rc.value == chain
+            check rc.value[0] == chain
 
         let root = pq.db.getKey((rVid,rVid)).to(Hash256)
         block:
           let rc = chain.partUntwigPath(root, path)
           check rc.isOk
           if rc.isOk:
-            check rc.value == pyl
+            check rc.value == Opt.some(pyl)
 
-        check chain.partUntwigPathOk(root, path, pyl).isOk
+        check chain.partUntwigPathOk(root, path, Opt.some pyl).isOk
 
 # ------------------------------------------------------------------------------
 # Test