diff --git a/nimbus/db/aristo/aristo_api.nim b/nimbus/db/aristo/aristo_api.nim index b487604fa..709fb2b98 100644 --- a/nimbus/db/aristo/aristo_api.nim +++ b/nimbus/db/aristo/aristo_api.nim @@ -330,16 +330,20 @@ type AristoApiPartAccountTwig* = proc(db: AristoDbRef; accPath: Hash256; - ): Result[seq[Blob], AristoError] + ): Result[(seq[Blob],bool), AristoError] {.noRaise.} ## This function returns a chain of rlp-encoded nodes along the argument - ## path `(root,path)`. + ## path `(root,path)` followed by a `true` value if the `path` argument + ## exists in the database. If the argument `path` is not on the database, + ## a partial path will be returned follwed by a `false` value. + ## + ## Errors will only be returned for invalid paths. AristoApiPartGenericTwig* = proc(db: AristoDbRef; root: VertexID; path: openArray[byte]; - ): Result[seq[Blob], AristoError] + ): Result[(seq[Blob],bool), AristoError] {.noRaise.} ## Variant of `partAccountTwig()`. ## @@ -350,30 +354,33 @@ type proc(db: AristoDbRef; accPath: Hash256; stoPath: Hash256; - ): Result[seq[Blob], AristoError] + ): Result[(seq[Blob],bool), AristoError] {.noRaise.} - ## Variant of `partAccountTwig()`. + ## Variant of `partAccountTwig()`. Note that the function always returns + ## an error unless the `accPath` is valid. AristoApiPartUntwigGeneric* = proc(chain: openArray[Blob]; root: Hash256; path: openArray[byte]; - ): Result[Blob,AristoError] + ): Result[Opt[Blob],AristoError] {.noRaise.} - ## Follow and verify the argument `chain` up unlil the last entry - ## which must be a leaf node. Extract the payload and pass it on - ## as return code. + ## Follow and verify the argument `chain` up unlil the last entry which + ## must be a leaf node. Extract the payload and pass it on as return + ## code. If a `Opt.none()` result is returned then the `path` argument + ## does provably not exist relative to `chain`. AristoApiPartUntwigGenericOk* = proc(chain: openArray[Blob]; root: Hash256; path: openArray[byte]; - payload: openArray[byte]; + payload: Opt[Blob]; ): Result[void,AristoError] {.noRaise.} - ## Variant of `partUntwigGeneric()`. The function verifis the argument + ## Variant of `partUntwigGeneric()`. The function verifies the argument ## `chain` of rlp-encoded nodes against the `path` and `payload` - ## arguments. + ## arguments. If `payload` is passed `Opt.none()`, then the function is + ## subject to proving that the `path` does not exist relaive to `chain`. ## ## Note: This function provides a functionality comparable to the ## `isValidBranch()` function from `hexary.nim`. @@ -382,7 +389,7 @@ type proc(chain: openArray[Blob]; root: Hash256; path: Hash256; - ): Result[Blob,AristoError] + ): Result[Opt[Blob],AristoError] {.noRaise.} ## Variant of `partUntwigGeneric()`. @@ -390,7 +397,7 @@ type proc(chain: openArray[Blob]; root: Hash256; path: Hash256; - payload: openArray[byte]; + payload: Opt[Blob]; ): Result[void,AristoError] {.noRaise.} ## Variant of `partUntwigGenericOk()`. @@ -974,7 +981,7 @@ func init*( result = api.partUntwigGeneric(a, b, c) profApi.partUntwigGenericOk = - proc(a: openArray[Blob]; b: Hash256; c, d: openArray[byte]): auto = + proc(a: openArray[Blob]; b:Hash256; c:openArray[byte]; d:Opt[Blob]): auto = AristoApiProfPartUntwigGenericOkFn.profileRunner: result = api.partUntwigGenericOk(a, b, c, d) @@ -984,7 +991,7 @@ func init*( result = api.partUntwigPath(a, b, c) profApi.partUntwigPathOk = - proc(a: openArray[Blob]; b, c: Hash256; d: openArray[byte]): auto = + proc(a: openArray[Blob]; b, c: Hash256; d: Opt[Blob]): auto = AristoApiProfPartUntwigPathOkFn.profileRunner: result = api.partUntwigPathOk(a, b, c, d) diff --git a/nimbus/db/aristo/aristo_check/check_twig.nim b/nimbus/db/aristo/aristo_check/check_twig.nim index 88e6ece42..998263c71 100644 --- a/nimbus/db/aristo/aristo_check/check_twig.nim +++ b/nimbus/db/aristo/aristo_check/check_twig.nim @@ -27,7 +27,7 @@ proc checkTwig*( let proof = ? db.partGenericTwig(root, path) key = ? db.computeKey (root,root) - pyl = ? proof.partUntwigGeneric(key.to(Hash256), path) + pyl = ? proof[0].partUntwigGeneric(key.to(Hash256), path) ok() @@ -40,7 +40,7 @@ proc checkTwig*( proof = ? db.partStorageTwig(accPath, stoPath) vid = ? db.fetchStorageID accPath key = ? db.computeKey (VertexID(1),vid) - pyl = ? proof.partUntwigPath(key.to(Hash256), stoPath) + pyl = ? proof[0].partUntwigPath(key.to(Hash256), stoPath) ok() diff --git a/nimbus/db/aristo/aristo_desc/desc_error.nim b/nimbus/db/aristo/aristo_desc/desc_error.nim index aa492a0bc..0429fefb6 100644 --- a/nimbus/db/aristo/aristo_desc/desc_error.nim +++ b/nimbus/db/aristo/aristo_desc/desc_error.nim @@ -170,6 +170,7 @@ type PartChkVidTabCoreRootMissing PartChkVidTabVidMissing PartChnBranchPathExhausted + PartChnBranchVoidEdge PartChnExtPfxMismatch PartChnLeafPathMismatch PartChnNodeConvError diff --git a/nimbus/db/aristo/aristo_part.nim b/nimbus/db/aristo/aristo_part.nim index f5fdbd5ef..100f14845 100644 --- a/nimbus/db/aristo/aristo_part.nim +++ b/nimbus/db/aristo/aristo_part.nim @@ -70,19 +70,28 @@ proc partGenericTwig*( db: AristoDbRef; root: VertexID; path: NibblesBuf; - ): Result[seq[Blob], AristoError] = + ): Result[(seq[Blob],bool), AristoError] = ## This function returns a chain of rlp-encoded nodes along the argument - ## path `(root,path)`. + ## path `(root,path)` followed by a `true` value if the `path` argument + ## exists in the database. If the argument `path` is not on the database, + ## a partial path will be returned follwed by a `false` value. + ## + ## Errors will only be returned for invalid paths. ## var chain: seq[Blob] - ? db.chainRlpNodes((root,root), path, chain) - ok chain + let rc = db.chainRlpNodes((root,root), path, chain) + if rc.isOk: + ok((chain, true)) + elif rc.error in ChainRlpNodesNoEntry: + ok((chain, false)) + else: + err(rc.error) proc partGenericTwig*( db: AristoDbRef; root: VertexID; path: openArray[byte]; - ): Result[seq[Blob], AristoError] = + ): Result[(seq[Blob],bool), AristoError] = ## Variant of `partGenericTwig()`. ## ## Note: This function provides a functionality comparable to the @@ -93,7 +102,7 @@ proc partGenericTwig*( proc partAccountTwig*( db: AristoDbRef; accPath: Hash256; - ): Result[seq[Blob], AristoError] = + ): Result[(seq[Blob],bool), AristoError] = ## Variant of `partGenericTwig()`. db.partGenericTwig(VertexID(1), NibblesBuf.fromBytes accPath.data) @@ -101,8 +110,9 @@ proc partStorageTwig*( db: AristoDbRef; accPath: Hash256; stoPath: Hash256; - ): Result[seq[Blob], AristoError] = - ## Variant of `partGenericTwig()`. + ): Result[(seq[Blob],bool), AristoError] = + ## Variant of `partGenericTwig()`. Note that the function always returns an + ## error unless the `accPath` is valid. let vid = ? db.fetchStorageID accPath db.partGenericTwig(vid, NibblesBuf.fromBytes stoPath.data) @@ -112,11 +122,19 @@ proc partUntwigGeneric*( chain: openArray[Blob]; root: Hash256; path: openArray[byte]; - ): Result[Blob,AristoError] = - ## Verify the chain of rlp-encoded nodes and return the payload. + ): Result[Opt[Blob],AristoError] = + ## Verify the chain of rlp-encoded nodes and return the payload. If a + ## `Opt.none()` result is returned then the `path` argument does provably + ## not exist relative to `chain`. try: - let nibbles = NibblesBuf.fromBytes path - return chain.trackRlpNodes(root.to(HashKey), nibbles, start=true) + let + nibbles = NibblesBuf.fromBytes path + rc = chain.trackRlpNodes(root.to(HashKey), nibbles, start=true) + if rc.isOk: + return ok(Opt.some rc.value) + if rc.error in TrackRlpNodesNoEntry: + return ok(Opt.none Blob) + return err(rc.error) except RlpError: return err(PartTrkRlpError) @@ -124,7 +142,7 @@ proc partUntwigPath*( chain: openArray[Blob]; root: Hash256; path: Hash256; - ): Result[Blob,AristoError] = + ): Result[Opt[Blob],AristoError] = ## Variant of `partUntwigGeneric()`. chain.partUntwigGeneric(root, path.data) @@ -133,7 +151,7 @@ proc partUntwigGenericOk*( chain: openArray[Blob]; root: Hash256; path: openArray[byte]; - payload: openArray[byte]; + payload: Opt[Blob]; ): Result[void,AristoError] = ## Verify the argument `chain` of rlp-encoded nodes against the `path` ## and `payload` arguments. @@ -150,7 +168,7 @@ proc partUntwigPathOk*( chain: openArray[Blob]; root: Hash256; path: Hash256; - payload: openArray[byte]; + payload: Opt[Blob]; ): Result[void,AristoError] = ## Variant of `partUntwigGenericOk()`. chain.partUntwigGenericOk(root, path.data, payload) diff --git a/nimbus/db/aristo/aristo_part/part_chain_rlp.nim b/nimbus/db/aristo/aristo_part/part_chain_rlp.nim index d23f71419..9baf63826 100644 --- a/nimbus/db/aristo/aristo_part/part_chain_rlp.nim +++ b/nimbus/db/aristo/aristo_part/part_chain_rlp.nim @@ -15,6 +15,16 @@ import results, ".."/[aristo_desc, aristo_get, aristo_utils, aristo_compute, aristo_serialise] +const + ChainRlpNodesNoEntry* = { + PartChnLeafPathMismatch, PartChnExtPfxMismatch, PartChnBranchVoidEdge} + ## Partial path errors that can be used to proof that a path does + ## not exists. + + TrackRlpNodesNoEntry* = {PartTrkLinkExpected, PartTrkLeafPfxMismatch} + ## This is the opposite of `ChainRlpNodesNoEntry` when verifying that a + ## node does not exist. + # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ @@ -53,6 +63,8 @@ proc chainRlpNodes*( let nibble = path[nChewOff] rest = path.slice(nChewOff+1) + if not vtx.bVid[nibble].isValid: + return err(PartChnBranchVoidEdge) # Recursion! db.chainRlpNodes((rvid.root,vtx.bVid[nibble]), rest, chain) diff --git a/nimbus/db/core_db/base.nim b/nimbus/db/core_db/base.nim index 32b5d8342..932aa3693 100644 --- a/nimbus/db/core_db/base.nim +++ b/nimbus/db/core_db/base.nim @@ -212,7 +212,7 @@ proc verify*( proof: openArray[Blob]; root: Hash256; path: openArray[byte]; - ): CoreDbRc[Blob] = + ): CoreDbRc[Opt[Blob]] = ## This function os the counterpart of any of the `proof()` functions. Given ## the argument chain of rlp-encoded nodes `proof`, this function verifies ## that the chain represents a partial MPT starting with a root node state @@ -243,7 +243,7 @@ proc verifyOk*( proof: openArray[Blob]; root: Hash256; path: openArray[byte]; - payload: openArray[byte]; + payload: Opt[Blob]; ): CoreDbRc[void] = ## Variant of `verify()` which directly checks the argument `payload` ## against what would be the return code in `verify()`. @@ -267,7 +267,7 @@ proc verify*( proof: openArray[Blob]; root: Hash256; path: Hash256; - ): CoreDbRc[Blob] = + ): CoreDbRc[Opt[Blob]] = ## Variant of `verify()`. template mpt: untyped = when db is CoreDbRef: @@ -288,7 +288,7 @@ proc verifyOk*( proof: openArray[Blob]; root: Hash256; path: Hash256; - payload: openArray[byte]; + payload: Opt[Blob]; ): CoreDbRc[void] = ## Variant of `verifyOk()`. template mpt: untyped = @@ -432,9 +432,12 @@ proc getGeneric*( proc proof*( mpt: CoreDbMptRef; key: openArray[byte]; - ): CoreDbRc[seq[Blob]] = + ): CoreDbRc[(seq[Blob],bool)] = ## On the generic MPT, collect the nodes along the `key` interpreted as - ## path. Return these path nodes as a chain of rlp-encoded blobs. + ## path. Return these path nodes as a chain of rlp-encoded blobs followed + ## by a bool value which is `true` if the `key` path exists in the database, + ## and `false` otherwise. In the latter case, the chain of rlp-encoded blobs + ## are the nodes proving that the `key` path does not exist. ## mpt.setTrackNewApi MptProofFn result = block: @@ -547,9 +550,12 @@ proc getAccounts*(ctx: CoreDbCtxRef): CoreDbAccRef = proc proof*( acc: CoreDbAccRef; accPath: Hash256; - ): CoreDbRc[seq[Blob]] = + ): CoreDbRc[(seq[Blob],bool)] = ## On the accounts MPT, collect the nodes along the `accPath` interpreted as - ## path. Return these path nodes as a chain of rlp-encoded blobs. + ## path. Return these path nodes as a chain of rlp-encoded blobs followed + ## by a bool value which is `true` if the `key` path exists in the database, + ## and `false` otherwise. In the latter case, the chain of rlp-encoded blobs + ## are the nodes proving that the `key` path does not exist. ## acc.setTrackNewApi AccProofFn result = block: @@ -671,10 +677,16 @@ proc slotProof*( acc: CoreDbAccRef; accPath: Hash256; stoPath: Hash256; - ): CoreDbRc[seq[Blob]] = + ): CoreDbRc[(seq[Blob],bool)] = ## On the storage MPT related to the argument account `acPath`, collect the ## nodes along the `stoPath` interpreted as path. Return these path nodes as - ## a chain of rlp-encoded blobs. + ## a chain of rlp-encoded blobs followed by a bool value which is `true` if + ## the `key` path exists in the database, and `false` otherwise. In the + ## latter case, the chain of rlp-encoded blobs are the nodes proving that + ## the `key` path does not exist. + ## + ## Note that the function always returns an error unless the `accPath` is + ## valid. ## acc.setTrackNewApi AccSlotProofFn result = block: diff --git a/nimbus/db/core_db/base/api_tracking.nim b/nimbus/db/core_db/base/api_tracking.nim index 638b72c16..904858f4e 100644 --- a/nimbus/db/core_db/base/api_tracking.nim +++ b/nimbus/db/core_db/base/api_tracking.nim @@ -132,6 +132,11 @@ func toStr(rc: CoreDbRc[seq[Blob]]): string = if rc.isOk: "ok([" & rc.value.mapIt("[#" & $it.len & "]").join(",") & "])" else: "err(" & rc.error.toStr & ")" +func toStr(rc: CoreDbRc[(seq[Blob],bool)]): string = + if rc.isOk: "ok([" & rc.value[0].mapIt("[#" & $it.len & "]").join(",") & + "]," & $rc.value[1] & ")" + else: "err(" & rc.error.toStr & ")" + func toStr(rc: CoreDbRc[Hash256]): string = if rc.isOk: "ok(" & rc.value.toStr & ")" else: "err(" & rc.error.toStr & ")" diff --git a/tests/test_aristo/test_portal_proof.nim b/tests/test_aristo/test_portal_proof.nim index 8af4e39b0..d3d027c75 100644 --- a/tests/test_aristo/test_portal_proof.nim +++ b/tests/test_aristo/test_portal_proof.nim @@ -25,6 +25,7 @@ import type ProofData = ref object chain: seq[Blob] + missing: bool error: AristoError hike: Hike @@ -122,6 +123,14 @@ func asExtension(b: Blob; path: Hash256): Blob = else: b +when false: + # just keep for potential debugging + proc sq(s: string): string = + ## For long strings print `begin..end` only + let n = (s.len + 1) div 2 + result = if s.len < 20: s else: s[0 .. 5] & ".." & s[s.len-8 .. ^1] + result &= "[" & (if 0 < n: "#" & $n else: "") & "]" + # ------------------------------------------------------------------------------ # Private test functions # ------------------------------------------------------------------------------ @@ -152,13 +161,33 @@ proc testCreatePortalProof(node: JsonNode, testStatusIMPL: var TestStatus) = # Create proof chains for (path,proof) in sample.pairs: let rc = ps.db.partAccountTwig path - check rc.isOk == (proof.error == AristoError 0) - if rc.isOk: - proof.chain = rc.value + if proof.error == AristoError(0): + check rc.isOk and rc.value[1] == true + proof.chain = rc.value[0] + elif proof.error != HikeBranchMissingEdge: + # Note that this is a partial data base and in this case the proof for a + # non-existing entry might not work properly when the vertex is missing. + check rc.isOk and rc.value[1] == false + proof.chain = rc.value[0] + proof.missing = true # Verify proof chains for (path,proof) in sample.pairs: - if proof.error == AristoError 0: + if proof.missing: + # Proof for missing entries + let + rVid = proof.hike.root + root = ps.db.getKey((rVid,rVid)).to(Hash256) + chain = proof.chain + + block: + let rc = proof.chain.partUntwigPath(root, path) + check rc.isOk and rc.value.isNone + + # Just for completeness (same a above combined into a single function) + check proof.chain.partUntwigPathOk(root, path, Opt.none Blob).isOk + + elif proof.error == AristoError 0: let rVid = proof.hike.root pyl = proof.hike.legs[^1].wp.vtx.lData.payloadAsBlob(ps) @@ -174,9 +203,8 @@ proc testCreatePortalProof(node: JsonNode, testStatusIMPL: var TestStatus) = # Create the same proof again which must result into the same as before block: let rc = pq.db.partAccountTwig path - check rc.isOk - if rc.isOk: - check rc.value == proof.chain + if rc.isOk and rc.value[1] == true: + check rc.value[0] == proof.chain # Verify proof let root = pq.db.getKey((rVid,rVid)).to(Hash256) @@ -184,10 +212,10 @@ proc testCreatePortalProof(node: JsonNode, testStatusIMPL: var TestStatus) = let rc = proof.chain.partUntwigPath(root, path) check rc.isOk if rc.isOk: - check rc.value == pyl + check rc.value == Opt.some(pyl) # Just for completeness (same a above combined into a single function) - check proof.chain.partUntwigPathOk(root, path, pyl).isOk + check proof.chain.partUntwigPathOk(root, path, Opt.some pyl).isOk # Extension nodes are rare, so there is one created, inserted and the # previous test repeated. @@ -204,18 +232,18 @@ proc testCreatePortalProof(node: JsonNode, testStatusIMPL: var TestStatus) = # Re-create proof again block: let rc = pq.db.partAccountTwig path - check rc.isOk - if rc.isOk: - check rc.value == chain + check rc.isOk and rc.value[1] == true + if rc.isOk and rc.value[1] == true: + check rc.value[0] == chain let root = pq.db.getKey((rVid,rVid)).to(Hash256) block: let rc = chain.partUntwigPath(root, path) check rc.isOk if rc.isOk: - check rc.value == pyl + check rc.value == Opt.some(pyl) - check chain.partUntwigPathOk(root, path, pyl).isOk + check chain.partUntwigPathOk(root, path, Opt.some pyl).isOk # ------------------------------------------------------------------------------ # Test