status-im
/
nim-eth
mirror of https://github.com/status-im/nim-eth.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

In the incomplete-db node-existence check, don't use contains. (#603) 

* In the incomplete-db node-existence check, don't use contains.

(Using contains led to a problem with CaptureDB.)

* In the incomplete-db check, just checking len > 0 isn't right.

* Oh, I needed the AssertionDefect thing too.

* Need this when compiling under older versions of Nim.

* Sometimes we want missing nodes to be errors, sometimes not.
This commit is contained in:
Adam Spitz 2023-04-28 12:04:33 -04:00 committed by GitHub
parent f5dd26eac0
commit 4b818e8307
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 47 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
eth/trie/hexary.nim
View File

@ -15,6 +15,7 @@ type
db*: DB db*: DB
root: TrieNodeKey root: TrieNodeKey
isPruning: bool isPruning: bool
shouldMissingNodesBeErrors: bool
SecureHexaryTrie* = distinct HexaryTrie SecureHexaryTrie* = distinct HexaryTrie
@ -31,14 +32,15 @@ proc expectHash(r: Rlp): seq[byte] =
raise newException(RlpTypeMismatch, raise newException(RlpTypeMismatch,
"RLP expected to be a Keccak hash value, but has an incorrect length") "RLP expected to be a Keccak hash value, but has an incorrect length")
type MissingNodeError* = ref object of Defect when (NimMajor, NimMinor, NimPatch) < (1, 4, 0):
type AssertionDefect = AssertionError
type MissingNodeError* = ref object of AssertionDefect
path*: NibblesSeq path*: NibblesSeq
nodeHashBytes*: seq[byte] nodeHashBytes*: seq[byte]
proc dbGet(db: DB, data: openArray[byte]): seq[byte] proc dbGet(db: DB, data: openArray[byte]): seq[byte]
{.gcsafe, raises: [Defect].} = {.gcsafe, raises: [Defect].} =
# Useful for debugging:
# doAssert(db.contains(data), "dbGet, db must contain the data")
db.get(data) db.get(data)
proc dbGet(db: DB, key: Rlp): seq[byte] = proc dbGet(db: DB, key: Rlp): seq[byte] =
@ -52,41 +54,39 @@ proc dbPut(db: DB, data: openArray[byte]): TrieNodeKey
# the missing node. So here we need the path to be passed in, and if the # the missing node. So here we need the path to be passed in, and if the
# node is missing we'll raise an exception to get that information up to # node is missing we'll raise an exception to get that information up to
# where it's needed. # where it's needed.
proc getPossiblyMissingNode(db: DB, data: openArray[byte], fullPath: NibblesSeq, pathIndex: int): seq[byte] proc getPossiblyMissingNode(db: DB, data: openArray[byte], fullPath: NibblesSeq, pathIndex: int, errorIfMissing: bool): seq[byte]
{.gcsafe, raises: [Defect].} = {.gcsafe, raises: [Defect].} =
# FIXME-Adam: This causes some tests to fail in nimbus-eth1; I'm not let nodeBytes = db.get(data) # need to call this before the call to contains, otherwise CaptureDB complains
# sure why. I need to figure it out, though, because we need this if nodeBytes.len > 0 or not errorIfMissing:
# behaviour. nodeBytes
# else:
# if db.contains(data): raise MissingNodeError(path: fullPath.slice(0, pathIndex), nodeHashBytes: @data)
# db.get(data)
# else:
# raise MissingNodeError(path: fullPath.slice(0, pathIndex), nodeHashBytes: @data)
db.get(data)
proc getPossiblyMissingNode(db: DB, key: Rlp, fullPath: NibblesSeq, pathIndex: int): seq[byte] = proc getPossiblyMissingNode(db: DB, key: Rlp, fullPath: NibblesSeq, pathIndex: int, errorIfMissing: bool): seq[byte] =
getPossiblyMissingNode(db, key.expectHash, fullPath, pathIndex) getPossiblyMissingNode(db, key.expectHash, fullPath, pathIndex, errorIfMissing)
converter toTrieNodeKey(hash: KeccakHash): TrieNodeKey = converter toTrieNodeKey(hash: KeccakHash): TrieNodeKey =
result.hash = hash result.hash = hash
result.usedBytes = 32 result.usedBytes = 32
proc initHexaryTrie*(db: DB, rootHash: KeccakHash, isPruning = true): HexaryTrie = proc initHexaryTrie*(db: DB, rootHash: KeccakHash, isPruning = true, shouldMissingNodesBeErrors = false): HexaryTrie =
result.db = db result.db = db
result.root = rootHash result.root = rootHash
result.isPruning = isPruning result.isPruning = isPruning
result.shouldMissingNodesBeErrors = shouldMissingNodesBeErrors
template initSecureHexaryTrie*(db: DB, rootHash: KeccakHash, isPruning = true): SecureHexaryTrie = template initSecureHexaryTrie*(db: DB, rootHash: KeccakHash, isPruning = true, shouldMissingNodesBeErrors = false): SecureHexaryTrie =
SecureHexaryTrie initHexaryTrie(db, rootHash, isPruning) SecureHexaryTrie initHexaryTrie(db, rootHash, isPruning, shouldMissingNodesBeErrors)
proc initHexaryTrie*(db: DB, isPruning = true): HexaryTrie proc initHexaryTrie*(db: DB, isPruning = true, shouldMissingNodesBeErrors = false): HexaryTrie
{.raises: [Defect].} = {.raises: [Defect].} =
result.db = db result.db = db
result.root = result.db.dbPut(emptyRlp) result.root = result.db.dbPut(emptyRlp)
result.isPruning = isPruning result.isPruning = isPruning
result.shouldMissingNodesBeErrors = shouldMissingNodesBeErrors
template initSecureHexaryTrie*(db: DB, isPruning = true): SecureHexaryTrie = template initSecureHexaryTrie*(db: DB, isPruning = true, shouldMissingNodesBeErrors = false): SecureHexaryTrie =
SecureHexaryTrie initHexaryTrie(db, isPruning) SecureHexaryTrie initHexaryTrie(db, isPruning, shouldMissingNodesBeErrors)
proc rootHash*(t: HexaryTrie): KeccakHash = proc rootHash*(t: HexaryTrie): KeccakHash =
t.root.hash t.root.hash
@ -113,11 +113,11 @@ template keyToLocalBytes(db: DB, k: TrieNodeKey): seq[byte] =
template extensionNodeKey(r: Rlp): auto = template extensionNodeKey(r: Rlp): auto =
hexPrefixDecode r.listElem(0).toBytes hexPrefixDecode r.listElem(0).toBytes
proc getLookup(db: DB, elem: Rlp, fullPath: NibblesSeq, pathIndex: int): Rlp = proc getLookup(db: DB, elem: Rlp, fullPath: NibblesSeq, pathIndex: int, errorIfMissing: bool): Rlp =
if elem.isList: elem if elem.isList: elem
else: rlpFromBytes(getPossiblyMissingNode(db, elem.expectHash, fullPath, pathIndex)) else: rlpFromBytes(getPossiblyMissingNode(db, elem.expectHash, fullPath, pathIndex, errorIfMissing))
proc getAux(db: DB, nodeRlp: Rlp, fullPath: NibblesSeq, pathIndex: int): seq[byte] proc getAux(db: DB, nodeRlp: Rlp, fullPath: NibblesSeq, pathIndex: int, errorIfMissing: bool): seq[byte]
{.gcsafe, raises: [RlpError, Defect].} = {.gcsafe, raises: [RlpError, Defect].} =
if not nodeRlp.hasData or nodeRlp.isEmpty: if not nodeRlp.hasData or nodeRlp.isEmpty:
return return
@ -133,8 +133,8 @@ proc getAux(db: DB, nodeRlp: Rlp, fullPath: NibblesSeq, pathIndex: int): seq[byt
if sharedNibbles == path.len and isLeaf: if sharedNibbles == path.len and isLeaf:
return value.toBytes return value.toBytes
elif not isLeaf: elif not isLeaf:
let nextLookup = getLookup(db, value, fullPath, pathIndex + sharedNibbles) let nextLookup = getLookup(db, value, fullPath, pathIndex + sharedNibbles, errorIfMissing)
return getAux(db, nextLookup, fullPath, pathIndex + sharedNibbles) return getAux(db, nextLookup, fullPath, pathIndex + sharedNibbles, errorIfMissing)
return return
of 17: of 17:
@ -144,20 +144,17 @@ proc getAux(db: DB, nodeRlp: Rlp, fullPath: NibblesSeq, pathIndex: int): seq[byt
if branch.isEmpty: if branch.isEmpty:
return return
else: else:
let nextLookup = getLookup(db, branch, fullPath, pathIndex + 1) let nextLookup = getLookup(db, branch, fullPath, pathIndex + 1, errorIfMissing)
return getAux(db, nextLookup, fullPath, pathIndex + 1) return getAux(db, nextLookup, fullPath, pathIndex + 1, errorIfMissing)
else: else:
raise newException(CorruptedTrieDatabase, raise newException(CorruptedTrieDatabase,
"HexaryTrie node with an unexpected number of children") "HexaryTrie node with an unexpected number of children")
proc getAuxByHash(db: DB, node: TrieNodeKey, fullPath: NibblesSeq, pathIndex: int): seq[byte] =
var nodeRlp = rlpFromBytes keyToLocalBytes(db, node)
return getAux(db, nodeRlp, fullPath, pathIndex)
proc get*(self: HexaryTrie; key: openArray[byte]): seq[byte] = proc get*(self: HexaryTrie; key: openArray[byte]): seq[byte] =
return getAuxByHash(self.db, self.root, initNibbleRange(key), 0) var nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root)
return getAux(self.db, nodeRlp, initNibbleRange(key), 0, self.shouldMissingNodesBeErrors)
proc getKeysAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]]): seq[byte] = proc getKeysAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]], errorIfMissing: bool): seq[byte] =
while stack.len > 0: while stack.len > 0:
let (nodeRlp, path) = stack.pop() let (nodeRlp, path) = stack.pop()
if not nodeRlp.hasData or nodeRlp.isEmpty: if not nodeRlp.hasData or nodeRlp.isEmpty:
@ -175,7 +172,7 @@ proc getKeysAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]]):
else: else:
let let
value = nodeRlp.listElem(1) value = nodeRlp.listElem(1)
nextLookup = getLookup(db, value, key, key.len) nextLookup = getLookup(db, value, key, key.len, errorIfMissing)
stack.add((nextLookup, key)) stack.add((nextLookup, key))
of 17: of 17:
for i in 0 ..< 16: for i in 0 ..< 16:
@ -183,7 +180,7 @@ proc getKeysAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]]):
if not branch.isEmpty: if not branch.isEmpty:
var key = path.cloneAndReserveNibble() var key = path.cloneAndReserveNibble()
key.replaceLastNibble(i.byte) key.replaceLastNibble(i.byte)
let nextLookup = getLookup(db, branch, key, key.len) let nextLookup = getLookup(db, branch, key, key.len, errorIfMissing)
stack.add((nextLookup, key)) stack.add((nextLookup, key))
var lastElem = nodeRlp.listElem(16) var lastElem = nodeRlp.listElem(16)
@ -199,9 +196,9 @@ iterator keys*(self: HexaryTrie): seq[byte] =
nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root) nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root)
stack = @[(nodeRlp, initNibbleRange([]))] stack = @[(nodeRlp, initNibbleRange([]))]
while stack.len > 0: while stack.len > 0:
yield getKeysAux(self.db, stack) yield getKeysAux(self.db, stack, self.shouldMissingNodesBeErrors)
proc getValuesAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]]): seq[byte] = proc getValuesAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]], errorIfMissing: bool): seq[byte] =
while stack.len > 0: while stack.len > 0:
let (nodeRlp, path) = stack.pop() let (nodeRlp, path) = stack.pop()
if not nodeRlp.hasData or nodeRlp.isEmpty: if not nodeRlp.hasData or nodeRlp.isEmpty:
@ -218,7 +215,7 @@ proc getValuesAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]])
doAssert(key.len mod 2 == 0) doAssert(key.len mod 2 == 0)
return value.toBytes return value.toBytes
else: else:
let nextLookup = getLookup(db, value, key, key.len) let nextLookup = getLookup(db, value, key, key.len, errorIfMissing)
stack.add((nextLookup, key)) stack.add((nextLookup, key))
of 17: of 17:
for i in 0 ..< 16: for i in 0 ..< 16:
@ -226,7 +223,7 @@ proc getValuesAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]])
if not branch.isEmpty: if not branch.isEmpty:
var key = path.cloneAndReserveNibble() var key = path.cloneAndReserveNibble()
key.replaceLastNibble(i.byte) key.replaceLastNibble(i.byte)
let nextLookup = getLookup(db, branch, key, key.len) let nextLookup = getLookup(db, branch, key, key.len, errorIfMissing)
stack.add((nextLookup, key)) stack.add((nextLookup, key))
var lastElem = nodeRlp.listElem(16) var lastElem = nodeRlp.listElem(16)
@ -241,9 +238,9 @@ iterator values*(self: HexaryTrie): seq[byte] =
nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root) nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root)
stack = @[(nodeRlp, initNibbleRange([]))] stack = @[(nodeRlp, initNibbleRange([]))]
while stack.len > 0: while stack.len > 0:
yield getValuesAux(self.db, stack) yield getValuesAux(self.db, stack, self.shouldMissingNodesBeErrors)
proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]]): (seq[byte], seq[byte]) = proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]], errorIfMissing: bool): (seq[byte], seq[byte]) =
while stack.len > 0: while stack.len > 0:
let (nodeRlp, path) = stack.pop() let (nodeRlp, path) = stack.pop()
if not nodeRlp.hasData or nodeRlp.isEmpty: if not nodeRlp.hasData or nodeRlp.isEmpty:
@ -260,7 +257,7 @@ proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]]):
doAssert(key.len mod 2 == 0) doAssert(key.len mod 2 == 0)
return (key.getBytes, value.toBytes) return (key.getBytes, value.toBytes)
else: else:
let nextLookup = getLookup(db, value, key, key.len) let nextLookup = getLookup(db, value, key, key.len, errorIfMissing)
stack.add((nextLookup, key)) stack.add((nextLookup, key))
of 17: of 17:
for i in 0 ..< 16: for i in 0 ..< 16:
@ -268,7 +265,7 @@ proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]]):
if not branch.isEmpty: if not branch.isEmpty:
var key = path.cloneAndReserveNibble() var key = path.cloneAndReserveNibble()
key.replaceLastNibble(i.byte) key.replaceLastNibble(i.byte)
let nextLookup = getLookup(db, branch, key, key.len) let nextLookup = getLookup(db, branch, key, key.len, errorIfMissing)
stack.add((nextLookup, key)) stack.add((nextLookup, key))
var lastElem = nodeRlp.listElem(16) var lastElem = nodeRlp.listElem(16)
@ -287,7 +284,7 @@ iterator pairs*(self: HexaryTrie): (seq[byte], seq[byte]) =
# perhaps a Nim bug #9778 # perhaps a Nim bug #9778
# cannot yield the helper proc directly # cannot yield the helper proc directly
# it will cut the yield in half # it will cut the yield in half
let res = getPairsAux(self.db, stack) let res = getPairsAux(self.db, stack, self.shouldMissingNodesBeErrors)
yield res yield res
iterator replicate*(self: HexaryTrie): (seq[byte], seq[byte]) = iterator replicate*(self: HexaryTrie): (seq[byte], seq[byte]) =
@ -422,7 +419,7 @@ proc replaceValue(data: Rlp, key: NibblesSeq, value: openArray[byte]): seq[byte]
proc isTwoItemNode(self: HexaryTrie; r: Rlp, fullPath: NibblesSeq, pathIndex: int): bool = proc isTwoItemNode(self: HexaryTrie; r: Rlp, fullPath: NibblesSeq, pathIndex: int): bool =
if r.isBlob: if r.isBlob:
let resolved = getPossiblyMissingNode(self.db, r, fullPath, pathIndex) let resolved = getPossiblyMissingNode(self.db, r, fullPath, pathIndex, self.shouldMissingNodesBeErrors)
let rlp = rlpFromBytes(resolved) let rlp = rlpFromBytes(resolved)
return rlp.isList and rlp.listLen == 2 return rlp.isList and rlp.listLen == 2
else: else:
@ -456,7 +453,7 @@ proc deleteAux(self: var HexaryTrie;
return false return false
var toDelete = if origRlp.isList: origRlp var toDelete = if origRlp.isList: origRlp
else: rlpFromBytes getPossiblyMissingNode(self.db, origRlp, fullPath, pathIndex) else: rlpFromBytes getPossiblyMissingNode(self.db, origRlp, fullPath, pathIndex, self.shouldMissingNodesBeErrors)
let b = self.deleteAt(toDelete, fullPath, pathIndex) let b = self.deleteAt(toDelete, fullPath, pathIndex)
@ -473,7 +470,7 @@ proc graft(self: var HexaryTrie; r: Rlp, fullPath: NibblesSeq, pathIndexToThePar
if not value.isList: if not value.isList:
let nodeKey = value.expectHash let nodeKey = value.expectHash
var resolvedData = getPossiblyMissingNode(self.db, nodeKey, fullPath, pathIndexToTheParent + origPath.len) var resolvedData = getPossiblyMissingNode(self.db, nodeKey, fullPath, pathIndexToTheParent + origPath.len, self.shouldMissingNodesBeErrors)
self.prune(nodeKey) self.prune(nodeKey)
value = rlpFromBytes resolvedData value = rlpFromBytes resolvedData
@ -610,7 +607,7 @@ proc mergeAtAux(self: var HexaryTrie, output: var RlpWriter, orig: Rlp,
var resolved = orig var resolved = orig
var isRemovable = false var isRemovable = false
if not (orig.isList or orig.isEmpty): if not (orig.isList or orig.isEmpty):
resolved = rlpFromBytes getPossiblyMissingNode(self.db, orig, fullPath, pathIndex) resolved = rlpFromBytes getPossiblyMissingNode(self.db, orig, fullPath, pathIndex, self.shouldMissingNodesBeErrors)
isRemovable = true isRemovable = true
let b = self.mergeAt(resolved, fullPath, pathIndex, value, not isRemovable) let b = self.mergeAt(resolved, fullPath, pathIndex, value, not isRemovable)
@ -706,7 +703,7 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
proc put*(self: var HexaryTrie; key, value: openArray[byte]) = proc put*(self: var HexaryTrie; key, value: openArray[byte]) =
let root = self.root.hash let root = self.root.hash
var rootBytes = getPossiblyMissingNode(self.db, root.data, NibblesSeq(), 0) var rootBytes = getPossiblyMissingNode(self.db, root.data, NibblesSeq(), 0, self.shouldMissingNodesBeErrors)
doAssert rootBytes.len > 0 doAssert rootBytes.len > 0
let newRootBytes = self.mergeAt(rlpFromBytes(rootBytes), root, let newRootBytes = self.mergeAt(rlpFromBytes(rootBytes), root,