status-im
/
nim-eth
mirror of https://github.com/status-im/nim-eth.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Revert missing hexary trie node work (#686) 

* Revert "In the incomplete-db node-existence check, don't use contains. (#603)"

This reverts commit 4b818e8307.

* Revert "Some changes to make hexary.nim better able to handle incomplete DBs (#602)"

This reverts commit f5dd26eac0.

* Revert "Added maybeGet, for working with incomplete DBs. (#595)"

This reverts commit 4754543605.
This commit is contained in:
Jacek Sieka 2024-05-28 10:05:57 +02:00 committed by GitHub
parent 258798873c
commit 50b3d0193c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 97 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
eth/trie/db.nim
View File

@ -9,7 +9,7 @@
{.push raises: [].} {.push raises: [].}
import import
std/[options, tables, hashes, sets], std/[tables, hashes, sets],
"."/[trie_defs, db_tracing] "."/[trie_defs, db_tracing]
type type
@ -242,15 +242,6 @@ proc contains*(db: TrieDatabaseRef, key: openArray[byte]): bool =
if db.containsProc != nil: if db.containsProc != nil:
result = db.containsProc(db.obj, key) result = db.containsProc(db.obj, key)
proc maybeGet*(db: TrieDatabaseRef, key: openArray[byte]): Option[seq[byte]] =
# FIXME-Adam: Could duplicate the structure of get, but for now let's just
# do this (I still don't know whether this overall approach makes any sense
# at all.)
if db.contains(key):
some(db.get(key))
else:
none[seq[byte]]()
# TransactionID imitate subset of JournalDB behaviour # TransactionID imitate subset of JournalDB behaviour
# but there is no need to rollback or dispose # but there is no need to rollback or dispose
# TransactionID, because it will be handled elsewhere # TransactionID, because it will be handled elsewhere

341
eth/trie/hexary.nim
View File

@ -6,7 +6,7 @@
# at your option. This file may not be copied, modified, or distributed except according to those terms. # at your option. This file may not be copied, modified, or distributed except according to those terms.
import import
std/[options, tables], std/tables,
nimcrypto/[keccak, hash], nimcrypto/[keccak, hash],
../rlp, ../rlp,
"."/[trie_defs, nibbles, db] "."/[trie_defs, nibbles, db]
@ -22,7 +22,6 @@ type
db*: DB db*: DB
root: TrieNodeKey root: TrieNodeKey
isPruning: bool isPruning: bool
shouldMissingNodesBeErrors: bool
SecureHexaryTrie* = distinct HexaryTrie SecureHexaryTrie* = distinct HexaryTrie
@ -39,58 +38,32 @@ proc expectHash(r: Rlp): seq[byte] =
raise newException(RlpTypeMismatch, raise newException(RlpTypeMismatch,
"RLP expected to be a Keccak hash value, but has an incorrect length") "RLP expected to be a Keccak hash value, but has an incorrect length")
type MissingNodeError* = ref object of AssertionDefect
path*: NibblesSeq
nodeHashBytes*: seq[byte]
proc dbGet(db: DB, data: openArray[byte]): seq[byte]
{.gcsafe, raises: [].} =
db.get(data)
proc dbGet(db: DB, key: Rlp): seq[byte] =
dbGet(db, key.expectHash)
proc dbPut(db: DB, data: openArray[byte]): TrieNodeKey proc dbPut(db: DB, data: openArray[byte]): TrieNodeKey
{.gcsafe, raises: [].} {.gcsafe, raises: [].}
# For stateless mode, it's possible for nodes to be missing from the DB, template get(db: DB, key: Rlp): seq[byte] =
# and we need the higher-level code to be able to find out the *path* to db.get(key.expectHash)
# the missing node. So here we need the path to be passed in, and if the
# node is missing we'll raise an exception to get that information up to
# where it's needed.
proc getPossiblyMissingNode(db: DB, data: openArray[byte], fullPath: NibblesSeq, pathIndex: int, errorIfMissing: bool): seq[byte]
{.gcsafe, raises: [].} =
let nodeBytes = db.get(data) # need to call this before the call to contains, otherwise CaptureDB complains
if nodeBytes.len > 0 or not errorIfMissing:
nodeBytes
else:
raise MissingNodeError(path: fullPath.slice(0, pathIndex), nodeHashBytes: @data)
proc getPossiblyMissingNode(db: DB, key: Rlp, fullPath: NibblesSeq, pathIndex: int, errorIfMissing: bool): seq[byte] =
getPossiblyMissingNode(db, key.expectHash, fullPath, pathIndex, errorIfMissing)
converter toTrieNodeKey(hash: KeccakHash): TrieNodeKey = converter toTrieNodeKey(hash: KeccakHash): TrieNodeKey =
result.hash = hash result.hash = hash
result.usedBytes = 32 result.usedBytes = 32
proc initHexaryTrie*(db: DB, rootHash: KeccakHash, isPruning = true, shouldMissingNodesBeErrors = false): HexaryTrie = proc initHexaryTrie*(db: DB, rootHash: KeccakHash, isPruning = true): HexaryTrie =
result.db = db result.db = db
result.root = rootHash result.root = rootHash
result.isPruning = isPruning result.isPruning = isPruning
result.shouldMissingNodesBeErrors = shouldMissingNodesBeErrors
template initSecureHexaryTrie*(db: DB, rootHash: KeccakHash, isPruning = true, shouldMissingNodesBeErrors = false): SecureHexaryTrie = template initSecureHexaryTrie*(db: DB, rootHash: KeccakHash, isPruning = true): SecureHexaryTrie =
SecureHexaryTrie initHexaryTrie(db, rootHash, isPruning, shouldMissingNodesBeErrors) SecureHexaryTrie initHexaryTrie(db, rootHash, isPruning)
proc initHexaryTrie*(db: DB, isPruning = true, shouldMissingNodesBeErrors = false): HexaryTrie proc initHexaryTrie*(db: DB, isPruning = true): HexaryTrie
{.raises: [].} = {.raises: [].} =
result.db = db result.db = db
result.root = result.db.dbPut(emptyRlp) result.root = result.db.dbPut(emptyRlp)
result.isPruning = isPruning result.isPruning = isPruning
result.shouldMissingNodesBeErrors = shouldMissingNodesBeErrors
template initSecureHexaryTrie*(db: DB, isPruning = true, shouldMissingNodesBeErrors = false): SecureHexaryTrie = template initSecureHexaryTrie*(db: DB, isPruning = true): SecureHexaryTrie =
SecureHexaryTrie initHexaryTrie(db, isPruning, shouldMissingNodesBeErrors) SecureHexaryTrie initHexaryTrie(db, isPruning)
proc rootHash*(t: HexaryTrie): KeccakHash = proc rootHash*(t: HexaryTrie): KeccakHash =
t.root.hash t.root.hash
@ -112,21 +85,27 @@ proc getLocalBytes(x: TrieNodeKey): seq[byte] =
template keyToLocalBytes(db: DB, k: TrieNodeKey): seq[byte] = template keyToLocalBytes(db: DB, k: TrieNodeKey): seq[byte] =
if k.len < 32: k.getLocalBytes if k.len < 32: k.getLocalBytes
else: dbGet(db, k.asDbKey) else: db.get(k.asDbKey)
template extensionNodeKey(r: Rlp): auto = template extensionNodeKey(r: Rlp): auto =
hexPrefixDecode r.listElem(0).toBytes hexPrefixDecode r.listElem(0).toBytes
proc getLookup(db: DB, elem: Rlp, fullPath: NibblesSeq, pathIndex: int, errorIfMissing: bool): Rlp = proc getAux(db: DB, nodeRlp: Rlp, path: NibblesSeq): seq[byte]
if elem.isList: elem {.gcsafe, raises: [RlpError].}
else: rlpFromBytes(getPossiblyMissingNode(db, elem.expectHash, fullPath, pathIndex, errorIfMissing))
proc getAux(db: DB, nodeRlp: Rlp, fullPath: NibblesSeq, pathIndex: int, errorIfMissing: bool): seq[byte] proc getAuxByHash(db: DB, node: TrieNodeKey, path: NibblesSeq): seq[byte] =
var nodeRlp = rlpFromBytes keyToLocalBytes(db, node)
return getAux(db, nodeRlp, path)
template getLookup(elem: untyped): untyped =
if elem.isList: elem
else: rlpFromBytes(get(db, elem.expectHash))
proc getAux(db: DB, nodeRlp: Rlp, path: NibblesSeq): seq[byte]
{.gcsafe, raises: [RlpError].} = {.gcsafe, raises: [RlpError].} =
if not nodeRlp.hasData or nodeRlp.isEmpty: if not nodeRlp.hasData or nodeRlp.isEmpty:
return return
let path = fullPath.slice(pathIndex)
case nodeRlp.listLen case nodeRlp.listLen
of 2: of 2:
let (isLeaf, k) = nodeRlp.extensionNodeKey let (isLeaf, k) = nodeRlp.extensionNodeKey
@ -137,8 +116,8 @@ proc getAux(db: DB, nodeRlp: Rlp, fullPath: NibblesSeq, pathIndex: int, errorIfM
if sharedNibbles == path.len and isLeaf: if sharedNibbles == path.len and isLeaf:
return value.toBytes return value.toBytes
elif not isLeaf: elif not isLeaf:
let nextLookup = getLookup(db, value, fullPath, pathIndex + sharedNibbles, errorIfMissing) let nextLookup = value.getLookup
return getAux(db, nextLookup, fullPath, pathIndex + sharedNibbles, errorIfMissing) return getAux(db, nextLookup, path.slice(sharedNibbles))
return return
of 17: of 17:
@ -148,17 +127,16 @@ proc getAux(db: DB, nodeRlp: Rlp, fullPath: NibblesSeq, pathIndex: int, errorIfM
if branch.isEmpty: if branch.isEmpty:
return return
else: else:
let nextLookup = getLookup(db, branch, fullPath, pathIndex + 1, errorIfMissing) let nextLookup = branch.getLookup
return getAux(db, nextLookup, fullPath, pathIndex + 1, errorIfMissing) return getAux(db, nextLookup, path.slice(1))
else: else:
raise newException(CorruptedTrieDatabase, raise newException(CorruptedTrieDatabase,
"HexaryTrie node with an unexpected number of children") "HexaryTrie node with an unexpected number of children")
proc get*(self: HexaryTrie; key: openArray[byte]): seq[byte] = proc get*(self: HexaryTrie; key: openArray[byte]): seq[byte] =
var nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root) return getAuxByHash(self.db, self.root, initNibbleRange(key))
return getAux(self.db, nodeRlp, initNibbleRange(key), 0, self.shouldMissingNodesBeErrors)
proc getKeysAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]], errorIfMissing: bool): seq[byte] = proc getKeysAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]]): seq[byte] =
while stack.len > 0: while stack.len > 0:
let (nodeRlp, path) = stack.pop() let (nodeRlp, path) = stack.pop()
if not nodeRlp.hasData or nodeRlp.isEmpty: if not nodeRlp.hasData or nodeRlp.isEmpty:
@ -176,15 +154,15 @@ proc getKeysAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]], e
else: else:
let let
value = nodeRlp.listElem(1) value = nodeRlp.listElem(1)
nextLookup = getLookup(db, value, key, key.len, errorIfMissing) nextLookup = value.getLookup
stack.add((nextLookup, key)) stack.add((nextLookup, key))
of 17: of 17:
for i in 0 ..< 16: for i in 0 ..< 16:
var branch = nodeRlp.listElem(i) var branch = nodeRlp.listElem(i)
if not branch.isEmpty: if not branch.isEmpty:
let nextLookup = branch.getLookup
var key = path.cloneAndReserveNibble() var key = path.cloneAndReserveNibble()
key.replaceLastNibble(i.byte) key.replaceLastNibble(i.byte)
let nextLookup = getLookup(db, branch, key, key.len, errorIfMissing)
stack.add((nextLookup, key)) stack.add((nextLookup, key))
var lastElem = nodeRlp.listElem(16) var lastElem = nodeRlp.listElem(16)
@ -200,35 +178,31 @@ iterator keys*(self: HexaryTrie): seq[byte] =
nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root) nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root)
stack = @[(nodeRlp, initNibbleRange([]))] stack = @[(nodeRlp, initNibbleRange([]))]
while stack.len > 0: while stack.len > 0:
yield getKeysAux(self.db, stack, self.shouldMissingNodesBeErrors) yield getKeysAux(self.db, stack)
proc getValuesAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]], errorIfMissing: bool): seq[byte] = proc getValuesAux(db: DB, stack: var seq[Rlp]): seq[byte] =
while stack.len > 0: while stack.len > 0:
let (nodeRlp, path) = stack.pop() let nodeRlp = stack.pop()
if not nodeRlp.hasData or nodeRlp.isEmpty: if not nodeRlp.hasData or nodeRlp.isEmpty:
continue continue
case nodeRlp.listLen case nodeRlp.listLen
of 2: of 2:
let let
(isLeaf, k) = nodeRlp.extensionNodeKey (isLeaf, _) = nodeRlp.extensionNodeKey
key = path & k
value = nodeRlp.listElem(1) value = nodeRlp.listElem(1)
if isLeaf: if isLeaf:
doAssert(key.len mod 2 == 0)
return value.toBytes return value.toBytes
else: else:
let nextLookup = getLookup(db, value, key, key.len, errorIfMissing) let nextLookup = value.getLookup
stack.add((nextLookup, key)) stack.add(nextLookup)
of 17: of 17:
for i in 0 ..< 16: for i in 0 ..< 16:
var branch = nodeRlp.listElem(i) var branch = nodeRlp.listElem(i)
if not branch.isEmpty: if not branch.isEmpty:
var key = path.cloneAndReserveNibble() let nextLookup = branch.getLookup
key.replaceLastNibble(i.byte) stack.add(nextLookup)
let nextLookup = getLookup(db, branch, key, key.len, errorIfMissing)
stack.add((nextLookup, key))
var lastElem = nodeRlp.listElem(16) var lastElem = nodeRlp.listElem(16)
if not lastElem.isEmpty: if not lastElem.isEmpty:
@ -240,11 +214,11 @@ proc getValuesAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]],
iterator values*(self: HexaryTrie): seq[byte] = iterator values*(self: HexaryTrie): seq[byte] =
var var
nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root) nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root)
stack = @[(nodeRlp, initNibbleRange([]))] stack = @[nodeRlp]
while stack.len > 0: while stack.len > 0:
yield getValuesAux(self.db, stack, self.shouldMissingNodesBeErrors) yield getValuesAux(self.db, stack)
proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]], errorIfMissing: bool): (seq[byte], seq[byte]) = proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]]): (seq[byte], seq[byte]) =
while stack.len > 0: while stack.len > 0:
let (nodeRlp, path) = stack.pop() let (nodeRlp, path) = stack.pop()
if not nodeRlp.hasData or nodeRlp.isEmpty: if not nodeRlp.hasData or nodeRlp.isEmpty:
@ -261,15 +235,15 @@ proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]],
doAssert(key.len mod 2 == 0) doAssert(key.len mod 2 == 0)
return (key.getBytes, value.toBytes) return (key.getBytes, value.toBytes)
else: else:
let nextLookup = getLookup(db, value, key, key.len, errorIfMissing) let nextLookup = value.getLookup
stack.add((nextLookup, key)) stack.add((nextLookup, key))
of 17: of 17:
for i in 0 ..< 16: for i in 0 ..< 16:
var branch = nodeRlp.listElem(i) var branch = nodeRlp.listElem(i)
if not branch.isEmpty: if not branch.isEmpty:
let nextLookup = branch.getLookup
var key = path.cloneAndReserveNibble() var key = path.cloneAndReserveNibble()
key.replaceLastNibble(i.byte) key.replaceLastNibble(i.byte)
let nextLookup = getLookup(db, branch, key, key.len, errorIfMissing)
stack.add((nextLookup, key)) stack.add((nextLookup, key))
var lastElem = nodeRlp.listElem(16) var lastElem = nodeRlp.listElem(16)
@ -288,7 +262,7 @@ iterator pairs*(self: HexaryTrie): (seq[byte], seq[byte]) =
# perhaps a Nim bug #9778 # perhaps a Nim bug #9778
# cannot yield the helper proc directly # cannot yield the helper proc directly
# it will cut the yield in half # it will cut the yield in half
let res = getPairsAux(self.db, stack, self.shouldMissingNodesBeErrors) let res = getPairsAux(self.db, stack)
yield res yield res
iterator replicate*(self: HexaryTrie): (seq[byte], seq[byte]) = iterator replicate*(self: HexaryTrie): (seq[byte], seq[byte]) =
@ -344,15 +318,14 @@ proc getKeys*(self: HexaryTrie): seq[seq[byte]] =
for k in self.keys: for k in self.keys:
result.add k result.add k
template getNode(db: DB, elem: Rlp): untyped = template getNode(elem: untyped): untyped =
if elem.isList: @(elem.rawData) if elem.isList: @(elem.rawData)
else: dbGet(db, elem.expectHash) else: get(db, elem.expectHash)
proc getBranchAux(db: DB, node: openArray[byte], fullPath: NibblesSeq, pathIndex: int, output: var seq[seq[byte]]) = proc getBranchAux(db: DB, node: openArray[byte], path: NibblesSeq, output: var seq[seq[byte]]) =
var nodeRlp = rlpFromBytes node var nodeRlp = rlpFromBytes node
if not nodeRlp.hasData or nodeRlp.isEmpty: return if not nodeRlp.hasData or nodeRlp.isEmpty: return
let path = fullPath.slice(pathIndex)
case nodeRlp.listLen case nodeRlp.listLen
of 2: of 2:
let (isLeaf, k) = nodeRlp.extensionNodeKey let (isLeaf, k) = nodeRlp.extensionNodeKey
@ -360,16 +333,16 @@ proc getBranchAux(db: DB, node: openArray[byte], fullPath: NibblesSeq, pathIndex
if sharedNibbles == k.len: if sharedNibbles == k.len:
let value = nodeRlp.listElem(1) let value = nodeRlp.listElem(1)
if not isLeaf: if not isLeaf:
let nextLookup = getNode(db, value) let nextLookup = value.getNode
output.add nextLookup output.add nextLookup
getBranchAux(db, nextLookup, fullPath, pathIndex + sharedNibbles, output) getBranchAux(db, nextLookup, path.slice(sharedNibbles), output)
of 17: of 17:
if path.len != 0: if path.len != 0:
var branch = nodeRlp.listElem(path[0].int) var branch = nodeRlp.listElem(path[0].int)
if not branch.isEmpty: if not branch.isEmpty:
let nextLookup = getNode(db, branch) let nextLookup = branch.getNode
output.add nextLookup output.add nextLookup
getBranchAux(db, nextLookup, fullPath, pathIndex + 1, output) getBranchAux(db, nextLookup, path.slice(1), output)
else: else:
raise newException(CorruptedTrieDatabase, raise newException(CorruptedTrieDatabase,
"HexaryTrie node with an unexpected number of children") "HexaryTrie node with an unexpected number of children")
@ -378,7 +351,7 @@ proc getBranch*(self: HexaryTrie; key: openArray[byte]): seq[seq[byte]] =
result = @[] result = @[]
var node = keyToLocalBytes(self.db, self.root) var node = keyToLocalBytes(self.db, self.root)
result.add node result.add node
getBranchAux(self.db, node, initNibbleRange(key), 0, result) getBranchAux(self.db, node, initNibbleRange(key), result)
proc dbDel(t: var HexaryTrie, data: openArray[byte]) = proc dbDel(t: var HexaryTrie, data: openArray[byte]) =
if data.len >= 32: t.prune(data.keccakHash.data) if data.len >= 32: t.prune(data.keccakHash.data)
@ -421,9 +394,9 @@ proc replaceValue(data: Rlp, key: NibblesSeq, value: openArray[byte]): seq[byte]
r.append value r.append value
return r.finish() return r.finish()
proc isTwoItemNode(self: HexaryTrie; r: Rlp, fullPath: NibblesSeq, pathIndex: int): bool = proc isTwoItemNode(self: HexaryTrie; r: Rlp): bool =
if r.isBlob: if r.isBlob:
let resolved = getPossiblyMissingNode(self.db, r, fullPath, pathIndex, self.shouldMissingNodesBeErrors) let resolved = self.db.get(r)
let rlp = rlpFromBytes(resolved) let rlp = rlpFromBytes(resolved)
return rlp.isList and rlp.listLen == 2 return rlp.isList and rlp.listLen == 2
else: else:
@ -442,24 +415,18 @@ proc findSingleChild(r: Rlp; childPos: var byte): Rlp =
return zeroBytesRlp return zeroBytesRlp
inc i inc i
proc deleteAt(self: var HexaryTrie; proc deleteAt(self: var HexaryTrie; origRlp: Rlp, key: NibblesSeq): seq[byte]
origRlp: Rlp,
fullPath: NibblesSeq,
pathIndex: int): seq[byte]
{.gcsafe, raises: [RlpError].} {.gcsafe, raises: [RlpError].}
proc deleteAux(self: var HexaryTrie; proc deleteAux(self: var HexaryTrie; rlpWriter: var RlpWriter;
rlpWriter: var RlpWriter; origRlp: Rlp; path: NibblesSeq): bool =
origRlp: Rlp;
fullPath: NibblesSeq,
pathIndex: int): bool =
if origRlp.isEmpty: if origRlp.isEmpty:
return false return false
var toDelete = if origRlp.isList: origRlp var toDelete = if origRlp.isList: origRlp
else: rlpFromBytes getPossiblyMissingNode(self.db, origRlp, fullPath, pathIndex, self.shouldMissingNodesBeErrors) else: rlpFromBytes self.db.get(origRlp)
let b = self.deleteAt(toDelete, fullPath, pathIndex) let b = self.deleteAt(toDelete, path)
if b.len == 0: if b.len == 0:
return false return false
@ -467,14 +434,14 @@ proc deleteAux(self: var HexaryTrie;
rlpWriter.appendAndSave(b, self.db) rlpWriter.appendAndSave(b, self.db)
return true return true
proc graft(self: var HexaryTrie; r: Rlp, fullPath: NibblesSeq, pathIndexToTheParent: int): seq[byte] = proc graft(self: var HexaryTrie; r: Rlp): seq[byte] =
doAssert r.isList and r.listLen == 2 doAssert r.isList and r.listLen == 2
var (_, origPath) = r.extensionNodeKey var (_, origPath) = r.extensionNodeKey
var value = r.listElem(1) var value = r.listElem(1)
if not value.isList: if not value.isList:
let nodeKey = value.expectHash let nodeKey = value.expectHash
var resolvedData = getPossiblyMissingNode(self.db, nodeKey, fullPath, pathIndexToTheParent + origPath.len, self.shouldMissingNodesBeErrors) var resolvedData = self.db.get(nodeKey)
self.prune(nodeKey) self.prune(nodeKey)
value = rlpFromBytes resolvedData value = rlpFromBytes resolvedData
@ -487,8 +454,6 @@ proc graft(self: var HexaryTrie; r: Rlp, fullPath: NibblesSeq, pathIndexToThePar
return rlpWriter.finish return rlpWriter.finish
proc mergeAndGraft(self: var HexaryTrie; proc mergeAndGraft(self: var HexaryTrie;
fullPath: NibblesSeq;
pathIndexToTheParent: int,
soleChild: Rlp, childPos: byte): seq[byte] = soleChild: Rlp, childPos: byte): seq[byte] =
var output = initRlpList(2) var output = initRlpList(2)
if childPos == 16: if childPos == 16:
@ -499,61 +464,45 @@ proc mergeAndGraft(self: var HexaryTrie;
output.append(soleChild) output.append(soleChild)
result = output.finish() result = output.finish()
if self.isTwoItemNode(soleChild, fullPath, pathIndexToTheParent + 1): if self.isTwoItemNode(soleChild):
result = self.graft(rlpFromBytes(result), fullPath, pathIndexToTheParent) result = self.graft(rlpFromBytes(result))
# If the key is present, returns the RLP bytes for a node that proc deleteAt(self: var HexaryTrie; origRlp: Rlp, key: NibblesSeq): seq[byte]
# omits this key. Returns an empty seq if the key is absent.
proc deleteAt(self: var HexaryTrie; origRlp: Rlp, fullPath: NibblesSeq, pathIndex: int): seq[byte]
{.gcsafe, raises: [RlpError].} = {.gcsafe, raises: [RlpError].} =
if origRlp.isEmpty: if origRlp.isEmpty:
# It's empty RLP, so the key is absent, so no change necessary.
return return
doAssert origRlp.isTrieBranch doAssert origRlp.isTrieBranch
let origBytes = @(origRlp.rawData) let origBytes = @(origRlp.rawData)
let path = fullPath.slice(pathIndex)
if origRlp.listLen == 2: if origRlp.listLen == 2:
let (isLeaf, k) = origRlp.extensionNodeKey let (isLeaf, k) = origRlp.extensionNodeKey
if k == path and isLeaf: if k == key and isLeaf:
# This is the leaf for the key we're looking for.
# Omitting this key from the leaf means we're
# left with empty RLP.
self.dbDel origBytes self.dbDel origBytes
return emptyRlp return emptyRlp
if path.startsWith(k): if key.startsWith(k):
# This extension node gets us *partway* to the desired var
# key, but not all the way. rlpWriter = initRlpList(2)
let path = origRlp.listElem(0) path = origRlp.listElem(0)
let value = origRlp.listElem(1) value = origRlp.listElem(1)
# Create RLP for a new 2-item node that omits the key we're
# trying to delete.
var rlpWriter = initRlpList(2)
rlpWriter.append(path) rlpWriter.append(path)
if not self.deleteAux(rlpWriter, value, fullPath, pathIndex + k.len): if not self.deleteAux(rlpWriter, value, key.slice(k.len)):
# Key is absent in the value, so never mind.
return return
# We don't need the original node anymore, since we're about to
# replace it with a modified one.
self.dbDel origBytes self.dbDel origBytes
var finalBytes = rlpWriter.finish var finalBytes = rlpWriter.finish
var rlp = rlpFromBytes(finalBytes) var rlp = rlpFromBytes(finalBytes)
# We already knew that *this* node is a 2-item node; now if self.isTwoItemNode(rlp.listElem(1)):
# we check to see if the modified *child* is also a 2-item return self.graft(rlp)
# node, because if so, we can graft it.
if self.isTwoItemNode(rlp.listElem(1), fullPath, pathIndex + k.len):
return self.graft(rlp, fullPath, pathIndex)
return finalBytes return finalBytes
else: else:
return return
else: else:
if path.len == 0 and origRlp.listElem(16).isEmpty: if key.len == 0 and origRlp.listElem(16).isEmpty:
self.dbDel origBytes self.dbDel origBytes
var foundChildPos: byte var foundChildPos: byte
let singleChild = origRlp.findSingleChild(foundChildPos) let singleChild = origRlp.findSingleChild(foundChildPos)
if singleChild.hasData and foundChildPos != 16: if singleChild.hasData and foundChildPos != 16:
result = self.mergeAndGraft(fullPath, pathIndex + 1, singleChild, foundChildPos) result = self.mergeAndGraft(singleChild, foundChildPos)
else: else:
var rlpRes = initRlpList(17) var rlpRes = initRlpList(17)
var iter = origRlp var iter = origRlp
@ -566,12 +515,12 @@ proc deleteAt(self: var HexaryTrie; origRlp: Rlp, fullPath: NibblesSeq, pathInde
return rlpRes.finish return rlpRes.finish
else: else:
var rlpWriter = initRlpList(17) var rlpWriter = initRlpList(17)
let keyHead = int(path[0]) let keyHead = int(key[0])
var i = 0 var i = 0
var origCopy = origRlp var origCopy = origRlp
for elem in items(origCopy): for elem in items(origCopy):
if i == keyHead: if i == keyHead:
if not self.deleteAux(rlpWriter, elem, fullPath, pathIndex + 1): if not self.deleteAux(rlpWriter, elem, key.slice(1)):
return return
else: else:
rlpWriter.append(elem) rlpWriter.append(elem)
@ -583,48 +532,47 @@ proc deleteAt(self: var HexaryTrie; origRlp: Rlp, fullPath: NibblesSeq, pathInde
var foundChildPos: byte var foundChildPos: byte
let singleChild = resultRlp.findSingleChild(foundChildPos) let singleChild = resultRlp.findSingleChild(foundChildPos)
if singleChild.hasData: if singleChild.hasData:
result = self.mergeAndGraft(fullPath, pathIndex + 1, singleChild, foundChildPos) result = self.mergeAndGraft(singleChild, foundChildPos)
proc del*(self: var HexaryTrie; key: openArray[byte]) = proc del*(self: var HexaryTrie; key: openArray[byte]) =
var var
rootBytes = keyToLocalBytes(self.db, self.root) rootBytes = keyToLocalBytes(self.db, self.root)
rootRlp = rlpFromBytes rootBytes rootRlp = rlpFromBytes rootBytes
var newRootBytes = self.deleteAt(rootRlp, initNibbleRange(key), 0) var newRootBytes = self.deleteAt(rootRlp, initNibbleRange(key))
if newRootBytes.len > 0: if newRootBytes.len > 0:
if rootBytes.len < 32: if rootBytes.len < 32:
self.prune(self.root.asDbKey) self.prune(self.root.asDbKey)
self.root = self.db.dbPut(newRootBytes) self.root = self.db.dbPut(newRootBytes)
proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash, proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
fullPath: NibblesSeq, pathIndex: int, value: openArray[byte], key: NibblesSeq, value: openArray[byte],
isInline = false): seq[byte] isInline = false): seq[byte]
{.gcsafe, raises: [RlpError].} {.gcsafe, raises: [RlpError].}
proc mergeAt(self: var HexaryTrie, rlp: Rlp, proc mergeAt(self: var HexaryTrie, rlp: Rlp,
fullPath: NibblesSeq, pathIndex: int, value: openArray[byte], key: NibblesSeq, value: openArray[byte],
isInline = false): seq[byte] = isInline = false): seq[byte] =
self.mergeAt(rlp, rlp.rawData.keccakHash, fullPath, pathIndex, value, isInline) self.mergeAt(rlp, rlp.rawData.keccakHash, key, value, isInline)
proc mergeAtAux(self: var HexaryTrie, output: var RlpWriter, orig: Rlp, proc mergeAtAux(self: var HexaryTrie, output: var RlpWriter, orig: Rlp,
fullPath: NibblesSeq, pathIndex: int, value: openArray[byte]) = key: NibblesSeq, value: openArray[byte]) =
var resolved = orig var resolved = orig
var isRemovable = false var isRemovable = false
if not (orig.isList or orig.isEmpty): if not (orig.isList or orig.isEmpty):
resolved = rlpFromBytes getPossiblyMissingNode(self.db, orig, fullPath, pathIndex, self.shouldMissingNodesBeErrors) resolved = rlpFromBytes self.db.get(orig)
isRemovable = true isRemovable = true
let b = self.mergeAt(resolved, fullPath, pathIndex, value, not isRemovable) let b = self.mergeAt(resolved, key, value, not isRemovable)
output.appendAndSave(b, self.db) output.appendAndSave(b, self.db)
proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash, proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
fullPath: NibblesSeq, pathIndex: int, value: openArray[byte], key: NibblesSeq, value: openArray[byte],
isInline = false): seq[byte] isInline = false): seq[byte]
{.gcsafe, raises: [RlpError].} = {.gcsafe, raises: [RlpError].} =
let path = fullPath.slice(pathIndex)
template origWithNewValue: auto = template origWithNewValue: auto =
self.prune(origHash.data) self.prune(origHash.data)
replaceValue(orig, path, value) replaceValue(orig, key, value)
if orig.isEmpty: if orig.isEmpty:
return origWithNewValue() return origWithNewValue()
@ -634,15 +582,15 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
let (isLeaf, k) = orig.extensionNodeKey let (isLeaf, k) = orig.extensionNodeKey
var origValue = orig.listElem(1) var origValue = orig.listElem(1)
if k == path and isLeaf: if k == key and isLeaf:
return origWithNewValue() return origWithNewValue()
let sharedNibbles = sharedPrefixLen(path, k) let sharedNibbles = sharedPrefixLen(key, k)
if sharedNibbles == k.len and not isLeaf: if sharedNibbles == k.len and not isLeaf:
var r = initRlpList(2) var r = initRlpList(2)
r.append orig.listElem(0) r.append orig.listElem(0)
self.mergeAtAux(r, origValue, fullPath, pathIndex + k.len, value) self.mergeAtAux(r, origValue, key.slice(k.len), value)
return r.finish return r.finish
if orig.rawData.len >= 32: if orig.rawData.len >= 32:
@ -658,7 +606,7 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
top.append hexPrefixEncode(k.slice(0, sharedNibbles), false) top.append hexPrefixEncode(k.slice(0, sharedNibbles), false)
top.appendAndSave(bottom.finish, self.db) top.appendAndSave(bottom.finish, self.db)
return self.mergeAt(rlpFromBytes(top.finish), fullPath, pathIndex, value, true) return self.mergeAt(rlpFromBytes(top.finish), key, value, true)
else: else:
# Create a branch node # Create a branch node
var branches = initRlpList(17) var branches = initRlpList(17)
@ -682,22 +630,22 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
branches.append "" branches.append ""
branches.append "" branches.append ""
return self.mergeAt(rlpFromBytes(branches.finish), fullPath, pathIndex, value, true) return self.mergeAt(rlpFromBytes(branches.finish), key, value, true)
else: else:
if path.len == 0: if key.len == 0:
return origWithNewValue() return origWithNewValue()
if isInline: if isInline:
self.prune(origHash.data) self.prune(origHash.data)
let n = path[0] let n = key[0]
var i = 0 var i = 0
var r = initRlpList(17) var r = initRlpList(17)
var origCopy = orig var origCopy = orig
for elem in items(origCopy): for elem in items(origCopy):
if i == int(n): if i == int(n):
self.mergeAtAux(r, elem, fullPath, pathIndex + 1, value) self.mergeAtAux(r, elem, key.slice(1), value)
else: else:
r.append(elem) r.append(elem)
inc i inc i
@ -707,11 +655,11 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
proc put*(self: var HexaryTrie; key, value: openArray[byte]) = proc put*(self: var HexaryTrie; key, value: openArray[byte]) =
let root = self.root.hash let root = self.root.hash
var rootBytes = getPossiblyMissingNode(self.db, root.data, NibblesSeq(), 0, self.shouldMissingNodesBeErrors) var rootBytes = self.db.get(root.data)
doAssert rootBytes.len > 0 doAssert rootBytes.len > 0
let newRootBytes = self.mergeAt(rlpFromBytes(rootBytes), root, let newRootBytes = self.mergeAt(rlpFromBytes(rootBytes), root,
initNibbleRange(key), 0, value) initNibbleRange(key), value)
if rootBytes.len < 32: if rootBytes.len < 32:
self.prune(root.data) self.prune(root.data)
@ -747,100 +695,3 @@ proc isValidBranch*(branch: seq[seq[byte]], rootHash: KeccakHash, key, value: se
var trie = initHexaryTrie(db, rootHash) var trie = initHexaryTrie(db, rootHash)
result = trie.get(key) == value result = trie.get(key) == value
# The code below has a lot of duplication with the code above; I needed
# versions of get/put/del that don't just assume that all the nodes exist.
# Maybe there's some way to eliminate the duplication without screwing
# up performance? But for now I don't want to meddle with the existing
# code, for fear of breaking it. --Adam, Nov. 2022
proc db*(self: SecureHexaryTrie): TrieDatabaseRef = HexaryTrie(self).db
template maybeKeyToLocalBytes(db: DB, k: TrieNodeKey): Option[seq[byte]] =
if k.len < 32:
some(k.getLocalBytes)
else:
db.maybeGet(k.asDbKey)
proc maybeGetLookup(db: DB, elem: Rlp): Option[Rlp] =
if elem.isList:
some(elem)
else:
let h = elem.expectHash
let maybeBytes = db.maybeGet(h)
if maybeBytes.isNone:
none[Rlp]()
else:
let bytes = maybeBytes.get
some(rlpFromBytes(bytes))
proc maybeGetAux(db: DB, nodeRlp: Rlp, fullPath: NibblesSeq, pathIndex: int): Option[seq[byte]]
{.gcsafe, raises: [RlpError].} =
# FIXME-Adam: do I need to distinguish between these two cases?
if not nodeRlp.hasData:
let zero: seq[byte] = @[]
return some(zero)
# return none[seq[byte]]()
if nodeRlp.isEmpty:
# FIXME-Adam: I am REALLY not sure this is the right thing to do. But toGenesisHeader
# failing is a pretty clear indication. So let's try this. I wonder whether the
# above case needs to do this too.
let zero: seq[byte] = @[]
return some(zero)
# return none[seq[byte]]()
let path = fullPath.slice(pathIndex)
case nodeRlp.listLen
of 2:
let (isLeaf, k) = nodeRlp.extensionNodeKey
let sharedNibbles = sharedPrefixLen(path, k)
if sharedNibbles == k.len:
let value = nodeRlp.listElem(1)
if sharedNibbles == path.len and isLeaf:
return some(value.toBytes)
elif not isLeaf:
let maybeNextLookup = maybeGetLookup(db, value)
if maybeNextLookup.isNone:
return none[seq[byte]]()
else:
return maybeGetAux(db, maybeNextLookup.get, fullPath, pathIndex + sharedNibbles)
else:
raise newException(RlpError, "isLeaf is true but the shared nibbles didn't exhaust the path?")
else:
let zero: seq[byte] = @[]
return some(zero)
of 17:
if path.len == 0:
return some(nodeRlp.listElem(16).toBytes)
var branch = nodeRlp.listElem(path[0].int)
if branch.isEmpty:
let zero: seq[byte] = @[]
return some(zero)
else:
let maybeNextLookup = maybeGetLookup(db, branch)
if maybeNextLookup.isNone:
return none[seq[byte]]()
else:
return maybeGetAux(db, maybeNextLookup.get, fullPath, pathIndex + 1)
else:
raise newException(CorruptedTrieDatabase,
"HexaryTrie node with an unexpected number of children")
proc maybeGetAuxByHash(db: DB, node: TrieNodeKey, fullPath: NibblesSeq, pathIndex: int): Option[seq[byte]] =
let maybeBytes = maybeKeyToLocalBytes(db, node)
if maybeBytes.isNone:
return none[seq[byte]]()
else:
let bytes = maybeBytes.get
var nodeRlp = rlpFromBytes(bytes)
return maybeGetAux(db, nodeRlp, fullPath, pathIndex)
proc maybeGet*(self: HexaryTrie; key: openArray[byte]): Option[seq[byte]] =
return maybeGetAuxByHash(self.db, self.root, initNibbleRange(key), 0)
proc maybeGet*(self: SecureHexaryTrie; key: openArray[byte]): Option[seq[byte]] =
return maybeGet(HexaryTrie(self), key.keccakHash.data)