mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-02-01 06:45:35 +00:00
b793f0de8d
* Redefine `seq[Blob]` => `seq[SnapProof]` for `snap/1` protocol why: Proof nodes are traded as `Blob` type items rather than Nim objects. So the RLP transcoder must not extra wrap proofs which are of type seq[Blob]. Without custom encoding one would produce a `list(blob(item1), blob(item2) ..)` instead of `list(item1, item2 ..)`. * Limit leaf extractor by RLP size rather than number of items why: To be used serving `snap/1` requests, the result of function `hexaryRangeLeafsProof()` is limited by the maximal space needed to serialise the result which will be part of the `snap/1` repsonse. * Let the range extractor `hexaryRangeLeafsProof()` return RLP list sizes why: When collecting accounts, the size oft the accounts list when encoded as RLP is continually updated. So the summed up value is available anyway. For the proof nodes list, there are not many (~ 10) so summing up is not expensive here.
368 lines
12 KiB
Nim
368 lines
12 KiB
Nim
# nimbus-eth1
|
|
# Copyright (c) 2021 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or distributed
|
|
# except according to those terms.
|
|
|
|
import
|
|
std/[sequtils, tables],
|
|
chronicles,
|
|
eth/[common, p2p, trie/db, trie/nibbles],
|
|
../../../../db/[select_backend, storage_types],
|
|
../../../protocol,
|
|
../../range_desc,
|
|
"."/[hexary_desc, hexary_error, hexary_import, hexary_nearby,
|
|
hexary_paths, rocky_bulk_load]
|
|
|
|
{.push raises: [].}
|
|
|
|
logScope:
|
|
topics = "snap-db"
|
|
|
|
const
|
|
extraTraceMessages = false or true
|
|
|
|
RockyBulkCache* = "accounts.sst"
|
|
## Name of temporary file to accomodate SST records for `rocksdb`
|
|
|
|
type
|
|
SnapDbRef* = ref object
|
|
## Global, re-usable descriptor
|
|
keyMap: Table[RepairKey,uint] ## For debugging only (will go away)
|
|
db: TrieDatabaseRef ## General database
|
|
rocky: RocksStoreRef ## Set if rocksdb is available
|
|
|
|
SnapDbBaseRef* = ref object of RootRef
|
|
## Session descriptor
|
|
xDb: HexaryTreeDbRef ## Hexary database, memory based
|
|
base: SnapDbRef ## Back reference to common parameters
|
|
root*: NodeKey ## Session DB root node key
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private debugging helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
template noPpError(info: static[string]; code: untyped) =
|
|
try:
|
|
code
|
|
except ValueError as e:
|
|
raiseAssert "Inconveivable (" & info & "): " & e.msg
|
|
except KeyError as e:
|
|
raiseAssert "Not possible (" & info & "): " & e.msg
|
|
except CatchableError as e:
|
|
raiseAssert "Ooops (" & info & ") " & $e.name & ": " & e.msg
|
|
|
|
proc toKey(a: RepairKey; pv: SnapDbRef): uint =
|
|
if not a.isZero:
|
|
noPpError("pp(RepairKey)"):
|
|
if not pv.keyMap.hasKey(a):
|
|
pv.keyMap[a] = pv.keyMap.len.uint + 1
|
|
result = pv.keyMap[a]
|
|
|
|
proc toKey(a: RepairKey; ps: SnapDbBaseRef): uint =
|
|
a.toKey(ps.base)
|
|
|
|
proc toKey(a: NodeKey; ps: SnapDbBaseRef): uint =
|
|
a.to(RepairKey).toKey(ps)
|
|
|
|
#proc toKey(a: NodeTag; ps: SnapDbBaseRef): uint =
|
|
# a.to(NodeKey).toKey(ps)
|
|
|
|
proc ppImpl(a: RepairKey; pv: SnapDbRef): string =
|
|
"$" & $a.toKey(pv)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Debugging, pretty printing
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc pp*(a: NodeKey; ps: SnapDbBaseRef): string =
|
|
if a.isZero: "ø" else:"$" & $a.toKey(ps)
|
|
|
|
proc pp*(a: RepairKey; ps: SnapDbBaseRef): string =
|
|
if a.isZero: "ø" elif a.isNodeKey: "$" & $a.toKey(ps) else: "@" & $a.toKey(ps)
|
|
|
|
proc pp*(a: NodeTag; ps: SnapDbBaseRef): string =
|
|
a.to(NodeKey).pp(ps)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private helper
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc clearRockyCacheFile(rocky: RocksStoreRef): bool =
|
|
if not rocky.isNil:
|
|
# A cache file might hang about from a previous crash
|
|
try:
|
|
discard rocky.clearCacheFile(RockyBulkCache)
|
|
return true
|
|
except OSError as e:
|
|
error "Cannot clear rocksdb cache", exception=($e.name), msg=e.msg
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public constructor
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc init*(
|
|
T: type SnapDbRef;
|
|
db: TrieDatabaseRef
|
|
): T =
|
|
## Main object constructor
|
|
T(db: db)
|
|
|
|
proc init*(
|
|
T: type SnapDbRef;
|
|
db: ChainDb
|
|
): T =
|
|
## Variant of `init()` allowing bulk import on rocksdb backend
|
|
result = T(db: db.trieDB, rocky: db.rocksStoreRef)
|
|
if not result.rocky.clearRockyCacheFile():
|
|
result.rocky = nil
|
|
|
|
proc init*(
|
|
T: type HexaryTreeDbRef;
|
|
pv: SnapDbRef;
|
|
): T =
|
|
## Constructor for inner hexary trie database
|
|
let xDb = HexaryTreeDbRef()
|
|
xDb.keyPp = proc(key: RepairKey): string = key.ppImpl(pv) # will go away
|
|
return xDb
|
|
|
|
proc init*(
|
|
T: type HexaryTreeDbRef;
|
|
ps: SnapDbBaseRef;
|
|
): T =
|
|
## Constructor variant
|
|
HexaryTreeDbRef.init(ps.base)
|
|
|
|
# ---------------
|
|
|
|
proc init*(
|
|
ps: SnapDbBaseRef;
|
|
pv: SnapDbRef;
|
|
root: NodeKey;
|
|
) =
|
|
## Session base constructor
|
|
ps.base = pv
|
|
ps.root = root
|
|
ps.xDb = HexaryTreeDbRef.init(pv)
|
|
|
|
proc init*(
|
|
T: type SnapDbBaseRef;
|
|
ps: SnapDbBaseRef;
|
|
root: NodeKey;
|
|
): T =
|
|
## Variant of session base constructor
|
|
new result
|
|
result.init(ps.base, root)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public getters
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc hexaDb*(ps: SnapDbBaseRef): HexaryTreeDbRef =
|
|
## Getter, low level access to underlying session DB
|
|
ps.xDb
|
|
|
|
proc rockDb*(ps: SnapDbBaseRef): RocksStoreRef =
|
|
## Getter, low level access to underlying persistent rock DB interface
|
|
ps.base.rocky
|
|
|
|
proc kvDb*(ps: SnapDbBaseRef): TrieDatabaseRef =
|
|
## Getter, low level access to underlying persistent key-value DB
|
|
ps.base.db
|
|
|
|
proc kvDb*(pv: SnapDbRef): TrieDatabaseRef =
|
|
## Getter, low level access to underlying persistent key-value DB
|
|
pv.db
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public functions, select sub-tables for persistent storage
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc toAccountsKey*(a: NodeKey): ByteArray33 =
|
|
a.ByteArray32.snapSyncAccountKey.data
|
|
|
|
proc toStorageSlotsKey*(a: NodeKey): ByteArray33 =
|
|
a.ByteArray32.snapSyncStorageSlotKey.data
|
|
|
|
proc toStateRootKey*(a: NodeKey): ByteArray33 =
|
|
a.ByteArray32.snapSyncStateRootKey.data
|
|
|
|
template toOpenArray*(k: ByteArray32): openArray[byte] =
|
|
k.toOpenArray(0, 31)
|
|
|
|
template toOpenArray*(k: ByteArray33): openArray[byte] =
|
|
k.toOpenArray(0, 32)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc dbBackendRocksDb*(pv: SnapDbRef): bool =
|
|
## Returns `true` if rocksdb features are available
|
|
not pv.rocky.isNil
|
|
|
|
proc dbBackendRocksDb*(ps: SnapDbBaseRef): bool =
|
|
## Returns `true` if rocksdb features are available
|
|
not ps.base.rocky.isNil
|
|
|
|
proc mergeProofs*(
|
|
ps: SnapDbBaseRef; ## Session database
|
|
peer: Peer; ## For log messages
|
|
proof: seq[SnapProof]; ## Node records
|
|
freeStandingOk = false; ## Remove freestanding nodes
|
|
): Result[void,HexaryError]
|
|
{.gcsafe, raises: [RlpError,KeyError].} =
|
|
## Import proof records (as received with snap message) into a hexary trie
|
|
## of the repair table. These hexary trie records can be extended to a full
|
|
## trie at a later stage and used for validating account data.
|
|
let
|
|
db = ps.hexaDb
|
|
var
|
|
nodes: HashSet[RepairKey]
|
|
refs = @[ps.root.to(RepairKey)].toHashSet
|
|
|
|
for n,rlpRec in proof:
|
|
let report = db.hexaryImport(rlpRec.data, nodes, refs)
|
|
if report.error != NothingSerious:
|
|
let error = report.error
|
|
trace "mergeProofs()", peer, item=n, proofs=proof.len, error
|
|
return err(error)
|
|
|
|
# Remove free standing nodes (if any)
|
|
if 0 < nodes.len:
|
|
let rest = nodes - refs
|
|
if 0 < rest.len:
|
|
if freeStandingOk:
|
|
trace "mergeProofs() detected unrelated nodes", peer, nodes=nodes.len
|
|
discard
|
|
else:
|
|
# Delete unreferenced nodes
|
|
for nodeKey in nodes:
|
|
db.tab.del(nodeKey)
|
|
trace "mergeProofs() ignoring unrelated nodes", peer, nodes=nodes.len
|
|
|
|
ok()
|
|
|
|
|
|
proc verifyLowerBound*(
|
|
ps: SnapDbBaseRef; ## Database session descriptor
|
|
peer: Peer; ## For log messages
|
|
base: NodeTag; ## Before or at first account entry in `data`
|
|
first: NodeTag; ## First account key
|
|
): Result[void,HexaryError]
|
|
{.gcsafe, raises: [CatchableError].} =
|
|
## Verify that `base` is to the left of the first leaf entry and there is
|
|
## nothing in between.
|
|
var error: HexaryError
|
|
|
|
let rc = base.hexaryNearbyRight(ps.root, ps.hexaDb)
|
|
if rc.isErr:
|
|
error = rc.error
|
|
elif first == rc.value:
|
|
return ok()
|
|
else:
|
|
error = LowerBoundProofError
|
|
|
|
when extraTraceMessages:
|
|
trace "verifyLowerBound()", peer, base=base.to(NodeKey).pp,
|
|
first=first.to(NodeKey).pp, error
|
|
err(error)
|
|
|
|
|
|
proc verifyNoMoreRight*(
|
|
ps: SnapDbBaseRef; ## Database session descriptor
|
|
peer: Peer; ## For log messages
|
|
base: NodeTag; ## Before or at first account entry in `data`
|
|
): Result[void,HexaryError]
|
|
{.gcsafe, raises: [CatchableError].} =
|
|
## Verify that there is are no more leaf entries to the right of and
|
|
## including `base`.
|
|
let
|
|
root = ps.root.to(RepairKey)
|
|
base = base.to(NodeKey)
|
|
if base.hexaryPath(root, ps.hexaDb).hexaryNearbyRightMissing(ps.hexaDb):
|
|
return ok()
|
|
|
|
let error = LowerBoundProofError
|
|
when extraTraceMessages:
|
|
trace "verifyLeftmostBound()", peer, base=base.pp, error
|
|
err(error)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Debugging (and playing with the hexary database)
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc assignPrettyKeys*(xDb: HexaryTreeDbRef; root: NodeKey) =
|
|
## Prepare for pretty pringing/debugging. Run early enough this function
|
|
## sets the root key to `"$"`, for instance.
|
|
if not xDb.keyPp.isNil:
|
|
noPpError("validate(1)"):
|
|
# Make keys assigned in pretty order for printing
|
|
let rootKey = root.to(RepairKey)
|
|
discard xDb.keyPp rootKey
|
|
var keysList = toSeq(xDb.tab.keys)
|
|
if xDb.tab.hasKey(rootKey):
|
|
keysList = @[rootKey] & keysList
|
|
for key in keysList:
|
|
let node = xDb.tab[key]
|
|
discard xDb.keyPp key
|
|
case node.kind:
|
|
of Branch: (for w in node.bLink: discard xDb.keyPp w)
|
|
of Extension: discard xDb.keyPp node.eLink
|
|
of Leaf: discard
|
|
|
|
proc assignPrettyKeys*(ps: SnapDbBaseRef) =
|
|
## Variant of `assignPrettyKeys()`
|
|
ps.hexaDb.assignPrettyKeys(ps.root)
|
|
|
|
proc dumpPath*(ps: SnapDbBaseRef; key: NodeTag): seq[string] =
|
|
## Pretty print helper compiling the path into the repair tree for the
|
|
## argument `key`.
|
|
noPpError("dumpPath"):
|
|
let rPath= key.hexaryPath(ps.root, ps.hexaDb)
|
|
result = rPath.path.mapIt(it.pp(ps.hexaDb)) & @["(" & rPath.tail.pp & ")"]
|
|
|
|
proc dumpHexaDB*(xDb: HexaryTreeDbRef; root: NodeKey; indent = 4): string =
|
|
## Dump the entries from the a generic accounts trie. These are
|
|
## key value pairs for
|
|
## ::
|
|
## Branch: ($1,b(<$2,$3,..,$17>,))
|
|
## Extension: ($18,e(832b5e..06e697,$19))
|
|
## Leaf: ($20,l(cc9b5d..1c3b4,f84401..f9e5129d[#70]))
|
|
##
|
|
## where keys are typically represented as `$<id>` or `¶<id>` or `ø`
|
|
## depending on whether a key is final (`$<id>`), temporary (`¶<id>`)
|
|
## or unset/missing (`ø`).
|
|
##
|
|
## The node types are indicated by a letter after the first key before
|
|
## the round brackets
|
|
## ::
|
|
## Branch: 'b', 'þ', or 'B'
|
|
## Extension: 'e', '€', or 'E'
|
|
## Leaf: 'l', 'ł', or 'L'
|
|
##
|
|
## Here a small letter indicates a `Static` node which was from the
|
|
## original `proofs` list, a capital letter indicates a `Mutable` node
|
|
## added on the fly which might need some change, and the decorated
|
|
## letters stand for `Locked` nodes which are like `Static` ones but
|
|
## added later (typically these nodes are update `Mutable` nodes.)
|
|
##
|
|
## Beware: dumping a large database is not recommended
|
|
xDb.pp(root, indent)
|
|
|
|
proc dumpHexaDB*(ps: SnapDbBaseRef; indent = 4): string =
|
|
## Ditto
|
|
ps.hexaDb.pp(ps.root, indent)
|
|
|
|
proc hexaryPpFn*(ps: SnapDbBaseRef): HexaryPpFn =
|
|
## Key mapping function used in `HexaryTreeDB`
|
|
ps.hexaDb.keyPp
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|