mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-02-07 09:44:06 +00:00
1b4572ed3b
* Re-arrange fetching storage slots in batch module why; Previously, fetching partial slot ranges first has a chance of terminating the worker peer 9due to network error) while there were many inheritable storage slots on the queue. Now, inheritance is checked first, then full slot ranges and finally partial ranges. * Update logging * Bundled node information for healing into single object `NodeSpecs` why: Previously, partial paths and node keys were kept in separate variables. This approach was error prone due to copying/reassembling function argument objects. As all partial paths, keys, and node data types are more or less handled as `Blob`s over the network (using Eth/6x, or Snap/1) it makes sense to hold these `Blob`s as named field in a single object (even if not all fields are active for the current purpose.) * For good housekeeping, using `NodeKey` type only for account keys why: previously, a mixture of `NodeKey` and `Hash256` was used. Now, only state or storage root keys use the `Hash256` type. * Always accept latest pivot (and not a slightly older one) why; For testing it was tried to use a slightly older pivot state root than available. Some anecdotal tests seemed to suggest an advantage so that more peers are willing to serve on that older pivot. But this could not be confirmed in subsequent tests (still anecdotal, though.) As a side note, the distance of the latest pivot to its predecessor is at least 128 (or whatever the constant `minPivotBlockDistance` is assigned to.) * Reshuffle name components for some file and function names why: Clarifies purpose: "storages" becomes: "storage slots" "store" becomes: "range fetch" * Stash away currently unused modules in sub-folder named "notused"
203 lines
6.4 KiB
Nim
203 lines
6.4 KiB
Nim
# nimbus-eth1
|
|
# Copyright (c) 2021 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or distributed
|
|
# except according to those terms.
|
|
|
|
import
|
|
std/[algorithm, tables],
|
|
chronicles,
|
|
eth/[common, trie/db],
|
|
../../../../db/kvstore_rocksdb,
|
|
../../range_desc,
|
|
"."/[hexary_desc, hexary_error, rocky_bulk_load, snapdb_desc]
|
|
|
|
{.push raises: [Defect].}
|
|
|
|
logScope:
|
|
topics = "snap-db"
|
|
|
|
type
|
|
AccountsGetFn* = proc(key: openArray[byte]): Blob {.gcsafe.}
|
|
## The `get()` function for the accounts trie
|
|
|
|
StorageSlotsGetFn* = proc(acc: NodeKey; key: openArray[byte]): Blob {.gcsafe.}
|
|
## The `get()` function for the storage trie depends on the current account
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc convertTo(key: RepairKey; T: type NodeKey): T =
|
|
## Might be lossy, check before use
|
|
discard result.init(key.ByteArray33[1 .. 32])
|
|
|
|
proc convertTo(key: RepairKey; T: type NodeTag): T =
|
|
## Might be lossy, check before use
|
|
UInt256.fromBytesBE(key.ByteArray33[1 .. 32]).T
|
|
|
|
proc toAccountsKey(a: RepairKey): ByteArray32 =
|
|
a.convertTo(NodeKey).toAccountsKey
|
|
|
|
proc toStorageSlotsKey(a: RepairKey): ByteArray33 =
|
|
a.convertTo(NodeKey).toStorageSlotsKey
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public functions: get
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc persistentAccountsGetFn*(db: TrieDatabaseRef): AccountsGetFn =
|
|
return proc(key: openArray[byte]): Blob =
|
|
var nodeKey: NodeKey
|
|
if nodeKey.init(key):
|
|
return db.get(nodeKey.toAccountsKey.toOpenArray)
|
|
|
|
proc persistentStorageSlotsGetFn*(db: TrieDatabaseRef): StorageSlotsGetFn =
|
|
return proc(accKey: NodeKey; key: openArray[byte]): Blob =
|
|
var nodeKey: NodeKey
|
|
if nodeKey.init(key):
|
|
return db.get(nodeKey.toStorageSlotsKey.toOpenArray)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public functions: store/put
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc persistentAccountsPut*(
|
|
db: HexaryTreeDbRef;
|
|
base: TrieDatabaseRef
|
|
): Result[void,HexaryDbError] =
|
|
## Bulk load using transactional `put()`
|
|
let dbTx = base.beginTransaction
|
|
defer: dbTx.commit
|
|
|
|
for (key,value) in db.tab.pairs:
|
|
if not key.isNodeKey:
|
|
let error = UnresolvedRepairNode
|
|
trace "Unresolved node in repair table", error
|
|
return err(error)
|
|
base.put(key.toAccountsKey.toOpenArray, value.convertTo(Blob))
|
|
ok()
|
|
|
|
proc persistentStorageSlotsPut*(
|
|
db: HexaryTreeDbRef;
|
|
base: TrieDatabaseRef
|
|
): Result[void,HexaryDbError] =
|
|
## Bulk load using transactional `put()`
|
|
let dbTx = base.beginTransaction
|
|
defer: dbTx.commit
|
|
|
|
for (key,value) in db.tab.pairs:
|
|
if not key.isNodeKey:
|
|
let error = UnresolvedRepairNode
|
|
trace "Unresolved node in repair table", error
|
|
return err(error)
|
|
base.put(key.toStorageSlotsKey.toOpenArray, value.convertTo(Blob))
|
|
ok()
|
|
|
|
|
|
proc persistentAccountsPut*(
|
|
db: HexaryTreeDbRef;
|
|
rocky: RocksStoreRef
|
|
): Result[void,HexaryDbError]
|
|
{.gcsafe, raises: [Defect,OSError,KeyError].} =
|
|
## SST based bulk load on `rocksdb`.
|
|
if rocky.isNil:
|
|
return err(NoRocksDbBackend)
|
|
let bulker = RockyBulkLoadRef.init(rocky)
|
|
defer: bulker.destroy()
|
|
if not bulker.begin(RockyBulkCache):
|
|
let error = CannotOpenRocksDbBulkSession
|
|
trace "Rocky hexary session initiation failed",
|
|
error, info=bulker.lastError()
|
|
return err(error)
|
|
|
|
#let keyList = toSeq(db.tab.keys)
|
|
# .filterIt(it.isNodeKey)
|
|
# .mapIt(it.convertTo(NodeTag))
|
|
# .sorted(cmp)
|
|
var
|
|
keyList = newSeq[NodeTag](db.tab.len)
|
|
inx = 0
|
|
for repairKey in db.tab.keys:
|
|
if repairKey.isNodeKey:
|
|
keyList[inx] = repairKey.convertTo(NodeTag)
|
|
inx.inc
|
|
if inx < db.tab.len:
|
|
return err(UnresolvedRepairNode)
|
|
keyList.sort(cmp)
|
|
|
|
for n,nodeTag in keyList:
|
|
let
|
|
nodeKey = nodeTag.to(NodeKey)
|
|
data = db.tab[nodeKey.to(RepairKey)].convertTo(Blob)
|
|
if not bulker.add(nodeKey.toAccountsKey.toOpenArray, data):
|
|
let error = AddBulkItemFailed
|
|
trace "Rocky hexary bulk load failure",
|
|
n, len=db.tab.len, error, info=bulker.lastError()
|
|
return err(error)
|
|
|
|
if bulker.finish().isErr:
|
|
let error = CommitBulkItemsFailed
|
|
trace "Rocky hexary commit failure",
|
|
len=db.tab.len, error, info=bulker.lastError()
|
|
return err(error)
|
|
ok()
|
|
|
|
|
|
proc persistentStorageSlotsPut*(
|
|
db: HexaryTreeDbRef;
|
|
rocky: RocksStoreRef
|
|
): Result[void,HexaryDbError]
|
|
{.gcsafe, raises: [Defect,OSError,KeyError].} =
|
|
## SST based bulk load on `rocksdb`.
|
|
if rocky.isNil:
|
|
return err(NoRocksDbBackend)
|
|
let bulker = RockyBulkLoadRef.init(rocky)
|
|
defer: bulker.destroy()
|
|
if not bulker.begin(RockyBulkCache):
|
|
let error = CannotOpenRocksDbBulkSession
|
|
trace "Rocky hexary session initiation failed",
|
|
error, info=bulker.lastError()
|
|
return err(error)
|
|
|
|
#let keyList = toSeq(db.tab.keys)
|
|
# .filterIt(it.isNodeKey)
|
|
# .mapIt(it.convertTo(NodeTag))
|
|
# .sorted(cmp)
|
|
var
|
|
keyList = newSeq[NodeTag](db.tab.len)
|
|
inx = 0
|
|
for repairKey in db.tab.keys:
|
|
if repairKey.isNodeKey:
|
|
keyList[inx] = repairKey.convertTo(NodeTag)
|
|
inx.inc
|
|
if inx < db.tab.len:
|
|
return err(UnresolvedRepairNode)
|
|
keyList.sort(cmp)
|
|
|
|
for n,nodeTag in keyList:
|
|
let
|
|
nodeKey = nodeTag.to(NodeKey)
|
|
data = db.tab[nodeKey.to(RepairKey)].convertTo(Blob)
|
|
if not bulker.add(nodeKey.toStorageSlotsKey.toOpenArray, data):
|
|
let error = AddBulkItemFailed
|
|
trace "Rocky hexary bulk load failure",
|
|
n, len=db.tab.len, error, info=bulker.lastError()
|
|
return err(error)
|
|
|
|
if bulker.finish().isErr:
|
|
let error = CommitBulkItemsFailed
|
|
trace "Rocky hexary commit failure",
|
|
len=db.tab.len, error, info=bulker.lastError()
|
|
return err(error)
|
|
ok()
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|
|
|