mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-01-13 22:04:52 +00:00
7bbb0f4421
When running the import, currently blocks are loaded in batches into a `seq` then passed to the importer as such. In reality, blocks are still processed one by one, so the batching does not offer any performance advantage. It does however require that the client wastes memory, up to several GB, on the block sequence while they're waiting to be processed. This PR introduces a persister that accepts these potentially large blocks one by one and at the same time removes a number of redundant / unnecessary copies, assignments and resets that were slowing down the import process in general.
85 lines
2.5 KiB
Nim
85 lines
2.5 KiB
Nim
# fluffy
|
|
# Copyright (c) 2024 Status Research & Development GmbH
|
|
# Licensed and distributed under either of
|
|
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
|
|
{.push raises: [].}
|
|
|
|
import
|
|
std/os,
|
|
stew/io2,
|
|
results,
|
|
eth/common/blocks,
|
|
../network/history/validation/historical_hashes_accumulator,
|
|
../eth_data/era1
|
|
|
|
type Era1DB* = ref object
|
|
## The Era1 database manages a collection of era files that together make up
|
|
## a linear history of pre-merge execution chain data.
|
|
path: string
|
|
network: string
|
|
accumulator: FinishedHistoricalHashesAccumulator
|
|
files: seq[Era1File]
|
|
|
|
proc getEra1File(db: Era1DB, era: Era1): Result[Era1File, string] =
|
|
for f in db.files:
|
|
if f.blockIdx.startNumber.era == era:
|
|
return ok(f)
|
|
|
|
if era > mergeBlockNumber.era():
|
|
return err("Selected era1 past pre-merge data")
|
|
|
|
let
|
|
root = db.accumulator.historicalEpochs[era.int]
|
|
name = era1FileName(db.network, era, Digest(data: root))
|
|
path = db.path / name
|
|
|
|
if not isFile(path):
|
|
return err("No such era file")
|
|
|
|
# TODO: The open call does not do full verification. It is assumed here that
|
|
# trusted files are used. We might want to add a full validation option.
|
|
let f = Era1File.open(path).valueOr:
|
|
return err(error)
|
|
|
|
if db.files.len > 16: # TODO LRU
|
|
close(db.files[0])
|
|
db.files.delete(0)
|
|
|
|
db.files.add(f)
|
|
ok(f)
|
|
|
|
proc new*(
|
|
T: type Era1DB,
|
|
path: string,
|
|
network: string,
|
|
accumulator: FinishedHistoricalHashesAccumulator,
|
|
): Era1DB =
|
|
Era1DB(path: path, network: network, accumulator: accumulator)
|
|
|
|
proc getEthBlock*(
|
|
db: Era1DB, blockNumber: uint64, res: var Block
|
|
): Result[void, string] =
|
|
let f = ?db.getEra1File(blockNumber.era)
|
|
|
|
f.getEthBlock(blockNumber, res)
|
|
|
|
proc getBlockTuple*(
|
|
db: Era1DB, blockNumber: uint64, res: var BlockTuple
|
|
): Result[void, string] =
|
|
let f = ?db.getEra1File(blockNumber.era)
|
|
|
|
f.getBlockTuple(blockNumber, res)
|
|
|
|
proc getAccumulator*(
|
|
db: Era1DB, blockNumber: uint64
|
|
): Result[EpochRecordCached, string] =
|
|
## Get the Epoch Record that the block with `blockNumber` is part of.
|
|
# TODO: Probably want this `EpochRecordCached` also actually cached in
|
|
# the Era1File or EraDB object.
|
|
let f = ?db.getEra1File(blockNumber.era)
|
|
|
|
f.buildAccumulator()
|