mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-02-05 16:54:00 +00:00
7bbb0f4421
When running the import, currently blocks are loaded in batches into a `seq` then passed to the importer as such. In reality, blocks are still processed one by one, so the batching does not offer any performance advantage. It does however require that the client wastes memory, up to several GB, on the block sequence while they're waiting to be processed. This PR introduces a persister that accepts these potentially large blocks one by one and at the same time removes a number of redundant / unnecessary copies, assignments and resets that were slowing down the import process in general.
54 lines
1.8 KiB
Nim
54 lines
1.8 KiB
Nim
# Nimbus
|
|
# Copyright (c) 2021-2024 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or distributed except
|
|
# according to those terms.
|
|
|
|
import results, eth/common, ../../nimbus/db/era1_db
|
|
|
|
var noisy* = false
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public undump
|
|
# ------------------------------------------------------------------------------
|
|
|
|
iterator undumpBlocksEra1*(
|
|
dir: string,
|
|
least = low(uint64), # First block to extract
|
|
stopAfter = high(uint64), # Last block to extract
|
|
doAssertOk = false;
|
|
): seq[EthBlock] =
|
|
let db = Era1DbRef.init(dir, "mainnet").expect("Era files present")
|
|
defer:
|
|
db.dispose()
|
|
|
|
# TODO it would be a lot more natural for this iterator to return 1 block at
|
|
# a time and let the consumer do the chunking
|
|
const blocksPerYield = 192
|
|
var tmp = newSeqOfCap[EthBlock](blocksPerYield)
|
|
var blk: Block
|
|
for i in 0 ..< stopAfter:
|
|
db.getEthBlock(least + i, blk).isOkOr:
|
|
if doAssertOk:
|
|
doAssert i > 0, "expected at least one block"
|
|
break
|
|
|
|
tmp.add move(blk)
|
|
|
|
# Genesis block requires a chunk of its own, for compatibility with current
|
|
# test setup (a bit weird, that...)
|
|
if tmp.len mod blocksPerYield == 0 or tmp[0].header.number == 0:
|
|
yield tmp
|
|
tmp.setLen(0)
|
|
|
|
if tmp.len > 0:
|
|
yield tmp
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|