2023-11-01 03:32:09 +00:00
|
|
|
# Nimbus
|
2024-02-01 14:11:41 +00:00
|
|
|
# Copyright (c) 2020-2024 Status Research & Development GmbH
|
2023-11-01 03:32:09 +00:00
|
|
|
# Licensed under either of
|
|
|
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
|
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
|
|
|
# http://opensource.org/licenses/MIT)
|
|
|
|
|
# at your option. This file may not be copied, modified, or distributed except
|
|
|
|
|
# according to those terms.
|
|
|
|
|
|
2019-01-06 15:21:34 +00:00
|
|
|
# use this module to quickly populate db with data from geth/parity
|
|
|
|
|
|
|
|
|
|
import
|
2024-02-01 14:11:41 +00:00
|
|
|
std/os,
|
2023-08-04 11:10:09 +00:00
|
|
|
chronicles,
|
2022-12-02 04:39:12 +00:00
|
|
|
../nimbus/errors,
|
|
|
|
|
../nimbus/core/chain,
|
2023-08-04 11:10:09 +00:00
|
|
|
../nimbus/common,
|
2024-06-05 15:08:29 +00:00
|
|
|
../nimbus/db/opts,
|
2023-08-04 11:10:09 +00:00
|
|
|
../nimbus/db/[core_db/persistent, storage_types],
|
|
|
|
|
configuration # must be late (compilation annoyance)
|
2019-01-06 15:21:34 +00:00
|
|
|
|
2021-09-14 09:49:31 +00:00
|
|
|
when defined(graphql):
|
|
|
|
|
import graphql_downloader
|
|
|
|
|
else:
|
|
|
|
|
import downloader
|
|
|
|
|
|
2023-08-04 11:10:09 +00:00
|
|
|
# `lmdb` is not used, anymore
|
|
|
|
|
#
|
|
|
|
|
# const
|
|
|
|
|
# manualCommit = nimbus_db_backend == "lmdb"
|
|
|
|
|
#
|
|
|
|
|
# template persistToDb(db: ChainDB, body: untyped) =
|
|
|
|
|
# when manualCommit:
|
|
|
|
|
# if not db.txBegin(): doAssert(false)
|
|
|
|
|
# body
|
|
|
|
|
# when manualCommit:
|
|
|
|
|
# if not db.txCommit(): doAssert(false)
|
|
|
|
|
|
|
|
|
|
template persistToDb(db: CoreDbRef, body: untyped) =
|
|
|
|
|
block: body
|
2024-02-01 14:11:41 +00:00
|
|
|
|
2024-06-05 20:52:04 +00:00
|
|
|
proc contains(kvt: CoreDxKvtRef; key: openArray[byte]): bool =
|
|
|
|
|
kvt.hasKey(key).expect "valid bool"
|
|
|
|
|
|
2020-07-21 06:15:06 +00:00
|
|
|
proc main() {.used.} =
|
2019-01-06 15:21:34 +00:00
|
|
|
# 97 block with uncles
|
|
|
|
|
# 46147 block with first transaction
|
|
|
|
|
# 46400 block with transaction
|
|
|
|
|
# 46402 block with first contract: failed
|
|
|
|
|
# 47205 block with first success contract
|
|
|
|
|
# 48712 block with 5 transactions
|
|
|
|
|
# 48915 block with contract
|
|
|
|
|
# 49018 first problematic block
|
|
|
|
|
# 49439 first block with contract call
|
|
|
|
|
# 52029 first block with receipts logs
|
|
|
|
|
# 66407 failed transaction
|
|
|
|
|
|
2020-06-19 10:52:19 +00:00
|
|
|
let conf = configuration.getConfiguration()
|
2023-08-04 11:10:09 +00:00
|
|
|
let com = CommonRef.new(
|
2024-06-05 15:08:29 +00:00
|
|
|
newCoreDbRef(DefaultDbPersistent, conf.dataDir, DbOptions.init()),
|
2024-05-20 10:17:51 +00:00
|
|
|
conf.netId, networkParams(conf.netId))
|
2019-01-06 15:21:34 +00:00
|
|
|
|
|
|
|
|
# move head to block number ...
|
2019-01-08 11:29:56 +00:00
|
|
|
if conf.head != 0.u256:
|
2024-02-01 14:11:41 +00:00
|
|
|
var parentBlock = requestBlock(conf.head, { DownloadAndValidate })
|
2022-12-02 04:39:12 +00:00
|
|
|
discard com.db.setHead(parentBlock.header)
|
2019-01-06 15:21:34 +00:00
|
|
|
|
2024-06-05 20:52:04 +00:00
|
|
|
let kvt = com.db.newKvt()
|
|
|
|
|
if canonicalHeadHashKey().toOpenArray notin kvt:
|
2023-08-04 11:10:09 +00:00
|
|
|
persistToDb(com.db):
|
2022-12-02 04:39:12 +00:00
|
|
|
com.initializeEmptyDb()
|
2024-06-05 20:52:04 +00:00
|
|
|
doAssert(canonicalHeadHashKey().toOpenArray in kvt)
|
2019-01-06 15:21:34 +00:00
|
|
|
|
2022-12-02 04:39:12 +00:00
|
|
|
var head = com.db.getCanonicalHead()
|
2019-01-06 15:21:34 +00:00
|
|
|
var blockNumber = head.blockNumber + 1
|
2022-12-02 04:39:12 +00:00
|
|
|
var chain = newChain(com)
|
2019-01-06 15:21:34 +00:00
|
|
|
|
2019-01-08 11:29:56 +00:00
|
|
|
let numBlocksToCommit = conf.numCommits
|
2019-01-06 15:21:34 +00:00
|
|
|
|
|
|
|
|
var headers = newSeqOfCap[BlockHeader](numBlocksToCommit)
|
|
|
|
|
var bodies = newSeqOfCap[BlockBody](numBlocksToCommit)
|
|
|
|
|
var one = 1.u256
|
|
|
|
|
|
|
|
|
|
var numBlocks = 0
|
2019-01-08 11:29:56 +00:00
|
|
|
var counter = 0
|
2024-02-01 14:11:41 +00:00
|
|
|
var retryCount = 0
|
2019-01-08 11:29:56 +00:00
|
|
|
|
2019-07-07 10:12:01 +00:00
|
|
|
while true:
|
2024-02-01 14:11:41 +00:00
|
|
|
var thisBlock: Block
|
|
|
|
|
try:
|
|
|
|
|
thisBlock = requestBlock(blockNumber, { DownloadAndValidate })
|
|
|
|
|
except CatchableError as e:
|
|
|
|
|
if retryCount < 3:
|
|
|
|
|
warn "Unable to get block data via JSON-RPC API", error = e.msg
|
|
|
|
|
inc retryCount
|
|
|
|
|
sleep(1000)
|
|
|
|
|
continue
|
|
|
|
|
else:
|
|
|
|
|
raise e
|
2019-01-06 15:21:34 +00:00
|
|
|
|
|
|
|
|
headers.add thisBlock.header
|
|
|
|
|
bodies.add thisBlock.body
|
2019-03-11 13:21:09 +00:00
|
|
|
info "REQUEST HEADER", blockNumber=blockNumber, txs=thisBlock.body.transactions.len
|
2019-07-07 10:12:01 +00:00
|
|
|
|
2019-01-06 15:21:34 +00:00
|
|
|
inc numBlocks
|
|
|
|
|
blockNumber += one
|
|
|
|
|
|
|
|
|
|
if numBlocks == numBlocksToCommit:
|
2023-08-04 11:10:09 +00:00
|
|
|
persistToDb(com.db):
|
2024-05-31 07:13:56 +00:00
|
|
|
let res = chain.persistBlocks(headers, bodies)
|
|
|
|
|
res.isOkOr:
|
|
|
|
|
raise newException(ValidationError, "Error when validating blocks: " & res.error)
|
2019-01-06 15:21:34 +00:00
|
|
|
numBlocks = 0
|
|
|
|
|
headers.setLen(0)
|
|
|
|
|
bodies.setLen(0)
|
|
|
|
|
|
2019-01-08 11:29:56 +00:00
|
|
|
inc counter
|
|
|
|
|
if conf.maxBlocks != 0 and counter >= conf.maxBlocks:
|
|
|
|
|
break
|
|
|
|
|
|
2019-01-06 15:21:34 +00:00
|
|
|
if numBlocks > 0:
|
2023-08-04 11:10:09 +00:00
|
|
|
persistToDb(com.db):
|
2024-05-31 07:13:56 +00:00
|
|
|
let res = chain.persistBlocks(headers, bodies)
|
|
|
|
|
res.isOkOr:
|
|
|
|
|
raise newException(ValidationError, "Error when validating blocks: " & res.error)
|
2019-01-06 15:21:34 +00:00
|
|
|
|
2019-01-08 11:29:56 +00:00
|
|
|
when isMainModule:
|
|
|
|
|
var message: string
|
|
|
|
|
|
|
|
|
|
## Processing command line arguments
|
2020-06-19 10:52:19 +00:00
|
|
|
if configuration.processArguments(message) != Success:
|
2024-02-01 14:11:41 +00:00
|
|
|
if len(message) > 0:
|
|
|
|
|
echo message
|
|
|
|
|
echo "Usage: persist --datadir=<DATA_DIR> --maxblocks=<MAX_BLOCKS> --head=<HEAD> --numcommits=<NUM_COMMITS> --netid=<NETWORK_ID>"
|
2019-01-08 11:29:56 +00:00
|
|
|
quit(QuitFailure)
|
|
|
|
|
else:
|
|
|
|
|
if len(message) > 0:
|
|
|
|
|
echo message
|
|
|
|
|
quit(QuitSuccess)
|
|
|
|
|
|
2019-01-09 12:10:58 +00:00
|
|
|
try:
|
|
|
|
|
main()
|
2023-08-04 11:10:09 +00:00
|
|
|
except CatchableError:
|
2019-01-09 12:10:58 +00:00
|
|
|
echo getCurrentExceptionMsg()
|
