status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Move premix's downloader and parser to fluffy/tools/eth_data_exporter (#2709) 

* Move premix's downloader and parser to fluffy/tools/eth_data_exporter

* Run nph formatter
This commit is contained in:
andri lim 2024-10-08 10:07:24 +07:00 committed by GitHub
parent 76c2a75a53
commit 22134fcf4c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 105 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
fluffy/tools/eth_data_exporter.nim
View File

@ -50,10 +50,10 @@ import
json_rpc/rpcclient, json_rpc/rpcclient,
snappy, snappy,
ncli/e2store, ncli/e2store,
../../premix/[downloader, parser],
../network/history/[history_content, validation/historical_hashes_accumulator], ../network/history/[history_content, validation/historical_hashes_accumulator],
../eth_data/[history_data_json_store, history_data_ssz_e2s, era1], ../eth_data/[history_data_json_store, history_data_ssz_e2s, era1],
eth_data_exporter/[exporter_conf, exporter_common, cl_data_exporter] eth_data_exporter/[exporter_conf, exporter_common, cl_data_exporter],
eth_data_exporter/[downloader, parser]
from eth/common/eth_types_rlp import rlpHash from eth/common/eth_types_rlp import rlpHash
# Need to be selective due to the `Block` type conflict from downloader # Need to be selective due to the `Block` type conflict from downloader
@ -64,7 +64,7 @@ chronicles.formatIt(IoErrorCode):
proc downloadHeader(client: RpcClient, i: uint64): headers.Header = proc downloadHeader(client: RpcClient, i: uint64): headers.Header =
try: try:
let jsonHeader = requestHeader(i, some(client)) let jsonHeader = requestHeader(i, client)
parseBlockHeader(jsonHeader) parseBlockHeader(jsonHeader)
except CatchableError as e: except CatchableError as e:
fatal "Error while requesting BlockHeader", error = e.msg, number = i fatal "Error while requesting BlockHeader", error = e.msg, number = i
@ -72,13 +72,13 @@ proc downloadHeader(client: RpcClient, i: uint64): headers.Header =
proc downloadBlock(i: uint64, client: RpcClient): downloader.Block = proc downloadBlock(i: uint64, client: RpcClient): downloader.Block =
try: try:
return requestBlock(i, flags = {DownloadReceipts}, client = some(client)) return requestBlock(i, client)
except CatchableError as e: except CatchableError as e:
fatal "Error while requesting Block", error = e.msg, number = i fatal "Error while requesting Block", error = e.msg, number = i
quit 1 quit 1
proc writeHeadersToJson(config: ExporterConf, client: RpcClient) = proc writeHeadersToJson(config: ExporterConf, client: RpcClient) =
let fh = createAndOpenFile(string config.dataDir, string config.fileName) let fh = createAndOpenFile(string config.dataDir, config.fileName)
try: try:
var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true) var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true)
@ -101,7 +101,7 @@ proc writeHeadersToJson(config: ExporterConf, client: RpcClient) =
quit 1 quit 1
proc writeBlocksToJson(config: ExporterConf, client: RpcClient) = proc writeBlocksToJson(config: ExporterConf, client: RpcClient) =
let fh = createAndOpenFile(string config.dataDir, string config.fileName) let fh = createAndOpenFile(string config.dataDir, config.fileName)
try: try:
var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true) var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true)
@ -208,7 +208,7 @@ proc cmdExportEra1(config: ExporterConf) =
# TODO: Not sure about the errors that can occur here. But the whole # TODO: Not sure about the errors that can occur here. But the whole
# block requests over json-rpc should be reworked here (and can be # block requests over json-rpc should be reworked here (and can be
# used in the bridge also then) # used in the bridge also then)
requestBlock(blockNumber, flags = {DownloadReceipts}, client = some(client)) requestBlock(blockNumber, client)
except CatchableError as e: except CatchableError as e:
error "Failed retrieving block, skip creation of era1 file", error "Failed retrieving block, skip creation of era1 file",
blockNumber, era, error = e.msg blockNumber, era, error = e.msg

79
fluffy/tools/eth_data_exporter/downloader.nim Normal file
View File

@ -0,0 +1,79 @@
# Nimbus
# Copyright (c) 2020-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
import
std/[json, strutils],
json_rpc/[rpcclient],
eth/common/blocks,
eth/common/receipts,
chronicles,
./parser
logScope:
topics = "downloader"
type Block* = object
header*: Header
body*: BlockBody
receipts*: seq[Receipt]
jsonData*: JsonNode
proc request*(methodName: string, params: JsonNode, client: RpcClient): JsonNode =
let res = waitFor client.call(methodName, params)
JrpcConv.decode(res.string, JsonNode)
proc requestBlockBody(
n: JsonNode, blockNumber: BlockNumber, client: RpcClient
): BlockBody =
let txs = n["transactions"]
if txs.len > 0:
result.transactions = newSeqOfCap[Transaction](txs.len)
for tx in txs:
let txn = parseTransaction(tx)
validateTxSenderAndHash(tx, txn)
result.transactions.add txn
let uncles = n["uncles"]
if uncles.len > 0:
result.uncles = newSeqOfCap[Header](uncles.len)
let blockNumber = blockNumber.to0xHex
for i in 0 ..< uncles.len:
let idx = i.to0xHex
let uncle =
request("eth_getUncleByBlockNumberAndIndex", %[%blockNumber, %idx], client)
if uncle.kind == JNull:
error "requested uncle not available", blockNumber = blockNumber, uncleIdx = i
raise newException(ValueError, "Error when retrieving block uncles")
result.uncles.add parseBlockHeader(uncle)
proc requestReceipts(n: JsonNode, client: RpcClient): seq[Receipt] =
let txs = n["transactions"]
if txs.len > 0:
result = newSeqOfCap[Receipt](txs.len)
for tx in txs:
let txHash = tx["hash"]
let rec = request("eth_getTransactionReceipt", %[txHash], client)
if rec.kind == JNull:
error "requested receipt not available", txHash = txHash
raise newException(ValueError, "Error when retrieving block receipts")
result.add parseReceipt(rec)
proc requestHeader*(blockNumber: BlockNumber, client: RpcClient): JsonNode =
result = request("eth_getBlockByNumber", %[%blockNumber.to0xHex, %true], client)
if result.kind == JNull:
error "requested block not available", blockNumber = blockNumber
raise newException(ValueError, "Error when retrieving block header")
proc requestBlock*(blockNumber: BlockNumber, client: RpcClient): Block =
let header = requestHeader(blockNumber, client)
result.jsonData = header
result.header = parseBlockHeader(header)
result.body = requestBlockBody(header, blockNumber, client)
result.receipts = requestReceipts(header, client)

35
premix/parser.nim → fluffy/tools/eth_data_exporter/parser.nim
View File

@ -9,11 +9,14 @@
# according to those terms. # according to those terms.
import import
json, strutils, os, eth/common/transaction_utils, std/[json, strutils, os],
eth/common, httputils, nimcrypto/utils, eth/common/transaction_utils,
stint, stew/byteutils eth/common/blocks,
eth/common/eth_types_rlp,
import ../nimbus/transaction, ../nimbus/utils/ec_recover httputils,
nimcrypto/utils,
stint,
stew/byteutils
from stew/objects import checkedEnumAssign from stew/objects import checkedEnumAssign
@ -30,7 +33,8 @@ func encodeQuantity(value: SomeUnsignedInt): string =
func hexToInt*(s: string, T: typedesc[SomeInteger]): T = func hexToInt*(s: string, T: typedesc[SomeInteger]): T =
var i = 0 var i = 0
if s[i] == '0' and (s[i+1] in {'x', 'X'}): inc(i, 2) if s[i] == '0' and (s[i + 1] in {'x', 'X'}):
inc(i, 2)
if s.len - i > sizeof(T) * 2: if s.len - i > sizeof(T) * 2:
raise newException(ValueError, "input hex too big for destination int") raise newException(ValueError, "input hex too big for destination int")
while i < s.len: while i < s.len:
@ -46,8 +50,7 @@ proc to0xHex*(x: UInt256): string =
proc to0xHex*(x: string): string = proc to0xHex*(x: string): string =
"0x" & toLowerAscii(x) "0x" & toLowerAscii(x)
type type SomeData* = Address | Bloom | Bytes8
SomeData* = EthAddress | BloomFilter | BlockNonce
proc fromJson*(n: JsonNode, name: string, x: var SomeData) = proc fromJson*(n: JsonNode, name: string, x: var SomeData) =
let node = n[name] let node = n[name]
@ -58,7 +61,7 @@ proc fromJson*(n: JsonNode, name: string, x: var SomeData) =
hexToByteArray(node["value"].getStr(), x.data) hexToByteArray(node["value"].getStr(), x.data)
doAssert(x.to0xHex == toLowerAscii(node["value"].getStr()), name) doAssert(x.to0xHex == toLowerAscii(node["value"].getStr()), name)
proc fromJson*(n: JsonNode, name: string, x: var (Hash256|Bytes32)) = proc fromJson*(n: JsonNode, name: string, x: var (Hash32 | Bytes32)) =
let node = n[name] let node = n[name]
if node.kind == JString: if node.kind == JString:
hexToByteArray(node.getStr(), x.data) hexToByteArray(node.getStr(), x.data)
@ -67,7 +70,7 @@ proc fromJson*(n: JsonNode, name: string, x: var (Hash256|Bytes32)) =
hexToByteArray(node["value"].getStr(), x.data) hexToByteArray(node["value"].getStr(), x.data)
doAssert(x.to0xHex == toLowerAscii(node["value"].getStr()), name) doAssert(x.to0xHex == toLowerAscii(node["value"].getStr()), name)
proc fromJson*(n: JsonNode, name: string, x: var Blob) = proc fromJson*(n: JsonNode, name: string, x: var seq[byte]) =
x = hexToSeqByte(n[name].getStr()) x = hexToSeqByte(n[name].getStr())
doAssert(x.to0xHex == toLowerAscii(n[name].getStr()), name) doAssert(x.to0xHex == toLowerAscii(n[name].getStr()), name)
@ -107,7 +110,7 @@ proc fromJson*(n: JsonNode, name: string, x: var TxType) =
else: else:
x = hexToInt(node.getStr(), int).TxType x = hexToInt(node.getStr(), int).TxType
proc fromJson*[T: Bytes32|Hash32](n: JsonNode, name: string, x: var seq[T]) = proc fromJson*[T: Bytes32 | Hash32](n: JsonNode, name: string, x: var seq[T]) =
let node = n[name] let node = n[name]
var h: T var h: T
x = newSeqOfCap[T](node.len) x = newSeqOfCap[T](node.len)
@ -115,7 +118,7 @@ proc fromJson*[T: Bytes32|Hash32](n: JsonNode, name: string, x: var seq[T]) =
hexToByteArray(v.getStr(), h.data) hexToByteArray(v.getStr(), h.data)
x.add h x.add h
proc parseBlockHeader*(n: JsonNode): BlockHeader = proc parseBlockHeader*(n: JsonNode): Header =
n.fromJson "parentHash", result.parentHash n.fromJson "parentHash", result.parentHash
n.fromJson "sha3Uncles", result.ommersHash n.fromJson "sha3Uncles", result.ommersHash
n.fromJson "miner", result.coinbase n.fromJson "miner", result.coinbase
@ -154,7 +157,7 @@ proc parseTransaction*(n: JsonNode): Transaction =
n.fromJson "gas", tx.gasLimit n.fromJson "gas", tx.gasLimit
if n["to"].kind != JNull: if n["to"].kind != JNull:
var to: EthAddress var to: Address
n.fromJson "to", to n.fromJson "to", to
tx.to = Opt.some(to) tx.to = Opt.some(to)
@ -197,7 +200,7 @@ proc parseWithdrawal*(n: JsonNode): Withdrawal =
proc validateTxSenderAndHash*(n: JsonNode, tx: Transaction) = proc validateTxSenderAndHash*(n: JsonNode, tx: Transaction) =
var sender = tx.recoverSender().expect("valid signature") var sender = tx.recoverSender().expect("valid signature")
var fromAddr: EthAddress var fromAddr: Address
n.fromJson "from", fromAddr n.fromJson "from", fromAddr
doAssert sender.to0xHex == fromAddr.to0xHex doAssert sender.to0xHex == fromAddr.to0xHex
doAssert n["hash"].getStr() == tx.rlpHash().to0xHex doAssert n["hash"].getStr() == tx.rlpHash().to0xHex
@ -231,7 +234,7 @@ proc parseReceipt*(n: JsonNode): Receipt =
raise newException(ValueError, "Unknown receipt type") raise newException(ValueError, "Unknown receipt type")
if n.hasKey("root"): if n.hasKey("root"):
var hash: Hash256 var hash: Hash32
n.fromJson "root", hash n.fromJson "root", hash
rec.isHash = true rec.isHash = true
rec.hash = hash rec.hash = hash
@ -246,7 +249,7 @@ proc parseReceipt*(n: JsonNode): Receipt =
rec.logs = parseLogs(n["logs"]) rec.logs = parseLogs(n["logs"])
rec rec
proc headerHash*(n: JsonNode): Hash256 = proc headerHash*(n: JsonNode): Hash32 =
n.fromJson "hash", result n.fromJson "hash", result
proc parseAccount*(n: JsonNode): Account = proc parseAccount*(n: JsonNode): Account =

150
premix/downloader.nim
View File

@ -1,150 +0,0 @@
# Nimbus
# Copyright (c) 2020-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
import
std/[json, strutils],
json_rpc/[rpcclient], httputils,
eth/common, chronicles,
../nimbus/utils/utils,
./parser
logScope:
topics = "downloader"
type
Block* = object
header*: BlockHeader
body*: BlockBody
traces*: JsonNode
receipts*: seq[Receipt]
jsonData*: JsonNode
DownloadFlags* = enum
DownloadReceipts
DownloadTxTrace
DownloadAndValidate
proc request*(
methodName: string,
params: JsonNode,
client: Option[RpcClient] = none[RpcClient]()): JsonNode =
if client.isSome():
let res = waitFor client.unsafeGet().call(methodName, params)
result = JrpcConv.decode(res.string, JsonNode)
else:
var client = newRpcHttpClient()
#client.httpMethod(MethodPost)
waitFor client.connect("127.0.0.1", Port(8545), false)
let res = waitFor client.call(methodName, params)
result = JrpcConv.decode(res.string, JsonNode)
waitFor client.close()
proc requestBlockBody(
n: JsonNode,
blockNumber: BlockNumber,
client: Option[RpcClient] = none[RpcClient]()): BlockBody =
let txs = n["transactions"]
if txs.len > 0:
result.transactions = newSeqOfCap[Transaction](txs.len)
for tx in txs:
let txn = parseTransaction(tx)
validateTxSenderAndHash(tx, txn)
result.transactions.add txn
let uncles = n["uncles"]
if uncles.len > 0:
result.uncles = newSeqOfCap[BlockHeader](uncles.len)
let blockNumber = blockNumber.to0xHex
for i in 0 ..< uncles.len:
let idx = i.to0xHex
let uncle = request("eth_getUncleByBlockNumberAndIndex", %[%blockNumber, %idx], client)
if uncle.kind == JNull:
error "requested uncle not available", blockNumber=blockNumber, uncleIdx=i
raise newException(ValueError, "Error when retrieving block uncles")
result.uncles.add parseBlockHeader(uncle)
proc requestReceipts(
n: JsonNode,
client: Option[RpcClient] = none[RpcClient]()): seq[Receipt] =
let txs = n["transactions"]
if txs.len > 0:
result = newSeqOfCap[Receipt](txs.len)
for tx in txs:
let txHash = tx["hash"]
let rec = request("eth_getTransactionReceipt", %[txHash], client)
if rec.kind == JNull:
error "requested receipt not available", txHash=txHash
raise newException(ValueError, "Error when retrieving block receipts")
result.add parseReceipt(rec)
proc requestTxTraces(
n: JsonNode,
client: Option[RpcClient] = none[RpcClient]()): JsonNode =
result = newJArray()
let txs = n["transactions"]
if txs.len == 0: return
for tx in txs:
let txHash = tx["hash"]
let txTrace = request("debug_traceTransaction", %[txHash], client)
if txTrace.kind == JNull:
error "requested trace not available", txHash=txHash
raise newException(ValueError, "Error when retrieving transaction trace")
result.add txTrace
proc requestHeader*(
blockNumber: BlockNumber,
client: Option[RpcClient] = none[RpcClient]()): JsonNode =
result = request("eth_getBlockByNumber", %[%blockNumber.to0xHex, %true], client)
if result.kind == JNull:
error "requested block not available", blockNumber=blockNumber
raise newException(ValueError, "Error when retrieving block header")
proc requestBlock*(
blockNumber: BlockNumber,
flags: set[DownloadFlags] = {},
client: Option[RpcClient] = none[RpcClient]()): Block =
let header = requestHeader(blockNumber, client)
result.jsonData = header
result.header = parseBlockHeader(header)
result.body = requestBlockBody(header, blockNumber, client)
if DownloadTxTrace in flags:
result.traces = requestTxTraces(header, client)
if DownloadReceipts in flags:
result.receipts = requestReceipts(header, client)
if DownloadAndValidate in flags:
let
receiptsRoot = calcReceiptsRoot(result.receipts).to0xHex
receiptsRootOK = result.header.receiptsRoot.to0xHex
if receiptsRoot != receiptsRootOK:
debug "wrong receipt root", receiptsRoot, receiptsRootOK, blockNumber
raise newException(ValueError, "Error when validating receipt root")
if DownloadAndValidate in flags:
let
txRoot = calcTxRoot(result.body.transactions).to0xHex
txRootOK = result.header.txRoot.to0xHex
ommersHash = rlpHash(result.body.uncles).to0xHex
ommersHashOK = result.header.ommersHash.to0xHex
headerHash = rlpHash(result.header).to0xHex
headerHashOK = header["hash"].getStr().toLowerAscii
if txRoot != txRootOK:
debug "wrong tx root", txRoot, txRootOK, blockNumber
raise newException(ValueError, "Error when validating tx root")
if ommersHash != ommersHashOK:
debug "wrong ommers hash", ommersHash, ommersHashOK, blockNumber
raise newException(ValueError, "Error when validating ommers hash")
if headerHash != headerHashOK:
debug "wrong header hash", headerHash, headerHashOK, blockNumber
raise newException(ValueError, "Error when validating block header hash")