Move premix's downloader and parser to fluffy/tools/eth_data_exporter (#2709)
* Move premix's downloader and parser to fluffy/tools/eth_data_exporter * Run nph formatter
This commit is contained in:
andri lim
GitHub
parent
76c2a75a53
commit
22134fcf4c
|
|
@ -50,10 +50,10 @@ import
|
|||
json_rpc/rpcclient,
|
||||
snappy,
|
||||
ncli/e2store,
|
||||
../../premix/[downloader, parser],
|
||||
../network/history/[history_content, validation/historical_hashes_accumulator],
|
||||
../eth_data/[history_data_json_store, history_data_ssz_e2s, era1],
|
||||
eth_data_exporter/[exporter_conf, exporter_common, cl_data_exporter]
|
||||
eth_data_exporter/[exporter_conf, exporter_common, cl_data_exporter],
|
||||
eth_data_exporter/[downloader, parser]
|
||||
|
||||
from eth/common/eth_types_rlp import rlpHash
|
||||
# Need to be selective due to the `Block` type conflict from downloader
|
||||
|
|
@ -64,7 +64,7 @@ chronicles.formatIt(IoErrorCode):
|
|||
|
||||
proc downloadHeader(client: RpcClient, i: uint64): headers.Header =
|
||||
try:
|
||||
let jsonHeader = requestHeader(i, some(client))
|
||||
let jsonHeader = requestHeader(i, client)
|
||||
parseBlockHeader(jsonHeader)
|
||||
except CatchableError as e:
|
||||
fatal "Error while requesting BlockHeader", error = e.msg, number = i
|
||||
|
|
@ -72,13 +72,13 @@ proc downloadHeader(client: RpcClient, i: uint64): headers.Header =
|
|||
|
||||
proc downloadBlock(i: uint64, client: RpcClient): downloader.Block =
|
||||
try:
|
||||
return requestBlock(i, flags = {DownloadReceipts}, client = some(client))
|
||||
return requestBlock(i, client)
|
||||
except CatchableError as e:
|
||||
fatal "Error while requesting Block", error = e.msg, number = i
|
||||
quit 1
|
||||
|
||||
proc writeHeadersToJson(config: ExporterConf, client: RpcClient) =
|
||||
let fh = createAndOpenFile(string config.dataDir, string config.fileName)
|
||||
let fh = createAndOpenFile(string config.dataDir, config.fileName)
|
||||
|
||||
try:
|
||||
var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true)
|
||||
|
|
@ -101,7 +101,7 @@ proc writeHeadersToJson(config: ExporterConf, client: RpcClient) =
|
|||
quit 1
|
||||
|
||||
proc writeBlocksToJson(config: ExporterConf, client: RpcClient) =
|
||||
let fh = createAndOpenFile(string config.dataDir, string config.fileName)
|
||||
let fh = createAndOpenFile(string config.dataDir, config.fileName)
|
||||
|
||||
try:
|
||||
var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true)
|
||||
|
|
@ -208,7 +208,7 @@ proc cmdExportEra1(config: ExporterConf) =
|
|||
# TODO: Not sure about the errors that can occur here. But the whole
|
||||
# block requests over json-rpc should be reworked here (and can be
|
||||
# used in the bridge also then)
|
||||
requestBlock(blockNumber, flags = {DownloadReceipts}, client = some(client))
|
||||
requestBlock(blockNumber, client)
|
||||
except CatchableError as e:
|
||||
error "Failed retrieving block, skip creation of era1 file",
|
||||
blockNumber, era, error = e.msg
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
# Nimbus
|
||||
# Copyright (c) 2020-2024 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed except
|
||||
# according to those terms.
|
||||
|
||||
import
|
||||
std/[json, strutils],
|
||||
json_rpc/[rpcclient],
|
||||
eth/common/blocks,
|
||||
eth/common/receipts,
|
||||
chronicles,
|
||||
./parser
|
||||
|
||||
logScope:
|
||||
topics = "downloader"
|
||||
|
||||
type Block* = object
|
||||
header*: Header
|
||||
body*: BlockBody
|
||||
receipts*: seq[Receipt]
|
||||
jsonData*: JsonNode
|
||||
|
||||
proc request*(methodName: string, params: JsonNode, client: RpcClient): JsonNode =
|
||||
let res = waitFor client.call(methodName, params)
|
||||
JrpcConv.decode(res.string, JsonNode)
|
||||
|
||||
proc requestBlockBody(
|
||||
n: JsonNode, blockNumber: BlockNumber, client: RpcClient
|
||||
): BlockBody =
|
||||
let txs = n["transactions"]
|
||||
if txs.len > 0:
|
||||
result.transactions = newSeqOfCap[Transaction](txs.len)
|
||||
for tx in txs:
|
||||
let txn = parseTransaction(tx)
|
||||
validateTxSenderAndHash(tx, txn)
|
||||
result.transactions.add txn
|
||||
|
||||
let uncles = n["uncles"]
|
||||
if uncles.len > 0:
|
||||
result.uncles = newSeqOfCap[Header](uncles.len)
|
||||
let blockNumber = blockNumber.to0xHex
|
||||
for i in 0 ..< uncles.len:
|
||||
let idx = i.to0xHex
|
||||
let uncle =
|
||||
request("eth_getUncleByBlockNumberAndIndex", %[%blockNumber, %idx], client)
|
||||
if uncle.kind == JNull:
|
||||
error "requested uncle not available", blockNumber = blockNumber, uncleIdx = i
|
||||
raise newException(ValueError, "Error when retrieving block uncles")
|
||||
result.uncles.add parseBlockHeader(uncle)
|
||||
|
||||
proc requestReceipts(n: JsonNode, client: RpcClient): seq[Receipt] =
|
||||
let txs = n["transactions"]
|
||||
if txs.len > 0:
|
||||
result = newSeqOfCap[Receipt](txs.len)
|
||||
for tx in txs:
|
||||
let txHash = tx["hash"]
|
||||
let rec = request("eth_getTransactionReceipt", %[txHash], client)
|
||||
if rec.kind == JNull:
|
||||
error "requested receipt not available", txHash = txHash
|
||||
raise newException(ValueError, "Error when retrieving block receipts")
|
||||
result.add parseReceipt(rec)
|
||||
|
||||
proc requestHeader*(blockNumber: BlockNumber, client: RpcClient): JsonNode =
|
||||
result = request("eth_getBlockByNumber", %[%blockNumber.to0xHex, %true], client)
|
||||
if result.kind == JNull:
|
||||
error "requested block not available", blockNumber = blockNumber
|
||||
raise newException(ValueError, "Error when retrieving block header")
|
||||
|
||||
proc requestBlock*(blockNumber: BlockNumber, client: RpcClient): Block =
|
||||
let header = requestHeader(blockNumber, client)
|
||||
result.jsonData = header
|
||||
result.header = parseBlockHeader(header)
|
||||
result.body = requestBlockBody(header, blockNumber, client)
|
||||
result.receipts = requestReceipts(header, client)
|
||||
|
|
@ -9,11 +9,14 @@
|
|||
# according to those terms.
|
||||
|
||||
import
|
||||
json, strutils, os, eth/common/transaction_utils,
|
||||
eth/common, httputils, nimcrypto/utils,
|
||||
stint, stew/byteutils
|
||||
|
||||
import ../nimbus/transaction, ../nimbus/utils/ec_recover
|
||||
std/[json, strutils, os],
|
||||
eth/common/transaction_utils,
|
||||
eth/common/blocks,
|
||||
eth/common/eth_types_rlp,
|
||||
httputils,
|
||||
nimcrypto/utils,
|
||||
stint,
|
||||
stew/byteutils
|
||||
|
||||
from stew/objects import checkedEnumAssign
|
||||
|
||||
|
|
@ -30,7 +33,8 @@ func encodeQuantity(value: SomeUnsignedInt): string =
|
|||
|
||||
func hexToInt*(s: string, T: typedesc[SomeInteger]): T =
|
||||
var i = 0
|
||||
if s[i] == '0' and (s[i+1] in {'x', 'X'}): inc(i, 2)
|
||||
if s[i] == '0' and (s[i + 1] in {'x', 'X'}):
|
||||
inc(i, 2)
|
||||
if s.len - i > sizeof(T) * 2:
|
||||
raise newException(ValueError, "input hex too big for destination int")
|
||||
while i < s.len:
|
||||
|
|
@ -46,8 +50,7 @@ proc to0xHex*(x: UInt256): string =
|
|||
proc to0xHex*(x: string): string =
|
||||
"0x" & toLowerAscii(x)
|
||||
|
||||
type
|
||||
SomeData* = EthAddress | BloomFilter | BlockNonce
|
||||
type SomeData* = Address | Bloom | Bytes8
|
||||
|
||||
proc fromJson*(n: JsonNode, name: string, x: var SomeData) =
|
||||
let node = n[name]
|
||||
|
|
@ -58,7 +61,7 @@ proc fromJson*(n: JsonNode, name: string, x: var SomeData) =
|
|||
hexToByteArray(node["value"].getStr(), x.data)
|
||||
doAssert(x.to0xHex == toLowerAscii(node["value"].getStr()), name)
|
||||
|
||||
proc fromJson*(n: JsonNode, name: string, x: var (Hash256|Bytes32)) =
|
||||
proc fromJson*(n: JsonNode, name: string, x: var (Hash32 | Bytes32)) =
|
||||
let node = n[name]
|
||||
if node.kind == JString:
|
||||
hexToByteArray(node.getStr(), x.data)
|
||||
|
|
@ -67,7 +70,7 @@ proc fromJson*(n: JsonNode, name: string, x: var (Hash256|Bytes32)) =
|
|||
hexToByteArray(node["value"].getStr(), x.data)
|
||||
doAssert(x.to0xHex == toLowerAscii(node["value"].getStr()), name)
|
||||
|
||||
proc fromJson*(n: JsonNode, name: string, x: var Blob) =
|
||||
proc fromJson*(n: JsonNode, name: string, x: var seq[byte]) =
|
||||
x = hexToSeqByte(n[name].getStr())
|
||||
doAssert(x.to0xHex == toLowerAscii(n[name].getStr()), name)
|
||||
|
||||
|
|
@ -107,7 +110,7 @@ proc fromJson*(n: JsonNode, name: string, x: var TxType) =
|
|||
else:
|
||||
x = hexToInt(node.getStr(), int).TxType
|
||||
|
||||
proc fromJson*[T: Bytes32|Hash32](n: JsonNode, name: string, x: var seq[T]) =
|
||||
proc fromJson*[T: Bytes32 | Hash32](n: JsonNode, name: string, x: var seq[T]) =
|
||||
let node = n[name]
|
||||
var h: T
|
||||
x = newSeqOfCap[T](node.len)
|
||||
|
|
@ -115,7 +118,7 @@ proc fromJson*[T: Bytes32|Hash32](n: JsonNode, name: string, x: var seq[T]) =
|
|||
hexToByteArray(v.getStr(), h.data)
|
||||
x.add h
|
||||
|
||||
proc parseBlockHeader*(n: JsonNode): BlockHeader =
|
||||
proc parseBlockHeader*(n: JsonNode): Header =
|
||||
n.fromJson "parentHash", result.parentHash
|
||||
n.fromJson "sha3Uncles", result.ommersHash
|
||||
n.fromJson "miner", result.coinbase
|
||||
|
|
@ -154,7 +157,7 @@ proc parseTransaction*(n: JsonNode): Transaction =
|
|||
n.fromJson "gas", tx.gasLimit
|
||||
|
||||
if n["to"].kind != JNull:
|
||||
var to: EthAddress
|
||||
var to: Address
|
||||
n.fromJson "to", to
|
||||
tx.to = Opt.some(to)
|
||||
|
||||
|
|
@ -197,7 +200,7 @@ proc parseWithdrawal*(n: JsonNode): Withdrawal =
|
|||
|
||||
proc validateTxSenderAndHash*(n: JsonNode, tx: Transaction) =
|
||||
var sender = tx.recoverSender().expect("valid signature")
|
||||
var fromAddr: EthAddress
|
||||
var fromAddr: Address
|
||||
n.fromJson "from", fromAddr
|
||||
doAssert sender.to0xHex == fromAddr.to0xHex
|
||||
doAssert n["hash"].getStr() == tx.rlpHash().to0xHex
|
||||
|
|
@ -231,7 +234,7 @@ proc parseReceipt*(n: JsonNode): Receipt =
|
|||
raise newException(ValueError, "Unknown receipt type")
|
||||
|
||||
if n.hasKey("root"):
|
||||
var hash: Hash256
|
||||
var hash: Hash32
|
||||
n.fromJson "root", hash
|
||||
rec.isHash = true
|
||||
rec.hash = hash
|
||||
|
|
@ -246,7 +249,7 @@ proc parseReceipt*(n: JsonNode): Receipt =
|
|||
rec.logs = parseLogs(n["logs"])
|
||||
rec
|
||||
|
||||
proc headerHash*(n: JsonNode): Hash256 =
|
||||
proc headerHash*(n: JsonNode): Hash32 =
|
||||
n.fromJson "hash", result
|
||||
|
||||
proc parseAccount*(n: JsonNode): Account =
|
||||
|
|
@ -1,150 +0,0 @@
|
|||
# Nimbus
|
||||
# Copyright (c) 2020-2024 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed except
|
||||
# according to those terms.
|
||||
|
||||
import
|
||||
std/[json, strutils],
|
||||
json_rpc/[rpcclient], httputils,
|
||||
eth/common, chronicles,
|
||||
../nimbus/utils/utils,
|
||||
./parser
|
||||
|
||||
logScope:
|
||||
topics = "downloader"
|
||||
|
||||
type
|
||||
Block* = object
|
||||
header*: BlockHeader
|
||||
body*: BlockBody
|
||||
traces*: JsonNode
|
||||
receipts*: seq[Receipt]
|
||||
jsonData*: JsonNode
|
||||
|
||||
DownloadFlags* = enum
|
||||
DownloadReceipts
|
||||
DownloadTxTrace
|
||||
DownloadAndValidate
|
||||
|
||||
proc request*(
|
||||
methodName: string,
|
||||
params: JsonNode,
|
||||
client: Option[RpcClient] = none[RpcClient]()): JsonNode =
|
||||
if client.isSome():
|
||||
let res = waitFor client.unsafeGet().call(methodName, params)
|
||||
result = JrpcConv.decode(res.string, JsonNode)
|
||||
else:
|
||||
var client = newRpcHttpClient()
|
||||
#client.httpMethod(MethodPost)
|
||||
waitFor client.connect("127.0.0.1", Port(8545), false)
|
||||
let res = waitFor client.call(methodName, params)
|
||||
result = JrpcConv.decode(res.string, JsonNode)
|
||||
waitFor client.close()
|
||||
|
||||
proc requestBlockBody(
|
||||
n: JsonNode,
|
||||
blockNumber: BlockNumber,
|
||||
client: Option[RpcClient] = none[RpcClient]()): BlockBody =
|
||||
let txs = n["transactions"]
|
||||
if txs.len > 0:
|
||||
result.transactions = newSeqOfCap[Transaction](txs.len)
|
||||
for tx in txs:
|
||||
let txn = parseTransaction(tx)
|
||||
validateTxSenderAndHash(tx, txn)
|
||||
result.transactions.add txn
|
||||
|
||||
let uncles = n["uncles"]
|
||||
if uncles.len > 0:
|
||||
result.uncles = newSeqOfCap[BlockHeader](uncles.len)
|
||||
let blockNumber = blockNumber.to0xHex
|
||||
for i in 0 ..< uncles.len:
|
||||
let idx = i.to0xHex
|
||||
let uncle = request("eth_getUncleByBlockNumberAndIndex", %[%blockNumber, %idx], client)
|
||||
if uncle.kind == JNull:
|
||||
error "requested uncle not available", blockNumber=blockNumber, uncleIdx=i
|
||||
raise newException(ValueError, "Error when retrieving block uncles")
|
||||
result.uncles.add parseBlockHeader(uncle)
|
||||
|
||||
proc requestReceipts(
|
||||
n: JsonNode,
|
||||
client: Option[RpcClient] = none[RpcClient]()): seq[Receipt] =
|
||||
let txs = n["transactions"]
|
||||
if txs.len > 0:
|
||||
result = newSeqOfCap[Receipt](txs.len)
|
||||
for tx in txs:
|
||||
let txHash = tx["hash"]
|
||||
let rec = request("eth_getTransactionReceipt", %[txHash], client)
|
||||
if rec.kind == JNull:
|
||||
error "requested receipt not available", txHash=txHash
|
||||
raise newException(ValueError, "Error when retrieving block receipts")
|
||||
result.add parseReceipt(rec)
|
||||
|
||||
proc requestTxTraces(
|
||||
n: JsonNode,
|
||||
client: Option[RpcClient] = none[RpcClient]()): JsonNode =
|
||||
result = newJArray()
|
||||
let txs = n["transactions"]
|
||||
if txs.len == 0: return
|
||||
for tx in txs:
|
||||
let txHash = tx["hash"]
|
||||
let txTrace = request("debug_traceTransaction", %[txHash], client)
|
||||
if txTrace.kind == JNull:
|
||||
error "requested trace not available", txHash=txHash
|
||||
raise newException(ValueError, "Error when retrieving transaction trace")
|
||||
result.add txTrace
|
||||
|
||||
proc requestHeader*(
|
||||
blockNumber: BlockNumber,
|
||||
client: Option[RpcClient] = none[RpcClient]()): JsonNode =
|
||||
result = request("eth_getBlockByNumber", %[%blockNumber.to0xHex, %true], client)
|
||||
if result.kind == JNull:
|
||||
error "requested block not available", blockNumber=blockNumber
|
||||
raise newException(ValueError, "Error when retrieving block header")
|
||||
|
||||
proc requestBlock*(
|
||||
blockNumber: BlockNumber,
|
||||
flags: set[DownloadFlags] = {},
|
||||
client: Option[RpcClient] = none[RpcClient]()): Block =
|
||||
let header = requestHeader(blockNumber, client)
|
||||
result.jsonData = header
|
||||
result.header = parseBlockHeader(header)
|
||||
result.body = requestBlockBody(header, blockNumber, client)
|
||||
|
||||
if DownloadTxTrace in flags:
|
||||
result.traces = requestTxTraces(header, client)
|
||||
|
||||
if DownloadReceipts in flags:
|
||||
result.receipts = requestReceipts(header, client)
|
||||
if DownloadAndValidate in flags:
|
||||
let
|
||||
receiptsRoot = calcReceiptsRoot(result.receipts).to0xHex
|
||||
receiptsRootOK = result.header.receiptsRoot.to0xHex
|
||||
if receiptsRoot != receiptsRootOK:
|
||||
debug "wrong receipt root", receiptsRoot, receiptsRootOK, blockNumber
|
||||
raise newException(ValueError, "Error when validating receipt root")
|
||||
|
||||
if DownloadAndValidate in flags:
|
||||
let
|
||||
txRoot = calcTxRoot(result.body.transactions).to0xHex
|
||||
txRootOK = result.header.txRoot.to0xHex
|
||||
ommersHash = rlpHash(result.body.uncles).to0xHex
|
||||
ommersHashOK = result.header.ommersHash.to0xHex
|
||||
headerHash = rlpHash(result.header).to0xHex
|
||||
headerHashOK = header["hash"].getStr().toLowerAscii
|
||||
|
||||
if txRoot != txRootOK:
|
||||
debug "wrong tx root", txRoot, txRootOK, blockNumber
|
||||
raise newException(ValueError, "Error when validating tx root")
|
||||
|
||||
if ommersHash != ommersHashOK:
|
||||
debug "wrong ommers hash", ommersHash, ommersHashOK, blockNumber
|
||||
raise newException(ValueError, "Error when validating ommers hash")
|
||||
|
||||
if headerHash != headerHashOK:
|
||||
debug "wrong header hash", headerHash, headerHashOK, blockNumber
|
||||
raise newException(ValueError, "Error when validating block header hash")
|
||||
Loading…
Reference in New Issue