diff --git a/fluffy/tools/eth_data_exporter.nim b/fluffy/tools/eth_data_exporter.nim index 9d505f026..f750a8c32 100644 --- a/fluffy/tools/eth_data_exporter.nim +++ b/fluffy/tools/eth_data_exporter.nim @@ -50,10 +50,10 @@ import json_rpc/rpcclient, snappy, ncli/e2store, - ../../premix/[downloader, parser], ../network/history/[history_content, validation/historical_hashes_accumulator], ../eth_data/[history_data_json_store, history_data_ssz_e2s, era1], - eth_data_exporter/[exporter_conf, exporter_common, cl_data_exporter] + eth_data_exporter/[exporter_conf, exporter_common, cl_data_exporter], + eth_data_exporter/[downloader, parser] from eth/common/eth_types_rlp import rlpHash # Need to be selective due to the `Block` type conflict from downloader @@ -64,7 +64,7 @@ chronicles.formatIt(IoErrorCode): proc downloadHeader(client: RpcClient, i: uint64): headers.Header = try: - let jsonHeader = requestHeader(i, some(client)) + let jsonHeader = requestHeader(i, client) parseBlockHeader(jsonHeader) except CatchableError as e: fatal "Error while requesting BlockHeader", error = e.msg, number = i @@ -72,13 +72,13 @@ proc downloadHeader(client: RpcClient, i: uint64): headers.Header = proc downloadBlock(i: uint64, client: RpcClient): downloader.Block = try: - return requestBlock(i, flags = {DownloadReceipts}, client = some(client)) + return requestBlock(i, client) except CatchableError as e: fatal "Error while requesting Block", error = e.msg, number = i quit 1 proc writeHeadersToJson(config: ExporterConf, client: RpcClient) = - let fh = createAndOpenFile(string config.dataDir, string config.fileName) + let fh = createAndOpenFile(string config.dataDir, config.fileName) try: var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true) @@ -101,7 +101,7 @@ proc writeHeadersToJson(config: ExporterConf, client: RpcClient) = quit 1 proc writeBlocksToJson(config: ExporterConf, client: RpcClient) = - let fh = createAndOpenFile(string config.dataDir, string config.fileName) + let fh = createAndOpenFile(string config.dataDir, config.fileName) try: var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true) @@ -208,7 +208,7 @@ proc cmdExportEra1(config: ExporterConf) = # TODO: Not sure about the errors that can occur here. But the whole # block requests over json-rpc should be reworked here (and can be # used in the bridge also then) - requestBlock(blockNumber, flags = {DownloadReceipts}, client = some(client)) + requestBlock(blockNumber, client) except CatchableError as e: error "Failed retrieving block, skip creation of era1 file", blockNumber, era, error = e.msg diff --git a/fluffy/tools/eth_data_exporter/downloader.nim b/fluffy/tools/eth_data_exporter/downloader.nim new file mode 100644 index 000000000..0c31ac5d3 --- /dev/null +++ b/fluffy/tools/eth_data_exporter/downloader.nim @@ -0,0 +1,79 @@ +# Nimbus +# Copyright (c) 2020-2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +import + std/[json, strutils], + json_rpc/[rpcclient], + eth/common/blocks, + eth/common/receipts, + chronicles, + ./parser + +logScope: + topics = "downloader" + +type Block* = object + header*: Header + body*: BlockBody + receipts*: seq[Receipt] + jsonData*: JsonNode + +proc request*(methodName: string, params: JsonNode, client: RpcClient): JsonNode = + let res = waitFor client.call(methodName, params) + JrpcConv.decode(res.string, JsonNode) + +proc requestBlockBody( + n: JsonNode, blockNumber: BlockNumber, client: RpcClient +): BlockBody = + let txs = n["transactions"] + if txs.len > 0: + result.transactions = newSeqOfCap[Transaction](txs.len) + for tx in txs: + let txn = parseTransaction(tx) + validateTxSenderAndHash(tx, txn) + result.transactions.add txn + + let uncles = n["uncles"] + if uncles.len > 0: + result.uncles = newSeqOfCap[Header](uncles.len) + let blockNumber = blockNumber.to0xHex + for i in 0 ..< uncles.len: + let idx = i.to0xHex + let uncle = + request("eth_getUncleByBlockNumberAndIndex", %[%blockNumber, %idx], client) + if uncle.kind == JNull: + error "requested uncle not available", blockNumber = blockNumber, uncleIdx = i + raise newException(ValueError, "Error when retrieving block uncles") + result.uncles.add parseBlockHeader(uncle) + +proc requestReceipts(n: JsonNode, client: RpcClient): seq[Receipt] = + let txs = n["transactions"] + if txs.len > 0: + result = newSeqOfCap[Receipt](txs.len) + for tx in txs: + let txHash = tx["hash"] + let rec = request("eth_getTransactionReceipt", %[txHash], client) + if rec.kind == JNull: + error "requested receipt not available", txHash = txHash + raise newException(ValueError, "Error when retrieving block receipts") + result.add parseReceipt(rec) + +proc requestHeader*(blockNumber: BlockNumber, client: RpcClient): JsonNode = + result = request("eth_getBlockByNumber", %[%blockNumber.to0xHex, %true], client) + if result.kind == JNull: + error "requested block not available", blockNumber = blockNumber + raise newException(ValueError, "Error when retrieving block header") + +proc requestBlock*(blockNumber: BlockNumber, client: RpcClient): Block = + let header = requestHeader(blockNumber, client) + result.jsonData = header + result.header = parseBlockHeader(header) + result.body = requestBlockBody(header, blockNumber, client) + result.receipts = requestReceipts(header, client) diff --git a/premix/parser.nim b/fluffy/tools/eth_data_exporter/parser.nim similarity index 91% rename from premix/parser.nim rename to fluffy/tools/eth_data_exporter/parser.nim index 3271ad0a4..16ff4ce11 100644 --- a/premix/parser.nim +++ b/fluffy/tools/eth_data_exporter/parser.nim @@ -9,11 +9,14 @@ # according to those terms. import - json, strutils, os, eth/common/transaction_utils, - eth/common, httputils, nimcrypto/utils, - stint, stew/byteutils - -import ../nimbus/transaction, ../nimbus/utils/ec_recover + std/[json, strutils, os], + eth/common/transaction_utils, + eth/common/blocks, + eth/common/eth_types_rlp, + httputils, + nimcrypto/utils, + stint, + stew/byteutils from stew/objects import checkedEnumAssign @@ -30,7 +33,8 @@ func encodeQuantity(value: SomeUnsignedInt): string = func hexToInt*(s: string, T: typedesc[SomeInteger]): T = var i = 0 - if s[i] == '0' and (s[i+1] in {'x', 'X'}): inc(i, 2) + if s[i] == '0' and (s[i + 1] in {'x', 'X'}): + inc(i, 2) if s.len - i > sizeof(T) * 2: raise newException(ValueError, "input hex too big for destination int") while i < s.len: @@ -46,8 +50,7 @@ proc to0xHex*(x: UInt256): string = proc to0xHex*(x: string): string = "0x" & toLowerAscii(x) -type - SomeData* = EthAddress | BloomFilter | BlockNonce +type SomeData* = Address | Bloom | Bytes8 proc fromJson*(n: JsonNode, name: string, x: var SomeData) = let node = n[name] @@ -58,7 +61,7 @@ proc fromJson*(n: JsonNode, name: string, x: var SomeData) = hexToByteArray(node["value"].getStr(), x.data) doAssert(x.to0xHex == toLowerAscii(node["value"].getStr()), name) -proc fromJson*(n: JsonNode, name: string, x: var (Hash256|Bytes32)) = +proc fromJson*(n: JsonNode, name: string, x: var (Hash32 | Bytes32)) = let node = n[name] if node.kind == JString: hexToByteArray(node.getStr(), x.data) @@ -67,7 +70,7 @@ proc fromJson*(n: JsonNode, name: string, x: var (Hash256|Bytes32)) = hexToByteArray(node["value"].getStr(), x.data) doAssert(x.to0xHex == toLowerAscii(node["value"].getStr()), name) -proc fromJson*(n: JsonNode, name: string, x: var Blob) = +proc fromJson*(n: JsonNode, name: string, x: var seq[byte]) = x = hexToSeqByte(n[name].getStr()) doAssert(x.to0xHex == toLowerAscii(n[name].getStr()), name) @@ -107,7 +110,7 @@ proc fromJson*(n: JsonNode, name: string, x: var TxType) = else: x = hexToInt(node.getStr(), int).TxType -proc fromJson*[T: Bytes32|Hash32](n: JsonNode, name: string, x: var seq[T]) = +proc fromJson*[T: Bytes32 | Hash32](n: JsonNode, name: string, x: var seq[T]) = let node = n[name] var h: T x = newSeqOfCap[T](node.len) @@ -115,7 +118,7 @@ proc fromJson*[T: Bytes32|Hash32](n: JsonNode, name: string, x: var seq[T]) = hexToByteArray(v.getStr(), h.data) x.add h -proc parseBlockHeader*(n: JsonNode): BlockHeader = +proc parseBlockHeader*(n: JsonNode): Header = n.fromJson "parentHash", result.parentHash n.fromJson "sha3Uncles", result.ommersHash n.fromJson "miner", result.coinbase @@ -154,7 +157,7 @@ proc parseTransaction*(n: JsonNode): Transaction = n.fromJson "gas", tx.gasLimit if n["to"].kind != JNull: - var to: EthAddress + var to: Address n.fromJson "to", to tx.to = Opt.some(to) @@ -197,7 +200,7 @@ proc parseWithdrawal*(n: JsonNode): Withdrawal = proc validateTxSenderAndHash*(n: JsonNode, tx: Transaction) = var sender = tx.recoverSender().expect("valid signature") - var fromAddr: EthAddress + var fromAddr: Address n.fromJson "from", fromAddr doAssert sender.to0xHex == fromAddr.to0xHex doAssert n["hash"].getStr() == tx.rlpHash().to0xHex @@ -231,7 +234,7 @@ proc parseReceipt*(n: JsonNode): Receipt = raise newException(ValueError, "Unknown receipt type") if n.hasKey("root"): - var hash: Hash256 + var hash: Hash32 n.fromJson "root", hash rec.isHash = true rec.hash = hash @@ -246,7 +249,7 @@ proc parseReceipt*(n: JsonNode): Receipt = rec.logs = parseLogs(n["logs"]) rec -proc headerHash*(n: JsonNode): Hash256 = +proc headerHash*(n: JsonNode): Hash32 = n.fromJson "hash", result proc parseAccount*(n: JsonNode): Account = diff --git a/premix/downloader.nim b/premix/downloader.nim deleted file mode 100644 index a06712ac1..000000000 --- a/premix/downloader.nim +++ /dev/null @@ -1,150 +0,0 @@ -# Nimbus -# Copyright (c) 2020-2024 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or -# http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or -# http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -import - std/[json, strutils], - json_rpc/[rpcclient], httputils, - eth/common, chronicles, - ../nimbus/utils/utils, - ./parser - -logScope: - topics = "downloader" - -type - Block* = object - header*: BlockHeader - body*: BlockBody - traces*: JsonNode - receipts*: seq[Receipt] - jsonData*: JsonNode - - DownloadFlags* = enum - DownloadReceipts - DownloadTxTrace - DownloadAndValidate - -proc request*( - methodName: string, - params: JsonNode, - client: Option[RpcClient] = none[RpcClient]()): JsonNode = - if client.isSome(): - let res = waitFor client.unsafeGet().call(methodName, params) - result = JrpcConv.decode(res.string, JsonNode) - else: - var client = newRpcHttpClient() - #client.httpMethod(MethodPost) - waitFor client.connect("127.0.0.1", Port(8545), false) - let res = waitFor client.call(methodName, params) - result = JrpcConv.decode(res.string, JsonNode) - waitFor client.close() - -proc requestBlockBody( - n: JsonNode, - blockNumber: BlockNumber, - client: Option[RpcClient] = none[RpcClient]()): BlockBody = - let txs = n["transactions"] - if txs.len > 0: - result.transactions = newSeqOfCap[Transaction](txs.len) - for tx in txs: - let txn = parseTransaction(tx) - validateTxSenderAndHash(tx, txn) - result.transactions.add txn - - let uncles = n["uncles"] - if uncles.len > 0: - result.uncles = newSeqOfCap[BlockHeader](uncles.len) - let blockNumber = blockNumber.to0xHex - for i in 0 ..< uncles.len: - let idx = i.to0xHex - let uncle = request("eth_getUncleByBlockNumberAndIndex", %[%blockNumber, %idx], client) - if uncle.kind == JNull: - error "requested uncle not available", blockNumber=blockNumber, uncleIdx=i - raise newException(ValueError, "Error when retrieving block uncles") - result.uncles.add parseBlockHeader(uncle) - -proc requestReceipts( - n: JsonNode, - client: Option[RpcClient] = none[RpcClient]()): seq[Receipt] = - let txs = n["transactions"] - if txs.len > 0: - result = newSeqOfCap[Receipt](txs.len) - for tx in txs: - let txHash = tx["hash"] - let rec = request("eth_getTransactionReceipt", %[txHash], client) - if rec.kind == JNull: - error "requested receipt not available", txHash=txHash - raise newException(ValueError, "Error when retrieving block receipts") - result.add parseReceipt(rec) - -proc requestTxTraces( - n: JsonNode, - client: Option[RpcClient] = none[RpcClient]()): JsonNode = - result = newJArray() - let txs = n["transactions"] - if txs.len == 0: return - for tx in txs: - let txHash = tx["hash"] - let txTrace = request("debug_traceTransaction", %[txHash], client) - if txTrace.kind == JNull: - error "requested trace not available", txHash=txHash - raise newException(ValueError, "Error when retrieving transaction trace") - result.add txTrace - -proc requestHeader*( - blockNumber: BlockNumber, - client: Option[RpcClient] = none[RpcClient]()): JsonNode = - result = request("eth_getBlockByNumber", %[%blockNumber.to0xHex, %true], client) - if result.kind == JNull: - error "requested block not available", blockNumber=blockNumber - raise newException(ValueError, "Error when retrieving block header") - -proc requestBlock*( - blockNumber: BlockNumber, - flags: set[DownloadFlags] = {}, - client: Option[RpcClient] = none[RpcClient]()): Block = - let header = requestHeader(blockNumber, client) - result.jsonData = header - result.header = parseBlockHeader(header) - result.body = requestBlockBody(header, blockNumber, client) - - if DownloadTxTrace in flags: - result.traces = requestTxTraces(header, client) - - if DownloadReceipts in flags: - result.receipts = requestReceipts(header, client) - if DownloadAndValidate in flags: - let - receiptsRoot = calcReceiptsRoot(result.receipts).to0xHex - receiptsRootOK = result.header.receiptsRoot.to0xHex - if receiptsRoot != receiptsRootOK: - debug "wrong receipt root", receiptsRoot, receiptsRootOK, blockNumber - raise newException(ValueError, "Error when validating receipt root") - - if DownloadAndValidate in flags: - let - txRoot = calcTxRoot(result.body.transactions).to0xHex - txRootOK = result.header.txRoot.to0xHex - ommersHash = rlpHash(result.body.uncles).to0xHex - ommersHashOK = result.header.ommersHash.to0xHex - headerHash = rlpHash(result.header).to0xHex - headerHashOK = header["hash"].getStr().toLowerAscii - - if txRoot != txRootOK: - debug "wrong tx root", txRoot, txRootOK, blockNumber - raise newException(ValueError, "Error when validating tx root") - - if ommersHash != ommersHashOK: - debug "wrong ommers hash", ommersHash, ommersHashOK, blockNumber - raise newException(ValueError, "Error when validating ommers hash") - - if headerHash != headerHashOK: - debug "wrong header hash", headerHash, headerHashOK, blockNumber - raise newException(ValueError, "Error when validating block header hash")