From 90f91e5ebd4c395c2eb2514f51a71bd6b757ff21 Mon Sep 17 00:00:00 2001 From: Kim De Mey Date: Thu, 5 Jan 2023 15:26:58 +0100 Subject: [PATCH] Reorganize eth data files parsing code (#1407) --- .../history_data_json_store.nim} | 94 +++++++------------ .../history_data_seeding.nim | 4 +- fluffy/eth_data/history_data_ssz_e2s.nim | 81 ++++++++++++++++ fluffy/fluffy.nim | 4 +- fluffy/rpc/rpc_portal_debug_api.nim | 1 + fluffy/scripts/test_portal_testnet.nim | 7 +- .../mainnet/test_accumulator_root.nim | 4 +- fluffy/tests/test_accumulator.nim | 4 +- fluffy/tests/test_history_validation.nim | 4 +- fluffy/tools/eth_data_exporter.nim | 42 +-------- 10 files changed, 136 insertions(+), 109 deletions(-) rename fluffy/{data/history_data_parser.nim => eth_data/history_data_json_store.nim} (69%) rename fluffy/{data => eth_data}/history_data_seeding.nim (98%) create mode 100644 fluffy/eth_data/history_data_ssz_e2s.nim diff --git a/fluffy/data/history_data_parser.nim b/fluffy/eth_data/history_data_json_store.nim similarity index 69% rename from fluffy/data/history_data_parser.nim rename to fluffy/eth_data/history_data_json_store.nim index cecf3ec49..9bec2bcd0 100644 --- a/fluffy/data/history_data_parser.nim +++ b/fluffy/eth_data/history_data_json_store.nim @@ -1,5 +1,5 @@ # Nimbus - Portal Network -# Copyright (c) 2022 Status Research & Development GmbH +# Copyright (c) 2022-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -11,16 +11,15 @@ import json_serialization, json_serialization/std/tables, stew/[byteutils, io2, results], chronicles, eth/[rlp, common/eth_types], - ncli/e2store, ../../nimbus/common/[chain_config, genesis], ../network/history/[history_content, accumulator] export results, tables -# Helper calls to parse history data from json files. Format currently -# unspecified and likely to change. -# Perhaps https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md -# can be interesting here too. +# Helper calls to read/write history data from/to json files. +# Format is currently unspecified and likely to change. + +# Reading JSON history data type BlockData* = object @@ -188,64 +187,43 @@ proc getGenesisHeader*(id: NetworkId = MainNet): BlockHeader = except RlpError: raise (ref Defect)(msg: "Genesis should be valid") -proc toString*(v: IoErrorCode): string = - try: ioErrorMsg(v) - except Exception as e: raiseAssert e.msg -proc readAccumulator*(file: string): Result[FinishedAccumulator, string] = - let encodedAccumulator = ? readAllFile(file).mapErr(toString) +# Writing JSON history data - try: - ok(SSZ.decode(encodedAccumulator, FinishedAccumulator)) - except SszError as e: - err("Failed decoding accumulator: " & e.msg) +type + HeaderRecord* = object + header: string + number: uint64 + BlockRecord* = object + header: string + body: string + receipts: string + number: uint64 -proc readEpochAccumulator*(file: string): Result[EpochAccumulator, string] = - let encodedAccumulator = ? readAllFile(file).mapErr(toString) +proc writeHeaderRecord*( + writer: var JsonWriter, header: BlockHeader) + {.raises: [IOError, Defect].} = + let + dataRecord = HeaderRecord( + header: rlp.encode(header).to0xHex(), + number: header.blockNumber.truncate(uint64)) - try: - ok(SSZ.decode(encodedAccumulator, EpochAccumulator)) - except SszError as e: - err("Decoding epoch accumulator failed: " & e.msg) + headerHash = to0xHex(rlpHash(header).data) -proc readEpochAccumulatorCached*(file: string): Result[EpochAccumulatorCached, string] = - let encodedAccumulator = ? readAllFile(file).mapErr(toString) + writer.writeField(headerHash, dataRecord) - try: - ok(SSZ.decode(encodedAccumulator, EpochAccumulatorCached)) - except SszError as e: - err("Decoding epoch accumulator failed: " & e.msg) +proc writeBlockRecord*( + writer: var JsonWriter, + header: BlockHeader, body: BlockBody, receipts: seq[Receipt]) + {.raises: [IOError, Defect].} = + let + dataRecord = BlockRecord( + header: rlp.encode(header).to0xHex(), + body: encode(body).to0xHex(), + receipts: encode(receipts).to0xHex(), + number: header.blockNumber.truncate(uint64)) -const - # Using the e2s format to store data, but without the specific structure - # like in an era file, as we currently don't really need that. - # See: https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md - # Added one type for now, with numbers not formally specified. - # Note: - # Snappy compression for `ExecutionBlockHeaderRecord` only helps for the - # first ~1M (?) block headers, after that there is no gain so we don't do it. - ExecutionBlockHeaderRecord* = [byte 0xFF, 0x00] + headerHash = to0xHex(rlpHash(header).data) -proc readBlockHeaders*(file: string): Result[seq[BlockHeader], string] = - let fh = ? openFile(file, {OpenFlags.Read}).mapErr(toString) - defer: discard closeFile(fh) - - var data: seq[byte] - var blockHeaders: seq[BlockHeader] - while true: - let header = readRecord(fh, data).valueOr: - break - - if header.typ == ExecutionBlockHeaderRecord: - let blockHeader = - try: - rlp.decode(data, BlockHeader) - except RlpError as e: - return err("Invalid block header in " & file & ": " & e.msg) - - blockHeaders.add(blockHeader) - else: - warn "Skipping record, not a block header", typ = toHex(header.typ) - - ok(blockHeaders) + writer.writeField(headerHash, dataRecord) diff --git a/fluffy/data/history_data_seeding.nim b/fluffy/eth_data/history_data_seeding.nim similarity index 98% rename from fluffy/data/history_data_seeding.nim rename to fluffy/eth_data/history_data_seeding.nim index fcb035371..e9452c828 100644 --- a/fluffy/data/history_data_seeding.nim +++ b/fluffy/eth_data/history_data_seeding.nim @@ -1,5 +1,5 @@ # # Nimbus - Portal Network -# # Copyright (c) 2022 Status Research & Development GmbH +# # Copyright (c) 2022-2023 Status Research & Development GmbH # # Licensed and distributed under either of # # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -13,7 +13,7 @@ import eth/common/eth_types, eth/rlp, ../network/wire/portal_protocol, ../network/history/[history_content, accumulator], - ./history_data_parser + "."/[history_data_json_store, history_data_ssz_e2s] export results diff --git a/fluffy/eth_data/history_data_ssz_e2s.nim b/fluffy/eth_data/history_data_ssz_e2s.nim new file mode 100644 index 000000000..c2a7d1a83 --- /dev/null +++ b/fluffy/eth_data/history_data_ssz_e2s.nim @@ -0,0 +1,81 @@ +# Nimbus - Portal Network +# Copyright (c) 2022-2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [Defect].} + +import + stew/[byteutils, io2, results], chronicles, + eth/[rlp, common/eth_types], + ncli/e2store, + ../network/history/[history_content, accumulator] + +export results + +proc toString*(v: IoErrorCode): string = + try: ioErrorMsg(v) + except Exception as e: raiseAssert e.msg + +# Reading SSZ data from files + +proc readAccumulator*(file: string): Result[FinishedAccumulator, string] = + let encodedAccumulator = ? readAllFile(file).mapErr(toString) + + try: + ok(SSZ.decode(encodedAccumulator, FinishedAccumulator)) + except SszError as e: + err("Failed decoding accumulator: " & e.msg) + +proc readEpochAccumulator*(file: string): Result[EpochAccumulator, string] = + let encodedAccumulator = ? readAllFile(file).mapErr(toString) + + try: + ok(SSZ.decode(encodedAccumulator, EpochAccumulator)) + except SszError as e: + err("Decoding epoch accumulator failed: " & e.msg) + +proc readEpochAccumulatorCached*(file: string): Result[EpochAccumulatorCached, string] = + let encodedAccumulator = ? readAllFile(file).mapErr(toString) + + try: + ok(SSZ.decode(encodedAccumulator, EpochAccumulatorCached)) + except SszError as e: + err("Decoding epoch accumulator failed: " & e.msg) + +# Reading data in e2s format + +const + # Using the e2s format to store data, but without the specific structure + # like in an era file, as we currently don't really need that. + # See: https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md + # Added one type for now, with numbers not formally specified. + # Note: + # Snappy compression for `ExecutionBlockHeaderRecord` only helps for the + # first ~1M (?) block headers, after that there is no gain so we don't do it. + ExecutionBlockHeaderRecord* = [byte 0xFF, 0x00] + +proc readBlockHeaders*(file: string): Result[seq[BlockHeader], string] = + let fh = ? openFile(file, {OpenFlags.Read}).mapErr(toString) + defer: discard closeFile(fh) + + var data: seq[byte] + var blockHeaders: seq[BlockHeader] + while true: + let header = readRecord(fh, data).valueOr: + break + + if header.typ == ExecutionBlockHeaderRecord: + let blockHeader = + try: + rlp.decode(data, BlockHeader) + except RlpError as e: + return err("Invalid block header in " & file & ": " & e.msg) + + blockHeaders.add(blockHeader) + else: + warn "Skipping record, not a block header", typ = toHex(header.typ) + + ok(blockHeaders) diff --git a/fluffy/fluffy.nim b/fluffy/fluffy.nim index 0172a7f72..4a14e8169 100644 --- a/fluffy/fluffy.nim +++ b/fluffy/fluffy.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2021-2022 Status Research & Development GmbH +# Copyright (c) 2021-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -31,7 +31,7 @@ import light_client_network ], ./network/wire/[portal_stream, portal_protocol_config], - ./data/[history_data_seeding, history_data_parser], + ./eth_data/history_data_ssz_e2s, ./content_db proc initializeBridgeClient(maybeUri: Option[string]): Option[BridgeClient] = diff --git a/fluffy/rpc/rpc_portal_debug_api.nim b/fluffy/rpc/rpc_portal_debug_api.nim index 6a91c294c..ffad527c7 100644 --- a/fluffy/rpc/rpc_portal_debug_api.nim +++ b/fluffy/rpc/rpc_portal_debug_api.nim @@ -11,6 +11,7 @@ import json_rpc/[rpcproxy, rpcserver], stew/byteutils, ../network/wire/portal_protocol, ../network/network_seed, + ../eth_data/history_data_seeding, ".."/[content_db, seed_db] export rpcserver diff --git a/fluffy/scripts/test_portal_testnet.nim b/fluffy/scripts/test_portal_testnet.nim index 57a2c6d4f..2a297d8e6 100644 --- a/fluffy/scripts/test_portal_testnet.nim +++ b/fluffy/scripts/test_portal_testnet.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2021 Status Research & Development GmbH +# Copyright (c) 2021-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -14,7 +14,10 @@ import ../../nimbus/rpc/[hexstrings, rpc_types], ../rpc/portal_rpc_client, ../rpc/eth_rpc_client, - ../data/[history_data_seeding, history_data_parser], + ../eth_data/[ + history_data_seeding, + history_data_json_store, + history_data_ssz_e2s], ../network/history/[history_content, accumulator], ../seed_db diff --git a/fluffy/tests/portal_spec_tests/mainnet/test_accumulator_root.nim b/fluffy/tests/portal_spec_tests/mainnet/test_accumulator_root.nim index 4d6cd625d..b8da6f993 100644 --- a/fluffy/tests/portal_spec_tests/mainnet/test_accumulator_root.nim +++ b/fluffy/tests/portal_spec_tests/mainnet/test_accumulator_root.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2022 Status Research & Development GmbH +# Copyright (c) 2022-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -12,7 +12,7 @@ import unittest2, stint, stew/byteutils, eth/common/eth_types_rlp, - ../../../data/history_data_parser, + ../../../eth_data/history_data_json_store, ../../../network/history/[history_content, accumulator] suite "Header Accumulator Root": diff --git a/fluffy/tests/test_accumulator.nim b/fluffy/tests/test_accumulator.nim index 2647be3fa..7de0d928d 100644 --- a/fluffy/tests/test_accumulator.nim +++ b/fluffy/tests/test_accumulator.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2022 Status Research & Development GmbH +# Copyright (c) 2022-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -12,7 +12,7 @@ import unittest2, stint, eth/common/eth_types_rlp, - ../data/history_data_parser, + ../eth_data/history_data_json_store, ../network/history/[history_content, accumulator], ./test_helpers diff --git a/fluffy/tests/test_history_validation.nim b/fluffy/tests/test_history_validation.nim index 0734bddd3..1b7e991d1 100644 --- a/fluffy/tests/test_history_validation.nim +++ b/fluffy/tests/test_history_validation.nim @@ -1,5 +1,5 @@ # Nimbus - Portal Network -# Copyright (c) 2022 Status Research & Development GmbH +# Copyright (c) 2022-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -14,7 +14,7 @@ import stew/[byteutils, results], eth/[common/eth_types, rlp], ../common/common_types, - ../data/history_data_parser, + ../eth_data/history_data_json_store, ../network/history/history_network const diff --git a/fluffy/tools/eth_data_exporter.nim b/fluffy/tools/eth_data_exporter.nim index 4eabfc319..e19aaa5b1 100644 --- a/fluffy/tools/eth_data_exporter.nim +++ b/fluffy/tools/eth_data_exporter.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2022 Status Research & Development GmbH +# Copyright (c) 2022-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -49,7 +49,7 @@ import ../seed_db, ../../premix/[downloader, parser], ../network/history/[history_content, accumulator], - ../data/history_data_parser + ../eth_data/[history_data_json_store, history_data_ssz_e2s] # Need to be selective due to the `Block` type conflict from downloader from ../network/history/history_network import encode @@ -184,16 +184,6 @@ type desc: "Number of the last block header to be exported" name: "end-block" .}: uint64 - HeaderRecord = object - header: string - number: uint64 - - BlockRecord = object - header: string - body: string - receipts: string - number: uint64 - proc parseCmdArg*(T: type StorageMode, p: TaintedString): T {.raises: [Defect, ConfigurationError].} = if p == "db": @@ -207,32 +197,6 @@ proc parseCmdArg*(T: type StorageMode, p: TaintedString): T proc completeCmdArg*(T: type StorageMode, val: TaintedString): seq[string] = return @[] -proc writeHeaderRecord( - writer: var JsonWriter, header: BlockHeader) - {.raises: [IOError, Defect].} = - let - dataRecord = HeaderRecord( - header: rlp.encode(header).to0xHex(), - number: header.blockNumber.truncate(uint64)) - - headerHash = to0xHex(rlpHash(header).data) - - writer.writeField(headerHash, dataRecord) - -proc writeBlockRecord( - writer: var JsonWriter, blck: Block) - {.raises: [IOError, Defect].} = - let - dataRecord = BlockRecord( - header: rlp.encode(blck.header).to0xHex(), - body: encode(blck.body).to0xHex(), - receipts: encode(blck.receipts).to0xHex(), - number: blck.header.blockNumber.truncate(uint64)) - - headerHash = to0xHex(rlpHash(blck.header).data) - - writer.writeField(headerHash, dataRecord) - proc downloadHeader(client: RpcClient, i: uint64): BlockHeader = let blockNumber = u256(i) try: @@ -309,7 +273,7 @@ proc writeBlocksToJson(config: ExporterConf, client: RpcClient) = writer.beginRecord() for i in config.startBlock..config.endBlock: let blck = downloadBlock(i, client) - writer.writeBlockRecord(blck) + writer.writeBlockRecord(blck.header, blck.body, blck.receipts) if ((i - config.startBlock) mod 8192) == 0 and i != config.startBlock: info "Downloaded 8192 new blocks", currentBlock = i writer.endRecord()