Reorganize eth data files parsing code (#1407)

fluffy/eth_data/history_data_json_store.nim

@@ -1,5 +1,5 @@
 # Nimbus - Portal Network
-# Copyright (c) 2022 Status Research & Development GmbH
+# Copyright (c) 2022-2023 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -11,16 +11,15 @@ import
   json_serialization, json_serialization/std/tables,
   stew/[byteutils, io2, results], chronicles,
   eth/[rlp, common/eth_types],
-  ncli/e2store,
   ../../nimbus/common/[chain_config, genesis],
   ../network/history/[history_content, accumulator]
 
 export results, tables
 
-# Helper calls to parse history data from json files. Format currently
-# unspecified and likely to change.
-# Perhaps https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
-# can be interesting here too.
+# Helper calls to read/write history data from/to json files.
+# Format is currently unspecified and likely to change.
+
+# Reading JSON history data
 
 type
   BlockData* = object
@@ -188,64 +187,43 @@ proc getGenesisHeader*(id: NetworkId = MainNet): BlockHeader =
   except RlpError:
     raise (ref Defect)(msg: "Genesis should be valid")
 
-proc toString*(v: IoErrorCode): string =
-  try: ioErrorMsg(v)
-  except Exception as e: raiseAssert e.msg
 
-proc readAccumulator*(file: string): Result[FinishedAccumulator, string] =
-  let encodedAccumulator = ? readAllFile(file).mapErr(toString)
+# Writing JSON history data
 
-  try:
-    ok(SSZ.decode(encodedAccumulator, FinishedAccumulator))
-  except SszError as e:
-    err("Failed decoding accumulator: " & e.msg)
+type
+  HeaderRecord* = object
+    header: string
+    number: uint64
 
+  BlockRecord* = object
+    header: string
+    body: string
+    receipts: string
+    number: uint64
 
-proc readEpochAccumulator*(file: string): Result[EpochAccumulator, string] =
-  let encodedAccumulator = ? readAllFile(file).mapErr(toString)
+proc writeHeaderRecord*(
+    writer: var JsonWriter, header: BlockHeader)
+    {.raises: [IOError, Defect].} =
+  let
+    dataRecord = HeaderRecord(
+      header: rlp.encode(header).to0xHex(),
+      number: header.blockNumber.truncate(uint64))
 
-  try:
-    ok(SSZ.decode(encodedAccumulator, EpochAccumulator))
-  except SszError as e:
-    err("Decoding epoch accumulator failed: " & e.msg)
+    headerHash = to0xHex(rlpHash(header).data)
 
-proc readEpochAccumulatorCached*(file: string): Result[EpochAccumulatorCached, string] =
-  let encodedAccumulator = ? readAllFile(file).mapErr(toString)
+  writer.writeField(headerHash, dataRecord)
 
-  try:
-    ok(SSZ.decode(encodedAccumulator, EpochAccumulatorCached))
-  except SszError as e:
-    err("Decoding epoch accumulator failed: " & e.msg)
+proc writeBlockRecord*(
+    writer: var JsonWriter,
+    header: BlockHeader, body: BlockBody, receipts: seq[Receipt])
+    {.raises: [IOError, Defect].} =
+  let
+    dataRecord = BlockRecord(
+      header: rlp.encode(header).to0xHex(),
+      body: encode(body).to0xHex(),
+      receipts: encode(receipts).to0xHex(),
+      number: header.blockNumber.truncate(uint64))
 
-const
-  # Using the e2s format to store data, but without the specific structure
-  # like in an era file, as we currently don't really need that.
-  # See: https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
-  # Added one type for now, with numbers not formally specified.
-  # Note:
-  # Snappy compression for `ExecutionBlockHeaderRecord` only helps for the
-  # first ~1M (?) block headers, after that there is no gain so we don't do it.
-  ExecutionBlockHeaderRecord* = [byte 0xFF, 0x00]
+    headerHash = to0xHex(rlpHash(header).data)
 
-proc readBlockHeaders*(file: string): Result[seq[BlockHeader], string] =
-  let fh = ? openFile(file, {OpenFlags.Read}).mapErr(toString)
-  defer: discard closeFile(fh)
-
-  var data: seq[byte]
-  var blockHeaders: seq[BlockHeader]
-  while true:
-    let header = readRecord(fh, data).valueOr:
-      break
-
-    if header.typ == ExecutionBlockHeaderRecord:
-      let blockHeader =
-        try:
-          rlp.decode(data, BlockHeader)
-        except RlpError as e:
-          return err("Invalid block header in " & file & ": " & e.msg)
-
-      blockHeaders.add(blockHeader)
-    else:
-      warn "Skipping record, not a block header", typ = toHex(header.typ)
-
-  ok(blockHeaders)
+  writer.writeField(headerHash, dataRecord)

fluffy/eth_data/history_data_seeding.nim

@@ -1,5 +1,5 @@
 # # Nimbus - Portal Network
-# # Copyright (c) 2022 Status Research & Development GmbH
+# # Copyright (c) 2022-2023 Status Research & Development GmbH
 # # Licensed and distributed under either of
 # #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 # #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -13,7 +13,7 @@ import
   eth/common/eth_types, eth/rlp,
   ../network/wire/portal_protocol,
   ../network/history/[history_content, accumulator],
-  ./history_data_parser
+  "."/[history_data_json_store, history_data_ssz_e2s]
 
 export results
 

fluffy/eth_data/history_data_ssz_e2s.nim

@@ -0,0 +1,81 @@
+# Nimbus - Portal Network
+# Copyright (c) 2022-2023 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [Defect].}
+
+import
+  stew/[byteutils, io2, results], chronicles,
+  eth/[rlp, common/eth_types],
+  ncli/e2store,
+  ../network/history/[history_content, accumulator]
+
+export results
+
+proc toString*(v: IoErrorCode): string =
+  try: ioErrorMsg(v)
+  except Exception as e: raiseAssert e.msg
+
+# Reading SSZ data from files
+
+proc readAccumulator*(file: string): Result[FinishedAccumulator, string] =
+  let encodedAccumulator = ? readAllFile(file).mapErr(toString)
+
+  try:
+    ok(SSZ.decode(encodedAccumulator, FinishedAccumulator))
+  except SszError as e:
+    err("Failed decoding accumulator: " & e.msg)
+
+proc readEpochAccumulator*(file: string): Result[EpochAccumulator, string] =
+  let encodedAccumulator = ? readAllFile(file).mapErr(toString)
+
+  try:
+    ok(SSZ.decode(encodedAccumulator, EpochAccumulator))
+  except SszError as e:
+    err("Decoding epoch accumulator failed: " & e.msg)
+
+proc readEpochAccumulatorCached*(file: string): Result[EpochAccumulatorCached, string] =
+  let encodedAccumulator = ? readAllFile(file).mapErr(toString)
+
+  try:
+    ok(SSZ.decode(encodedAccumulator, EpochAccumulatorCached))
+  except SszError as e:
+    err("Decoding epoch accumulator failed: " & e.msg)
+
+# Reading data in e2s format
+
+const
+  # Using the e2s format to store data, but without the specific structure
+  # like in an era file, as we currently don't really need that.
+  # See: https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
+  # Added one type for now, with numbers not formally specified.
+  # Note:
+  # Snappy compression for `ExecutionBlockHeaderRecord` only helps for the
+  # first ~1M (?) block headers, after that there is no gain so we don't do it.
+  ExecutionBlockHeaderRecord* = [byte 0xFF, 0x00]
+
+proc readBlockHeaders*(file: string): Result[seq[BlockHeader], string] =
+  let fh = ? openFile(file, {OpenFlags.Read}).mapErr(toString)
+  defer: discard closeFile(fh)
+
+  var data: seq[byte]
+  var blockHeaders: seq[BlockHeader]
+  while true:
+    let header = readRecord(fh, data).valueOr:
+      break
+
+    if header.typ == ExecutionBlockHeaderRecord:
+      let blockHeader =
+        try:
+          rlp.decode(data, BlockHeader)
+        except RlpError as e:
+          return err("Invalid block header in " & file & ": " & e.msg)
+
+      blockHeaders.add(blockHeader)
+    else:
+      warn "Skipping record, not a block header", typ = toHex(header.typ)
+
+  ok(blockHeaders)

fluffy/fluffy.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2021-2022 Status Research & Development GmbH
+# Copyright (c) 2021-2023 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -31,7 +31,7 @@ import
     light_client_network
   ],
   ./network/wire/[portal_stream, portal_protocol_config],
-  ./data/[history_data_seeding, history_data_parser],
+  ./eth_data/history_data_ssz_e2s,
   ./content_db
 
 proc initializeBridgeClient(maybeUri: Option[string]): Option[BridgeClient] =

fluffy/rpc/rpc_portal_debug_api.nim

@@ -11,6 +11,7 @@ import
   json_rpc/[rpcproxy, rpcserver], stew/byteutils,
   ../network/wire/portal_protocol,
   ../network/network_seed,
+  ../eth_data/history_data_seeding,
   ".."/[content_db, seed_db]
 
 export rpcserver

fluffy/scripts/test_portal_testnet.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2021 Status Research & Development GmbH
+# Copyright (c) 2021-2023 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -14,7 +14,10 @@ import
   ../../nimbus/rpc/[hexstrings, rpc_types],
   ../rpc/portal_rpc_client,
   ../rpc/eth_rpc_client,
-  ../data/[history_data_seeding, history_data_parser],
+  ../eth_data/[
+    history_data_seeding,
+    history_data_json_store,
+    history_data_ssz_e2s],
   ../network/history/[history_content, accumulator],
   ../seed_db
 

fluffy/tests/portal_spec_tests/mainnet/test_accumulator_root.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2022 Status Research & Development GmbH
+# Copyright (c) 2022-2023 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -12,7 +12,7 @@
 import
   unittest2, stint, stew/byteutils,
   eth/common/eth_types_rlp,
-  ../../../data/history_data_parser,
+  ../../../eth_data/history_data_json_store,
   ../../../network/history/[history_content, accumulator]
 
 suite "Header Accumulator Root":

fluffy/tests/test_accumulator.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2022 Status Research & Development GmbH
+# Copyright (c) 2022-2023 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -12,7 +12,7 @@
 import
   unittest2, stint,
   eth/common/eth_types_rlp,
-  ../data/history_data_parser,
+  ../eth_data/history_data_json_store,
   ../network/history/[history_content, accumulator],
   ./test_helpers
 

fluffy/tests/test_history_validation.nim

@@ -1,5 +1,5 @@
 # Nimbus - Portal Network
-# Copyright (c) 2022 Status Research & Development GmbH
+# Copyright (c) 2022-2023 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -14,7 +14,7 @@ import
   stew/[byteutils, results],
   eth/[common/eth_types, rlp],
   ../common/common_types,
-  ../data/history_data_parser,
+  ../eth_data/history_data_json_store,
   ../network/history/history_network
 
 const

fluffy/tools/eth_data_exporter.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2022 Status Research & Development GmbH
+# Copyright (c) 2022-2023 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -49,7 +49,7 @@ import
   ../seed_db,
   ../../premix/[downloader, parser],
   ../network/history/[history_content, accumulator],
-  ../data/history_data_parser
+  ../eth_data/[history_data_json_store, history_data_ssz_e2s]
 
 # Need to be selective due to the `Block` type conflict from downloader
 from ../network/history/history_network import encode
@@ -184,16 +184,6 @@ type
         desc: "Number of the last block header to be exported"
         name: "end-block" .}: uint64
 
-  HeaderRecord = object
-    header: string
-    number: uint64
-
-  BlockRecord = object
-    header: string
-    body: string
-    receipts: string
-    number: uint64
-
 proc parseCmdArg*(T: type StorageMode, p: TaintedString): T
     {.raises: [Defect, ConfigurationError].} =
   if p == "db":
@@ -207,32 +197,6 @@ proc parseCmdArg*(T: type StorageMode, p: TaintedString): T
 proc completeCmdArg*(T: type StorageMode, val: TaintedString): seq[string] =
   return @[]
 
-proc writeHeaderRecord(
-    writer: var JsonWriter, header: BlockHeader)
-    {.raises: [IOError, Defect].} =
-  let
-    dataRecord = HeaderRecord(
-      header: rlp.encode(header).to0xHex(),
-      number: header.blockNumber.truncate(uint64))
-
-    headerHash = to0xHex(rlpHash(header).data)
-
-  writer.writeField(headerHash, dataRecord)
-
-proc writeBlockRecord(
-    writer: var JsonWriter, blck: Block)
-    {.raises: [IOError, Defect].} =
-  let
-    dataRecord = BlockRecord(
-      header: rlp.encode(blck.header).to0xHex(),
-      body: encode(blck.body).to0xHex(),
-      receipts: encode(blck.receipts).to0xHex(),
-      number: blck.header.blockNumber.truncate(uint64))
-
-    headerHash = to0xHex(rlpHash(blck.header).data)
-
-  writer.writeField(headerHash, dataRecord)
-
 proc downloadHeader(client: RpcClient, i: uint64): BlockHeader =
   let blockNumber = u256(i)
   try:
@@ -309,7 +273,7 @@ proc writeBlocksToJson(config: ExporterConf, client: RpcClient) =
     writer.beginRecord()
     for i in config.startBlock..config.endBlock:
       let blck = downloadBlock(i, client)
-      writer.writeBlockRecord(blck)
+      writer.writeBlockRecord(blck.header, blck.body, blck.receipts)
       if ((i - config.startBlock) mod 8192) == 0 and i != config.startBlock:
         info "Downloaded 8192 new blocks", currentBlock = i
     writer.endRecord()