Reorganize eth data files parsing code (#1407)
This commit is contained in:
parent
4bf4aeba94
commit
90f91e5ebd
|
@ -1,5 +1,5 @@
|
|||
# Nimbus - Portal Network
|
||||
# Copyright (c) 2022 Status Research & Development GmbH
|
||||
# Copyright (c) 2022-2023 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
|
@ -11,16 +11,15 @@ import
|
|||
json_serialization, json_serialization/std/tables,
|
||||
stew/[byteutils, io2, results], chronicles,
|
||||
eth/[rlp, common/eth_types],
|
||||
ncli/e2store,
|
||||
../../nimbus/common/[chain_config, genesis],
|
||||
../network/history/[history_content, accumulator]
|
||||
|
||||
export results, tables
|
||||
|
||||
# Helper calls to parse history data from json files. Format currently
|
||||
# unspecified and likely to change.
|
||||
# Perhaps https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
|
||||
# can be interesting here too.
|
||||
# Helper calls to read/write history data from/to json files.
|
||||
# Format is currently unspecified and likely to change.
|
||||
|
||||
# Reading JSON history data
|
||||
|
||||
type
|
||||
BlockData* = object
|
||||
|
@ -188,64 +187,43 @@ proc getGenesisHeader*(id: NetworkId = MainNet): BlockHeader =
|
|||
except RlpError:
|
||||
raise (ref Defect)(msg: "Genesis should be valid")
|
||||
|
||||
proc toString*(v: IoErrorCode): string =
|
||||
try: ioErrorMsg(v)
|
||||
except Exception as e: raiseAssert e.msg
|
||||
|
||||
proc readAccumulator*(file: string): Result[FinishedAccumulator, string] =
|
||||
let encodedAccumulator = ? readAllFile(file).mapErr(toString)
|
||||
# Writing JSON history data
|
||||
|
||||
try:
|
||||
ok(SSZ.decode(encodedAccumulator, FinishedAccumulator))
|
||||
except SszError as e:
|
||||
err("Failed decoding accumulator: " & e.msg)
|
||||
type
|
||||
HeaderRecord* = object
|
||||
header: string
|
||||
number: uint64
|
||||
|
||||
BlockRecord* = object
|
||||
header: string
|
||||
body: string
|
||||
receipts: string
|
||||
number: uint64
|
||||
|
||||
proc readEpochAccumulator*(file: string): Result[EpochAccumulator, string] =
|
||||
let encodedAccumulator = ? readAllFile(file).mapErr(toString)
|
||||
proc writeHeaderRecord*(
|
||||
writer: var JsonWriter, header: BlockHeader)
|
||||
{.raises: [IOError, Defect].} =
|
||||
let
|
||||
dataRecord = HeaderRecord(
|
||||
header: rlp.encode(header).to0xHex(),
|
||||
number: header.blockNumber.truncate(uint64))
|
||||
|
||||
try:
|
||||
ok(SSZ.decode(encodedAccumulator, EpochAccumulator))
|
||||
except SszError as e:
|
||||
err("Decoding epoch accumulator failed: " & e.msg)
|
||||
headerHash = to0xHex(rlpHash(header).data)
|
||||
|
||||
proc readEpochAccumulatorCached*(file: string): Result[EpochAccumulatorCached, string] =
|
||||
let encodedAccumulator = ? readAllFile(file).mapErr(toString)
|
||||
writer.writeField(headerHash, dataRecord)
|
||||
|
||||
try:
|
||||
ok(SSZ.decode(encodedAccumulator, EpochAccumulatorCached))
|
||||
except SszError as e:
|
||||
err("Decoding epoch accumulator failed: " & e.msg)
|
||||
proc writeBlockRecord*(
|
||||
writer: var JsonWriter,
|
||||
header: BlockHeader, body: BlockBody, receipts: seq[Receipt])
|
||||
{.raises: [IOError, Defect].} =
|
||||
let
|
||||
dataRecord = BlockRecord(
|
||||
header: rlp.encode(header).to0xHex(),
|
||||
body: encode(body).to0xHex(),
|
||||
receipts: encode(receipts).to0xHex(),
|
||||
number: header.blockNumber.truncate(uint64))
|
||||
|
||||
const
|
||||
# Using the e2s format to store data, but without the specific structure
|
||||
# like in an era file, as we currently don't really need that.
|
||||
# See: https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
|
||||
# Added one type for now, with numbers not formally specified.
|
||||
# Note:
|
||||
# Snappy compression for `ExecutionBlockHeaderRecord` only helps for the
|
||||
# first ~1M (?) block headers, after that there is no gain so we don't do it.
|
||||
ExecutionBlockHeaderRecord* = [byte 0xFF, 0x00]
|
||||
headerHash = to0xHex(rlpHash(header).data)
|
||||
|
||||
proc readBlockHeaders*(file: string): Result[seq[BlockHeader], string] =
|
||||
let fh = ? openFile(file, {OpenFlags.Read}).mapErr(toString)
|
||||
defer: discard closeFile(fh)
|
||||
|
||||
var data: seq[byte]
|
||||
var blockHeaders: seq[BlockHeader]
|
||||
while true:
|
||||
let header = readRecord(fh, data).valueOr:
|
||||
break
|
||||
|
||||
if header.typ == ExecutionBlockHeaderRecord:
|
||||
let blockHeader =
|
||||
try:
|
||||
rlp.decode(data, BlockHeader)
|
||||
except RlpError as e:
|
||||
return err("Invalid block header in " & file & ": " & e.msg)
|
||||
|
||||
blockHeaders.add(blockHeader)
|
||||
else:
|
||||
warn "Skipping record, not a block header", typ = toHex(header.typ)
|
||||
|
||||
ok(blockHeaders)
|
||||
writer.writeField(headerHash, dataRecord)
|
|
@ -1,5 +1,5 @@
|
|||
# # Nimbus - Portal Network
|
||||
# # Copyright (c) 2022 Status Research & Development GmbH
|
||||
# # Copyright (c) 2022-2023 Status Research & Development GmbH
|
||||
# # Licensed and distributed under either of
|
||||
# # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
|
@ -13,7 +13,7 @@ import
|
|||
eth/common/eth_types, eth/rlp,
|
||||
../network/wire/portal_protocol,
|
||||
../network/history/[history_content, accumulator],
|
||||
./history_data_parser
|
||||
"."/[history_data_json_store, history_data_ssz_e2s]
|
||||
|
||||
export results
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
# Nimbus - Portal Network
|
||||
# Copyright (c) 2022-2023 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
||||
|
||||
{.push raises: [Defect].}
|
||||
|
||||
import
|
||||
stew/[byteutils, io2, results], chronicles,
|
||||
eth/[rlp, common/eth_types],
|
||||
ncli/e2store,
|
||||
../network/history/[history_content, accumulator]
|
||||
|
||||
export results
|
||||
|
||||
proc toString*(v: IoErrorCode): string =
|
||||
try: ioErrorMsg(v)
|
||||
except Exception as e: raiseAssert e.msg
|
||||
|
||||
# Reading SSZ data from files
|
||||
|
||||
proc readAccumulator*(file: string): Result[FinishedAccumulator, string] =
|
||||
let encodedAccumulator = ? readAllFile(file).mapErr(toString)
|
||||
|
||||
try:
|
||||
ok(SSZ.decode(encodedAccumulator, FinishedAccumulator))
|
||||
except SszError as e:
|
||||
err("Failed decoding accumulator: " & e.msg)
|
||||
|
||||
proc readEpochAccumulator*(file: string): Result[EpochAccumulator, string] =
|
||||
let encodedAccumulator = ? readAllFile(file).mapErr(toString)
|
||||
|
||||
try:
|
||||
ok(SSZ.decode(encodedAccumulator, EpochAccumulator))
|
||||
except SszError as e:
|
||||
err("Decoding epoch accumulator failed: " & e.msg)
|
||||
|
||||
proc readEpochAccumulatorCached*(file: string): Result[EpochAccumulatorCached, string] =
|
||||
let encodedAccumulator = ? readAllFile(file).mapErr(toString)
|
||||
|
||||
try:
|
||||
ok(SSZ.decode(encodedAccumulator, EpochAccumulatorCached))
|
||||
except SszError as e:
|
||||
err("Decoding epoch accumulator failed: " & e.msg)
|
||||
|
||||
# Reading data in e2s format
|
||||
|
||||
const
|
||||
# Using the e2s format to store data, but without the specific structure
|
||||
# like in an era file, as we currently don't really need that.
|
||||
# See: https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
|
||||
# Added one type for now, with numbers not formally specified.
|
||||
# Note:
|
||||
# Snappy compression for `ExecutionBlockHeaderRecord` only helps for the
|
||||
# first ~1M (?) block headers, after that there is no gain so we don't do it.
|
||||
ExecutionBlockHeaderRecord* = [byte 0xFF, 0x00]
|
||||
|
||||
proc readBlockHeaders*(file: string): Result[seq[BlockHeader], string] =
|
||||
let fh = ? openFile(file, {OpenFlags.Read}).mapErr(toString)
|
||||
defer: discard closeFile(fh)
|
||||
|
||||
var data: seq[byte]
|
||||
var blockHeaders: seq[BlockHeader]
|
||||
while true:
|
||||
let header = readRecord(fh, data).valueOr:
|
||||
break
|
||||
|
||||
if header.typ == ExecutionBlockHeaderRecord:
|
||||
let blockHeader =
|
||||
try:
|
||||
rlp.decode(data, BlockHeader)
|
||||
except RlpError as e:
|
||||
return err("Invalid block header in " & file & ": " & e.msg)
|
||||
|
||||
blockHeaders.add(blockHeader)
|
||||
else:
|
||||
warn "Skipping record, not a block header", typ = toHex(header.typ)
|
||||
|
||||
ok(blockHeaders)
|
|
@ -1,5 +1,5 @@
|
|||
# Nimbus
|
||||
# Copyright (c) 2021-2022 Status Research & Development GmbH
|
||||
# Copyright (c) 2021-2023 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
|
@ -31,7 +31,7 @@ import
|
|||
light_client_network
|
||||
],
|
||||
./network/wire/[portal_stream, portal_protocol_config],
|
||||
./data/[history_data_seeding, history_data_parser],
|
||||
./eth_data/history_data_ssz_e2s,
|
||||
./content_db
|
||||
|
||||
proc initializeBridgeClient(maybeUri: Option[string]): Option[BridgeClient] =
|
||||
|
|
|
@ -11,6 +11,7 @@ import
|
|||
json_rpc/[rpcproxy, rpcserver], stew/byteutils,
|
||||
../network/wire/portal_protocol,
|
||||
../network/network_seed,
|
||||
../eth_data/history_data_seeding,
|
||||
".."/[content_db, seed_db]
|
||||
|
||||
export rpcserver
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
# Nimbus
|
||||
# Copyright (c) 2021 Status Research & Development GmbH
|
||||
# Copyright (c) 2021-2023 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
|
@ -14,7 +14,10 @@ import
|
|||
../../nimbus/rpc/[hexstrings, rpc_types],
|
||||
../rpc/portal_rpc_client,
|
||||
../rpc/eth_rpc_client,
|
||||
../data/[history_data_seeding, history_data_parser],
|
||||
../eth_data/[
|
||||
history_data_seeding,
|
||||
history_data_json_store,
|
||||
history_data_ssz_e2s],
|
||||
../network/history/[history_content, accumulator],
|
||||
../seed_db
|
||||
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
# Nimbus
|
||||
# Copyright (c) 2022 Status Research & Development GmbH
|
||||
# Copyright (c) 2022-2023 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
|
@ -12,7 +12,7 @@
|
|||
import
|
||||
unittest2, stint, stew/byteutils,
|
||||
eth/common/eth_types_rlp,
|
||||
../../../data/history_data_parser,
|
||||
../../../eth_data/history_data_json_store,
|
||||
../../../network/history/[history_content, accumulator]
|
||||
|
||||
suite "Header Accumulator Root":
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
# Nimbus
|
||||
# Copyright (c) 2022 Status Research & Development GmbH
|
||||
# Copyright (c) 2022-2023 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
|
@ -12,7 +12,7 @@
|
|||
import
|
||||
unittest2, stint,
|
||||
eth/common/eth_types_rlp,
|
||||
../data/history_data_parser,
|
||||
../eth_data/history_data_json_store,
|
||||
../network/history/[history_content, accumulator],
|
||||
./test_helpers
|
||||
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
# Nimbus - Portal Network
|
||||
# Copyright (c) 2022 Status Research & Development GmbH
|
||||
# Copyright (c) 2022-2023 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
|
@ -14,7 +14,7 @@ import
|
|||
stew/[byteutils, results],
|
||||
eth/[common/eth_types, rlp],
|
||||
../common/common_types,
|
||||
../data/history_data_parser,
|
||||
../eth_data/history_data_json_store,
|
||||
../network/history/history_network
|
||||
|
||||
const
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
# Nimbus
|
||||
# Copyright (c) 2022 Status Research & Development GmbH
|
||||
# Copyright (c) 2022-2023 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
|
@ -49,7 +49,7 @@ import
|
|||
../seed_db,
|
||||
../../premix/[downloader, parser],
|
||||
../network/history/[history_content, accumulator],
|
||||
../data/history_data_parser
|
||||
../eth_data/[history_data_json_store, history_data_ssz_e2s]
|
||||
|
||||
# Need to be selective due to the `Block` type conflict from downloader
|
||||
from ../network/history/history_network import encode
|
||||
|
@ -184,16 +184,6 @@ type
|
|||
desc: "Number of the last block header to be exported"
|
||||
name: "end-block" .}: uint64
|
||||
|
||||
HeaderRecord = object
|
||||
header: string
|
||||
number: uint64
|
||||
|
||||
BlockRecord = object
|
||||
header: string
|
||||
body: string
|
||||
receipts: string
|
||||
number: uint64
|
||||
|
||||
proc parseCmdArg*(T: type StorageMode, p: TaintedString): T
|
||||
{.raises: [Defect, ConfigurationError].} =
|
||||
if p == "db":
|
||||
|
@ -207,32 +197,6 @@ proc parseCmdArg*(T: type StorageMode, p: TaintedString): T
|
|||
proc completeCmdArg*(T: type StorageMode, val: TaintedString): seq[string] =
|
||||
return @[]
|
||||
|
||||
proc writeHeaderRecord(
|
||||
writer: var JsonWriter, header: BlockHeader)
|
||||
{.raises: [IOError, Defect].} =
|
||||
let
|
||||
dataRecord = HeaderRecord(
|
||||
header: rlp.encode(header).to0xHex(),
|
||||
number: header.blockNumber.truncate(uint64))
|
||||
|
||||
headerHash = to0xHex(rlpHash(header).data)
|
||||
|
||||
writer.writeField(headerHash, dataRecord)
|
||||
|
||||
proc writeBlockRecord(
|
||||
writer: var JsonWriter, blck: Block)
|
||||
{.raises: [IOError, Defect].} =
|
||||
let
|
||||
dataRecord = BlockRecord(
|
||||
header: rlp.encode(blck.header).to0xHex(),
|
||||
body: encode(blck.body).to0xHex(),
|
||||
receipts: encode(blck.receipts).to0xHex(),
|
||||
number: blck.header.blockNumber.truncate(uint64))
|
||||
|
||||
headerHash = to0xHex(rlpHash(blck.header).data)
|
||||
|
||||
writer.writeField(headerHash, dataRecord)
|
||||
|
||||
proc downloadHeader(client: RpcClient, i: uint64): BlockHeader =
|
||||
let blockNumber = u256(i)
|
||||
try:
|
||||
|
@ -309,7 +273,7 @@ proc writeBlocksToJson(config: ExporterConf, client: RpcClient) =
|
|||
writer.beginRecord()
|
||||
for i in config.startBlock..config.endBlock:
|
||||
let blck = downloadBlock(i, client)
|
||||
writer.writeBlockRecord(blck)
|
||||
writer.writeBlockRecord(blck.header, blck.body, blck.receipts)
|
||||
if ((i - config.startBlock) mod 8192) == 0 and i != config.startBlock:
|
||||
info "Downloaded 8192 new blocks", currentBlock = i
|
||||
writer.endRecord()
|
||||
|
|
Loading…
Reference in New Issue