Change test data and propagation format (#1115)

* Change test data and propagation format

* Add tool to download data from local node
This commit is contained in:
KonradStaniec 2022-06-08 15:14:01 +02:00 committed by GitHub
parent ee77d704bc
commit 730013bde3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 193 additions and 33 deletions

View File

@ -91,6 +91,19 @@ make fluffy-tools
./build/blockwalk --block-hash:0xf6bfad56d1a45d1661506343dd1e511b5d7e17565b3ec293125ff0890b9709e5
```
One can also use the `eth-data-exporter` tool to download history data from
local geth instance into the format which is suitable for propagating data into
fluffy client e.g:
```bash
make fluffy-tools
./build/eth_data_exporter --initial-block:1 --end-block: 10 --data-dir:"/userDirectory/"
```
Will download blocks from 1 to 10 into user provided director in one json file.
### Run fluffy test suite
```bash
# From the nimbus-eth1 repository

View File

@ -71,11 +71,8 @@ func readBlockData(
return err("Invalid hex for rlp block data, number " &
$blockData.number & ": " & e.msg)
# The data is currently formatted as an rlp encoded `EthBlock`, thus
# containing header, txs and uncles: [header, txs, uncles]. No receipts are
# available.
# TODO: Change to format to rlp data as it gets stored and send over the
# network over the network. I.e. [header, [txs, uncles], receipts]
# Data is formatted as it gets stored and send over the
# network. I.e. [header, [txs, uncles], receipts]
if rlp.enterList():
var blockHash: BlockHash
try:
@ -106,24 +103,16 @@ func readBlockData(
contentType: blockBody,
blockBodyKey: contentKeyType)
# Note: Temporary until the data format gets changed.
let blockBody = BlockBody(
transactions: rlp.read(seq[Transaction]),
uncles: rlp.read(seq[BlockHeader]))
let rlpdata = encode(blockBody)
res.add((contentKey, @(rlp.rawData())))
rlp.skipElem()
res.add((contentKey, rlpdata))
# res.add((contentKey, @(rlp.rawData())))
# rlp.skipElem()
block:
let contentKey = ContentKey(
contentType: receipts,
receiptsKey: contentKeyType)
# Note: No receipts yet in the data set
# block:
# let contentKey = ContentKey(
# contentType: receipts,
# receiptsKey: contentKeyType)
# res.add((contentKey, @(rlp.rawData())))
# rlp.skipElem()
res.add((contentKey, @(rlp.rawData())))
rlp.skipElem()
except RlpError as e:
return err("Invalid rlp data, number " & $blockData.number & ": " & e.msg)

File diff suppressed because one or more lines are too long

View File

@ -1,10 +1,10 @@
{
"0x88e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6": {
"rlp": "0xf90216f90211a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d493479405a56e2d52c817161883f50c441c3228cfe54d9fa0d67e4d450343046425ae4271474353857ab860dbc0a1dde64b41b5cd3a532bf3a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff80000001821388808455ba422499476574682f76312e302e302f6c696e75782f676f312e342e32a0969b900de27b6ac6a67742365dd65f55a0526c41fd18e1b16f1a1215c2e66f5988539bd4979fef1ec4c0c0",
"rlp": "0xf90218f90211a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d493479405a56e2d52c817161883f50c441c3228cfe54d9fa0d67e4d450343046425ae4271474353857ab860dbc0a1dde64b41b5cd3a532bf3a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff80000001821388808455ba422499476574682f76312e302e302f6c696e75782f676f312e342e32a0969b900de27b6ac6a67742365dd65f55a0526c41fd18e1b16f1a1215c2e66f5988539bd4979fef1ec4c2c0c0c0",
"number": 1
},
"0xb495a1d7e6663152ae92708da4843337b958146015a2802f4193a410044698c9": {
"rlp": "0xf9021df90218a088e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794dd2f1e6e498202e86d8f5442af596580a4f03c2ca04943d941637411107494da9ec8bc04359d731bfd08b72b4d0edcbd4cd2ecb341a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff00100002821388808455ba4241a0476574682f76312e302e302d30636463373634372f6c696e75782f676f312e34a02f0790c5aa31ab94195e1f6443d645af5b75c46c04fbf9911711198a0ce8fdda88b853fa261a86aa9ec0c0",
"rlp": "0xf9021ff90218a088e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794dd2f1e6e498202e86d8f5442af596580a4f03c2ca04943d941637411107494da9ec8bc04359d731bfd08b72b4d0edcbd4cd2ecb341a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff00100002821388808455ba4241a0476574682f76312e302e302d30636463373634372f6c696e75782f676f312e34a02f0790c5aa31ab94195e1f6443d645af5b75c46c04fbf9911711198a0ce8fdda88b853fa261a86aa9ec2c0c0c0",
"number": 2
}
}

View File

@ -0,0 +1,157 @@
# Nimbus
# Copyright (c) 2022 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# Tool to download chain history data from local node, and save it to the json
# file.
# Data of each block is rlp encoded list of:
# [blockHeader, [block_transactions, block_uncles], block_receipts]
# Json file has following format:
# {
# "hexEncodedBlockHash: {
# "rlp": "hex of rlp encoded list [blockHeader, [block_transactions, block_uncles], block_receipts]",
# "number": "block number"
# },
# ...,
# ...,
# }
#
#
{.push raises: [Defect].}
import
std/[json, typetraits, strutils, os],
confutils,
stew/[byteutils, io2],
json_serialization,
faststreams, chronicles,
eth/[common, rlp], chronos,
eth/common/eth_types_json_serialization,
../../premix/downloader
proc defaultDataDir*(): string =
let dataDir = when defined(windows):
"AppData" / "Roaming" / "EthData"
elif defined(macosx):
"Library" / "Application Support" / "EthData"
else:
".cache" / "ethData"
getHomeDir() / dataDir
const
defaultDataDirDesc = defaultDataDir()
defaultFileName = "eth-history-data.json"
type
ExporterConf* = object
logLevel* {.
defaultValue: LogLevel.INFO
defaultValueDesc: $LogLevel.INFO
desc: "Sets the log level"
name: "log-level" .}: LogLevel
initialBlock* {.
desc: "Number of first block which should be downloaded"
defaultValue: 0
name: "initial-block" .}: uint64
endBlock* {.
desc: "Number of last block which should be downloaded"
defaultValue: 0
name: "end-block" .}: uint64
dataDir* {.
desc: "The directory where generated file will be placed"
defaultValue: defaultDataDir()
defaultValueDesc: $defaultDataDirDesc
name: "data-dir" .}: OutDir
filename* {.
desc: "default name of the file with history data"
defaultValue: defaultFileName
name: "filename" .}: string
DataRecord = object
rlp: string
number: uint64
proc writeBlock(writer: var JsonWriter, blck: Block) {.raises: [IOError, Defect].} =
let
enc = rlp.encodeList(blck.header, blck.body, blck.receipts)
asHex = to0xHex(enc)
dataRecord = DataRecord(rlp: asHex, number: cast[uint64](blck.header.blockNumber))
headerHash = to0xHex(rlpHash(blck.header).data)
writer.writeField(headerHash, dataRecord)
proc downloadBlock(i: uint64): Block =
let num = u256(i)
try:
# premix has hardcoded making request to local host which is "127.0.0.1:8545"
# which is defult port of geth json rpc server
return requestBlock(num, flags = {DownloadReceipts})
except CatchableError as e:
fatal "Error while requesting Block", error = e.msg
quit 1
proc createAndOpenFile(config: ExporterConf): OutputStreamHandle =
# Creates directory and file specified in config, if file already exists
# program is aborted with info to user, to avoid losing data
let filePath = config.dataDir / config.filename
if isFile(filePath):
fatal "File under provided path already exists and would be overwritten",
path = filePath
quit 1
let res = createPath(distinctBase(config.dataDir))
if res.isErr():
fatal "Error occurred while creating directory", error = res.error
quit 1
try:
# this means that each time file be overwritten, but it is ok for such one
# off toll
return fileOutput(filePath)
except IOError as e:
fatal "Error occurred while opening the file", error = e.msg
quit 1
proc run(config: ExporterConf) =
let fh = createAndOpenFile(config)
try:
var writer = JsonWriter[DefaultFlavor].init(fh.s)
writer.beginRecord()
for i in config.initialBlock..config.endBlock:
let blck = downloadBlock(i)
writer.writeBlock(blck)
writer.endRecord()
info "File successfully written"
except IOError as e:
fatal "Error occoured while writing to file", error = e.msg
quit 1
finally:
try:
fh.close()
except IOError as e:
fatal "Error occoured while closing file", error = e.msg
quit 1
when isMainModule:
{.pop.}
let config = ExporterConf.load()
{.push raises: [Defect].}
if (config.endBlock < config.initialBlock):
fatal "Initial block number should be smaller than end block number",
initialBlock = config.initialBlock,
endBlock = config.endBlock
quit 1
setLogLevel(config.logLevel)
run(config)

View File

@ -67,6 +67,7 @@ task fluffy, "Build fluffy":
task fluffy_tools, "Build fluffy tools":
buildBinary "portalcli", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"
buildBinary "blockwalk", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"
buildBinary "eth_data_exporter", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"
task utp_test_app, "Build uTP test app":
buildBinary "utp_test_app", "fluffy/tools/utp_testing/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"