Change test data and propagation format (#1115)

* Change test data and propagation format

* Add tool to download data from local node

fluffy/README.md

@@ -91,6 +91,19 @@ make fluffy-tools
 ./build/blockwalk --block-hash:0xf6bfad56d1a45d1661506343dd1e511b5d7e17565b3ec293125ff0890b9709e5
 ```
 
+One can also use the `eth-data-exporter` tool to download history data from
+local geth instance into the format which is suitable for propagating data into
+fluffy client e.g:
+
+```bash
+make fluffy-tools
+
+./build/eth_data_exporter --initial-block:1 --end-block: 10 --data-dir:"/userDirectory/"
+```
+
+Will download blocks from 1 to 10 into user provided director in one json file.
+
+
 ### Run fluffy test suite
 ```bash
 # From the nimbus-eth1 repository

fluffy/populate_db.nim

@@ -71,11 +71,8 @@ func readBlockData(
       return err("Invalid hex for rlp block data, number " &
         $blockData.number & ": " & e.msg)
 
-  # The data is currently formatted as an rlp encoded `EthBlock`, thus
+  # Data is formatted as it gets stored and send over the
-  # containing header, txs and uncles: [header, txs, uncles]. No receipts are
+  # network. I.e. [header, [txs, uncles], receipts]
-  # available.
-  # TODO: Change to format to rlp data as it gets stored and send over the
-  # network over the network. I.e. [header, [txs, uncles], receipts]
   if rlp.enterList():
     var blockHash: BlockHash
     try:
@@ -106,24 +103,16 @@ func readBlockData(
           contentType: blockBody,
           blockBodyKey: contentKeyType)
 
-        # Note: Temporary until the data format gets changed.
+        res.add((contentKey, @(rlp.rawData())))
-        let blockBody = BlockBody(
+        rlp.skipElem()
-          transactions: rlp.read(seq[Transaction]),
-          uncles: rlp.read(seq[BlockHeader]))
-        let rlpdata = encode(blockBody)
 
-        res.add((contentKey, rlpdata))
+      block:
-        # res.add((contentKey, @(rlp.rawData())))
+        let contentKey = ContentKey(
-        # rlp.skipElem()
+          contentType: receipts,
+          receiptsKey: contentKeyType)
 
-      # Note: No receipts yet in the data set
+        res.add((contentKey, @(rlp.rawData())))
-      # block:
+        rlp.skipElem()
-        # let contentKey = ContentKey(
-        #   contentType: receipts,
-        #   receiptsKey: contentKeyType)
-
-        # res.add((contentKey, @(rlp.rawData())))
-        # rlp.skipElem()
 
     except RlpError as e:
       return err("Invalid rlp data, number " & $blockData.number & ": " & e.msg)

fluffy/tests/blocks/mainnet_blocks_1-2.json

@@ -1,10 +1,10 @@
 {
   "0x88e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6": {
-    "rlp": "0xf90216f90211a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d493479405a56e2d52c817161883f50c441c3228cfe54d9fa0d67e4d450343046425ae4271474353857ab860dbc0a1dde64b41b5cd3a532bf3a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff80000001821388808455ba422499476574682f76312e302e302f6c696e75782f676f312e342e32a0969b900de27b6ac6a67742365dd65f55a0526c41fd18e1b16f1a1215c2e66f5988539bd4979fef1ec4c0c0",
+    "rlp": "0xf90218f90211a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d493479405a56e2d52c817161883f50c441c3228cfe54d9fa0d67e4d450343046425ae4271474353857ab860dbc0a1dde64b41b5cd3a532bf3a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff80000001821388808455ba422499476574682f76312e302e302f6c696e75782f676f312e342e32a0969b900de27b6ac6a67742365dd65f55a0526c41fd18e1b16f1a1215c2e66f5988539bd4979fef1ec4c2c0c0c0",
     "number": 1
   },
   "0xb495a1d7e6663152ae92708da4843337b958146015a2802f4193a410044698c9": {
-    "rlp": "0xf9021df90218a088e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794dd2f1e6e498202e86d8f5442af596580a4f03c2ca04943d941637411107494da9ec8bc04359d731bfd08b72b4d0edcbd4cd2ecb341a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff00100002821388808455ba4241a0476574682f76312e302e302d30636463373634372f6c696e75782f676f312e34a02f0790c5aa31ab94195e1f6443d645af5b75c46c04fbf9911711198a0ce8fdda88b853fa261a86aa9ec0c0",
+    "rlp": "0xf9021ff90218a088e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794dd2f1e6e498202e86d8f5442af596580a4f03c2ca04943d941637411107494da9ec8bc04359d731bfd08b72b4d0edcbd4cd2ecb341a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008503ff00100002821388808455ba4241a0476574682f76312e302e302d30636463373634372f6c696e75782f676f312e34a02f0790c5aa31ab94195e1f6443d645af5b75c46c04fbf9911711198a0ce8fdda88b853fa261a86aa9ec2c0c0c0",
     "number": 2
   }
 }

fluffy/tools/eth_data_exporter.nim

@@ -0,0 +1,157 @@
+# Nimbus
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+# Tool to download chain history data from local node, and save it to the json
+# file.
+# Data of each block is rlp encoded list of: 
+# [blockHeader, [block_transactions, block_uncles], block_receipts]
+# Json file has following format:
+# {
+#   "hexEncodedBlockHash: {
+#      "rlp": "hex of rlp encoded list [blockHeader, [block_transactions, block_uncles], block_receipts]",
+#      "number": "block number"
+#    },
+#   ...,
+#   ...,
+# }
+#
+#
+
+{.push raises: [Defect].}
+
+import
+  std/[json, typetraits, strutils, os],
+  confutils,
+  stew/[byteutils, io2],
+  json_serialization,
+  faststreams, chronicles,
+  eth/[common, rlp], chronos,
+  eth/common/eth_types_json_serialization,
+  ../../premix/downloader
+
+proc defaultDataDir*(): string =
+  let dataDir = when defined(windows):
+    "AppData" / "Roaming" / "EthData"
+  elif defined(macosx):
+    "Library" / "Application Support" / "EthData"
+  else:
+    ".cache" / "ethData"
+
+  getHomeDir() / dataDir
+
+const
+  defaultDataDirDesc = defaultDataDir()
+  defaultFileName = "eth-history-data.json"
+
+type
+  ExporterConf* = object
+    logLevel* {.
+      defaultValue: LogLevel.INFO
+      defaultValueDesc: $LogLevel.INFO
+      desc: "Sets the log level"
+      name: "log-level" .}: LogLevel
+    initialBlock* {.
+      desc: "Number of first block which should be downloaded"
+      defaultValue: 0
+      name: "initial-block" .}: uint64
+    endBlock* {.
+      desc: "Number of last block which should be downloaded"
+      defaultValue: 0
+      name: "end-block" .}: uint64
+    dataDir* {.
+      desc: "The directory where generated file will be placed"
+      defaultValue: defaultDataDir()
+      defaultValueDesc: $defaultDataDirDesc
+      name: "data-dir" .}: OutDir
+    filename* {.
+      desc: "default name of the file with history data"
+      defaultValue: defaultFileName
+      name: "filename" .}: string
+
+  DataRecord = object
+    rlp: string
+    number: uint64
+
+proc writeBlock(writer: var JsonWriter, blck: Block) {.raises: [IOError, Defect].} =
+  let 
+    enc = rlp.encodeList(blck.header, blck.body, blck.receipts)
+    asHex = to0xHex(enc)
+    dataRecord = DataRecord(rlp: asHex, number: cast[uint64](blck.header.blockNumber))
+    headerHash = to0xHex(rlpHash(blck.header).data)
+
+  writer.writeField(headerHash, dataRecord)
+
+proc downloadBlock(i: uint64): Block =
+  let num = u256(i)
+  try:
+    # premix has hardcoded making request to local host which is "127.0.0.1:8545"
+    # which is defult port of geth json rpc server
+    return requestBlock(num, flags = {DownloadReceipts})
+  except CatchableError as e:
+    fatal "Error while requesting Block", error = e.msg
+    quit 1
+
+proc createAndOpenFile(config: ExporterConf): OutputStreamHandle =
+  # Creates directory and file specified in config, if file already exists 
+  # program is aborted with info to user, to avoid losing data
+
+  let filePath = config.dataDir / config.filename
+
+  if isFile(filePath):
+    fatal "File under provided path already exists and would be overwritten",
+      path = filePath
+    quit 1
+
+  let res = createPath(distinctBase(config.dataDir))
+
+  if res.isErr():
+    fatal "Error occurred while creating directory", error = res.error
+    quit 1
+
+  try:
+    # this means that each time file be overwritten, but it is ok for such one
+    # off toll
+    return fileOutput(filePath)
+  except IOError as e:
+    fatal "Error occurred while opening the file", error = e.msg
+    quit 1
+
+proc run(config: ExporterConf) =
+  let fh = createAndOpenFile(config)
+
+  try:
+    var writer = JsonWriter[DefaultFlavor].init(fh.s)
+    writer.beginRecord()
+    for i in config.initialBlock..config.endBlock:
+      let blck = downloadBlock(i)
+      writer.writeBlock(blck)
+    writer.endRecord()
+    info "File successfully written"
+  except IOError as e:
+    fatal "Error occoured while writing to file", error = e.msg
+    quit 1
+  finally:
+    try:
+      fh.close()
+    except IOError as e:
+      fatal "Error occoured while closing file", error = e.msg
+      quit 1
+
+when isMainModule:
+  {.pop.}
+  let config = ExporterConf.load()
+  {.push raises: [Defect].}
+
+  if (config.endBlock < config.initialBlock):
+    fatal "Initial block number should be smaller than end block number",
+      initialBlock = config.initialBlock,
+      endBlock = config.endBlock
+    quit 1
+
+  setLogLevel(config.logLevel)
+
+  run(config)

nimbus.nimble

@@ -67,6 +67,7 @@ task fluffy, "Build fluffy":
 task fluffy_tools, "Build fluffy tools":
   buildBinary "portalcli", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"
   buildBinary "blockwalk", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"
+  buildBinary "eth_data_exporter", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"
 
 task utp_test_app, "Build uTP test app":
   buildBinary "utp_test_app", "fluffy/tools/utp_testing/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"