Add blockwalk tool to test range of blocks on Portal network (#1020)

Makefile

@@ -170,7 +170,7 @@ fluffy-test: | build deps
 
 # builds the fluffy tools
 fluffy-tools: | build deps
-	$(ENV_SCRIPT) nim portalcli $(NIM_PARAMS) nimbus.nims
+	$(ENV_SCRIPT) nim fluffy_tools $(NIM_PARAMS) nimbus.nims
 
 # builds the fluffy tools
 utp-test-app: | build deps

fluffy/rpc/eth_rpc_client.nim

@@ -8,9 +8,10 @@
 import
   std/os,
   json_rpc/rpcclient,
+  json_rpc/errors, # TODO: should be exported in json_rpc/clients/httpclient
   web3/conversions, # sigh
   ../../nimbus/rpc/[rpc_types, hexstrings, rpc_utils]
 
-export rpcclient, rpc_types
+export rpcclient, rpc_types, errors
 
 createRpcSigs(RpcClient, currentSourcePath.parentDir / "rpc_calls" / "rpc_eth_calls.nim")

fluffy/tools/blockwalk.nim

@@ -0,0 +1,103 @@
+# Nimbus
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+# Testing tool to verify that a specific range of blocks can be fetched from
+# the Portal network.
+
+{.push raises: [Defect].}
+
+import
+  confutils, chronicles, chronicles/topics_registry, stew/byteutils,
+  eth/common/eth_types,
+  ../../nimbus/rpc/[hexstrings, rpc_types], ../../nimbus/errors,
+  ../rpc/eth_rpc_client
+
+type
+  BlockWalkConf* = object
+    logLevel* {.
+      defaultValue: LogLevel.INFO
+      defaultValueDesc: $LogLevel.INFO
+      desc: "Sets the log level"
+      name: "log-level" .}: LogLevel
+
+    rpcAddress* {.
+      desc: "Address of the JSON-RPC service"
+      defaultValue: "127.0.0.1"
+      name: "rpc-address" .}: string
+
+    rpcPort* {.
+      defaultValue: 8545
+      desc: "Port of the JSON-RPC service"
+      name: "rpc-port" .}: uint16
+
+    blockHash* {.
+      desc: "The block hash from where to start walking the blocks backwards"
+      name: "block-hash" .}: Hash256
+
+proc parseCmdArg*(T: type Hash256, p: TaintedString): T
+    {.raises: [Defect, ConfigurationError].} =
+  var hash: Hash256
+  try:
+    hexToByteArray(p, hash.data)
+  except ValueError:
+    raise newException(ConfigurationError, "Invalid Hash256")
+
+  return hash
+
+proc completeCmdArg*(T: type Hash256, val: TaintedString): seq[string] =
+  return @[]
+
+proc walkBlocks(client: RpcClient, startHash: Hash256) {.async.} =
+  var parentHash = startHash
+  var blockNumber = ""
+
+  # Should be 0x0, but block 0 does not exist in the json data file
+  while blockNumber != "0x1":
+    let parentBlockOpt =
+      try:
+        await client.eth_getBlockByHash(parentHash.ethHashStr(), false)
+      except ValidationError as e:
+        # RpcPostError when for example timing out on the request. Could retry
+        # in this case.
+        fatal "Error occured on JSON-RPC request", error = e.msg
+        quit 1
+      except RpcPostError as e:
+        # ValidationError from buildBlockObject, should not occur with proper
+        # blocks
+        fatal "Error occured on JSON-RPC request", error = e.msg
+        quit 1
+
+    if parentBlockOpt.isNone():
+      fatal "Failed getting parent block", hash = parentHash.data.toHex()
+      quit 1
+
+    let parentBlock = parentBlockOpt.get()
+    if parentBlock.number.isNone() or parentBlock.hash.isNone():
+      fatal "Parent block cannot be a pending block",
+        hash = parentHash.data.toHex()
+      quit 1
+
+    blockNumber = parentBlock.number.get().string
+    parentHash = parentBlock.parentHash
+
+    echo "Block number: " & blockNumber
+    echo "Block hash: " & "0x" & parentBlock.hash.get().data.toHex()
+
+proc run(config: BlockWalkConf) {.async.} =
+  let client = newRpcHttpClient()
+  await client.connect(config.rpcAddress, Port(config.rpcPort), false)
+
+  await walkBlocks(client, config.blockHash)
+
+when isMainModule:
+  {.pop.}
+  let config = BlockWalkConf.load()
+  {.push raises: [Defect].}
+
+  setLogLevel(config.logLevel)
+
+  waitFor run(config)

fluffy/tools/nim.cfg

@@ -0,0 +1,2 @@
+-d:"chronicles_runtime_filtering=on"
+-d:"chronicles_disable_thread_id"

nimbus.nimble

@@ -51,8 +51,9 @@ task test, "Run tests":
 task fluffy, "Build fluffy":
   buildBinary "fluffy", "fluffy/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"
 
-task portalcli, "Build portalcli":
+task fluffy_tools, "Build fluffy tools":
   buildBinary "portalcli", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"
+  buildBinary "blockwalk", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"
 
 task utp_test_app, "Build uTP test app":
   buildBinary "utp_test_app", "fluffy/tools/utp_testing/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"