fixes and add hive simulators written in nim

now we have hive simulators written in nim:
- ethereum/consensus
- ethereum/graphql

Using these simulators, we can debug test cases without have to run
hive or docker.

hive_integration/README.md

@@ -93,3 +93,29 @@ The number of passes and fails output at the time of writing (2021-04-26) is:
     devp2p/eth:              0 pass,     1 fail,     1 total
     ethereum/rpc:            3 pass,    35 fail,    38 total
     ethereum/sync:           0 pass,     1 fail,     1 total
+
+## Nim simulators without docker
+
+We have rewrite some of the hive simulators in Nim to aid debugging.
+It is assumed you already install nimbus dependencies via nimble.
+In the future, we will provide more instructions how to run these
+simulators using local dependencies.
+
+On Windows you might need to add `-d:disable_libbacktrace` compiler switch.
+Working directory is nimbus-eth1 root directory. And you can see the result
+in a markdown file with the same name with the simulator.
+
+- ethereum/consensus
+  - first you need to run `extract_consensus_data`
+    ```nim
+    nim c -r -d:release hive_integration/nodocker/consensus/extract_consensus_data
+    ```
+  - then you can run the simulator
+    ```nim
+    nim c -r -d:release hive_integration/nodocker/consensus/consensus_sim
+    ```
+
+- ethereum/graphql
+  ```nim
+  nim c -r -d:release hive_integration/nodocker/graphql/graphql_sim
+  ```

hive_integration/nodocker/consensus/consensus_sim.nim

@@ -0,0 +1,50 @@
+# Nimbus
+# Copyright (c) 2021 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+import
+  std/[os, parseopt, json],
+  eth/[common, p2p, trie/db], stew/byteutils,
+  ../../../nimbus/db/db_chain,
+  ../../../nimbus/[genesis, config, conf_utils],
+  ../sim_utils
+
+proc processNode(genesisFile, chainFile,
+                 lastBlockHash: string, testStatusIMPL: var TestStatus) =
+  var msg: string
+  var opt = initOptParser("--customnetwork:" & genesisFile)
+  let res = processArguments(msg, opt)
+  if res != Success:
+    echo msg
+    quit(QuitFailure)
+
+  let
+    conf = getConfiguration()
+    chainDB = newBaseChainDB(newMemoryDb(),
+      pruneTrie = false,
+      id = toPublicNetwork(conf.net.networkId)
+    )
+
+  initializeEmptyDb(chainDB)
+  discard importRlpBlock(chainFile, chainDB)
+  let head = chainDB.getCanonicalHead()
+  let blockHash = "0x" & head.blockHash.data.toHex
+  check blockHash == lastBlockHash
+
+proc main() =
+  let caseFolder = if paramCount() == 0:
+                     "consensus_data"
+                   else:
+                     paramStr(1)
+
+  runTest("Consensus", caseFolder):
+    let node = parseFile(fileName)
+    processNode(fileName, node["chainfile"].getStr,
+      node["lastblockhash"].getStr, testStatusIMPL)
+
+main()

hive_integration/nodocker/consensus/extract_consensus_data.nim

@@ -1,3 +1,12 @@
+# Nimbus
+# Copyright (c) 2021 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
 import
   std/[json, os, strutils, parseopt, terminal],
   stew/byteutils
@@ -143,13 +152,18 @@ proc processNetWork(network: string): JsonNode =
   n["chainId"]             = newJInt(1)
   result = n
 
-proc extractChainData(n: JsonNode): ChainData =
+proc extractChainData(n: JsonNode, chainFile: string): ChainData =
   let gen = n["genesisBlockHeader"]
-  var ngen = newJObject()
+  var genesis = newJObject()
   for x in genFields:
-    ngen[x] = gen[x]
-  ngen["alloc"] = n["pre"]
+    genesis[x] = gen[x]
+  genesis["alloc"] = n["pre"]
+
+  var ngen = newJObject()
+  ngen["genesis"] = genesis
   ngen["config"] = processNetwork(n["network"].getStr)
+  ngen["lastblockhash"] = n["lastblockhash"]
+  ngen["chainfile"] = %chainFile
   result.genesis = ngen
 
   let blks = n["blocks"]
@@ -160,8 +174,12 @@ proc extractChainData(n: JsonNode): ChainData =
 
 proc processFile(fileName, outPath: string): int =
   let n = json.parseFile(fileName)
+  let (folder, name) = fileName.splitPath()
+  let last = folder.splitPath().tail
+
   for name, unit in n:
-    let cd = extractChainData(unit)
+    let name = last & "_" & name
+    let cd = extractChainData(unit, outPath / name & "_chain.rlp")
     writeFile(outPath / name & "_config.json", cd.genesis.pretty)
     writeFile(outPath / name & "_chain.rlp", cd.blocksRlp)
     inc result
@@ -209,4 +227,3 @@ proc main() =
   echo "\ntest data generated: ", count
 
 main()
-

hive_integration/nodocker/graphql/graphql_sim.nim

@@ -1,10 +1,19 @@
+# Nimbus
+# Copyright (c) 2021 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
 import
-  std/[os, parseopt, json, unittest], chronicles,
+  std/[os, parseopt, json],
   eth/[p2p, trie/db], ../../../nimbus/db/db_chain,
   eth/p2p/rlpx_protocols/eth_protocol,
   ../../../nimbus/[genesis, config, conf_utils],
   ../../../nimbus/graphql/ethapi, ../../../tests/test_helpers,
-  graphql
+  graphql, ../sim_utils
 
 const
   baseFolder  = "hive_integration" / "nodocker" / "graphql"
@@ -76,10 +85,8 @@ proc main() =
   importRlpBlock(blocksFile, chainDB)
   let ctx = setupGraphqlContext(chainDB, ethNode)
 
-  suite "graphql":
-    for fileName in walkDirRec(caseFolder):
-      let node = parseFile(fileName)
-      test fileName:
-        ctx.processNode(node, fileName, testStatusIMPL)
+  runTest("GraphQL", caseFolder):
+    let node = parseFile(fileName)
+    ctx.processNode(node, fileName, testStatusIMPL)
 
 main()

hive_integration/nodocker/graphql/init/genesis.json

@@ -1,4 +1,5 @@
 {
+  "genesis": {
     "coinbase"   : "0x8888f1f195afa192cfee860698584c030f4c9db1",
     "difficulty" : "0x020000",
     "extraData"  : "0x42",
@@ -12,4 +13,5 @@
             "balance" : "0x09184e72a000"
         }
     }
-}
+  }
+}

hive_integration/nodocker/sim_utils.nim

@@ -0,0 +1,40 @@
+# Nimbus
+# Copyright (c) 2021 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+import
+  std/[tables, strutils, unittest],
+  testutils/markdown_reports
+
+export
+  tables, strutils, unittest,
+  markdown_reports
+
+template runTest*(suiteName: string, caseFolder: string, body: untyped) =
+  disableParamFiltering()
+  suite suiteName:
+    var status = initOrderedTable[string, OrderedTable[string, Status]]()
+    for fileName {.inject.} in walkDirRec(caseFolder):
+      if not fileName.endsWith(".json"):
+        continue
+
+      let (folder, name) = fileName.splitPath()
+      let last = folder.splitPath().tail
+      if last notin status:
+        status[last] = initOrderedTable[string, Status]()
+
+      test fileName:
+        # we set this here because exceptions might be raised in the handler
+        status[last][name] = Status.Fail
+        body
+        if testStatusIMPL == OK:
+          status[last][name] = Status.OK
+        elif testStatusIMPL == SKIPPED:
+          status[last][name] = Status.Skip
+
+    generateReport(suiteName, status)