diff --git a/hive_integration/README.md b/hive_integration/README.md index f4d6b5222..c05cd7310 100644 --- a/hive_integration/README.md +++ b/hive_integration/README.md @@ -93,3 +93,29 @@ The number of passes and fails output at the time of writing (2021-04-26) is: devp2p/eth: 0 pass, 1 fail, 1 total ethereum/rpc: 3 pass, 35 fail, 38 total ethereum/sync: 0 pass, 1 fail, 1 total + +## Nim simulators without docker + +We have rewrite some of the hive simulators in Nim to aid debugging. +It is assumed you already install nimbus dependencies via nimble. +In the future, we will provide more instructions how to run these +simulators using local dependencies. + +On Windows you might need to add `-d:disable_libbacktrace` compiler switch. +Working directory is nimbus-eth1 root directory. And you can see the result +in a markdown file with the same name with the simulator. + +- ethereum/consensus + - first you need to run `extract_consensus_data` + ```nim + nim c -r -d:release hive_integration/nodocker/consensus/extract_consensus_data + ``` + - then you can run the simulator + ```nim + nim c -r -d:release hive_integration/nodocker/consensus/consensus_sim + ``` + +- ethereum/graphql + ```nim + nim c -r -d:release hive_integration/nodocker/graphql/graphql_sim + ``` diff --git a/hive_integration/nodocker/consensus/consensus_sim.nim b/hive_integration/nodocker/consensus/consensus_sim.nim new file mode 100644 index 000000000..8b7fee58e --- /dev/null +++ b/hive_integration/nodocker/consensus/consensus_sim.nim @@ -0,0 +1,50 @@ +# Nimbus +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +import + std/[os, parseopt, json], + eth/[common, p2p, trie/db], stew/byteutils, + ../../../nimbus/db/db_chain, + ../../../nimbus/[genesis, config, conf_utils], + ../sim_utils + +proc processNode(genesisFile, chainFile, + lastBlockHash: string, testStatusIMPL: var TestStatus) = + var msg: string + var opt = initOptParser("--customnetwork:" & genesisFile) + let res = processArguments(msg, opt) + if res != Success: + echo msg + quit(QuitFailure) + + let + conf = getConfiguration() + chainDB = newBaseChainDB(newMemoryDb(), + pruneTrie = false, + id = toPublicNetwork(conf.net.networkId) + ) + + initializeEmptyDb(chainDB) + discard importRlpBlock(chainFile, chainDB) + let head = chainDB.getCanonicalHead() + let blockHash = "0x" & head.blockHash.data.toHex + check blockHash == lastBlockHash + +proc main() = + let caseFolder = if paramCount() == 0: + "consensus_data" + else: + paramStr(1) + + runTest("Consensus", caseFolder): + let node = parseFile(fileName) + processNode(fileName, node["chainfile"].getStr, + node["lastblockhash"].getStr, testStatusIMPL) + +main() diff --git a/hive_integration/nodocker/consensus/extract_consensus_data.nim b/hive_integration/nodocker/consensus/extract_consensus_data.nim index f0f945664..0996fa7de 100644 --- a/hive_integration/nodocker/consensus/extract_consensus_data.nim +++ b/hive_integration/nodocker/consensus/extract_consensus_data.nim @@ -1,3 +1,12 @@ +# Nimbus +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + import std/[json, os, strutils, parseopt, terminal], stew/byteutils @@ -143,13 +152,18 @@ proc processNetWork(network: string): JsonNode = n["chainId"] = newJInt(1) result = n -proc extractChainData(n: JsonNode): ChainData = +proc extractChainData(n: JsonNode, chainFile: string): ChainData = let gen = n["genesisBlockHeader"] - var ngen = newJObject() + var genesis = newJObject() for x in genFields: - ngen[x] = gen[x] - ngen["alloc"] = n["pre"] + genesis[x] = gen[x] + genesis["alloc"] = n["pre"] + + var ngen = newJObject() + ngen["genesis"] = genesis ngen["config"] = processNetwork(n["network"].getStr) + ngen["lastblockhash"] = n["lastblockhash"] + ngen["chainfile"] = %chainFile result.genesis = ngen let blks = n["blocks"] @@ -160,8 +174,12 @@ proc extractChainData(n: JsonNode): ChainData = proc processFile(fileName, outPath: string): int = let n = json.parseFile(fileName) + let (folder, name) = fileName.splitPath() + let last = folder.splitPath().tail + for name, unit in n: - let cd = extractChainData(unit) + let name = last & "_" & name + let cd = extractChainData(unit, outPath / name & "_chain.rlp") writeFile(outPath / name & "_config.json", cd.genesis.pretty) writeFile(outPath / name & "_chain.rlp", cd.blocksRlp) inc result @@ -209,4 +227,3 @@ proc main() = echo "\ntest data generated: ", count main() - diff --git a/hive_integration/nodocker/graphql/graphql_sim.nim b/hive_integration/nodocker/graphql/graphql_sim.nim index e965550b9..d29021217 100644 --- a/hive_integration/nodocker/graphql/graphql_sim.nim +++ b/hive_integration/nodocker/graphql/graphql_sim.nim @@ -1,10 +1,19 @@ +# Nimbus +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + import - std/[os, parseopt, json, unittest], chronicles, + std/[os, parseopt, json], eth/[p2p, trie/db], ../../../nimbus/db/db_chain, eth/p2p/rlpx_protocols/eth_protocol, ../../../nimbus/[genesis, config, conf_utils], ../../../nimbus/graphql/ethapi, ../../../tests/test_helpers, - graphql + graphql, ../sim_utils const baseFolder = "hive_integration" / "nodocker" / "graphql" @@ -76,10 +85,8 @@ proc main() = importRlpBlock(blocksFile, chainDB) let ctx = setupGraphqlContext(chainDB, ethNode) - suite "graphql": - for fileName in walkDirRec(caseFolder): - let node = parseFile(fileName) - test fileName: - ctx.processNode(node, fileName, testStatusIMPL) + runTest("GraphQL", caseFolder): + let node = parseFile(fileName) + ctx.processNode(node, fileName, testStatusIMPL) main() diff --git a/hive_integration/nodocker/graphql/init/genesis.json b/hive_integration/nodocker/graphql/init/genesis.json index 719b23333..d26036f5a 100644 --- a/hive_integration/nodocker/graphql/init/genesis.json +++ b/hive_integration/nodocker/graphql/init/genesis.json @@ -1,4 +1,5 @@ { + "genesis": { "coinbase" : "0x8888f1f195afa192cfee860698584c030f4c9db1", "difficulty" : "0x020000", "extraData" : "0x42", @@ -12,4 +13,5 @@ "balance" : "0x09184e72a000" } } -} \ No newline at end of file + } +} diff --git a/hive_integration/nodocker/sim_utils.nim b/hive_integration/nodocker/sim_utils.nim new file mode 100644 index 000000000..8ba86b49a --- /dev/null +++ b/hive_integration/nodocker/sim_utils.nim @@ -0,0 +1,40 @@ +# Nimbus +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +import + std/[tables, strutils, unittest], + testutils/markdown_reports + +export + tables, strutils, unittest, + markdown_reports + +template runTest*(suiteName: string, caseFolder: string, body: untyped) = + disableParamFiltering() + suite suiteName: + var status = initOrderedTable[string, OrderedTable[string, Status]]() + for fileName {.inject.} in walkDirRec(caseFolder): + if not fileName.endsWith(".json"): + continue + + let (folder, name) = fileName.splitPath() + let last = folder.splitPath().tail + if last notin status: + status[last] = initOrderedTable[string, Status]() + + test fileName: + # we set this here because exceptions might be raised in the handler + status[last][name] = Status.Fail + body + if testStatusIMPL == OK: + status[last][name] = Status.OK + elif testStatusIMPL == SKIPPED: + status[last][name] = Status.Skip + + generateReport(suiteName, status)