modify hive simulators to run in CI

hive_integration/nodocker/consensus/consensus_sim.nim

@@ -8,44 +8,54 @@
 # those terms.
 
 import
-  std/[os, strformat, json],
+  std/[os, json, strutils, times],
-  eth/[common, trie/db], stew/byteutils,
+  eth/[common, trie/db, p2p], stew/byteutils,
   ../../../nimbus/db/db_chain,
-  ../../../nimbus/[genesis, config, conf_utils],
+  ../../../nimbus/[genesis, chain_config, conf_utils],
-  ../sim_utils
+  ../sim_utils,
+  ./extract_consensus_data
 
-proc processNode(genesisFile, chainFile,
+proc processChainData(cd: ChainData): TestStatus =
-                 lastBlockHash: string, testStatusIMPL: var TestStatus) =
+  var np: NetworkParams
+  doAssert decodeNetworkParams(cd.genesis, np)
 
   let
-    conf = makeConfig(@["--custom-network:" & genesisFile])
+    networkId = NetworkId(np.config.chainId)
     chainDB = newBaseChainDB(newMemoryDB(),
       pruneTrie = false,
-      conf.networkId,
+      networkId,
-      conf.networkParams
+      np
     )
 
   initializeEmptyDb(chainDB)
-  discard importRlpBlock(chainFile, chainDB)
+  discard importRlpBlock(cd.blocksRlp, chainDB, "consensus_sim")
   let head = chainDB.getCanonicalHead()
   let blockHash = "0x" & head.blockHash.data.toHex
-  check blockHash == lastBlockHash
+  if blockHash == cd.lastBlockHash:
+    TestStatus.OK
+  else:
+    TestStatus.Failed
 
 proc main() =
-  let caseFolder = if paramCount() == 0:
+  const basePath = "tests" / "fixtures" / "eth_tests" / "BlockchainTests"
-                     "consensus_data"
+  var stat: SimStat
-                   else:
+  let start = getTime()
-                     paramStr(1)
 
-  if not caseFolder.dirExists:
+  for fileName in walkDirRec(basePath):
-    # Handy early error message and stop directive
+    if not fileName.endsWith(".json"):
-    let progname = getAppFilename().extractFilename
+      continue
-    quit(&"*** {progname}: Not a case folder: {caseFolder}")
 
-  runTest("Consensus", caseFolder):
+    let n = json.parseFile(fileName)
-    # Variable `fileName` is injected by `runTest()`
+    for name, unit in n:
-    let node = parseFile(fileName)
+      if "loopMul" in name:
-    processNode(fileName, node["chainfile"].getStr,
+        inc stat.skipped
-      node["lastblockhash"].getStr, testStatusIMPL)
+        continue
+
+      let cd = extractChainData(unit)
+      let status = processChainData(cd)
+      stat.inc(name, status)
+
+  let elpd = getTime() - start
+  print(stat, elpd, "consensus")
 
 main()

hive_integration/nodocker/consensus/extract_consensus_data.nim

@@ -8,17 +8,14 @@
 # those terms.
 
 import
-  std/[json, os, strutils, parseopt, terminal],
+  std/[json, strutils],
   stew/byteutils
 
 type
-  ChainData = object
+  ChainData* = object
-    genesis: JsonNode
+    genesis*: string
-    blocksRlp: seq[byte]
+    lastBlockHash*: string
-
+    blocksRlp*: seq[byte]
-  Config = object
-    outPath: string
-    filter: string
 
 const genFields = [
   "nonce",
@@ -179,7 +176,7 @@ proc optionalField(n: string, genesis, gen: JsonNode) =
   if n in gen:
     genesis[n] = gen[n]
 
-proc extractChainData(n: JsonNode, chainFile: string): ChainData =
+proc extractChainData*(n: JsonNode): ChainData =
   let gen = n["genesisBlockHeader"]
   var genesis = newJObject()
   for x in genFields:
@@ -191,68 +188,11 @@ proc extractChainData(n: JsonNode, chainFile: string): ChainData =
   var ngen = newJObject()
   ngen["genesis"] = genesis
   ngen["config"] = processNetwork(n["network"].getStr)
-  ngen["lastblockhash"] = n["lastblockhash"]
+  result.lastblockhash = n["lastblockhash"].getStr
-  ngen["chainfile"] = %chainFile
+  result.genesis = $ngen
-  result.genesis = ngen
 
   let blks = n["blocks"]
   for x in blks:
     let hex = x["rlp"].getStr
     let bytes = hexToSeqByte(hex)
     result.blocksRlp.add bytes
-
-proc processFile(fileName, outPath: string): int =
-  let n = json.parseFile(fileName)
-  let (folder, name) = fileName.splitPath()
-  let last = folder.splitPath().tail
-
-  for name, unit in n:
-    let name = last & "_" & name
-    let cd = extractChainData(unit, outPath / name & "_chain.rlp")
-    writeFile(outPath / name & "_config.json", cd.genesis.pretty)
-    writeFile(outPath / name & "_chain.rlp", cd.blocksRlp)
-    inc result
-
-proc initConfig(): Config =
-  result.outPath = "consensus_data"
-
-proc processArguments(conf: var Config) =
-  var opt = initOptParser()
-
-  for kind, key, value in opt.getopt():
-    case kind
-    of cmdArgument:
-      conf.filter = key
-    of cmdLongOption, cmdShortOption:
-      case key.toLowerAscii()
-      of "o": conf.outPath = value
-      else:
-        var msg = "Unknown option " & key
-        if value.len > 0: msg = msg & " : " & value
-        quit(QuitFailure)
-    of cmdEnd:
-      doAssert(false)
-
-proc main() =
-  const basePath = "tests" / "fixtures" / "eth_tests" / "BlockchainTests"
-
-  var conf = initConfig()
-  processArguments(conf)
-  createDir(conf.outPath)
-
-  var count = 0
-  for fileName in walkDirRec(basePath):
-    if not fileName.endsWith(".json"):
-      continue
-    let (_, name) = fileName.splitPath()
-    if conf.filter notin fileName:
-      continue
-
-    terminal.eraseLine(stdout)
-    stdout.write "\r"
-    stdout.write name
-    count += processFile(fileName, conf.outPath)
-
-  echo "\ntest data generated: ", count
-
-main()

hive_integration/nodocker/engine/engine_sim.nim

@@ -1,19 +1,21 @@
 import
   test_env,
   engine_tests,
-  chronos,
+  unittest2,
-  unittest2
+  ../sim_utils
-
-proc runTest(x: TestSpec, testStatusIMPL: var TestStatus) =
-  var t = setupELClient()
-  t.setRealTTD(x.ttd)
-  x.run(t, testStatusIMPL)
-  t.stopELClient()
 
 proc main() =
-  suite "Engine Tests":
+  var stat: SimStat
+  let start = getTime()
+
   for x in engineTestList:
-      test x.name:
+    var t = setupELClient()
-        runTest(x, testStatusIMPL)
+    t.setRealTTD(x.ttd)
+    let status = x.run(t)
+    t.stopELClient()
+    stat.inc(x.name, status)
+
+  let elpd = getTime() - start
+  print(stat, elpd, "engine")
 
 main()

hive_integration/nodocker/engine/engine_tests.nim

@@ -14,15 +14,26 @@ import
 type
   TestSpec* = object
     name*: string
-    run*: proc(t: TestEnv, testStatusIMPL: var TestStatus)
+    run*: proc(t: TestEnv): TestStatus
     ttd*: int64
 
 const
   prevRandaoContractAddr = hexToByteArray[20]("0000000000000000000000000000000000000316")
 
+template testCond(expr: untyped) =
+  if not (expr):
+    return TestStatus.Failed
+
+template testCond(expr, body: untyped) =
+  if not (expr):
+    body
+    return TestStatus.Failed
+
 # Invalid Terminal Block in ForkchoiceUpdated:
 # Client must reject ForkchoiceUpdated directives if the referenced HeadBlockHash does not meet the TTD requirement.
-proc invalidTerminalBlockForkchoiceUpdated(t: TestEnv, testStatusIMPL: var TestStatus) =
+proc invalidTerminalBlockForkchoiceUpdated(t: TestEnv): TestStatus =
+  result = TestStatus.OK
+
   let
     gHash = Web3BlockHash t.gHeader.blockHash.data
     forkchoiceState = ForkchoiceStateV1(
@@ -36,28 +47,29 @@ proc invalidTerminalBlockForkchoiceUpdated(t: TestEnv, testStatusIMPL: var TestS
   # Execution specification:
   # {payloadStatus: {status: INVALID_TERMINAL_BLOCK, latestValidHash: null, validationError: errorMessage | null}, payloadId: null}
   # either obtained from the Payload validation process or as a result of validating a PoW block referenced by forkchoiceState.headBlockHash
-  check res.isOk
+  testCond res.isOk
-
-  if res.isErr:
-    return
 
   let s = res.get()
-  check s.payloadStatus.status == PayloadExecutionStatus.invalid_terminal_block
+  testCond s.payloadStatus.status == PayloadExecutionStatus.invalid_terminal_block
-  check s.payloadStatus.latestValidHash.isNone
+  testCond s.payloadStatus.latestValidHash.isNone
-  check s.payloadId.isNone
+  testCond s.payloadId.isNone
 
   # ValidationError is not validated since it can be either null or a string message
 
 # Invalid GetPayload Under PoW: Client must reject GetPayload directives under PoW.
-proc invalidGetPayloadUnderPoW(t: TestEnv, testStatusIMPL: var TestStatus) =
+proc invalidGetPayloadUnderPoW(t: TestEnv): TestStatus =
+  result = TestStatus.OK
+
   # We start in PoW and try to get an invalid Payload, which should produce an error but nothing should be disrupted.
   let id = PayloadID [1.byte, 2,3,4,5,6,7,8]
   let res = t.rpcClient.getPayloadV1(id)
-  check res.isErr
+  testCond res.isErr
 
 # Invalid Terminal Block in NewPayload:
 # Client must reject NewPayload directives if the referenced ParentHash does not meet the TTD requirement.
-proc invalidTerminalBlockNewPayload(t: TestEnv, testStatusIMPL: var TestStatus) =
+proc invalidTerminalBlockNewPayload(t: TestEnv): TestStatus =
+  result = TestStatus.OK
+
   let gBlock = t.gHeader
   let payload = ExecutableData(
     parentHash:   gBlock.blockHash,
@@ -75,23 +87,20 @@ proc invalidTerminalBlockNewPayload(t: TestEnv, testStatusIMPL: var TestStatus)
   # Execution specification:
   # {status: INVALID_TERMINAL_BLOCK, latestValidHash: null, validationError: errorMessage | null}
   # if terminal block conditions are not satisfied
-  check res.isOk
+  testCond res.isOk
-  if res.isErr:
-    return
 
   let s = res.get()
-  check s.status == PayloadExecutionStatus.invalid_terminal_block
+  testCond s.status == PayloadExecutionStatus.invalid_terminal_block
-  check s.latestValidHash.isNone
+  testCond s.latestValidHash.isNone
+
+proc unknownHeadBlockHash(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc unknownHeadBlockHash(t: TestEnv, testStatusIMPL: var TestStatus) =
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-
-  if not ok:
-    return
 
   var randomHash: Hash256
-  check nimcrypto.randomBytes(randomHash.data) == 32
+  testCond nimcrypto.randomBytes(randomHash.data) == 32
 
   let clMock = t.clMock
   let forkchoiceStateUnknownHeadHash = ForkchoiceStateV1(
@@ -101,16 +110,14 @@ proc unknownHeadBlockHash(t: TestEnv, testStatusIMPL: var TestStatus) =
   )
 
   var res = t.rpcClient.forkchoiceUpdatedV1(forkchoiceStateUnknownHeadHash)
-  check res.isOk
+  testCond res.isOk
-  if res.isErr:
-    return
 
   let s = res.get()
   # Execution specification::
   # - {payloadStatus: {status: SYNCING, latestValidHash: null, validationError: null}, payloadId: null}
   #   if forkchoiceState.headBlockHash references an unknown payload or a payload that can't be validated
   #   because requisite data for the validation is missing
-  check s.payloadStatus.status == PayloadExecutionStatus.syncing
+  testCond s.payloadStatus.status == PayloadExecutionStatus.syncing
 
   # Test again using PayloadAttributes, should also return SYNCING and no PayloadID
   let timestamp = uint64 clMock.latestExecutedPayload.timestamp
@@ -119,22 +126,19 @@ proc unknownHeadBlockHash(t: TestEnv, testStatusIMPL: var TestStatus) =
   )
 
   res = t.rpcClient.forkchoiceUpdatedV1(forkchoiceStateUnknownHeadHash, some(payloadAttr))
-  check res.isOk
+  testCond res.isOk
-  if res.isErr:
+  testCond s.payloadStatus.status == PayloadExecutionStatus.syncing
-    return
+  testCond s.payloadId.isNone
-  check s.payloadStatus.status == PayloadExecutionStatus.syncing
+
-  check s.payloadId.isNone
+proc unknownSafeBlockHash(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc unknownSafeBlockHash(t: TestEnv, testStatusIMPL: var TestStatus) =
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
 
   let clMock = t.clMock
   let client = t.rpcClient
@@ -157,18 +161,17 @@ proc unknownSafeBlockHash(t: TestEnv, testStatusIMPL: var TestStatus) =
       return res.isErr
   ))
 
-  check produceSingleBlockRes
+  testCond produceSingleBlockRes
+
+proc unknownFinalizedBlockHash(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc unknownFinalizedBlockHash(t: TestEnv, testStatusIMPL: var TestStatus) =
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
 
   let clMock = t.clMock
   let client = t.rpcClient
@@ -200,19 +203,17 @@ proc unknownFinalizedBlockHash(t: TestEnv, testStatusIMPL: var TestStatus) =
       return res.isErr
   ))
 
-  check produceSingleBlockRes
+  testCond produceSingleBlockRes
+
+proc preTTDFinalizedBlockHash(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc preTTDFinalizedBlockHash(t: TestEnv, testStatusIMPL: var TestStatus) =
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
-  if not produce5BlockRes:
-    return
 
   let
     gHash = Web3BlockHash t.gHeader.blockHash.data
@@ -229,24 +230,19 @@ proc preTTDFinalizedBlockHash(t: TestEnv, testStatusIMPL: var TestStatus) =
   # if not defined on re-orgs to a point before the latest finalized block.
 
   res = client.forkchoiceUpdatedV1(clMock.latestForkchoice)
-  check res.isOk
+  testCond res.isOk
-  if res.isErr:
-    return
-
   let s = res.get()
-  check s.payloadStatus.status == PayloadExecutionStatus.valid
+  testCond s.payloadStatus.status == PayloadExecutionStatus.valid
+
+proc badHashOnExecPayload(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc badHashOnExecPayload(t: TestEnv, testStatusIMPL: var TestStatus) =
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
-  if not produce5BlockRes:
-    return
 
   type
     Shadow = ref object
@@ -274,9 +270,7 @@ proc badHashOnExecPayload(t: TestEnv, testStatusIMPL: var TestStatus) =
       let s = res.get()
       s.status == PayloadExecutionStatus.invalid_block_hash
   ))
-  check produceSingleBlockRes
+  testCond produceSingleBlockRes
-  if not produceSingleBlockRes:
-    return
 
   # Lastly, attempt to build on top of the invalid payload
   produceSingleBlockRes = clMock.produceSingleBlock(BlockProcessCallbacks(
@@ -295,20 +289,18 @@ proc badHashOnExecPayload(t: TestEnv, testStatusIMPL: var TestStatus) =
       let s = res.get()
       s.status != PayloadExecutionStatus.valid
   ))
-  check produceSingleBlockRes
+  testCond produceSingleBlockRes
+
+proc parentHashOnExecPayload(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc parentHashOnExecPayload(t: TestEnv, testStatusIMPL: var TestStatus) =
   # Wait until TTD is reached by this client
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
-  if not produce5BlockRes:
-    return
 
   let clMock = t.clMock
   let client = t.rpcClient
@@ -326,31 +318,29 @@ proc parentHashOnExecPayload(t: TestEnv, testStatusIMPL: var TestStatus) =
       let s = res.get()
       s.status == PayloadExecutionStatus.invalid_block_hash
   ))
-  check produceSingleBlockRes
+  testCond produceSingleBlockRes
 
-proc invalidPayloadTestCaseGen(payloadField: string): proc (t: TestEnv, testStatusIMPL: var TestStatus) =
+proc invalidPayloadTestCaseGen(payloadField: string): proc (t: TestEnv): TestStatus =
-  return proc (t: TestEnv, testStatusIMPL: var TestStatus) =
+  return proc (t: TestEnv): TestStatus =
-    discard
+    result = TestStatus.SKIPPED
 
 # Test to verify Block information available at the Eth RPC after NewPayload
-proc blockStatusExecPayload(t: TestEnv, testStatusIMPL: var TestStatus) =
+proc blockStatusExecPayload(t: TestEnv): TestStatus =
+  result = TestStatus.OK
+
   # Wait until TTD is reached by this client
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
-  if not produce5BlockRes:
-    return
 
   let clMock = t.clMock
   let client = t.rpcClient
   var produceSingleBlockRes = clMock.produceSingleBlock(BlockProcessCallbacks(
     onNewPayloadBroadcast: proc(): bool =
-      # TODO: Ideally, we would need to check that the newPayload returned VALID
+      # TODO: Ideally, we would need to testCond that the newPayload returned VALID
       var lastHeader: EthBlockHeader
       var hRes = client.latestHeader(lastHeader)
       if hRes.isErr:
@@ -381,20 +371,18 @@ proc blockStatusExecPayload(t: TestEnv, testStatusIMPL: var TestStatus) =
 
       return true
   ))
-  check produceSingleBlockRes
+  testCond produceSingleBlockRes
+
+proc blockStatusHeadBlock(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc blockStatusHeadBlock(t: TestEnv, testStatusIMPL: var TestStatus) =
   # Wait until TTD is reached by this client
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
-  if not produce5BlockRes:
-    return
 
   let clMock = t.clMock
   let client = t.rpcClient
@@ -415,20 +403,18 @@ proc blockStatusHeadBlock(t: TestEnv, testStatusIMPL: var TestStatus) =
         return false
       return true
   ))
-  check produceSingleBlockRes
+  testCond produceSingleBlockRes
+
+proc blockStatusSafeBlock(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc blockStatusSafeBlock(t: TestEnv, testStatusIMPL: var TestStatus) =
   # Wait until TTD is reached by this client
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
-  if not produce5BlockRes:
-    return
 
   let clMock = t.clMock
   let client = t.rpcClient
@@ -449,20 +435,18 @@ proc blockStatusSafeBlock(t: TestEnv, testStatusIMPL: var TestStatus) =
         return false
       return true
   ))
-  check produceSingleBlockRes
+  testCond produceSingleBlockRes
+
+proc blockStatusFinalizedBlock(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc blockStatusFinalizedBlock(t: TestEnv, testStatusIMPL: var TestStatus) =
   # Wait until TTD is reached by this client
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
-  if not produce5BlockRes:
-    return
 
   let clMock = t.clMock
   let client = t.rpcClient
@@ -483,20 +467,18 @@ proc blockStatusFinalizedBlock(t: TestEnv, testStatusIMPL: var TestStatus) =
         return false
       return true
   ))
-  check produceSingleBlockRes
+  testCond produceSingleBlockRes
+
+proc blockStatusReorg(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc blockStatusReorg(t: TestEnv, testStatusIMPL: var TestStatus) =
   # Wait until TTD is reached by this client
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
-  if not produce5BlockRes:
-    return
 
   let clMock = t.clMock
   let client = t.rpcClient
@@ -545,7 +527,7 @@ proc blockStatusReorg(t: TestEnv, testStatusIMPL: var TestStatus) =
           get=latestValidHash.toHex
         return false
 
-      # Check that we reorg to the previous block
+      # testCond that we reorg to the previous block
       hRes = client.latestHeader(currHeader)
       if hRes.isErr:
         error "unable to get latest header", msg=hRes.error
@@ -582,31 +564,27 @@ proc blockStatusReorg(t: TestEnv, testStatusIMPL: var TestStatus) =
         return false
       return true
   ))
-  check produceSingleBlockRes
+  testCond produceSingleBlockRes
+
+proc reExecPayloads(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc reExecPayloads(t: TestEnv, testStatusIMPL: var TestStatus) =
   # Wait until this client catches up with latest PoS
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # How many Payloads we are going to re-execute
   var payloadReExecCount = 10
 
   # Create those blocks
   let produceBlockRes = t.clMock.produceBlocks(payloadReExecCount, BlockProcessCallbacks())
-  check produceBlockRes
+  testCond produceBlockRes
-  if not produceBlockRes:
-    return
 
   # Re-execute the payloads
   let client = t.rpcClient
   var hRes = client.blockNumber()
-  check hRes.isOk
+  testCond hRes.isOk:
-  if hRes.isErr:
     error "unable to get blockNumber", msg=hRes.error
-    return
 
   let lastBlock = int(hRes.get)
   info "Started re-executing payloads at block", number=lastBlock
@@ -619,33 +597,26 @@ proc reExecPayloads(t: TestEnv, testStatusIMPL: var TestStatus) =
     if clMock.executedPayloadHistory.hasKey(uint64 i):
       let payload = clMock.executedPayloadHistory[uint64 i]
       let res = client.newPayloadV1(payload)
-      check res.isOk
+      testCond res.isOk:
-      if res.isErr:
         error "FAIL (%s): Unable to re-execute valid payload", msg=res.error
-        return
 
       let s = res.get()
-      check s.status == PayloadExecutionStatus.valid
+      testCond s.status == PayloadExecutionStatus.valid:
-      if s.status != PayloadExecutionStatus.valid:
         error "Unexpected status after re-execute valid payload", status=s.status
-        return
     else:
-      check false
+      testCond true:
         error "(test issue) Payload does not exist", index=i
-      return
 
-proc multipleNewCanonicalPayloads(t: TestEnv, testStatusIMPL: var TestStatus) =
+proc multipleNewCanonicalPayloads(t: TestEnv): TestStatus =
+  result = TestStatus.OK
+
   # Wait until TTD is reached by this client
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
-  if not produce5BlockRes:
-    return
 
   let clMock = t.clMock
   let client = t.rpcClient
@@ -676,14 +647,14 @@ proc multipleNewCanonicalPayloads(t: TestEnv, testStatusIMPL: var TestStatus) =
       return true
   ))
   # At the end the CLMocker continues to try to execute fcU with the original payload, which should not fail
-  check produceSingleBlockRes
+  testCond produceSingleBlockRes
+
+proc outOfOrderPayloads(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc outOfOrderPayloads(t: TestEnv, testStatusIMPL: var TestStatus) =
   # Wait until TTD is reached by this client
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # First prepare payloads on a first client, which will also contain multiple transactions
 
@@ -710,38 +681,32 @@ proc outOfOrderPayloads(t: TestEnv, testStatusIMPL: var TestStatus) =
           return false
       return true
   ))
-  check produceBlockRes
+  testCond produceBlockRes
 
   let expectedBalance = amountPerTx * u256(payloadCount*txPerPayload)
 
-  # Check balance on this first client
+  # testCond balance on this first client
   let balRes = client.balanceAt(recipient)
-  check balRes.isOk
+  testCond balRes.isOk:
-  if balRes.isErr:
     error "Error while getting balance of funded account"
-    return
 
   let bal = balRes.get()
-  check expectedBalance == bal
+  testCond expectedBalance == bal
-  if expectedBalance != bal:
-    return
 
   # TODO: this section need multiple client
 
-proc transactionReorg(t: TestEnv, testStatusIMPL: var TestStatus) =
+proc transactionReorg(t: TestEnv): TestStatus =
+  result = TestStatus.OK
+
   # Wait until TTD is reached by this client
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
-  if not produce5BlockRes:
-    return
 
-  # Create transactions that modify the state in order to check after the reorg.
+  # Create transactions that modify the state in order to testCond after the reorg.
   const
     txCount      = 5
     contractAddr = hexToByteArray[20]("0000000000000000000000000000000000000317")
@@ -762,23 +727,17 @@ proc transactionReorg(t: TestEnv, testStatusIMPL: var TestStatus) =
 
     # Send the transaction
     let res = client.sendTransaction(tx)
-    check res.isOk
+    testCond res.isOk:
-    if res.isErr:
       error "Unable to send transaction", msg=res.error
-      return
 
     # Produce the block containing the transaction
     var blockRes = clMock.produceSingleBlock(BlockProcessCallbacks())
-    check blockRes
+    testCond blockRes
-    if not blockRes:
-      return
 
     # Get the receipt
     let rr = client.txReceipt(rlpHash(tx))
-    check rr.isOk
+    testCond rr.isOk:
-    if rr.isErr:
       error "Unable to obtain transaction receipt", msg=rr.error
-      return
 
     receipts[i] = rr.get()
 
@@ -787,13 +746,11 @@ proc transactionReorg(t: TestEnv, testStatusIMPL: var TestStatus) =
     let storageKey = i.u256
 
     var rr = client.storageAt(contractAddr, storageKey)
-    check rr.isOk
+    testCond rr.isOk:
-    if rr.isErr:
       error "Could not get storage", msg=rr.error
-      return
 
     let valueWithTxApplied = rr.get()
-    check valueWithTxApplied == 1.u256
+    testCond valueWithTxApplied == 1.u256
     if valueWithTxApplied != 1.u256:
       error "Expected storage not set after transaction", valueWithTxApplied
       return
@@ -803,16 +760,12 @@ proc transactionReorg(t: TestEnv, testStatusIMPL: var TestStatus) =
     var reorgBlock: EthBlockHeader
     let blockRes = client.headerByNumber(number - 1, reorgBlock)
     rr = client.storageAt(contractAddr, storageKey, reorgBlock.blockNumber)
-    check rr.isOk
+    testCond rr.isOk:
-    if rr.isErr:
       error "could not get storage", msg= rr.error
-      return
 
     let valueWithoutTxApplied = rr.get()
-    check valueWithoutTxApplied == 0.u256
+    testCond valueWithoutTxApplied == 0.u256:
-    if valueWithoutTxApplied != 0.u256:
       error "Storage not unset before transaction!", valueWithoutTxApplied
-      return
 
     # Re-org back to a previous block where the tx is not included using forkchoiceUpdated
     let rHash = Web3BlockHash reorgBlock.blockHash.data
@@ -823,44 +776,32 @@ proc transactionReorg(t: TestEnv, testStatusIMPL: var TestStatus) =
     )
 
     var res = client.forkchoiceUpdatedV1(reorgForkchoice)
-    check res.isOk
+    testCond res.isOk:
-    if res.isErr:
       error "Could not send forkchoiceUpdatedV1", msg=res.error
-      return
 
     var s = res.get()
-    check s.payloadStatus.status == PayloadExecutionStatus.valid
+    testCond s.payloadStatus.status == PayloadExecutionStatus.valid:
-    if s.payloadStatus.status != PayloadExecutionStatus.valid:
       error "Could not send forkchoiceUpdatedV1", status=s.payloadStatus.status
-      return
 
-    # Check storage again using `latest`, should be unset
+    # testCond storage again using `latest`, should be unset
     rr = client.storageAt( contractAddr, storageKey)
-    check rr.isOk
+    testCond rr.isOk:
-    if rr.isErr:
       error "could not get storage", msg= rr.error
-      return
 
     let valueAfterReOrgBeforeTxApplied = rr.get()
-    check valueAfterReOrgBeforeTxApplied == 0.u256
+    testCond valueAfterReOrgBeforeTxApplied == 0.u256:
-    if valueAfterReOrgBeforeTxApplied != 0.u256:
       error "Storage not unset after re-org", valueAfterReOrgBeforeTxApplied
-      return
 
     # Re-send latest forkchoice to test next transaction
     res = client.forkchoiceUpdatedV1(clMock.latestForkchoice)
-    check res.isOk
+    testCond res.isOk:
-    if res.isErr:
       error "Could not send forkchoiceUpdatedV1", msg=res.error
-      return
 
     s = res.get()
-    check s.payloadStatus.status == PayloadExecutionStatus.valid
+    testCond s.payloadStatus.status == PayloadExecutionStatus.valid:
-    if s.payloadStatus.status != PayloadExecutionStatus.valid:
       error "Could not send forkchoiceUpdatedV1", status=s.payloadStatus.status
-      return
 
-proc checkPrevRandaoValue(t: TestEnv, expectedPrevRandao: Hash256, blockNumber: uint64): bool =
+proc testCondPrevRandaoValue(t: TestEnv, expectedPrevRandao: Hash256, blockNumber: uint64): bool =
   let storageKey = blockNumber.u256
   let client = t.rpcClient
 
@@ -877,32 +818,28 @@ proc checkPrevRandaoValue(t: TestEnv, expectedPrevRandao: Hash256, blockNumber:
     return false
   true
 
-proc sidechainReorg(t: TestEnv, testStatusIMPL: var TestStatus) =
+proc sidechainReorg(t: TestEnv): TestStatus =
+  result = TestStatus.OK
+
   # Wait until TTD is reached by this client
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Produce blocks before starting the test
   let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
-  check produce5BlockRes
+  testCond produce5BlockRes
-  if not produce5BlockRes:
-    return
 
   let
     client = t.rpcClient
     clMock = t.clMock
 
-  # Produce two payloads, send fcU with first payload, check transaction outcome, then reorg, check transaction outcome again
+  # Produce two payloads, send fcU with first payload, testCond transaction outcome, then reorg, testCond transaction outcome again
 
   # This single transaction will change its outcome based on the payload
   let tx = t.makeNextTransaction(prevRandaoContractAddr, 0.u256)
   let rr = client.sendTransaction(tx)
-  check rr.isOk
+  testCond rr.isOk:
-  if rr.isErr:
     error "Unable to send transaction", msg=rr.error
-    return
 
   let singleBlockRes = clMock.produceSingleBlock(BlockProcessCallbacks(
     onNewPayloadBroadcast: proc(): bool =
@@ -966,21 +903,21 @@ proc sidechainReorg(t: TestEnv, testStatusIMPL: var TestStatus) =
         return false
 
       # PrevRandao should be the alternative prevRandao we sent
-      return checkPrevRandaoValue(t, alternativePrevRandao, uint64 alternativePayload.blockNumber)
+      return testCondPrevRandaoValue(t, alternativePrevRandao, uint64 alternativePayload.blockNumber)
   ))
 
-  check singleBlockRes
+  testCond singleBlockRes
   # The reorg actually happens after the CLMocker continues,
   # verify here that the reorg was successful
   let latestBlockNum = cLMock.latestFinalizedNumber.uint64
-  check checkPrevRandaoValue(t, clMock.prevRandaoHistory[latestBlockNum], latestBlockNum)
+  testCond testCondPrevRandaoValue(t, clMock.prevRandaoHistory[latestBlockNum], latestBlockNum)
+
+proc suggestedFeeRecipient(t: TestEnv): TestStatus =
+  result = TestStatus.OK
 
-proc suggestedFeeRecipient(t: TestEnv, testStatusIMPL: var TestStatus) =
   # Wait until TTD is reached by this client
   let ok = waitFor t.clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Amount of transactions to send
   const
@@ -988,7 +925,7 @@ proc suggestedFeeRecipient(t: TestEnv, testStatusIMPL: var TestStatus) =
 
   # Verify that, in a block with transactions, fees are accrued by the suggestedFeeRecipient
   var feeRecipient: EthAddress
-  check nimcrypto.randomBytes(feeRecipient) == 20
+  testCond nimcrypto.randomBytes(feeRecipient) == 20
 
   let
     client = t.rpcClient
@@ -999,130 +936,107 @@ proc suggestedFeeRecipient(t: TestEnv, testStatusIMPL: var TestStatus) =
     # Empty self tx
     let tx = t.makeNextTransaction(vaultAccountAddr, 0.u256)
     let res = client.sendTransaction(tx)
-    check res.isOk
+    testCond res.isOk:
-    if res.isErr:
       error "unable to send transaction", msg=res.error
-      return
 
   # Produce the next block with the fee recipient set
   clMock.nextFeeRecipient = feeRecipient
-  check clMock.produceSingleBlock(BlockProcessCallbacks())
+  testCond clMock.produceSingleBlock(BlockProcessCallbacks())
 
-  # Calculate the fees and check that they match the balance of the fee recipient
+  # Calculate the fees and testCond that they match the balance of the fee recipient
   var blockIncluded: EthBlock
   var rr = client.latestBlock(blockIncluded)
-  check rr.isOk
+  testCond rr.isOk:
-  if rr.isErr:
     error "unable to get latest block", msg=rr.error
-    return
 
-  check blockIncluded.txs.len == txCount
+  testCond blockIncluded.txs.len == txCount:
-  if blockIncluded.txs.len != txCount:
     error "not all transactions were included in block",
       expected=txCount,
       get=blockIncluded.txs.len
-    return
 
-  check blockIncluded.header.coinbase == feeRecipient
+  testCond blockIncluded.header.coinbase == feeRecipient:
-  if blockIncluded.header.coinbase != feeRecipient:
     error "feeRecipient was not set as coinbase",
       expected=feeRecipient.toHex,
       get=blockIncluded.header.coinbase.toHex
-    return
 
   var feeRecipientFees = 0.u256
   for tx in blockIncluded.txs:
     let effGasTip = tx.effectiveGasTip(blockIncluded.header.fee)
     let tr = client.txReceipt(rlpHash(tx))
-    check tr.isOk
+    testCond tr.isOk:
-    if tr.isErr:
       error "unable to obtain receipt", msg=tr.error
-      return
 
     let rec = tr.get()
     let gasUsed = UInt256.fromHex(rec.gasUsed.string)
     feeRecipientFees = feeRecipientFees  + effGasTip.u256 * gasUsed
 
   var br = client.balanceAt(feeRecipient)
-  check br.isOk
+  testCond br.isOk
 
   var feeRecipientBalance = br.get()
-  check feeRecipientBalance == feeRecipientFees
+  testCond feeRecipientBalance == feeRecipientFees:
-  if feeRecipientBalance != feeRecipientFees:
     error "balance does not match fees",
       feeRecipientBalance, feeRecipientFees
 
   # Produce another block without txns and get the balance again
   clMock.nextFeeRecipient = feeRecipient
-  check clMock.produceSingleBlock(BlockProcessCallbacks())
+  testCond clMock.produceSingleBlock(BlockProcessCallbacks())
 
   br = client.balanceAt(feeRecipient)
-  check br.isOk
+  testCond br.isOk
   feeRecipientBalance = br.get()
-  check feeRecipientBalance == feeRecipientFees
+  testCond feeRecipientBalance == feeRecipientFees:
-  if feeRecipientBalance != feeRecipientFees:
     error "balance does not match fees",
       feeRecipientBalance, feeRecipientFees
 
-proc prevRandaoOpcodeTx(t: TestEnv, testStatusIMPL: var TestStatus) =
+proc prevRandaoOpcodeTx(t: TestEnv): TestStatus =
+  result = TestStatus.OK
+
   let
     client = t.rpcClient
     clMock = t.clMock
     tx = t.makeNextTransaction(prevRandaoContractAddr, 0.u256)
     rr = client.sendTransaction(tx)
 
-  check rr.isOk
+  testCond rr.isOk:
-  if rr.isErr:
     error "Unable to send transaction", msg=rr.error
-    return
 
   # Wait until TTD is reached by this client
   let ok = waitFor clMock.waitForTTD()
-  check ok
+  testCond ok
-  if not ok:
-    return
 
   # Ideally all blocks up until TTD must have a DIFFICULTY opcode tx in it
   let nr = client.blockNumber()
-  check nr.isOk
+  testCond nr.isOk:
-  if nr.isErr:
     error "Unable to get latest block number", msg=nr.error
-    return
 
   let ttdBlockNumber = nr.get()
 
   # Start
   for i in ttdBlockNumber..ttdBlockNumber:
-    # First check that the block actually contained the transaction
+    # First testCond that the block actually contained the transaction
     var blk: EthBlock
     let res = client.blockByNumber(i, blk)
-    check res.isOk
+    testCond res.isOk:
-    if res.isErr:
       error "Unable to get block", msg=res.error
-      return
 
-    check blk.txs.len > 0
+    testCond blk.txs.len > 0:
-    if blk.txs.len == 0:
       error "(Test issue) no transactions went in block"
-      return
 
     let storageKey = i.u256
     let rr = client.storageAt(prevRandaoContractAddr, storageKey)
-    check rr.isOk
+    testCond rr.isOk:
-    if rr.isErr:
       error "Unable to get storage", msg=rr.error
-      return
 
     let opcodeValueAtBlock = rr.get()
-    if opcodeValueAtBlock != 2.u256:
+    testCond opcodeValueAtBlock == 2.u256:
       error "Incorrect difficulty value in block",
         expect=2,
         get=opcodeValueAtBlock
-      return
 
-proc postMergeSync(t: TestEnv, testStatusIMPL: var TestStatus) =
+proc postMergeSync(t: TestEnv): TestStatus =
+  result = TestStatus.SKIPPED
   # TODO: need multiple client
-  discard
 
 const engineTestList* = [
   TestSpec(
@@ -1165,7 +1079,7 @@ const engineTestList* = [
     name: "ParentHash==BlockHash on NewPayload",
     run:  parentHashOnExecPayload,
   ),
-  #[TestSpec(
+  TestSpec(
     name: "Invalid ParentHash NewPayload",
     run:  invalidPayloadTestCaseGen("ParentHash"),
   ),
@@ -1220,7 +1134,7 @@ const engineTestList* = [
   TestSpec(
     name: "Invalid Transaction Value NewPayload",
     run:  invalidPayloadTestCaseGen("Transaction/Value"),
-  ),]#
+  ),
 
   # Eth RPC Status on ForkchoiceUpdated Events
   TestSpec(
@@ -1276,11 +1190,11 @@ const engineTestList* = [
 
   # TODO: debug and fix
   # PrevRandao opcode tests
- #TestSpec(
+  TestSpec(
- #  name: "PrevRandao Opcode Transactions",
+    name: "PrevRandao Opcode Transactions",
- #  run:  prevRandaoOpcodeTx,
+    run:  prevRandaoOpcodeTx,
- #  ttd:  10,
+    ttd:  10,
- #),
+  ),
 
   # Multi-Client Sync tests
   TestSpec(

hive_integration/nodocker/graphql/graphql_sim.nim

@@ -8,7 +8,7 @@
 # those terms.
 
 import
-  std/[os, json],
+  std/[os, json, times],
   eth/[p2p, trie/db], ../../../nimbus/db/db_chain,
   ../../../nimbus/sync/protocol_ethxx,
   ../../../nimbus/[genesis, config, conf_utils, context],
@@ -22,7 +22,11 @@ const
   genesisFile = baseFolder / "init" / "genesis.json"
   caseFolder  = baseFolder / "testcases"
 
-proc processNode(ctx: GraphqlRef, node: JsonNode, fileName: string, testStatusIMPL: var TestStatus) =
+template testCond(expr: untyped) =
+  if not (expr):
+    result = TestStatus.Failed
+    
+proc processNode(ctx: GraphqlRef, node: JsonNode, fileName: string): TestStatus =
   let request = node["request"]
   let responses = node["responses"]
   let statusCode = node["statusCode"].getInt()
@@ -30,11 +34,12 @@ proc processNode(ctx: GraphqlRef, node: JsonNode, fileName: string, testStatusIM
   let savePoint = ctx.getNameCounter()
   let res = ctx.parseQuery(request.getStr())
 
+  result = TestStatus.OK
   block:
     if res.isErr:
       if statusCode == 200:
         debugEcho res.error
-      check statusCode != 200
+      testCond statusCode != 200
       break
 
     let resp = JsonRespStream.new()
@@ -42,11 +47,11 @@ proc processNode(ctx: GraphqlRef, node: JsonNode, fileName: string, testStatusIM
     if r.isErr:
       if statusCode == 200:
         debugEcho r.error
-      check statusCode != 200
+      testCond statusCode != 200
       break
 
-    check statusCode == 200
+    testCond statusCode == 200
-    check r.isOk
+    testCond r.isOk
 
     let nimbus = resp.getString()
     var resultOK = false
@@ -56,7 +61,7 @@ proc processNode(ctx: GraphqlRef, node: JsonNode, fileName: string, testStatusIM
         resultOK = true
         break
 
-    check resultOK
+    testCond resultOK
     if not resultOK:
       debugEcho "NIMBUS RESULT: ", nimbus
       for x in responses:
@@ -83,8 +88,19 @@ proc main() =
   discard importRlpBlock(blocksFile, chainDB)
   let ctx = setupGraphqlContext(chainDB, ethNode, txPool)
 
-  runTest("GraphQL", caseFolder):
+  var stat: SimStat
+  let start = getTime()
+  for fileName {.inject.} in walkDirRec(
+                 caseFolder, yieldFilter = {pcFile,pcLinkToFile}):
+    if not fileName.endsWith(".json"):
+      continue
+
+    let (folder, name) = fileName.splitPath()
     let node = parseFile(fileName)
-    ctx.processNode(node, fileName, testStatusIMPL)
+    let status = ctx.processNode(node, fileName)
+    stat.inc(name, status)
+
+  let elpd = getTime() - start
+  print(stat, elpd, "graphql")
 
 main()

hive_integration/nodocker/rpc/callsigs.nim

@@ -1,18 +0,0 @@
-# Nimbus
-# Copyright (c) 2021 Status Research & Development GmbH
-# Licensed under either of
-#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-# at your option.
-# This file may not be copied, modified, or distributed except according to
-# those terms.
-
-import
-  std/[os, strutils],
-  eth/[common],
-  json_rpc/[rpcclient],
-  ../../../nimbus/rpc/[hexstrings, rpc_types]
-
-template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
-const sigPath = sourceDir / ".." / ".." / ".." / "tests" / "rpcclient" / "ethcallsigs.nim"
-createRpcSigs(RpcClient, sigPath)

hive_integration/nodocker/rpc/client.nim

@@ -15,7 +15,9 @@ import
   json_rpc/[rpcclient],
   ../../../nimbus/[utils, transaction],
   ../../../nimbus/rpc/hexstrings,
-  "."/[callsigs]
+  ../../../tests/rpcclient/eth_api
+
+export eth_api
 
 proc fromHex(x: type Hash256, hex: EthHashStr): Hash256 =
   hexToByteArray(hex.string, result.data)

hive_integration/nodocker/rpc/ethclient.nim

@@ -13,8 +13,11 @@ import
   stew/byteutils,
   stint,
   chronos,
+  unittest2,
   json_rpc/[rpcclient],
-  "."/[vault, client, callsigs]
+  "."/[vault, client]
+
+export client
 
 const
   gasPrice* = 30000000000 # 30 Gwei or 30 * pow(10, 9)
@@ -25,6 +28,10 @@ type
     vault*: Vault
     client*: RpcClient
 
+  TestSpec* = object
+    name*: string
+    run*: proc(t: TestEnv): Future[TestStatus]
+
 func eth(n: int): UInt256 {.compileTime.} =
   n.u256 * pow(10.u256, 18)
 
@@ -35,7 +42,7 @@ func ethAddr(x: string): EthAddress =
   hexToByteArray[20](x)
 
 # envTest make sure the env is set up properly for subsequent tests
-proc envTest*(t: TestEnv): Future[bool] {.async.} =
+proc envTest(t: TestEnv): Future[TestStatus] {.async.} =
   let client = t.client
   let res = await client.web3_clientVersion()
 
@@ -52,14 +59,14 @@ proc envTest*(t: TestEnv): Future[bool] {.async.} =
     let expected = u256(x[1])
     if res != expected:
       debugEcho "expected: $1, got $2" % [x[0], $res]
-      return false
+      return TestStatus.Failed
 
-  result = true
+  result = TestStatus.OK
 
 # balanceAndNonceAtTest creates a new account and transfers funds to it.
 # It then tests if the balance and nonce of the sender and receiver
 # address are updated correct.
-proc balanceAndNonceAtTest*(t: TestEnv) {.async.} =
+proc balanceAndNonceAtTest(t: TestEnv): Future[TestStatus] {.async.} =
   let
     sourceAddr  = await t.vault.createAccount(1.eth)
     sourceNonce = 0.AccountNonce
@@ -69,3 +76,15 @@ proc balanceAndNonceAtTest*(t: TestEnv) {.async.} =
   let sourceAddressBalanceBefore = t.client.balanceAt(sourceAddr)
 
   # TODO: complete this test
+  result = TestStatus.OK
+
+const testList* = [
+  TestSpec(
+    name: "env is set up properly for subsequent tests",
+    run: envTest
+  ),
+  TestSpec(
+    name: "balance and nonce update correctly",
+    run: balanceAndNonceAtTest
+  )
+]

hive_integration/nodocker/rpc/rpc_sim.nim

@@ -8,18 +8,19 @@
 # those terms.
 
 import
-  std/os, asynctest,
+  std/[os, times],
   eth/[trie/db],
-  eth/p2p as ethP2p,
+  eth/p2p as ethp2p,
   stew/shims/net as stewNet,
   stew/results,
   chronos, json_rpc/[rpcserver, rpcclient],
   ../../../nimbus/db/db_chain,
-  ../../../nimbus/p2p/protocol_ethxx,
+  ../../../nimbus/sync/protocol_ethxx,
-  ../../../nimbus/[config, context, genesis],
+  ../../../nimbus/[config, context, genesis, utils/tx_pool],
   ../../../nimbus/rpc/[common, p2p, debug],
   ../../../tests/test_helpers,
-  "."/[callsigs, ethclient, vault, client]
+  "."/[ethclient, vault, client],
+  ../sim_utils
 
 const
   initPath = "hive_integration" / "nodocker" / "rpc" / "init"
@@ -31,18 +32,22 @@ proc manageAccounts(ctx: EthContext, conf: NimbusConf) =
       echo res.error()
       quit(QuitFailure)
 
-proc setupRpcServer(ctx: EthContext, chainDB: BaseChainDB, ethNode: EthereumNode, conf: NimbusConf): RpcServer  =
+proc setupRpcServer(ctx: EthContext, chainDB: BaseChainDB,
+                    ethNode: EthereumNode, txPool: TxPoolRef,
+                    conf: NimbusConf): RpcServer  =
   let rpcServer = newRpcHttpServer([initTAddress(conf.rpcAddress, conf.rpcPort)])
   setupCommonRpc(ethNode, conf, rpcServer)
-  setupEthRpc(ethNode, ctx, chainDB, rpcServer)
+  setupEthRpc(ethNode, ctx, chainDB, txPool, rpcServer)
 
   rpcServer.start()
   rpcServer
 
-proc setupWsRpcServer(ctx: EthContext, chainDB: BaseChainDB, ethNode: EthereumNode, conf: NimbusConf): RpcServer  =
+proc setupWsRpcServer(ctx: EthContext, chainDB: BaseChainDB,
+                      ethNode: EthereumNode, txPool: TxPoolRef,
+                      conf: NimbusConf): RpcServer  =
   let rpcServer = newRpcWebSocketServer(initTAddress(conf.wsAddress, conf.wsPort))
   setupCommonRpc(ethNode, conf, rpcServer)
-  setupEthRpc(ethNode, ctx, chainDB, rpcServer)
+  setupEthRpc(ethNode, ctx, chainDB, txPool, rpcServer)
 
   rpcServer.start()
   rpcServer
@@ -56,9 +61,18 @@ proc runRpcTest() =
     vault : newVault(chainID, gasPrice, client)
   )
 
-  suite "JSON RPC Test Over HTTP":
+  var stat: SimStat
-    test "env test":
+  let start = getTime()
-      check await envTest(testEnv)
+  for x in testList:
+    try:
+      let status = waitFor x.run(testEnv)
+      stat.inc(x.name, status)
+    except ValueError as ex:
+      stat.inc(x.name, TestStatus.Failed)
+      echo ex.msg
+
+  let elpd = getTime() - start
+  print(stat, elpd, "rpc")
 
 proc main() =
   let conf = makeConfig(@[
@@ -89,7 +103,9 @@ proc main() =
 
   chainDB.populateProgress()
   chainDB.initializeEmptyDb()
-  let rpcServer = setupRpcServer(ethCtx, chainDB, ethNode, conf)
+
+  let txPool = TxPoolRef.new(chainDB, conf.engineSigner)
+  let rpcServer = setupRpcServer(ethCtx, chainDB, ethNode, txPool, conf)
   runRpcTest()
 
 main()

hive_integration/nodocker/sim_utils.nim

@@ -8,35 +8,36 @@
 # those terms.
 
 import
-  std/[tables, strutils, unittest],
+  std/[tables, strutils, times],
-  testutils/markdown_reports
+  unittest2
 
 export
-  tables, strutils, unittest,
+  tables, strutils, unittest2
-  markdown_reports
 
-template runTest*(suiteName: string, caseFolder: string, body: untyped) =
+type
-  disableParamFiltering()
+  SimStat* = object
-  suite suiteName:
+    ok*: int
-    var status = initOrderedTable[string, OrderedTable[string, Status]]()
+    skipped*: int
-    for fileName {.inject.} in walkDirRec(
+    failed*: int
-                    caseFolder, yieldFilter = {pcFile,pcLinkToFile}):
 
-      if not fileName.endsWith(".json"):
+proc inc*(stat: var SimStat, name: string, status: TestStatus) =
-        continue
+  echo name, ", ", status
+  if status == OK:
+    inc stat.ok
+  elif status == SKIPPED:
+    inc stat.skipped
+  else:
+    inc stat.failed
 
-      let (folder, name) = fileName.splitPath()
+proc `$`*(stat: SimStat): string =
-      let last = folder.splitPath().tail
+  "ok: $1, skipped: $2, failed: $3" % [$stat.ok, $stat.skipped, $stat.failed]
-      if last notin status:
-        status[last] = initOrderedTable[string, Status]()
 
-      test fileName:
+proc print*(stat: SimStat, dur: Duration, name: string) =
-        # we set this here because exceptions might be raised in the handler
+  var f = open(name & ".md", fmWrite)
-        status[last][name] = Status.Fail
+  f.write("* " & name)
-        body
+  f.write("\n")
-        if testStatusIMPL == OK:
+  f.write("  - " & $stat)
-          status[last][name] = Status.OK
+  f.write("\n")
-        elif testStatusIMPL == SKIPPED:
+  f.write("  - " & $dur)
-          status[last][name] = Status.Skip
+  f.write("\n")
-
+  f.close()
-    generateReport(suiteName, status)

nimbus/conf_utils.nim

@@ -17,16 +17,10 @@ type
   EthHeader = object
     header: BlockHeader
 
-proc importRlpBlock*(importFile: string; chainDB: BaseChainDB): bool =
+proc importRlpBlock*(blocksRlp: openArray[byte]; chainDB: BaseChainDB; importFile: string = ""): bool =
-  let res = io2.readAllBytes(importFile)
-  if res.isErr:
-    error "failed to import",
-      fileName = importFile
-    return false
-
   var
     # the encoded rlp can contains one or more blocks
-    rlp = rlpFromBytes(res.get)
+    rlp = rlpFromBytes(blocksRlp)
     chain = newChain(chainDB, extraValidation = true)
     errorCount = 0
   let
@@ -57,3 +51,13 @@ proc importRlpBlock*(importFile: string; chainDB: BaseChainDB): bool =
       errorCount.inc
 
   return errorCount == 0
+
+proc importRlpBlock*(importFile: string; chainDB: BaseChainDB): bool =
+  let res = io2.readAllBytes(importFile)
+  if res.isErr:
+    error "failed to import",
+      fileName = importFile
+    return false
+    
+  importRlpBlock(res.get, chainDB, importFile)
+  

nimbus/db/select_backend.nim

@@ -7,6 +7,7 @@ export kvstore
 # be reconsidered when making more changes here.
 
 type DbBackend = enum
+  none,
   sqlite,
   rocksdb,
   lmdb
@@ -55,5 +56,8 @@ elif dbBackend == lmdb:
   #      https://github.com/status-im/nim-beacon-chain/pull/809
 
   {.error: "lmdb deprecated, needs reimplementing".}
+elif dbBackend == none:
+  discard
 
-export database_backend
+when dbBackend != none:
+  export database_backend