Generate markdown test reports

tests/all_tests.nim

@@ -50,4 +50,4 @@ import # Refactor state transition unit tests
 #   ./official/test_fixture_shuffling,
 #   ./official/test_fixture_bls
 
-summarizeLongTests()
+summarizeLongTests("AllTests")

tests/official/all_fixtures_require_ssz.nim

@@ -22,4 +22,4 @@ import
   ./test_fixture_operations_proposer_slashings,
   ./test_fixture_operations_voluntary_exit
 
-summarizeLongTests()
+summarizeLongTests("FixtureAll")

tests/official/test_fixture_bls.nim

@@ -49,7 +49,7 @@ proc readValue*(r: var JsonReader, a: var Domain) {.inline.} =
 # TODO: json tests were removed
 const BLSDir = JsonTestsDir/"general"/"phase0"/"bls"
 
-suite "Official - BLS tests":
+suiteReport "Official - BLS tests":
   timedTest "Private to public key conversion":
     for file in walkDirRec(BLSDir/"priv_to_pub"):
       let t = parseTest(file, Json, BLSPrivToPub)

tests/official/test_fixture_const_sanity_check.nim

@@ -121,5 +121,5 @@ proc checkConfig() =
       else:
         check: ConstsToCheck[constant] == value.getBiggestInt().uint64()
 
-suite "Official - 0.10.1 - constants & config " & preset():
+suiteReport "Official - 0.10.1 - constants & config " & preset():
   checkConfig()

tests/official/test_fixture_operations_attestations.nim

@@ -65,6 +65,6 @@ proc runTest(identifier: string) =
 
   `testImpl _ operations_attestations _ identifier`()
 
-suite "Official - Operations - Attestations " & preset():
+suiteReport "Official - Operations - Attestations " & preset():
   for kind, path in walkDir(OperationsAttestationsDir, true):
     runTest(path)

tests/official/test_fixture_operations_attester_slashings.nim

@@ -67,7 +67,7 @@ proc runTest(identifier: string) =
 
   `testImpl _ operations_attester_slashing _ identifier`()
 
-suite "Official - Operations - Attester slashing " & preset():
+suiteReport "Official - Operations - Attester slashing " & preset():
   # TODO these are both valid and check BLS signatures, which isn't working
   # since 0.10.x introduces new BLS signing/verifying interface with domain
   # in particular handled differently through compute_signing_root() rather

tests/official/test_fixture_operations_block_header.nim

@@ -62,6 +62,6 @@ proc runTest(identifier: string) =
 
   `testImpl _ blockheader _ identifier`()
 
-suite "Official - Operations - Block header " & preset():
+suiteReport "Official - Operations - Block header " & preset():
   for kind, path in walkDir(OpBlockHeaderDir, true):
     runTest(path)

tests/official/test_fixture_operations_deposits.nim

@@ -60,7 +60,7 @@ proc runTest(identifier: string) =
 
   `testImpl _ operations_deposits _ identifier`()
 
-suite "Official - Operations - Deposits " & preset():
+suiteReport "Official - Operations - Deposits " & preset():
   # TODO
   const expected_failures = ["valid_sig_but_forked_state"]
 

tests/official/test_fixture_operations_proposer_slashings.nim

@@ -65,6 +65,6 @@ proc runTest(identifier: string) =
 
   `testImpl_proposer_slashing _ identifier`()
 
-suite "Official - Operations - Proposer slashing " & preset():
+suiteReport "Official - Operations - Proposer slashing " & preset():
   for kind, path in walkDir(OpProposerSlashingDir, true):
     runTest(path)

tests/official/test_fixture_operations_voluntary_exit.nim

@@ -63,6 +63,6 @@ proc runTest(identifier: string) =
 
   `testImpl _ voluntary_exit _ identifier`()
 
-suite "Official - Operations - Voluntary exit " & preset():
+suiteReport "Official - Operations - Voluntary exit " & preset():
   for kind, path in walkDir(OpVoluntaryExitDir, true):
     runTest(path)

tests/official/test_fixture_sanity_blocks.nim

@@ -62,7 +62,7 @@ proc runTest(identifier: string) =
 
   `testImpl _ blck _ identifier`()
 
-suite "Official - Sanity - Blocks " & preset():
+suiteReport "Official - Sanity - Blocks " & preset():
   # Failing due to signature checking in indexed validation checking pending
   # 0.10 BLS verification API with new domain handling.
   const expected_failures =

tests/official/test_fixture_sanity_slots.nim

@@ -44,6 +44,6 @@ proc runTest(identifier: string) =
 
   `testImpl _ slots _ identifier`()
 
-suite "Official - Sanity - Slots " & preset():
+suiteReport "Official - Sanity - Slots " & preset():
   for kind, path in walkDir(SanitySlotsDir, true):
     runTest(path)

tests/official/test_fixture_ssz_consensus_objects.nim

@@ -103,5 +103,7 @@ proc runSSZtests() =
           else:
             raise newException(ValueError, "Unsupported test: " & sszType)
 
-suite "Official - 0.10.1 - SSZ consensus objects " & preset():
+suiteReport "Official - 0.10.1 - SSZ consensus objects " & preset():
   runSSZtests()
+
+summarizeLongTests("FixtureSSZConsensus")

tests/official/test_fixture_ssz_generic_types.nim

@@ -302,5 +302,7 @@ proc runSSZtests() =
   # test "Testing " & name & " inputs (" & $T & ") - invalid":
   #   const path = SSZDir/name/"invalid"
 
-suite "Official - SSZ generic types":
+suiteReport "Official - SSZ generic types":
   runSSZtests()
+
+summarizeLongTests("FixtureSSZGeneric")

tests/official/test_fixture_state_transition_epoch.nim

@@ -33,7 +33,7 @@ template runSuite(suiteDir, testName: string, transitionProc: untyped{ident}, us
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
   proc `suiteImpl _ transitionProc`() =
-    suite "Official - Epoch Processing - " & testName & preset():
+    suiteReport "Official - Epoch Processing - " & testName & preset():
       for testDir in walkDirRec(suiteDir, yieldFilter = {pcDir}):
 
         let unitTestName = testDir.rsplit(DirSep, 1)[1]

tests/spec_block_processing/test_genesis.nim

@@ -27,7 +27,7 @@ import
 #   - MIN_GENESIS_TIME is not implemented
 #   - is_valid_genesis_state is not implemented
 
-suite "[Unit - Spec - Genesis] Genesis block checks " & preset():
+suiteReport "[Unit - Spec - Genesis] Genesis block checks " & preset():
   timedTest "is_valid_genesis_state for a valid state":
     discard initGenesisState(
       num_validators = MIN_GENESIS_ACTIVE_VALIDATOR_COUNT,

tests/spec_block_processing/test_process_attestation.nim

@@ -20,7 +20,7 @@ import
   ../mocking/[mock_genesis, mock_attestations, mock_state, mock_blocks],
   ../testutil
 
-suite "[Unit - Spec - Block processing] Attestations " & preset():
+suiteReport "[Unit - Spec - Block processing] Attestations " & preset():
 
   const NumValidators = uint64(8) * SLOTS_PER_EPOCH
   let genesisState = initGenesisState(NumValidators)

tests/spec_block_processing/test_process_deposits.nim

@@ -23,7 +23,7 @@ import
   ../mocking/[mock_deposits, mock_genesis],
   ../testutil, ../helpers/math_helpers
 
-suite "[Unit - Spec - Block processing] Deposits " & preset():
+suiteReport "[Unit - Spec - Block processing] Deposits " & preset():
 
   const NumValidators = uint64 5 * SLOTS_PER_EPOCH
   let genesisState = initGenesisState(NumValidators)

tests/spec_epoch_processing/test_process_justification_and_finalization.nim

@@ -212,7 +212,7 @@ proc finalizeOn12(state: var BeaconState, epoch: Epoch, sufficient_support: bool
     doAssert state.current_justified_checkpoint == c2    # still old current
     doAssert state.finalized_checkpoint == old_finalized # no new finalized checkpoint
 
-suite "[Unit - Spec - Epoch processing] Justification and Finalization " & preset():
+suiteReport "[Unit - Spec - Epoch processing] Justification and Finalization " & preset():
   echo "   Finalization rules are detailed at https://github.com/protolambda/eth2-docs#justification-and-finalization"
 
   const NumValidators = uint64(8) * SLOTS_PER_EPOCH

tests/test_attestation_pool.nim

@@ -18,7 +18,7 @@ import
   ../beacon_chain/ssz
 
 when const_preset == "minimal": # Too much stack space used on mainnet
-  suite "Attestation pool processing" & preset():
+  suiteReport "Attestation pool processing" & preset():
     ## For now just test that we can compile and execute block processing with
     ## mock data.
 

tests/test_beacon_chain_db.nim

@@ -13,7 +13,7 @@ import  options, unittest, sequtils,
   # test utilies
   ./testutil, ./testblockutil
 
-suite "Beacon chain DB" & preset():
+suiteReport "Beacon chain DB" & preset():
   timedTest "empty database" & preset():
     var
       db = init(BeaconChainDB, kvStore MemoryStoreRef.init())

tests/test_beacon_node.nim

@@ -12,7 +12,7 @@ import unittest, ./testutil
 when false:
   import ../beacon_chain/beacon_node
 
-suite "Beacon node":
+suiteReport "Beacon node":
   # Compile test
 
   timedTest "Compile":

tests/test_beaconstate.nim

@@ -13,7 +13,7 @@ import
   ../beacon_chain/spec/[beaconstate, datatypes, digest],
   ../beacon_chain/extras
 
-suite "Beacon state" & preset():
+suiteReport "Beacon state" & preset():
   timedTest "Smoke test initialize_beacon_state_from_eth1" & preset():
     let state = initialize_beacon_state_from_eth1(
       Eth2Digest(), 0,

tests/test_block_pool.nim

@@ -13,7 +13,7 @@ import
   ../beacon_chain/spec/[datatypes, digest, helpers, validator],
   ../beacon_chain/[beacon_node_types, block_pool, ssz]
 
-suite "BlockRef and helpers" & preset():
+suiteReport "BlockRef and helpers" & preset():
   timedTest "isAncestorOf sanity" & preset():
     let
       s0 = BlockRef(slot: Slot(0))
@@ -51,7 +51,7 @@ suite "BlockRef and helpers" & preset():
       s4.getAncestorAt(Slot(3)) == s2
       s4.getAncestorAt(Slot(4)) == s4
 
-suite "BlockSlot and helpers" & preset():
+suiteReport "BlockSlot and helpers" & preset():
   timedTest "atSlot sanity" & preset():
     let
       s0 = BlockRef(slot: Slot(0))
@@ -83,7 +83,7 @@ suite "BlockSlot and helpers" & preset():
       s24.parent.parent == s22
 
 when const_preset == "minimal": # Too much stack space used on mainnet
-  suite "Block pool processing" & preset():
+  suiteReport "Block pool processing" & preset():
     setup:
       var
         db = makeTestDB(SLOTS_PER_EPOCH)
@@ -236,7 +236,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet
         tmpState.blck == b1Add.parent
         tmpState.data.data.slot == bs1.parent.slot
 
-  suite "BlockPool finalization tests" & preset():
+  suiteReport "BlockPool finalization tests" & preset():
     setup:
       var
         db = makeTestDB(SLOTS_PER_EPOCH)

tests/test_discovery_helpers.nim

@@ -5,7 +5,7 @@ import
   eth/keys, eth/p2p/enode, libp2p/multiaddress,
   ../beacon_chain/eth2_discovery
 
-suite "Discovery v5 utilities":
+suiteReport "Discovery v5 utilities":
   timedTest "Multiaddress to ENode":
     let addrStr = "/ip4/178.128.140.61/tcp/9000/p2p/16Uiu2HAmL5A5DAiiupFi6sUTF6Zq1TCKf6Pd5T8oFt9opQJqLqTQ"
     let ma = MultiAddress.init addrStr

tests/test_helpers.nim

@@ -11,7 +11,7 @@ import
   unittest, ./testutil,
   ../beacon_chain/spec/[helpers]
 
-suite "Spec helpers":
+suiteReport "Spec helpers":
   timedTest "integer_squareroot":
     check:
       integer_squareroot(0'u64) == 0'u64

tests/test_honest_validator.nim

@@ -4,7 +4,7 @@ import
   unittest, stint, ./testutil,
   ../beacon_chain/spec/network
 
-suite "Honest validator":
+suiteReport "Honest validator":
   timedTest "Attestation topics":
     check:
       getAttestationTopic(0) == "/eth2/index0_beacon_attestation/ssz"

tests/test_interop.nim

@@ -119,7 +119,7 @@ let depositsConfig = [
   )
 ]
 
-suite "Interop":
+suiteReport "Interop":
   timedTest "Mocked start private key":
     for i, k in privateKeys:
       let

tests/test_peer_pool.nim

@@ -36,7 +36,7 @@ proc init*(t: typedesc[PeerTest], id: string = "",
 proc close*(peer: PeerTest) =
   peer.future.complete()
 
-suite "PeerPool testing suite":
+suiteReport "PeerPool testing suite":
   timedTest "addPeerNoWait() test":
     const peersCount = [
       [10, 5, 5, 10, 5, 5],

tests/test_ssz.nim

@@ -75,7 +75,7 @@ type
 proc toDigest[N: static int](x: array[N, byte]): Eth2Digest =
   result.data[0 .. N-1] = x
 
-suite "SSZ navigator":
+suiteReport "SSZ navigator":
   timedTest "simple object fields":
     var foo = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
     let encoded = SSZ.encode(foo)
@@ -101,7 +101,7 @@ suite "SSZ navigator":
     let root2 = hash_tree_root(leaves2)
     check $root2 == "9FB7D518368DC14E8CC588FB3FD2749BEEF9F493FEF70AE34AF5721543C67173"
 
-suite "SSZ dynamic navigator":
+suiteReport "SSZ dynamic navigator":
   timedTest "navigating fields":
     var fooOrig = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
     let fooEncoded = SSZ.encode(fooOrig)

tests/test_state_transition.nim

@@ -13,7 +13,7 @@ import
   ../beacon_chain/spec/[beaconstate, datatypes, digest, validator],
   ../beacon_chain/[extras, state_transition, ssz]
 
-suite "Block processing" & preset():
+suiteReport "Block processing" & preset():
   ## For now just test that we can compile and execute block processing with
   ## mock data.
 

tests/test_sync_manager.nim

@@ -894,7 +894,7 @@ proc syncManagerFailureTest(): Future[bool] {.async.} =
     doAssert(checkRequest(peer, i, 10000, 20, 1) == true)
   result = true
 
-suite "SyncManager test suite":
+suiteReport "SyncManager test suite":
   timedTest "PeerSlot tests":
     check waitFor(peerSlotTests()) == true
   timedTest "PeerGroup tests":

tests/test_sync_protocol.nim

@@ -12,7 +12,7 @@ import unittest, ./testutil
 when false:
   import ../beacon_chain/sync_protocol
 
-suite "Sync protocol":
+suiteReport "Sync protocol":
   # Compile test
 
   timedTest "Compile":

tests/test_validator.nim

@@ -9,6 +9,6 @@ import
   ../beacon_chain/spec/[helpers, datatypes, digest, validator, beaconstate],
   ../beacon_chain/extras
 
-suite "Validators":
+suiteReport "Validators":
   # TODO: Empty!
   discard

tests/test_zero_signature.nim

@@ -17,7 +17,7 @@ import
 # and https://github.com/ethereum/eth2.0-specs/issues/1396
 # don't blow up.
 
-suite "Zero signature sanity checks":
+suiteReport "Zero signature sanity checks":
   # Using signature directly triggers a bug
   # in object_serialization/stew: https://github.com/status-im/nim-beacon-chain/issues/396
 

tests/testutil.nim

@@ -6,8 +6,8 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  algorithm, strformat, stats, times, std/monotimes, stew/endians2,
-  chronicles,
+  algorithm, strformat, stats, times, tables, std/monotimes, stew/endians2,
+  testutils/markdown_reports, chronicles,
   ../beacon_chain/[beacon_chain_db, block_pool, extras, ssz, kvstore, beacon_node_types],
   ../beacon_chain/spec/[digest, beaconstate, datatypes],
   testblockutil
@@ -47,8 +47,10 @@ template withTimerRet*(stats: var RunningStat, body: untyped): untyped =
   tmp
 
 var testTimes: seq[TestDuration]
+var status = initOrderedTable[string, OrderedTable[string, Status]]()
+var last: string
 
-proc summarizeLongTests*() =
+proc summarizeLongTests*(name: string) =
   # TODO clean-up and make machine-readable/storable the output
   # TODO this is too hard-coded and mostly a demo for using the
   # timedTest wrapper template for unittest
@@ -62,12 +64,30 @@ proc summarizeLongTests*() =
     if i >= 10:
       break
 
+  status.sort do (a: (string, OrderedTable[string, Status]),
+                  b: (string, OrderedTable[string, Status])) -> int: cmp(a[0], b[0])
+
+  generateReport(name & "-" & const_preset, status, width=90)
+
+template suiteReport*(name, body) =
+  last = name
+  status[last] = initOrderedTable[string, Status]()
+  suite name:
+    body
+
 template timedTest*(name, body) =
   var f: float
   test name:
+    status[last][name] = Status.Fail
+
     withTimer f:
       body
 
+    status[last][name] = case testStatusIMPL
+                         of OK: Status.OK
+                         of FAILED: Status.Fail
+                         of SKIPPED: Status.Skip
+
   # TODO reached for a failed test; maybe defer or similar
   # TODO noto thread-safe as-is
   testTimes.add (f, name)