diff --git a/tests/all_tests.nim b/tests/all_tests.nim index a278bb137..b3ed41f35 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -50,4 +50,4 @@ import # Refactor state transition unit tests # ./official/test_fixture_shuffling, # ./official/test_fixture_bls -summarizeLongTests() +summarizeLongTests("AllTests") diff --git a/tests/official/all_fixtures_require_ssz.nim b/tests/official/all_fixtures_require_ssz.nim index 9fc892752..e2f12f8e4 100644 --- a/tests/official/all_fixtures_require_ssz.nim +++ b/tests/official/all_fixtures_require_ssz.nim @@ -22,4 +22,4 @@ import ./test_fixture_operations_proposer_slashings, ./test_fixture_operations_voluntary_exit -summarizeLongTests() +summarizeLongTests("FixtureAll") diff --git a/tests/official/test_fixture_bls.nim b/tests/official/test_fixture_bls.nim index 44ee479ed..efeb10bf5 100644 --- a/tests/official/test_fixture_bls.nim +++ b/tests/official/test_fixture_bls.nim @@ -49,7 +49,7 @@ proc readValue*(r: var JsonReader, a: var Domain) {.inline.} = # TODO: json tests were removed const BLSDir = JsonTestsDir/"general"/"phase0"/"bls" -suite "Official - BLS tests": +suiteReport "Official - BLS tests": timedTest "Private to public key conversion": for file in walkDirRec(BLSDir/"priv_to_pub"): let t = parseTest(file, Json, BLSPrivToPub) diff --git a/tests/official/test_fixture_const_sanity_check.nim b/tests/official/test_fixture_const_sanity_check.nim index 7df5f9582..5f43ac4bb 100644 --- a/tests/official/test_fixture_const_sanity_check.nim +++ b/tests/official/test_fixture_const_sanity_check.nim @@ -121,5 +121,5 @@ proc checkConfig() = else: check: ConstsToCheck[constant] == value.getBiggestInt().uint64() -suite "Official - 0.10.1 - constants & config " & preset(): +suiteReport "Official - 0.10.1 - constants & config " & preset(): checkConfig() diff --git a/tests/official/test_fixture_operations_attestations.nim b/tests/official/test_fixture_operations_attestations.nim index 5c3f8b10a..d30b79d25 100644 --- a/tests/official/test_fixture_operations_attestations.nim +++ b/tests/official/test_fixture_operations_attestations.nim @@ -65,6 +65,6 @@ proc runTest(identifier: string) = `testImpl _ operations_attestations _ identifier`() -suite "Official - Operations - Attestations " & preset(): +suiteReport "Official - Operations - Attestations " & preset(): for kind, path in walkDir(OperationsAttestationsDir, true): runTest(path) diff --git a/tests/official/test_fixture_operations_attester_slashings.nim b/tests/official/test_fixture_operations_attester_slashings.nim index b6886e5b5..d44d29658 100644 --- a/tests/official/test_fixture_operations_attester_slashings.nim +++ b/tests/official/test_fixture_operations_attester_slashings.nim @@ -67,7 +67,7 @@ proc runTest(identifier: string) = `testImpl _ operations_attester_slashing _ identifier`() -suite "Official - Operations - Attester slashing " & preset(): +suiteReport "Official - Operations - Attester slashing " & preset(): # TODO these are both valid and check BLS signatures, which isn't working # since 0.10.x introduces new BLS signing/verifying interface with domain # in particular handled differently through compute_signing_root() rather diff --git a/tests/official/test_fixture_operations_block_header.nim b/tests/official/test_fixture_operations_block_header.nim index 9584bdf6c..aa5d846f0 100644 --- a/tests/official/test_fixture_operations_block_header.nim +++ b/tests/official/test_fixture_operations_block_header.nim @@ -62,6 +62,6 @@ proc runTest(identifier: string) = `testImpl _ blockheader _ identifier`() -suite "Official - Operations - Block header " & preset(): +suiteReport "Official - Operations - Block header " & preset(): for kind, path in walkDir(OpBlockHeaderDir, true): runTest(path) diff --git a/tests/official/test_fixture_operations_deposits.nim b/tests/official/test_fixture_operations_deposits.nim index 4803a6608..3f86e661d 100644 --- a/tests/official/test_fixture_operations_deposits.nim +++ b/tests/official/test_fixture_operations_deposits.nim @@ -60,7 +60,7 @@ proc runTest(identifier: string) = `testImpl _ operations_deposits _ identifier`() -suite "Official - Operations - Deposits " & preset(): +suiteReport "Official - Operations - Deposits " & preset(): # TODO const expected_failures = ["valid_sig_but_forked_state"] diff --git a/tests/official/test_fixture_operations_proposer_slashings.nim b/tests/official/test_fixture_operations_proposer_slashings.nim index 5c8739fc6..9e2841346 100644 --- a/tests/official/test_fixture_operations_proposer_slashings.nim +++ b/tests/official/test_fixture_operations_proposer_slashings.nim @@ -65,6 +65,6 @@ proc runTest(identifier: string) = `testImpl_proposer_slashing _ identifier`() -suite "Official - Operations - Proposer slashing " & preset(): +suiteReport "Official - Operations - Proposer slashing " & preset(): for kind, path in walkDir(OpProposerSlashingDir, true): runTest(path) diff --git a/tests/official/test_fixture_operations_voluntary_exit.nim b/tests/official/test_fixture_operations_voluntary_exit.nim index 5609fb5ae..392b0cbb5 100644 --- a/tests/official/test_fixture_operations_voluntary_exit.nim +++ b/tests/official/test_fixture_operations_voluntary_exit.nim @@ -63,6 +63,6 @@ proc runTest(identifier: string) = `testImpl _ voluntary_exit _ identifier`() -suite "Official - Operations - Voluntary exit " & preset(): +suiteReport "Official - Operations - Voluntary exit " & preset(): for kind, path in walkDir(OpVoluntaryExitDir, true): runTest(path) diff --git a/tests/official/test_fixture_sanity_blocks.nim b/tests/official/test_fixture_sanity_blocks.nim index a74f72d4f..9dc58cc2a 100644 --- a/tests/official/test_fixture_sanity_blocks.nim +++ b/tests/official/test_fixture_sanity_blocks.nim @@ -62,7 +62,7 @@ proc runTest(identifier: string) = `testImpl _ blck _ identifier`() -suite "Official - Sanity - Blocks " & preset(): +suiteReport "Official - Sanity - Blocks " & preset(): # Failing due to signature checking in indexed validation checking pending # 0.10 BLS verification API with new domain handling. const expected_failures = diff --git a/tests/official/test_fixture_sanity_slots.nim b/tests/official/test_fixture_sanity_slots.nim index 3dc0e5655..b5e7f0ee7 100644 --- a/tests/official/test_fixture_sanity_slots.nim +++ b/tests/official/test_fixture_sanity_slots.nim @@ -44,6 +44,6 @@ proc runTest(identifier: string) = `testImpl _ slots _ identifier`() -suite "Official - Sanity - Slots " & preset(): +suiteReport "Official - Sanity - Slots " & preset(): for kind, path in walkDir(SanitySlotsDir, true): runTest(path) diff --git a/tests/official/test_fixture_ssz_consensus_objects.nim b/tests/official/test_fixture_ssz_consensus_objects.nim index 28c79b151..8498eb089 100644 --- a/tests/official/test_fixture_ssz_consensus_objects.nim +++ b/tests/official/test_fixture_ssz_consensus_objects.nim @@ -103,5 +103,7 @@ proc runSSZtests() = else: raise newException(ValueError, "Unsupported test: " & sszType) -suite "Official - 0.10.1 - SSZ consensus objects " & preset(): +suiteReport "Official - 0.10.1 - SSZ consensus objects " & preset(): runSSZtests() + +summarizeLongTests("FixtureSSZConsensus") diff --git a/tests/official/test_fixture_ssz_generic_types.nim b/tests/official/test_fixture_ssz_generic_types.nim index f7f948fc7..e4e8e9ba4 100644 --- a/tests/official/test_fixture_ssz_generic_types.nim +++ b/tests/official/test_fixture_ssz_generic_types.nim @@ -302,5 +302,7 @@ proc runSSZtests() = # test "Testing " & name & " inputs (" & $T & ") - invalid": # const path = SSZDir/name/"invalid" -suite "Official - SSZ generic types": +suiteReport "Official - SSZ generic types": runSSZtests() + +summarizeLongTests("FixtureSSZGeneric") diff --git a/tests/official/test_fixture_state_transition_epoch.nim b/tests/official/test_fixture_state_transition_epoch.nim index 6d248d109..cd163bd6d 100644 --- a/tests/official/test_fixture_state_transition_epoch.nim +++ b/tests/official/test_fixture_state_transition_epoch.nim @@ -33,7 +33,7 @@ template runSuite(suiteDir, testName: string, transitionProc: untyped{ident}, us # https://github.com/nim-lang/Nim/issues/12084#issue-486866402 proc `suiteImpl _ transitionProc`() = - suite "Official - Epoch Processing - " & testName & preset(): + suiteReport "Official - Epoch Processing - " & testName & preset(): for testDir in walkDirRec(suiteDir, yieldFilter = {pcDir}): let unitTestName = testDir.rsplit(DirSep, 1)[1] diff --git a/tests/spec_block_processing/test_genesis.nim b/tests/spec_block_processing/test_genesis.nim index e7ec023ed..ce6df9ec9 100644 --- a/tests/spec_block_processing/test_genesis.nim +++ b/tests/spec_block_processing/test_genesis.nim @@ -27,7 +27,7 @@ import # - MIN_GENESIS_TIME is not implemented # - is_valid_genesis_state is not implemented -suite "[Unit - Spec - Genesis] Genesis block checks " & preset(): +suiteReport "[Unit - Spec - Genesis] Genesis block checks " & preset(): timedTest "is_valid_genesis_state for a valid state": discard initGenesisState( num_validators = MIN_GENESIS_ACTIVE_VALIDATOR_COUNT, diff --git a/tests/spec_block_processing/test_process_attestation.nim b/tests/spec_block_processing/test_process_attestation.nim index 4c9745540..e914530b3 100644 --- a/tests/spec_block_processing/test_process_attestation.nim +++ b/tests/spec_block_processing/test_process_attestation.nim @@ -20,7 +20,7 @@ import ../mocking/[mock_genesis, mock_attestations, mock_state, mock_blocks], ../testutil -suite "[Unit - Spec - Block processing] Attestations " & preset(): +suiteReport "[Unit - Spec - Block processing] Attestations " & preset(): const NumValidators = uint64(8) * SLOTS_PER_EPOCH let genesisState = initGenesisState(NumValidators) diff --git a/tests/spec_block_processing/test_process_deposits.nim b/tests/spec_block_processing/test_process_deposits.nim index c5a2613b5..765e0aa8d 100644 --- a/tests/spec_block_processing/test_process_deposits.nim +++ b/tests/spec_block_processing/test_process_deposits.nim @@ -23,7 +23,7 @@ import ../mocking/[mock_deposits, mock_genesis], ../testutil, ../helpers/math_helpers -suite "[Unit - Spec - Block processing] Deposits " & preset(): +suiteReport "[Unit - Spec - Block processing] Deposits " & preset(): const NumValidators = uint64 5 * SLOTS_PER_EPOCH let genesisState = initGenesisState(NumValidators) diff --git a/tests/spec_epoch_processing/test_process_justification_and_finalization.nim b/tests/spec_epoch_processing/test_process_justification_and_finalization.nim index 2b36e0755..decf8d2ca 100644 --- a/tests/spec_epoch_processing/test_process_justification_and_finalization.nim +++ b/tests/spec_epoch_processing/test_process_justification_and_finalization.nim @@ -212,7 +212,7 @@ proc finalizeOn12(state: var BeaconState, epoch: Epoch, sufficient_support: bool doAssert state.current_justified_checkpoint == c2 # still old current doAssert state.finalized_checkpoint == old_finalized # no new finalized checkpoint -suite "[Unit - Spec - Epoch processing] Justification and Finalization " & preset(): +suiteReport "[Unit - Spec - Epoch processing] Justification and Finalization " & preset(): echo " Finalization rules are detailed at https://github.com/protolambda/eth2-docs#justification-and-finalization" const NumValidators = uint64(8) * SLOTS_PER_EPOCH diff --git a/tests/test_attestation_pool.nim b/tests/test_attestation_pool.nim index 6b036cd17..3aa69fe2c 100644 --- a/tests/test_attestation_pool.nim +++ b/tests/test_attestation_pool.nim @@ -18,7 +18,7 @@ import ../beacon_chain/ssz when const_preset == "minimal": # Too much stack space used on mainnet - suite "Attestation pool processing" & preset(): + suiteReport "Attestation pool processing" & preset(): ## For now just test that we can compile and execute block processing with ## mock data. diff --git a/tests/test_beacon_chain_db.nim b/tests/test_beacon_chain_db.nim index cc94ef1b1..28bf53ee4 100644 --- a/tests/test_beacon_chain_db.nim +++ b/tests/test_beacon_chain_db.nim @@ -13,7 +13,7 @@ import options, unittest, sequtils, # test utilies ./testutil, ./testblockutil -suite "Beacon chain DB" & preset(): +suiteReport "Beacon chain DB" & preset(): timedTest "empty database" & preset(): var db = init(BeaconChainDB, kvStore MemoryStoreRef.init()) diff --git a/tests/test_beacon_node.nim b/tests/test_beacon_node.nim index 632146b7e..67d13a9fd 100644 --- a/tests/test_beacon_node.nim +++ b/tests/test_beacon_node.nim @@ -12,7 +12,7 @@ import unittest, ./testutil when false: import ../beacon_chain/beacon_node -suite "Beacon node": +suiteReport "Beacon node": # Compile test timedTest "Compile": diff --git a/tests/test_beaconstate.nim b/tests/test_beaconstate.nim index 13edaa804..b539ce9a8 100644 --- a/tests/test_beaconstate.nim +++ b/tests/test_beaconstate.nim @@ -13,7 +13,7 @@ import ../beacon_chain/spec/[beaconstate, datatypes, digest], ../beacon_chain/extras -suite "Beacon state" & preset(): +suiteReport "Beacon state" & preset(): timedTest "Smoke test initialize_beacon_state_from_eth1" & preset(): let state = initialize_beacon_state_from_eth1( Eth2Digest(), 0, diff --git a/tests/test_block_pool.nim b/tests/test_block_pool.nim index 20bb2ec6a..1239c09c9 100644 --- a/tests/test_block_pool.nim +++ b/tests/test_block_pool.nim @@ -13,7 +13,7 @@ import ../beacon_chain/spec/[datatypes, digest, helpers, validator], ../beacon_chain/[beacon_node_types, block_pool, ssz] -suite "BlockRef and helpers" & preset(): +suiteReport "BlockRef and helpers" & preset(): timedTest "isAncestorOf sanity" & preset(): let s0 = BlockRef(slot: Slot(0)) @@ -51,7 +51,7 @@ suite "BlockRef and helpers" & preset(): s4.getAncestorAt(Slot(3)) == s2 s4.getAncestorAt(Slot(4)) == s4 -suite "BlockSlot and helpers" & preset(): +suiteReport "BlockSlot and helpers" & preset(): timedTest "atSlot sanity" & preset(): let s0 = BlockRef(slot: Slot(0)) @@ -83,7 +83,7 @@ suite "BlockSlot and helpers" & preset(): s24.parent.parent == s22 when const_preset == "minimal": # Too much stack space used on mainnet - suite "Block pool processing" & preset(): + suiteReport "Block pool processing" & preset(): setup: var db = makeTestDB(SLOTS_PER_EPOCH) @@ -236,7 +236,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet tmpState.blck == b1Add.parent tmpState.data.data.slot == bs1.parent.slot - suite "BlockPool finalization tests" & preset(): + suiteReport "BlockPool finalization tests" & preset(): setup: var db = makeTestDB(SLOTS_PER_EPOCH) diff --git a/tests/test_discovery_helpers.nim b/tests/test_discovery_helpers.nim index 6df2d4c5a..4fbc69f65 100644 --- a/tests/test_discovery_helpers.nim +++ b/tests/test_discovery_helpers.nim @@ -5,7 +5,7 @@ import eth/keys, eth/p2p/enode, libp2p/multiaddress, ../beacon_chain/eth2_discovery -suite "Discovery v5 utilities": +suiteReport "Discovery v5 utilities": timedTest "Multiaddress to ENode": let addrStr = "/ip4/178.128.140.61/tcp/9000/p2p/16Uiu2HAmL5A5DAiiupFi6sUTF6Zq1TCKf6Pd5T8oFt9opQJqLqTQ" let ma = MultiAddress.init addrStr diff --git a/tests/test_helpers.nim b/tests/test_helpers.nim index af12ffa4e..ac0c3d42a 100644 --- a/tests/test_helpers.nim +++ b/tests/test_helpers.nim @@ -11,7 +11,7 @@ import unittest, ./testutil, ../beacon_chain/spec/[helpers] -suite "Spec helpers": +suiteReport "Spec helpers": timedTest "integer_squareroot": check: integer_squareroot(0'u64) == 0'u64 diff --git a/tests/test_honest_validator.nim b/tests/test_honest_validator.nim index 7036cfced..5bae259ca 100644 --- a/tests/test_honest_validator.nim +++ b/tests/test_honest_validator.nim @@ -4,7 +4,7 @@ import unittest, stint, ./testutil, ../beacon_chain/spec/network -suite "Honest validator": +suiteReport "Honest validator": timedTest "Attestation topics": check: getAttestationTopic(0) == "/eth2/index0_beacon_attestation/ssz" diff --git a/tests/test_interop.nim b/tests/test_interop.nim index f60afdec6..47d8ed27d 100644 --- a/tests/test_interop.nim +++ b/tests/test_interop.nim @@ -119,7 +119,7 @@ let depositsConfig = [ ) ] -suite "Interop": +suiteReport "Interop": timedTest "Mocked start private key": for i, k in privateKeys: let diff --git a/tests/test_peer_pool.nim b/tests/test_peer_pool.nim index b689ad270..9a845dd9a 100644 --- a/tests/test_peer_pool.nim +++ b/tests/test_peer_pool.nim @@ -36,7 +36,7 @@ proc init*(t: typedesc[PeerTest], id: string = "", proc close*(peer: PeerTest) = peer.future.complete() -suite "PeerPool testing suite": +suiteReport "PeerPool testing suite": timedTest "addPeerNoWait() test": const peersCount = [ [10, 5, 5, 10, 5, 5], diff --git a/tests/test_ssz.nim b/tests/test_ssz.nim index a796ec103..e032fb1ea 100644 --- a/tests/test_ssz.nim +++ b/tests/test_ssz.nim @@ -75,7 +75,7 @@ type proc toDigest[N: static int](x: array[N, byte]): Eth2Digest = result.data[0 .. N-1] = x -suite "SSZ navigator": +suiteReport "SSZ navigator": timedTest "simple object fields": var foo = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64))) let encoded = SSZ.encode(foo) @@ -101,7 +101,7 @@ suite "SSZ navigator": let root2 = hash_tree_root(leaves2) check $root2 == "9FB7D518368DC14E8CC588FB3FD2749BEEF9F493FEF70AE34AF5721543C67173" -suite "SSZ dynamic navigator": +suiteReport "SSZ dynamic navigator": timedTest "navigating fields": var fooOrig = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64))) let fooEncoded = SSZ.encode(fooOrig) diff --git a/tests/test_state_transition.nim b/tests/test_state_transition.nim index a2c1b210f..855c992af 100644 --- a/tests/test_state_transition.nim +++ b/tests/test_state_transition.nim @@ -13,7 +13,7 @@ import ../beacon_chain/spec/[beaconstate, datatypes, digest, validator], ../beacon_chain/[extras, state_transition, ssz] -suite "Block processing" & preset(): +suiteReport "Block processing" & preset(): ## For now just test that we can compile and execute block processing with ## mock data. diff --git a/tests/test_sync_manager.nim b/tests/test_sync_manager.nim index 3073427f3..9406a7e72 100644 --- a/tests/test_sync_manager.nim +++ b/tests/test_sync_manager.nim @@ -894,7 +894,7 @@ proc syncManagerFailureTest(): Future[bool] {.async.} = doAssert(checkRequest(peer, i, 10000, 20, 1) == true) result = true -suite "SyncManager test suite": +suiteReport "SyncManager test suite": timedTest "PeerSlot tests": check waitFor(peerSlotTests()) == true timedTest "PeerGroup tests": diff --git a/tests/test_sync_protocol.nim b/tests/test_sync_protocol.nim index 37f695d66..6078a6012 100644 --- a/tests/test_sync_protocol.nim +++ b/tests/test_sync_protocol.nim @@ -12,7 +12,7 @@ import unittest, ./testutil when false: import ../beacon_chain/sync_protocol -suite "Sync protocol": +suiteReport "Sync protocol": # Compile test timedTest "Compile": diff --git a/tests/test_validator.nim b/tests/test_validator.nim index 652f4d46b..8d5b0b880 100644 --- a/tests/test_validator.nim +++ b/tests/test_validator.nim @@ -9,6 +9,6 @@ import ../beacon_chain/spec/[helpers, datatypes, digest, validator, beaconstate], ../beacon_chain/extras -suite "Validators": +suiteReport "Validators": # TODO: Empty! discard diff --git a/tests/test_zero_signature.nim b/tests/test_zero_signature.nim index c259ab859..391a5615d 100644 --- a/tests/test_zero_signature.nim +++ b/tests/test_zero_signature.nim @@ -17,7 +17,7 @@ import # and https://github.com/ethereum/eth2.0-specs/issues/1396 # don't blow up. -suite "Zero signature sanity checks": +suiteReport "Zero signature sanity checks": # Using signature directly triggers a bug # in object_serialization/stew: https://github.com/status-im/nim-beacon-chain/issues/396 diff --git a/tests/testutil.nim b/tests/testutil.nim index 936ec50cb..50e71193d 100644 --- a/tests/testutil.nim +++ b/tests/testutil.nim @@ -6,8 +6,8 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. import - algorithm, strformat, stats, times, std/monotimes, stew/endians2, - chronicles, + algorithm, strformat, stats, times, tables, std/monotimes, stew/endians2, + testutils/markdown_reports, chronicles, ../beacon_chain/[beacon_chain_db, block_pool, extras, ssz, kvstore, beacon_node_types], ../beacon_chain/spec/[digest, beaconstate, datatypes], testblockutil @@ -47,8 +47,10 @@ template withTimerRet*(stats: var RunningStat, body: untyped): untyped = tmp var testTimes: seq[TestDuration] +var status = initOrderedTable[string, OrderedTable[string, Status]]() +var last: string -proc summarizeLongTests*() = +proc summarizeLongTests*(name: string) = # TODO clean-up and make machine-readable/storable the output # TODO this is too hard-coded and mostly a demo for using the # timedTest wrapper template for unittest @@ -62,12 +64,30 @@ proc summarizeLongTests*() = if i >= 10: break + status.sort do (a: (string, OrderedTable[string, Status]), + b: (string, OrderedTable[string, Status])) -> int: cmp(a[0], b[0]) + + generateReport(name & "-" & const_preset, status, width=90) + +template suiteReport*(name, body) = + last = name + status[last] = initOrderedTable[string, Status]() + suite name: + body + template timedTest*(name, body) = var f: float test name: + status[last][name] = Status.Fail + withTimer f: body + status[last][name] = case testStatusIMPL + of OK: Status.OK + of FAILED: Status.Fail + of SKIPPED: Status.Skip + # TODO reached for a failed test; maybe defer or similar # TODO noto thread-safe as-is testTimes.add (f, name)