Generate markdown test reports
This commit is contained in:
parent
04234bf90e
commit
6ba7b4b117
|
@ -50,4 +50,4 @@ import # Refactor state transition unit tests
|
|||
# ./official/test_fixture_shuffling,
|
||||
# ./official/test_fixture_bls
|
||||
|
||||
summarizeLongTests()
|
||||
summarizeLongTests("AllTests")
|
||||
|
|
|
@ -22,4 +22,4 @@ import
|
|||
./test_fixture_operations_proposer_slashings,
|
||||
./test_fixture_operations_voluntary_exit
|
||||
|
||||
summarizeLongTests()
|
||||
summarizeLongTests("FixtureAll")
|
||||
|
|
|
@ -49,7 +49,7 @@ proc readValue*(r: var JsonReader, a: var Domain) {.inline.} =
|
|||
# TODO: json tests were removed
|
||||
const BLSDir = JsonTestsDir/"general"/"phase0"/"bls"
|
||||
|
||||
suite "Official - BLS tests":
|
||||
suiteReport "Official - BLS tests":
|
||||
timedTest "Private to public key conversion":
|
||||
for file in walkDirRec(BLSDir/"priv_to_pub"):
|
||||
let t = parseTest(file, Json, BLSPrivToPub)
|
||||
|
|
|
@ -121,5 +121,5 @@ proc checkConfig() =
|
|||
else:
|
||||
check: ConstsToCheck[constant] == value.getBiggestInt().uint64()
|
||||
|
||||
suite "Official - 0.10.1 - constants & config " & preset():
|
||||
suiteReport "Official - 0.10.1 - constants & config " & preset():
|
||||
checkConfig()
|
||||
|
|
|
@ -65,6 +65,6 @@ proc runTest(identifier: string) =
|
|||
|
||||
`testImpl _ operations_attestations _ identifier`()
|
||||
|
||||
suite "Official - Operations - Attestations " & preset():
|
||||
suiteReport "Official - Operations - Attestations " & preset():
|
||||
for kind, path in walkDir(OperationsAttestationsDir, true):
|
||||
runTest(path)
|
||||
|
|
|
@ -67,7 +67,7 @@ proc runTest(identifier: string) =
|
|||
|
||||
`testImpl _ operations_attester_slashing _ identifier`()
|
||||
|
||||
suite "Official - Operations - Attester slashing " & preset():
|
||||
suiteReport "Official - Operations - Attester slashing " & preset():
|
||||
# TODO these are both valid and check BLS signatures, which isn't working
|
||||
# since 0.10.x introduces new BLS signing/verifying interface with domain
|
||||
# in particular handled differently through compute_signing_root() rather
|
||||
|
|
|
@ -62,6 +62,6 @@ proc runTest(identifier: string) =
|
|||
|
||||
`testImpl _ blockheader _ identifier`()
|
||||
|
||||
suite "Official - Operations - Block header " & preset():
|
||||
suiteReport "Official - Operations - Block header " & preset():
|
||||
for kind, path in walkDir(OpBlockHeaderDir, true):
|
||||
runTest(path)
|
||||
|
|
|
@ -60,7 +60,7 @@ proc runTest(identifier: string) =
|
|||
|
||||
`testImpl _ operations_deposits _ identifier`()
|
||||
|
||||
suite "Official - Operations - Deposits " & preset():
|
||||
suiteReport "Official - Operations - Deposits " & preset():
|
||||
# TODO
|
||||
const expected_failures = ["valid_sig_but_forked_state"]
|
||||
|
||||
|
|
|
@ -65,6 +65,6 @@ proc runTest(identifier: string) =
|
|||
|
||||
`testImpl_proposer_slashing _ identifier`()
|
||||
|
||||
suite "Official - Operations - Proposer slashing " & preset():
|
||||
suiteReport "Official - Operations - Proposer slashing " & preset():
|
||||
for kind, path in walkDir(OpProposerSlashingDir, true):
|
||||
runTest(path)
|
||||
|
|
|
@ -63,6 +63,6 @@ proc runTest(identifier: string) =
|
|||
|
||||
`testImpl _ voluntary_exit _ identifier`()
|
||||
|
||||
suite "Official - Operations - Voluntary exit " & preset():
|
||||
suiteReport "Official - Operations - Voluntary exit " & preset():
|
||||
for kind, path in walkDir(OpVoluntaryExitDir, true):
|
||||
runTest(path)
|
||||
|
|
|
@ -62,7 +62,7 @@ proc runTest(identifier: string) =
|
|||
|
||||
`testImpl _ blck _ identifier`()
|
||||
|
||||
suite "Official - Sanity - Blocks " & preset():
|
||||
suiteReport "Official - Sanity - Blocks " & preset():
|
||||
# Failing due to signature checking in indexed validation checking pending
|
||||
# 0.10 BLS verification API with new domain handling.
|
||||
const expected_failures =
|
||||
|
|
|
@ -44,6 +44,6 @@ proc runTest(identifier: string) =
|
|||
|
||||
`testImpl _ slots _ identifier`()
|
||||
|
||||
suite "Official - Sanity - Slots " & preset():
|
||||
suiteReport "Official - Sanity - Slots " & preset():
|
||||
for kind, path in walkDir(SanitySlotsDir, true):
|
||||
runTest(path)
|
||||
|
|
|
@ -103,5 +103,7 @@ proc runSSZtests() =
|
|||
else:
|
||||
raise newException(ValueError, "Unsupported test: " & sszType)
|
||||
|
||||
suite "Official - 0.10.1 - SSZ consensus objects " & preset():
|
||||
suiteReport "Official - 0.10.1 - SSZ consensus objects " & preset():
|
||||
runSSZtests()
|
||||
|
||||
summarizeLongTests("FixtureSSZConsensus")
|
||||
|
|
|
@ -302,5 +302,7 @@ proc runSSZtests() =
|
|||
# test "Testing " & name & " inputs (" & $T & ") - invalid":
|
||||
# const path = SSZDir/name/"invalid"
|
||||
|
||||
suite "Official - SSZ generic types":
|
||||
suiteReport "Official - SSZ generic types":
|
||||
runSSZtests()
|
||||
|
||||
summarizeLongTests("FixtureSSZGeneric")
|
||||
|
|
|
@ -33,7 +33,7 @@ template runSuite(suiteDir, testName: string, transitionProc: untyped{ident}, us
|
|||
# https://github.com/nim-lang/Nim/issues/12084#issue-486866402
|
||||
|
||||
proc `suiteImpl _ transitionProc`() =
|
||||
suite "Official - Epoch Processing - " & testName & preset():
|
||||
suiteReport "Official - Epoch Processing - " & testName & preset():
|
||||
for testDir in walkDirRec(suiteDir, yieldFilter = {pcDir}):
|
||||
|
||||
let unitTestName = testDir.rsplit(DirSep, 1)[1]
|
||||
|
|
|
@ -27,7 +27,7 @@ import
|
|||
# - MIN_GENESIS_TIME is not implemented
|
||||
# - is_valid_genesis_state is not implemented
|
||||
|
||||
suite "[Unit - Spec - Genesis] Genesis block checks " & preset():
|
||||
suiteReport "[Unit - Spec - Genesis] Genesis block checks " & preset():
|
||||
timedTest "is_valid_genesis_state for a valid state":
|
||||
discard initGenesisState(
|
||||
num_validators = MIN_GENESIS_ACTIVE_VALIDATOR_COUNT,
|
||||
|
|
|
@ -20,7 +20,7 @@ import
|
|||
../mocking/[mock_genesis, mock_attestations, mock_state, mock_blocks],
|
||||
../testutil
|
||||
|
||||
suite "[Unit - Spec - Block processing] Attestations " & preset():
|
||||
suiteReport "[Unit - Spec - Block processing] Attestations " & preset():
|
||||
|
||||
const NumValidators = uint64(8) * SLOTS_PER_EPOCH
|
||||
let genesisState = initGenesisState(NumValidators)
|
||||
|
|
|
@ -23,7 +23,7 @@ import
|
|||
../mocking/[mock_deposits, mock_genesis],
|
||||
../testutil, ../helpers/math_helpers
|
||||
|
||||
suite "[Unit - Spec - Block processing] Deposits " & preset():
|
||||
suiteReport "[Unit - Spec - Block processing] Deposits " & preset():
|
||||
|
||||
const NumValidators = uint64 5 * SLOTS_PER_EPOCH
|
||||
let genesisState = initGenesisState(NumValidators)
|
||||
|
|
|
@ -212,7 +212,7 @@ proc finalizeOn12(state: var BeaconState, epoch: Epoch, sufficient_support: bool
|
|||
doAssert state.current_justified_checkpoint == c2 # still old current
|
||||
doAssert state.finalized_checkpoint == old_finalized # no new finalized checkpoint
|
||||
|
||||
suite "[Unit - Spec - Epoch processing] Justification and Finalization " & preset():
|
||||
suiteReport "[Unit - Spec - Epoch processing] Justification and Finalization " & preset():
|
||||
echo " Finalization rules are detailed at https://github.com/protolambda/eth2-docs#justification-and-finalization"
|
||||
|
||||
const NumValidators = uint64(8) * SLOTS_PER_EPOCH
|
||||
|
|
|
@ -18,7 +18,7 @@ import
|
|||
../beacon_chain/ssz
|
||||
|
||||
when const_preset == "minimal": # Too much stack space used on mainnet
|
||||
suite "Attestation pool processing" & preset():
|
||||
suiteReport "Attestation pool processing" & preset():
|
||||
## For now just test that we can compile and execute block processing with
|
||||
## mock data.
|
||||
|
||||
|
|
|
@ -13,7 +13,7 @@ import options, unittest, sequtils,
|
|||
# test utilies
|
||||
./testutil, ./testblockutil
|
||||
|
||||
suite "Beacon chain DB" & preset():
|
||||
suiteReport "Beacon chain DB" & preset():
|
||||
timedTest "empty database" & preset():
|
||||
var
|
||||
db = init(BeaconChainDB, kvStore MemoryStoreRef.init())
|
||||
|
|
|
@ -12,7 +12,7 @@ import unittest, ./testutil
|
|||
when false:
|
||||
import ../beacon_chain/beacon_node
|
||||
|
||||
suite "Beacon node":
|
||||
suiteReport "Beacon node":
|
||||
# Compile test
|
||||
|
||||
timedTest "Compile":
|
||||
|
|
|
@ -13,7 +13,7 @@ import
|
|||
../beacon_chain/spec/[beaconstate, datatypes, digest],
|
||||
../beacon_chain/extras
|
||||
|
||||
suite "Beacon state" & preset():
|
||||
suiteReport "Beacon state" & preset():
|
||||
timedTest "Smoke test initialize_beacon_state_from_eth1" & preset():
|
||||
let state = initialize_beacon_state_from_eth1(
|
||||
Eth2Digest(), 0,
|
||||
|
|
|
@ -13,7 +13,7 @@ import
|
|||
../beacon_chain/spec/[datatypes, digest, helpers, validator],
|
||||
../beacon_chain/[beacon_node_types, block_pool, ssz]
|
||||
|
||||
suite "BlockRef and helpers" & preset():
|
||||
suiteReport "BlockRef and helpers" & preset():
|
||||
timedTest "isAncestorOf sanity" & preset():
|
||||
let
|
||||
s0 = BlockRef(slot: Slot(0))
|
||||
|
@ -51,7 +51,7 @@ suite "BlockRef and helpers" & preset():
|
|||
s4.getAncestorAt(Slot(3)) == s2
|
||||
s4.getAncestorAt(Slot(4)) == s4
|
||||
|
||||
suite "BlockSlot and helpers" & preset():
|
||||
suiteReport "BlockSlot and helpers" & preset():
|
||||
timedTest "atSlot sanity" & preset():
|
||||
let
|
||||
s0 = BlockRef(slot: Slot(0))
|
||||
|
@ -83,7 +83,7 @@ suite "BlockSlot and helpers" & preset():
|
|||
s24.parent.parent == s22
|
||||
|
||||
when const_preset == "minimal": # Too much stack space used on mainnet
|
||||
suite "Block pool processing" & preset():
|
||||
suiteReport "Block pool processing" & preset():
|
||||
setup:
|
||||
var
|
||||
db = makeTestDB(SLOTS_PER_EPOCH)
|
||||
|
@ -236,7 +236,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet
|
|||
tmpState.blck == b1Add.parent
|
||||
tmpState.data.data.slot == bs1.parent.slot
|
||||
|
||||
suite "BlockPool finalization tests" & preset():
|
||||
suiteReport "BlockPool finalization tests" & preset():
|
||||
setup:
|
||||
var
|
||||
db = makeTestDB(SLOTS_PER_EPOCH)
|
||||
|
|
|
@ -5,7 +5,7 @@ import
|
|||
eth/keys, eth/p2p/enode, libp2p/multiaddress,
|
||||
../beacon_chain/eth2_discovery
|
||||
|
||||
suite "Discovery v5 utilities":
|
||||
suiteReport "Discovery v5 utilities":
|
||||
timedTest "Multiaddress to ENode":
|
||||
let addrStr = "/ip4/178.128.140.61/tcp/9000/p2p/16Uiu2HAmL5A5DAiiupFi6sUTF6Zq1TCKf6Pd5T8oFt9opQJqLqTQ"
|
||||
let ma = MultiAddress.init addrStr
|
||||
|
|
|
@ -11,7 +11,7 @@ import
|
|||
unittest, ./testutil,
|
||||
../beacon_chain/spec/[helpers]
|
||||
|
||||
suite "Spec helpers":
|
||||
suiteReport "Spec helpers":
|
||||
timedTest "integer_squareroot":
|
||||
check:
|
||||
integer_squareroot(0'u64) == 0'u64
|
||||
|
|
|
@ -4,7 +4,7 @@ import
|
|||
unittest, stint, ./testutil,
|
||||
../beacon_chain/spec/network
|
||||
|
||||
suite "Honest validator":
|
||||
suiteReport "Honest validator":
|
||||
timedTest "Attestation topics":
|
||||
check:
|
||||
getAttestationTopic(0) == "/eth2/index0_beacon_attestation/ssz"
|
||||
|
|
|
@ -119,7 +119,7 @@ let depositsConfig = [
|
|||
)
|
||||
]
|
||||
|
||||
suite "Interop":
|
||||
suiteReport "Interop":
|
||||
timedTest "Mocked start private key":
|
||||
for i, k in privateKeys:
|
||||
let
|
||||
|
|
|
@ -36,7 +36,7 @@ proc init*(t: typedesc[PeerTest], id: string = "",
|
|||
proc close*(peer: PeerTest) =
|
||||
peer.future.complete()
|
||||
|
||||
suite "PeerPool testing suite":
|
||||
suiteReport "PeerPool testing suite":
|
||||
timedTest "addPeerNoWait() test":
|
||||
const peersCount = [
|
||||
[10, 5, 5, 10, 5, 5],
|
||||
|
|
|
@ -75,7 +75,7 @@ type
|
|||
proc toDigest[N: static int](x: array[N, byte]): Eth2Digest =
|
||||
result.data[0 .. N-1] = x
|
||||
|
||||
suite "SSZ navigator":
|
||||
suiteReport "SSZ navigator":
|
||||
timedTest "simple object fields":
|
||||
var foo = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
|
||||
let encoded = SSZ.encode(foo)
|
||||
|
@ -101,7 +101,7 @@ suite "SSZ navigator":
|
|||
let root2 = hash_tree_root(leaves2)
|
||||
check $root2 == "9FB7D518368DC14E8CC588FB3FD2749BEEF9F493FEF70AE34AF5721543C67173"
|
||||
|
||||
suite "SSZ dynamic navigator":
|
||||
suiteReport "SSZ dynamic navigator":
|
||||
timedTest "navigating fields":
|
||||
var fooOrig = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
|
||||
let fooEncoded = SSZ.encode(fooOrig)
|
||||
|
|
|
@ -13,7 +13,7 @@ import
|
|||
../beacon_chain/spec/[beaconstate, datatypes, digest, validator],
|
||||
../beacon_chain/[extras, state_transition, ssz]
|
||||
|
||||
suite "Block processing" & preset():
|
||||
suiteReport "Block processing" & preset():
|
||||
## For now just test that we can compile and execute block processing with
|
||||
## mock data.
|
||||
|
||||
|
|
|
@ -894,7 +894,7 @@ proc syncManagerFailureTest(): Future[bool] {.async.} =
|
|||
doAssert(checkRequest(peer, i, 10000, 20, 1) == true)
|
||||
result = true
|
||||
|
||||
suite "SyncManager test suite":
|
||||
suiteReport "SyncManager test suite":
|
||||
timedTest "PeerSlot tests":
|
||||
check waitFor(peerSlotTests()) == true
|
||||
timedTest "PeerGroup tests":
|
||||
|
|
|
@ -12,7 +12,7 @@ import unittest, ./testutil
|
|||
when false:
|
||||
import ../beacon_chain/sync_protocol
|
||||
|
||||
suite "Sync protocol":
|
||||
suiteReport "Sync protocol":
|
||||
# Compile test
|
||||
|
||||
timedTest "Compile":
|
||||
|
|
|
@ -9,6 +9,6 @@ import
|
|||
../beacon_chain/spec/[helpers, datatypes, digest, validator, beaconstate],
|
||||
../beacon_chain/extras
|
||||
|
||||
suite "Validators":
|
||||
suiteReport "Validators":
|
||||
# TODO: Empty!
|
||||
discard
|
||||
|
|
|
@ -17,7 +17,7 @@ import
|
|||
# and https://github.com/ethereum/eth2.0-specs/issues/1396
|
||||
# don't blow up.
|
||||
|
||||
suite "Zero signature sanity checks":
|
||||
suiteReport "Zero signature sanity checks":
|
||||
# Using signature directly triggers a bug
|
||||
# in object_serialization/stew: https://github.com/status-im/nim-beacon-chain/issues/396
|
||||
|
||||
|
|
|
@ -6,8 +6,8 @@
|
|||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
||||
|
||||
import
|
||||
algorithm, strformat, stats, times, std/monotimes, stew/endians2,
|
||||
chronicles,
|
||||
algorithm, strformat, stats, times, tables, std/monotimes, stew/endians2,
|
||||
testutils/markdown_reports, chronicles,
|
||||
../beacon_chain/[beacon_chain_db, block_pool, extras, ssz, kvstore, beacon_node_types],
|
||||
../beacon_chain/spec/[digest, beaconstate, datatypes],
|
||||
testblockutil
|
||||
|
@ -47,8 +47,10 @@ template withTimerRet*(stats: var RunningStat, body: untyped): untyped =
|
|||
tmp
|
||||
|
||||
var testTimes: seq[TestDuration]
|
||||
var status = initOrderedTable[string, OrderedTable[string, Status]]()
|
||||
var last: string
|
||||
|
||||
proc summarizeLongTests*() =
|
||||
proc summarizeLongTests*(name: string) =
|
||||
# TODO clean-up and make machine-readable/storable the output
|
||||
# TODO this is too hard-coded and mostly a demo for using the
|
||||
# timedTest wrapper template for unittest
|
||||
|
@ -62,12 +64,30 @@ proc summarizeLongTests*() =
|
|||
if i >= 10:
|
||||
break
|
||||
|
||||
status.sort do (a: (string, OrderedTable[string, Status]),
|
||||
b: (string, OrderedTable[string, Status])) -> int: cmp(a[0], b[0])
|
||||
|
||||
generateReport(name & "-" & const_preset, status, width=90)
|
||||
|
||||
template suiteReport*(name, body) =
|
||||
last = name
|
||||
status[last] = initOrderedTable[string, Status]()
|
||||
suite name:
|
||||
body
|
||||
|
||||
template timedTest*(name, body) =
|
||||
var f: float
|
||||
test name:
|
||||
status[last][name] = Status.Fail
|
||||
|
||||
withTimer f:
|
||||
body
|
||||
|
||||
status[last][name] = case testStatusIMPL
|
||||
of OK: Status.OK
|
||||
of FAILED: Status.Fail
|
||||
of SKIPPED: Status.Skip
|
||||
|
||||
# TODO reached for a failed test; maybe defer or similar
|
||||
# TODO noto thread-safe as-is
|
||||
testTimes.add (f, name)
|
||||
|
|
Loading…
Reference in New Issue