set block header tests to automatically enumerate all relevant tests (#717)

* set block header tests to automatically enumerate all relevant tests

* add several attester slashing tests and run all tests by default, rather than only explicitly listed test vectors

* print something relating to skipped tests

* set voluntary exits to automatically enumerate all tests

tests/official/test_fixture_operations_attester_slashings.nim

@@ -20,13 +20,13 @@ import
 
 const OpAttSlashingDir = SszTestsDir/const_preset/"phase0"/"operations"/"attester_slashing"/"pyspec_tests"
 
-template runTest(identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OpAttSlashingDir / astToStr(identifier)
+  let testDir = OpAttSlashingDir / identifier
 
   proc `testImpl _ operations_attester_slashing _ identifier`() =
 
@@ -39,7 +39,7 @@ template runTest(identifier: untyped) =
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & astToStr(identifier):
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       var attesterSlashingRef: ref AttesterSlashing
       new attesterSlashingRef
@@ -68,9 +68,6 @@ template runTest(identifier: untyped) =
   `testImpl _ operations_attester_slashing _ identifier`()
 
 suite "Official - Operations - Attester slashing " & preset():
-  runTest(success_double)
-  runTest(success_surround)
-  when false:
   # TODO these are both valid and check BLS signatures, which isn't working
   # since 0.10.x introduces new BLS signing/verifying interface with domain
   # in particular handled differently through compute_signing_root() rather
@@ -78,17 +75,10 @@ suite "Official - Operations - Attester slashing " & preset():
   # visible issue it now is because another bug had been masking it wherein
   # crypto.nim's bls_verify(...) call had been creating false positives, in
   # which cases signature checks had been incorrectly passing.
-    runTest(success_already_exited_recent)
+  const expected_failures =
-    runTest(success_already_exited_long_ago)
+    ["success_already_exited_recent", "success_already_exited_long_ago"]
-  runTest(invalid_sig_1)
+  for kind, path in walkDir(OpAttSlashingDir, true):
-  when false: # TODO - https://github.com/status-im/nim-beacon-chain/issues/429
+    if path in expected_failures:
-    runTest(invalid_sig_2)
+      echo "Skipping test: ", path
-  runTest(invalid_sig_1_and_2)
+      continue
-  runTest(same_data)
+    runTest(path)
-  runTest(no_double_or_surround)
-  runTest(participants_already_slashed)
-  when false: # TODO - https://github.com/status-im/nim-beacon-chain/issues/429
-    runTest(att1_bad_extra_index)
-    runTest(att1_bad_replaced_index)
-    runTest(att2_bad_extra_index)
-    runTest(att2_bad_replaced_index)

tests/official/test_fixture_operations_block_header.nim

@@ -20,26 +20,23 @@ import
 
 const OpBlockHeaderDir = SszTestsDir/const_preset/"phase0"/"operations"/"block_header"/"pyspec_tests"
 
-template runTest(identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OpBlockHeaderDir / astToStr(identifier)
+  let testDir = OpBlockHeaderDir / identifier
 
   proc `testImpl _ blockheader _ identifier`() =
 
-    var flags: UpdateFlags
     var prefix: string
-    if not existsFile(testDir/"meta.yaml"):
-      flags.incl skipValidation
     if existsFile(testDir/"post.ssz"):
       prefix = "[Valid]   "
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & astToStr(identifier):
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       var blck: ref BeaconBlock
       new blck
@@ -55,10 +52,10 @@ template runTest(identifier: untyped) =
         postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
 
       if postRef.isNil:
-        let done = process_block_header(stateRef[], blck[], flags, cache)
+        let done = process_block_header(stateRef[], blck[], {}, cache)
         doAssert done == false, "We didn't expect this invalid block header to be processed."
       else:
-        let done = process_block_header(stateRef[], blck[], flags, cache)
+        let done = process_block_header(stateRef[], blck[], {}, cache)
         doAssert done, "Valid block header not processed"
         check: stateRef.hash_tree_root() == postRef.hash_tree_root()
         reportDiff(stateRef, postRef)
@@ -66,8 +63,5 @@ template runTest(identifier: untyped) =
   `testImpl _ blockheader _ identifier`()
 
 suite "Official - Operations - Block header " & preset():
-  runTest(success_block_header)
+  for kind, path in walkDir(OpBlockHeaderDir, true):
-  runTest(invalid_slot_block_header)
+    runTest(path)
-  when false: # skipValidation needs to be split https://github.com/status-im/nim-beacon-chain/issues/407
-    runTest(invalid_parent_root)
-  runTest(proposer_slashed)

tests/official/test_fixture_operations_voluntary_exit.nim

@@ -20,13 +20,13 @@ import
 
 const OpVoluntaryExitDir = SszTestsDir/const_preset/"phase0"/"operations"/"voluntary_exit"/"pyspec_tests"
 
-template runTest(identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OpVoluntaryExitDir / astToStr(identifier)
+  let testDir = OpVoluntaryExitDir / identifier
 
   proc `testImpl _ voluntary_exit _ identifier`() =
 
@@ -39,7 +39,7 @@ template runTest(identifier: untyped) =
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & astToStr(identifier):
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       var voluntaryExit: ref SignedVoluntaryExit
       new voluntaryExit
@@ -64,18 +64,5 @@ template runTest(identifier: untyped) =
   `testImpl _ voluntary_exit _ identifier`()
 
 suite "Official - Operations - Voluntary exit " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/voluntary_exit/pyspec_tests
+  for kind, path in walkDir(OpVoluntaryExitDir, true):
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/voluntary_exit/pyspec_tests
+    runTest(path)
-  runTest(success)
-
-  when false:
-    # TODO not sure how this particularly could falsely succeed
-    runTest(invalid_signature)
-
-  runTest(validator_invalid_validator_index)
-  runTest(validator_already_exited)
-  runTest(success_exit_queue)
-  runTest(validator_exit_in_future)
-  runTest(default_exit_epoch_subsequent_exit)
-  runTest(validator_not_active_long_enough)
-  runTest(validator_not_active)