switch attestations, proposer slashings, and slots sanity tests to automatically iterate across all available test vectors rather than hard-coding them

tests/official/test_fixture_operations_attestations.nim

@@ -20,13 +20,13 @@ import
 
 const OperationsAttestationsDir = SszTestsDir/const_preset/"phase0"/"operations"/"attestation"/"pyspec_tests"
 
-template runTest(testName: string, identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OperationsAttestationsDir / astToStr(identifier)
+  let testDir = OperationsAttestationsDir / identifier
 
   proc `testImpl _ operations_attestations _ identifier`() =
 
@@ -39,7 +39,7 @@ template runTest(testName: string, identifier: untyped) =
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & testName & " (" & astToStr(identifier) & ")":
+    timedTest prefix & identifier:
       var stateRef, postRef: ref BeaconState
       var attestationRef: ref Attestation
       new attestationRef
@@ -66,27 +66,5 @@ template runTest(testName: string, identifier: untyped) =
   `testImpl _ operations_attestations _ identifier`()
 
 suite "Official - Operations - Attestations " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/attestation/pyspec_tests
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/attestation/pyspec_tests
-  runTest("after_epoch_slots", after_epoch_slots)
-  runTest("bad source root", bad_source_root)
-  runTest("before inclusion delay", before_inclusion_delay)
-  runTest("empty aggregation bits", empty_aggregation_bits)
-  runTest("future target epoch", future_target_epoch)
-  runTest("invalid attestation signature", invalid_attestation_signature)
-  runTest("invalid current source root", invalid_current_source_root)
-  runTest("invalid index", invalid_index)
-  runTest("mismatched target and slot", mismatched_target_and_slot)
-  runTest("new source epoch", new_source_epoch)
-  runTest("old source epoch", old_source_epoch)
-  runTest("old target epoch", old_target_epoch)
-  runTest("source root is target root", source_root_is_target_root)
-  runTest("success", success)
-  runTest("success multi-proposer index interations",
-    success_multi_proposer_index_iterations)
-  runTest("success previous epoch", success_previous_epoch)
-  runTest("too few aggregation bits", too_few_aggregation_bits)
-  runTest("too many aggregation bits", too_many_aggregation_bits)
-  runTest("wrong index for committee signature",
-    wrong_index_for_committee_signature)
-  runTest("wrong index for slot", wrong_index_for_slot)
+  for kind, path in walkDir(OperationsAttestationsDir, true):
+    runTest(path)

tests/official/test_fixture_operations_proposer_slashings.nim

@@ -20,13 +20,13 @@ import
 
 const OpProposerSlashingDir = SszTestsDir/const_preset/"phase0"/"operations"/"proposer_slashing"/"pyspec_tests"
 
-template runTest(identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OpProposerSlashingDir / astToStr(identifier)
+  let testDir = OpProposerSlashingDir / identifier
 
   proc `testImpl_proposer_slashing _ identifier`() =
 
@@ -66,15 +66,5 @@ template runTest(identifier: untyped) =
   `testImpl_proposer_slashing _ identifier`()
 
 suite "Official - Operations - Proposer slashing " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/proposer_slashing/pyspec_tests
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/proposer_slashing/pyspec_tests
-  runTest(epochs_are_different)
-  runTest(headers_are_same)
-  runTest(invalid_proposer_index)
-  runTest(invalid_sig_1)
-  runTest(invalid_sig_1_and_2)
-  runTest(invalid_sig_2)
-  runTest(proposer_is_not_activated)
-  runTest(proposer_is_slashed)
-  runTest(proposer_is_withdrawn)
-  runTest(success)
+  for kind, path in walkDir(OpProposerSlashingDir, true):
+    runTest(path)

tests/official/test_fixture_sanity_slots.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  os, unittest,
+  os, strutils, unittest,
   # Beacon chain internals
   ../../beacon_chain/spec/datatypes,
   ../../beacon_chain/state_transition,
@@ -20,16 +20,17 @@ import
 
 const SanitySlotsDir = SszTestsDir/const_preset/"phase0"/"sanity"/"slots"/"pyspec_tests"
 
-template runTest(testName: string, identifier: untyped, num_slots: uint64): untyped =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
-
-  const testDir = SanitySlotsDir / astToStr(identifier)
+  let
+    testDir = SanitySlotsDir / identifier
+    num_slots = readLines(testDir / "slots.yaml", 2)[0].parseInt.uint64
 
   proc `testImpl _ slots _ identifier`() =
-    timedTest "Slots - " & testName & " (" & astToStr(identifier) & ")":
+    timedTest "Slots - " & identifier:
       var stateRef, postRef: ref BeaconState
       new stateRef
       new postRef
@@ -43,18 +44,6 @@ template runTest(testName: string, identifier: untyped, num_slots: uint64): unty
 
   `testImpl _ slots _ identifier`()
 
-# 1 slot
-# ---------------------------------------------------------------
-
 suite "Official - Sanity - Slots " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/sanity/slots/pyspec_tests
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/sanity/slots/pyspec_tests
-  runTest("Advance 1 slot", slots_1, 1)
-  runTest("Advance 2 slots", slots_2, 2)
-  runTest("Advance an empty epoch", empty_epoch, SLOTS_PER_EPOCH)
-
-  const DoubleEpoch = SLOTS_PER_EPOCH.uint64*2 # workaround undeclared identifier "double_empty_epoch"
-  runTest("Advance 2 empty epochs", double_empty_epoch, DoubleEpoch)
-
-  # This starts in the middle of an epoch
-  runTest("Advance over an epoch boundary", over_epoch_boundary, SLOTS_PER_EPOCH)
+  for kind, path in walkDir(SanitySlotsDir, true):
+    runTest(path)