finish conversion of manual iteration to automatic iteration over EF test vectors per section

tests/official/test_fixture_operations_deposits.nim

@@ -20,13 +20,13 @@ import
 
 const OperationsDepositsDir = SszTestsDir/const_preset/"phase0"/"operations"/"deposit"/"pyspec_tests"
 
-template runTest(testName: string, identifier: untyped) =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = OperationsDepositsDir / astToStr(identifier)
+  let testDir = OperationsDepositsDir / identifier
 
   proc `testImpl _ operations_deposits _ identifier`() =
 
@@ -39,7 +39,7 @@ template runTest(testName: string, identifier: untyped) =
     else:
       prefix = "[Invalid] "
 
-    timedTest prefix & testName & " (" & astToStr(identifier) & ")":
+    timedTest prefix & " " & identifier:
       var stateRef, postRef: ref BeaconState
       var depositRef: ref Deposit
       new depositRef
@@ -61,19 +61,10 @@ template runTest(testName: string, identifier: untyped) =
   `testImpl _ operations_deposits _ identifier`()
 
 suite "Official - Operations - Deposits " & preset():
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/deposit/pyspec_tests
-  # https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/deposit/pyspec_tests
-  runTest("bad merkle proof", bad_merkle_proof)
-  runTest("invalid signature new deposit", invalid_sig_new_deposit)
-  runTest("invalid signature other version", invalid_sig_other_version)
-  runTest("invalid signature top-up", invalid_sig_top_up)
-  runTest("invalid withdrawal credentials top-up",
-    invalid_withdrawal_credentials_top_up)
-  runTest("new deposit max", new_deposit_max)
-  runTest("new deposit over max", new_deposit_over_max)
-  runTest("new deposit under max", new_deposit_under_max)
-  runTest("success top-up", success_top_up)
-  when false:
-    # TODO
-    runTest("valid signature but forked state", valid_sig_but_forked_state)
-  runTest("wrong deposit for deposit count", wrong_deposit_for_deposit_count)
+  # TODO
+  const expected_failures = ["valid_sig_but_forked_state"]
+
+  for kind, path in walkDir(OperationsDepositsDir, true):
+    if path in expected_failures:
+      continue
+    runTest(path)

tests/official/test_fixture_sanity_blocks.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  os, unittest,
+  os, sequtils, unittest,
   # Beacon chain internals
   ../../beacon_chain/spec/[crypto, datatypes],
   ../../beacon_chain/[ssz, state_transition, extras],
@@ -20,13 +20,13 @@ import
 
 const SanityBlocksDir = SszTestsDir/const_preset/"phase0"/"sanity"/"blocks"/"pyspec_tests"
 
-template runTest(identifier: string, num_blocks: int): untyped =
+proc runTest(identifier: string) =
   # We wrap the tests in a proc to avoid running out of globals
   # in the future: Nim supports up to 3500 globals
   # but unittest with the macro/templates put everything as globals
   # https://github.com/nim-lang/Nim/issues/12084#issue-486866402
 
-  const testDir = SanityBlocksDir / identifier
+  let testDir = SanityBlocksDir / identifier
 
   proc `testImpl _ blck _ identifier`() =
     let prefix = if existsFile(testDir/"post.ssz"):
@@ -43,7 +43,9 @@ template runTest(identifier: string, num_blocks: int): untyped =
         new postRef
         postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
 
-      for i in 0 ..< num_blocks:
+      # In test cases with more than 10 blocks the first 10 aren't 0-prefixed,
+      # so purely lexicographic sorting wouldn't sort properly.
+      for i in 0 ..< toSeq(walkPattern(testDir/"blocks_*.ssz")).len:
         let blck = parseTest(testDir/"blocks_" & $i & ".ssz", SSZ, SignedBeaconBlock)
 
         if postRef.isNil:
@@ -54,43 +56,18 @@ template runTest(identifier: string, num_blocks: int): untyped =
           let success = state_transition(stateRef[], blck.message, flags = {skipValidation})
           doAssert success, "Failure when applying block " & $i
 
-          # Checks:
-          # check: stateRef.hash_tree_root() == postRef.hash_tree_root()
-          if i == num_blocks - 1: reportDiff(stateRef, postRef)
+      # check: stateRef.hash_tree_root() == postRef.hash_tree_root()
+      if not postRef.isNil:
+        reportDiff(stateRef, postRef)
 
   `testImpl _ blck _ identifier`()
 
 suite "Official - Sanity - Blocks " & preset():
+  # Failing due to signature checking in indexed validation checking pending
+  # 0.10 BLS verification API with new domain handling.
   const expected_failures = ["attester_slashing"]
-  runTest("attestation", 2)
 
-  when false:
-    # Failing due to signature checking in indexed validation checking pending
-    # 0.10 BLS verification API with new domain handling.
-    runTest("attester_slashing", 1)
-  echo "Skipping test: attester_slashing"
-
-  runTest("balance_driven_status_transitions", 1)
-  runTest("deposit_in_block", 1)
-  runTest("deposit_top_up", 1)
-  runTest("empty_block_transition", 1)
-  runTest("empty_epoch_transition", 1)
-
-  when const_preset=="minimal":
-    runTest("empty_epoch_transition_not_finalizing", 1)
-    runTest("eth1_data_votes_consensus", 17)
-    runTest("eth1_data_votes_no_consensus", 16)
-
-  runTest("expected_deposit_in_block", 1)
-  runTest("high_proposer_index", 1)
-  runTest("historical_batch", 1)
-  runTest("invalid_block_sig", 1)
-  runTest("invalid_state_root", 1)
-  runTest("prev_slot_block_transition", 1)
-  runTest("proposer_after_inactive_index", 1)
-  runTest("proposer_slashing", 1)
-  runTest("same_slot_block_transition", 1)
-  runTest("skipped_slots", 1)
-  runTest("voluntary_exit", 2)
-  when const_preset=="minimal":
-    runTest("zero_block_sig", 1)
+  for kind, path in walkDir(SanityBlocksDir, true):
+    if path in expected_failures:
+      continue
+    runTest(path)