Merge pull request #2576 from ralexstokes/consolidate-like-operations-tests

Allow spec tests author to batch tests under one handler name

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py

@@ -77,6 +77,13 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
         required=False,
         help="specify presets to run with. Allows all if no preset names are specified.",
     )
+    parser.add_argument(
+        "-c",
+        "--collect-only",
+        action="store_true",
+        default=False,
+        help="if set only print tests to generate, do not actually run the test and dump the target data",
+    )
 
     args = parser.parse_args()
     output_dir = args.output_dir
@@ -100,12 +107,15 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
     if len(presets) != 0:
         print(f"Filtering test-generator runs to only include presets: {', '.join(presets)}")
 
+    collect_only = args.collect_only
+    collected_test_count = 0
     generated_test_count = 0
     skipped_test_count = 0
     provider_start = time.time()
     for tprov in test_providers:
-        # runs anything that we don't want to repeat for every test case.
+        if not collect_only:
-        tprov.prepare()
+            # runs anything that we don't want to repeat for every test case.
+            tprov.prepare()
 
         for test_case in tprov.make_cases():
             case_dir = (
@@ -115,6 +125,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
             )
             incomplete_tag_file = case_dir / "INCOMPLETE"
 
+            collected_test_count += 1
+            if collect_only:
+                print(f"Collected test at: {case_dir}")
+                continue
+
             if case_dir.exists():
                 if not args.force and not incomplete_tag_file.exists():
                     skipped_test_count += 1
@@ -193,11 +208,14 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
     provider_end = time.time()
     span = round(provider_end - provider_start, 2)
 
-    summary_message = f"completed generation of {generator_name} with {generated_test_count} tests"
+    if collect_only:
-    summary_message += f" ({skipped_test_count} skipped tests)"
+        print(f"Collected {collected_test_count} tests in total")
-    if span > TIME_THRESHOLD_TO_PRINT:
+    else:
-        summary_message += f" in {span} seconds"
+        summary_message = f"completed generation of {generator_name} with {generated_test_count} tests"
-    print(summary_message)
+        summary_message += f" ({skipped_test_count} skipped tests)"
+        if span > TIME_THRESHOLD_TO_PRINT:
+            summary_message += f" in {span} seconds"
+        print(summary_message)
 
 
 def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML):

tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py

@@ -1,6 +1,6 @@
 from importlib import import_module
 from inspect import getmembers, isfunction
-from typing import Any, Callable, Dict, Iterable, Optional
+from typing import Any, Callable, Dict, Iterable, Optional, List, Union
 
 from eth2spec.utils import bls
 from eth2spec.test.helpers.constants import ALL_PRESETS, TESTGEN_FORKS
@@ -59,8 +59,10 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any,
 def get_provider(create_provider_fn: Callable[[SpecForkName, PresetBaseName, str, str], TestProvider],
                  fork_name: SpecForkName,
                  preset_name: PresetBaseName,
-                 all_mods: Dict[str, Dict[str, str]]) -> Iterable[TestProvider]:
+                 all_mods: Dict[str, Dict[str, Union[List[str], str]]]) -> Iterable[TestProvider]:
     for key, mod_name in all_mods[fork_name].items():
+        if not isinstance(mod_name, List):
+            mod_name = [mod_name]
         yield create_provider_fn(
             fork_name=fork_name,
             preset_name=preset_name,
@@ -75,16 +77,17 @@ def get_create_provider_fn(runner_name: str) -> Callable[[SpecForkName, str, str
         return
 
     def create_provider(fork_name: SpecForkName, preset_name: PresetBaseName,
-                        handler_name: str, tests_src_mod_name: str) -> TestProvider:
+                        handler_name: str, tests_src_mod_name: List[str]) -> TestProvider:
         def cases_fn() -> Iterable[TestCase]:
-            tests_src = import_module(tests_src_mod_name)
+            for mod_name in tests_src_mod_name:
-            return generate_from_tests(
+                tests_src = import_module(mod_name)
-                runner_name=runner_name,
+                yield from generate_from_tests(
-                handler_name=handler_name,
+                    runner_name=runner_name,
-                src=tests_src,
+                    handler_name=handler_name,
-                fork_name=fork_name,
+                    src=tests_src,
-                preset_name=preset_name,
+                    fork_name=fork_name,
-            )
+                    preset_name=preset_name,
+                )
 
         return TestProvider(prepare=prepare_fn, make_cases=cases_fn)
     return create_provider

tests/generators/operations/main.py

@@ -12,9 +12,9 @@ if __name__ == "__main__":
         'voluntary_exit',
     ]}
     altair_mods = {
-        **{key: 'eth2spec.test.altair.block_processing.sync_aggregate.test_process_' + key for key in [
+        **{'sync_aggregate': [
-            'sync_aggregate',
+            'eth2spec.test.altair.block_processing.sync_aggregate.test_process_' + key
-            'sync_aggregate_random',
+            for key in ['sync_aggregate', 'sync_aggregate_random']
         ]},
         **phase_0_mods,
     }  # also run the previous phase 0 tests