From 387113b2f4011c27d0cfc2beb57afb7666ce0c69 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 30 Aug 2021 13:36:01 -0700 Subject: [PATCH] add "collect only" mode to spec test generator --- .../gen_helpers/gen_base/gen_runner.py | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 765fc501b..6850af188 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -77,6 +77,13 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): required=False, help="specify presets to run with. Allows all if no preset names are specified.", ) + parser.add_argument( + "-c", + "--collect-only", + action="store_true", + default=False, + help="if set only print tests to generate, do not actually run the test and dump the target data", + ) args = parser.parse_args() output_dir = args.output_dir @@ -100,12 +107,15 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): if len(presets) != 0: print(f"Filtering test-generator runs to only include presets: {', '.join(presets)}") + collect_only = args.collect_only + collected_test_count = 0 generated_test_count = 0 skipped_test_count = 0 provider_start = time.time() for tprov in test_providers: - # runs anything that we don't want to repeat for every test case. - tprov.prepare() + if not collect_only: + # runs anything that we don't want to repeat for every test case. + tprov.prepare() for test_case in tprov.make_cases(): case_dir = ( @@ -115,6 +125,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): ) incomplete_tag_file = case_dir / "INCOMPLETE" + collected_test_count += 1 + if collect_only: + print(f"Collected test at: {case_dir}") + continue + if case_dir.exists(): if not args.force and not incomplete_tag_file.exists(): skipped_test_count += 1 @@ -193,11 +208,14 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): provider_end = time.time() span = round(provider_end - provider_start, 2) - summary_message = f"completed generation of {generator_name} with {generated_test_count} tests" - summary_message += f" ({skipped_test_count} skipped tests)" - if span > TIME_THRESHOLD_TO_PRINT: - summary_message += f" in {span} seconds" - print(summary_message) + if collect_only: + print(f"Collected {collected_test_count} tests in total") + else: + summary_message = f"completed generation of {generator_name} with {generated_test_count} tests" + summary_message += f" ({skipped_test_count} skipped tests)" + if span > TIME_THRESHOLD_TO_PRINT: + summary_message += f" in {span} seconds" + print(summary_message) def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML):