Merge pull request #3986 from jtraglia/gen-collect-only

Add `gen-modcheck` CI check

.github/workflows/run-tests.yml

@@ -110,3 +110,18 @@ jobs:
         with:
           name: test-reports-${{ matrix.version }}
           path: tests/core/pyspec/test-reports
+
+  gen-modcheck:
+   runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs]
+   steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12.4'
+          cache: ''
+      - name: Install pyspec requirements
+        run: make install_test
+      - name: Run generators with --modcheck
+        run: make generate_tests modcheck=true

Makefile

@@ -158,6 +158,9 @@ lint_generators: pyspec
 	. venv/bin/activate; cd $(TEST_GENERATORS_DIR); \
 	flake8 --config $(CURRENT_DIR)/flake8.ini
 
+# If set to true, it will not run generator tests.
+modcheck ?= false
+
 # Runs a generator, identified by param 1
 define run_generator
 	# Started!
@@ -176,7 +179,7 @@ define run_generator
 	. venv/bin/activate; \
 	pip3 install ../../../dist/eth2spec-*.whl; \
 	pip3 install 'eth2spec[generator]'; \
-	python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR); \
+	python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) $(if $(filter true,$(modcheck)),--modcheck); \
 	echo "generator $(1) finished"
 endef
 

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py

@@ -194,14 +194,17 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
         help="specify forks to run with. Allows all if no fork names are specified.",
     )
     parser.add_argument(
-        "-c",
-        "--collect-only",
+        "--modcheck",
         action="store_true",
         default=False,
-        help="if set only print tests to generate, do not actually run the test and dump the target data",
+        help="check generator modules, do not run any tests.",
     )
-
     args = parser.parse_args()
+
+    # Bail here if we are checking modules.
+    if args.modcheck:
+        return
+
     output_dir = args.output_dir
     if not args.force:
         file_mode = "x"
@@ -229,8 +232,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
     if len(presets) != 0:
         print(f"Filtering test-generator runs to only include forks: {', '.join(forks)}")
 
-    collect_only = args.collect_only
-
     diagnostics_obj = Diagnostics()
     provider_start = time.time()
 
@@ -238,9 +239,8 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
         all_test_case_params = []
 
     for tprov in test_providers:
-        if not collect_only:
-            # runs anything that we don't want to repeat for every test case.
-            tprov.prepare()
+        # Runs anything that we don't want to repeat for every test case.
+        tprov.prepare()
 
         for test_case in tprov.make_cases():
             # If preset list is assigned, filter by presets.
@@ -276,14 +276,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
     provider_end = time.time()
     span = round(provider_end - provider_start, 2)
 
-    if collect_only:
-        print(f"Collected {diagnostics_obj.collected_test_count} tests in total")
-    else:
-        summary_message = f"completed generation of {generator_name} with {diagnostics_obj.generated_test_count} tests"
-        summary_message += f" ({diagnostics_obj.skipped_test_count} skipped tests)"
-        if span > TIME_THRESHOLD_TO_PRINT:
-            summary_message += f" in {span} seconds"
-        print(summary_message)
+    summary_message = f"completed generation of {generator_name} with {diagnostics_obj.generated_test_count} tests"
+    summary_message += f" ({diagnostics_obj.skipped_test_count} skipped tests)"
+    if span > TIME_THRESHOLD_TO_PRINT:
+        summary_message += f" in {span} seconds"
+    print(summary_message)
 
     diagnostics_output = {
         "collected_test_count": diagnostics_obj.collected_test_count,