Add gen-collect-only CI check

.github/workflows/run-tests.yml

@@ -24,7 +24,7 @@ on:
     - cron: '0 0 * * *'
 
 jobs:
-  table_of_contents:
+  table-of-contents:
    runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs]
    steps:
       - name: Checkout repository
@@ -71,7 +71,7 @@ jobs:
 
   pyspec-tests:
     runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs]
-    needs: [lint,codespell,table_of_contents]
+    needs: [lint, codespell, table-of-contents]
     strategy:
       matrix:
         version: ["phase0", "altair", "bellatrix", "capella", "deneb", "electra", "whisk", "eip7594"]
@@ -104,9 +104,24 @@ jobs:
       - name: Install pyspec requirements
         run: make install_test
       - name: test-${{ matrix.version }}
-        run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{env.spec_test_preset_type}}
+        run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{ env.spec_test_preset_type }}
       - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: test-reports-${{ matrix.version }}
           path: tests/core/pyspec/test-reports
+
+  gen-collect-only:
+   runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs]
+   steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12.4'
+          cache: ''
+      - name: Install pyspec requirements
+        run: make install_test
+      - name: Run generators with --collect-only
+        run: make generate_tests collect_only=true

Makefile

@@ -159,6 +159,9 @@ lint_generators: pyspec
 	. venv/bin/activate; cd $(TEST_GENERATORS_DIR); \
 	flake8 --config $(LINTER_CONFIG_FILE)
 
+# If set to true, it will not run generator tests.
+collect_only ?= false
+
 # Runs a generator, identified by param 1
 define run_generator
 	# Started!
@@ -177,7 +180,7 @@ define run_generator
 	. venv/bin/activate; \
 	pip3 install ../../../dist/eth2spec-*.whl; \
 	pip3 install 'eth2spec[generator]'; \
-	python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR); \
+	python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) $(if $(filter true,$(collect_only)),--collect-only); \
 	echo "generator $(1) finished"
 endef
 

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py

@@ -248,6 +248,10 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
             print(f"Collected test at: {case_dir}")
             diagnostics_obj.collected_test_count += 1
 
+            # Bail here if we do not want to generate.
+            if collect_only:
+                continue
+
             is_skip, diagnostics_obj = should_skip_case_dir(case_dir, args.force, diagnostics_obj)
             if is_skip:
                 continue