Merge pull request #2247 from ethereum/gen_runner_log_and_incomplete_tag

Enhance gen_runner error handler + check with `INCOMPLETE` temporary file

Makefile

@@ -27,6 +27,7 @@ COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html
 
 CURRENT_DIR = ${CURDIR}
 LINTER_CONFIG_FILE = $(CURRENT_DIR)/linter.ini
+GENERATOR_ERROR_LOG_FILE = $(CURRENT_DIR)/$(TEST_VECTOR_DIR)/testgen_error_log.txt
 
 export DAPP_SKIP_BUILD:=1
 export DAPP_SRC:=$(SOLIDITY_DEPOSIT_CONTRACT_DIR)
@@ -35,7 +36,8 @@ export DAPP_JSON:=build/combined.json
 
 .PHONY: clean partial_clean all test citest lint generate_tests pyspec install_test open_cov \
         install_deposit_contract_tester test_deposit_contract install_deposit_contract_compiler \
-        compile_deposit_contract test_compile_deposit_contract check_toc
+        compile_deposit_contract test_compile_deposit_contract check_toc \
+        detect_generator_incomplete detect_generator_error_log
 
 all: $(PY_SPEC_ALL_TARGETS)
 
@@ -171,3 +173,9 @@ $(TEST_VECTOR_DIR)/:
 # (creation of output dir is a dependency)
 gen_%: $(TEST_VECTOR_DIR)
 	$(call run_generator,$*)
+
+detect_generator_incomplete: $(TEST_VECTOR_DIR)
+	find $(TEST_VECTOR_DIR) -name "INCOMPLETE"
+
+detect_generator_error_log: $(TEST_VECTOR_DIR)
+	[ -f $(GENERATOR_ERROR_LOG_FILE) ] && echo "[ERROR] $(GENERATOR_ERROR_LOG_FILE) file exists" || echo "[PASSED] error log file does not exist"

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py

@@ -1,3 +1,5 @@
+import os
+import shutil
 import argparse
 from pathlib import Path
 import sys
@@ -102,8 +104,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
     yaml = YAML(pure=True)
     yaml.default_flow_style = None
 
+    log_file = Path(output_dir) / 'testgen_error_log.txt'
+
     print(f"Generating tests into {output_dir}")
     print(f"Reading configs from {args.configs_path}")
+    print(f'Error log file: {log_file}')
 
     configs = args.config_list
     if configs is None:
@@ -126,14 +131,27 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
                 / Path(test_case.runner_name) / Path(test_case.handler_name)
                 / Path(test_case.suite_name) / Path(test_case.case_name)
             )
+            incomplete_tag_file = case_dir / "INCOMPLETE"
+
             if case_dir.exists():
-                if not args.force:
+                if not args.force and not incomplete_tag_file.exists():
                     print(f'Skipping already existing test: {case_dir}')
                     continue
-                print(f'Warning, output directory {case_dir} already exist,'
-                      f' old files are not deleted but will be overwritten when a new version is produced')
+                else:
+                    print(f'Warning, output directory {case_dir} already exist,'
+                          f' old files will be deleted and it will generate test vector files with the latest version')
+                    # Clear the existing case_dir folder
+                    shutil.rmtree(case_dir)
 
             print(f'Generating test: {case_dir}')
+
+            written_part = False
+
+            # Add `INCOMPLETE` tag file to indicate that the test generation has not completed.
+            case_dir.mkdir(parents=True, exist_ok=True)
+            with incomplete_tag_file.open("w") as f:
+                f.write("\n")
+
             try:
                 def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]):
                     # make sure the test case directory is created before any test part is written.
@@ -143,7 +161,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
                     except IOError as e:
                         sys.exit(f'Error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}')
 
-                written_part = False
                 meta = dict()
 
                 try:
@@ -157,6 +174,7 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
                             output_part("ssz", name, dump_ssz_fn(data, name, file_mode))
                 except SkippedTest as e:
                     print(e)
+                    shutil.rmtree(case_dir)
                     continue
 
                 # Once all meta data is collected (if any), write it to a meta data file.
@@ -166,10 +184,22 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
 
                 if not written_part:
                     print(f"test case {case_dir} did not produce any test case parts")
-
             except Exception as e:
                 print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}")
                 traceback.print_exc()
+                # Write to log file
+                with log_file.open("a+") as f:
+                    f.write(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}")
+                    traceback.print_exc(file=f)
+                    f.write('\n')
+            else:
+                # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder.
+                if not written_part:
+                    shutil.rmtree(case_dir)
+                else:
+                    # Only remove `INCOMPLETE` tag file
+                    os.remove(incomplete_tag_file)
+
     print(f"completed {generator_name}")
 
 

tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py

@@ -284,6 +284,7 @@ def test_invalid_signature_previous_committee(spec, state):
 @with_all_phases_except([PHASE0, PHASE1])
 @spec_state_test
 @always_bls
+@with_configs([MINIMAL], reason="too slow")
 def test_valid_signature_future_committee(spec, state):
     # NOTE: the `state` provided is at genesis and the process to select
     # sync committees currently returns the same committee for the first and second