status-im/eth2.0-specs
2
0
Fork 0
mirror of https://github.com/status-im/eth2.0-specs.git synced 2025-02-02 22:03:37 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3111 from ethereum/testgen-fix

Browse Source 
Fix --preset-list argument and enhance error output
This commit is contained in:
Hsiao-Wei Wang 2022-11-18 17:18:39 +08:00 committed by GitHub
commit a1d259addf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py
View File

@ -4,7 +4,6 @@ import time
import shutil import shutil
import argparse import argparse
from pathlib import Path from pathlib import Path
from filelock import FileLock
import sys import sys
import json import json
from typing import Iterable, AnyStr, Any, Callable from typing import Iterable, AnyStr, Any, Callable
@ -13,6 +12,7 @@ from ruamel.yaml import (
YAML, YAML,
) )
from filelock import FileLock
from snappy import compress from snappy import compress
from eth2spec.test import context from eth2spec.test import context
@ -141,6 +141,10 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
tprov.prepare() tprov.prepare()
for test_case in tprov.make_cases(): for test_case in tprov.make_cases():
# If preset list is assigned, filter by presets.
if len(presets) != 0 and test_case.preset_name not in presets:
continue
case_dir = ( case_dir = (
Path(output_dir) / Path(test_case.preset_name) / Path(test_case.fork_name) Path(output_dir) / Path(test_case.preset_name) / Path(test_case.fork_name)
/ Path(test_case.runner_name) / Path(test_case.handler_name) / Path(test_case.runner_name) / Path(test_case.handler_name)
@ -179,7 +183,16 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
try: try:
fn(case_dir) fn(case_dir)
except IOError as e: except IOError as e:
sys.exit(f'Error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}') error_message = (
f'[Error] error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}'
)
# Write to error log file
with log_file.open("a+") as f:
f.write(error_message)
traceback.print_exc(file=f)
f.write('\n')
sys.exit(error_message)
meta = dict() meta = dict()
@ -210,13 +223,13 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
if not written_part: if not written_part:
print(f"test case {case_dir} did not produce any test case parts") print(f"test case {case_dir} did not produce any test case parts")
except Exception as e: except Exception as e:
print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") error_message = f"[ERROR] failed to generate vector(s) for test {case_dir}: {e}"
traceback.print_exc() # Write to error log file
# Write to log file
with log_file.open("a+") as f: with log_file.open("a+") as f:
f.write(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") f.write(error_message)
traceback.print_exc(file=f) traceback.print_exc(file=f)
f.write('\n') f.write('\n')
traceback.print_exc()
else: else:
# If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder. # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder.
if not written_part: if not written_part:

4
tests/generators/README.md
View File

@ -186,7 +186,7 @@ if __name__ == "__main__":
ALTAIR: altair_mods, ALTAIR: altair_mods,
} }
run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods) run_state_test_generators(runner_name="sanity", all_mods=all_mods)
``` ```
Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `eth2spec.gen_helpers.gen_from_tests.gen.run_state_test_generators` utility. Note that this helper generates all available tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner. Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `eth2spec.gen_helpers.gen_from_tests.gen.run_state_test_generators` utility. Note that this helper generates all available tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner.
@ -210,7 +210,7 @@ To add a new test generator that builds `New Tests`:
with any dependencies it may need. Leave it empty if your generator has none. with any dependencies it may need. Leave it empty if your generator has none.
3. Your generator is assumed to have a `main.py` file in its root. 3. Your generator is assumed to have a `main.py` file in its root.
By adding the base generator to your requirements, you can make a generator really easily. See docs below. By adding the base generator to your requirements, you can make a generator really easily. See docs below.
4. Your generator is called with `-o some/file/path/for_testing/can/be_anything -c some/other/path/to_configs/`. 4. Your generator is called with `-o some/file/path/for_testing/can/be_anything --preset-list mainnet minimal`.
The base generator helps you handle this; you only have to define test case providers. The base generator helps you handle this; you only have to define test case providers.
5. Finally, add any linting or testing commands to the 5. Finally, add any linting or testing commands to the
[circleci config file](../../.circleci/config.yml) if desired to increase code quality. [circleci config file](../../.circleci/config.yml) if desired to increase code quality.