status-im/eth2.0-specs
2
0
Fork 0
mirror of https://github.com/status-im/eth2.0-specs.git synced 2025-02-02 22:03:37 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3111 from ethereum/testgen-fix

Browse Source 
Fix --preset-list argument and enhance error output
This commit is contained in:
Hsiao-Wei Wang 2022-11-18 17:18:39 +08:00 committed by GitHub
commit a1d259addf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py
View File

@ -4,7 +4,6 @@ import time
import shutil
import argparse
from pathlib import Path
from filelock import FileLock
import sys
import json
from typing import Iterable, AnyStr, Any, Callable
@ -13,6 +12,7 @@ from ruamel.yaml import (
YAML,
)
from filelock import FileLock
from snappy import compress
from eth2spec.test import context
@ -141,6 +141,10 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
tprov.prepare()
for test_case in tprov.make_cases():
# If preset list is assigned, filter by presets.
if len(presets) != 0 and test_case.preset_name not in presets:
continue
case_dir = (
Path(output_dir) / Path(test_case.preset_name) / Path(test_case.fork_name)
/ Path(test_case.runner_name) / Path(test_case.handler_name)
@ -179,7 +183,16 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
try:
fn(case_dir)
except IOError as e:
sys.exit(f'Error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}')
error_message = (
f'[Error] error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}'
)
# Write to error log file
with log_file.open("a+") as f:
f.write(error_message)
traceback.print_exc(file=f)
f.write('\n')
sys.exit(error_message)
meta = dict()
@ -210,13 +223,13 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
if not written_part:
print(f"test case {case_dir} did not produce any test case parts")
except Exception as e:
print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}")
traceback.print_exc()
# Write to log file
error_message = f"[ERROR] failed to generate vector(s) for test {case_dir}: {e}"
# Write to error log file
with log_file.open("a+") as f:
f.write(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}")
f.write(error_message)
traceback.print_exc(file=f)
f.write('\n')
traceback.print_exc()
else:
# If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder.
if not written_part:

4
tests/generators/README.md
View File

@ -186,7 +186,7 @@ if __name__ == "__main__":
ALTAIR: altair_mods,
}
run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods)
run_state_test_generators(runner_name="sanity", all_mods=all_mods)
```
Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `eth2spec.gen_helpers.gen_from_tests.gen.run_state_test_generators` utility. Note that this helper generates all available tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner.
@ -210,7 +210,7 @@ To add a new test generator that builds `New Tests`:
with any dependencies it may need. Leave it empty if your generator has none.
3. Your generator is assumed to have a `main.py` file in its root.
By adding the base generator to your requirements, you can make a generator really easily. See docs below.
4. Your generator is called with `-o some/file/path/for_testing/can/be_anything -c some/other/path/to_configs/`.
4. Your generator is called with `-o some/file/path/for_testing/can/be_anything --preset-list mainnet minimal`.
The base generator helps you handle this; you only have to define test case providers.
5. Finally, add any linting or testing commands to the
[circleci config file](../../.circleci/config.yml) if desired to increase code quality.