From fb82472b46414b1b93dbd875beb0a58b4ead099e Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 May 2021 18:29:24 +0200 Subject: [PATCH] update test generators --- Makefile | 2 +- .../gen_helpers/gen_base/gen_runner.py | 47 ++++------------- .../gen_helpers/gen_base/gen_typing.py | 6 +-- .../gen_helpers/gen_from_tests/gen.py | 52 ++++++++++--------- tests/core/pyspec/eth2spec/test/context.py | 2 +- .../pyspec/eth2spec/test/helpers/constants.py | 2 +- tests/generators/bls/main.py | 5 +- tests/generators/epoch_processing/main.py | 8 +-- tests/generators/finality/main.py | 8 +-- tests/generators/fork_choice/main.py | 8 +-- tests/generators/forks/main.py | 25 +++++---- tests/generators/genesis/main.py | 7 +-- tests/generators/operations/main.py | 8 +-- tests/generators/rewards/main.py | 8 +-- tests/generators/sanity/main.py | 9 +--- tests/generators/shuffling/main.py | 34 ++++++------ tests/generators/ssz_generic/main.py | 5 +- tests/generators/ssz_static/main.py | 32 ++++-------- tests/generators/transition/main.py | 14 ++--- 19 files changed, 102 insertions(+), 180 deletions(-) diff --git a/Makefile b/Makefile index 511100e75..007f49f5b 100644 --- a/Makefile +++ b/Makefile @@ -167,7 +167,7 @@ define run_generator if ! test -d venv; then python3 -m venv venv; fi; \ . venv/bin/activate; \ pip3 install -r requirements.txt; \ - python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) -c $(CURRENT_DIR)/$(CONFIGS_DIR); \ + python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR); \ echo "generator $(1) finished" endef diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index e4f1318ae..2b02d1b5c 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -34,18 +34,6 @@ def validate_output_dir(path_str): return path -def validate_configs_dir(path_str): - path = Path(path_str) - - if not path.exists(): - raise argparse.ArgumentTypeError("Configs directory must exist") - - if not path.is_dir(): - raise argparse.ArgumentTypeError("Config path must lead to a directory") - - return path - - def run_generator(generator_name, test_providers: Iterable[TestProvider]): """ Implementation for a general test generator. @@ -76,22 +64,14 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): default=False, help="if set re-generate and overwrite test files if they already exist", ) - parser.add_argument( - "-c", - "--configs-path", - dest="configs_path", - required=True, - type=validate_configs_dir, - help="specify the path of the configs directory", - ) parser.add_argument( "-l", - "--config-list", - dest="config_list", + "--preset-list", + dest="preset_list", nargs='*', type=str, required=False, - help="specify configs to run with. Allows all if no config names are specified.", + help="specify presets to run with. Allows all if no preset names are specified.", ) args = parser.parse_args() @@ -107,27 +87,22 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): log_file = Path(output_dir) / 'testgen_error_log.txt' print(f"Generating tests into {output_dir}") - print(f"Reading configs from {args.configs_path}") print(f'Error log file: {log_file}') - configs = args.config_list - if configs is None: - configs = [] + presets = args.preset_list + if presets is None: + presets = [] - if len(configs) != 0: - print(f"Filtering test-generator runs to only include configs: {', '.join(configs)}") + if len(presets) != 0: + print(f"Filtering test-generator runs to only include presets: {', '.join(presets)}") for tprov in test_providers: - # loads configuration etc. - config_name = tprov.prepare(args.configs_path) - if len(configs) != 0 and config_name not in configs: - print(f"skipping tests with config '{config_name}' since it is filtered out") - continue + # runs anything that we don't want to repeat for every test case. + tprov.prepare() - print(f"generating tests with config '{config_name}' ...") for test_case in tprov.make_cases(): case_dir = ( - Path(output_dir) / Path(config_name) / Path(test_case.fork_name) + Path(output_dir) / Path(test_case.preset_name) / Path(test_case.fork_name) / Path(test_case.runner_name) / Path(test_case.handler_name) / Path(test_case.suite_name) / Path(test_case.case_name) ) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py index 34bd71db1..669238d1c 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py @@ -19,6 +19,7 @@ TestCasePart = NewType("TestCasePart", Tuple[str, str, Any]) @dataclass class TestCase(object): fork_name: str + preset_name: str runner_name: str handler_name: str suite_name: str @@ -28,8 +29,7 @@ class TestCase(object): @dataclass class TestProvider(object): - # Prepares the context with a configuration, loaded from the given config path. - # fn(config path) => chosen config name - prepare: Callable[[str], str] + # Prepares the context for the provider as a whole, as opposed to per-test-case changes. + prepare: Callable[[], None] # Retrieves an iterable of cases, called after prepare() make_cases: Callable[[], Iterable[TestCase]] diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index bc63b212b..db3f9e949 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -1,18 +1,18 @@ -from importlib import reload, import_module +from importlib import import_module from inspect import getmembers, isfunction from typing import Any, Callable, Dict, Iterable, Optional -from eth2spec.config import config_util from eth2spec.utils import bls -from eth2spec.test.helpers.constants import ALL_CONFIGS, TESTGEN_FORKS -from eth2spec.test.helpers.typing import SpecForkName, ConfigName +from eth2spec.test.helpers.constants import ALL_PRESETS, TESTGEN_FORKS +from eth2spec.test.helpers.typing import SpecForkName, PresetBaseName from eth2spec.gen_helpers.gen_base import gen_runner from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider def generate_from_tests(runner_name: str, handler_name: str, src: Any, - fork_name: SpecForkName, bls_active: bool = True, + fork_name: SpecForkName, preset_name: PresetBaseName, + bls_active: bool = True, phase: Optional[str]=None) -> Iterable[TestCase]: """ Generate a list of test cases by running tests from the given src in generator-mode. @@ -21,8 +21,10 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, :param src: to retrieve tests from (discovered using inspect.getmembers). :param fork_name: the folder name for these tests. (if multiple forks are applicable, indicate the last fork) + :param preset_name: to select a preset. Tests that do not support the preset will be skipped. :param bls_active: optional, to override BLS switch preference. Defaults to True. :param phase: optional, to run tests against a particular spec version. Default to `fork_name` value. + Set to the pre-fork (w.r.t. fork_name) in multi-fork tests. :return: an iterable of test cases. """ fn_names = [ @@ -44,40 +46,36 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, yield TestCase( fork_name=fork_name, + preset_name=preset_name, runner_name=runner_name, handler_name=handler_name, suite_name='pyspec_tests', case_name=case_name, # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent. - case_fn=lambda: tfn(generator_mode=True, phase=phase, bls_active=bls_active) + case_fn=lambda: tfn(generator_mode=True, phase=phase, preset=preset_name, bls_active=bls_active) ) -def get_provider(create_provider_fn: Callable[[SpecForkName, str, str, ConfigName], TestProvider], - config_name: ConfigName, +def get_provider(create_provider_fn: Callable[[SpecForkName, PresetBaseName, str, str], TestProvider], fork_name: SpecForkName, + preset_name: PresetBaseName, all_mods: Dict[str, Dict[str, str]]) -> Iterable[TestProvider]: for key, mod_name in all_mods[fork_name].items(): yield create_provider_fn( fork_name=fork_name, + preset_name=preset_name, handler_name=key, tests_src_mod_name=mod_name, - config_name=config_name, ) -def get_create_provider_fn( - runner_name: str, config_name: ConfigName, specs: Iterable[Any] -) -> Callable[[SpecForkName, str, str, ConfigName], TestProvider]: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - for spec in specs: - reload(spec) +def get_create_provider_fn(runner_name: str) -> Callable[[SpecForkName, str, str, PresetBaseName], TestProvider]: + def prepare_fn() -> None: bls.use_milagro() - return config_name + return - def create_provider(fork_name: SpecForkName, handler_name: str, - tests_src_mod_name: str, config_name: ConfigName) -> TestProvider: + def create_provider(fork_name: SpecForkName, preset_name: PresetBaseName, + handler_name: str, tests_src_mod_name: str) -> TestProvider: def cases_fn() -> Iterable[TestCase]: tests_src = import_module(tests_src_mod_name) return generate_from_tests( @@ -85,22 +83,26 @@ def get_create_provider_fn( handler_name=handler_name, src=tests_src, fork_name=fork_name, + preset_name=preset_name, ) return TestProvider(prepare=prepare_fn, make_cases=cases_fn) return create_provider -def run_state_test_generators(runner_name: str, specs: Iterable[Any], all_mods: Dict[str, Dict[str, str]]) -> None: +def run_state_test_generators(runner_name: str, + all_mods: Dict[str, Dict[str, str]], + presets: Iterable[PresetBaseName] = ALL_PRESETS, + forks: Iterable[SpecForkName] = TESTGEN_FORKS) -> None: """ - Generate all available state tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner. + Generate all available state tests of `TESTGEN_FORKS` forks of `ALL_PRESETS` presets of the given runner. """ - for config_name in ALL_CONFIGS: - for fork_name in TESTGEN_FORKS: + for preset_name in presets: + for fork_name in forks: if fork_name in all_mods: gen_runner.run_generator(runner_name, get_provider( - create_provider_fn=get_create_provider_fn(runner_name, config_name, specs), - config_name=config_name, + create_provider_fn=get_create_provider_fn(runner_name), fork_name=fork_name, + preset_name=preset_name, all_mods=all_mods, )) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index e174d4841..362f64abf 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -414,7 +414,7 @@ def with_config_overrides(config_overrides): test_config = {k: config_types[k](v) for k, v in tmp_config.items()} # Output the config for test vectors (TODO: check config YAML encoding) - yield 'config', test_config + yield 'config', 'data', test_config spec.config = spec.Configuration(**test_config) diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index 2f7cb403a..4e98845c4 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -31,4 +31,4 @@ FORKS_BEFORE_MERGE = (PHASE0,) MAINNET = PresetBaseName('mainnet') MINIMAL = PresetBaseName('minimal') -ALL_CONFIGS = (MINIMAL, MAINNET) +ALL_PRESETS = (MINIMAL, MAINNET) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 3d67c9db3..3ebaa1354 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -358,10 +358,10 @@ def case05_aggregate_verify(): def create_provider(handler_name: str, test_case_fn: Callable[[], Iterable[Tuple[str, Dict[str, Any]]]]) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: + def prepare_fn() -> None: # Nothing to load / change in spec. Maybe in future forks. # Put the tests into the general config category, to not require any particular configuration. - return 'general' + return def cases_fn() -> Iterable[gen_typing.TestCase]: for data in test_case_fn(): @@ -369,6 +369,7 @@ def create_provider(handler_name: str, (case_name, case_content) = data yield gen_typing.TestCase( fork_name=PHASE0, + preset_name='general', runner_name='bls', handler_name=handler_name, suite_name='small', diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 2aa6381ff..a3d0f82be 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,13 +1,7 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.altair import spec as spec_altair -from eth2spec.merge import spec as spec_merge from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE -specs = (spec_phase0, spec_altair, spec_merge) - - if __name__ == "__main__": phase_0_mods = {key: 'eth2spec.test.phase0.epoch_processing.test_process_' + key for key in [ 'justification_and_finalization', @@ -45,4 +39,4 @@ if __name__ == "__main__": MERGE: merge_mods, } - run_state_test_generators(runner_name="epoch_processing", specs=specs, all_mods=all_mods) + run_state_test_generators(runner_name="epoch_processing", all_mods=all_mods) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index 148ddef96..39f454338 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,13 +1,7 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.altair import spec as spec_altair -from eth2spec.merge import spec as spec_merge from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE -specs = (spec_phase0, spec_altair, spec_merge) - - if __name__ == "__main__": phase_0_mods = {'finality': 'eth2spec.test.phase0.finality.test_finality'} altair_mods = phase_0_mods # No additional Altair specific finality tests @@ -19,4 +13,4 @@ if __name__ == "__main__": MERGE: spec_merge, } - run_state_test_generators(runner_name="finality", specs=specs, all_mods=all_mods) + run_state_test_generators(runner_name="finality", all_mods=all_mods) diff --git a/tests/generators/fork_choice/main.py b/tests/generators/fork_choice/main.py index ae15caa1d..f162d9564 100644 --- a/tests/generators/fork_choice/main.py +++ b/tests/generators/fork_choice/main.py @@ -1,13 +1,7 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.altair import spec as spec_altair -from eth2spec.merge import spec as spec_merge from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE -specs = (spec_phase0, spec_altair, spec_merge) - - if __name__ == "__main__": phase_0_mods = {key: 'eth2spec.test.phase0.fork_choice.test_' + key for key in [ 'get_head', @@ -23,4 +17,4 @@ if __name__ == "__main__": MERGE: merge_mods, } - run_state_test_generators(runner_name="fork_choice", specs=specs, all_mods=all_mods) + run_state_test_generators(runner_name="fork_choice", all_mods=all_mods) diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index 71a4a54de..a25eecec3 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -1,23 +1,19 @@ -from importlib import reload from typing import Iterable from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MINIMAL, MAINNET -from eth2spec.config import config_util -from eth2spec.test.altair.fork import test_fork as test_altair_forks -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.altair import spec as spec_altair +from eth2spec.test.helpers.typing import SpecForkName, PresetBaseName +from eth2spec.test.altair.fork import test_altair_fork_basic +from eth2spec.test.altair.fork import test_altair_fork_random from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests -def create_provider(tests_src, config_name: str, phase: str, fork_name: str) -> gen_typing.TestProvider: +def create_provider(tests_src, preset_name: PresetBaseName, + phase: SpecForkName, fork_name: SpecForkName) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_altair) - return config_name + def prepare_fn() -> None: + return def cases_fn() -> Iterable[gen_typing.TestCase]: return generate_from_tests( @@ -25,6 +21,7 @@ def create_provider(tests_src, config_name: str, phase: str, fork_name: str) -> handler_name='fork', src=tests_src, fork_name=fork_name, + preset_name=preset_name, phase=phase, ) @@ -33,6 +30,8 @@ def create_provider(tests_src, config_name: str, phase: str, fork_name: str) -> if __name__ == "__main__": gen_runner.run_generator("forks", [ - create_provider(test_altair_forks, MINIMAL, PHASE0, ALTAIR), - create_provider(test_altair_forks, MAINNET, PHASE0, ALTAIR), + create_provider(test_altair_fork_basic, MINIMAL, PHASE0, ALTAIR), + create_provider(test_altair_fork_basic, MAINNET, PHASE0, ALTAIR), + create_provider(test_altair_fork_random, MINIMAL, PHASE0, ALTAIR), + create_provider(test_altair_fork_random, MAINNET, PHASE0, ALTAIR), ]) diff --git a/tests/generators/genesis/main.py b/tests/generators/genesis/main.py index 3b4d045a3..8e0294bf0 100644 --- a/tests/generators/genesis/main.py +++ b/tests/generators/genesis/main.py @@ -1,12 +1,7 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.altair import spec as spec_altair from eth2spec.test.helpers.constants import PHASE0, ALTAIR -specs = (spec_phase0, spec_altair) - - if __name__ == "__main__": phase_0_mods = {key: 'eth2spec.test.phase0.genesis.test_' + key for key in [ 'initialization', @@ -18,4 +13,4 @@ if __name__ == "__main__": ALTAIR: altair_mods, } - run_state_test_generators(runner_name="genesis", specs=specs, all_mods=all_mods) + run_state_test_generators(runner_name="genesis", all_mods=all_mods) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 316ccdf10..554d0b30a 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,13 +1,7 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.altair import spec as spec_altair -from eth2spec.merge import spec as spec_merge from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE -specs = (spec_phase0, spec_altair, spec_merge) - - if __name__ == "__main__": phase_0_mods = {key: 'eth2spec.test.phase0.block_processing.test_process_' + key for key in [ 'attestation', @@ -46,4 +40,4 @@ if __name__ == "__main__": MERGE: merge_mods, } - run_state_test_generators(runner_name="operations", specs=specs, all_mods=all_mods) + run_state_test_generators(runner_name="operations", all_mods=all_mods) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index 8e50732e1..a3072daca 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,13 +1,7 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.altair import spec as spec_altair -from eth2spec.merge import spec as spec_merge from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE -specs = (spec_phase0, spec_altair, spec_merge) - - if __name__ == "__main__": phase_0_mods = {key: 'eth2spec.test.phase0.rewards.test_' + key for key in [ 'basic', @@ -28,4 +22,4 @@ if __name__ == "__main__": MERGE: merge_mods, } - run_state_test_generators(runner_name="rewards", specs=specs, all_mods=all_mods) + run_state_test_generators(runner_name="rewards", all_mods=all_mods) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 3bed6672d..8caedc8e5 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,14 +1,7 @@ -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.altair import spec as spec_altair -from eth2spec.merge import spec as spec_merge from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE - from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -specs = (spec_phase0, spec_altair, spec_merge) - - if __name__ == "__main__": phase_0_mods = {key: 'eth2spec.test.phase0.sanity.test_' + key for key in [ 'blocks', @@ -29,4 +22,4 @@ if __name__ == "__main__": MERGE: merge_mods, } - run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods) + run_state_test_generators(runner_name="sanity", all_mods=all_mods) diff --git a/tests/generators/shuffling/main.py b/tests/generators/shuffling/main.py index 73b714760..1b363f67c 100644 --- a/tests/generators/shuffling/main.py +++ b/tests/generators/shuffling/main.py @@ -1,15 +1,14 @@ from eth_utils import to_tuple from typing import Iterable -from importlib import reload from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing +from eth2spec.test.helpers.typing import SpecForkName, PresetBaseName -from eth2spec.config import config_util -from eth2spec.phase0 import spec as spec -from eth2spec.test.helpers.constants import PHASE0 +from eth2spec.phase0 import mainnet as spec_mainnet, minimal as spec_minimal +from eth2spec.test.helpers.constants import PHASE0, MINIMAL, MAINNET -def shuffling_case_fn(seed, count): +def shuffling_case_fn(spec, seed, count): yield 'mapping', 'data', { 'seed': '0x' + seed.hex(), 'count': count, @@ -17,28 +16,33 @@ def shuffling_case_fn(seed, count): } -def shuffling_case(seed, count): - return f'shuffle_0x{seed.hex()}_{count}', lambda: shuffling_case_fn(seed, count) +def shuffling_case(spec, seed, count): + return f'shuffle_0x{seed.hex()}_{count}', lambda: shuffling_case_fn(spec, seed, count) @to_tuple -def shuffling_test_cases(): +def shuffling_test_cases(spec): for seed in [spec.hash(seed_init_value.to_bytes(length=4, byteorder='little')) for seed_init_value in range(30)]: for count in [0, 1, 2, 3, 5, 10, 33, 100, 1000, 9999]: - yield shuffling_case(seed, count) + yield shuffling_case(spec, seed, count) -def create_provider(config_name: str) -> gen_typing.TestProvider: +def create_provider(preset_name: PresetBaseName) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec) - return config_name + def prepare_fn() -> None: + return def cases_fn() -> Iterable[gen_typing.TestCase]: + if preset_name == MAINNET: + spec = spec_mainnet + elif preset_name == MINIMAL: + spec = spec_minimal + else: + raise Exception(f"unrecognized preset: {preset_name}") for (case_name, case_fn) in shuffling_test_cases(): yield gen_typing.TestCase( fork_name=PHASE0, + preset_name=preset_name, runner_name='shuffling', handler_name='core', suite_name='shuffle', @@ -50,4 +54,4 @@ def create_provider(config_name: str) -> gen_typing.TestProvider: if __name__ == "__main__": - gen_runner.run_generator("shuffling", [create_provider("minimal"), create_provider("mainnet")]) + gen_runner.run_generator("shuffling", [create_provider(MINIMAL), create_provider(MAINNET)]) diff --git a/tests/generators/ssz_generic/main.py b/tests/generators/ssz_generic/main.py index e6cd3d976..2e96ce2e8 100644 --- a/tests/generators/ssz_generic/main.py +++ b/tests/generators/ssz_generic/main.py @@ -11,13 +11,14 @@ from eth2spec.test.helpers.constants import PHASE0 def create_provider(handler_name: str, suite_name: str, case_maker) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - return "general" + def prepare_fn() -> None: + return def cases_fn() -> Iterable[gen_typing.TestCase]: for (case_name, case_fn) in case_maker(): yield gen_typing.TestCase( fork_name=PHASE0, + preset_name="general", runner_name='ssz_generic', handler_name=handler_name, suite_name=suite_name, diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index d86636e85..42dbfd0f1 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -1,16 +1,12 @@ from random import Random from typing import Iterable -from importlib import reload from inspect import getmembers, isclass from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.debug import random_value, encode -from eth2spec.config import config_util -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.altair import spec as spec_altair -from eth2spec.merge import spec as spec_merge from eth2spec.test.helpers.constants import ALTAIR, MERGE, TESTGEN_FORKS, MINIMAL, MAINNET +from eth2spec.test.context import spec_targets from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, @@ -40,7 +36,7 @@ def get_spec_ssz_types(spec): ] -def ssz_static_cases(fork_name: str, seed: int, name, ssz_type, +def ssz_static_cases(fork_name: str, preset_name: str, seed: int, name, ssz_type, mode: random_value.RandomizationMode, chaos: bool, count: int): random_mode_name = mode.to_name() @@ -50,6 +46,7 @@ def ssz_static_cases(fork_name: str, seed: int, name, ssz_type, for i in range(count): yield gen_typing.TestCase( fork_name=fork_name, + preset_name=preset_name, runner_name='ssz_static', handler_name=name, suite_name=f"ssz_{random_mode_name}{'_chaos' if chaos else ''}", @@ -58,26 +55,17 @@ def ssz_static_cases(fork_name: str, seed: int, name, ssz_type, ) -def create_provider(fork_name, config_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool, +def create_provider(fork_name, preset_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool, cases_if_random: int) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - # Apply changes to presets, this affects some of the vector types. - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_altair) - reload(spec_merge) - return config_name + def prepare_fn() -> None: + return def cases_fn() -> Iterable[gen_typing.TestCase]: count = cases_if_random if chaos or mode.is_changing() else 1 - spec = spec_phase0 - if fork_name == ALTAIR: - spec = spec_altair - elif fork_name == MERGE: - spec = spec_merge + spec = spec_targets[preset_name][fork_name] for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types(spec)): - yield from ssz_static_cases(fork_name, seed * 1000 + i, name, ssz_type, mode, chaos, count) + yield from ssz_static_cases(fork_name, preset_name, seed * 1000 + i, name, ssz_type, mode, chaos, count) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) @@ -95,6 +83,6 @@ if __name__ == "__main__": seed += 1 for fork in TESTGEN_FORKS: gen_runner.run_generator("ssz_static", [ - create_provider(fork, config_name, seed, mode, chaos, cases_if_random) - for (seed, config_name, mode, chaos, cases_if_random) in settings + create_provider(fork, preset_name, seed, mode, chaos, cases_if_random) + for (seed, preset_name, mode, chaos, cases_if_random) in settings ]) diff --git a/tests/generators/transition/main.py b/tests/generators/transition/main.py index b7fd7b0a8..2ded56a13 100644 --- a/tests/generators/transition/main.py +++ b/tests/generators/transition/main.py @@ -1,23 +1,16 @@ -from importlib import reload from typing import Iterable from eth2spec.test.helpers.constants import ALTAIR, MINIMAL, MAINNET, PHASE0 -from eth2spec.config import config_util from eth2spec.test.altair.transition import test_transition as test_altair_transition -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.altair import spec as spec_altair from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests -def create_provider(tests_src, config_name: str, pre_fork_name: str, post_fork_name: str) -> gen_typing.TestProvider: +def create_provider(tests_src, preset_name: str, pre_fork_name: str, post_fork_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_altair) - return config_name + def prepare_fn() -> None: + return def cases_fn() -> Iterable[gen_typing.TestCase]: return generate_from_tests( @@ -26,6 +19,7 @@ def create_provider(tests_src, config_name: str, pre_fork_name: str, post_fork_n src=tests_src, fork_name=post_fork_name, phase=pre_fork_name, + preset_name=preset_name, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)