diff --git a/setup.py b/setup.py index df95d99d3..6cfa0910a 100644 --- a/setup.py +++ b/setup.py @@ -562,13 +562,12 @@ setup( url="https://github.com/ethereum/eth2.0-specs", include_package_data=False, package_data={'configs': ['*.yaml'], - 'specs': ['**/*.md'], 'eth2spec': ['VERSION.txt']}, package_dir={ "eth2spec": "tests/core/pyspec/eth2spec", "configs": "configs", - "specs": "specs" + "specs": "specs", }, packages=find_packages(where='tests/core/pyspec') + ['configs', 'specs'], py_modules=["eth2spec"], diff --git a/tests/core/gen_helpers/gen_from_tests/gen.py b/tests/core/gen_helpers/gen_from_tests/gen.py deleted file mode 100644 index 902b0954a..000000000 --- a/tests/core/gen_helpers/gen_from_tests/gen.py +++ /dev/null @@ -1,40 +0,0 @@ -from inspect import getmembers, isfunction -from typing import Any, Iterable - -from gen_base.gen_typing import TestCase - - -def generate_from_tests(runner_name: str, handler_name: str, src: Any, - fork_name: str, bls_active: bool = True) -> Iterable[TestCase]: - """ - Generate a list of test cases by running tests from the given src in generator-mode. - :param runner_name: to categorize the test in general as. - :param handler_name: to categorize the test specialization as. - :param src: to retrieve tests from (discovered using inspect.getmembers). - :param fork_name: to run tests against particular phase and/or fork. - (if multiple forks are applicable, indicate the last fork) - :param bls_active: optional, to override BLS switch preference. Defaults to True. - :return: an iterable of test cases. - """ - fn_names = [ - name for (name, _) in getmembers(src, isfunction) - if name.startswith('test_') - ] - print("generating test vectors from tests source: %s" % src.__name__) - for name in fn_names: - tfn = getattr(src, name) - - # strip off the `test_` - case_name = name - if case_name.startswith('test_'): - case_name = case_name[5:] - - yield TestCase( - fork_name=fork_name, - runner_name=runner_name, - handler_name=handler_name, - suite_name='pyspec_tests', - case_name=case_name, - # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent. - case_fn=lambda: tfn(generator_mode=True, phase=fork_name, bls_active=bls_active) - ) diff --git a/tests/core/gen_helpers/requirements.txt b/tests/core/gen_helpers/requirements.txt deleted file mode 100644 index e7cdd30ea..000000000 --- a/tests/core/gen_helpers/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -ruamel.yaml==0.16.5 -eth-utils==1.6.0 -pytest>=4.4 diff --git a/tests/core/gen_helpers/setup.py b/tests/core/gen_helpers/setup.py deleted file mode 100644 index e9fc1a787..000000000 --- a/tests/core/gen_helpers/setup.py +++ /dev/null @@ -1,11 +0,0 @@ -from distutils.core import setup - -setup( - name='gen_helpers', - packages=['gen_base', 'gen_from_tests'], - install_requires=[ - "ruamel.yaml==0.16.5", - "eth-utils==1.6.0", - "pytest>=4.4", - ] -) diff --git a/tests/core/gen_helpers/README.md b/tests/core/pyspec/eth2spec/gen_helpers/README.md similarity index 96% rename from tests/core/gen_helpers/README.md rename to tests/core/pyspec/eth2spec/gen_helpers/README.md index 20b48db83..d39ee66ae 100644 --- a/tests/core/gen_helpers/README.md +++ b/tests/core/pyspec/eth2spec/gen_helpers/README.md @@ -4,7 +4,7 @@ A util to quickly write new test suite generators with. -See [Generators documentation](../../generators/README.md) for integration details. +See [Generators documentation](../../../../generators/README.md) for integration details. Options: diff --git a/tests/core/gen_helpers/gen_base/__init__.py b/tests/core/pyspec/eth2spec/gen_helpers/__init__.py similarity index 100% rename from tests/core/gen_helpers/gen_base/__init__.py rename to tests/core/pyspec/eth2spec/gen_helpers/__init__.py diff --git a/tests/core/gen_helpers/gen_from_tests/__init__.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/__init__.py similarity index 100% rename from tests/core/gen_helpers/gen_from_tests/__init__.py rename to tests/core/pyspec/eth2spec/gen_helpers/gen_base/__init__.py diff --git a/tests/core/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py similarity index 95% rename from tests/core/gen_helpers/gen_base/gen_runner.py rename to tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index a22073c00..26427c6f8 100644 --- a/tests/core/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -8,11 +8,11 @@ from ruamel.yaml import ( YAML, ) -from gen_base.gen_typing import TestProvider - from eth2spec.test import context from eth2spec.test.exceptions import SkippedTest +from .gen_typing import TestProvider + # Flag that the runner does NOT run test via pytest context.is_pytest = False @@ -119,10 +119,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): print(f"generating tests with config '{config_name}' ...") for test_case in tprov.make_cases(): - case_dir = Path(output_dir) / Path(config_name) / Path(test_case.fork_name) \ - / Path(test_case.runner_name) / Path(test_case.handler_name) \ - / Path(test_case.suite_name) / Path(test_case.case_name) - + case_dir = ( + Path(output_dir) / Path(config_name) / Path(test_case.fork_name) + / Path(test_case.runner_name) / Path(test_case.handler_name) + / Path(test_case.suite_name) / Path(test_case.case_name) + ) if case_dir.exists(): if not args.force: print(f'Skipping already existing test: {case_dir}') diff --git a/tests/core/gen_helpers/gen_base/gen_typing.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py similarity index 100% rename from tests/core/gen_helpers/gen_base/gen_typing.py rename to tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/__init__.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py new file mode 100644 index 000000000..67d29b194 --- /dev/null +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -0,0 +1,99 @@ +from importlib import reload, import_module +from inspect import getmembers, isfunction +from typing import Any, Callable, Dict, Iterable + +from eth2spec.config import config_util +from eth2spec.utils import bls +from eth2spec.test.context import ALL_CONFIGS, TESTGEN_FORKS, SpecForkName, ConfigName + +from eth2spec.gen_helpers.gen_base import gen_runner +from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider + + +def generate_from_tests(runner_name: str, handler_name: str, src: Any, + fork_name: SpecForkName, bls_active: bool = True) -> Iterable[TestCase]: + """ + Generate a list of test cases by running tests from the given src in generator-mode. + :param runner_name: to categorize the test in general as. + :param handler_name: to categorize the test specialization as. + :param src: to retrieve tests from (discovered using inspect.getmembers). + :param fork_name: to run tests against particular phase and/or fork. + (if multiple forks are applicable, indicate the last fork) + :param bls_active: optional, to override BLS switch preference. Defaults to True. + :return: an iterable of test cases. + """ + fn_names = [ + name for (name, _) in getmembers(src, isfunction) + if name.startswith('test_') + ] + print("generating test vectors from tests source: %s" % src.__name__) + for name in fn_names: + tfn = getattr(src, name) + + # strip off the `test_` + case_name = name + if case_name.startswith('test_'): + case_name = case_name[5:] + + yield TestCase( + fork_name=fork_name, + runner_name=runner_name, + handler_name=handler_name, + suite_name='pyspec_tests', + case_name=case_name, + # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent. + case_fn=lambda: tfn(generator_mode=True, phase=fork_name, bls_active=bls_active) + ) + + +def get_provider(create_provider_fn: Callable[[SpecForkName, str, str, ConfigName], TestProvider], + config_name: ConfigName, + fork_name: SpecForkName, + all_mods: Dict[str, Dict[str, str]]) -> Iterable[TestProvider]: + for key, mod_name in all_mods[fork_name].items(): + yield create_provider_fn( + fork_name=fork_name, + handler_name=key, + tests_src_mod_name=mod_name, + config_name=config_name, + ) + + +def get_create_provider_fn( + runner_name: str, config_name: ConfigName, specs: Iterable[Any] +) -> Callable[[SpecForkName, str, str, ConfigName], TestProvider]: + def prepare_fn(configs_path: str) -> str: + config_util.prepare_config(configs_path, config_name) + for spec in specs: + reload(spec) + bls.use_milagro() + return config_name + + def create_provider(fork_name: SpecForkName, handler_name: str, + tests_src_mod_name: str, config_name: ConfigName) -> TestProvider: + def cases_fn() -> Iterable[TestCase]: + tests_src = import_module(tests_src_mod_name) + return generate_from_tests( + runner_name=runner_name, + handler_name=handler_name, + src=tests_src, + fork_name=fork_name, + ) + + return TestProvider(prepare=prepare_fn, make_cases=cases_fn) + return create_provider + + +def run_state_test_generators(runner_name: str, specs: Iterable[Any], all_mods: Dict[str, Dict[str, str]]) -> None: + """ + Generate all available state tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner. + """ + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(runner_name, get_provider( + create_provider_fn=get_create_provider_fn(runner_name, config_name, specs), + config_name=config_name, + fork_name=fork_name, + all_mods=all_mods, + )) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 4f07a790a..5c2a3bf4d 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -37,6 +37,10 @@ ALL_PHASES = (PHASE0, PHASE1, LIGHTCLIENT_PATCH) MAINNET = ConfigName('mainnet') MINIMAL = ConfigName('minimal') +ALL_CONFIGS = (MINIMAL, MAINNET) + +# The forks that output to the test vectors. +TESTGEN_FORKS = (PHASE0, LIGHTCLIENT_PATCH) # TODO: currently phases are defined as python modules. # It would be better if they would be more well-defined interfaces for stronger typing. diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/__init__.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/generators/README.md b/tests/generators/README.md index 077a8443c..26094bbae 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -78,9 +78,8 @@ It's recommended to extend the base-generator. Create a `requirements.txt` in the root of your generator directory: ``` -../../core/gen_helpers -../../core/config_helpers -../../core/pyspec +pytest>=4.4 +../../../ ``` The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself in order to prevent code duplication and outdated tests. @@ -103,7 +102,7 @@ Write a `main.py` file. The shuffling test generator is a good minimal starting ```python from eth2spec.phase0 import spec as spec from eth_utils import to_tuple -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from preset_loader import loader from typing import Iterable @@ -163,35 +162,40 @@ To extend this, one could decide to parametrize the `shuffling_test_cases` funct Another example, to generate tests from pytests: ```python -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH - def prepare_fn(configs_path: str) -> str: - presets = loader.load_presets(configs_path, config_name) - spec_phase0.apply_constants_preset(presets) - spec_phase1.apply_constants_preset(presets) - return config_name +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators - def cases_fn() -> Iterable[gen_typing.TestCase]: - return generate_from_tests( - runner_name='epoch_processing', - handler_name=handler_name, - src=tests_src, - fork_name='phase0' - ) - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": - gen_runner.run_generator("epoch_processing", [ - create_provider('justification_and_finalization', test_process_justification_and_finalization, 'minimal'), - ... - ]) + phase_0_mods = {key: 'eth2spec.test.phase0.sanity.test_' + key for key in [ + 'blocks', + 'slots', + ]} + lightclient_patch_mods = {**{key: 'eth2spec.test.lightclient_patch.sanity.test_' + key for key in [ + 'blocks', + ]}, **phase_0_mods} # also run the previous phase 0 tests + phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [ + 'blocks', # more phase 1 specific block tests + 'shard_blocks', + ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods) ``` -Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `generate_from_tests` utility. - +Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `eth2spec.gen_helpers.gen_from_tests.gen.run_state_test_generators` utility. Note that this helper generates all available tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner. Recommendations: - You can have more than just one test provider. @@ -200,8 +204,7 @@ Recommendations: - Use config `minimal` for performance and simplicity, but also implement a suite with the `mainnet` config where necessary. - You may be able to write your test case provider in a way where it does not make assumptions on constants. If so, you can generate test cases with different configurations for the same scenario (see example). -- See [`tests/core/gen_helpers/README.md`](../core/gen_helpers/README.md) for command line options for generators. - +- See [`tests/core/gen_helpers/README.md`](../core/pyspec/eth2spec/gen_helpers/README.md) for command line options for generators. ## How to add a new test generator @@ -216,8 +219,8 @@ To add a new test generator that builds `New Tests`: 4. Your generator is called with `-o some/file/path/for_testing/can/be_anything -c some/other/path/to_configs/`. The base generator helps you handle this; you only have to define test case providers. 5. Finally, add any linting or testing commands to the - [circleci config file](../.circleci/config.yml) if desired to increase code quality. - Or add it to the [`Makefile`](../Makefile), if it can be run locally. + [circleci config file](../../.circleci/config.yml) if desired to increase code quality. + Or add it to the [`Makefile`](../../Makefile), if it can be run locally. *Note*: You do not have to change the makefile. However, if necessary (e.g. not using Python, or mixing in other languages), submit an issue, and it can be a special case. diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 6552b8654..fecc2df7d 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -13,7 +13,7 @@ import milagro_bls_binding as milagro_bls from eth2spec.utils import bls from eth2spec.test.context import PHASE0 -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing def to_bytes(i): diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt index 5f830773a..df386450b 100644 --- a/tests/generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,4 +1,2 @@ -py_ecc==5.1.0 -eth-utils==1.6.0 -../../core/gen_helpers +pytest>=4.4 ../../../ diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index ea7639605..50a1e2b57 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,34 +1,11 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests -from importlib import reload, import_module -from eth2spec.config import config_util +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='epoch_processing', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -44,21 +21,22 @@ if __name__ == "__main__": 'historical_roots_update', 'participation_record_updates', ]} + lightclient_patch_mods = { + **{key: 'eth2spec.test.lightclient_patch.epoch_processing.test_process_' + key for key in [ + 'sync_committee_updates', + ]}, + **phase_0_mods, + } # also run the previous phase 0 tests phase_1_mods = {**{key: 'eth2spec.test.phase1.epoch_processing.test_process_' + key for key in [ 'reveal_deadlines', 'challenge_deadlines', 'custody_final_updates', ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) - gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() - ]) - gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + run_state_test_generators(runner_name="epoch_processing", specs=specs, all_mods=all_mods) diff --git a/tests/generators/epoch_processing/requirements.txt b/tests/generators/epoch_processing/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/epoch_processing/requirements.txt +++ b/tests/generators/epoch_processing/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index ef2d8293f..8b961f9f4 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,43 +1,23 @@ -from typing import Iterable -from importlib import reload - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests - -from eth2spec.test.context import PHASE0, PHASE1 -from eth2spec.test.phase0.finality import test_finality -from eth2spec.config import config_util +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: - - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - return generate_from_tests( - runner_name='finality', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": - # No additional phase 1 specific rewards tests, yet. - key = 'finality' - gen_runner.run_generator("finality", [ - create_provider(PHASE0, 'finality', test_finality, 'minimal'), - create_provider(PHASE0, 'finality', test_finality, 'mainnet'), - create_provider(PHASE1, 'finality', test_finality, 'minimal'), - create_provider(PHASE1, 'finality', test_finality, 'mainnet'), - ]) + phase_0_mods = {'finality': 'eth2spec.test.phase0.finality.test_finality'} + # No additional lightclient_patch or phase 1 specific finality tests, yet. + lightclient_patch_mods = phase_0_mods + phase_1_mods = phase_0_mods + + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + run_state_test_generators(runner_name="finality", specs=specs, all_mods=all_mods) diff --git a/tests/generators/finality/requirements.txt b/tests/generators/finality/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/finality/requirements.txt +++ b/tests/generators/finality/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/genesis/main.py b/tests/generators/genesis/main.py index ce055b44a..854af6572 100644 --- a/tests/generators/genesis/main.py +++ b/tests/generators/genesis/main.py @@ -3,8 +3,8 @@ from typing import Iterable from eth2spec.test.context import PHASE0 from eth2spec.test.phase0.genesis import test_initialization, test_validity -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests from eth2spec.phase0 import spec as spec from importlib import reload from eth2spec.config import config_util diff --git a/tests/generators/genesis/requirements.txt b/tests/generators/genesis/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/genesis/requirements.txt +++ b/tests/generators/genesis/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 1acf45e47..00a58288f 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,34 +1,11 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests -from importlib import reload, import_module -from eth2spec.config import config_util +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='operations', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -40,6 +17,12 @@ if __name__ == "__main__": 'proposer_slashing', 'voluntary_exit', ]} + lightclient_patch_mods = { + **{key: 'eth2spec.test.lightclient_patch.block_processing.test_process_' + key for key in [ + 'sync_committee', + ]}, + **phase_0_mods, + } # also run the previous phase 0 tests phase_1_mods = {**{key: 'eth2spec.test.phase1.block_processing.test_process_' + key for key in [ 'attestation', 'chunk_challenge', @@ -49,15 +32,10 @@ if __name__ == "__main__": 'shard_transition', ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) - gen_runner.run_generator(f"operations", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"operations", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"operations", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() - ]) - gen_runner.run_generator(f"operations", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + run_state_test_generators(runner_name="operations", specs=specs, all_mods=all_mods) diff --git a/tests/generators/operations/requirements.txt b/tests/generators/operations/requirements.txt index a6ea61aea..df386450b 100644 --- a/tests/generators/operations/requirements.txt +++ b/tests/generators/operations/requirements.txt @@ -1,3 +1,2 @@ -eth-utils==1.6.0 -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index 23d0633b0..4124587cc 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,34 +1,11 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests -from importlib import reload, import_module -from eth2spec.config import config_util +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='rewards', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -37,18 +14,14 @@ if __name__ == "__main__": 'leak', 'random', ]} - # No additional phase 1 specific rewards tests, yet. + # No additional lightclient_patch or phase 1 specific rewards tests, yet. + lightclient_patch_mods = phase_0_mods phase_1_mods = phase_0_mods - gen_runner.run_generator(f"rewards", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"rewards", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"rewards", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() - ]) - gen_runner.run_generator(f"rewards", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + run_state_test_generators(runner_name="rewards", specs=specs, all_mods=all_mods) diff --git a/tests/generators/rewards/requirements.txt b/tests/generators/rewards/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/rewards/requirements.txt +++ b/tests/generators/rewards/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 83166f0cf..5155798ff 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,34 +1,12 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests -from importlib import reload, import_module -from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH + +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='sanity', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -36,20 +14,18 @@ if __name__ == "__main__": 'blocks', 'slots', ]} + lightclient_patch_mods = {**{key: 'eth2spec.test.lightclient_patch.sanity.test_' + key for key in [ + 'blocks', + ]}, **phase_0_mods} # also run the previous phase 0 tests phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [ 'blocks', # more phase 1 specific block tests 'shard_blocks', ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) - gen_runner.run_generator(f"sanity", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"sanity", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"sanity", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() - ]) - gen_runner.run_generator(f"sanity", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods) diff --git a/tests/generators/sanity/requirements.txt b/tests/generators/sanity/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/sanity/requirements.txt +++ b/tests/generators/sanity/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/shuffling/main.py b/tests/generators/shuffling/main.py index 6069de77a..091207bca 100644 --- a/tests/generators/shuffling/main.py +++ b/tests/generators/shuffling/main.py @@ -2,7 +2,7 @@ from eth_utils import to_tuple from typing import Iterable from importlib import reload -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.config import config_util from eth2spec.phase0 import spec as spec diff --git a/tests/generators/shuffling/requirements.txt b/tests/generators/shuffling/requirements.txt index a6ea61aea..df386450b 100644 --- a/tests/generators/shuffling/requirements.txt +++ b/tests/generators/shuffling/requirements.txt @@ -1,3 +1,2 @@ -eth-utils==1.6.0 -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/ssz_generic/main.py b/tests/generators/ssz_generic/main.py index 8cfb2e3eb..737f8cda6 100644 --- a/tests/generators/ssz_generic/main.py +++ b/tests/generators/ssz_generic/main.py @@ -1,5 +1,5 @@ from typing import Iterable -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing import ssz_basic_vector import ssz_bitlist import ssz_bitvector diff --git a/tests/generators/ssz_generic/requirements.txt b/tests/generators/ssz_generic/requirements.txt index af061a3b1..df386450b 100644 --- a/tests/generators/ssz_generic/requirements.txt +++ b/tests/generators/ssz_generic/requirements.txt @@ -1,3 +1,2 @@ -eth-utils==1.6.0 -../../core/gen_helpers +pytest>=4.4 ../../../ diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 38ff18615..8b18705e7 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -3,7 +3,7 @@ from typing import Iterable from importlib import reload from inspect import getmembers, isclass -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.debug import random_value, encode from eth2spec.config import config_util diff --git a/tests/generators/ssz_static/requirements.txt b/tests/generators/ssz_static/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/ssz_static/requirements.txt +++ b/tests/generators/ssz_static/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../