Merge pull request #2199 from ethereum/turn_off_phase1_testgen

[testgen] turn off phase 1, turn on HF1, and refactoring

setup.py

@@ -562,13 +562,12 @@ setup(
     url="https://github.com/ethereum/eth2.0-specs",
     include_package_data=False,
     package_data={'configs': ['*.yaml'],
-                 
                   'specs': ['**/*.md'],
                   'eth2spec': ['VERSION.txt']},
     package_dir={
         "eth2spec": "tests/core/pyspec/eth2spec",
         "configs": "configs",
-        "specs": "specs"
+        "specs": "specs",
     },
     packages=find_packages(where='tests/core/pyspec') + ['configs', 'specs'],
     py_modules=["eth2spec"],

tests/core/gen_helpers/gen_from_tests/gen.py

@@ -1,40 +0,0 @@
-from inspect import getmembers, isfunction
-from typing import Any, Iterable
-
-from gen_base.gen_typing import TestCase
-
-
-def generate_from_tests(runner_name: str, handler_name: str, src: Any,
-                        fork_name: str, bls_active: bool = True) -> Iterable[TestCase]:
-    """
-    Generate a list of test cases by running tests from the given src in generator-mode.
-    :param runner_name: to categorize the test in general as.
-    :param handler_name: to categorize the test specialization as.
-    :param src: to retrieve tests from (discovered using inspect.getmembers).
-    :param fork_name: to run tests against particular phase and/or fork.
-           (if multiple forks are applicable, indicate the last fork)
-    :param bls_active: optional, to override BLS switch preference. Defaults to True.
-    :return: an iterable of test cases.
-    """
-    fn_names = [
-        name for (name, _) in getmembers(src, isfunction)
-        if name.startswith('test_')
-    ]
-    print("generating test vectors from tests source: %s" % src.__name__)
-    for name in fn_names:
-        tfn = getattr(src, name)
-
-        # strip off the `test_`
-        case_name = name
-        if case_name.startswith('test_'):
-            case_name = case_name[5:]
-
-        yield TestCase(
-            fork_name=fork_name,
-            runner_name=runner_name,
-            handler_name=handler_name,
-            suite_name='pyspec_tests',
-            case_name=case_name,
-            # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent.
-            case_fn=lambda: tfn(generator_mode=True, phase=fork_name, bls_active=bls_active)
-        )

tests/core/gen_helpers/requirements.txt

@@ -1,3 +0,0 @@
-ruamel.yaml==0.16.5
-eth-utils==1.6.0
-pytest>=4.4

tests/core/gen_helpers/setup.py

@@ -1,11 +0,0 @@
-from distutils.core import setup
-
-setup(
-    name='gen_helpers',
-    packages=['gen_base', 'gen_from_tests'],
-    install_requires=[
-        "ruamel.yaml==0.16.5",
-        "eth-utils==1.6.0",
-        "pytest>=4.4",
-    ]
-)

tests/core/pyspec/eth2spec/gen_helpers/README.md

@@ -4,7 +4,7 @@
 
 A util to quickly write new test suite generators with.
 
-See [Generators documentation](../../generators/README.md) for integration details.
+See [Generators documentation](../../../../generators/README.md) for integration details.
 
 Options:
 

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py

@@ -8,11 +8,11 @@ from ruamel.yaml import (
     YAML,
 )
 
-from gen_base.gen_typing import TestProvider
-
 from eth2spec.test import context
 from eth2spec.test.exceptions import SkippedTest
 
+from .gen_typing import TestProvider
+
 
 # Flag that the runner does NOT run test via pytest
 context.is_pytest = False
@@ -119,10 +119,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
 
         print(f"generating tests with config '{config_name}' ...")
         for test_case in tprov.make_cases():
-            case_dir = Path(output_dir) / Path(config_name) / Path(test_case.fork_name) \
-                       / Path(test_case.runner_name) / Path(test_case.handler_name) \
-                       / Path(test_case.suite_name) / Path(test_case.case_name)
-
+            case_dir = (
+                Path(output_dir) / Path(config_name) / Path(test_case.fork_name)
+                / Path(test_case.runner_name) / Path(test_case.handler_name)
+                / Path(test_case.suite_name) / Path(test_case.case_name)
+            )
             if case_dir.exists():
                 if not args.force:
                     print(f'Skipping already existing test: {case_dir}')

tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py

@@ -0,0 +1,99 @@
+from importlib import reload, import_module
+from inspect import getmembers, isfunction
+from typing import Any, Callable, Dict, Iterable
+
+from eth2spec.config import config_util
+from eth2spec.utils import bls
+from eth2spec.test.context import ALL_CONFIGS, TESTGEN_FORKS, SpecForkName, ConfigName
+
+from eth2spec.gen_helpers.gen_base import gen_runner
+from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider
+
+
+def generate_from_tests(runner_name: str, handler_name: str, src: Any,
+                        fork_name: SpecForkName, bls_active: bool = True) -> Iterable[TestCase]:
+    """
+    Generate a list of test cases by running tests from the given src in generator-mode.
+    :param runner_name: to categorize the test in general as.
+    :param handler_name: to categorize the test specialization as.
+    :param src: to retrieve tests from (discovered using inspect.getmembers).
+    :param fork_name: to run tests against particular phase and/or fork.
+           (if multiple forks are applicable, indicate the last fork)
+    :param bls_active: optional, to override BLS switch preference. Defaults to True.
+    :return: an iterable of test cases.
+    """
+    fn_names = [
+        name for (name, _) in getmembers(src, isfunction)
+        if name.startswith('test_')
+    ]
+    print("generating test vectors from tests source: %s" % src.__name__)
+    for name in fn_names:
+        tfn = getattr(src, name)
+
+        # strip off the `test_`
+        case_name = name
+        if case_name.startswith('test_'):
+            case_name = case_name[5:]
+
+        yield TestCase(
+            fork_name=fork_name,
+            runner_name=runner_name,
+            handler_name=handler_name,
+            suite_name='pyspec_tests',
+            case_name=case_name,
+            # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent.
+            case_fn=lambda: tfn(generator_mode=True, phase=fork_name, bls_active=bls_active)
+        )
+
+
+def get_provider(create_provider_fn: Callable[[SpecForkName, str, str, ConfigName], TestProvider],
+                 config_name: ConfigName,
+                 fork_name: SpecForkName,
+                 all_mods: Dict[str, Dict[str, str]]) -> Iterable[TestProvider]:
+    for key, mod_name in all_mods[fork_name].items():
+        yield create_provider_fn(
+            fork_name=fork_name,
+            handler_name=key,
+            tests_src_mod_name=mod_name,
+            config_name=config_name,
+        )
+
+
+def get_create_provider_fn(
+        runner_name: str, config_name: ConfigName, specs: Iterable[Any]
+) -> Callable[[SpecForkName, str, str, ConfigName], TestProvider]:
+    def prepare_fn(configs_path: str) -> str:
+        config_util.prepare_config(configs_path, config_name)
+        for spec in specs:
+            reload(spec)
+        bls.use_milagro()
+        return config_name
+
+    def create_provider(fork_name: SpecForkName, handler_name: str,
+                        tests_src_mod_name: str, config_name: ConfigName) -> TestProvider:
+        def cases_fn() -> Iterable[TestCase]:
+            tests_src = import_module(tests_src_mod_name)
+            return generate_from_tests(
+                runner_name=runner_name,
+                handler_name=handler_name,
+                src=tests_src,
+                fork_name=fork_name,
+            )
+
+        return TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+    return create_provider
+
+
+def run_state_test_generators(runner_name: str, specs: Iterable[Any], all_mods: Dict[str, Dict[str, str]]) -> None:
+    """
+    Generate all available state tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner.
+    """
+    for config_name in ALL_CONFIGS:
+        for fork_name in TESTGEN_FORKS:
+            if fork_name in all_mods:
+                gen_runner.run_generator(runner_name, get_provider(
+                    create_provider_fn=get_create_provider_fn(runner_name, config_name, specs),
+                    config_name=config_name,
+                    fork_name=fork_name,
+                    all_mods=all_mods,
+                ))

tests/core/pyspec/eth2spec/test/context.py

@@ -37,6 +37,10 @@ ALL_PHASES = (PHASE0, PHASE1, LIGHTCLIENT_PATCH)
 MAINNET = ConfigName('mainnet')
 MINIMAL = ConfigName('minimal')
 
+ALL_CONFIGS = (MINIMAL, MAINNET)
+
+# The forks that output to the test vectors.
+TESTGEN_FORKS = (PHASE0, LIGHTCLIENT_PATCH)
 
 # TODO: currently phases are defined as python modules.
 # It would be better if they would be more well-defined interfaces for stronger typing.

tests/generators/README.md

@@ -78,9 +78,8 @@ It's recommended to extend the base-generator.
 
 Create a `requirements.txt` in the root of your generator directory:
 ```
-../../core/gen_helpers
-../../core/config_helpers
-../../core/pyspec
+pytest>=4.4
+../../../
 ```
 
 The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself in order to prevent code duplication and outdated tests.
@@ -103,7 +102,7 @@ Write a `main.py` file. The shuffling test generator is a good minimal starting
 ```python
 from eth2spec.phase0 import spec as spec
 from eth_utils import to_tuple
-from gen_base import gen_runner, gen_typing
+from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
 from preset_loader import loader
 from typing import Iterable
 
@@ -163,35 +162,40 @@ To extend this, one could decide to parametrize the `shuffling_test_cases` funct
 Another example, to generate tests from pytests:
 
 ```python
-def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
+from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.lightclient_patch import spec as spec_lightclient_patch
+from eth2spec.phase1 import spec as spec_phase1
+from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH
 
-    def prepare_fn(configs_path: str) -> str:
-        presets = loader.load_presets(configs_path, config_name)
-        spec_phase0.apply_constants_preset(presets)
-        spec_phase1.apply_constants_preset(presets)
-        return config_name
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
 
-    def cases_fn() -> Iterable[gen_typing.TestCase]:
-        return generate_from_tests(
-            runner_name='epoch_processing',
-            handler_name=handler_name,
-            src=tests_src,
-            fork_name='phase0'
-        )
 
-    return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+specs = (spec_phase0, spec_lightclient_patch, spec_phase1)
 
 
 if __name__ == "__main__":
-    gen_runner.run_generator("epoch_processing", [
-        create_provider('justification_and_finalization', test_process_justification_and_finalization, 'minimal'),
-        ...
-    ])
+    phase_0_mods = {key: 'eth2spec.test.phase0.sanity.test_' + key for key in [
+        'blocks',
+        'slots',
+    ]}
+    lightclient_patch_mods = {**{key: 'eth2spec.test.lightclient_patch.sanity.test_' + key for key in [
+        'blocks',
+    ]}, **phase_0_mods}  # also run the previous phase 0 tests
+    phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [
+        'blocks',  # more phase 1 specific block tests
+        'shard_blocks',
+    ]}, **phase_0_mods}  # also run the previous phase 0 tests (but against phase 1 spec)
 
+    all_mods = {
+        PHASE0: phase_0_mods,
+        LIGHTCLIENT_PATCH: lightclient_patch_mods,
+        PHASE1: phase_1_mods,
+    }
+
+    run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods)
 ```
 
-Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `generate_from_tests` utility.
-
+Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `eth2spec.gen_helpers.gen_from_tests.gen.run_state_test_generators` utility. Note that this helper generates all available tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner.
 
 Recommendations:
 - You can have more than just one test provider.
@@ -200,8 +204,7 @@ Recommendations:
 - Use config `minimal` for performance and simplicity, but also implement a suite with the `mainnet` config where necessary. 
 - You may be able to write your test case provider in a way where it does not make assumptions on constants.
   If so, you can generate test cases with different configurations for the same scenario (see example). 
-- See [`tests/core/gen_helpers/README.md`](../core/gen_helpers/README.md) for command line options for generators.
-
+- See [`tests/core/gen_helpers/README.md`](../core/pyspec/eth2spec/gen_helpers/README.md) for command line options for generators.
 
 ## How to add a new test generator
 
@@ -216,8 +219,8 @@ To add a new test generator that builds `New Tests`:
 4. Your generator is called with `-o some/file/path/for_testing/can/be_anything -c some/other/path/to_configs/`.
  The base generator helps you handle this; you only have to define test case providers.
 5. Finally, add any linting or testing commands to the
- [circleci config file](../.circleci/config.yml) if desired to increase code quality.
- Or add it to the [`Makefile`](../Makefile), if it can be run locally. 
+ [circleci config file](../../.circleci/config.yml) if desired to increase code quality.
+ Or add it to the [`Makefile`](../../Makefile), if it can be run locally.
 
 *Note*: You do not have to change the makefile.
 However, if necessary (e.g. not using Python, or mixing in other languages), submit an issue, and it can be a special case.

tests/generators/bls/main.py

@@ -13,7 +13,7 @@ import milagro_bls_binding as milagro_bls
 
 from eth2spec.utils import bls
 from eth2spec.test.context import PHASE0
-from gen_base import gen_runner, gen_typing
+from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
 
 
 def to_bytes(i):

tests/generators/bls/requirements.txt

@@ -1,4 +1,2 @@
-py_ecc==5.1.0
-eth-utils==1.6.0
-../../core/gen_helpers
+pytest>=4.4
 ../../../

tests/generators/epoch_processing/main.py

@@ -1,34 +1,11 @@
-from typing import Iterable
-
-from gen_base import gen_runner, gen_typing
-from gen_from_tests.gen import generate_from_tests
-from importlib import reload, import_module
-from eth2spec.config import config_util
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
 from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.lightclient_patch import spec as spec_lightclient_patch
 from eth2spec.phase1 import spec as spec_phase1
-from eth2spec.test.context import PHASE0, PHASE1
-from eth2spec.utils import bls
+from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH
 
 
-def create_provider(fork_name: str, handler_name: str,
-                    tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
-    def prepare_fn(configs_path: str) -> str:
-        config_util.prepare_config(configs_path, config_name)
-        reload(spec_phase0)
-        reload(spec_phase1)
-        bls.use_milagro()
-        return config_name
-
-    def cases_fn() -> Iterable[gen_typing.TestCase]:
-        tests_src = import_module(tests_src_mod_name)
-        return generate_from_tests(
-            runner_name='epoch_processing',
-            handler_name=handler_name,
-            src=tests_src,
-            fork_name=fork_name,
-        )
-
-    return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+specs = (spec_phase0, spec_lightclient_patch, spec_phase1)
 
 
 if __name__ == "__main__":
@@ -44,21 +21,22 @@ if __name__ == "__main__":
         'historical_roots_update',
         'participation_record_updates',
     ]}
+    lightclient_patch_mods = {
+        **{key: 'eth2spec.test.lightclient_patch.epoch_processing.test_process_' + key for key in [
+            'sync_committee_updates',
+        ]},
+        **phase_0_mods,
+    }  # also run the previous phase 0 tests
     phase_1_mods = {**{key: 'eth2spec.test.phase1.epoch_processing.test_process_' + key for key in [
         'reveal_deadlines',
         'challenge_deadlines',
         'custody_final_updates',
     ]}, **phase_0_mods}  # also run the previous phase 0 tests (but against phase 1 spec)
 
-    gen_runner.run_generator(f"epoch_processing", [
-        create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items()
-    ])
-    gen_runner.run_generator(f"epoch_processing", [
-        create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items()
-    ])
-    gen_runner.run_generator(f"epoch_processing", [
-        create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items()
-    ])
-    gen_runner.run_generator(f"epoch_processing", [
-        create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items()
-    ])
+    all_mods = {
+        PHASE0: phase_0_mods,
+        LIGHTCLIENT_PATCH: lightclient_patch_mods,
+        PHASE1: phase_1_mods,
+    }
+
+    run_state_test_generators(runner_name="epoch_processing", specs=specs, all_mods=all_mods)

tests/generators/epoch_processing/requirements.txt

@@ -1,2 +1,2 @@
-../../core/gen_helpers
-../../../
+pytest>=4.4
+../../../

tests/generators/finality/main.py

@@ -1,43 +1,23 @@
-from typing import Iterable
-from importlib import reload
-
-from gen_base import gen_runner, gen_typing
-from gen_from_tests.gen import generate_from_tests
-
-from eth2spec.test.context import PHASE0, PHASE1
-from eth2spec.test.phase0.finality import test_finality
-from eth2spec.config import config_util
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
 from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.lightclient_patch import spec as spec_lightclient_patch
 from eth2spec.phase1 import spec as spec_phase1
-from eth2spec.utils import bls
+from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH
 
 
-def create_provider(fork_name: str, handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
-
-    def prepare_fn(configs_path: str) -> str:
-        config_util.prepare_config(configs_path, config_name)
-        reload(spec_phase0)
-        reload(spec_phase1)
-        bls.use_milagro()
-        return config_name
-
-    def cases_fn() -> Iterable[gen_typing.TestCase]:
-        return generate_from_tests(
-            runner_name='finality',
-            handler_name=handler_name,
-            src=tests_src,
-            fork_name=fork_name,
-        )
-
-    return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+specs = (spec_phase0, spec_lightclient_patch, spec_phase1)
 
 
 if __name__ == "__main__":
-    # No additional phase 1 specific rewards tests, yet.
-    key = 'finality'
-    gen_runner.run_generator("finality", [
-        create_provider(PHASE0, 'finality', test_finality, 'minimal'),
-        create_provider(PHASE0, 'finality', test_finality, 'mainnet'),
-        create_provider(PHASE1, 'finality', test_finality, 'minimal'),
-        create_provider(PHASE1, 'finality', test_finality, 'mainnet'),
-    ])
+    phase_0_mods = {'finality': 'eth2spec.test.phase0.finality.test_finality'}
+    # No additional lightclient_patch or phase 1 specific finality tests, yet.
+    lightclient_patch_mods = phase_0_mods
+    phase_1_mods = phase_0_mods
+
+    all_mods = {
+        PHASE0: phase_0_mods,
+        LIGHTCLIENT_PATCH: lightclient_patch_mods,
+        PHASE1: phase_1_mods,
+    }
+
+    run_state_test_generators(runner_name="finality", specs=specs, all_mods=all_mods)

tests/generators/finality/requirements.txt

@@ -1,2 +1,2 @@
-../../core/gen_helpers
-../../../
+pytest>=4.4
+../../../

tests/generators/genesis/main.py

@@ -3,8 +3,8 @@ from typing import Iterable
 from eth2spec.test.context import PHASE0
 from eth2spec.test.phase0.genesis import test_initialization, test_validity
 
-from gen_base import gen_runner, gen_typing
-from gen_from_tests.gen import generate_from_tests
+from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
+from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests
 from eth2spec.phase0 import spec as spec
 from importlib import reload
 from eth2spec.config import config_util

tests/generators/genesis/requirements.txt

@@ -1,2 +1,2 @@
-../../core/gen_helpers
-../../../
+pytest>=4.4
+../../../

tests/generators/operations/main.py

@@ -1,34 +1,11 @@
-from typing import Iterable
-
-from gen_base import gen_runner, gen_typing
-from gen_from_tests.gen import generate_from_tests
-from importlib import reload, import_module
-from eth2spec.config import config_util
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
 from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.lightclient_patch import spec as spec_lightclient_patch
 from eth2spec.phase1 import spec as spec_phase1
-from eth2spec.test.context import PHASE0, PHASE1
-from eth2spec.utils import bls
+from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH
 
 
-def create_provider(fork_name: str, handler_name: str,
-                    tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
-    def prepare_fn(configs_path: str) -> str:
-        config_util.prepare_config(configs_path, config_name)
-        reload(spec_phase0)
-        reload(spec_phase1)
-        bls.use_milagro()
-        return config_name
-
-    def cases_fn() -> Iterable[gen_typing.TestCase]:
-        tests_src = import_module(tests_src_mod_name)
-        return generate_from_tests(
-            runner_name='operations',
-            handler_name=handler_name,
-            src=tests_src,
-            fork_name=fork_name,
-        )
-
-    return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+specs = (spec_phase0, spec_lightclient_patch, spec_phase1)
 
 
 if __name__ == "__main__":
@@ -40,6 +17,12 @@ if __name__ == "__main__":
         'proposer_slashing',
         'voluntary_exit',
     ]}
+    lightclient_patch_mods = {
+        **{key: 'eth2spec.test.lightclient_patch.block_processing.test_process_' + key for key in [
+            'sync_committee',
+        ]},
+        **phase_0_mods,
+    }  # also run the previous phase 0 tests
     phase_1_mods = {**{key: 'eth2spec.test.phase1.block_processing.test_process_' + key for key in [
         'attestation',
         'chunk_challenge',
@@ -49,15 +32,10 @@ if __name__ == "__main__":
         'shard_transition',
     ]}, **phase_0_mods}  # also run the previous phase 0 tests (but against phase 1 spec)
 
-    gen_runner.run_generator(f"operations", [
-        create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items()
-    ])
-    gen_runner.run_generator(f"operations", [
-        create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items()
-    ])
-    gen_runner.run_generator(f"operations", [
-        create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items()
-    ])
-    gen_runner.run_generator(f"operations", [
-        create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items()
-    ])
+    all_mods = {
+        PHASE0: phase_0_mods,
+        LIGHTCLIENT_PATCH: lightclient_patch_mods,
+        PHASE1: phase_1_mods,
+    }
+
+    run_state_test_generators(runner_name="operations", specs=specs, all_mods=all_mods)

tests/generators/operations/requirements.txt

@@ -1,3 +1,2 @@
-eth-utils==1.6.0
-../../core/gen_helpers
-../../../
+pytest>=4.4
+../../../

tests/generators/rewards/main.py

@@ -1,34 +1,11 @@
-from typing import Iterable
-
-from gen_base import gen_runner, gen_typing
-from gen_from_tests.gen import generate_from_tests
-from importlib import reload, import_module
-from eth2spec.config import config_util
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
 from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.lightclient_patch import spec as spec_lightclient_patch
 from eth2spec.phase1 import spec as spec_phase1
-from eth2spec.test.context import PHASE0, PHASE1
-from eth2spec.utils import bls
+from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH
 
 
-def create_provider(fork_name: str, handler_name: str,
-                    tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
-    def prepare_fn(configs_path: str) -> str:
-        config_util.prepare_config(configs_path, config_name)
-        reload(spec_phase0)
-        reload(spec_phase1)
-        bls.use_milagro()
-        return config_name
-
-    def cases_fn() -> Iterable[gen_typing.TestCase]:
-        tests_src = import_module(tests_src_mod_name)
-        return generate_from_tests(
-            runner_name='rewards',
-            handler_name=handler_name,
-            src=tests_src,
-            fork_name=fork_name,
-        )
-
-    return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+specs = (spec_phase0, spec_lightclient_patch, spec_phase1)
 
 
 if __name__ == "__main__":
@@ -37,18 +14,14 @@ if __name__ == "__main__":
         'leak',
         'random',
     ]}
-    # No additional phase 1 specific rewards tests, yet.
+    # No additional lightclient_patch or phase 1 specific rewards tests, yet.
+    lightclient_patch_mods = phase_0_mods
     phase_1_mods = phase_0_mods
 
-    gen_runner.run_generator(f"rewards", [
-        create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items()
-    ])
-    gen_runner.run_generator(f"rewards", [
-        create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items()
-    ])
-    gen_runner.run_generator(f"rewards", [
-        create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items()
-    ])
-    gen_runner.run_generator(f"rewards", [
-        create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items()
-    ])
+    all_mods = {
+        PHASE0: phase_0_mods,
+        LIGHTCLIENT_PATCH: lightclient_patch_mods,
+        PHASE1: phase_1_mods,
+    }
+
+    run_state_test_generators(runner_name="rewards", specs=specs, all_mods=all_mods)

tests/generators/rewards/requirements.txt

@@ -1,2 +1,2 @@
-../../core/gen_helpers
-../../../
+pytest>=4.4
+../../../

tests/generators/sanity/main.py

@@ -1,34 +1,12 @@
-from typing import Iterable
-
-from gen_base import gen_runner, gen_typing
-from gen_from_tests.gen import generate_from_tests
-from importlib import reload, import_module
-from eth2spec.config import config_util
 from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.lightclient_patch import spec as spec_lightclient_patch
 from eth2spec.phase1 import spec as spec_phase1
-from eth2spec.test.context import PHASE0, PHASE1
-from eth2spec.utils import bls
+from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH
+
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
 
 
-def create_provider(fork_name: str, handler_name: str,
-                    tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
-    def prepare_fn(configs_path: str) -> str:
-        config_util.prepare_config(configs_path, config_name)
-        reload(spec_phase0)
-        reload(spec_phase1)
-        bls.use_milagro()
-        return config_name
-
-    def cases_fn() -> Iterable[gen_typing.TestCase]:
-        tests_src = import_module(tests_src_mod_name)
-        return generate_from_tests(
-            runner_name='sanity',
-            handler_name=handler_name,
-            src=tests_src,
-            fork_name=fork_name,
-        )
-
-    return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+specs = (spec_phase0, spec_lightclient_patch, spec_phase1)
 
 
 if __name__ == "__main__":
@@ -36,20 +14,18 @@ if __name__ == "__main__":
         'blocks',
         'slots',
     ]}
+    lightclient_patch_mods = {**{key: 'eth2spec.test.lightclient_patch.sanity.test_' + key for key in [
+        'blocks',
+    ]}, **phase_0_mods}  # also run the previous phase 0 tests
     phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [
         'blocks',  # more phase 1 specific block tests
         'shard_blocks',
     ]}, **phase_0_mods}  # also run the previous phase 0 tests (but against phase 1 spec)
 
-    gen_runner.run_generator(f"sanity", [
-        create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items()
-    ])
-    gen_runner.run_generator(f"sanity", [
-        create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items()
-    ])
-    gen_runner.run_generator(f"sanity", [
-        create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items()
-    ])
-    gen_runner.run_generator(f"sanity", [
-        create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items()
-    ])
+    all_mods = {
+        PHASE0: phase_0_mods,
+        LIGHTCLIENT_PATCH: lightclient_patch_mods,
+        PHASE1: phase_1_mods,
+    }
+
+    run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods)

tests/generators/sanity/requirements.txt

@@ -1,2 +1,2 @@
-../../core/gen_helpers
-../../../
+pytest>=4.4
+../../../

tests/generators/shuffling/main.py

@@ -2,7 +2,7 @@ from eth_utils import to_tuple
 from typing import Iterable
 from importlib import reload
 
-from gen_base import gen_runner, gen_typing
+from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
 
 from eth2spec.config import config_util
 from eth2spec.phase0 import spec as spec

tests/generators/shuffling/requirements.txt

@@ -1,3 +1,2 @@
-eth-utils==1.6.0
-../../core/gen_helpers
-../../../
+pytest>=4.4
+../../../

tests/generators/ssz_generic/main.py

@@ -1,5 +1,5 @@
 from typing import Iterable
-from gen_base import gen_runner, gen_typing
+from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
 import ssz_basic_vector
 import ssz_bitlist
 import ssz_bitvector

tests/generators/ssz_generic/requirements.txt

@@ -1,3 +1,2 @@
-eth-utils==1.6.0
-../../core/gen_helpers
+pytest>=4.4
 ../../../

tests/generators/ssz_static/main.py

@@ -3,7 +3,7 @@ from typing import Iterable
 from importlib import reload
 from inspect import getmembers, isclass
 
-from gen_base import gen_runner, gen_typing
+from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
 
 from eth2spec.debug import random_value, encode
 from eth2spec.config import config_util

tests/generators/ssz_static/requirements.txt

@@ -1,2 +1,2 @@
-../../core/gen_helpers
-../../../
+pytest>=4.4
+../../../