implement epoch processing test-gen, bugfix tests

test_generators/epoch_processing/README.md

@@ -0,0 +1,11 @@
+# Epoch processing
+
+Epoch processing covers the sub-transitions during an epoch change.
+
+An epoch-processing test-runner can consume these sub-transition test-suites,
+ and handle different kinds of epoch sub-transitions by processing the cases using the specified test handler.
+
+Information on the format of the tests can be found in the [epoch-processing test formats documentation](../../specs/test_formats/epoch_processing/README.md).
+
+ 
+

test_generators/epoch_processing/main.py

@@ -0,0 +1,38 @@
+from eth2spec.test.epoch_processing import (
+    test_process_crosslinks,
+    test_process_registry_updates
+)
+
+from gen_base import gen_runner, gen_suite, gen_typing
+from gen_from_tests.gen import generate_from_tests
+from typing import Callable, Iterable
+from preset_loader import loader
+from eth2spec.phase0 import spec
+
+
+def create_suite(transition_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]]) \
+        -> Callable[[str], gen_typing.TestSuiteOutput]:
+    def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput:
+        presets = loader.load_presets(configs_path, config_name)
+        spec.apply_constants_preset(presets)
+
+        return ("%s_%s" % (transition_name, config_name), transition_name, gen_suite.render_suite(
+            title="%s epoch processing" % transition_name,
+            summary="Test suite for %s type epoch processing" % transition_name,
+            forks_timeline="testing",
+            forks=["phase0"],
+            config=config_name,
+            runner="epoch_processing",
+            handler=transition_name,
+            test_cases=get_cases()))
+    return suite_definition
+
+
+if __name__ == "__main__":
+    gen_runner.run_generator("epoch_processing", [
+        create_suite('crosslinks',       'minimal', lambda: generate_from_tests(test_process_crosslinks)),
+        # To be updated to support mainnet config.
+        # create_suite('crosslinks',       'mainnet', lambda: generate_from_tests(test_process_crosslinks)),
+        create_suite('registry_updates', 'minimal', lambda: generate_from_tests(test_process_registry_updates)),
+        create_suite('registry_updates', 'mainnet', lambda: generate_from_tests(test_process_registry_updates)),
+    ])

test_generators/epoch_processing/requirements.txt

@@ -0,0 +1,4 @@
+eth-utils==1.6.0
+../../test_libs/gen_helpers
+../../test_libs/config_helpers
+../../test_libs/pyspec

test_generators/operations/suite_creator.py

@@ -1,39 +0,0 @@
-from typing import Callable, Iterable
-from inspect import getmembers, isfunction
-from gen_base import gen_suite, gen_typing
-from preset_loader import loader
-from eth2spec.phase0 import spec
-
-
-def generate_from_tests(pkg):
-    fn_names = [
-        name for (name, _) in getmembers(pkg, isfunction)
-        if name.startswith('test_')
-    ]
-    out = []
-    print("generating test vectors from tests package: %s" % pkg.__name__)
-    for name in fn_names:
-        tfn = getattr(pkg, name)
-        try:
-            out.append(tfn(generator_mode=True, bls_active=True))
-        except AssertionError:
-            print("ERROR: failed to generate vector from test: %s (pkg: %s)" % (name, pkg.__name__))
-    return out
-
-
-def create_suite(operation_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]])\
-        -> Callable[[str], gen_typing.TestSuiteOutput]:
-    def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput:
-        presets = loader.load_presets(configs_path, config_name)
-        spec.apply_constants_preset(presets)
-
-        return ("%s_%s" % (operation_name, config_name), operation_name, gen_suite.render_suite(
-            title="%s operation" % operation_name,
-            summary="Test suite for %s type operation processing" % operation_name,
-            forks_timeline="testing",
-            forks=["phase0"],
-            config=config_name,
-            runner="operations",
-            handler=operation_name,
-            test_cases=get_cases()))
-    return suite_definition

test_libs/gen_helpers/gen_from_tests/gen.py

@@ -0,0 +1,22 @@
+from inspect import getmembers, isfunction
+
+def generate_from_tests(src, bls_active=True):
+    """
+    Generate a list of test cases by running tests from the given src in generator-mode.
+    :param src: to retrieve tests from (discovered using inspect.getmembers)
+    :param bls_active: optional, to override BLS switch preference. Defaults to True.
+    :return: the list of test cases.
+    """
+    fn_names = [
+        name for (name, _) in getmembers(src, isfunction)
+        if name.startswith('test_')
+    ]
+    out = []
+    print("generating test vectors from tests source: %s" % src.__name__)
+    for name in fn_names:
+        tfn = getattr(src, name)
+        try:
+            out.append(tfn(generator_mode=True, bls_active=bls_active))
+        except AssertionError:
+            print("ERROR: failed to generate vector from test: %s (src: %s)" % (name, src.__name__))
+    return out

test_libs/gen_helpers/setup.py

@@ -2,7 +2,7 @@ from distutils.core import setup
 
 setup(
     name='gen_helpers',
-    packages=['gen_base'],
+    packages=['gen_base', 'gen_from_tests'],
     install_requires=[
         "ruamel.yaml==0.15.96",
         "eth-utils==1.6.0"

test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py

@@ -4,6 +4,7 @@ from eth2spec.phase0.spec import (
     get_current_epoch,
     is_active_validator,
 )
+from eth2spec.test.helpers.block import apply_empty_block
 from eth2spec.test.helpers.state import next_epoch
 from eth2spec.test.context import spec_state_test
 
@@ -19,15 +20,13 @@ def test_activation(state):
     state.validator_registry[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE
     assert not is_active_validator(state.validator_registry[index], get_current_epoch(state))
 
+    for _ in range(spec.ACTIVATION_EXIT_DELAY):
+        next_epoch(state)
+
     yield 'pre', state
 
-    blocks = []
+    next_epoch(state)
-    for _ in range(spec.ACTIVATION_EXIT_DELAY + 1):
+    yield 'trigger_block', apply_empty_block(state)
-        block = next_epoch(state)
-        blocks.append(block)
-
-    # provide extra type hinting here, since it is wrapped in a list.
-    yield 'blocks', blocks, [spec.BeaconBlock]
 
     yield 'post', state
 
@@ -48,15 +47,13 @@ def test_ejection(state):
     # Mock an ejection
     state.validator_registry[index].effective_balance = spec.EJECTION_BALANCE
 
+    for _ in range(spec.ACTIVATION_EXIT_DELAY):
+        next_epoch(state)
+
     yield 'pre', state
 
-    blocks = []
+    next_epoch(state)
-    for _ in range(spec.ACTIVATION_EXIT_DELAY + 1):
+    yield 'trigger_block', apply_empty_block(state)
-        block = next_epoch(state)
-        blocks.append(block)
-
-    # provide extra type hinting here, since it is wrapped in a list.
-    yield 'blocks', blocks, [spec.BeaconBlock]
 
     yield 'post', state