Merge pull request #2221 from ethereum/hf1-fork-tests-truly-pass

Make `test_fork.py` truly pass (on #2220)

tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py

@@ -1,6 +1,6 @@
 from importlib import reload, import_module
 from inspect import getmembers, isfunction
-from typing import Any, Callable, Dict, Iterable
+from typing import Any, Callable, Dict, Iterable, Optional
 
 from eth2spec.config import config_util
 from eth2spec.utils import bls
@@ -11,7 +11,8 @@ from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider
 
 
 def generate_from_tests(runner_name: str, handler_name: str, src: Any,
-                        fork_name: SpecForkName, bls_active: bool = True) -> Iterable[TestCase]:
+                        fork_name: SpecForkName, bls_active: bool = True,
+                        phase: Optional[str]=None) -> Iterable[TestCase]:
     """
     Generate a list of test cases by running tests from the given src in generator-mode.
     :param runner_name: to categorize the test in general as.
@@ -20,12 +21,17 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any,
     :param fork_name: to run tests against particular phase and/or fork.
            (if multiple forks are applicable, indicate the last fork)
     :param bls_active: optional, to override BLS switch preference. Defaults to True.
+    :param phase: optional, specific phase name
     :return: an iterable of test cases.
     """
     fn_names = [
         name for (name, _) in getmembers(src, isfunction)
         if name.startswith('test_')
     ]
+
+    if phase is None:
+        phase = fork_name
+
     print("generating test vectors from tests source: %s" % src.__name__)
     for name in fn_names:
         tfn = getattr(src, name)
@@ -42,7 +48,7 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any,
             suite_name='pyspec_tests',
             case_name=case_name,
             # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent.
-            case_fn=lambda: tfn(generator_mode=True, phase=fork_name, bls_active=bls_active)
+            case_fn=lambda: tfn(generator_mode=True, phase=phase, bls_active=bls_active)
         )
 
 

tests/core/pyspec/eth2spec/test/context.py

@@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca
         # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper.
         # Decide based on performance/consistency results later.
         state = phases[PHASE1].upgrade_to_phase1(state)
-    elif spec.fork == LIGHTCLIENT_PATCH and not fork_test:  # do not upgrade if spec ttttest
+    elif spec.fork == LIGHTCLIENT_PATCH and not is_fork_test:  # do not upgrade if spec test
         state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state)
 
     return state
@@ -294,6 +294,7 @@ def fork_test(fn):
     def entry(*args, **kw):
         # override fork test setting
         kw['fork_test'] = True
+        return fn(*args, **kw)
     return entry
 
 

tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py

@@ -1,5 +1,5 @@
 from eth2spec.test.context import (
-    LIGHTCLIENT_PATCH,
+    PHASE0, LIGHTCLIENT_PATCH,
     with_phases,
     with_custom_state, fork_test,
     spec_test, with_state,
@@ -53,39 +53,39 @@ def run_fork_test(spec, pre_state):
     yield 'post', post_state
 
 
-@with_phases(([LIGHTCLIENT_PATCH]))
-@with_state
-@spec_test
 @fork_test
+@with_phases(([PHASE0]))
+@spec_test
+@with_state
 @with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_base_state(spec, phases, state):
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
 
 
-@with_phases(([LIGHTCLIENT_PATCH]))
-@with_state
-@spec_test
 @fork_test
+@with_phases(([PHASE0]))
+@spec_test
+@with_state
 @with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_next_epoch(spec, phases, state):
     next_epoch(spec, state)
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
 
 
-@with_phases(([LIGHTCLIENT_PATCH]))
-@with_state
-@spec_test
 @fork_test
+@with_phases(([PHASE0]))
+@spec_test
+@with_state
 @with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_next_epoch_with_block(spec, phases, state):
     next_epoch_via_block(spec, state)
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
 
 
-@with_phases(([LIGHTCLIENT_PATCH]))
-@with_state
-@spec_test
 @fork_test
+@with_phases(([PHASE0]))
+@spec_test
+@with_state
 @with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_many_next_epoch(spec, phases, state):
     for _ in range(3):
@@ -93,28 +93,28 @@ def test_fork_many_next_epoch(spec, phases, state):
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
 
 
-@with_phases(([LIGHTCLIENT_PATCH]))
+@fork_test
+@with_phases(([PHASE0]))
 @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE)
 @spec_test
-@fork_test
 @with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_random_low_balances(spec, phases, state):
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
 
 
-@with_phases(([LIGHTCLIENT_PATCH]))
+@fork_test
+@with_phases(([PHASE0]))
 @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE)
 @spec_test
-@fork_test
 @with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_random_misc_balances(spec, phases, state):
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
 
 
-@with_phases(([LIGHTCLIENT_PATCH]))
+@fork_test
+@with_phases(([PHASE0]))
 @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE)
 @spec_test
-@fork_test
 @with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_random_large_validator_set(spec, phases, state):
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)

tests/generators/forks/main.py

@@ -1,33 +1,31 @@
 from importlib import reload
 from typing import Iterable
 
-from eth2spec.test.context import LIGHTCLIENT_PATCH
+from eth2spec.test.context import PHASE0, LIGHTCLIENT_PATCH, MINIMAL, MAINNET
 from eth2spec.config import config_util
 from eth2spec.test.lightclient_patch.fork import test_fork as test_altair_forks
 from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.lightclient_patch import spec as spec_lightclient_patch
 
 from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
 from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests
 
 
-pre_specs = {
+def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider:
-    LIGHTCLIENT_PATCH: spec_phase0,
-}
-
-
-def create_provider(fork_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
 
     def prepare_fn(configs_path: str) -> str:
         config_util.prepare_config(configs_path, config_name)
-        reload(pre_specs[fork_name])
+        reload(spec_phase0)
+        reload(spec_lightclient_patch)
         return config_name
 
     def cases_fn() -> Iterable[gen_typing.TestCase]:
         return generate_from_tests(
-            runner_name='forks',
+            runner_name='fork',
-            handler_name='core',
+            handler_name='fork',
             src=tests_src,
-            fork_name=fork_name,
+            fork_name=LIGHTCLIENT_PATCH,
+            phase=PHASE0,
         )
 
     return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
@@ -35,6 +33,6 @@ def create_provider(fork_name: str, tests_src, config_name: str) -> gen_typing.T
 
 if __name__ == "__main__":
     gen_runner.run_generator("forks", [
-        create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'),
+        create_provider(test_altair_forks, MINIMAL),
-        create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'),
+        create_provider(test_altair_forks, MAINNET),
     ])