working through test gens

tests/core/pyspec/eth2spec/test/context.py

@@ -70,7 +70,7 @@ class SpecForks(TypedDict, total=False):
 
 
 def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int],
-                   spec: Spec, phases: SpecForks):
+                   spec: Spec, phases: SpecForks, is_fork_test: bool):
 
     p0 = phases[PHASE0]
     balances = balances_fn(p0)
@@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca
         # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper.
         # Decide based on performance/consistency results later.
         state = phases[PHASE1].upgrade_to_phase1(state)
-    elif spec.fork == LIGHTCLIENT_PATCH:  # not generalizing this just yet, unclear final spec fork/patch order.
+    elif spec.fork == LIGHTCLIENT_PATCH and not fork_test:  # do not upgrade if spec ttttest
         state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state)
 
     return state
@@ -98,10 +98,11 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]],
         def entry(*args, spec: Spec, phases: SpecForks, **kw):
             # make a key for the state
             # genesis fork version separates configs during test-generation runtime.
-            key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn)
+            is_fork_test = kw.pop('fork_test') if 'fork_test' in kw else False
+            key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn, is_fork_test)
             global _custom_state_cache_dict
             if key not in _custom_state_cache_dict:
-                state = _prepare_state(balances_fn, threshold_fn, spec, phases)
+                state = _prepare_state(balances_fn, threshold_fn, spec, phases, is_fork_test)
                 _custom_state_cache_dict[key] = state.get_backing()
 
             # Take an entry out of the LRU.
@@ -287,6 +288,15 @@ def bls_switch(fn):
     return entry
 
 
+def fork_test(fn):
+    """
+    """
+    def entry(*args, **kw):
+        # override fork test setting
+        kw['fork_test'] = True
+    return entry
+
+
 def disable_process_reveal_deadlines(fn):
     """
     Decorator to make a function execute with `process_reveal_deadlines` OFF.

tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py

@@ -1,20 +1,22 @@
-from operator import attrgetter
-
 from eth2spec.test.context import (
-    PHASE0, LIGHTCLIENT_PATCH,
+    LIGHTCLIENT_PATCH,
-    spec_state_test, with_phases,
+    with_phases,
-    with_custom_state,
+    with_custom_state, fork_test,
     spec_test, with_state,
     low_balances, misc_balances, large_validator_set,
 )
+from eth2spec.test.utils import with_meta_tags
 from eth2spec.test.helpers.state import (
-    next_slots,
     next_epoch,
     next_epoch_via_block,
-    transition_to_slot_via_block,
 )
 
 
+HF1_FORK_TEST_META_TAGS = {
+    'fork': 'altair',
+}
+
+
 def run_fork_test(spec, pre_state):
     yield 'pre', pre_state
 
@@ -51,54 +53,68 @@ def run_fork_test(spec, pre_state):
     yield 'post', post_state
 
 
-@with_phases(([PHASE0]))
+@with_phases(([LIGHTCLIENT_PATCH]))
 @with_state
 @spec_test
+@fork_test
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_base_state(spec, phases, state):
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
 
 
-@with_phases(([PHASE0]))
+@with_phases(([LIGHTCLIENT_PATCH]))
 @with_state
 @spec_test
+@fork_test
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_next_epoch(spec, phases, state):
     next_epoch(spec, state)
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
 
 
-@with_phases(([PHASE0]))
+@with_phases(([LIGHTCLIENT_PATCH]))
 @with_state
 @spec_test
+@fork_test
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_next_epoch_with_block(spec, phases, state):
     next_epoch_via_block(spec, state)
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
 
 
-@with_phases(([PHASE0]))
+@with_phases(([LIGHTCLIENT_PATCH]))
 @with_state
 @spec_test
+@fork_test
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_many_next_epoch(spec, phases, state):
     for _ in range(3):
         next_epoch(spec, state)
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
 
 
-@with_phases(([PHASE0]))
+@with_phases(([LIGHTCLIENT_PATCH]))
 @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE)
 @spec_test
+@fork_test
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_random_low_balances(spec, phases, state):
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
 
 
-@with_phases(([PHASE0]))
+@with_phases(([LIGHTCLIENT_PATCH]))
 @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE)
 @spec_test
+@fork_test
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_random_misc_balances(spec, phases, state):
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
 
 
-@with_phases(([PHASE0]))
+@with_phases(([LIGHTCLIENT_PATCH]))
 @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE)
 @spec_test
+@fork_test
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
 def test_fork_random_large_validator_set(spec, phases, state):
     yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)

tests/formats/forks/README.md

@@ -0,0 +1,47 @@
+# Forks
+
+The aim of the fork tests is to ensure that a pre-fork state can be transformed
+ into a valid post-fork state, utilizing the `upgrade` function found in the relevant `fork.md` spec.
+
+There is only one handler: `core`. Each fork (after genesis) is handled with the same format,
+ and the particular fork boundary being tested is noted in `meta.yaml`.
+
+## Test case format
+
+### `meta.yaml`
+
+A yaml file to signify which fork boundary is being tested.
+
+```yaml
+fork: str    -- Fork being transitioned to
+```
+
+#### Fork strings
+
+Key of valid `fork` strings that might be found in `meta.yaml`
+
+| String ID | Pre-fork | Post-fork | Function |
+| - | - | - | - |
+| `altair` | Phase 0 | Altair | `upgrade_to_lightclient_patch` |
+
+### `pre.yaml`
+
+A YAML-encoded `BeaconState`, the state before running the fork transition.
+
+Also available as `pre.ssz`.
+
+### `post.yaml`
+
+A YAML-encoded `BeaconState`, the state after applying the fork transition.
+
+Also available as `post.ssz`.
+
+*Note*: This type is the `BeaconState` after the fork and is *not* the same type as `pre`.
+
+## Processing
+
+To process this test, pass `pre` into the upgrade function defined by the `fork` in `meta.yaml`.
+
+## Condition
+
+The resulting state should match the expected `post`.

tests/generators/forks/main.py

@@ -0,0 +1,40 @@
+from importlib import reload
+from typing import Iterable
+
+from eth2spec.test.context import LIGHTCLIENT_PATCH
+from eth2spec.config import config_util
+from eth2spec.test.lightclient_patch.fork import test_fork as test_altair_forks
+from eth2spec.phase0 import spec as spec_phase0
+
+from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
+from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests
+
+
+pre_specs = {
+    LIGHTCLIENT_PATCH: spec_phase0,
+}
+
+
+def create_provider(fork_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
+
+    def prepare_fn(configs_path: str) -> str:
+        config_util.prepare_config(configs_path, config_name)
+        reload(pre_specs[fork_name])
+        return config_name
+
+    def cases_fn() -> Iterable[gen_typing.TestCase]:
+        return generate_from_tests(
+            runner_name='forks',
+            handler_name='core',
+            src=tests_src,
+            fork_name=fork_name,
+        )
+
+    return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+
+
+if __name__ == "__main__":
+    gen_runner.run_generator("forks", [
+        create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'),
+        create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'),
+    ])

tests/generators/forks/requirements.txt

@@ -0,0 +1,2 @@
+pytest>=4.4
+../../../