add transition spec test format

tests/core/pyspec/eth2spec/test/altair/transition/test_transition.py

@@ -0,0 +1,51 @@
+from eth2spec.test.context import (
+    fork_transition_test,
+    single_phase,
+    with_custom_state,
+    default_activation_threshold,
+    low_balances,
+)
+from eth2spec.test.helpers.constants import PHASE0, ALTAIR
+from eth2spec.test.helpers.state import state_transition_and_sign_block
+from eth2spec.test.helpers.block import build_empty_block_for_next_slot
+
+
+@fork_transition_test(PHASE0, ALTAIR, fork_epoch=2)
+def test_normal_transition(state, fork_epoch, spec, post_spec, pre_tag, post_tag):
+    yield "pre", state
+
+    blocks = []
+    for slot in range(state.slot, fork_epoch * spec.SLOTS_PER_EPOCH):
+        block = build_empty_block_for_next_slot(spec, state)
+        state_transition_and_sign_block(spec, state, block)
+        blocks.append(pre_tag(block))
+
+    state = post_spec.upgrade_to_altair(state)
+
+    assert state.fork.epoch == fork_epoch
+    assert state.fork.previous_version == post_spec.GENESIS_FORK_VERSION
+    assert state.fork.current_version == post_spec.ALTAIR_FORK_VERSION
+
+    block = build_empty_block_for_next_slot(post_spec, state)
+    state_transition_and_sign_block(post_spec, state, block)
+    blocks.append(post_tag(block))
+
+    yield "blocks", blocks
+    yield "post", state
+
+
+@fork_transition_test(PHASE0, ALTAIR)
+def test_normal_transition_with_manual_fork_epoch(state, spec, post_spec, pre_tag, post_tag):
+    fork_epoch = 2
+    yield "fork_epoch", "meta", fork_epoch
+
+    # run test with computed fork_epoch...
+
+
+@fork_transition_test(PHASE0, ALTAIR, fork_epoch=2)
+@with_custom_state(low_balances, default_activation_threshold)
+@single_phase
+def test_normal_transition_with_low_balances(state, fork_epoch, spec, post_spec, pre_tag, post_tag):
+    yield "pre", state
+
+    # run test with custom state...

tests/core/pyspec/eth2spec/test/context.py

@@ -11,7 +11,7 @@ from .helpers.constants import (
     ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_MERGE,
 )
 from .helpers.genesis import create_genesis_state
-from .utils import vector_test, with_meta_tags
+from .utils import vector_test, with_meta_tags, build_transition_test
 
 from random import Random
 from typing import Any, Callable, Sequence, TypedDict, Protocol
@@ -383,3 +383,38 @@ def is_post_merge(spec):
 
 with_altair_and_later = with_phases([ALTAIR])  # TODO: include Merge, but not until Merge work is rebased.
 with_merge_and_later = with_phases([MERGE])
+
+
+def fork_transition_test(pre_fork_name, post_fork_name, fork_epoch=None):
+    """
+    A decorator to construct a "transition" test from one fork of the eth2 spec
+    to another.
+
+    Decorator assumes a transition from the `pre_fork_name` fork to the
+    `post_fork_name` fork. The user can supply a `fork_epoch` at which the
+    fork occurs or they must compute one (yielding to the generator) during the test
+    if more custom behavior is desired.
+
+    A test using this decorator should expect to receive as parameters:
+    `state`: the default state constructed for the `pre_fork_name` fork
+        according to the `with_state` decorator.
+    `fork_epoch`: the `fork_epoch` provided to this decorator, if given.
+    `spec`: the version of the eth2 spec corresponding to `pre_fork_name`.
+    `post_spec`: the version of the eth2 spec corresponding to `post_fork_name`.
+    `pre_tag`: a function to tag data as belonging to `pre_fork_name` fork.
+        Used to discriminate data during consumption of the generated spec tests.
+    `post_tag`: a function to tag data as belonging to `post_fork_name` fork.
+        Used to discriminate data during consumption of the generated spec tests.
+    """
+    def _wrapper(fn):
+        @with_phases([pre_fork_name], other_phases=[post_fork_name])
+        @spec_test
+        @with_state
+        def _adapter(*args, **kwargs):
+            wrapped = build_transition_test(fn,
+                                            pre_fork_name,
+                                            post_fork_name,
+                                            fork_epoch=fork_epoch)
+            return wrapped(*args, **kwargs)
+        return _adapter
+    return _wrapper

tests/core/pyspec/eth2spec/test/utils.py

@@ -1,5 +1,6 @@
+import inspect
 from typing import Dict, Any
-from eth2spec.utils.ssz.ssz_typing import View
+from eth2spec.utils.ssz.ssz_typing import View, boolean, Container
 from eth2spec.utils.ssz.ssz_impl import serialize
 
 
@@ -93,3 +94,48 @@ def with_meta_tags(tags: Dict[str, Any]):
                     yield k, 'meta', v
         return entry
     return runner
+
+
+class FlaggedContainer(Container):
+    flag: boolean
+    obj: Container
+
+
+def build_transition_test(fn, pre_fork_name, post_fork_name, fork_epoch=None):
+    """
+    Handles the inner plumbing to generate `transition_test`s.
+    See that decorator in `context.py` for more information.
+    """
+    def _adapter(*args, **kwargs):
+        post_spec = kwargs["phases"][post_fork_name]
+
+        def pre_tag(obj):
+            return FlaggedContainer(flag=False, obj=obj)
+
+        def post_tag(obj):
+            return FlaggedContainer(flag=True, obj=obj)
+
+        yield "post_fork", "meta", post_fork_name
+
+        has_fork_epoch = False
+        if fork_epoch:
+            kwargs["fork_epoch"] = fork_epoch
+            has_fork_epoch = True
+            yield "fork_epoch", "meta", fork_epoch
+
+        # massage args to handle an optional custom state using
+        # `with_custom_state` decorator
+        expected_args = inspect.getfullargspec(fn)
+        if "phases" not in expected_args.kwonlyargs:
+            kwargs.pop("phases", None)
+
+        for part in fn(*args,
+                       post_spec=post_spec,
+                       pre_tag=pre_tag,
+                       post_tag=post_tag,
+                       **kwargs):
+            if part[0] == "fork_epoch":
+                has_fork_epoch = True
+            yield part
+        assert has_fork_epoch
+    return _adapter

tests/formats/transition/README.md

@@ -0,0 +1,72 @@
+# Transition testing
+
+Transition tests to cover processing the chain across a fork boundary.
+
+Each test case contains a `post_fork` key in the `meta.yaml` that indicates the target fork which also fixes the fork the test begins in.
+
+Clients should assume forks happen sequentially in the following manner:
+
+0. `phase0`
+1. `altair`
+
+For example, if a test case has `post_fork` of `altair`, the test consumer should assume the test begins in `phase0` and use that specification to process the initial state and any blocks up until the fork epoch. After the fork happens, the test consumer should use the specification according to the `altair` fork to process the remaining data.
+
+## Encoding notes
+
+This test type contains objects that span fork boundaries.
+In general, it may not be clear which objects belong to which fork so each
+object is prefixed with a SSZ `boolean` to indicate if the object belongs to the post fork or if it belongs to the initial fork.
+This "flagged" data should be used to select the appropriate version of the spec when interpreting the enclosed object.
+
+```python
+class FlaggedContainer(Container):
+    flag: boolean
+    obj: Container
+```
+
+If `flag` is `False`, then the `obj` belongs to the **initial** fork.
+If `flag` is `True`, then the `obj` belongs to the **post** fork.
+
+Unless stated otherwise, all references to spec types below refer to SSZ-snappy
+encoded data `obj` with the relevant `flag` set:
+`FlaggedContainer(flag=flag, obj=obj)`.
+
+For example, when testing the fork from Phase 0 to Altair, an Altair block is given
+as the encoding of `FlaggedContainer(flag=True, obj=SignedBeaconBlock())` where
+`SignedBeaconBlock` is the type defined in the Altair spec.
+
+## Test case format
+
+### `meta.yaml`
+
+```yaml
+post_fork: string              -- String name of the spec after the fork.
+fork_epoch: int                -- The epoch at which the fork takes place.
+blocks_count: int              -- The number of blocks processed in this test.
+```
+
+*Note*: There may be a fork transition function to run at the `fork_epoch`. Refer to the specs for the relevant fork for further details.
+
+### `pre.ssz_snappy`
+
+A SSZ-snappy encoded `BeaconState` according to the specification of the initial fork, the state before running the block transitions.
+
+*NOTE*: This object is _not_ "flagged" as it is assumed to always belong to the post fork.
+
+### `blocks_<index>.ssz_snappy`
+
+A series of files, with `<index>` in range `[0, blocks_count)`.
+Blocks must be processed in order, following the main transition function
+(i.e. process slot and epoch transitions in between blocks as normal).
+
+Blocks are encoded as `SignedBeaconBlock`s from the relevant spec version indicated by flag data as described in the `Encoding notes`.
+
+### `post.ssz_snappy`
+
+A SSZ-snappy encoded `BeaconState` according to the specification of the post fork, the state after running the block transitions.
+
+*NOTE*: This object is _not_ "flagged" as it is assumed to always belong to the post fork.
+
+## Condition
+
+The resulting state should match the expected `post` state.

tests/generators/transition/main.py

@@ -0,0 +1,42 @@
+from importlib import reload
+from typing import Iterable
+
+from eth2spec.test.helpers.constants import ALTAIR, MINIMAL, MAINNET, PHASE0
+from eth2spec.config import config_util
+from eth2spec.test.altair.transition import test_transition as test_altair_transition
+from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.altair import spec as spec_altair
+
+from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
+from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests
+
+
+def create_provider(tests_src, config_name: str, pre_fork_name: str, post_fork_name: str) -> gen_typing.TestProvider:
+
+    def prepare_fn(configs_path: str) -> str:
+        config_util.prepare_config(configs_path, config_name)
+        reload(spec_phase0)
+        reload(spec_altair)
+        return config_name
+
+    def cases_fn() -> Iterable[gen_typing.TestCase]:
+        return generate_from_tests(
+            runner_name='transition',
+            handler_name='core',
+            src=tests_src,
+            fork_name=post_fork_name,
+            phase=pre_fork_name,
+        )
+
+    return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+
+
+TRANSITION_TESTS = ((PHASE0, ALTAIR, test_altair_transition),)
+
+
+if __name__ == "__main__":
+    for pre_fork, post_fork, transition_test_module in TRANSITION_TESTS:
+        gen_runner.run_generator("transition", [
+            create_provider(transition_test_module, MINIMAL, pre_fork, post_fork),
+            create_provider(transition_test_module, MAINNET, pre_fork, post_fork),
+        ])

tests/generators/transition/requirements.txt

@@ -0,0 +1,2 @@
+pytest>=4.4
+../../../[generator]