Merge pull request #2363 from ralexstokes/add-altair-fork-transition-test

Add fork transition spec tests

tests/core/pyspec/eth2spec/test/altair/transition/test_transition.py

@@ -0,0 +1,244 @@
+from eth2spec.test.context import fork_transition_test
+from eth2spec.test.helpers.constants import PHASE0, ALTAIR
+from eth2spec.test.helpers.state import state_transition_and_sign_block, next_slot
+from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_empty_block, sign_block
+
+
+def _state_transition_and_sign_block_at_slot(spec, state):
+    """
+    Cribbed from ``transition_unsigned_block`` helper
+    where the early parts of the state transition have already
+    been applied to ``state``.
+
+    Used to produce a block during an irregular state transition.
+    """
+    block = build_empty_block(spec, state)
+
+    assert state.latest_block_header.slot < block.slot
+    assert state.slot == block.slot
+    spec.process_block(state, block)
+    block.state_root = state.hash_tree_root()
+    return sign_block(spec, state, block)
+
+
+def _all_blocks(_):
+    return True
+
+
+def _skip_slots(*slots):
+    """
+    Skip making a block if its slot is
+    passed as an argument to this filter
+    """
+    def f(state_at_prior_slot):
+        return state_at_prior_slot.slot + 1 not in slots
+    return f
+
+
+def _no_blocks(_):
+    return False
+
+
+def _only_at(slot):
+    """
+    Only produce a block if its slot is ``slot``.
+    """
+    def f(state_at_prior_slot):
+        return state_at_prior_slot.slot + 1 == slot
+    return f
+
+
+def _state_transition_across_slots(spec, state, to_slot, block_filter=_all_blocks):
+    assert state.slot < to_slot
+    while state.slot < to_slot:
+        should_make_block = block_filter(state)
+        if should_make_block:
+            block = build_empty_block_for_next_slot(spec, state)
+            signed_block = state_transition_and_sign_block(spec, state, block)
+            yield signed_block
+        else:
+            next_slot(spec, state)
+
+
+def _do_altair_fork(state, spec, post_spec, fork_epoch, with_block=True):
+    spec.process_slots(state, state.slot + 1)
+
+    assert state.slot % spec.SLOTS_PER_EPOCH == 0
+    assert spec.compute_epoch_at_slot(state.slot) == fork_epoch
+
+    state = post_spec.upgrade_to_altair(state)
+
+    assert state.fork.epoch == fork_epoch
+    assert state.fork.previous_version == post_spec.GENESIS_FORK_VERSION
+    assert state.fork.current_version == post_spec.ALTAIR_FORK_VERSION
+
+    if with_block:
+        return state, _state_transition_and_sign_block_at_slot(post_spec, state)
+    else:
+        return state, None
+
+
+@fork_transition_test(PHASE0, ALTAIR, fork_epoch=2)
+def test_normal_transition(state, fork_epoch, spec, post_spec, pre_tag, post_tag):
+    """
+    Transition from the initial ``state`` to the epoch after the ``fork_epoch``,
+    producing blocks for every slot along the way.
+    """
+    yield "pre", state
+
+    assert spec.get_current_epoch(state) < fork_epoch
+
+    # regular state transition until fork:
+    to_slot = fork_epoch * spec.SLOTS_PER_EPOCH - 1
+    blocks = []
+    blocks.extend([
+        pre_tag(block) for block in
+        _state_transition_across_slots(spec, state, to_slot)
+    ])
+
+    # irregular state transition to handle fork:
+    state, block = _do_altair_fork(state, spec, post_spec, fork_epoch)
+    blocks.append(post_tag(block))
+
+    # continue regular state transition with new spec into next epoch
+    to_slot = post_spec.SLOTS_PER_EPOCH + state.slot
+    blocks.extend([
+        post_tag(block) for block in
+        _state_transition_across_slots(post_spec, state, to_slot)
+    ])
+
+    assert state.slot % post_spec.SLOTS_PER_EPOCH == 0
+    assert post_spec.compute_epoch_at_slot(state.slot) == fork_epoch + 1
+
+    slots_with_blocks = [block.message.slot for block in blocks]
+    assert len(set(slots_with_blocks)) == len(slots_with_blocks)
+    assert set(range(1, state.slot + 1)) == set(slots_with_blocks)
+
+    yield "blocks", blocks
+    yield "post", state
+
+
+@fork_transition_test(PHASE0, ALTAIR, fork_epoch=2)
+def test_transition_missing_first_post_block(state, fork_epoch, spec, post_spec, pre_tag, post_tag):
+    """
+    Transition from the initial ``state`` to the epoch after the ``fork_epoch``,
+    producing blocks for every slot along the way except for the first block
+    of the new fork.
+    """
+    yield "pre", state
+
+    assert spec.get_current_epoch(state) < fork_epoch
+
+    # regular state transition until fork:
+    to_slot = fork_epoch * spec.SLOTS_PER_EPOCH - 1
+    blocks = []
+    blocks.extend([
+        pre_tag(block) for block in
+        _state_transition_across_slots(spec, state, to_slot)
+    ])
+
+    # irregular state transition to handle fork:
+    state, _ = _do_altair_fork(state, spec, post_spec, fork_epoch, with_block=False)
+
+    # continue regular state transition with new spec into next epoch
+    to_slot = post_spec.SLOTS_PER_EPOCH + state.slot
+    blocks.extend([
+        post_tag(block) for block in
+        _state_transition_across_slots(post_spec, state, to_slot)
+    ])
+
+    assert state.slot % post_spec.SLOTS_PER_EPOCH == 0
+    assert post_spec.compute_epoch_at_slot(state.slot) == fork_epoch + 1
+
+    slots_with_blocks = [block.message.slot for block in blocks]
+    assert len(set(slots_with_blocks)) == len(slots_with_blocks)
+    expected_slots = set(range(1, state.slot + 1)).difference(set([fork_epoch * spec.SLOTS_PER_EPOCH]))
+    assert expected_slots == set(slots_with_blocks)
+
+    yield "blocks", blocks
+    yield "post", state
+
+
+@fork_transition_test(PHASE0, ALTAIR, fork_epoch=2)
+def test_transition_missing_last_pre_fork_block(state, fork_epoch, spec, post_spec, pre_tag, post_tag):
+    """
+    Transition from the initial ``state`` to the epoch after the ``fork_epoch``,
+    producing blocks for every slot along the way except for the last block
+    of the old fork.
+    """
+    yield "pre", state
+
+    assert spec.get_current_epoch(state) < fork_epoch
+
+    # regular state transition until fork:
+    last_slot_of_pre_fork = fork_epoch * spec.SLOTS_PER_EPOCH - 1
+    to_slot = last_slot_of_pre_fork
+    blocks = []
+    blocks.extend([
+        pre_tag(block) for block in
+        _state_transition_across_slots(spec, state, to_slot, block_filter=_skip_slots(last_slot_of_pre_fork))
+    ])
+
+    # irregular state transition to handle fork:
+    state, block = _do_altair_fork(state, spec, post_spec, fork_epoch)
+    blocks.append(post_tag(block))
+
+    # continue regular state transition with new spec into next epoch
+    to_slot = post_spec.SLOTS_PER_EPOCH + state.slot
+    blocks.extend([
+        post_tag(block) for block in
+        _state_transition_across_slots(post_spec, state, to_slot)
+    ])
+
+    assert state.slot % post_spec.SLOTS_PER_EPOCH == 0
+    assert post_spec.compute_epoch_at_slot(state.slot) == fork_epoch + 1
+
+    slots_with_blocks = [block.message.slot for block in blocks]
+    assert len(set(slots_with_blocks)) == len(slots_with_blocks)
+    expected_slots = set(range(1, state.slot + 1)).difference(set([last_slot_of_pre_fork]))
+    assert expected_slots == set(slots_with_blocks)
+
+    yield "blocks", blocks
+    yield "post", state
+
+
+@fork_transition_test(PHASE0, ALTAIR, fork_epoch=2)
+def test_transition_only_blocks_post_fork(state, fork_epoch, spec, post_spec, pre_tag, post_tag):
+    """
+    Transition from the initial ``state`` to the epoch after the ``fork_epoch``,
+    skipping blocks for every slot along the way except for the first block
+    in the ending epoch.
+    """
+    yield "pre", state
+
+    assert spec.get_current_epoch(state) < fork_epoch
+
+    # regular state transition until fork:
+    last_slot_of_pre_fork = fork_epoch * spec.SLOTS_PER_EPOCH - 1
+    to_slot = last_slot_of_pre_fork
+    blocks = []
+    blocks.extend([
+        pre_tag(block) for block in
+        _state_transition_across_slots(spec, state, to_slot, block_filter=_no_blocks)
+    ])
+
+    # irregular state transition to handle fork:
+    state, _ = _do_altair_fork(state, spec, post_spec, fork_epoch, with_block=False)
+
+    # continue regular state transition with new spec into next epoch
+    to_slot = post_spec.SLOTS_PER_EPOCH + state.slot
+    last_slot = (fork_epoch + 1) * post_spec.SLOTS_PER_EPOCH
+    blocks.extend([
+        post_tag(block) for block in
+        _state_transition_across_slots(post_spec, state, to_slot, block_filter=_only_at(last_slot))
+    ])
+
+    assert state.slot % post_spec.SLOTS_PER_EPOCH == 0
+    assert post_spec.compute_epoch_at_slot(state.slot) == fork_epoch + 1
+
+    slots_with_blocks = [block.message.slot for block in blocks]
+    assert len(slots_with_blocks) == 1
+    assert slots_with_blocks[0] == last_slot
+
+    yield "blocks", blocks
+    yield "post", state

tests/core/pyspec/eth2spec/test/context.py

@@ -11,7 +11,7 @@ from .helpers.constants import (
     ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_MERGE,
 )
 from .helpers.genesis import create_genesis_state
-from .utils import vector_test, with_meta_tags
+from .utils import vector_test, with_meta_tags, build_transition_test
 
 from random import Random
 from typing import Any, Callable, Sequence, TypedDict, Protocol
@@ -383,3 +383,38 @@ def is_post_merge(spec):
 
 with_altair_and_later = with_phases([ALTAIR])  # TODO: include Merge, but not until Merge work is rebased.
 with_merge_and_later = with_phases([MERGE])
+
+
+def fork_transition_test(pre_fork_name, post_fork_name, fork_epoch=None):
+    """
+    A decorator to construct a "transition" test from one fork of the eth2 spec
+    to another.
+
+    Decorator assumes a transition from the `pre_fork_name` fork to the
+    `post_fork_name` fork. The user can supply a `fork_epoch` at which the
+    fork occurs or they must compute one (yielding to the generator) during the test
+    if more custom behavior is desired.
+
+    A test using this decorator should expect to receive as parameters:
+    `state`: the default state constructed for the `pre_fork_name` fork
+        according to the `with_state` decorator.
+    `fork_epoch`: the `fork_epoch` provided to this decorator, if given.
+    `spec`: the version of the eth2 spec corresponding to `pre_fork_name`.
+    `post_spec`: the version of the eth2 spec corresponding to `post_fork_name`.
+    `pre_tag`: a function to tag data as belonging to `pre_fork_name` fork.
+        Used to discriminate data during consumption of the generated spec tests.
+    `post_tag`: a function to tag data as belonging to `post_fork_name` fork.
+        Used to discriminate data during consumption of the generated spec tests.
+    """
+    def _wrapper(fn):
+        @with_phases([pre_fork_name], other_phases=[post_fork_name])
+        @spec_test
+        @with_state
+        def _adapter(*args, **kwargs):
+            wrapped = build_transition_test(fn,
+                                            pre_fork_name,
+                                            post_fork_name,
+                                            fork_epoch=fork_epoch)
+            return wrapped(*args, **kwargs)
+        return _adapter
+    return _wrapper

tests/core/pyspec/eth2spec/test/utils.py

@@ -1,3 +1,4 @@
+import inspect
 from typing import Dict, Any
 from eth2spec.utils.ssz.ssz_typing import View
 from eth2spec.utils.ssz.ssz_impl import serialize
@@ -93,3 +94,50 @@ def with_meta_tags(tags: Dict[str, Any]):
                     yield k, 'meta', v
         return entry
     return runner
+
+
+def build_transition_test(fn, pre_fork_name, post_fork_name, fork_epoch=None):
+    """
+    Handles the inner plumbing to generate `transition_test`s.
+    See that decorator in `context.py` for more information.
+    """
+    def _adapter(*args, **kwargs):
+        post_spec = kwargs["phases"][post_fork_name]
+
+        pre_fork_counter = 0
+
+        def pre_tag(obj):
+            nonlocal pre_fork_counter
+            pre_fork_counter += 1
+            return obj
+
+        def post_tag(obj):
+            return obj
+
+        yield "post_fork", "meta", post_fork_name
+
+        has_fork_epoch = False
+        if fork_epoch:
+            kwargs["fork_epoch"] = fork_epoch
+            has_fork_epoch = True
+            yield "fork_epoch", "meta", fork_epoch
+
+        # massage args to handle an optional custom state using
+        # `with_custom_state` decorator
+        expected_args = inspect.getfullargspec(fn)
+        if "phases" not in expected_args.kwonlyargs:
+            kwargs.pop("phases", None)
+
+        for part in fn(*args,
+                       post_spec=post_spec,
+                       pre_tag=pre_tag,
+                       post_tag=post_tag,
+                       **kwargs):
+            if part[0] == "fork_epoch":
+                has_fork_epoch = True
+            yield part
+        assert has_fork_epoch
+
+        if pre_fork_counter > 0:
+            yield "fork_block", "meta", pre_fork_counter - 1
+    return _adapter

tests/formats/transition/README.md

@@ -0,0 +1,72 @@
+# Transition testing
+
+Transition tests to cover processing the chain across a fork boundary.
+
+Each test case contains a `post_fork` key in the `meta.yaml` that indicates the target fork which also fixes the fork the test begins in.
+
+Clients should assume forks happen sequentially in the following manner:
+
+0. `phase0`
+1. `altair`
+
+For example, if a test case has `post_fork` of `altair`, the test consumer should assume the test begins in `phase0` and use that specification to process the initial state and any blocks up until the fork epoch. After the fork happens, the test consumer should use the specification according to the `altair` fork to process the remaining data.
+
+## Test case format
+
+### `meta.yaml`
+
+```yaml
+post_fork: string              -- String name of the spec after the fork.
+fork_epoch: int                -- The epoch at which the fork takes place.
+fork_block: int                -- Optional. The `<index>` of the last block on the initial fork.
+blocks_count: int              -- The number of blocks processed in this test.
+```
+
+*Note*: There may be a fork transition function to run at the `fork_epoch`.
+Refer to the specs for the relevant fork for further details.
+
+### `pre.ssz_snappy`
+
+A SSZ-snappy encoded `BeaconState` according to the specification of
+the initial fork, the state before running the block transitions.
+
+### `blocks_<index>.ssz_snappy`
+
+A series of files, with `<index>` in range `[0, blocks_count)`.
+Blocks must be processed in order, following the main transition function
+(i.e. process slot and epoch transitions in between blocks as normal).
+
+Blocks are encoded as `SignedBeaconBlock`s from the relevant spec version
+as indicated by the `post_fork` and `fork_block` data in the `meta.yaml`.
+
+As blocks span fork boundaires, a `fork_block` number is given in
+the `meta.yaml` to help resolve which blocks belong to which fork.
+
+The `fork_block` is the index in the test data of the **last** block
+of the **initial** fork.
+
+To demonstrate, the following diagram shows slots with `_` and blocks
+in those slots as `x`. The fork happens at the epoch delineated by the `|`.
+
+```
+x   x     x x
+_ _ _ _ | _ _ _ _
+```
+
+The `blocks_count` value in the `meta.yaml` in this case is `4` where the
+`fork_block` value in the `meta.yaml` is `1`. If this particular example were
+testing the fork from Phase 0 to Altair, blocks with indices `0, 1` represent
+`SignedBeaconBlock`s defined in the Phase 0 spec and blocks with indices `2, 3`
+represent `SignedBeaconBlock`s defined in the Altair spec.
+
+*Note*: If `fork_block` is missing, then all block data should be
+interpreted as belonging to the post fork.
+
+### `post.ssz_snappy`
+
+A SSZ-snappy encoded `BeaconState` according to the specification of
+the post fork, the state after running the block transitions.
+
+## Condition
+
+The resulting state should match the expected `post` state.

tests/generators/transition/main.py

@@ -0,0 +1,42 @@
+from importlib import reload
+from typing import Iterable
+
+from eth2spec.test.helpers.constants import ALTAIR, MINIMAL, MAINNET, PHASE0
+from eth2spec.config import config_util
+from eth2spec.test.altair.transition import test_transition as test_altair_transition
+from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.altair import spec as spec_altair
+
+from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
+from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests
+
+
+def create_provider(tests_src, config_name: str, pre_fork_name: str, post_fork_name: str) -> gen_typing.TestProvider:
+
+    def prepare_fn(configs_path: str) -> str:
+        config_util.prepare_config(configs_path, config_name)
+        reload(spec_phase0)
+        reload(spec_altair)
+        return config_name
+
+    def cases_fn() -> Iterable[gen_typing.TestCase]:
+        return generate_from_tests(
+            runner_name='transition',
+            handler_name='core',
+            src=tests_src,
+            fork_name=post_fork_name,
+            phase=pre_fork_name,
+        )
+
+    return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+
+
+TRANSITION_TESTS = ((PHASE0, ALTAIR, test_altair_transition),)
+
+
+if __name__ == "__main__":
+    for pre_fork, post_fork, transition_test_module in TRANSITION_TESTS:
+        gen_runner.run_generator("transition", [
+            create_provider(transition_test_module, MINIMAL, pre_fork, post_fork),
+            create_provider(transition_test_module, MAINNET, pre_fork, post_fork),
+        ])

tests/generators/transition/requirements.txt

@@ -0,0 +1,2 @@
+pytest>=4.4
+../../../[generator]