Merge branch 'dev' into leak-scores

setup.py

@@ -449,7 +449,7 @@ class PySpecCommand(Command):
                     specs/phase1/beacon-chain.md
                     specs/phase1/shard-transition.md
                     specs/phase1/fork-choice.md
-                    specs/phase1/phase1-fork.md
+                    specs/phase1/fork.md
                     specs/phase1/shard-fork-choice.md
                     specs/phase1/validator.md
                 """
@@ -460,7 +460,7 @@ class PySpecCommand(Command):
                     specs/phase0/validator.md
                     specs/phase0/weak-subjectivity.md
                     specs/lightclient/beacon-chain.md
-                    specs/lightclient/lightclient-fork.md
+                    specs/lightclient/fork.md
                 """
                 # TODO: add specs/lightclient/sync-protocol.md back when the GeneralizedIndex helpers are included.
             else:

tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py

@@ -1,6 +1,6 @@
 from importlib import reload, import_module
 from inspect import getmembers, isfunction
-from typing import Any, Callable, Dict, Iterable
+from typing import Any, Callable, Dict, Iterable, Optional
 
 from eth2spec.config import config_util
 from eth2spec.utils import bls
@@ -11,21 +11,27 @@ from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider
 
 
 def generate_from_tests(runner_name: str, handler_name: str, src: Any,
-                        fork_name: SpecForkName, bls_active: bool = True) -> Iterable[TestCase]:
+                        fork_name: SpecForkName, bls_active: bool = True,
+                        phase: Optional[str]=None) -> Iterable[TestCase]:
     """
     Generate a list of test cases by running tests from the given src in generator-mode.
     :param runner_name: to categorize the test in general as.
     :param handler_name: to categorize the test specialization as.
     :param src: to retrieve tests from (discovered using inspect.getmembers).
-    :param fork_name: to run tests against particular phase and/or fork.
+    :param fork_name: the folder name for these tests.
            (if multiple forks are applicable, indicate the last fork)
     :param bls_active: optional, to override BLS switch preference. Defaults to True.
+    :param phase: optional, to run tests against a particular spec version. Default to `fork_name` value.
     :return: an iterable of test cases.
     """
     fn_names = [
         name for (name, _) in getmembers(src, isfunction)
         if name.startswith('test_')
     ]
+
+    if phase is None:
+        phase = fork_name
+
     print("generating test vectors from tests source: %s" % src.__name__)
     for name in fn_names:
         tfn = getattr(src, name)
@@ -42,7 +48,7 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any,
             suite_name='pyspec_tests',
             case_name=case_name,
             # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent.
-            case_fn=lambda: tfn(generator_mode=True, phase=fork_name, bls_active=bls_active)
+            case_fn=lambda: tfn(generator_mode=True, phase=phase, bls_active=bls_active)
         )
 
 

tests/core/pyspec/eth2spec/test/context.py

@@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca
         # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper.
         # Decide based on performance/consistency results later.
         state = phases[PHASE1].upgrade_to_phase1(state)
-    elif spec.fork == LIGHTCLIENT_PATCH:  # not generalizing this just yet, unclear final spec fork/patch order.
+    elif spec.fork == LIGHTCLIENT_PATCH:
         state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state)
 
     return state
@@ -340,12 +340,13 @@ def with_phases(phases, other_phases=None):
 
             available_phases = set(run_phases)
             if other_phases is not None:
-                available_phases += set(other_phases)
+                available_phases |= set(other_phases)
 
             # TODO: test state is dependent on phase0 but is immediately transitioned to phase1.
             #  A new state-creation helper for phase 1 may be in place, and then phase1+ tests can run without phase0
             available_phases.add(PHASE0)
 
+            # Populate all phases for multi-phase tests
             phase_dir = {}
             if PHASE0 in available_phases:
                 phase_dir[PHASE0] = spec_phase0

tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py

@@ -0,0 +1,113 @@
+from eth2spec.test.context import (
+    PHASE0, LIGHTCLIENT_PATCH,
+    with_phases,
+    with_custom_state,
+    spec_test, with_state,
+    low_balances, misc_balances, large_validator_set,
+)
+from eth2spec.test.utils import with_meta_tags
+from eth2spec.test.helpers.state import (
+    next_epoch,
+    next_epoch_via_block,
+)
+
+
+HF1_FORK_TEST_META_TAGS = {
+    'fork': 'altair',
+}
+
+
+def run_fork_test(post_spec, pre_state):
+    yield 'pre', pre_state
+
+    post_state = post_spec.upgrade_to_lightclient_patch(pre_state)
+
+    # Stable fields
+    stable_fields = [
+        'genesis_time', 'genesis_validators_root', 'slot',
+        # History
+        'latest_block_header', 'block_roots', 'state_roots', 'historical_roots',
+        # Eth1
+        'eth1_data', 'eth1_data_votes', 'eth1_deposit_index',
+        # Registry
+        'validators', 'balances',
+        # Randomness
+        'randao_mixes',
+        # Slashings
+        'slashings',
+        # Finality
+        'justification_bits', 'previous_justified_checkpoint', 'current_justified_checkpoint', 'finalized_checkpoint',
+    ]
+    for field in stable_fields:
+        assert getattr(pre_state, field) == getattr(post_state, field)
+
+    # Modified fields
+    modified_fields = ['fork']
+    for field in modified_fields:
+        assert getattr(pre_state, field) != getattr(post_state, field)
+
+    assert pre_state.fork.current_version == post_state.fork.previous_version
+    assert post_state.fork.current_version == post_spec.LIGHTCLIENT_PATCH_FORK_VERSION
+    assert post_state.fork.epoch == post_spec.get_current_epoch(post_state)
+
+    yield 'post', post_state
+
+
+@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH])
+@spec_test
+@with_state
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
+def test_fork_base_state(spec, phases, state):
+    yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
+
+
+@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH])
+@spec_test
+@with_state
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
+def test_fork_next_epoch(spec, phases, state):
+    next_epoch(spec, state)
+    yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
+
+
+@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH])
+@spec_test
+@with_state
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
+def test_fork_next_epoch_with_block(spec, phases, state):
+    next_epoch_via_block(spec, state)
+    yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
+
+
+@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH])
+@spec_test
+@with_state
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
+def test_fork_many_next_epoch(spec, phases, state):
+    for _ in range(3):
+        next_epoch(spec, state)
+    yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
+
+
+@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH])
+@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE)
+@spec_test
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
+def test_fork_random_low_balances(spec, phases, state):
+    yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
+
+
+@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH])
+@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE)
+@spec_test
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
+def test_fork_random_misc_balances(spec, phases, state):
+    yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)
+
+
+@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH])
+@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE)
+@spec_test
+@with_meta_tags(HF1_FORK_TEST_META_TAGS)
+def test_fork_random_large_validator_set(spec, phases, state):
+    yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state)

tests/formats/forks/README.md

@@ -0,0 +1,47 @@
+# Forks
+
+The aim of the fork tests is to ensure that a pre-fork state can be transformed
+ into a valid post-fork state, utilizing the `upgrade` function found in the relevant `fork.md` spec.
+
+There is only one handler: `fork`. Each fork (after genesis) is handled with the same format,
+ and the particular fork boundary being tested is noted in `meta.yaml`.
+
+## Test case format
+
+### `meta.yaml`
+
+A yaml file to signify which fork boundary is being tested.
+
+```yaml
+fork: str    -- Fork being transitioned to
+```
+
+#### Fork strings
+
+Key of valid `fork` strings that might be found in `meta.yaml`
+
+| String ID | Pre-fork | Post-fork | Function |
+| - | - | - | - |
+| `altair` | Phase 0 | Altair | `upgrade_to_lightclient_patch` |
+
+### `pre.yaml`
+
+A YAML-encoded `BeaconState`, the state before running the fork transition.
+
+Also available as `pre.ssz`.
+
+### `post.yaml`
+
+A YAML-encoded `BeaconState`, the state after applying the fork transition.
+
+Also available as `post.ssz`.
+
+*Note*: This type is the `BeaconState` after the fork and is *not* the same type as `pre`.
+
+## Processing
+
+To process this test, pass `pre` into the upgrade function defined by the `fork` in `meta.yaml`.
+
+## Condition
+
+The resulting state should match the expected `post`.

tests/generators/forks/main.py

@@ -0,0 +1,38 @@
+from importlib import reload
+from typing import Iterable
+
+from eth2spec.test.context import PHASE0, LIGHTCLIENT_PATCH, MINIMAL, MAINNET
+from eth2spec.config import config_util
+from eth2spec.test.lightclient_patch.fork import test_fork as test_altair_forks
+from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.lightclient_patch import spec as spec_lightclient_patch
+
+from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
+from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests
+
+
+def create_provider(tests_src, config_name: str, phase: str, fork_name: str) -> gen_typing.TestProvider:
+
+    def prepare_fn(configs_path: str) -> str:
+        config_util.prepare_config(configs_path, config_name)
+        reload(spec_phase0)
+        reload(spec_lightclient_patch)
+        return config_name
+
+    def cases_fn() -> Iterable[gen_typing.TestCase]:
+        return generate_from_tests(
+            runner_name='fork',
+            handler_name='fork',
+            src=tests_src,
+            fork_name=fork_name,
+            phase=phase,
+        )
+
+    return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+
+
+if __name__ == "__main__":
+    gen_runner.run_generator("forks", [
+        create_provider(test_altair_forks, MINIMAL, PHASE0, LIGHTCLIENT_PATCH),
+        create_provider(test_altair_forks, MAINNET, PHASE0, LIGHTCLIENT_PATCH),
+    ])

tests/generators/forks/requirements.txt

@@ -0,0 +1,2 @@
+pytest>=4.4
+../../../