Merge pull request #2240 from ethereum/altair-genesis-tests

Enable Altair genesis tests + rename `SpecLightclient` -> `SpecAltair` + disable too slow test

specs/altair/fork.md

@@ -9,7 +9,7 @@
 
 - [Introduction](#introduction)
 - [Configuration](#configuration)
-- [Fork to Light-client patch](#fork-to-light-client-patch)
+- [Fork to Altair](#fork-to-altair)
   - [Fork trigger](#fork-trigger)
   - [Upgrading the state](#upgrading-the-state)
 
@@ -28,7 +28,7 @@ Warning: this configuration is not definitive.
 | `ALTAIR_FORK_VERSION` | `Version('0x01000000')` |
 | `ALTAIR_FORK_SLOT` | `Slot(0)` **TBD** |
 
-## Fork to Light-client patch
+## Fork to Altair
 
 ### Fork trigger
 

tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py

@@ -1,7 +1,9 @@
 from eth2spec.test.context import (
     PHASE0, PHASE1,
-    with_all_phases_except,
+    MINIMAL,
     spec_state_test,
+    with_all_phases_except,
+    with_configs,
 )
 from eth2spec.test.helpers.state import transition_to
 from eth2spec.test.helpers.epoch_processing import (
@@ -11,6 +13,7 @@ from eth2spec.test.helpers.epoch_processing import (
 
 @with_all_phases_except([PHASE0, PHASE1])
 @spec_state_test
+@with_configs([MINIMAL], reason="too slow")
 def test_sync_committees_progress(spec, state):
     current_epoch = spec.get_current_epoch(state)
     # NOTE: if not in the genesis epoch, period math below needs to be

tests/core/pyspec/eth2spec/test/context.py

@@ -58,7 +58,7 @@ class SpecPhase1(Spec):
     ...
 
 
-class SpecLightclient(Spec):
+class SpecAltair(Spec):
     ...
 
 
@@ -66,7 +66,7 @@ class SpecLightclient(Spec):
 class SpecForks(TypedDict, total=False):
     PHASE0: SpecPhase0
     PHASE1: SpecPhase1
-    ALTAIR: SpecLightclient
+    ALTAIR: SpecAltair
 
 
 def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int],

tests/core/pyspec/eth2spec/test/phase0/genesis/test_initialization.py

@@ -1,14 +1,29 @@
-from eth2spec.test.context import PHASE0, spec_test, with_phases, single_phase
+from eth2spec.test.context import (
+    MINIMAL,
+    is_post_altair,
+    single_phase,
+    spec_test,
+    with_configs,
+    with_all_phases,
+)
 from eth2spec.test.helpers.deposits import (
     prepare_full_genesis_deposits,
     prepare_random_genesis_deposits,
 )
 
 
-@with_phases(([PHASE0]))
+def get_post_altair_description(spec):
+    return f"Although it's not phase 0, we may use {spec.fork} spec to start testnets."
+
+
+@with_all_phases
 @spec_test
 @single_phase
+@with_configs([MINIMAL], reason="too slow")
 def test_initialize_beacon_state_from_eth1(spec):
+    if is_post_altair(spec):
+        yield 'description', 'meta', get_post_altair_description(spec)
+
     deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT
     deposits, deposit_root, _ = prepare_full_genesis_deposits(
         spec,
@@ -38,10 +53,14 @@ def test_initialize_beacon_state_from_eth1(spec):
     yield 'state', state
 
 
-@with_phases([PHASE0])
+@with_all_phases
 @spec_test
 @single_phase
+@with_configs([MINIMAL], reason="too slow")
 def test_initialize_beacon_state_some_small_balances(spec):
+    if is_post_altair(spec):
+        yield 'description', 'meta', get_post_altair_description(spec)
+
     main_deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT
     main_deposits, _, deposit_data_list = prepare_full_genesis_deposits(
         spec, spec.MAX_EFFECTIVE_BALANCE,
@@ -79,10 +98,14 @@ def test_initialize_beacon_state_some_small_balances(spec):
     yield 'state', state
 
 
-@with_phases([PHASE0])
+@with_all_phases
 @spec_test
 @single_phase
+@with_configs([MINIMAL], reason="too slow")
 def test_initialize_beacon_state_one_topup_activation(spec):
+    if is_post_altair(spec):
+        yield 'description', 'meta', get_post_altair_description(spec)
+
     # Submit all but one deposit as MAX_EFFECTIVE_BALANCE
     main_deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - 1
     main_deposits, _, deposit_data_list = prepare_full_genesis_deposits(
@@ -125,10 +148,14 @@ def test_initialize_beacon_state_one_topup_activation(spec):
     yield 'state', state
 
 
-@with_phases([PHASE0])
+@with_all_phases
 @spec_test
 @single_phase
+@with_configs([MINIMAL], reason="too slow")
 def test_initialize_beacon_state_random_invalid_genesis(spec):
+    if is_post_altair(spec):
+        yield 'description', 'meta', get_post_altair_description(spec)
+
     # Make a bunch of random deposits
     deposits, _, deposit_data_list = prepare_random_genesis_deposits(
         spec,
@@ -149,10 +176,14 @@ def test_initialize_beacon_state_random_invalid_genesis(spec):
     yield 'state', state
 
 
-@with_phases([PHASE0])
+@with_all_phases
 @spec_test
 @single_phase
+@with_configs([MINIMAL], reason="too slow")
 def test_initialize_beacon_state_random_valid_genesis(spec):
+    if is_post_altair(spec):
+        yield 'description', 'meta', get_post_altair_description(spec)
+
     # Make a bunch of random deposits
     random_deposits, _, deposit_data_list = prepare_random_genesis_deposits(
         spec,

tests/core/pyspec/eth2spec/test/phase0/genesis/test_validity.py

@@ -1,9 +1,20 @@
-from eth2spec.test.context import PHASE0, spec_test, with_phases, single_phase
+from eth2spec.test.context import (
+    MINIMAL,
+    is_post_altair,
+    spec_test,
+    single_phase,
+    with_configs,
+    with_all_phases,
+)
 from eth2spec.test.helpers.deposits import (
     prepare_full_genesis_deposits,
 )
 
 
+def get_post_altair_description(spec):
+    return f"Although it's not phase 0, we may use {spec.fork} spec to start testnets."
+
+
 def create_valid_beacon_state(spec):
     deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT
     deposits, _, _ = prepare_full_genesis_deposits(
@@ -30,39 +41,55 @@ def run_is_valid_genesis_state(spec, state, valid=True):
     assert is_valid == valid
 
 
-@with_phases([PHASE0])
+@with_all_phases
 @spec_test
 @single_phase
+@with_configs([MINIMAL], reason="too slow")
 def test_is_valid_genesis_state_true(spec):
+    if is_post_altair(spec):
+        yield 'description', 'meta', get_post_altair_description(spec)
+
     state = create_valid_beacon_state(spec)
 
     yield from run_is_valid_genesis_state(spec, state, valid=True)
 
 
-@with_phases([PHASE0])
+@with_all_phases
 @spec_test
 @single_phase
+@with_configs([MINIMAL], reason="too slow")
 def test_is_valid_genesis_state_false_invalid_timestamp(spec):
+    if is_post_altair(spec):
+        yield 'description', 'meta', get_post_altair_description(spec)
+
     state = create_valid_beacon_state(spec)
     state.genesis_time = spec.MIN_GENESIS_TIME - 1
 
     yield from run_is_valid_genesis_state(spec, state, valid=False)
 
 
-@with_phases([PHASE0])
+@with_all_phases
 @spec_test
 @single_phase
+@with_configs([MINIMAL], reason="too slow")
 def test_is_valid_genesis_state_true_more_balance(spec):
+    if is_post_altair(spec):
+        yield 'description', 'meta', get_post_altair_description(spec)
+
     state = create_valid_beacon_state(spec)
     state.validators[0].effective_balance = spec.MAX_EFFECTIVE_BALANCE + 1
 
     yield from run_is_valid_genesis_state(spec, state, valid=True)
 
 
-@with_phases([PHASE0])
+@with_all_phases
 @spec_test
 @single_phase
+@with_configs([MINIMAL], reason="too slow")
 def test_is_valid_genesis_state_true_one_more_validator(spec):
+    if is_post_altair(spec):
+        yield 'description', 'meta', get_post_altair_description(spec)
+
     deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + 1
     deposits, _, _ = prepare_full_genesis_deposits(
         spec,
@@ -78,10 +105,14 @@ def test_is_valid_genesis_state_true_one_more_validator(spec):
     yield from run_is_valid_genesis_state(spec, state, valid=True)
 
 
-@with_phases([PHASE0])
+@with_all_phases
 @spec_test
 @single_phase
+@with_configs([MINIMAL], reason="too slow")
 def test_is_valid_genesis_state_false_not_enough_validator(spec):
+    if is_post_altair(spec):
+        yield 'description', 'meta', get_post_altair_description(spec)
+
     deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - 1
     deposits, _, _ = prepare_full_genesis_deposits(
         spec,

tests/formats/genesis/initialization.md

@@ -17,6 +17,7 @@ An integer. The timestamp of the block, in seconds.
 A yaml file to help read the deposit count:
 
 ```yaml
+description: string    -- Optional. Description of test case, purely for debugging purposes.
 deposits_count: int    -- Amount of deposits.
 ```
 

tests/formats/genesis/validity.md

@@ -4,6 +4,14 @@ Tests if a genesis state is valid, i.e. if it counts as trigger to launch.
 
 ## Test case format
 
+### `meta.yaml`
+
+A yaml file to help read the deposit count:
+
+```yaml
+description: string    -- Optional. Description of test case, purely for debugging purposes.
+```
+
 ### `genesis.ssz_snappy`
 
 An SSZ-snappy encoded `BeaconState`, the state to validate as genesis candidate.

tests/generators/genesis/main.py

@@ -1,37 +1,24 @@
-from typing import Iterable
-
-from eth2spec.test.context import PHASE0
-from eth2spec.test.phase0.genesis import test_initialization, test_validity
-
-from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
-from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests
-from eth2spec.phase0 import spec as spec
-from importlib import reload
-from eth2spec.config import config_util
-from eth2spec.utils import bls
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
+from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.altair import spec as spec_altair
+from eth2spec.phase1 import spec as spec_phase1
+from eth2spec.test.context import PHASE0, PHASE1, ALTAIR
 
 
-def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
-
-    def prepare_fn(configs_path: str) -> str:
-        config_util.prepare_config(configs_path, config_name)
-        reload(spec)
-        bls.use_milagro()
-        return config_name
-
-    def cases_fn() -> Iterable[gen_typing.TestCase]:
-        return generate_from_tests(
-            runner_name='genesis',
-            handler_name=handler_name,
-            src=tests_src,
-            fork_name=PHASE0,
-        )
-
-    return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
+specs = (spec_phase0, spec_altair, spec_phase1)
 
 
 if __name__ == "__main__":
-    gen_runner.run_generator("genesis", [
-        create_provider('initialization', test_initialization, 'minimal'),
-        create_provider('validity', test_validity, 'minimal'),
-    ])
+    phase_0_mods = {key: 'eth2spec.test.phase0.genesis.test_' + key for key in [
+        'initialization',
+        'validity',
+    ]}
+    altair_mods = phase_0_mods
+    phase_1_mods = phase_0_mods
+    all_mods = {
+        PHASE0: phase_0_mods,
+        ALTAIR: altair_mods,
+        PHASE1: phase_1_mods,
+    }
+
+    run_state_test_generators(runner_name="genesis", specs=specs, all_mods=all_mods)