Merge pull request #1957 from ethereum/testgenphase1

Enable test generation for phase1

.circleci/config.yml

@@ -130,8 +130,11 @@ jobs:
           key: v3-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_pyspec_cached_venv
       - run:
-          name: Run linter
+          name: Run linter for pyspec
           command: make lint
+      - run:
+          name: Run linter for test generators
+          command: make lint_generators
   build_deposit_contract:
     docker:
       - image: ethereum/solc:0.6.11-alpine

Makefile

@@ -1,6 +1,7 @@
 SPEC_DIR = ./specs
 SSZ_DIR = ./ssz
 TEST_LIBS_DIR = ./tests/core
+TEST_GENERATORS_DIR = ./tests/generators
 PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
 TEST_VECTOR_DIR = ../eth2.0-spec-tests/tests
 GENERATOR_DIR = ./tests/generators
@@ -113,6 +114,10 @@ lint: pyspec
 	flake8  --config $(LINTER_CONFIG_FILE) ./eth2spec \
 	&& mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.phase1
 
+lint_generators: pyspec
+	. venv/bin/activate; cd $(TEST_GENERATORS_DIR); \
+	flake8  --config $(LINTER_CONFIG_FILE)
+
 compile_deposit_contract:
 	@cd $(SOLIDITY_DEPOSIT_CONTRACT_DIR)
 	@git submodule update --recursive --init

configs/mainnet/phase0.yaml

@@ -2,6 +2,7 @@
 # Note: the intention of this file (for now) is to illustrate what a mainnet configuration could look like.
 # Some of these constants may still change before the launch of Phase 0.
 
+CONFIG_NAME: "mainnet"
 
 # Misc
 # ---------------------------------------------------------------

configs/mainnet/phase1.yaml

@@ -1,5 +1,6 @@
 # Mainnet preset - phase 1
 
+CONFIG_NAME: "mainnet"
 
 # phase1-fork
 # ---------------------------------------------------------------

configs/minimal/phase0.yaml

@@ -1,5 +1,6 @@
 # Minimal preset
 
+CONFIG_NAME: "minimal"
 
 # Misc
 # ---------------------------------------------------------------

configs/minimal/phase1.yaml

@@ -1,5 +1,6 @@
 # Minimal preset - phase 1
 
+CONFIG_NAME: "minimal"
 
 # phase1-fork
 # ---------------------------------------------------------------

setup.py

@@ -119,6 +119,8 @@ from eth2spec.utils import bls
 from eth2spec.utils.hash_function import hash
 
 SSZObject = TypeVar('SSZObject', bound=View)
+
+CONFIG_NAME = 'mainnet'
 '''
 PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0
 from eth2spec.config.config_util import apply_constants_config
@@ -151,6 +153,8 @@ reload(phase0)
 SSZVariableName = str
 GeneralizedIndex = NewType('GeneralizedIndex', int)
 SSZObject = TypeVar('SSZObject', bound=View)
+
+CONFIG_NAME = 'mainnet'
 '''
 SUNDRY_CONSTANTS_FUNCTIONS = '''
 def ceillog2(x: int) -> uint64:

tests/core/gen_helpers/gen_base/gen_runner.py

@@ -2,6 +2,7 @@ import argparse
 from pathlib import Path
 import sys
 from typing import Iterable, AnyStr, Any, Callable
+import traceback
 
 from ruamel.yaml import (
     YAML,
@@ -9,6 +10,13 @@ from ruamel.yaml import (
 
 from gen_base.gen_typing import TestProvider
 
+from eth2spec.test import context
+from eth2spec.test.exceptions import SkippedTest
+
+
+# Flag that the runner does NOT run test via pytest
+context.is_pytest = False
+
 
 def validate_output_dir(path_str):
     path = Path(path_str)
@@ -134,14 +142,20 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
 
                 written_part = False
                 meta = dict()
-                for (name, out_kind, data) in test_case.case_fn():
-                    written_part = True
-                    if out_kind == "meta":
-                        meta[name] = data
-                    if out_kind == "data":
-                        output_part("data", name, dump_yaml_fn(data, name, file_mode, yaml))
-                    if out_kind == "ssz":
-                        output_part("ssz", name, dump_ssz_fn(data, name, file_mode))
+
+                try:
+                    for (name, out_kind, data) in test_case.case_fn():
+                        written_part = True
+                        if out_kind == "meta":
+                            meta[name] = data
+                        if out_kind == "data":
+                            output_part("data", name, dump_yaml_fn(data, name, file_mode, yaml))
+                        if out_kind == "ssz":
+                            output_part("ssz", name, dump_ssz_fn(data, name, file_mode))
+                except SkippedTest as e:
+                    print(e)
+                    continue
+
                 # Once all meta data is collected (if any), write it to a meta data file.
                 if len(meta) != 0:
                     written_part = True
@@ -152,6 +166,7 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
 
             except Exception as e:
                 print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}")
+                traceback.print_exc()
     print(f"completed {generator_name}")
 
 

tests/core/gen_helpers/requirements.txt

@@ -1,2 +1,3 @@
 ruamel.yaml==0.16.5
 eth-utils==1.6.0
+pytest>=4.4

tests/core/gen_helpers/setup.py

@@ -5,6 +5,7 @@ setup(
     packages=['gen_base', 'gen_from_tests'],
     install_requires=[
         "ruamel.yaml==0.16.5",
-        "eth-utils==1.6.0"
+        "eth-utils==1.6.0",
+        "pytest>=4.4",
     ]
 )

tests/core/pyspec/eth2spec/config/config_util.py

@@ -54,6 +54,8 @@ def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]:
             out[k] = [int(item) if item.isdigit() else item for item in v]
         elif isinstance(v, str) and v.startswith("0x"):
             out[k] = bytes.fromhex(v[2:])
+        elif k == "CONFIG_NAME":
+            out[k] = str(v)
         else:
             out[k] = int(v)
     return out

tests/core/pyspec/eth2spec/debug/random_value.py

@@ -68,9 +68,8 @@ def get_random_ssz_object(rng: Random,
         else:
             return typ(get_random_bytes_list(rng, rng.randint(0, min(max_bytes_length, typ.limit()))))
     if issubclass(typ, ByteVector):
-        # Sanity, don't generate absurdly big random values
-        # If a client is aiming to performance-test, they should create a benchmark suite.
-        assert typ.type_byte_length() <= max_bytes_length
+        # Random byte vectors can be bigger than max bytes size, e.g. custody chunk data.
+        # No max-bytes-length limitation here.
         if mode == RandomizationMode.mode_zero:
             return typ(b'\x00' * typ.type_byte_length())
         elif mode == RandomizationMode.mode_max:

tests/core/pyspec/eth2spec/test/context.py

@@ -1,9 +1,11 @@
+import pytest
+
 from eth2spec.phase0 import spec as spec_phase0
 from eth2spec.phase1 import spec as spec_phase1
 from eth2spec.utils import bls
 
+from .exceptions import SkippedTest
 from .helpers.genesis import create_genesis_state
-
 from .utils import vector_test, with_meta_tags
 
 from random import Random
@@ -22,11 +24,16 @@ def reload_specs():
 # Some of the Spec module functionality is exposed here to deal with phase-specific changes.
 
 SpecForkName = NewType("SpecForkName", str)
+ConfigName = NewType("ConfigName", str)
 
 PHASE0 = SpecForkName('phase0')
 PHASE1 = SpecForkName('phase1')
 ALL_PHASES = (PHASE0, PHASE1)
 
+MAINNET = ConfigName('mainnet')
+MINIMAL = ConfigName('minimal')
+
+
 # TODO: currently phases are defined as python modules.
 # It would be better if they would be more well-defined interfaces for stronger typing.
 
@@ -153,7 +160,7 @@ def low_single_balance(spec):
 
 def large_validator_set(spec):
     """
-    Helper method to create a series of default balances.
+    Helper method to create a large series of default balances.
     Usage: `@with_custom_state(balances_fn=default_balances, ...)`
     """
     num_validators = 2 * spec.SLOTS_PER_EPOCH * spec.MAX_COMMITTEES_PER_SLOT * spec.TARGET_COMMITTEE_SIZE
@@ -184,6 +191,17 @@ def single_phase(fn):
 DEFAULT_BLS_ACTIVE = True
 
 
+is_pytest = True
+
+
+def dump_skipping_message(reason: str) -> None:
+    message = f"[Skipped test] {reason}"
+    if is_pytest:
+        pytest.skip(message)
+    else:
+        raise SkippedTest(message)
+
+
 def spec_test(fn):
     # Bls switch must be wrapped by vector_test,
     # to fully go through the yielded bls switch data, before setting back the BLS setting.
@@ -255,6 +273,24 @@ def bls_switch(fn):
     return entry
 
 
+def disable_process_reveal_deadlines(fn):
+    """
+    Decorator to make a function execute with `process_reveal_deadlines` OFF.
+    This is for testing long-range epochs transition without considering the reveal-deadline slashing effect.
+    """
+    def entry(*args, spec: Spec, **kw):
+        if hasattr(spec, 'process_reveal_deadlines'):
+            old_state = spec.process_reveal_deadlines
+            spec.process_reveal_deadlines = lambda state: None
+
+        yield from fn(*args, spec=spec, **kw)
+
+        if hasattr(spec, 'process_reveal_deadlines'):
+            spec.process_reveal_deadlines = old_state
+
+    return with_meta_tags({'reveal_deadlines_setting': 1})(entry)
+
+
 def with_all_phases(fn):
     """
     A decorator for running a test with every phase
@@ -284,7 +320,8 @@ def with_phases(phases, other_phases=None):
             if 'phase' in kw:
                 phase = kw.pop('phase')
                 if phase not in phases:
-                    return
+                    dump_skipping_message(f"doesn't support this fork: {phase}")
+                    return None
                 run_phases = [phase]
 
             available_phases = set(run_phases)
@@ -309,3 +346,33 @@ def with_phases(phases, other_phases=None):
             return ret
         return wrapper
     return decorator
+
+
+def with_configs(configs, reason=None):
+    def decorator(fn):
+        def wrapper(*args, spec: Spec, **kw):
+            available_configs = set(configs)
+            if spec.CONFIG_NAME not in available_configs:
+                message = f"doesn't support this config: {spec.CONFIG_NAME}."
+                if reason is not None:
+                    message = f"{message} Reason: {reason}"
+                dump_skipping_message(message)
+                return None
+
+            return fn(*args, spec=spec, **kw)
+        return wrapper
+    return decorator
+
+
+def only_full_crosslink(fn):
+    def is_full_crosslink(spec, state):
+        epoch = spec.compute_epoch_at_slot(state.slot)
+        return spec.get_committee_count_per_slot(state, epoch) >= spec.get_active_shard_count(state)
+
+    def wrapper(*args, spec: Spec, state: Any, **kw):
+        # TODO: update condition to "phase1+" if we have phase2
+        if spec.fork == PHASE1 and not is_full_crosslink(spec, state):
+            dump_skipping_message("only for full crosslink")
+            return None
+        return fn(*args, spec=spec, state=state, **kw)
+    return wrapper

tests/core/pyspec/eth2spec/test/exceptions.py

@@ -0,0 +1,2 @@
+class SkippedTest(Exception):
+    ...

tests/core/pyspec/eth2spec/test/helpers/custody.py

@@ -179,15 +179,6 @@ def get_sample_shard_transition(spec, start_slot, block_lengths):
     return shard_transition
 
 
-def get_custody_secret(spec, state, validator_index, epoch=None):
-    period = spec.get_custody_period_for_validator(validator_index, epoch if epoch is not None
-                                                   else spec.get_current_epoch(state))
-    epoch_to_sign = spec.get_randao_epoch_for_custody_period(period, validator_index)
-    domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign)
-    signing_root = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain)
-    return bls.Sign(privkeys[validator_index], signing_root)
-
-
 def get_custody_slashable_test_vector(spec, custody_secret, length, slashable=True):
     test_vector = get_custody_test_vector(length)
     offset = 0

tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py

@@ -35,8 +35,3 @@ def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks
     shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot)
     shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks)
     return shard_transition
-
-
-def is_full_crosslink(spec, state):
-    epoch = spec.compute_epoch_at_slot(state.slot)
-    return spec.get_committee_count_per_slot(state, epoch) >= spec.get_active_shard_count(state)

tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py

@@ -193,9 +193,6 @@ def test_participants_already_slashed(spec, state):
     yield from run_attester_slashing_processing(spec, state, attester_slashing, False)
 
 
-# Some of the following tests are phase0 only: phase 1 lists participants with bitfields instead of index list.
-
-
 @with_all_phases
 @spec_state_test
 @always_bls

tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py

@@ -21,10 +21,13 @@ from eth2spec.test.helpers.deposits import prepare_state_and_deposit
 from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee
 
 from eth2spec.test.context import (
-    PHASE0, PHASE1,
-    spec_test,
-    spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases,
-    with_custom_state, single_phase,
+    PHASE0, PHASE1, MINIMAL,
+    spec_test, spec_state_test, dump_skipping_message,
+    with_phases, with_all_phases, single_phase,
+    expect_assertion_error, always_bls,
+    disable_process_reveal_deadlines,
+    with_configs,
+    with_custom_state,
     large_validator_set,
 )
 
@@ -90,6 +93,8 @@ def test_empty_block_transition(spec, state):
 
 
 @with_all_phases
+@with_configs([MINIMAL],
+              reason="mainnet config leads to larger validator set than limit of public/private keys pre-generated")
 @spec_test
 @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE)
 @single_phase
@@ -316,6 +321,8 @@ def test_empty_epoch_transition(spec, state):
 
 
 @with_all_phases
+@with_configs([MINIMAL],
+              reason="mainnet config leads to larger validator set than limit of public/private keys pre-generated")
 @spec_test
 @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE)
 @single_phase
@@ -338,10 +345,9 @@ def test_empty_epoch_transition_large_validator_set(spec, state):
 @with_all_phases
 @spec_state_test
 def test_empty_epoch_transition_not_finalizing(spec, state):
-    # Don't run for non-minimal configs, it takes very long, and the effect
-    # of calling finalization/justification is just the same as with the minimal configuration.
     if spec.SLOTS_PER_EPOCH > 8:
-        return
+        return dump_skipping_message("Skip mainnet config for saving time."
+                                     " Minimal config suffice to cover the target-of-test.")
 
     # copy for later balance lookups.
     pre_balances = list(state.balances)
@@ -527,9 +533,8 @@ def test_attester_slashing(spec, state):
 @with_all_phases
 @spec_state_test
 def test_duplicate_attester_slashing(spec, state):
-    # Skip test if config cannot handle multiple AttesterSlashings per block
     if spec.MAX_ATTESTER_SLASHINGS < 2:
-        return
+        return dump_skipping_message("Skip test if config cannot handle multiple AttesterSlashings per block")
 
     attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True)
     attester_slashings = [attester_slashing, attester_slashing.copy()]
@@ -556,9 +561,8 @@ def test_duplicate_attester_slashing(spec, state):
 @with_all_phases
 @spec_state_test
 def test_multiple_attester_slashings_no_overlap(spec, state):
-    # Skip test if config cannot handle multiple AttesterSlashings per block
     if spec.MAX_ATTESTER_SLASHINGS < 2:
-        return
+        return dump_skipping_message("Skip test if config cannot handle multiple AttesterSlashings per block")
 
     # copy for later balance lookups.
     pre_state = state.copy()
@@ -597,9 +601,8 @@ def test_multiple_attester_slashings_no_overlap(spec, state):
 @with_all_phases
 @spec_state_test
 def test_multiple_attester_slashings_partial_overlap(spec, state):
-    # Skip test if config cannot handle multiple AttesterSlashings per block
     if spec.MAX_ATTESTER_SLASHINGS < 2:
-        return
+        return dump_skipping_message("Skip test if config cannot handle multiple AttesterSlashings per block")
 
     # copy for later balance lookups.
     pre_state = state.copy()
@@ -809,8 +812,9 @@ def prepare_signed_exits(spec, state, indices):
 # exceeding the minimal-config randao mixes memory size.
 # Applies to all voluntary-exit sanity block tests.
 
-@with_phases([PHASE0])
+@with_all_phases
 @spec_state_test
+@disable_process_reveal_deadlines
 def test_voluntary_exit(spec, state):
     validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1]
 
@@ -858,8 +862,9 @@ def test_double_validator_exit_same_block(spec, state):
     yield 'post', None
 
 
-@with_phases([PHASE0])
+@with_all_phases
 @spec_state_test
+@disable_process_reveal_deadlines
 def test_multiple_different_validator_exits_same_block(spec, state):
     validator_indices = [
         spec.get_active_validator_indices(state, spec.get_current_epoch(state))[i]
@@ -935,9 +940,9 @@ def test_historical_batch(spec, state):
 @with_all_phases
 @spec_state_test
 def test_eth1_data_votes_consensus(spec, state):
-    # Don't run when it will take very, very long to simulate. Minimal configuration suffices.
     if spec.EPOCHS_PER_ETH1_VOTING_PERIOD > 2:
-        return
+        return dump_skipping_message("Skip test if config with longer `EPOCHS_PER_ETH1_VOTING_PERIOD` for saving time."
+                                     " Minimal config suffice to cover the target-of-test.")
 
     voting_period_slots = spec.EPOCHS_PER_ETH1_VOTING_PERIOD * spec.SLOTS_PER_EPOCH
 
@@ -979,9 +984,9 @@ def test_eth1_data_votes_consensus(spec, state):
 @with_all_phases
 @spec_state_test
 def test_eth1_data_votes_no_consensus(spec, state):
-    # Don't run when it will take very, very long to simulate. Minimal configuration suffices.
     if spec.EPOCHS_PER_ETH1_VOTING_PERIOD > 2:
-        return
+        return dump_skipping_message("Skip test if config with longer `EPOCHS_PER_ETH1_VOTING_PERIOD` for saving time."
+                                     " Minimal config suffice to cover the target-of-test.")
 
     voting_period_slots = spec.EPOCHS_PER_ETH1_VOTING_PERIOD * spec.SLOTS_PER_EPOCH
 

tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py

@@ -9,9 +9,12 @@ from eth2spec.test.helpers.attestations import (
 from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot
 from eth2spec.test.context import (
     PHASE0,
-    with_all_phases_except,
-    spec_state_test,
+    MINIMAL,
     expect_assertion_error,
+    disable_process_reveal_deadlines,
+    spec_state_test,
+    with_all_phases_except,
+    with_configs,
 )
 from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing
 
@@ -67,6 +70,8 @@ def run_custody_chunk_response_processing(spec, state, custody_response, valid=T
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@with_configs([MINIMAL], reason="too slow")
+@disable_process_reveal_deadlines
 def test_challenge_appended(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + 1)  # Make len(offset_slots) == 1
@@ -89,6 +94,8 @@ def test_challenge_appended(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_challenge_empty_element_replaced(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + 1)  # Make len(offset_slots) == 1
@@ -113,6 +120,8 @@ def test_challenge_empty_element_replaced(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_duplicate_challenge(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + 1)  # Make len(offset_slots) == 1
@@ -137,6 +146,8 @@ def test_duplicate_challenge(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_second_challenge(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + 1)  # Make len(offset_slots) == 1
@@ -163,6 +174,8 @@ def test_second_challenge(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_multiple_epochs_custody(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3)
@@ -186,6 +199,8 @@ def test_multiple_epochs_custody(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_many_epochs_custody(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20)
@@ -209,6 +224,8 @@ def test_many_epochs_custody(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_off_chain_attestation(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH)
@@ -228,6 +245,8 @@ def test_off_chain_attestation(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_custody_response(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH)
@@ -258,6 +277,8 @@ def test_custody_response(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_custody_response_chunk_index_2(spec, state):
     transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH)
 
@@ -287,6 +308,8 @@ def test_custody_response_chunk_index_2(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_custody_response_multiple_epochs(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3)
@@ -317,6 +340,8 @@ def test_custody_response_multiple_epochs(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_custody_response_many_epochs(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20)

tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py

@@ -1,18 +1,21 @@
 from eth2spec.test.helpers.custody import (
     get_valid_custody_slashing,
-    get_custody_secret,
     get_custody_slashable_shard_transition,
 )
 from eth2spec.test.helpers.attestations import (
     get_valid_on_time_attestation,
 )
+from eth2spec.test.helpers.keys import privkeys
 from eth2spec.utils.ssz.ssz_typing import ByteList
 from eth2spec.test.helpers.state import get_balance, transition_to
 from eth2spec.test.context import (
     PHASE0,
+    MINIMAL,
     with_all_phases_except,
     spec_state_test,
     expect_assertion_error,
+    disable_process_reveal_deadlines,
+    with_configs,
 )
 from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing
 
@@ -77,7 +80,12 @@ def run_standard_custody_slashing_test(spec,
     if block_lengths is None:
         block_lengths = [2**15 // 3] * len(offset_slots)
 
-    custody_secret = get_custody_secret(spec, state, validator_index)
+    custody_secret = spec.get_custody_secret(
+        state,
+        validator_index,
+        privkeys[validator_index],
+        spec.get_current_epoch(state),
+    )
     shard_transition, slashable_test_vector = get_custody_slashable_shard_transition(
         spec,
         state.slot,
@@ -106,30 +114,40 @@ def run_standard_custody_slashing_test(spec,
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_custody_slashing(spec, state):
     yield from run_standard_custody_slashing_test(spec, state)
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_incorrect_custody_slashing(spec, state):
     yield from run_standard_custody_slashing_test(spec, state, correct=False)
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_multiple_epochs_custody(spec, state):
     yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 3)
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_many_epochs_custody(spec, state):
     yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 5)
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@disable_process_reveal_deadlines
+@with_configs([MINIMAL], reason="too slow")
 def test_invalid_custody_slashing(spec, state):
     yield from run_standard_custody_slashing_test(
         spec,

tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py

@@ -1,6 +1,7 @@
 from eth2spec.test.context import (
     PHASE0,
     with_all_phases_except,
+    only_full_crosslink,
     spec_state_test,
 )
 from eth2spec.test.helpers.attestations import (
@@ -10,7 +11,6 @@ from eth2spec.test.helpers.attestations import (
 )
 from eth2spec.test.helpers.shard_transitions import (
     run_shard_transitions_processing,
-    is_full_crosslink,
 )
 from eth2spec.test.helpers.shard_block import (
     build_shard_block,
@@ -92,31 +92,22 @@ def run_successful_crosslink_tests(spec, state, target_len_offset_slot):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_basic_crosslinks(spec, state):
-    if not is_full_crosslink(spec, state):
-        # Skip this test
-        return
-
     yield from run_successful_crosslink_tests(spec, state, target_len_offset_slot=1)
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_multiple_offset_slots(spec, state):
-    if not is_full_crosslink(spec, state):
-        # Skip this test
-        return
-
     yield from run_successful_crosslink_tests(spec, state, target_len_offset_slot=2)
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_no_winning_root(spec, state):
-    if not is_full_crosslink(spec, state):
-        # Skip this test
-        return
-
     state, shard, target_shard_slot = get_initial_env(spec, state, target_len_offset_slot=1)
     init_slot = state.slot
 
@@ -163,11 +154,8 @@ def test_no_winning_root(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_wrong_shard_transition_root(spec, state):
-    if not is_full_crosslink(spec, state):
-        # Skip this test
-        return
-
     state, shard, target_shard_slot = get_initial_env(spec, state, target_len_offset_slot=1)
     init_slot = state.slot
 

tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py

@@ -8,8 +8,10 @@ from eth2spec.test.helpers.attestations import (
 from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot
 from eth2spec.test.context import (
     PHASE0,
-    with_all_phases_except,
+    MINIMAL,
     spec_state_test,
+    with_all_phases_except,
+    with_configs,
 )
 from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing
 from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with
@@ -25,6 +27,7 @@ def run_process_challenge_deadlines(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@with_configs([MINIMAL], reason="too slow")
 def test_validator_slashed_after_chunk_challenge(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + 1)  # Make len(offset_slots) == 1

tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py

@@ -4,7 +4,9 @@ from eth2spec.test.helpers.custody import (
 from eth2spec.test.helpers.state import transition_to
 from eth2spec.test.context import (
     PHASE0,
+    MINIMAL,
     with_all_phases_except,
+    with_configs,
     spec_state_test,
 )
 from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with
@@ -17,6 +19,7 @@ def run_process_challenge_deadlines(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@with_configs([MINIMAL], reason="too slow")
 def test_validator_slashed_after_reveal_deadline(spec, state):
     assert state.validators[0].slashed == 0
     transition_to(spec, state, spec.get_randao_epoch_for_custody_period(0, 0) * spec.SLOTS_PER_EPOCH)
@@ -36,6 +39,7 @@ def test_validator_slashed_after_reveal_deadline(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@with_configs([MINIMAL], reason="too slow")
 def test_validator_not_slashed_after_reveal(spec, state):
     transition_to(spec, state, spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH)
     custody_key_reveal = get_valid_custody_key_reveal(spec, state)

tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py

@@ -1,14 +1,15 @@
 from typing import Dict, Sequence
 
 from eth2spec.test.context import (
-    PHASE0,
+    PHASE0, MINIMAL,
     with_all_phases_except,
     spec_state_test,
+    only_full_crosslink,
+    with_configs,
 )
 from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
 from eth2spec.test.helpers.block import build_empty_block
 from eth2spec.test.helpers.custody import (
-    get_custody_secret,
     get_custody_slashable_test_vector,
     get_valid_chunk_challenge,
     get_valid_custody_chunk_response,
@@ -16,13 +17,13 @@ from eth2spec.test.helpers.custody import (
     get_valid_custody_slashing,
     get_valid_early_derived_secret_reveal,
 )
+from eth2spec.test.helpers.keys import privkeys
 from eth2spec.test.helpers.shard_block import (
     build_shard_block,
     get_committee_index_of_shard,
     get_sample_shard_block_body,
     get_shard_transitions,
 )
-from eth2spec.test.helpers.shard_transitions import is_full_crosslink
 from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to
 
 
@@ -99,12 +100,8 @@ def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, comm
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_process_beacon_block_with_normal_shard_transition(spec, state):
-    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     transition_to_valid_shard_slot(spec, state)
 
     target_len_offset_slot = 1
@@ -117,12 +114,8 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_process_beacon_block_with_empty_proposal_transition(spec, state):
-    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     transition_to_valid_shard_slot(spec, state)
 
     target_len_offset_slot = 1
@@ -140,12 +133,8 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_with_shard_transition_with_custody_challenge_and_response(spec, state):
-    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     transition_to_valid_shard_slot(spec, state)
 
     # build shard block
@@ -178,6 +167,7 @@ def test_with_shard_transition_with_custody_challenge_and_response(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@with_configs([MINIMAL])
 def test_custody_key_reveal(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH)
@@ -202,12 +192,8 @@ def test_early_derived_secret_reveal(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_custody_slashing(spec, state):
-    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     transition_to_valid_shard_slot(spec, state)
 
     # Build shard block
@@ -215,7 +201,12 @@ def test_custody_slashing(spec, state):
     committee_index = get_committee_index_of_shard(spec, state, state.slot, shard)
     # Create slashable shard block body
     validator_index = spec.get_beacon_committee(state, state.slot, committee_index)[0]
-    custody_secret = get_custody_secret(spec, state, validator_index)
+    custody_secret = spec.get_custody_secret(
+        state,
+        validator_index,
+        privkeys[validator_index],
+        spec.get_current_epoch(state),
+    )
     slashable_body = get_custody_slashable_test_vector(spec, custody_secret, length=100, slashable=True)
     shard_block = build_shard_block(spec, state, shard, body=slashable_body, slot=state.slot, signed=True)
     shard_block_dict: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]}

tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py

@@ -4,12 +4,12 @@ from eth2spec.test.context import (
     expect_assertion_error,
     spec_state_test,
     with_all_phases_except,
+    only_full_crosslink,
 )
 from eth2spec.test.helpers.shard_block import (
     build_shard_block,
     sign_shard_block,
 )
-from eth2spec.test.helpers.shard_transitions import is_full_crosslink
 from eth2spec.test.helpers.state import next_slot, transition_to_valid_shard_slot, transition_to
 
 
@@ -46,11 +46,8 @@ def run_shard_blocks(spec, shard_state, signed_shard_block, beacon_parent_state,
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_valid_shard_block(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
 
@@ -68,11 +65,8 @@ def test_valid_shard_block(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_invalid_shard_parent_root(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
 
@@ -87,11 +81,8 @@ def test_invalid_shard_parent_root(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_invalid_beacon_parent_root(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -105,11 +96,8 @@ def test_invalid_beacon_parent_root(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_invalid_slot(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -124,11 +112,8 @@ def test_invalid_slot(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_invalid_proposer_index(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -148,12 +133,8 @@ def test_invalid_proposer_index(spec, state):
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_out_of_bound_offset(spec, state):
-    # TODO: Handle this edge case properly
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -173,11 +154,8 @@ def test_out_of_bound_offset(spec, state):
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_invalid_offset(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     # 4 is not in `SHARD_BLOCK_OFFSETS`
@@ -195,11 +173,8 @@ def test_invalid_offset(spec, state):
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_empty_block_body(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -217,11 +192,8 @@ def test_empty_block_body(spec, state):
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_invalid_signature(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -239,11 +211,8 @@ def test_invalid_signature(spec, state):
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_max_offset(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -259,11 +228,8 @@ def test_max_offset(spec, state):
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_pending_shard_parent_block(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     # Block N
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)

tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py

@@ -1,6 +1,6 @@
 from eth2spec.utils.ssz.ssz_impl import hash_tree_root
 
-from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls
+from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls, only_full_crosslink
 from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
 from eth2spec.test.helpers.shard_block import (
     build_shard_block,
@@ -8,7 +8,6 @@ from eth2spec.test.helpers.shard_block import (
     get_committee_index_of_shard,
 )
 from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root, get_genesis_forkchoice_store
-from eth2spec.test.helpers.shard_transitions import is_full_crosslink
 from eth2spec.test.helpers.state import state_transition_and_sign_block
 from eth2spec.test.helpers.block import build_empty_block
 
@@ -209,11 +208,8 @@ def create_simple_fork(spec, state, store, shard):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_shard_simple_fork(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     spec.PHASE_1_GENESIS_SLOT = 0  # NOTE: mock genesis slot here
     state = spec.upgrade_to_phase1(state)
     shard = spec.Shard(1)
@@ -237,11 +233,8 @@ def test_shard_simple_fork(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_shard_latest_messages_for_different_shards(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     spec.PHASE_1_GENESIS_SLOT = 0  # NOTE: mock genesis slot here
     state = spec.upgrade_to_phase1(state)
     shard_0 = spec.Shard(0)

tests/formats/README.md

@@ -165,6 +165,9 @@ bls_setting: int     -- optional, can have 3 different values:
                                  but there is no change of outcome when running the test if BLS is ON or OFF.
                             1: known as "BLS required" - if the test validity is strictly dependent on BLS being ON
                             2: known as "BLS ignored"  - if the test validity is strictly dependent on BLS being OFF
+reveal_deadlines_setting:   -- optional, can have 2 different values:
+                            0: default, `process_reveal_deadlines` is ON.
+                            1: `process_reveal_deadlines` is OFF.
 ```
 
 

tests/formats/sanity/blocks.md

@@ -7,9 +7,10 @@ Sanity tests to cover a series of one or more blocks being processed, aiming to
 ### `meta.yaml`
 
 ```yaml
-description: string    -- Optional. Description of test case, purely for debugging purposes.
-bls_setting: int       -- see general test-format spec.
-blocks_count: int      -- the number of blocks processed in this test.
+description: string            -- Optional. Description of test case, purely for debugging purposes.
+bls_setting: int               -- see general test-format spec.
+reveal_deadlines_setting: int  -- see general test-format spec.
+blocks_count: int              -- the number of blocks processed in this test.
 ```
 
 

tests/generators/epoch_processing/main.py

@@ -1,24 +1,17 @@
 from typing import Iterable
 
-from eth2spec.phase0 import spec as spec_phase0
-from eth2spec.phase1 import spec as spec_phase1
-from eth2spec.test.phase0.epoch_processing import (
-    test_process_final_updates,
-    test_process_justification_and_finalization,
-    test_process_registry_updates,
-    test_process_rewards_and_penalties,
-    test_process_slashings
-)
 from gen_base import gen_runner, gen_typing
 from gen_from_tests.gen import generate_from_tests
-from importlib import reload
+from importlib import reload, import_module
 from eth2spec.config import config_util
-from eth2spec.test.context import PHASE0
+from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.phase1 import spec as spec_phase1
+from eth2spec.test.context import PHASE0, PHASE1
 from eth2spec.utils import bls
 
 
-def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
-
+def create_provider(fork_name: str, handler_name: str,
+                    tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
     def prepare_fn(configs_path: str) -> str:
         config_util.prepare_config(configs_path, config_name)
         reload(spec_phase0)
@@ -27,27 +20,40 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin
         return config_name
 
     def cases_fn() -> Iterable[gen_typing.TestCase]:
+        tests_src = import_module(tests_src_mod_name)
         return generate_from_tests(
             runner_name='epoch_processing',
             handler_name=handler_name,
             src=tests_src,
-            fork_name=PHASE0,
+            fork_name=fork_name,
         )
 
     return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
 
 
 if __name__ == "__main__":
-    gen_runner.run_generator("epoch_processing", [
-        create_provider('final_updates', test_process_final_updates, 'minimal'),
-        create_provider('final_updates', test_process_final_updates, 'mainnet'),
-        create_provider('justification_and_finalization', test_process_justification_and_finalization, 'minimal'),
-        create_provider('justification_and_finalization', test_process_justification_and_finalization, 'mainnet'),
-        create_provider('registry_updates', test_process_registry_updates, 'minimal'),
-        create_provider('registry_updates', test_process_registry_updates, 'mainnet'),
-        create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'minimal'),
-        # runs full epochs filled with data, with uncached ssz. Disabled for now.
-        # create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'mainnet'),
-        create_provider('slashings', test_process_slashings, 'minimal'),
-        create_provider('slashings', test_process_slashings, 'mainnet'),
+    phase_0_mods = {key: 'eth2spec.test.phase0.epoch_processing.test_process_' + key for key in [
+        'final_updates',
+        'justification_and_finalization',
+        'registry_updates',
+        'rewards_and_penalties',
+        'slashings',
+    ]}
+    phase_1_mods = {**{key: 'eth2spec.test.phase1.epoch_processing.test_process_' + key for key in [
+        'challenge_deadlines',
+        'custody_final_updates',
+        'reveal_deadlines',
+    ]}, **phase_0_mods}  # also run the previous phase 0 tests (but against phase 1 spec)
+
+    gen_runner.run_generator(f"epoch_processing", [
+        create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items()
+    ])
+    gen_runner.run_generator(f"epoch_processing", [
+        create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items()
+    ])
+    gen_runner.run_generator(f"epoch_processing", [
+        create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items()
+    ])
+    gen_runner.run_generator(f"epoch_processing", [
+        create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items()
     ])

tests/generators/finality/main.py

@@ -4,7 +4,7 @@ from importlib import reload
 from gen_base import gen_runner, gen_typing
 from gen_from_tests.gen import generate_from_tests
 
-from eth2spec.test.context import PHASE0
+from eth2spec.test.context import PHASE0, PHASE1
 from eth2spec.test.phase0.finality import test_finality
 from eth2spec.config import config_util
 from eth2spec.phase0 import spec as spec_phase0
@@ -12,7 +12,7 @@ from eth2spec.phase1 import spec as spec_phase1
 from eth2spec.utils import bls
 
 
-def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
+def create_provider(fork_name: str, handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
 
     def prepare_fn(configs_path: str) -> str:
         config_util.prepare_config(configs_path, config_name)
@@ -26,14 +26,18 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin
             runner_name='finality',
             handler_name=handler_name,
             src=tests_src,
-            fork_name=PHASE0,
+            fork_name=fork_name,
         )
 
     return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
 
 
 if __name__ == "__main__":
+    # No additional phase 1 specific rewards tests, yet.
+    key = 'finality'
     gen_runner.run_generator("finality", [
-        create_provider('finality', test_finality, 'minimal'),
-        create_provider('finality', test_finality, 'mainnet'),
+        create_provider(PHASE0, 'finality', test_finality, 'minimal'),
+        create_provider(PHASE0, 'finality', test_finality, 'mainnet'),
+        create_provider(PHASE1, 'finality', test_finality, 'minimal'),
+        create_provider(PHASE1, 'finality', test_finality, 'mainnet'),
     ])

tests/generators/operations/main.py

@@ -1,26 +1,17 @@
 from typing import Iterable
 
-from eth2spec.test.phase0.block_processing import (
-    test_process_attestation,
-    test_process_attester_slashing,
-    test_process_block_header,
-    test_process_deposit,
-    test_process_proposer_slashing,
-    test_process_voluntary_exit,
-)
-
 from gen_base import gen_runner, gen_typing
 from gen_from_tests.gen import generate_from_tests
-from importlib import reload
+from importlib import reload, import_module
 from eth2spec.config import config_util
 from eth2spec.phase0 import spec as spec_phase0
 from eth2spec.phase1 import spec as spec_phase1
-from eth2spec.test.context import PHASE0
+from eth2spec.test.context import PHASE0, PHASE1
 from eth2spec.utils import bls
 
 
-def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
-
+def create_provider(fork_name: str, handler_name: str,
+                    tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
     def prepare_fn(configs_path: str) -> str:
         config_util.prepare_config(configs_path, config_name)
         reload(spec_phase0)
@@ -29,28 +20,44 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin
         return config_name
 
     def cases_fn() -> Iterable[gen_typing.TestCase]:
+        tests_src = import_module(tests_src_mod_name)
         return generate_from_tests(
             runner_name='operations',
             handler_name=handler_name,
             src=tests_src,
-            fork_name=PHASE0,
+            fork_name=fork_name,
         )
 
     return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
 
 
 if __name__ == "__main__":
-    gen_runner.run_generator("operations", [
-        create_provider('attestation', test_process_attestation, 'minimal'),
-        create_provider('attestation', test_process_attestation, 'mainnet'),
-        create_provider('attester_slashing', test_process_attester_slashing, 'minimal'),
-        create_provider('attester_slashing', test_process_attester_slashing, 'mainnet'),
-        create_provider('block_header', test_process_block_header, 'minimal'),
-        create_provider('block_header', test_process_block_header, 'mainnet'),
-        create_provider('deposit', test_process_deposit, 'minimal'),
-        create_provider('deposit', test_process_deposit, 'mainnet'),
-        create_provider('proposer_slashing', test_process_proposer_slashing, 'minimal'),
-        create_provider('proposer_slashing', test_process_proposer_slashing, 'mainnet'),
-        create_provider('voluntary_exit', test_process_voluntary_exit, 'minimal'),
-        create_provider('voluntary_exit', test_process_voluntary_exit, 'mainnet'),
+    phase_0_mods = {key: 'eth2spec.test.phase0.block_processing.test_process_' + key for key in [
+        'attestation',
+        'attester_slashing',
+        'block_header',
+        'deposit',
+        'proposer_slashing',
+        'voluntary_exit',
+    ]}
+    phase_1_mods = {**{key: 'eth2spec.test.phase1.block_processing.test_process_' + key for key in [
+        'attestation',
+        'chunk_challenge',
+        'custody_key_reveal',
+        'custody_slashing',
+        'early_derived_secret_reveal',
+        'shard_transition',
+    ]}, **phase_0_mods}  # also run the previous phase 0 tests (but against phase 1 spec)
+
+    gen_runner.run_generator(f"operations", [
+        create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items()
+    ])
+    gen_runner.run_generator(f"operations", [
+        create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items()
+    ])
+    gen_runner.run_generator(f"operations", [
+        create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items()
+    ])
+    gen_runner.run_generator(f"operations", [
+        create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items()
     ])

tests/generators/rewards/main.py

@@ -1,22 +1,17 @@
 from typing import Iterable
 
-from eth2spec.phase0 import spec as spec_phase0
-from eth2spec.phase1 import spec as spec_phase1
-from eth2spec.test.phase0.rewards import (
-    test_basic,
-    test_leak,
-    test_random,
-)
 from gen_base import gen_runner, gen_typing
 from gen_from_tests.gen import generate_from_tests
-from importlib import reload
+from importlib import reload, import_module
 from eth2spec.config import config_util
-from eth2spec.test.context import PHASE0
+from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.phase1 import spec as spec_phase1
+from eth2spec.test.context import PHASE0, PHASE1
 from eth2spec.utils import bls
 
 
-def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider:
-
+def create_provider(fork_name: str, handler_name: str,
+                    tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
     def prepare_fn(configs_path: str) -> str:
         config_util.prepare_config(configs_path, config_name)
         reload(spec_phase0)
@@ -25,22 +20,35 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider:
         return config_name
 
     def cases_fn() -> Iterable[gen_typing.TestCase]:
+        tests_src = import_module(tests_src_mod_name)
         return generate_from_tests(
             runner_name='rewards',
-            handler_name='core',
+            handler_name=handler_name,
             src=tests_src,
-            fork_name=PHASE0,
+            fork_name=fork_name,
         )
 
     return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
 
 
 if __name__ == "__main__":
-    gen_runner.run_generator("rewards", [
-        create_provider(test_basic, 'minimal'),
-        create_provider(test_basic, 'mainnet'),
-        create_provider(test_leak, 'minimal'),
-        create_provider(test_leak, 'mainnet'),
-        create_provider(test_random, 'minimal'),
-        create_provider(test_random, 'mainnet'),
+    phase_0_mods = {key: 'eth2spec.test.phase0.rewards.test_' + key for key in [
+        'basic',
+        'leak',
+        'random',
+    ]}
+    # No additional phase 1 specific rewards tests, yet.
+    phase_1_mods = phase_0_mods
+
+    gen_runner.run_generator(f"rewards", [
+        create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items()
+    ])
+    gen_runner.run_generator(f"rewards", [
+        create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items()
+    ])
+    gen_runner.run_generator(f"rewards", [
+        create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items()
+    ])
+    gen_runner.run_generator(f"rewards", [
+        create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items()
     ])

tests/generators/sanity/main.py

@@ -1,19 +1,17 @@
 from typing import Iterable
-from importlib import reload
 
 from gen_base import gen_runner, gen_typing
 from gen_from_tests.gen import generate_from_tests
-
-from eth2spec.test.context import PHASE0
-from eth2spec.test.phase0.sanity import test_blocks, test_slots
+from importlib import reload, import_module
 from eth2spec.config import config_util
 from eth2spec.phase0 import spec as spec_phase0
 from eth2spec.phase1 import spec as spec_phase1
+from eth2spec.test.context import PHASE0, PHASE1
 from eth2spec.utils import bls
 
 
-def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
-
+def create_provider(fork_name: str, handler_name: str,
+                    tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
     def prepare_fn(configs_path: str) -> str:
         config_util.prepare_config(configs_path, config_name)
         reload(spec_phase0)
@@ -22,20 +20,36 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin
         return config_name
 
     def cases_fn() -> Iterable[gen_typing.TestCase]:
+        tests_src = import_module(tests_src_mod_name)
         return generate_from_tests(
             runner_name='sanity',
             handler_name=handler_name,
             src=tests_src,
-            fork_name=PHASE0,
+            fork_name=fork_name,
         )
 
     return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
 
 
 if __name__ == "__main__":
-    gen_runner.run_generator("sanity", [
-        create_provider('blocks', test_blocks, 'minimal'),
-        create_provider('blocks', test_blocks, 'mainnet'),
-        create_provider('slots', test_slots, 'minimal'),
-        create_provider('slots', test_slots, 'mainnet'),
+    phase_0_mods = {key: 'eth2spec.test.phase0.sanity.test_' + key for key in [
+        'blocks',
+        'slots',
+    ]}
+    phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [
+        'blocks',  # more phase 1 specific block tests
+        'shard_blocks',
+    ]}, **phase_0_mods}  # also run the previous phase 0 tests (but against phase 1 spec)
+
+    gen_runner.run_generator(f"sanity", [
+        create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items()
+    ])
+    gen_runner.run_generator(f"sanity", [
+        create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items()
+    ])
+    gen_runner.run_generator(f"sanity", [
+        create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items()
+    ])
+    gen_runner.run_generator(f"sanity", [
+        create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items()
     ])

tests/generators/ssz_generic/ssz_boolean.py

@@ -12,4 +12,3 @@ def invalid_cases():
     yield "byte_rev_nibble", invalid_test_case(lambda: b'\x10')
     yield "byte_0x80", invalid_test_case(lambda: b'\x80')
     yield "byte_full", invalid_test_case(lambda: b'\xff')
-

tests/generators/ssz_generic/ssz_container.py

@@ -85,7 +85,7 @@ def valid_cases():
 def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]):
     return b[:offset_index] + \
            (change(int.from_bytes(b[offset_index:offset_index + 4], byteorder='little')) & 0xffffffff) \
-               .to_bytes(length=4, byteorder='little') + \
+           .to_bytes(length=4, byteorder='little') + \
            b[offset_index + 4:]
 
 

tests/generators/ssz_static/main.py

@@ -7,8 +7,9 @@ from gen_base import gen_runner, gen_typing
 
 from eth2spec.debug import random_value, encode
 from eth2spec.config import config_util
-from eth2spec.phase0 import spec
-from eth2spec.test.context import PHASE0
+from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.phase1 import spec as spec_phase1
+from eth2spec.test.context import PHASE0, PHASE1
 from eth2spec.utils.ssz.ssz_typing import Container
 from eth2spec.utils.ssz.ssz_impl import (
     hash_tree_root,
@@ -16,11 +17,12 @@ from eth2spec.utils.ssz.ssz_impl import (
 )
 
 
-MAX_BYTES_LENGTH = 100
+MAX_BYTES_LENGTH = 1000
 MAX_LIST_LENGTH = 10
 
 
-def create_test_case(rng: Random, typ, mode: random_value.RandomizationMode, chaos: bool) -> Iterable[gen_typing.TestCasePart]:
+def create_test_case(rng: Random, typ,
+                     mode: random_value.RandomizationMode, chaos: bool) -> Iterable[gen_typing.TestCasePart]:
     value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos)
     yield "value", "data", encode.encode(value)
     yield "serialized", "ssz", serialize(value)
@@ -30,14 +32,15 @@ def create_test_case(rng: Random, typ, mode: random_value.RandomizationMode, cha
     yield "roots", "data", roots_data
 
 
-def get_spec_ssz_types():
+def get_spec_ssz_types(spec):
     return [
         (name, value) for (name, value) in getmembers(spec, isclass)
         if issubclass(value, Container) and value != Container  # only the subclasses, not the imported base class
     ]
 
 
-def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.RandomizationMode, chaos: bool, count: int):
+def ssz_static_cases(fork_name: str, seed: int, name, ssz_type,
+                     mode: random_value.RandomizationMode, chaos: bool, count: int):
     random_mode_name = mode.to_name()
 
     # Reproducible RNG
@@ -45,7 +48,7 @@ def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.Randomization
 
     for i in range(count):
         yield gen_typing.TestCase(
-            fork_name=PHASE0,
+            fork_name=fork_name,
             runner_name='ssz_static',
             handler_name=name,
             suite_name=f"ssz_{random_mode_name}{'_chaos' if chaos else ''}",
@@ -54,19 +57,23 @@ def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.Randomization
         )
 
 
-def create_provider(config_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool,
+def create_provider(fork_name, config_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool,
                     cases_if_random: int) -> gen_typing.TestProvider:
     def prepare_fn(configs_path: str) -> str:
         # Apply changes to presets, this affects some of the vector types.
         config_util.prepare_config(configs_path, config_name)
-        reload(spec)
+        reload(spec_phase0)
+        reload(spec_phase1)
         return config_name
 
     def cases_fn() -> Iterable[gen_typing.TestCase]:
         count = cases_if_random if chaos or mode.is_changing() else 1
+        spec = spec_phase0
+        if fork_name == PHASE1:
+            spec = spec_phase1
 
-        for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types()):
-            yield from ssz_static_cases(seed * 1000 + i, name, ssz_type, mode, chaos, count)
+        for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types(spec)):
+            yield from ssz_static_cases(fork_name, seed * 1000 + i, name, ssz_type, mode, chaos, count)
 
     return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
 
@@ -80,10 +87,11 @@ if __name__ == "__main__":
         seed += 1
     settings.append((seed, "minimal", random_value.RandomizationMode.mode_random, True, 30))
     seed += 1
-    settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5))
-    seed += 1
+    # settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5))
+    # seed += 1
 
-    gen_runner.run_generator("ssz_static", [
-        create_provider(config_name, seed, mode, chaos, cases_if_random)
-        for (seed, config_name, mode, chaos, cases_if_random) in settings
-    ])
+    for fork in [PHASE0, PHASE1]:
+        gen_runner.run_generator("ssz_static", [
+            create_provider(fork, config_name, seed, mode, chaos, cases_if_random)
+            for (seed, config_name, mode, chaos, cases_if_random) in settings
+        ])