Update testing context

1. Add `with_configs` decorator to assign available configs
2. Add `only_full_crosslink` decorator to detect if the configuation can
do full crosslinking
3. Add `context.is_pytest` flag: True if calling via pytest. False if
calling from test generator.

configs/mainnet/phase0.yaml

@@ -2,6 +2,7 @@
 # Note: the intention of this file (for now) is to illustrate what a mainnet configuration could look like.
 # Some of these constants may still change before the launch of Phase 0.
 
+CONFIG_NAME: "mainnet"
 
 # Misc
 # ---------------------------------------------------------------

configs/mainnet/phase1.yaml

@@ -1,5 +1,6 @@
 # Mainnet preset - phase 1
 
+CONFIG_NAME: "mainnet"
 
 # phase1-fork
 # ---------------------------------------------------------------

configs/minimal/phase0.yaml

@@ -1,5 +1,6 @@
 # Minimal preset
 
+CONFIG_NAME: "minimal"
 
 # Misc
 # ---------------------------------------------------------------

configs/minimal/phase1.yaml

@@ -1,5 +1,6 @@
 # Minimal preset - phase 1
 
+CONFIG_NAME: "minimal"
 
 # phase1-fork
 # ---------------------------------------------------------------

setup.py

@@ -119,6 +119,8 @@ from eth2spec.utils import bls
 from eth2spec.utils.hash_function import hash
 
 SSZObject = TypeVar('SSZObject', bound=View)
+
+CONFIG_NAME = 'mainnet'
 '''
 PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0
 from eth2spec.config.config_util import apply_constants_config
@@ -150,6 +152,8 @@ reload(phase0)
 SSZVariableName = str
 GeneralizedIndex = NewType('GeneralizedIndex', int)
 SSZObject = TypeVar('SSZObject', bound=View)
+
+CONFIG_NAME = 'mainnet'
 '''
 SUNDRY_CONSTANTS_FUNCTIONS = '''
 def ceillog2(x: uint64) -> int:

tests/core/gen_helpers/gen_base/gen_runner.py

@@ -10,6 +10,13 @@ from ruamel.yaml import (
 
 from gen_base.gen_typing import TestProvider
 
+from eth2spec.test import context
+from eth2spec.test.exceptions import SkippedTest
+
+
+# Flag that the runner does NOT run test via pytest
+context.is_pytest = False
+
 
 def validate_output_dir(path_str):
     path = Path(path_str)
@@ -136,12 +143,8 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
                 written_part = False
                 meta = dict()
 
-                output = test_case.case_fn()
-                # If the output is `None`, that means the testcase doesn't support this fork or this config.
-                if output is None:
-                    continue
-
-                for (name, out_kind, data) in output:
+                try:
+                    for (name, out_kind, data) in test_case.case_fn():
                         written_part = True
                         if out_kind == "meta":
                             meta[name] = data
@@ -149,6 +152,10 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
                             output_part("data", name, dump_yaml_fn(data, name, file_mode, yaml))
                         if out_kind == "ssz":
                             output_part("ssz", name, dump_ssz_fn(data, name, file_mode))
+                except SkippedTest as e:
+                    print(e)
+                    continue
+
                 # Once all meta data is collected (if any), write it to a meta data file.
                 if len(meta) != 0:
                     written_part = True

tests/core/gen_helpers/requirements.txt

@@ -1,2 +1,3 @@
 ruamel.yaml==0.16.5
 eth-utils==1.6.0
+pytest==3.6.1

tests/core/gen_helpers/setup.py

@@ -5,6 +5,7 @@ setup(
     packages=['gen_base', 'gen_from_tests'],
     install_requires=[
         "ruamel.yaml==0.16.5",
-        "eth-utils==1.6.0"
+        "eth-utils==1.6.0",
+        "pytest==3.6.1",
     ]
 )

tests/core/pyspec/eth2spec/config/config_util.py

@@ -54,6 +54,8 @@ def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]:
             out[k] = [int(item) if item.isdigit() else item for item in v]
         elif isinstance(v, str) and v.startswith("0x"):
             out[k] = bytes.fromhex(v[2:])
+        elif k == "CONFIG_NAME":
+            out[k] = str(v)
         else:
             out[k] = int(v)
     return out

tests/core/pyspec/eth2spec/test/context.py

@@ -1,9 +1,11 @@
+import pytest
+
 from eth2spec.phase0 import spec as spec_phase0
 from eth2spec.phase1 import spec as spec_phase1
 from eth2spec.utils import bls
 
+from .exceptions import SkippedTest
 from .helpers.genesis import create_genesis_state
-
 from .utils import vector_test, with_meta_tags
 
 from random import Random
@@ -22,11 +24,16 @@ def reload_specs():
 # Some of the Spec module functionality is exposed here to deal with phase-specific changes.
 
 SpecForkName = NewType("SpecForkName", str)
+ConfigName = NewType("ConfigName", str)
 
 PHASE0 = SpecForkName('phase0')
 PHASE1 = SpecForkName('phase1')
 ALL_PHASES = (PHASE0, PHASE1)
 
+MAINNET = ConfigName('mainnet')
+MINIMAL = ConfigName('minimal')
+
+
 # TODO: currently phases are defined as python modules.
 # It would be better if they would be more well-defined interfaces for stronger typing.
 
@@ -175,6 +182,17 @@ def single_phase(fn):
 DEFAULT_BLS_ACTIVE = True
 
 
+is_pytest = True
+
+
+def dump_skipping_message(reason: str) -> None:
+    message = f"[Skipped test] {reason}"
+    if is_pytest:
+        pytest.skip(message)
+    else:
+        raise SkippedTest(message)
+
+
 def spec_test(fn):
     # Bls switch must be wrapped by vector_test,
     # to fully go through the yielded bls switch data, before setting back the BLS setting.
@@ -275,7 +293,8 @@ def with_phases(phases, other_phases=None):
             if 'phase' in kw:
                 phase = kw.pop('phase')
                 if phase not in phases:
-                    return
+                    dump_skipping_message(f"doesn't support this fork: {phase}")
+                    return None
                 run_phases = [phase]
 
             available_phases = set(run_phases)
@@ -300,3 +319,30 @@ def with_phases(phases, other_phases=None):
             return ret
         return wrapper
     return decorator
+
+
+def with_configs(configs):
+    def decorator(fn):
+        def wrapper(*args, spec: Spec, **kw):
+            available_configs = set(configs)
+            if spec.CONFIG_NAME not in available_configs:
+                dump_skipping_message(f"doesn't support this config: {spec.CONFIG_NAME}")
+                return None
+
+            return fn(*args, spec=spec, **kw)
+        return wrapper
+    return decorator
+
+
+def only_full_crosslink(fn):
+    def is_full_crosslink(spec, state):
+        epoch = spec.compute_epoch_at_slot(state.slot)
+        return spec.get_committee_count_per_slot(state, epoch) >= spec.get_active_shard_count(state)
+
+    def wrapper(*args, spec: Spec, state: Any, **kw):
+        # TODO: update condition to "phase1+" if we have phase2
+        if spec.fork == PHASE1 and not is_full_crosslink(spec, state):
+            dump_skipping_message("only for full crosslink")
+            return None
+        return fn(*args, spec=spec, state=state, **kw)
+    return wrapper

tests/core/pyspec/eth2spec/test/exceptions.py

@@ -0,0 +1,2 @@
+class SkippedTest(Exception):
+    ...

tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py

@@ -35,8 +35,3 @@ def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks
     shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot)
     shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks)
     return shard_transition
-
-
-def is_full_crosslink(spec, state):
-    epoch = spec.compute_epoch_at_slot(state.slot)
-    return spec.get_committee_count_per_slot(state, epoch) >= spec.get_active_shard_count(state)

tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py

@@ -9,9 +9,9 @@ from eth2spec.test.helpers.attestations import (
 from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot
 from eth2spec.test.context import (
     PHASE0,
-    with_all_phases_except,
-    spec_state_test,
     expect_assertion_error,
+    spec_state_test,
+    with_all_phases_except,
 )
 from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing
 

tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py

@@ -1,6 +1,7 @@
 from eth2spec.test.context import (
     PHASE0,
     with_all_phases_except,
+    only_full_crosslink,
     spec_state_test,
 )
 from eth2spec.test.helpers.attestations import (
@@ -10,7 +11,6 @@ from eth2spec.test.helpers.attestations import (
 )
 from eth2spec.test.helpers.shard_transitions import (
     run_shard_transitions_processing,
-    is_full_crosslink,
 )
 from eth2spec.test.helpers.shard_block import (
     build_shard_block,
@@ -92,31 +92,22 @@ def run_successful_crosslink_tests(spec, state, target_len_offset_slot):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_basic_crosslinks(spec, state):
-    if not is_full_crosslink(spec, state):
-        # Skip this test
-        return
-
     yield from run_successful_crosslink_tests(spec, state, target_len_offset_slot=1)
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_multiple_offset_slots(spec, state):
-    if not is_full_crosslink(spec, state):
-        # Skip this test
-        return
-
     yield from run_successful_crosslink_tests(spec, state, target_len_offset_slot=2)
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_no_winning_root(spec, state):
-    if not is_full_crosslink(spec, state):
-        # Skip this test
-        return
-
     state, shard, target_shard_slot = get_initial_env(spec, state, target_len_offset_slot=1)
     init_slot = state.slot
 
@@ -163,11 +154,8 @@ def test_no_winning_root(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_wrong_shard_transition_root(spec, state):
-    if not is_full_crosslink(spec, state):
-        # Skip this test
-        return
-
     state, shard, target_shard_slot = get_initial_env(spec, state, target_len_offset_slot=1)
     init_slot = state.slot
 

tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py

@@ -1,9 +1,11 @@
 from typing import Dict, Sequence
 
 from eth2spec.test.context import (
-    PHASE0,
+    PHASE0, MINIMAL,
     with_all_phases_except,
     spec_state_test,
+    only_full_crosslink,
+    with_configs,
 )
 from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
 from eth2spec.test.helpers.block import build_empty_block
@@ -22,7 +24,6 @@ from eth2spec.test.helpers.shard_block import (
     get_sample_shard_block_body,
     get_shard_transitions,
 )
-from eth2spec.test.helpers.shard_transitions import is_full_crosslink
 from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to
 
 
@@ -99,12 +100,8 @@ def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, comm
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_process_beacon_block_with_normal_shard_transition(spec, state):
-    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     transition_to_valid_shard_slot(spec, state)
 
     target_len_offset_slot = 1
@@ -117,12 +114,8 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_process_beacon_block_with_empty_proposal_transition(spec, state):
-    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     transition_to_valid_shard_slot(spec, state)
 
     target_len_offset_slot = 1
@@ -140,12 +133,8 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_with_shard_transition_with_custody_challenge_and_response(spec, state):
-    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     transition_to_valid_shard_slot(spec, state)
 
     # build shard block
@@ -178,6 +167,7 @@ def test_with_shard_transition_with_custody_challenge_and_response(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@with_configs([MINIMAL])
 def test_custody_key_reveal(spec, state):
     transition_to_valid_shard_slot(spec, state)
     transition_to(spec, state, state.slot + spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH)
@@ -202,12 +192,8 @@ def test_early_derived_secret_reveal(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_custody_slashing(spec, state):
-    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     transition_to_valid_shard_slot(spec, state)
 
     # Build shard block

tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py

@@ -4,12 +4,12 @@ from eth2spec.test.context import (
     expect_assertion_error,
     spec_state_test,
     with_all_phases_except,
+    only_full_crosslink,
 )
 from eth2spec.test.helpers.shard_block import (
     build_shard_block,
     sign_shard_block,
 )
-from eth2spec.test.helpers.shard_transitions import is_full_crosslink
 from eth2spec.test.helpers.state import next_slot, transition_to_valid_shard_slot, transition_to
 
 
@@ -46,11 +46,8 @@ def run_shard_blocks(spec, shard_state, signed_shard_block, beacon_parent_state,
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_valid_shard_block(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
 
@@ -68,11 +65,8 @@ def test_valid_shard_block(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_invalid_shard_parent_root(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
 
@@ -87,11 +81,8 @@ def test_invalid_shard_parent_root(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_invalid_beacon_parent_root(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -105,11 +96,8 @@ def test_invalid_beacon_parent_root(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_invalid_slot(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -124,11 +112,8 @@ def test_invalid_slot(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_invalid_proposer_index(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -148,12 +133,8 @@ def test_invalid_proposer_index(spec, state):
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_out_of_bound_offset(spec, state):
-    # TODO: Handle this edge case properly
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -173,11 +154,8 @@ def test_out_of_bound_offset(spec, state):
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_invalid_offset(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     # 4 is not in `SHARD_BLOCK_OFFSETS`
@@ -195,11 +173,8 @@ def test_invalid_offset(spec, state):
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_empty_block_body(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -217,11 +192,8 @@ def test_empty_block_body(spec, state):
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_invalid_signature(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -239,11 +211,8 @@ def test_invalid_signature(spec, state):
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_max_offset(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)
     shard = 0
@@ -259,11 +228,8 @@ def test_max_offset(spec, state):
 @with_all_phases_except([PHASE0])
 @spec_state_test
 @always_bls
+@only_full_crosslink
 def test_pending_shard_parent_block(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     # Block N
     beacon_state = state.copy()
     transition_to_valid_shard_slot(spec, beacon_state)

tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py

@@ -1,6 +1,6 @@
 from eth2spec.utils.ssz.ssz_impl import hash_tree_root
 
-from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls
+from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls, only_full_crosslink
 from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
 from eth2spec.test.helpers.shard_block import (
     build_shard_block,
@@ -8,7 +8,6 @@ from eth2spec.test.helpers.shard_block import (
     get_committee_index_of_shard,
 )
 from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root
-from eth2spec.test.helpers.shard_transitions import is_full_crosslink
 from eth2spec.test.helpers.state import state_transition_and_sign_block
 from eth2spec.test.helpers.block import build_empty_block
 
@@ -209,11 +208,8 @@ def create_simple_fork(spec, state, store, shard):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_shard_simple_fork(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     spec.PHASE_1_GENESIS_SLOT = 0  # NOTE: mock genesis slot here
     state = spec.upgrade_to_phase1(state)
     shard = spec.Shard(1)
@@ -237,11 +233,8 @@ def test_shard_simple_fork(spec, state):
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
+@only_full_crosslink
 def test_shard_latest_messages_for_different_shards(spec, state):
-    if not is_full_crosslink(spec, state):
-        # skip
-        return
-
     spec.PHASE_1_GENESIS_SLOT = 0  # NOTE: mock genesis slot here
     state = spec.upgrade_to_phase1(state)
     shard_0 = spec.Shard(0)

tests/generators/sanity/main.py

@@ -54,4 +54,3 @@ if __name__ == "__main__":
     # gen_runner.run_generator(f"sanity", [
     #     create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods
     # ])
-