From 74a2a1c0d4a7706fd77da2b85769c7b355c20562 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 4 Sep 2020 02:40:36 +0800 Subject: [PATCH 1/3] Enable all tests --- tests/generators/epoch_processing/main.py | 14 ++++++-------- tests/generators/operations/main.py | 14 ++++++-------- tests/generators/sanity/main.py | 14 ++++++-------- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index cf87e7451..d4114b93b 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -47,14 +47,12 @@ if __name__ == "__main__": gen_runner.run_generator(f"epoch_processing", [ create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods ]) - # TODO: disabled for testing - # gen_runner.run_generator(f"epoch_processing", [ - # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods - # ]) + gen_runner.run_generator(f"epoch_processing", [ + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + ]) gen_runner.run_generator(f"epoch_processing", [ create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods ]) - # Disabled for now - # gen_runner.run_generator(f"epoch_processing", [ - # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods - # ]) + gen_runner.run_generator(f"epoch_processing", [ + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + ]) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index c04ad2671..0ecc63f46 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -51,14 +51,12 @@ if __name__ == "__main__": gen_runner.run_generator(f"operations", [ create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods ]) - # TODO: disabled for testing - # gen_runner.run_generator(f"operations", [ - # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods - # ]) + gen_runner.run_generator(f"operations", [ + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + ]) gen_runner.run_generator(f"operations", [ create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods ]) - # Disabled for now - # gen_runner.run_generator(f"operations", [ - # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods - # ]) + gen_runner.run_generator(f"operations", [ + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + ]) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 032339970..c2669ffdc 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -43,14 +43,12 @@ if __name__ == "__main__": gen_runner.run_generator(f"sanity", [ create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods ]) - # TODO: disabled for testing - # gen_runner.run_generator(f"sanity", [ - # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods - # ]) + gen_runner.run_generator(f"sanity", [ + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + ]) gen_runner.run_generator(f"sanity", [ create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods ]) - # Disabled for now - # gen_runner.run_generator(f"sanity", [ - # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods - # ]) + gen_runner.run_generator(f"sanity", [ + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + ]) From a0d646e1e06ca431c24b4045d0b3485845407609 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 4 Sep 2020 02:41:09 +0800 Subject: [PATCH 2/3] Add decorators to diable testing with mainnet config --- tests/core/pyspec/eth2spec/test/context.py | 7 +++++-- .../test_process_chunk_challenge.py | 13 +++++++++++++ .../test_process_custody_slashing.py | 7 +++++++ .../test_process_challenge_deadlines.py | 5 ++++- .../test_process_reveal_deadlines.py | 4 ++++ 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index c2bbff341..7f58c6610 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -329,12 +329,15 @@ def disable_process_reveal_deadlines(fn): return with_meta_tags({'reveal_deadlines_setting': 1})(entry) -def with_configs(configs): +def with_configs(configs, reason=None): def decorator(fn): def wrapper(*args, spec: Spec, **kw): available_configs = set(configs) if spec.CONFIG_NAME not in available_configs: - dump_skipping_message(f"doesn't support this config: {spec.CONFIG_NAME}") + message = f"doesn't support this config: {spec.CONFIG_NAME}." + if reason is not None: + message = f"{message} Reason: {reason}" + dump_skipping_message(message) return None return fn(*args, spec=spec, **kw) diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py index ecc704184..e916010b2 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py @@ -9,10 +9,12 @@ from eth2spec.test.helpers.attestations import ( from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot from eth2spec.test.context import ( PHASE0, + MINIMAL, expect_assertion_error, disable_process_reveal_deadlines, spec_state_test, with_all_phases_except, + with_configs, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing @@ -68,6 +70,7 @@ def run_custody_chunk_response_processing(spec, state, custody_response, valid=T @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") @disable_process_reveal_deadlines def test_challenge_appended(spec, state): transition_to_valid_shard_slot(spec, state) @@ -92,6 +95,7 @@ def test_challenge_appended(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_challenge_empty_element_replaced(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -117,6 +121,7 @@ def test_challenge_empty_element_replaced(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_duplicate_challenge(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -142,6 +147,7 @@ def test_duplicate_challenge(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_second_challenge(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -169,6 +175,7 @@ def test_second_challenge(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_multiple_epochs_custody(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) @@ -193,6 +200,7 @@ def test_multiple_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_many_epochs_custody(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) @@ -217,6 +225,7 @@ def test_many_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_off_chain_attestation(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) @@ -237,6 +246,7 @@ def test_off_chain_attestation(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_response(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) @@ -268,6 +278,7 @@ def test_custody_response(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_response_chunk_index_2(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) @@ -298,6 +309,7 @@ def test_custody_response_chunk_index_2(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_response_multiple_epochs(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) @@ -329,6 +341,7 @@ def test_custody_response_multiple_epochs(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_response_many_epochs(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py index 6836685b8..fc7efa5bc 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py @@ -10,10 +10,12 @@ from eth2spec.utils.ssz.ssz_typing import ByteList from eth2spec.test.helpers.state import get_balance, transition_to from eth2spec.test.context import ( PHASE0, + MINIMAL, with_all_phases_except, spec_state_test, expect_assertion_error, disable_process_reveal_deadlines, + with_configs, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing @@ -113,6 +115,7 @@ def run_standard_custody_slashing_test(spec, @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_slashing(spec, state): yield from run_standard_custody_slashing_test(spec, state) @@ -120,6 +123,7 @@ def test_custody_slashing(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_incorrect_custody_slashing(spec, state): yield from run_standard_custody_slashing_test(spec, state, correct=False) @@ -127,6 +131,7 @@ def test_incorrect_custody_slashing(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_multiple_epochs_custody(spec, state): yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 3) @@ -134,6 +139,7 @@ def test_multiple_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_many_epochs_custody(spec, state): yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 5) @@ -141,6 +147,7 @@ def test_many_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_invalid_custody_slashing(spec, state): yield from run_standard_custody_slashing_test( spec, diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py index 828ace91c..e270ff615 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py @@ -8,8 +8,10 @@ from eth2spec.test.helpers.attestations import ( from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot from eth2spec.test.context import ( PHASE0, - with_all_phases_except, + MINIMAL, spec_state_test, + with_all_phases_except, + with_configs, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with @@ -25,6 +27,7 @@ def run_process_challenge_deadlines(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_validator_slashed_after_chunk_challenge(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py index da1a60446..5777e184a 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py @@ -4,7 +4,9 @@ from eth2spec.test.helpers.custody import ( from eth2spec.test.helpers.state import transition_to from eth2spec.test.context import ( PHASE0, + MINIMAL, with_all_phases_except, + with_configs, spec_state_test, ) from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with @@ -17,6 +19,7 @@ def run_process_challenge_deadlines(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_validator_slashed_after_reveal_deadline(spec, state): assert state.validators[0].slashed == 0 transition_to(spec, state, spec.get_randao_epoch_for_custody_period(0, 0) * spec.SLOTS_PER_EPOCH) @@ -36,6 +39,7 @@ def test_validator_slashed_after_reveal_deadline(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_validator_not_slashed_after_reveal(spec, state): transition_to(spec, state, spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH) custody_key_reveal = get_valid_custody_key_reveal(spec, state) From 916ae9e309f0c08faade851cda361715ff08d198 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 4 Sep 2020 21:26:16 +0800 Subject: [PATCH 3/3] Enable more rewards and finality tests --- tests/generators/finality/main.py | 14 +++++++++----- tests/generators/rewards/main.py | 15 ++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index dca0ecb8d..ef2d8293f 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -4,7 +4,7 @@ from importlib import reload from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from eth2spec.test.context import PHASE0 +from eth2spec.test.context import PHASE0, PHASE1 from eth2spec.test.phase0.finality import test_finality from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 @@ -12,7 +12,7 @@ from eth2spec.phase1 import spec as spec_phase1 from eth2spec.utils import bls -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(fork_name: str, handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) @@ -26,14 +26,18 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin runner_name='finality', handler_name=handler_name, src=tests_src, - fork_name=PHASE0, + fork_name=fork_name, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": + # No additional phase 1 specific rewards tests, yet. + key = 'finality' gen_runner.run_generator("finality", [ - create_provider('finality', test_finality, 'minimal'), - create_provider('finality', test_finality, 'mainnet'), + create_provider(PHASE0, 'finality', test_finality, 'minimal'), + create_provider(PHASE0, 'finality', test_finality, 'mainnet'), + create_provider(PHASE1, 'finality', test_finality, 'minimal'), + create_provider(PHASE1, 'finality', test_finality, 'mainnet'), ]) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index 30554358b..b07092d68 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -45,15 +45,12 @@ if __name__ == "__main__": gen_runner.run_generator(f"rewards", [ create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods ]) - # TODO: disabled for testing - # gen_runner.run_generator(f"rewards", [ - # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods - # ]) + gen_runner.run_generator(f"rewards", [ + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods + ]) gen_runner.run_generator(f"rewards", [ create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods ]) - # Disabled for now - # gen_runner.run_generator(f"rewards", [ - # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods - # ]) - + gen_runner.run_generator(f"rewards", [ + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods + ])