From 50fb3da0729429179e6a24abaf1150b6c023a94b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 20:31:06 +0800 Subject: [PATCH 1/7] Make test_fork.py truly pass --- tests/core/pyspec/eth2spec/test/context.py | 3 +- .../test/lightclient_patch/fork/test_fork.py | 45 ++++++++++--------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 9c45049a2..f3d8e2a8f 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases[PHASE1].upgrade_to_phase1(state) - elif spec.fork == LIGHTCLIENT_PATCH and not fork_test: # do not upgrade if spec ttttest + elif spec.fork == LIGHTCLIENT_PATCH and not is_fork_test: # do not upgrade if spec ttttest state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state) return state @@ -294,6 +294,7 @@ def fork_test(fn): def entry(*args, **kw): # override fork test setting kw['fork_test'] = True + return fn(*args, **kw) return entry diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index 527ad746c..88ff68a06 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -1,5 +1,5 @@ from eth2spec.test.context import ( - LIGHTCLIENT_PATCH, + PHASE0, LIGHTCLIENT_PATCH, with_phases, with_custom_state, fork_test, spec_test, with_state, @@ -53,39 +53,39 @@ def run_fork_test(spec, pre_state): yield 'post', post_state -@with_phases(([LIGHTCLIENT_PATCH])) -@with_state -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_base_state(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_state -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_next_epoch(spec, phases, state): next_epoch(spec, state) yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_state -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_next_epoch_with_block(spec, phases, state): next_epoch_via_block(spec, state) yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_state -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_many_next_epoch(spec, phases, state): for _ in range(3): @@ -93,28 +93,29 @@ def test_fork_many_next_epoch(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_low_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@spec_test @fork_test +@with_phases(([PHASE0])) +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_misc_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@spec_test @fork_test +@with_phases(([PHASE0])) +@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_large_validator_set(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) From f97ea9e172e88a7527e7c7fa97e104a90df99a69 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 20:55:39 +0800 Subject: [PATCH 2/7] Generate basic tests. Still having problem with generating `with_custom_state` tests --- .../gen_helpers/gen_from_tests/gen.py | 11 +++++++--- tests/generators/forks/main.py | 21 +++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index 67d29b194..f04bf46a7 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -1,6 +1,6 @@ from importlib import reload, import_module from inspect import getmembers, isfunction -from typing import Any, Callable, Dict, Iterable +from typing import Any, Callable, Dict, Iterable, Optional from eth2spec.config import config_util from eth2spec.utils import bls @@ -11,7 +11,7 @@ from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider def generate_from_tests(runner_name: str, handler_name: str, src: Any, - fork_name: SpecForkName, bls_active: bool = True) -> Iterable[TestCase]: + fork_name: SpecForkName, bls_active: bool = True, phase: Optional[str]=None) -> Iterable[TestCase]: """ Generate a list of test cases by running tests from the given src in generator-mode. :param runner_name: to categorize the test in general as. @@ -20,12 +20,17 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, :param fork_name: to run tests against particular phase and/or fork. (if multiple forks are applicable, indicate the last fork) :param bls_active: optional, to override BLS switch preference. Defaults to True. + :param phase: optional, specific phase name :return: an iterable of test cases. """ fn_names = [ name for (name, _) in getmembers(src, isfunction) if name.startswith('test_') ] + + if phase is None: + phase = fork_name + print("generating test vectors from tests source: %s" % src.__name__) for name in fn_names: tfn = getattr(src, name) @@ -42,7 +47,7 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, suite_name='pyspec_tests', case_name=case_name, # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent. - case_fn=lambda: tfn(generator_mode=True, phase=fork_name, bls_active=bls_active) + case_fn=lambda: tfn(generator_mode=True, phase=phase, bls_active=bls_active) ) diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index a32ece712..d39af802a 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -1,7 +1,7 @@ from importlib import reload from typing import Iterable -from eth2spec.test.context import LIGHTCLIENT_PATCH +from eth2spec.test.context import PHASE0, LIGHTCLIENT_PATCH, MINIMAL, MAINNET from eth2spec.config import config_util from eth2spec.test.lightclient_patch.fork import test_fork as test_altair_forks from eth2spec.phase0 import spec as spec_phase0 @@ -10,24 +10,20 @@ from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests -pre_specs = { - LIGHTCLIENT_PATCH: spec_phase0, -} - - -def create_provider(fork_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) - reload(pre_specs[fork_name]) + reload(spec_phase0) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: return generate_from_tests( - runner_name='forks', - handler_name='core', + runner_name='fork', + handler_name='fork', src=tests_src, - fork_name=fork_name, + fork_name=LIGHTCLIENT_PATCH, + phase=PHASE0, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) @@ -35,6 +31,5 @@ def create_provider(fork_name: str, tests_src, config_name: str) -> gen_typing.T if __name__ == "__main__": gen_runner.run_generator("forks", [ - create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'), - create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'), + create_provider(test_altair_forks, MINIMAL), ]) From 7a10c7108ad5b21efac726afe2a5799383457a3f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 21:17:02 +0800 Subject: [PATCH 3/7] Fix decorator calls --- .../eth2spec/test/lightclient_patch/fork/test_fork.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index 88ff68a06..c5b9b8981 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -95,9 +95,8 @@ def test_fork_many_next_epoch(spec, phases, state): @fork_test @with_phases(([PHASE0])) -@spec_test @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@with_state +@spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_low_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) @@ -106,7 +105,7 @@ def test_fork_random_low_balances(spec, phases, state): @fork_test @with_phases(([PHASE0])) @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@with_state +@spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_misc_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) @@ -115,7 +114,7 @@ def test_fork_random_misc_balances(spec, phases, state): @fork_test @with_phases(([PHASE0])) @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@with_state +@spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_large_validator_set(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) From f025ec40c584e45517aeed1f93ce4fa4db1f0f3d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 21:21:32 +0800 Subject: [PATCH 4/7] Fix linter error --- tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index f04bf46a7..057aa1a2c 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -11,7 +11,8 @@ from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider def generate_from_tests(runner_name: str, handler_name: str, src: Any, - fork_name: SpecForkName, bls_active: bool = True, phase: Optional[str]=None) -> Iterable[TestCase]: + fork_name: SpecForkName, bls_active: bool = True, + phase: Optional[str]=None) -> Iterable[TestCase]: """ Generate a list of test cases by running tests from the given src in generator-mode. :param runner_name: to categorize the test in general as. From 5e864af67a161878c294a41dce579a20f98b62bd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 21:32:37 +0800 Subject: [PATCH 5/7] Reload Altair spec --- tests/generators/forks/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index d39af802a..dd4c0051c 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -5,6 +5,7 @@ from eth2spec.test.context import PHASE0, LIGHTCLIENT_PATCH, MINIMAL, MAINNET from eth2spec.config import config_util from eth2spec.test.lightclient_patch.fork import test_fork as test_altair_forks from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests @@ -15,6 +16,7 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: From f71a3c6b22fe23f28eb49216618a20ef194d1025 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 21:34:45 +0800 Subject: [PATCH 6/7] Generate with mainnet config --- tests/generators/forks/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index dd4c0051c..b22707786 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -34,4 +34,5 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: if __name__ == "__main__": gen_runner.run_generator("forks", [ create_provider(test_altair_forks, MINIMAL), + create_provider(test_altair_forks, MAINNET), ]) From 956a7a2ef1a20da3472307768d41e28b69c80840 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 9 Mar 2021 16:09:08 -0700 Subject: [PATCH 7/7] Update tests/core/pyspec/eth2spec/test/context.py --- tests/core/pyspec/eth2spec/test/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index f3d8e2a8f..916c74e6a 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases[PHASE1].upgrade_to_phase1(state) - elif spec.fork == LIGHTCLIENT_PATCH and not is_fork_test: # do not upgrade if spec ttttest + elif spec.fork == LIGHTCLIENT_PATCH and not is_fork_test: # do not upgrade if spec test state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state) return state