status-im/eth2.0-specs
2
0
Fork 0
mirror of https://github.com/status-im/eth2.0-specs.git synced 2025-01-20 07:29:02 +00:00
Code Issues Projects Releases Wiki Activity

update test generators

Browse Source
This commit is contained in:
protolambda 2021-05-18 18:29:24 +02:00
parent a57ff5fc01
commit fb82472b46
No known key found for this signature in database
GPG Key ID: EC89FDBB2B4C7623
19 changed files with 102 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -167,7 +167,7 @@ define run_generator
if ! test -d venv; then python3 -m venv venv; fi; \ if ! test -d venv; then python3 -m venv venv; fi; \
. venv/bin/activate; \ . venv/bin/activate; \
pip3 install -r requirements.txt; \ pip3 install -r requirements.txt; \
python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) -c $(CURRENT_DIR)/$(CONFIGS_DIR); \ python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR); \
echo "generator $(1) finished" echo "generator $(1) finished"
endef endef

47
tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py
View File

@ -34,18 +34,6 @@ def validate_output_dir(path_str):
return path return path
def validate_configs_dir(path_str):
path = Path(path_str)
if not path.exists():
raise argparse.ArgumentTypeError("Configs directory must exist")
if not path.is_dir():
raise argparse.ArgumentTypeError("Config path must lead to a directory")
return path
def run_generator(generator_name, test_providers: Iterable[TestProvider]): def run_generator(generator_name, test_providers: Iterable[TestProvider]):
""" """
Implementation for a general test generator. Implementation for a general test generator.
@ -76,22 +64,14 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
default=False, default=False,
help="if set re-generate and overwrite test files if they already exist", help="if set re-generate and overwrite test files if they already exist",
) )
parser.add_argument(
"-c",
"--configs-path",
dest="configs_path",
required=True,
type=validate_configs_dir,
help="specify the path of the configs directory",
)
parser.add_argument( parser.add_argument(
"-l", "-l",
"--config-list", "--preset-list",
dest="config_list", dest="preset_list",
nargs='*', nargs='*',
type=str, type=str,
required=False, required=False,
help="specify configs to run with. Allows all if no config names are specified.", help="specify presets to run with. Allows all if no preset names are specified.",
) )
args = parser.parse_args() args = parser.parse_args()
@ -107,27 +87,22 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
log_file = Path(output_dir) / 'testgen_error_log.txt' log_file = Path(output_dir) / 'testgen_error_log.txt'
print(f"Generating tests into {output_dir}") print(f"Generating tests into {output_dir}")
print(f"Reading configs from {args.configs_path}")
print(f'Error log file: {log_file}') print(f'Error log file: {log_file}')
configs = args.config_list presets = args.preset_list
if configs is None: if presets is None:
configs = [] presets = []
if len(configs) != 0: if len(presets) != 0:
print(f"Filtering test-generator runs to only include configs: {', '.join(configs)}") print(f"Filtering test-generator runs to only include presets: {', '.join(presets)}")
for tprov in test_providers: for tprov in test_providers:
# loads configuration etc. # runs anything that we don't want to repeat for every test case.
config_name = tprov.prepare(args.configs_path) tprov.prepare()
if len(configs) != 0 and config_name not in configs:
print(f"skipping tests with config '{config_name}' since it is filtered out")
continue
print(f"generating tests with config '{config_name}' ...")
for test_case in tprov.make_cases(): for test_case in tprov.make_cases():
case_dir = ( case_dir = (
Path(output_dir) / Path(config_name) / Path(test_case.fork_name) Path(output_dir) / Path(test_case.preset_name) / Path(test_case.fork_name)
/ Path(test_case.runner_name) / Path(test_case.handler_name) / Path(test_case.runner_name) / Path(test_case.handler_name)
/ Path(test_case.suite_name) / Path(test_case.case_name) / Path(test_case.suite_name) / Path(test_case.case_name)
) )

6
tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py
View File

@ -19,6 +19,7 @@ TestCasePart = NewType("TestCasePart", Tuple[str, str, Any])
@dataclass @dataclass
class TestCase(object): class TestCase(object):
fork_name: str fork_name: str
preset_name: str
runner_name: str runner_name: str
handler_name: str handler_name: str
suite_name: str suite_name: str
@ -28,8 +29,7 @@ class TestCase(object):
@dataclass @dataclass
class TestProvider(object): class TestProvider(object):
# Prepares the context with a configuration, loaded from the given config path. # Prepares the context for the provider as a whole, as opposed to per-test-case changes.
# fn(config path) => chosen config name prepare: Callable[[], None]
prepare: Callable[[str], str]
# Retrieves an iterable of cases, called after prepare() # Retrieves an iterable of cases, called after prepare()
make_cases: Callable[[], Iterable[TestCase]] make_cases: Callable[[], Iterable[TestCase]]

52
tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py
View File

@ -1,18 +1,18 @@
from importlib import reload, import_module from importlib import import_module
from inspect import getmembers, isfunction from inspect import getmembers, isfunction
from typing import Any, Callable, Dict, Iterable, Optional from typing import Any, Callable, Dict, Iterable, Optional
from eth2spec.config import config_util
from eth2spec.utils import bls from eth2spec.utils import bls
from eth2spec.test.helpers.constants import ALL_CONFIGS, TESTGEN_FORKS from eth2spec.test.helpers.constants import ALL_PRESETS, TESTGEN_FORKS
from eth2spec.test.helpers.typing import SpecForkName, ConfigName from eth2spec.test.helpers.typing import SpecForkName, PresetBaseName
from eth2spec.gen_helpers.gen_base import gen_runner from eth2spec.gen_helpers.gen_base import gen_runner
from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider
def generate_from_tests(runner_name: str, handler_name: str, src: Any, def generate_from_tests(runner_name: str, handler_name: str, src: Any,
fork_name: SpecForkName, bls_active: bool = True, fork_name: SpecForkName, preset_name: PresetBaseName,
bls_active: bool = True,
phase: Optional[str]=None) -> Iterable[TestCase]: phase: Optional[str]=None) -> Iterable[TestCase]:
""" """
Generate a list of test cases by running tests from the given src in generator-mode. Generate a list of test cases by running tests from the given src in generator-mode.
@ -21,8 +21,10 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any,
:param src: to retrieve tests from (discovered using inspect.getmembers). :param src: to retrieve tests from (discovered using inspect.getmembers).
:param fork_name: the folder name for these tests. :param fork_name: the folder name for these tests.
(if multiple forks are applicable, indicate the last fork) (if multiple forks are applicable, indicate the last fork)
:param preset_name: to select a preset. Tests that do not support the preset will be skipped.
:param bls_active: optional, to override BLS switch preference. Defaults to True. :param bls_active: optional, to override BLS switch preference. Defaults to True.
:param phase: optional, to run tests against a particular spec version. Default to `fork_name` value. :param phase: optional, to run tests against a particular spec version. Default to `fork_name` value.
Set to the pre-fork (w.r.t. fork_name) in multi-fork tests.
:return: an iterable of test cases. :return: an iterable of test cases.
""" """
fn_names = [ fn_names = [
@ -44,40 +46,36 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any,
yield TestCase( yield TestCase(
fork_name=fork_name, fork_name=fork_name,
preset_name=preset_name,
runner_name=runner_name, runner_name=runner_name,
handler_name=handler_name, handler_name=handler_name,
suite_name='pyspec_tests', suite_name='pyspec_tests',
case_name=case_name, case_name=case_name,
# TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent. # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent.
case_fn=lambda: tfn(generator_mode=True, phase=phase, bls_active=bls_active) case_fn=lambda: tfn(generator_mode=True, phase=phase, preset=preset_name, bls_active=bls_active)
) )
def get_provider(create_provider_fn: Callable[[SpecForkName, str, str, ConfigName], TestProvider], def get_provider(create_provider_fn: Callable[[SpecForkName, PresetBaseName, str, str], TestProvider],
config_name: ConfigName,
fork_name: SpecForkName, fork_name: SpecForkName,
preset_name: PresetBaseName,
all_mods: Dict[str, Dict[str, str]]) -> Iterable[TestProvider]: all_mods: Dict[str, Dict[str, str]]) -> Iterable[TestProvider]:
for key, mod_name in all_mods[fork_name].items(): for key, mod_name in all_mods[fork_name].items():
yield create_provider_fn( yield create_provider_fn(
fork_name=fork_name, fork_name=fork_name,
preset_name=preset_name,
handler_name=key, handler_name=key,
tests_src_mod_name=mod_name, tests_src_mod_name=mod_name,
config_name=config_name,
) )
def get_create_provider_fn( def get_create_provider_fn(runner_name: str) -> Callable[[SpecForkName, str, str, PresetBaseName], TestProvider]:
runner_name: str, config_name: ConfigName, specs: Iterable[Any] def prepare_fn() -> None:
) -> Callable[[SpecForkName, str, str, ConfigName], TestProvider]:
def prepare_fn(configs_path: str) -> str:
config_util.prepare_config(configs_path, config_name)
for spec in specs:
reload(spec)
bls.use_milagro() bls.use_milagro()
return config_name return
def create_provider(fork_name: SpecForkName, handler_name: str, def create_provider(fork_name: SpecForkName, preset_name: PresetBaseName,
tests_src_mod_name: str, config_name: ConfigName) -> TestProvider: handler_name: str, tests_src_mod_name: str) -> TestProvider:
def cases_fn() -> Iterable[TestCase]: def cases_fn() -> Iterable[TestCase]:
tests_src = import_module(tests_src_mod_name) tests_src = import_module(tests_src_mod_name)
return generate_from_tests( return generate_from_tests(
@ -85,22 +83,26 @@ def get_create_provider_fn(
handler_name=handler_name, handler_name=handler_name,
src=tests_src, src=tests_src,
fork_name=fork_name, fork_name=fork_name,
preset_name=preset_name,
) )
return TestProvider(prepare=prepare_fn, make_cases=cases_fn) return TestProvider(prepare=prepare_fn, make_cases=cases_fn)
return create_provider return create_provider
def run_state_test_generators(runner_name: str, specs: Iterable[Any], all_mods: Dict[str, Dict[str, str]]) -> None: def run_state_test_generators(runner_name: str,
all_mods: Dict[str, Dict[str, str]],
presets: Iterable[PresetBaseName] = ALL_PRESETS,
forks: Iterable[SpecForkName] = TESTGEN_FORKS) -> None:
""" """
Generate all available state tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner. Generate all available state tests of `TESTGEN_FORKS` forks of `ALL_PRESETS` presets of the given runner.
""" """
for config_name in ALL_CONFIGS: for preset_name in presets:
for fork_name in TESTGEN_FORKS: for fork_name in forks:
if fork_name in all_mods: if fork_name in all_mods:
gen_runner.run_generator(runner_name, get_provider( gen_runner.run_generator(runner_name, get_provider(
create_provider_fn=get_create_provider_fn(runner_name, config_name, specs), create_provider_fn=get_create_provider_fn(runner_name),
config_name=config_name,
fork_name=fork_name, fork_name=fork_name,
preset_name=preset_name,
all_mods=all_mods, all_mods=all_mods,
)) ))

2
tests/core/pyspec/eth2spec/test/context.py
View File

@ -414,7 +414,7 @@ def with_config_overrides(config_overrides):
test_config = {k: config_types[k](v) for k, v in tmp_config.items()} test_config = {k: config_types[k](v) for k, v in tmp_config.items()}
# Output the config for test vectors (TODO: check config YAML encoding) # Output the config for test vectors (TODO: check config YAML encoding)
yield 'config', test_config yield 'config', 'data', test_config
spec.config = spec.Configuration(**test_config) spec.config = spec.Configuration(**test_config)

2
tests/core/pyspec/eth2spec/test/helpers/constants.py
View File

@ -31,4 +31,4 @@ FORKS_BEFORE_MERGE = (PHASE0,)
MAINNET = PresetBaseName('mainnet') MAINNET = PresetBaseName('mainnet')
MINIMAL = PresetBaseName('minimal') MINIMAL = PresetBaseName('minimal')
ALL_CONFIGS = (MINIMAL, MAINNET) ALL_PRESETS = (MINIMAL, MAINNET)

5
tests/generators/bls/main.py
View File

@ -358,10 +358,10 @@ def case05_aggregate_verify():
def create_provider(handler_name: str, def create_provider(handler_name: str,
test_case_fn: Callable[[], Iterable[Tuple[str, Dict[str, Any]]]]) -> gen_typing.TestProvider: test_case_fn: Callable[[], Iterable[Tuple[str, Dict[str, Any]]]]) -> gen_typing.TestProvider:
def prepare_fn(configs_path: str) -> str: def prepare_fn() -> None:
# Nothing to load / change in spec. Maybe in future forks. # Nothing to load / change in spec. Maybe in future forks.
# Put the tests into the general config category, to not require any particular configuration. # Put the tests into the general config category, to not require any particular configuration.
return 'general' return
def cases_fn() -> Iterable[gen_typing.TestCase]: def cases_fn() -> Iterable[gen_typing.TestCase]:
for data in test_case_fn(): for data in test_case_fn():
@ -369,6 +369,7 @@ def create_provider(handler_name: str,
(case_name, case_content) = data (case_name, case_content) = data
yield gen_typing.TestCase( yield gen_typing.TestCase(
fork_name=PHASE0, fork_name=PHASE0,
preset_name='general',
runner_name='bls', runner_name='bls',
handler_name=handler_name, handler_name=handler_name,
suite_name='small', suite_name='small',

8
tests/generators/epoch_processing/main.py
View File

@ -1,13 +1,7 @@
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
from eth2spec.phase0 import spec as spec_phase0
from eth2spec.altair import spec as spec_altair
from eth2spec.merge import spec as spec_merge
from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE
specs = (spec_phase0, spec_altair, spec_merge)
if __name__ == "__main__": if __name__ == "__main__":
phase_0_mods = {key: 'eth2spec.test.phase0.epoch_processing.test_process_' + key for key in [ phase_0_mods = {key: 'eth2spec.test.phase0.epoch_processing.test_process_' + key for key in [
'justification_and_finalization', 'justification_and_finalization',
@ -45,4 +39,4 @@ if __name__ == "__main__":
MERGE: merge_mods, MERGE: merge_mods,
} }
run_state_test_generators(runner_name="epoch_processing", specs=specs, all_mods=all_mods) run_state_test_generators(runner_name="epoch_processing", all_mods=all_mods)

8
tests/generators/finality/main.py
View File

@ -1,13 +1,7 @@
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
from eth2spec.phase0 import spec as spec_phase0
from eth2spec.altair import spec as spec_altair
from eth2spec.merge import spec as spec_merge
from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE
specs = (spec_phase0, spec_altair, spec_merge)
if __name__ == "__main__": if __name__ == "__main__":
phase_0_mods = {'finality': 'eth2spec.test.phase0.finality.test_finality'} phase_0_mods = {'finality': 'eth2spec.test.phase0.finality.test_finality'}
altair_mods = phase_0_mods # No additional Altair specific finality tests altair_mods = phase_0_mods # No additional Altair specific finality tests
@ -19,4 +13,4 @@ if __name__ == "__main__":
MERGE: spec_merge, MERGE: spec_merge,
} }
run_state_test_generators(runner_name="finality", specs=specs, all_mods=all_mods) run_state_test_generators(runner_name="finality", all_mods=all_mods)

8
tests/generators/fork_choice/main.py
View File

@ -1,13 +1,7 @@
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
from eth2spec.phase0 import spec as spec_phase0
from eth2spec.altair import spec as spec_altair
from eth2spec.merge import spec as spec_merge
from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE
specs = (spec_phase0, spec_altair, spec_merge)
if __name__ == "__main__": if __name__ == "__main__":
phase_0_mods = {key: 'eth2spec.test.phase0.fork_choice.test_' + key for key in [ phase_0_mods = {key: 'eth2spec.test.phase0.fork_choice.test_' + key for key in [
'get_head', 'get_head',
@ -23,4 +17,4 @@ if __name__ == "__main__":
MERGE: merge_mods, MERGE: merge_mods,
} }
run_state_test_generators(runner_name="fork_choice", specs=specs, all_mods=all_mods) run_state_test_generators(runner_name="fork_choice", all_mods=all_mods)

25
tests/generators/forks/main.py
View File

@ -1,23 +1,19 @@
from importlib import reload
from typing import Iterable from typing import Iterable
from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MINIMAL, MAINNET from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MINIMAL, MAINNET
from eth2spec.config import config_util from eth2spec.test.helpers.typing import SpecForkName, PresetBaseName
from eth2spec.test.altair.fork import test_fork as test_altair_forks from eth2spec.test.altair.fork import test_altair_fork_basic
from eth2spec.phase0 import spec as spec_phase0 from eth2spec.test.altair.fork import test_altair_fork_random
from eth2spec.altair import spec as spec_altair
from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests
def create_provider(tests_src, config_name: str, phase: str, fork_name: str) -> gen_typing.TestProvider: def create_provider(tests_src, preset_name: PresetBaseName,
phase: SpecForkName, fork_name: SpecForkName) -> gen_typing.TestProvider:
def prepare_fn(configs_path: str) -> str: def prepare_fn() -> None:
config_util.prepare_config(configs_path, config_name) return
reload(spec_phase0)
reload(spec_altair)
return config_name
def cases_fn() -> Iterable[gen_typing.TestCase]: def cases_fn() -> Iterable[gen_typing.TestCase]:
return generate_from_tests( return generate_from_tests(
@ -25,6 +21,7 @@ def create_provider(tests_src, config_name: str, phase: str, fork_name: str) ->
handler_name='fork', handler_name='fork',
src=tests_src, src=tests_src,
fork_name=fork_name, fork_name=fork_name,
preset_name=preset_name,
phase=phase, phase=phase,
) )
@ -33,6 +30,8 @@ def create_provider(tests_src, config_name: str, phase: str, fork_name: str) ->
if __name__ == "__main__": if __name__ == "__main__":
gen_runner.run_generator("forks", [ gen_runner.run_generator("forks", [
create_provider(test_altair_forks, MINIMAL, PHASE0, ALTAIR), create_provider(test_altair_fork_basic, MINIMAL, PHASE0, ALTAIR),
create_provider(test_altair_forks, MAINNET, PHASE0, ALTAIR), create_provider(test_altair_fork_basic, MAINNET, PHASE0, ALTAIR),
create_provider(test_altair_fork_random, MINIMAL, PHASE0, ALTAIR),
create_provider(test_altair_fork_random, MAINNET, PHASE0, ALTAIR),
]) ])

7
tests/generators/genesis/main.py
View File

@ -1,12 +1,7 @@
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
from eth2spec.phase0 import spec as spec_phase0
from eth2spec.altair import spec as spec_altair
from eth2spec.test.helpers.constants import PHASE0, ALTAIR from eth2spec.test.helpers.constants import PHASE0, ALTAIR
specs = (spec_phase0, spec_altair)
if __name__ == "__main__": if __name__ == "__main__":
phase_0_mods = {key: 'eth2spec.test.phase0.genesis.test_' + key for key in [ phase_0_mods = {key: 'eth2spec.test.phase0.genesis.test_' + key for key in [
'initialization', 'initialization',
@ -18,4 +13,4 @@ if __name__ == "__main__":
ALTAIR: altair_mods, ALTAIR: altair_mods,
} }
run_state_test_generators(runner_name="genesis", specs=specs, all_mods=all_mods) run_state_test_generators(runner_name="genesis", all_mods=all_mods)

8
tests/generators/operations/main.py
View File

@ -1,13 +1,7 @@
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
from eth2spec.phase0 import spec as spec_phase0
from eth2spec.altair import spec as spec_altair
from eth2spec.merge import spec as spec_merge
from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE
specs = (spec_phase0, spec_altair, spec_merge)
if __name__ == "__main__": if __name__ == "__main__":
phase_0_mods = {key: 'eth2spec.test.phase0.block_processing.test_process_' + key for key in [ phase_0_mods = {key: 'eth2spec.test.phase0.block_processing.test_process_' + key for key in [
'attestation', 'attestation',
@ -46,4 +40,4 @@ if __name__ == "__main__":
MERGE: merge_mods, MERGE: merge_mods,
} }
run_state_test_generators(runner_name="operations", specs=specs, all_mods=all_mods) run_state_test_generators(runner_name="operations", all_mods=all_mods)

8
tests/generators/rewards/main.py
View File

@ -1,13 +1,7 @@
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
from eth2spec.phase0 import spec as spec_phase0
from eth2spec.altair import spec as spec_altair
from eth2spec.merge import spec as spec_merge
from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE
specs = (spec_phase0, spec_altair, spec_merge)
if __name__ == "__main__": if __name__ == "__main__":
phase_0_mods = {key: 'eth2spec.test.phase0.rewards.test_' + key for key in [ phase_0_mods = {key: 'eth2spec.test.phase0.rewards.test_' + key for key in [
'basic', 'basic',
@ -28,4 +22,4 @@ if __name__ == "__main__":
MERGE: merge_mods, MERGE: merge_mods,
} }
run_state_test_generators(runner_name="rewards", specs=specs, all_mods=all_mods) run_state_test_generators(runner_name="rewards", all_mods=all_mods)

9
tests/generators/sanity/main.py
View File

@ -1,14 +1,7 @@
from eth2spec.phase0 import spec as spec_phase0
from eth2spec.altair import spec as spec_altair
from eth2spec.merge import spec as spec_merge
from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE
from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
specs = (spec_phase0, spec_altair, spec_merge)
if __name__ == "__main__": if __name__ == "__main__":
phase_0_mods = {key: 'eth2spec.test.phase0.sanity.test_' + key for key in [ phase_0_mods = {key: 'eth2spec.test.phase0.sanity.test_' + key for key in [
'blocks', 'blocks',
@ -29,4 +22,4 @@ if __name__ == "__main__":
MERGE: merge_mods, MERGE: merge_mods,
} }
run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods) run_state_test_generators(runner_name="sanity", all_mods=all_mods)

34
tests/generators/shuffling/main.py
View File

@ -1,15 +1,14 @@
from eth_utils import to_tuple from eth_utils import to_tuple
from typing import Iterable from typing import Iterable
from importlib import reload
from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
from eth2spec.test.helpers.typing import SpecForkName, PresetBaseName
from eth2spec.config import config_util from eth2spec.phase0 import mainnet as spec_mainnet, minimal as spec_minimal
from eth2spec.phase0 import spec as spec from eth2spec.test.helpers.constants import PHASE0, MINIMAL, MAINNET
from eth2spec.test.helpers.constants import PHASE0
def shuffling_case_fn(seed, count): def shuffling_case_fn(spec, seed, count):
yield 'mapping', 'data', { yield 'mapping', 'data', {
'seed': '0x' + seed.hex(), 'seed': '0x' + seed.hex(),
'count': count, 'count': count,
@ -17,28 +16,33 @@ def shuffling_case_fn(seed, count):
} }
def shuffling_case(seed, count): def shuffling_case(spec, seed, count):
return f'shuffle_0x{seed.hex()}_{count}', lambda: shuffling_case_fn(seed, count) return f'shuffle_0x{seed.hex()}_{count}', lambda: shuffling_case_fn(spec, seed, count)
@to_tuple @to_tuple
def shuffling_test_cases(): def shuffling_test_cases(spec):
for seed in [spec.hash(seed_init_value.to_bytes(length=4, byteorder='little')) for seed_init_value in range(30)]: for seed in [spec.hash(seed_init_value.to_bytes(length=4, byteorder='little')) for seed_init_value in range(30)]:
for count in [0, 1, 2, 3, 5, 10, 33, 100, 1000, 9999]: for count in [0, 1, 2, 3, 5, 10, 33, 100, 1000, 9999]:
yield shuffling_case(seed, count) yield shuffling_case(spec, seed, count)
def create_provider(config_name: str) -> gen_typing.TestProvider: def create_provider(preset_name: PresetBaseName) -> gen_typing.TestProvider:
def prepare_fn(configs_path: str) -> str: def prepare_fn() -> None:
config_util.prepare_config(configs_path, config_name) return
reload(spec)
return config_name
def cases_fn() -> Iterable[gen_typing.TestCase]: def cases_fn() -> Iterable[gen_typing.TestCase]:
if preset_name == MAINNET:
spec = spec_mainnet
elif preset_name == MINIMAL:
spec = spec_minimal
else:
raise Exception(f"unrecognized preset: {preset_name}")
for (case_name, case_fn) in shuffling_test_cases(): for (case_name, case_fn) in shuffling_test_cases():
yield gen_typing.TestCase( yield gen_typing.TestCase(
fork_name=PHASE0, fork_name=PHASE0,
preset_name=preset_name,
runner_name='shuffling', runner_name='shuffling',
handler_name='core', handler_name='core',
suite_name='shuffle', suite_name='shuffle',
@ -50,4 +54,4 @@ def create_provider(config_name: str) -> gen_typing.TestProvider:
if __name__ == "__main__": if __name__ == "__main__":
gen_runner.run_generator("shuffling", [create_provider("minimal"), create_provider("mainnet")]) gen_runner.run_generator("shuffling", [create_provider(MINIMAL), create_provider(MAINNET)])

5
tests/generators/ssz_generic/main.py
View File

@ -11,13 +11,14 @@ from eth2spec.test.helpers.constants import PHASE0
def create_provider(handler_name: str, suite_name: str, case_maker) -> gen_typing.TestProvider: def create_provider(handler_name: str, suite_name: str, case_maker) -> gen_typing.TestProvider:
def prepare_fn(configs_path: str) -> str: def prepare_fn() -> None:
return "general" return
def cases_fn() -> Iterable[gen_typing.TestCase]: def cases_fn() -> Iterable[gen_typing.TestCase]:
for (case_name, case_fn) in case_maker(): for (case_name, case_fn) in case_maker():
yield gen_typing.TestCase( yield gen_typing.TestCase(
fork_name=PHASE0, fork_name=PHASE0,
preset_name="general",
runner_name='ssz_generic', runner_name='ssz_generic',
handler_name=handler_name, handler_name=handler_name,
suite_name=suite_name, suite_name=suite_name,

32
tests/generators/ssz_static/main.py
View File

@ -1,16 +1,12 @@
from random import Random from random import Random
from typing import Iterable from typing import Iterable
from importlib import reload
from inspect import getmembers, isclass from inspect import getmembers, isclass
from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
from eth2spec.debug import random_value, encode from eth2spec.debug import random_value, encode
from eth2spec.config import config_util
from eth2spec.phase0 import spec as spec_phase0
from eth2spec.altair import spec as spec_altair
from eth2spec.merge import spec as spec_merge
from eth2spec.test.helpers.constants import ALTAIR, MERGE, TESTGEN_FORKS, MINIMAL, MAINNET from eth2spec.test.helpers.constants import ALTAIR, MERGE, TESTGEN_FORKS, MINIMAL, MAINNET
from eth2spec.test.context import spec_targets
from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_typing import Container
from eth2spec.utils.ssz.ssz_impl import ( from eth2spec.utils.ssz.ssz_impl import (
hash_tree_root, hash_tree_root,
@ -40,7 +36,7 @@ def get_spec_ssz_types(spec):
] ]
def ssz_static_cases(fork_name: str, seed: int, name, ssz_type, def ssz_static_cases(fork_name: str, preset_name: str, seed: int, name, ssz_type,
mode: random_value.RandomizationMode, chaos: bool, count: int): mode: random_value.RandomizationMode, chaos: bool, count: int):
random_mode_name = mode.to_name() random_mode_name = mode.to_name()
@ -50,6 +46,7 @@ def ssz_static_cases(fork_name: str, seed: int, name, ssz_type,
for i in range(count): for i in range(count):
yield gen_typing.TestCase( yield gen_typing.TestCase(
fork_name=fork_name, fork_name=fork_name,
preset_name=preset_name,
runner_name='ssz_static', runner_name='ssz_static',
handler_name=name, handler_name=name,
suite_name=f"ssz_{random_mode_name}{'_chaos' if chaos else ''}", suite_name=f"ssz_{random_mode_name}{'_chaos' if chaos else ''}",
@ -58,26 +55,17 @@ def ssz_static_cases(fork_name: str, seed: int, name, ssz_type,
) )
def create_provider(fork_name, config_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool, def create_provider(fork_name, preset_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool,
cases_if_random: int) -> gen_typing.TestProvider: cases_if_random: int) -> gen_typing.TestProvider:
def prepare_fn(configs_path: str) -> str: def prepare_fn() -> None:
# Apply changes to presets, this affects some of the vector types. return
config_util.prepare_config(configs_path, config_name)
reload(spec_phase0)
reload(spec_altair)
reload(spec_merge)
return config_name
def cases_fn() -> Iterable[gen_typing.TestCase]: def cases_fn() -> Iterable[gen_typing.TestCase]:
count = cases_if_random if chaos or mode.is_changing() else 1 count = cases_if_random if chaos or mode.is_changing() else 1
spec = spec_phase0 spec = spec_targets[preset_name][fork_name]
if fork_name == ALTAIR:
spec = spec_altair
elif fork_name == MERGE:
spec = spec_merge
for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types(spec)): for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types(spec)):
yield from ssz_static_cases(fork_name, seed * 1000 + i, name, ssz_type, mode, chaos, count) yield from ssz_static_cases(fork_name, preset_name, seed * 1000 + i, name, ssz_type, mode, chaos, count)
return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
@ -95,6 +83,6 @@ if __name__ == "__main__":
seed += 1 seed += 1
for fork in TESTGEN_FORKS: for fork in TESTGEN_FORKS:
gen_runner.run_generator("ssz_static", [ gen_runner.run_generator("ssz_static", [
create_provider(fork, config_name, seed, mode, chaos, cases_if_random) create_provider(fork, preset_name, seed, mode, chaos, cases_if_random)
for (seed, config_name, mode, chaos, cases_if_random) in settings for (seed, preset_name, mode, chaos, cases_if_random) in settings
]) ])

14
tests/generators/transition/main.py
View File

@ -1,23 +1,16 @@
from importlib import reload
from typing import Iterable from typing import Iterable
from eth2spec.test.helpers.constants import ALTAIR, MINIMAL, MAINNET, PHASE0 from eth2spec.test.helpers.constants import ALTAIR, MINIMAL, MAINNET, PHASE0
from eth2spec.config import config_util
from eth2spec.test.altair.transition import test_transition as test_altair_transition from eth2spec.test.altair.transition import test_transition as test_altair_transition
from eth2spec.phase0 import spec as spec_phase0
from eth2spec.altair import spec as spec_altair
from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests
def create_provider(tests_src, config_name: str, pre_fork_name: str, post_fork_name: str) -> gen_typing.TestProvider: def create_provider(tests_src, preset_name: str, pre_fork_name: str, post_fork_name: str) -> gen_typing.TestProvider:
def prepare_fn(configs_path: str) -> str: def prepare_fn() -> None:
config_util.prepare_config(configs_path, config_name) return
reload(spec_phase0)
reload(spec_altair)
return config_name
def cases_fn() -> Iterable[gen_typing.TestCase]: def cases_fn() -> Iterable[gen_typing.TestCase]:
return generate_from_tests( return generate_from_tests(
@ -26,6 +19,7 @@ def create_provider(tests_src, config_name: str, pre_fork_name: str, post_fork_n
src=tests_src, src=tests_src,
fork_name=post_fork_name, fork_name=post_fork_name,
phase=pre_fork_name, phase=pre_fork_name,
preset_name=preset_name,
) )
return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)