status-im
/
eth2.0-specs
mirror of https://github.com/status-im/eth2.0-specs.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update test gen to output phase 1 test vectors. Mainnet disabled for now.

This commit is contained in:
protolambda 2020-07-07 21:21:43 +02:00
parent 6b1d4839d8
commit 3c374b1e16
No known key found for this signature in database
GPG Key ID: EC89FDBB2B4C7623
8 changed files with 125 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/core/gen_helpers/gen_base/gen_runner.py
View File

@ -2,6 +2,7 @@ import argparse
from pathlib import Path from pathlib import Path
import sys import sys
from typing import Iterable, AnyStr, Any, Callable from typing import Iterable, AnyStr, Any, Callable
import traceback
from ruamel.yaml import ( from ruamel.yaml import (
YAML, YAML,
@ -152,6 +153,7 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
except Exception as e: except Exception as e:
print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}")
traceback.print_exc()
print(f"completed {generator_name}") print(f"completed {generator_name}")

5
tests/core/pyspec/eth2spec/debug/random_value.py
View File

@ -68,9 +68,8 @@ def get_random_ssz_object(rng: Random,
else: else:
return typ(get_random_bytes_list(rng, rng.randint(0, min(max_bytes_length, typ.limit())))) return typ(get_random_bytes_list(rng, rng.randint(0, min(max_bytes_length, typ.limit()))))
if issubclass(typ, ByteVector): if issubclass(typ, ByteVector):
# Sanity, don't generate absurdly big random values # Random byte vectors can be bigger than max bytes size, e.g. custody chunk data.
# If a client is aiming to performance-test, they should create a benchmark suite. # No max-bytes-length limitation here.
assert typ.type_byte_length() <= max_bytes_length
if mode == RandomizationMode.mode_zero: if mode == RandomizationMode.mode_zero:
return typ(b'\x00' * typ.type_byte_length()) return typ(b'\x00' * typ.type_byte_length())
elif mode == RandomizationMode.mode_max: elif mode == RandomizationMode.mode_max:

0
tests/core/pyspec/eth2spec/test/phase1/epoch_processing/__init__.py Normal file
View File

59
tests/generators/epoch_processing/main.py
View File

@ -1,23 +1,15 @@
from typing import Iterable from typing import Iterable
from eth2spec.phase0 import spec as spec_phase0
from eth2spec.phase1 import spec as spec_phase1
from eth2spec.test.phase0.epoch_processing import (
test_process_final_updates,
test_process_justification_and_finalization,
test_process_registry_updates,
test_process_rewards_and_penalties,
test_process_slashings
)
from gen_base import gen_runner, gen_typing from gen_base import gen_runner, gen_typing
from gen_from_tests.gen import generate_from_tests from gen_from_tests.gen import generate_from_tests
from importlib import reload from importlib import reload, import_module
from eth2spec.config import config_util from eth2spec.config import config_util
from eth2spec.test.context import PHASE0 from eth2spec.phase0 import spec as spec_phase0
from eth2spec.phase1 import spec as spec_phase1
from eth2spec.test.context import PHASE0, PHASE1
def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
def prepare_fn(configs_path: str) -> str: def prepare_fn(configs_path: str) -> str:
config_util.prepare_config(configs_path, config_name) config_util.prepare_config(configs_path, config_name)
reload(spec_phase0) reload(spec_phase0)
@ -25,27 +17,42 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin
return config_name return config_name
def cases_fn() -> Iterable[gen_typing.TestCase]: def cases_fn() -> Iterable[gen_typing.TestCase]:
tests_src = import_module(tests_src_mod_name)
return generate_from_tests( return generate_from_tests(
runner_name='epoch_processing', runner_name='epoch_processing',
handler_name=handler_name, handler_name=handler_name,
src=tests_src, src=tests_src,
fork_name=PHASE0, fork_name=fork_name,
) )
return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
if __name__ == "__main__": if __name__ == "__main__":
gen_runner.run_generator("epoch_processing", [ phase_0_mods = [(key, 'eth2spec.test.phase0.epoch_processing.test_process_' + key) for key in [
create_provider('final_updates', test_process_final_updates, 'minimal'), 'final_updates',
create_provider('final_updates', test_process_final_updates, 'mainnet'), 'justification_and_finalization',
create_provider('justification_and_finalization', test_process_justification_and_finalization, 'minimal'), 'registry_updates',
create_provider('justification_and_finalization', test_process_justification_and_finalization, 'mainnet'), 'rewards_and_penalties',
create_provider('registry_updates', test_process_registry_updates, 'minimal'), 'slashings',
create_provider('registry_updates', test_process_registry_updates, 'mainnet'), ]]
create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'minimal'), phase_1_mods = [(key, 'eth2spec.test.phase1.epoch_processing.test_process_' + key) for key in [
# runs full epochs filled with data, with uncached ssz. Disabled for now. 'challenge_deadlines',
# create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'mainnet'), 'custody_final_updates',
create_provider('slashings', test_process_slashings, 'minimal'), 'reveal_deadlines',
create_provider('slashings', test_process_slashings, 'mainnet'), ]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec)
gen_runner.run_generator(f"epoch_processing", [
create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods
]) ])
# TODO: disabled for testing
# gen_runner.run_generator(f"epoch_processing", [
# create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods
# ])
gen_runner.run_generator(f"epoch_processing", [
create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods
])
# Disabled for now
# gen_runner.run_generator(f"epoch_processing", [
# create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods
# ])

11
tests/generators/operations/main.py
View File

@ -1,9 +1,8 @@
from typing import Iterable from typing import Iterable
from importlib import import_module
from gen_base import gen_runner, gen_typing from gen_base import gen_runner, gen_typing
from gen_from_tests.gen import generate_from_tests from gen_from_tests.gen import generate_from_tests
from importlib import reload from importlib import reload, import_module
from eth2spec.config import config_util from eth2spec.config import config_util
from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase0 import spec as spec_phase0
from eth2spec.phase1 import spec as spec_phase1 from eth2spec.phase1 import spec as spec_phase1
@ -47,17 +46,17 @@ if __name__ == "__main__":
'shard_transition', 'shard_transition',
]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec) ]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec)
gen_runner.run_generator(f"phase0/operations", [ gen_runner.run_generator(f"operations", [
create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods
]) ])
# TODO: disabled for testing # TODO: disabled for testing
# gen_runner.run_generator(f"phase0/operations", [ # gen_runner.run_generator(f"operations", [
# create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods # create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods
# ]) # ])
gen_runner.run_generator(f"phase1/operations", [ gen_runner.run_generator(f"operations", [
create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods
]) ])
# Disabled for now # Disabled for now
# gen_runner.run_generator(f"phase1/operations", [ # gen_runner.run_generator(f"operations", [
# create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods # create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods
# ]) # ])

53
tests/generators/rewards/main.py
View File

@ -1,21 +1,15 @@
from typing import Iterable from typing import Iterable
from eth2spec.phase0 import spec as spec_phase0
from eth2spec.phase1 import spec as spec_phase1
from eth2spec.test.phase0.rewards import (
test_basic,
test_leak,
test_random,
)
from gen_base import gen_runner, gen_typing from gen_base import gen_runner, gen_typing
from gen_from_tests.gen import generate_from_tests from gen_from_tests.gen import generate_from_tests
from importlib import reload from importlib import reload, import_module
from eth2spec.config import config_util from eth2spec.config import config_util
from eth2spec.test.context import PHASE0 from eth2spec.phase0 import spec as spec_phase0
from eth2spec.phase1 import spec as spec_phase1
from eth2spec.test.context import PHASE0, PHASE1
def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
def prepare_fn(configs_path: str) -> str: def prepare_fn(configs_path: str) -> str:
config_util.prepare_config(configs_path, config_name) config_util.prepare_config(configs_path, config_name)
reload(spec_phase0) reload(spec_phase0)
@ -23,22 +17,41 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider:
return config_name return config_name
def cases_fn() -> Iterable[gen_typing.TestCase]: def cases_fn() -> Iterable[gen_typing.TestCase]:
tests_src = import_module(tests_src_mod_name)
return generate_from_tests( return generate_from_tests(
runner_name='rewards', runner_name='rewards',
handler_name='core', handler_name=handler_name,
src=tests_src, src=tests_src,
fork_name=PHASE0, fork_name=fork_name,
) )
return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
if __name__ == "__main__": if __name__ == "__main__":
gen_runner.run_generator("rewards", [ phase_0_mods = [(key, 'eth2spec.test.phase0.rewards.test_'+key) for key in [
create_provider(test_basic, 'minimal'), 'basic',
create_provider(test_basic, 'mainnet'), 'basic',
create_provider(test_leak, 'minimal'), 'leak',
create_provider(test_leak, 'mainnet'), 'leak',
create_provider(test_random, 'minimal'), 'random',
create_provider(test_random, 'mainnet'), 'random',
]]
# No additional phase 1 specific rewards tests, yet.
phase_1_mods = phase_0_mods
gen_runner.run_generator(f"rewards", [
create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods
]) ])
# TODO: disabled for testing
# gen_runner.run_generator(f"rewards", [
# create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods
# ])
gen_runner.run_generator(f"rewards", [
create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods
])
# Disabled for now
# gen_runner.run_generator(f"rewards", [
# create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods
# ])

40
tests/generators/sanity/main.py
View File

@ -1,18 +1,15 @@
from typing import Iterable from typing import Iterable
from importlib import reload
from gen_base import gen_runner, gen_typing from gen_base import gen_runner, gen_typing
from gen_from_tests.gen import generate_from_tests from gen_from_tests.gen import generate_from_tests
from importlib import reload, import_module
from eth2spec.test.context import PHASE0
from eth2spec.test.phase0.sanity import test_blocks, test_slots
from eth2spec.config import config_util from eth2spec.config import config_util
from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase0 import spec as spec_phase0
from eth2spec.phase1 import spec as spec_phase1 from eth2spec.phase1 import spec as spec_phase1
from eth2spec.test.context import PHASE0, PHASE1
def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
def prepare_fn(configs_path: str) -> str: def prepare_fn(configs_path: str) -> str:
config_util.prepare_config(configs_path, config_name) config_util.prepare_config(configs_path, config_name)
reload(spec_phase0) reload(spec_phase0)
@ -20,20 +17,39 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin
return config_name return config_name
def cases_fn() -> Iterable[gen_typing.TestCase]: def cases_fn() -> Iterable[gen_typing.TestCase]:
tests_src = import_module(tests_src_mod_name)
return generate_from_tests( return generate_from_tests(
runner_name='sanity', runner_name='sanity',
handler_name=handler_name, handler_name=handler_name,
src=tests_src, src=tests_src,
fork_name=PHASE0, fork_name=fork_name,
) )
return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
if __name__ == "__main__": if __name__ == "__main__":
gen_runner.run_generator("sanity", [ phase_0_mods = [(key, 'eth2spec.test.phase0.sanity.test_' + key) for key in [
create_provider('blocks', test_blocks, 'minimal'), 'blocks',
create_provider('blocks', test_blocks, 'mainnet'), 'slots',
create_provider('slots', test_slots, 'minimal'), ]]
create_provider('slots', test_slots, 'mainnet'), phase_1_mods = [(key, 'eth2spec.test.phase1.sanity.test_' + key) for key in [
'blocks', # more phase 1 specific block tests
'shard_blocks',
]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec)
gen_runner.run_generator(f"sanity", [
create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods
]) ])
# TODO: disabled for testing
# gen_runner.run_generator(f"sanity", [
# create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods
# ])
gen_runner.run_generator(f"sanity", [
create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods
])
# Disabled for now
# gen_runner.run_generator(f"sanity", [
# create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods
# ])

38
tests/generators/ssz_static/main.py
View File

@ -7,8 +7,9 @@ from gen_base import gen_runner, gen_typing
from eth2spec.debug import random_value, encode from eth2spec.debug import random_value, encode
from eth2spec.config import config_util from eth2spec.config import config_util
from eth2spec.phase0 import spec from eth2spec.phase0 import spec as spec_phase0
from eth2spec.test.context import PHASE0 from eth2spec.phase1 import spec as spec_phase1
from eth2spec.test.context import PHASE0, PHASE1
from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_typing import Container
from eth2spec.utils.ssz.ssz_impl import ( from eth2spec.utils.ssz.ssz_impl import (
hash_tree_root, hash_tree_root,
@ -16,7 +17,7 @@ from eth2spec.utils.ssz.ssz_impl import (
) )
MAX_BYTES_LENGTH = 100 MAX_BYTES_LENGTH = 1000
MAX_LIST_LENGTH = 10 MAX_LIST_LENGTH = 10
@ -30,14 +31,14 @@ def create_test_case(rng: Random, typ, mode: random_value.RandomizationMode, cha
yield "roots", "data", roots_data yield "roots", "data", roots_data
def get_spec_ssz_types(): def get_spec_ssz_types(spec):
return [ return [
(name, value) for (name, value) in getmembers(spec, isclass) (name, value) for (name, value) in getmembers(spec, isclass)
if issubclass(value, Container) and value != Container # only the subclasses, not the imported base class if issubclass(value, Container) and value != Container # only the subclasses, not the imported base class
] ]
def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.RandomizationMode, chaos: bool, count: int): def ssz_static_cases(fork_name: str, seed: int, name, ssz_type, mode: random_value.RandomizationMode, chaos: bool, count: int):
random_mode_name = mode.to_name() random_mode_name = mode.to_name()
# Reproducible RNG # Reproducible RNG
@ -45,7 +46,7 @@ def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.Randomization
for i in range(count): for i in range(count):
yield gen_typing.TestCase( yield gen_typing.TestCase(
fork_name=PHASE0, fork_name=fork_name,
runner_name='ssz_static', runner_name='ssz_static',
handler_name=name, handler_name=name,
suite_name=f"ssz_{random_mode_name}{'_chaos' if chaos else ''}", suite_name=f"ssz_{random_mode_name}{'_chaos' if chaos else ''}",
@ -54,19 +55,23 @@ def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.Randomization
) )
def create_provider(config_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool, def create_provider(fork_name, config_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool,
cases_if_random: int) -> gen_typing.TestProvider: cases_if_random: int) -> gen_typing.TestProvider:
def prepare_fn(configs_path: str) -> str: def prepare_fn(configs_path: str) -> str:
# Apply changes to presets, this affects some of the vector types. # Apply changes to presets, this affects some of the vector types.
config_util.prepare_config(configs_path, config_name) config_util.prepare_config(configs_path, config_name)
reload(spec) reload(spec_phase0)
reload(spec_phase1)
return config_name return config_name
def cases_fn() -> Iterable[gen_typing.TestCase]: def cases_fn() -> Iterable[gen_typing.TestCase]:
count = cases_if_random if chaos or mode.is_changing() else 1 count = cases_if_random if chaos or mode.is_changing() else 1
spec = spec_phase0
if fork_name == PHASE1:
spec = spec_phase1
for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types()): for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types(spec)):
yield from ssz_static_cases(seed * 1000 + i, name, ssz_type, mode, chaos, count) yield from ssz_static_cases(fork_name, seed * 1000 + i, name, ssz_type, mode, chaos, count)
return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
@ -80,10 +85,11 @@ if __name__ == "__main__":
seed += 1 seed += 1
settings.append((seed, "minimal", random_value.RandomizationMode.mode_random, True, 30)) settings.append((seed, "minimal", random_value.RandomizationMode.mode_random, True, 30))
seed += 1 seed += 1
settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) # settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5))
seed += 1 # seed += 1
gen_runner.run_generator("ssz_static", [ for fork in [PHASE0, PHASE1]:
create_provider(config_name, seed, mode, chaos, cases_if_random) gen_runner.run_generator("ssz_static", [
for (seed, config_name, mode, chaos, cases_if_random) in settings create_provider(fork, config_name, seed, mode, chaos, cases_if_random)
]) for (seed, config_name, mode, chaos, cases_if_random) in settings
])