Update test gen to output phase 1 test vectors. Mainnet disabled for now.
This commit is contained in:
protolambda
parent
6b1d4839d8
commit
3c374b1e16
|
|
@ -2,6 +2,7 @@ import argparse
|
|||
from pathlib import Path
|
||||
import sys
|
||||
from typing import Iterable, AnyStr, Any, Callable
|
||||
import traceback
|
||||
|
||||
from ruamel.yaml import (
|
||||
YAML,
|
||||
|
|
@ -152,6 +153,7 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
|
|||
|
||||
except Exception as e:
|
||||
print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}")
|
||||
traceback.print_exc()
|
||||
print(f"completed {generator_name}")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -68,9 +68,8 @@ def get_random_ssz_object(rng: Random,
|
|||
else:
|
||||
return typ(get_random_bytes_list(rng, rng.randint(0, min(max_bytes_length, typ.limit()))))
|
||||
if issubclass(typ, ByteVector):
|
||||
# Sanity, don't generate absurdly big random values
|
||||
# If a client is aiming to performance-test, they should create a benchmark suite.
|
||||
assert typ.type_byte_length() <= max_bytes_length
|
||||
# Random byte vectors can be bigger than max bytes size, e.g. custody chunk data.
|
||||
# No max-bytes-length limitation here.
|
||||
if mode == RandomizationMode.mode_zero:
|
||||
return typ(b'\x00' * typ.type_byte_length())
|
||||
elif mode == RandomizationMode.mode_max:
|
||||
|
|
|
|||
|
|
@ -1,23 +1,15 @@
|
|||
from typing import Iterable
|
||||
|
||||
from eth2spec.phase0 import spec as spec_phase0
|
||||
from eth2spec.phase1 import spec as spec_phase1
|
||||
from eth2spec.test.phase0.epoch_processing import (
|
||||
test_process_final_updates,
|
||||
test_process_justification_and_finalization,
|
||||
test_process_registry_updates,
|
||||
test_process_rewards_and_penalties,
|
||||
test_process_slashings
|
||||
)
|
||||
from gen_base import gen_runner, gen_typing
|
||||
from gen_from_tests.gen import generate_from_tests
|
||||
from importlib import reload
|
||||
from importlib import reload, import_module
|
||||
from eth2spec.config import config_util
|
||||
from eth2spec.test.context import PHASE0
|
||||
from eth2spec.phase0 import spec as spec_phase0
|
||||
from eth2spec.phase1 import spec as spec_phase1
|
||||
from eth2spec.test.context import PHASE0, PHASE1
|
||||
|
||||
|
||||
def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
|
||||
|
||||
def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
|
||||
def prepare_fn(configs_path: str) -> str:
|
||||
config_util.prepare_config(configs_path, config_name)
|
||||
reload(spec_phase0)
|
||||
|
|
@ -25,27 +17,42 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin
|
|||
return config_name
|
||||
|
||||
def cases_fn() -> Iterable[gen_typing.TestCase]:
|
||||
tests_src = import_module(tests_src_mod_name)
|
||||
return generate_from_tests(
|
||||
runner_name='epoch_processing',
|
||||
handler_name=handler_name,
|
||||
src=tests_src,
|
||||
fork_name=PHASE0,
|
||||
fork_name=fork_name,
|
||||
)
|
||||
|
||||
return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
gen_runner.run_generator("epoch_processing", [
|
||||
create_provider('final_updates', test_process_final_updates, 'minimal'),
|
||||
create_provider('final_updates', test_process_final_updates, 'mainnet'),
|
||||
create_provider('justification_and_finalization', test_process_justification_and_finalization, 'minimal'),
|
||||
create_provider('justification_and_finalization', test_process_justification_and_finalization, 'mainnet'),
|
||||
create_provider('registry_updates', test_process_registry_updates, 'minimal'),
|
||||
create_provider('registry_updates', test_process_registry_updates, 'mainnet'),
|
||||
create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'minimal'),
|
||||
# runs full epochs filled with data, with uncached ssz. Disabled for now.
|
||||
# create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'mainnet'),
|
||||
create_provider('slashings', test_process_slashings, 'minimal'),
|
||||
create_provider('slashings', test_process_slashings, 'mainnet'),
|
||||
phase_0_mods = [(key, 'eth2spec.test.phase0.epoch_processing.test_process_' + key) for key in [
|
||||
'final_updates',
|
||||
'justification_and_finalization',
|
||||
'registry_updates',
|
||||
'rewards_and_penalties',
|
||||
'slashings',
|
||||
]]
|
||||
phase_1_mods = [(key, 'eth2spec.test.phase1.epoch_processing.test_process_' + key) for key in [
|
||||
'challenge_deadlines',
|
||||
'custody_final_updates',
|
||||
'reveal_deadlines',
|
||||
]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec)
|
||||
|
||||
gen_runner.run_generator(f"epoch_processing", [
|
||||
create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods
|
||||
])
|
||||
# TODO: disabled for testing
|
||||
# gen_runner.run_generator(f"epoch_processing", [
|
||||
# create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods
|
||||
# ])
|
||||
gen_runner.run_generator(f"epoch_processing", [
|
||||
create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods
|
||||
])
|
||||
# Disabled for now
|
||||
# gen_runner.run_generator(f"epoch_processing", [
|
||||
# create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods
|
||||
# ])
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
from typing import Iterable
|
||||
|
||||
from importlib import import_module
|
||||
from gen_base import gen_runner, gen_typing
|
||||
from gen_from_tests.gen import generate_from_tests
|
||||
from importlib import reload
|
||||
from importlib import reload, import_module
|
||||
from eth2spec.config import config_util
|
||||
from eth2spec.phase0 import spec as spec_phase0
|
||||
from eth2spec.phase1 import spec as spec_phase1
|
||||
|
|
@ -47,17 +46,17 @@ if __name__ == "__main__":
|
|||
'shard_transition',
|
||||
]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec)
|
||||
|
||||
gen_runner.run_generator(f"phase0/operations", [
|
||||
gen_runner.run_generator(f"operations", [
|
||||
create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods
|
||||
])
|
||||
# TODO: disabled for testing
|
||||
# gen_runner.run_generator(f"phase0/operations", [
|
||||
# gen_runner.run_generator(f"operations", [
|
||||
# create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods
|
||||
# ])
|
||||
gen_runner.run_generator(f"phase1/operations", [
|
||||
gen_runner.run_generator(f"operations", [
|
||||
create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods
|
||||
])
|
||||
# Disabled for now
|
||||
# gen_runner.run_generator(f"phase1/operations", [
|
||||
# gen_runner.run_generator(f"operations", [
|
||||
# create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods
|
||||
# ])
|
||||
|
|
|
|||
|
|
@ -1,21 +1,15 @@
|
|||
from typing import Iterable
|
||||
|
||||
from eth2spec.phase0 import spec as spec_phase0
|
||||
from eth2spec.phase1 import spec as spec_phase1
|
||||
from eth2spec.test.phase0.rewards import (
|
||||
test_basic,
|
||||
test_leak,
|
||||
test_random,
|
||||
)
|
||||
from gen_base import gen_runner, gen_typing
|
||||
from gen_from_tests.gen import generate_from_tests
|
||||
from importlib import reload
|
||||
from importlib import reload, import_module
|
||||
from eth2spec.config import config_util
|
||||
from eth2spec.test.context import PHASE0
|
||||
from eth2spec.phase0 import spec as spec_phase0
|
||||
from eth2spec.phase1 import spec as spec_phase1
|
||||
from eth2spec.test.context import PHASE0, PHASE1
|
||||
|
||||
|
||||
def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider:
|
||||
|
||||
def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
|
||||
def prepare_fn(configs_path: str) -> str:
|
||||
config_util.prepare_config(configs_path, config_name)
|
||||
reload(spec_phase0)
|
||||
|
|
@ -23,22 +17,41 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider:
|
|||
return config_name
|
||||
|
||||
def cases_fn() -> Iterable[gen_typing.TestCase]:
|
||||
tests_src = import_module(tests_src_mod_name)
|
||||
return generate_from_tests(
|
||||
runner_name='rewards',
|
||||
handler_name='core',
|
||||
handler_name=handler_name,
|
||||
src=tests_src,
|
||||
fork_name=PHASE0,
|
||||
fork_name=fork_name,
|
||||
)
|
||||
|
||||
return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
gen_runner.run_generator("rewards", [
|
||||
create_provider(test_basic, 'minimal'),
|
||||
create_provider(test_basic, 'mainnet'),
|
||||
create_provider(test_leak, 'minimal'),
|
||||
create_provider(test_leak, 'mainnet'),
|
||||
create_provider(test_random, 'minimal'),
|
||||
create_provider(test_random, 'mainnet'),
|
||||
phase_0_mods = [(key, 'eth2spec.test.phase0.rewards.test_'+key) for key in [
|
||||
'basic',
|
||||
'basic',
|
||||
'leak',
|
||||
'leak',
|
||||
'random',
|
||||
'random',
|
||||
]]
|
||||
# No additional phase 1 specific rewards tests, yet.
|
||||
phase_1_mods = phase_0_mods
|
||||
|
||||
gen_runner.run_generator(f"rewards", [
|
||||
create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods
|
||||
])
|
||||
# TODO: disabled for testing
|
||||
# gen_runner.run_generator(f"rewards", [
|
||||
# create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods
|
||||
# ])
|
||||
gen_runner.run_generator(f"rewards", [
|
||||
create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods
|
||||
])
|
||||
# Disabled for now
|
||||
# gen_runner.run_generator(f"rewards", [
|
||||
# create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods
|
||||
# ])
|
||||
|
||||
|
|
|
|||
|
|
@ -1,18 +1,15 @@
|
|||
from typing import Iterable
|
||||
from importlib import reload
|
||||
|
||||
from gen_base import gen_runner, gen_typing
|
||||
from gen_from_tests.gen import generate_from_tests
|
||||
|
||||
from eth2spec.test.context import PHASE0
|
||||
from eth2spec.test.phase0.sanity import test_blocks, test_slots
|
||||
from importlib import reload, import_module
|
||||
from eth2spec.config import config_util
|
||||
from eth2spec.phase0 import spec as spec_phase0
|
||||
from eth2spec.phase1 import spec as spec_phase1
|
||||
from eth2spec.test.context import PHASE0, PHASE1
|
||||
|
||||
|
||||
def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider:
|
||||
|
||||
def create_provider(fork_name: str, handler_name: str, tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider:
|
||||
def prepare_fn(configs_path: str) -> str:
|
||||
config_util.prepare_config(configs_path, config_name)
|
||||
reload(spec_phase0)
|
||||
|
|
@ -20,20 +17,39 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin
|
|||
return config_name
|
||||
|
||||
def cases_fn() -> Iterable[gen_typing.TestCase]:
|
||||
tests_src = import_module(tests_src_mod_name)
|
||||
return generate_from_tests(
|
||||
runner_name='sanity',
|
||||
handler_name=handler_name,
|
||||
src=tests_src,
|
||||
fork_name=PHASE0,
|
||||
fork_name=fork_name,
|
||||
)
|
||||
|
||||
return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
gen_runner.run_generator("sanity", [
|
||||
create_provider('blocks', test_blocks, 'minimal'),
|
||||
create_provider('blocks', test_blocks, 'mainnet'),
|
||||
create_provider('slots', test_slots, 'minimal'),
|
||||
create_provider('slots', test_slots, 'mainnet'),
|
||||
phase_0_mods = [(key, 'eth2spec.test.phase0.sanity.test_' + key) for key in [
|
||||
'blocks',
|
||||
'slots',
|
||||
]]
|
||||
phase_1_mods = [(key, 'eth2spec.test.phase1.sanity.test_' + key) for key in [
|
||||
'blocks', # more phase 1 specific block tests
|
||||
'shard_blocks',
|
||||
]] + phase_0_mods # also run the previous phase 0 tests (but against phase 1 spec)
|
||||
|
||||
gen_runner.run_generator(f"sanity", [
|
||||
create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods
|
||||
])
|
||||
# TODO: disabled for testing
|
||||
# gen_runner.run_generator(f"sanity", [
|
||||
# create_provider(key, mod_name, 'mainnet') for key, mod_name in phase_0_mods
|
||||
# ])
|
||||
gen_runner.run_generator(f"sanity", [
|
||||
create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods
|
||||
])
|
||||
# Disabled for now
|
||||
# gen_runner.run_generator(f"sanity", [
|
||||
# create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods
|
||||
# ])
|
||||
|
||||
|
|
|
|||
|
|
@ -7,8 +7,9 @@ from gen_base import gen_runner, gen_typing
|
|||
|
||||
from eth2spec.debug import random_value, encode
|
||||
from eth2spec.config import config_util
|
||||
from eth2spec.phase0 import spec
|
||||
from eth2spec.test.context import PHASE0
|
||||
from eth2spec.phase0 import spec as spec_phase0
|
||||
from eth2spec.phase1 import spec as spec_phase1
|
||||
from eth2spec.test.context import PHASE0, PHASE1
|
||||
from eth2spec.utils.ssz.ssz_typing import Container
|
||||
from eth2spec.utils.ssz.ssz_impl import (
|
||||
hash_tree_root,
|
||||
|
|
@ -16,7 +17,7 @@ from eth2spec.utils.ssz.ssz_impl import (
|
|||
)
|
||||
|
||||
|
||||
MAX_BYTES_LENGTH = 100
|
||||
MAX_BYTES_LENGTH = 1000
|
||||
MAX_LIST_LENGTH = 10
|
||||
|
||||
|
||||
|
|
@ -30,14 +31,14 @@ def create_test_case(rng: Random, typ, mode: random_value.RandomizationMode, cha
|
|||
yield "roots", "data", roots_data
|
||||
|
||||
|
||||
def get_spec_ssz_types():
|
||||
def get_spec_ssz_types(spec):
|
||||
return [
|
||||
(name, value) for (name, value) in getmembers(spec, isclass)
|
||||
if issubclass(value, Container) and value != Container # only the subclasses, not the imported base class
|
||||
]
|
||||
|
||||
|
||||
def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.RandomizationMode, chaos: bool, count: int):
|
||||
def ssz_static_cases(fork_name: str, seed: int, name, ssz_type, mode: random_value.RandomizationMode, chaos: bool, count: int):
|
||||
random_mode_name = mode.to_name()
|
||||
|
||||
# Reproducible RNG
|
||||
|
|
@ -45,7 +46,7 @@ def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.Randomization
|
|||
|
||||
for i in range(count):
|
||||
yield gen_typing.TestCase(
|
||||
fork_name=PHASE0,
|
||||
fork_name=fork_name,
|
||||
runner_name='ssz_static',
|
||||
handler_name=name,
|
||||
suite_name=f"ssz_{random_mode_name}{'_chaos' if chaos else ''}",
|
||||
|
|
@ -54,19 +55,23 @@ def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.Randomization
|
|||
)
|
||||
|
||||
|
||||
def create_provider(config_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool,
|
||||
def create_provider(fork_name, config_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool,
|
||||
cases_if_random: int) -> gen_typing.TestProvider:
|
||||
def prepare_fn(configs_path: str) -> str:
|
||||
# Apply changes to presets, this affects some of the vector types.
|
||||
config_util.prepare_config(configs_path, config_name)
|
||||
reload(spec)
|
||||
reload(spec_phase0)
|
||||
reload(spec_phase1)
|
||||
return config_name
|
||||
|
||||
def cases_fn() -> Iterable[gen_typing.TestCase]:
|
||||
count = cases_if_random if chaos or mode.is_changing() else 1
|
||||
spec = spec_phase0
|
||||
if fork_name == PHASE1:
|
||||
spec = spec_phase1
|
||||
|
||||
for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types()):
|
||||
yield from ssz_static_cases(seed * 1000 + i, name, ssz_type, mode, chaos, count)
|
||||
for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types(spec)):
|
||||
yield from ssz_static_cases(fork_name, seed * 1000 + i, name, ssz_type, mode, chaos, count)
|
||||
|
||||
return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn)
|
||||
|
||||
|
|
@ -80,10 +85,11 @@ if __name__ == "__main__":
|
|||
seed += 1
|
||||
settings.append((seed, "minimal", random_value.RandomizationMode.mode_random, True, 30))
|
||||
seed += 1
|
||||
settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5))
|
||||
seed += 1
|
||||
# settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5))
|
||||
# seed += 1
|
||||
|
||||
gen_runner.run_generator("ssz_static", [
|
||||
create_provider(config_name, seed, mode, chaos, cases_if_random)
|
||||
for (seed, config_name, mode, chaos, cases_if_random) in settings
|
||||
])
|
||||
for fork in [PHASE0, PHASE1]:
|
||||
gen_runner.run_generator("ssz_static", [
|
||||
create_provider(fork, config_name, seed, mode, chaos, cases_if_random)
|
||||
for (seed, config_name, mode, chaos, cases_if_random) in settings
|
||||
])
|
||||
|
|
|
|||
Loading…
Reference in New Issue