diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks_random.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks_random.py index ffc37c7c2..9b10ab355 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks_random.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks_random.py @@ -1,34 +1,240 @@ +import itertools +from random import Random +from typing import Callable +from tests.core.pyspec.eth2spec.test.context import default_activation_threshold +from eth2spec.test.helpers.multi_operations import ( + build_random_block_from_state, +) from eth2spec.test.helpers.state import ( next_epoch, next_slot, + ensure_state_has_validators_across_lifecycle, + state_transition_and_sign_block, +) +from eth2spec.test.helpers.random import ( + randomize_state, ) from eth2spec.test.context import ( with_all_phases, - spec_state_test, + always_bls, + spec_test, + with_custom_state, + default_activation_threshold, + single_phase, + misc_balances, ) +# primitives +## epochs -def generate_randomized_scenarios(): - # TODO: WIP schema +def _epochs_until_leak(spec): + return spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + + +def _epochs_for_shard_committee_period(spec): + return spec.config.SHARD_COMMITTEE_PERIOD + + +## slots + +def _last_slot_in_epoch(spec): + return spec.SLOTS_PER_EPOCH - 1 + + +def _random_slot_in_epoch(rng): + def f(spec): + return rng.randrange(1, spec.SLOTS_PER_EPOCH - 2) + return f + + +def _penultimate_slot_in_epoch(spec): + return spec.SLOTS_PER_EPOCH - 2 + + +## blocks + +def _no_block(_spec, _pre_state, _signed_blocks): + return None + + +def _random_block_for_next_slot(spec, pre_state, _signed_blocks): + return build_random_block_from_state(spec, pre_state) + + +## validations + +def _no_op_validation(spec, state): + return True + + +def _validate_is_leaking(spec, state): + return spec.is_in_inactivity_leak(state) + + +# transitions + +def _no_op_transition(): + return {} + + +def _epoch_transition(n=0): return { - # ("randomize_state", "ensure all validator states present: pending/deposited, activated, exited, slashed"), - # ("randomized balances", "ensure distribution of bals"), - # ("transition to leak if not already, maybe", "assert is or is not leaking"), - "setup": [], - "epochs_to_skip": 0, # 0, 1, 2, N, EPOCHS_TO_INACTIVITY_LEAK, - "slots_to_skip": 0, # 0, 1, 2, N, SLOTS_PER_EPOCH - 1, - "transitions": [ # TODO: consider large numbers of blocks, load on generated data - { - "block_producer": lambda spec, state: spec.SignedBeaconBlock(), - "epochs_to_skip": 0, # 0, 1, 2, N, EPOCHS_TO_INACTIVITY_LEAK, - "slots_to_skip": 0, # 0, 1, 2, N, SLOTS_PER_EPOCH - 1, - } - ], + "epochs_to_skip": n, } -def id_from_scenario(test_description): - return '-'.join(':'.join((str(k),str(v))) for k,v in test_description.items()) +def _slot_transition(n=0): + return { + "slots_to_skip": n, + } + + +def _transition_to_leaking(): + return { + "epochs_to_skip": _epochs_until_leak, + "validation": _validate_is_leaking, + } + + +## block transitions + +def _transition_with_random_block(epochs=None, slots=None): + """ + Build a block transition with randomized data. + Provide optional sub-transitions to advance some + number of epochs or slots before applying the random block. + """ + transition = { + "block_producer": _random_block_for_next_slot, + } + if epochs: + transition.update(epochs) + if slots: + transition.update(slots) + return transition + + +# setup and test gen + +def _randomized_scenario_setup(): + """ + Return a sequence of pairs of ("mutator", "validator"), + a function that accepts (spec, state) arguments and performs some change + and a function that accepts (spec, state) arguments and validates some change was made. + """ + def _skip_epochs(epoch_producer): + def f(spec, state): + """ + The unoptimized spec implementation is too slow to advance via ``next_epoch``. + Instead, just overwrite the ``state.slot`` and continue... + """ + epochs_to_skip = epoch_producer(spec) + slots_to_skip = epochs_to_skip * spec.SLOTS_PER_EPOCH + state.slot += slots_to_skip + return f + + return ( + # NOTE: the block randomization function assumes at least 1 shard committee period + # so advance the state before doing anything else. + (_skip_epochs(_epochs_for_shard_committee_period), _no_op_validation), + (randomize_state, ensure_state_has_validators_across_lifecycle), + ) + + +def _normalize_transition(transition): + """ + Provide "empty" or "no op" sub-transitions + to a given transition. + """ + if isinstance(transition, Callable): + transition = transition() + if "epochs_to_skip" not in transition: + transition["epochs_to_skip"] = 0 + if "slots_to_skip" not in transition: + transition["slots_to_skip"] = 0 + if "block_producer" not in transition: + transition["block_producer"] = _no_block + if "validation" not in transition: + transition["validation"] = _no_op_validation + return transition + + +def _normalize_scenarios(scenarios): + """ + "Normalize" a "scenario" so that a producer of a test case + does not need to provide every expected key/value. + """ + for scenario in scenarios: + if "setup" not in scenario: + scenario["setup"] = _randomized_scenario_setup() + + transitions = scenario["transitions"] + for i, transition in enumerate(transitions): + transitions[i] = _normalize_transition(transition) + + +def _generate_randomized_scenarios(): + """ + Generates a set of randomized testing scenarios. + Return a sequence of "scenarios" where each scenario: + 1. Provides some setup + 2. Provides a sequence of transitions that mutate the state in some way, + possibly yielding blocks along the way + NOTE: scenarios are "normalized" with empty/no-op elements before returning + to the test generation to facilitate brevity when writing scenarios by hand. + NOTE: the main block driver builds a block for the **next** slot, so + the slot transitions are offset by -1 to target certain boundaries. + """ + rng = Random(1336) + leak_transitions = (_no_op_transition, _transition_to_leaking) + + # go forward 0 or 1 epochs + epochs_set = (_epoch_transition(n=0), _epoch_transition(n=1)) + # within those epochs, go forward to: + slots_set = ( + # the first slot in an epoch (see note in docstring about offsets...) + _slot_transition(_last_slot_in_epoch), + # the second slot in an epoch + _slot_transition(n=0), + # some random number of slots, but not at epoch boundaries + _slot_transition(_random_slot_in_epoch(rng)), + # the last slot in an epoch (see note in docstring about offsets...) + _slot_transition(_penultimate_slot_in_epoch), + ) + # build a set of block transitions from combinations of sub-transitions + block_transitions = list( + _transition_with_random_block(epochs=epochs, slots=slots) + for epochs, slots in itertools.product(epochs_set, slots_set) + ) + + # and preface each block transition with the possible leak transitions + # (... either no leak or transition to a leak before applying the block transition) + scenarios = [ + {"transitions": list(t)} + for t in itertools.product(leak_transitions, block_transitions) + ] + _normalize_scenarios(scenarios) + return scenarios + + +def _id_from_scenario(test_description): + """ + Construct a test name for ``pytest`` infra. + """ + def _to_id_part(prefix, x): + suffix = str(x) + if isinstance(x, Callable): + suffix = x.__name__ + return f"{prefix}{suffix}" + + def _id_from_transition(transition): + return ",".join(( + _to_id_part("epochs:", transition["epochs_to_skip"]), + _to_id_part("slots:", transition["slots_to_skip"]), + _to_id_part("with-block:", transition["block_producer"]) + )) + + return "|".join(map(_id_from_transition, test_description["transitions"])) def pytest_generate_tests(metafunc): @@ -36,8 +242,8 @@ def pytest_generate_tests(metafunc): Pytest hook to generate test cases from dynamically computed data """ generated_name = "test_description" - generated_values = generate_randomized_scenarios() - metafunc.parametrize(generated_name, generated_values, ids=id_from_scenario, scope="module") + generated_values = _generate_randomized_scenarios() + metafunc.parametrize(generated_name, generated_values, ids=_id_from_scenario, scope="module") def pytest_generate_tests_adapter(f): @@ -51,17 +257,47 @@ def pytest_generate_tests_adapter(f): return wrapper +def _iter_temporal(spec, callable_or_int): + """ + Intended to advance some number of {epochs, slots}. + Caller can provide a constant integer or a callable deriving a number from + the ``spec`` under consideration. + """ + numeric = callable_or_int + if isinstance(callable_or_int, Callable): + numeric = callable_or_int(spec) + for i in range(numeric): + yield i + + @pytest_generate_tests_adapter @with_all_phases -@spec_state_test +@with_custom_state(balances_fn=misc_balances, threshold_fn=default_activation_threshold) +@spec_test +@single_phase +@always_bls def test_harness_for_randomized_blocks(spec, state, test_description): for mutation, validation in test_description["setup"]: mutation(spec, state) validation(spec, state) - for _ in range(len(test_description["epochs_to_skip"])): - next_epoch(spec, state) - for _ in range(len(test_description["slots_to_skip"])): - next_slot(spec, state) + + yield "pre", state + + blocks = [] for transition in test_description["transitions"]: - # TODO apply transition - pass + epochs_to_skip = _iter_temporal(spec, transition["epochs_to_skip"]) + for _ in epochs_to_skip: + next_epoch(spec, state) + slots_to_skip = _iter_temporal(spec, transition["slots_to_skip"]) + for _ in slots_to_skip: + next_slot(spec, state) + + block = transition["block_producer"](spec, state, blocks) + if block: + signed_block = state_transition_and_sign_block(spec, state, block) + blocks.append(signed_block) + + assert transition["validation"](spec, state) + + yield "blocks", blocks + yield "post", state